fix(protocol): remove auto-detection of frame mode and ensure extra chunks are always used for valid waveform footer

.dockerignore

@@ -1,28 +0,0 @@
-.git
-.gitignore
-
-.venv
-venv
-env
-__pycache__
-*.pyc
-*.pyo
-*.pyd
-.pytest_cache
-.mypy_cache
-.ruff_cache
-
-*.db
-*.db-wal
-*.db-shm
-*.sqlite
-*.sqlite3
-
-sfm/data
-bridges/captures
-example-events
-captures
-logs
-
-.DS_Store
-Thumbs.db

.gitignore

@@ -1,6 +1,6 @@
 /bridges/captures/
 /example-events/
-/tests/fixtures/
+
 /manuals/
 
 # Python build artifacts

Dockerfile

@@ -1,31 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-# tzdata is required for the TZ env var to take effect (python:slim
-# omits the timezone database).  Without it, datetime.now() / logging
-# / matplotlib all stay in UTC regardless of TZ.  Default zone gets
-# set further down via ENV; users override per-deployment via the
-# `TZ` env var in docker-compose.
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends curl tzdata && \
-    rm -rf /var/lib/apt/lists/*
-
-# Default display timezone — applied to server logs, datetime.now(),
-# matplotlib rendered timestamps, and any naïve-vs-aware datetime
-# conversions in the PDF renderer.  Override via TZ env var in
-# docker-compose; storage in the DB is always UTC regardless.
-ENV TZ=America/New_York
-
-COPY pyproject.toml requirements.txt ./
-COPY minimateplus ./minimateplus
-COPY micromate    ./micromate
-COPY sfm          ./sfm
-COPY bridges      ./bridges
-COPY scripts      ./scripts
-
-RUN pip install --no-cache-dir -e .
-
-EXPOSE 8200
-
-CMD ["python", "-m", "uvicorn", "sfm.server:app", "--host", "0.0.0.0", "--port", "8200"]

README.md

@@ -1,11 +1,7 @@
-# seismo-relay  `v0.31.0`
+# seismo-relay  `v0.12.1`
 
 A ground-up replacement for **Blastware** — Instantel's aging Windows-only
-software for managing seismographs.  Supports both the **MiniMate Plus
+software for managing MiniMate Plus seismographs.
-(Series III)** and the **Micromate (Series IV / "Thor")** families:
-Series III via the live RS-232 / TCP wire protocol *and* Blastware ACH file
-ingest; Series IV currently via Thor TXT-paired IDF file ingest, with the
-binary codec on the roadmap.
 
 Built in Python. Runs on Windows, Linux, or macOS. Connects to instruments
 over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
@@ -14,51 +10,6 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
 > pipeline working end-to-end over TCP/cellular. ACH Auto Call Home server
 > handles inbound unit connections, downloads events, and persists everything
 > to a SQLite database. SFM REST API exposes device control and DB queries.
-> **As of v0.14.3 (2026-05-05): SUB 5A bulk waveform protocol is verified
-> byte-perfect against Blastware captures across 2-sec, 3-sec, and 10-sec
-> events.** Generated `.G10` / `.AB0` files open cleanly in Blastware with
-> full Event Reports, frequency analysis, and waveform plots.
-> **v0.16.0 (2026-05-11)** adds BW ASCII report ingestion to
-> `/db/import/blastware_file` — paired with **series3-watcher v1.5.0**,
-> every Blastware ACH event lands in SeismoDb with device-authoritative
-> peaks, project metadata, sensor self-check, and ZC/Time-of-Peak data,
-> without depending on the still-undecoded waveform body codec.
-> **v0.18.0 (2026-05-19)** adds Thor / Micromate Series IV ingest at
-> `/db/import/idf_file` — paired with **thor-watcher v0.3.0**, every
-> `.IDFH` / `.IDFW` event file (plus its `.txt` sidecar) lands in
-> SeismoDb the same way BW events do.  See
-> [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md) for
-> the IDF format reference and reverse-engineering plan.
-> **v0.19.0 (2026-05-20)** separates Series III and Series IV at the
-> code level: new `micromate/` package alongside `minimateplus/`, new
-> `events.device_family` DB column ("series3" / "series4") so the UI
-> and storage layer dispatch deterministically instead of sniffing
-> filenames.  Self-applying migration backfills existing rows from the
-> binary filename extension.
-> **v0.20.0 (2026-05-28)** closes out the Event-Report PDF iteration
-> started in v0.17.x: histogram layouts render correctly against BW
-> reference PDFs, the ASCII parser handles real-world edge cases
-> (`OORANGE`, `>100 Hz`, histogram timestamps), and per-channel ZC
-> Freq is surfaced in both modals (event browser + main webapp).
-> Adds a server-wide `TZ` env var so operator-visible timestamps
-> render in local time instead of UTC.  New
-> `scripts/backfill_sidecars.py --reparse-txt` lets parser fixes be
-> applied retroactively to existing events without re-forwarding,
-> using the `.TXT` files preserved at ingest time.
-> **v0.21.0 (2026-05-29)** is the Thor / Series IV decoder release —
-> `micromate/idf_file.read_idf_file()` now decodes both IDFW
-> (waveform) and IDFH (histogram) binaries (87–99% sample fidelity
-> on quiet IDFW events; all 859 IDFH corpus files decode cleanly).
-> A new `micromate/idf_to_bw_report.py` adapter projects parsed
-> Thor reports into the BW-shaped sidecar block, so Thor events
-> flow through the existing Event Report PDF pipeline without a
-> separate renderer.  Terra-View v0.13.0 ships in parallel and
-> closes Phase 1 of the SFM integration — see its CHANGELOG.
-> **v0.22.0 (2026-07-03)** adds the SFM side of the full-snapshot
-> bundle: `GET /db/snapshot` (WAL-safe DB copy), `GET /db/waveforms/
-> recent.zip` (recent events' waveform files), and a gated
-> `POST /db/restore` (`SFM_DB_RESTORE_ENABLED`, dev-only).  Terra-View
-> v0.17.0 drives them to pull a one-pass prod→dev refresh.
 > See [CHANGELOG.md](CHANGELOG.md) for full version history.
 
 ---
@@ -67,36 +18,26 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
 
 ```
 seismo-relay/
-├── seismo_lab.py              ← Main GUI (Bridge + Analyzer + Download + Console tabs)
+├── seismo_lab.py              ← Main GUI (Bridge + Analyzer + Console tabs)
 │
-├── minimateplus/              ← Series III (MiniMate Plus) client library
+├── minimateplus/              ← MiniMate Plus client library
 │   ├── transport.py           ←   SerialTransport, TcpTransport, SocketTransport
 │   ├── protocol.py            ←   DLE frame layer, SUB command dispatch
-│   ├── client.py              ←   High-level client (connect, get_events, delete_all_events, push_config, get_call_home_config, …)
+│   ├── client.py              ←   High-level client (connect, get_events, push_config, …)
 │   ├── framing.py             ←   Frame builders, DLE codec, S3FrameParser
-│   ├── models.py              ←   DeviceInfo, Event, ComplianceConfig, MonitorLogEntry, CallHomeConfig, …
+│   └── models.py              ←   DeviceInfo, Event, ComplianceConfig, MonitorLogEntry, …
-│   ├── bw_ascii_report.py     ←   Parse BW per-event ASCII reports (.TXT sidecars)
-│   ├── event_file_io.py       ←   Read BW binaries, write .sfm.json sidecars
-│   └── blastware_file.py      ←   Write events to Blastware-compatible .AB0 files
-│
-├── micromate/                 ← Series IV (Micromate / Thor) client library (NEW v0.19)
-│   ├── models.py              ←   IdfEvent, IdfReport, IdfPeaks, IdfProjectInfo, IdfSensorCheck (mic in native dB(L))
-│   ├── idf_ascii_report.py    ←   Parse Thor .IDFW.txt / .IDFH.txt event sidecars
-│   ├── idf_file.py            ←   Binary codec for .IDFW + .IDFH (v0.21.0+)
-│   └── idf_to_bw_report.py    ←   Adapter projecting Thor IDF into the BW report shape (v0.21.0+)
 │
 ├── sfm/                       ← SFM REST API server (FastAPI, port 8200)
-│   ├── server.py              ←   Live device endpoints + DB query + ingest endpoints + caching
+│   ├── server.py              ←   All device + DB endpoints
-│   ├── database.py            ←   SeismoDb — SQLite persistence (events, monitor_log, ach_sessions)
+│   ├── database.py            ←   SeismoDb — SQLite persistence layer
-│   ├── waveform_store.py      ←   On-disk store for BW + IDF event binaries + .sfm.json sidecars
+│   └── sfm_webapp.html        ←   Embedded web UI (served at /)
-│   └── sfm_webapp.html        ←   Embedded web UI with Call Home config tab
 │
 ├── bridges/
 │   ├── ach_server.py          ←   Inbound ACH call-home server (main production server)
 │   ├── ach_mitm.py            ←   Transparent MITM proxy for capturing BW sessions
 │   ├── s3-bridge/             ←   RS-232 serial bridge (capture tool)
 │   ├── tcp_serial_bridge.py   ←   Local TCP↔serial bridge (bench testing)
-│   ├── gui_bridge.py          ←   Standalone bridge GUI with raw capture checkboxes
+│   ├── gui_bridge.py          ←   Standalone bridge GUI
 │   └── raw_capture.py         ←   Simple raw capture tool
 │
 ├── parsers/
@@ -105,8 +46,7 @@ seismo-relay/
 │   └── frame_db.py            ←   SQLite frame database
 │
 └── docs/
-    ├── instantel_protocol_reference.md  ← Series III protocol spec (the Rosetta Stone)
+    └── instantel_protocol_reference.md  ← Reverse-engineered protocol spec
-    └── idf_protocol_reference.md         ← Series IV (Thor IDF) format reference + codec RE plan
 ```
 
 ---
@@ -161,28 +101,21 @@ python seismo_lab.py
 Each call dials the device, does its work, and closes the connection. TCP
 connections are retried once on `ProtocolError` to handle cold-boot timing.
 
-**In-memory caching** — frequently-polled endpoints avoid redundant TCP round-trips
+**Caching** — frequently-polled endpoints are cached in-process to avoid
-via a thread-safe `_LiveCache` (plain Python dict + `threading.Lock`):
+redundant TCP round-trips:
 
-| Method | URL | Cache Strategy |
+| Method | URL | Cache |
-|--------|-----|---|
+|--------|-----|-------|
 | `GET` | `/device/info` | Indefinite; invalidated by `POST /device/config` |
 | `GET` | `/device/events` | Count-probe fast path (~2s); full download only when new events detected |
 | `GET` | `/device/event/{idx}/waveform` | Permanent per event index |
-| `GET` | `/device/monitor/status` | 30-second TTL; invalidated by monitor start/stop |
+| `GET` | `/device/monitor/status` | 30-second TTL |
-| `GET` | `/device/call_home` | Fresh read from device (not cached) |
 | `POST` | `/device/connect` | — |
-| `POST` | `/device/config` | Writes compliance config; invalidates info + events cache |
+| `POST` | `/device/config` | Writes compliance config; invalidates cache |
-| `POST` | `/device/config/project` | Patches project/client/operator/sensor_location strings |
+| `POST` | `/device/monitor/start` | Sends SUB 0x96 |
-| `POST` | `/device/monitor/start` | Sends SUB 0x96; immediately evicts status cache |
+| `POST` | `/device/monitor/stop` | Sends SUB 0x97 |
-| `POST` | `/device/monitor/stop` | Sends SUB 0x97; immediately evicts status cache |
-| `POST` | `/device/call_home` | Reads, patches specified fields, writes back to device |
 
-**Cache bypass** — All cached endpoints accept `?force=true` to skip the cache and
+All cached endpoints accept `?force=true` to bypass the cache.
-force a fresh read from the device.
-
-**Cache stats** — `GET /cache/stats` returns hit/miss counts and TTL info; `DELETE /cache/device`
-clears the device cache immediately.
 
 Transport query params (supply one set):
 ```
@@ -198,35 +131,11 @@ Query the SQLite database written by `ach_server.py`. All read-only except
 | Method | URL | Description |
 |--------|-----|-------------|
 | `GET` | `/db/units` | All known serials with summary stats |
-| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger).  Response rows include `device_family` ("series3" / "series4") so clients dispatch on unit type without sniffing filenames. |
+| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger) |
 | `GET` | `/db/monitor_log` | Monitoring intervals |
 | `GET` | `/db/sessions` | ACH call-home session history |
 | `PATCH` | `/db/events/{id}/false_trigger?value=true` | Flag / unflag false triggers |
 
-### File ingest endpoints
-
-Used by watcher daemons to push field-collected event files into the SFM DB
-+ waveform store.  Both accept multipart uploads of binary event files
-optionally paired with their ASCII sidecar reports; both dedup by
-`(serial, timestamp)` and UPSERT device-authoritative fields on re-import.
-
-| Method | URL | Description |
-|--------|-----|-------------|
-| `POST` | `/db/import/blastware_file` | Series III: `.AB0*` / `.N00` binaries + paired `_ASCII.TXT`.  Source: `series3-watcher`. |
-| `POST` | `/db/import/idf_file` | Series IV: `.IDFH` / `.IDFW` binaries + paired `.IDFW.txt` / `.IDFH.txt`.  Source: `thor-watcher`. |
-
-### DB snapshot / restore endpoints
-
-Back the "full snapshot bundle" prod→dev refresh Terra-View orchestrates
-(Settings → Database).  Snapshot/zip are read-only; restore is gated and
-dev-only.
-
-| Method | URL | Description |
-|--------|-----|-------------|
-| `GET` | `/db/snapshot` | WAL-safe point-in-time copy of `seismo_relay.db` via the SQLite online backup API. |
-| `GET` | `/db/waveforms/recent.zip?n=…` | Zips the on-disk files (event file + `.h5` / sidecars) for the *n* most-recent events. |
-| `POST` | `/db/restore` | Validate-first restore of an uploaded DB + waveforms; takes a WAL-safe pre-restore safety backup, path-traversal-guards the zip.  Gated by `SFM_DB_RESTORE_ENABLED` (dormant → 404). |
-
 ---
 
 ## minimateplus library
@@ -243,33 +152,21 @@ client = MiniMateClient(transport=TcpTransport("1.2.3.4", 12345), timeout=30.0)
 
 with client:
     # Read
-    info     = client.connect()               # DeviceInfo — serial, firmware, compliance config
+    info    = client.connect()               # DeviceInfo — serial, firmware, compliance config
-    count    = client.count_events()          # Number of stored events
+    count   = client.count_events()          # Number of stored events
-    keys     = client.list_event_keys()       # Fast browse walk — event keys only, no download
+    keys    = client.list_event_keys()       # Fast browse walk — event keys only, no download
-    events   = client.get_events()            # Full download: headers + peaks + metadata
+    events  = client.get_events()            # Full download: headers + peaks + metadata
-    monitor  = client.get_monitor_status()    # Battery, memory, is_monitoring flag
+    monitor = client.get_monitor_status()    # Battery, memory, is_monitoring flag
-    log      = client.get_monitor_log_entries()  # Monitoring intervals (partial 0x2C records)
+    log     = client.get_monitor_log_entries()  # Monitoring intervals (partial 0x2C records)
-    ach_cfg  = client.get_call_home_config()  # Auto Call Home settings (SUB 0x2C)
 
     # Write
     client.apply_config(
         sample_rate=1024,
-        recording_mode="Continuous",         # Single Shot / Continuous / Histogram / Histogram+Continuous
-        histogram_interval_sec=15,            # 2, 5, 15, 60, 300, 900
         trigger_level_geo=0.5,
-        geo_range="Normal",                   # Normal (10.000 in/s) / Sensitive (1.25 in/s)
         project="Bridge Inspection 2026",
         client_name="City of Portland",
         operator="B. Harrison",
     )
-    
-    client.set_call_home_config(
-        auto_call_home_enabled=True,
-        after_event_recorded=True,
-        at_specified_times=True,
-        time1_hour=18, time1_min=30,          # 6:30 PM
-        time2_hour=6, time2_min=0,            # 6:00 AM
-    )
 
     # Control
     client.start_monitoring()                # SUB 0x96
@@ -277,88 +174,26 @@ with client:
     client.delete_all_events()              # Erase all (SUB 0xA3 → 0x1C → 0x06 → 0xA2)
 ```
 
-`get_events()` runs the full per-event sequence:
+`get_events()` runs the full per-event sequence: `1E → 0A → 0C → 5A → 1F`.
-`1E → 0A → 1E(arm token=0xFE) → 0C → 1F(arm) → POLL×3 → 5A → 1F(browse)`.
+SUB 5A bulk stream provides `client`, `operator`, and `sensor_location` as they
-SUB 5A bulk stream walks chunks bounded by the `end_offset` extracted from
+existed at record time — not backfilled from the current compliance config.
-the STRT record at byte 17 of the probe response — no over-reading, no
-chunk-count cap. Project / client / operator / sensor location strings come
-from the dedicated metadata pages at counter `0x1002` and `0x1004`,
-read once per session (they reflect the compliance setup at session start,
-not per individual event).
-
----
-
-## micromate library
-
-Series IV / Thor support, sibling to `minimateplus`.  Currently scoped to
-offline-file ingest from Thor's TXT exporter; live-device protocol is
-deferred until the binary codec is cracked.
-
-```python
-from micromate import IdfEvent, parse_idf_report
-
-# Parse a .IDFW.txt / .IDFH.txt sidecar (1014 example files round-trip cleanly)
-text = open("UM11719_20231219162723.IDFW.txt").read()
-report_dict = parse_idf_report(text)        # permissive dict
-
-# Wrap into a typed event using the device-native binary filename
-event = IdfEvent.from_report(report_dict, "UM11719_20231219162723.IDFW")
-
-event.serial                     # "UM11719"
-event.kind                       # "Waveform" or "Histogram"
-event.peaks.transverse_ips       # 0.0251  (in/s, native unit)
-event.peaks.mic_pspl_dbl         # 99.4    (dB(L), Thor's native mic unit — NOT psi)
-event.project_info.project       # "UPMC Presby-Loc 3-Level1-1R Elevator Rm"
-event.sensor_check.tran          # True (passed self-check)
-event.firmware_version           # "Micromate ISEE 11.0AK"
-event.calibration_text           # "November 22, 2023 by Instantel"
-
-# Bridge to the existing minimateplus.Event shape for the DB / sidecar paths
-# (waveform_key is a 16-byte sha256 prefix when ingesting from a binary file)
-bridged_event = event.to_minimateplus_event(waveform_key=b"\x00" * 16)
-```
-
-The binary codec (`.IDFW` / `.IDFH` event files themselves) is on the
-roadmap — see [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md)
-for everything known so far, the two observed file signatures, and the
-reverse-engineering plan.  The `micromate/idf_file.py` stub is where
-`read_idf_file()` will land.
 
 ---
 
 ## Database
 
-`ach_server.py` and the file-ingest endpoints write to
+`ach_server.py` writes to `bridges/captures/seismo_relay.db` (SQLite, WAL mode).
-`bridges/captures/seismo_relay.db` (SQLite, WAL mode) via the `SeismoDb`
+Three tables, all unit-keyed by serial number:
-persistence layer.  Three tables, all unit-keyed by serial number:
 
 | Table | Key | Contents |
 |-------|-----|----------|
-| `ach_sessions` | UUID | Per-call-home audit record: serial, timestamp, peer IP, events_downloaded, monitor_entries, duration_seconds |
+| `ach_sessions` | UUID | Per-call-home audit record: serial, peer IP, events_downloaded, duration |
-| `events` | UUID, UNIQUE(serial, timestamp) | Triggered events: timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location strings, sample_rate, record_type, false_trigger flag, **`device_family`** ("series3" / "series4"), `blastware_filename` (binary at-rest in `waveforms/`), sidecar references |
+| `events` | UUID, UNIQUE(serial, waveform_key) | Triggered events: timestamp, PPV per channel, project/client/operator strings, false_trigger flag |
-| `monitor_log` | UUID, UNIQUE(serial, start_time) | Monitoring intervals: serial, waveform_key, start_time, stop_time, duration_seconds, geo_threshold_ips |
+| `monitor_log` | UUID, UNIQUE(serial, waveform_key) | Monitoring intervals: start/stop time, duration, geo threshold |
 
-**Deduplication is by `(serial, timestamp)`** — the device clock is the
+Deduplication is by `(serial, waveform_key)` — repeat call-homes or re-runs
-stable natural key.  Repeat call-homes or re-runs UPSERT the row in place,
+never produce duplicate rows. Post-erase key reuse is handled automatically
-refreshing every device-authoritative field (peaks, project strings,
+via the high-water mark in `ach_state.json`.
-sample_rate, file references) so the latest writer wins.  `false_trigger`
-and `device_family` are preserved across UPSERTs.  Earlier versions used
-`(serial, waveform_key)` for dedup, but the device's event-key counter
-resets to `0x01110000` after every erase, so timestamps are the correct
-dedup field.  Migration handles the transition transparently on first
-startup.
-
-**`device_family` (added v0.19.0)** discriminates Series III from Series
-IV at the SQL level.  Set by every import path; the UI dispatches on it
-to render mic units correctly (Series III: psi → dBL conversion; Series
-IV: native dBL passthrough).  Existing rows are backfilled at first
-startup of v0.19.0+ by sniffing the binary filename extension.
-
-The on-disk waveform store lives at `bridges/captures/waveforms/<serial>/`
-and holds the original event binaries (BW `.AB0*` / `.N00` for Series III,
-`.IDFH` / `.IDFW` for Series IV) plus their `.sfm.json` review/metadata
-sidecars.  Series III events also produce `.a5.pkl` source-frame pickles
-and `.h5` clean-waveform exports; Series IV doesn't yet (pending codec).
 
 ---
 
@@ -396,27 +231,6 @@ Full protocol documentation: [`docs/instantel_protocol_reference.md`](docs/insta
 
 ---
 
-## Compliance Config Features
-
-The REST API and web UI expose full control over device compliance settings:
-
-- **Recording Mode** (Single Shot / Continuous / Histogram / Histogram+Continuous)
-- **Sample Rate** (1024 / 2048 / 4096 sps)
-- **Record Time** (float, seconds)
-- **Histogram Interval** (2s, 5s, 15s, 1m, 5m, 15m) — when recording mode includes histogram
-- **Geo Trigger Levels** (float, in/s per channel)
-- **Geo Maximum Range** (Normal 10.000 in/s / Sensitive 1.250 in/s per channel)
-- **Project / Client / Operator / Sensor Location** (ASCII strings)
-
-Auto Call Home config:
-- **Auto Call Home Enable** (bool)
-- **Dial String** (read-only; 40-byte ASCII)
-- **Trigger on Event** (bool)
-- **Scheduled Call-Ins** (two time slots with HH:MM each)
-- **Retry Settings** (count, delay, connection timeout, warm-up time)
-
----
-
 ## Requirements
 
 ```bash
@@ -438,176 +252,17 @@ Use **com0com** or **VSPD** to create the virtual COM pair on Windows.
 
 ---
 
-## Key Features
+## Roadmap
 
-**Series III (MiniMate Plus) device support:**
+- [x] Full read pipeline — device info, compliance config, event download with true event-time metadata
-- [x] Full read/write/erase pipelines over RS-232 or TCP/cellular
+- [x] Write commands — push compliance config, trigger thresholds, project strings to device
-- [x] Compliance config (recording mode, sample rate, histogram interval, geo sensitivity, project strings)
+- [x] Erase all events — confirmed erase sequence from live MITM capture
-- [x] Auto Call Home config (read/write ACH settings, dial string, time slots, retries)
+- [x] Monitor control — start/stop monitoring, read battery/memory/status
-- [x] Monitor control (start/stop, status polling, battery/memory)
+- [x] Monitor log entries — decode partial 0x2C records (continuous monitoring intervals)
-- [x] Monitor log entries (continuous monitoring intervals without full waveform download)
+- [x] ACH inbound server — accept call-home connections, download events, dedup by key
-- [x] Blastware file ingest at `/db/import/blastware_file` (paired with `series3-watcher`)
+- [x] SQLite persistence — events, monitor log, and session history in `seismo_relay.db`
-
+- [x] SFM REST API — device control + DB query endpoints, live device cache
-**Series IV (Micromate / Thor) device support:**
+- [ ] Terra-view integration — seismo-relay router, unit detail page, VISON-style event listing
-- [x] Thor IDF file ingest at `/db/import/idf_file` (paired with `thor-watcher`, v0.18.0+)
+- [ ] Vibration summary reports — highest legit PPV per project → Word doc (false trigger filtering first)
-- [x] Native `IdfEvent` / `IdfReport` typed models — mic in dB(L), full title strings, sensor self-check, calibration, firmware version
+- [ ] Compliance config encoder — build raw write payloads from a `ComplianceConfig` object
-- [x] Parser verified against 1,014 paired `.txt` sidecars in `thor-watcher/example-data/`
+- [ ] Modem manager — push RV50/RV55 configs via Sierra Wireless API
-- [x] Binary `.IDFW` / `.IDFH` codec — ✅ v0.21.0.  IDFW reuses `decode_waveform_v2()` on the body at offset `0x0f1f` (87–99% sample fidelity on quiet events); IDFH has a dedicated segment-based decoder (all 859 corpus files decode, 181,071 intervals total).  See `micromate/idf_file.py` + `docs/idf_protocol_reference.md`.
-- [ ] Live-device protocol — pending codec
-
-**Data persistence:**
-- [x] SQLite database (`seismo_relay.db`) with `events`, `monitor_log`, `ach_sessions` tables
-- [x] Per-row `device_family` column ("series3" / "series4") for clean UI / unit-of-measurement dispatch (v0.19.0+)
-- [x] Deduplication by `(serial, timestamp)` — natural key handles post-erase counter resets
-- [x] UPSERT on re-import refreshes every device-authoritative field (peaks, project, sample_rate); preserves operator review state (`false_trigger`)
-- [x] Post-erase key-reuse detection (tracks high-water mark in `ach_state.json`)
-
-**REST API:**
-- [x] Live device endpoints with in-memory caching (`_LiveCache`)
-- [x] Cache statistics (`/cache/stats`) and manual invalidation (`/cache/device`)
-- [x] DB query endpoints (units, events, monitor_log, sessions, false_trigger PATCH)
-- [x] Call Home config read/write endpoints
-- [x] Blastware file download endpoint (`/device/event/{index}/blastware_file`)
-- [x] Import endpoints for both device families (`/db/import/blastware_file`, `/db/import/idf_file`)
-
-**File output (v0.7+, byte-perfect as of v0.14.3):**
-- [x] Blastware-compatible `.AB0` / `.G10` file generation (waveform + metadata)
-- [x] Multi-channel waveform decode from SUB 5A bulk stream
-- [x] Second-resolution timestamp encoding in Blastware filename
-- [x] **Byte-perfect against BW reference captures** (verified across 2-sec / 3-sec / 10-sec event durations, both event 0 and event N continuation events)
-- [x] STRT-bounded chunk walk + correct event-N probe counter + partial DLE stuffing of `0x10` in 5A params (the four fixes that landed in v0.14.0–v0.14.3)
-
-**Capture tools:**
-- [x] Serial-to-TCP bridge with raw BW/S3 capture (s3_bridge.py, defaults to auto-capture)
-- [x] GUI bridge with raw capture checkboxes (gui_bridge.py)
-- [x] ACH inbound server with bidirectional capture (ach_server.py saves raw_tx + raw_rx)
-- [x] Transparent TCP MITM proxy for live BW session capture (ach_mitm.py)
-
-**Analysis tools:**
-- [x] s3_analyzer.py — session parser, frame differ, Claude export
-- [x] gui_analyzer.py — standalone analyzer GUI
-- [x] frame_db.py — SQLite frame database for capture analysis
-
-**seismo_lab.py GUI:**
-- [x] Bridge tab — Serial/TCP mode selector with raw capture options
-- [x] Analyzer tab — BW/S3 capture playback and differencing
-- [x] Download tab — Live wire-byte capture during event download
-- [x] Console tab — Logging and diagnostics
-
-## Roadmap (Future)
-
-> **Where it stands *today*** — an honest per-capability maturity assessment,
-> what to rely on, known issues, and the gap to a real tool:
-> [`docs/sfm_tool_status.md`](docs/sfm_tool_status.md).  This section covers
-> where it is *going*.
-
-### Strategic direction — where this is going
-
-seismo-relay is being built as a **suite of cooperating components**
-that together replace and improve on Blastware's role.  Three logical
-tiers:
-
-1. **SFM** (device-side) — owns the active connection to a physical
-   unit.  Today: `minimateplus/`, `/device/*` HTTP endpoints,
-   `seismo_lab.py`.  Future: live Thor / Micromate support.
-2. **SDM** (data-side) — owns the database, waveform store, ingest
-   pipelines, and the read-API that Terra-View consumes.  Today this
-   code lives under `sfm/` for historical reasons; the role has
-   migrated and the eventual rename is on the long-tail cleanup list.
-3. **Codec library** — pure data-interpretation: `minimateplus/*_codec.py`,
-   `bw_ascii_report.py`, `micromate/idf_*.py`.  Used by both SFM and
-   SDM, depends on neither.
-
-Terra-View is downstream of SDM for fleet listings, event detail, etc.
-The long-term vision adds a **second link** from Terra-View → SFM for
-direct device interaction (see below).
-
-The codec work in this repo isn't trying to replace BW's network
-layer — BW's ACH file forwarding and Thor's IDF call-home are
-battle-tested.  The value is in the receiving and processing side: turn
-the stream of binary+ASCII pairs into something users can search,
-filter, alert on, and report from.
-
-### Terra-View ↔ SFM device control (the long-term vision)
-
-Today Terra-View only reads from SDM (event listings, dashboards,
-project reports).  When a unit goes missing — operator notices in the
-Terra-View dashboard — there's no way to *do* anything from the UI.
-The path of least resistance is to RDP into a Windows box and open
-Blastware, which defeats the purpose of having Terra-View.
-
-Target experience:
-- Operator notices a unit in Terra-View dashboard hasn't called in.
-- Clicks unit detail → "Connect to Device" button.
-- Terra-View opens an embedded view (modal or side-panel) that talks
-  to SFM's `/device/*` endpoints over the network.
-- Live view: device clock, battery, memory, current monitor status.
-- Actions: start/stop monitoring, push compliance config changes, pull
-  fresh events, run a sensor self-check, change call-home settings.
-- Audit log: every connect / action recorded in SDM for the unit
-  history.
-
-Implementation steps (concrete):
-- [ ] **SFM authentication & authorization layer.**  Today `/device/*`
-      endpoints are unauthenticated — anyone on the network can call
-      them.  Need at minimum a token-based auth, ideally with a "who
-      can connect to which units" mapping.  Hard prerequisite for
-      letting Terra-View users into the control surface.
-- [ ] **Terra-View "Connect to Device" entry point** on the unit
-      detail page.  Renders only when unit has connection info on file
-      and the user has permission.
-- [ ] **Embedded live-monitor view** in Terra-View — equivalent to
-      `seismo_lab.py`'s Bridge tab, but in the browser.  Polls SFM's
-      `/device/monitor/status` on an interval; sends start/stop via
-      `/device/monitor/{start,stop}`.
-- [ ] **Action history** — every connect / push / action call records
-      a row in `unit_history`, viewable on the unit detail page.
-- [ ] **Series IV live-device support in SFM** — currently `/device/*`
-      only supports MiniMate Plus.  Blocks "Connect to Device" for
-      Thor units until done.  Depends on Thor wire-protocol capture
-      and a `micromate/` parallel of the `minimateplus/` modules.
-
-### High-impact (unblocks product features)
-
-- [ ] **Series III waveform body codec reverse-engineering.**  The 5A bulk-stream body is some kind of compressed/encoded format (not raw int16 LE as previously assumed — see §7.6.1 retraction in `docs/instantel_protocol_reference.md`).  Structural framing is ~50% decoded on branch `claude/codec-re-cBGNe` (tagged-block walker, segment counters); per-byte sample mapping is still open.  Until this lands, the in-app waveform viewer renders garbage and BW-import peak values fall back to `_peaks_from_samples()` saturation noise.  Workaround: pair every BW-imported event with its `_ASCII.TXT` so the device-authoritative peaks land in the DB regardless of codec.
-- [x] **Series IV (Thor IDF) binary codec reverse-engineering.** ✅ v0.21.0 — `micromate/idf_file.read_idf_file()` decodes both IDFW (waveform body at offset `0x0f1f`, reusing `decode_waveform_v2()`; 87–99% sample fidelity on quiet events) and IDFH (dedicated segment-based decoder: all 859 corpus files decode, 181,071 intervals, peaks within ~1.8% of sidecar values).  `WaveformStore.save_imported_idf` now also projects parsed Thor data into a `bw_report` block via `micromate/idf_to_bw_report.py` so Thor events render in the existing Event Report PDF pipeline without a separate renderer.
-- [ ] **In-app waveform viewer accuracy.**  Depends on Series III codec decode.  Plot.v1 JSON pipeline + viewer skeleton already exist; will start showing real waveforms automatically once `_decode_a5_waveform` produces correct samples.  Series IV waveforms come online when the IDF codec lands.
-- [ ] **Series IV live-device support.**  Once the IDF binary is decoded, extend `micromate/` with `transport.py` / `framing.py` / `protocol.py` / `client.py` mirroring the `minimateplus/` package layout — depends on capturing Thor's wire protocol (TCP / RS-232 captures TBD).
-- [ ] **Terra-view integration** — seismo-relay router, unit detail page, VISON-style event listing.
-- [ ] **Vibration summary reports** — highest legit PPV per project → Word doc (false-trigger filtering first).
-
-### BW ASCII report parser enhancements (built in v0.16.0)
-
-- [x] **PPV field misses on certain TXT formats.** ✅ v0.20.0 — root cause was the `OORANGE` (Out Of Range) saturation marker that BW writes when a channel exceeds its full-scale; `_parse_number()` returned None for the non-numeric value.  Parser now substitutes `geo_range_ips` as a lower bound + sets `ppv_saturated` flag.  All 5 prod events (T190LD5Q.LK0W, T438L713.RY0W, K557L3YM.OE0W, + 2 others) now parse cleanly.
-- [x] **Histogram-specific structural fields.** ✅ v0.20.0 — `Histogram Start/Stop Time+Date`, `Number of Intervals`, `Interval Size`, per-channel `Peak Time` + `Peak Date`, and `Peak Vector Sum Date` all parse now.  Land in the sidecar's `bw_report.histogram` block.
-- [ ] **Histogram interval bin-table parsing.**  Trailing 792-row table (per-interval Peak/Freq per channel + MicL) in histogram TXTs is unparsed.  Probably too big for the sidecar JSON; may want a separate `.histogram.h5` companion file.
-- [x] **`>100 Hz` value parsing.** ✅ v0.20.0 — parser now mirrors the OORANGE pattern: stores 100.0 on `zc_freq_hz` + sets `zc_freq_above_range` flag.  PDF + both modals render `>100 Hz` instead of `—`.
-
-### Ingestion gaps
-
-- [ ] **MLG forwarding.**  `series3-watcher` forwards event binaries + their `_ASCII.TXT` reports, but skips `.MLG` per-unit monitor log files entirely.  Adding an `POST /db/import/mlg_file` endpoint + watcher scan path would populate `monitor_log` for non-ACH-routed units (coverage queries, "was this unit monitoring on date X" lookups).
-- [ ] **0C-record raw bytes persistence in the sidecar.**  Currently on branch `claude/codec-re-cBGNe` as commit `a187124`; cherry-pick if useful as a standalone fix.  Preserves the 210-byte 0C record under `extensions.raw_records.waveform_record_b64` so future field-offset analysis (Peak Acceleration / Time of Peak / etc. — the fields BW computes client-side from samples) can run offline.
-
-### Operational
-
-- [ ] **`series3-watcher` file archive manager** — 90-day-old events moved to `<watch_folder>_archive/<year>/<month>/` subfolders.  Plan drafted in `claude/codec-re-cBGNe`'s plan-mode session; awaiting a 5-minute test on whether Blastware UI walks subfolders before any code lands (determines layout: in-place subfolders vs sibling archive).
-- [ ] **Compliance config encoder** — build raw write payloads from a `ComplianceConfig` object.
-- [ ] **Modem manager** — push RV50/RV55 configs via Sierra Wireless API.
-- [ ] **Call Home dial_string write support** (requires DLE escaping for embedded control characters).
-- [ ] **Histogram mode recording support** (5A stream analysis for mode 0x03 — separate from histogram ASCII parsing above).
-
-### Test coverage
-
-- [ ] Verify 30-sec event download — body may exceed `0xFFFF` and force the device into a different `end_key` encoding (none of the 2/3/10-sec test cases hit this boundary).
-- [ ] Histogram mode (0x03) write via SFM — confirmed working for Single Shot / Continuous / Histogram+Continuous; Histogram (0x03) needs a live test from a non-Histogram starting state.
-
-### Lower-priority cleanups
-
-- [ ] Compliance write anchor-9 cleanup — when changing recording_mode via SFM, a spurious `0x10` may persist after Histogram→other mode transitions.  Doesn't affect device operation but differs from BW's byte-perfect output.
-- [ ] Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring).
-- [ ] Call Home — map time slots 3/4 offsets; confirm `modem_power_relay_enabled`.
-- [ ] RV55 DCD/DTR — newer RV55 firmware doesn't assert DCD by default; units don't resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred).
-- [ ] **NULL-timestamp duplicate-row dedup.**  A small handful of events (2 known on prod as of 2026-05-22) have `events.timestamp IS NULL` because the codec couldn't extract a timestamp from the binary footer.  The `UNIQUE(serial, timestamp)` constraint doesn't fire on `NULL` (SQL semantics: `NULL ≠ NULL`), so every `--force` backfill INSERTs a new row instead of UPSERTing the existing one.  Cleanup: a one-shot SQL query that keeps only the newest row per `(serial, blastware_filename)` and deletes the rest.  Longer-term: extend the unique key to `(serial, COALESCE(timestamp, blastware_filename))` or reject inserts with NULL timestamp.
-- [ ] **Histogram body sub-format with `byte[5] != 0`.**  ~3 events on prod (`T190LD5Q.LD0H`, `O121L4L1.GU0H`) use a histogram body my walker doesn't recognize — the first block has `byte[5] = 0x01` or `0x07` instead of `0x00`, and the entire body lacks the `1e 0a 00 00` tail signature.  Codec returns 0 valid blocks; their DB PVS comes from the bw_report ASCII overlay (which BW computed from the same binary, so the DB columns are correct).  Only the `.h5` waveform plot is empty.  Cracking the sub-format would unlock the plot.  Needs binary+ASCII pairs from a few `byte[5]!=0` events; same RE approach as the K558 case.
-- [ ] **Histogram body sub-format with `byte[5] == 0x00` but undecodable.**  Observed 2026-05-28 on BE17353 (S353) events: `S353L4H2.FZ0H`, `S353L4H2.P00H`, `S353L4H3.7O0H`, `S353L4H3.E10H`.  Body starts `00 00 00 01 0a 00 XX 00 ...` which LOOKS like a valid histogram block header (marker 0x000a at byte[4:6] ✓, byte[5]=0x00 normal-format ✓), but the walker finds zero data blocks across the whole body.  Likely an extra header before the block stream OR a different tail signature than `1e 0a 00 00`.  Smaller body lengths (1900-2100 bytes) suggest these may be short-recording histogram variants.  Same operational impact as the byte[5]!=0 case: event ingests cleanly, DB peaks correct via bw_report overlay, only the chart is empty.  Worth dumping a hex view of one body to diagnose.
-- [ ] **Sensor-check waveform extraction from the BW binary.**  BW's Event Report PDFs include a narrow panel on the right side of the waveform plot showing each channel's response to the sensor self-check signal (a damped sinusoid for geo, sawtooth-at-test-freq for mic).  Our parser captures the test RESULTS (`test_freq_hz`, `test_ratio`, `test_amplitude_mv`, `test_results` pass/fail) and the PDF + modal display them as text — but BW's per-sample sensor-check waveform isn't accessible to us today.  Two paths to add it:  (a) RE the binary to find where the sensor-check samples are stored — could be a section before STRT, after the footer, or in a separate sub-record; protocol reference doesn't currently mention it.  (b) If samples aren't in the binary, synthesize a representative waveform from the test parameters (damped sinusoid at `test_freq_hz` with damping from `test_ratio`).  Path (a) is the honest answer; path (b) is decorative.  Until either lands, the text-only sensor-check display in the report is fine.

analysis/README.md

@@ -1,66 +0,0 @@
-# analysis/ — exploratory scripts for waveform-body RE
-
-**These are scratch.** Run them, read them, copy them, but don't trust
-them as documentation.  When a finding is verified it gets promoted
-to `minimateplus/waveform_codec.py` and `tests/test_waveform_codec.py`;
-when it's wrong it stays here as a fossil.
-
-Authoritative status lives in:
-
-- `docs/waveform_codec_re_status.md` (current truth, working note)
-- `minimateplus/waveform_codec.py` (verified implementation + docstring)
-- `tests/test_waveform_codec.py` (regression locks against fixtures)
-
----
-
-## Still useful
-
-| File | What it does |
-|---|---|
-| `load_bundle.py` | Fixture loader.  Parses BW binary + ASCII TXT into a `Bundle` dataclass with samples, metadata, body bytes.  Used by most other scripts here. |
-| `verify_tran.py` | Verifies `decode_tran_initial` against fixture ground truth across all events.  Useful when you change the decoder and want a quick sanity check. |
-| `inspect_5_11.py` | Inspects the 5-11-26 high-amplitude bundle's body structure, prints metadata, peaks, and block counts. |
-| `walk_5_11.py` | Walks blocks for the 5-11-26 bundle and prints offset/tag/length/data. |
-| `seg1_blocks.py` | Dumps all blocks in segment 1 of each event.  The starting point for cracking multi-segment Tran continuation. |
-| `full_tran.py` | Multi-segment Tran decoder attempt (broken — diverges at sample ~512).  Useful as a starting scaffold for the next experiment. |
-| `multi_segment.py` | Earlier multi-segment attempt with different segment-header consumption strategies.  Records what didn't work. |
-| `test_rle.py` | Tests `00 NN` interpretation as zero-RLE with different divisor values.  Documents how the RLE rule was confirmed. |
-
-## Superseded — keep for archaeology
-
-| File | Superseded by |
-|---|---|
-| `walk_v2.py` … `walk_v5.py` | `walk_v6.py` and ultimately `minimateplus/waveform_codec.walk_body`.  Each version represents one round of refinement.  Don't read in isolation — read the diff between them to see what was learned. |
-| `walk_chunks.py` | `walk_v6.py` / production walker |
-| `decode_v1.py` | First naive decoder attempt.  Wrong but readable. |
-
-## Pure exploration — read if curious
-
-| File | What it explored |
-|---|---|
-| `inspect_body.py` | Byte-frequency stats per event.  Established that bytes 0x00 / 0x10 dominate. |
-| `find_blocks.py` | Searched for repeating 2-byte tag patterns. |
-| `find_signal_runs.py` | Searched for stretches of bytes that "look like a smooth signal" (small inter-byte deltas).  Found the `20 NN` literal blocks. |
-| `dump_head.py`, `dump_trailer.py`, `dump_around.py` | Hex dumpers at various body positions. |
-| `compare_cd.py` | Byte-diff between event-c and event-d (same length, similar signal).  Used to identify structural vs data bytes. |
-| `brute_force.py` | Tested 96 combinations of channel-permutation × nibble-order × sign-convention × init-from-header on the quiet bundle.  All failed because the quiet bundle had T[0]=T[1]=0, making the preamble undetectable. |
-| `try_nibbles.py`, `try_layouts.py` | Earlier channel-interleaving hypotheses.  All wrong. |
-| `test_tran_continue.py` | Test of "Tran continues uninterrupted across `30 04` blocks" hypothesis.  Disproven. |
-
----
-
-## Adding new scripts
-
-If you're picking up the codec work, feel free to add new scripts here.
-Suggested conventions:
-
-- Start the filename with what you're testing: `test_<hypothesis>.py`,
-  `verify_<piece>.py`, `inspect_<region>.py`.
-- Print enough output that the reader can see exactly which events
-  match / diverge and where.
-- When a finding is solid, move the verified logic to
-  `minimateplus/waveform_codec.py` and add a regression test in
-  `tests/test_waveform_codec.py` — don't leave the truth only in
-  this directory.
-- If a script is fully superseded, leave it in place (don't delete) —
-  the fossil record is useful when re-evaluating hypotheses later.

analysis/brute_force.py

@@ -1,93 +0,0 @@
-"""Brute-force test channel permutations / nibble orders on event-d (simplest signal)."""
-import sys
-import itertools
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-from minimateplus.waveform_codec import walk_body
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def decode(body, channel_perm, nibble_order, sign_mode, init_from_header):
-    """Try one decoder configuration on event-d. Returns first 8 cumulative samples per channel."""
-    blocks = walk_body(body)
-    # Initial values from bytes [4:7] if init_from_header else 0
-    if init_from_header:
-        init = [body[4] if body[4] < 128 else body[4] - 256,
-                body[5] if body[5] < 128 else body[5] - 256,
-                body[6] if body[6] < 128 else body[6] - 256,
-                0]
-    else:
-        init = [0, 0, 0, 0]
-    cur = list(init)
-    out = [[init[0]], [init[1]], [init[2]], [init[3]]]  # sample 0 = init
-    nibble_idx = 0  # within delta stream; channel = channel_perm[nibble_idx % 4]
-
-    # Walk only the 10 NN data blocks
-    for blk in blocks:
-        if blk.tag_hi != 0x10:
-            continue
-        for byte in blk.data:
-            if nibble_order == 'high_first':
-                nib1, nib2 = (byte >> 4) & 0xF, byte & 0xF
-            else:
-                nib1, nib2 = byte & 0xF, (byte >> 4) & 0xF
-            for nib in (nib1, nib2):
-                if sign_mode == 'signed':
-                    delta = s4(nib)
-                else:
-                    delta = nib
-                ch = channel_perm[nibble_idx % 4]
-                cur[ch] += delta
-                if (nibble_idx + 1) % 4 == 0:
-                    out[0].append(cur[0])
-                    out[1].append(cur[1])
-                    out[2].append(cur[2])
-                    out[3].append(cur[3])
-                nibble_idx += 1
-                if len(out[0]) >= 16:
-                    return out
-    return out
-
-
-def best_match(pred, truth, n=10):
-    """Sum of squared differences in first n samples."""
-    n = min(n, len(pred), len(truth))
-    return sum((pred[i] - truth[i])**2 for i in range(n))
-
-
-def main():
-    b = load_bundle("event-d")
-    # truth in 16-count units
-    tr = {ch: [round(v * 200) for v in b.samples[ch]] for ch in ("Tran", "Vert", "Long")}
-
-    print("Truth event-d first 10 samples:")
-    for ch in ("Tran", "Vert", "Long"):
-        print(f"  {ch}: {tr[ch][:10]}")
-
-    # Test 96 combinations
-    best = []
-    for perm in itertools.permutations([0, 1, 2, 3]):
-        for nibble_order in ('high_first', 'low_first'):
-            for sign in ('signed', 'unsigned'):
-                for init_h in (False, True):
-                    decoded = decode(b.body, perm, nibble_order, sign, init_h)
-                    # Score as TVL channel-sum
-                    score = sum(
-                        best_match(decoded[i], tr[ch], n=10)
-                        for i, ch in enumerate(("Tran", "Vert", "Long"))
-                        if i < 3
-                    )
-                    label = f"perm={perm} nib={nibble_order[:1]} sign={sign[:3]} init={init_h}"
-                    best.append((score, label, decoded))
-
-    best.sort(key=lambda x: x[0])
-    print(f"\nTop 10 configurations:")
-    for s, lbl, dec in best[:10]:
-        print(f"  score={s:>5}  {lbl}  T={dec[0][:8]}  V={dec[1][:8]}  L={dec[2][:8]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/compare_cd.py

@@ -1,42 +0,0 @@
-"""Compare event-c and event-d (same N_samples) to find header vs data bytes."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def main():
-    bc = load_bundle("event-c")
-    bd = load_bundle("event-d")
-
-    # Compare prefixes
-    nc, nd = len(bc.body), len(bd.body)
-    n = min(nc, nd)
-    diffs = []
-    for i in range(n):
-        if bc.body[i] != bd.body[i]:
-            diffs.append(i)
-    print(f"event-c body={nc}, event-d body={nd}")
-    print(f"Total diffs (first {n}): {len(diffs)}")
-
-    # Show common prefix
-    same_prefix = 0
-    for i in range(n):
-        if bc.body[i] == bd.body[i]:
-            same_prefix += 1
-        else:
-            break
-    print(f"Common prefix length: {same_prefix}")
-    print(f"event-c prefix: {bc.body[:same_prefix].hex(' ')}")
-
-    # Look for runs of common bytes
-    print(f"\nFirst 32 diff positions: {diffs[:32]}")
-
-    # Show the "diff fingerprint" of the first 100 bytes
-    print(f"\n  pos    c     d")
-    for i in range(0, 100):
-        marker = " " if bc.body[i] == bd.body[i] else "*"
-        bd_b = bd.body[i] if i < nd else None
-        print(f"  {i:>3}  {bc.body[i]:02x}{marker}  {bd_b:02x}" if bd_b is not None else f"  {i:>3}  {bc.body[i]:02x}{marker}")
-
-if __name__ == "__main__":
-    main()

analysis/decode_v1.py

@@ -1,99 +0,0 @@
-"""
-Decoder v1: nibble-pair signed deltas in 10 NN blocks, 4-channel round-robin.
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def walk_blocks(body, start):
-    i = start
-    blocks = []
-    while i + 1 < len(body):
-        t0, t1 = body[i], body[i + 1]
-        if t0 == 0x10 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 // 2 + 2
-            data = bytes(body[i + 2 : i + length])
-            blocks.append(("10", t1, data))
-            i += length
-        elif t0 == 0x20 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 + 2
-            data = bytes(body[i + 2 : i + length])
-            blocks.append(("20", t1, data))
-            i += length
-        elif t0 == 0x00 and t1 % 4 == 0:
-            blocks.append(("00", t1, b""))
-            i += 2
-        elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0x10:
-            length = t1 * 4
-            data = bytes(body[i + 2 : i + length])
-            blocks.append(("30", t1, data))
-            i += length
-        elif t0 == 0x40 and t1 == 0x02:
-            length = 20
-            data = bytes(body[i + 2 : i + length])
-            blocks.append(("40", t1, data))
-            i += length
-        else:
-            blocks.append(("??", t0, bytes(body[i:i+8])))
-            break
-    return blocks
-
-
-def decode_v1(body, start, n_samples):
-    """Decode by accumulating nibble-pair deltas from all 10 NN blocks."""
-    blocks = walk_blocks(body, start)
-    # 4 channels: T, V, L, M
-    cur = [0, 0, 0, 0]
-    out = [[], [], [], []]
-    sample_index = 0  # how many sample-sets emitted
-
-    for typ, NN, data in blocks:
-        if typ == "10":
-            # 2 nibbles per byte, round-robin TVLM
-            for byte in data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    ch = sample_index % 4
-                    cur[ch] += s4(nib)
-                    out[ch].append(cur[ch])
-                    sample_index = (sample_index + 1) // 4 * 4 + (sample_index + 1) % 4  # ?
-                    sample_index += 1
-                    # We emit per-nibble, but the structure is unclear
-        elif typ == "20":
-            # int8 absolute or delta?
-            for byte in data:
-                v = byte if byte < 128 else byte - 256
-                ch = sample_index % 4
-                cur[ch] = v  # treat as absolute
-                out[ch].append(cur[ch])
-                sample_index += 1
-    return out
-
-
-def main():
-    b = load_bundle("event-c")
-    body = b.body
-    truth_T = [round(v * 200) for v in b.samples["Tran"]]
-    truth_V = [round(v * 200) for v in b.samples["Vert"]]
-    truth_L = [round(v * 200) for v in b.samples["Long"]]
-
-    # Find start
-    for s in range(15):
-        if body[s] == 0x10 and body[s+1] % 4 == 0 and 0 < body[s+1] <= 0xFC:
-            start = s
-            break
-
-    blocks = walk_blocks(body, start)
-    # Print block-by-block what's in each
-    print(f"Total blocks: {len(blocks)}")
-    bytes_processed = 0
-    for typ, NN, data in blocks[:30]:
-        print(f"  type={typ} NN=0x{NN:02x} data_len={len(data)} data_hex={data[:32].hex(' ')}{'...' if len(data) > 32 else ''}")
-
-
-if __name__ == "__main__":
-    main()

analysis/dump_around.py

@@ -1,27 +0,0 @@
-"""Dump body bytes around a specific offset."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def dump_around(name: str, center: int, radius: int = 96):
-    b = load_bundle(name)
-    body = b.body
-    start = max(0, center - radius)
-    end = min(len(body), center + radius)
-    print(f"\n=== {name} body[{start}:{end}] (full body={len(body)}) ===")
-    for i in range(start, end, 32):
-        row = body[i:i+32]
-        marker = "  <-- center" if i <= center < i+32 else ""
-        print(f"  +{i:>5}  {row.hex(' ')}{marker}")
-
-
-def main():
-    # Look at the trailer transitions
-    trailer_starts = {"event-a": 7047, "event-b": 6475, "event-c": 4043, "event-d": 3941}
-    for name, off in trailer_starts.items():
-        dump_around(name, off, 96)
-
-
-if __name__ == "__main__":
-    main()

analysis/dump_head.py

@@ -1,18 +0,0 @@
-"""Dump the START of each body in 32-byte rows."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def main():
-    for name in ("event-a", "event-c"):
-        b = load_bundle(name)
-        body = b.body
-        print(f"\n=== {name} body[0:512] (full body={len(body)}, samples={len(b.samples['Tran'])}) ===")
-        for i in range(0, min(512, len(body)), 32):
-            row = body[i:i+32]
-            print(f"  +{i:>5}  {row.hex(' ')}")
-
-
-if __name__ == "__main__":
-    main()

analysis/dump_trailer.py

@@ -1,24 +0,0 @@
-"""Dump body bytes split into 32-byte rows starting from `start_offset`."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def dump(body: bytes, name: str, start: int, n_rows: int = 30):
-    print(f"\n=== {name} body[{start}:] (full body={len(body)}) ===")
-    end = min(start + 32 * n_rows, len(body))
-    for i in range(start, end, 32):
-        row = body[i:i+32]
-        print(f"  +{i:>5}  {row.hex(' ')}")
-
-
-def main():
-    for name in ("event-a", "event-b", "event-c", "event-d"):
-        b = load_bundle(name)
-        # Print the LAST ~600 bytes of the body to see the tail structure
-        start = max(0, len(b.body) - 32 * 12)
-        dump(b.body, name, start, 12)
-
-
-if __name__ == "__main__":
-    main()

analysis/find_blocks.py

@@ -1,41 +0,0 @@
-"""Search for structural repetition in the body bytes."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def find_pattern_offsets(body: bytes, pattern: bytes, max_count=20):
-    out = []
-    i = 0
-    while True:
-        i = body.find(pattern, i)
-        if i < 0:
-            break
-        out.append(i)
-        i += 1
-        if len(out) >= max_count:
-            break
-    return out
-
-
-def main():
-    for name in ("event-a", "event-b", "event-c", "event-d"):
-        b = load_bundle(name)
-        body = b.body
-        print(f"\n=== {name} (body={len(body)}, N_samples={len(b.samples['Tran'])}) ===")
-
-        # Try to find repeating substructures (look for 4-byte 0x10-prefixed markers)
-        for prefix in [b"\x10\x10", b"\x10\x04", b"\x10\x08", b"\x10\x0c", b"\x10\x18",
-                       b"\x10\x14", b"\x10\x20", b"\x10\x40", b"\x10\x80", b"\x10\x00",
-                       b"\x10\x01", b"\x10\x03", b"\x10\xf0", b"\xf1\x10", b"\x00\x10",
-                       b"\x40\x02", b"\x20\x04", b"\x30\x04", b"\x30\x08", b"\x00\x1a"]:
-            offs = find_pattern_offsets(body, prefix, max_count=200)
-            if 1 <= len(offs) <= 1000:
-                # Print first 10 offsets
-                first = offs[:6]
-                last = offs[-3:]
-                print(f"  '{prefix.hex()}' x{len(offs):>4}  first={first} last={last}")
-
-
-if __name__ == "__main__":
-    main()

analysis/find_signal_runs.py

@@ -1,34 +0,0 @@
-"""Find body byte ranges that look like absolute int8 sample data (smooth waveform)."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def looks_like_smooth_int8(buf):
-    """Convert bytes to int8 and check if successive deltas are small (waveform-like)."""
-    if len(buf) < 8:
-        return 0.0
-    vals = [b if b < 128 else b - 256 for b in buf]
-    diffs = [abs(vals[i+1] - vals[i]) for i in range(len(vals)-1)]
-    avg_diff = sum(diffs) / len(diffs)
-    return avg_diff
-
-
-def main():
-    for name in ("event-a", "event-c"):
-        b = load_bundle(name)
-        body = b.body
-        # Scan with sliding window of 64 bytes; find segments where the bytes look like a smooth wave
-        win = 64
-        scores = []
-        for i in range(len(body) - win):
-            scores.append((i, looks_like_smooth_int8(body[i:i+win])))
-        # Lowest avg_diff means smoothest
-        scores.sort(key=lambda x: x[1])
-        print(f"\n=== {name} (body={len(body)}) — smoothest 10 windows ===")
-        for off, s in scores[:10]:
-            print(f"  +{off:>5}  avg_diff={s:.2f}  bytes={body[off:off+24].hex(' ')}")
-
-
-if __name__ == "__main__":
-    main()

analysis/full_tran.py

@@ -1,76 +0,0 @@
-"""Full Tran decoder: continues across segment headers using T_delta from header bytes [0:2]."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def decode_full_tran(body):
-    if len(body) < 7 or body[0:3] != b"\x00\x02\x00":
-        return None
-    T0 = int.from_bytes(body[3:5], "big", signed=True)
-    T1 = int.from_bytes(body[5:7], "big", signed=True)
-
-    i = 7
-    while i + 1 < len(body) and body[i] not in (0x00, 0x10, 0x20, 0x30, 0x40):
-        i += 1
-
-    blocks = walk_body(body, i)
-    T = [T0, T1]
-    cur = T1
-    for blk in blocks:
-        if blk.tag_hi == 0x40:
-            # Segment header carries 2 T deltas (int16 BE each) at bytes [0:2] and [2:4]
-            if len(blk.data) >= 4:
-                delta1 = int.from_bytes(blk.data[0:2], "big", signed=True)
-                cur += delta1
-                T.append(cur)
-                delta2 = int.from_bytes(blk.data[2:4], "big", signed=True)
-                cur += delta2
-                T.append(cur)
-        elif blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += s4(nib)
-                    T.append(cur)
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur += i8(byte)
-                T.append(cur)
-        elif blk.tag_hi == 0x00:
-            for _ in range(blk.tag_lo):
-                T.append(cur)
-        # 30 NN: skip for now
-    return T
-
-
-def main():
-    for stem in ("M529LL1L.V70", "M529LL1L.JQ0", "M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        truth_T = [round(v*200) for v in samples["Tran"]]
-        n_truth = len(truth_T)
-
-        decoded = decode_full_tran(body)
-        n = min(len(decoded), n_truth)
-        matches = sum(1 for i in range(n) if decoded[i] == truth_T[i])
-        div_at = -1
-        for i in range(n):
-            if decoded[i] != truth_T[i]:
-                div_at = i
-                break
-        print(f"{stem}: decoded={len(decoded)}, truth={n_truth}, matches={matches}/{n}, first div={div_at}")
-
-
-if __name__ == "__main__":
-    main()

analysis/inspect_5_11.py

@@ -1,50 +0,0 @@
-"""Quick inspection of the new high-amplitude events."""
-import os, re, sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-ROOT = "tests/fixtures/5-11-26"
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        bin_path = os.path.join(ROOT, stem)
-        txt_path = bin_path + ".TXT"
-        with open(bin_path, "rb") as f:
-            raw = f.read()
-        body = raw[43:-26]
-        meta, samples = _parse_txt(txt_path)
-        n = len(samples["Tran"])
-
-        print(f"\n=== {stem} ===")
-        print(f"  file={len(raw)}, body={len(body)}, N_samples={n}")
-        print(f"  rectime={meta.get('Record Time')} pretrig={meta.get('Pre-trigger Length')}")
-        print(f"  PPV(T,V,L)={meta.get('Tran PPV')} / {meta.get('Vert PPV')} / {meta.get('Long PPV')}")
-        # Show first few non-trivial samples
-        print(f"  First 5 truth samples (in/s):")
-        for i in range(5):
-            print(f"    T={samples['Tran'][i]:8.3f}  V={samples['Vert'][i]:8.3f}  "
-                  f"L={samples['Long'][i]:8.3f}  M={samples['MicL'][i]:8.3f}")
-        # Peak sample positions
-        for ch in ("Tran", "Vert", "Long"):
-            vals = samples[ch]
-            peak_i = max(range(n), key=lambda i: abs(vals[i]))
-            print(f"  {ch}: peak {vals[peak_i]:.3f} at sample {peak_i} (t={peak_i/1024:.3f}s)")
-        # Body structure
-        start = find_data_start(body)
-        blocks = walk_body(body, start)
-        types = {}
-        for b in blocks:
-            types[b.tag_hi] = types.get(b.tag_hi, 0) + 1
-        print(f"  body start={start}, total blocks walked: {len(blocks)}")
-        print(f"  block tag counts: {types}")
-        # How far the walker got
-        if blocks:
-            last = blocks[-1]
-            walked = last.offset + last.length
-            print(f"  walker stopped at offset {walked}/{len(body)} ({100*walked/len(body):.0f}%)")
-
-
-if __name__ == "__main__":
-    main()

analysis/inspect_body.py

@@ -1,23 +0,0 @@
-"""Print raw body hex + byte-distribution stats for one event."""
-from collections import Counter
-import sys
-
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def main():
-    for name in ("event-a", "event-b", "event-c", "event-d"):
-        b = load_bundle(name)
-        body = b.body
-        print(f"\n=== {name} ({len(body)} body bytes) ===")
-        print(f"  STRT: {b.strt.hex()}")
-        print(f"  body[0:64]:   {body[:64].hex()}")
-        print(f"  body[64:128]: {body[64:128].hex()}")
-        print(f"  body[-32:]:   {body[-32:].hex()}")
-        cnt = Counter(body)
-        print(f"  top 16 bytes: {[(f'0x{k:02x}', f'{v/len(body):.2%}') for k,v in cnt.most_common(16)]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/load_bundle.py

@@ -1,144 +0,0 @@
-"""
-load_bundle.py — extract body bytes from BW binary + parse sample columns from TXT.
-
-Used by the codec reverse-engineering scripts in this directory.
-"""
-from __future__ import annotations
-
-import os
-import re
-from dataclasses import dataclass
-
-
-BUNDLE_ROOT = os.path.join(
-    os.path.dirname(__file__), "..", "tests", "fixtures", "decode-re-5-8-26"
-)
-
-
-@dataclass
-class Bundle:
-    name: str
-    bin_path: str
-    txt_path: str
-    bin: bytes
-    body: bytes  # bytes between STRT (43) and footer (last 26)
-    strt: bytes  # 21-byte STRT record
-    samples: dict  # {"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}
-    sample_rate: int
-    rectime_sec: float
-    pretrig_sec: float
-    geo_range_ips: float
-    ppv: dict  # {"Tran": float, "Vert": float, "Long": float}
-    mic_pspl: float
-    serial: str
-
-
-def _parse_txt(path: str) -> dict:
-    with open(path, "r", encoding="utf-8", errors="replace") as f:
-        text = f.read()
-
-    meta = {}
-    samples = {"Tran": [], "Vert": [], "Long": [], "MicL": []}
-
-    # Find header line that starts the columns ("Tran   Vert   Long   MicL").
-    # Then every line after is sample data (4 tab-separated floats).
-    lines = text.splitlines()
-    header_idx = None
-    for i, line in enumerate(lines):
-        if "Tran" in line and "Vert" in line and "Long" in line and "MicL" in line:
-            # The columns header.  Sample lines start a few lines later.
-            header_idx = i
-            break
-    if header_idx is None:
-        raise ValueError(f"no Tran/Vert/Long/MicL header in {path}")
-
-    # Parse meta — quoted lines with "Field : value"
-    for line in lines[:header_idx]:
-        m = re.match(r'^"([^"]+)\s*:\s*([^"]*)"', line.strip())
-        if m:
-            k, v = m.group(1).strip(), m.group(2).strip()
-            meta[k] = v
-
-    # Parse samples
-    for line in lines[header_idx + 1 :]:
-        line = line.strip()
-        if not line:
-            continue
-        parts = re.split(r"\s+", line)
-        if len(parts) < 4:
-            continue
-        try:
-            t = float(parts[0])
-            v = float(parts[1])
-            l = float(parts[2])
-            m = float(parts[3])
-        except ValueError:
-            continue
-        samples["Tran"].append(t)
-        samples["Vert"].append(v)
-        samples["Long"].append(l)
-        samples["MicL"].append(m)
-
-    return meta, samples
-
-
-def load_bundle(name: str) -> Bundle:
-    folder = os.path.join(BUNDLE_ROOT, name)
-    files = os.listdir(folder)
-    bin_name = next(f for f in files if not f.endswith(".TXT"))
-    txt_name = next(f for f in files if f.endswith(".TXT"))
-
-    bin_path = os.path.join(folder, bin_name)
-    txt_path = os.path.join(folder, txt_name)
-
-    with open(bin_path, "rb") as f:
-        binary = f.read()
-
-    # Header is 22 bytes; STRT at [22:43]; footer at last 26 bytes.
-    strt = binary[22:43]
-    body = binary[43:-26]
-
-    meta, samples = _parse_txt(txt_path)
-
-    sample_rate = int(re.search(r"(\d+)", meta.get("Sample Rate", "1024")).group(1))
-    rectime_sec = float(re.search(r"([\d.]+)", meta.get("Record Time", "3.0")).group(1))
-    pretrig_sec = float(re.search(r"-?[\d.]+", meta.get("Pre-trigger Length", "0")).group(0))
-    geo_range_ips = float(re.search(r"([\d.]+)", meta.get("Geo Range", "10.0")).group(1))
-    serial = meta.get("Serial Number", "").strip()
-
-    def _f(s):
-        return float(re.search(r"-?[\d.]+", s).group(0))
-
-    ppv = {
-        "Tran": _f(meta.get("Tran PPV", "0")),
-        "Vert": _f(meta.get("Vert PPV", "0")),
-        "Long": _f(meta.get("Long PPV", "0")),
-    }
-    mic_pspl = _f(meta.get("MicL PSPL", "0"))
-
-    return Bundle(
-        name=name,
-        bin_path=bin_path,
-        txt_path=txt_path,
-        bin=binary,
-        body=body,
-        strt=strt,
-        samples=samples,
-        sample_rate=sample_rate,
-        rectime_sec=rectime_sec,
-        pretrig_sec=pretrig_sec,
-        geo_range_ips=geo_range_ips,
-        ppv=ppv,
-        mic_pspl=mic_pspl,
-        serial=serial,
-    )
-
-
-if __name__ == "__main__":
-    for name in ("event-a", "event-b", "event-c", "event-d"):
-        b = load_bundle(name)
-        n = len(b.samples["Tran"])
-        print(f"{name}: body={len(b.body):>6}  N_samples={n}  rate={b.sample_rate}  "
-              f"rectime={b.rectime_sec}  pretrig={b.pretrig_sec}  range={b.geo_range_ips}  "
-              f"PPV(T,V,L)={b.ppv['Tran']:.3f},{b.ppv['Vert']:.3f},{b.ppv['Long']:.3f}  "
-              f"MicL={b.mic_pspl}")

analysis/multi_segment.py

@@ -1,81 +0,0 @@
-"""Decode Tran across multiple segments by resetting at 40 02 headers."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def decode_full_tran(body):
-    """Decode all Tran samples in the body, walking through segments."""
-    if len(body) < 7 or body[0:3] != b"\x00\x02\x00":
-        return None
-    T0 = int.from_bytes(body[3:5], "big", signed=True)
-    T1 = int.from_bytes(body[5:7], "big", signed=True)
-
-    # Locate first tag
-    i = 7
-    while i + 1 < len(body) and body[i] not in (0x00, 0x10, 0x20, 0x30, 0x40):
-        i += 1
-
-    blocks = walk_body(body, i)
-    T = [T0, T1]
-    cur = T1
-    for bi, blk in enumerate(blocks):
-        if blk.tag_hi == 0x40:
-            # Segment header — try interpreting bytes [0:2] as new T anchor
-            if len(blk.data) >= 2:
-                new_anchor = int.from_bytes(blk.data[0:2], "big", signed=True)
-                # The next sample IS this anchor value, NOT a delta from cur.
-                T.append(new_anchor)
-                cur = new_anchor
-        elif blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += s4(nib)
-                    T.append(cur)
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur += i8(byte)
-                T.append(cur)
-        elif blk.tag_hi == 0x00:
-            # RLE: append NN zero deltas
-            for _ in range(blk.tag_lo):
-                T.append(cur)
-        # 30 NN: skip
-    return T
-
-
-def main():
-    for stem in ("M529LL1L.V70", "M529LL1L.JQ0", "M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        truth_T = [round(v*200) for v in samples["Tran"]]
-        n_truth = len(truth_T)
-
-        decoded = decode_full_tran(body)
-        n = min(len(decoded), n_truth)
-        matches = sum(1 for i in range(n) if decoded[i] == truth_T[i])
-        # Find first divergence
-        div_at = -1
-        for i in range(n):
-            if decoded[i] != truth_T[i]:
-                div_at = i
-                break
-        print(f"{stem}: decoded={len(decoded)}, truth={n_truth}, matches={matches}/{n}, first div={div_at}")
-        if div_at >= 0 and div_at < 30:
-            print(f"  truth around div [{max(0,div_at-3)}:{div_at+8}]: {truth_T[max(0,div_at-3):div_at+8]}")
-            print(f"  pred  around div [{max(0,div_at-3)}:{div_at+8}]: {decoded[max(0,div_at-3):div_at+8]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/seg1_blocks.py

@@ -1,28 +0,0 @@
-"""Dump all blocks in segment 1 of each event with their data."""
-import sys
-sys.path.insert(0, ".")
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1L.JQ0", "M529LL1L.V70"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        blocks = walk_body(body, find_data_start(body))
-
-        # Find segment 1 (between first and second 40 02)
-        seg40_indices = [i for i, b in enumerate(blocks) if b.tag_hi == 0x40]
-        if len(seg40_indices) < 2:
-            print(f"\n{stem}: only {len(seg40_indices)} segment headers found")
-            seg1_blocks = blocks[seg40_indices[0]:] if seg40_indices else []
-        else:
-            seg1_blocks = blocks[seg40_indices[0]:seg40_indices[1]+1]
-        print(f"\n=== {stem} segment 1 ({len(seg1_blocks)} blocks) ===")
-        for b in seg1_blocks[:25]:
-            tag = f"{b.tag_hi:02x}{b.tag_lo:02x}"
-            print(f"  off={b.offset:>5} {tag} NN=0x{b.tag_lo:02x}({b.tag_lo:>3}) len={b.length:>3}  data={b.data[:16].hex(' ')}{'...' if len(b.data)>16 else ''}")
-
-
-if __name__ == "__main__":
-    main()

analysis/test_30nn_12bit.py

@@ -1,195 +0,0 @@
-"""Test 12-bit signed packed deltas hypothesis for 30 NN blocks across all loud events.
-
-For each 30 NN block in each event, identify what samples it should cover
-(based on the cumulative delta count up to that point) and compare the
-truth deltas against various 12-bit packing schemes.
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-CHANNEL_ORDER = ["Vert", "Long", "MicL", "Tran"]  # rotation after initial T
-
-
-def s12(v):
-    """Sign-extend a 12-bit unsigned value to signed int."""
-    return v if v < 0x800 else v - 0x1000
-
-
-def unpack_12bit_be(data):
-    """4 deltas in 6 bytes, BE order: byte[0:1.5], byte[1.5:3], byte[3:4.5], byte[4.5:6]."""
-    # bits 0..47 (MSB-first), split into 4 × 12-bit
-    val = int.from_bytes(data, "big")
-    out = []
-    for i in range(4):
-        d = (val >> (12 * (3 - i))) & 0xFFF
-        out.append(s12(d))
-    return out
-
-
-def unpack_12bit_le(data):
-    """4 deltas in 6 bytes, LE order: bytes packed as 2 × 24-bit groups."""
-    out = []
-    # First 3 bytes contain 2 deltas
-    b0, b1, b2 = data[0], data[1], data[2]
-    d0 = b0 | ((b1 & 0x0F) << 8)
-    d1 = (b1 >> 4) | (b2 << 4)
-    out.append(s12(d0))
-    out.append(s12(d1))
-    # Next 3 bytes contain 2 more deltas
-    b3, b4, b5 = data[3], data[4], data[5]
-    d2 = b3 | ((b4 & 0x0F) << 8)
-    d3 = (b4 >> 4) | (b5 << 4)
-    out.append(s12(d2))
-    out.append(s12(d3))
-    return out
-
-
-def unpack_12bit_be_per_triplet(data):
-    """4 deltas as 2 triplets of (high4, low8) BE within each 3-byte group."""
-    out = []
-    b0, b1, b2 = data[0], data[1], data[2]
-    d0 = (b0 << 4) | (b1 >> 4)
-    d1 = ((b1 & 0x0F) << 8) | b2
-    out.append(s12(d0))
-    out.append(s12(d1))
-    b3, b4, b5 = data[3], data[4], data[5]
-    d2 = (b3 << 4) | (b4 >> 4)
-    d3 = ((b4 & 0x0F) << 8) | b5
-    out.append(s12(d2))
-    out.append(s12(d3))
-    return out
-
-
-def truth_deltas_for_block(blocks, block_idx, event_truth, channel):
-    """For a 30 NN block at block_idx, determine which samples it covers and
-    return the truth deltas for those samples.
-
-    Walks through all blocks before block_idx (within the same segment) and
-    counts how many deltas have been emitted for *channel*, starting from the
-    segment's anchor pair.
-    """
-    # Find the segment header that contains this block.
-    seg_header_idx = None
-    for j in range(block_idx, -1, -1):
-        if blocks[j].tag_hi == 0x40:
-            seg_header_idx = j
-            break
-    if seg_header_idx is None:
-        # block is in the initial T segment; samples count from sample 2.
-        first_sample_in_segment = 2
-    else:
-        # Anchor pair covers samples [N, N+1] for some N.  Subsequent deltas
-        # are samples [N+2, N+2+1, ...].  We don't actually need to know N
-        # for this test — just the relative position within the segment.
-        first_sample_in_segment = 2  # anchor=0,1; deltas start at 2
-
-    # Count deltas from segment-data start to block_idx.
-    delta_count = 0
-    start_block = seg_header_idx + 1 if seg_header_idx is not None else 0
-    for j in range(start_block, block_idx):
-        blk = blocks[j]
-        if blk.tag_hi == 0x10:
-            delta_count += blk.tag_lo  # NN nibbles = NN deltas
-        elif blk.tag_hi == 0x20:
-            delta_count += blk.tag_lo  # NN int8 deltas
-        elif blk.tag_hi == 0x00:
-            delta_count += blk.tag_lo  # RLE zero deltas
-    # Now the 30 NN block carries NN deltas.
-    nn = blocks[block_idx].tag_lo
-    # First sample affected: segment first_sample + delta_count.
-    # But we ALSO need to know which segment this is, since the segment maps
-    # to a specific channel and a specific starting absolute sample index.
-    return first_sample_in_segment + delta_count, nn
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1L.JQ0", "M529LL1L.V70",
-                 "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        blocks = walk_body(body, find_data_start(body))
-        seg_idx = [i for i, b in enumerate(blocks) if b.tag_hi == 0x40]
-
-        # Find all 30 NN blocks in DATA section (not trailer).
-        thirty_blocks = []
-        for bi, b in enumerate(blocks):
-            if b.tag_hi != 0x30:
-                continue
-            # Determine which segment this is in
-            seg_num = None
-            for k, hi in enumerate(seg_idx):
-                next_hi = seg_idx[k + 1] if k + 1 < len(seg_idx) else len(blocks)
-                if hi < bi < next_hi:
-                    seg_num = k
-                    break
-            if seg_num is None and seg_idx and bi < seg_idx[0]:
-                seg_num = -1  # initial T segment
-            thirty_blocks.append((bi, b, seg_num))
-
-        if not thirty_blocks:
-            continue
-
-        print(f"\n=== {stem} ===")
-        for bi, b, seg_num in thirty_blocks:
-            # Channel for this segment
-            if seg_num == -1:
-                channel = "Tran"
-                seg_label = "initial T"
-            else:
-                channel = CHANNEL_ORDER[seg_num % 4]
-                seg_label = f"seg {seg_num}"
-
-            # Count deltas before this block within the same segment.
-            seg_header_idx = seg_idx[seg_num] if seg_num >= 0 else -1
-            start_block = seg_header_idx + 1 if seg_header_idx >= 0 else 0
-            delta_count = 0
-            for j in range(start_block, bi):
-                blk = blocks[j]
-                if blk.tag_hi in (0x10, 0x20, 0x00):
-                    delta_count += blk.tag_lo
-
-            # First sample this 30 NN block affects (within the segment)
-            # = anchor positions + delta_count + 2 (since anchor pair was samples 0,1)
-            # But the segment's first absolute sample index in the channel is
-            # (seg_num // 4) * 512 (approximately) if segment 0 is the first V seg.
-            cycle = (seg_num // 4) if seg_num >= 0 else 0
-            base = cycle * 512 + 2  # +2 for anchor pair
-            sample_idx = base + delta_count
-            truth_ch = [round(v * 200) for v in samples[channel]]
-            nn = b.tag_lo
-
-            if sample_idx + nn >= len(truth_ch):
-                print(f"  block @ {b.offset} ({seg_label} {channel}): out of truth range")
-                continue
-
-            # Get the previous sample so we can compute truth deltas
-            if sample_idx == 0:
-                prev = 0
-            else:
-                prev = truth_ch[sample_idx - 1]
-            truth_deltas = []
-            for k in range(nn):
-                truth_deltas.append(truth_ch[sample_idx + k] - (prev if k == 0 else truth_ch[sample_idx + k - 1]))
-
-            # Try each packing
-            schemes = [
-                ("12-bit BE contiguous", unpack_12bit_be(b.data)),
-                ("12-bit LE per-triplet", unpack_12bit_le(b.data)),
-                ("12-bit BE per-triplet", unpack_12bit_be_per_triplet(b.data)),
-            ]
-            print(f"  block @ {b.offset:>5} ({seg_label} {channel}, samples {sample_idx}..{sample_idx+nn-1}):")
-            print(f"    data:  {b.data.hex(' ')}")
-            print(f"    truth: {truth_deltas}")
-            for name, pred in schemes:
-                match = "✓" if pred == truth_deltas else " "
-                n_match = sum(1 for x, y in zip(pred, truth_deltas) if x == y)
-                print(f"    {match}{n_match}/4  {name}: {pred}")
-
-
-if __name__ == "__main__":
-    main()

analysis/test_30nn_hybrid.py

@@ -1,132 +0,0 @@
-"""Test the '30 NN data = high-nibbles + int8 low-bytes' hypothesis.
-
-Layout for `30 04` (6 data bytes, 4 deltas):
-  bytes [0:2] = 16 bits = 4 × 4-bit high-nibbles (MSB first)
-  bytes [2:6] = 4 × int8 low bytes
-  Each delta = 12-bit signed = sign-extend((high_nibble << 8) | low_byte)
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def sign_extend_12(v):
-    return v if v < 0x800 else v - 0x1000
-
-
-def decode_30nn(data):
-    """4 × 12-bit signed deltas (high nibble + low byte).
-    bytes[0:2] hold the 4 high nibbles (MSB first); bytes[2:6] hold the low bytes.
-    """
-    if len(data) < 6:
-        return []
-    # Read high nibbles from bytes 0-1 (4 nibbles MSB-first)
-    high_word = (data[0] << 8) | data[1]
-    high_nibbles = [
-        (high_word >> 12) & 0xF,
-        (high_word >> 8) & 0xF,
-        (high_word >> 4) & 0xF,
-        high_word & 0xF,
-    ]
-    out = []
-    for i in range(4):
-        v = (high_nibbles[i] << 8) | data[2 + i]
-        out.append(sign_extend_12(v))
-    return out
-
-
-def simulate_up_to(blocks, target_block_idx, t_preamble):
-    """Run decoder up to block_idx; return per-channel sample lists.
-    NOW with 30 NN decoded too."""
-    out = {"Tran": [], "Vert": [], "Long": [], "MicL": []}
-    out["Tran"].extend(t_preamble)
-    cur = {"Tran": t_preamble[-1], "Vert": None, "Long": None, "MicL": None}
-    rotation = ["Vert", "Long", "MicL", "Tran"]
-    current_channel = "Tran"
-    seg_counter = -1
-    for j in range(target_block_idx):
-        blk = blocks[j]
-        if blk.tag_hi == 0x40:
-            seg_counter += 1
-            prev = "Tran" if seg_counter == 0 else rotation[(seg_counter - 1) % 4]
-            new_ch = rotation[seg_counter % 4]
-            if cur[prev] is not None:
-                d0 = int.from_bytes(blk.data[0:2], "big", signed=True)
-                d1 = int.from_bytes(blk.data[2:4], "big", signed=True)
-                cur[prev] += d0; out[prev].append(cur[prev])
-                cur[prev] += d1; out[prev].append(cur[prev])
-            c0 = int.from_bytes(blk.data[14:16], "big", signed=True)
-            c1 = int.from_bytes(blk.data[16:18], "big", signed=True)
-            out[new_ch].extend([c0, c1])
-            cur[new_ch] = c1
-            current_channel = new_ch
-        elif blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur[current_channel] += s4(nib)
-                    out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur[current_channel] += i8(byte)
-                out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x00:
-            for _ in range(blk.tag_lo):
-                out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x30:
-            # NEW: decode 30 NN
-            deltas = decode_30nn(blk.data)
-            for d in deltas:
-                cur[current_channel] += d
-                out[current_channel].append(cur[current_channel])
-    return out, current_channel
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1L.JQ0", "M529LL1L.V70",
-                 "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        blocks = walk_body(body, find_data_start(body))
-        t0 = int.from_bytes(body[3:5], "big", signed=True)
-        t1 = int.from_bytes(body[5:7], "big", signed=True)
-        thirty_blocks = [(j, b) for j, b in enumerate(blocks) if b.tag_hi == 0x30]
-        if not thirty_blocks:
-            continue
-        print(f"\n=== {stem} ===")
-        for j, blk in thirty_blocks:
-            pred, ch = simulate_up_to(blocks, j, [t0, t1])
-            cur_before = pred[ch][-1]
-            truth = [round(v * 200) for v in samples[ch]]
-            n_pred = len(pred[ch])
-            nn = blk.tag_lo
-            if n_pred + nn > len(truth):
-                continue
-            # Decode this 30 NN block with hypothesis
-            pred_deltas = decode_30nn(blk.data)
-            # Compute truth deltas relative to cur_before
-            truth_deltas = []
-            prev = cur_before
-            for k in range(nn):
-                truth_deltas.append(truth[n_pred + k] - prev)
-                prev = truth[n_pred + k]
-            n_match = sum(1 for a, b in zip(pred_deltas, truth_deltas) if a == b)
-            tag = "✓" if pred_deltas == truth_deltas else " "
-            print(f"  block @ {blk.offset:>5} (chan={ch}, NN={nn}):")
-            print(f"    data:  {blk.data.hex(' ')}")
-            print(f"    truth: {truth_deltas}")
-            print(f"    pred:  {pred_deltas}  {tag}{n_match}/{nn}")
-
-
-if __name__ == "__main__":
-    main()

analysis/test_30nn_v2.py

@@ -1,141 +0,0 @@
-"""Test 30 NN packing by running the real decoder up to each 30 NN block,
-recording how many samples have been produced for each channel at that point,
-then checking truth deltas immediately after."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def s12(v):
-    return v if v < 0x800 else v - 0x1000
-
-
-def unpack_12bit_be_contiguous(data):
-    out = []
-    val = int.from_bytes(data, "big")
-    n = len(data) * 8 // 12
-    for i in range(n):
-        d = (val >> (12 * (n - 1 - i))) & 0xFFF
-        out.append(s12(d))
-    return out
-
-
-def unpack_12bit_per_triplet_be(data):
-    out = []
-    for i in range(0, len(data), 3):
-        if i + 2 >= len(data):
-            break
-        b0, b1, b2 = data[i], data[i + 1], data[i + 2]
-        d0 = (b0 << 4) | (b1 >> 4)
-        d1 = ((b1 & 0x0F) << 8) | b2
-        out.append(s12(d0))
-        out.append(s12(d1))
-    return out
-
-
-def simulate_up_to(blocks, target_block_idx, t_preamble):
-    """Run the decoder up to block_idx; return per-channel sample lists."""
-    out = {"Tran": [], "Vert": [], "Long": [], "MicL": []}
-    out["Tran"].extend(t_preamble)
-    cur = {"Tran": t_preamble[-1], "Vert": None, "Long": None, "MicL": None}
-    rotation = ["Vert", "Long", "MicL", "Tran"]
-    seg_idx = [j for j, b in enumerate(blocks) if b.tag_hi == 0x40]
-
-    # Determine which channel we're CURRENTLY decoding into
-    current_channel = "Tran"
-    seg_counter = -1  # incremented at each 40 02
-
-    for j in range(target_block_idx):
-        blk = blocks[j]
-        if blk.tag_hi == 0x40:
-            # Switch: extend prev channel, set up new channel
-            seg_counter += 1
-            prev = "Tran" if seg_counter == 0 else rotation[(seg_counter - 1) % 4]
-            new_ch = rotation[seg_counter % 4]
-            if cur[prev] is not None:
-                d0 = int.from_bytes(blk.data[0:2], "big", signed=True)
-                d1 = int.from_bytes(blk.data[2:4], "big", signed=True)
-                cur[prev] += d0; out[prev].append(cur[prev])
-                cur[prev] += d1; out[prev].append(cur[prev])
-            c0 = int.from_bytes(blk.data[14:16], "big", signed=True)
-            c1 = int.from_bytes(blk.data[16:18], "big", signed=True)
-            out[new_ch].extend([c0, c1])
-            cur[new_ch] = c1
-            current_channel = new_ch
-        elif blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur[current_channel] += s4(nib)
-                    out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur[current_channel] += i8(byte)
-                out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x00:
-            for _ in range(blk.tag_lo):
-                out[current_channel].append(cur[current_channel])
-        elif blk.tag_hi == 0x30:
-            # Skip for now — we want to know what comes next
-            pass
-
-    return out, current_channel
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1L.JQ0", "M529LL1L.V70",
-                 "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        blocks = walk_body(body, find_data_start(body))
-        t0 = int.from_bytes(body[3:5], "big", signed=True)
-        t1 = int.from_bytes(body[5:7], "big", signed=True)
-
-        # Find all 30 NN blocks in data section
-        thirty_blocks = [(j, b) for j, b in enumerate(blocks) if b.tag_hi == 0x30]
-        if not thirty_blocks:
-            continue
-
-        print(f"\n=== {stem} ===")
-        for j, blk in thirty_blocks:
-            pred, ch = simulate_up_to(blocks, j, [t0, t1])
-            n_pred = len(pred[ch])
-            # The 30 NN block carries NN deltas for channel `ch` starting at sample n_pred
-            truth = [round(v * 200) for v in samples[ch]]
-            if n_pred >= len(truth):
-                continue
-            # Truth deltas: truth[n_pred] - cur, truth[n_pred+1] - truth[n_pred], ...
-            cur_val = pred[ch][-1]
-            nn = blk.tag_lo
-            truth_deltas = []
-            prev = cur_val
-            for k in range(min(nn, len(truth) - n_pred)):
-                truth_deltas.append(truth[n_pred + k] - prev)
-                prev = truth[n_pred + k]
-
-            print(f"  block @ {blk.offset:>5} (chan={ch}, after sample {n_pred-1}, "
-                  f"NN={nn}, last_val={cur_val}):")
-            print(f"    data:  {blk.data.hex(' ')}")
-            print(f"    truth: {truth_deltas}")
-            schemes = [
-                ("12-bit BE contiguous", unpack_12bit_be_contiguous(blk.data)),
-                ("12-bit per-triplet BE", unpack_12bit_per_triplet_be(blk.data)),
-            ]
-            for name, pred_deltas in schemes:
-                n_match = sum(1 for a, b in zip(pred_deltas, truth_deltas) if a == b)
-                tag = "✓" if pred_deltas == truth_deltas else " "
-                print(f"    {tag}{n_match}/{nn}  {name}: {pred_deltas[:nn]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/test_rle.py

@@ -1,86 +0,0 @@
-"""Test: 00 NN markers might be RLE for zero-deltas in current channel."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def decode_with_rle(body):
-    """Decode Tran assuming:
-    - preamble[3:5], [5:7] = T[0], T[1]
-    - All 10 NN / 20 NN blocks until segment_header (40 02) are Tran deltas
-    - 00 NN markers are RLE: NN/4 zero T deltas (or NN, or NN/2 — try them)
-    """
-    if len(body) < 9 or body[0:3] != b"\x00\x02\x00":
-        return None, None, None
-    T0 = int.from_bytes(body[3:5], "big", signed=True)
-    T1 = int.from_bytes(body[5:7], "big", signed=True)
-
-    # Find first tag (might be 00 NN, 10 NN, or 20 NN)
-    i = 7
-    while i + 1 < len(body):
-        if body[i] in (0x00, 0x10, 0x20):
-            break
-        i += 1
-    start = i
-
-    blocks = walk_body(body, start)
-
-    results = {}
-    for rle_div in (4, 2, 1):  # try different RLE interpretations
-        T = [T0, T1]
-        cur = T1
-        for blk in blocks:
-            if blk.tag_hi == 0x40:
-                break
-            if blk.tag_hi == 0x10:
-                for byte in blk.data:
-                    for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                        cur += s4(nib)
-                        T.append(cur)
-            elif blk.tag_hi == 0x20:
-                for byte in blk.data:
-                    cur += i8(byte)
-                    T.append(cur)
-            elif blk.tag_hi == 0x00:
-                # RLE of zero deltas
-                n_zeros = blk.tag_lo // rle_div
-                for _ in range(n_zeros):
-                    T.append(cur)
-            # 30 NN: skip for now
-        results[rle_div] = T
-    return results, T0, T1
-
-
-def main():
-    for stem in ("M529LL1L.V70", "M529LL1L.JQ0", "M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        truth_T = [round(v*200) for v in samples["Tran"]]
-
-        results, T0, T1 = decode_with_rle(body)
-        print(f"\n=== {stem} (T[0]={T0}, T[1]={T1}) ===")
-        for rle_div, T in results.items():
-            n = min(len(T), len(truth_T))
-            matches = sum(1 for i in range(n) if T[i] == truth_T[i])
-            # Find first divergence
-            div_at = -1
-            for i in range(n):
-                if T[i] != truth_T[i]:
-                    div_at = i
-                    break
-            print(f"  rle_div={rle_div}: decoded {len(T)}, matches {matches}/{n}, first div at sample {div_at}")
-
-
-if __name__ == "__main__":
-    main()

analysis/test_tran_continue.py

@@ -1,71 +0,0 @@
-"""Test: does the second '20 NN' block in SS0 continue Tran samples?"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def main():
-    stem = "M529LL1A.SS0"
-    path = f"tests/fixtures/5-11-26/{stem}"
-    with open(path, "rb") as f:
-        body = f.read()[43:-26]
-    _, samples = _parse_txt(path + ".TXT")
-    truth_T_16 = [round(v * 200) for v in samples["Tran"]]
-
-    # Preamble
-    T0 = int.from_bytes(body[3:5], "big", signed=True)
-    T1 = int.from_bytes(body[5:7], "big", signed=True)
-
-    # Walk blocks
-    start = find_data_start(body)
-    blocks = walk_body(body, start)
-
-    print(f"=== {stem} ===  T[0]={T0} T[1]={T1}")
-
-    # Hypothesis: Tran continues through ALL 10 NN and 20 NN blocks
-    # in order, until the next 40 02 segment header (which resets).
-    T = [T0, T1]
-    cur = T1
-    decoded_count = 2  # T[0], T[1] from preamble
-    for bi, blk in enumerate(blocks):
-        if blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += s4(nib)
-                    T.append(cur)
-                    decoded_count += 1
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur += i8(byte)
-                T.append(cur)
-                decoded_count += 1
-        elif blk.tag_hi == 0x40:
-            # Segment header — stop here for this test
-            break
-        # 00 and 30 NN don't contribute to Tran (in this hypothesis)
-
-    # Compare to truth
-    print(f"  Decoded {len(T)} T samples up to first 40 02")
-    matches = sum(1 for i in range(min(len(T), len(truth_T_16))) if T[i] == truth_T_16[i])
-    print(f"  Matches in first {min(len(T), len(truth_T_16))}: {matches}")
-    # Print first divergence
-    for i in range(min(len(T), len(truth_T_16))):
-        if T[i] != truth_T_16[i]:
-            print(f"  First divergence: sample {i}: pred={T[i]}, truth={truth_T_16[i]}")
-            # Show context
-            print(f"    pred  [{i-3}:{i+5}]: {T[max(0,i-3):i+5]}")
-            print(f"    truth [{i-3}:{i+5}]: {truth_T_16[max(0,i-3):i+5]}")
-            break
-
-
-if __name__ == "__main__":
-    main()

analysis/try_layouts.py

@@ -1,67 +0,0 @@
-"""Try various nibble-level channel interleavings to find which one matches truth."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def run_decoder(body, layout, skip, n_channels=4):
-    """layout: function nibble_index -> channel_index. Returns list-of-lists per channel."""
-    out = [[] for _ in range(n_channels)]
-    cur = [0] * n_channels
-    nibbles = []
-    for byte in body[skip:]:
-        nibbles.append((byte >> 4) & 0xF)
-        nibbles.append(byte & 0xF)
-    for i, n in enumerate(nibbles):
-        ch = layout(i)
-        cur[ch] += s4(n)
-        out[ch].append(cur[ch])
-    return out
-
-
-def cmp(pred, truth, n=24):
-    n = min(n, len(pred), len(truth))
-    return [(pred[i], truth[i]) for i in range(n)]
-
-
-def main():
-    b = load_bundle("event-c")
-    truth_T = [round(v * 200) for v in b.samples["Tran"]]
-    truth_V = [round(v * 200) for v in b.samples["Vert"]]
-    truth_L = [round(v * 200) for v in b.samples["Long"]]
-    print(f"T truth[0:10]: {truth_T[:10]}")
-    print(f"V truth[0:10]: {truth_V[:10]}")
-    print(f"L truth[0:10]: {truth_L[:10]}")
-
-    # Try several nibble->channel layouts (4 channels)
-    layouts = {
-        "interleaved TVLM (0,1,2,3,0,1,2,3,...)": lambda i: i % 4,
-        "interleaved VLMT": lambda i: (i + 3) % 4,
-        "interleaved LMTV": lambda i: (i + 2) % 4,
-        "interleaved MTVL": lambda i: (i + 1) % 4,
-        "byte-based TV LM TV LM (high T low V byte0; high L low M byte1)": lambda i: i % 4,
-        # "chunks of 8 nibbles per channel": each channel gets 8 nibbles in a row
-        "chunks-8 TVLM": lambda i: (i // 8) % 4,
-        "chunks-16 TVLM": lambda i: (i // 16) % 4,
-        # planar (full channel sequential)
-        "planar T(0..N) V(N..2N) L(2N..3N) M(3N..4N)": None,  # special
-    }
-
-    for label, layout_fn in layouts.items():
-        if layout_fn is None:
-            continue
-        for skip in (0, 4, 7, 8, 9, 11, 14):
-            out = run_decoder(b.body, layout_fn, skip)
-            # Check first 8 cumulative on each channel
-            print(f"  skip={skip:2}  {label}")
-            print(f"    T_cum[0:10]: {out[0][:10]}")
-            print(f"    V_cum[0:10]: {out[1][:10]}")
-            print(f"    L_cum[0:10]: {out[2][:10]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/try_nibbles.py

@@ -1,73 +0,0 @@
-"""Try decoding body as 4-bit signed nibble deltas, 4-channel round-robin."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-CHANNELS = ("Tran", "Vert", "Long", "MicL")
-
-
-def s4(n):
-    """Sign-extend a 4-bit unsigned to int (0..7 → 0..7, 8..F → -8..-1)."""
-    return n if n < 8 else n - 16
-
-
-def decode_nibbles(body: bytes, skip_bytes: int = 7, n_channels: int = 4):
-    """Read body as 2 nibbles per byte; accumulate as deltas for n_channels round-robin."""
-    out = [[] for _ in range(n_channels)]
-    cur = [0] * n_channels
-    ch = 0
-    nibbles = []
-    for byte in body[skip_bytes:]:
-        nibbles.append((byte >> 4) & 0xF)
-        nibbles.append(byte & 0xF)
-    for n in nibbles:
-        cur[ch] += s4(n)
-        out[ch].append(cur[ch])
-        ch = (ch + 1) % n_channels
-    return out
-
-
-def cmp_to_truth(pred, truth, scale=16):
-    """Compare predicted ints (in 16-count units) to truth (in 16-count units = txt * 200).
-    Return (max_abs_err, mean_abs_err, n_compared).
-    """
-    n = min(len(pred), len(truth))
-    errs = []
-    for i in range(n):
-        p = pred[i]
-        t = truth[i]
-        errs.append(abs(p - t))
-    if not errs:
-        return None
-    return (max(errs), sum(errs) / len(errs), n)
-
-
-def main():
-    for name in ("event-a", "event-c"):
-        b = load_bundle(name)
-        # Convert TXT samples (in/s) to 16-count units (multiply by 200, since 0.005 in/s = 1)
-        # WAIT: 0.005 in/s = 16 ADC counts. 1 count = 0.000305 in/s.
-        # So in 1-count units: count = txt * (1/0.0003052) ≈ txt * 3276.7
-        # But TXT only has 0.005 resolution so equivalent to 16-count units = txt * 200.
-        truth_in_16 = {ch: [round(v * 200) for v in b.samples[ch]] for ch in CHANNELS[:3]}
-        # MicL is in dB, skip for now
-
-        # Try decoder with skip_bytes = 7
-        decoded = decode_nibbles(b.body, skip_bytes=7, n_channels=4)
-        print(f"\n=== {name} ===")
-        print(f"  body={len(b.body)}, nibbles={2*(len(b.body)-7)}, samples_per_ch={len(decoded[0])}")
-        print(f"  truth samples per ch: {len(truth_in_16['Tran'])}")
-        # Print first 24 of each
-        for i, chan in enumerate(CHANNELS):
-            pred_first = decoded[i][:24]
-            if chan in truth_in_16:
-                truth_first = truth_in_16[chan][:24]
-                print(f"  {chan} pred: {pred_first}")
-                print(f"  {chan} truth: {truth_first}")
-            else:
-                print(f"  {chan} pred: {pred_first}  (truth in dB, skipped)")
-
-
-if __name__ == "__main__":
-    main()

analysis/verify_full_decode.py

@@ -1,32 +0,0 @@
-"""Verify decode_waveform_v2 against BW ASCII truth for all fixtures."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import decode_waveform_v2
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0",
-                 "M529LL1L.JQ0", "M529LL1L.V70"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            body = f.read()[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        decoded = decode_waveform_v2(body)
-        if decoded is None:
-            print(f"{stem}: decoder returned None")
-            continue
-
-        print(f"\n=== {stem} ===")
-        for ch in ("Tran", "Vert", "Long"):
-            truth = [round(v * 200) for v in samples[ch]]
-            pred = decoded[ch]
-            n = min(len(pred), len(truth))
-            matches = sum(1 for i in range(n) if pred[i] == truth[i])
-            div = next((i for i in range(n) if pred[i] != truth[i]), -1)
-            print(f"  {ch}: decoded={len(pred):>5}  truth={len(truth):>5}  "
-                  f"matches={matches:>5}/{n:<5}  first div={div}")
-
-
-if __name__ == "__main__":
-    main()

analysis/verify_quiet_bundle.py

@@ -1,55 +0,0 @@
-"""Run decode_waveform_v2 against the 5-8-26 quiet bundle to test the
-'quiet events should decode fully' hypothesis."""
-import os, sys
-sys.path.insert(0, ".")
-from minimateplus.waveform_codec import decode_waveform_v2, walk_body, find_data_start
-from analysis.load_bundle import _parse_txt
-
-
-def main():
-    base = "tests/fixtures/decode-re-5-8-26"
-    for evt in sorted(os.listdir(base)):
-        folder = os.path.join(base, evt)
-        if not os.path.isdir(folder):
-            continue
-        # Find the binary (not .TXT)
-        bin_name = next(
-            (f for f in os.listdir(folder) if not f.endswith(".TXT")),
-            None,
-        )
-        if not bin_name:
-            continue
-        bin_path = os.path.join(folder, bin_name)
-        txt_path = bin_path + ".TXT"
-        if not os.path.exists(txt_path):
-            # Sometimes the TXT name differs slightly
-            for f in os.listdir(folder):
-                if f.endswith(".TXT"):
-                    txt_path = os.path.join(folder, f)
-                    break
-        with open(bin_path, "rb") as f:
-            body = f.read()[43:-26]
-        decoded = decode_waveform_v2(body)
-        _, samples = _parse_txt(txt_path)
-
-        # Count 30 NN blocks
-        blocks = walk_body(body, find_data_start(body))
-        n_30 = sum(1 for b in blocks if b.tag_hi == 0x30)
-        n_40 = sum(1 for b in blocks if b.tag_hi == 0x40)
-
-        print(f"\n=== {evt} === body={len(body)}  segments={n_40}  '30 NN' blocks={n_30}")
-        if decoded is None:
-            print("  decoder returned None")
-            continue
-        for ch in ("Tran", "Vert", "Long"):
-            truth = [round(v * 200) for v in samples[ch]]
-            pred = decoded[ch]
-            n = min(len(pred), len(truth))
-            matches = sum(1 for i in range(n) if pred[i] == truth[i])
-            div = next((i for i in range(n) if pred[i] != truth[i]), -1)
-            print(f"  {ch}: decoded={len(pred):>5}  truth={len(truth):>5}  "
-                  f"matches={matches:>5}/{n:<5}  first div={div}")
-
-
-if __name__ == "__main__":
-    main()

analysis/verify_tran.py

@@ -1,71 +0,0 @@
-"""Verify: preamble[3:7] = Tran[0], Tran[1] as int16 BE in 16-count units.
-And first 20/10 NN block = Tran deltas starting at sample 2.
-"""
-import os, sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import _parse_txt
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def s4(n):
-    return n if n < 8 else n - 16
-
-
-def i8(b):
-    return b if b < 128 else b - 256
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        path = f"tests/fixtures/5-11-26/{stem}"
-        with open(path, "rb") as f:
-            raw = f.read()
-        body = raw[43:-26]
-        _, samples = _parse_txt(path + ".TXT")
-        truth_T_16 = [round(v * 200) for v in samples["Tran"]]
-
-        # Preamble parse
-        T0_pre = int.from_bytes(body[3:5], "big", signed=True)
-        T1_pre = int.from_bytes(body[5:7], "big", signed=True)
-        print(f"\n=== {stem} ===")
-        print(f"  Preamble T[0]={T0_pre} (truth {truth_T_16[0]})  T[1]={T1_pre} (truth {truth_T_16[1]})  match={T0_pre==truth_T_16[0] and T1_pre==truth_T_16[1]}")
-
-        # First block
-        start = find_data_start(body)
-        blocks = walk_body(body, start)
-        if not blocks:
-            print(f"  no blocks found")
-            continue
-
-        # Assume first block = Tran deltas from sample 2
-        first = blocks[0]
-        T = [T0_pre, T1_pre]
-        cur_T = T1_pre
-        if first.tag_hi == 0x10:
-            # Nibble pairs
-            for byte in first.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur_T += s4(nib)
-                    T.append(cur_T)
-        elif first.tag_hi == 0x20:
-            # int8 per byte
-            for byte in first.data:
-                cur_T += i8(byte)
-                T.append(cur_T)
-
-        # Compare against truth
-        n_check = min(len(T), len(truth_T_16))
-        match_count = sum(1 for i in range(n_check) if T[i] == truth_T_16[i])
-        print(f"  First block type=0x{first.tag_hi:02x} NN=0x{first.tag_lo:02x} len={len(first.data)} → {len(T)} T samples decoded")
-        print(f"  Tran predicted[0:10]: {T[:10]}")
-        print(f"  Tran truth    [0:10]: {truth_T_16[:10]}")
-        print(f"  Matches in first {n_check}: {match_count} / {n_check}")
-        # Show where it diverges
-        for i in range(n_check):
-            if T[i] != truth_T_16[i]:
-                print(f"  First divergence: sample {i}: pred={T[i]}, truth={truth_T_16[i]}")
-                break
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_5_11.py

@@ -1,20 +0,0 @@
-"""Walk blocks of the new 5-11-26 events and look at what comes after Tran block."""
-import sys
-sys.path.insert(0, ".")
-from minimateplus.waveform_codec import walk_body, find_data_start
-
-
-def main():
-    for stem in ("M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0"):
-        with open(f"tests/fixtures/5-11-26/{stem}", "rb") as f:
-            raw = f.read()
-        body = raw[43:-26]
-        start = find_data_start(body)
-        blocks = walk_body(body, start)
-        print(f"\n=== {stem} === body={len(body)} start={start} blocks walked={len(blocks)}")
-        for i, b in enumerate(blocks[:20]):
-            print(f"  block[{i:>2}] @ {b.offset:>5} tag={b.tag_hi:02x} NN=0x{b.tag_lo:02x}({b.tag_lo}) len={b.length} data[:24]={b.data[:24].hex(' ')}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_chunks.py

@@ -1,44 +0,0 @@
-"""Walk the body assuming chunks delimited by 0x10 NN tags. Print each chunk's structure."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def walk(body: bytes, start_offset: int = 7, max_chunks: int = 30):
-    """Find all positions where byte = 0x10 followed by a multiple-of-4 byte. Print chunks."""
-    chunks = []
-    i = start_offset
-    while i < len(body) - 1:
-        # Find next `10 NN` where NN is multiple of 4 (and not preceded by another 0x10 immediately, which would be data).
-        if body[i] == 0x10 and (body[i+1] % 4 == 0):
-            chunks.append(i)
-        i += 1
-    return chunks
-
-
-def main():
-    for name in ("event-c", "event-d"):
-        b = load_bundle(name)
-        body = b.body
-        positions = []
-        i = 7  # skip 7-byte preamble
-        while i < len(body) - 1:
-            if body[i] == 0x10 and body[i+1] % 4 == 0 and body[i+1] > 0:
-                positions.append(i)
-                i += 2  # skip past tag
-            else:
-                i += 1
-        print(f"\n=== {name} ===  body={len(body)}, total `10 NN` (NN%4==0, NN>0) tags: {len(positions)}")
-        # Print first 20 chunks: show position, NN, gap to next tag
-        for k in range(min(30, len(positions))):
-            pos = positions[k]
-            NN = body[pos + 1]
-            next_pos = positions[k+1] if k+1 < len(positions) else len(body)
-            gap = next_pos - pos
-            data_bytes = body[pos+2 : next_pos]
-            print(f"  chunk[{k:>3}] @ {pos:>5}  NN=0x{NN:02x} ({NN:>3}, NN/2={NN//2})  gap={gap:>3}  "
-                  f"data={data_bytes[:24].hex(' ')}{'...' if len(data_bytes) > 24 else ''}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_v2.py

@@ -1,50 +0,0 @@
-"""Deterministic chunk walker: each chunk = [10 NN][NN/2 bytes data][2 bytes trailer]."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def walk_chunks(body: bytes, start: int = 7):
-    """Yield (offset, NN, data_bytes, trailer_bytes) tuples."""
-    i = start
-    while i + 1 < len(body):
-        if body[i] != 0x10:
-            break
-        NN = body[i + 1]
-        if NN == 0 or NN > 0x80 or NN % 4 != 0:
-            break
-        chunk_len = NN // 2 + 4
-        if i + chunk_len > len(body):
-            break
-        data = bytes(body[i + 2 : i + 2 + NN // 2])
-        trailer = bytes(body[i + 2 + NN // 2 : i + chunk_len])
-        yield (i, NN, data, trailer)
-        i += chunk_len
-
-
-def main():
-    for name in ("event-c", "event-d", "event-a", "event-b"):
-        b = load_bundle(name)
-        body = b.body
-        chunks = list(walk_chunks(body))
-        print(f"\n=== {name} ===  body={len(body)}  N_samples={len(b.samples['Tran'])}")
-        print(f"  chunks parsed: {len(chunks)}")
-        if chunks:
-            last = chunks[-1]
-            end_of_walk = last[0] + last[1] // 2 + 4
-            print(f"  walk ended at offset {end_of_walk} (= {len(body) - end_of_walk} bytes from end)")
-            # Stats
-            total_data_bytes = sum(len(c[2]) for c in chunks)
-            print(f"  total data bytes: {total_data_bytes}, total nibbles: {2*total_data_bytes}")
-            if name in ("event-c", "event-d"):
-                ratio = (2 * total_data_bytes) / (len(b.samples['Tran']) * 4)
-                print(f"  nibbles per (sample × channel): {ratio:.3f}")
-            # Sum of trailer second-byte
-            trailer_sums = [c[3][-1] if c[3] else None for c in chunks]
-            print(f"  first 10 chunks: {[(c[0], c[1], c[3].hex()) for c in chunks[:10]]}")
-            # Print last 10 chunks (likely transition to trailer)
-            print(f"  last 10 chunks: {[(c[0], c[1], c[3].hex()) for c in chunks[-10:]]}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_v3.py

@@ -1,51 +0,0 @@
-"""Walk chunks; auto-detect preamble length by finding first 10 NN."""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def walk_chunks(body, start, max_NN=0x80):
-    chunks = []
-    i = start
-    while i + 1 < len(body):
-        if body[i] != 0x10:
-            break
-        NN = body[i + 1]
-        if NN == 0 or NN > max_NN or NN % 4 != 0:
-            break
-        chunk_len = NN // 2 + 4
-        if i + chunk_len > len(body):
-            break
-        data = bytes(body[i + 2 : i + 2 + NN // 2])
-        trailer = bytes(body[i + 2 + NN // 2 : i + chunk_len])
-        chunks.append((i, NN, data, trailer))
-        i += chunk_len
-    return chunks, i
-
-
-def find_first_chunk_start(body):
-    """Locate first byte that begins a `10 NN` chunk (NN ∈ multiples of 4, 4..0x7C)."""
-    for i in range(20):
-        if body[i] == 0x10 and body[i + 1] % 4 == 0 and 0 < body[i + 1] <= 0x7C:
-            return i
-    return -1
-
-
-def main():
-    for name in ("event-c", "event-d", "event-a", "event-b"):
-        b = load_bundle(name)
-        body = b.body
-        start = find_first_chunk_start(body)
-        chunks, end = walk_chunks(body, start)
-        print(f"\n=== {name} ===  body={len(body)}  N_samples={len(b.samples['Tran'])}  start={start}")
-        print(f"  chunks parsed: {len(chunks)}, walk ended at {end}")
-        if chunks:
-            print(f"  first 5 chunks: {[(c[0], c[1], c[3].hex()) for c in chunks[:5]]}")
-            print(f"  last 5 chunks: {[(c[0], c[1], c[3].hex()) for c in chunks[-5:]]}")
-            print(f"  bytes around end of walk: {body[end-4:end+12].hex(' ')}")
-        else:
-            print(f"  bytes at start: {body[start:start+16].hex(' ')}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_v4.py

@@ -1,75 +0,0 @@
-"""
-Walker v4: alternate [10 NN] data chunks and [00 NN] (or other) marker tags.
-
-Hypothesis:
-- [10 NN]: data block, length NN/2 + 2 bytes (2-byte tag + NN/2 bytes data)
-- [00 NN]: 2-byte marker block (no data)
-- [20/30/40 NN]: special blocks with type-dependent length
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-
-
-def walk(body, start):
-    i = start
-    blocks = []
-    while i + 1 < len(body):
-        t0 = body[i]
-        t1 = body[i + 1]
-        if t0 == 0x10 and t1 % 4 == 0 and 0 < t1 <= 0x80:
-            # data chunk: length NN/2 + 2
-            length = t1 // 2 + 2
-            blocks.append((i, "10", t1, bytes(body[i + 2 : i + length]), length))
-            i += length
-        elif t0 == 0x00 and t1 % 4 == 0:
-            # 2-byte marker
-            blocks.append((i, "00", t1, b"", 2))
-            i += 2
-        elif t0 == 0x20 and t1 % 4 == 0:
-            # type 2 — try length 2+t1/2 (similar to 10) OR fixed
-            length = t1 // 2 + 2
-            blocks.append((i, "20", t1, bytes(body[i + 2 : i + length]), length))
-            i += length
-        elif t0 == 0x30 and t1 % 4 == 0:
-            length = t1 // 2 + 2
-            blocks.append((i, "30", t1, bytes(body[i + 2 : i + length]), length))
-            i += length
-        elif t0 == 0x40 and t1 == 0x02:
-            # Special "footer transition" block — try fixed 22 bytes
-            length = 22
-            blocks.append((i, "40", t1, bytes(body[i + 2 : i + length]), length))
-            i += length
-        else:
-            # Unknown tag — stop
-            blocks.append((i, "??", t0, bytes(body[i:i+8]), 0))
-            break
-    return blocks, i
-
-
-def main():
-    for name in ("event-c", "event-d", "event-a", "event-b"):
-        b = load_bundle(name)
-        body = b.body
-        # Auto-detect start
-        for s in range(15):
-            if body[s] == 0x10 and body[s+1] % 4 == 0 and 0 < body[s+1] <= 0x80:
-                start = s
-                break
-        else:
-            start = 7
-        blocks, end = walk(body, start)
-        # Categorize
-        from collections import Counter
-        types = Counter(b[1] for b in blocks)
-        print(f"\n=== {name} === body={len(body)} N={len(b.samples['Tran'])}  start={start}")
-        print(f"  total blocks: {len(blocks)}, walk ended at {end}/{len(body)}")
-        print(f"  type counts: {dict(types)}")
-        # Print last 5 blocks
-        print(f"  last 5 blocks: {[(bb[0], bb[1], bb[2]) for bb in blocks[-5:]]}")
-        if end < len(body):
-            print(f"  bytes at end: {body[end:end+24].hex(' ')}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_v5.py

@@ -1,83 +0,0 @@
-"""
-Walker v5: flexible NN range and multiple block-type lengths.
-
-Hypothesis:
-- [10 NN]: 4-bit-delta data block, length = NN/2 + 2
-- [20 NN]: 8-bit-literal data block, length = NN + 2
-- [00 NN]: 2-byte marker (no payload)
-- [30 NN]: trailer/summary block, length = NN*4
-- [40 NN]: footer-marker block, fixed 22 bytes
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-from collections import Counter
-
-
-def walk(body, start, max_blocks=10000):
-    i = start
-    blocks = []
-    while i + 1 < len(body) and len(blocks) < max_blocks:
-        t0 = body[i]
-        t1 = body[i + 1]
-        if t0 == 0x10 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 // 2 + 2
-            if i + length > len(body):
-                break
-            data = bytes(body[i + 2 : i + length])
-            blocks.append((i, "10", t1, data, length))
-            i += length
-        elif t0 == 0x20 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 + 2
-            if i + length > len(body):
-                break
-            data = bytes(body[i + 2 : i + length])
-            blocks.append((i, "20", t1, data, length))
-            i += length
-        elif t0 == 0x00 and t1 % 4 == 0:
-            # 2-byte marker
-            blocks.append((i, "00", t1, b"", 2))
-            i += 2
-        elif t0 == 0x30 and t1 % 4 == 0:
-            length = t1 * 4
-            if i + length > len(body):
-                break
-            data = bytes(body[i + 2 : i + length])
-            blocks.append((i, "30", t1, data, length))
-            i += length
-        elif t0 == 0x40 and t1 == 0x02:
-            length = 22
-            if i + length > len(body):
-                break
-            data = bytes(body[i + 2 : i + length])
-            blocks.append((i, "40", t1, data, length))
-            i += length
-        else:
-            blocks.append((i, "??", t0, bytes(body[i:i+8]), 0))
-            break
-    return blocks, i
-
-
-def main():
-    for name in ("event-c", "event-d", "event-a", "event-b"):
-        b = load_bundle(name)
-        body = b.body
-        for s in range(15):
-            if body[s] == 0x10 and body[s+1] % 4 == 0 and 0 < body[s+1] <= 0xFC:
-                start = s; break
-        else:
-            start = 7
-        blocks, end = walk(body, start)
-        types = Counter(bb[1] for bb in blocks)
-        print(f"\n=== {name} === body={len(body)} N={len(b.samples['Tran'])}  start={start}")
-        print(f"  total blocks: {len(blocks)}, walk ended at {end}/{len(body)}")
-        print(f"  type counts: {dict(types)}")
-        if blocks and blocks[-1][1] == "??":
-            print(f"  stopped at byte: 0x{blocks[-1][2]:02x}, prev 5 blocks: {[(bb[0], bb[1], bb[2]) for bb in blocks[-6:-1]]}")
-        # Sum payload sizes by type
-        payload_sizes = {t: sum(len(bb[3]) for bb in blocks if bb[1] == t) for t in types}
-        print(f"  payload bytes by type: {payload_sizes}")
-
-
-if __name__ == "__main__":
-    main()

analysis/walk_v6.py

@@ -1,68 +0,0 @@
-"""
-Walker v6: handle 40 02 blocks correctly (length 20).
-
-Block formats:
-- [10 NN]: 4-bit nibble delta data, length = NN/2 + 2
-- [20 NN]: int8 literal data, length = NN + 2
-- [00 NN]: 2-byte marker
-- [30 NN]: trailer/summary block, length = NN*4
-- [40 02]: segment header, fixed length 20
-"""
-import sys
-sys.path.insert(0, ".")
-from analysis.load_bundle import load_bundle
-from collections import Counter
-
-
-def walk(body, start, max_blocks=10000):
-    i = start
-    blocks = []
-    while i + 1 < len(body) and len(blocks) < max_blocks:
-        t0 = body[i]
-        t1 = body[i + 1]
-        if t0 == 0x10 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 // 2 + 2
-        elif t0 == 0x20 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 + 2
-        elif t0 == 0x00 and t1 % 4 == 0:
-            length = 2
-        elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0x10:
-            length = t1 * 4
-        elif t0 == 0x40 and t1 == 0x02:
-            length = 20
-        else:
-            blocks.append((i, "??", t0, bytes(body[i:i+8]), 0))
-            break
-        if i + length > len(body):
-            break
-        data = bytes(body[i + 2 : i + length])
-        blocks.append((i, f"{t0:02x}", t1, data, length))
-        i += length
-    return blocks, i
-
-
-def main():
-    for name in ("event-c", "event-d", "event-a", "event-b"):
-        b = load_bundle(name)
-        body = b.body
-        for s in range(15):
-            if body[s] == 0x10 and body[s+1] % 4 == 0 and 0 < body[s+1] <= 0xFC:
-                start = s; break
-        else:
-            start = 7
-        blocks, end = walk(body, start)
-        types = Counter(bb[1] for bb in blocks)
-        print(f"\n=== {name} === body={len(body)} N={len(b.samples['Tran'])}  start={start}")
-        print(f"  total blocks: {len(blocks)}, walk ended at {end}/{len(body)}")
-        print(f"  type counts: {dict(types)}")
-        if blocks and blocks[-1][1] == "??":
-            print(f"  stopped at byte: 0x{blocks[-1][2]:02x} at offset {blocks[-1][0]}")
-            print(f"  prev 5 blocks: {[(bb[0], bb[1], bb[2]) for bb in blocks[-6:-1]]}")
-            print(f"  bytes around stop: {body[end-4:end+24].hex(' ')}")
-        # Sum
-        payload_sizes = {t: sum(len(bb[3]) for bb in blocks if bb[1] == t) for t in types}
-        print(f"  payload bytes by type: {payload_sizes}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/corpus_accuracy.py

@@ -1,65 +0,0 @@
-"""Run read_idf_file across the corpus and report per-channel accuracy vs sidecars."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from micromate.idf_file import read_idf_file
-from analysis_idf.recon import load_sidecar_samples
-
-
-def sidecar_path(idfw: Path) -> Path:
-    return idfw.parent / "TXT" / f"{idfw.name}.txt"
-
-
-def main():
-    root = REPO / "tests/fixtures/THORDATA_example"
-    files = [f for f in root.rglob("*.IDFW") if not str(f).endswith(".CDB")]
-    files.sort()
-    GEO_LSB = 0.0003
-
-    n_ok = n_skip = 0
-    overall = {"Tran": [], "Vert": [], "Long": []}
-
-    for f in files:
-        try:
-            res = read_idf_file(f)
-        except Exception:
-            n_skip += 1
-            continue
-        sc_path = sidecar_path(f)
-        if not sc_path.exists():
-            n_skip += 1
-            continue
-        try:
-            sc = load_sidecar_samples(sc_path)
-        except Exception:
-            n_skip += 1
-            continue
-
-        per_file = {}
-        for ch in ("Tran", "Vert", "Long"):
-            sc_counts = [int(round(v / GEO_LSB)) for v in sc[ch]]
-            dec = res.samples.get(ch, [])
-            n = min(len(sc_counts), len(dec))
-            if n == 0:
-                per_file[ch] = 0.0
-                continue
-            exact = sum(1 for i in range(n) if sc_counts[i] == dec[i])
-            pct = 100.0 * exact / n
-            per_file[ch] = pct
-            overall[ch].append(pct)
-        n_ok += 1
-
-    print(f"Processed {n_ok} files (skipped {n_skip})")
-    print("Per-channel exact-match % (mean / min / max):")
-    for ch, vals in overall.items():
-        if vals:
-            avg = sum(vals) / len(vals)
-            print(f"  {ch}: mean={avg:.2f}%  min={min(vals):.2f}%  max={max(vals):.2f}%  n={len(vals)}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/diff_trail.py

@@ -1,49 +0,0 @@
-"""Find where decoded-vs-sidecar diverges for each channel."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from minimateplus.waveform_codec import decode_waveform_v2
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def main():
-    buf = TARGET.read_bytes()
-    sc = load_sidecar_samples(TXT)
-    decoded = decode_waveform_v2(buf[0x0f1f:])
-    GEO_LSB = 0.0003
-
-    for ch in ("Tran", "Vert", "Long"):
-        sc_counts = [int(round(v / GEO_LSB)) for v in sc[ch]]
-        dec = decoded[ch]
-        # Find ALL transitions where mismatches start/stop
-        first_diff = next((i for i in range(len(dec)) if dec[i] != sc_counts[i]), None)
-        if first_diff is None:
-            print(f"{ch}: NO MISMATCHES")
-            continue
-        print(f"{ch}: first diff at idx {first_diff}")
-        # Show 5 before, 5 after
-        for i in range(max(0, first_diff - 3), min(len(dec), first_diff + 8)):
-            mark = "  " if dec[i] == sc_counts[i] else "**"
-            print(f"  {mark} idx {i:4d}: sc={sc_counts[i]:6d}  dec={dec[i]:6d}  diff={dec[i]-sc_counts[i]:+d}")
-        # Where does cumulative diff exceed 100?
-        cum_match_run = 0
-        max_match_run = 0
-        match_run_start = 0
-        diff_count = 0
-        for i in range(len(dec)):
-            if dec[i] == sc_counts[i]:
-                cum_match_run += 1
-                max_match_run = max(max_match_run, cum_match_run)
-            else:
-                cum_match_run = 0
-                diff_count += 1
-        print(f"  total mismatches: {diff_count}/{len(dec)}, longest run of matches: {max_match_run}")
-        print()
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/e2e_idfh.py

@@ -1,48 +0,0 @@
-"""End-to-end IDFH ingest verification."""
-from __future__ import annotations
-import sys
-import tempfile
-import json
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from sfm.waveform_store import WaveformStore
-
-
-def main():
-    idfh = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM13981/UM13981_20220805075441.IDFH"
-    txt  = idfh.parent / "TXT" / f"{idfh.name}.txt"
-
-    with tempfile.TemporaryDirectory() as td:
-        store = WaveformStore(Path(td))
-        ev, rec = store.save_imported_idf(
-            idfh.read_bytes(),
-            idfh,
-            idf_report_text=txt.read_text(errors="replace"),
-        )
-        print("=== save_imported_idf (IDFH) ===")
-        print(f"  serial:        {rec['serial']}")
-        print(f"  filename:      {rec['filename']}")
-        print(f"  filesize:      {rec['filesize']}")
-        print(f"  h5:            {rec['hdf5_filename']}")  # expect None for histogram
-        print(f"  sidecar:       {rec['sidecar_filename']}")
-        print()
-        print("=== Event ===")
-        print(f"  timestamp:     {ev.timestamp}")
-        print(f"  record_type:   {ev.record_type}")
-        print(f"  sample_rate:   {ev.sample_rate}")
-        print()
-        # Inspect sidecar to confirm intervals were stashed
-        sc_path = Path(td) / "UM13981" / f"{idfh.name}.sfm.json"
-        sc = json.loads(sc_path.read_text())
-        intervals = sc.get("extensions", {}).get("idf_intervals", [])
-        print(f"  sidecar intervals: {len(intervals)}")
-        if intervals:
-            print(f"  first interval:    {intervals[0]}")
-            print(f"  last interval:     {intervals[-1]}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/e2e_no_txt.py

@@ -1,40 +0,0 @@
-"""Verify the had_report=False path: ingest IDFW with no .txt."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-import tempfile
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from sfm.waveform_store import WaveformStore
-
-
-def main():
-    idfw = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162723.IDFW"
-    with tempfile.TemporaryDirectory() as td:
-        store = WaveformStore(Path(td))
-        ev, rec = store.save_imported_idf(
-            idfw.read_bytes(),
-            idfw,
-            serial_hint=None,
-            idf_report_text=None,        # ← no .txt!
-        )
-        print("=== IDFW without .txt ingest ===")
-        print(f"  serial:        {rec['serial']}")
-        print(f"  timestamp:     {ev.timestamp}")
-        print(f"  sample_rate:   {ev.sample_rate}")
-        print(f"  record_type:   {ev.record_type}")
-        print(f"  rectime_sec:   {ev.rectime_seconds}")
-        nT = len(ev.raw_samples.get('Tran', [])) if ev.raw_samples else 0
-        nV = len(ev.raw_samples.get('Vert', [])) if ev.raw_samples else 0
-        nL = len(ev.raw_samples.get('Long', [])) if ev.raw_samples else 0
-        nM = len(ev.raw_samples.get('MicL', [])) if ev.raw_samples else 0
-        print(f"  raw_samples:   Tran={nT} Vert={nV} Long={nL} MicL={nM}")
-        if ev.peak_values:
-            print(f"  peak_values:   tran={ev.peak_values.tran} vert={ev.peak_values.vert} long={ev.peak_values.long}")
-        print(f"  h5 written:    {rec['hdf5_filename']}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/e2e_report.py

@@ -1,102 +0,0 @@
-"""End-to-end Thor report PDF rendering.
-
-Ingests an IDFW + .txt via save_imported_idf, runs gather_report_data
-(faking a minimal DB row), and renders the PDF to disk.
-"""
-from __future__ import annotations
-import sys
-import tempfile
-import json
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from sfm.waveform_store import WaveformStore
-from sfm import report_pdf
-
-
-class FakeDb:
-    """Stand-in for SeismoDb.get_event(); the renderer only needs a few cols."""
-    def __init__(self, event):
-        self.event = event
-
-    def get_event(self, _id):
-        return self.event
-
-
-def main():
-    base = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719"
-    idfw = base / "UM11719_20231219162723.IDFW"
-    txt  = base / "TXT" / f"{idfw.name}.txt"
-
-    with tempfile.TemporaryDirectory() as td:
-        store = WaveformStore(Path(td))
-        ev, rec = store.save_imported_idf(
-            idfw.read_bytes(),
-            idfw,
-            idf_report_text=txt.read_text(errors="replace"),
-        )
-        print(f"save_imported_idf: h5={rec['hdf5_filename']}, sidecar={rec['sidecar_filename']}")
-
-        # Verify sidecar has bw_report block
-        sc_path = Path(td) / "UM11719" / f"{idfw.name}.sfm.json"
-        sc = json.loads(sc_path.read_text())
-        bw = sc.get("bw_report", {})
-        print(f"  bw_report.available: {bw.get('available')}")
-        print(f"  bw_report.peaks.tran.ppv_ips: {bw.get('peaks', {}).get('tran', {}).get('ppv_ips')}")
-        print(f"  bw_report.mic.pspl_dbl: {bw.get('mic', {}).get('pspl_dbl')}")
-        print(f"  bw_report.histogram.n_intervals: {bw.get('histogram', {}).get('n_intervals')}")
-
-        # Build a DB-row-shaped dict from the Event for gather_report_data
-        import datetime
-        ts = ev.timestamp
-        ts_iso = None
-        if ts is not None:
-            try:
-                ts_iso = datetime.datetime(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second).isoformat()
-            except Exception:
-                pass
-        fake_row = {
-            "serial":              "UM11719",
-            "blastware_filename":  rec["filename"],
-            "record_type":         "Waveform",
-            "timestamp":           ts_iso,
-            "sample_rate":         ev.sample_rate,
-            "project":             ev.project_info.project if ev.project_info else None,
-            "client":              ev.project_info.client  if ev.project_info else None,
-            "operator":            ev.project_info.operator if ev.project_info else None,
-            "sensor_location":     ev.project_info.sensor_location if ev.project_info else None,
-            "created_at":          None,
-        }
-
-        rd = report_pdf.gather_report_data(FakeDb(fake_row), store, event_id="test-1")
-        print()
-        print(f"=== ReportData ===")
-        print(f"  event_id:           {rd.event_id}")
-        print(f"  serial:             {rd.serial}")
-        print(f"  record_type:        {rd.record_type}")
-        print(f"  event_datetime:     {rd.event_datetime_str}")
-        print(f"  trigger:            {rd.trigger_source}")
-        print(f"  geo_range:          {rd.geo_range_str}")
-        print(f"  sample_rate:        {rd.sample_rate_str}")
-        print(f"  firmware:           {rd.firmware}")
-        print(f"  calibration:        {rd.calibration_date} by {rd.calibration_by}")
-        print(f"  battery:            {rd.battery_volts}")
-        print(f"  PVS:                {rd.peak_vector_sum_ips} in/s at {rd.peak_vector_sum_time_s} sec")
-        print(f"  mic_pspl_dbl:       {rd.mic_pspl_dbl}")
-        print(f"  mic_zc_freq_hz:     {rd.mic_zc_freq_hz}")
-        print(f"  channel_stats:      {len(rd.channel_stats)} rows")
-        for cs in rd.channel_stats:
-            print(f"    {cs['name']}: PPV={cs['ppv_ips']} ZC={cs['zc_freq_hz']} ToP={cs['time_of_peak_s']} Acc={cs['peak_accel_g']} Disp={cs['peak_disp_in']} Test={cs['sensor_check']}")
-
-        # Render the PDF
-        out_path = REPO / "analysis_idf" / "thor_report.pdf"
-        pdf_bytes = report_pdf.render_event_report_pdf(rd)
-        out_path.write_bytes(pdf_bytes)
-        print()
-        print(f"  PDF written: {out_path} ({len(pdf_bytes)} bytes)")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/e2e_report_idfh.py

@@ -1,91 +0,0 @@
-"""End-to-end Thor IDFH histogram report PDF rendering."""
-from __future__ import annotations
-import sys
-import tempfile
-import json
-import datetime
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from sfm.waveform_store import WaveformStore
-from sfm import report_pdf
-
-
-class FakeDb:
-    def __init__(self, event):
-        self.event = event
-
-    def get_event(self, _id):
-        return self.event
-
-
-def main():
-    # Use the multi-interval IDFH (81 + trigger row)
-    idfh = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM13981/UM13981_20220805075441.IDFH"
-    txt  = idfh.parent / "TXT" / f"{idfh.name}.txt"
-
-    with tempfile.TemporaryDirectory() as td:
-        store = WaveformStore(Path(td))
-        ev, rec = store.save_imported_idf(
-            idfh.read_bytes(),
-            idfh,
-            idf_report_text=txt.read_text(errors="replace"),
-        )
-        print(f"save_imported_idf: h5={rec['hdf5_filename']}, sidecar={rec['sidecar_filename']}")
-
-        sc_path = Path(td) / "UM13981" / f"{idfh.name}.sfm.json"
-        sc = json.loads(sc_path.read_text())
-        bw = sc.get("bw_report", {})
-        hist = bw.get("histogram", {})
-        print(f"  bw_report.histogram.start:           {hist.get('start')}")
-        print(f"  bw_report.histogram.stop:            {hist.get('stop')}")
-        print(f"  bw_report.histogram.n_intervals:     {hist.get('n_intervals')}")
-        print(f"  bw_report.histogram.interval_size:   {hist.get('interval_size')}")
-        print(f"  bw_report.histogram.interval_size_s: {hist.get('interval_size_s')}")
-        print(f"  bw_report.peaks.tran.ppv_ips:        {bw.get('peaks', {}).get('tran', {}).get('ppv_ips')}")
-
-        ts = ev.timestamp
-        ts_iso = None
-        if ts is not None:
-            try:
-                ts_iso = datetime.datetime(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second).isoformat()
-            except Exception:
-                pass
-        fake_row = {
-            "serial":              "UM13981",
-            "blastware_filename":  rec["filename"],
-            "record_type":         "Histogram",
-            "timestamp":           ts_iso,
-            "sample_rate":         ev.sample_rate,
-            "project":             ev.project_info.project if ev.project_info else None,
-            "client":              ev.project_info.client  if ev.project_info else None,
-            "operator":            ev.project_info.operator if ev.project_info else None,
-            "sensor_location":     ev.project_info.sensor_location if ev.project_info else None,
-            "created_at":          None,
-        }
-        rd = report_pdf.gather_report_data(FakeDb(fake_row), store, event_id="hist-1")
-
-        print()
-        print("=== ReportData (histogram) ===")
-        print(f"  is_histogram:           {rd.is_histogram}")
-        print(f"  histogram_start:        {rd.histogram_start_str}")
-        print(f"  histogram_stop:         {rd.histogram_stop_str}")
-        print(f"  histogram_n_intervals:  {rd.histogram_n_intervals}")
-        print(f"  histogram_interval_size:{rd.histogram_interval_size}")
-        print(f"  histogram_interval_times[:3]: {rd.histogram_interval_times[:3]}")
-        print(f"  histogram_interval_times[-2:]: {rd.histogram_interval_times[-2:]}")
-        print(f"  channel_stats: {len(rd.channel_stats)} rows")
-        for cs in rd.channel_stats:
-            print(f"    {cs['name']}: PPV={cs['ppv_ips']} ZC={cs['zc_freq_hz']} peak_date={cs['peak_date']} peak_time={cs['peak_time']}")
-
-        pdf_bytes = report_pdf.render_event_report_pdf(rd)
-        out_path = REPO / "analysis_idf" / "thor_report_idfh.pdf"
-        out_path.write_bytes(pdf_bytes)
-        print()
-        print(f"  PDF written: {out_path} ({len(pdf_bytes)} bytes)")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/e2e_save_idf.py

@@ -1,52 +0,0 @@
-"""End-to-end ingest test: feed an IDFW + .txt to save_imported_idf in a tmp store."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-import tempfile
-import shutil
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from sfm.waveform_store import WaveformStore
-
-
-def main():
-    idfw = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162723.IDFW"
-    txt  = idfw.parent / "TXT" / f"{idfw.name}.txt"
-
-    with tempfile.TemporaryDirectory() as td:
-        store = WaveformStore(Path(td))
-        ev, rec = store.save_imported_idf(
-            idfw.read_bytes(),
-            idfw,
-            serial_hint=None,
-            idf_report_text=txt.read_text(errors="replace"),
-        )
-        print("=== Save result ===")
-        print(f"  serial:    {rec['serial']}")
-        print(f"  filename:  {rec['filename']}")
-        print(f"  filesize:  {rec['filesize']}")
-        print(f"  h5:        {rec['hdf5_filename']}")
-        print(f"  sidecar:   {rec['sidecar_filename']}")
-        print()
-        print("=== Event ===")
-        print(f"  serial:        {ev.serial if hasattr(ev,'serial') else '(n/a)'}")
-        print(f"  timestamp:     {ev.timestamp}")
-        print(f"  sample_rate:   {ev.sample_rate}")
-        print(f"  record_type:   {ev.record_type}")
-        print(f"  rectime_sec:   {ev.rectime_seconds}")
-        print(f"  raw_samples:   Tran={len(ev.raw_samples.get('Tran', [])) if ev.raw_samples else 0}, Vert={len(ev.raw_samples.get('Vert', [])) if ev.raw_samples else 0}, Long={len(ev.raw_samples.get('Long', [])) if ev.raw_samples else 0}, MicL={len(ev.raw_samples.get('MicL', [])) if ev.raw_samples else 0}")
-        if ev.peak_values:
-            print(f"  peaks (txt):   Tran={ev.peak_values.tran} Vert={ev.peak_values.vert} Long={ev.peak_values.long}")
-        print()
-
-        # Verify the h5 file actually got written
-        h5path = Path(td) / "UM11719" / f"{idfw.name}.h5"
-        print(f"  h5 exists:     {h5path.exists()}  size={h5path.stat().st_size if h5path.exists() else 0}")
-        sidecar = Path(td) / "UM11719" / f"{idfw.name}.sfm.json"
-        print(f"  sidecar exists:{sidecar.exists()}  size={sidecar.stat().st_size if sidecar.exists() else 0}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/idfh_decode.py

@@ -1,137 +0,0 @@
-"""Decode IDFH histogram intervals + verify against sidecar."""
-from __future__ import annotations
-import sys
-import struct
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-
-SEGMENT_MAGIC = b"\x02\xda\x0a\x00\x00\x00"
-SEGMENT_SIZE = 732   # = 10-byte header + 10 × 72-byte intervals + 2-byte tail
-INTERVAL_SIZE = 72
-CHANNELS = ("Tran", "Vert", "Long", "MicL")
-
-
-def decode_interval(buf72: bytes) -> dict:
-    """Decode one 72-byte interval into per-channel min/max/halfp."""
-    out = {}
-    for i, ch in enumerate(CHANNELS):
-        block = buf72[i*16 : (i+1)*16]
-        mn = struct.unpack_from(">h", block, 0)[0]
-        mx = struct.unpack_from(">h", block, 2)[0]
-        sb = struct.unpack_from(">h", block, 4)[0]
-        halfp = struct.unpack_from(">H", block, 6)[0]
-        f10 = struct.unpack_from(">H", block, 10)[0]
-        f14 = struct.unpack_from(">H", block, 14)[0]
-        peak_count = max(abs(mn), abs(mx))
-        out[ch] = {
-            "min":     mn,
-            "max":     mx,
-            "field4":  sb,
-            "halfp":   halfp,
-            "field10": f10,
-            "field14": f14,
-            "peak":    peak_count,
-            "freq_hz": (512.0 / halfp) if halfp > 5 else None,
-        }
-    out["_tail"] = buf72[64:].hex(" ")
-    return out
-
-
-def walk_idfh(buf: bytes) -> list:
-    """Walk all interval records in an IDFH file."""
-    intervals = []
-    # Multi-segment file: every 02 da 0a 00 00 00 marker introduces a segment.
-    # Single-interval file: just one body header at 0xf96 of form ?? ?? 0a 00 00 00.
-    # Find them all.
-    i = 0
-    while True:
-        j = buf.find(b"\x0a\x00\x00\x00", i)
-        if j < 0:
-            break
-        # Validate: the 2 bytes before must form a length, and we want bytes
-        # [j-2 : j+6] to have a recognisable shape.  Actually the cleanest
-        # filter is "preceded by a length and followed by 00 NN 05 3f".
-        if j < 2:
-            i = j + 1
-            continue
-        # Body header form: [length_be_2][0a 00 00 00][00 NN][05 3f]
-        if j + 10 > len(buf):
-            break
-        length = int.from_bytes(buf[j-2:j], "big")
-        # Verify the segment-marker shape: [length_be][0a 00 00 00][00 NN][05 3f]
-        if buf[j+4] != 0x00:
-            i = j + 1
-            continue
-        if buf[j+6:j+8] != b"\x05\x3f":
-            i = j + 1
-            continue
-        # Header layout (10 bytes): [length_be 2B][0a 00 00 00 4B][00 NN 2B][05 3f 2B]
-        # Followed by N interval records of 72 bytes each, then 2 tail bytes.
-        # length value = (N × 72) + 10  (counts bytes from 0x0a... through interval data).
-        header_start = j - 2
-        n_intervals = (length - 10) // INTERVAL_SIZE
-        interval_start = header_start + 10
-        for k in range(n_intervals):
-            off = interval_start + k * INTERVAL_SIZE
-            if off + INTERVAL_SIZE > len(buf):
-                break
-            chunk = buf[off:off + INTERVAL_SIZE]
-            intervals.append({"offset": off, **decode_interval(chunk)})
-        i = header_start + length + 2
-    return intervals
-
-
-def main():
-    # Test against multi-segment IDFH
-    target = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM13981/UM13981_20220805075441.IDFH"
-    sc_path = target.parent / "TXT" / f"{target.name}.txt"
-    buf = target.read_bytes()
-    intervals = walk_idfh(buf)
-    print(f"=== {target.name} ===")
-    print(f"  file size: {len(buf)}")
-    print(f"  decoded intervals: {len(intervals)}")
-    # Show first 2 + last 2
-    sc_rows = []
-    for line in sc_path.read_text(errors="replace").splitlines():
-        if line.startswith("2022-") or line.startswith("2023-"):
-            sc_rows.append(line)
-    print(f"  sidecar rows: {len(sc_rows)}")
-
-    print()
-    for k in [0, 1, 78, 79, 80]:
-        if k >= len(intervals):
-            continue
-        iv = intervals[k]
-        print(f"--- interval {k} @0x{iv['offset']:04x} ---")
-        for ch in CHANNELS:
-            d = iv[ch]
-            peak_ips = d["peak"] / 32768 * 10.0
-            print(f"  {ch}: peak={d['peak']:5d} ({peak_ips:.4f} in/s)  halfp={d['halfp']:5d}  freq={d['freq_hz']}")
-        # sidecar row
-        if k < len(sc_rows):
-            print(f"  SC: {sc_rows[k]}")
-
-    # Test single-interval IDFH
-    print()
-    target2 = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162648.IDFH"
-    sc2 = target2.parent / "TXT" / f"{target2.name}.txt"
-    buf2 = target2.read_bytes()
-    intervals2 = walk_idfh(buf2)
-    print(f"=== {target2.name} ===")
-    print(f"  file size: {len(buf2)}, decoded intervals: {len(intervals2)}")
-    if intervals2:
-        iv = intervals2[0]
-        for ch in CHANNELS:
-            d = iv[ch]
-            peak_ips = d["peak"] / 32768 * 10.0
-            print(f"  {ch}: peak={d['peak']:5d} ({peak_ips:.4f} in/s)  halfp={d['halfp']:5d}  freq={d['freq_hz']}")
-        sc_rows2 = [l for l in sc2.read_text(errors='replace').splitlines() if l.startswith("2023-")]
-        if sc_rows2:
-            print(f"  SC: {sc_rows2[0]}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/idfh_period.py

@@ -1,41 +0,0 @@
-"""Find IDFH interval period via auto-correlation of structural patterns."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-from collections import Counter
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-
-def main():
-    target = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM13981/UM13981_20220805075441.IDFH"
-    buf = target.read_bytes()
-    body_start = 0xF96
-    body_end   = 0x270C
-    body = buf[body_start:body_end]
-    print(f"body size: {len(body)} bytes (file {len(buf)} bytes)")
-
-    # For each candidate interval size, count how many bytes at fixed offsets within
-    # each interval are zero (consistent column-zero pattern indicates correct size).
-    print()
-    print("=== zero-column score by interval size (higher = more likely) ===")
-    best = []
-    for sz in range(16, 100):
-        n = len(body) // sz
-        if n < 30:
-            continue
-        # For each column position within an interval, count how many of n intervals have zero
-        score = 0
-        for col in range(sz):
-            zeros = sum(1 for i in range(n) if body[i*sz + col] == 0)
-            if zeros >= n * 0.9:
-                score += 1
-        best.append((score, sz, n))
-    best.sort(reverse=True)
-    for score, sz, n in best[:10]:
-        print(f"  size={sz:3d}  n_intervals={n}  consistently-zero-cols={score}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/per_file_detail.py

@@ -1,40 +0,0 @@
-"""Per-file accuracy + sample-count details."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from micromate.idf_file import read_idf_file
-from analysis_idf.recon import load_sidecar_samples
-
-
-def main():
-    root = REPO / "tests/fixtures/THORDATA_example"
-    files = sorted([f for f in root.rglob("*.IDFW") if not str(f).endswith(".CDB")])
-    GEO_LSB = 0.0003
-    # Limit to first 15 successful files for detail.
-    shown = 0
-    for f in files:
-        try:
-            res = read_idf_file(f)
-        except Exception:
-            continue
-        sc_path = f.parent / "TXT" / f"{f.name}.txt"
-        if not sc_path.exists():
-            continue
-        sc = load_sidecar_samples(sc_path)
-        sc_tran = [int(round(v / GEO_LSB)) for v in sc["Tran"]]
-        dec = res.samples.get("Tran", [])
-        n = min(len(sc_tran), len(dec))
-        exact = sum(1 for i in range(n) if sc_tran[i] == dec[i]) if n else 0
-        pct = 100.0 * exact / n if n else 0.0
-        print(f"{f.name:40s}  size={f.stat().st_size:6d}  sc_n={len(sc_tran):4d}  dec_n={len(dec):4d}  exact={pct:.1f}%")
-        shown += 1
-        if shown >= 20:
-            break
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/probe_boundary.py

@@ -1,64 +0,0 @@
-"""Look at what's at the divergence boundary."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from minimateplus.waveform_codec import walk_body, find_data_start, parse_segment_header
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def main():
-    buf = TARGET.read_bytes()
-    body = buf[0x0f1f:]
-    start = find_data_start(body)
-    print(f"data_start: {start}  (= file offset 0x{0x0f1f + start:04x})")
-
-    blocks = walk_body(body, start)
-    print(f"{len(blocks)} blocks total")
-    print()
-
-    # First 25 blocks
-    print("=== first 30 blocks ===")
-    for i, b in enumerate(blocks[:30]):
-        body_off = 0x0f1f + b.offset
-        if b.tag_hi == 0x40:
-            hdr = parse_segment_header(b)
-            print(f"  [{i:3d}] @0x{body_off:04x}  {b.kind}  (segment header)  counter={hdr['counter'] if hdr else '?'}  field2={hdr['field2'].hex() if hdr else '?'}  anchor={hdr['anchor_bytes'].hex() if hdr else '?'}  tail={hdr['tail'].hex() if hdr else '?'}")
-        else:
-            print(f"  [{i:3d}] @0x{body_off:04x}  {b.kind}  len={b.length}  data={b.data[:16].hex()}")
-    print()
-
-    # Cumulative sample counts per block to find which block contains sample 254
-    print("=== cumulative samples through blocks ===")
-    cur_ch = "Tran"
-    rotation = ["Vert", "Long", "MicL", "Tran"]
-    seg_count = 0
-    samples_in_curseg = 2  # preamble Tran[0], Tran[1]
-    for i, b in enumerate(blocks[:30]):
-        if b.tag_hi == 0x40:
-            seg_count += 1
-            prev_ch = cur_ch
-            cur_ch = rotation[(seg_count - 1) % 4]
-            print(f"  [{i:3d}] 40 02 -> end of {prev_ch} segment, start {cur_ch} (segment {seg_count})")
-            samples_in_curseg = 2  # anchors
-        elif (b.tag_hi & 0xF0) == 0x10:
-            nn = ((b.tag_hi & 0x0F) << 8) | b.tag_lo
-            samples_in_curseg += nn
-            print(f"  [{i:3d}] {b.kind} nibble: +{nn} samples, ch={cur_ch}, ch_total~{samples_in_curseg}")
-        elif (b.tag_hi & 0xF0) == 0x20:
-            nn = ((b.tag_hi & 0x0F) << 8) | b.tag_lo
-            samples_in_curseg += nn
-            print(f"  [{i:3d}] {b.kind} int8: +{nn} samples, ch={cur_ch}, ch_total~{samples_in_curseg}")
-        elif b.tag_hi == 0x00:
-            samples_in_curseg += b.tag_lo
-            print(f"  [{i:3d}] {b.kind} RLE: +{b.tag_lo}, ch={cur_ch}, ch_total~{samples_in_curseg}")
-        elif b.tag_hi == 0x30:
-            samples_in_curseg += b.tag_lo
-            print(f"  [{i:3d}] {b.kind} packed12: +{b.tag_lo} samples, ch={cur_ch}, ch_total~{samples_in_curseg}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/recon.py

@@ -1,89 +0,0 @@
-"""Reconnaissance helpers for cracking the Thor IDFW binary."""
-from __future__ import annotations
-
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-TARGET = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162723.IDFW"
-TXT = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/TXT/UM11719_20231219162723.IDFW.txt"
-
-
-def hex_at(buf: bytes, off: int, n: int = 32) -> str:
-    chunk = buf[off : off + n]
-    hexs = " ".join(f"{b:02x}" for b in chunk)
-    asc = "".join(chr(b) if 32 <= b < 127 else "." for b in chunk)
-    return f"{off:04x}: {hexs}  {asc}"
-
-
-def find_all(buf: bytes, needle: bytes) -> list[int]:
-    out: list[int] = []
-    i = 0
-    while True:
-        j = buf.find(needle, i)
-        if j < 0:
-            break
-        out.append(j)
-        i = j + 1
-    return out
-
-
-def load_sidecar_samples(path: Path) -> dict[str, list[float]]:
-    """Parse the txt sample table — Tran/Vert/Long/MicL."""
-    out = {"Tran": [], "Vert": [], "Long": [], "MicL": []}
-    in_block = False
-    for line in path.read_text(errors="replace").splitlines():
-        if not in_block:
-            if line.strip() == "Waveform Data Channels":
-                in_block = True
-            continue
-        if line.startswith("Waveform Data USB Channels"):
-            break
-        parts = line.split("\t")
-        # First row is the header "\tTran\tVert\tLong\tMicL"
-        if len(parts) >= 5 and parts[1] == "Tran":
-            continue
-        if len(parts) < 5:
-            continue
-        try:
-            out["Tran"].append(float(parts[1]))
-            out["Vert"].append(float(parts[2]))
-            out["Long"].append(float(parts[3]))
-            out["MicL"].append(float(parts[4]))
-        except ValueError:
-            continue
-    return out
-
-
-def main():
-    buf = TARGET.read_bytes()
-    samples = load_sidecar_samples(TXT)
-    print(f"file size: {len(buf)} bytes")
-    print(f"sample rows: Tran={len(samples['Tran'])} Vert={len(samples['Vert'])} Long={len(samples['Long'])} MicL={len(samples['MicL'])}")
-    print(f"first 6 Tran samples: {samples['Tran'][:6]}")
-    print(f"first 6 Vert samples: {samples['Vert'][:6]}")
-    print(f"first 6 Long samples: {samples['Long'][:6]}")
-    print(f"first 6 MicL samples: {samples['MicL'][:6]}")
-
-    print()
-    print("=== BW magic '00 02 00' positions ===")
-    hits = find_all(buf, b"\x00\x02\x00")
-    print(f"{len(hits)} hits")
-    for h in hits[:20]:
-        print(hex_at(buf, h, 24))
-
-    print()
-    print("=== '40 02' segment-header positions ===")
-    hits = find_all(buf, b"\x40\x02")
-    print(f"{len(hits)} hits")
-    for h in hits:
-        ctx_pre = buf[max(0, h - 4): h].hex()
-        ctx_post = buf[h: h + 20].hex()
-        # Show byte preceding to help identify real headers vs casual occurrences
-        print(f"  0x{h:04x}  pre={ctx_pre}  post={ctx_post}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/seg_resync.py

@@ -1,40 +0,0 @@
-"""Find each segment boundary in the channel and check if errors reset there."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from minimateplus.waveform_codec import decode_waveform_v2
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def main():
-    buf = TARGET.read_bytes()
-    sc = load_sidecar_samples(TXT)
-    decoded = decode_waveform_v2(buf[0x0f1f:])
-    GEO_LSB = 0.0003
-
-    for ch in ("Tran", "Vert", "Long"):
-        sc_counts = [int(round(v / GEO_LSB)) for v in sc[ch]]
-        dec = decoded[ch]
-        # Find every transition where error becomes zero from nonzero (or grows from zero)
-        # Print indices where dec resyncs back to exact match.
-        n = min(len(sc_counts), len(dec))
-        events = []
-        prev_match = True
-        for i in range(n):
-            match = sc_counts[i] == dec[i]
-            if match != prev_match:
-                kind = "RESYNC" if match else "DIVERGE"
-                events.append((i, kind, sc_counts[i], dec[i]))
-                prev_match = match
-        print(f"{ch}: {len(events)} transitions")
-        for i, kind, sc_v, dec_v in events[:20]:
-            print(f"  idx {i:4d}  {kind:8s}  sc={sc_v:6d}  dec={dec_v:6d}  diff={dec_v-sc_v:+d}")
-        print()
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/smoke_idfh.py

@@ -1,46 +0,0 @@
-"""Smoke-test read_idf_file on IDFH across the corpus."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from micromate.idf_file import read_idf_file
-
-
-def main():
-    target = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162648.IDFH"
-    result = read_idf_file(target)
-    ev = result.event
-    print(f"=== {target.name} ===")
-    print(f"  signature:   {result.signature}")
-    print(f"  serial:      {ev.serial}")
-    print(f"  timestamp:   {ev.timestamp}")
-    print(f"  sample_rate: {ev.sample_rate}")
-    print(f"  kind:        {ev.kind}")
-    print(f"  intervals:   {len(result.intervals or [])}")
-    print(f"  peaks:       T={ev.peaks.transverse_ips:.4f} V={ev.peaks.vertical_ips:.4f} L={ev.peaks.longitudinal_ips:.4f}")
-    print()
-
-    root = REPO / "tests/fixtures/THORDATA_example"
-    files = list(root.rglob("*.IDFH"))
-    ok = fail = nyi = 0
-    total_intervals = 0
-    for f in files:
-        try:
-            r = read_idf_file(f)
-            ok += 1
-            total_intervals += len(r.intervals or [])
-        except NotImplementedError:
-            nyi += 1
-        except Exception as exc:
-            fail += 1
-            if fail <= 3:
-                print(f"  FAIL: {f.name}: {type(exc).__name__}: {exc}")
-    print(f"Corpus: {len(files)} IDFH files | ok={ok} fail={fail} nyi={nyi}")
-    print(f"Total intervals decoded: {total_intervals}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/smoke_test.py

@@ -1,48 +0,0 @@
-"""Smoke-test read_idf_file across the sample corpus."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from micromate.idf_file import read_idf_file, geo_count_to_ips, mic_count_to_psi
-
-
-def main():
-    target = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719/UM11719_20231219162723.IDFW"
-    result = read_idf_file(target)
-    ev = result.event
-    print(f"=== {target.name} ===")
-    print(f"  signature: {result.signature}")
-    print(f"  serial:    {ev.serial}")
-    print(f"  timestamp: {ev.timestamp}")
-    print(f"  sample_rate: {ev.sample_rate}")
-    print(f"  record_time: {ev.record_time_sec}")
-    print(f"  calibration: {result.binary_metadata.calibration_date}")
-    print(f"  Tran samples: {len(result.samples['Tran'])}, peak_ips={ev.peaks.transverse_ips:.4f}")
-    print(f"  Vert samples: {len(result.samples['Vert'])}, peak_ips={ev.peaks.vertical_ips:.4f}")
-    print(f"  Long samples: {len(result.samples['Long'])}, peak_ips={ev.peaks.longitudinal_ips:.4f}")
-    print(f"  MicL samples: {len(result.samples['MicL'])}")
-    print()
-
-    # Corpus sweep
-    root = REPO / "tests/fixtures/THORDATA_example"
-    files = [f for f in root.rglob("*.IDFW") if not str(f).endswith(".CDB")]
-    ok = fail = nyi = 0
-    for f in files:
-        try:
-            r = read_idf_file(f)
-            ok += 1
-        except NotImplementedError:
-            nyi += 1
-        except Exception as exc:
-            fail += 1
-            if fail <= 5:
-                print(f"  FAIL: {f.name}: {type(exc).__name__}: {exc}")
-    print()
-    print(f"Corpus: {len(files)} IDFW files | ok={ok} fail={fail} not-implemented={nyi}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/test_adapter.py

@@ -1,47 +0,0 @@
-"""Verify build_bw_report_from_idf against a known sidecar."""
-from __future__ import annotations
-import json
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from micromate.idf_ascii_report import parse_idf_report
-from micromate.idf_to_bw_report import build_bw_report_from_idf
-from micromate.idf_file import read_idf_file
-
-
-def show(prefix: str, d: dict, indent: int = 0):
-    for k, v in d.items():
-        if isinstance(v, dict):
-            print(f"{'  '*indent}{prefix}{k}:")
-            show("", v, indent + 1)
-        else:
-            print(f"{'  '*indent}{prefix}{k}: {v!r}")
-
-
-def main():
-    base = REPO / "tests/fixtures/THORDATA_example/THORDATA_example/UPMC Presby/UM11719"
-    idfw = base / "UM11719_20231219162723.IDFW"
-    txt  = base / "TXT" / f"{idfw.name}.txt"
-
-    report_dict = parse_idf_report(txt.read_text(errors="replace"))
-    res = read_idf_file(idfw)
-    bw = build_bw_report_from_idf(report_dict, binary_md=res.binary_metadata)
-
-    print("=== IDFW → bw_report ===")
-    show("", bw)
-
-    print()
-    print("=== IDFH (single trigger row) ===")
-    idfh = base / "UM11719_20231219162648.IDFH"
-    txt_h = base / "TXT" / f"{idfh.name}.txt"
-    rh = parse_idf_report(txt_h.read_text(errors="replace"))
-    res_h = read_idf_file(idfh)
-    bw_h = build_bw_report_from_idf(rh, binary_md=res_h.binary_metadata, intervals=res_h.intervals)
-    show("", bw_h)
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/trace_path.py

@@ -1,73 +0,0 @@
-"""Trace Tran sample-by-sample to find exactly where the codec drifts."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def s4(n: int) -> int:
-    return n if n < 8 else n - 16
-
-
-def i8(b: int) -> int:
-    return b if b < 128 else b - 256
-
-
-def main():
-    buf = TARGET.read_bytes()
-    sc = load_sidecar_samples(TXT)
-    GEO_LSB = 0.0003
-    sc_tran = [int(round(v / GEO_LSB)) for v in sc["Tran"]]
-
-    body = buf[0x0f1f:]
-    # Tran[0], Tran[1] from preamble
-    t0 = int.from_bytes(body[3:5], "big", signed=True)
-    t1 = int.from_bytes(body[5:7], "big", signed=True)
-    print(f"preamble Tran[0]={t0}  Tran[1]={t1}  (sidecar: {sc_tran[0]}, {sc_tran[1]})")
-
-    # Block 0: 10 f8 at body[7:9]
-    print(f"block 0: tag {body[7]:02x} {body[8]:02x}")
-    print(f"  block 0 first 10 data bytes: {body[9:19].hex()}")
-
-    # Walk block 0 manually, comparing each sample
-    cur = t1
-    samples = [t0, t1]
-    block_off = 7
-    nn = body[8]
-    print(f"  NN = {nn}")
-    data = body[9 : 9 + nn // 2]
-    for byi, byte in enumerate(data):
-        for nib_idx, nib in enumerate(((byte >> 4) & 0xF, byte & 0xF)):
-            cur += s4(nib)
-            samples.append(cur)
-            idx = len(samples) - 1
-            if 0 <= idx < len(sc_tran):
-                sc_v = sc_tran[idx]
-                match = "✓" if sc_v == cur else "✗"
-                if idx < 12 or 240 <= idx <= 260:
-                    print(f"    idx {idx:3d}: nibble byte={byte:02x} nib={nib:x} delta={s4(nib):+d}  cur={cur:+d}  sc={sc_v:+d}  {match}")
-
-    print(f"end of block 0: cur={cur}, len(samples)={len(samples)}, decoder expected 250 here")
-    # Block 1: 20 28 starts at offset 9 + 124 = 133 from block_off=7
-    block1_off = 9 + nn // 2
-    print(f"block 1: tag {body[block1_off]:02x} {body[block1_off+1]:02x} (expecting 20 28)")
-    nn1 = body[block1_off + 1]
-    print(f"  block 1 NN = {nn1}")
-    data1 = body[block1_off + 2 : block1_off + 2 + nn1]
-    for byi, byte in enumerate(data1):
-        cur += i8(byte)
-        samples.append(cur)
-        idx = len(samples) - 1
-        if idx < len(sc_tran):
-            sc_v = sc_tran[idx]
-            match = "✓" if sc_v == cur else "✗"
-            if 248 <= idx <= 295:
-                print(f"    idx {idx:3d}: int8 byte={byte:02x} delta={i8(byte):+d}  cur={cur:+d}  sc={sc_v:+d}  {match}")
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/try_codec.py

@@ -1,42 +0,0 @@
-"""Feed candidate body offsets to the BW codec and compare with sidecar."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from minimateplus.waveform_codec import decode_waveform_v2, walk_body, find_data_start
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def main():
-    buf = TARGET.read_bytes()
-    sc = load_sidecar_samples(TXT)
-    # Sidecar samples in 0.0003 counts (Thor geo LSB).
-    sc_tran = [int(round(v / 0.0003)) for v in sc["Tran"][:30]]
-    sc_vert = [int(round(v / 0.0003)) for v in sc["Vert"][:30]]
-    sc_long = [int(round(v / 0.0003)) for v in sc["Long"][:30]]
-    sc_micl = [int(round(v / 1e-6)) for v in sc["MicL"][:30]]  # 1 µ unit for mic? Will iterate.
-    print(f"sidecar Tran (counts): {sc_tran}")
-    print(f"sidecar Vert (counts): {sc_vert}")
-    print(f"sidecar Long (counts): {sc_long}")
-    print(f"sidecar MicL (×1e-6):  {sc_micl}")
-    print()
-
-    # Try candidate body start offsets.
-    for off in (0x0f1f, 0x1057, 0x11f1, 0x1333, 0x1bde, 0x0d30):
-        print(f"=== body @ 0x{off:04x} ===")
-        body = buf[off:]
-        decoded = decode_waveform_v2(body)
-        if not decoded:
-            print("  decode_waveform_v2 returned None")
-            continue
-        for ch in ("Tran", "Vert", "Long", "MicL"):
-            arr = decoded.get(ch, [])
-            print(f"  {ch}[{len(arr)}]: {arr[:20]}")
-        print()
-
-
-if __name__ == "__main__":
-    main()

analysis_idf/verify_full.py

@@ -1,51 +0,0 @@
-"""Verify decode_waveform_v2 against sidecar across all 2304 samples per channel."""
-from __future__ import annotations
-import sys
-from pathlib import Path
-
-REPO = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO))
-
-from minimateplus.waveform_codec import decode_waveform_v2
-from analysis_idf.recon import TARGET, TXT, load_sidecar_samples
-
-
-def main():
-    buf = TARGET.read_bytes()
-    sc = load_sidecar_samples(TXT)
-    body = buf[0x0f1f:]
-    decoded = decode_waveform_v2(body)
-
-    print(f"Sidecar lengths: Tran={len(sc['Tran'])} Vert={len(sc['Vert'])} Long={len(sc['Long'])} MicL={len(sc['MicL'])}")
-    print(f"Decoded lengths: Tran={len(decoded['Tran'])} Vert={len(decoded['Vert'])} Long={len(decoded['Long'])} MicL={len(decoded['MicL'])}")
-    print()
-
-    GEO_LSB = 0.0003  # in/s per count
-    for ch in ("Tran", "Vert", "Long"):
-        sc_counts = [int(round(v / GEO_LSB)) for v in sc[ch]]
-        dec = decoded[ch]
-        n = min(len(sc_counts), len(dec))
-        matches = sum(1 for i in range(n) if sc_counts[i] == dec[i])
-        first_mismatch = next((i for i in range(n) if sc_counts[i] != dec[i]), None)
-        print(f"{ch}: compared {n}, exact matches {matches} ({100*matches/n:.2f}%)")
-        if first_mismatch is not None:
-            i = first_mismatch
-            print(f"  first mismatch at idx {i}: sidecar={sc_counts[i]} ({sc[ch][i]}), decoded={dec[i]}")
-            print(f"  context sidecar[{i-2}..{i+5}]: {sc_counts[max(0,i-2):i+5]}")
-            print(f"  context decoded[{i-2}..{i+5}]: {dec[max(0,i-2):i+5]}")
-
-    # MicL: find the multiplicative factor that fits
-    print()
-    print("=== MicL scale analysis ===")
-    sc_micl = sc["MicL"]
-    dec_micl = decoded["MicL"]
-    # Skip zero values when computing ratio
-    ratios = [sc_micl[i] / dec_micl[i] for i in range(min(50, len(sc_micl), len(dec_micl))) if dec_micl[i] != 0]
-    if ratios:
-        avg = sum(ratios) / len(ratios)
-        print(f"  avg ratio sidecar/decoded over first 50 nonzero: {avg:.4e} (n={len(ratios)})")
-        print(f"  ratios sample: {[f'{r:.4e}' for r in ratios[:6]]}")
-
-
-if __name__ == "__main__":
-    main()

bridges/ach_server.py

@@ -70,77 +70,42 @@ from minimateplus.transport import SocketTransport
 from minimateplus.client import MiniMateClient
 from minimateplus.models import DeviceInfo, Event, MonitorLogEntry
 from sfm.database import SeismoDb
-from sfm.waveform_store import WaveformStore
 
 log = logging.getLogger("ach_server")
 
-# ── Per-unit state (downloaded events index) ──────────────────────────────────
+# ── Per-unit state (downloaded-key set) ───────────────────────────────────────
 # Persisted as <output_dir>/ach_state.json
-# Format (current — v2):
+# Format:
 #   {
 #     "BE11529": {
-#       "downloaded_events": {              # key_hex → ISO timestamp string
+#       "downloaded_keys":   ["01110000", "0111245a"],  # hex keys already on disk
-#         "01110000": "2026-04-11T00:42:17",
+#       "max_downloaded_key": "0111245a",               # highest key ever seen
-#         "0111245a": "2026-04-11T01:04:30"
+#       "last_seen": "2026-04-11T01:04:36"
-#       },
-#       "max_downloaded_key": "0111245a",
-#       "last_seen": "2026-04-11T01:04:36",
-#       "serial":    "BE11529",
-#       "peer":      "63.43.212.232:51920"
 #     }
 #   }
 #
-# Why (key, timestamp) and not key alone:
+# Key-based deduplication works well within a single "key generation" (between
-#   The device's event-key counter resets to 0x01110000 after every memory
+# erases).  After the device memory is erased the event counter resets to
-#   erase (internal or external).  A bare-key dedup (the v1 format) cannot
+# 0x01110000, so the first new event has the SAME key as the very first event
-#   distinguish a re-recorded event with the same key from one we already
+# we ever downloaded.  We detect this situation with max_downloaded_key:
-#   downloaded.  The 0C waveform record's timestamp IS unique per physical
-#   event, so we pair (key, timestamp) and treat a key with a different
-#   timestamp as a new event regardless of `max_downloaded_key`.
 #
-# Legacy v1 format (`downloaded_keys: list[str]` only) is auto-migrated on
+#   if max(current_device_keys) < max_downloaded_key
-# read: the keys are kept under a sentinel of "" (empty string) timestamp so
+#       → device was wiped and keys have restarted → treat all device keys as new
-# the (key, timestamp) compare always sees a mismatch and forces a one-time
+#
-# re-download.  After that pass the state is rewritten in v2 form.
+# After our own erase (--clear-after-download) we also explicitly clear
+# downloaded_keys and max_downloaded_key so the next session starts fresh.
 
 _state_lock = threading.Lock()
 
 
 def _load_state(state_path: Path) -> dict:
-    """
+    if state_path.exists():
-    Load ach_state.json, transparently migrating any legacy
+        try:
-    `downloaded_keys: list` entries into the v2 `downloaded_events: dict`
+            with open(state_path) as f:
-    schema.  Returns the migrated state.
+                return json.load(f)
-    """
+        except Exception:
-    if not state_path.exists():
+            pass
-        return {}
+    return {}
-    try:
-        with open(state_path) as f:
-            state = json.load(f)
-    except Exception:
-        return {}
-
-    # Per-unit migration: legacy list → dict-with-empty-timestamps
-    for unit_key, unit_state in list(state.items()):
-        if not isinstance(unit_state, dict):
-            continue
-        if "downloaded_events" in unit_state:
-            continue
-        legacy_keys = unit_state.get("downloaded_keys")
-        if isinstance(legacy_keys, list):
-            unit_state["downloaded_events"] = {k: "" for k in legacy_keys}
-            log.info(
-                "ach_state: migrated %s from v1 (downloaded_keys list) → v2 "
-                "(downloaded_events dict, %d keys with empty timestamps; "
-                "they will re-validate on next session)",
-                unit_key, len(legacy_keys),
-            )
-        else:
-            unit_state["downloaded_events"] = {}
-        # keep legacy field for one cycle; cleared on next save
-        unit_state.pop("downloaded_keys", None)
-
-    return state
 
 
 def _save_state(state_path: Path, state: dict) -> None:
@@ -174,12 +139,8 @@ class AchSession:
         max_events: Optional[int],
         state_path: Path,
         db: "SeismoDb",
-        store: "WaveformStore",
         clear_after_download: bool = False,
         restart_monitoring: bool = False,
-        rescue_stop_monitoring: bool = False,
-        rescue_disable_ach: bool = False,
-        force_redownload: bool = False,
     ) -> None:
         self.sock                 = sock
         self.peer                 = peer
@@ -189,17 +150,8 @@ class AchSession:
         self.max_events           = max_events
         self.state_path           = state_path
         self.db                   = db
-        self.store                = store
         self.clear_after_download = clear_after_download
         self.restart_monitoring   = restart_monitoring
-        # Rescue actions for a runaway unit — fired before the event walk.
-        self.rescue_stop_monitoring = rescue_stop_monitoring
-        self.rescue_disable_ach     = rescue_disable_ach
-        # `force_redownload` tells this session to ignore ach_state and
-        # re-download every event currently on the device, regardless of any
-        # (key, timestamp) match.  Useful as a manual override when state has
-        # become inconsistent with what's actually on disk / in the DB.
-        self.force_redownload     = force_redownload
 
     def run(self) -> None:
         ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -295,41 +247,6 @@ class AchSession:
         root_logger.addHandler(fh)
 
         try:
-            # ── Step 1.5: rescue actions ──────────────────────────────────────
-            # Fired BEFORE the event walk so a runaway unit is quieted as early
-            # in the session as possible.  A unit whose geophone sits above the
-            # trigger threshold records back-to-back and, with ACH set to "after
-            # event recorded", re-dials every time — saturating its own firmware
-            # so it never services inbound requests.  See
-            # docs/runbooks/wedged_unit_recovery.md.
-            #
-            # Each action is independently guarded: a failure here must not
-            # abort the download that follows.
-            if self.rescue_stop_monitoring or self.rescue_disable_ach:
-                rescue: dict = {"peer": self.peer, "ts": ts}
-
-                if self.rescue_stop_monitoring:
-                    log.info("Step 1.5: RESCUE — stop monitoring (SUB 0x97)")
-                    try:
-                        client.stop_monitoring()
-                        rescue["stop_monitoring"] = "ok"
-                        log.info("  stop monitoring OK — device should stop recording")
-                    except Exception as exc:
-                        rescue["stop_monitoring"] = f"failed: {exc}"
-                        log.error("  stop monitoring FAILED: %s", exc)
-
-                if self.rescue_disable_ach:
-                    log.info("Step 1.5: RESCUE — disable auto call home (SUB 0x2C/0x7E/0x7F)")
-                    try:
-                        client.set_call_home_config(auto_call_home_enabled=False)
-                        rescue["disable_ach"] = "ok"
-                        log.info("  disable ACH OK — unit should stop calling home")
-                    except Exception as exc:
-                        rescue["disable_ach"] = f"failed: {exc}"
-                        log.error("  disable ACH FAILED: %s", exc)
-
-                _save_json(session_dir / "rescue.json", rescue)
-
             # ── Step 2: device info ───────────────────────────────────────────
             device_info = None
             if not self.events_only:
@@ -356,20 +273,11 @@ class AchSession:
             state = _load_state(self.state_path)
             unit_key = serial or self.peer  # fall back to IP if no serial
             unit_state = state.get(unit_key, {})
-
+            seen_keys: set[str] = set(unit_state.get("downloaded_keys", []))
-            # downloaded_events is the v2 (key_hex → timestamp_iso) dict.
+            # Highest event key ever downloaded from this unit (hex string, 8 chars).
-            # Empty-string timestamps are migrated v1 entries — they force a
+            # Used to detect post-erase key reuse — see comment block above.
-            # one-time re-download because the (key, timestamp) compare always
-            # mismatches against any non-empty timestamp from a fresh 0C read.
-            seen_events: dict[str, str] = dict(unit_state.get("downloaded_events", {}))
             max_seen_key: str = unit_state.get("max_downloaded_key", "00000000")
 
-            if self.force_redownload:
-                log.info("  --force-redownload-all set — ignoring %d cached "
-                         "(key, timestamp) entries for this session",
-                         len(seen_events))
-                seen_events = {}
-
             # Walk the event index (browse-mode, no 5A) to get the actual current
             # key list.  The SUB 08 event_count field is a lifetime "total events
             # ever recorded" counter that does NOT decrement on erase — confirmed
@@ -382,10 +290,11 @@ class AchSession:
                 log.warning("  list_event_keys failed: %s -- falling back to full download", exc)
                 device_keys = None
 
+            # Use the walk result as our authoritative current count.
             current_count = len(device_keys) if device_keys is not None else 0
 
-            log.info("  Unit has %d stored event(s); %d (key, ts) entr(ies) previously downloaded",
+            log.info("  Unit has %d stored event(s); %d key(s) previously downloaded",
-                     current_count, len(seen_events))
+                     current_count, len(seen_keys))
 
             if device_keys is not None and current_count == 0:
                 log.info("  [OK] No events on device -- nothing to download")
@@ -393,29 +302,75 @@ class AchSession:
                 return
 
             if device_keys is not None:
-                # ── Post-erase detection (best-effort, key-only signal) ───────
+                # ── Post-erase detection ──────────────────────────────────────
-                # After erase the device's key counter resets to 01110000.
+                # After the device memory is erased, new events start from key
-                # If the device's current max key is below our high-water mark
+                # 01110000 again — the same keys we already downloaded.  Detect
-                # we know erase happened.  This catches the cleanest case but
+                # this by comparing the device's current highest key against the
-                # does NOT catch erase-then-record-many-events (where the new
+                # historical maximum.  If the device has rolled back below our
-                # max may climb past the old max).  The (key, timestamp) check
+                # high-water mark, its counter was reset and we must treat all
-                # in get_events() is what handles those.
+                # its keys as new, regardless of what seen_keys contains.
                 if device_keys and max_seen_key != "00000000":
-                    max_device_key = max(device_keys)
+                    max_device_key = max(device_keys)   # lexicographic; safe because
+                                                        # keys share the same 4-char prefix
                     if max_device_key < max_seen_key:
                         log.info(
                             "  Post-erase reset detected: "
                             "device max key %s < historical max %s "
-                            "-- discarding stale (key, ts) state for this session",
+                            "-- treating all device keys as new",
                             max_device_key, max_seen_key,
                         )
-                        seen_events = {}
+                        seen_keys = set()   # discard stale dedup info for this session
 
-                # Note: no early-exit "all already downloaded" short-circuit
+                new_key_set = set(device_keys) - seen_keys
-                # here.  Without per-event timestamps we cannot tell whether
+                log.info("  Device has %d key(s): %d new, %d already seen",
-                # device_keys ⊆ seen_events.keys() actually means we have
+                         len(device_keys), len(new_key_set), len(device_keys) - len(new_key_set))
-                # those physical events.  get_events() will read 0C on its
+                if not new_key_set:
-                # skip path and decide per event.
+                    log.info("  [OK] All events already downloaded -- nothing to do")
+                    # Refresh state timestamp; preserve max_seen_key unchanged.
+                    state[unit_key] = {
+                        "downloaded_keys": sorted(seen_keys | set(device_keys)),
+                        "max_downloaded_key": max_seen_key,
+                        "last_seen":         datetime.datetime.now().isoformat(),
+                        "serial":            serial,
+                        "peer":              self.peer,
+                    }
+                    _save_state(self.state_path, state)
+
+                    # ── Erase even when no new events (if requested) ──────────
+                    # Blastware ACH always erases after every session — even when
+                    # nothing new was downloaded.  Without the erase the device
+                    # still sees stored events in its memory and immediately
+                    # retries the call-home, causing the looping we observed.
+                    # Only erase when device actually has events stored; skip
+                    # the erase if device_keys is empty (nothing to erase).
+                    if self.clear_after_download and device_keys:
+                        log.info(
+                            "  Clearing device memory (--clear-after-download, "
+                            "no new events but device has %d stored)...",
+                            len(device_keys),
+                        )
+                        try:
+                            client.delete_all_events()
+                            log.info("  [OK] Device memory cleared")
+                            # Reset state so the next session starts fresh.
+                            state[unit_key] = {
+                                "downloaded_keys":    [],
+                                "max_downloaded_key": "00000000",
+                                "last_seen":          datetime.datetime.now().isoformat(),
+                                "serial":             serial,
+                                "peer":               self.peer,
+                            }
+                            _save_state(self.state_path, state)
+                        except Exception as exc:
+                            log.error(
+                                "  [WARN] Event deletion failed: %s -- events NOT cleared",
+                                exc,
+                            )
+
+                    log.info("Session complete (no new events) -> %s", session_dir)
+                    return
+            else:
+                new_key_set = None  # unknown; proceed with full download
 
             # Apply max_events cap
             # stop_idx: when we know the count from list_event_keys, use it as
@@ -433,67 +388,27 @@ class AchSession:
                     )
 
             try:
-                # Pass `seen_events` (key → ISO timestamp) so the client can
-                # read 0C on its skip path and only skip 5A when the per-event
-                # timestamp matches what we already have on disk.  When force_-
-                # redownload is set, seen_events was already cleared above.
-                #
-                # Filter out empty-string timestamps (legacy v1 entries) — the
-                # client's 0C-on-skip-path only trusts entries with a
-                # populated timestamp; otherwise it falls through to a full
-                # 5A download.
-                skip_dict = {k: ts for k, ts in seen_events.items() if ts}
-
                 all_events = client.get_events(
                     full_waveform=True,
                     stop_after_index=stop_idx,
-                    skip_waveform_for_events=skip_dict if skip_dict else None,
+                    skip_waveform_for_keys=seen_keys if seen_keys else None,
                 )
 
-                # New events are those that came back with _a5_frames populated
+                # Filter to events whose keys we haven't saved before.
-                # (= 5A actually ran on this session).  Skipped events have
-                # _a5_frames = None because the client matched (key, timestamp)
-                # against skip_dict and bypassed 5A.
                 new_events = [
                     e for e in all_events
-                    if getattr(e, "_a5_frames", None)
+                    if e._waveform_key is None
+                    or e._waveform_key.hex() not in seen_keys
                 ]
                 skipped = len(all_events) - len(new_events)
 
-                log.info("  [OK] Walked %d event(s): %d downloaded, %d skipped (matched (key, ts) in state)",
+                log.info("  [OK] Downloaded %d event(s): %d new, %d skipped (already seen)",
                          len(all_events), len(new_events), skipped)
-
+                if skipped:
-                # ── Persist event file + A5 sidecar to the waveform store ──
+                    log.info("  (skipped %d already-downloaded event(s))", skipped)
-                # Saves ride alongside the existing JSON dump so the on-disk
-                # event file and events.json reference the same set of events.
-                waveform_records: dict[str, dict] = {}
-                for ev in new_events:
-                    if not ev._a5_frames:
-                        continue
-                    try:
-                        rec = self.store.save(
-                            ev,
-                            serial=serial or "UNKNOWN",
-                            a5_frames=ev._a5_frames,
-                        )
-                        if ev._waveform_key is not None:
-                            waveform_records[ev._waveform_key.hex()] = rec
-                        log.info(
-                            "  [WAVE] saved %s (%d bytes)",
-                            rec["filename"], rec["filesize"],
-                        )
-                    except Exception as exc:
-                        key_hex = ev._waveform_key.hex() if ev._waveform_key else "????????"
-                        log.warning(
-                            "  [WARN] Waveform store save failed for %s: %s",
-                            key_hex, exc,
-                        )
 
                 if new_events:
-                    _save_json(
+                    _save_json(session_dir / "events.json", [_event_to_dict(e) for e in new_events])
-                        session_dir / "events.json",
-                        [_event_to_dict(e, waveform_records) for e in new_events],
-                    )
 
                     for ev in new_events:
                         pv = ev.peak_values
@@ -552,11 +467,7 @@ class AchSession:
                 _session_start = datetime.datetime.now()
                 try:
                     _ev_ins, _ev_skip = self.db.insert_events(
-                        new_events,
+                        new_events, serial=serial or self.peer, session_id=None
-                        serial=serial or self.peer,
-                        session_id=None,
-                        waveform_records=waveform_records,
-                        device_family="series3",
                     )
                     _ml_ins, _ml_skip = self.db.insert_monitor_log(
                         new_monitor_entries, session_id=None
@@ -591,64 +502,35 @@ class AchSession:
                         )
 
                 # ── Update persistent state ───────────────────────────────────
-                # Build a fresh (key → ISO timestamp) map from THIS session's
+                # Include both triggered-event keys and monitor-log keys in the
-                # results.  For each event currently on the device, prefer the
+                # downloaded set so they are not re-processed on the next call-home.
-                # timestamp we just observed (from 0C); fall back to whatever
+                current_event_keys = [
-                # was already in seen_events for that key (so we don't lose an
+                    e._waveform_key.hex()
-                # entry just because get_events skipped it on the (key, ts)
+                    for e in all_events
-                # match path).
+                    if e._waveform_key is not None
-                def _ts_iso(ev) -> str:
+                ]
-                    ts = getattr(ev, "timestamp", None)
+                current_monitor_keys = [e.key for e in new_monitor_entries]
-                    if ts is None:
+                current_keys = current_event_keys + current_monitor_keys
-                        return ""
-                    try:
-                        return datetime.datetime(
-                            ts.year, ts.month, ts.day,
-                            ts.hour or 0, ts.minute or 0, ts.second or 0,
-                        ).isoformat()
-                    except Exception:
-                        return str(ts)
-
-                current_events_map: dict[str, str] = {}
-                for ev in all_events:
-                    if ev._waveform_key is None:
-                        continue
-                    key_hex = ev._waveform_key.hex()
-                    ts_iso  = _ts_iso(ev) or seen_events.get(key_hex, "")
-                    current_events_map[key_hex] = ts_iso
-
-                # Monitor-log entries don't have a 0C-style timestamp, but
-                # they DO have a start_time; use that so the monitor-log keys
-                # are properly entered into the (key, ts) map.
-                for ml in new_monitor_entries:
-                    key_hex = ml.key
-                    ts = ml.start_time
-                    ts_iso = ts.isoformat() if ts else seen_events.get(key_hex, "")
-                    # If a triggered event already populated this key, keep
-                    # whichever has a non-empty timestamp.
-                    if key_hex not in current_events_map or not current_events_map[key_hex]:
-                        current_events_map[key_hex] = ts_iso
 
                 if erased_successfully:
-                    updated_events: dict[str, str] = {}
+                    # Device memory is clear. Reset downloaded_keys and the
+                    # high-water mark so the next call-home starts fresh and
+                    # doesn't mis-identify the recycled key 01110000 as "seen".
+                    updated_keys = []
                     new_max_key  = "00000000"
                     log.info(
                         "  State reset after erase -- next session will download "
                         "from key 0 (device counter resets after erase)"
                     )
                 else:
-                    # Merge: keep prior (key, ts) entries we still have evidence
+                    # Normal (no erase): union of previously-seen + all keys on
-                    # of (for survivors of any partial failure), plus this
+                    # device now.  Includes already-seen survivors so we never
-                    # session's authoritative (key, ts) pairs.
+                    # re-download them if the device somehow keeps old records.
-                    updated_events = dict(seen_events)
+                    updated_keys = sorted(set(seen_keys) | set(current_keys))
-                    updated_events.update(current_events_map)
+                    new_max_key  = updated_keys[-1] if updated_keys else max_seen_key
-                    new_max_key  = (
-                        max(updated_events.keys())
-                        if updated_events else max_seen_key
-                    )
 
                 state[unit_key] = {
-                    "downloaded_events":  updated_events,
+                    "downloaded_keys":   updated_keys,
                     "max_downloaded_key": new_max_key,
                     "last_seen":          datetime.datetime.now().isoformat(),
                     "serial":             serial,
@@ -710,10 +592,7 @@ def _device_info_to_dict(d: DeviceInfo) -> dict:
     }
 
 
-def _event_to_dict(
+def _event_to_dict(e: Event) -> dict:
-    e: Event,
-    waveform_records: Optional[dict[str, dict]] = None,
-) -> dict:
     pv = e.peak_values
     pi = e.project_info
     peaks = {}
@@ -732,11 +611,6 @@ def _event_to_dict(
             for ch, vals in e.raw_samples.items()
         }
         samples["__note__"] = "first 20 sample-sets only; see raw_rx.bin for full waveform"
-
-    rec: dict = {}
-    if waveform_records and e._waveform_key is not None:
-        rec = waveform_records.get(e._waveform_key.hex(), {}) or {}
-
     return {
         "timestamp":       str(e.timestamp) if e.timestamp else None,
         "project":         pi.project         if pi else None,
@@ -745,9 +619,6 @@ def _event_to_dict(
         "sensor_location": pi.sensor_location if pi else None,
         "peaks":           peaks,
         "raw_samples_preview": samples,
-        "blastware_filename":   rec.get("filename"),
-        "blastware_filesize":   rec.get("filesize"),
-        "a5_pickle_filename":   rec.get("a5_pickle_filename"),
     }
 
 
@@ -769,7 +640,6 @@ def serve(args: argparse.Namespace) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
     state_path = output_dir / "ach_state.json"
     db = SeismoDb(output_dir / "seismo_relay.db")
-    store = WaveformStore(output_dir / "waveforms")
 
     server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -787,14 +657,6 @@ def serve(args: argparse.Namespace) -> None:
     print(f"  Max events per session: {max_ev if max_ev else 'unlimited'}")
     print(f"  Clear device after download: {'YES' if args.clear_after_download else 'no'}")
     print(f"  Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}")
-    _stop_mon = args.stop_monitoring or args.rescue
-    _dis_ach  = args.disable_ach or args.rescue
-    print(f"  RESCUE stop monitoring on connect:  {'YES' if _stop_mon else 'no'}")
-    print(f"  RESCUE disable auto call home:      {'YES' if _dis_ach else 'no'}")
-    if _stop_mon and args.restart_monitoring:
-        print("  !! --restart-monitoring will re-start the unit after download,")
-        print("     undoing --stop-monitoring.  Drop one of them.")
-    print(f"  Force re-download all (ignore state): {'YES' if args.force_redownload_all else 'no'}")
     print(f"{'='*60}")
     print(f"\n  Point your test unit's ACEmanager call-home settings to:")
     print(f"    Remote Host: <this machine's LAN IP>")
@@ -832,12 +694,8 @@ def serve(args: argparse.Namespace) -> None:
                     max_events=max_ev,
                     state_path=state_path,
                     db=db,
-                    store=store,
                     clear_after_download=args.clear_after_download,
                     restart_monitoring=args.restart_monitoring,
-                    rescue_stop_monitoring=args.stop_monitoring or args.rescue,
-                    rescue_disable_ach=args.disable_ach or args.rescue,
-                    force_redownload=args.force_redownload_all,
                 )
                 t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}")
                 t.start()
@@ -911,32 +769,6 @@ def parse_args() -> argparse.Namespace:
             "DCD on disconnect — without this the unit stays idle after a call-home."
         ),
     )
-    p.add_argument(
-        "--stop-monitoring",
-        action="store_true",
-        default=False,
-        help=(
-            "RESCUE: send SUB 0x97 (stop monitoring) immediately after the "
-            "handshake, before any event download. Use on a unit that is "
-            "recording back-to-back because of a stuck-triggered geophone."
-        ),
-    )
-    p.add_argument(
-        "--disable-ach",
-        action="store_true",
-        default=False,
-        help=(
-            "RESCUE: disable Auto Call Home on the device (SUB 0x2C read → "
-            "0x7E write → 0x7F confirm) immediately after the handshake. The "
-            "unit stops dialing out until ACH is explicitly re-enabled."
-        ),
-    )
-    p.add_argument(
-        "--rescue",
-        action="store_true",
-        default=False,
-        help="Shorthand for --stop-monitoring --disable-ach.",
-    )
     p.add_argument(
         "--clear-after-download",
         action="store_true",
@@ -948,17 +780,6 @@ def parse_args() -> argparse.Namespace:
             "This mirrors the standard Blastware ACH workflow."
         ),
     )
-    p.add_argument(
-        "--force-redownload-all",
-        action="store_true",
-        default=False,
-        help=(
-            "Manual override: ignore ach_state.json's downloaded_events map "
-            "for this session and re-download every event currently on the "
-            "device, regardless of (key, timestamp) match.  Useful when state "
-            "has become inconsistent with the on-disk waveform store / DB."
-        ),
-    )
     p.add_argument(
         "--verbose", "-v",
         action="store_true",

bridges/mm_link.py

@@ -1,338 +0,0 @@
-#!/usr/bin/env python3
-"""
-mm_link.py — a "perfect modem" between THOR and a Micromate, with a readable
-log and deliberate fault injection.
-
-Why
----
-THOR gives almost no visibility into a connection: a refresh button, two poll
-intervals, and no way to see whether a check succeeded, timed out, or was never
-sent.  When a unit "won't stay connected" there is nothing to look at.
-
-This sits where the cellular modem would sit and answers the question directly:
-
-  * **What is THOR actually doing?**  Every frame is decoded and timestamped —
-    `POLL`, `MONITOR_STATUS`, `SETUP_NAME_READ` — not a hex dump.
-  * **Is it even trying?**  Silence is visible: the log shows gaps.
-  * **How does it behave when the link misbehaves?**  Faults can be injected on
-    demand, which a real cell link will not do on cue.
-
-Point THOR at this host and port exactly as if it were a modem (Communication:
-TCP, IP: <this host>, Port: <--listen>).
-
-Fault injection
----------------
-Write a mode into the control file (default `mm_link.ctl`) and it takes effect
-on the next byte:
-
-    echo pass      > mm_link.ctl    # normal relay
-    echo blackhole > mm_link.ctl    # TCP stays up, bytes are swallowed
-    echo drop      > mm_link.ctl    # close the connection abruptly (RST-ish)
-    echo delay:2.0 > mm_link.ctl    # forward, but 2 s late in both directions
-    echo onewaydev > mm_link.ctl    # THOR->unit passes, unit->THOR is swallowed
-
-**`blackhole` is the one that matters.**  It reproduces the classic cellular
-failure: the socket is still open as far as both ends are concerned, but nothing
-crosses.  A client that relies on TCP to tell it the peer is gone will sit there
-until the OS keepalive fires — which by default is about two hours.
-
-Usage
------
-    python3 bridges/mm_link.py --serial /dev/ttyACM0 --baud 115200 \\
-        --listen 12345 --logdir ~/mm-captures
-
-Writes, per session:
-    <logdir>/mmlink_<ts>/session.log   decoded, timestamped, human-readable
-    <logdir>/mmlink_<ts>/raw_bw.bin    THOR -> unit, raw
-    <logdir>/mmlink_<ts>/raw_s3.bin    unit -> THOR, raw
-
-The raw pair loads straight into `scratch/mm_frame_parse.py`.
-"""
-
-from __future__ import annotations
-
-import argparse
-import datetime
-import os
-import socket
-import sys
-import threading
-import time
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scratch"))
-try:
-    from mm_frame_parse import SUBNAME, destuff  # noqa: F401
-except Exception:                                 # pragma: no cover
-    SUBNAME = {}
-
-import errno
-import select
-import termios
-
-DLE, STX, ETX, ACK = 0x10, 0x02, 0x03, 0x41
-
-_BAUD = {9600: termios.B9600, 19200: termios.B19200, 38400: termios.B38400,
-         57600: termios.B57600, 115200: termios.B115200}
-
-
-class SerialPort:
-    """Minimal raw serial port on stdlib termios — no pyserial dependency.
-
-    The bench hosts are whatever is to hand; requiring a pip install on someone
-    else's machine is a poor trade for the ~30 lines this saves.
-    """
-
-    def __init__(self, path: str, baud: int):
-        if baud not in _BAUD:
-            raise ValueError(f"unsupported baud {baud}; pick one of {sorted(_BAUD)}")
-        self.fd = os.open(path, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
-        a = termios.tcgetattr(self.fd)
-        a[0] = 0                                             # iflag: no translation
-        a[1] = 0                                             # oflag: raw
-        a[2] = termios.CS8 | termios.CREAD | termios.CLOCAL  # cflag: 8N1, ignore modem lines
-        a[3] = 0                                             # lflag: non-canonical, no echo
-        a[4] = a[5] = _BAUD[baud]
-        a[6] = list(a[6])
-        a[6][termios.VMIN] = 0
-        a[6][termios.VTIME] = 0
-        termios.tcsetattr(self.fd, termios.TCSANOW, a)
-        termios.tcflush(self.fd, termios.TCIOFLUSH)
-
-    def read(self, n: int) -> bytes:
-        r, _, _ = select.select([self.fd], [], [], 0.2)
-        if not r:
-            return b""
-        try:
-            return os.read(self.fd, n)
-        except OSError as e:
-            if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
-                return b""
-            raise
-
-    def write(self, data: bytes) -> None:
-        while data:
-            try:
-                data = data[os.write(self.fd, data):]
-            except OSError as e:
-                if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
-                    select.select([], [self.fd], [], 0.2)
-                    continue
-                raise
-
-    def close(self) -> None:
-        try:
-            os.close(self.fd)
-        except OSError:
-            pass
-
-
-def name_of(sub: int, is_request: bool) -> str:
-    if is_request:
-        return SUBNAME.get(sub, f"SUB_{sub:02X}")
-    return "rsp " + SUBNAME.get(0xFF - sub, f"SUB_{0xFF - sub:02X}")
-
-
-class FrameSniffer:
-    """Accumulate bytes and report complete frames, without altering the stream."""
-
-    def __init__(self, is_request: bool):
-        self.is_request = is_request
-        self.buf = bytearray()
-
-    def feed(self, data: bytes):
-        """Yield (sub, payload_len) for each complete frame seen."""
-        self.buf.extend(data)
-        while True:
-            start = -1
-            for i, b in enumerate(self.buf):
-                if self.is_request and b == ACK and i + 1 < len(self.buf) and self.buf[i + 1] == STX:
-                    start = i
-                    break
-                if not self.is_request and b == STX:
-                    start = i
-                    break
-            if start < 0:
-                if len(self.buf) > 8192:
-                    del self.buf[:-16]
-                return
-            j = start + (2 if self.is_request else 1)
-            end = -1
-            while j < len(self.buf):
-                if self.buf[j] == DLE and j + 1 < len(self.buf):
-                    j += 2
-                    continue
-                if self.buf[j] == ETX:
-                    end = j
-                    break
-                j += 1
-            if end < 0:
-                return                      # wait for more bytes
-            body = self.buf[start:end + 1]
-            del self.buf[:end + 1]
-            # SUB sits at a fixed spot past the leading framing -- but it is
-            # DLE-escaped when its own value is 0x02/0x03/0x04/0x10, so a raw
-            # read reports 0x10 for those.  SUB 0x02 was being logged as
-            # "SUB_10" until this was handled.
-            off = 5 if self.is_request else 3
-            if len(body) > off:
-                sub = body[off]
-                if sub == DLE and len(body) > off + 1:
-                    sub = body[off + 1]
-                yield sub, len(body)
-
-
-class Link:
-    def __init__(self, args):
-        self.args = args
-        self.mode = "pass"
-        self.delay = 0.0
-        self.ctl = Path(args.control)
-        self.session: Path | None = None
-        self.log_fh = None
-        self.raw = {}
-        self.t0 = time.time()
-        self.counts = {}
-
-    # ── logging ────────────────────────────────────────────────────────────
-    def open_session(self):
-        ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-        self.session = Path(self.args.logdir) / f"mmlink_{ts}"
-        self.session.mkdir(parents=True, exist_ok=True)
-        self.log_fh = open(self.session / "session.log", "a", buffering=1)
-        self.raw = {
-            "bw": open(self.session / "raw_bw.bin", "ab"),
-            "s3": open(self.session / "raw_s3.bin", "ab"),
-        }
-        self.say(f"=== session {ts} — serial {self.args.serial} @ {self.args.baud} ===")
-
-    def say(self, text: str):
-        line = f"{datetime.datetime.now().strftime('%H:%M:%S.%f')[:-3]}  {text}"
-        print(line, flush=True)
-        if self.log_fh:
-            self.log_fh.write(line + "\n")
-
-    # ── control file ───────────────────────────────────────────────────────
-    def poll_control(self):
-        while True:
-            try:
-                if self.ctl.exists():
-                    want = self.ctl.read_text().strip().lower()
-                    if want.startswith("delay:"):
-                        d = float(want.split(":", 1)[1])
-                        if ("delay", d) != (self.mode, self.delay):
-                            self.mode, self.delay = "delay", d
-                            self.say(f"*** MODE -> delay {d}s ***")
-                    elif want and want != self.mode:
-                        self.mode, self.delay = want, 0.0
-                        self.say(f"*** MODE -> {want} ***")
-            except Exception:
-                pass
-            time.sleep(0.25)
-
-    # ── the relay ──────────────────────────────────────────────────────────
-    def pump(self, src, dst, tag: str, is_request: bool, stop: threading.Event):
-        sniff = FrameSniffer(is_request)
-        arrow = "THOR->unit" if is_request else "unit->THOR"
-        last = time.time()
-        while not stop.is_set():
-            timed_out = False
-            try:
-                data = src.recv(4096) if isinstance(src, socket.socket) else src.read(4096)
-            except TimeoutError:
-                timed_out = True
-                # socket.timeout subclasses OSError, so it MUST be caught first.
-                # Treating it as a dead socket closes the connection after 200 ms
-                # of quiet -- which is exactly what `blackhole` produces, so the
-                # relay killed the link it was supposed to be faking a fault on.
-                data = b""
-            except OSError:
-                break
-            if isinstance(src, socket.socket) and data == b"" and not timed_out:
-                self.say(f"{arrow}: peer closed the connection")
-                break
-            if not data:
-                if time.time() - last > self.args.quiet_after and self.counts:
-                    self.say(f"--- {self.args.quiet_after:.0f}s with no traffic ---")
-                    last = time.time()
-                continue
-            last = time.time()
-
-            self.raw[tag].write(data)
-            self.raw[tag].flush()
-            for sub, ln in sniff.feed(data):
-                label = name_of(sub, is_request)
-                self.counts[label] = self.counts.get(label, 0) + 1
-                self.say(f"{arrow}  {label:<20} ({ln} B)"
-                         + ("" if self.mode == "pass" else f"   [mode={self.mode}]"))
-
-            mode = self.mode
-            if mode == "drop":
-                self.say(f"{arrow}: DROPPING the connection (fault injection)")
-                stop.set()
-                break
-            if mode == "blackhole":
-                continue                      # swallow, keep the socket open
-            if mode == "onewaydev" and not is_request:
-                continue                      # unit's replies never reach THOR
-            if mode == "delay" and self.delay:
-                time.sleep(self.delay)
-            try:
-                if isinstance(dst, socket.socket):
-                    dst.sendall(data)
-                else:
-                    dst.write(data)
-            except OSError:
-                break
-        stop.set()
-
-    def serve(self):
-        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        srv.bind(("0.0.0.0", self.args.listen))
-        srv.listen(5)
-        self.open_session()
-        self.say(f"listening on 0.0.0.0:{self.args.listen}   control file: {self.ctl}")
-        self.say("point THOR at this host/port as Communication=TCP")
-        threading.Thread(target=self.poll_control, daemon=True).start()
-
-        while True:
-            conn, addr = srv.accept()
-            conn.settimeout(0.2)
-            self.say(f"+++ THOR connected from {addr[0]}:{addr[1]} +++")
-            try:
-                ser = SerialPort(self.args.serial, self.args.baud)
-            except OSError as e:
-                self.say(f"!!! cannot open {self.args.serial}: {e}")
-                conn.close()
-                continue
-            stop = threading.Event()
-            ts = [
-                threading.Thread(target=self.pump, args=(conn, ser, "bw", True, stop), daemon=True),
-                threading.Thread(target=self.pump, args=(ser, conn, "s3", False, stop), daemon=True),
-            ]
-            for t in ts:
-                t.start()
-            for t in ts:
-                t.join()
-            conn.close()
-            ser.close()
-            summary = ", ".join(f"{k}x{v}" for k, v in sorted(self.counts.items()))
-            self.say(f"--- connection closed.  frames this session: {summary or 'none'} ---")
-
-
-def main():
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--serial", default="/dev/ttyACM0")
-    ap.add_argument("--baud", type=int, default=115200)
-    ap.add_argument("--listen", type=int, default=12345)
-    ap.add_argument("--logdir", default=os.path.expanduser("~/mm-captures"))
-    ap.add_argument("--control", default="mm_link.ctl")
-    ap.add_argument("--quiet-after", type=float, default=30.0,
-                    help="log a marker after this many seconds of silence")
-    Link(ap.parse_args()).serve()
-
-
-if __name__ == "__main__":
-    main()

bridges/mm_probe.py

@@ -1,226 +0,0 @@
-#!/usr/bin/env python3
-"""
-mm_probe.py — answer "why can't we reach this unit?" in one command.
-
-THOR reports a failed connection as "disconnected" and nothing else.  That single
-word covers at least four completely different faults with four different fixes,
-and telling them apart is the difference between a modem reboot and a site visit:
-
-  * **connection refused**   something answered and said no — wrong port, or the
-                             modem is refusing a further session
-  * **connect timed out**    nothing answered at all — trusted-IP whitelist,
-                             firewall, or the modem is off the network
-  * **connected, no reply**  the MODEM answered but the unit did not.  The TCP
-                             path is fine; the modem is not forwarding to serial.
-                             This is the signature of a wedged transparent-TCP
-                             session, and it is the one THOR cannot distinguish
-                             from any of the others
-  * **replied**              the unit is alive; the problem is upstream software
-
-Read-only.  It sends `POLL`, then optionally `SERIAL` and the state read — the
-same three commands THOR's own connection check uses — and never writes.
-
-Usage
------
-    python3 bridges/mm_probe.py 63.45.161.30:9034
-    python3 bridges/mm_probe.py 10.0.0.8:12345 --timeout 5
-    python3 bridges/mm_probe.py <host:port> --slots 3
-
-`--slots N` opens N connections at once and reports how many the far end accepts.
-A transparent-TCP modem typically serves **one** session; if the first succeeds
-and the rest are refused or hang, that confirms the single-slot behaviour and
-explains why a leaked session takes a unit offline until the slot frees.
-
-Works for both series: a Series III reply opens `DLE STX`, a Micromate reply
-opens with a bare `STX`, so the probe also tells you which one answered.
-"""
-
-from __future__ import annotations
-
-import argparse
-import socket
-import sys
-import time
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-from minimateplus.framing import build_bw_frame  # noqa: E402
-
-DLE, STX, ETX = 0x10, 0x02, 0x03
-
-
-def destuff(raw: bytes) -> bytes:
-    """Strip framing and DLE escapes; return the payload without its checksum."""
-    i = 1 if raw and raw[0] == STX else (2 if len(raw) > 1 and raw[1] == STX else 0)
-    out = bytearray()
-    while i < len(raw):
-        b = raw[i]
-        if b == DLE and i + 1 < len(raw):
-            out.append(raw[i + 1])
-            i += 2
-            continue
-        if b == ETX:
-            break
-        out.append(b)
-        i += 1
-    return bytes(out[:-1]) if len(out) > 1 else b""
-
-
-# Reads are two-step on Series III: a probe at offset 0, then a data read at the
-# block's length.  THOR sends these offsets, and they also work on a Micromate.
-OFFSETS = {0x5B: 0x0030, 0x15: 0x000A, 0x49: 0xFFFF}
-
-
-def exchange(sock: socket.socket, sub: int, timeout: float) -> tuple[bytes, float]:
-    sock.sendall(build_bw_frame(sub, OFFSETS.get(sub, 0)))
-    t0 = time.time()
-    buf, deadline = b"", t0 + timeout
-    sock.settimeout(0.3)
-    while time.time() < deadline:
-        try:
-            chunk = sock.recv(4096)
-            if not chunk:
-                break
-            buf += chunk
-            if buf.endswith(bytes([ETX])) and len(buf) > 8:
-                break
-        except TimeoutError:
-            continue
-        except OSError:
-            break
-    return buf, time.time() - t0
-
-
-def step(n: int, label: str, result: str) -> None:
-    print(f"  [{n}] {label:.<28} {result}")
-
-
-def probe(host: str, port: int, timeout: float) -> int:
-    print(f"\ntarget {host}:{port}   (read-only: POLL, SERIAL, state)\n")
-
-    # ── 1. TCP ────────────────────────────────────────────────────────────
-    t0 = time.time()
-    try:
-        sock = socket.create_connection((host, port), timeout=timeout)
-    except ConnectionRefusedError:
-        step(1, "TCP connect", f"REFUSED after {1000*(time.time()-t0):.0f} ms")
-        print("\nverdict: something answered and actively refused.")
-        print("  Not a silent firewall drop — the host is reachable.")
-        print("  Wrong port, the service is down, or the modem is refusing")
-        print("  an additional session because its one slot is in use.")
-        return 2
-    except (TimeoutError, socket.timeout):
-        step(1, "TCP connect", f"TIMED OUT after {time.time()-t0:.1f} s")
-        print("\nverdict: nothing answered at all.")
-        print("  A silent drop, which is what a trusted-IP whitelist looks")
-        print("  like — it discards rather than refuses.  Check the modem's")
-        print("  Trusted IPs (and note a VPN changes the IP you arrive from),")
-        print("  the firewall, and whether the modem is on the network.")
-        return 3
-    except OSError as e:
-        step(1, "TCP connect", f"FAILED: {e}")
-        return 4
-    step(1, "TCP connect", f"ok ({1000*(time.time()-t0):.0f} ms)")
-
-    # ── 2. POLL ───────────────────────────────────────────────────────────
-    try:
-        raw, dt = exchange(sock, 0x5B, timeout)
-    except OSError as e:
-        step(2, "POLL", f"send failed: {e}")
-        sock.close()
-        return 4
-
-    if not raw:
-        step(2, "POLL", f"NO REPLY in {timeout:.1f} s")
-        print("\nverdict: the MODEM answered but the unit did not.")
-        print("  TCP is fine end to end — something accepted the connection.")
-        print("  What is missing is the serial side.  Most likely the modem is")
-        print("  not forwarding to its serial port, which is what a wedged")
-        print("  transparent-TCP session looks like: the slot is held by a")
-        print("  connection that never closed.")
-        print("\n  Try, in order:")
-        print("    1. ACEmanager -> TCP Idle Timeout.  If 0/disabled, a stale")
-        print("       session holds the slot forever.  2 minutes is the value")
-        print("       this project standardised on.")
-        print("    2. Reboot the modem.  If that fixes it, the modem was")
-        print("       holding state and the timeout is the permanent fix.")
-        print("    3. Check the unit's own screen — serial cable, power.")
-        sock.close()
-        return 5
-
-    series = "Series III (DLE STX)" if raw[0] == DLE else "Micromate (bare STX)"
-    step(2, "POLL", f"reply {len(raw)} B in {1000*dt:.0f} ms")
-    p = destuff(raw)
-    ok = len(p) > 3 and p[2] == 0xFF - 0x5B
-    step(3, "frame", f"{'valid' if ok else 'MALFORMED'}, {series}")
-    if not ok:
-        print("\nverdict: something replied, but not a seismograph.")
-        print("  Another service is on this port, or the modem is in a mode")
-        print("  that injects its own text (check Quiet Mode / AT echo).")
-        print(f"  first bytes: {raw[:16].hex(' ')}")
-        sock.close()
-        return 6
-
-    # ── 3. identity + state ───────────────────────────────────────────────
-    for n, (sub, label) in enumerate(((0x15, "serial"), (0x49, "state")), start=4):
-        try:
-            r, dt = exchange(sock, sub, timeout)
-            d = destuff(r)[5:]
-            if sub == 0x15:
-                # serial is a null-terminated run; a further field follows it
-                serial = bytes(d[11:]).split(b"\x00")[0]
-                step(n, label, serial.decode("ascii", "replace") or "(empty)")
-            else:
-                step(n, label, "MONITORING" if len(d) > 11 and d[11] else "idle")
-        except OSError:
-            step(n, label, "no reply")
-
-    sock.close()
-    print("\nverdict: the unit is alive and answering.")
-    print("  If THOR still shows it disconnected, the fault is in THOR, not")
-    print("  the network or the device.")
-    return 0
-
-
-def slots(host: str, port: int, n: int, timeout: float) -> None:
-    print(f"\nopening {n} simultaneous connections to {host}:{port}\n")
-    held = []
-    for i in range(n):
-        try:
-            s = socket.create_connection((host, port), timeout=timeout)
-            held.append(s)
-            step(i + 1, f"connection {i+1}", "accepted")
-        except ConnectionRefusedError:
-            step(i + 1, f"connection {i+1}", "REFUSED")
-        except (TimeoutError, socket.timeout):
-            step(i + 1, f"connection {i+1}", "timed out")
-        except OSError as e:
-            step(i + 1, f"connection {i+1}", f"failed: {e}")
-    print(f"\n{len(held)} of {n} accepted.")
-    if len(held) == 1:
-        print("  Single-slot behaviour confirmed — this far end serves ONE")
-        print("  session at a time.  A connection that is never closed takes")
-        print("  the unit offline until the idle timeout frees the slot.")
-    for s in held:
-        s.close()
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("target", help="host:port, e.g. 63.45.161.30:9034")
-    ap.add_argument("--timeout", type=float, default=10.0)
-    ap.add_argument("--slots", type=int, metavar="N",
-                    help="open N simultaneous connections to test single-slot behaviour")
-    a = ap.parse_args()
-    host, _, port = a.target.rpartition(":")
-    if not host:
-        ap.error("target must be host:port")
-    if a.slots:
-        slots(host, int(port), a.slots, a.timeout)
-        return 0
-    return probe(host, int(port), a.timeout)
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

docs/histogram_codec_re_status.md

@@ -1,212 +0,0 @@
-> ## SUPERSEDED 2026-08-25 — the block is uniformly BIG-ENDIAN
->
-> The `uint8` peak / `annotation` byte model described below is wrong,
-> though it decoded quiet data correctly.  The real layout:
->
-> - **Every per-channel field is `uint16` big-endian.**  `T_peak` is
->   `[5:7]`, `T_halfperiod` `[7:9]`, `V_peak` `[9:11]`, and so on.
->   Only `block_ctr` at `[2:4]` is little-endian.
-> - The **marker is `block[4]` alone**, not a `uint16 LE` at `[4:6]`.
->   Testing `[4:6] == 10` forced `block[5] == 0`, which is exactly what
->   capped every geo peak at one byte (255 counts = 1.275 in/s).
-> - The **"annotation" byte was never an annotation** — it is the high
->   byte of the big-endian half-period.  That is why it was non-zero
->   precisely on the sub-Hz intervals Blastware renders as `<1.0`.
-> - The **final block of the stream carries tail `9c 06 00 42`** instead
->   of `1e 0a 00 00`, and arbitrary bytes at `[21:23]`.  Rejecting it
->   dropped the last interval of nearly every histogram — often the one
->   holding the event peak, so the file's PPV read low.
->
-> Verified against 1211 production histograms paired with their Blastware
-> ASCII exports: **1211/1211 decode exactly** (interval count plus every
-> per-interval peak), and 842,442 per-interval frequency comparisons match
-> with zero mismatches.  The uint8 model scored 1204/1211 — the seven
-> failures are exactly the files containing a peak above 1.275 in/s.
->
-> The section below is retained as the reasoning trail.
-
-# Histogram body codec — FULLY DECODED (2026-05-20)
-
-Clean working status doc for the MiniMate Plus histogram-mode event
-body codec.  Companion to `waveform_codec_re_status.md`.  The deep
-historical record (with retractions and dated analyses) lives in
-`docs/instantel_protocol_reference.md §7.6.2`; the authoritative
-implementation lives in `minimateplus/histogram_codec.py`.
-
-## TL;DR
-
-**The codec is fully decoded.**  Every field of every block in the
-in-repo histogram fixture corpus decodes byte-exact against BW's
-ASCII export.
-
-26 regression tests pass against ~3,500 blocks across 5 in-repo
-fixtures, plus a synthetic regression block taken from a real
-BE9558 prod event to lock in the uint8-peak interpretation.
-
-**Important correction (2026-05-21):** the per-channel peak count
-is `uint8` at byte[6]/[10]/[14]/[18], NOT `uint16 LE` at byte[6:8]
-etc.  The N844 fixture corpus the original RE was done against has
-zero values in bytes [7]/[11]/[15]/[19] for every block, so the
-two interpretations happened to be equivalent.  Cross-correlating
-non-N844 events (BE9558 Tran-drift, BE18003 Histogram+Continuous)
-against BW's per-interval ASCII export — 4 channels × ~1400 blocks
-per event × multiple events = 100% byte-exact only when the peak
-is read as uint8.  Reading as uint16 LE produced peaks up to 268
-in/s per channel and 35× inflated PVS sums when first deployed to
-prod (rolled back, root-caused, and fixed in commit 7183b95+1).
-
-## Body format
-
-```
-body = [stream of 32-byte data blocks] + [small trailing remnant]
-```
-
-Each block represents one histogram interval.  Block layout:
-
-```
-[0]    0x00                      always-zero tag
-[1]    segment_id (uint8)        0x00..0x03 — 256 blocks per segment
-[2:4]  block_ctr (uint16 LE)     resets each segment (0x0100, 0x0101, …)
-[4:6]  0x000a (uint16 LE)        constant marker (= 10)
-[6]    T_peak_count   uint8      Tran peak (count × 0.005 → in/s at Normal,
-                                  max 1.275 in/s — fits in uint8)
-[7]    T_annotation   uint8      empirically non-zero on intervals with sub-Hz
-                                  or unmeasurable freq; meaning not fully RE'd
-[8:10] T_halfperiod   uint16 LE  Tran half-period in samples
-                                  (freq_Hz = 512 / halfp; ≤ 5 means ">100 Hz")
-[10]   V_peak_count   uint8      Vert peak
-[11]   V_annotation   uint8
-[12:14] V_halfperiod  uint16 LE  Vert freq half-period
-[14]   L_peak_count   uint8      Long peak
-[15]   L_annotation   uint8
-[16:18] L_halfperiod  uint16 LE  Long freq half-period
-[18]   M_peak_count   uint8      MicL peak count
-                                  (dB via waveform_codec.mic_count_to_db)
-[19]   M_annotation   uint8
-[20:22] M_halfperiod  uint16 LE  MicL freq half-period
-[22:24] 0x00 0x00                constant
-[24:28] 4-byte variable          purpose unknown — possibly CRC,
-                                  timestamp delta, or psi(L) numeric;
-                                  not needed for waveform reconstruction
-[28:32] 0x1e 0x0a 0x00 0x00      constant block-end signature
-```
-
-Reliable block-identification anchor:
-```python
-block[22:24] == b"\x00\x00" and block[28:32] == b"\x1e\x0a\x00\x00"
-```
-(The `1e 0a 00 00` constant tail is the most distinctive signature.)
-
-## Per-channel encoding
-
-| Channel | Peak encoding | Frequency encoding |
-|---|---|---|
-| Tran | count × 0.005 = in/s at Normal range | `freq_Hz = 512 / halfperiod` |
-| Vert | same | same |
-| Long | same | same |
-| MicL | count → dB via `mic_count_to_db(count)` (same formula as waveform codec) | same |
-
-**`>100 Hz` sentinel**: when halfperiod ≤ 5 (giving ≥100 Hz from the
-512/halfp formula), BW displays `>100 Hz`.  Codec's `half_period_to_hz`
-returns `None` in this range.
-
-## Verified facts (cross-checked against fixture corpus)
-
-Example: N844L6Z8.ZR0H block 130 → all 8 decoded fields byte-exact:
-
-```
-binary samples [10, 6, 24, 4, 18, 5, 21, 5, 9]
-TXT row        [0.030, 21, 0.020, 28, 0.025, 24, 0.040, 0.000, 95.92, 57]
-
-slot[0] = 10                                  marker
-slot[1] = 6  × 0.005 = 0.030 in/s         ✓ T_peak
-slot[2] = 24 → 512/24 = 21.3 → 21 Hz      ✓ T_freq
-slot[3] = 4  × 0.005 = 0.020 in/s         ✓ V_peak
-slot[4] = 18 → 512/18 = 28.4 → 28 Hz      ✓ V_freq
-slot[5] = 5  × 0.005 = 0.025 in/s         ✓ L_peak
-slot[6] = 21 → 512/21 = 24.4 → 24 Hz      ✓ L_freq
-slot[7] = 5  → 81.94 + 20·log10(5) = 95.92 dB  ✓ M_peak
-slot[8] = 9  → 512/9 = 56.9 → 57 Hz       ✓ M_freq
-```
-
-## Verified test coverage
-
-`tests/test_histogram_codec.py` (24 tests):
-
-- Block walking: yields one record per `.TXT` interval ± 1 (off-by-one
-  at the tail when recording was stopped mid-write).  Segment-ID
-  groups of 256 blocks confirmed.
-- Geo peaks: every block of N844L20G, N844L6Z8, N844L6XE, N844L23B
-  matches `.TXT` within the 0.0005 in/s quantization step.
-- Geo freqs: every block of N844L6Z8 and N844L6XE matches `.TXT`
-  within 1 Hz (BW display rounds).  `>100 Hz` sentinel handled correctly.
-- Mic dB: every block of N844L6XE, N844L23B, N844L6Z8 matches `.TXT`
-  within 0.1 dB (BW display precision).
-- Mic freq: matches `.TXT` within 1 Hz across active blocks.
-
-## What's NOT yet decoded
-
-- **Annotation bytes (`block[7]/[11]/[15]/[19]`)**.  Empirically
-  non-zero on intervals where the per-channel ZC frequency comes
-  out as `N/A` or sub-Hz (`<1.0`, `1.X`).  Hypothesis tested in the
-  RE session: byte != 0 ↔ sub-Hz freq.  Only ~50% correlation
-  across the K558 corpus, so the relationship is more complex.
-  Possibilities: time-of-peak-within-interval, halfp extension for
-  very-long-period signals, or a debug/diagnostic field the firmware
-  writes opportunistically.  Doesn't affect peak amplitudes or
-  waveform reconstruction.  Captured as `record["annotations"]` for
-  future RE.
-- **4-byte variable metadata field (bytes 24:28)**.  Not needed for
-  waveform reconstruction.  Speculation: per-block CRC, sub-second
-  timestamp offset, or a Mic psi(L) count not in the 9 samples.
-  Punt until something needs it.
-- **Geo PVS (TXT col 7, e.g. "0.040 in/s")**.  Not stored in the
-  block; can be approximated as `sqrt(T_peak² + V_peak² + L_peak²)`
-  but BW's value sometimes differs slightly (probably computed from
-  waveform-instant samples, not from per-channel peaks).  Punt — the
-  `.h5` consumers don't need PVS as a sample channel.
-- **Mic psi(L) value (TXT col 8)**.  TXT shows it as a small psi value
-  derived from the dB measurement.  Not in the 9 samples.  Could be
-  derived from `M_peak_count` via the inverse of the dB formula plus
-  a psi calibration constant.  Defer.
-
-## Output shape
-
-`decode_histogram_body` returns the standard 4-channel dict that
-mirrors `waveform_codec.decode_waveform_v2`'s output:
-
-```python
-{
-    "Tran": [peak_count_per_interval, ...],   # 16-count units (LSB = 0.005 in/s)
-    "Vert": [..., ...],
-    "Long": [..., ...],
-    "MicL": [..., ...],                       # raw ADC counts
-}
-```
-
-Run through `waveform_codec.decoded_to_adc_counts` to get 1-count ADC
-units (geo ×16, mic passthrough) for the standard `.h5` writer.
-
-For the full per-interval record with frequencies + metadata, use
-`decode_histogram_body_full()`.
-
-## Where it's wired
-
-- `minimateplus/event_file_io.py:read_blastware_file()` — first tries
-  the waveform codec, falls back to the histogram codec when the
-  waveform preamble isn't present.  Same output shape, same
-  downstream pipeline.
-- `scripts/backfill_sidecars.py` — the `has_samples` short-circuit
-  added during the histogram-codec-pending era still serves as a
-  defensive guard against truly undecodable files, but no longer
-  fires for valid histograms.
-
-## Companion reference
-
-- `docs/waveform_codec_re_status.md` — sibling status doc for the
-  much-more-complex waveform-mode codec.
-- `docs/instantel_protocol_reference.md §7.6.2` — historical
-  protocol-reference entry.  Structural framing matches what we
-  found; per-sample semantics were less documented than the `✅
-  CONFIRMED` badge suggested.  This doc supersedes §7.6.2 where they
-  conflict on confidence level.

docs/idf_protocol_reference.md

@@ -1,563 +0,0 @@
-# IDF Protocol Reference — Thor / Micromate Series IV
-
-Starting-point reference for reverse-engineering Instantel's Micromate
-Series IV event-file format.  Sibling to
-[instantel_protocol_reference.md](instantel_protocol_reference.md) (the
-Series III "Rosetta Stone") — this doc holds what we know so far and
-the open questions still to crack.
-
-> ⚠ **The "Status (2026-05-28)" block below is SUPERSEDED.**  Its geo LSB
-> (0.0003), its IDFH scale (`/32768 × 10`), its fixed body offset (`0x0f1f`)
-> and its "87–99% byte-exact / loud events truncate" caveat were all wrong or
-> incomplete.  See **[Verified against Thor's own exports
-> (2026-09-10)](#verified-against-thors-own-exports-2026-09-10)** — the
-> decoder is now per-sample exact on 1,057,536/1,057,536 samples.  The block
-> is kept only for the reverse-engineering trail.
-
-**Status (2026-05-28, SUPERSEDED):** ASCII text sidecar fully decoded (1,014
-sample files round-trip).  **Thor IDFW** binary now decodes via
-`micromate.idf_file.read_idf_file()` — reuses the BW segment-rotated
-block codec verbatim at fixed body offset `0x0f1f`; metadata (serial,
-timestamp, sample_rate, record_time, calibration_date) extracted from
-the binary header.  Sample fidelity is 87–99% byte-exact on quiet
-events; loud events hit the BW codec's known walker-stops-early
-limitation.  Residual ~3% drift on per-sample deltas (likely a
-Thor-specific 12-bit delta refinement not yet modelled).
-
-**Thor IDFH histograms also decoded.**  Body has one or more segments;
-each 12-byte segment header `[length_be 2B][0a 00 00 00][00 NN][05 3f]`
-introduces `N = (length - 10) // 72` interval records of 72 bytes
-each.  Each interval = 4 × 16-byte per-channel records:
-`[int16 min][int16 max][int16 ??][uint16 halfp][2B 00][uint16 ??][2B 00][uint16 ??]`.
-Geo peak `= max(|min|, |max|) / 32768 × 10` in/s (matches sidecar
-~1.8%); freq `= 512 / halfp` Hz (None for halfp ≤ 5 → ">100"
-sentinel).  Corpus: **all 859 Thor IDFH files decode, 181,071
-intervals**.  Wired through `read_idf_file()` →
-`save_imported_idf()` → sidecar's `extensions.idf_intervals`.
-
-**Note on the BE9439 outliers in the example corpus:** Two files
-(`BE9439_20200713131747.IDFW` and `BE9439_20200713124251.IDFH`) are
-**Series III Blastware** binaries, not Thor.  Provenance: TMI tried
-to use Thor to manage auto-call-homes for Series III units; the
-experiment didn't work out, but it did leave a few BW event files
-in Thor's per-serial directory structure with `.IDFW`/`.IDFH`
-extensions — Thor's forwarder applied its own naming convention to
-the BW bodies it was relaying.  Their header `10 00 01 80 00 00
-Instantel STRT ff fe <end_key> <start_key>` is the BW SUB 5A STRT
-record, not a Thor body preamble.  The reader detects them by
-signature and raises `NotImplementedError` pointing callers at
-`read_blastware_file()`, which extracts BW-format peaks from them.
-
-**Still NYI for Thor IDFH:** per-channel `int16 field4` (possibly
-time-of-peak); the two uint16 fields (probably PVS contributions);
-8-byte interval tail (PVS data); mic dB(L) exact conversion constant.
-
-## Verified against Thor's own exports (2026-09-10)
-
-**The series-4 decoder is now per-sample exact.**  1,057,536 / 1,057,536
-geophone samples across all 153 genuine Thor waveform files reproduce Thor's
-own CSV export exactly; histogram peaks land within 2% on 858/858 files
-(median error −0.004%).
-
-### Ground truth — it was there all along
-
-Thor writes `TXT/`, `CSV/`, `XML/` and `PDF/` exports beside every binary:
-
-```
-<serial dir>/UM13981_20220207084555.IDFW
-<serial dir>/CSV/UM13981_20220207084555.IDFW.csv
-```
-
-The **CSV carries a per-sample block** — four columns (Tran, Vert, Long, Mic)
-in in/s and psi, after the 2-column report header.  That is the series-4
-equivalent of Blastware's `_ASCII.TXT` exports, and it gives 1,012 paired
-files (152 IDFW + 860 IDFH).  Earlier notes in this file and in
-`micromate/idf_file.py` asserted "Thor has no ASCII ground truth in the
-corpus"; that was wrong, and it is why the decoder sat pinned to a
-superseded walker with a scaling constant nobody could check.
-
-Harness: `scratch/verify_thor_against_csv.py`.
-
-### Geo LSB = 0.000310308 in/s per count (NOT 0.0003)
-
-The old 0.0003 was read off the smallest non-zero sample in the exports —
-but that is Thor's **4-decimal display rounding of the LSB, not the LSB**.
-It read every series-4 geophone sample **3.3% low**.  The quantisation
-ladder gives it away: counts 1..6 export as 0.0003, 0.0006, 0.0009, 0.0012,
-0.0016, 0.0019 — an LSB of exactly 0.0003 would end 0.0015, 0.0018.
-
-Each exported sample constrains the LSB to the window that rounds to its
-printed value.  Intersecting 991,415 such constraints gives
-
-```
-LSB ∈ [0.000310307933, 0.000310308057]      width 1.2e-10
-```
-
-so `_GEO_LSB_IPS = 0.000310308`, i.e. full scale 10.0 in/s = **32226.05
-counts**.  Corroboration: an IDFH interval that never recorded keeps its
-min/max accumulator at its ±full-scale seed, and that seed is
-`(min=+32226, max=-32226)`.  ⚠ The tempting closed form `10.0/32226` is
-very slightly wrong — it lands 4.5e-10 above the feasible window and loses
-78 boundary samples while never winning one.  **Series III uses 32000 counts
-for the same 10.0 in/s, so the two generations do not share a scale.**
-
-Independently confirmed on 8 production units (UM6047, UM11402, UM11719,
-UM12947, UM13981, UM14133, UM20146, UM20147): every unit's median PPV error
-against its device-reported peak moved from −3.3% to within ±0.03%.  It is a
-global constant, not a per-unit calibration.
-
-### IDFH segment header: the counter is a uint16, and it is cumulative
-
-```
-[length_be 2B][0a 00 00 00][counter_be 2B][05 3f]
-```
-
-`counter` is the **0-based cumulative index of the last interval in the
-segment** — 9, 19, 29, ... for the usual 10-intervals-per-segment layout
-(`length` = 730).
-
-The validator used to require `counter`'s high byte to be `0x00`.  That
-silently **capped every histogram at 250 intervals**: once the cumulative
-counter passed 255 the high byte went non-zero and every later segment was
-rejected.  Any run longer than ~4 hours lost its tail — frequently the part
-holding the event peak, so the file's PPV read low.  **540 of 858 corpus
-files were affected**; fixing it moved histogram peaks from 48.3% to 93.8%
-within 0.5% of Thor's reported PPV.
-
-### Unwritten interval slots carry a ±full-scale seed
-
-An interval the device reserved but never wrote keeps `min = +32226`,
-`max = -32226` on all four channels — `min > max`, impossible for real data.
-Decoded naively it yields a 10.0 in/s peak on every channel and, being a
-max-over-intervals, poisons the whole file's PPV.  Rare but real: exactly 1
-of 497,611 corpus intervals, and it inflated that file's Long PPV from
-0.0081 to 10.0 in/s.  The inversion is all-or-nothing across channels (0
-partial cases), so requiring every channel to be inverted is a safe test.
-
-### Record mode `00 00` — raw int16 absolute (MODE_RAW16)
-
-The record chain's mode field at `off+8` takes a fourth value:
-
-| mode | meaning | header |
-|---|---|---|
-| `02 00` | deltas + two int16 anchors | 14 B |
-| `01 00` | absolute, tagged blocks | 10 B |
-| `00 03` | raw 12-bit absolute, untagged | 10 B |
-| **`00 00`** | **raw int16 BE absolute, untagged** | **10 B** |
-
-A `MODE_RAW16` record with `length = 1032` carries exactly
-`(1032 - 8) / 2 = 512` samples and reproduced Thor's export **512/512
-exactly** on first test.  Thor uses it for segment 0 (the pre-trigger
-window) on some events.  Before this mode existed the record fell through
-the dispatch unhandled, so the channel silently lost its first 512 samples —
-which is what produced the "loud events truncate" symptom.
-
-`MODE_ABSOLUTE` is also valid as a **preamble** (the implicit segment-0 Tran
-record); its tagged blocks start at `body[3]`, not `body[7]`, because its
-header is 10 bytes rather than 14.
-
-### Body offset is not fixed at 0x0f1f — and 0x0f1f is really a record + 7
-
-A "body offset" is `<record start> + 7`, so that `body[0]` is the segment
-index and `body[1:3]` is the mode.  The canonical `0x0f1f` is simply the
-record at `0x0f18`.
-
-Searching for the literal preamble `00 02 00` finds only MODE_DELTA bodies,
-and worse, it **matches the `[seg][mode]` bytes inside any record header**,
-so the scan could pick a candidate part-way down the chain.  That decodes a
-plausible-looking but rotation-shifted body which drops each channel's
-segment 0 — the real cause of the remaining truncations.
-
-`_find_waveform_body_offset()` now anchors on record headers (the
-`<channel_id> 00 00` signature at `+4`, validated with `is_record()`),
-takes the **chain head** — a record no other record's length field points at
-— and trial-decodes `head + 7`, preferring the candidate where all four
-channels come out the same length.
-
-⚠ Do **not** scan for candidate preambles instead: `MODE_RAW16` is
-`00 00`, so every run of three zero bytes looks like a body start and each
-costs a full trial decode (~0.5 s/file measured, vs 6 ms/file now).
-
-### `40 NN` is not capped at NN=8 (2026-09-11)
-
-`data_block_len()` rejected any `40 NN` int16 block with `NN > 0x08`.  The cap
-had no evidence behind it — every corpus available when it was written used
-only NN ∈ {1, 2, 3, 4, 8}, so it was never exercised.  Loud events use much
-wider blocks:
-
-| corpus | `40 NN` values | walker stops |
-|---|---|---|
-| first + 3-channel corpora | 1, 2, 3, 4, 8 | none |
-| UM12947 2025-07..09 | 2, 4, 8, **12, 16, 20 … 196** | every value > 8 |
-
-Because `walk_body`/`run` stop at the first unrecognised tag rather than
-raising, this surfaced as **silently short channels** — e.g. Tran 1812 /
-Vert 2132 / Long 2324 on a file whose export has 2324 for all three.  The
-real bound is the buffer (and the caller's record end), not a magic constant.
-
-Verified against Thor's exports for UM12947 (2025-07-14 … 2025-09-25, 167
-waveforms): length mismatches **22 → 0**, and **1,476,242 / 1,476,249**
-samples exact.
-
-⚠ These events are **not** truncated recordings, which was the competing
-hypothesis — the exports carry the full sample count.
-
-**The 7 residual samples are Thor's rounding, not ours.**  Each differs by
-exactly one 4th-decimal tick (e.g. decoded 3.3551 vs export 3.3550).
-Intersecting the per-sample rounding constraints over this corpus is
-**infeasible** — the binding pair (count 2013 → 0.6247, count 4351 → 1.3501)
-contradict by 2.3e-11, i.e. 7e-5 relative.  No single linear LSB can
-reproduce every printed value, so Thor is not doing plain round-half-up on
-`count × LSB`.  Do not retune `_GEO_LSB_IPS` to chase these; it is already
-pinned to ~1e-11.
-
-### Mic-disabled units are a distinct shape (2026-09-10, second corpus)
-
-Some units run with the microphone disabled — **3 channels, not 4** — and that
-changes two structural things.  Confirmed on the `9-10-26-csv-req` corpus
-(UM11402, UM12947, UM20147): 139/139 waveforms and 877/877 histograms.
-
-**Waveform: the body starts earlier.**  A 3-channel unit has a shorter fixed
-header and puts its record chain head at **`0x0dba`**, below the old
-`_BODY_SCAN_FLOOR` of `0x0E00`.  The head was therefore invisible to the scan,
-which fell through to the *Vert* segment-0 record and decoded a body shifted
-one position around the channel rotation.  The signature is unmistakable:
-
-```
-Tran 3072 / Vert 2560 / Long 3072 / MicL 0      <- Vert exactly 512 short
-```
-
-46 of 139 files in that corpus were affected; all 46 became per-sample exact
-once the floor dropped to `0x0C00`.  Note the body-offset scoring also had to
-stop requiring four channels — `len(lengths) >= 3`, not `== 4`, or `equal` is
-permanently False for these events and the pick falls back to raw sample count.
-
-**Histogram: the interval record is 56 bytes, not 72.**
-
-```
-interval_size = 16 × n_channels + 8        (72 for 4 channels, 56 for 3)
-```
-
-It is **not a constant**, and it cannot be inferred from `length` alone.
-Derive the interval count from the segment counter — it is cumulative, so
-`n = counter - previous_counter` — and then `stride = (length - 10) / n`.
-`n_channels` follows from `(stride - 8) / 16`.
-
-Assuming 72 read 7 intervals out of each 10-interval segment and then walked
-off alignment into garbage that decoded as ~10 in/s peaks — inflating those
-files' PPV by up to 191,000%.  Fixing it moved the second corpus from 56.6% to
-**100.0%** of histograms within 2% of Thor's reported PPV, and recovered 4
-files that previously decoded no intervals at all.
-
-### What is still open
-
-- ~~23 of 575 production IDFW files~~ — **RESOLVED 2026-09-11.** Production
-  IDFW is now **575/575** with zero truncations and zero decode failures
-  (median PPV error −0.0007%).  See "`40 NN` is not capped at NN=8" above.
-
-- Mic → psi scale is still the rough `2.14e-6` regression, not derived.
-- Per-channel `int16 field4` in the IDFH interval record (possibly
-  time-of-peak) and the 8-byte tail (PVS data) remain undecoded.
-
-⚠ **Thor's histogram PPV has a display floor of 0.0050 in/s.**  In the
-production store 6,080 sidecar PPV values are exactly 0.0050 (next most
-common value: 275 occurrences), and **41.4% of IDFH sidecars report a
-component PPV larger than their own vector sum** — geometrically impossible.
-On those quiet files the decoder's ~0.0025 in/s is *more* accurate than the
-reference; do not "fix" the decoder to match it.
-
-### Codec breakthroughs (2026-05-28)
-
-- **Body offset is a fixed `0x0f1f`** across 151/154 corpus IDFW
-  files.  Preceded by a 4-byte record-type marker (`46 00 00 00`)
-  + magic preamble `00 02 00 [Tran[0] BE] [Tran[1] BE]`.
-- **Sample stream is BW's segment-rotated block codec verbatim.**
-  Thor reuses `10 NN` (nibble), `20 NN` (int8), `00 NN` (RLE),
-  `30 NN` (packed12), `40 02` (segment header) tags with the same
-  semantics.  Channel rotation Tran→Vert→Long→MicL.
-- **Geo LSB = 0.0003 in/s** (not BW's 0.005), because Thor's 16-bit
-  ADC range maps to 10 in/s without the 16-count BW quantization step.
-- **Mic ≈ 2.14×10⁻⁶ psi/count** (rough scale; refine after channel
-  block calibration constants are decoded).
-- **BW compliance anchor `\xbe\x80\x00\x00\x00\x00` reappears at
-  IDFW offset 0x952** — sample_rate at anchor−6 (uint16 BE),
-  record_time at anchor+6 (float32 BE), same layout as BW.
-- **Event timestamp at offset 0x97A** — 8 bytes `[day][month]
-  [year_be][unk][hour][min][sec]`.  Stop-time mirrors at 0x982.
-- **Serial as null-terminated ASCII at 0x14E**.
-- **Calibration date** at 0x194–0x197 (day, month, year_be).
-- Per-sample residual drift of ~3% suggests Thor encodes int8/nibble
-  deltas with an extra refinement bit that BW doesn't carry —
-  unsolved; errors resync within a few samples so cumulative impact
-  is small.
-
----
-
-## File model
-
-### Filename convention
-
-```
-<SERIAL>_<YYYYMMDDHHMMSS>.<KIND>
-```
-
-- **SERIAL** — literal device serial, two-letter prefix + numeric
-  suffix.  Examples seen: `UM11719`, `UM13981`, `UM20147`, `BE9439`.
-  Unlike Series III BW filenames (`M529LK44.AB0`, base-36 stem),
-  Series IV filenames carry the serial in plain text.
-- **YYYYMMDDHHMMSS** — 14-char ASCII timestamp in **device local
-  time** (no timezone marker).
-- **KIND** — `IDFH` for histograms, `IDFW` for waveforms.
-
-The `.IDFH.txt` / `.IDFW.txt` ASCII sidecar lives in a `TXT/`
-**subfolder** of the unit's directory, not alongside the binary.
-This pairing convention is encoded in
-`event_forwarder.idf_report_path()`.
-
-### Directory layout
-
-```
-C:\THORDATA\
-└── <Project>\
-    └── <UM####>\                  ← unit serial dir
-        ├── UM12345_20260520100000.MLG     ← monitor log (not events)
-        ├── UM12345_20260520100000.IDFH    ← histogram event (binary)
-        ├── UM12345_20260520100000.IDFW    ← waveform event (binary)
-        ├── UM12345_20260520100000.IDFW.CDB ← cache-DB variant (skip)
-        ├── TXT\
-        │   ├── UM12345_20260520100000.IDFH.txt    ← histogram ASCII sidecar
-        │   └── UM12345_20260520100000.IDFW.txt    ← waveform  ASCII sidecar
-        ├── CSV\, HTML\, PDF\, XML\        ← operator-facing derived exports
-        └── ...
-```
-
-The `.IDFW.CDB` files share the binary's basename but appear to be a
-separate cache/database variant.  Their first 8 bytes match the
-**old**-firmware Thor signature (see below) regardless of which
-signature the paired `.IDFW` uses.  Purpose unknown; sizes vary
-wildly (observed 123 B → 40,491 B).  Thor-watcher's forwarder
-deliberately skips them.
-
-### Sample corpus
-
-The `thor-watcher/example-data/THORDATA_example/` tree carries
-**1,014 paired .IDFW / .IDFH + .txt files** spanning 2020–2023
-across nine units (UM11719, UM13981, UM20147, …, plus BE9439 from
-2020).  This is the reverse-engineering ground truth.
-
----
-
-## ASCII sidecar (`.IDFW.txt` / `.IDFH.txt`) — fully decoded
-
-Shape: plain text, one `"Key : Value"` line per metadata field,
-followed for waveforms by a tab-separated sample table headed by
-the literal line `Waveform Data Channels`.  Parsed by
-[`micromate/idf_ascii_report.py`](../micromate/idf_ascii_report.py).
-See [`micromate/models.py`](../micromate/models.py) for the typed
-`IdfReport` shape.
-
-### Notable conventions
-
-- **Units are native to Thor** — geophone in **in/s**, microphone in
-  **dB(L)** (not psi like Series III BW reports), frequency in Hz,
-  acceleration in g, displacement in in.
-- **Below-threshold readings** appear as the literal string
-  `<0.005 in/s` (155 occurrences in the sample corpus) — the parser
-  strips the `<` and treats the numeric remainder as the value.
-- **Out-of-range / not-measured** values appear as `N/A` — parser
-  drops the field rather than letting the string leak into a numeric
-  column.
-- **Firmware string** observed: `Micromate ISEE 11.0AK`.
-- **TitleString1..4** are operator-defined free-text slots; Thor's
-  default labels map them to Location / Client / Company / Notes,
-  which the parser surfaces as `project` / `client` / `operator` /
-  `notes`.
-- **Histogram sidecars** use `HistogramStartDate` / `HistogramStartTime`
-  in place of waveform's `EventDate` / `EventTime`.  Parser falls
-  through to either.
-- **Histogram tabular block** lacks the `Waveform Data Channels`
-  marker; instead it's a multi-line column header followed by
-  per-interval rows (`<date> <time> <tran-ppv> <freq> ...`).  Parser
-  silently ignores lines after the metadata block since they lack a
-  colon-separated `key : value` shape (the timestamps DO contain
-  colons but produce garbage keys that don't collide with any
-  recognised field).
-
----
-
-## Binary header signatures (observed)
-
-Hex dump of the first 32 bytes across 1,014 sample files reveals
-**two distinct file signatures**, both anchored by the literal
-ASCII string `"\x00Instantel\x00"` at offset 6–16:
-
-### Signature A — newer firmware (1,012 files, 99.8% of corpus)
-
-```
-00000000: 0012 0100 0000 496e 7374 616e 7465 6c00   ......Instantel.
-00000010: 0000 a695 002e b500 4f70 6572 6174 6f72   ........Operator
-                                ^^^^^^^^^^^^^^^^
-                                operator/title string starts at 0x18
-```
-
-Header bytes 0–5: `00 12 01 00 00 00`.  Followed immediately by the
-8-byte ASCII tag, then 6 unknown bytes, then ASCII operator-supplied
-strings (Operator name, etc.) and on through the project / client /
-title strings.  No `STRT` record observed in this layout.
-
-### Signature B — older firmware (2 files: BE9439 from 2020)
-
-```
-00000000: 1000 0180 0000 496e 7374 616e 7465 6c00   ......Instantel.
-00000010: 072c 0012 0300 5354 5254 fffe 0111 2340   .,....STRT....#@
-                          ^^^^^^^^^                ^^^^^^^^^
-                          STRT magic               4-byte end_key
-00000020: 0111 0000 2e5f 00ac 4600 0000 0200 0000   ....._..F.......
-          ^^^^^^^^^             ^^^
-          4-byte start_key      0x46 (BW WAVEHDR record-type marker)
-```
-
-Header bytes 0–5: `10 00 01 80 00 00`.  The structure after the
-`Instantel` magic is **byte-for-byte identical to a BW SUB 5A
-probe-response STRT record** as documented in
-[instantel_protocol_reference.md → "SUB 5A — STRT record encodes
-end_offset"](instantel_protocol_reference.md).  Specifically:
-
-| Offset | Bytes               | Meaning (per BW reference)          |
-|--------|---------------------|--------------------------------------|
-| 0x14   | `53 54 52 54`       | `STRT` magic                         |
-| 0x18   | `ff fe`             | STRT sentinel                        |
-| 0x1A   | `01 11 23 40`       | `end_key` (4 bytes)                  |
-| 0x1E   | `01 11 00 00`       | `start_key` (4 bytes)                |
-| 0x26   | `46`                | `0x46` waveform-record type marker   |
-
-**Hypothesis:** Older Micromate firmware writes a wrapped BW-format
-event into the `.IDFW` file — essentially the same on-disk shape as
-a Series III device, with the new filename convention applied at
-export time.  Newer firmware (signature A) abandoned the
-BW-compatible layout for an Instantel-specific format.
-
-If that hypothesis holds, the 2 signature-B files can already be
-parsed via `minimateplus/event_file_io.read_blastware_file()` — worth
-testing.  The 1,012 signature-A files are the real reverse-engineering
-target.
-
-### `.IDFW.CDB` cache files
-
-Always carry signature B (`10 00 01 80 ...`), even when the paired
-`.IDFW` carries signature A.  Plausible explanation: the CDB is an
-internal Thor cache-database export that retains the legacy BW-style
-record layout regardless of the user-facing `.IDFW` format version.
-Not currently consumed by the forwarder.
-
----
-
-## File-size patterns (Signature A, the main target)
-
-Survey of 1,012 signature-A files:
-
-| Event type   | Typical size      | Source of variance                           |
-|--------------|-------------------|----------------------------------------------|
-| `.IDFW` 2-sec | 9,200 – 10,500 B | Operator-supplied strings (TitleString1..4) of varying length |
-| `.IDFH`       | 2,944 – 4,076 B  | Histogram interval count (record duration / interval) |
-
-**Naive arithmetic for 2-sec waveform:**
-- 4 channels × 2 sec × 1024 sps = 8,192 samples
-- At 2 bytes/sample (int16) = 16,384 sample bytes → file would be > 16 KB
-- Observed: ~9–10 KB
-- → samples are likely **1 byte each** (int8 quantised), **or** stored
-  with bit-packing / delta encoding, **or** only one channel's
-  full-rate samples are stored with the others reconstructed
-  arithmetically.  Verifying this is the **first RE milestone**.
-
-Project-string–length variance (~1 KB across the corpus) is consistent
-with the file carrying a single copy of each TitleString1..4 plus
-operator + setup-name as null-padded ASCII regions.
-
----
-
-## Open questions
-
-The reverse-engineering targets, roughly in dependency order:
-
-1. **Sample encoding (signature A)** — int8? int16 LE/BE? Bit-packed?
-   Delta-coded?  Per-channel interleaved or sequential blocks?
-2. **Header field layout (signature A)** — where do sample_rate,
-   record_time, channel count, and per-channel peaks live in the
-   binary?  The ASCII sidecar gives the device-authoritative values,
-   so binary fields can be confirmed by diff.
-3. **Operator-string offsets** — `Operator` at 0x18 is the first
-   visible string in signature-A files; the rest (project, client,
-   notes, setup) follow.  Need to map exact offsets and null-padding
-   conventions.
-4. **Signature-B → BW codec compatibility** — does
-   `minimateplus/event_file_io.read_blastware_file()` actually parse
-   the 2 BE9439 signature-B files as-is?  If yes, the OLD-format
-   ingest is free.
-5. **`.IDFW.CDB` purpose** — is it an internal Thor cache, a
-   ring-buffer dump, or something else?  Worth a single small effort
-   to characterise so we know what we're skipping.
-6. **Footer / checksum** — every BW event file has a footer; does
-   IDF?  Where does the per-channel sample block end?
-
----
-
-## Reverse-engineering playbook (when we start)
-
-The Series III BW codec took ~2 months of MITM wire captures
-because we didn't have ground-truth metadata.  Thor's situation is
-**substantially better**:
-
-- **Ground truth is on disk.**  Every binary in `example-data/`
-  has a paired `.IDFW.txt` carrying the full decoded sample table
-  (`Waveform Data Channels` block — see any sample file in
-  `thor-watcher/example-data/.../TXT/`).  Aligning binary bytes
-  to the table's float-per-row values gives an immediate per-byte
-  hypothesis test.
-- **Cross-event diffing.**  1,012 signature-A samples from 9 units
-  spanning 4 years means any field that varies between events is
-  immediately localisable.  Fields that are constant across all
-  files (firmware ID, channel labels, format-version word) are also
-  immediately localisable by complementary search.
-- **No protocol surface.**  Files at rest, not a wire dialect.  No
-  DLE stuffing, no inner-frame parsing, no probe/data two-step.
-
-Suggested first session (2-4 hours): hand-decode `UM11719_20231219162723.IDFW`
-(10,290 bytes) against its `TXT/UM11719_20231219162723.IDFW.txt`
-sample table (the 2-sec waveform at 1024 sps × 4 channels = 8,192
-sample rows).  Find the first per-channel sample value (`0.0003` in
-the Tran column at t=0) in the binary.  Confirms sample encoding.
-Everything else flows from there.
-
----
-
-## Code seams ready to receive the codec
-
-When the codec lands, it goes into
-[`micromate/idf_file.py`](../micromate/idf_file.py) (currently a
-stub raising `NotImplementedError`).  Public API:
-
-```python
-from micromate import IdfEvent
-from micromate.idf_file import read_idf_file
-
-event: IdfEvent = read_idf_file(Path("UM11719_20231219163444.IDFW"))
-# event.peaks.transverse_ips, event.timestamp, event.raw_samples, ...
-```
-
-The ingest pipeline (`WaveformStore.save_imported_idf`) currently
-builds the `IdfEvent` from the `.txt` parser only.  Once
-`read_idf_file()` works, the binary becomes authoritative; the
-`.txt` parser drops to fast-path metadata cross-check.  Operators
-who don't enable Thor's TXT exporter still get fully populated
-events.
-
----
-
-## See also
-
-- [instantel_protocol_reference.md](instantel_protocol_reference.md) — Series III BW protocol reference (the Rosetta Stone).  STRT record format, DLE framing, BW filename encoding.
-- [`micromate/idf_ascii_report.py`](../micromate/idf_ascii_report.py) — `.txt` sidecar parser.
-- [`micromate/models.py`](../micromate/models.py) — `IdfEvent`, `IdfReport` typed dataclasses.
-- [`micromate/idf_file.py`](../micromate/idf_file.py) — placeholder for the binary codec.
-- [`thor-watcher/example-data/THORDATA_example/`](../../thor-watcher/example-data/) — 1,014 paired binary + .txt files for codec validation.

docs/offset_investigation.md

@@ -1,987 +0,0 @@
-# The "offset" fault — investigation journal
-
-> ## ⚠ CORRECTED 2026-08-28 (same day) — the v1 detector was wrong
->
-> Brian pushed back on the finding that offsets "come and go": in the field,
-> once a unit develops one it stays broken until the geophone is replaced.
-> He was right, and the challenge exposed **two real flaws** in the v1 detector:
->
-> 1. **It scored only the axis with the largest peak.** A real event on one axis
->    hid a persistent pedestal on another. BE12599 on 2026-08-21 read "clean"
->    solely because Long had a 1.065 in/s event — Tran was sitting at
->    **+0.4732 in/s** at that moment and was never examined.
-> 2. **It used the MEAN**, which a real transient perturbs. The **median** is the
->    resting baseline — most samples sit at it, so a blast does not move it.
->    Same event, Long channel: mean **+0.0783** vs median **-0.0050**.
->
-> Both flaws manufactured false recoveries. The corrected detector
-> (`scratch/offset_scan2.py`, per-channel median) shows the pedestal is
-> **persistent**, exactly as the field experience says. See §2b and §3b.
->
-> **Then Brian proposed a better detector still** — measure the floor during
-> the *pre-trigger* window, and require it to hold across pre/middle/end.
-> That is now the detector of record (§2c). Final answer: **5 of 45 units
-> (11%)**, stable across a 2x threshold range.
->
-> Sections below that were written against v1 are marked; v1 numbers are kept
-> for the reasoning trail, not as current fact.
-
-A running record of the **offset** hardware fault on Instantel Series III
-seismographs: a geophone channel whose trace sits displaced from zero rather
-than centred on it.
-
-This is a *journal*, not a spec. Findings are dated, dead ends are kept with
-the reason they died, and every number says where it came from. When something
-here is superseded, strike it and say why rather than deleting it — the point
-is that a future session can tell what was actually established from what was
-merely believed at the time.
-
-Companion material:
-- `scratch/offset_scan.py` — the detector
-- `scratch/verify_against_ascii.py` — decoder verification harness
-- `docs/instantel_protocol_reference.md` — wire protocol, incl. the
-  unimplemented `SUB 0x0E` this investigation now wants
-
----
-
-## TL;DR (current state, 2026-08-28)
-
-- **It is real device data, not a decode bug.** Settled early and confirmed
-  against Blastware's own ASCII exports.
-- **Base rate: 5–6 of 45 units (11–13%)** across the full DL2 archive,
-  2018–2026. This *confirms* the earlier 2-of-21 (9.5%) estimate from the much
-  smaller Terra-View DB — survivorship bias from deleted events had **not**
-  concealed a wave of cases.
-- **The fault is bimodal, not a drift continuum.** A unit is either clean or
-  grossly off. Loosening the amplitude threshold 11× adds no new units.
-- **The unit's own sensor check cannot see it.** 102 offset events, zero
-  sensor-check failures. Do not try to use it as a screen.
-- **Cause is still unsettled.** Instantel's autozero fixes the minority of
-  cases; the rest are hardware. We cannot yet tell which is which remotely.
-- **The histogram corpus (63,535 files, 9.7x the waveforms) is now scanned too** —
-  see §8b. It independently confirms BE18438 and BE9558 with a clean 2.5x
-  separation, but detects only **2 of the 5** confirmed units, cannot attribute a
-  channel, and resolves time to ~a month. **A negative histogram result is not
-  evidence of health** — DC leakage into the interval peak varies 45x between units.
-- **`offset_scan3.py` has a label defect** (§8b): its spread gate discards 18.8% of
-  high-|pre| rows onto units currently counted as clean. Re-cut before quoting any
-  precision number again.
-- **Best open lead:** `SUB 0x0E` (channel sensor data, 8 channels × 10 bytes,
-  unimplemented) may carry the very numbers Instantel says to check against
-  **2027–2069**. Untested.
-
----
-
-## 1. What the fault looks like
-
-A healthy geophone trace is centred on zero. An offset channel is parked away
-from zero, so the channel **mean approaches its own peak**. In Blastware the
-signature is "parallel lines above or below the zero line" (Instantel's own
-wording).
-
-Consequences observed in the field:
-- The unit can **self-trigger on its own offset** when the displacement exceeds
-  the geo trigger level, producing streams of junk events with no ground
-  motion. Instantel has a separate FAQ for this symptom (13-0-22, *"Unit
-  triggers continuously without activity"*).
-- Recorded PPV for that channel is meaningless while the fault persists.
-
----
-
-## 2. The detector
-
-Implemented in `scratch/offset_scan.py`. Operates on raw BW binaries only — no
-DB, no sidecars.
-
-```
-for each series-3 waveform binary:
-    decode -> per-channel ADC counts
-    dominant axis = channel with the largest |peak|
-    flag when   |mean| / peak  >  0.70
-          and   |mean|         >= 0.90 x the unit's geo trigger level
-episodes = per-serial runs of flagged events, split on a >12 h gap
-```
-
-Why each term:
-
-| term | purpose |
-|---|---|
-| `\|mean\|/peak > 0.7` | the discriminator. A DC-parked trace has mean ≈ peak. |
-| `\|mean\| >= 0.9 × trigger` | amplitude floor — suppresses quiet traces where mean and peak are both tiny and the ratio is meaningless. |
-| dominant axis only | the fault is per-channel; scoring all three dilutes it. |
-| 12 h episode gap | separates deployments/visits rather than counting events. |
-
-Trigger level comes from a paired `_ASCII.TXT` when one exists, else the
-per-serial median learned from that unit's ASCII files, else 0.2 in/s.
-
-**Known limitation.** Event traces contain real ground motion, so this can only
-see offsets large enough to *dominate* the trace. A mild offset on a real blast
-is invisible. Instantel's A/D-mode check (§5) is the only thing that sees the
-mild end. Our base rate is therefore a **gross-offset** rate.
-
-### 2b. Detector v2 — per-channel median (CURRENT)
-
-`scratch/offset_scan2.py`. Supersedes the above.
-
-```
-for each series-3 waveform binary:
-    for each geo channel independently:
-        pedestal = median(samples)          # resting baseline, robust to blasts
-        flag the CHANNEL when |pedestal| >= 0.025 in/s   (5 A/D counts)
-a unit has a real fault when a channel is flagged on >=3 CONSECUTIVE events
-```
-
-Why median: a DC pedestal shifts every sample, so it moves the median. A real
-event moves only a minority of samples, so it does not. This removes the need
-for the `m/p` ratio guard entirely — that guard existed only to compensate for
-using the mean.
-
-Why per-channel: the fault is on one geophone axis. Scoring only the dominant
-axis means any event with motion elsewhere hides it.
-
-Why "3 consecutive": the 0.025 in/s floor is only ~2x a healthy channel's
-resting median (observed 0.010-0.015), so isolated flags are noise. Persistence
-is the discriminator — and it is what the field experience predicts.
-
----
-
-## 3b. Archive results, corrected (v2)
-
-| | v1 (dominant axis, mean) | **v2 (per-channel median)** |
-|---|---|---|
-| units with any flagged event | 6 of 45 | 19 of 45 |
-| **units with a sustained pedestal (>=3 consecutive)** | — | **8 of 45 (18%)** |
-| runs of >=3 consecutive | — | 29 |
-| runs of 1-2 events (noise) | — | 69 |
-
-Units with a sustained pedestal: **BE9558, BA10895, BE11007, BE11529, BE12599,
-BE13117, BE18003, BE18438**. BA10895 and BE18003 were invisible to v1.
-
-**The affected channel is most often Vert**, which v1 got wrong — it named
-whichever axis had the largest peak. BE13117 and BE18438 are both Vert faults.
-
-Longest / clearest runs:
-
-| unit | ch | span | events | median in/s |
-|---|---|---|---|---|
-| BE13117 | Vert | 2023-05-03 → 05-04 | 194 | 0.035 → **1.915** |
-| BE18438 | Vert | 2026-02-25 → 02-26 | 75 | 0.180 → 0.370 |
-| BE9558 | Vert | 2020-02-11 (6 h) | 33 | 0.065 → 0.090 |
-| BE12599 | Tran | 2026-08-14 → 08-23 | 8 | 0.030 → **0.565** |
-| BE18003 | Vert | 2021-03-17 → 06-11 | 3 | 0.040 → 0.060 |
-
-BE12599 began **2026-08-14**, not 08-17 as v1 reported, and was still faulting
-at the last event in the archive.
-
-### 2c. Detector v3 — PRE-TRIGGER floor + constant-floor test (CURRENT)
-
-`scratch/offset_scan3.py`. Brian's method, and better than v2 for a reason
-worth naming: **the pre-trigger window is definitionally quiet** — it is the
-buffer captured before the trigger fired — whereas a whole-record median is
-merely *robust* to the event. `pretrig_samples` comes from the STRT record.
-
-```
-per channel:
-    pre    = median of the first pretrig_samples samples
-    mid    = median of the middle third
-    end    = median of the final third
-    spread = max(pre,mid,end) - min(pre,mid,end)
-
-    offset when  |pre| >= floor  AND  spread <= 0.02 in/s
-real fault when a channel is flagged on >=3 CONSECUTIVE events
-```
-
-A DC offset is a **constant floor** — present before the trigger, during, and
-after. The spread test rejects transients (settling, handling, a long event
-tail) that move one segment relative to the others, which is what v2's
-whole-record median could not do.
-
-**The empirical noise floor justifies the threshold.** Across 19,244
-non-flagged channel-events the pre-trigger floor distributes as:
-
-| floor | share |
-|---|---|
-| −1 unit (−0.005) | 18.4% |
-| **0.000** | **62.7%** |
-| +1 unit (+0.005) | 13.4% |
-
-**94.5% within ±1 quantisation unit; median exactly +0.0000, mean −0.0008.**
-So there is **no systematic zero-point bias in the decoder** — an independent
-confirmation of the 32000-count scale. A healthy channel really does read
-0.000, and "any constant floor that is not 0.000" is the right signal, with
-±1 unit of slack for quantisation.
-
-**The result is threshold-insensitive**, which is what distinguishes a real
-signal from a tuned one:
-
-| floor | units flagged | sustained units |
-|---|---|---|
-| 2 units (0.010) | 34 | 15 ← into the noise |
-| 3 units (0.015) | 26 | 8 |
-| **4 units (0.020)** | 17 | **5** |
-| **5 units (0.025)** — Instantel's | 12 | **5** |
-| **8 units (0.040)** | 8 | **5** |
-
-### FINAL RESULT: 5 of 45 units (11%)
-
-**BE9558, BE11529, BE12599, BE13117, BE18438.**
-
-Unchanged across a 2x threshold range. BE11007 and BA10895 drop out — the
-spread test identifies them as transients, not pedestals.
-
-The 11% headline happens to match v1's, but the reasoning and the unit list
-differ: v1 included BE11007 and named the wrong *channel* on most units.
-
----
-
-## 3. Archive results (2026-08-28)
-
-Source: DL2 event export, 6,577 **unique** series-3 waveforms, 45 units.
-See [`dl2-archive`](#8-data-and-tooling) for the `Sent/` mirror trap.
-
-**283 suspect events, 15 episodes, 6 of 45 units (13.3%).**
-Excluding BE11007 (§4, likely not an offset at all): **5 of 45 = 11.1%**.
-
-### Threshold sensitivity — the bimodality result
-
-Re-scoring the same corpus at a range of amplitude floors, with two
-ratio cut-offs (1 A/D count = 0.005 in/s, see §5):
-
-| \|offset\| floor | m/p > 0.7 | m/p > 0.9 |
-|---|---|---|
-| 5 cts (0.025 in/s) — *Instantel's own* | 333 ev / 6 units | 279 ev / **5 units** |
-| 10 cts (0.050) | 294 / 6 | 274 / 5 |
-| 20 cts (0.100) | 250 / 5 | 244 / 4 |
-| 40 cts (0.200) | 209 / 5 | 203 / 4 |
-| 80 cts (0.400) | 152 / 4 | 148 / 2 |
-| 160 cts (0.800) | 144 / 2 | 141 / 1 |
-
-Relaxing the floor by 11× (0.27 → 0.025 in/s) adds ~14% more events and **no
-new units**. There is no population of mild offsets hiding below our threshold
-*in event data*. Either a unit is clean or it is grossly off.
-
----
-
-## 4. Per-unit case files
-
-Ordered by severity. `m/p` medians are on the offending channel.
-
-### BE13117 — one violent day, never again
-`145 / 454 events (32%)`, **1 episode**, 2023-05-04, 6.8 h.
-Offset climbed **0.393 → 1.875 in/s within the episode**. `m/p` median
-**0.996** — the trace is almost pure DC. No recurrence in the rest of its 454
-events. No ASCII files in the archive, so no calibration history.
-
-### BE18438 — recurring, months apart
-`87 / 293 (30%)`, **2 episodes**: 2025-11-15 (1.2 h, n=12, 0.279 → 0.369) and
-2026-02-25 (**28.8 h**, n=75, 0.183 → 0.366). `m/p` median 0.967.
-Clean across all 196 events preceding its 2025-08-12 calibration.
-
-### BE9558 — six years apart
-`38 / 196 (19%)`, **4 episodes**: 2020-02-11 (6.3 h, n=33, but only
-0.068 → 0.086 — very mild), then 2026-04-14, 2026-04-29, 2026-05-04
-(0.28–0.45). `m/p` median 0.919. Calibrated 2026-06-26; 0/7 events flagged
-after, but n=7 is far too small to call it fixed.
-
-### BE12599 — the live case ⚠
-`6 / 77 (8%)`, **6 single-event episodes, one per day at exactly 05:00**,
-2026-08-17 → 2026-08-23. Offset rose 0.383 → 0.565 then fell back to 0.345.
-`m/p` ≈ 0.965, geo trigger 0.3 in/s — **the offset exceeds the trigger level,
-so the unit is triggering on its own fault**. Last calibrated 2025-08-12.
-
-This is the most recent and the most useful: a currently-faulting unit is the
-natural experiment for the re-zero-vs-repair question (§7).
-
-### BE11529 — marginal
-`4 / 99 (4%)`, 1 episode 2025-07-08, 0.4 h, offsets only 0.051 → 0.058 in/s.
-`m/p` median 0.959, so DC-dominated, but the magnitude is near the noise of
-this method. Treat as unconfirmed.
-
-### BE11007 — probably NOT an offset
-`3 / 70 (4%)`, 1 episode 2022-01-17, offsets 5.500 → 6.904 in/s — by far the
-largest. But `m/p` is only **0.719–0.738** against ≥0.9 for every other unit,
-and the peaks are 7.6–9.4 in/s on a 10 in/s range. That reads as a **large
-real blast with asymmetric ground motion**, not a parked trace. Excluded from
-the headline base rate.
-
----
-
-## 5. Instantel's own procedure and thresholds
-
-From two Instantel technical-support FAQs supplied 2026-08-28
-(answers **13-0-21** *"How to determine offsets"* and **12-0-10** *"Removing
-offsets on an Instantel Series III monitor"*; created 2008/2007, last updated
-2009-03-06).
-
-### Identifying (13-0-21)
-
-1. Create or use an event with the **manual minimum trigger** set for the
-   connected geophone and microphone — i.e. an event that recorded no real data.
-2. Save it and open in Blastware.
-3. An offset shows as **parallel lines above or below the zero line**.
-4. Put the unit in **A/D mode** — on Series III, press and hold `OPTION`, then
-   press `START MONITOR`.
-5. **Display counts higher than 5**, with no vibration or overpressure present,
-   indicate an offset.
-
-### Removing — the autozero (12-0-10)
-
-1. Be in a **quiet area with low vibration**.
-2. Power on the Blastmate III / Minimate Plus.
-3. Connect the geophone and microphone — **LINEAR mic only**.
-   ⚠ *Do not connect an "A" weight microphone, regardless of what the monitor
-   displays.*
-4. Press `Test`.
-5. Wait for the **Sensor Check** results to appear.
-6. Press `OPTION` and `START MONITOR` **simultaneously**.
-7. `Performing Autozero` appears; press `Enter`.
-8. Confirm the sensors are properly connected; press `Enter`.
-9. Wait for the autozero to complete.
-10. Press `Enter` twice → Main Menu, *Ready To Monitor*, offset corrected.
-
-### The go/no-go number — 2027 to 2069
-
-> When you perform an Autozero on any Series III unit, the lists of numbers in
-> the **X1 and X8 gains should all be between 2027 and 2069**. If not, repeat
-> the Autozero. **If the numbers are extremely out of the specified range, then
-> the unit should be sent in for repair.**
->
-> If this process does not remove the offset problem, return the unit **and
-> sensors** to Instantel for repair.
-
-This is the documented explanation for the field experience (Brian's dad,
-2026-08-28) that **a re-zero works maybe 10% of the time** — the autozero only
-recovers units whose zero reference is still near-correct.
-
-### Scale derivation (inference, well-supported — not proven)
-
-2048 is 12-bit midscale. Our codec's geo full scale is 32000 internal counts =
-10 in/s, with 1 decoder unit = 16 counts = exactly 0.005 in/s
-(`geo-full-scale-is-32000-counts`). ±2000 A/D counts about 2048 therefore maps
-to ±10 in/s at **0.005 in/s per A/D count**. That makes:
-
-- Instantel's ">5 counts" threshold ≈ **0.025 in/s**
-- the 2027–2069 window = **±21 counts = ±0.105 in/s** of tolerated zero error
-
-Consistent and mutually corroborating, but we have not confirmed the A/D-count
-scale directly from a device reading.
-
----
-
-## 6. Ruled out — keep these dead
-
-### Condensation / humidity — DEAD (2026-08-25)
-Proposed, then killed by its own controls: BE18438 stayed flat across a 10-hour
-overnight gap, and only 2 of 21 units showed the fault while 19 sat in the same
-weather. The apparent "diurnal cycle" was an artifact of binning by hour-of-day
-across two days. See `waveform-dc-offset-is-real-device-data`.
-
-### Clipping as a false-positive source — RULED OUT (2026-08-28)
-A rail-hitting trace would fake an offset (mean → peak). It isn't happening:
-median suspect peak is only **10% of full scale**, p90 is 18.6%. Only BE11007's
-3 events exceed 50% FS, and none reach 98%.
-
-### The sensor check as a predictor — DOES NOT WORK (2026-08-28)
-Tested on 102 offset events across 4 units:
-
-| unit | state | n | failed | median ratio | median freq |
-|---|---|---|---|---|---|
-| BE11529 | offset | 4 | **0** | 3.90 | 7.6 |
-| BE11529 | clean | 14 | 0 | 3.80 | 7.5 |
-| BE12599 | offset | 6 | **0** | 4.00 | 7.4 |
-| BE12599 | clean | 13 | 0 | 4.00 | 7.6 |
-| BE18438 | offset | 87 | **0** | 3.70 | 7.6 |
-| BE18438 | clean | 25 | 0 | 3.80 | 7.5 |
-| BE9558 | offset | 5 | **0** | 3.90 | 7.8 |
-| BE9558 | clean | 44 | 0 | 3.80 | 7.5 |
-
-Zero failures on either side and indistinguishable ratios/frequencies. The
-swing test measures geophone frequency response and damping — it never examines
-DC zero. **A grossly offset unit passes its own self-check.** This is why the
-fault goes unnoticed until somebody looks at waveforms.
-
-### "Offsets are transient / come and go on their own" — RETRACTED 2026-08-28
-v1 reported episodes lasting hours that ended spontaneously. **This was an
-artifact of the v1 detector** (see the banner at the top). With the per-channel
-median, the pedestal persists. Every clear case reads clean again only after a
-multi-day-to-multi-month gap consistent with service: BE13117 6 days, BE18438
-24 days, BE9558 63 days **with a confirmed Instantel calibration inside the
-gap**. BE12599 never reads clean — it is still faulting at the end of the
-archive. This matches the operational experience: once a unit develops an
-offset it stays broken until the geophone is replaced.
-
-### "Offsets develop N months after calibration" — CONFOUNDED, NOT A FINDING
-Tempting, and it looked strong:
-
-| unit | suspect before latest cal | after |
-|---|---|---|
-| BE18438 | 0 / 196 | 87 / 97 |
-| BE12599 | 0 / 62 | 6 / 15 |
-| BE11529 | 0 / 82 | 4 / 17 |
-| BE9558 | 38 / 189 | 0 / 7 |
-
-But bucketing suspects by months-since-calibration gives **one unit per bucket**:
-`0–3mo={BE11529}`, `3–6 & 6–9mo={BE18438}`, `9–12mo={BE9558}`,
-`12–15mo={BE12599}`. The apparent "51% failure rate at 6–9 months" is entirely
-BE18438's single February 2026 episode. Five units with roughly one episode
-each cannot support a population trend. **Do not re-derive this.**
-
-Also note: all affected units are calibrated on a **~12–13 month cadence**, so
-"sent to Instantel" is the routine annual schedule, not evidence of a
-fault-driven return.
-
----
-
-## 7. Open questions
-
-### Q1 — Is it a latched bad zero or analog degradation?
-The question that decides everything. A latched zero is correctable (possibly
-over the wire); degradation means a repair. Instantel's 2027–2069 rule implies
-*both* populations exist, with the split roughly 10/90 in the field.
-
-**BE12599 is the natural experiment** — faulting as of 2026-08-23. Read its
-values, run the autozero, read them again.
-
-### Q2 — Can we read the autozero numbers over the wire? (best lead)
-Instantel says to check *"the lists of numbers in the **X1 and X8 gains**"* —
-4 sensors × 2 gains = **8 channels**. The protocol reference already documents
-an unimplemented command with exactly that shape:
-
-```
-SUB 0x0E  ->  RSP 0xF1     "channel sensor data"
-  2-step read; channel selector in params[6:8] = 0x0000..0x0007
-  data length 0x0A (10 bytes) per channel
-```
-
-Blastware's *Unit Channel Test* sequence:
-`POLL×N → 0x15 → 0x01 → 0x08 → 0x01 → 0x0E×8 → 0x98×2 → 0x0E×8`
-— note the **second `0x0E` pass carries live ADC readings**.
-
-**Hypothesis (untested):** `0x0E` returns the numbers Instantel wants compared
-against 2027–2069. If true, SFM could diagnose an offset remotely *and* predict
-whether a re-zero will succeed — converting a 10%/90% shipping gamble into a
-decision made before packing a box.
-
-**How to test.** `bridges/ach_mitm.py` is a generic TCP proxy:
-
-```bash
-python bridges/ach_mitm.py --bw-host <MODEM_IP> --bw-port 9034 --listen-port 9999
-```
-
-Point Blastware at the proxy and run **Unit Channel Test**.
-⚠ In this topology the output filenames are reversed — the tool labels the
-*connecting* side "unit", so `raw_s3_*.bin` holds Blastware's bytes and
-`raw_bw_*.bin` the unit's.
-
-Capture priority: (1) BE12599 while faulting, (2) a known-good unit as control,
-(3) before/after an autozero on the same unit. Eight 10-byte payloads with an
-expected value near 2048 is a very constrained puzzle.
-
-### Q3 — What is the mild-offset rate?
-Unmeasurable from event files (§2). Only the A/D-mode check sees it. Would
-need a fleet sweep in A/D mode, or Q2 to succeed.
-
-### Q4 — Does an offset recur on the same unit after service?
-BE9558 shows episodes in 2020 and 2026; BE18438 twice in four months. Suggestive
-of recurrence, but service records aren't in the data — only calibration dates.
-
----
-
-## 8. Data and tooling
-
-| what | where |
-|---|---|
-| detector | `scratch/offset_scan.py` |
-| current results | `/home/serversdown/dl2-archive/offset_archive.csv` |
-| earlier candidate list (Terra-View DB, 274 events) | `scratch/offset_candidates.csv` |
-| archive working copy | `/home/serversdown/dl2-archive/files/` |
-| archive source | NAS `DeathStar` 10.0.0.2, `/volume1/Uploads/TMI/DL2-Event-backup-8-25-26/Event/autocall home/` |
-
-⚠ **The DL2 export keeps a byte-identical `Sent/` mirror of its root.** 13,077
-waveform paths are 6,577 distinct files. Always dedupe by basename — this
-doubled two reported figures before it was caught.
-
----
-
-## 8b. The histogram corpus — the other 90% of the archive (2026-09-04)
-
-Every result above §8 comes from **waveform** files. `offset_scan3.py` filters on
-`\.[A-Za-z0-9]{2}0[Ww]$`, so the corpus it scanned is 6,577 unique binaries. The
-archive also holds **63,535 unique histograms** — 9.7x more files — which the
-pre-trigger method cannot touch, because a histogram carries no samples: only a
-per-interval, per-channel peak and half-period.
-
-`scratch/offset_hist_scan.py` scans them. **63,505 of 63,535 decoded (99.95%),
-43 units, 77.9M intervals.** Two of the 45 units have no histograms at all.
-Output: `/home/serversdown/dl2-archive/offset_hist.csv` (190,515 channel-rows).
-
-### The premise, and how far it actually holds
-
-A histogram file is hours of continuous monitoring, so most of its intervals are
-definitionally quiet, and a channel parked off zero cannot report a peak below
-its own displacement. The signal is real — two within-unit contrasts, siblings
-unmoved in both:
-
-| unit | channel | in-episode floor | outside | waveform \|pre\| same window |
-|---|---|---|---|---|
-| BE18438 | Vert | 0.0350 | 0.0050 | +0.18 .. +0.37 |
-| BE12599 | Tran | 0.0250 | 0.0050 | +0.03 .. +0.49 |
-
-But the **leakage from a waveform pedestal into the histogram floor is bimodal,
-not merely partial**: measured ratio ~0.9 on BE18438 Vert, ~0.7 on BE9558,
-**~0.02 on BE12599** — two orders of magnitude on one instrument. The device
-evidently measures each interval peak against a running baseline, and how much
-DC survives that varies per unit. **Consequence: a negative histogram result
-carries almost no information.** Do not read "clean in the histograms" as clean.
-
-### The detector that survived
-
-    dmin(file, ch) = min[ch] - min over the other two geo channels, SAME file
-    gates (both hard):  n_intervals >= 60   AND   mic_p5 <= 5 raw counts
-    day statistic:      median of dmin over that day's qualifying files
-    flag day at        dmin >= 0.020 in/s (4 A/D counts)
-    episode at         >= 3 CONSECUTIVE observed days
-
-**Result: BE18438|Vert, BE9558|Tran, BE9558|Long.** Threshold-insensitive —
-the journal's own test for a real signal against a tuned one — and this is the
-first operating point in the investigation that passes it cleanly. The identical
-answer holds across: statistic `min` or `p5`; length gate 10/30/60/120/300; mic
-gate 3/5/8/10; threshold 0.015–0.035 (a 2.3x span); persistence K = 2,3,4,5,7.
-
-Separation, ranked by highest floor sustained over 3 consecutive gated days
-across all 135 unit-channels:
-
-| unit-channel | best3 |
-|---|---|
-| BE18438 Vert | 0.1650 |
-| BE9558 Long | 0.0350 |
-| BE9558 Tran | 0.0250 |
-| *(2.5x gap)* | |
-| BE7145 Tran | 0.0100 |
-| entire rest of fleet | <= 0.0050 (one quantisation count) |
-
-Day-level false alarm: **37 of 99,432 gated unit-channel-days = 0.037%.**
-
-### What it does NOT do — read this before trusting it
-
-- **It finds 2 of the 5 confirmed units, not 5.** The site-quiet gate is what
-  makes it work and it is also what costs BE11529 and BE12599. BE11529's
-  four-day single-axis ramp (Tran 0.025 -> 0.055, both siblings pinned at 0.005)
-  is the most offset-shaped thing in the corpus outside the two detections, and
-  the gate discards it.
-- **The positive class is two units.** Every threshold here is fitted to
-  BE18438 and BE9558, which contribute 22 of the 37 flagged days in the entire
-  corpus. No cross-validation is possible at n=2.
-- **Per-channel attribution is NOT established.** Rotating the three geo channel
-  labels within each file — preserving every value, file and day, destroying
-  only channel identity — reproduces the episode *count* with p = 0.769 and the
-  label agreement at p = 0.038–0.077. Report a **unit and a window**; do not
-  name a geophone axis on the strength of this detector alone.
-- **Timing resolution is ~1 month, not ~1 day.** A 30-day label shift still
-  scores 2 of 9 episode hits; the signal dies only past ~60 days. The day-level
-  series look far crisper than they are.
-- **Ground truth here is a sibling detector, not a service record.** Agreement
-  between the two corpora is corroboration of a shared method. Nothing in this
-  section has been checked against an actual repair, calibration or RMA.
-
-### Dead ends — keep these dead
-
-- **Absolute floor (min / p1 / p5 / p10 / p25, thresholded alone) — RETIRED.**
-  Not fleet-comparable and mostly not about the channel. Scoring each cell using
-  *only the other two channels* — a statistic containing zero information about
-  the suspect channel — reaches AUC 0.746 against the same labels, versus 0.872
-  for the absolute floor itself. **66% of its apparent discrimination is "that
-  day was noisy at that site."** Interval size alone moves its p99 7x (0.0350 at
-  1 min vs 0.0050 at 2 s). And of all files with any channel above 0.025, 56.5%
-  have **all three** channels above it — common-mode, i.e. the wrong physics.
-- **Zero-fraction — STRUCTURALLY IMPOSSIBLE, not merely weak.** The device never
-  reports a zero histogram interval peak. The value is a max over hundreds of
-  samples of a channel that always carries at least 1 count of noise, so it is
-  clamped at 1 A/D count (0.005 in/s). There is no zero to count.
-- **Interval size, sample rate, geo range, firmware — refuted as confounds for
-  the differential.** All four are *file-level scalars*: they move all three geo
-  channels together, so they cannot produce a single-channel lift and the
-  within-file differential is immune to them by construction. Geo range is
-  identical across the three geo channels in **63,535 of 63,535** binaries.
-  (Interval size remains fatal to the *absolute*-floor version, above.)
-
-### Two findings that are independent of the histogram detector
-
-**1. `offset_scan3.py`'s `spread <= 0.02` gate is discarding real signal.**
-It rejects **113 of the 600 channel-rows with |pre| >= 0.025 (18.8%)**, and the
-rejections are not random — 92 of them fall across 41 unit-channels currently
-labelled NEGATIVE. Four would become sustained positives under an
-amplitude-only >=3-consecutive rule: **BE12599|Long (run of 8), BE18003|Vert
-(4), BA10895|Vert (3), BE12844|Tran (3).** Until this is re-cut, the fleet label
-is **three-state — POSITIVE / NEGATIVE / SPREAD-REJECTED(unknown)** — and the
-third state should be excluded from both TP and FP counts rather than silently
-scored as healthy. Every precision figure computed against the two-state label,
-in this section and in §3, is affected.
-
-**2. The waveform corpus sees ~7% of the days a unit was deployed.** 2,627
-(unit, day) observations against the histogram corpus's 35,105 — 13.4x — with a
-per-unit median ratio of 0.070. BE12599, a confirmed unit, is waveform-observed
-on 39 of its 1,666 histogram-observed days (**2.3%**). Any statement of the form
-"the fault was absent before date X" that rests on waveform coverage alone is
-much weaker than its event count suggests.
-
-### BA10895 — reclassified (see also §4)
-
-Previously dismissed as a transient. The histogram record shows its **Vert**
-quiet-minute floor at 0.005 on 62/62 qualifying files from 2023-07-07, then
-0.010–0.015 on 48/58 files from 2023-08-03 to 08-27, while Tran moves on 2/58
-and Long on 9/58 and the site mic floor never leaves 1–3 counts. Independently,
-**42 of its 85 waveform events (49.4%) are single-axis-dominant** — one geo peak
->= 10x both siblings and >= 0.05 in/s — the **highest rate in the 45-unit
-fleet** (BE13117 36.1%, BE18438 29.4%), and **100% of it on Vert**. Vert
-excursions of 0.1–1.5 in/s with Tran/Long at 0.005–0.035 are not ground motion.
-
-This is a genuine Vert-channel hardware fault, but **not the classic pedestal** —
-the differential is only one A/D count. Caveat: its entire histogram record is a
-single 52-day deployment ending 2023-08-27, so nothing says whether it
-persisted, was serviced, or resolved.
-
-The other six marginal units — BE11007, BE17354, BE18004, BE18104, BE9557,
-BE18003 — are **clean**. All seven cap at +0.005 to +0.007 (one A/D count)
-lifetime under the quiet-site gate, against +0.175 for BE18438 Vert and +0.062
-for BE9558 Long. Three individual waveform flags fall in windows with **zero**
-histogram coverage and are NO-DATA, not clean: BE18004|Tran 2024-10-16,
-BE9557|Tran 2021-06-28, BE9557|Vert 2025-06-12.
-
-### Still open in this section
-
-- **The 11 thin-coverage units were not screened** (BE10202, BE11462, BE13779,
-  BE15760, BA15957, BE16754, BE16758, BE8081, BE8626, BA9229, BE9887 — each
-  under 20 waveform events, several with hundreds of histograms). This is the
-  population most likely to hold a previously unknown offset, and it is the one
-  slice of the plan that did not run. BE11462 was incidentally scored clean by
-  the full-archive pass; BE10202 has no histogram files at all.
-- **No completeness audit was run** over the above.
-- Re-cutting the ground truth three-state (finding 1) and re-scoring everything
-  against it.
-
----
-
-## 8c. Mechanism — five hypotheses tested, all dead (2026-09-06)
-
-**The mechanism is still unknown.** Five campaigns, ~105 effectively independent
-tests, seven nominally significant results against **5.2 expected by chance**
-under a global null. Every one died to its own confound analysis. What the
-campaign bought is a set of *shape constraints* and a long list of dead ends.
-
-### ⚠ Two things retracted from this journal
-
-**1. "Polarity is perfectly consistent — 11 of 11, zero mixed cases."** That is
-a **tautology of the spread gate**, not a property of the fault. `spread <= 0.02`
-requires pre/mid/end to agree, which forces one sign. Amplitude-only at the same
-0.025 threshold: **12 of 53 unit-channels are mixed**, including BE18438|Vert
-(88+/1−) and BE9558|Vert (1+/35−). Withdrawn.
-
-**2. "5 of 45 units, unchanged across a 2x threshold range."** The
-threshold-insensitivity is also a property of the gate. Amplitude-only gives
-**9 units at 0.020, 8 at 0.025** (adding BA10895, BE12844, BE18003), 5 at 0.040.
-The fleet is **8–9 units, not 5**.
-
-**3. "Persistent — it stays until the geophone is serviced."** Weakened, not
-withdrawn. There are **23 recoveries after runs of >=3 flagged events, median
-gap 6.03 days**, three inside ten minutes. BE18438|Vert reads `pre=mid=end=
-+0.0000` on 2026-02-10, +0.185→+0.370 across 02-25/26, and `+0.0000` again on
-2026-03-22 — identical Project, Seis Loc, calibration date, geo range and
-trigger throughout. The one thing that cannot be excluded is a **field
-autozero**: it is a button sequence at the unit and writes nothing into the
-event header. So "persistent" may be "persistent unless somebody pressed the
-buttons," and the archive cannot tell those apart.
-
-### The one positive finding: onset is a RAMP, minutes to hours
-
-Both onsets resolvable at minute cadence are ramps. **BE18438|Vert,
-2026-02-20** — the histogram corpus collapses a 14 d 21 h waveform bracket to
-**one minute**:
-
-```
-~14,200 consecutive quiet minutes at 0.000–0.005   (ten full daily files)
-09:32 +0.005   09:39 +0.045   10:20 +0.125   16:00 +0.165
-09:33 +0.010   09:42 +0.070   13:13 +0.150   20:17 +0.185  plateau
-```
-
-**50% of the excursion in 7 minutes**, the rest asymptotic over ~10 h, **>=25
-distinct one-minute intermediates**. Validated **75/75** against Blastware's own
-ASCII export. Its 2025-11-15 onset is the same shape over 2.7 h. BE13117 stage B
-is a 91-minute monotone rise, +0.035 → +1.745 in/s over ~40 samples.
-
-**This kills both poles of the original dichotomy** (journal Q1): not an
-instantaneous latched step (a bad autozero, a stuck trim-DAC), and not slow
-component degradation over days or weeks.
-
-⚠ It rests on **2 of 45 instruments**. Clopper-Pearson on 4/4 resolved onsets
-gives 95% CI [0.40, 1.00] — a mixed population with up to 60% true steps is not
-excluded. BE13117 has zero paired ASCII, so its ramp rests on our decoder alone.
-
-### The methodological corollary — more important than the finding
-
-**A waveform-only bracket manufactures the appearance of a step, and the spread
-gate is blind to onsets by construction.**
-
-The offset is what fires the trigger, so no waveform event can exist until the
-ramp has nearly reached the trigger level. BE18438's first event of each episode
-sits at 0.280 against a 0.300 trigger, and 0.185 against 0.200. At daily cadence
-against a 3 h ramp, P(catching an intermediate) = **0.125**.
-
-And `spread <= 0.02` rejects any record in which the floor is *moving* — which
-is exactly what an onset is. **The gate rejected the very BE18438 record where
-the ramp is visible.** If the operational goal is catching a fault early, before
-the unit floods the store with junk events, the current detector is the wrong
-shape for the job.
-
-### The surviving shape
-
-An **electrical, reversible, two-time-constant settling process** (~10 min and
-~hours), saturating at a ceiling, with occasional sub-3-minute discrete jumps
-superposed (BE18438 2026-02-26: 13:24 pre +0.180 / mid +0.240 / end +0.255 →
-13:27 +0.325, identical metadata). That is the signature of a **bias or leakage
-path charging a high-impedance node** — the class of fault Instantel's autozero
-recovers ~10% of the time, and what the X1/X8 gains measure.
-
-**It is a shape constraint, not a mechanism. Do not write it up as one.**
-
-### Dead — with the evidence, so none of this is re-derived
-
-| Killed | Evidence |
-|---|---|
-| **Latched step at onset** | >=25 one-minute intermediates over ~10 h, ASCII-validated. Direct observation, not a test. |
-| **Slow degradation over days/weeks** | Same observation — bulk of the excursion in 7 min to 2.7 h. |
-| **Thermal driving of pedestal magnitude** | BE13117, 365-count pedestal, n=128: full-day modulation **−0.42% ± 0.42%**, 95% CI [−1.25%, +0.40%]. Healthy-fleet seasonal zero drift totals **~0.3 A/D counts** — 15x to 1200x too small. Best-powered result in the campaign. |
-| **Ground-motion shock** | 30-day window-max percentile ranks 0.03/0.98/0.15/0.01/0.68/0.24, median **0.194** against a null of 0.5. **0 of 7 events >=9 in/s** was followed by an onset within 30 d. BE12599 hit 10.220 in/s (2023-11) and 10.005 (2025-04) and did not onset until 2026-08-14. |
-| **Handling / redeployment** | **0 of 9** onsets had a Project/Client/Seis Loc change. Widened to 30 d: 2 observed vs 4.90 expected, P(X>=2)=0.995 — *depleted*, the wrong direction. The apparent gap effect (p=0.035) died on histogram coverage: BE18438's "59.7-day gap" contains 122 histogram files; true silence 0.52 d. |
-| **Mechanical resonance / damping change** | BE18438|Vert at a 64-count pedestal (3x outside Instantel's ±21): ΔTest-Freq **CI [−0.090, +0.021]** against 0.127 Hz for a real calibration. Block permutation p=0.658. |
-| **Accumulated-duty threshold** | ~4 clean units logged more monitoring than the largest positive onset dose; BE18193 logged **13.45M intervals, 6.2x**. A counterexample — no power argument weakens it. |
-| **Firmware** | **14,338 of 14,340** exports read `V 10.72-8.17`. A constant cannot explain a variable. |
-| **Unit age** | Serial rank-sum 118.0 vs null 115.0, p=0.549; unchanged on the 8-unit re-cut (p=0.586). Serial is a poor age proxy anyway (Spearman +0.113 against archive entry). |
-| **Strong seasonal clustering** | 25 onsets, exposure-weighted permutation **p=0.59**. Excludes >=75%-in-one-season only; a 2x seasonal hazard is *not* excluded. |
-
-Also retire two overstated bounds. H6's dose-response exclusion "|r| > 0.03" is
-a **10x overstatement** once clustering is corrected — the honest bound is
-|r| > 0.1–0.3, so a real r=0.2 is not excluded. And **any statistic quoted
-per-event**: 512 flagged channel-events collapse to **4.9 effective independent
-observations** (unequal-cluster design effect 104.6 at ICC=1), and **55% of the
-flagged corpus is one instrument on two calendar days** (BE13117, 2023-05-03/04).
-
-### Power — read every negative in this section as bounded
-
-Fisher exact, 5 positives of 45, one-sided α=0.05, exposure a third of the fleet:
-
-| relative risk | power |
-|---|---|
-| 1.5 | 0.059 |
-| 2 | 0.112 |
-| 3 | 0.231 |
-| 6 | 0.497 |
-| 15 | 0.753 |
-
-80% power needs **RR ≈ 13–20**. Even a *perfect* split reaches p<0.05 only if
-the exposed group is <=25 of 45 units. **This archive can detect only
-near-deterministic unit-level causes.** Every negative above excludes a strong
-effect, not a real one.
-
-### What this archive can NEVER answer
-
-- **The A/D zero and the X1/X8 gains.** The 2027–2069 numbers appear in no file,
-  header or decoded record. They exist only on a live device behind `SUB 0x0E`.
-  Q1 is structurally unanswerable from data.
-- **Unit-level vs component-level cause.** **Zero of 14,340** exports carry a
-  geophone or sensor serial. Q4 is dead — there is no way to know whether the
-  same physical geophone came back after service.
-- **Service history.** The only service-adjacent field is `Calibration: <date>`
-  — 30 distinct dates fleet-wide, none before 2023, ASCII corpus entirely
-  2025–26. BE9558's 2020 and BE13117's 2023 episodes have no calibration record.
-- **Temperature.** Zero exports carry it. Battery Level is a verified coarse
-  thermometer (+0.204 V winter over summer, 20/20 unit-years, p=9.5e−7, matching
-  lead-acid tempco) but quantised at 0.1 V ≈ 10 °C — useless within a day. The
-  archive can *bound* thermal; it can never *test* it.
-- **BE13117 specifically** — 55% of the flagged corpus, the largest pedestal at
-  1.92 in/s, **zero** ASCII exports, histogram record ending eight months before
-  its episode. The most informative case in the archive is permanently outside
-  every metadata test.
-- **The mild-offset rate**, and therefore the base rate's denominator. Event
-  files only see offsets large enough to dominate the trace.
-
-### The experiment to run — `SUB 0x0E`, one afternoon
-
-Point Blastware at `bridges/ach_mitm.py` and run **Unit Channel Test** against
-(1) a faulting unit, (2) a known-good control, (3) the same unit before and
-after an autozero. BW's sequence is `0x0E x8 → 0x98 x2 → 0x0E x8`, the second
-pass carrying live ADC. Eight 10-byte payloads with expected values near 2048 is
-a very constrained puzzle.
-
-- **Proves:** whether the X1/X8 gains are readable over the wire, and whether
-  the fault sits at or upstream of the ADC zero reference. Gains walk out of
-  2027–2069 with the pedestal → the fault *is* the zero reference, Q1 answered.
-  Gains hold while the trace moves → the fault is downstream, look at the front
-  end.
-- **§8c hands it a falsifiable time course:** poll at ~1-minute cadence and the
-  numbers should **ramp over minutes-to-hours, not step**. If they step while
-  the trace ramps, the two are decoupled.
-- **Payoff:** converts the 10%/90% ship-it-or-not gamble into a decision made
-  before packing a box, remotely, for the whole fleet.
-- ⚠ In the MITM topology filenames are reversed — `raw_s3_*.bin` holds
-  Blastware's bytes.
-
-**Second: swap the geophone** between a faulted base and a healthy one. Fault
-follows the sensor → element or cable. Fault stays with the base → front-end
-board. One afternoon, zero code, and it settles the one question the archive is
-permanently blind to.
-
-**Third: log a faulting unit for 72 h untouched.** Every recovery we have is
-confounded by a possible field autozero. A shelf and a logger settles whether
-the fault genuinely self-reverses.
-
-**Fourth, free: re-cut the fleet label** — drop the spread gate, re-score
-amplitude-only, screen the 11 unscreened thin-coverage units. Might reach 9–10
-positives. Be honest about the gain: power against "older half carries 3x the
-hazard" rises only 0.23 → 0.30.
-
-**Highest-value item overall, and not an experiment: the RMA/repair records.**
-Which unit went back, when, what was done (autozero vs geophone replaced vs
-board), and the geophone serial fitted. "Same channel after a documented
-geophone *replacement*" is component-level-negative in one observation.
-
----
-
-### 8d. The non-motion test — Brian's "it doesn't cross zero" (2026-09-07)
-
-Looking at BE12599's 2026-08-09 event, Brian noted it reports no ZC frequency
-**because the trace never crosses zero**. That observation is the best detector
-in this investigation, and it comes from physics rather than a threshold.
-
-A geophone is a velocity sensor with no DC response, so its output over a record
-must integrate to ~zero — the ground does not relocate. Real motion therefore
-sits roughly half below zero. Anything electrical is one-sided.
-
-    mp = |mean| / peak        ~0 for motion, ~1 for a fault
-    frac_neg = share of samples < 0
-
-`scratch/nonmotion_scan.py`, all 6,577 waveforms, 19,731 channel-rows.
-Restricted to peak >= 0.05 in/s (n = 12,068), the distribution is **bimodal
-with an empty middle**:
-
-| mp band | channel-events |
-|---|---|
-| 0.0–0.1 | 11,384 |
-| 0.1–0.2 | 293 |
-| **0.15–0.85 (dead zone)** | **131 = 1.09%** |
-| 0.9–1.0 | 278 |
-
-At `mp >= 0.8` with >=3 events it returns **exactly the five confirmed units** —
-BE9558, BE11529, BE12599, BE13117, BE18438 — stable from 0.5 to 0.9. Two
-detectors on entirely different principles agreeing on the unit list is the
-strongest corroboration that list has.
-
-**BE11007 is settled: NOT an offset.** It reaches mp 0.75–0.89, but with
-`frac_neg = 0.99` at peaks of **7.4–9.4 in/s** — parked *negative* during a
-near-full-scale blast. §4's guess was right. `mp` alone cannot separate a
-pedestal from a large one-sided blast; pair it with a peak ceiling or with
-sign-consistency across events.
-
-⚠ **Not a rediscovery of the retracted v1 detector.** v1 scored only the
-largest-peak axis and used the mean as a *baseline estimator* where the median
-was required. Here the mean is the signal itself, per channel — that is what the
-physics licenses.
-
-**Correction to §8c.** That section says the spread gate is "blind to onsets by
-construction." Too strong: of 87 BE18438|Vert events at mp >= 0.5 the gate
-rejected **one** — the transitional record. It does not lose onsets
-systematically; it loses the transition specifically.
-
-### 8e. BE12599 — a connector, not a geophone (2026-09-07)
-
-Waveform shapes across its August episode, measured rather than eyeballed:
-
-| date | channel | shape |
-|---|---|---|
-| Aug 09 05:29 | Long | **unipolar +**, 0/2304 samples below zero, decay tau **26 ms** |
-| Aug 09 05:35 | Long | unipolar +, 3 spikes at irregular gaps (744, 1032 ms), tau **38 ms** |
-| Aug 14 05:00 | Long | single lobe, bipolar, tau **118 ms** |
-| Aug 17–23 | Tran | **flat DC pedestal**, sd/level 0.015–0.020, 0 zero crossings |
-
-**Unipolar impulses with an RC tail are not mechanical.** Fast rise, exponential
-decay, one polarity, irregular timing — that is charge dumped into a
-capacitively-coupled input and draining through the input resistance. The
-progression 26 ms -> 118 ms -> never recovers, over 14 days, is a leakage path
-worsening.
-
-**And the fault moved channels** — Long on Aug 9/14, Tran on Aug 17–23, Long
-again on Aug 21 (1.065 in/s) while Tran held its pedestal. Vert stayed clean
-throughout. **A failing geophone element cannot hop channels. A connector can.**
-
-That single fact explains what had been puzzling:
-- **The sensor self-check keeps passing** (7.4/7.5/7.6 Hz, ratios 3.6–4.2, all
-  four channels Passed, on the very events where Long throws 0.5 in/s spikes).
-  The swing test drives the element; the element is fine. The fault is in the
-  wiring to it.
-- **Why Instantel's autozero fixes only ~10%** — it cannot fix a connector.
-- **Why onset "ramps" over minutes to hours** — contact resistance drifting.
-
-All seven Aug 17–23 events are stamped **05:00:14**, the same second, and their
-filename extensions run `8E → WE → KE → 8E → WE → KE → 8E` — the documented
-3-day cycle for a fixed daily time. Clock-scheduled, not physically triggered:
-the modem powers up, draws a surge, and a marginal connection responds.
-
-**Field action: inspect and photograph the geophone connector BEFORE reseating
-anything** — an intermittent contact clears the moment it is disturbed.
-
-⚠ Scoped to BE12599. BE18438's onset was a smooth 7-minute ramp with no spikes,
-which looks like a different failure mode wearing the same signature.
-
----
-
-### ⚠ Serial prefixes — four of these units are BlastMates, not MiniMates
-
-Corrected 2026-09-06, after Brian queried "BA10895?" against a report that
-said BE10895. He was right. The BW filename encodes the serial **number
-only** — `L895` -> 10895 — and every offset scanner synthesised the family
-prefix as `"BE"`. Four of the 43 archive units are **BA** (BlastMate, the
-MiniMate Plus's bigger sibling; same Series III, byte-identical data):
-
-**BA9229, BA10060, BA10895, BA15957.**
-
-Read off the file bodies, which carry the serial verbatim. No analysis
-changed — grouping was always on the numeric part, and no unit number maps
-to two serials — but every earlier reference to "BE10895" and the other
-three is a label error and has been corrected throughout this document.
-
-The same assumption was live in two production sites and is fixed
-(`sfm/waveform_store.py`, `minimateplus/client.py`): the store would have
-filed a BlastMate under a unit that does not exist, and the monitor-log
-decoder lost the geo threshold along with the serial. See commit `9ceff65`.
-
----
-
-## 9. Chronology
-
-| date | event |
-|---|---|
-| 2026-08-25 | Reported as a *waveform decode bug* — traces with a DC offset. Investigation shows the offset is **real device data**; the decoder is correct. |
-| 2026-08-25 | Brian relays his dad's description: a known hardware fault called an "offset"; usually sent to Instantel. |
-| 2026-08-25 | First detection pass over the Terra-View DB: **2 of 21 units**, 274 events, 11 months. Flagged as vulnerable to survivorship bias — flooded events were routinely deleted. |
-| 2026-08-25 | Condensation hypothesis proposed, then **killed by its own controls**. |
-| 2026-08-25 | Parked pending the multi-year archive. |
-| 2026-08-28 | DL2 archive pulled (33 GB, 546k files; 6.6 GB working set). |
-| 2026-08-28 | Archive scan: **6 of 45 units**, 283 events, 15 episodes. Prior base rate **confirmed**, not overturned. |
-| 2026-08-28 | Clipping ruled out; `m/p` established as the discriminator; BE11007 reclassified as probably a real blast. |
-| 2026-08-28 | Calibration-timing correlation attempted and **rejected as confounded**. |
-| 2026-08-28 | Instantel FAQs supplied: autozero procedure, the **2027–2069** window, the **>5 counts** threshold. Explains the ~10% re-zero success rate. |
-| 2026-08-28 | Bimodality established; sensor check proven **blind** to offsets; `SUB 0x0E` identified as the best open lead. |
-| 2026-08-28 | **v1 detector retracted.** Brian challenged the "come and go" finding against field experience. Two flaws found: dominant-axis-only scoring and mean-instead-of-median. Corrected detector shows persistent pedestals on **8 of 45 units**, and the gaps are service windows. |
-| 2026-08-28 | **Detector v3 (Brian's method):** pre-trigger floor + pre/mid/end consistency. Healthy channels proven to sit at 0.000 +/-1 unit (94.5%), confirming no decoder zero-point bias. Final: **5 of 45 units (11%)**, threshold-insensitive. |
-| 2026-09-04 | **Histogram corpus scanned** — 63,505 of 63,535 files, 43 units, 77.9M intervals (9.7x the waveform corpus). `scratch/offset_hist_scan.py`. |
-| 2026-09-04 | Absolute-floor statistic **retired**: 66% of its discrimination is a day/site confound (other-channels-only AUC 0.746 vs 0.872). Zero-fraction shown **structurally impossible** — the device clamps every interval peak at >= 1 count. |
-| 2026-09-04 | Site-quiet-gated cross-channel differential established: **BE18438 Vert, BE9558 Tran+Long**, threshold-insensitive over a 2.3x span. Finds only **2 of the 5** confirmed units — leakage into the histogram floor is bimodal (0.9 to 0.02), so a negative result carries almost no information. Per-channel attribution **not** established (channel-scramble p = 0.769). |
-| 2026-09-04 | **BA10895 reclassified** from transient to a genuine Vert fault of a different subtype — 49.4% single-axis-dominant events, the highest in the fleet, 100% on Vert. The other six marginal units are clean. |
-| 2026-09-04 | **Defect found in `offset_scan3.py`**: its `spread <= 0.02` gate discards 18.8% of rows with \|pre\| >= 0.025, concentrated on 41 negative unit-channels; 4 would be sustained positives without it. The fleet label is three-state, not two. |
-| 2026-09-06 | **Four units relabelled BA, not BE** — BA9229, BA10060, BA10895, BA15957 are BlastMates. The BW filename carries only the serial number; the family prefix must be read from the file body. Fixed in the scanners and in two production sites. |
-| 2026-09-06 | **Mechanism campaign — five hypotheses, all dead.** Thermal, ground-motion shock, handling/redeployment, accumulated duty, unit age, firmware and a mechanical element fault are each refuted or bounded. 7 nominally significant results against 5.2 expected by chance. |
-| 2026-09-06 | **Onset is a RAMP of minutes-to-hours, not a step** — BE18438 Vert resolved to one-minute cadence, 50% of the excursion in 7 min, >=25 intermediates, ASCII-validated 75/75. Kills both a latched digital step AND slow component degradation. Surviving shape: a reversible two-time-constant settling process — a bias/leakage path charging a high-impedance node. |
-| 2026-09-06 | **Polarity consistency RETRACTED** (a tautology of the spread gate; amplitude-only gives 12 of 53 unit-channels mixed) and the fleet **re-cut to 8–9 units, not 5**. "Persistent until serviced" weakened: 23 recoveries, median gap 6 days — though a field autozero cannot be excluded. |
-| 2026-09-06 | The spread gate is **blind to onsets by construction** — it rejects a moving floor, which is what an onset is. It rejected the very record in which the ramp is visible. |
-| 2026-09-07 | **The non-motion test** (Brian: "it doesn't cross zero"). `\|mean\|/peak` is bimodal with a 1.09% dead zone and returns exactly the 5 confirmed units from physics, not a threshold. Independent corroboration of the unit list. **BE11007 settled as NOT an offset** — a one-sided 9 in/s blast. |
-| 2026-09-07 | **BE12599 is a connector fault, not a geophone fault.** Unipolar spikes with a 26→118 ms RC tail progressing to a flat pedestal, and the fault MOVES between Long and Tran while the sensor self-check passes on every event. An element cannot hop channels; a connector can. Inspect before reseating. |

docs/ri8507_compliance_curve.md

@@ -1,135 +0,0 @@
-# USBM RI8507 / OSMRE Blasting Compliance Curve — Reference
-
-Reference for the **velocity-vs-frequency blasting compliance chart** Blastware
-draws on its Event Report ("USBM RI8507 And OSMRE"), and how seismo-relay
-reproduces it. Implemented in [`sfm/compliance.py`](../sfm/compliance.py); the
-spectral (FFT) side lives in [`waveform_fft.py`](../waveform_fft.py).
-
-Reverse-engineered 2026-09-14 against 7 BE12844 (MiniMate Plus) events, each
-with a Blastware Event Report + FFT Report as ground truth. Curve values from
-USBM RI8507 Appendix B and 30 CFR 816.67.
-
----
-
-## What it is
-
-Two closely-related sources for the same limit curve:
-
-- **USBM RI8507** — Bureau of Mines *Report of Investigations 8507* (Siskind
-  et al., 1980), *"Structure Response and Damage Produced by Ground Vibration
-  From Surface Mine Blasting."* The curve is **Figure B-1**, Appendix B
-  ("Alternative Blasting Level Criteria"), p.73–74.
-- **OSMRE / OSM** — the Office of Surface Mining Reclamation and Enforcement
-  codified it as **30 CFR 816.67, Figure 1**. "CFR" = the U.S. Code of Federal
-  Regulations. Same curve, regulatory force.
-
-The chart plots each geophone channel's significant vibration cycles as
-`(frequency, peak velocity)` points against this limit. A point **below** the
-line passes; **above** fails.
-
----
-
-## The limit curve
-
-A structure has a resonance band (~4–12 Hz for whole structures) where it is
-most vulnerable, so the safe velocity is **lower** at those frequencies and
-**higher** away from them. The curve captures this by alternating two kinds of
-bound:
-
-- **Constant-velocity** segments — a flat horizontal line at a fixed PPV.
-- **Constant-displacement** segments — a fixed peak *displacement* `d`. For
-  simple harmonic motion, peak velocity `v = 2πf·d`, so on a velocity-vs-
-  frequency **log-log** plot this is a straight line of slope +1 (velocity rises
-  with frequency). This is why the low- and high-frequency bounds are sloped.
-
-### Two lines — structure type
-
-RI8507 gives two lines for two interior-wall constructions (Table 13, p.67):
-
-| line | construction | plateau PPV |
-|---|---|---|
-| **Drywall** (solid) | modern gypsum wallboard | **0.75 in/s** |
-| **Plaster** (dashed) | older plaster on wood lath | **0.50 in/s** |
-
-Plaster-on-lath is more damage-prone, hence the lower limit. You apply **one**
-line depending on the monitored structure.
-
-### The four segments (Figure B-1, p.74)
-
-Going low → high frequency, each line is:
-
-1. **Ultimate low-frequency bound** — constant displacement **0.030 in**
-   (`v = 2πf·0.030`). Only relevant below ~4 Hz.
-2. **Plateau** — constant velocity **0.75** (Drywall) / **0.50** (plaster) in/s.
-3. **Rising diagonal** — constant displacement **0.008 in** (`v = 2πf·0.008`),
-   climbing from the plateau up to the high-frequency cap.
-4. **High-frequency cap** — constant velocity **2.0 in/s** above ~40 Hz.
-
-The segments are drawn **continuous**: each bound is used over the frequency
-range where it is the binding (lowest) limit, and consecutive bounds meet where
-they are equal — so there are no vertical steps. Transition frequencies come
-straight from the values (`f = V / (2π·d)`):
-
-| transition | formula | Drywall | Plaster |
-|---|---|---|---|
-| 0.030 in → plateau | `V_mid / (2π·0.030)` | 3.98 Hz | 2.65 Hz |
-| plateau → 0.008 in | `V_mid / (2π·0.008)` | 14.92 Hz | 9.95 Hz |
-| 0.008 in → 2.0 in/s | `2.0 / (2π·0.008)` | 39.79 Hz | 39.79 Hz |
-
-Because both lines share the same **0.008 in** rising diagonal, above ~15 Hz
-they lie on the *same* line (both reach 2.0 in/s at ~40 Hz) — RI8507's literal
-construction merges them there. Blastware renders the dashed line as a separate
-parallel diagonal, but that is cosmetic: above ~15 Hz both structure types carry
-the identical limit, so compliance is unaffected.
-
-> ⚠ RI8507's *Table 13* is a simpler two-range criterion with a **sharp
-> discontinuity at 40 Hz** (flat plateau, then a jump to 2.0). Figure B-1 is the
-> **smoothed** version that adds the 0.008 in transition — that is the one drawn
-> on reports and implemented here.
-
----
-
-## The compliance scatter (the points)
-
-The cloud is **not** the FFT spectrum. It is a per-cycle, time-domain measure by
-the **zero-crossing method** (`channel_compliance_points`):
-
-- Split the channel's waveform at its zero crossings.
-- Each half-cycle contributes one point: **frequency** `= 1 / (2 · half-period)`
-  (from the samples between the two crossings), **velocity** `= peak |amplitude|`
-  in that half-cycle.
-
-This yields ~90–110 points per channel, and — by construction — each channel's
-**highest** point equals that channel's PPV. Verified against Blastware: the
-cloud shape, density, and ceiling all match.
-
-### Why not the FFT?
-
-A broadband blast spreads its energy across many FFT bins, so no single bin
-reaches the time-domain peak — the FFT amplitudes come out ~10× below the
-compliance-chart velocities. The compliance chart is a *per-cycle peak* view;
-the **FFT** is a separate analysis (Blastware's *FFT Report*), reproduced by
-[`waveform_fft.py`](../waveform_fft.py) and used for the dominant-frequency
-readout and the #10 FFT view — not for this scatter.
-
----
-
-## Implementation
-
-- `sfm/compliance.py`
-  - `limit_at(freq, curve)` — the limit PPV at a frequency (`curve` = `"Drywall"`
-    or `"Plaster"`); curves are data in `_CURVES`, so more standards can be added.
-  - `channel_compliance_points(samples, sps)` — the zero-crossing scatter.
-  - `draw_compliance_chart(ax, channels, sps)` — matplotlib rendering (both
-    limit lines + per-channel scatter, Blastware's tick scales and channel
-    markers: Tran `+` red, Vert `×` green, Long `o` blue).
-- Tests: `tests/test_compliance.py`.
-
----
-
-## Sources
-
-- USBM **RI8507** (Siskind, Stagg, Kopp, Dowding, 1980), Appendix B / Figure B-1,
-  p.73–74; Table 13, p.67. (`ref-stuff/usbm-ri8507-ground_vibration.pdf`.)
-- **30 CFR 816.67**, "Use of explosives: Control of adverse effects," Figure 1 —
-  <https://www.ecfr.gov/current/title-30/chapter-VII/subchapter-K/part-816/section-816.67>

docs/runbooks/wedged_unit_recovery.md

@@ -1,579 +0,0 @@
-# Runbook — Recovering a wedged unit stuck in a call-home loop
-
-**Incidents:** BE9558H at `166.246.130.1:9034`, 2026-05-17 (Method B) ·
-BE12599 at `166.246.64.226:9034`, 2026-09-16 (Method A).
-
-A field unit with a stuck-triggered geophone (or any hardware fault causing
-constant event triggering) will record events back-to-back, and if Auto Call
-Home is set to "After Event Recorded" the device will dial the office BW
-ACH server in a tight loop. Combined with a Sierra Wireless modem in
-bidirectional serial-TCP mode, this makes the unit effectively unreachable
-from SFM — every TCP connection we open gets killed when the modem flips
-from server-mode to client-mode to honor the device's next AT dial command.
-
-This runbook describes how to break the loop and recover control.
-
----
-
-## ⚠ Two cures for one disease — intercept first
-
-Both incidents below are the **same failure**: a geophone offset crosses the
-trigger level, the unit records back-to-back, ACH set to "after event
-recorded" dials continuously, and the unit becomes unreachable because its
-modem is in client mode almost all of the time.
-
-There are two ways to get a Stop Monitoring command into it.
-
-| | **A — intercept the call** (preferred) | **B — catch it between calls** (original) |
-|---|---|---|
-| Idea | Be the server it dials.  Point the modem's Destination at our own ACH server and answer it. | Clear the Destination so it stops dialing, then race a Stop into the gap. |
-| Needs inbound? | **No — the unit calls us** | Yes: working inbound TCP to the modem |
-| Determinism | Deterministic — it dials every ~75 s, we only have to be listening | A race.  BE9558H took ~7 h of attempts before one landed. |
-| Tool | `bridges/ach_server.py --stop-monitoring` | `scripts/slow_drip.sh` |
-| Proven on | BE12599, 2026-09-16 | BE9558H, 2026-05-17 |
-
-**Method A is the standard procedure now.**  The unit won't answer us because
-it is on the phone — so stop dialing it and be the one it calls.  It rings,
-we pick up, take its data, and tell it to stop calling here.
-
-Method B is kept because it is proven, and because A needs a listener the
-modem can actually reach (public IP + forwarded port).  When you have that,
-don't race it — intercept it.
-
----
-
-## Symptoms
-
-- Terra-View / SFM `/device/info` either hangs or fails on `count_events()`.
-- `/device/monitor/status` and `/device/rescue` return 502 (protocol timeout
-  waiting for POLL response) or 503 (TCP connect refused).
-- ACEmanager serial log shows repeating
-  `Connect to IP: <BW_IP> Port: <BW_PORT>` → `Shutdown TCP socket` cycles
-  every 30-60 seconds.
-- Spam-mode endpoints (`/device/stop_monitoring_spam`) report many
-  `sent_ok` but the device's monitoring state never changes.
-- `slow_drip` reports `[Errno 32] Broken pipe` after sending the preamble
-  but before completing the drip loop.
-
-If you see *all* of these, the unit is in this exact failure mode.
-
----
-
-## Method A (preferred) — intercept the call
-
-You need **ACEmanager access** and a host the modem can dial: public IP with
-the listener's port forwarded to it.
-
-### A1 — start the listener BEFORE touching the modem
-
-```bash
-cd /home/serversdown/seismo-relay
-tmux new -s rescue
-.venv/bin/python -u bridges/ach_server.py --port 12345 \
-  -o bridges/captures/<unit>-diag --stop-monitoring -v
-```
-
-⚠ **Listener first, always.**  A Destination pointed at a dead port is the
-worst state available — the device still dials, the modem still flips to
-client mode, inbound stays blocked, and nothing gets delivered.
-
-Do **not** add `--events-only` (it silently breaks dedup — see gotchas), and
-do **not** add `--disable-ach` yet (see A4).
-
-### A2 — point the modem at it
-
-ACEmanager → **Serial → Port Configuration**:
-
-| Field | Set to |
-|---|---|
-| **Destination Address** | the listener's public IP |
-| **Destination Port** | the listener's port (e.g. `12345`) |
-
-Apply.  The modem auto-dials its Destination whenever serial data arrives
-while the serial port is closed — so the unit's own retry cycle now lands on
-you instead of nowhere.
-
-### A3 — answer, and stop the bleeding
-
-Within ~75 s you should see a call-in.  `--stop-monitoring` fires SUB 0x97 at
-step 1.5 — after the handshake, **before** the event walk — so the recording
-halts at the earliest possible moment in the session.  Confirm via
-`rescue.json` in the session directory:
-
-```json
-{"peer": "166.246.64.226:60921", "stop_monitoring": "ok"}
-```
-
-That is the bleeding stopped.  Everything after this is cleanup.
-
-### A4 — drain the backlog, THEN disable ACH
-
-⚠ **Order matters, and it is counter-intuitive.**  Stopping monitoring also
-removes your call-in trigger: ACH fires on "after event recorded", so with
-recording stopped the unit has no reason to dial again.  The backlog sitting
-in its memory does **not** re-arm it.
-
-So if the stored events are worth keeping — and on a fault unit they usually
-are, they're the evidence — drain them across however many call-ins it takes
-*before* you silence it.  Only then add `--disable-ach` (or use
-`scripts/rescue_device.sh <host> <port> --no-erase`).
-
-If the unit has gone quiet and you still need it, cycling the modem produces
-a call-in, and a unit with a scheduled daily call will dial at its configured
-time regardless.
-
-### A5 — restore the Destination, and confirm you did
-
-Put `Destination Address` back to `0.0.0.0` (or the office Instantel ACH
-server) once you are finished, and only stop the listener after that is done.
-
-### A6 — do NOT re-enable ACH until the hardware fault is repaired
-
-Otherwise the loop restarts the moment monitoring resumes and you run this
-runbook again.
-
----
-
-## Method B (fallback) — catch it between calls
-
-The original 2026-05 procedure.  Use when you cannot stand up a listener the
-modem can reach.  You need **ACEmanager access** to the unit's modem.
-
-### Step 1: stop the modem's mode-flipping
-
-In ACEmanager → **Serial → Port Configuration**:
-
-| Field | Set to |
-|---|---|
-| **Destination Address** | clear (blank) |
-| **Destination Port** | `0` |
-
-Click **Apply**. This removes the modem's auto-dial-out target. The device's
-AT dial commands now error back at the modem instead of triggering a
-mode-flip, so the modem stays in TCP-server mode permanently and our inbound
-TCP sessions stay alive.
-
-*(Optional belt-and-suspenders: also add the BW server's port to
-**Security → Port Filtering - Outbound** as a blocked port, with
-Outbound Port Filtering Mode = Blocked Ports.)*
-
-### Step 2: stop monitoring on the device (slow drip)
-
-From the SFM host:
-
-```bash
-/home/serversdown/seismo-relay/scripts/slow_drip.sh <DEVICE_IP> <PORT>
-```
-
-Defaults are 120s duration with a drip every 3s. Watch the response:
-
-- `duration_s ≈ 120` and `drips_sent ≈ 40` → session held the full duration ✓
-- `bytes_received > 0` → device is responding ✓ (this is the success signal)
-
-If `duration_s` is small or `send_error: "Broken pipe"`, Step 1 didn't take
-hold — re-check ACEmanager, may need to reboot the modem after Apply.
-
-### Step 3: confirm monitoring stopped
-
-```bash
-curl 'http://localhost:8200/device/monitor/status?host=<DEVICE_IP>&tcp_port=<PORT>&force=true'
-# expect: {"is_monitoring": false, ...}
-```
-
-### Step 4: disable ACH at the device level + erase corrupted events
-
-Either fire the rescue endpoint:
-
-```bash
-/home/serversdown/seismo-relay/scripts/rescue_device.sh <DEVICE_IP> <PORT>
-```
-
-Or do the two steps manually:
-
-```bash
-# Disable ACH in the device's compliance config
-curl -X POST 'http://localhost:8200/device/call_home?host=<DEVICE_IP>&tcp_port=<PORT>' \
-  -H 'Content-Type: application/json' \
-  -d '{"auto_call_home_enabled": false}'
-
-# Erase corrupted event chain
-curl -X POST 'http://localhost:8200/device/events/erase?host=<DEVICE_IP>&tcp_port=<PORT>'
-```
-
-You can also do this via the SFM standalone UI → **Call Home** tab → set
-`Enable Auto Call Home` to `Disabled` → **Write to Device**.
-
-### Step 5: restore modem config (housekeeping)
-
-Once the device-side ACH is disabled, restore the modem's Destination
-Address and Port to the original values (e.g. `50.197.32.92` / `12345`) in
-ACEmanager. The modem will resume normal bidirectional behavior, but the
-unit won't issue any dial commands until ACH is explicitly re-enabled on
-the device.
-
-### Step 6: do NOT re-enable ACH on this unit until the underlying hardware
-fault is repaired. If you do, the call-home loop starts again immediately
-and you'll be running this runbook a second time.
-
----
-
-## Why this works — the failure mode explained
-
-The Sierra Wireless RV50/RV55 serial port operates in one of two TCP modes
-at any moment:
-
-- **Server mode** — listens on `Device Port` (e.g. 9034), bridges inbound
-  TCP to the device's serial port. This is what we need to interact with
-  the device.
-- **Client mode** — when the device sends an AT dial command on its serial
-  TX line, the modem opens an outbound TCP to `Destination Address:Port`
-  and bridges that to serial.
-
-A serial port in this configuration is **bidirectional**: the modem flips
-between server and client modes on demand. When the device's firmware is
-healthy and only dials occasionally, this works fine.
-
-When the unit is constantly triggering events and ACH is set to "After
-Event Recorded", the device sends an AT dial command every few seconds.
-Each one causes the modem to:
-
-1. Drop any active inbound TCP session
-2. Flip to client mode
-3. Attempt outbound TCP to `Destination Address:Port`
-4. Hang for up to a minute waiting for it to succeed/fail
-5. Drop back to server mode
-
-**During the entire hang, no inbound TCP can establish.** Even between
-hangs, the modem closes any existing inbound session before flipping. So
-any tool that needs more than a few seconds of held TCP (e.g. POLL +
-config read + write) gets repeatedly kicked off.
-
-Clearing `Destination Address` removes step 3-4 from the cycle: the modem
-has nowhere to dial, so it doesn't flip modes when it receives an AT dial
-command. The serial port effectively becomes server-only, and inbound TCP
-sessions can stay open as long as needed.
-
-**This is a modem-layer issue, not a device firmware issue.** The device
-is alive and responsive the whole time — confirmed in the BE9558H
-recovery by 990 bytes of S3 responses received over a 120s slow-drip
-session once the modem was no longer mode-flipping.
-
----
-
-## Why simpler approaches don't work
-
-| Approach | Why it fails |
-|---|---|
-| Standard `/device/info` | Triggers `count_events()` 1E/1F walk, takes 90s+ and hits corrupted event chain in this scenario |
-| `/device/rescue` race loop | Gets 502 (protocol timeout) because the modem closes the TCP before the POLL handshake can complete |
-| `/device/stop_monitoring_blind` (single frame) | Even if the bytes leave the wire, the device's protocol parser ignores write commands without a preceding POLL handshake (early-version bug, now fixed by including POLL preamble in blind sends) |
-| `/device/stop_monitoring_spam` (sub-second cadence) | Each session is killed by the modem's mode-flip before the device can drain its UART RX buffer; high-rate spam also risks UART FIFO overrun on the device side |
-| Outbound port firewall block alone | Stops the outbound TCP from succeeding, but doesn't stop the modem from *trying* and mode-flipping. Reduces but doesn't eliminate the contention. |
-| Modem reboot | Temporary — as soon as the device starts triggering again, the loop resumes within seconds |
-
-The combination of `slow_drip` + cleared `Destination Address` works because:
-
-1. The modem stops mode-flipping → TCP session stays open for the full
-   drip duration
-2. Slow drip rate → device's UART RX FIFO never overflows even if
-   firmware is busy with event recording
-3. The drip is `SESSION_RESET + STOP_MONITORING` every 3s → many
-   independent chances for the parser to land one valid frame
-4. Once one Stop Monitoring is parsed, event recording halts → firmware
-   has CPU to spare → subsequent operations are trivially easy
-
----
-
-## Tooling reference
-
-All endpoints live in `seismo-relay/sfm/server.py`. All scripts live in
-`seismo-relay/scripts/` and default to SFM direct (`http://localhost:8200`),
-overridable via `SFM_BASE_URL`.
-
-### Endpoints added during BE9558H recovery
-
-| Endpoint | Purpose |
-|---|---|
-| `GET /device/events/storage_range` | SUB 0x06 — first/last event keys, `is_empty` flag. ~2s, no event walk. |
-| `GET /device/events/index` | SUB 0x08 — lifetime event counter (does NOT decrement on erase). ~2s. |
-| `POST /device/events/erase` | Full erase sequence 0xA3 → 0x1C → 0x06 → 0xA2. |
-| `POST /device/rescue` | Disable ACH + erase in one TCP session. Short timeouts for race-loop usage. |
-| `POST /device/stop_monitoring_blind` | Fire-and-forget Stop with full POLL preamble (single attempt). |
-| `POST /device/stop_monitoring_spam` | Server-side tight retry loop, sub-second cadence, duration-bounded. |
-| `POST /device/stop_monitoring_slow_drip` | One held TCP session, slow trickle of stop frames. **The endpoint that saved BE9558H.** |
-
-Also changed: default protocol recv timeout dropped from 30s → 10s in
-`_build_client`. Added `connect_timeout` knob to same. Cleaned up
-unhandled-exception path in `/device/monitor/status` so it returns 502
-instead of 500 on protocol timeouts.
-
-### Scripts
-
-| Script | Purpose |
-|---|---|
-| `scripts/rescue_device.sh` | Race-loop wrapper around `/device/rescue` |
-| `scripts/blind_stop.sh` | Race-loop wrapper around `/device/stop_monitoring_blind` |
-| `scripts/spam_stop.sh` | Single-call burst hammer (`/device/stop_monitoring_spam`) |
-| `scripts/slow_drip.sh` | Single-call held-session drip (`/device/stop_monitoring_slow_drip`) |
-| `scripts/watch_unit.sh` | Passive periodic reachability check, logs to file |
-
----
-
-## Incident log — BE9558H, 2026-05-16/17
-
-What was wrong: Long-axis geophone developed an offset, constantly above
-trigger threshold → constant event recording → after-event ACH set →
-modem dialing office BW server (`50.197.32.92:12345`) every 30-60s.
-Local event chain corrupted (`next_boundary 0x100EE exceeds uint16`).
-
-Diagnostic path:
-
-1. `/device/info` slow, choked on event walk
-2. Built lightweight probe endpoints (`storage_range`, `index`) — useful
-   but didn't reach the wedged unit
-3. Built `/device/rescue` with short timeouts — got 502 (POLL no response)
-4. Built `/device/stop_monitoring_blind` — first version was a false
-   positive (no POLL preamble); fixed by including
-   `SESSION_RESET+POLL_PROBE+SESSION_RESET+POLL_DATA` in the dump
-5. Verified blind stop works on bench unit
-6. Built `/device/stop_monitoring_spam` — 420 successful sends over
-   5 min, zero behavior change on field unit
-7. Inspected ACEmanager logs → saw outbound dial-out attempts every ~30s,
-   confirmed device was not fully locked up
-8. Added outbound port-12345 firewall block → outbound attempts now fail
-   instantly but contention persisted
-9. Built `/device/stop_monitoring_slow_drip` — session died at 3s with
-   broken pipe (modem closing on us)
-10. Looked at full ACEmanager Port Configuration → **found
-    `Destination Address: 50.197.32.92` configured**, realized every AT
-    dial command was triggering a modem mode-flip that killed our inbound
-11. Cleared Destination Address + Port → slow_drip held 120s, device
-    responded with 990 bytes, 39 stop commands acked
-12. Disabled ACH at device level via `/device/call_home`, erased events
-
-Final state: device IDLE, memory 958.1 / 960 KB free, ACH disabled at
-device level, modem destination cleared (to be restored after physical
-service).
-
-Total time from "i was wondering if its possible to" first attempt to
-recovery: ~7 hours of intermittent debugging across one evening.
-
----
-
-# Second incident — BE12599, 2026-09-16/17
-
-**Unit:** BE12599 at `166.246.64.226:9034`, RV50, job *I-80 North Fork Bridge
-— Abut 1 West* (Fay Company).  Same job as BE9558H, which is a coincidence.
-
-**Fault:** the connector fault documented in `docs/offset_investigation.md`
-§8e progressed until the Tran pedestal reached **0.400 in/s** — its trigger
-level.  Constant triggering → constant recording → ACH "after event recorded"
-→ continuous dialing.  Same disease as BE9558H.
-
-**Same disease, inverted cure.**  Method B's Step 1 *did* work — clearing the
-Destination stopped the dial-outs, confirmed in the ALEOS log.  It was Step 2
-that didn't land, and rather than keep racing we turned the rescue around:
-gave the unit a different server to call, and answered it.
-
-Total time ≈ 5 h, of which ~90 min went to two red herrings documented below.
-Much of the rest was rediscovering the May procedure, which is why the
-"two cures" table now sits at the top of this file.
-
----
-
-## Turn on ALEOS_SERIAL debug FIRST
-
-This is the single highest-value diagnostic and it should be step zero on any
-future incident.  ACEmanager → **Admin → Log → ALEOS_SERIAL log level →
-DEBUG**, then view the serial log.
-
-It is the only thing that tells you what the *device* is actually saying.
-Everything before we did this was guesswork.
-
-## What the log showed — the unit is on the phone
-
-Every ~75 seconds, verbatim:
-
-```
-ALEOS_SERIAL_HIF: 29 byte(s) in buffer: 'ATQ1^MATE0^MATS0=2^M^MRADIO RING^M'
-ALEOS_SERIAL_HMC: TCP recvhost fd 65535 len 29  state TCPMode::kClosed
-ALEOS_SERIAL_HMC: tcpmode trying to send to invalid socket
-ALEOS_SERIAL_HMC: Connect to IP: 0.0.0.0 Port 0
-ALEOS_SERIAL_HMC: Initialize Auto answer on port 9034
-ALEOS_SERIAL_HMC: Cannot connect to 0.0.0.0
-```
-
-Read that carefully:
-
-- `ATQ1` (quiet) / `ATE0` (echo off) / `ATS0=2` (auto-answer after 2 rings).
-  **There is no `ATD`.**  The device is not dialing — it is trying to
-  *configure* its modem.
-- The modem's serial port is in TCP data mode, so it never interprets these
-  as AT commands.  It treats them as payload and tries to ship them to a TCP
-  socket that does not exist.
-- The device therefore never receives `OK`, never progresses, and **retries
-  the identical 29 bytes forever**.
-
-**While it is in this state it is busy placing a call, not listening for
-us.**  This is almost certainly what BE9558H was doing too — we simply never
-turned on ALEOS_SERIAL debug in May to look.  It is not a different disease;
-it is the same one, seen properly for the first time.
-
-It is also the argument for Method A in one picture: the unit is mid-dial
-every ~75 s, and our inbound Stop has to thread the gaps between those
-attempts.  Give it somewhere to dial and the problem inverts into a
-deterministic one.
-
-### Why `slow_drip` lied
-
-`slow_drip` returned the *success* signature except for the one field that
-mattered:
-
-```json
-{"duration_s":120.0,"drips_sent":38,"bytes_sent":920,
- "bytes_received":0,"send_error":null}
-```
-
-Full duration, no broken pipe — but zero bytes back.  Cause is in the log
-above: each 75 s cycle re-runs `Initialize Auto answer on port 9034`, which
-orphans the held session (`data in for unknown reason 3 removing from
-select`, `OnMsg recv error: 107 - Transport endpoint is not connected`).  Our
-local TCP stayed open so `sendall` never raised — but the modem stopped
-bridging after the first re-init, so every drip after that went into a socket
-nobody was reading.
-
-⚠ **`send_error: null` + full duration is NOT success.  Only
-`bytes_received > 0` is success.**
-
-⚠ **In fairness to slow_drip: it got exactly one attempt here**, run ~90 s
-after a modem reboot, with a dead session visible in the log at 20:19:17 in
-that same window.  BE9558H took hours of attempts before one landed.  Method B
-was not ruled out on BE12599 so much as abandoned in favour of something that
-doesn't need luck.
-
----
-
-## ⚠ Two red herrings that cost ~90 minutes
-
-### 1. The trusted-IP whitelist (this was the real reason inbound never worked)
-
-The RV50s run with **Security → Trusted IPs (Friends List) enabled**.  A
-source IP that is not on the list is dropped **silently** — inbound presents
-as `Connection error: timed out`, never a refusal.
-
-Brian's dev-box public IP is **dynamic** and had changed, so `tmi-dev` was no
-longer whitelisted.  Every inbound attempt failed identically across four
-different modem and device states, which looked exactly like the BE9558H
-mode-flipping symptom and sent us chasing modem configuration for over an
-hour.
-
-**Check this before diagnosing anything else.**  Note that SFM in Docker
-egresses via the *host's public IP*, not its LAN IP.
-
-### 2. A 502 from SFM does not mean TCP connected
-
-`sfm/server.py` raises **502 for both** failure classes:
-
-```python
-raise HTTPException(status_code=502, detail=f"Protocol error: {exc}")
-raise HTTPException(status_code=502, detail=f"Connection error: {exc}")
-```
-
-We read an early 502 as "TCP connected, modem bridged, device mute" and built
-a whole theory on it.  It was almost certainly a connect timeout.
-**Always read the `detail` string** — "connect failed" and "device didn't
-answer" are completely different problems and the status code will not
-separate them.
-
----
-
-## What actually worked — invert the direction
-
-The key observation is in the log above:
-
-> `TCP recvhost ... state TCPMode::kClosed` → `Connect to IP: 0.0.0.0 Port 0`
-
-**The modem auto-dials its Destination whenever serial data arrives while
-closed.**  So instead of fighting for inbound, give it somewhere to dial:
-point `Destination Address` at our own `ach_server` and the device's own
-75-second attempts become **device-initiated sessions the modem bridges
-correctly**.  No race, no contention, worst case a 75-second wait.
-
-### Procedure
-
-1. **Run the rescue server** on a host the modem can reach (public IP +
-   forwarded port):
-
-   ```bash
-   cd /home/serversdown/seismo-relay
-   .venv/bin/python -u bridges/ach_server.py --port 12345 \
-     -o bridges/captures/<unit>-diag --stop-monitoring -v
-   ```
-
-2. **Point the modem at it** — ACEmanager → Serial → Port Configuration →
-   `Destination Address` = your public IP, `Destination Port` = 12345.
-
-3. **Wait for the call-in.**  `--stop-monitoring` fires SUB 0x97 at step 1.5,
-   after the handshake and *before* the event walk.  Confirm via
-   `rescue.json` in the session directory:
-
-   ```json
-   {"peer": "166.246.64.226:60921", "stop_monitoring": "ok"}
-   ```
-
-4. **Restore the modem's Destination** once you are done, then finish the
-   device side (disable ACH, erase) through whichever channel works.
-
-On BE12599 the first call-in landed at 20:58:11 and reported
-`stop_monitoring: ok`; a second at 20:58:20 confirmed it.  `is_monitoring:
-false` was still true **6½ hours later** — the fix is durable.
-
----
-
-## Hard-won gotchas (do not re-derive)
-
-- **Never leave the Destination pointed at a host with nothing listening.**
-  That is the worst state available: the device still dials, the modem still
-  flips, inbound stays blocked, and nothing is delivered.  An 8-minute gap
-  with the listener down produced a spurious inbound timeout that cost
-  another round of misdiagnosis.
-
-- **Stopping monitoring removes your call-in channel.**  ACH is "after event
-  recorded"; no new events means no new dials.  The backlog sitting in memory
-  does *not* re-arm it.  After a successful stop the unit goes quiet and you
-  need the modem cycled (works — produced a call-in), the scheduled daily call
-  (BE12599 calls at **05:00:14 device-local**, per §8e), or working inbound.
-  **Plan the order before you fire the stop.**
-
-- **`--events-only` silently breaks dedup.**  It skips the device-info step,
-  so the serial is never read; `ach_state.json` then keys on
-  `peer:ephemeral_port`, which is unique per connection.  Every session looks
-  like a new unit, starts from key 0, and re-downloads the same event.  Four
-  sessions on BE12599 downloaded the identical event four times and made zero
-  progress on the backlog.  Events also file as `serial=UNKNOWN` with a
-  `M000…` BW filename (serial_numeric 0) instead of `N599…`.
-  **Do not use `--events-only` when you intend to download anything.**
-
-- **`/device/events/index` reported `lifetime_count: 0`** on a unit with years
-  of history.  Suspected decode bug in the SUB 0x08 field offset — do not
-  trust that number.  The 88-byte payload is preserved in the `raw_hex` field
-  if someone wants to chase it.
-
-- **Memory used cross-checks the event keys exactly:**
-  `last_key − buffer_start = memory_total − memory_free`.  On BE12599:
-  `0x011230ec − 0x01110000 = 78,060` and `983,028 − 904,968 = 78,060`.
-  Useful sanity check that you are reading the keys right.
-
----
-
-## Final state (2026-09-17 ~01:30 local)
-
-- `is_monitoring: false`, held 6½ hours
-- Battery 6.76 V
-- Memory 78,060 / 983,028 bytes used (8%)
-- `first_key 01121728`, `last_key 011230ec` — ~6.6 KB of addressable event
-  chain, roughly 3 events
-- ACH still **enabled** — to be disabled after the backlog is preserved
-- Modem Destination still pointed at tmi-dev — to be restored
-- ⚠ **Do not re-enable ACH until the connector is serviced.**  Tran is still
-  sitting at 0.400 and the loop restarts the moment monitoring resumes.

docs/sfm_tool_status.md

@@ -1,150 +0,0 @@
-# SFM — where it actually stands as a tool
-
-**Status as of 2026-09-20 (v0.31.0).**  This is the honest assessment, not the
-roadmap — `README.md § Roadmap` covers where it is *going*.  Expect this file to
-go stale; re-date it when you revise it.
-
----
-
-## The framing
-
-SFM is **three different things wearing one name**, at three very different
-levels of maturity:
-
-| | what it is | maturity |
-|---|---|---|
-| **The codec library** | `minimateplus/`, `micromate/` — bytes in, `Event` out | **Production.** Verified per-sample at scale. |
-| **SDM — the data side** | the DB, waveform store, `/db/*`, ingest | **Production.** Terra-View depends on it daily. |
-| **SFM — the device side** | `/device/*`, live connections to units | **Emergency-grade.** Works, but manual, unauthenticated, and thinly tested. |
-| **The lab** | `seismo_lab.py`, `scratch/`, the Inspector | **Research artifacts.** Useful, not products. |
-
-Brian's own description — *"right now it's an emergency tool and a research
-project"* — is accurate, and it applies specifically to the **device side**.
-The data side is not an emergency tool; it has been carrying production for
-months.
-
-Most confusion about "is SFM reliable?" comes from answering for the wrong
-tier.
-
----
-
-## 1. What you can rely on
-
-### Production-grade — trust it
-
-- **Series-3 decode.** 14,338 / 14,338 files decode per-sample exact against
-  preserved Blastware ASCII exports, 45 units, files back to 2018.
-- **Series-4 (Thor) decode.** 1,057,536 / 1,057,536 geo samples exact against
-  Thor's own CSV exports; production IDFW 575/575 with zero truncations.
-- **Histogram decode.** 1,211 / 1,211 production histograms exact, including
-  842,442 per-interval frequency comparisons with zero mismatches.
-- **The ingest path.** `/db/import/blastware_file` and `/db/import/idf_file`
-  fed by the watchers — this is how prod actually gets its data, and it has
-  been running unattended for months.
-- **`/db/*` read API.** Always-on, consumed by Terra-View for every fleet
-  listing, event detail and report.
-- **The waveform store** — `.h5` + `.sfm.json` sidecars + retained raw
-  binaries, with operator review state preserved across regeneration.
-- **`bridges/ach_server.py`** — speaks the full BW protocol to calling units.
-  Proven in the field, including as a rescue tool (see the runbook).
-
-### Emergency-grade — works, but you are the error handling
-
-- **`/device/*` live endpoints.**  They do what they say.  But they are
-  synchronous, unauthenticated, and a single cellular download can exceed the
-  60 s timeouts that sit in front of them.
-- **The rescue ladder** (`rescue`, `stop_monitoring_*`, `events/erase`).
-  Each has worked in a real incident — but each has been used a handful of
-  times, by one person, with the runbook open.
-- **The standalone webapp.**  Perfectly usable, and as of v0.31.0 the cheap
-  probes and rescue actions are reachable without curl.  No auth of any kind.
-
-### Research artifacts — useful, not products
-
-- **`seismo_lab.py`** — 2,789 lines of Tkinter (Bridge / Analyzer / Query DB /
-  Inspector).  Desktop-only, single-user, no tests.
-- **`scratch/`** — the verification harnesses (`verify_against_ascii.py`,
-  `verify_thor_against_csv.py`) and the offset detector (`offset_scan3.py`).
-  These produced the numbers the production claims rest on, so they matter —
-  but they are analysis scripts, not maintained code.
-- **`docs/offset_investigation.md`** — an open investigation, not a feature.
-
----
-
-## 2. What to use when
-
-| you want to… | use | notes |
-|---|---|---|
-| Know if a unit is monitoring / its battery / memory | `GET /device/monitor/status?force=true` | ~2 s |
-| Know whether ACH is on | `GET /device/call_home` | ~2 s.  **Not** `/device/events`. |
-| See how full a unit's buffer is | `GET /device/events/storage_range` | ~2 s, no chain walk |
-| Stop a runaway unit | Diagnostics tab → Stop Monitoring | see the runbook first |
-| Reach a unit that will not answer | **point its modem at an `ach_server` and answer its call** | runbook Method A — do not race it |
-| List a unit's stored events | Events tab → Load events | **slow**, and broken past 64 KB (below) |
-| Get event data into the DB | the watcher → `/db/import/*` path | not the live walk |
-
-The single most useful habit: **the cheap probes are cheap and the event walk
-is not.**  Reaching for `/device/events` to answer a yes/no question about a
-unit is the mistake that motivated the v0.31.0 webapp changes.
-
----
-
-## 3. Known issues
-
-| issue | impact | status |
-|---|---|---|
-| **5A walk dies once a unit's buffer crosses 64 KB** | `/device/events` 500s; event body never downloads | Known, documented in `CLAUDE.md`.  Needs a BW capture of a spanning event to fix properly. |
-| **No auth on SFM at all** | 21 `/device/*` endpoints, including destructive ones, open to anything that reaches the port | Design agreed (Terra-View as authenticated jump host); not built. |
-| **Swagger try-it-out is live on destructive endpoints** | `POST /device/events/erase` is one click away at `:8200/docs` | Partially mitigated: the webapp's erase now requires typing the serial.  `/docs` itself is unguarded. |
-| **`SUB 0x08` lifetime counter reads 0** | `/device/events/index` returns a meaningless number | Suspected field-offset bug.  Surfaced in the UI as "unreliable". |
-| **Long device operations are synchronous** | 60 s timeouts in `routers/sfm.py` and the reverse proxy; a full download exceeds both | Known design constraint.  Must be POST-starts-job / GET-polls before any remote lab. |
-| **`backfill_sidecars.py --force` silently inserts DB rows** | store files with no DB row get one; the dry-run does not report the count | Known.  Avoid `--force` — `TOOL_VERSION` gates regeneration anyway. |
-| **14 sensitive-range files show an exact 8× discrepancy** | 10.0 / 1.25 — a units problem, not a decode problem | Open, not blocking. |
-| **16 failing tests on `dev`** | 15 need gitignored fixture bundles; 1 is real (`sc["peak_values"]["transverse"]` returns `None` where `0.0` is expected) | The real one shipped in v0.31.0. |
-
----
-
-## 4. What stands between this and a real tool
-
-Roughly in dependency order — each unblocks the ones below it.
-
-**1. Authentication.**  Everything else is gated on this.  SFM has none, and
-the modem IP whitelist gives zero protection because SFM *is* the whitelisted
-origin.  The agreed design delegates rather than builds: Terra-View becomes the
-authenticated jump host (`/api/sfm/*` already inherits deny-by-default operator
-auth), and the `8200:8200` publish is dropped so Terra-View is the only door.
-
-**2. Async long operations.**  POST starts a job, GET polls.  Retrofitting this
-after building a remote lab on top of synchronous endpoints would be far worse
-than designing for it now.
-
-**3. Confirm-guards on the remaining destructive endpoints.**  Auth answers
-*who*, not *did you mean it*.  The webapp's erase is guarded; the other seven
-destructive POSTs and `/docs` are not.
-
-**4. The 5A page-boundary fix.**  Until this lands, live event download is
-unreliable on exactly the units most likely to need attention — the ones that
-have been recording heavily.  Wants a Blastware capture of an event spanning a
-page boundary before the chunk-addressing half is trustworthy.
-
-**5. A live Thor / Micromate client.**  The device side is MiniMate-only.
-Series-4 units can only be read from forwarded files, so half the fleet has no
-live path at all.
-
-**6. Test coverage that runs from a clean checkout.**  15 of 16 current
-failures are missing fixture bundles.  A test suite that cannot go green on a
-fresh clone cannot gate anything.
-
-**7. The SDM rename.**  Cosmetic relative to the above, but the longer `sfm/`
-holds the data-side code the more the tiers blur.  ~30–50 files here, ~10–15 in
-Terra-View, plus a Docker volume migration.  Do it when the codebase is quiet.
-
----
-
-## The short version
-
-The **data side is a real tool already**.  The **device side is a set of sharp
-instruments** that work in the hands of the person who wrote them, with the
-runbook open.  The gap between those two states is mostly **auth, async, and
-guardrails** — not protocol work.  The protocol is the part that is actually
-finished.

docs/superpowers/plans/2026-08-21-waveform-ft-detection-phase-a.md

@@ -1,628 +0,0 @@
-# Waveform-shape FT detection — Phase A (seismo-relay) Implementation Plan
-
-> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
-
-**Goal:** Compute per-event waveform-shape metrics (crest factor + points-near-peak) in SFM and store them as `events` columns, populated at ingest and by a backfill script, so Terra-View can read them from `/db/events`.
-
-**Architecture:** A pure DSP module (`sfm/shape_metrics.py`) turns decoded `.h5` samples into shape metrics. New nullable `events.shape_*` columns are added via the existing `_migrate` ADD COLUMN loop. The three `WaveformStore.save*` paths compute shape from the just-written `.h5` and hand it to `insert_events`; a backfill script does the same over existing events. Mirrors exactly how per-channel ZC frequency was added.
-
-**Tech Stack:** Python 3.10, numpy, h5py, sqlite3 (raw), pytest. seismo-relay venv: `/home/serversdown/seismo-relay/.venv/bin/python3`.
-
-## Global Constraints
-
-- Metrics are read from the `.h5` `samples/{Tran,Vert,Long}` float32 arrays (physical in/s). The measured channel is the max-|peak| geophone channel.
-- All new columns are nullable; histogram records and events without usable samples store NULL (Terra-View falls back to cheap signals). Legacy rows stay valid.
-- Crest factor = `max(|x|) / rms(x)`; near-peak count = number of samples with `|x| ≥ 0.5·peak`. Threshold `0.5` is a module constant so calibration can tune it.
-- No manual migration: columns are added in `SeismoDb._migrate`, run at `SeismoDb()` construction.
-- `/db/events` needs no change — it returns all columns via `SELECT *` (verify with a test).
-- Run tests with `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest`.
-
----
-
-### Task 1: Shape DSP module
-
-**Files:**
-- Create: `sfm/shape_metrics.py`
-- Test: `tests/test_shape_metrics.py`
-
-**Interfaces:**
-- Produces:
-  - `channel_shape(x) -> dict | None` — `{"crest_factor": float, "near_peak_count": int, "sample_count": int}` or None for unusable input (size < 2, flat, all-zero).
-  - `shape_from_samples(chans: dict[str, ArrayLike]) -> dict | None` — picks the max-peak geophone channel; returns `{"crest_factor","near_peak_count","sample_count","axis"}` or None.
-  - `shape_from_h5(path) -> dict | None` — reads `samples/{Tran,Vert,Long}` and delegates to `shape_from_samples`; None on any read error.
-  - Constant `NEAR_PEAK_FRACTION = 0.5`.
-
-- [ ] **Step 1: Write the failing test**
-
-```python
-# tests/test_shape_metrics.py
-import numpy as np
-from sfm.shape_metrics import channel_shape, shape_from_samples
-
-def test_needle_spike_high_crest_few_near_peak():
-    x = np.zeros(1024); x[500] = 1.0          # one isolated spike
-    s = channel_shape(x)
-    assert s["sample_count"] == 1024
-    assert s["crest_factor"] > 15             # peak towers over rms
-    assert s["near_peak_count"] <= 3          # almost nothing near the peak
-
-def test_ringing_low_crest_many_near_peak():
-    t = np.arange(1024)
-    x = np.sin(2*np.pi*t/32) * np.exp(-t/4000)  # decaying oscillation
-    s = channel_shape(x)
-    assert s["crest_factor"] < 6
-    assert s["near_peak_count"] > 30          # many samples near the peak
-
-def test_channel_shape_none_for_unusable():
-    assert channel_shape(np.zeros(1024)) is None   # flat / all-zero
-    assert channel_shape(np.array([1.0])) is None  # too short
-
-def test_shape_from_samples_picks_max_peak_axis():
-    chans = {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": np.zeros(1024)}
-    chans["Long"][10] = 0.5
-    chans["Vert"] = np.sin(np.arange(1024)/5) * 0.01
-    s = shape_from_samples(chans)
-    assert s["axis"] == "Long"                # Long has the biggest peak
-    assert s["near_peak_count"] <= 3
-
-def test_shape_from_samples_none_when_no_geo():
-    assert shape_from_samples({"MicL": np.ones(1024)}) is None
-```
-
-- [ ] **Step 2: Run test to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics.py -q`
-Expected: FAIL — `ModuleNotFoundError: sfm.shape_metrics`.
-
-- [ ] **Step 3: Write minimal implementation**
-
-```python
-# sfm/shape_metrics.py
-"""Waveform-shape metrics for false-trigger detection.
-
-A false trigger is an isolated impulse (quiet → spike → quiet); a real event
-rings for many cycles. Two numbers separate them: crest factor (how far the
-peak stands above the typical sample) and how many samples sit near the peak.
-"""
-from __future__ import annotations
-import numpy as np
-
-_GEO_CHANNELS = ("Tran", "Vert", "Long")
-NEAR_PEAK_FRACTION = 0.5   # a sample "near the peak" is >= this * peak amplitude
-
-
-def channel_shape(x) -> dict | None:
-    x = np.asarray(x, dtype=float)
-    if x.size < 2:
-        return None
-    peak = float(np.max(np.abs(x)))
-    if peak <= 0:
-        return None
-    rms = float(np.sqrt(np.mean(x ** 2)))
-    if rms <= 0:
-        return None
-    near = int(np.sum(np.abs(x) >= NEAR_PEAK_FRACTION * peak))
-    return {"crest_factor": peak / rms, "near_peak_count": near,
-            "sample_count": int(x.size)}
-
-
-def shape_from_samples(chans: dict) -> dict | None:
-    best_axis, best_peak, best_x = None, -1.0, None
-    for ax in _GEO_CHANNELS:
-        x = chans.get(ax)
-        if x is None:
-            continue
-        x = np.asarray(x, dtype=float)
-        if x.size < 2:
-            continue
-        p = float(np.max(np.abs(x)))
-        if p > best_peak:
-            best_axis, best_peak, best_x = ax, p, x
-    if best_axis is None:
-        return None
-    s = channel_shape(best_x)
-    if s is None:
-        return None
-    s["axis"] = best_axis
-    return s
-
-
-def shape_from_h5(path) -> dict | None:
-    import h5py
-    try:
-        with h5py.File(path, "r") as f:
-            chans = {ax: f[f"samples/{ax}"][:] for ax in _GEO_CHANNELS
-                     if f"samples/{ax}" in f}
-    except Exception:
-        return None
-    return shape_from_samples(chans)
-```
-
-- [ ] **Step 4: Run test to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics.py -q`
-Expected: PASS (5 tests).
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add sfm/shape_metrics.py tests/test_shape_metrics.py
-git commit -m "feat(shape): crest-factor + points-near-peak waveform metrics"
-```
-
----
-
-### Task 2: shape_from_h5 round-trips a real .h5
-
-**Files:**
-- Test: `tests/test_shape_metrics_h5.py`
-
-**Interfaces:**
-- Consumes: `sfm.shape_metrics.shape_from_h5`; `h5py`.
-
-- [ ] **Step 1: Write the failing test** (writes a tiny .h5 the same shape SFM writes, then reads it back)
-
-```python
-# tests/test_shape_metrics_h5.py
-import numpy as np, h5py
-from sfm.shape_metrics import shape_from_h5
-
-def _write_h5(path, chans):
-    with h5py.File(path, "w") as f:
-        g = f.create_group("samples")
-        for k, v in chans.items():
-            g.create_dataset(k, data=np.asarray(v, dtype="float32"))
-
-def test_shape_from_h5_reads_dominant_axis(tmp_path):
-    p = tmp_path / "ev.h5"
-    long = np.zeros(1024, dtype="float32"); long[100] = 0.48
-    _write_h5(p, {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": long,
-                  "MicL": np.ones(1024)})
-    s = shape_from_h5(str(p))
-    assert s["axis"] == "Long" and s["near_peak_count"] <= 3
-
-def test_shape_from_h5_none_on_missing_or_degenerate(tmp_path):
-    assert shape_from_h5(str(tmp_path / "nope.h5")) is None
-    p = tmp_path / "degen.h5"
-    _write_h5(p, {"Tran": np.zeros(1), "Vert": np.zeros(1), "Long": np.zeros(1)})
-    assert shape_from_h5(str(p)) is None
-```
-
-- [ ] **Step 2: Run to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics_h5.py -q`
-Expected: FAIL (assertion or, if Task 1 incomplete, import error).
-
-- [ ] **Step 3: Implementation** — none needed; `shape_from_h5` already exists from Task 1. If a test fails, fix `shape_from_h5` (not the test).
-
-- [ ] **Step 4: Run to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics_h5.py -q`
-Expected: PASS (2 tests).
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add tests/test_shape_metrics_h5.py
-git commit -m "test(shape): shape_from_h5 round-trips a real .h5"
-```
-
----
-
-### Task 3: Add shape columns to the events schema + migration
-
-**Files:**
-- Modify: `sfm/database.py` — `_SCHEMA` CREATE TABLE `events` (after `mic_zc_above_range`); `_migrate` ADD COLUMN loop (the tuple around line 205-218).
-- Test: `tests/test_shape_columns.py`
-
-**Interfaces:**
-- Produces: `events` columns `shape_crest_factor REAL`, `shape_near_peak_count INTEGER`, `shape_sample_count INTEGER`, `shape_axis TEXT`.
-
-- [ ] **Step 1: Write the failing test**
-
-```python
-# tests/test_shape_columns.py
-import sqlite3
-from sfm.database import SeismoDb
-
-_SHAPE_COLS = {"shape_crest_factor", "shape_near_peak_count",
-               "shape_sample_count", "shape_axis"}
-
-def _cols(db):
-    with sqlite3.connect(db.db_path) as c:
-        return {r[1] for r in c.execute("PRAGMA table_info(events)")}
-
-def test_fresh_db_has_shape_columns(tmp_path):
-    db = SeismoDb(tmp_path / "s.db")
-    assert _SHAPE_COLS <= _cols(db)
-
-def test_existing_db_migrates_shape_columns(tmp_path):
-    p = tmp_path / "s.db"
-    db = SeismoDb(p)
-    with sqlite3.connect(p) as c:      # simulate an older DB missing the columns
-        for col in _SHAPE_COLS:
-            c.execute(f"ALTER TABLE events DROP COLUMN {col}")
-    assert not (_SHAPE_COLS <= _cols(SeismoDb(p)))  # sanity: dropped
-    SeismoDb(p)                        # re-open triggers _migrate
-    assert _SHAPE_COLS <= _cols(SeismoDb(p))
-```
-
-> Note: sqlite `DROP COLUMN` needs sqlite ≥ 3.35 (bundled py3.10 has it). If the runner's sqlite lacks it, replace the "simulate older DB" block with building a table without the columns; keep the assertion that re-open adds them.
-
-- [ ] **Step 2: Run to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_columns.py -q`
-Expected: FAIL — columns absent.
-
-- [ ] **Step 3: Implementation**
-
-In `_SCHEMA`, after the `mic_zc_above_range      INTEGER,` line in the `events` CREATE TABLE, add:
-
-```
-    shape_crest_factor      REAL,                   -- peak / rms of the triggering channel
-    shape_near_peak_count   INTEGER,                -- samples >= 0.5 * peak (FT: few; real: many)
-    shape_sample_count      INTEGER,                -- total samples (to normalize near_peak_count)
-    shape_axis              TEXT,                   -- geophone channel measured ("Tran"/"Vert"/"Long")
-```
-
-In `_migrate`, extend the ADD COLUMN tuple (the `for col, ddl in (...)` list) with:
-
-```python
-            ("shape_crest_factor",    "REAL"),
-            ("shape_near_peak_count", "INTEGER"),
-            ("shape_sample_count",    "INTEGER"),
-            ("shape_axis",            "TEXT"),
-```
-
-- [ ] **Step 4: Run to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_columns.py -q`
-Expected: PASS.
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add sfm/database.py tests/test_shape_columns.py
-git commit -m "feat(db): shape_* columns on events (+ auto-migrate)"
-```
-
----
-
-### Task 4: insert_events persists shape from the waveform record
-
-**Files:**
-- Modify: `sfm/database.py` — `insert_events` INSERT (column list + placeholders + values) and the UPSERT `UPDATE` block.
-- Test: `tests/test_insert_events_shape.py`
-
-**Interfaces:**
-- Consumes: a `waveform_records` rec dict that may carry `shape_crest_factor`, `shape_near_peak_count`, `shape_sample_count`, `shape_axis`.
-- Produces: those four values stored on the row; refreshed on UPSERT.
-
-- [ ] **Step 1: Write the failing test**
-
-```python
-# tests/test_insert_events_shape.py
-from sfm.database import SeismoDb
-from tests.helpers_events import make_event  # existing helper used by other insert tests
-
-def test_insert_stores_shape_from_record(tmp_path):
-    db = SeismoDb(tmp_path / "s.db")
-    ev = make_event(serial="BE1", key="0111abcd")
-    rec = {ev._waveform_key.hex(): {
-        "filename": "F.CE0W", "filesize": 10,
-        "shape_crest_factor": 34.0, "shape_near_peak_count": 3,
-        "shape_sample_count": 1024, "shape_axis": "Long"}}
-    db.insert_events([ev], serial="BE1", waveform_records=rec)
-    row = db.query_events(serial="BE1")[0]
-    assert row["shape_crest_factor"] == 34.0
-    assert row["shape_near_peak_count"] == 3
-    assert row["shape_axis"] == "Long"
-```
-
-> If `tests/helpers_events.make_event` doesn't exist, build the `Event` inline the way `tests/test_zc_freq_columns.py` does (copy its event-construction helper). Keep the assertion identical.
-
-- [ ] **Step 2: Run to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_insert_events_shape.py -q`
-Expected: FAIL — `KeyError`/`sqlite3.OperationalError` (columns not in INSERT) or values are NULL.
-
-- [ ] **Step 3: Implementation**
-
-In `insert_events` INSERT: add `shape_crest_factor, shape_near_peak_count, shape_sample_count, shape_axis` to the column list, add four `?` placeholders, and add these to the VALUES tuple (after the `mic_zc_above_range` value):
-
-```python
-                            rec.get("shape_crest_factor"),
-                            rec.get("shape_near_peak_count"),
-                            rec.get("shape_sample_count"),
-                            rec.get("shape_axis"),
-```
-
-In the UPSERT `UPDATE ... SET`: add
-
-```sql
-                               shape_crest_factor    = COALESCE(?, shape_crest_factor),
-                               shape_near_peak_count = COALESCE(?, shape_near_peak_count),
-                               shape_sample_count    = COALESCE(?, shape_sample_count),
-                               shape_axis            = COALESCE(?, shape_axis),
-```
-
-and the matching params (before `serial, ts`):
-
-```python
-                            rec.get("shape_crest_factor") if rec else None,
-                            rec.get("shape_near_peak_count") if rec else None,
-                            rec.get("shape_sample_count") if rec else None,
-                            rec.get("shape_axis") if rec else None,
-```
-
-(`COALESCE` on UPSERT so a re-import that lacks samples doesn't wipe a previously-computed shape.)
-
-- [ ] **Step 4: Run to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_insert_events_shape.py -q`
-Expected: PASS.
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add sfm/database.py tests/test_insert_events_shape.py
-git commit -m "feat(db): insert_events persists shape_* from waveform record"
-```
-
----
-
-### Task 5: Populate shape at ingest (the three save paths)
-
-**Files:**
-- Modify: `sfm/waveform_store.py` — in `save`, `save_imported_bw`, `save_imported_idf`, after the `.h5` is written, add its shape to the returned `rec` dict.
-- Test: `tests/test_save_shape.py`
-
-**Interfaces:**
-- Consumes: `sfm.shape_metrics.shape_from_h5`.
-- Produces: `save*` return dicts carry `shape_crest_factor / shape_near_peak_count / shape_sample_count / shape_axis` (present only when the `.h5` had usable samples).
-
-- [ ] **Step 1: Write the failing test** (drives the BW-import path, which the existing suite already exercises)
-
-```python
-# tests/test_save_shape.py
-from sfm.waveform_store import WaveformStore
-from tests.helpers_bw import sample_bw_bytes, sample_serial  # reuse existing import-test fixtures
-
-def test_save_imported_bw_attaches_shape(tmp_path):
-    store = WaveformStore(tmp_path / "waveforms")
-    ev, rec = store.save_imported_bw(sample_bw_bytes(), serial=sample_serial())
-    # A real BW waveform → shape present with a geo axis.
-    assert rec.get("shape_axis") in ("Tran", "Vert", "Long")
-    assert rec["shape_crest_factor"] > 0
-    assert rec["shape_sample_count"] > 200
-```
-
-> Reuse whatever fixture `tests/test_save_imported_bw*.py` already uses for BW bytes; match its import. If the existing BW fixture produces a degenerate/short waveform, use the fixture from the test that asserts a full h5.
-
-- [ ] **Step 2: Run to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_save_shape.py -q`
-Expected: FAIL — `rec` has no `shape_*` keys.
-
-- [ ] **Step 3: Implementation**
-
-Add a helper near the top of `WaveformStore` methods (module-level import `from sfm.shape_metrics import shape_from_h5`). In each of `save`, `save_imported_bw`, `save_imported_idf`, immediately before building/returning the `rec` dict — and only when the `.h5` was written (i.e. `hdf5_filename`/`hdf5_path` is set) — compute and merge:
-
-```python
-        shape = shape_from_h5(hdf5_path) if hdf5_filename else None
-        # ... in the returned rec dict literal, add:
-        #   **(shape and {
-        #       "shape_crest_factor":    shape["crest_factor"],
-        #       "shape_near_peak_count": shape["near_peak_count"],
-        #       "shape_sample_count":    shape["sample_count"],
-        #       "shape_axis":            shape["axis"],
-        #   } or {}),
-```
-
-Concretely, after each method computes `hdf5_filename`, add before its `return {...}`:
-
-```python
-        _shape = shape_from_h5(hdf5_path) if hdf5_filename else None
-        _shape_rec = {
-            "shape_crest_factor":    _shape["crest_factor"],
-            "shape_near_peak_count": _shape["near_peak_count"],
-            "shape_sample_count":    _shape["sample_count"],
-            "shape_axis":            _shape["axis"],
-        } if _shape else {}
-```
-
-and spread `**_shape_rec` into the returned dict. (`save_imported_bw`/`save_imported_idf` use their own hdf5 path variable names — use whichever local holds the written `.h5` path in each method.)
-
-- [ ] **Step 4: Run to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_save_shape.py -q`
-Expected: PASS.
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add sfm/waveform_store.py tests/test_save_shape.py
-git commit -m "feat(ingest): compute shape from the written .h5 in all save paths"
-```
-
----
-
-### Task 6: Backfill script for existing events
-
-**Files:**
-- Create: `scripts/backfill_event_shape.py` (mirror `scripts/backfill_event_zc_freq.py`, but read the `.h5` for samples instead of the sidecar).
-- Test: `tests/test_backfill_event_shape.py`
-
-**Interfaces:**
-- Produces: `backfill_shape(db, store, *, dry_run=False) -> dict` with counts `{"updated","skipped_no_h5","skipped_no_samples"}`; `main(argv)` CLI mirroring the zc-freq script's args (`--db-path`, `--store-root`, `--dry-run`).
-
-- [ ] **Step 1: Write the failing test**
-
-```python
-# tests/test_backfill_event_shape.py
-import numpy as np, h5py
-from sfm.database import SeismoDb
-from sfm.waveform_store import WaveformStore
-from scripts.backfill_event_shape import backfill_shape
-from tests.helpers_events import make_event   # or inline as in test_zc_freq_columns
-
-def _h5(path, long):
-    with h5py.File(path, "w") as f:
-        g = f.create_group("samples")
-        for k in ("Tran", "Vert"): g.create_dataset(k, data=np.zeros(1024, "float32"))
-        g.create_dataset("Long", data=np.asarray(long, "float32"))
-
-def test_backfill_updates_shape_and_is_idempotent(tmp_path):
-    db = SeismoDb(tmp_path / "s.db")
-    store = WaveformStore(tmp_path / "waveforms")
-    ev = make_event(serial="BE1", key="0111abcd")
-    db.insert_events([ev], serial="BE1",
-                     waveform_records={ev._waveform_key.hex():
-                                       {"filename": "F.CE0W", "filesize": 10}})
-    # place the .h5 where store.paths_for expects it
-    long = np.zeros(1024); long[100] = 0.48
-    _h5(store.hdf5_path_for("BE1", "F.CE0W"), long)
-
-    c1 = backfill_shape(db, store)
-    assert c1["updated"] == 1
-    row = db.query_events(serial="BE1")[0]
-    assert row["shape_axis"] == "Long" and row["shape_near_peak_count"] <= 3
-    c2 = backfill_shape(db, store)                  # idempotent: re-run overwrites same values
-    assert db.query_events(serial="BE1")[0]["shape_crest_factor"] == row["shape_crest_factor"]
-```
-
-> Match `make_event` / column names to whatever `tests/test_zc_freq_columns.py` uses. `store.hdf5_path_for(serial, filename)` is the existing helper that returns the `.h5` path.
-
-- [ ] **Step 2: Run to verify it fails**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_backfill_event_shape.py -q`
-Expected: FAIL — `ModuleNotFoundError: scripts.backfill_event_shape`.
-
-- [ ] **Step 3: Implementation** (mirror the zc-freq script structure)
-
-```python
-#!/usr/bin/env python3
-"""Backfill events.shape_* from each event's .h5 waveform samples. Idempotent."""
-from __future__ import annotations
-import argparse, logging, sys
-from pathlib import Path
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-from sfm.database import SeismoDb
-from sfm.waveform_store import WaveformStore
-from sfm.shape_metrics import shape_from_h5
-
-log = logging.getLogger("backfill_event_shape")
-
-def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict:
-    counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0}
-    for row in db.query_events(limit=1_000_000):
-        serial, filename = row.get("serial"), row.get("blastware_filename")
-        if not serial or not filename:
-            counts["skipped_no_h5"] += 1; continue
-        h5_path = store.hdf5_path_for(serial, filename)
-        if not h5_path.exists():
-            counts["skipped_no_h5"] += 1; continue
-        shape = shape_from_h5(h5_path)
-        if shape is None:
-            counts["skipped_no_samples"] += 1; continue
-        if not dry_run:
-            with db._connect() as conn:
-                conn.execute(
-                    "UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, "
-                    "shape_sample_count=?, shape_axis=? WHERE id=?",
-                    (shape["crest_factor"], shape["near_peak_count"],
-                     shape["sample_count"], shape["axis"], row["id"]))
-        counts["updated"] += 1
-    log.info("backfill_shape: %s", counts)
-    return counts
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--db-path", required=True)
-    ap.add_argument("--store-root", required=True)
-    ap.add_argument("--dry-run", action="store_true")
-    a = ap.parse_args(argv)
-    logging.basicConfig(level=logging.INFO)
-    counts = backfill_shape(SeismoDb(a.db_path), WaveformStore(a.store_root), dry_run=a.dry_run)
-    print(counts)
-    return 0
-
-if __name__ == "__main__":
-    raise SystemExit(main())
-```
-
-> `store.hdf5_path_for` and `db._connect` are existing internals used the same way by other scripts. If `hdf5_path_for` isn't public, use `store.paths_for(...)` sibling `.h5` path exactly as `save()` derives `hdf5_path`.
-
-- [ ] **Step 4: Run to verify it passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_backfill_event_shape.py -q`
-Expected: PASS.
-
-- [ ] **Step 5: Run the full suite + commit**
-
-```bash
-/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest -q
-git add scripts/backfill_event_shape.py tests/test_backfill_event_shape.py
-git commit -m "feat(scripts): backfill events.shape_* from .h5 samples"
-```
-
----
-
-### Task 7: Confirm /db/events carries shape + version bump
-
-**Files:**
-- Test: `tests/test_db_events_exposes_shape.py`
-- Modify: `pyproject.toml` version; `sfm/server.py` version string; `CHANGELOG.md`.
-
-**Interfaces:**
-- Consumes: the running `/db/events` route (already returns `SELECT *`).
-
-- [ ] **Step 1: Write the failing test** (guards that the feed dict includes the new keys)
-
-```python
-# tests/test_db_events_exposes_shape.py
-from sfm.database import SeismoDb
-def test_query_events_row_includes_shape_keys(tmp_path):
-    db = SeismoDb(tmp_path / "s.db")
-    # query_events returns dict(row); a fresh insert has the keys (values may be None)
-    from tests.helpers_events import make_event
-    db.insert_events([make_event(serial="BE1", key="0111abcd")], serial="BE1")
-    row = db.query_events(serial="BE1")[0]
-    for k in ("shape_crest_factor","shape_near_peak_count","shape_sample_count","shape_axis"):
-        assert k in row
-```
-
-- [ ] **Step 2: Run to verify it fails / passes**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_db_events_exposes_shape.py -q`
-Expected: PASS immediately if Task 3 landed (columns present in `SELECT *`). If it fails, the columns weren't added — fix Task 3. (This task is the guard, not new behavior.)
-
-- [ ] **Step 3: Version bump**
-
-Bump `pyproject.toml` `version` 0.23.0 → 0.24.0; set `sfm/server.py` `version="0.24.0"`; add a `## v0.24.0` CHANGELOG entry ("waveform-shape metrics on events: crest factor + points-near-peak, ingest + backfill").
-
-- [ ] **Step 4: Full suite**
-
-Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest -q`
-Expected: PASS (no regressions).
-
-- [ ] **Step 5: Commit**
-
-```bash
-git add tests/test_db_events_exposes_shape.py pyproject.toml sfm/server.py CHANGELOG.md
-git commit -m "chore(release): v0.24.0 — waveform-shape metrics on events"
-```
-
----
-
-## Self-Review
-
-**Spec coverage:** SFM shape columns (Tasks 3–4) ✓; DSP crest + near-peak (Task 1) ✓; ingest population (Task 5) ✓; backfill (Task 6) ✓; `/db/events` exposure (Task 7) ✓; NULL for histogram/no-sample events (Tasks 1/5/6 return None → NULL) ✓; 94%-coverage / series-4 fallback handled by NULL-then-Terra-View-fallback (Phase B) ✓. Terra-View scoring, 3-state review, twin flagging, export Notes column → **Phase B plan** (separate, depends on this feed). Calibration → Phase C.
-
-**Placeholder scan:** No TBD/TODO; every code step has real code. The two "reuse existing fixture" notes point at concrete existing tests (`test_zc_freq_columns.py`, `test_save_imported_bw*.py`) rather than leaving blanks.
-
-**Type consistency:** `shape_from_h5`/`shape_from_samples`/`channel_shape` return the same dict keys (`crest_factor`, `near_peak_count`, `sample_count`, `axis`) throughout; the DB columns (`shape_crest_factor`, `shape_near_peak_count`, `shape_sample_count`, `shape_axis`) and rec keys match across Tasks 4–6.
-
-## Deferred to Phase B (terra-view, separate plan)
-
-Scoring service combining shape + cheap signals; suspicion column + reason chips; `reviewed_real` mirror + 3-state review; twin-aware flag propagation; Notes-column export + Maximums "(excludes N flagged)". Written once this feed is live so column names/values are real.

docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md

@@ -1,264 +0,0 @@
-# Phase B2-A (seismo-relay) — reviewed_real + 3-state mirror + twin propagation
-
-> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
-
-**Goal:** Give SFM a persisted, queryable `reviewed_real` flag alongside `false_trigger` (mirrored from the sidecar review block, mutually exclusive), and propagate a review to an event's histogram/waveform twin so flagging one flags both.
-
-**Architecture:** Mirror the existing `false_trigger` mechanism. New `events.reviewed_real` column (auto-migrate). `update_event_review` mirrors BOTH flags from the sidecar review block and enforces mutual exclusivity on the columns. A `find_twins` matcher (same serial + identical peak_vector_sum + timestamp within a window) drives `propagate_review_to_twins`, which the sidecar-PATCH endpoint calls after mirroring the primary. Terra-View reads `reviewed_real` from `/db/events` (SELECT *).
-
-**Tech Stack:** Python 3.10, raw sqlite3, FastAPI, pytest. Runner: `/home/serversdown/seismo-relay/.venv/bin/python3`.
-
-## Global Constraints
-
-- 3 review states are **mutually exclusive**: an event is `false_trigger=1` XOR `reviewed_real=1` XOR neither. Setting one true forces the other's column to 0.
-- The sidecar JSON stays the source of truth for full review state; the `false_trigger`/`reviewed_real` columns are derived indexes (like today). B2-A propagates twins at the **column** level (what the feed/peak/export read); twin sidecars are not rewritten (known limitation — noted).
-- Twin match = **same serial AND identical `peak_vector_sum` (exact equality) AND `timestamp` within ± window (default 300 s), excluding the event itself.** Identical PVS is the safety anchor.
-- Known pre-existing test failures (~16, missing gitignored fixtures under `tests/fixtures/`) are unrelated — confirm zero NEW failures, don't try to fix them.
-- Run tests with `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest`.
-
----
-
-### Task 1: `reviewed_real` column on events
-
-**Files:** Modify `sfm/database.py` (`_SCHEMA` CREATE TABLE `events` + the Migration-1 rebuild `CREATE TABLE` + the `_migrate` ADD COLUMN loop). Test: `tests/test_reviewed_real_column.py`.
-
-**Interfaces:** Produces `events.reviewed_real INTEGER NOT NULL DEFAULT 0`.
-
-- [ ] **Step 1: Failing test**
-```python
-# tests/test_reviewed_real_column.py
-import sqlite3
-from sfm.database import SeismoDb
-def _cols(db):
-    with sqlite3.connect(db.db_path) as c:
-        return {r[1] for r in c.execute("PRAGMA table_info(events)")}
-def test_fresh_db_has_reviewed_real(tmp_path):
-    assert "reviewed_real" in _cols(SeismoDb(tmp_path/"s.db"))
-def test_existing_db_migrates_reviewed_real(tmp_path):
-    p = tmp_path/"s.db"; db = SeismoDb(p)
-    with sqlite3.connect(p) as c:
-        c.execute("ALTER TABLE events DROP COLUMN reviewed_real")
-    assert "reviewed_real" not in _cols(db)      # dropped (read via existing handle/connection)
-    SeismoDb(p)                                   # re-open migrates
-    assert "reviewed_real" in _cols(SeismoDb(p))
-```
-> If sqlite < 3.35 lacks DROP COLUMN, fall back to building a table without the column and asserting re-open adds it (same as the shape-columns test did).
-
-- [ ] **Step 2: Run → FAIL** (`pytest tests/test_reviewed_real_column.py -q`).
-- [ ] **Step 3: Implement**
-  - In `_SCHEMA` `events` CREATE TABLE, after the `false_trigger ... DEFAULT 0,` line add: `    reviewed_real           INTEGER NOT NULL DEFAULT 0,  -- 0=no, 1=operator-confirmed real (mutually exclusive with false_trigger)`
-  - In the `_migrate` ADD COLUMN loop tuple add: `("reviewed_real", "INTEGER NOT NULL DEFAULT 0"),`
-  - **Do NOT** add it to the Migration-1 rebuild `CREATE TABLE events (...)` block — that block uses a positional `INSERT ... SELECT * FROM events_old` and, by convention, contains only the columns that existed when Migration 1 was written; every later column is added by the ADD COLUMN loop only. Adding it there crashes `_migrate` on genuinely legacy (pre-Migration-1) DBs.
-- [ ] **Step 4: Run → PASS.**
-- [ ] **Step 5: Commit** `feat(db): reviewed_real column on events (+ auto-migrate)`
-
----
-
-### Task 2: `update_event_review` mirrors both flags + mutual exclusivity
-
-**Files:** Modify `sfm/database.py` `update_event_review`. Test: `tests/test_update_event_review_reviewed_real.py`.
-
-**Interfaces:** Consumes a `review` dict that may carry `false_trigger` and/or `reviewed_real` (bools). Produces mutually-exclusive column state.
-
-- [ ] **Step 1: Failing test**
-```python
-# tests/test_update_event_review_reviewed_real.py
-from sfm.database import SeismoDb
-from minimateplus.models import Event
-def _ins(db, eid_key="01110000", serial="BE1"):
-    ev = Event(index=0); ev._waveform_key = bytes.fromhex(eid_key)
-    db.insert_events([ev], serial=serial)
-    return db.query_events(serial=serial)[0]["id"]
-def test_confirm_real_sets_and_clears_ft(tmp_path):
-    db = SeismoDb(tmp_path/"s.db"); eid = _ins(db)
-    db.update_event_review(eid, {"false_trigger": True})
-    assert db.get_event(eid)["false_trigger"] == 1
-    db.update_event_review(eid, {"reviewed_real": True})   # confirming real clears FT
-    row = db.get_event(eid)
-    assert row["reviewed_real"] == 1 and row["false_trigger"] == 0
-def test_flag_ft_clears_reviewed_real(tmp_path):
-    db = SeismoDb(tmp_path/"s.db"); eid = _ins(db)
-    db.update_event_review(eid, {"reviewed_real": True})
-    db.update_event_review(eid, {"false_trigger": True})
-    row = db.get_event(eid)
-    assert row["false_trigger"] == 1 and row["reviewed_real"] == 0
-```
-> Build the Event inline like `tests/test_zc_freq_columns.py` if the import differs.
-
-- [ ] **Step 2: Run → FAIL.**
-- [ ] **Step 3: Implement** — replace the body of `update_event_review` so it handles both keys:
-```python
-    if not isinstance(review, dict):
-        return False
-    has_ft = "false_trigger" in review
-    has_real = "reviewed_real" in review
-    if not has_ft and not has_real:
-        with self._connect() as conn:
-            row = conn.execute("SELECT 1 FROM events WHERE id=?", (event_id,)).fetchone()
-        return row is not None
-    sets = {}
-    if has_ft:
-        sets["false_trigger"] = 1 if review.get("false_trigger") else 0
-    if has_real:
-        sets["reviewed_real"] = 1 if review.get("reviewed_real") else 0
-    # mutual exclusivity: a true in one forces the other column to 0
-    if sets.get("false_trigger") == 1:
-        sets["reviewed_real"] = 0
-    if sets.get("reviewed_real") == 1:
-        sets["false_trigger"] = 0
-    assign = ", ".join(f"{k}=?" for k in sets)
-    params = list(sets.values()) + [event_id]
-    with self._connect() as conn:
-        cur = conn.execute(f"UPDATE events SET {assign} WHERE id=?", params)
-    return cur.rowcount > 0
-```
-- [ ] **Step 4: Run → PASS** (+ run `tests/test_*false_trigger*`/existing review tests to confirm no regression).
-- [ ] **Step 5: Commit** `feat(db): update_event_review mirrors reviewed_real + enforces 3-state exclusivity`
-
----
-
-### Task 3: `find_twins` matcher
-
-**Files:** Modify `sfm/database.py` (add `find_twins`). Test: `tests/test_find_twins.py`.
-
-**Interfaces:** Produces `find_twins(event_id, *, window_seconds=300) -> list[dict]` — same serial, identical peak_vector_sum, timestamp within ±window, excluding self.
-
-- [ ] **Step 1: Failing test**
-```python
-# tests/test_find_twins.py
-import datetime
-from sfm.database import SeismoDb
-from minimateplus.models import Event, Timestamp
-def _ins(db, key, serial, pvs, ts):
-    ev = Event(index=0); ev._waveform_key = bytes.fromhex(key)
-    ev.timestamp = ts
-    # peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly
-    db.insert_events([ev], serial=serial)
-    row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0]
-    import sqlite3
-    with sqlite3.connect(db.db_path) as c:
-        c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"]))
-    return row["id"]
-def test_find_twins_matches_same_serial_pvs_near_time(tmp_path):
-    db = SeismoDb(tmp_path/"s.db")
-    base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5)
-    twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45)
-    far  = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=21, minute=0,  second=0)
-    a = _ins(db, "01110001", "BE1", 0.4763, base)
-    b = _ins(db, "01110002", "BE1", 0.4763, twin)   # twin: same serial+pvs, 40s apart
-    c = _ins(db, "01110003", "BE1", 0.4763, far)    # same pvs but >window away
-    d = _ins(db, "01110004", "BE1", 0.9999, twin)   # near time but different pvs
-    ids = {r["id"] for r in db.find_twins(a, window_seconds=300)}
-    assert ids == {b}
-```
-> Adjust the Event/Timestamp construction to match how `tests/test_waveform_store.py::_make_synthetic_event` builds them if fields differ.
-
-- [ ] **Step 2: Run → FAIL.**
-- [ ] **Step 3: Implement**
-```python
-    def find_twins(self, event_id: str, *, window_seconds: int = 300) -> list[dict]:
-        row = self.get_event(event_id)
-        if not row:
-            return []
-        serial = row.get("serial"); pvs = row.get("peak_vector_sum"); ts = row.get("timestamp")
-        if serial is None or pvs is None or not ts:
-            return []
-        try:
-            t = datetime.datetime.fromisoformat(ts.replace(" ", "T"))
-        except ValueError:
-            return []
-        lo = (t - datetime.timedelta(seconds=window_seconds)).isoformat()
-        hi = (t + datetime.timedelta(seconds=window_seconds)).isoformat()
-        with self._connect() as conn:
-            rows = conn.execute(
-                "SELECT * FROM events WHERE serial=? AND id!=? AND peak_vector_sum=? "
-                "AND timestamp BETWEEN ? AND ?",
-                (serial, event_id, pvs, lo, hi),
-            ).fetchall()
-        return [dict(r) for r in rows]
-```
-- [ ] **Step 4: Run → PASS.**
-- [ ] **Step 5: Commit** `feat(db): find_twins (serial + identical PVS + time window)`
-
----
-
-### Task 4: propagate a review to twins + wire into the sidecar-PATCH endpoint
-
-**Files:** Modify `sfm/database.py` (add `propagate_review_to_twins`); `sfm/server.py` (`db_event_sidecar_patch`). Test: `tests/test_twin_propagation.py`.
-
-**Interfaces:** `propagate_review_to_twins(event_id, *, window_seconds=300) -> list[str]` copies the event's `false_trigger`/`reviewed_real` columns onto each twin; returns twin ids.
-
-- [ ] **Step 1: Failing test** (DB-level)
-```python
-# tests/test_twin_propagation.py — reuse the _ins helper pattern from test_find_twins
-def test_propagate_copies_flags_to_twins(tmp_path):
-    db = SeismoDb(tmp_path/"s.db")
-    # (build primary + twin via the _ins helper as in test_find_twins)
-    # flag the primary FT, then propagate:
-    db.update_event_review(primary_id, {"false_trigger": True})
-    moved = db.propagate_review_to_twins(primary_id)
-    assert twin_id in moved
-    assert db.get_event(twin_id)["false_trigger"] == 1
-```
-- [ ] **Step 2: Run → FAIL.**
-- [ ] **Step 3: Implement**
-  - `sfm/database.py`:
-```python
-    def propagate_review_to_twins(self, event_id: str, *, window_seconds: int = 300) -> list[str]:
-        row = self.get_event(event_id)
-        if not row:
-            return []
-        ft = 1 if row.get("false_trigger") else 0
-        real = 1 if row.get("reviewed_real") else 0
-        twins = self.find_twins(event_id, window_seconds=window_seconds)
-        moved = []
-        with self._connect() as conn:
-            for tw in twins:
-                conn.execute("UPDATE events SET false_trigger=?, reviewed_real=? WHERE id=?",
-                             (ft, real, tw["id"]))
-                moved.append(tw["id"])
-        return moved
-```
-  - `sfm/server.py` `db_event_sidecar_patch`: after the existing `_get_db().update_event_review(event_id, new_sidecar.get("review", {}))`, add:
-```python
-        # Propagate the review to the event's histogram/waveform twin(s) so
-        # flagging one flags both (column-level; twins share serial+PVS+near time).
-        try:
-            _get_db().propagate_review_to_twins(event_id)
-        except Exception as exc:
-            log.warning("twin review-propagation failed for %s: %s", event_id, exc)
-```
-  (Guard the `if body.review is not None:` block so propagation only runs when review changed.)
-- [ ] **Step 4: Run → PASS.**
-- [ ] **Step 5: Commit** `feat(review): propagate false_trigger/reviewed_real to twins on sidecar PATCH`
-
----
-
-### Task 5: expose in feed guard + version bump
-
-**Files:** Test `tests/test_reviewed_real_in_feed.py`; `pyproject.toml`, `sfm/server.py` version, `CHANGELOG.md`.
-
-- [ ] **Step 1: Guard test** — a `query_events` row dict includes `reviewed_real` (SELECT * returns it).
-```python
-from sfm.database import SeismoDb
-from minimateplus.models import Event
-def test_query_events_includes_reviewed_real(tmp_path):
-    db = SeismoDb(tmp_path/"s.db")
-    ev = Event(index=0); ev._waveform_key = bytes.fromhex("01110000")
-    db.insert_events([ev], serial="BE1")
-    assert "reviewed_real" in db.query_events(serial="BE1")[0]
-```
-- [ ] **Step 2: Run → PASS** (columns already present from Task 1).
-- [ ] **Step 3: Bump** `pyproject.toml` 0.24.0 → 0.25.0; `sfm/server.py` version="0.25.0"; add `## v0.25.0` CHANGELOG entry ("reviewed_real 3-state review flag + histogram/waveform twin review-propagation").
-- [ ] **Step 4: Full suite** — confirm zero NEW failures beyond the ~16 pre-existing.
-- [ ] **Step 5: Commit** `chore(release): v0.25.0 — reviewed_real + twin review-propagation`
-
----
-
-## Self-Review
-
-**Spec coverage:** reviewed_real column (T1) ✓; mirror + mutual exclusivity (T2) ✓; twin match serial+identical-PVS+window (T3) ✓; twin propagation wired into the review path (T4) ✓; feed exposure + version (T5) ✓. Twin **sidecar** rewrite intentionally deferred (column-level only — documented in Global Constraints); this is the SFM half — the 3-state UI + confirm-real PATCH is the separate **B2-B (terra-view)** plan.
-
-**Placeholder scan:** none — every step has real code (the two "adjust Event construction to match test_waveform_store" notes point at a concrete existing helper).
-
-**Type consistency:** `false_trigger`/`reviewed_real` INTEGER columns used identically across T1/T2/T4; `find_twins`→`propagate_review_to_twins` both key on the same match; server calls the DB methods by the exact names T2–T4 define.

docs/superpowers/plans/2026-09-17-rescue-listener.md

@@ -1,134 +0,0 @@
-# Plan — "Rescue Listener": a first-class tool for the inverted rescue
-
-**Status:** proposal, not started.  Written 2026-09-17 ~01:40 local, straight
-off the BE12599 incident.  Open questions at the bottom need Brian's answer
-before anything is built.
-
-**Background:** `docs/runbooks/wedged_unit_recovery.md`, "Second incident —
-BE12599".  The manual version of this worked; this plan is about making it a
-tool instead of a sequence of remembered steps at 1 AM.
-
----
-
-## The problem, stated plainly
-
-When a unit is wedged in the BE12599 mode — geophone offset above trigger,
-recording back-to-back, ACH dialing constantly, device stuck repeating an AT
-modem-init string and therefore **deaf to S3 over inbound** — the only channel
-that works is the one the *device* opens.
-
-Recovering it currently means:
-
-1. Remember that `bridges/ach_server.py` exists and takes the right flags
-2. Start it by hand on a box the modem can reach, with a public port forwarded
-3. Go into ACEmanager and repoint the modem's Destination
-4. Watch a terminal for a call-in
-5. Read `rescue.json` to find out whether it worked
-6. Go back into ACEmanager and repoint the modem to where it belongs
-7. **Not forget step 6**, because leaving the Destination pointed at a dead
-   listener is worse than never having started
-
-That is six manual steps and one landmine, executed under pressure while a
-unit floods the office server.
-
-## What the tool should be
-
-**A "rescue listener" an operator can start for one unit, which handles
-whatever that unit says when it calls in, and refuses to go away until the
-operator confirms the modem has been pointed back.**
-
-Lifecycle:
-
-1. **Start** — operator names the target unit and starts a rescue listener.
-   The tool reports the exact address/port to enter in ACEmanager, plus the
-   actions it will take.
-2. **Operator repoints the modem** to that address.
-3. **Wait** — listener sits there.  Live status: "waiting for call-in",
-   elapsed, last-seen.
-4. **Act** — on call-in, run the configured rescue actions automatically,
-   in a safe order, each independently guarded.  Report per-action outcome.
-5. **Hold** — the listener **stays up** and keeps reporting, because the
-   modem is still pointed at it.
-6. **Confirm & stop** — the operator explicitly confirms the Destination has
-   been restored (to `0.0.0.0`, or to the office Instantel ACH server).
-   Only then does the listener shut down.
-
-Step 6 is the whole point of making this a tool.  It is the step that is
-easiest to skip and most expensive to skip.
-
-## Default action set
-
-Ordered deliberately — see "order matters" below.
-
-| # | Action | Default | Why |
-|---|---|---|---|
-| 1 | **Stop monitoring** (SUB 0x97) | ✅ on | Halts recording; ends the trigger→record→dial loop at its source.  Already implemented as `--stop-monitoring`. |
-| 2 | **Drain events** to a diagnostics store | ⚙ configurable | The backlog is usually evidence, not garbage — see the BE12599 offset investigation.  Must NOT land in the prod SFM DB. |
-| 3 | **Disable ACH** (SUB 0x2C/0x7E/0x7F) | ❌ off by default | Stops the dialing — **and stops your only channel**.  Opt-in, and ideally gated on step 1 having succeeded. |
-| 4 | **Erase events** | ❌ off by default | Destructive.  Only after a verified drain. |
-
-### Order matters — the lesson from BE12599
-
-Stopping monitoring *removes the call-in trigger*.  ACH fires on "after event
-recorded"; with recording stopped, the unit has no reason to dial again, even
-though the backlog is still sitting in its memory.  So a naive
-"stop + disable + erase, all at once" rescue can silence the unit before
-you've collected anything, leaving you with no channel and a device full of
-evidence.
-
-The tool should either sequence around this or warn loudly about it.  My
-instinct is: **stop monitoring immediately** (it's the bleeding), then drain
-across however many call-ins it takes, and treat disable-ACH/erase as a
-separate, explicit "finish" action once the operator is satisfied.
-
-## Where it should live — open question, with a proposal
-
-The natural tier is **SFM** (device-side, per the three-tier model in
-CLAUDE.md).  But the rescue listener must be reachable *from the cellular
-network*, which is a deployment constraint SFM's usual profile doesn't have.
-
-**Proposal worth considering:** run it at the office, beside the real Instantel
-ACH server, on a **different port** (e.g. 12346 while Instantel holds 12345).
-Then the ACEmanager change is a **port change, not an IP change** — smaller,
-faster, less to get wrong, and trivially reversible.  It also means the office
-public IP (already stable and known) is the destination, rather than whatever
-Brian's dynamic home IP happens to be that week.
-
-The tmi-dev approach used on BE12599 worked, but required a router forward and
-ran into the dynamic-IP problem in the same session.
-
-## Open questions
-
-1. **Where does it run?**  Office beside Instantel ACH (port swap), SFM on the
-   NAS, or ad-hoc on tmi-dev?  Affects everything else.
-2. **What drives it?**  Terra-View admin page (fits "operator UI"), an SFM
-   endpoint pair (`POST /device/rescue_listener/start` + `/stop` + `/status`),
-   or a CLI wrapper?  A long-lived listener doesn't fit the request/response
-   endpoint shape well — probably needs a background task with a status poll.
-3. **How does it identify the unit?**  It can't know the serial until the
-   device calls in and the handshake reads it.  Allowlist by modem IP?  Accept
-   anything and report what showed up?
-4. **Where do drained events go?**  A per-incident diagnostics store
-   (`bridges/captures/<unit>-diag`) seems right — explicitly *not* the prod
-   SFM DB.  Does that store need to be a first-class thing with its own
-   retention, or is a directory fine?
-5. **How is "confirm the modem is repointed" verified?**  Operator attestation
-   (a button), or can we actually probe it?  If the listener stops seeing
-   call-ins that's weak evidence; if inbound to the unit starts working that's
-   stronger.
-6. **Multi-unit?**  One listener per incident, or one listener that handles any
-   unit that dials in?  Probably the former for safety.
-7. **Timeout / abandonment policy.**  If nobody ever confirms, does it run
-   forever?  Alert after N hours?
-
-## What already exists
-
-- `bridges/ach_server.py` — the listener itself, with `--stop-monitoring`,
-  `--disable-ach`, `--rescue` (added on `feat/ach-rescue-on-connect`, commit
-  `9f1050b`), `--clear-after-download`, `--max-events`, `--allow-ip`.
-- Per-session `rescue.json` recording per-action outcomes.
-- Isolated per-output-dir SQLite + waveform store, so a diagnostics capture is
-  already separate from prod by construction.
-
-So the gap is not protocol work — it's lifecycle, operator surface, and the
-confirmation gate.  Most of the risk is in questions 1 and 2.

docs/waveform_codec_re_status.md

@@ -1,350 +0,0 @@
-> ## SUPERSEDED 2026-08-25 — the body is a RECORD CHAIN
->
-> The tag-dispatch model described in this document — `40 NN` segment headers,
-> tagless headers, channel rotation — is **wrong**.  It produced nearly-correct
-> output only because the block table happens to tile the data sections.
->
-> The body is a chain of self-delimiting per-channel records: `off+2` is a
-> uint16 BE length, `next = off + 2 + len`, and the chain ends on a record whose
-> chan_id is `0x06`.  A 3-valued mode enum at `off+8` selects delta / absolute /
-> raw-12-bit semantics.  `40 NN` is an ordinary int16 BE data block.
->
-> See the record-chain section of `docs/instantel_protocol_reference.md` §7.6.1
-> and the implementation in `minimateplus/waveform_codec.py`.
->
-> Result: all four channels equal length in 1388/1388 files (was 156/1388);
-> ASCII sample-count exact 75/75, fully exact 73/75; device PPV 1306/1306.
->
-> This document is retained as the reasoning trail.
-
-# Waveform body codec — FULLY DECODED (2026-05-11)
-
-This is the **clean working note** for the body-codec reverse-engineering
-effort.  It supersedes scattered claims elsewhere when they conflict.
-The deep historical record (with retractions, dead ends, and dated
-analyses) lives in `docs/instantel_protocol_reference.md §7.6.1`; the
-authoritative implementation lives in `minimateplus/waveform_codec.py`.
-
-## TL;DR
-
-**The codec is fully decoded.**  Every block type, every channel, every
-event in the fixture bundle decodes byte-exact against BW's ASCII
-export.
-
-| Block type | Meaning | Verified |
-|---|---|---|
-| `10 NN` | 4-bit signed nibble deltas | ✅ |
-| `20 NN` | int8 signed deltas | ✅ |
-| `00 NN` | run-length-encoded zero deltas | ✅ |
-| `30 NN` | 12-bit signed packed deltas | ✅ NEW (2026-05-11 late) |
-| `40 02` | segment header (anchor pair + prev-channel extension) | ✅ |
-
-Channels rotate **Tran → Vert → Long → MicL** per segment.  Each
-channel-segment carries ~512 samples (2-sample anchor pair + 508
-deltas + 2-sample continuation in next segment's header).
-
-## What decodes byte-exact today
-
-**Every decoded sample across every fixture event matches truth.  Zero
-divergences.**
-
-| Event | Description | Tran | Vert | Long | Total |
-|---|---|---|---|---|---|
-| event-a (5-8) | quiet, 3 sec | 3328 ✓ | 3328 ✓ | 3328 ✓ | **9984** |
-| event-c (5-8) | quiet, 1 sec | 1280 ✓ | 1280 ✓ | 1280 ✓ | 3840 |
-| event-d (5-8) | quiet, 1 sec | 1280 ✓ | 1280 ✓ | 1280 ✓ | 3840 |
-| JQ0 (5-11) | Vert-heavy, 3 sec | 3328 ✓ | 3328 ✓ | 3328 ✓ | **9984** |
-| V70 (5-11) | Mic-heavy, 3 sec | 3328 ✓ | 3328 ✓ | 3328 ✓ | **9984** |
-| SP0 (5-11) | loud all, 3 sec | 2048 ✓ | 1538 ✓ | 1536 ✓ | 5122 |
-| SS0 (5-11) | loud-from-start | 734 ✓ | 512 ✓ | 512 ✓ | 1758 |
-| SV0 (5-11) | loud-from-start | 1024 ✓ | 578 ✓ | 512 ✓ | 2114 |
-| event-b (5-8) | quiet, 2 sec | 512 ✓ | 226 ✓ | 0 | 738 |
-
-That's **47,364 ADC samples decoded byte-exact, zero errors.**
-
-Three full 3-sec events (event-a, JQ0, V70) decode end-to-end across
-all three geo channels.
-
-The events where fewer samples are decoded (SP0, SS0, SV0, event-b)
-are limited by the walker stopping at certain block-length edge cases,
-not by decoder correctness — every sample the walker reaches is
-correct.
-
-## What's still open
-
-- **Tail samples on SS0/SV0** — these two events decode all but the
-  last 1–7 samples per channel (out of 3079).  Likely the same
-  "last segment is truncated" pattern.  Minor; doesn't affect the
-  bulk of the data.
-
-## Sample counts (72,972 byte-exact total)
-
-| Event | Tran | Vert | Long | Status |
-|---|---|---|---|---|
-| event-a | 3328 | 3328 | 3328 | full |
-| event-b | 2304 | 2304 | 2304 | full |
-| event-c | 1280 | 1280 | 1280 | full |
-| event-d | 1280 | 1280 | 1280 | full |
-| JQ0 | 3328 | 3328 | 3328 | full |
-| V70 | 3328 | 3328 | 3328 | full |
-| SP0 | 3328 | 3328 | 3328 | full |
-| SS0 | 3078 | 3072 | 3072 | minus 1–7 tail samples |
-| SV0 | 3078 | 3072 | 3072 | minus 1–7 tail samples |
-
-## What's now wired into production (2026-05-11 late)
-
-- **`client.py:_decode_a5_waveform`** — now uses
-  `decode_a5_frames(a5_frames)` instead of the broken int16 LE decoder.
-  `event.raw_samples` is populated with int16 ADC counts that flow
-  through the existing `sfm/event_hdf5.py` scaling pipeline unchanged.
-  Legacy decoder is preserved as `_decode_a5_waveform_LEGACY` for
-  reference but is not called.
-
-- **MicL → dB(L) conversion** — exposed as
-  `waveform_codec.mic_count_to_db(count)`.  Verified against BW
-  display values (count=1 → 81.94 dB; count=813 → 140.14 dB; matches
-  the V70 mic-heavy fixture exactly).
-
-- **`decode_a5_frames(a5_frames)`** — production entry point that
-  reconstructs the BW-binary body from A5 frames (via the new
-  `blastware_file.extract_body_bytes` helper) and runs the verified
-  codec.  Returns the same `raw_samples` dict shape the consumers
-  already expect.
-
-## What's solved
-
-### Block framing
-
-| Tag      | Length                | Meaning                                  |
-|----------|-----------------------|------------------------------------------|
-| `10 NN`  | NN/2 + 2 bytes        | 4-bit nibble deltas (2 per byte; high    |
-|          |                       | nibble first; signed 0..7 / 8..F = -8..-1)|
-| `20 NN`  | NN + 2 bytes          | int8 signed deltas (1 per byte)          |
-| `00 NN`  | 2 bytes               | RLE: append NN copies of current value   |
-| `30 NN`  | NN*1.5 + 2 in data    | 12-bit signed deltas (see below).        |
-|          | section, NN*4 trailer |                                          |
-| `40 NN`  | 2*NN + 16 bytes       | Segment header (NN = prev-channel deltas)|
-
-NN is always a multiple of 4.
-
-**Wide-NN forms.**  `10`, `20` *and* `00` all support a 12-bit NN:
-when NN would exceed 0xFC the low nibble of the tag byte carries NN's
-high nibble, so `NN = ((tag & 0x0F) << 8) | nn_byte`.  Confirmed for
-`1X`/`2X` in 2026-05-11 and for `0X` (RLE) in 2026-08-25 — e.g.
-`01 0c` = a 268-sample zero-delta run.
-
-**`40 NN` is variable width.**  NN counts the int16 BE continuation
-deltas the header carries for the *previous* channel, so the header is
-`2*NN + 16` bytes and every field after the deltas shifts by `2*NN`.
-`40 01` (18 B) and `40 03` (22 B) both occur alongside the common
-`40 02` (20 B).  Confirmed 2026-08-25.
-
-Implementation: `walk_body()` in `minimateplus/waveform_codec.py`.
-
-### 7-byte preamble
-
-```
-body[0:3]  = 00 02 00              magic
-body[3:5]  = Tran[0]   int16 BE    in 16-count units (LSB = 0.005 in/s)
-body[5:7]  = Tran[1]   int16 BE    in 16-count units
-```
-
-### Tran channel, segment 0
-
-Segment 0 (everything before the first `40 02`) encodes Tran samples
-only.  Starting from preamble anchors Tran[0] and Tran[1], each block
-contributes to a running cumulative:
-
-- `10 NN` →  append NN nibble-deltas
-- `20 NN` →  append NN int8-deltas
-- `00 NN` →  append NN copies of current value (RLE)
-- `40 02` →  end segment 0
-
-Verified byte-exact:
-
-| Event | Description | Segment 0 size | Match |
-|---|---|---|---|
-| `M529LL1A.SP0` | Loud, 0.25 s pretrig | 510 | 510/510 ✓ |
-| `M529LL1A.SV0` | Loud from sample 0 | 58 | 58/58 ✓ (stops at first `30 NN`) |
-| `M529LL1A.SS0` | Loud from sample 0 | 42 | 42/42 ✓ (stops at first `30 04`) |
-| `M529LL1L.JQ0` | Vert-heavy | 510 | 510/510 ✓ |
-| `M529LL1L.V70` | Mic-heavy (140 dB) | 510 | 510/510 ✓ |
-
-Implementation: `decode_tran_initial()`.
-
-### Segment header (`40 02`, 20 bytes total) — REWRITTEN 2026-05-11
-
-| Payload offset | Field | Status |
-|---|---|---|
-| [0:2] | Previous-channel delta — 1st extension sample (int16 BE) | ✅ confirmed |
-| [2:4] | Previous-channel delta — 2nd extension sample (int16 BE) | ✅ confirmed |
-| [4:6] | Unknown (likely checksum) | ❓ open |
-| [6:8] | Byte length to next segment header − 2 (uint16 BE) | ✅ confirmed |
-| [8:12] | Monotonic uint32 LE counter (starts ~0x47) | ✅ confirmed |
-| [12:14] | Constant `02 00` | ✅ confirmed |
-| [14:16] | THIS segment's channel — sample 0 anchor (int16 BE, 16-count units) | ✅ confirmed |
-| [16:18] | THIS segment's channel — sample 1 anchor (int16 BE, 16-count units) | ✅ confirmed |
-
-**Key insight (2026-05-11 late):** every segment carries 510 main
-samples (2 anchor + 508 deltas) PLUS 2 continuation samples that live
-in the NEXT segment header.  So each channel-segment effectively spans
-512 sample-sets.  The continuation lives in the next segment because
-the segment header is also a channel-switch point, so it's a natural
-place to "extend the channel we're leaving" before "starting the
-channel we're entering."
-
-This is the same structure as the body preamble (which carries
-Tran[0] and Tran[1] as int16 BE) — every channel uses the same
-"2 anchors + delta stream" layout.
-
-## Channel rotation — VERIFIED 2026-05-11
-
-```
-(initial body)  →  Tran samples 0..509       (preamble + delta blocks)
-segment 0 hdr  ext+anchor →  Vert samples 0..511   ← anchor in hdr [14:18]
-segment 1 hdr  ext+anchor →  Long samples 0..511
-segment 2 hdr  ext+anchor →  Mic  samples 0..511
-segment 3 hdr  ext+anchor →  Tran samples 510..1021 (continuation)
-segment 4 hdr  ext+anchor →  Vert samples 512..1023
-segment 5 hdr  ext+anchor →  Long samples 512..1023
-segment 6 hdr  ext+anchor →  Mic  samples 512..1023
-segment 7 hdr  ext+anchor →  Tran samples 1022..1533
-...
-```
-
-Implementation: `decode_waveform_v2()` returns
-`{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}` with
-each channel's samples in 16-count units.  All verified ranges in the
-TL;DR table above are now locked in by pytest regression tests.
-
-## What's still open
-
-1. **`30 NN` block content.**  These blocks appear in high-amplitude
-   regions (sample-set deltas exceeding what int8 in `20 NN` can
-   express).  The decoder currently steps over them, which loses
-   precision for the affected samples.  Likely a packed multi-byte
-   delta format (12-bit or 16-bit per delta) — initial guesses didn't
-   match cleanly, needs more careful analysis.
-
-2. **MicL decoding.**  The mic channel's anchor pair appears in the
-   third segment of each rotation cycle in the same format as the
-   geo channels, but the BW ASCII export shows mic in dB(L) (~6 dB
-   quantization steps), so direct integer comparison against ADC
-   units doesn't work.  Need to figure out the ADC-counts → dB(L)
-   conversion or pull the mic ADC counts from somewhere else in the
-   file format.
-
-3. **Walker fix for event-b.**  The original quiet bundle's event-b
-   still bails out partway through.  Lower priority since the other
-   7 events walk cleanly.
-
-4. **Variable-prefix segment descriptors** (found 2026-08-25).
-   3 of 75 ground-truth production events still truncate.  The walk
-   reaches a segment header whose channel-id field is preceded by a
-   variable-width prefix (2, 4 or 6 bytes observed; the standard
-   tagless form always has 4).  These also carry an `01 00` marker
-   instead of `02 00`.  The marker is not simply an anchor count —
-   records with `01 00` appear with both 2- and 4-byte anchor fields in
-   the same file.  Examples: `BE12599/N599LPNB.JF0W` @1155,
-   `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485.
-
-## Segment header: channel id and tagless form — 2026-08-25
-
-The 4-byte field previously read as a "monotonic uint32 LE counter" is
-`[channel_id][00][00][segment_index]`, with `0x46`=Tran `0x47`=Vert
-`0x48`=Long `0x49`=MicL.  Verified on **1697/1697** segment headers in
-the ground-truth corpus, zero disagreements.  `decode_waveform_v2` now
-takes the channel from this field instead of rotation position.
-
-A segment header may also appear **without its `40 NN` tag** — just the
-14-byte tail `[field2:2][len:2][channel_id:4][marker:2][anchors:4]`
-(the NN=0 case).  `is_tagless_segment_header()` detects it from the six
-bytes at `[4:10]`.
-
-## Geo scale: full scale is 32000 counts — 2026-08-25
-
-One decoder unit (16 ADC counts) is exactly 0.005 in/s, so Normal range
-(10.000 in/s) is `10.0 / (0.005/16)` = **32000** ADC counts.  Consumers
-that divided by 32768 read every geophone sample 2.34% low.  Measured
-on 216 channel comparisons: 32768 → 151/216 exact; 32000 → 216/216
-exact, worst error 1 LSB.
-
-**Scope — not waveform-specific.**  The scale is applied where ADC
-counts become physical units, which every event passes through
-regardless of source codec:
-
-| source | median ratio ours/device, 32768 | with 32000 |
-|---|---|---|
-| series-3 waveform (vs ASCII sample table) | 0.9766 | **1.0000** |
-| series-3 histogram (vs ASCII PPV, n=1137) | 0.9766 | **1.0000** |
-| series-4 Thor IDF (vs device peak, n=1468) | 0.960 | **0.983** |
-
-The four block-framing fixes are waveform-only — `histogram_codec` is
-untouched by them.
-
-## Ground-truth corpus (2026-08-25)
-
-Beyond the bundled fixtures, the production waveform store keeps each
-event's original Blastware ASCII export at
-`<store>/<serial>/<filename>_ASCII.TXT`.  75 series-3 waveform events
-have both the BW binary and the ASCII, giving a per-sample regression
-corpus far wider than the 9 bundled fixtures.  Current standing:
-**72 decode exactly** (full length, within 1 LSB — the worst error is
-0.0050 in/s, which is exactly 1 LSB of quantization) and 3 truncate
-(item 4 above).  Zero events have full-length value errors.
-
-## `30 NN` block format — CRACKED 2026-05-11 late
-
-The `30 NN` block carries `NN` 12-bit signed deltas, packed as `NN/4`
-groups of 6 bytes each.  Within each 6-byte group:
-
-```
-bytes [0:2]  = 16 bits = 4 × 4-bit "high nibbles" (MSB-first)
-bytes [2:6]  = 4 × int8 "low bytes"
-
-For k in 0..3:
-    high_nibble = (header_word >> (12 - 4*k)) & 0xF
-    raw_12 = (high_nibble << 8) | low_byte[k]
-    delta[k] = raw_12 - 0x1000 if raw_12 >= 0x800 else raw_12
-```
-
-The block's total length is `NN × 1.5 + 2` bytes (tag included).  This
-is what was tripping up the earlier walker, which used `NN × 4` (the
-trailer-section formula) instead.
-
-Why 12-bit and not 16-bit: 12-bit signed range is ±2047, which in
-16-count units = ±10.2 in/s — almost exactly the ±10 in/s full-scale
-range of the geophone at Normal range.  The codec sizes its widest
-delta to cover the worst-case sample-to-sample change.
-
-Verified against all 14 `30 NN` blocks across the bundled fixture
-events.  Every delta decodes byte-exact against BW's ASCII export.
-
-## Test fixtures
-
-Committed under `tests/fixtures/`:
-
-- `decode-re-5-8-26/event-a..event-d/`: original quiet bundle (4 events,
-  PPV < 1 in/s).  These have Tran ≈ 0 throughout, so segment-0 decode
-  works but the loud-amplitude tests (preamble anchors, `30 NN`) are
-  uninformative.
-- `5-11-26/M529LL1A.{SP0,SS0,SV0}`: loud bundle (PPV 6-7 in/s on all
-  channels).  These cracked the Tran codec.
-- `5-11-26/M529LL1L.{JQ0,V70}`: targeted captures.  JQ0 is Vert-heavy,
-  V70 is Mic-heavy (140 dB).  These cracked the `00 NN` RLE rule.
-
-Each fixture has a `.TXT` Blastware ASCII export as ground truth.
-
-## Tests
-
-`tests/test_waveform_codec.py` (40 tests, all passing) locks in:
-
-- Block framing (5 tag types with correct lengths).
-- Walker contiguity (no gaps or overlaps).
-- Segment header parsing (counter monotonicity, fixed-pattern check).
-- `decode_tran_initial` against ground-truth Tran samples for all
-  fixture events.
-
-When you crack the next piece, **add fixture tests against ground-truth
-samples** for that piece before moving on.  Don't let unverified code
-ship without a regression lock-in.

micromate/__init__.py

@@ -1,48 +0,0 @@
-"""
-micromate — Instantel Micromate (Series IV) device library.
-
-Sibling of ``minimateplus`` (the Series III library).  Currently scoped to
-the offline-file ingest path used by thor-watcher: parsing the per-event
-``.IDFH``/``.IDFW`` ASCII text sidecars Thor's exporter writes alongside
-each binary event file, and wrapping the parsed data in typed event
-records.
-
-Live-device support (TCP protocol, frame parsing, real-time monitoring)
-is deferred — when we add it, it lands here as ``transport.py`` /
-``framing.py`` / ``protocol.py`` / ``client.py``, mirroring the
-``minimateplus`` package layout.
-
-Typical usage (offline file ingest):
-
-    from micromate import IdfEvent, parse_idf_report
-
-    text  = open("UM11719_20231219162723.IDFW.txt").read()
-    rep   = parse_idf_report(text)                       # dict
-    event = IdfEvent.from_report(rep, "UM11719_20231219162723.IDFW")
-    print(event.serial, event.peaks.transverse_ips, event.mic_pspl_dbl)
-"""
-
-from .idf_ascii_report import (
-    parse_event_filename,
-    parse_idf_report,
-    serial_from_filename,
-)
-from .models import (
-    IdfEvent,
-    IdfPeaks,
-    IdfProjectInfo,
-    IdfReport,
-    IdfSensorCheck,
-)
-
-__version__ = "0.1.0"
-__all__ = [
-    "IdfEvent",
-    "IdfPeaks",
-    "IdfProjectInfo",
-    "IdfReport",
-    "IdfSensorCheck",
-    "parse_event_filename",
-    "parse_idf_report",
-    "serial_from_filename",
-]

micromate/idf_ascii_report.py

@@ -1,330 +0,0 @@
-"""
-micromate/idf_ascii_report.py — parse Thor (Micromate Series IV) IDF ASCII reports.
-
-Thor exports a `.IDFW.txt` or `.IDFH.txt` sidecar next to each `.IDFW`
-(waveform) or `.IDFH` (histogram) event binary.  Each sidecar is a
-plain-text file with `"Key : Value"` lines covering the full device-
-authoritative event metadata — PPV per channel, ZC Freq, Time of Peak,
-Peak Acceleration / Displacement, sensor self-check results, project
-strings, calibration date, battery level, etc. — followed by a raw
-waveform-samples block headed by the literal line "Waveform Data Channels".
-
-This is the Thor analogue of `minimateplus/bw_ascii_report.py` for the
-Blastware (Series III) report format.  The parser is intentionally
-permissive: we extract everything we recognise into a flat dict and
-silently ignore anything we don't.  Downstream callers parse units
-(`"0.2119 in/s"` → 0.2119) only on the fields they need.
-
-Example input (truncated):
-
-    "EventType : Full Waveform"
-    "SampleRate : 1024 sps"
-    "EventTime : 16:27:23"
-    "EventDate : 2023-12-19"
-    "TranPPV : 0.0251 in/s"
-    "VertPPV : 0.2119 in/s"
-    "LongPPV : 0.0282 in/s"
-    "PeakVectorSum : 0.2131 in/s"
-    "MicPSPL : 99.4 dB(L)"
-    "TranZCFreq : 6.5 Hz"
-    "SerialNumber : UM11719"
-    "Version : Micromate ISEE 11.0AK"
-    "FileName : UM11719_20231219162723.IDFW"
-    "BatteryLevel : 3.8 volts"
-    "Calibration : November 22, 2023 by Instantel"
-    "TranTestResults : Passed"
-    "TitleString1 : UPMC Presby-Loc 3-Level1-1R Elevator Rm"
-    Waveform Data Channels
-        Tran    Vert    Long    MicL
-        0.0003  -0.0003  0.0003  0.00013
-        ...
-"""
-
-from __future__ import annotations
-
-import datetime
-import re
-from typing import Any, Dict, Optional, Tuple, Union
-
-
-# Lines look like:  "Key : Value"   (quotes literal, single ":" separator)
-_LINE_RE = re.compile(r'^\s*"?([^":]+?)"?\s*:\s*"?(.*?)"?\s*$')
-
-# Marker that ends the metadata block — everything after is raw sample data.
-_WAVEFORM_BLOCK_MARKER = "waveform data channels"
-
-
-def _normalize_key(raw: str) -> str:
-    """Convert "TranPPV" / "PreTriggerLength" → snake_case."""
-    s = raw.strip()
-    # Insert underscore between lower→upper / digit→letter transitions
-    s = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", s)
-    s = re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", "_", s)
-    s = s.replace("-", "_").replace(" ", "_")
-    return s.lower()
-
-
-def _strip_unit_suffix(value: str) -> str:
-    """Return the numeric part of values like "0.2119 in/s" → "0.2119".
-
-    Also strips Thor's below/above-threshold prefixes:
-      "<0.005 in/s"  → "0.005"   (below-noise-floor reading)
-      ">100 Hz"      → "100"     (above-measurement-range reading)
-    """
-    parts = value.strip().split()
-    token = parts[0] if parts else value.strip()
-    if token.startswith("<") or token.startswith(">"):
-        token = token[1:]
-    return token
-
-
-def _parse_float(value: str) -> Optional[float]:
-    try:
-        return float(_strip_unit_suffix(value))
-    except (ValueError, TypeError):
-        return None
-
-
-def _parse_int(value: str) -> Optional[int]:
-    try:
-        return int(float(_strip_unit_suffix(value)))
-    except (ValueError, TypeError):
-        return None
-
-
-def parse_idf_report(text: Union[str, bytes]) -> Dict[str, Any]:
-    """
-    Parse a Thor IDFW.txt / IDFH.txt sidecar.
-
-    Returns a flat dict with two kinds of entries:
-
-      - **Raw fields** — every `Key : Value` line, keyed by snake_case
-        of the original key, value as a string (unit suffix preserved).
-        Lets callers grab any field we haven't explicitly normalised.
-
-      - **Derived fields** — a curated set with parsed types:
-          * `serial_number`     str
-          * `event_type`        str  ("Full Waveform" / "Full Histogram")
-          * `event_datetime`    ISO-8601 string ("YYYY-MM-DDTHH:MM:SS") when
-                                 both EventDate and EventTime are present
-          * `sample_rate`       int  (samples/sec)
-          * `tran_ppv`,`vert_ppv`,`long_ppv` float (in/s)
-          * `mic_ppv`           float (dB or psi — same units as MicPSPL)
-          * `peak_vector_sum`   float (in/s)
-          * `tran_zc_freq`,`vert_zc_freq`,`long_zc_freq` float (Hz)
-          * `record_time_sec`   float (seconds)
-          * `pre_trigger_sec`   float (seconds)
-          * `project`           str  (from TitleString1 — Thor's location)
-          * `client`            str  (TitleString2)
-          * `operator`          str  (TitleString3 — company/operator)
-          * `notes`             str  (TitleString4)
-          * `setup`             str
-          * `version`           str  (firmware)
-          * `battery_volts`     float
-          * `calibration_text`  str  (e.g. "November 22, 2023 by Instantel")
-          * `tran_test_passed`, `vert_test_passed`, `long_test_passed`,
-            `mic_test_passed`  bool  ("Passed" → True; anything else → False)
-          * `filename`          str  (FileName line — useful sanity check)
-
-    Stops parsing at the literal "Waveform Data Channels" line; the
-    raw-samples block is left to whoever wants to decode the binary.
-
-    Input may be `str` or `bytes` (`utf-8`/`latin-1` tolerant).
-    """
-    if isinstance(text, bytes):
-        try:
-            text = text.decode("utf-8")
-        except UnicodeDecodeError:
-            text = text.decode("latin-1", errors="replace")
-
-    raw: Dict[str, str] = {}
-
-    for line in text.splitlines():
-        stripped = line.strip()
-        if not stripped:
-            continue
-        if stripped.lower().startswith(_WAVEFORM_BLOCK_MARKER):
-            break
-        m = _LINE_RE.match(stripped)
-        if not m:
-            continue
-        key = _normalize_key(m.group(1))
-        value = m.group(2).strip()
-        # Multi-value lines (Channel, Units, etc.) — coalesce by appending.
-        if key in raw:
-            raw[key] = raw[key] + "; " + value
-        else:
-            raw[key] = value
-
-    out: Dict[str, Any] = dict(raw)  # keep all raw fields
-
-    # ── Derived fields ───────────────────────────────────────────────────────
-
-    def _take(*candidates: str) -> Optional[str]:
-        for c in candidates:
-            if c in raw:
-                return raw[c]
-        return None
-
-    # Event identity
-    if "serial_number" in raw:
-        out["serial_number"] = raw["serial_number"]
-    if "event_type" in raw:
-        out["event_type"] = raw["event_type"]
-    if "file_name" in raw:
-        out["filename"] = raw["file_name"]
-
-    # Combined date+time.  Waveform sidecars use "EventDate" / "EventTime";
-    # histogram sidecars use "HistogramStartDate" / "HistogramStartTime".
-    # Prefer the event_* names when both are present.
-    ed = raw.get("event_date") or raw.get("histogram_start_date")
-    et = raw.get("event_time") or raw.get("histogram_start_time")
-    if ed and et:
-        try:
-            dt = datetime.datetime.strptime(f"{ed} {et}", "%Y-%m-%d %H:%M:%S")
-            out["event_datetime"] = dt.isoformat()
-        except ValueError:
-            pass
-
-    # Numeric scalars.  For every field we typify here, we MUST drop the
-    # raw string copy from `out` when parsing fails — Thor writes things
-    # like "<0.005 in/s" (below threshold) and "N/A" (not measured) that
-    # would otherwise linger in `out` as strings, sneak into SQLite REAL
-    # columns via permissive type affinity, and then crash the JS
-    # frontend on `.toFixed(...)`.
-    int_fields = ("sample_rate",)
-    for key in int_fields:
-        v = raw.get(key)
-        if v is None:
-            continue
-        iv = _parse_int(v)
-        if iv is not None:
-            out[key] = iv
-        else:
-            out.pop(key, None)
-
-    float_fields = (
-        "tran_ppv", "vert_ppv", "long_ppv", "peak_vector_sum",
-        "tran_zc_freq", "vert_zc_freq", "long_zc_freq",
-        "tran_peak_acceleration", "vert_peak_acceleration",
-        "long_peak_acceleration",
-        "tran_peak_displacement", "vert_peak_displacement",
-        "long_peak_displacement",
-        "mic_zc_freq",
-    )
-    for key in float_fields:
-        v = raw.get(key)
-        if v is None:
-            continue
-        fv = _parse_float(v)
-        if fv is not None:
-            out[key] = fv
-        else:
-            out.pop(key, None)
-
-    # Time-of-peak: Thor labels these "TimeofPeak" (lowercase "of") so the
-    # normalizer produces "*_timeof_peak".  Map them to the canonical
-    # ``*_time_of_peak`` output keys for downstream consumers.
-    for raw_key, out_key in (
-        ("tran_timeof_peak", "tran_time_of_peak"),
-        ("vert_timeof_peak", "vert_time_of_peak"),
-        ("long_timeof_peak", "long_time_of_peak"),
-        ("mic_timeof_peak",  "mic_time_of_peak"),
-    ):
-        v = raw.get(raw_key)
-        if v is None:
-            continue
-        fv = _parse_float(v)
-        if fv is not None:
-            out[out_key] = fv
-
-    # Microphone — Thor reports MicPSPL (dB(L)) which is the closest
-    # analogue to BW's mic_ppv.  The raw "99.4 dB(L)" string stays in
-    # `out` under the original `mic_pspl` key for display; the parsed
-    # float goes in `mic_ppv`.
-    mic = raw.get("mic_pspl")
-    if mic is not None:
-        fv = _parse_float(mic)
-        if fv is not None:
-            out["mic_ppv"] = fv
-
-    # Record / pre-trigger duration — same drop-on-failure discipline.
-    rt = raw.get("record_time")
-    if rt is not None:
-        fv = _parse_float(rt)
-        if fv is not None:
-            out["record_time_sec"] = fv
-    pt = raw.get("pre_trigger_length")
-    if pt is not None:
-        fv = _parse_float(pt)
-        if fv is not None:
-            out["pre_trigger_sec"] = fv
-
-    # Project / client / operator / location strings.  Thor's title
-    # strings are operator-defined; conventional mapping (per Thor's
-    # default TitleNote labels in the example data):
-    #   TitleString1 = Location  → project (sensor location identifier)
-    #   TitleString2 = Client    → client
-    #   TitleString3 = Company   → operator (the monitoring company)
-    #   TitleString4 = Notes     → notes
-    out["project"]  = _take("title_string1")
-    out["client"]   = _take("title_string2")
-    out["operator"] = _take("title_string3", "operator")
-    out["notes"]    = _take("title_string4", "post_event_note")
-
-    if "setup" in raw:
-        out["setup"] = raw["setup"]
-    if "version" in raw:
-        out["version"] = raw["version"]
-
-    # Battery (e.g. "3.8 volts" → 3.8)
-    bl = raw.get("battery_level")
-    if bl is not None:
-        fv = _parse_float(bl)
-        if fv is not None:
-            out["battery_volts"] = fv
-
-    # Calibration line is free-form (e.g. "November 22, 2023 by Instantel").
-    if "calibration" in raw:
-        out["calibration_text"] = raw["calibration"]
-
-    # Sensor self-check results — bool flags
-    for key, out_key in (
-        ("tran_test_results", "tran_test_passed"),
-        ("vert_test_results", "vert_test_passed"),
-        ("long_test_results", "long_test_passed"),
-        ("mic_test_results",  "mic_test_passed"),
-    ):
-        v = raw.get(key)
-        if v is not None:
-            out[out_key] = v.strip().lower() == "passed"
-
-    return out
-
-
-def serial_from_filename(name: str) -> Optional[str]:
-    """Convenience: pull the serial prefix from a Thor event filename.
-
-    Thor uses the literal serial as the filename prefix:
-      UM11719_20231219163444.IDFW  →  "UM11719"
-      BE9439_20200713124251.IDFH   →  "BE9439"
-    """
-    m = re.match(r"^([A-Z]{2}\d+)_\d{14}\.(IDFH|IDFW)(?:\.txt)?$",
-                 name, re.IGNORECASE)
-    return m.group(1).upper() if m else None
-
-
-def parse_event_filename(name: str) -> Optional[Tuple[str, datetime.datetime, str]]:
-    """Parse `<SERIAL>_<YYYYMMDDHHMMSS>.<KIND>` → (serial, datetime, kind).
-
-    `kind` is "IDFH" or "IDFW" (upper-case).  Returns None on no match.
-    """
-    m = re.match(r"^([A-Z]{2}\d+)_(\d{14})\.(IDFH|IDFW)$",
-                 name, re.IGNORECASE)
-    if not m:
-        return None
-    try:
-        ts = datetime.datetime.strptime(m.group(2), "%Y%m%d%H%M%S")
-    except ValueError:
-        return None
-    return m.group(1).upper(), ts, m.group(3).upper()

micromate/idf_file.py

@@ -1,723 +0,0 @@
-"""
-micromate/idf_file.py — Thor IDF binary codec.
-
-Decodes the Instantel Micromate Series IV ``.IDFW`` (waveform) and
-``.IDFH`` (histogram) binary on-disk format.  Sister module to
-``minimateplus/event_file_io.py``.
-
-Status (2026-05-28):
-
-- **Genuine Series IV / Thor binaries** are all signed
-  ``00 12 01 00 00 00 Instantel\\0`` (sig-A in earlier notes).  Two
-  Series III (Blastware) binaries appear in the example corpus
-  (``BE9439_*``) — they share the ``.IDFW``/``.IDFH`` extension by
-  filing convention but carry a BW STRT header (``10 00 01 80 00 00
-  Instantel STRT...``) and are NOT Thor data.  The reader detects
-  them by signature and raises NotImplementedError pointing callers
-  at ``minimateplus.event_file_io.read_blastware_file()``.
-- **IDFW waveform body** reuses the BW segment-rotated block codec
-  verbatim.  Body always starts at file offset ``0x0f1f``.  Samples
-  decoded via ``minimateplus.waveform_codec.decode_waveform_v2``
-  with 87–99% byte-exact match against ``.IDFW.txt`` sidecar (quiet
-  events).  Loud events hit the BW codec's known walker-stops-early
-  limit.  Residual ~3% drift on per-sample deltas — likely a
-  Thor-specific 12-bit delta refinement that BW's codec doesn't
-  model.  Geo LSB = 0.0003 in/s; mic factor ~2.14e-6 psi/count.
-- **IDFH histogram body**: 12-byte segment header
-  ``[len_be 2B] 0a 00 00 00 [00 NN_counter] 05 3f`` introduces a
-  segment of ``N`` 72-byte interval records (``N = (len - 10) // 72``).
-  Each record holds 4 × 16-byte per-channel min/max/halfp + 8-byte
-  tail.  Geo peaks via ``max(|min|, |max|) / 32768 × 10`` in/s
-  (matches sidecar within ~1.8%), freq via ``512 / halfp`` Hz.
-  **All 859 Thor IDFH files in the corpus decode (181,071 intervals).**
-- Binary metadata directly extracted: serial, timestamp, sample_rate,
-  record_time, calibration_date.  Other fields fall back to the paired
-  ``.IDFW.txt`` / ``.IDFH.txt`` sidecar (consumed by
-  ``WaveformStore.save_imported_idf``).
-
-The full reverse-engineering writeup lives in
-``docs/idf_protocol_reference.md``.
-"""
-
-from __future__ import annotations
-
-import datetime
-import struct
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Optional, Union
-
-# Thor IDFW bodies use the series-3 record-chain decoder.
-#
-# This was previously pinned to the SUPERSEDED tag-dispatch walker
-# (`decode_waveform_legacy`) on the stated grounds that "Thor has no ASCII
-# ground truth in the corpus and its geo scaling is separately suspect".
-# Both premises were false: Thor writes a per-sample CSV export next to every
-# binary (see scratch/verify_thor_against_csv.py), and the scaling is now
-# resolved (see _GEO_LSB_IPS).  Measured against that ground truth on
-# 2026-09-10, the record chain beats the legacy walker outright:
-#
-#     channel truncation   55/153 files -> 3/153
-#     files exact           98/153      -> 150/153
-#     per-sample exact      99.781%     -> 99.854%
-#
-# The legacy walker stops at the first unrecognised tag and returns whatever
-# channels it had, so its failure mode is silent short channels rather than an
-# error.  Do not re-pin it.
-from minimateplus.waveform_codec import _MODES, decode_waveform_v2, is_record
-
-from .models import IdfEvent, IdfPeaks, IdfReport
-
-
-# Genuine Series IV / Thor IDF binary signature: 6 bytes, then ASCII "Instantel".
-_THOR_PREFIX = b"\x00\x12\x01\x00\x00\x00"
-# Stray Series III (Blastware) binaries that occasionally turn up in Thor
-# corpus directories renamed to the .IDFW/.IDFH convention.  Their header
-# (`10 00 01 80 00 00 Instantel STRT ...`) is byte-for-byte a BW SUB 5A
-# STRT record, not a Thor binary.  Detected so we can refuse-and-route
-# rather than mis-parse.
-_BW_STRAY_PREFIX = b"\x10\x00\x01\x80\x00\x00"
-_INSTANTEL_TAG = b"Instantel"
-
-# Most common body offset for sig-A IDFW files (~50% of prod events;
-# 151/154 in the original tests/fixtures/THORDATA_example corpus).  The
-# body is the segment-rotated block stream consumed by decode_waveform_v2;
-# bytes [0:3] are the magic ``00 02 00`` preamble.  Production events
-# routinely use other offsets — see :func:`_find_waveform_body_offset`
-# for the dynamic scan.  This constant survives only as the priority hint.
-_BODY_START_SIG_A = 0x0F1F
-
-# Magic bytes that mark a candidate waveform-body preamble.
-_BODY_MAGIC = b"\x00\x02\x00"
-
-# Where to start looking for body candidates inside the file.  Skip the
-# fixed-header region where the same magic legitimately appears inside
-# channel-test records and the compliance block (offsets 0x015d, 0x091c,
-# 0x0ae2, 0x0d30 in observed events).
-# Lowered from 0x0E00 to 0x0C00 (2026-09-10).  Three-channel events -- mic
-# disabled -- have a shorter fixed header and put their record chain head at
-# 0x0dba, below the old floor.  The head was therefore invisible to the scan,
-# which fell through to the *Vert* segment-0 record and decoded a body shifted
-# one position around the channel rotation.  46 of 139 files in the
-# 9-10-26-csv-req corpus were affected; all 46 became per-sample exact once
-# the head was reachable.  The floor still skips the fixed-header region,
-# where `is_record()` can match channel-test records (0x015d, 0x091c, 0x0ae2).
-_BODY_SCAN_FLOOR = 0x0C00
-
-# Cap on trial decodes per file.  Chain-head detection normally yields one
-# or two candidates; the cap only bounds the worst case on a corrupt file.
-_MAX_BODY_CANDIDATES = 16
-
-# Geophone count → in/s.
-#
-# The old value 0.0003 was read off the smallest non-zero sample in the
-# sidecar corpus, but that sample is Thor's *4-decimal display rounding* of
-# the true LSB, not the LSB itself.  It read every series-4 geophone sample
-# 3.3% low.  The quantisation ladder gives it away: counts 1..6 export as
-# 0.0003, 0.0006, 0.0009, 0.0012, 0.0016, 0.0019 — an LSB of exactly 0.0003
-# would end 0.0015, 0.0018.
-#
-# The value below maximises exact 4-dp agreement over 1,046,016 paired
-# samples (454 channel-events, 2 units) at 99.854%, versus 50.7% for 0.0003.
-# It is a global constant, not a per-unit calibration: all 8 UM units in the
-# production store independently agree to within ±0.07% on their
-# device-reported PPV.  1/LSB = 3222.6 counts per in/s.
-#
-# The value is pinned, not guessed.  Each exported sample constrains the LSB
-# to the window that rounds to the printed 4-dp figure; intersecting 991,415
-# such constraints (clean channel-events only) gives
-#
-#     LSB in [0.000310307933, 0.000310308057]     width 1.2e-10
-#
-# 0.000310308 sits at the centre of that window.  Equivalent full scale is
-# 10.0 in/s / 0.000310308 = 32226.05 counts.
-#
-# Corroboration from the device: an IDFH interval that never recorded keeps
-# its min/max accumulator at its ±full-scale seed, and that seed is
-# (min=+32226, max=-32226) — the same magnitude, independently.  Note the
-# tempting closed form 10.0/32226 is very slightly WRONG: it lands 4.5e-10
-# above the feasible window and loses 78 boundary samples to the literal
-# value while never winning one.  Series-3 uses 32000 counts for the same
-# 10.0 in/s, so the two generations do NOT share a scale.
-#
-# Ground truth + harness: scratch/verify_thor_against_csv.py
-_GEO_LSB_IPS = 0.000310308
-
-# Microphone count → psi, derived from sidecar regression on 50 sample
-# pairs from UM11719_20231219162723.IDFW (mic-heavy event).
-_MIC_LSB_PSI = 2.14e-6
-
-# IDFH histogram constants.
-# Bytes per interval record = 16 per channel + an 8-byte tail, so a
-# 4-channel unit uses 72 and a mic-disabled 3-channel unit uses 56.  It is
-# NOT a constant: derive it per segment from the interval counter (see
-# decode_idfh_body).  This value survives only as the 4-channel default.
-_IDFH_INTERVAL_SIZE = 72        # bytes per per-interval record (4 channels)
-_IDFH_CHANNEL_BLOCK = 16        # bytes per channel inside an interval record
-_IDFH_INTERVAL_TAIL = 8         # bytes after the per-channel blocks
-_IDFH_SEGMENT_HEADER = 10       # bytes: [len_be 2B][0a 00 00 00 4B][00 NN 2B][05 3f 2B]
-_IDFH_SEGMENT_TAIL   = 2        # bytes after the interval data block, before next marker
-_IDFH_HALFP_FREQ_NUM = 512.0    # freq_hz = NUM / halfp; halfp ≤ 5 means ">100 Hz" sentinel
-_IDFH_CHANNELS = ("Tran", "Vert", "Long", "MicL")
-
-
-# ─── Binary metadata extraction ─────────────────────────────────────────────
-
-
-@dataclass
-class IdfBinaryMetadata:
-    """Fields recoverable from the sig-A binary header (no .txt needed)."""
-    serial:           Optional[str] = None
-    event_datetime:   Optional[datetime.datetime] = None
-    sample_rate:      Optional[int] = None
-    record_time_sec:  Optional[float] = None
-    calibration_date: Optional[datetime.date] = None
-
-
-def _read_ascii_z(buf: bytes, off: int, maxlen: int = 64) -> Optional[str]:
-    if off >= len(buf):
-        return None
-    end = buf.find(b"\x00", off, off + maxlen)
-    if end < 0:
-        end = min(off + maxlen, len(buf))
-    s = buf[off:end].decode("ascii", errors="replace").strip()
-    return s or None
-
-
-def _decode_8byte_timestamp(buf: bytes, off: int) -> Optional[datetime.datetime]:
-    """Layout: ``[day][month][year_hi][year_lo][unknown][hour][min][sec]``."""
-    if off + 8 > len(buf):
-        return None
-    day, mon, yh, yl, _unk, hr, mn, sc = buf[off : off + 8]
-    year = (yh << 8) | yl
-    if not (2015 <= year <= 2050 and 1 <= mon <= 12 and 1 <= day <= 31
-            and 0 <= hr < 24 and 0 <= mn < 60 and 0 <= sc < 60):
-        return None
-    try:
-        return datetime.datetime(year, mon, day, hr, mn, sc)
-    except ValueError:
-        return None
-
-
-def extract_binary_metadata(buf: bytes) -> IdfBinaryMetadata:
-    """Pull serial/timestamp/sample_rate/record_time/calibration from the
-    sig-A binary header.
-
-    Field positions confirmed against UM11719_20231219162723.IDFW; stable
-    across the 151-file sig-A corpus.
-    """
-    md = IdfBinaryMetadata()
-
-    # Serial: null-terminated ASCII at 0x14E.
-    md.serial = _read_ascii_z(buf, 0x14E, maxlen=16)
-
-    # Sample rate + record time live in a BW-compatible compliance block.
-    # Locate the 6-byte anchor `be 80 00 00 00 00` and read offsets relative
-    # to it: anchor-6 = sample_rate uint16 BE; anchor+6 = record_time float32 BE.
-    anchor = buf.find(b"\xbe\x80\x00\x00\x00\x00", 0x800, 0xA00)
-    if anchor > 0:
-        sr_bytes = buf[anchor - 6 : anchor - 4]
-        if len(sr_bytes) == 2:
-            sr = int.from_bytes(sr_bytes, "big")
-            if sr in (256, 512, 1024, 2048, 4096):
-                md.sample_rate = sr
-        rt_bytes = buf[anchor + 6 : anchor + 10]
-        if len(rt_bytes) == 4:
-            try:
-                rt = struct.unpack(">f", rt_bytes)[0]
-                if 0.1 <= rt <= 600.0:
-                    md.record_time_sec = float(rt)
-            except struct.error:
-                pass
-
-    # Event timestamp: 8 bytes.  Position differs between IDFW (0x97A) and
-    # IDFH (0x9F8); scan a small range and accept the first valid decode.
-    for off in (0x97A, 0x9F8):
-        ts = _decode_8byte_timestamp(buf, off)
-        if ts is not None:
-            md.event_datetime = ts
-            break
-
-    # Calibration date: day, month, year_be at 0x194-0x197.
-    if len(buf) > 0x197:
-        day, mon = buf[0x194], buf[0x195]
-        year = int.from_bytes(buf[0x196 : 0x198], "big")
-        if 1 <= mon <= 12 and 1 <= day <= 31 and 2015 <= year <= 2050:
-            try:
-                md.calibration_date = datetime.date(year, mon, day)
-            except ValueError:
-                pass
-
-    return md
-
-
-# ─── Sample decoder + unit conversion ───────────────────────────────────────
-
-
-def _find_waveform_body_offset(buf: bytes) -> Optional[int]:
-    """Pick the file offset of the waveform body by trial-decoding every
-    ``00 02 00`` magic position past the fixed-header region.
-
-    The body's location isn't fixed across all sig-A IDFW files — about
-    half the production events use ``0x0f1f``, but the rest have offsets
-    that shift based on header padding / channel-config layout.  We
-    auto-detect by:
-
-      1. Find every ``00 02 00`` occurrence past ``_BODY_SCAN_FLOOR``.
-      2. Try ``decode_waveform_v2()`` on each candidate.
-      3. Pick the offset whose decoded sample count is largest.
-
-    Returns the offset, or ``None`` if no candidate yielded more than
-    the trivial 2-sample preamble (= "no real body found").
-
-    Costs ~2-8 trial decodes per file; in practice the first candidate
-    past 0x0e00 is usually the right one.
-    """
-    if len(buf) < _BODY_SCAN_FLOOR + 8:
-        return None
-
-    # 1. Locate every plausible per-channel record header.  A header carries
-    #    [len 2B][channel_id][00][00] at +2..+6, so anchor the search on the
-    #    three-byte ``<cid> 00 00`` signature and validate with is_record().
-    #    Scanning candidate *preambles* instead is not viable: MODE_RAW16 is
-    #    ``00 00``, so every run of three zero bytes would look like a body
-    #    start and each would cost a full trial decode (~0.5 s/file measured).
-    floor = max(0, _BODY_SCAN_FLOOR - 7)
-    starts: list = []
-    for cid in (0x46, 0x47, 0x48, 0x49):
-        sig = bytes((cid, 0x00, 0x00))
-        i = floor
-        while True:
-            j = buf.find(sig, i)
-            if j < 0:
-                break
-            i = j + 1
-            q = j - 4
-            if q >= floor and is_record(buf, q):
-                starts.append(q)
-    if not starts:
-        return None
-    starts.sort()
-
-    # 2. A body begins at the head of a record chain -- a record that no other
-    #    record's length field points at.  The head's own payload is the
-    #    implicit segment-0 Tran record, and the body offset is head + 7 (past
-    #    [len 2B][cid][00][00][seg]) so that body[1:3] lands on the mode.
-    ends = {q + 2 + int.from_bytes(buf[q + 2 : q + 4], "big") for q in starts}
-    heads = [q for q in starts if q not in ends] or starts[:1]
-
-    # 3. Trial-decode each head and keep the best.  Prefer a candidate where
-    #    all four channels come out the same length: scoring on raw sample
-    #    count alone picks false positives sitting *inside* a record header,
-    #    which decode a plausible-looking but rotation-shifted body that
-    #    silently drops each channel's segment 0.
-    best = None
-    best_off = None
-    for head in heads[:_MAX_BODY_CANDIDATES]:
-        j = head + 7
-        if j + 3 > len(buf) or (buf[j + 1], buf[j + 2]) not in _MODES:
-            continue
-        try:
-            decoded = decode_waveform_v2(buf[j:])
-        except Exception:
-            continue
-        if not decoded:
-            continue
-        lengths = [len(v) for v in decoded.values() if v]
-        total = sum(len(v) for v in decoded.values())
-        # A "real" body has more than just the 2-sample preamble.
-        if total <= 2:
-            continue
-        # >= 3 rather than == 4: a mic-disabled event has only the three geo
-        # channels, and demanding four made `equal` permanently False for
-        # them, leaving the pick to raw sample count alone.
-        equal = len(lengths) >= 3 and len(set(lengths)) == 1
-        score = (equal, total)
-        if best is None or score > best:
-            best, best_off = score, j
-    return best_off
-
-
-def _decode_waveform_samples(buf: bytes) -> Optional[dict]:
-    """Decode samples from the sig-A waveform body.
-
-    Returns the raw decoder counts dict — geo LSB = 0.0003 in/s, mic in
-    its own count unit (see :func:`mic_count_to_psi`).  Returns None if
-    no usable body is found.
-
-    Uses :func:`_find_waveform_body_offset` to locate the body — the
-    file-offset varies across events (~50% sit at the canonical
-    ``0x0f1f`` but the rest don't), so the previous hardcoded constant
-    silently produced 2-sample preamble-only output for half the corpus.
-    """
-    off = _find_waveform_body_offset(buf)
-    if off is None:
-        return None
-    return decode_waveform_v2(buf[off:])
-
-
-def geo_count_to_ips(count: int) -> float:
-    """Convert a Thor geo decoder count to in/s.  LSB = 0.0003 in/s."""
-    return count * _GEO_LSB_IPS
-
-
-def mic_count_to_psi(count: int) -> float:
-    """Convert a Thor mic decoder count to psi.  Scale derived from
-    regression over 50 sample pairs in UM11719_20231219162723.IDFW;
-    consistent to ~5%.  Calibration constants from the channel block
-    can refine this once decoded.
-    """
-    return count * _MIC_LSB_PSI
-
-
-# ─── IDFH histogram decoder ─────────────────────────────────────────────────
-
-
-@dataclass
-class IdfhInterval:
-    """One decoded histogram interval (typically one minute of monitoring)."""
-    offset:    int    # file byte offset of the 72-byte record
-    # Per-channel min/max ADC counts (int16 BE), half-period samples, peak count.
-    # Peak = max(|min|, |max|).  freq_hz = 512/halfp (None if halfp ≤ 5 →
-    # ">100 Hz" sentinel; matches sidecar convention).
-    tran_min:    int
-    tran_max:    int
-    tran_halfp:  int
-    vert_min:    int
-    vert_max:    int
-    vert_halfp:  int
-    long_min:    int
-    long_max:    int
-    long_halfp:  int
-    micl_min:    int
-    micl_max:    int
-    micl_halfp:  int
-    # 4 on a normal unit; 3 when the microphone is disabled, in which case the
-    # micl_* fields are absent from the record and read as zero.
-    n_channels:  int = 4
-
-    def has_channel(self, channel: str) -> bool:
-        return channel != "MicL" or self.n_channels >= 4
-
-    def peak_count(self, channel: str) -> int:
-        mn = getattr(self, f"{channel.lower()}_min")
-        mx = getattr(self, f"{channel.lower()}_max")
-        return max(abs(mn), abs(mx))
-
-    def peak_ips(self, channel: str) -> float:
-        """Convert peak count to in/s (geo channels only)."""
-        # Same geo LSB as the waveform path — verified independently against
-        # the IDFH exports: as peak magnitude rises (and 4-dp quantisation
-        # noise falls) the implied LSB converges on 0.0003103, matching
-        # _GEO_LSB_IPS.  The old 10.0/32768 read histogram peaks 1.7% low.
-        return self.peak_count(channel) * _GEO_LSB_IPS
-
-    def freq_hz(self, channel: str) -> Optional[float]:
-        halfp = getattr(self, f"{channel.lower()}_halfp")
-        if halfp <= 5:
-            return None
-        return _IDFH_HALFP_FREQ_NUM / halfp
-
-
-def _is_unwritten_interval(interval: "IdfhInterval") -> bool:
-    """True for an interval slot the device reserved but never wrote.
-
-    Thor seeds each interval's per-channel accumulators at ``min = +full
-    scale`` and ``max = -full scale`` and then narrows them as samples
-    arrive.  A slot that never recorded keeps that seed, so ``min > max`` —
-    impossible for real data.  Such a record decodes to a full-scale
-    10.0 in/s peak on every channel and, being a max-over-intervals, poisons
-    the whole file's PPV.
-
-    Rare but real: exactly 1 of 497,611 corpus intervals, and it inflated
-    that file's Long PPV from 0.0081 to 10.0 in/s.  The inversion is always
-    all-or-nothing across channels (0 partial cases in the corpus), so
-    requiring every channel to be inverted keeps this from ever firing on
-    genuine data.
-    """
-    pairs = [
-        (interval.tran_min, interval.tran_max),
-        (interval.vert_min, interval.vert_max),
-        (interval.long_min, interval.long_max),
-    ]
-    if interval.has_channel("MicL"):
-        pairs.append((interval.micl_min, interval.micl_max))
-    return all(mn > mx for mn, mx in pairs)
-
-
-def _decode_idfh_interval(buf72: bytes, offset: int,
-                          n_channels: int = 4) -> IdfhInterval:
-    """Decode one interval record into per-channel min/max/halfp.
-
-    The record is ``n_channels`` × 16-byte blocks plus an 8-byte tail, so it
-    is 72 bytes on a normal unit and 56 when the microphone is disabled.
-    Missing channels read as zero.
-    """
-    import struct
-    fields = []
-    for i in range(4):
-        if i >= n_channels:
-            fields.extend([0, 0, 0])
-            continue
-        block = buf72[i * 16 : (i + 1) * 16]
-        mn = struct.unpack_from(">h", block, 0)[0]
-        mx = struct.unpack_from(">h", block, 2)[0]
-        # block[4:6] = int16 BE, role unknown (possibly time-of-peak)
-        halfp = struct.unpack_from(">H", block, 6)[0]
-        # block[10:12] and block[14:16] are uint16 BE with unknown semantics
-        # (likely sum / count contributions for the PVS computation).
-        fields.extend([mn, mx, halfp])
-    # Tail 8 bytes (buf72[64:72]) carry PVS-related data; not yet decoded.
-    return IdfhInterval(
-        offset=offset,
-        tran_min=fields[0], tran_max=fields[1], tran_halfp=fields[2],
-        vert_min=fields[3], vert_max=fields[4], vert_halfp=fields[5],
-        long_min=fields[6], long_max=fields[7], long_halfp=fields[8],
-        micl_min=fields[9], micl_max=fields[10], micl_halfp=fields[11],
-        n_channels=n_channels,
-    )
-
-
-def decode_idfh_body(buf: bytes) -> list:
-    """Walk an IDFH file and decode every interval record.
-
-    The body has one or more segments; each segment header is 12 bytes:
-    ``[length_be 2B][0a 00 00 00][counter_be 2B][05 3f]`` where ``length``
-    is bytes from the magic through the end of the interval block
-    (= 10 + 72 × n_intervals).  Segments are separated by a 2-byte tail
-    + next-segment 2-byte prefix (the bytes before the next length field).
-
-    ``counter`` is a **uint16 BE cumulative interval index** — the 0-based
-    index of the LAST interval in this segment.  Segments carry 10
-    intervals each, so it runs 9, 19, 29, ... across the file.
-
-    ⚠ This validator used to require ``buf[j + 4] == 0x00``, i.e. that the
-    counter's high byte was zero.  That silently capped every histogram at
-    **250 intervals**: the moment the cumulative counter passed 255 the high
-    byte went non-zero and every later segment was rejected, so any
-    monitoring run longer than ~4 hours lost its tail — frequently the part
-    holding the event peak, which is why those files' PPV read low.  540 of
-    858 corpus files were affected.  Do not reinstate that check.
-    """
-    intervals: list = []
-    i = 0
-    prev_counter = -1          # so the first segment's n = counter + 1
-    while True:
-        j = buf.find(b"\x0a\x00\x00\x00", i)
-        if j < 0 or j < 2:
-            break
-        # Validate: [length_be][0a 00 00 00][counter_be][05 3f].  The counter
-        # is deliberately NOT constrained — see the note above.
-        if buf[j + 6 : j + 8] != b"\x05\x3f":
-            i = j + 1
-            continue
-        length = int.from_bytes(buf[j - 2 : j], "big")
-        counter = int.from_bytes(buf[j + 4 : j + 6], "big")
-        header_start = j - 2
-        if length < _IDFH_SEGMENT_HEADER or header_start + length > len(buf):
-            # Truncated / bogus length — not a real segment header.
-            i = j + 1
-            continue
-        # The counter is the cumulative index of this segment's LAST interval,
-        # so the interval count is its delta from the previous segment.  That
-        # gives the record stride, which is NOT fixed: 16 bytes per channel
-        # plus an 8-byte tail, so 72 for a 4-channel unit and 56 for a
-        # mic-disabled 3-channel one.  Assuming 72 unconditionally made every
-        # 3-channel histogram read 7 intervals per 10-interval segment,
-        # walking off alignment into garbage that decoded as ~10 in/s peaks.
-        n = counter - prev_counter
-        if n <= 0:
-            i = j + 1
-            continue
-        stride = (length - _IDFH_SEGMENT_HEADER) // n
-        n_channels, remainder = divmod(stride - _IDFH_INTERVAL_TAIL,
-                                       _IDFH_CHANNEL_BLOCK)
-        if remainder or not (1 <= n_channels <= 4):
-            i = j + 1
-            continue
-        interval_start = header_start + _IDFH_SEGMENT_HEADER
-        for k in range(n):
-            off = interval_start + k * stride
-            if off + stride > len(buf):
-                break
-            chunk = buf[off : off + stride]
-            interval = _decode_idfh_interval(chunk, off, n_channels)
-            if _is_unwritten_interval(interval):
-                # Reserved-but-never-recorded slot: the min/max accumulators
-                # still hold their ±full-scale seed.  Counting it would
-                # fabricate a 10.0 in/s peak on every channel.
-                continue
-            intervals.append(interval)
-        prev_counter = counter
-        # Advance past this segment + the 2-byte tail.
-        i = header_start + length + _IDFH_SEGMENT_TAIL
-    return intervals
-
-
-# ─── Top-level reader ───────────────────────────────────────────────────────
-
-
-@dataclass
-class IdfReadResult:
-    """Return type for :func:`read_idf_file`.
-
-    For waveforms (``.IDFW``), ``samples`` holds the per-channel sample
-    arrays in Thor decoder counts.  For histograms (``.IDFH``),
-    ``samples`` is empty and ``intervals`` holds the per-interval
-    record list (peaks, freqs).
-    """
-    event:           IdfEvent
-    samples:         dict   # {"Tran": [...], ...} for IDFW; empty for IDFH
-    binary_metadata: IdfBinaryMetadata
-    signature:       str    # always "thor" for now (sig-A genuine Thor)
-    intervals:       Optional[list] = None  # list[IdfhInterval] for IDFH; None for IDFW
-
-
-def read_idf_file(
-    path: Union[str, Path],
-    *,
-    data: Optional[bytes] = None,
-) -> IdfReadResult:
-    """Parse a Thor ``.IDFW`` binary into an ``IdfEvent`` + decoded samples.
-
-    Currently implements signature-A waveforms only.  Signature-B
-    (old-firmware) and ``.IDFH`` histograms raise NotImplementedError;
-    use the paired ``.IDFW.txt`` / ``.IDFH.txt`` sidecar for those via
-    ``parse_idf_report()``.
-
-    Returns an :class:`IdfReadResult`.  The caller converts int sample
-    counts to physical units via :func:`geo_count_to_ips` /
-    :func:`mic_count_to_psi`.
-
-    ``path`` is used for filename in error messages and ``.IDFH`` vs
-    ``.IDFW`` suffix detection.  When ``data`` is supplied the disk
-    read is skipped — useful for ingest paths that already have the
-    bytes in memory and where the file may not exist on disk yet.
-    """
-    p = Path(path)
-    buf = data if data is not None else p.read_bytes()
-
-    if len(buf) < 16 or buf[6:16] != _INSTANTEL_TAG + b"\x00":
-        raise ValueError(f"{p.name}: not an IDF file (missing Instantel magic)")
-
-    sig_prefix = buf[:6]
-    if sig_prefix == _THOR_PREFIX:
-        signature = "thor"
-    elif sig_prefix == _BW_STRAY_PREFIX:
-        raise NotImplementedError(
-            f"{p.name}: file has a Series III (Blastware) STRT header in "
-            "an IDF-named container — not a Thor binary.  Route through "
-            "minimateplus.event_file_io.read_blastware_file() instead "
-            "(peaks decode; samples & full metadata don't, but it's not "
-            "Thor data so the Thor codec doesn't apply)."
-        )
-    else:
-        raise ValueError(f"{p.name}: unknown IDF signature {sig_prefix.hex()}")
-
-    is_histogram = p.suffix.upper() == ".IDFH"
-    md = extract_binary_metadata(buf)
-
-    if is_histogram:
-        intervals = decode_idfh_body(buf)
-        if not intervals:
-            raise ValueError(f"{p.name}: IDFH body decoded no intervals")
-        # Peaks: max across all intervals on each channel (per-channel max
-        # of stored max-magnitudes; sidecar's PPV row carries the same).
-        peak_tran = max((iv.peak_ips("Tran") for iv in intervals), default=0.0)
-        peak_vert = max((iv.peak_ips("Vert") for iv in intervals), default=0.0)
-        peak_long = max((iv.peak_ips("Long") for iv in intervals), default=0.0)
-        # Mic peak in psi — Thor stores per-interval mic ADC counts in the
-        # binary; convert the max count to psi via the per-count factor.
-        # Skip on a mic-disabled (3-channel) unit: those records carry no mic
-        # block at all, so peak_count("MicL") would report a synthetic zero.
-        mic_peak_count = max(
-            (iv.peak_count("MicL") for iv in intervals if iv.has_channel("MicL")),
-            default=0,
-        )
-        mic_peak_psi = mic_count_to_psi(mic_peak_count) if mic_peak_count else None
-        rep = IdfReport(
-            serial_number=md.serial,
-            event_type="Full Histogram",
-            event_datetime=md.event_datetime,
-            filename=p.name,
-            sample_rate=md.sample_rate,
-            record_time_sec=md.record_time_sec,
-        )
-        peaks = IdfPeaks(
-            transverse_ips=peak_tran,
-            vertical_ips=peak_vert,
-            longitudinal_ips=peak_long,
-            peak_vector_sum_ips=None,
-            mic_pspl_dbl=None,         # IDFH binary doesn't carry the dB(L) value
-            mic_pspl_psi=mic_peak_psi,
-        )
-        event = IdfEvent(
-            serial=md.serial or "UNKNOWN",
-            timestamp=md.event_datetime or datetime.datetime(1970, 1, 1),
-            kind="Histogram",
-            filename=p.name,
-            sample_rate=md.sample_rate,
-            record_time_sec=md.record_time_sec,
-            peaks=peaks,
-            report=rep,
-        )
-        return IdfReadResult(
-            event=event,
-            samples={},
-            binary_metadata=md,
-            signature=signature,
-            intervals=intervals,
-        )
-
-    # Waveform path.
-    decoded = _decode_waveform_samples(buf)
-    if decoded is None:
-        raise ValueError(f"{p.name}: waveform body codec failed")
-
-    rep = IdfReport(
-        serial_number=md.serial,
-        event_type="Full Waveform",
-        event_datetime=md.event_datetime,
-        filename=p.name,
-        sample_rate=md.sample_rate,
-        record_time_sec=md.record_time_sec,
-    )
-
-    def _peak_ips(ch: str) -> float:
-        arr = decoded.get(ch, [])
-        return geo_count_to_ips(max((abs(v) for v in arr), default=0))
-
-    # Mic peak psi from binary: max absolute MicL ADC count × 2.14e-6 psi/count.
-    mic_arr = decoded.get("MicL", [])
-    mic_peak_count = max((abs(v) for v in mic_arr), default=0)
-    mic_peak_psi = mic_count_to_psi(mic_peak_count) if mic_peak_count else None
-
-    peaks = IdfPeaks(
-        transverse_ips=_peak_ips("Tran"),
-        vertical_ips=_peak_ips("Vert"),
-        longitudinal_ips=_peak_ips("Long"),
-        # PVS requires aligned per-sample √(T²+V²+L²); leave None — the
-        # sidecar carries it and the bridge picks it up if present.
-        peak_vector_sum_ips=None,
-        mic_pspl_dbl=None,             # binary IDFW doesn't carry the dB(L) value;
-                                       # sidecar .txt fills it via IdfReport.from_dict
-        mic_pspl_psi=mic_peak_psi,
-    )
-
-    event = IdfEvent(
-        serial=md.serial or "UNKNOWN",
-        timestamp=md.event_datetime or datetime.datetime(1970, 1, 1),
-        kind="Waveform",
-        filename=p.name,
-        sample_rate=md.sample_rate,
-        record_time_sec=md.record_time_sec,
-        peaks=peaks,
-        report=rep,
-    )
-
-    return IdfReadResult(
-        event=event,
-        samples=decoded,
-        binary_metadata=md,
-        signature=signature,
-    )

micromate/idf_to_bw_report.py

@@ -1,323 +0,0 @@
-"""
-micromate/idf_to_bw_report.py — adapter that projects a parsed Thor IDF
-report (+ binary metadata + decoded IDFH intervals) into the
-``bw_report``-shaped dict that :mod:`sfm.report_pdf.gather_report_data`
-consumes.
-
-Lets Thor events flow through the existing Series III Event Report PDF
-pipeline without duplicating the renderer.  Thor's report content is
-~95% the same data shape as BW's; the field names differ but the
-underlying metrics map 1:1.
-
-Caveats
-───────
-
-- **Mic units** — Thor records ``MicPSPL`` natively in dB(L).  This
-  adapter sets ``bw_report.mic.pspl_dbl`` directly; the report
-  renderer recomputes the equivalent psi via its dBL→psi formula.
-- **Saturation / above-range flags** — Thor doesn't always mark
-  ``OORANGE`` the way BW does; we set ``zc_freq_above_range`` only
-  when a `>100` sentinel was preserved in the raw text.
-- **Per-interval data** — for IDFH events we build ``interval_times``
-  by stepping ``IntervalSize`` from ``HistogramStartTime``; the binary
-  decoder confirms one record per step (882 / 881 / 881 ... across
-  the corpus).
-- **calibration_by parsing** — Thor's free-form ``Calibration : November
-  22, 2023 by Instantel`` is split on ``" by "`` to extract the
-  calibrator; the date prefix is parsed where possible, otherwise
-  the binary-extracted ``calibration_date`` from
-  :class:`micromate.idf_file.IdfBinaryMetadata` wins.
-"""
-
-from __future__ import annotations
-
-import datetime
-import re
-from typing import Any, Dict, List, Optional
-
-
-# ─── Helpers ────────────────────────────────────────────────────────────────
-
-
-_NUM_RE = re.compile(r"-?\d+(?:\.\d+)?")
-
-
-def _parse_first_number(s: Optional[str]) -> Optional[float]:
-    """Pull the first numeric token from a string like ``"0.1500 in/s"``."""
-    if s is None:
-        return None
-    m = _NUM_RE.search(str(s))
-    if not m:
-        return None
-    try:
-        return float(m.group(0))
-    except ValueError:
-        return None
-
-
-def _parse_interval_size_s(s: Optional[str]) -> Optional[float]:
-    """``"60 sec"`` → 60.0, ``"5 min"`` → 300.0, ``"1 hour"`` → 3600."""
-    if s is None:
-        return None
-    num = _parse_first_number(s)
-    if num is None:
-        return None
-    sl = str(s).lower()
-    if "hour" in sl or "hr" in sl:
-        return num * 3600.0
-    if "min" in sl:
-        return num * 60.0
-    return num   # default to seconds
-
-
-def _parse_calibration(text: Optional[str]) -> tuple[Optional[str], Optional[str]]:
-    """Split ``"November 22, 2023 by Instantel"`` → (ISO date, calibrator).
-
-    Returns ``(None, None)`` if neither half parses.
-    """
-    if not text:
-        return None, None
-    parts = str(text).split(" by ", 1)
-    date_part = parts[0].strip() if parts else None
-    by_part = parts[1].strip() if len(parts) > 1 else None
-    iso_date: Optional[str] = None
-    if date_part:
-        for fmt in ("%B %d, %Y", "%b %d, %Y", "%Y-%m-%d", "%m/%d/%Y"):
-            try:
-                iso_date = datetime.datetime.strptime(date_part, fmt).date().isoformat()
-                break
-            except ValueError:
-                continue
-    return iso_date, by_part
-
-
-def _channel_peaks(idf: Dict[str, Any], ch_lc: str) -> Dict[str, Any]:
-    """Map ``tran_ppv`` / ``tran_zc_freq`` / ... → bw_report.peaks.tran shape."""
-    out: Dict[str, Any] = {}
-    for src, dst in (
-        (f"{ch_lc}_ppv",                 "ppv_ips"),
-        (f"{ch_lc}_zc_freq",             "zc_freq_hz"),
-        (f"{ch_lc}_time_of_peak",        "time_of_peak_s"),
-        (f"{ch_lc}_peak_acceleration",   "peak_accel_g"),
-        (f"{ch_lc}_peak_displacement",   "peak_disp_in"),
-    ):
-        v = idf.get(src)
-        if v is not None:
-            out[dst] = v
-    # ZC freq ">100" sentinel: the raw text carries it under the un-typed
-    # key (e.g. ``raw["tran_zc_freq"]`` would be ``">100"``), and our parser
-    # dropped the typed entry.  Detect that case and flag.
-    raw_zc = idf.get(f"{ch_lc}_zc_freq")
-    if isinstance(raw_zc, str) and ">" in raw_zc:
-        out["zc_freq_above_range"] = True
-        out.pop("zc_freq_hz", None)
-    return out
-
-
-def _sensor_check(idf: Dict[str, Any], ch_lc: str) -> Dict[str, Any]:
-    out: Dict[str, Any] = {}
-    fr = idf.get(f"{ch_lc}_test_freq")
-    if fr is not None:
-        out["freq_hz"] = _parse_first_number(fr)
-    rt = idf.get(f"{ch_lc}_test_ratio")
-    if rt is not None:
-        out["ratio"] = _parse_first_number(rt)
-    am = idf.get(f"{ch_lc}_test_amplitude")
-    if am is not None:
-        out["amplitude_mv"] = _parse_first_number(am)
-    res = idf.get(f"{ch_lc}_test_results")
-    if res is not None:
-        out["result"] = str(res).strip()
-    return {k: v for k, v in out.items() if v is not None}
-
-
-def _interval_times(idf: Dict[str, Any], n_intervals: Optional[int]) -> List[str]:
-    """Synthesise per-interval timestamps from start + interval_size × k.
-
-    Returns ``[]`` when start time or interval size is unknown.
-    """
-    if not n_intervals:
-        return []
-    start_date = idf.get("histogram_start_date") or idf.get("event_date")
-    start_time = idf.get("histogram_start_time") or idf.get("event_time")
-    iv_str = idf.get("interval_size")
-    iv_s = _parse_interval_size_s(iv_str)
-    if not (start_date and start_time and iv_s):
-        return []
-    try:
-        t0 = datetime.datetime.strptime(f"{start_date} {start_time}", "%Y-%m-%d %H:%M:%S")
-    except ValueError:
-        return []
-    out = []
-    for k in range(int(n_intervals)):
-        t = t0 + datetime.timedelta(seconds=iv_s * (k + 1))
-        out.append(t.isoformat())
-    return out
-
-
-# ─── Top-level adapter ──────────────────────────────────────────────────────
-
-
-def build_bw_report_from_idf(
-    idf_report: Dict[str, Any],
-    *,
-    binary_md=None,
-    intervals: Optional[list] = None,
-    is_histogram: Optional[bool] = None,
-) -> Dict[str, Any]:
-    """Project a parsed IDF report dict (and optional binary metadata +
-    decoded IDFH intervals) into the BW report sidecar shape.
-
-    The returned dict is structurally identical to what
-    ``minimateplus.event_file_io._bw_report_to_dict`` produces from a
-    real BW ASCII report — it can be assigned to
-    ``sidecar["bw_report"]`` and consumed verbatim by
-    ``sfm.report_pdf.gather_report_data``.
-
-    ``intervals`` is the list of :class:`micromate.idf_file.IdfhInterval`
-    objects from :func:`micromate.idf_file.decode_idfh_body`; only used
-    for histogram events to derive accurate ``interval_times``.
-    """
-    if is_histogram is None:
-        et = str(idf_report.get("event_type", ""))
-        is_histogram = et.lower().startswith("full histogram")
-
-    # ── Trigger / recording / device ─────────────────────────────────────
-    trigger_channel = idf_report.get("trigger")
-    trigger_level   = _parse_first_number(idf_report.get("geo_trigger_level"))
-    geo_range_ips   = _parse_first_number(idf_report.get("geo_range"))
-
-    cal_iso, cal_by = _parse_calibration(idf_report.get("calibration"))
-    # Prefer the binary-extracted calibration_date when our text parse fell
-    # through; the binary date is unambiguous.
-    if cal_iso is None and binary_md is not None and binary_md.calibration_date:
-        cal_iso = binary_md.calibration_date.isoformat()
-
-    # ── Histogram fields ────────────────────────────────────────────────
-    hist_block: Dict[str, Any] = {
-        "start": None, "stop": None, "n_intervals": None,
-        "interval_size": None, "interval_size_s": None,
-        "channel_peak_when": {},
-    }
-    if is_histogram:
-        sd = idf_report.get("histogram_start_date")
-        st = idf_report.get("histogram_start_time")
-        if sd and st:
-            try:
-                hist_block["start"] = datetime.datetime.strptime(
-                    f"{sd} {st}", "%Y-%m-%d %H:%M:%S"
-                ).isoformat()
-            except ValueError:
-                pass
-        ed = idf_report.get("histogram_stop_date")
-        et_ = idf_report.get("histogram_stop_time")
-        if ed and et_:
-            try:
-                hist_block["stop"] = datetime.datetime.strptime(
-                    f"{ed} {et_}", "%Y-%m-%d %H:%M:%S"
-                ).isoformat()
-            except ValueError:
-                pass
-        n_raw = idf_report.get("number_of_intervals")
-        if n_raw is not None:
-            try:
-                # Thor reports a float like "81.04"; round to int (the BW
-                # report uses an int for the column).
-                hist_block["n_intervals"] = int(float(str(n_raw)))
-            except ValueError:
-                pass
-        # When the binary decoder gave us the actual interval count, prefer it.
-        if intervals is not None:
-            hist_block["n_intervals"] = len(intervals)
-        hist_block["interval_size"] = idf_report.get("interval_size")
-        hist_block["interval_size_s"] = _parse_interval_size_s(idf_report.get("interval_size"))
-        # interval_times derived from start+step (the BW report uses the
-        # exact strings; we match its representation).
-        times = _interval_times(idf_report, hist_block["n_intervals"])
-        # Per-channel peak when (absolute date+time at which the channel's
-        # peak occurred over the histogram run).  Thor splits this into
-        # ``TranPeakDate`` / ``TranPeakTime`` etc.
-        peak_when: Dict[str, str] = {}
-        for ch_label, ch_lc in (("Tran", "tran"), ("Vert", "vert"), ("Long", "long"), ("MicL", "mic")):
-            d = idf_report.get(f"{ch_lc}_peak_date")
-            t = idf_report.get(f"{ch_lc}_peak_time")
-            if d and t:
-                try:
-                    peak_when[ch_label] = datetime.datetime.strptime(
-                        f"{d} {t}", "%Y-%m-%d %H:%M:%S"
-                    ).isoformat()
-                except ValueError:
-                    continue
-        if peak_when:
-            hist_block["channel_peak_when"] = peak_when
-
-    # ── Mic block ────────────────────────────────────────────────────────
-    mic_block = {
-        "weighting":           "L",                   # Thor mic is ISEE Linear
-        "pspl_dbl":            idf_report.get("mic_ppv"),  # the dB(L) float
-        "pspl_saturated":      False,
-        "zc_freq_hz":          idf_report.get("mic_zc_freq"),
-        "zc_freq_above_range": isinstance(idf_report.get("mic_zc_freq"), str)
-                               and ">" in str(idf_report.get("mic_zc_freq")),
-        "time_of_peak_s":      idf_report.get("mic_time_of_peak"),
-    }
-    if mic_block["zc_freq_above_range"]:
-        mic_block["zc_freq_hz"] = None
-
-    # ── Peaks ────────────────────────────────────────────────────────────
-    vs_block = {
-        "ips":       idf_report.get("peak_vector_sum"),
-        "time_s":    _parse_first_number(idf_report.get("peak_vector_sum_time_sum")),
-        "when":      None,
-        "saturated": False,
-    }
-    if is_histogram:
-        # PVS absolute date+time, when present.
-        vs_d = idf_report.get("peak_vector_sum_date")
-        vs_t = idf_report.get("peak_vector_sum_time")
-        if vs_d and vs_t:
-            try:
-                vs_block["when"] = datetime.datetime.strptime(
-                    f"{vs_d} {vs_t}", "%Y-%m-%d %H:%M:%S"
-                ).isoformat()
-            except ValueError:
-                pass
-
-    return {
-        "available":  True,
-        "event_type": idf_report.get("event_type"),
-        "version":    idf_report.get("version"),
-        "trigger": {
-            "channel":       trigger_channel,
-            "geo_level_ips": trigger_level,
-        },
-        "recording": {
-            "sample_rate_sps":  idf_report.get("sample_rate"),
-            "record_time_s":    idf_report.get("record_time_sec"),
-            "pretrig_s":        idf_report.get("pre_trigger_sec"),
-            "stop_mode":        idf_report.get("record_stop_mode"),
-            "geo_range_ips":    geo_range_ips,
-            "units":            idf_report.get("units"),
-        },
-        "device": {
-            "battery_volts":    idf_report.get("battery_volts"),
-            "calibration_date": cal_iso,
-            "calibration_by":   cal_by,
-        },
-        "peaks": {
-            "tran":       _channel_peaks(idf_report, "tran"),
-            "vert":       _channel_peaks(idf_report, "vert"),
-            "long":       _channel_peaks(idf_report, "long"),
-            "vector_sum": vs_block,
-        },
-        "mic":          mic_block,
-        "sensor_check": {
-            "tran": _sensor_check(idf_report, "tran"),
-            "vert": _sensor_check(idf_report, "vert"),
-            "long": _sensor_check(idf_report, "long"),
-            "mic":  _sensor_check(idf_report, "mic"),
-        },
-        "histogram":    hist_block,
-        "monitor_log":  [],
-        "pc_sw_version": None,
-    }

micromate/models.py

@@ -1,398 +0,0 @@
-"""
-Micromate (Series IV / Thor) native data models.
-
-These are the right-shaped dataclasses for Thor data — Thor measures
-the microphone in dB(L) directly, so this model carries
-``mic_pspl_dbl`` rather than the pseudo-``psi`` shoehorn that
-``minimateplus.PeakValues`` uses for Series III BW data.
-
-The ingest pipeline today goes:
-
-    .IDFW.txt  →  parse_idf_report()  →  dict
-    dict       →  IdfEvent.from_report()  →  IdfEvent  (typed)
-    IdfEvent   →  IdfEvent.to_minimateplus_event()  →  shape DB / sidecar
-                                                     machinery expects
-
-The ``to_minimateplus_event()`` bridge is a temporary boundary — when we
-crack the binary IDF codec and have richer per-event data to store, the
-DB schema will grow Series-IV-specific columns and the bridge will
-shrink or disappear.
-"""
-
-from __future__ import annotations
-
-import datetime
-from dataclasses import dataclass, field
-from typing import Any, Dict, Optional, Tuple
-
-
-# ── IdfReport ─────────────────────────────────────────────────────────────────
-
-
-@dataclass
-class IdfReport:
-    """Typed wrapper around the dict returned by ``parse_idf_report``.
-
-    All fields optional — Thor's exporter is permissive and some IDF .txt
-    files (especially histograms) omit fields that waveform sidecars
-    include.  Use ``.raw`` for any field this dataclass hasn't surfaced
-    yet (the parser keeps every recognised key in the raw dict).
-    """
-
-    # Identity / kind
-    serial_number:     Optional[str] = None
-    event_type:        Optional[str] = None      # "Full Waveform" | "Full Histogram"
-    event_datetime:    Optional[datetime.datetime] = None
-    filename:          Optional[str] = None      # echoed by Thor's exporter
-
-    # Sampling / timing
-    sample_rate:       Optional[int]   = None    # samples/sec
-    record_time_sec:   Optional[float] = None
-    pre_trigger_sec:   Optional[float] = None
-
-    # Geophone peaks (in/s)
-    tran_ppv:          Optional[float] = None
-    vert_ppv:          Optional[float] = None
-    long_ppv:          Optional[float] = None
-    peak_vector_sum:   Optional[float] = None
-
-    # Microphone — Thor's native unit is dB(L), NOT psi.
-    mic_pspl_dbl:      Optional[float] = None
-
-    # Zero-crossing frequencies (Hz)
-    tran_zc_freq:      Optional[float] = None
-    vert_zc_freq:      Optional[float] = None
-    long_zc_freq:      Optional[float] = None
-    mic_zc_freq:       Optional[float] = None
-
-    # Per-channel time of peak (sec, since event start)
-    tran_time_of_peak: Optional[float] = None
-    vert_time_of_peak: Optional[float] = None
-    long_time_of_peak: Optional[float] = None
-    mic_time_of_peak:  Optional[float] = None
-
-    # Derived per-channel motion
-    tran_peak_acceleration: Optional[float] = None    # g
-    vert_peak_acceleration: Optional[float] = None
-    long_peak_acceleration: Optional[float] = None
-    tran_peak_displacement: Optional[float] = None    # in
-    vert_peak_displacement: Optional[float] = None
-    long_peak_displacement: Optional[float] = None
-
-    # Operator-supplied strings (Thor's TitleString1..4 → semantic slots)
-    project:           Optional[str] = None    # TitleString1
-    client:            Optional[str] = None    # TitleString2
-    operator:          Optional[str] = None    # TitleString3
-    notes:             Optional[str] = None    # TitleString4 / PostEventNote
-    setup:             Optional[str] = None    # setup file name
-
-    # Sensor self-check results
-    tran_test_passed:  Optional[bool] = None
-    vert_test_passed:  Optional[bool] = None
-    long_test_passed:  Optional[bool] = None
-    mic_test_passed:   Optional[bool] = None
-
-    # Device-fixed metadata
-    firmware_version:  Optional[str]   = None
-    calibration_text:  Optional[str]   = None
-    battery_volts:     Optional[float] = None
-
-    # Original parser dict — preserves every recognised key (including
-    # raw unit-suffixed strings) for forward-compatible field access.
-    raw: Dict[str, Any] = field(default_factory=dict, repr=False)
-
-    @classmethod
-    def from_dict(cls, d: Dict[str, Any]) -> "IdfReport":
-        """Build an IdfReport from the dict returned by ``parse_idf_report``."""
-        ed = d.get("event_datetime")
-        if isinstance(ed, str):
-            try:
-                ed = datetime.datetime.fromisoformat(ed)
-            except ValueError:
-                ed = None
-
-        return cls(
-            serial_number     = d.get("serial_number"),
-            event_type        = d.get("event_type"),
-            event_datetime    = ed if isinstance(ed, datetime.datetime) else None,
-            filename          = d.get("filename"),
-            sample_rate       = d.get("sample_rate"),
-            record_time_sec   = d.get("record_time_sec"),
-            pre_trigger_sec   = d.get("pre_trigger_sec"),
-            tran_ppv          = d.get("tran_ppv"),
-            vert_ppv          = d.get("vert_ppv"),
-            long_ppv          = d.get("long_ppv"),
-            peak_vector_sum   = d.get("peak_vector_sum"),
-            mic_pspl_dbl      = d.get("mic_ppv"),       # parser names it mic_ppv (legacy)
-            tran_zc_freq      = d.get("tran_zc_freq"),
-            vert_zc_freq      = d.get("vert_zc_freq"),
-            long_zc_freq      = d.get("long_zc_freq"),
-            mic_zc_freq       = d.get("mic_zc_freq"),
-            tran_time_of_peak = d.get("tran_time_of_peak"),
-            vert_time_of_peak = d.get("vert_time_of_peak"),
-            long_time_of_peak = d.get("long_time_of_peak"),
-            mic_time_of_peak  = d.get("mic_time_of_peak"),
-            tran_peak_acceleration = d.get("tran_peak_acceleration"),
-            vert_peak_acceleration = d.get("vert_peak_acceleration"),
-            long_peak_acceleration = d.get("long_peak_acceleration"),
-            tran_peak_displacement = d.get("tran_peak_displacement"),
-            vert_peak_displacement = d.get("vert_peak_displacement"),
-            long_peak_displacement = d.get("long_peak_displacement"),
-            project           = d.get("project"),
-            client            = d.get("client"),
-            operator          = d.get("operator"),
-            notes             = d.get("notes"),
-            setup             = d.get("setup"),
-            tran_test_passed  = d.get("tran_test_passed"),
-            vert_test_passed  = d.get("vert_test_passed"),
-            long_test_passed  = d.get("long_test_passed"),
-            mic_test_passed   = d.get("mic_test_passed"),
-            firmware_version  = d.get("version"),
-            calibration_text  = d.get("calibration_text"),
-            battery_volts     = d.get("battery_volts"),
-            raw               = d,
-        )
-
-
-# ── IdfPeaks / IdfProjectInfo / IdfSensorCheck (narrow grouping types) ───────
-
-
-@dataclass
-class IdfPeaks:
-    """Geophone + mic peak values for one Thor event.  Native Thor units.
-
-    Thor stores the mic peak in two parallel forms — ``mic_pspl_dbl`` is
-    what the sidecar's top-level ``MicPSPL`` header field carries (dB(L)),
-    used in the report header.  ``mic_pspl_psi`` is the psi value derived
-    either from the IDFW sample table / IDFH interval column 9, or from
-    the binary mic counts (~2.14e-6 psi/count).  Needed because the
-    BW-shaped ``PeakValues.micl`` consumed by ``event_hdf5.write_event_hdf5``
-    expects psi — feeding it dB(L) makes the h5 mic-chart scale factor
-    blow up.
-    """
-    transverse_ips:    Optional[float] = None    # in/s
-    vertical_ips:      Optional[float] = None    # in/s
-    longitudinal_ips:  Optional[float] = None    # in/s
-    peak_vector_sum_ips: Optional[float] = None  # in/s
-    mic_pspl_dbl:      Optional[float] = None    # dB(L)
-    mic_pspl_psi:      Optional[float] = None    # psi
-
-
-@dataclass
-class IdfProjectInfo:
-    """Operator-supplied strings from Thor's TitleString1..4."""
-    project:  Optional[str] = None
-    client:   Optional[str] = None
-    operator: Optional[str] = None
-    notes:    Optional[str] = None
-    setup:    Optional[str] = None
-
-
-@dataclass
-class IdfSensorCheck:
-    """Per-channel pass/fail from Thor's self-test."""
-    tran: Optional[bool] = None
-    vert: Optional[bool] = None
-    long: Optional[bool] = None
-    mic:  Optional[bool] = None
-
-
-# ── IdfEvent ─────────────────────────────────────────────────────────────────
-
-
-@dataclass
-class IdfEvent:
-    """A single Thor / Micromate Series IV event.
-
-    Built from a parsed .IDFW.txt or .IDFH.txt sidecar via
-    ``IdfEvent.from_report()``.  The filename is the authoritative
-    source for serial + timestamp + kind; the .txt provides
-    device-authoritative peak values, frequencies, project strings,
-    sensor self-check, firmware, calibration.
-    """
-
-    # Identity
-    serial:    str
-    timestamp: datetime.datetime
-    kind:      str                  # "Waveform" | "Histogram"
-    filename:  str                  # device-native binary filename, e.g. "UM11719_20231219163444.IDFW"
-
-    # Sampling / timing
-    sample_rate:     Optional[int]   = None
-    record_time_sec: Optional[float] = None
-    pre_trigger_sec: Optional[float] = None
-
-    # Peaks
-    peaks: IdfPeaks = field(default_factory=IdfPeaks)
-
-    # Per-channel frequencies (Hz)
-    tran_zc_freq: Optional[float] = None
-    vert_zc_freq: Optional[float] = None
-    long_zc_freq: Optional[float] = None
-    mic_zc_freq:  Optional[float] = None
-
-    # Project strings
-    project_info: IdfProjectInfo = field(default_factory=IdfProjectInfo)
-
-    # Sensor self-check
-    sensor_check: IdfSensorCheck = field(default_factory=IdfSensorCheck)
-
-    # Device-fixed
-    firmware_version: Optional[str]   = None
-    calibration_text: Optional[str]   = None
-    battery_volts:    Optional[float] = None
-
-    # The full parsed report — preserves anything not surfaced as a typed field
-    report: IdfReport = field(default_factory=IdfReport)
-
-    @classmethod
-    def from_report(
-        cls,
-        report: Any,
-        filename: str,
-    ) -> "IdfEvent":
-        """Build an IdfEvent from a parsed report (dict or IdfReport) and
-        the device-native binary filename.
-
-        The filename is authoritative for serial + timestamp + kind:
-        Thor's filenames are literal ``<SERIAL>_<YYYYMMDDHHMMSS>.<KIND>``
-        and the device's own clock is the canonical event timestamp.
-        If the report carries an ``event_datetime`` that differs from
-        what's in the filename, the report wins (it has finer-grained
-        device-reported time-of-trigger semantics).
-        """
-        from .idf_ascii_report import parse_event_filename
-
-        # Normalise input to IdfReport
-        if isinstance(report, IdfReport):
-            rep = report
-        elif isinstance(report, dict):
-            rep = IdfReport.from_dict(report)
-        else:
-            raise TypeError(
-                f"report must be IdfReport or dict; got {type(report).__name__}"
-            )
-
-        # Filename → (serial, timestamp, kind).  Required — fall back to
-        # report-supplied values only if filename parsing fails.
-        parsed = parse_event_filename(filename)
-        if parsed is not None:
-            fn_serial, fn_ts, fn_kind = parsed
-            kind = "Histogram" if fn_kind == "IDFH" else "Waveform"
-        else:
-            fn_serial = rep.serial_number or "UNKNOWN"
-            fn_ts     = rep.event_datetime or datetime.datetime(1970, 1, 1)
-            kind      = "Waveform" if (rep.event_type or "").lower().startswith("full waveform") else "Histogram"
-
-        # Prefer report's event_datetime (device-authoritative) over the filename.
-        ts = rep.event_datetime or fn_ts
-        serial = rep.serial_number or fn_serial
-
-        return cls(
-            serial=serial,
-            timestamp=ts,
-            kind=kind,
-            filename=filename,
-            sample_rate=rep.sample_rate,
-            record_time_sec=rep.record_time_sec,
-            pre_trigger_sec=rep.pre_trigger_sec,
-            peaks=IdfPeaks(
-                transverse_ips      = rep.tran_ppv,
-                vertical_ips        = rep.vert_ppv,
-                longitudinal_ips    = rep.long_ppv,
-                peak_vector_sum_ips = rep.peak_vector_sum,
-                mic_pspl_dbl        = rep.mic_pspl_dbl,
-            ),
-            tran_zc_freq=rep.tran_zc_freq,
-            vert_zc_freq=rep.vert_zc_freq,
-            long_zc_freq=rep.long_zc_freq,
-            mic_zc_freq=rep.mic_zc_freq,
-            project_info=IdfProjectInfo(
-                project=rep.project,
-                client=rep.client,
-                operator=rep.operator,
-                notes=rep.notes,
-                setup=rep.setup,
-            ),
-            sensor_check=IdfSensorCheck(
-                tran=rep.tran_test_passed,
-                vert=rep.vert_test_passed,
-                long=rep.long_test_passed,
-                mic=rep.mic_test_passed,
-            ),
-            firmware_version=rep.firmware_version,
-            calibration_text=rep.calibration_text,
-            battery_volts=rep.battery_volts,
-            report=rep,
-        )
-
-    # ── Bridge to minimateplus shape (for the existing DB / sidecar paths) ──
-
-    def to_minimateplus_event(self, waveform_key: bytes) -> Any:
-        """Project this Thor event into the shape ``minimateplus.Event``
-        carries, so it can flow through the existing
-        ``SeismoDb.insert_events()`` and ``event_to_sidecar_dict()``
-        machinery without those code paths needing to know about Thor.
-
-        Caveats of the bridge:
-          - ``PeakValues.micl`` carries the mic peak in **psi** (matching
-            BW's convention) — set from :attr:`IdfPeaks.mic_pspl_psi`,
-            with a dB(L)→psi fallback when only the dB(L) value is
-            available.  This is what the h5 writer's mic-scale-factor
-            logic needs.  The dB(L) value still flows through
-            ``bw_report.mic.pspl_dbl`` (set by the
-            ``idf_to_bw_report`` adapter) and the renderer reads it
-            from there for the report header.
-          - Many Thor-specific fields (Peak Acceleration / Displacement,
-            sensor self-check, calibration) don't have a slot in
-            ``Event``.  The full IdfReport is preserved on the
-            ``.sfm.json`` sidecar under ``extensions.idf_report`` via
-            ``save_imported_idf`` — that's the source of truth for them.
-        """
-        from minimateplus.models import (
-            Event, PeakValues, ProjectInfo, Timestamp,
-        )
-
-        ts_obj = Timestamp(
-            raw=bytes(9),
-            flag=0,
-            year=self.timestamp.year,
-            unknown_byte=0,
-            month=self.timestamp.month,
-            day=self.timestamp.day,
-            hour=self.timestamp.hour,
-            minute=self.timestamp.minute,
-            second=self.timestamp.second,
-        )
-        # Resolve mic peak as psi.  Priority: binary-derived mic_pspl_psi
-        # (set by read_idf_file) > dB(L)→psi fallback via standard formula
-        # (psi = 2.9e-9 × 10^(dBL/20)) > None.
-        mic_psi = self.peaks.mic_pspl_psi
-        if mic_psi is None and self.peaks.mic_pspl_dbl is not None:
-            mic_psi = 2.9e-9 * (10.0 ** (self.peaks.mic_pspl_dbl / 20.0))
-        pv = PeakValues(
-            tran=self.peaks.transverse_ips,
-            vert=self.peaks.vertical_ips,
-            long=self.peaks.longitudinal_ips,
-            micl=mic_psi,   # psi, matching BW's convention (h5 scaling depends on this)
-            peak_vector_sum=self.peaks.peak_vector_sum_ips,
-        )
-        pi = ProjectInfo(
-            setup_name=self.project_info.setup,
-            project=self.project_info.project,
-            client=self.project_info.client,
-            operator=self.project_info.operator,
-            sensor_location=None,           # Thor folds location into project string
-            notes=self.project_info.notes,
-        )
-        ev = Event(
-            index=0,
-            timestamp=ts_obj,
-            sample_rate=self.sample_rate,
-            peak_values=pv,
-            project_info=pi,
-            record_type=self.kind,
-            rectime_seconds=self.record_time_sec,
-        )
-        ev._waveform_key = waveform_key
-        return ev

micromate/sensor_check.py

@@ -1,89 +0,0 @@
-r"""Decode the Thor / Micromate (series-4) sensor self-check waveforms from an
-IDFW event binary.
-
-Reverse-engineered 2026-09-15 against 4 UM (Thor) oracle events.  The IDFW
-binary carries the sensor self-check in its fixed-header region (before the
-waveform body), as up to four records tagged ``01 0e 3c/3d/3e/3f`` — the SAME
-channel ids as the series-3 MiniMate Plus (Tran / Vert / Long / MicL), which is
-the physical self-test:
-
-  * 3c / 3d / 3e = Tran / Vert / Long geophone ring-downs (a damped impulse
-    response — resonant frequency + damping).
-  * 3f          = MicL pulse train (the mic's known-signal gain check).  Absent
-    on three-channel (mic-disabled) units.
-
-Record framing (per record)::
-
-    01 0e [id:1] [flags:3] [count:2 BE] [pad:10] [int16-BE samples × count]
-    \___ 18-byte header ___/
-
-Unlike series-3's delta-coded trailing block, series-4 stores each trace as a
-raw int16 big-endian array.  ``count`` (the 2-byte field at header offset +8)
-is the sample count; the record is padded to a fixed stride after that.
-"""
-from __future__ import annotations
-
-import struct
-from typing import Dict, List
-
-# Record id → channel.  Same ids/order as series-3 (minimateplus.sensor_check).
-_ID_TO_CHANNEL = {0x3C: "Tran", 0x3D: "Vert", 0x3E: "Long", 0x3F: "MicL"}
-_CHAIN_IDS = (0x3C, 0x3D, 0x3E, 0x3F)
-
-_MARKER = b"\x01\x0e"       # precedes the 1-byte channel id
-_HEADER_LEN = 18            # bytes from the marker start to the first sample
-_COUNT_OFF = 8              # 2-byte BE sample count, from the marker start
-_MAX_COUNT = 4000           # sanity cap (traces are ~70-200 samples)
-
-
-def _find_chain(raw: bytes):
-    """Locate the sensor-check record chain.  Returns a list of
-    ``(offset, id, count)`` for the first run of markers whose ids run
-    3c, 3d, 3e[, 3f] in order, or ``[]``.
-
-    Records are padded to a fixed stride, so the next marker is not at
-    ``header + count*2``; instead collect every ``01 0e [id]`` marker with a
-    sane count and take the first id-ordered run.  Validating the id sequence
-    (not a lone ``01 0e 3c``) keeps a stray marker in the waveform body from
-    matching — the real chain sits in the fixed header, ahead of the body.
-    """
-    n = len(raw)
-    markers = []
-    for p in range(n - _HEADER_LEN):
-        if raw[p:p + 2] == _MARKER and raw[p + 2] in _ID_TO_CHANNEL:
-            count = int.from_bytes(raw[p + _COUNT_OFF:p + _COUNT_OFF + 2], "big")
-            if 0 < count <= _MAX_COUNT:
-                markers.append((p, raw[p + 2], count))
-
-    for i, (off, rid, _c) in enumerate(markers):
-        if rid != 0x3C:
-            continue
-        run = [markers[i]]
-        for m in markers[i + 1:]:
-            if len(run) < len(_CHAIN_IDS) and m[1] == _CHAIN_IDS[len(run)]:
-                run.append(m)
-            else:
-                break
-        if len(run) >= 3:       # 3-channel (mic-disabled) units are valid
-            return run
-    return []
-
-
-def decode_idf_sensor_check(raw: bytes) -> Dict[str, List[int]]:
-    """Decode the sensor self-check traces from a Thor/Micromate IDFW binary.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}`` in
-    raw int16 ADC counts (MicL omitted on 3-channel units), or ``{}`` if the
-    binary carries no sensor-check chain (a non-IDF file, or an IDFH histogram).
-    """
-    chain = _find_chain(raw)
-    if not chain:
-        return {}
-    out: Dict[str, List[int]] = {}
-    for off, rid, count in chain:
-        start = off + _HEADER_LEN
-        blob = raw[start:start + count * 2]
-        if len(blob) < count * 2:
-            continue
-        out[_ID_TO_CHANNEL[rid]] = list(struct.unpack(">%dh" % count, blob))
-    return out

minimateplus/__init__.py

@@ -21,15 +21,7 @@ Typical usage (TCP / modem):
 
 from .client import MiniMateClient
 from .models import DeviceInfo, Event, MonitorLogEntry
-from .transport import CapturingTransport, SerialTransport, TcpTransport
+from .transport import SerialTransport, TcpTransport
 
 __version__ = "0.1.0"
-__all__ = [
+__all__ = ["MiniMateClient", "DeviceInfo", "Event", "MonitorLogEntry", "SerialTransport", "TcpTransport"]
-    "MiniMateClient",
-    "DeviceInfo",
-    "Event",
-    "MonitorLogEntry",
-    "SerialTransport",
-    "TcpTransport",
-    "CapturingTransport",
-]

minimateplus/binary_annotate.py

@@ -1,75 +0,0 @@
-"""Structural annotation of a Series-3 Blastware waveform binary.
-
-Pure, no I/O: takes the raw file bytes and returns a flat, gap-free tiling of
-labelled :class:`Span` regions for a hex viewer to paint.  Every byte is
-covered — anything the decoder can't account for becomes an ``unknown`` span,
-so undecoded regions (e.g. a stored spectral/FFT block, if one exists) stand
-out instead of hiding.
-
-File layout (see ``blastware_file.py``):  ``[header][21B STRT][body][26B footer]``.
-The body is the record chain walked by :func:`waveform_codec.walk_records`.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import List
-
-from .waveform_codec import walk_records
-
-_STRT_LEN = 21
-_FOOTER_LEN = 26
-
-
-@dataclass
-class Span:
-    start: int          # inclusive byte offset
-    end: int            # exclusive byte offset
-    label: str          # human-readable description
-    kind: str           # 'header' | 'strt' | 'sample' | 'footer' | 'unknown'
-
-
-def _tile(known: List[Span], total: int) -> List[Span]:
-    """Sort *known* spans and fill every gap with an ``unknown`` span, so the
-    result is a contiguous, non-overlapping tiling of ``[0, total)``.  Overlaps
-    are resolved by clamping to the running position (first writer wins)."""
-    out: List[Span] = []
-    pos = 0
-    for s in sorted(known, key=lambda x: (x.start, x.end)):
-        if s.end <= pos:
-            continue                       # fully behind — dropped overlap
-        start = max(s.start, pos)
-        if start > pos:
-            out.append(Span(pos, start, "unknown", "unknown"))
-        out.append(s if start == s.start else Span(start, s.end, s.label, s.kind))
-        pos = s.end
-    if pos < total:
-        out.append(Span(pos, total, "unknown", "unknown"))
-    return out
-
-
-def annotate_blastware_binary(raw: bytes) -> List[Span]:
-    """Annotate a Series-3 waveform binary into a gap-free list of spans."""
-    total = len(raw)
-    strt_pos = raw.find(b"STRT")
-    if strt_pos < 0:
-        return [Span(0, total, "unrecognized — no STRT record", "unknown")]
-
-    known: List[Span] = []
-    if strt_pos > 0:
-        known.append(Span(0, strt_pos, "File header", "header"))
-    known.append(Span(strt_pos, strt_pos + _STRT_LEN, "STRT record", "strt"))
-
-    body_start = strt_pos + _STRT_LEN
-    footer_start = total - _FOOTER_LEN
-    if footer_start >= body_start:
-        known.append(Span(footer_start, total, "File footer", "footer"))
-    else:
-        footer_start = total          # file too short for a footer
-
-    body = raw[body_start:footer_start]
-    for rec in walk_records(body):
-        hi, lo = rec["mode"]
-        label = f"{rec['channel']} record (seg {rec['segment_index']}, mode {hi:02x} {lo:02x})"
-        known.append(Span(body_start + rec["offset"], body_start + rec["end"], label, "sample"))
-
-    return _tile(known, total)

minimateplus/blastware_file.py

@@ -552,105 +552,6 @@ def classify_frame(frame: S3Frame) -> str:
 
 # ── Waveform file writer ───────────────────────────────────────────────────────────
 
-def extract_body_bytes(a5_frames):
-    """Reconstruct the Blastware-file body bytes from a list of A5 frames.
-
-    Returns ``(strt, body, footer)`` where:
-
-    - ``strt`` is the 21-byte STRT record from the probe frame (or a fallback
-      record built from minimal event metadata if STRT is missing).
-    - ``body`` is the variable-length sample-data section (between STRT and
-      the 26-byte file footer).  Empty if no frames decode.
-    - ``footer`` is the 26-byte file footer.
-
-    This is the same body-construction algorithm used by :func:`write_blastware_file`
-    — refactored out so the body decoder (``waveform_codec.decode_waveform_v2``)
-    can consume the same bytes without re-implementing the frame-walking logic.
-
-    Returns ``(b"", b"", b"")`` if *a5_frames* is empty.
-    """
-    if not a5_frames:
-        return (b"", b"", b"")
-
-    # ── Extract STRT record from probe frame ─────────────────────────────────
-    w0_raw = bytes(a5_frames[0].data[7:])
-    w0_stripped = _strip_inner_frame_dles(w0_raw)
-    strt_pos_stripped = w0_stripped.find(b"STRT")
-
-    if strt_pos_stripped >= 0:
-        strt = bytes(w0_stripped[strt_pos_stripped : strt_pos_stripped + 21])
-
-        # Walk raw bytes to find the raw-domain end of the STRT (= body start).
-        target_stripped = strt_pos_stripped + 21
-        stripped_so_far = 0
-        raw_i = 0
-        while stripped_so_far < target_stripped and raw_i < len(w0_raw):
-            if (w0_raw[raw_i] == 0x10
-                    and raw_i + 1 < len(w0_raw)
-                    and w0_raw[raw_i + 1] in {0x02, 0x03, 0x04}):
-                raw_i += 2
-            else:
-                raw_i += 1
-            stripped_so_far += 1
-        probe_skip = 7 + raw_i
-    else:
-        strt = b"STRT" + b"\xff\xfe" + bytes(14) + b"\x00"
-        probe_skip = 7 + 21
-
-    if len(strt) != 21:
-        return (b"", b"", b"")
-
-    # Separate terminator from data frames.
-    term_idx: Optional[int] = None
-    if a5_frames and a5_frames[-1].page_key != 0x0010:
-        term_idx = len(a5_frames) - 1
-
-    if term_idx is not None:
-        body_frames = a5_frames[:term_idx]
-        term_frame = a5_frames[term_idx]
-    else:
-        body_frames = a5_frames
-        term_frame = None
-
-    all_bytes = bytearray()
-    for fi, frame in enumerate(body_frames):
-        if fi == 0:
-            skip = probe_skip
-        elif fi in (1, 2):
-            skip = 13   # metadata pages
-        else:
-            skip = 12   # sample chunks
-        all_bytes.extend(_frame_body_bytes(frame, skip))
-
-    if term_frame is not None:
-        all_bytes.extend(_frame_body_bytes(term_frame, 11))
-
-    # Find the first valid `0e 08` footer marker.
-    footer_pos = -1
-    pos = 0
-    while True:
-        pos = bytes(all_bytes).find(b"\x0e\x08", pos)
-        if pos < 0 or pos + 26 > len(all_bytes):
-            break
-        yr = (all_bytes[pos + 4] << 8) | all_bytes[pos + 5]
-        if 2015 <= yr <= 2050:
-            footer_pos = pos
-            break
-        pos += 1
-
-    if footer_pos >= 0:
-        body = bytes(all_bytes[:footer_pos])
-        footer = bytes(all_bytes[footer_pos : footer_pos + 26])
-    elif len(all_bytes) >= 26:
-        body = bytes(all_bytes[:-26])
-        footer = bytes(all_bytes[-26:])
-    else:
-        body = bytes(all_bytes)
-        footer = b""
-
-    return (strt, body, footer)
-
-
 def write_blastware_file(
     event: Event,
     a5_frames: list[S3Frame],
@@ -738,7 +639,7 @@ def write_blastware_file(
         strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF])
         probe_skip = 7 + 21
 
-    log.debug(
+    log.warning(
         "write_blastware_file: strt_pos_stripped=%d  probe_skip=%d  "
         "probe_data_len=%d  strt_hex=%s",
         strt_pos_stripped if strt_pos_stripped >= 0 else -1,
@@ -771,10 +672,11 @@ def write_blastware_file(
     # Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a
     # subsequent event (a known get_events side-effect), the last frame will
     # not be the terminator and the footer will be mis-identified.
-    # TERM detection (v0.14.0): last frame if page_key != 0x0010 (sample marker)
     term_idx: Optional[int] = None
-    if a5_frames and a5_frames[-1].page_key != 0x0010:
+    for _i, _f in enumerate(a5_frames):
-        term_idx = len(a5_frames) - 1
+        if _f.page_key == 0x0000:
+            term_idx = _i
+            break
 
     if term_idx is not None:
         body_frames = a5_frames[:term_idx]
@@ -783,32 +685,38 @@ def write_blastware_file(
         body_frames = a5_frames
         term_frame  = None
 
-    # Frame contribution loop (v0.14.0 BW-exact walk).
+    log.warning(
-    # Skip values:
+        "write_blastware_file: %d body_frames  term_idx=%s",
-    #   probe (fi=0):           probe_skip
+        len(body_frames),
-    #   meta@0x1002 (fi=1):     13 (6-byte inner header)
+        str(term_idx) if term_idx is not None else "None",
-    #   meta@0x1004 (fi=2):     13 (6-byte inner header)
-    #   sample chunks (fi=3+):  12 (5-byte inner header)
-    last_fi = len(body_frames) - 1
-
-    log.debug(
-        "write_blastware_file: %d body_frames  last_fi=%d",
-        len(body_frames), last_fi,
     )
 
     all_bytes = bytearray()
 
     for fi, frame in enumerate(body_frames):
+        # All body frames contribute to the waveform body — no frames are skipped.
+        #
+        # Over TCP via cellular modem, _recv_5a_batch() correctly collects all
+        # A5 frames per chunk request (the device's ~1100-byte RS-232 response
+        # is forwarded as ~2 TCP segments of ~550 bytes each, each parsed as a
+        # separate S3 frame).  ALL of these frames contain ADC body data and
+        # must be included in the file — confirmed from 4-27-26 TCP capture
+        # analysis: contributions from all 14 frames → 6821 bytes → file 6864 bytes.
+        #
+        # Skip amounts (offsets into frame.data):
+        #   fi=0 (probe): probe_skip — skips the type_tag header + STRT record
+        #   fi=1:         13         — 7-byte frame.data prefix + 6 inner header bytes
+        #   fi>=2:        12         — 7-byte frame.data prefix + 5 inner header bytes
         if fi == 0:
             skip = probe_skip
-        elif fi in (1, 2):
+        elif fi == 1:
-            skip = 13   # metadata pages
+            skip = 13
         else:
-            skip = 12   # sample chunks
+            skip = 12
 
         contribution = _frame_body_bytes(frame, skip)
-        log.debug("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
+        log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
-                  fi, skip, len(frame.data), len(contribution))
+                    fi, skip, len(frame.data), len(contribution))
         all_bytes.extend(contribution)
 
     # Terminator contributes its content, which ends with the 26-byte footer.
@@ -816,7 +724,7 @@ def write_blastware_file(
     # one shorter than chunk frames' 5-byte inner header.  Confirmed 2026-04-21.
     if term_frame is not None:
         term_contribution = _frame_body_bytes(term_frame, 11)
-        log.debug(
+        log.warning(
             "write_blastware_file: term_frame data_len=%d  skip=11  "
             "contribution_len=%d  first8=%s",
             len(term_frame.data),
@@ -825,49 +733,17 @@ def write_blastware_file(
         )
         all_bytes.extend(term_contribution)
 
-    log.debug(
+    log.warning(
         "write_blastware_file: all_bytes total=%d  last28=%s",
         len(all_bytes),
         bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(),
     )
 
-    # NOTE: The "duplicate header+STRT strip" logic from v0.13.x has been
+    if len(all_bytes) >= 26:
-    # REMOVED in v0.14.2.  Under the v0.14.0 BW-exact 5A walk, body assembly
-    # is just contiguous concatenation of frame contributions in stream order
-    # (probe → meta@0x1002 → meta@0x1004 → samples → TERM), exactly as BW
-    # writes its files.  The previous strip was matching the `00 12 03 00 STRT`
-    # byte sequence in legitimate waveform data — sample chunks at counter
-    # 0x1000 and beyond often contain those bytes coincidentally — and
-    # zeroing 25 bytes of valid samples per match.  Compared to a known-good
-    # BW reference for the same 3-sec event 0, the strip introduced 26 bytes
-    # of zeros that BW did not have, then propagated alignment differences
-    # through the rest of the body.  See decode_test/5-1-26/bw vs SFM diff
-    # at file[0x1012..0x102B] (2026-05-04 analysis).
-
-    # Find the first valid 0e 08 footer marker (v0.14.0).
-    footer_pos = -1
-    pos = 0
-    while True:
-        pos = bytes(all_bytes).find(b"\x0e\x08", pos)
-        if pos < 0 or pos + 26 > len(all_bytes):
-            break
-        yr = (all_bytes[pos + 4] << 8) | all_bytes[pos + 5]
-        if 2015 <= yr <= 2050:
-            footer_pos = pos
-            break
-        pos += 1
-    if footer_pos >= 0:
-        body   = bytes(all_bytes[:footer_pos])
-        footer = bytes(all_bytes[footer_pos:footer_pos + 26])
-        log.debug(
-            "write_blastware_file: real 0e 08 footer at all_bytes[%d]; "
-            "truncating %d post-footer bytes",
-            footer_pos, len(all_bytes) - footer_pos - 26,
-        )
-    elif len(all_bytes) >= 26:
         body   = bytes(all_bytes[:-26])
         footer = bytes(all_bytes[-26:])
     else:
+        # Fallback: no terminator or very short stream → build footer from event metadata
         body = bytes(all_bytes)
         start_dt = _ts_from_model(event.timestamp)
         stop_dt: Optional[datetime.datetime] = None
@@ -878,7 +754,7 @@ def write_blastware_file(
             + _encode_ts_be(start_dt)
             + _encode_ts_be(stop_dt)
             + b"\x00\x01\x00\x02\x00\x00"
-            + b"\x00\x00"
+            + b"\x00\x00"   # CRC placeholder
         )
 
     # ── Write file ───────────────────────────────────────────────────────────

minimateplus/bw_ascii_report.py

@@ -1,738 +0,0 @@
-"""
-minimateplus/bw_ascii_report.py — parser for Blastware's per-event ASCII
-report (the .TXT file BW writes alongside each saved event binary).
-
-The ASCII export is the authoritative source for every "rich" per-event
-field that BW computes from the waveform but never persists in the BW
-binary itself:
-
-  - Per-channel PPV (Tran / Vert / Long / MicL)
-  - Peak Vector Sum + Peak Vector Sum Time
-  - Per-channel ZC Freq, Time of Peak, Peak Acceleration, Peak Displacement
-  - MicL PSPL, MicL Time of Peak, MicL ZC Freq
-  - Per-channel Sensor Self-Check (Test Freq / Test Ratio / Test Results)
-  - MicL Test Amplitude (mV)
-  - Battery, calibration date, monitor-log timestamps
-
-Persisting these values into the SFM database lets the monthly-summary
-review workflow ("show me events at Location X with PVS > 0.5") work
-without depending on the (still-undecoded) waveform body codec.
-
-Format (verified against decode-re/5-8-26 4-event bundle):
-
-  - One field per line, wrapped in double quotes:   `"Field Name : Value"`
-  - Field/value separator: literal ` : ` (space-colon-space).
-  - Some field names contain an internal `:` already (e.g. `"Project:"`),
-    so we split on the FIRST ` : ` only.
-  - Some fields have unit suffixes:  `"0.500 in/s"` / `"7.5 Hz"` / `"533 mv"`.
-  - A `"Monitor Log(s)"` marker line is followed by tab-separated rows
-    of `start_time<TAB>stop_time<TAB>description`.
-  - Final `"PC SW Version : ..."` line ends the metadata block.
-  - A blank line separates metadata from the sample table.
-  - Sample table starts with `   Tran   <TAB>   Vert   <TAB>...`, then
-    one row per sample (tab-separated, right-padded numeric values).
-  - Geo channel values are in in/s; MicL in dB(L) (or 0.000 below threshold).
-
-Because some metadata fields have whitespace quirks ("MicL  Time of
-Peak" has two spaces; the leading "Project:" value has its own colon),
-we normalise whitespace in the key before lookup.
-"""
-
-from __future__ import annotations
-
-import datetime
-import re
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Output dataclasses
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-@dataclass
-class ChannelStats:
-    """Per-channel derived stats, populated from an event report."""
-    ppv_ips:           Optional[float] = None      # in/s            (geo channels only)
-    zc_freq_hz:        Optional[float] = None      # Hz
-    time_of_peak_s:    Optional[float] = None      # seconds (relative to trigger; can be negative)
-    peak_accel_g:      Optional[float] = None      # g               (geo channels only)
-    peak_disp_in:      Optional[float] = None      # in              (geo channels only)
-    # When BW writes "OORANGE" (Out Of Range — truncated) for a PPV
-    # value, the true peak exceeded the channel's full-scale range.
-    # We substitute the range max (e.g. 10.000 in/s for Normal range)
-    # as a lower bound, and flag here so downstream UI / alerts know
-    # to render "> 10 in/s" or "saturated" instead of trusting the
-    # value as an exact measurement.
-    ppv_saturated:     bool = False
-    # Set when BW writes ">100 Hz" for ZC Freq — the zero-crossing
-    # algorithm's peak frequency exceeded the device's reporting
-    # ceiling (typically 100 Hz on V10.72).  zc_freq_hz gets the
-    # threshold (100.0) as a lower bound; downstream UI renders ">100".
-    zc_freq_above_range: bool = False
-
-
-@dataclass
-class MicStats:
-    """MicL-specific stats."""
-    weighting:         Optional[str]   = None      # e.g. "Linear Weighting"
-    pspl_dbl:          Optional[float] = None      # dB(L)
-    zc_freq_hz:        Optional[float] = None
-    time_of_peak_s:    Optional[float] = None
-    # Set when BW writes "OORANGE" for PSPL — mic exceeded its
-    # measurement range.  pspl_dbl gets the conservative upper bound
-    # 140 dBL (typical NL-43 max; some units cap at 148).  Consumers
-    # should render "> 140 dB(L)" or similar when this flag is set.
-    pspl_saturated:    bool = False
-    # Same semantics as ChannelStats.zc_freq_above_range — mic ZC
-    # peak exceeded device reporting ceiling.
-    zc_freq_above_range: bool = False
-
-
-@dataclass
-class SensorCheck:
-    """Per-channel sensor self-check result.
-
-    Geo channels report a frequency + ratio; MicL reports a frequency +
-    amplitude (mV).  All channels also have a Pass/Fail string.
-    """
-    test_freq_hz:      Optional[float] = None
-    test_ratio:        Optional[float] = None      # geo channels only
-    test_amplitude_mv: Optional[float] = None      # MicL only
-    test_results:      Optional[str]   = None      # "Passed" / "Failed"
-
-
-@dataclass
-class MonitorLogEntry:
-    """One row of the trailing Monitor Log(s) block."""
-    start_time:  Optional[datetime.datetime] = None
-    stop_time:   Optional[datetime.datetime] = None
-    description: Optional[str] = None
-
-
-# BW saturation marker — appears in PPV / Peak Vector Sum / similar
-# numeric fields when the underlying measurement exceeded the
-# channel's full-scale range (e.g., a geophone reading > 10 in/s at
-# Normal range, or a mic exceeding its sensitivity ceiling).  Treated
-# as "≥ range_max" + a saturated flag rather than discarded.
-# Appears as: ``"Tran PPV : OORANGE in/s"``
-_OORANGE_MARKERS = ("OORANGE", "OUT OF RANGE")
-
-
-def _is_oorange(value: str) -> bool:
-    """True when a BW numeric field is an Out-Of-Range saturation marker."""
-    s = value.strip().upper()
-    return any(m in s for m in _OORANGE_MARKERS)
-
-
-def _parse_above_range(value: str) -> Optional[float]:
-    """For BW "above-range" markers like ">100 Hz", return the threshold.
-
-    BW writes ZC Freq as ">100 Hz" when the zero-crossing algorithm sees
-    a peak too fast to count (device cuts off at 100 Hz).  Returns the
-    numeric portion after the '>' (e.g. 100.0), or None if `value` is
-    not an above-range marker.
-    """
-    s = value.strip()
-    if not s.startswith(">"):
-        return None
-    return _parse_number(s[1:])
-
-
-@dataclass
-class BwAsciiReport:
-    """Structured representation of one BW per-event ASCII export."""
-    # ── Identity ─────────────────────────────────────────────────────────────
-    event_type:        Optional[str] = None         # e.g. "Full Waveform"
-    serial:            Optional[str] = None         # e.g. "BE11529"
-    version:           Optional[str] = None         # firmware version line
-    file_name:         Optional[str] = None         # e.g. "M529LK44.AB0"
-    event_datetime:    Optional[datetime.datetime] = None  # parsed from Event Time + Event Date
-
-    # ── Trigger / recording config ──────────────────────────────────────────
-    trigger_channel:        Optional[str]   = None  # e.g. "Vert" or "From Unit"
-    geo_trigger_level_ips:  Optional[float] = None
-    pretrig_s:              Optional[float] = None  # negative seconds
-    record_time_s:          Optional[float] = None
-    record_stop_mode:       Optional[str]   = None
-    sample_rate_sps:        Optional[int]   = None
-    battery_volts:          Optional[float] = None
-    calibration_date:       Optional[datetime.date] = None
-    calibration_by:         Optional[str]   = None  # e.g. "Instantel"
-    units:                  Optional[str]   = None  # e.g. "in/s and dB(L)"
-
-    # ── Operator-supplied metadata ──────────────────────────────────────────
-    # Parsed by POSITION from the 4-line "User Notes" block BW writes
-    # between the `Units :` and `Geo Range :` lines.  Position-based so
-    # the values populate correctly even when an operator renames the
-    # labels in Blastware's Compliance Setup → Notes tab (the 4 labels
-    # are user-editable, e.g. "Seis Loc:" → "Building:" → "Site Address:").
-    # The original labels BW wrote are preserved in `user_note_labels`
-    # so terra-view can render them as the operator named them.
-    project:           Optional[str] = None     # position 1 (BW default label "Project:")
-    client:            Optional[str] = None     # position 2 (BW default label "Client:")
-    operator:          Optional[str] = None     # position 3 (BW default label "User Name:")
-    sensor_location:   Optional[str] = None     # position 4 (BW default label "Seis Loc:")
-
-    # Maps canonical slot name → the literal label BW wrote in the ASCII
-    # export.  Empty if the User Notes block wasn't present.  Example
-    # when the operator renamed slot 4 to "Building:":
-    #     {"project": "Project:", "client": "Client:",
-    #      "operator": "User Name:", "sensor_location": "Building:"}
-    user_note_labels:  Dict[str, str] = field(default_factory=dict)
-
-    # ── Geo channel scaling ─────────────────────────────────────────────────
-    geo_range_ips:     Optional[float] = None       # 10.000 / 1.250
-
-    # ── Per-channel derived stats (geo + mic) ───────────────────────────────
-    channels:          Dict[str, ChannelStats] = field(default_factory=dict)
-    mic:               MicStats = field(default_factory=MicStats)
-
-    # ── Vector sum ──────────────────────────────────────────────────────────
-    peak_vector_sum_ips:    Optional[float] = None
-    peak_vector_sum_time_s: Optional[float] = None
-    # Saturation flag — set when BW writes "OORANGE" for the PVS.  We
-    # then substitute sqrt(3) * geo_range_ips as a conservative upper
-    # bound (the theoretical maximum PVS when all 3 geo channels are
-    # simultaneously at full-scale).  Consumers should display this as
-    # ">{value} in/s" or similar.
-    peak_vector_sum_saturated: bool = False
-    # Histograms additionally have an absolute date+time for the PVS
-    # (it occurred at a specific interval).  Waveform reports show
-    # only the relative-time value above.
-    peak_vector_sum_when:   Optional[datetime.datetime] = None
-
-    # ── Histogram-specific fields (populated only when Event Type starts
-    # with 'Histogram' / 'Full Histogram' / 'Histogram + Continuous') ──
-    histogram_start:        Optional[datetime.datetime] = None
-    histogram_stop:         Optional[datetime.datetime] = None
-    histogram_n_intervals:  Optional[int]   = None      # e.g. 4, 1436
-    histogram_interval_size_str: Optional[str]   = None  # "1 minute" / "5 minutes" / "15 seconds"
-    histogram_interval_size_s:   Optional[float] = None  # parsed to seconds
-    # Per-channel absolute peak time+date (histogram-specific).  For
-    # waveform events these are None — those reports use the channel's
-    # time_of_peak_s (relative to trigger) instead.  Keyed by channel
-    # name ("Tran", "Vert", "Long", "MicL").
-    channel_peak_when:      Dict[str, datetime.datetime] = field(default_factory=dict)
-
-    # ── Sensor self-check (per channel) ─────────────────────────────────────
-    sensor_check:      Dict[str, SensorCheck] = field(default_factory=dict)
-
-    # ── Monitor log + tooling version ───────────────────────────────────────
-    monitor_log:       List[MonitorLogEntry] = field(default_factory=list)
-    pc_sw_version:     Optional[str] = None
-
-    # ── Sample table (optional; only parsed if requested) ───────────────────
-    # Each entry: (Tran, Vert, Long, MicL) in the report's units (geo
-    # channels in in/s, MicL in dB(L)).  None when parse_samples=False.
-    samples:           Optional[List[Tuple[float, float, float, float]]] = None
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Helpers
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-_KEY_NORMALISE_RE = re.compile(r"\s+")
-_NUMERIC_RE       = re.compile(r"^-?\d+(?:\.\d+)?")
-
-
-def _normalise_key(k: str) -> str:
-    """Collapse whitespace runs (incl. tabs) and strip — handles BW's
-    "MicL  Time of Peak" double-space and leading-colon quirks."""
-    return _KEY_NORMALISE_RE.sub(" ", k).strip()
-
-
-def _strip_quotes(line: str) -> str:
-    line = line.rstrip("\r\n")
-    if len(line) >= 2 and line.startswith('"') and line.endswith('"'):
-        return line[1:-1]
-    return line
-
-
-def _parse_number(value: str) -> Optional[float]:
-    """Pull the leading numeric portion out of a value like "0.500 in/s"."""
-    m = _NUMERIC_RE.match(value.strip())
-    if not m:
-        return None
-    try:
-        return float(m.group(0))
-    except ValueError:
-        return None
-
-
-def _parse_int(value: str) -> Optional[int]:
-    n = _parse_number(value)
-    return None if n is None else int(round(n))
-
-
-# Months exactly as BW writes them.
-_MONTHS = {
-    "January": 1, "February": 2, "March": 3, "April": 4,
-    "May": 5, "June": 6, "July": 7, "August": 8,
-    "September": 9, "October": 10, "November": 11, "December": 12,
-    # Short forms used in monitor-log rows ("Apr 23 /26").
-    "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "Jun": 6, "Jul": 7,
-    "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12,
-}
-
-
-def _parse_event_date(s: str) -> Optional[datetime.date]:
-    """Parse "April 23, 2026" or "May 8, 2026" → date."""
-    s = s.strip()
-    parts = s.replace(",", " ").split()
-    if len(parts) < 3:
-        return None
-    month_name, day_str, year_str = parts[0], parts[1], parts[2]
-    month = _MONTHS.get(month_name)
-    if month is None:
-        return None
-    try:
-        return datetime.date(int(year_str), month, int(day_str))
-    except ValueError:
-        return None
-
-
-def _parse_iso_date(s: str) -> Optional[datetime.date]:
-    """Parse "2026-05-16" → date.  Histograms use ISO format for their
-    Start Date / Stop Date / Peak Date fields; waveforms use the
-    "May 8, 2026" long form which `_parse_event_date` handles."""
-    s = s.strip()
-    try:
-        return datetime.date.fromisoformat(s)
-    except ValueError:
-        return None
-
-
-_INTERVAL_UNIT_SECONDS = {
-    "second": 1, "seconds": 1, "sec": 1, "secs": 1,
-    "minute": 60, "minutes": 60, "min": 60, "mins": 60,
-    "hour": 3600, "hours": 3600, "hr": 3600, "hrs": 3600,
-}
-
-
-def _parse_interval_size(s: str) -> Optional[float]:
-    """Parse "1 minute" / "5 minutes" / "15 seconds" / "2 seconds" → seconds.
-
-    Handles the BW Compliance Setup → Histogram Interval values verbatim
-    ("2 seconds", "5 seconds", "15 seconds", "1 minute", "5 minutes",
-    "15 minutes") plus a few defensive variants.
-    """
-    if not s:
-        return None
-    parts = s.strip().split()
-    if len(parts) < 2:
-        return None
-    try:
-        n = float(parts[0])
-    except ValueError:
-        return None
-    unit_per_s = _INTERVAL_UNIT_SECONDS.get(parts[1].lower())
-    if unit_per_s is None:
-        return None
-    return n * unit_per_s
-
-
-def _parse_event_time(s: str) -> Optional[datetime.time]:
-    """Parse "15:56:35" → time."""
-    s = s.strip()
-    try:
-        h, m, sec = s.split(":")
-        return datetime.time(int(h), int(m), int(sec))
-    except (ValueError, IndexError):
-        return None
-
-
-def _parse_calibration(value: str) -> Tuple[Optional[datetime.date], Optional[str]]:
-    """Parse "April 29, 2025 by Instantel" → (date, "Instantel")."""
-    parts = value.split(" by ", 1)
-    date = _parse_event_date(parts[0])
-    by = parts[1].strip() if len(parts) > 1 else None
-    return date, by
-
-
-def _parse_monitor_row(line: str) -> Optional[MonitorLogEntry]:
-    """Parse a tab-separated monitor log row.
-
-    Format: `<start>\t<stop>\t<desc>` where each timestamp is BW's
-    short form "Mon DD /YY HH:MM:SS" (e.g. "Apr 23 /26 15:46:16").
-    Year is encoded as a 2-digit suffix; we expand "/26" → 2026.
-    """
-    parts = line.split("\t")
-    if len(parts) < 2:
-        return None
-    start = _parse_monitor_ts(parts[0])
-    stop  = _parse_monitor_ts(parts[1])
-    desc  = parts[2].strip() if len(parts) > 2 else None
-    if start is None and stop is None and not desc:
-        return None
-    return MonitorLogEntry(start_time=start, stop_time=stop, description=desc)
-
-
-def _parse_monitor_ts(s: str) -> Optional[datetime.datetime]:
-    """Parse "Apr 23 /26 15:46:16" → datetime."""
-    s = s.strip()
-    parts = s.split()
-    if len(parts) < 4:
-        return None
-    month = _MONTHS.get(parts[0])
-    if month is None:
-        return None
-    try:
-        day = int(parts[1])
-        # parts[2] looks like "/26" → century-flip to 2026
-        yy = int(parts[2].lstrip("/"))
-        year = 2000 + yy if yy < 80 else 1900 + yy
-        h, m, sec = (int(x) for x in parts[3].split(":"))
-        return datetime.datetime(year, month, day, h, m, sec)
-    except (ValueError, IndexError):
-        return None
-
-
-# ── User-notes positional slot map ──────────────────────────────────────────
-#
-# Blastware's Compliance Setup → Notes tab shows four operator-supplied
-# fields whose LABELS the operator can rename (see screenshot in
-# project archive).  Defaults are "Project:" / "Client:" /
-# "User Name:" / "Seis Loc:", but an operator using a different
-# convention can rename them to anything ("Building:", "Site:",
-# "Address:", etc.).  The ASCII export reflects whatever the operator
-# typed, so label-based matching is fragile.
-#
-# What IS reliable: BW always writes the 4 user-notes lines in the
-# same order, contiguously between the `Units :` line and the
-# `Geo Range :` line.  We parse them by POSITION and preserve the
-# operator's labels in `report.user_note_labels` so terra-view can
-# render them as the operator intended.
-
-_USER_NOTE_SLOTS = ("project", "client", "operator", "sensor_location")
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Top-level parser
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-def parse_report(text: Union[str, bytes], *, parse_samples: bool = False) -> BwAsciiReport:
-    """Parse a BW per-event ASCII export into a structured BwAsciiReport.
-
-    Set ``parse_samples=True`` to also populate ``report.samples`` with
-    the trailing sample table.  Default False because the table is
-    huge and most callers only want metadata for indexing.
-    """
-    if isinstance(text, bytes):
-        text = text.decode("ascii", errors="replace")
-
-    report = BwAsciiReport()
-    # Pre-create channel stat slots so callers can rely on them existing.
-    for ch in ("Tran", "Vert", "Long", "MicL"):
-        report.channels.setdefault(ch, ChannelStats())
-        report.sensor_check.setdefault(ch, SensorCheck())
-
-    lines = text.splitlines()
-    i = 0
-    n = len(lines)
-
-    in_monitor_log_section = False
-    event_time_str: Optional[str] = None
-    event_date: Optional[datetime.date] = None
-
-    # User-notes block detection.  We enter the block after parsing
-    # the "Units :" line and exit on the "Geo Range :" line.  Inside,
-    # the first 4 unmatched `<label> : <value>` lines are assigned to
-    # the 4 canonical operator-supplied slots by POSITION (project,
-    # client, operator, sensor_location) regardless of what the
-    # operator named the labels in BW's Compliance Setup → Notes tab.
-    in_user_notes_block = False
-    user_note_position = 0
-
-    # Histogram-field staging — BW writes <Channel> Peak Time and
-    # <Channel> Peak Date on separate lines (and similarly Histogram
-    # Start Time / Date).  We stash the partial value when the time
-    # line arrives and combine it when the matching date line arrives.
-    _hist_start_time: Optional[datetime.time] = None
-    _hist_stop_time:  Optional[datetime.time] = None
-    _pending_peak_time: Dict[str, Optional[datetime.time]] = {}
-    _pvs_time_raw: Optional[str] = None  # last Peak Vector Sum Time value, raw
-
-    while i < n:
-        raw_line = lines[i]
-        i += 1
-        # Blank line marks the start of the sample table.
-        if raw_line.strip() == "":
-            break
-
-        line = _strip_quotes(raw_line)
-
-        # Monitor log section: "Monitor Log(s)" header followed by N rows
-        # (still inside double-quoted lines), terminated by a non-row line
-        # like "PC SW Version : ..." or a blank line.
-        if not in_monitor_log_section and line.strip() == "Monitor Log(s)":
-            in_monitor_log_section = True
-            continue
-        if in_monitor_log_section:
-            # Heuristic: monitor rows contain a tab; the next "Field : Value"
-            # line ends the section.
-            if "\t" in line:
-                entry = _parse_monitor_row(line)
-                if entry:
-                    report.monitor_log.append(entry)
-                continue
-            # Falls through to the field parser below; clear the flag.
-            in_monitor_log_section = False
-
-        # "Field : Value" — split on FIRST occurrence of " : "
-        idx = line.find(" : ")
-        if idx < 0:
-            continue
-        key = _normalise_key(line[:idx])
-        value = line[idx + 3 :].strip()
-
-        # ── Identity / config ────────────────────────────────────────────────
-        if   key == "Event Type":           report.event_type = value
-        elif key == "Serial Number":        report.serial = value
-        elif key == "Version":              report.version = value
-        elif key == "File Name":            report.file_name = value
-        elif key == "Event Time":           event_time_str = value
-        elif key == "Event Date":           event_date = _parse_event_date(value)
-
-        elif key == "Trigger":              report.trigger_channel = value
-        elif key == "Geo Trigger Level":    report.geo_trigger_level_ips = _parse_number(value)
-        elif key == "Pre-trigger Length":   report.pretrig_s = _parse_number(value)
-        elif key == "Record Time":          report.record_time_s = _parse_number(value)
-        elif key == "Record Stop Mode":     report.record_stop_mode = value
-        elif key == "Sample Rate":          report.sample_rate_sps = _parse_int(value)
-        elif key == "Battery Level":        report.battery_volts = _parse_number(value)
-        elif key == "Calibration":
-            report.calibration_date, report.calibration_by = _parse_calibration(value)
-        elif key == "Units":
-            report.units = value
-            # Entering the user-notes block.  Next ~4 lines until
-            # "Geo Range :" are the operator-supplied notes.
-            in_user_notes_block = True
-            user_note_position = 0
-
-        elif key == "Geo Range":
-            # Exiting the user-notes block.
-            in_user_notes_block = False
-            report.geo_range_ips = _parse_number(value)
-
-        # User-notes block: assign by position (operator may have
-        # renamed the labels, so we don't trust them).  Preserve the
-        # original labels in `user_note_labels` for downstream UIs
-        # (terra-view) that want to display them as the operator
-        # named them.
-        elif in_user_notes_block and user_note_position < len(_USER_NOTE_SLOTS):
-            slot = _USER_NOTE_SLOTS[user_note_position]
-            setattr(report, slot, value)
-            report.user_note_labels[slot] = key
-            user_note_position += 1
-
-        # ── Per-channel stats ────────────────────────────────────────────────
-        # All match the pattern "{Channel} <stat-name>"
-        elif key in (
-            "Tran PPV", "Vert PPV", "Long PPV",
-            "Tran ZC Freq", "Vert ZC Freq", "Long ZC Freq",
-            "Tran Time of Peak", "Vert Time of Peak", "Long Time of Peak",
-            "Tran Peak Acceleration", "Vert Peak Acceleration", "Long Peak Acceleration",
-            "Tran Peak Displacement", "Vert Peak Displacement", "Long Peak Displacement",
-        ):
-            ch_name, stat = key.split(" ", 1)
-            cs = report.channels.setdefault(ch_name, ChannelStats())
-            if stat == "PPV":
-                if _is_oorange(value):
-                    # Channel saturated — substitute range max as lower
-                    # bound; flag so downstream UI can render "> 10 in/s".
-                    cs.ppv_ips       = report.geo_range_ips
-                    cs.ppv_saturated = True
-                else:
-                    cs.ppv_ips = _parse_number(value)
-            elif stat == "ZC Freq":
-                # ">100 Hz" → store threshold + flag; numeric → parse normally
-                threshold = _parse_above_range(value)
-                if threshold is not None:
-                    cs.zc_freq_hz = threshold
-                    cs.zc_freq_above_range = True
-                else:
-                    cs.zc_freq_hz = _parse_number(value)
-            else:
-                num = _parse_number(value)
-                if   stat == "Time of Peak":        cs.time_of_peak_s = num
-                elif stat == "Peak Acceleration":   cs.peak_accel_g   = num
-                elif stat == "Peak Displacement":   cs.peak_disp_in   = num
-
-        # ── Histogram-specific fields ────────────────────────────────────────
-        # Histograms have Start/Stop time+date pairs + an interval count
-        # and size, plus per-channel absolute Peak Time/Date instead of
-        # the waveform's relative Time of Peak.
-        elif key == "Histogram Start Time":
-            _hist_start_time = _parse_event_time(value)
-        elif key == "Histogram Start Date":
-            _d = _parse_iso_date(value)
-            if _d and _hist_start_time:
-                report.histogram_start = datetime.datetime.combine(_d, _hist_start_time)
-        elif key == "Histogram Stop Time":
-            _hist_stop_time = _parse_event_time(value)
-        elif key == "Histogram Stop Date":
-            _d = _parse_iso_date(value)
-            if _d and _hist_stop_time:
-                report.histogram_stop = datetime.datetime.combine(_d, _hist_stop_time)
-        elif key == "Number of Intervals":
-            try:
-                report.histogram_n_intervals = int(float(value.strip()))
-            except ValueError:
-                pass
-        elif key == "Interval Size":
-            report.histogram_interval_size_str = value.strip()
-            report.histogram_interval_size_s   = _parse_interval_size(value)
-
-        # ── Per-channel histogram Peak Date / Peak Time ──
-        # Lines like "Tran Peak Time : 22:31:38" + "Tran Peak Date : 2026-05-16"
-        elif key in ("Tran Peak Time", "Vert Peak Time", "Long Peak Time", "MicL Time"):
-            ch_name = "MicL" if key == "MicL Time" else key.split(" ", 1)[0]
-            _pending_peak_time[ch_name] = _parse_event_time(value)
-        elif key in ("Tran Peak Date", "Vert Peak Date", "Long Peak Date", "MicL Date"):
-            ch_name = "MicL" if key == "MicL Date" else key.split(" ", 1)[0]
-            _d = _parse_iso_date(value)
-            _t = _pending_peak_time.get(ch_name)
-            if _d and _t:
-                report.channel_peak_when[ch_name] = datetime.datetime.combine(_d, _t)
-
-        # ── Vector Sum ───────────────────────────────────────────────────────
-        elif key == "Peak Vector Sum":
-            if _is_oorange(value):
-                # PVS saturated — conservative upper bound is
-                # sqrt(3) * geo_range_ips (all 3 channels at full-scale).
-                # Real PVS could be lower (channels rarely peak
-                # simultaneously) but never higher within the range.
-                if report.geo_range_ips is not None:
-                    import math as _math
-                    report.peak_vector_sum_ips = _math.sqrt(3) * report.geo_range_ips
-                report.peak_vector_sum_saturated = True
-            else:
-                report.peak_vector_sum_ips = _parse_number(value)
-        # BW writes the PVS-time label with a typo: "Peak Vector Sum TimeSum"
-        # (looks like Sum got appended twice).  Accept both forms.  Confirmed
-        # against actual BW output on 2026-05-27 — every PVS-time line in
-        # the field examples (T190, T438, K557) uses the typo'd label.
-        elif key in ("Peak Vector Sum Time", "Peak Vector Sum TimeSum"):
-            report.peak_vector_sum_time_s = _parse_number(value)
-            _pvs_time_raw = value
-        elif key == "Peak Vector Sum Date":
-            # Histogram-mode PVS gets paired with a date.  We may have
-            # captured 'Peak Vector Sum Time' as either a relative
-            # seconds float (waveform) or an HH:MM:SS string we
-            # interpreted as a number.  For histograms, BW writes
-            # "Peak Vector Sum Time : 22:33:52" which _parse_number
-            # parses as 22.0 (loses information).  When Peak Vector Sum
-            # Date arrives, re-parse the previous PVS time line as a
-            # clock time and combine into an absolute datetime.
-            _d = _parse_iso_date(value)
-            if _d and _pvs_time_raw is not None:
-                _t = _parse_event_time(_pvs_time_raw)
-                if _t:
-                    report.peak_vector_sum_when = datetime.datetime.combine(_d, _t)
-                    # The earlier seconds parse was bogus for histograms;
-                    # clear it so consumers don't think it's a real offset.
-                    report.peak_vector_sum_time_s = None
-
-        # ── Microphone block ────────────────────────────────────────────────
-        elif key == "Microphone":
-            report.mic.weighting = value
-        elif key == "MicL PSPL":
-            if _is_oorange(value):
-                # Mic saturated — substitute conservative upper bound 140 dBL.
-                report.mic.pspl_dbl       = 140.0
-                report.mic.pspl_saturated = True
-            else:
-                report.mic.pspl_dbl = _parse_number(value)
-            # Mirror onto the "MicL" entry in channels so callers querying
-            # `channels["MicL"].ppv_ips` see something — but it's dB(L), not
-            # in/s, so we store as-is in the MicStats and mark the channel.
-        elif key == "MicL Time of Peak":
-            report.mic.time_of_peak_s = _parse_number(value)
-            cs = report.channels.setdefault("MicL", ChannelStats())
-            cs.time_of_peak_s = report.mic.time_of_peak_s
-        elif key == "MicL ZC Freq":
-            threshold = _parse_above_range(value)
-            if threshold is not None:
-                report.mic.zc_freq_hz         = threshold
-                report.mic.zc_freq_above_range = True
-            else:
-                report.mic.zc_freq_hz = _parse_number(value)
-            cs = report.channels.setdefault("MicL", ChannelStats())
-            cs.zc_freq_hz          = report.mic.zc_freq_hz
-            cs.zc_freq_above_range = report.mic.zc_freq_above_range
-
-        # ── Sensor self-check ────────────────────────────────────────────────
-        elif key in (
-            "Tran Test Freq", "Vert Test Freq", "Long Test Freq", "MicL Test Freq",
-            "Tran Test Ratio", "Vert Test Ratio", "Long Test Ratio",
-            "MicL Test Amplitude",
-            "Tran Test Results", "Vert Test Results", "Long Test Results", "MicL Test Results",
-        ):
-            ch_name, stat = key.split(" ", 1)
-            sc = report.sensor_check.setdefault(ch_name, SensorCheck())
-            if   stat == "Test Freq":      sc.test_freq_hz      = _parse_number(value)
-            elif stat == "Test Ratio":     sc.test_ratio        = _parse_number(value)
-            elif stat == "Test Amplitude": sc.test_amplitude_mv = _parse_number(value)
-            elif stat == "Test Results":   sc.test_results      = value
-
-        # ── Trailer ─────────────────────────────────────────────────────────
-        elif key == "PC SW Version":
-            report.pc_sw_version = value
-
-        # Unknown keys are silently dropped — forward-compat for future
-        # BW versions that may add fields.
-
-    # Combine event date + time into a datetime
-    if event_date is not None and event_time_str is not None:
-        t = _parse_event_time(event_time_str)
-        if t is not None:
-            report.event_datetime = datetime.datetime.combine(event_date, t)
-
-    if parse_samples:
-        report.samples = _parse_sample_table(lines, i)
-
-    return report
-
-
-def _parse_sample_table(
-    lines: List[str], start: int,
-) -> List[Tuple[float, float, float, float]]:
-    """Parse the trailing sample table.
-
-    The table starts with a header row ("   Tran   <TAB>...") and continues
-    until EOF.  Each data row is a tab-separated quartet of numeric values.
-    """
-    samples: List[Tuple[float, float, float, float]] = []
-    seen_header = False
-    for line in lines[start:]:
-        line = line.rstrip("\r\n")
-        if not line.strip():
-            continue
-        cols = [c.strip() for c in line.split("\t") if c.strip()]
-        if not seen_header:
-            # Header row contains channel names; numeric rows don't.
-            if any(c in ("Tran", "Vert", "Long", "MicL") for c in cols):
-                seen_header = True
-            continue
-        if len(cols) < 4:
-            continue
-        try:
-            samples.append((
-                float(cols[0]), float(cols[1]),
-                float(cols[2]), float(cols[3]),
-            ))
-        except ValueError:
-            continue
-    return samples
-
-
-def parse_report_file(
-    path: Union[str, Path], *, parse_samples: bool = False,
-) -> BwAsciiReport:
-    """Convenience: read a .TXT file from disk and parse it."""
-    return parse_report(Path(path).read_bytes(), parse_samples=parse_samples)

minimateplus/client.py

@@ -30,7 +30,6 @@ from __future__ import annotations
 
 import datetime
 import logging
-import re
 import struct
 from typing import Optional
 
@@ -450,7 +449,7 @@ class MiniMateClient:
         proto.confirm_erase_all()
         log.info("delete_all_events: erase confirmed — device memory cleared")
 
-    def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None, skip_waveform_for_events: Optional[dict] = None, extra_chunks_after_metadata: int = 1) -> list[Event]:
+    def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None, extra_chunks_after_metadata: int = 1) -> list[Event]:
         """
         Download all stored events from the device using the confirmed
         1E → 0A → 0C → 5A → 1F event-iterator protocol.
@@ -498,24 +497,37 @@ class MiniMateClient:
         events: list[Event] = []
         idx = 0
 
-        # Legacy bare-key skip set is deprecated: the device's key counter
-        # resets to 0x01110000 after every memory erase, so a key in this set
-        # cannot be trusted to identify the same physical event across erases.
-        # If a caller still passes it, log a warning and ignore — full
-        # downloads will run for every event so the bug never silently bites.
-        if skip_waveform_for_keys:
-            log.warning(
-                "get_events: skip_waveform_for_keys is deprecated and unsafe "
-                "(post-erase key reuse); ignoring %d entries.  Use "
-                "skip_waveform_for_events={key: timestamp_iso} instead.",
-                len(skip_waveform_for_keys),
-            )
-        skip_evts: dict[str, str] = dict(skip_waveform_for_events or {})
-
         while data8[4:8] != b"\x00\x00\x00\x00":
             cur_key = key4   # key for this event's 0A/1E-arm/0C/5A calls
             log.info("get_events: record %d  key=%s", idx, cur_key.hex())
 
+            # Fast-advance path: if this key is already downloaded, skip
+            # 1E-arm/0C/POLL/5A entirely.  Only 0A + 1F(browse) are needed
+            # to advance the device's internal pointer to the next event.
+            # This is identical to the browse-mode walk in count_events().
+            if skip_waveform_for_keys and cur_key.hex() in skip_waveform_for_keys:
+                log.debug("get_events: key=%s already seen -- fast-advance only", cur_key.hex())
+                try:
+                    proto.read_waveform_header(cur_key)
+                except ProtocolError as exc:
+                    log.warning(
+                        "get_events: 0A failed for key=%s (skip path): %s -- stopping",
+                        cur_key.hex(), exc,
+                    )
+                    break
+                try:
+                    key4, data8 = proto.advance_event(browse=True)
+                except ProtocolError as exc:
+                    log.warning(
+                        "get_events: 1F failed for key=%s (skip path): %s -- stopping",
+                        cur_key.hex(), exc,
+                    )
+                    break
+                idx += 1
+                if stop_after_index is not None and idx > stop_after_index:
+                    break
+                continue
+
             ev = Event(index=idx)
             ev._waveform_key = cur_key
 
@@ -562,96 +574,72 @@ class MiniMateClient:
                         "get_events: 0C failed for key=%s: %s", cur_key.hex(), exc
                     )
 
-                # ── Skip-5A decision based on (key, timestamp) match ──────
+                # SUB 1F (download-arm) — send token=0xFE BEFORE POLL+5A to arm the
-                # If skip_waveform_for_events maps cur_key.hex() to a non-empty
+                # device's bulk stream state machine.  Cache the returned key as a
-                # ISO timestamp matching what we just read from 0C, this is
+                # fallback for loop iteration when 5A fails (see iteration block below).
-                # the same physical event we already have on disk — bypass
+                # Confirmed from 4-2-26 capture frames 66-67 (1F before frames 68-73 POLL).
-                # the 1F(arm)+POLL+5A bulk download.  Otherwise (no entry, or
-                # timestamp mismatch indicating post-erase reuse) fall through
-                # to the full download.
-                expected_ts = skip_evts.get(cur_key.hex(), "")
-                actual_ts   = _event_timestamp_iso(ev)
-                skip_5a     = bool(expected_ts and actual_ts and expected_ts == actual_ts)
-                if skip_5a:
-                    log.info(
-                        "get_events: key=%s (key, ts=%s) match — skipping 5A bulk download",
-                        cur_key.hex(), actual_ts,
-                    )
-
                 arm_key4: Optional[bytes] = None
-                a5_ok = False
+                try:
+                    arm_key4, _ = proto.advance_event(browse=False)   # arm 5A
+                    log.info("get_events: 1F(download) — 5A armed, arm_key=%s", arm_key4.hex())
+                except ProtocolError as exc:
+                    log.warning("get_events: 1F(download) arm failed: %s", exc)
 
-                if not skip_5a:
+                # POLL × 3 — BW sends 3 full POLL cycles between 1F and 5A.
-                    # SUB 1F (download-arm) — send token=0xFE BEFORE POLL+5A to arm the
+                # Confirmed from 4-2-26 BW TX capture (frames 68-73 before 5A at 74).
-                    # device's bulk stream state machine.  Cache the returned key as a
+                log.info("get_events: POLL × 3 before 5A")
-                    # fallback for loop iteration when 5A fails (see iteration block below).
+                for _p in range(3):
-                    # Confirmed from 4-2-26 capture frames 66-67 (1F before frames 68-73 POLL).
                     try:
-                        arm_key4, _ = proto.advance_event(browse=False)   # arm 5A
+                        proto.poll()
-                        log.info("get_events: 1F(download) — 5A armed, arm_key=%s", arm_key4.hex())
                     except ProtocolError as exc:
-                        log.warning("get_events: 1F(download) arm failed: %s", exc)
+                        log.warning("get_events: POLL %d failed: %s", _p, exc)
-
-                    # POLL × 3 — BW sends 3 full POLL cycles between 1F and 5A.
-                    # Confirmed from 4-2-26 BW TX capture (frames 68-73 before 5A at 74).
-                    log.info("get_events: POLL × 3 before 5A")
-                    for _p in range(3):
-                        try:
-                            proto.poll()
-                        except ProtocolError as exc:
-                            log.warning("get_events: POLL %d failed: %s", _p, exc)
 
                 # SUB 5A — bulk waveform stream (uses cur_key, the event set up by 0A+1E+0C).
                 # By default (full_waveform=False): stop after frame 7 for metadata only.
                 # When full_waveform=True: fetch all chunks and decode raw ADC samples.
-                #
+                a5_ok = False
-                # Bypassed when skip_5a is True — the event is left with
+                try:
-                # _a5_frames=None, which signals to the caller (e.g.
+                    if full_waveform:
-                # ach_server.py) that this event was matched by (key, ts) and
+                        log.info(
-                # already has a stored .file in the persistent waveform store.
+                            "get_events: 5A full waveform download for key=%s", cur_key.hex()
-                if not skip_5a:
-                    try:
-                        if full_waveform:
-                            log.info(
-                                "get_events: 5A full waveform download for key=%s", cur_key.hex()
-                            )
-                            a5_frames = proto.read_bulk_waveform_stream(
-                                cur_key, stop_after_metadata=False, max_chunks=128,
-                                include_terminator=True,
-                            )
-                            if a5_frames:
-                                a5_ok = True
-                                ev._a5_frames = a5_frames   # store for write_blastware_file
-                                _decode_a5_metadata_into(a5_frames, ev)
-                                _decode_a5_waveform(a5_frames, ev)
-                                log.info(
-                                    "get_events: 5A decoded %d sample-sets",
-                                    len((ev.raw_samples or {}).get("Tran", [])),
-                                )
-                        else:
-                            log.info(
-                                "get_events: 5A metadata-only download for key=%s", cur_key.hex()
-                            )
-                            a5_frames = proto.read_bulk_waveform_stream(
-                                cur_key, stop_after_metadata=True,
-                                include_terminator=True,
-                                extra_chunks_after_metadata=extra_chunks_after_metadata,
-                                max_chunks=128,
-                            )
-                            if a5_frames:
-                                a5_ok = True
-                                ev._a5_frames = a5_frames   # store for write_blastware_file
-                                _decode_a5_metadata_into(a5_frames, ev)
-                                log.debug(
-                                    "get_events: 5A metadata  client=%r  operator=%r",
-                                    ev.project_info.client if ev.project_info else None,
-                                    ev.project_info.operator if ev.project_info else None,
-                                )
-                    except ProtocolError as exc:
-                        log.warning(
-                            "get_events: 5A failed for key=%s: %s — metadata unavailable",
-                            cur_key.hex(), exc,
                         )
+                        a5_frames = proto.read_bulk_waveform_stream(
+                            cur_key, stop_after_metadata=False, max_chunks=128,
+                            include_terminator=True,
+                        )
+                        if a5_frames:
+                            a5_ok = True
+                            ev._a5_frames = a5_frames   # store for write_blastware_file
+                            _decode_a5_metadata_into(a5_frames, ev)
+                            _decode_a5_waveform(a5_frames, ev)
+                            log.info(
+                                "get_events: 5A decoded %d sample-sets",
+                                len((ev.raw_samples or {}).get("Tran", [])),
+                            )
+                    else:
+                        log.info(
+                            "get_events: 5A metadata-only download for key=%s", cur_key.hex()
+                        )
+                        a5_frames = proto.read_bulk_waveform_stream(
+                            cur_key, stop_after_metadata=True,
+                            include_terminator=True,
+                            extra_chunks_after_metadata=extra_chunks_after_metadata,
+                            max_chunks=128,
+                        )
+                        if a5_frames:
+                            a5_ok = True
+                            ev._a5_frames = a5_frames   # store for write_blastware_file
+                            _decode_a5_metadata_into(a5_frames, ev)
+                            log.debug(
+                                "get_events: 5A metadata  client=%r  operator=%r",
+                                ev.project_info.client if ev.project_info else None,
+                                ev.project_info.operator if ev.project_info else None,
+                            )
+                except ProtocolError as exc:
+                    log.warning(
+                        "get_events: 5A failed for key=%s: %s — metadata unavailable",
+                        cur_key.hex(), exc,
+                    )
 
                 # SUB 1F — loop iteration.
                 #
@@ -664,14 +652,7 @@ class MiniMateClient:
                 # Confirmed from 4-3-26 browse-mode captures: browse=True params
                 # are correct for multi-event iteration.  Conditional logic added
                 # 2026-04-06 to avoid post-failure state disruption.
-                #
+                if a5_ok:
-                # NEW 2026-05-06: when skip_5a=True we never entered the 5A
-                # state at all (we read 0A+1E(arm)+0C and chose to bypass).
-                # 1F(browse) is safe in this scenario — the device's iteration
-                # pointer is independent of the bulk-stream state machine, and
-                # we never put it into the half-attempted 5A state that the
-                # earlier "post-failure 1F disruption" warning is about.
-                if skip_5a or a5_ok:
                     # 5A succeeded — use browse 1F for reliable key advancement.
                     try:
                         key4, data8 = proto.advance_event(browse=True)
@@ -1193,27 +1174,6 @@ class MiniMateClient:
 # Pure functions: bytes → model field population.
 # Kept here (not in models.py) to isolate protocol knowledge from data shapes.
 
-def _event_timestamp_iso(event: Event) -> str:
-    """
-    Return a stable ISO-8601 string for the event's 0C-derived timestamp,
-    or "" if the event has no timestamp populated.
-
-    The format intentionally matches what `bridges/ach_server.py` writes
-    into `ach_state.json:downloaded_events[*]` so the (key, ts) compare
-    in get_events()'s skip path is a simple string equality.
-    """
-    ts = getattr(event, "timestamp", None)
-    if ts is None:
-        return ""
-    try:
-        return datetime.datetime(
-            ts.year, ts.month, ts.day,
-            ts.hour or 0, ts.minute or 0, ts.second or 0,
-        ).isoformat()
-    except Exception:
-        return str(ts)
-
-
 def _decode_serial_number(data: bytes) -> DeviceInfo:
     """
     Decode SUB EA (SERIAL_NUMBER_RESPONSE) payload into a new DeviceInfo.
@@ -1363,40 +1323,28 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
 
     Modifies event in-place.
     """
-    # ── Record type + format detection ────────────────────────────────────────
+    # ── Record type ───────────────────────────────────────────────────────────
-    # `record_type` is the user-facing label ("Waveform" for any triggered
+    # Decoded from byte[1] (sub_code) first so we can gate timestamp parsing.
-    # event regardless of timestamp-header layout).  `fmt` is the internal
-    # format code used to pick the right Timestamp parser; it stays
-    # internal and doesn't leak to the API / sidecar / UI.
     try:
         event.record_type = _extract_record_type(data)
     except Exception as exc:
         log.warning("waveform record type decode failed: %s", exc)
-    fmt = _detect_record_format(data)
 
     # ── Timestamp ─────────────────────────────────────────────────────────────
-    # Three timestamp-header layouts have been observed across BE11529
+    # 9-byte format for sub_code=0x10 Waveform records:
-    # firmware S338.17 — each picks a different Timestamp parser:
+    #   [day][sub_code][month][year:2 BE][unknown][hour][min][sec]
-    #   "single_shot": 9-byte  [day][0x10][month][year:2][unk][h][m][s]
+    # sub_code=0x10 and sub_code=0x03 have different timestamp byte layouts.
-    #   "continuous":  10-byte [0x10][day][0x10][month][year:2][unk][h][m][s]
+    # Both confirmed against Blastware event reports (BE11529, 2026-04-01 and 2026-04-03).
-    #   "short":       8-byte  [day][month][year:2][unk][h][m][s]
+    if event.record_type == "Waveform":
-    # All decoded into the same Timestamp dataclass — only the byte
-    # offsets differ.
-    if fmt == "single_shot":
         try:
             event.timestamp = Timestamp.from_waveform_record(data)
         except Exception as exc:
-            log.warning("single_shot record timestamp decode failed: %s", exc)
+            log.warning("waveform record timestamp decode failed: %s", exc)
-    elif fmt == "continuous":
+    elif event.record_type == "Waveform (Continuous)":
         try:
             event.timestamp = Timestamp.from_continuous_record(data)
         except Exception as exc:
             log.warning("continuous record timestamp decode failed: %s", exc)
-    elif fmt == "short":
-        try:
-            event.timestamp = Timestamp.from_short_record(data)
-        except Exception as exc:
-            log.warning("short record timestamp decode failed: %s", exc)
 
     # ── Peak values (per-channel PPV + Peak Vector Sum) ───────────────────────
     try:
@@ -1501,69 +1449,22 @@ def _decode_a5_waveform(
     (BULK_WAVEFORM_STREAM) frame payloads and populate event.raw_samples,
     event.total_samples, event.pretrig_samples, and event.rectime_seconds.
 
-    Wired up 2026-05-11 to the verified ``decode_waveform_v2`` codec (see
+    This requires ALL A5 frames (stop_after_metadata=False), not just the
-    ``minimateplus/waveform_codec.py`` and ``docs/waveform_codec_re_status.md``).
+    metadata-bearing subset.
-    Replaces the legacy int16 LE decoder, which produced full-scale ±32K
-    noise on every event because the body bytes are encoded, not raw
-    samples.
 
-    Output convention (preserved from the legacy decoder):
+    ── Waveform format (confirmed from 4-2-26 blast capture) ───────────────────
-      ``event.raw_samples`` is a dict with keys "Tran", "Vert", "Long",
+    The blast waveform is 4-channel interleaved signed 16-bit little-endian,
-      "MicL" mapping to lists of **int16 ADC counts**.  Multiply by
+    8 bytes per sample-set:
-      ``geo_range / 32768`` for geo channels to get in/s; use
-      :func:`minimateplus.waveform_codec.mic_count_to_db` for mic dB(L).
-
-    ``total_samples`` / ``pretrig_samples`` / ``rectime_seconds`` are set
-    to ``None`` so the caller backfills from compliance_config (the
-    authoritative source — STRT fields aren't reliable).
-    """
-    from .waveform_codec import decode_a5_frames
-
-    event.total_samples = None
-    event.pretrig_samples = None
-    event.rectime_seconds = None
-
-    if not frames_data:
-        log.debug("_decode_a5_waveform: no frames provided")
-        return
-
-    decoded = decode_a5_frames(frames_data)
-    if decoded is None:
-        log.warning("_decode_a5_waveform: codec returned no samples")
-        return
-
-    event.raw_samples = decoded
-    log.debug(
-        "_decode_a5_waveform: decoded %d/%d/%d/%d samples (T/V/L/M)",
-        len(decoded.get("Tran", [])),
-        len(decoded.get("Vert", [])),
-        len(decoded.get("Long", [])),
-        len(decoded.get("MicL", [])),
-    )
-
-
-def _decode_a5_waveform_LEGACY(
-    frames_data: list[S3Frame],
-    event: Event,
-) -> None:
-    """
-    LEGACY decoder — kept for reference only.  DO NOT CALL.
-
-    This is the int16 LE decoder that produced full-scale ±32K noise
-    on every event.  Retracted 2026-05-08; replaced 2026-05-11 with
-    the verified codec in :mod:`minimateplus.waveform_codec`.  See
-    ``docs/instantel_protocol_reference.md §7.6.1`` for the full history.
-
-    ── Waveform format (LEGACY — WRONG) ────────────────────────────────
-    Claimed 4-channel interleaved signed 16-bit little-endian, 8 bytes
-    per sample-set:
 
         [T_lo T_hi V_lo V_hi L_lo L_hi M_lo M_hi] × N
 
-    where T=Tran, V=Vert, L=Long, M=Mic.
+    where T=Tran, V=Vert, L=Long, M=Mic.  Channel ordering follows the
+    Blastware convention [Tran, Vert, Long, Mic] = [ch0, ch1, ch2, ch3].
 
-    The body bytes are actually a tagged delta+RLE stream — this
+    ⚠️  Channel ordering is a confirmed CONVENTION — the physical ordering on
-    interpretation was wrong.
+        the ADC mux is not independently verifiable from the saturating blast
+        captures we have.  The convention is consistent with Blastware labeling
+        (Tran is always the first channel field in the A5 STRT+waveform stream).
 
     ── Frame structure ──────────────────────────────────────────────────────────
     A5[0] (probe response):
@@ -1617,109 +1518,46 @@ def _decode_a5_waveform_LEGACY(
         log.warning("_decode_a5_waveform: STRT record truncated (%dB)", len(strt))
         return
 
-    # STRT byte layout (21 bytes; verified against M529LIY6 reference files
+    total_samples   = struct.unpack_from(">H", strt, 8)[0]
-    # and re-confirmed against live BE11529 captures, 2026-05-08):
+    pretrig_samples = struct.unpack_from(">H", strt, 16)[0]
-    #   [0:4]   b'STRT'
+    rectime_seconds = strt[18]
-    #   [4:6]   0xff 0xfe                      sentinel
-    #   [6:10]  end_key      4-byte BE         flash address where event ends
-    #   [10:14] start_key    4-byte BE         flash address where event starts
-    #   [14:18] device-specific (semantics not pinned; values vary across events
-    #           and don't hold authoritative total_samples / pretrig)
-    #   [18]    0x46         record-type marker (NOT rectime)
-    #   [19]    device-specific
-    #   [20]    sometimes rectime, sometimes 0 — not reliable
-    #
-    # AUTHORITATIVE values must come from compliance_config (sample_rate,
-    # record_time) and from end_offset - start_offset arithmetic (event size).
-    # Earlier code claimed STRT[8:10]=total_samples and STRT[16:18]=pretrig;
-    # those positions actually overlap end_key low-word and dev-specific bytes
-    # respectively.  We surface the address-derived event size so consumers
-    # can sanity-check chunk-loop bounds, but `total_samples` per channel must
-    # be derived externally (sample_rate × record_time, or computed from the
-    # decoded sample count below).
-    end_key   = strt[6:10]
-    start_key = strt[10:14]
-    end_offset_in_strt   = (end_key[2]   << 8) | end_key[3]
-    start_offset_in_strt = (start_key[2] << 8) | start_key[3]
-    is_event_1 = (start_offset_in_strt == 0x0000)
 
-    # Don't trust STRT for these — leave them as None so the caller can
+    event.total_samples   = total_samples
-    # backfill from compliance_config (the authoritative source).
+    event.pretrig_samples = pretrig_samples
-    event.total_samples   = None
+    event.rectime_seconds = rectime_seconds
-    event.pretrig_samples = None
-    event.rectime_seconds = None
 
     log.debug(
-        "_decode_a5_waveform: STRT start_key=%s end_key=%s "
+        "_decode_a5_waveform: STRT total_samples=%d pretrig=%d rectime=%ds",
-        "start_off=0x%04X end_off=0x%04X is_event_1=%s "
+        total_samples, pretrig_samples, rectime_seconds,
-        "dev-specific[14:18]=%s strt[20]=0x%02X",
-        start_key.hex(), end_key.hex(),
-        start_offset_in_strt, end_offset_in_strt, is_event_1,
-        strt[14:18].hex(), strt[20],
     )
 
     # ── Collect per-frame waveform bytes with global offset tracking ─────────
     # global_offset is the cumulative byte count across all frames, used to
     # compute the channel alignment at each frame boundary.
-    #
-    # Frame layout under the v0.14.0+ walk:
-    #   frames_data[0]            = probe response  (page_addr 0x0000;
-    #                                contains STRT + post-STRT data)
-    #   frames_data[1..2]         = (event 1 only) metadata pages
-    #                                page_addr = 0x1002 / 0x1004
-    #   frames_data[mid]          = sample chunks at flash addresses
-    #                                0x0600, 0x0800, …  (page_addr in
-    #                                {0x0600..0x1FFE})
-    #   frames_data[last]         = TERM response (page_key=0x0000)
-    #
-    # We identify metadata pages by their PAGE ADDRESS at db.data[4:6] (the
-    # 2-byte counter the device echoes back), NOT by content scan.  An earlier
-    # needle-based detection (b"Project:", b"Client:", etc.) was the wrong
-    # layer of abstraction:
-    #   • The actual metadata pages 0x1002 / 0x1004 do NOT contain ASCII
-    #     project strings on this firmware (S338.17 / BE11529).
-    #   • The strings physically live at flash address 0x1600 — which falls
-    #     inside the sample-chunk address range.  Skipping that frame would
-    #     drop a real sample chunk.
-    # BW handles the "samples region happens to contain string bytes" case
-    # by just rendering the bytes verbatim; we do the same.
-    _METADATA_PAGES = (b"\x10\x02", b"\x10\x04")
-
     chunks: list[tuple[int, bytes]] = []   # (frame_idx, waveform_bytes)
     global_offset = 0
 
     for fi, db in enumerate(frames_data):
-        page_addr = db.data[4:6] if len(db.data) >= 6 else b""
         w = db.data[7:]   # frame.data[7:]
 
-        # A5[0]: probe response.  Two cases:
+        # A5[0]: waveform begins after the 21-byte STRT record and 6-byte preamble.
-        #   - Event 1 (start_offset_in_strt == 0x0000): the bytes after STRT
+        # Layout: STRT(21B) + null-pad(2B) + 0xFF sentinel(4B) = 27 bytes total.
-        #     are the device's *pre-event reserved area* (flash 0x0046 to
-        #     0x0600), NOT samples.  We must skip them; samples begin at
-        #     the first dedicated chunk frame at counter=0x0600.
-        #   - Event N (continuation, start_offset != 0x0000): the bytes after
-        #     the STRT record ARE the first slice of real samples for the
-        #     event (BW's chunk loop addresses the probe as a sample chunk).
         if fi == 0:
             sp = w.find(b"STRT")
             if sp < 0:
                 continue
-            if is_event_1:
-                # No usable samples in the probe — pre-event reserved bytes.
-                continue
-            # Layout: STRT(21B) + null-pad(2B) + 0xFF sentinel(4B) = 27 bytes total.
             wave = w[sp + 27 :]
 
-        # Skip the dedicated metadata pages (event 1 only): page_addr 0x1002 / 0x1004.
+        # Frame 7 carries event-time metadata strings ("Project:", "Client:", …)
-        elif page_addr in _METADATA_PAGES:
+        # and no waveform ADC data.
-            log.debug(
+        elif fi == 7:
-                "_decode_a5_waveform: skipping metadata page fi=%d page_addr=%s",
-                fi, page_addr.hex(),
-            )
             continue
 
-        # Sample chunk (or TERM): strip the 8-byte per-frame header.
+        # Terminator frames have page_key=0x0000 and are excluded upstream
+        # (read_bulk_waveform_stream returns early on page_key==0).
+        # No hardcoded frame-index skip here — all non-metadata frames are data.
         else:
+            # Strip the 8-byte per-frame header (ctr + 6 zero bytes)
             if len(w) < 8:
                 continue
             wave = w[8:]
@@ -1733,8 +1571,10 @@ def _decode_a5_waveform_LEGACY(
     total_bytes = global_offset
     n_sets = total_bytes // 8
     log.debug(
-        "_decode_a5_waveform: %d chunks, %dB total → %d complete sample-sets",
+        "_decode_a5_waveform: %d chunks, %dB total → %d complete sample-sets "
-        len(chunks), total_bytes, n_sets,
+        "(%d of %d expected; %.0f%%)",
+        len(chunks), total_bytes, n_sets, n_sets, total_samples,
+        100.0 * n_sets / total_samples if total_samples else 0,
     )
 
     if n_sets == 0:
@@ -1792,85 +1632,38 @@ def _decode_a5_waveform_LEGACY(
         "Tran": tran,
         "Vert": vert,
         "Long": long_,
-        "MicL": mic,
+        "Mic":  mic,
     }
 
 
-def _detect_record_format(data: bytes) -> Optional[str]:
-    """
-    Detect which timestamp-header format a 210-byte 0C waveform record uses.
-
-    THREE formats observed on BE11529 firmware S338.17:
-
-      "single_shot" — 9-byte header:
-        [day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec]
-        sub_code=0x10 at byte [1].  Year at [3:5].
-
-      "continuous" — 10-byte header:
-        [0x10] [day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec]
-        marker 0x10 at byte [0] AND byte [2].  Year at [4:6].
-
-      "short" — 8-byte header (NEW 2026-05-01):
-        [day] [month] [year_BE:2] [unknown] [hour] [min] [sec]
-        No marker bytes.  Year at [2:4].
-
-    Each format has the year (uint16 BE) at a UNIQUE byte position, so we can
-    disambiguate by scanning each candidate position and picking the one
-    where the year falls in a sane range (2015..2050).
-
-    Returns "single_shot" / "continuous" / "short" or None if no format matches.
-    """
-    if len(data) < 8:
-        return None
-
-    def _sane_year(hi: int, lo: int) -> bool:
-        y = (hi << 8) | lo
-        return 2015 <= y <= 2050
-
-    # Order matters: prefer formats with stronger marker-byte evidence first.
-    if data[1] == 0x10 and len(data) >= 9 and _sane_year(data[3], data[4]):
-        return "single_shot"
-    if (data[0] == 0x10 and data[2] == 0x10
-            and len(data) >= 10 and _sane_year(data[4], data[5])):
-        return "continuous"
-    if _sane_year(data[2], data[3]):
-        return "short"
-    return None
-
-
 def _extract_record_type(data: bytes) -> Optional[str]:
     """
-    Return a user-facing name for a waveform record.  All three internal
+    Decode the recording mode from byte[1] of the 210-byte waveform record.
-    timestamp-header layouts represent the *same* user concept — a
-    triggered seismic event — so they all surface as just "Waveform".
 
-    The internal format code is preserved for parsing logic (timestamp
+    Byte[1] is the sub-record code that immediately follows the day byte in the
-    decoder selection) but doesn't leak into the API / UI / sidecar.
+    9-byte timestamp header at the start of each waveform record:
-    Callers that need the raw layout can call `_detect_record_format`
+      [day:1] [sub_code:1] [month:1] [year:2 BE] ...
-    directly.
 
-    Background: across BE11529 firmware S338.17 we've observed three
+    Confirmed codes (✅ 2026-04-01):
-    different byte layouts for the timestamp header at the start of the
+      0x10 → "Waveform"   (continuous / single-shot mode)
-    0C record (8 / 9 / 10 bytes, distinguished by the position of the
+
-    BE-encoded year and the presence of `0x10` marker bytes).  An older
+    Histogram mode code is not yet confirmed — a histogram event must be
-    revision of this code labelled them "Waveform" / "Waveform
+    captured with debug=true to identify it.  Returns None for unknown codes.
-    (Continuous)" / "Waveform (Short)", which created the false
-    impression that there were three distinct event "types" the user
-    could configure.  In reality the user only ever picks Single Shot
-    vs Continuous vs Histogram in the compliance config — the byte
-    layout is a firmware-internal detail that doesn't always correlate
-    with that choice.
     """
-    fmt = _detect_record_format(data)
+    if len(data) < 2:
-    if fmt in ("single_shot", "continuous", "short"):
+        return None
+    code = data[1]
+    if code == 0x10:
         return "Waveform"
-    if len(data) >= 3:
+    if code == 0x03:
-        log.warning(
+        # Continuous mode waveform record (confirmed by user — NOT a monitor log).
-            "_extract_record_type: unrecognized header: data[0:3]=%02X %02X %02X",
+        # The byte layout differs from 0x10 single-shot records: the timestamp
-            data[0], data[1], data[2],
+        # fields decode as garbage under the 0x10 waveform layout.
-        )
+        # TODO: confirm correct timestamp layout for 0x03 records from a known-time event.
-        return f"Unknown({data[0]:02X}.{data[1]:02X}.{data[2]:02X})"
+        return "Waveform (Continuous)"
-    return None
+    log.warning("_extract_record_type: unknown sub_code=0x%02X", code)
+    return f"Unknown(0x{code:02X})"
+
 
 def _extract_peak_floats(data: bytes) -> Optional[PeakValues]:
     """
@@ -2533,17 +2326,10 @@ def _decode_0a_partial_header(raw_data: bytes, index: int, key4: bytes) -> Optio
         ts2 = try_ts(raw_data[ts1_end + 1:ts1_end + 1 + ts_size])
 
     # Extract serial and geo threshold from "BE11529\0" and "Geo: X.XXX in/s\0".
-    #
-    # Match any two-letter family prefix, not a literal "BE" — a BlastMate
-    # reports "BA10895", and the old `find(b"BE")` returned -1 on one.  That
-    # skipped this whole block, so the geo threshold went missing along with
-    # the serial.  Requiring the NUL terminator in the pattern also makes the
-    # match stricter than the bare two-byte search it replaces.
     serial: Optional[str] = None
     geo_ips: Optional[float] = None
 
-    serial_match = re.search(rb"[A-Z]{2}\d{3,6}(?=\x00)", raw_data)
+    serial_pos = raw_data.find(b"BE")
-    serial_pos = serial_match.start() if serial_match else -1
     if serial_pos >= 0:
         # Read null-terminated serial starting at serial_pos.
         null_pos = raw_data.find(b"\x00", serial_pos)

minimateplus/framing.py

@@ -111,24 +111,20 @@ def build_5a_frame(offset_word: int, raw_params: bytes) -> bytes:
     verified against this algorithm on 2026-04-02).
 
     Args:
-        offset_word: 16-bit offset.  For probe/chunks/metadata pages this is
+        offset_word: 16-bit offset (0x1004 for probe/chunks, 0x005A for term).
-                     `0x1002`.  For the proper TERM frame this is computed by
+        raw_params:  10 or 11 params bytes (from bulk_waveform_params or
-                     `bulk_waveform_term_v2()` from the STRT-derived
+                     bulk_waveform_term_params). 0x10 bytes in params are
-                     `end_offset`.
+                     written RAW — NOT DLE-stuffed. Confirmed 2026-04-06 by
-        raw_params:  10, 11, or 12 params bytes (from `bulk_waveform_params`
+                     comparing wire bytes: BW sends bare `10 04` for chunk 1
-                     for probes/samples, `bulk_waveform_term_v2` for TERM, or
+                     (counter=0x1004), not stuffed `10 10 04`. Device reads
-                     a manually-built 12-byte block for the metadata pages
+                     params at fixed byte positions; stuffing shifts the bytes
-                     0x1002 / 0x1004).  See gotcha #3 below — params region
+                     and corrupts the counter, causing device to ignore the frame.
-                     uses partial DLE stuffing of 0x10 bytes.
 
     Returns:
         Complete frame bytes: [ACK][STX][stuffed_section][chk][ETX]
     """
-    if len(raw_params) not in (10, 11, 12):
+    if len(raw_params) not in (10, 11):
-        # 10 = termination params; 11 = regular probe / chunk params;
+        raise ValueError(f"raw_params must be 10 or 11 bytes, got {len(raw_params)}")
-        # 12 = metadata-page params (extra trailing 0x00 — BW byte-perfect quirk
-        # for the two fixed metadata reads at counter=0x1002 and 0x1004).
-        raise ValueError(f"raw_params must be 10/11/12 bytes, got {len(raw_params)}")
 
     # Build stuffed section between STX and checksum
     s = bytearray()
@@ -138,40 +134,8 @@ def build_5a_frame(offset_word: int, raw_params: bytes) -> bytes:
     s += b"\x00"                                 # field3
     s += bytes([(offset_word >> 8) & 0xFF,       # offset_hi — raw, NOT stuffed
                  offset_word & 0xFF])            # offset_lo
-    # Params — partial DLE stuffing of 0x10 bytes (CONFIRMED 2026-05-05).
+    for b in raw_params:                         # params — NOT DLE-stuffed (raw bytes, match BW wire format)
-    #
-    # The device's de-stuffing rule for params is:
-    #   • `10 10`        → de-stuffs to `10`
-    #   • `10 02/03/04`  → kept literal (these are inner-frame markers)
-    #   • `10 X` other   → de-stuffs to just `X`  (drops the 0x10)
-    #
-    # So for any 0x10 byte in the *logical* params that is followed by a
-    # byte NOT in {0x02, 0x03, 0x04, 0x10}, we must double the 0x10 on the
-    # wire (`10 X` → `10 10 X`) so the device's de-stuffer reproduces the
-    # original `10 X` pair.  Without this, counter values with `0x10` in
-    # the high byte (e.g. counter=0x1000 has params bytes `10 00`) are
-    # silently corrupted to `0x__00` on the device side, and the device
-    # responds for the wrong address — for counter=0x1000 it returns the
-    # probe response (counter=0x0000), which contains the file header +
-    # STRT.  That STRT block then lands in the assembled file body and
-    # Blastware rejects the file as malformed.
-    #
-    # Confirmed against BW capture 5-1-26 / bwcap3sec frame 20: params
-    # logical bytes `00 01 11 10 00 00 00 00 00 00 00` (counter=0x1000)
-    # are encoded on the wire as `00 01 11 10 10 00 00 00 00 00 00 00`.
-    # BW frames 13/14 (meta @ 0x1002 / 0x1004) leave `10 02` and `10 04`
-    # raw — the device handles those literal pairs correctly.
-    i = 0
-    while i < len(raw_params):
-        b = raw_params[i]
         s.append(b)
-        if (
-            b == 0x10
-            and i + 1 < len(raw_params)
-            and raw_params[i + 1] not in (0x02, 0x03, 0x04, 0x10)
-        ):
-            s.append(0x10)   # double the 0x10 so it survives device de-stuffing
-        i += 1
 
     # DLE-aware checksum: for 0x10 XX pairs count XX; for lone bytes count them
     chk, i = 0, 0
@@ -434,26 +398,28 @@ def bulk_waveform_params(key4: bytes, counter: int, *, is_probe: bool = False) -
 
 def bulk_waveform_term_params(key4: bytes, counter: int) -> bytes:
     """
-    ⛔ DEPRECATED — DO NOT USE IN NEW CODE.
+    Build the 10-byte params block for the SUB 5A termination request.
 
-    This is the v1 termination params helper, paired with the broken
+    The termination request uses offset=0x005A and a DIFFERENT params layout —
-    `_BULK_TERM_OFFSET = 0x005A` magic offset_word.  Together they produce a
+    the leading 0x00 byte is dropped, key4[0:2] shifts to params[0:2], and the
-    ~100-byte device-side terminator response that does NOT contain the
+    counter high byte is at params[2]:
-    partial-last-chunk waveform tail or the 26-byte file footer.  Files
-    reconstructed using this terminator are missing their last ~512 bytes of
-    waveform data and have a synthesized footer that disagrees with what BW
-    would have written.
 
-    **For new code, use `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`**
+      params[0]   = key4[0]
-    which computes the correct offset_word + params from the STRT-derived
+      params[1]   = key4[1]
-    `end_offset`.  v2 produces wire bytes that match BW exactly across all
+      params[2]   = (counter >> 8) & 0xFF
-    tested events (4-27-26 / 5-1-26 / 5-4-26 captures).
+      params[3:]  = zeros
 
-    This function is retained ONLY for the defensive fallback path in
+    Counter for the termination request = last_regular_counter + 0x0400.
-    `read_bulk_waveform_stream()` that triggers when STRT parsing fails or no
+
-    chunks are fetched (= a malformed event or an unexpected device state).
+    Confirmed from 1-2-26 BW TX capture: final request (frame 83) uses
-    The fallback already logs a WARNING when it activates; if you see that
+    offset=0x005A, params[0:3] = key4[0:2] + term_counter_hi.
-    warning, the bug is upstream — STRT should have been parseable.
+
+    Args:
+        key4:    4-byte waveform key.
+        counter: Termination counter (= last regular counter + 0x0400).
+
+    Returns:
+        10-byte params block.
     """
     if len(key4) != 4:
         raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
@@ -464,123 +430,6 @@ def bulk_waveform_term_params(key4: bytes, counter: int) -> bytes:
     return bytes(p)
 
 
-def bulk_waveform_term_v2(
-    key4: bytes,
-    end_offset: int,
-    last_chunk_counter: int,
-) -> tuple[int, bytes]:
-    """
-    Compute the SUB 5A TERM frame's offset_word and 10-byte params block.
-
-    Confirmed across 3 events (4-27-26 + 5-1-26 captures):
-
-      next_boundary = last_chunk_counter + 0x0200
-      offset_word   = end_offset - next_boundary    (residual byte count)
-      params[0]     = key4[0]                       (= 0x01 on every observed device)
-      params[1]     = key4[1]                       (= 0x11)
-      params[2]     = (next_boundary >> 8) & 0xFF
-      params[3]     = next_boundary & 0xFF
-      params[4:10]  = zeros
-
-    Verification:
-      | end_offset | last_chunk | next_boundary | offset_word | params[2:4] |
-      | 0x1ABE     | 0x1800     | 0x1A00        | 0x00BE      | 1A 00       |
-      | 0x21F2     | 0x1E00     | 0x2000        | 0x01F2      | 20 00       |
-      | 0x417E     | 0x3E38     | 0x4038        | 0x0146      | 40 38       |
-
-    The device receives `requested_address = (params[2] << 8) | offset_word`
-    and replies with `(end_offset - next_boundary)` bytes of waveform tail
-    starting at `next_boundary` — including the 26-byte file footer.
-
-    Args:
-        key4:               4-byte waveform key for this event.
-        end_offset:         Event-end pointer (= `(end_key[2] << 8) | end_key[3]`
-                            from the STRT record at data[23:27] of A5[0]).
-        last_chunk_counter: Counter of the last full 0x0200-byte chunk fetched
-                            (the chunk that covers [last_chunk_counter,
-                            last_chunk_counter + 0x0200)).
-
-    Returns:
-        (offset_word, params10) tuple.  Pass as
-        `build_5a_frame(offset_word, params)`.
-
-    Raises:
-        ValueError: on inconsistent inputs.
-    """
-    if len(key4) != 4:
-        raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
-    next_boundary = last_chunk_counter + 0x0200
-    if next_boundary > 0xFFFF:
-        raise ValueError(
-            f"next_boundary 0x{next_boundary:04X} exceeds uint16; check inputs"
-        )
-    if end_offset <= last_chunk_counter:
-        raise ValueError(
-            f"end_offset 0x{end_offset:04X} must be > "
-            f"last_chunk_counter 0x{last_chunk_counter:04X}"
-        )
-    offset_word = end_offset - next_boundary
-    if offset_word < 0:
-        # Last chunk overshot end_offset; caller should have stopped one chunk
-        # earlier.  Treat as zero residual.
-        offset_word = 0
-    if offset_word > 0xFFFF:
-        raise ValueError(
-            f"offset_word 0x{offset_word:04X} exceeds uint16"
-        )
-    p = bytearray(10)
-    p[0] = key4[0]
-    p[1] = key4[1]
-    p[2] = (next_boundary >> 8) & 0xFF
-    p[3] = next_boundary & 0xFF
-    return offset_word, bytes(p)
-
-
-# ── End-offset extraction from STRT record ────────────────────────────────────
-
-STRT_MARKER = b"STRT"
-
-
-def parse_strt_end_offset(a5_data: bytes) -> Optional[int]:
-    """
-    Extract the event-end offset from the STRT record in an A5 response payload.
-
-    The first A5 response (the probe response, or the first chunk for events
-    with non-zero start_key[2:4]) contains a STRT record at byte offset 17 of
-    `data`.  Layout:
-
-      data[17:21]  "STRT"
-      data[21:23]  ff fe        sentinel
-      data[23:27]  end_key      ← 4-byte key of where this event ENDS
-      data[27:31]  start_key
-      ...
-
-    Returns `(end_key[2] << 8) | end_key[3]` — the absolute device-buffer
-    address where the event ends.  Use this to bound the chunk loop and to
-    compute the TERM frame.
-
-    Verified end_offset values:
-      | event start_key | end_key      | end_offset |
-      | 01110000        | 01111ABE     | 0x1ABE     |
-      | 01110000        | 011121F2     | 0x21F2     |
-      | 011121F2        | 0111417E     | 0x417E     |
-
-    Args:
-        a5_data: The `data` field of an A5 response frame (frame.data).
-
-    Returns:
-        The end_offset (uint16) if STRT is found, else None.
-    """
-    pos = a5_data.find(STRT_MARKER)
-    if pos < 0 or pos + 10 > len(a5_data):
-        return None
-    # data[pos+4:pos+6] is "ff fe"; data[pos+6:pos+10] is end_key.
-    end_key = a5_data[pos + 6 : pos + 10]
-    if len(end_key) < 4:
-        return None
-    return (end_key[2] << 8) | end_key[3]
-
-
 # ── Pre-built POLL frames ─────────────────────────────────────────────────────
 #
 # POLL (SUB 0x5B) uses the same two-step pattern as all other reads — the
@@ -621,6 +470,7 @@ class S3Frame:
 
 
 # ── Streaming S3 frame parser ─────────────────────────────────────────────────
+
 class S3FrameParser:
     """
     Incremental byte-stream parser for S3→BW response frames.

minimateplus/histogram_codec.py

@@ -1,487 +0,0 @@
-"""
-histogram_codec.py — decoder for MiniMate Plus histogram-mode event bodies.
-
-FULLY DECODED 2026-05-20.  Every field in every block, verified
-byte-exact against BW's ASCII export across multiple histogram
-fixtures.
-
-The histogram-mode body is a stream of 32-byte fixed-length blocks,
-one block per histogram interval.  Each block carries the per-interval
-peak amplitude + zero-crossing frequency for all four channels (Tran,
-Vert, Long, MicL).
-
-────────────────────────────────────────────────────────────────────────────
-Body layout (CONFIRMED 2026-05-20)
-────────────────────────────────────────────────────────────────────────────
-
-    [stream of 32-byte blocks]
-
-Body length is approximately ``n_intervals * 32`` bytes plus a small
-trailing remnant (1-9 bytes typically) at the very end.  Walker should
-iterate 32-stride and stop before the tail.
-
-────────────────────────────────────────────────────────────────────────────
-32-byte block layout
-────────────────────────────────────────────────────────────────────────────
-
-    [0]    0x00                      always-zero tag
-    [1]    segment_id  (uint8)       0x00..0x03 - 256 blocks per segment
-    [2:4]  block_ctr  (uint16 LE)    resets each segment (0x0100, 0x0101, ...)
-    [4]    0x0a (uint8)              constant marker (= 10)
-    [5:7]  T_peak_count   uint16 BE  Tran peak (count x 0.005 -> in/s)
-    [7:9]  T_halfperiod   uint16 BE  Tran half-period in samples (freq = 512 / halfp)
-    [9:11] V_peak_count   uint16 BE
-    [11:13] V_halfperiod  uint16 BE
-    [13:15] L_peak_count  uint16 BE
-    [15:17] L_halfperiod  uint16 BE
-    [17:19] M_peak_count  uint16 BE  MicL peak (count -> dB via mic_count_to_db)
-    [19:21] M_halfperiod  uint16 BE  MicL half-period in samples
-    [21:23] 0x00 0x00                constant on standard blocks
-    [24:28] 4-byte variable          purpose unknown (possibly CRC or timestamp delta)
-    [28:32] block-end signature      see "Two block tails" below
-
-**Every per-channel field is uint16 BIG-endian** (confirmed 2026-08-25).
-Only ``block_ctr`` at [2:4] is little-endian.
-
-HISTORY - two earlier readings of this block were wrong in ways that
-cancelled out on quiet data:
-
-  1. *peak as uint16 LE at [6:8]* - produced 268 in/s peaks on any
-     interval whose next byte was non-zero.
-  2. *peak as uint8 at [6] with an "annotation" byte at [7]* - correct
-     for every peak below 256 counts (1.275 in/s), but it silently
-     **clipped larger peaks**: the final interval of
-     BE18193/T193LQ9K.OE0H reads 8.270 in/s in BW's export
-     (1654 counts = 0x0676) and decoded as 0x76 = 118 = 0.590 in/s.
-     The "annotation" byte was never an annotation - it is the high
-     byte of the big-endian half-period, which is why it was non-zero
-     exactly on the sub-Hz intervals BW renders as "<1.0".
-
-Both readings also forced ``block[5] == 0`` via a bogus ``uint16 LE``
-marker check at [4:6], which is what capped the peak at one byte.
-The marker is ``block[4]`` alone.
-
-Verified 2026-08-25 against 1211 production histograms paired with
-their Blastware ASCII exports: **1211/1211 decode exactly** (interval
-count plus every per-interval peak), and 842,442 per-interval
-frequency comparisons match with **zero** mismatches.
-
-Two block tails
----------------
-Standard blocks end with ``1e 0a 00 00``.  The **final block of the
-stream** ends with ``9c 06 00 42`` instead, and carries arbitrary bytes
-at [21:23].  Rejecting it dropped the last interval of nearly every
-histogram - and the last interval is frequently the one holding the
-event peak, so the file's reported PPV came out low.  Observed in 1206
-of 1211 production histograms, always positioned after every
-standard-tail block.
-
-Block-identification anchor: ``block[0] == 0x00`` AND
-``block[4] == 0x0A`` AND the tail is one of the two signatures above;
-standard-tail blocks additionally require ``block[22] == 0x00``.
-
-────────────────────────────────────────────────────────────────────────────
-Per-channel encoding
-────────────────────────────────────────────────────────────────────────────
-
-Geophone channels (Tran, Vert, Long):
-  - peak_count × 0.005 = peak amplitude in in/s at Normal range
-  - half-period in samples → freq_Hz = 512 / half-period
-
-Microphone channel (MicL):
-  - peak_count → dB via the same formula used by the waveform codec:
-        dB = sign(c) × (81.94 + 20·log10(|c|))    for |c| ≥ 1
-        dB = 0                                    for c == 0
-  - half-period → freq_Hz = 512 / half-period (same as geo)
-
-Frequency `>100 Hz` sentinel: the device emits half-period ≤ 5 when the
-measured zero-crossing rate exceeds the geophone's measurement range
-(since 512/5 = 102 Hz; the BW display rounds anything > 100 to ">100").
-
-────────────────────────────────────────────────────────────────────────────
-Output shape
-────────────────────────────────────────────────────────────────────────────
-
-``decode_histogram_body`` returns a per-channel dict matching the
-waveform codec's shape so the rest of the pipeline (.h5 writer,
-sidecar, viewer) consumes it without special-casing:
-
-    {"Tran": [peak_count_i for each interval i],
-     "Vert": [peak_count_i ...],
-     "Long": [peak_count_i ...],
-     "MicL": [peak_count_i ...]}
-
-Values are in **16-count units for geo** (LSB = 0.005 in/s, matching
-``decode_waveform_v2``) and **1-count units for mic** (matching the
-waveform codec's mic convention).  Run through
-``waveform_codec.decoded_to_adc_counts`` to scale geo to 1-count ADC.
-
-Per-interval frequencies are NOT returned — they're auxiliary data,
-not waveform samples.  Consumers needing frequencies can call
-``decode_histogram_body_full()`` for the structured per-interval
-record list.
-"""
-
-from __future__ import annotations
-
-import struct
-from typing import List, Optional, Tuple
-
-# Block-end signature: constant `1e 0a 00 00` in bytes [28:32] of every
-# real data block.  More distinctive than the byte-22 `00 00` (which
-# matches many false positives), so we anchor on this.
-_BLOCK_TAIL = b"\x1e\x0a\x00\x00"
-
-# The final block of a histogram stream ends with this instead.  It is a
-# real data block - same layout - and holds the last interval.  See the
-# module docstring, "Two block tails".
-_BLOCK_TAIL_TERMINAL = b"\x9c\x06\x00\x42"
-
-_BLOCK_SIZE = 32
-
-# Marker byte at block[4:6] of every histogram data block.  Used as
-# additional validation that we're looking at a real block.
-_BLOCK_MARKER = 10
-
-# Geo peak scaling: stored as "count × 0.005 in/s" where 1 count = one
-# 0.005 in/s display quantum.  Equivalent to the waveform codec's
-# 16-count-unit output (1 unit = 0.005 in/s = 16 ADC counts).
-_GEO_LSB_INS = 0.005
-
-# Frequency formula: freq_Hz = _FREQ_NUMERATOR / half_period_samples.
-# Empirically determined to be 512 (= sample_rate / 2, where sample rate
-# is 1024 sps for the standard MiniMate Plus configuration).
-_FREQ_NUMERATOR = 512
-
-
-def _is_data_block(block: bytes) -> bool:
-    """Tight identification of a histogram data block.
-
-    Accepts both tail signatures.  ``block[4]`` alone is the marker -
-    ``block[5]`` is the high byte of the Tran peak and is non-zero on any
-    interval above 1.275 in/s, so it must not be part of the marker test.
-    The ``block[22] == 0`` constraint is what keeps trailer content out,
-    but it applies only to standard-tail blocks: terminal blocks carry
-    arbitrary bytes there.
-    """
-    if len(block) < _BLOCK_SIZE:
-        return False
-    if block[0] != 0x00:
-        return False
-    if block[4] != _BLOCK_MARKER:
-        return False
-    # The 4-byte tail plus block[0]==0 and block[4]==0x0A is already six bytes
-    # of constraint — enough to keep trailer content out.  There is NO extra
-    # test on block[22]: it was documented as a constant 0x00 but carries data
-    # on loud blocks, and rejecting those threw away the interval holding the
-    # event peak.  BE18350/T350L7HR.NL0H is the proof: its block 92 has
-    # block[22]=0x26 and a Tran peak of 0x0563 = 1379 counts = 6.895 in/s,
-    # exactly the device-reported PPV, while the file decoded to 0.015 in/s.
-    return block[28:32] in (_BLOCK_TAIL, _BLOCK_TAIL_TERMINAL)
-
-
-def _decode_block(block: bytes) -> Optional[dict]:
-    """Decode one 32-byte histogram block.  Caller must have validated
-    with ``_is_data_block`` first.
-
-    Returns a record with per-channel peak counts (uint8) and
-    half-periods (uint16 LE).
-    """
-    # Every per-channel field is uint16 BIG-endian; only block_ctr is LE.
-    # See the module docstring for the two superseded readings and why
-    # each looked correct on quiet data.
-    def _be16(i: int) -> int:
-        return (block[i] << 8) | block[i + 1]
-
-    t_peak  = _be16(5)
-    t_halfp = _be16(7)
-    v_peak  = _be16(9)
-    v_halfp = _be16(11)
-    l_peak  = _be16(13)
-    l_halfp = _be16(15)
-    m_peak  = _be16(17)
-    m_halfp = _be16(19)
-    segment_id = block[1]
-    block_ctr  = block[2] | (block[3] << 8)
-    var_meta   = bytes(block[24:28])
-    return {
-        "segment_id":  segment_id,
-        "block_ctr":   block_ctr,
-        "t_peak":      t_peak,
-        "t_halfp":     t_halfp,
-        "v_peak":      v_peak,
-        "v_halfp":     v_halfp,
-        "l_peak":      l_peak,
-        "l_halfp":     l_halfp,
-        "m_peak":      m_peak,
-        "m_halfp":     m_halfp,
-        "meta_var":    var_meta,
-        "is_terminal": block[28:32] == _BLOCK_TAIL_TERMINAL,
-    }
-
-
-def walk_body(body: bytes) -> List[dict]:
-    """Walk the body and return one dict per histogram interval.
-
-    Iterates 32-byte strides from offset 0.  Yields a decoded record
-    for every block that passes ``_is_data_block`` validation.  Stops
-    when the remaining bytes are too short to form a complete block.
-
-    In Histogram+Continuous mode the body interleaves data blocks with
-    other 32-byte content (likely continuous-mode waveform blocks) that
-    fail the data-block validation; the walker naturally skips them
-    without losing 32-byte alignment.  Use ``block_ctr`` from each
-    returned record to map back to the original interval index — the
-    record list is sparse when other block types are interleaved.
-    """
-    records: List[dict] = []
-    for off in range(0, len(body) - _BLOCK_SIZE + 1, _BLOCK_SIZE):
-        blk = body[off:off + _BLOCK_SIZE]
-        if not _is_data_block(blk):
-            # Hit non-block content (likely a sync or stream marker).
-            # Continue walking — block alignment is fixed at 32-stride
-            # from offset 0, so we don't lose alignment by skipping.
-            continue
-        decoded = _decode_block(blk)
-        if decoded is None:
-            # Block validated as a histogram block but had peak fields
-            # outside the plausible range — undocumented extension.
-            # Skip rather than propagating bogus PVS contributions.
-            continue
-        records.append(decoded)
-    return records
-
-
-def _walk_auto(body: bytes) -> List[dict]:
-    """Pick the block model by signature strength, not by which returns first.
-
-    The multi-interval variant announces itself with consecutive block headers
-    at an exact ``12 + 20*n`` stride — far stronger evidence than a handful of
-    scattered standard-tail blocks, which a multi-interval body will also yield
-    by coincidence.  Dispatching on "whichever decoder returns something"
-    handed 193 BE18193 files to the standard walker and produced peaks of
-    149 in/s against a 10 in/s full scale.
-    """
-    if detect_multi_interval_stride(body):
-        recs = walk_multi_interval_blocks(body)
-        if recs:
-            return recs
-    return walk_body(body)
-
-
-def decode_histogram_body(body: bytes) -> Optional[dict]:
-    """Decode a histogram-mode body into per-channel peak-sample arrays.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}``
-    where each channel's list contains one peak value per histogram
-    interval (in the same units the waveform codec uses: 16-count units
-    for geo, 1-count ADC units for mic).  Returns ``None`` if the body
-    doesn't contain any valid histogram blocks.
-
-    To convert to physical units:
-      - Geo channels: ``count * 0.005`` = peak in in/s at Normal range
-        (or run through ``waveform_codec.decoded_to_adc_counts`` first
-         to get 1-count ADC values, then ``count / 32767 * 10.0`` for in/s)
-      - Mic channel:  use ``waveform_codec.mic_count_to_db(count)``
-    """
-    records = _walk_auto(body)
-    if not records:
-        return None
-    return {
-        "Tran": [r["t_peak"] for r in records],
-        "Vert": [r["v_peak"] for r in records],
-        "Long": [r["l_peak"] for r in records],
-        "MicL": [r["m_peak"] for r in records],
-    }
-
-
-def decode_histogram_body_full(body: bytes) -> Optional[List[dict]]:
-    """Decode a histogram-mode body into the full per-interval record list.
-
-    Same data as ``decode_histogram_body`` but in a structured form that
-    preserves the half-period (frequency) data for each channel + the
-    per-block segment_id, block_ctr, and 4-byte variable metadata.
-    Useful for diagnostic tools, sidecar enrichment, and future-codec
-    work.
-
-    Returns ``None`` if the body has no valid blocks.
-    """
-    records = _walk_auto(body)
-    return records if records else None
-
-
-def half_period_to_hz(halfp: int) -> Optional[float]:
-    """Convert a half-period in samples to frequency in Hz.
-
-    Returns ``None`` for half-period ≤ 5 — the device emits values in
-    that range when the measured zero-crossing rate exceeds 100 Hz
-    (the BW display reports `>100 Hz` for such cases).  Callers can
-    treat ``None`` as the `>100 Hz` sentinel.
-    """
-    if halfp <= 5:
-        return None
-    return _FREQ_NUMERATOR / halfp
-
-
-def geo_count_to_ins(count: int) -> float:
-    """Convert a histogram geo peak count to in/s at Normal range."""
-    return count * _GEO_LSB_INS
-
-
-# ── Multi-interval block variant (CONFIRMED 2026-08-26) ─────────────────────
-#
-# When the histogram interval is SHORTER than one minute, the device packs
-# several intervals into a single block so that every block still covers
-# exactly one minute of data:
-#
-#     interval size   intervals/block   stride
-#     1 minute        1                 32   <- the standard block above
-#     15 seconds      4                 92
-#     2 seconds       30                612
-#
-#     stride = 12 + n_intervals * 20
-#
-# Block layout:
-#     [0]      0x00
-#     [1]      segment_id (256 blocks per segment, same as the standard block)
-#     [2:4]    block_ctr uint16 LE (0x0100.., resets each segment)
-#     [4]      0x0a marker
-#     [5]      0x00
-#     [6 ...]  n x 20-byte interval records, each carrying 8 x uint16
-#              LITTLE-endian values:
-#                  T_peak, T_halfperiod, V_peak, V_halfperiod,
-#                  L_peak, L_halfperiod, M_peak, M_halfperiod
-#              then 2 more words; the first is 0x0000 on every real interval.
-#     [-6:]    6-byte block trailer
-#
-# ⚠ ENDIANNESS: the standard 32-byte block is BIG-endian.  This variant is
-# LITTLE-endian.  Do not share the accessor.
-#
-# These files previously decoded to nothing at all — 415 of them in the
-# production snapshot, 216 on BE18193 (2 s intervals) and 199 on BE9440
-# (15 s).  Before that they were being accepted by the WAVEFORM codec, which
-# returned garbage peaking up to 400x the device-reported PPV.
-#
-# Ground truth: BE9440/K440L3AQ.T70H (15 s intervals, 5,710 of them) decodes
-# against its Blastware ASCII export with 17,130/17,130 geo peak counts,
-# 22,840/22,840 frequencies and 5,710/5,710 mic dB(L) values matching exactly.
-
-_MULTI_HEADER_LEN   = 6
-_MULTI_RECORD_LEN   = 20
-_MULTI_TRAILER_LEN  = 6
-# At least 2 records: a 1-record block would have stride 12 + 20 = 32, which
-# collides with the standard big-endian block and mis-decodes it.
-_MULTI_MIN_RECORDS  = 2
-_MULTI_MAX_RECORDS  = 64
-
-# Geo full scale in 16-count units: 10.000 in/s / 0.005 = 2000.  A peak above
-# this is physically impossible and marks buffer garbage in a partial block.
-_GEO_MAX_COUNTS = 2000
-
-
-def _is_multi_header(body: bytes, off: int) -> bool:
-    return (off + _MULTI_HEADER_LEN <= len(body)
-            and body[off] == 0x00
-            and body[off + 4] == 0x0A
-            and body[off + 5] == 0x00)
-
-
-def detect_multi_interval_stride(body: bytes) -> Optional[int]:
-    """Block stride of a multi-interval histogram body, or None.
-
-    Found by locating the second block header; validated against
-    ``stride = 12 + n * 20`` and confirmed on a third block where present.
-    """
-    if not _is_multi_header(body, 0):
-        return None
-    lo = _MULTI_HEADER_LEN + _MULTI_TRAILER_LEN + _MULTI_RECORD_LEN * _MULTI_MIN_RECORDS
-    hi = _MULTI_HEADER_LEN + _MULTI_TRAILER_LEN + _MULTI_RECORD_LEN * _MULTI_MAX_RECORDS
-    for stride in range(lo, min(hi, len(body)) + 1, 2):
-        if (stride - 12) % _MULTI_RECORD_LEN:
-            continue
-        if not _is_multi_header(body, stride):
-            continue
-        # DECISIVE CHECK: consecutive blocks differ by exactly 1 in block_ctr.
-        # Without it this false-positives on ordinary standard-block bodies:
-        # those carry a header every 32 bytes, and 192 = 12 + 20*9 and
-        # 512 = 12 + 20*25 are both multiples of 32, so a stride "fits" while
-        # actually skipping 6 or 16 real blocks.  Sampling a standard body at
-        # stride 192 handed 9,082 files to the wrong decoder and produced peaks
-        # of 149 in/s against a 10 in/s full scale.
-        def _ctr(o: int) -> int:
-            return body[o + 2] | (body[o + 3] << 8)
-
-        if (_ctr(stride) - _ctr(0)) & 0xFFFF != 1:
-            continue
-        # Confirm on a third block WHEN ONE IS ACTUALLY PRESENT.  A body can
-        # be longer than two strides and still hold only two real blocks: a
-        # final *partial* block leaves trailing padding.  E.g. 51 intervals at
-        # 2 s = one full 30-interval block + a 21-interval remainder, in a
-        # 2787-byte body — long enough to demand a third header at 1224 that
-        # does not exist.  Requiring it unconditionally threw away the correct
-        # stride and the file decoded to nothing (BE18193 T193L0XM.CI0H).
-        # The block-counter check above is the decisive anti-false-positive
-        # test; this one is corroboration, so a missing third header means
-        # end-of-stream, not disqualification.
-        if (2 * stride + _MULTI_HEADER_LEN <= len(body)
-                and _is_multi_header(body, 2 * stride)):
-            if (_ctr(2 * stride) - _ctr(stride)) & 0xFFFF != 1:
-                continue
-        return stride
-    return None
-
-
-def walk_multi_interval_blocks(body: bytes,
-                               stride: Optional[int] = None) -> List[dict]:
-    """Decode a multi-interval histogram body into per-interval records."""
-    if stride is None:
-        stride = detect_multi_interval_stride(body)
-    if not stride:
-        return []
-    n_per_block = (stride - _MULTI_HEADER_LEN - _MULTI_TRAILER_LEN) // _MULTI_RECORD_LEN
-    if n_per_block < 1:
-        return []
-
-    def u16le(p: int) -> int:
-        return body[p] | (body[p + 1] << 8)
-
-    out: List[dict] = []
-    for off in range(0, len(body) - stride + 1, stride):
-        if not _is_multi_header(body, off):
-            break                      # end of the block run; trailer follows
-        for k in range(n_per_block):
-            q = off + _MULTI_HEADER_LEN + _MULTI_RECORD_LEN * k
-            out.append({
-                "_tail0": u16le(q + 16),
-                "segment_id": body[off + 1],
-                "block_ctr":  u16le(off + 2),
-                "t_peak": u16le(q),      "t_halfp": u16le(q + 2),
-                "v_peak": u16le(q + 4),  "v_halfp": u16le(q + 6),
-                "l_peak": u16le(q + 8),  "l_halfp": u16le(q + 10),
-                "m_peak": u16le(q + 12), "m_halfp": u16le(q + 14),
-                "meta_var": bytes(body[q + 16:q + 20]),
-                "is_terminal": False,
-            })
-    # A session ending mid-block leaves the remaining slots of the FINAL block
-    # filled with whatever was in the buffer.  Those decoded as peaks thousands
-    # of times the device-reported PPV, so they have to go — but only from the
-    # final block: a non-zero tail word occurs mid-file on real intervals, and
-    # trimming on that alone truncated four BE9440 files by up to 2,800
-    # intervals, while trimming purely from the end left garbage stranded
-    # behind one slot that happened to have a zero tail word.
-    #
-    # Within the final block, stop at the first slot that is not plausibly
-    # real: a non-zero tail word, or a geo peak above full scale.  16-count
-    # units put Normal-range full scale (10.000 in/s) at 2000 counts, so
-    # anything beyond that is physically impossible.
-    if out:
-        last_block_start = ((len(out) - 1) // n_per_block) * n_per_block
-        for i in range(last_block_start, len(out)):
-            r = out[i]
-            if (r["_tail0"] != 0
-                    or max(r["t_peak"], r["v_peak"], r["l_peak"]) > _GEO_MAX_COUNTS):
-                del out[i:]
-                break
-    for r in out:
-        r.pop("_tail0", None)
-    return out

minimateplus/models.py

@@ -201,58 +201,6 @@ class Timestamp:
             second=second,
         )
 
-    @classmethod
-    def from_short_record(cls, data: bytes) -> "Timestamp":
-        """
-        Decode an 8-byte timestamp header from a 210-byte waveform record.
-
-        Wire layout (✅ CONFIRMED 2026-05-01 against live SFM run on BE11529 in
-        Continuous mode, day-of-month = 1 May, raw: 01 05 07 ea 00 0d 15 25):
-          byte[0]:    day                 (uint8)
-          byte[1]:    month               (uint8)
-          bytes[2-3]: year                (big-endian uint16)
-          byte[4]:    unknown             (0x00 in observed sample)
-          byte[5]:    hour                (uint8)
-          byte[6]:    minute              (uint8)
-          byte[7]:    second              (uint8)
-
-        This is a third format observed in the wild — distinct from the 9-byte
-        (single-shot, sub_code=0x10 at [1]) and 10-byte (continuous, 0x10 at
-        [0] AND [2]) layouts.  No marker bytes; disambiguated by where the
-        year lands when scanned at byte 2/3/4.
-
-        Args:
-            data: at least 8 bytes; only the first 8 are consumed.
-
-        Returns:
-            Decoded Timestamp.
-
-        Raises:
-            ValueError: if data is fewer than 8 bytes.
-        """
-        if len(data) < 8:
-            raise ValueError(
-                f"Short record timestamp requires at least 8 bytes, got {len(data)}"
-            )
-        day          = data[0]
-        month        = data[1]
-        year         = struct.unpack_from(">H", data, 2)[0]
-        unknown_byte = data[4]
-        hour         = data[5]
-        minute       = data[6]
-        second       = data[7]
-        return cls(
-            raw=bytes(data[:8]),
-            flag=0,
-            year=year,
-            unknown_byte=unknown_byte,
-            month=month,
-            day=day,
-            hour=hour,
-            minute=minute,
-            second=second,
-        )
-
     @property
     def clock_set(self) -> bool:
         """False when year == 1995 (factory default / battery-lost state)."""
@@ -352,14 +300,6 @@ class PeakValues:
     long: Optional[float] = None             # Longitudinal PPV (in/s) ✅
     micl: Optional[float] = None             # Air overpressure (psi) 🔶 (units uncertain)
     peak_vector_sum: Optional[float] = None  # Scalar geo PVS (in/s) ✅
-    tran_zc_freq: Optional[float] = None
-    vert_zc_freq: Optional[float] = None
-    long_zc_freq: Optional[float] = None
-    mic_zc_freq:  Optional[float] = None
-    tran_zc_above_range: bool = False
-    vert_zc_above_range: bool = False
-    long_zc_above_range: bool = False
-    mic_zc_above_range:  bool = False
 
 
 # ── Project / operator metadata ───────────────────────────────────────────────
@@ -544,15 +484,6 @@ class Event:
     pretrig_samples: Optional[int] = None   # from STRT record: pre-trigger sample count
     rectime_seconds: Optional[int] = None   # from STRT record: record duration (seconds)
 
-    # Sensor self-check traces keyed by channel label — the short diagnostic
-    # waveforms the unit records when it pulses each sensor before monitoring
-    # (geophone ring-downs + a mic pulse train).  Decoded from the binary by
-    # the per-series decoder (minimateplus.sensor_check / micromate.sensor_check)
-    # and carried here so the .h5 writer can persist them device-agnostically.
-    # Raw ADC counts; the source series' scale differs but the trace is a
-    # shape diagnostic (rendered fit-to-box).  None when absent.
-    sensor_check: Optional[dict] = None     # {"Tran": [...], ..., "MicL": [...]}
-
     # ── Debug / introspection ─────────────────────────────────────────────────
     # Raw 210-byte waveform record bytes, set when debug mode is active.
     # Exposed by the SFM server via ?debug=true so field layouts can be verified.

minimateplus/protocol.py

@@ -35,8 +35,6 @@ from .framing import (
     token_params,
     bulk_waveform_params,
     bulk_waveform_term_params,
-    bulk_waveform_term_v2,
-    parse_strt_end_offset,
     POLL_PROBE,
     POLL_DATA,
     SESSION_RESET,
@@ -124,22 +122,16 @@ DATA_LENGTHS: dict[int, int] = {
 }
 
 # SUB 5A (BULK_WAVEFORM_STREAM) protocol constants.
-#
+# Confirmed from 1-2-26 BW TX capture analysis (2026-04-02).
-# 2026-05-01 minimal-fix: the chunk-counter walk is now bounded by the event's
+_BULK_CHUNK_OFFSET = 0x1004   # offset field for probe + all regular chunk requests ✅
-# `end_offset` extracted from the STRT record at data[23:27] of the probe
+_BULK_TERM_OFFSET  = 0x005A   # offset field for termination request ✅
-# response.  Without this bound the loop kept asking for chunks past the event
+_BULK_COUNTER_STEP = 0x0400   # chunk counter increment per chunk ✅
-# end and the device responded with post-event circular-buffer garbage,
+# Chunk counter formula: key4[2:4] + (chunk_num - 1) * 0x0400
-# corrupting reconstructed Blastware files for events ≥ 2 sec.
+# where key4[2:4] is the event's circular-buffer base offset ((key4[2]<<8)|key4[3]).
-#
+# Earlier captures showed 0x1004 for chunk 1 of key 01110000 — that was a Blastware
-# We keep the OLD 0x0400 chunk step here (BW actually uses 0x0200 — see §7.8.5
+# artifact.  For keys where key4[2:4] != 0x0000 (e.g. key 01111884) the old
-# of the protocol reference for the corrected understanding) because the
+# "n * 0x0400" formula sends counters from the wrong buffer region and the device
-# existing blastware_file.py builder relies on the 0x0400-step frame structure
+# returns data from a different event.  Confirmed correct 2026-04-24.
-# to produce valid files.  Switching to BW's 0x0200 step is a separate task
-# that also requires updating the file builder.
-# BW-exact protocol values (v0.14.0).  Verified against 4-27-26 + 5-1-26 captures.
-_BULK_CHUNK_OFFSET = 0x1002   # offset_word for probe + all chunk requests
-_BULK_TERM_OFFSET  = 0x005A   # offset_word for the legacy terminator (fallback only)
-_BULK_COUNTER_STEP = 0x0200   # chunk counter increment (matches chunk payload size)
 
 # Default timeout values (seconds).
 # MiniMate Plus is a slow device — keep these generous.
@@ -534,270 +526,223 @@ class MiniMateProtocol:
         self,
         key4: bytes,
         *,
-        stop_after_metadata: bool = True,   # DEPRECATED — no-op under BW-exact walk
+        stop_after_metadata: bool = True,
-        max_chunks: int = 256,              # safety cap only; loop is bounded by end_offset
+        max_chunks: int = 32,
         include_terminator: bool = False,
-        extra_chunks_after_metadata: int = 1,   # DEPRECATED — no-op
+        extra_chunks_after_metadata: int = 1,
     ) -> list[S3Frame]:
         """
-        Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event using
+        Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event.
-        Blastware's exact protocol.  REWRITTEN 2026-05-02 (v0.14.0).
 
-        Algorithm (matches BW captures across 2-sec / 3-sec / event-2):
+        The bulk waveform stream carries both raw ADC samples (large) and
+        event-time metadata strings ("Project:", "Client:", "User Name:",
+        "Seis Loc:", "Extended Notes") embedded in one of the middle frames
+        (confirmed: A5[7] of 9 for 1-2-26 capture).
 
-          1. Probe
+        Protocol is request-per-chunk, NOT a continuous stream:
-               - For events at start_key[2:4] = 0x0000 (first event after erase
+          1. Probe  (offset=_BULK_CHUNK_OFFSET, is_probe=True,  counter=0x0000)
-                 / wrap): probe at counter=0x0000 with full key in params.
+          2. Chunks (offset=_BULK_CHUNK_OFFSET, is_probe=False, counter+=0x0400)
-               - For continuation events (start_key[2:4] != 0): first chunk at
+          3. Loop until metadata found (stop_after_metadata=True) or max_chunks
-                 counter = start_key[2:4] + 0x0046; acts as both probe and
+          4. Termination (offset=_BULK_TERM_OFFSET, counter=last+_BULK_COUNTER_STEP)
-                 first sample chunk; response carries STRT.
+             Device responds with a final A5 frame (page_key=0x0000).
 
-          2. Parse end_offset from STRT record at data[23:27] of the probe response.
+        By default the termination frame (page_key=0x0000) is NOT included in the
+        returned list.  Pass include_terminator=True to append it; the blastware_file
+        writer needs the terminator frame's body to reconstruct the waveform file footer.
 
-          3. Read two fixed metadata pages at counter=0x1002 and counter=0x1004
+        Args:
-             — global session metadata (Project / Client / User Name / Seis Loc
+            key4:                4-byte waveform key from EVENT_HEADER (1E).
-             / Extended Notes ASCII strings).  Event 1 only; continuation
+            stop_after_metadata: If True (default), send termination as soon as
-             events skip these (BW caches them across the session).
+                                 b"Project:" is found in a frame's data — avoids
-
+                                 downloading the full ADC waveform payload (several
-          4. Walk sample chunks at 0x0200 increments, starting from 0x0600 for
+                                 hundred KB).  Set False to download everything.
-             event 1 or `start + 0x0046 + 0x0200` for continuation events.
+            max_chunks:          Safety cap on the number of chunk requests sent
-             Stop when `next_chunk + 0x0200 > end_offset`.
+                                 (default 32; a typical event uses 9 large frames).
-
+            include_terminator:  If True, append the terminator A5 frame
-          5. Send TERM frame with offset_word and params computed by
+                                 (page_key=0x0000) to the returned list.  The
-             `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`.
+                                 terminator carries the waveform file footer bytes.
-             The TERM response contains the partial last chunk (residual =
+                                 Default False preserves existing caller behaviour.
-             end_offset - next_boundary) including the 26-byte 0e 08 file
-             footer.
 
         Returns:
-            List of S3Frame objects from each A5 response (probe, metadata
+            List of S3Frame objects from each A5 response frame.  Frame indices
-            pages, sample chunks, optional TERM response).  Caller passes
+            match the request sequence: index 0 = probe response, index 1 = first
-            `include_terminator=True` (e.g. write_blastware_file) to keep the
+            chunk, etc.  If include_terminator=True, the last element is the
-            TERM response in the list — it's required to reconstruct the
+            terminator frame (page_key=0x0000).
-            file footer.
-
-        Deprecated kwargs:
-            stop_after_metadata: legacy "Project:"-string-based stop condition.
-                                 No-op under the BW-exact walk; the loop is
-                                 deterministically bounded by end_offset from
-                                 STRT.  Accepted for backward compat.
-            extra_chunks_after_metadata: same.
 
         Raises:
-            ProtocolError: on timeout / bad checksum / unexpected SUB.
+            ProtocolError: on timeout, bad checksum, or unexpected SUB.
+
+        Confirmed from 1-2-26 BW TX/RX captures (2026-04-02):
+          - probe + 8 regular chunks + 1 termination = 10 TX frames
+          - 9 large A5 responses + 1 terminator A5  = 10 RX frames
+          - page_key=0x0010 on large frames; page_key=0x0000 on terminator ✅
+          - "Project:" metadata at A5[7].data[626] ✅
         """
         if len(key4) != 4:
             raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
 
-        # Quietly accept and warn on deprecated kwargs.
+        rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM)   # 0xFF - 0x5A = 0xA5
-        if not stop_after_metadata:
-            log.debug("5A: stop_after_metadata=False is no-op under BW-exact walk")
-        if extra_chunks_after_metadata not in (0, 1):
-            log.debug("5A: extra_chunks_after_metadata=%d is no-op under BW-exact walk",
-                      extra_chunks_after_metadata)
-
-        rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM)   # 0xA5
         frames_data: list[S3Frame] = []
+        counter = 0
 
-        start_offset = (key4[2] << 8) | key4[3]
+        # BW counter formula (confirmed from 4-3-26 capture for key 0111245a,
-        is_event_1   = (start_offset == 0)
+        # and empirical live-device test 2026-04-06 for key 01110000):
+        #   counter for chunk n = max(key4[2:4], 0x0400) + (n - 1) * 0x0400
+        # key4[2:4] is the event's circular-buffer base offset.  The max() guard
+        # ensures chunk 1 never uses counter=0x0000 (which equals the probe address
+        # and causes the device to re-return STRT record data for the first chunk).
+        _key4_offset = (key4[2] << 8) | key4[3]
 
-        # ── Step 1: probe / first chunk ──────────────────────────────────────
+        # ── Step 1: probe ────────────────────────────────────────────────────
-        if is_event_1:
+        log.debug("5A probe  key=%s  key4_offset=0x%04X", key4.hex(), _key4_offset)
-            probe_counter = 0
+        params = bulk_waveform_params(key4, 0, is_probe=True)
-            probe_params  = bulk_waveform_params(key4, 0, is_probe=True)
+        self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
-            log.debug("5A probe (event-1)  key=%s  counter=0x0000", key4.hex())
+        self._parser.reset()        # reset bytes_fed counter before probe recv
-        else:
-            # Continuation events: first 5A request lands at counter = key[2:4]
-            # (i.e. the address of the off=0x46 WAVEHDR record returned by 1F).
-            # The probe response carries STRT at byte 17 with end_offset.
-            #
-            # Confirmed 2026-05-04 from 5-1-26 "copy 2nd address" capture
-            # (BW probes counter=0x2238 with key=01112238, STRT@17 end=0x417E)
-            # and 5-4-26 BW captures (2-sec event probes counter=0x2238).
-            #
-            # The earlier "+0x46" formula in the doc came from calling
-            # start_key the BOUNDARY (off=0x2C) key, but the iteration walk
-            # uses 1F's off=0x46 key as cur_key, which already incorporates
-            # the +0x46 offset relative to the boundary.  Adding it again
-            # caused the probe to overshoot, miss STRT, and run uncapped.
-            probe_counter = start_offset
-            probe_params  = bulk_waveform_params(key4, probe_counter)
-            log.debug(
-                "5A probe (event-N)  key=%s  counter=0x%04X",
-                key4.hex(), probe_counter,
-            )
-
-        self._send(build_5a_frame(_BULK_CHUNK_OFFSET, probe_params))
-        self._parser.reset()
         try:
-            rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False)
+            probe_batch = self._recv_5a_batch(rsp_sub)
         except TimeoutError:
             log.warning(
-                "5A probe TIMED OUT for key=%s — %d raw bytes received",
+                "5A probe TIMED OUT for key=%s — "
+                "%d raw bytes received (no complete A5 frame assembled)",
                 key4.hex(), self._parser.bytes_fed,
             )
             raise
-
+        frames_data.extend(probe_batch)
-        frames_data.append(rsp)
+        log.debug(
-        log.debug("5A A5[0] (probe)  page_key=0x%04X  %d bytes",
+            "5A probe: %d frame(s)  page_keys=%s",
-                  rsp.page_key, len(rsp.data))
+            len(probe_batch),
-
+            [f"0x{f.page_key:04X}" for f in probe_batch],
-        # ── Step 2: parse STRT end_offset from probe response ────────────────
-        end_offset = parse_strt_end_offset(rsp.data)
-        if end_offset is None:
-            log.warning(
-                "5A probe response did not contain a STRT record; "
-                "cannot bound chunk loop — falling back to max_chunks=%d cap",
-                max_chunks,
-            )
-            end_offset = 0xFFFF   # impossible value → loop runs to max_chunks
-        else:
-            log.info(
-                "5A STRT  start_offset=0x%04X  end_offset=0x%04X  size=0x%04X",
-                start_offset, end_offset, end_offset - start_offset,
-            )
-
-        # ── Step 3: metadata pages 0x1002 + 0x1004 (event 1 only) ────────────
-        # Confirmed from BW captures: BW reads these two fixed device-buffer
-        # pages immediately after the probe for events at start_key[2:4]=0.
-        # Continuation events skip them (BW caches across the session).
-        # Their content is global compliance-setup metadata: Project, Client,
-        # User Name, Seis Loc, Extended Notes.
-        if is_event_1:
-            for meta_counter in (0x1002, 0x1004):
-                # Metadata page params have an extra trailing 0x00 byte
-                # (12-byte params instead of 11) — empirical from BW captures.
-                # Checksum-neutral but matches BW byte-for-byte.
-                meta_params = bytes([
-                    0x00,
-                    key4[0], key4[1],
-                    (meta_counter >> 8) & 0xFF,
-                    meta_counter & 0xFF,
-                    0, 0, 0, 0, 0, 0, 0,
-                ])
-                log.debug("5A metadata page  counter=0x%04X", meta_counter)
-                self._send(build_5a_frame(_BULK_CHUNK_OFFSET, meta_params))
-                self._parser.reset()
-                try:
-                    meta_rsp = self._recv_one(
-                        expected_sub=rsp_sub, reset_parser=False, timeout=10.0,
-                    )
-                except TimeoutError:
-                    log.warning(
-                        "5A metadata page 0x%04X TIMED OUT — continuing",
-                        meta_counter,
-                    )
-                    continue
-                frames_data.append(meta_rsp)
-                log.debug(
-                    "5A meta@0x%04X  page_key=0x%04X  %d bytes",
-                    meta_counter, meta_rsp.page_key, len(meta_rsp.data),
-                )
-
-        # ── Step 4: sample chunk loop, bounded by end_offset ─────────────────
-        # Sample chunks start at:
-        #   event 1:        counter = 0x0600
-        #   event N (>0):   counter = probe_counter + 0x0200
-        #                            (probe was the first sample chunk)
-        if is_event_1:
-            counter = 0x0600
-        else:
-            counter = probe_counter + _BULK_COUNTER_STEP
-
-        last_chunk_counter: Optional[int] = (
-            probe_counter if not is_event_1 else None
         )
-        chunks_fetched = 0
 
-        while chunks_fetched < max_chunks:
+        # Log probe frame size for diagnostics.
-            # Stop when next chunk would straddle the event end.
+        # The device always needs extra_chunks_after_metadata chunks after the
-            if counter + _BULK_COUNTER_STEP > end_offset:
+        # metadata frame before termination to prime the valid waveform footer.
-                log.debug(
+        # This holds regardless of TCP frame size (1-frame vs 2-frame mode).
-                    "5A chunk loop done at counter=0x%04X (end=0x%04X); "
+        _effective_extra_chunks = extra_chunks_after_metadata
-                    "%d chunks fetched",
+        log.warning(
-                    counter, end_offset, chunks_fetched,
+            "5A probe data_len=%d  effective_extra_chunks=%d",
-                )
+            len(probe_batch[0].data),
-                break
+            _effective_extra_chunks,
+        )
 
-            params = bulk_waveform_params(key4, counter)
+        # ── Step 2: chunk loop ───────────────────────────────────────────────
-            log.debug("5A chunk #%d  counter=0x%04X", chunks_fetched + 1, counter)
+        # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400
+        # where _chunk_base = max(key4[2:4], 0x0400).
+        #
+        # For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a):
+        #   _chunk_base = 0x245a  →  chunk 1=0x245a, chunk 2=0x285a, ...
+        #   Confirmed from 4-3-26 capture.
+        #
+        # For events with key4[2:4] == 0 (e.g. key 01110000):
+        #   _chunk_base = max(0, 0x0400) = 0x0400
+        #   →  chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400)
+        #   CRITICAL: counter=0x0000 (same as the probe) causes the device to
+        #   re-return the STRT record data for chunk 1, making frame 1 look like
+        #   a second probe response (confirmed from server log: frame 1 len=1097,
+        #   contains STRT\xff\xfe, contributes zero body bytes after DLE-strip).
+        #   counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06).
+        _chunk_base = max(_key4_offset, _BULK_COUNTER_STEP)
+        for chunk_num in range(1, max_chunks + 1):
+            counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP
+            params  = bulk_waveform_params(key4, counter)
+            log.debug("5A chunk %d  counter=0x%04X", chunk_num, counter)
             self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
-            self._parser.reset()
+            self._parser.reset()        # reset bytes_fed for accurate per-chunk count
             try:
-                rsp = self._recv_one(
+                # Collect ALL frames from this chunk response.
-                    expected_sub=rsp_sub, reset_parser=False, timeout=10.0,
+                # Over TCP via modem, a single large A5 device response (~1100 bytes
-                )
+                # RS-232) is split across ~2 TCP segments, each parsed as its own
+                # complete S3 frame.  _recv_5a_batch gathers all of them so that
+                # every subsequent chunk request is paired with the correct response.
+                batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0)
             except TimeoutError:
                 raw = self._parser.bytes_fed
                 log.warning(
                     "5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d",
-                    chunks_fetched + 1, counter, raw,
+                    chunk_num, counter, raw,
                 )
                 if raw > 0 and frames_data:
+                    # Device sent a partial byte (likely a bare DLE/ETX end-of-stream
+                    # signal) but never completed a full frame.  Treat as graceful
+                    # stream end and fall through to the termination step.
                     log.warning(
-                        "5A unexpected end-of-stream — proceeding to TERM",
+                        "5A end-of-stream detected at chunk=%d (raw_bytes=%d, "
+                        "frames_collected=%d) — proceeding to termination",
+                        chunk_num, raw, len(frames_data),
                     )
                     break
                 raise
 
-            log.debug(
+            # Process all frames from this batch.
-                "5A RX chunk=%d page_key=0x%04X data_len=%d",
+            metadata_found = False
-                chunks_fetched + 1, rsp.page_key, len(rsp.data),
+            for rsp in batch:
-            )
-
-            if rsp.page_key == 0x0000:
-                # Device terminated mid-stream unexpectedly.
                 log.warning(
-                    "5A unexpected page_key=0x0000 mid-stream at counter=0x%04X",
+                    "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s",
-                    counter,
+                    chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data,
                 )
-                if include_terminator:
+                if rsp.page_key == 0x0000:
-                    frames_data.append(rsp)
+                    # Device unexpectedly terminated mid-stream.
-                return frames_data
+                    log.debug("5A page_key=0x0000 — device terminated early")
+                    if include_terminator:
+                        frames_data.append(rsp)
+                    return frames_data
+                frames_data.append(rsp)
+                if stop_after_metadata and b"Project:" in rsp.data:
+                    metadata_found = True
 
-            frames_data.append(rsp)
+            if metadata_found:
-            last_chunk_counter = counter
+                # Download extra_chunks_after_metadata more chunks after metadata.
-            counter += _BULK_COUNTER_STEP
+                # This primes the device to return the valid waveform footer in the
-            chunks_fetched += 1
+                # termination response — without it the terminator carries too few bytes
+                # (confirmed 2026-04-23).  The extra chunk data also belongs in the
+                # file body (confirmed from TCP capture analysis 2026-04-27).
+                log.debug("5A metadata found — fetching %d more chunk(s)",
+                          _effective_extra_chunks)
+                for _extra_n in range(_effective_extra_chunks):
+                    chunk_num += 1
+                    counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP
+                    params = bulk_waveform_params(key4, counter)
+                    self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
+                    try:
+                        extra_batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0)
+                        for ef in extra_batch:
+                            log.debug(
+                                "5A extra chunk page_key=0x%04X data_len=%d",
+                                ef.page_key, len(ef.data),
+                            )
+                            if ef.page_key == 0x0000:
+                                if include_terminator:
+                                    frames_data.append(ef)
+                                return frames_data
+                            frames_data.append(ef)
+                    except TimeoutError:
+                        log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1)
+                        break
+                break
         else:
             log.warning(
-                "5A reached max_chunks=%d at counter=0x%04X (end=0x%04X)",
+                "5A reached max_chunks=%d without end-of-stream; sending termination",
-                max_chunks, counter, end_offset,
+                max_chunks,
             )
 
-        # ── Step 5: TERM with proper end_offset-derived formula ──────────────
+        # ── Step 3: termination ──────────────────────────────────────────────
-        if last_chunk_counter is None or end_offset == 0xFFFF:
+        term_counter = counter + _BULK_COUNTER_STEP
-            # No STRT or no chunks fetched — fall back to legacy TERM.
+        term_params  = bulk_waveform_term_params(key4, term_counter)
-            log.warning(
+        log.debug(
-                "5A using legacy TERM (offset_word=0x005A); "
+            "5A termination  term_counter=0x%04X  offset=0x%04X",
-                "end_offset unavailable or no chunks fetched",
+            term_counter, _BULK_TERM_OFFSET,
-            )
+        )
-            legacy_counter = (last_chunk_counter or probe_counter) + _BULK_COUNTER_STEP
+        self._send(build_5a_frame(_BULK_TERM_OFFSET, term_params))
-            term_offset_word = _BULK_TERM_OFFSET   # 0x005A
-            term_params = bulk_waveform_term_params(key4, legacy_counter)
-        else:
-            term_offset_word, term_params = bulk_waveform_term_v2(
-                key4, end_offset, last_chunk_counter,
-            )
-            log.debug(
-                "5A TERM  offset_word=0x%04X  params[2:4]=%s  end=0x%04X  "
-                "last_chunk=0x%04X",
-                term_offset_word, term_params[2:4].hex(),
-                end_offset, last_chunk_counter,
-            )
-
-        self._send(build_5a_frame(term_offset_word, term_params))
         try:
-            term_rsp = self._recv_one(expected_sub=rsp_sub, timeout=10.0)
+            term_rsp = self._recv_one(expected_sub=rsp_sub)
-            log.info(
+            log.debug(
-                "5A TERM response  page_key=0x%04X  %d bytes",
+                "5A termination response  page_key=0x%04X  %d bytes",
                 term_rsp.page_key, len(term_rsp.data),
             )
             if include_terminator:
                 frames_data.append(term_rsp)
         except TimeoutError:
-            log.warning("5A no TERM response (timeout)")
+            log.debug("5A no termination response — device may have already closed")
 
         return frames_data
 
@@ -937,7 +882,7 @@ class MiniMateProtocol:
                 continue
 
             chunk = data_rsp.data[11:]
-            log.debug(
+            log.warning(
                 "read_compliance_config: frame %s  page=0x%04X  data=%d  cfg_chunk=%d  running_total=%d",
                 step_name, data_rsp.page_key, len(data_rsp.data),
                 len(chunk), len(config) + len(chunk),
@@ -957,18 +902,17 @@ class MiniMateProtocol:
         except TimeoutError:
             pass
 
-        log.info(
+        log.warning(
             "read_compliance_config: done — %d cfg bytes total",
             len(config),
         )
 
-        # Hex dump first 128 bytes — useful only for field-mapping work, not normal operation.
+        # Hex dump first 128 bytes for field mapping
-        if log.isEnabledFor(logging.DEBUG):
+        for row in range(0, min(len(config), 128), 16):
-            for row in range(0, min(len(config), 128), 16):
+            row_bytes = bytes(config[row:row + 16])
-                row_bytes = bytes(config[row:row + 16])
+            hex_part = ' '.join(f'{b:02x}' for b in row_bytes)
-                hex_part = ' '.join(f'{b:02x}' for b in row_bytes)
+            asc_part = ''.join(chr(b) if 32 <= b < 127 else '.' for b in row_bytes)
-                asc_part = ''.join(chr(b) if 32 <= b < 127 else '.' for b in row_bytes)
+            log.warning("  cfg[%04x]: %-48s  %s", row, hex_part, asc_part)
-                log.debug("  cfg[%04x]: %-48s  %s", row, hex_part, asc_part)
 
         return bytes(config)
 
@@ -1459,6 +1403,63 @@ class MiniMateProtocol:
         log.debug("TX %d bytes: %s", len(frame), frame.hex())
         self._transport.write(frame)
 
+    def _recv_5a_batch(
+        self,
+        expected_sub: int,
+        first_timeout: float = 10.0,
+        batch_timeout: float = 0.5,
+    ) -> list[S3Frame]:
+        """
+        Collect all S3 frames that arrive as part of one device response.
+
+        Over TCP via cellular modem, a single device A5 response (~1100 bytes of
+        RS-232 data) is forwarded in multiple TCP segments due to the modem's
+        data-forwarding timeout (~100-150 ms per segment).  Each TCP segment
+        contains a complete, valid S3 frame (~550 bytes).  Calling _recv_one()
+        once returns only the first segment's frame and misses the rest, causing
+        the chunk request/response pairing to cascade out of alignment.
+
+        This helper collects ALL frames before returning, by trying additional
+        short-timeout receives after the first frame arrives.
+
+        The caller must call self._parser.reset() before this method to ensure
+        bytes_fed is accurate; this method always uses reset_parser=False.
+
+        Args:
+            expected_sub:   Expected SUB byte for validation.
+            first_timeout:  Timeout for the mandatory first frame.  Should be
+                            generous (default 10 s) since the device may be slow.
+            batch_timeout:  Short timeout for subsequent frames.  Default 0.5 s
+                            — comfortably longer than the modem forwarding gap
+                            (~150 ms) but short enough to avoid stalling when
+                            only one frame is expected (probe, terminator).
+
+        Returns:
+            List of S3Frame objects in arrival order (at least one).
+
+        Raises:
+            TimeoutError:       If no frame arrives within first_timeout.
+            UnexpectedResponse: If any frame has the wrong SUB byte.
+        """
+        frames: list[S3Frame] = []
+        first = self._recv_one(
+            expected_sub=expected_sub,
+            reset_parser=False,
+            timeout=first_timeout,
+        )
+        frames.append(first)
+        while True:
+            try:
+                extra = self._recv_one(
+                    expected_sub=expected_sub,
+                    reset_parser=False,
+                    timeout=batch_timeout,
+                )
+                frames.append(extra)
+            except TimeoutError:
+                break
+        return frames
+
     def _recv_one(
         self,
         expected_sub: Optional[int] = None,

minimateplus/sensor_check.py

@@ -1,146 +0,0 @@
-r"""Decode the Blastware sensor self-check waveforms from a series-3 event binary.
-
-Reverse-engineered 2026-09-15 against 7 BE12844 (MiniMate Plus) oracle events.
-After the main waveform record-chain and the trailing metadata / per-channel
-calibration records, the binary carries four length-prefixed records tagged
-0x3c-0x3f: the sensor self-check traces the unit records when it pulses each
-sensor before monitoring.  Blastware draws these as the little waveforms in the
-"Sensor Check" strip on the right of the Event Report.
-
-  * 0x3c / 0x3d / 0x3e = Tran / Vert / Long geophone ring-downs (a damped
-    oscillation at the geophone's resonance, ~7-8 Hz at 1024 sps).
-  * 0x3f            = MicL, a pulse train at the mic self-test frequency
-    (~20 Hz), whose zero-crossing frequency is BW's mic "Channel Test" freq.
-
-Record framing (per record, all four chained by their length prefix)::
-
-    [len:2 BE][id:1][00 00][Nchan:1][12-byte header][delta stream][40 02][6B]
-    \_________________ payload (len bytes) _______________________________/
-
-The delta stream is ``payload[20 : len-8]`` (the ``40 02`` terminator sits at
-``len-8``, followed by 6 trailing bytes).  It uses the exact same 10/20/30/00
-delta-block tags as the main waveform codec
-(:mod:`minimateplus.waveform_codec`), decoded here from an implicit anchor of 0
-— so the traces come out in the same 16-count raw units as the main waveform
-(LSB = 0.005 in/s at Normal range for the geophones).
-"""
-from __future__ import annotations
-
-from typing import Dict, List
-
-from minimateplus.waveform_codec import walk_body
-
-# Record id → channel.  Order mirrors the trailing per-channel calibration
-# records (Tran / Vert / Long / MicL), confirmed against BW's sensor-check
-# frequencies on all 7 oracle events.
-_ID_TO_CHANNEL = {0x3C: "Tran", 0x3D: "Vert", 0x3E: "Long", 0x3F: "MicL"}
-_CHAIN_IDS = (0x3C, 0x3D, 0x3E, 0x3F)
-
-_HEADER_LEN = 20    # payload bytes before the delta stream
-_TRAILER_LEN = 8    # 40 02 terminator + 6 trailing bytes after the stream
-
-
-def _s4(nib: int) -> int:
-    """Sign-extend a 4-bit nibble delta."""
-    return nib - 16 if nib >= 8 else nib
-
-
-def _i8(byte: int) -> int:
-    """Sign-extend an 8-bit int delta."""
-    return byte - 256 if byte >= 128 else byte
-
-
-def _decode_delta_stream(buf: bytes) -> List[int]:
-    """Accumulate a 10/20/30/00 delta-block stream from an anchor of 0,
-    stopping at the 0x40 terminator.
-
-    Mirrors the block semantics in
-    :func:`minimateplus.waveform_codec.decode_waveform_v2` (fully decoded &
-    byte-exact as of 2026-05-11); see that module for the format details.
-    """
-    out: List[int] = []
-    cur = 0
-    for blk in walk_body(buf, 0):
-        fam = blk.tag_hi & 0xF0
-        if fam == 0x10:
-            # nibble deltas, high nibble first
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += _s4(nib)
-                    out.append(cur)
-        elif fam == 0x20:
-            # int8 deltas
-            for byte in blk.data:
-                cur += _i8(byte)
-                out.append(cur)
-        elif fam == 0x30:
-            # 12-bit signed deltas, packed as tag_lo/4 groups of 6 bytes
-            for g in range(blk.tag_lo // 4):
-                grp = blk.data[g * 6:(g + 1) * 6]
-                if len(grp) < 6:
-                    break
-                high_word = (grp[0] << 8) | grp[1]
-                for k in range(4):
-                    nib = (high_word >> (12 - 4 * k)) & 0xF
-                    v = (nib << 8) | grp[2 + k]
-                    if v >= 0x800:
-                        v -= 0x1000
-                    cur += v
-                    out.append(cur)
-        elif fam == 0x00:
-            # RLE zero-delta run (wide form carries the high nibble in the tag)
-            run = ((blk.tag_hi & 0x0F) << 8) | blk.tag_lo
-            out.extend([cur] * run)
-        elif fam == 0x40:
-            # segment / record terminator
-            break
-    return out
-
-
-def _find_chain(body: bytes):
-    """Locate the four length-prefixed sensor-check records.
-
-    Returns a list of ``(offset, id, length)`` or ``None``.  The chain is
-    validated by walking the ids 0x3c → 0x3d → 0x3e → 0x3f via their own length
-    prefixes, so a stray 0x3c byte in the waveform data cannot match.
-    """
-    for p in range(len(body) - 6):
-        if body[p + 2] == 0x3C and body[p + 3] == 0 and body[p + 4] == 0:
-            q = p
-            recs = []
-            ok = True
-            for expect in _CHAIN_IDS:
-                if q + 3 > len(body) or body[q + 2] != expect:
-                    ok = False
-                    break
-                length = int.from_bytes(body[q:q + 2], "big")
-                recs.append((q, expect, length))
-                q = q + 2 + length
-            if ok and len(recs) == 4:
-                return recs
-    return None
-
-
-def decode_sensor_check(raw: bytes) -> Dict[str, List[int]]:
-    """Decode the four sensor self-check traces from a series-3 event binary.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}`` in
-    raw decode units (same 16-count LSB as the main waveform), or ``{}`` if the
-    binary carries no sensor-check block (a histogram event, a non-series-3
-    file, or a unit/firmware that doesn't store it).
-    """
-    strt = raw.find(b"STRT")
-    if strt < 0 or len(raw) < strt + 21 + 26:
-        return {}
-    body = raw[strt + 21: len(raw) - 26]
-    chain = _find_chain(body)
-    if not chain:
-        return {}
-    out: Dict[str, List[int]] = {}
-    for off, rid, length in chain:
-        payload = body[off + 2: off + 2 + length]
-        if len(payload) < _HEADER_LEN + _TRAILER_LEN:
-            continue
-        stream = payload[_HEADER_LEN: length - _TRAILER_LEN]
-        out[_ID_TO_CHANNEL[rid]] = _decode_delta_stream(stream)
-    return out

minimateplus/transport.py

@@ -454,102 +454,3 @@ class SocketTransport(TcpTransport):
 
     def __repr__(self) -> str:
         return f"SocketTransport(peer={self.host!r})"
-
-
-# ── Capturing transport (MITM-style raw byte mirror) ──────────────────────────
-
-class CapturingTransport(BaseTransport):
-    """
-    Wraps another BaseTransport and mirrors every byte to two raw capture files:
-
-        raw_bw_<...>.bin  — bytes WE wrote to the device (BW-side TX)
-        raw_s3_<...>.bin  — bytes the device wrote back  (S3-side TX)
-
-    The file naming and on-wire byte layout are identical to the captures
-    produced by `bridges/ach_mitm.py`, so the resulting `.bin` files can be
-    loaded directly by the Analyzer (File > Open Capture) and parsed by the
-    same tooling used for genuine Blastware MITM captures.
-
-    All BaseTransport methods are forwarded to the inner transport; the only
-    side-effect is that successful read/write byte streams are appended to the
-    two open binary files.
-
-    Args:
-        inner:   An already-built BaseTransport (SerialTransport / TcpTransport).
-        bw_path: File path for the "BW TX" stream (bytes we send).  Opened "wb".
-        s3_path: File path for the "S3 TX" stream (bytes the device sends).
-                 Opened "wb".
-
-    Example:
-        with CapturingTransport(TcpTransport("1.2.3.4", 9034),
-                                "raw_bw.bin", "raw_s3.bin") as t:
-            client = MiniMateClient(transport=t)
-            client.connect()
-            client.get_events()
-        # both .bin files now hold the full bidirectional capture.
-    """
-
-    def __init__(self, inner: BaseTransport, bw_path: str, s3_path: str) -> None:
-        self._inner = inner
-        self._bw_path = bw_path
-        self._s3_path = s3_path
-        self._bw_fh = None
-        self._s3_fh = None
-        # Forward inner attrs so callers can introspect (e.g. .host, .port).
-        self.host = getattr(inner, "host", None)
-        self.port = getattr(inner, "port", None)
-
-    # ── BaseTransport interface ───────────────────────────────────────────────
-
-    def connect(self) -> None:
-        if self._bw_fh is None:
-            self._bw_fh = open(self._bw_path, "wb", buffering=0)
-        if self._s3_fh is None:
-            self._s3_fh = open(self._s3_path, "wb", buffering=0)
-        self._inner.connect()
-
-    def disconnect(self) -> None:
-        try:
-            self._inner.disconnect()
-        finally:
-            for fh_attr in ("_bw_fh", "_s3_fh"):
-                fh = getattr(self, fh_attr)
-                if fh is not None:
-                    try:
-                        fh.flush()
-                        fh.close()
-                    except Exception:
-                        pass
-                    setattr(self, fh_attr, None)
-
-    @property
-    def is_connected(self) -> bool:
-        return self._inner.is_connected
-
-    def write(self, data: bytes) -> None:
-        self._inner.write(data)
-        if data and self._bw_fh is not None:
-            try:
-                self._bw_fh.write(data)
-            except Exception:
-                pass
-
-    def read(self, n: int) -> bytes:
-        got = self._inner.read(n)
-        if got and self._s3_fh is not None:
-            try:
-                self._s3_fh.write(got)
-            except Exception:
-                pass
-        return got
-
-    @property
-    def bw_path(self) -> str:
-        return self._bw_path
-
-    @property
-    def s3_path(self) -> str:
-        return self._s3_path
-
-    def __repr__(self) -> str:
-        return f"CapturingTransport({self._inner!r}, bw={self._bw_path!r}, s3={self._s3_path!r})"

minimateplus/waveform_codec.py

@@ -1,948 +0,0 @@
-"""
-waveform_codec.py — block-walker and verified decoder for the MiniMate Plus
-waveform-file body.
-
-FULLY DECODED 2026-05-11.  Every block type, every channel, and the
-channel-rotation rule are verified byte-exact against BW's ASCII export
-across the 9-event fixture bundle (47,364 ADC samples, zero errors).
-
-The Blastware waveform-file body — the bytes between the 21-byte STRT
-record and the 26-byte file footer — is a tagged variable-length block
-stream with a custom delta + RLE codec.  (Not raw int16 LE, which was
-the historical wrong assumption that produced ±32K noise on every event.)
-
-Current status:
-
-- Block framing: ✅ solved (5 block types and lengths all confirmed)
-- Per-channel decode: ✅ solved (Tran / Vert / Long / MicL all byte-exact)
-- Channel rotation: ✅ Tran → Vert → Long → MicL per segment
-- Segment header: ✅ fully decoded (anchor pair + prev-channel extension)
-- 30 NN packed-delta block: ✅ NN × 12-bit signed deltas in NN/4 groups
-- MicL → dB(L) conversion: ✅ ``mic_count_to_db`` matches BW display
-- Production wiring: ✅ ``client.py:_decode_a5_waveform`` uses the new
-  codec (via ``decode_a5_frames``).  ``.h5`` sidecars now render
-  correctly.
-
-Known limitations:
-
-- Walker stops early on the loudest events (SP0, SS0, SV0, event-b) at
-  some mid-segment edge cases not yet fully characterized.  Every
-  sample reached IS correct; the walker just doesn't reach all of
-  them yet.  The cleanly-decoded subset is still ~5000–15000 samples
-  per loud event.
-
-────────────────────────────────────────────────────────────────────────────
-Body layout (CONFIRMED 2026-05-11 against 8 fixture events)
-────────────────────────────────────────────────────────────────────────────
-
-    [7-byte preamble] [stream of tagged blocks] [trailer]
-
-The preamble is always exactly 7 bytes:
-
-    body[0:3]  = 00 02 00              magic
-    body[3:5]  = Tran[0]   int16 BE    in 16-count units (LSB = 0.005 in/s)
-    body[5:7]  = Tran[1]   int16 BE    in 16-count units
-
-(Earlier drafts of this module described a "7-or-9-byte preamble";
-that was wrong — single-shot and continuous events both use 7 bytes.
-The "extra 2 bytes" on continuous events were the first ``00 NN`` RLE
-marker, not part of the preamble.)
-
-Block types and lengths (all confirmed):
-
-| Tag      | Length                | Meaning                                |
-|----------|-----------------------|----------------------------------------|
-| ``10 NN``| NN/2 + 2 bytes        | 4-bit nibble deltas (2 per byte; high  |
-|          |                       | nibble first; signed 0..7 / 8..F = -8..-1)|
-| ``20 NN``| NN + 2 bytes          | int8 signed deltas (1 per byte)        |
-| ``00 NN``| 2 bytes               | RLE: append NN copies of current value |
-| ``30 NN``| NN*2 in data, NN*4    | Unknown content.  Only in loud events. |
-|          | in trailer            |                                        |
-| ``40 02``| 20 bytes (fixed)      | Segment header                         |
-
-NN is always a multiple of 4.
-
-────────────────────────────────────────────────────────────────────────────
-Tran channel, segment 0 (CONFIRMED 2026-05-11)
-────────────────────────────────────────────────────────────────────────────
-
-Segment 0 — everything before the first ``40 02`` segment header — encodes
-Tran samples only.  Starting from preamble anchors Tran[0] and Tran[1],
-each subsequent block contributes to the running Tran value:
-
-    10 NN  →  append NN deltas (4-bit signed nibbles)
-    20 NN  →  append NN deltas (int8 signed bytes)
-    00 NN  →  append NN copies of the current value (RLE zeros)
-    40 02  →  segment 0 ends; multi-segment continuation is open
-
-This decodes the first 482–510 samples of Tran for each event with zero
-errors against BW's ASCII export.  The exact segment-0 sample count
-varies per event (it's bounded by a fixed device-flash byte budget, not
-a fixed sample count — quiet events fit more samples because zero
-deltas pack into ``00 NN`` markers compactly).
-
-Implementation: :func:`decode_tran_initial`.
-
-────────────────────────────────────────────────────────────────────────────
-Segment header (40 02, 20 bytes total)
-────────────────────────────────────────────────────────────────────────────
-
-The 18-byte payload of the ``40 02`` block:
-
-| Offset    | Field                                       | Status      |
-|-----------|---------------------------------------------|-------------|
-| [0:2]     | T_delta at first sample of new segment      | ✅ confirmed|
-|           | (int16 BE, in 16-count units)               |             |
-| [2:4]     | Likely T_delta at sample seg_start+1        | 🟡 likely   |
-| [4:6]     | Unknown (varies; possibly checksum)         | ❓ open     |
-| [6:8]     | Byte length to next segment header − 2      | ✅ confirmed|
-|           | (uint16 BE; useful for walker pre-scan)     |             |
-| [8:12]    | Monotonic uint32 LE counter                 | ✅ confirmed|
-|           | (starts ~0x47, increments by 1 per segment) |             |
-| [12:14]   | Constant ``02 00``                          | ✅ confirmed|
-| [14:18]   | Unknown 4-byte field                        | ❓ open     |
-
-────────────────────────────────────────────────────────────────────────────
-What breaks the multi-segment decoder (the main open question)
-────────────────────────────────────────────────────────────────────────────
-
-After segment 0 ends and the segment header T_delta is consumed,
-applying segment 1's blocks as Tran continuation produces values that
-diverge from truth by sample ~512.  The block structure inside segment
-1 is IDENTICAL to segment 0 (same alternating 10 NN / 00 NN pattern),
-and the delta budget matches the segment size exactly (V70 segment 1
-has 264 nibble-deltas + 244 RLE zeros = 508 = the segment's sample
-count).  But the cumulative is wrong.
-
-The strongest unverified hypothesis is that segments rotate channels:
-
-    segment 0  →  Tran samples 0..509
-    segment 1  →  Vert samples 0..507
-    segment 2  →  Long samples 0..507
-    segment 3  →  Mic  samples 0..507
-    segment 4  →  Tran samples 510..N (continuation)
-    ...
-
-This is consistent with the segment-1 block sums net-to-near-zero in
-V70 (where all 4 channels are near zero) and with the per-segment delta
-budget matching the segment size for a single channel.  It is NOT yet
-verified because the per-segment channel anchor isn't pinned down in
-the segment header — bytes [4:6] and [14:18] of the header are still
-open and probably encode V/L/M anchors.
-
-See ``docs/waveform_codec_re_status.md`` for the current working notes
-and the suggested next experiment ("segment-channel scoring analyzer").
-"""
-
-from __future__ import annotations
-
-import math
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-
-
-@dataclass
-class WaveformBlock:
-    """One tagged block parsed out of a Blastware waveform-file body."""
-    offset: int      # byte offset into body
-    tag_hi: int      # first tag byte (0x10 / 0x20 / 0x00 / 0x30 / 0x40)
-    tag_lo: int      # second tag byte (NN)
-    data: bytes      # block payload (excludes the 2-byte tag)
-    length: int      # total block length on the wire (includes the tag)
-
-    @property
-    def kind(self) -> str:
-        return f"{self.tag_hi:02x} {self.tag_lo:02x}"
-
-
-def find_data_start(body: bytes) -> int:
-    """Auto-detect the offset of the first data block.
-
-    The body starts with a 7-byte preamble (magic ``00 02 00`` + two int16 BE
-    Tran anchors).  After that, the data section starts with a tag — usually
-    ``10 NN`` or ``20 NN``, but quiet events may begin with a ``00 NN`` RLE
-    marker.  We return the offset of the first recognized tag.
-    """
-    # Try fixed offset 7 first (canonical preamble length).
-    if len(body) >= 9:
-        b, nn = body[7], body[8]
-        # Accept the same tag vocabulary ``walk_body`` accepts, including the
-        # wide-NN forms (``0X``/``1X``/``2X``) and the variable-width ``40 NN``
-        # segment header.
-        if ((b & 0xF0) in (0x00, 0x10, 0x20) and nn % 4 == 0
-                and ((b & 0x0F) != 0 or 0 < nn <= 0xFC)) \
-                or (b == 0x30 and nn % 4 == 0 and 0 < nn <= 0xFC) \
-                or (b == 0x40 and 0 < nn <= 0x08) \
-                or is_tagless_segment_header(body, 7):
-            return 7
-    # Fall back to scanning the first 20 bytes.
-    for i in range(min(20, len(body) - 1)):
-        b = body[i]
-        nn = body[i + 1]
-        if b in (0x10, 0x20) and nn % 4 == 0 and 0 < nn <= 0xFC:
-            return i
-    return -1
-
-
-# Channel-id byte carried in every segment header.  Previously mis-read as a
-# "monotonic uint32 LE counter"; it is really ``[channel][00][00][segment]``.
-# Verified 2026-08-25 on 1697/1697 segment headers across the ground-truth
-# corpus with zero disagreements against the decoded channel rotation.
-SEGMENT_CHANNEL_IDS = {0x46: "Tran", 0x47: "Vert", 0x48: "Long", 0x49: "MicL"}
-
-# A tagless segment header: the 14-byte tail of a ``40 NN`` header with no tag
-# and no previous-channel continuation deltas (the NN=0 case).
-_TAGLESS_HEADER_LEN = 14
-
-
-def is_tagless_segment_header(body: bytes, i: int) -> bool:
-    """True if a bare 14-byte segment header starts at *i*.
-
-    Layout ``[field2:2][len_to_next:2][channel_id:4][marker:2][anchors:4]``.
-    The discriminator is the 6 bytes at ``[4:10]``: a known channel id, two
-    zero bytes, a small segment index, and the ``01 00`` / ``02 00`` marker.
-    """
-    if i + _TAGLESS_HEADER_LEN > len(body):
-        return False
-    return (body[i + 4] in SEGMENT_CHANNEL_IDS
-            and body[i + 5] == 0x00 and body[i + 6] == 0x00
-            and body[i + 8] in (0x01, 0x02) and body[i + 9] == 0x00)
-
-
-def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]:
-    """Walk the tagged-block sequence starting at *start* (auto-detected by default).
-
-    Stops when an unrecognized tag is encountered or end of body is reached.
-    Returned blocks are in stream order.
-    """
-    if start is None:
-        start = find_data_start(body)
-        if start < 0:
-            return []
-
-    blocks: List[WaveformBlock] = []
-    i = start
-    while i + 1 < len(body):
-        t0 = body[i]
-        t1 = body[i + 1]
-        if t0 == 0x10 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 // 2 + 2
-        elif (t0 & 0xF0) == 0x10 and (t0 & 0x0F) != 0 and t1 % 4 == 0:
-            # Wide-NN nibble block: ``1X NN`` where X is the high nibble of a
-            # 12-bit NN value.  NN = ((t0 & 0x0F) << 8) | t1.  Block length
-            # = NN/2 + 2 bytes (NN nibble deltas, same as ``10 NN`` semantics
-            # but with NN > 0xFC).  Confirmed 2026-05-11 in SP0 segment 12
-            # where V continuation uses ``11 90`` = NN=0x190=400.
-            wide_nn = ((t0 & 0x0F) << 8) | t1
-            length = wide_nn // 2 + 2
-        elif t0 == 0x20 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            length = t1 + 2
-        elif (t0 & 0xF0) == 0x20 and (t0 & 0x0F) != 0 and t1 % 4 == 0:
-            # Wide-NN int8 block: ``2X NN`` extends NN to 12 bits the same way.
-            wide_nn = ((t0 & 0x0F) << 8) | t1
-            length = wide_nn + 2
-        elif (t0 & 0xF0) == 0x00 and t1 % 4 == 0:
-            # ``00 NN`` RLE zero-delta run, plus its wide form ``0X NN``
-            # (X != 0) which extends NN to 12 bits exactly like ``1X``/``2X``:
-            # NN = ((t0 & 0x0F) << 8) | t1.  A narrow run maxes out at
-            # NN=0xFC, so quiet stretches longer than 252 samples must use
-            # the wide form.  Confirmed 2026-08-25 against six production
-            # events (e.g. ``01 0c`` = 268 repeats in K558LKOF.460W).
-            length = 2
-        elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0xFC:
-            # Data-section ``30 NN`` blocks carry NN 12-bit signed deltas packed
-            # as NN/4 groups of (2-byte high-nibble field + 4 × int8 low byte).
-            # Length = NN/4 × 6 + 2 = NN × 1.5 + 2 (= 8 for NN=4, 14 for NN=8,
-            # 20 for NN=12, etc.).  Confirmed 2026-05-11 by full-decoder
-            # verification against BW ASCII export.
-            #
-            # Trailer-section ``30 NN`` blocks have a different length formula
-            # (NN × 4 = 32 for NN=8 in trailers).  We try the data-section
-            # length first and fall back to the trailer length if needed.
-            cand_data = t1 * 3 // 2 + 2
-            cand_trailer = t1 * 4
-            if (i + cand_data < len(body) - 1
-                    and body[i + cand_data] in (0x10, 0x20, 0x00, 0x30, 0x40)):
-                length = cand_data
-            else:
-                length = cand_trailer
-        elif t0 == 0x40 and 0 < t1 <= 0x08:
-            # ``40 NN`` segment header.  NN is the number of int16 BE
-            # continuation deltas the header carries for the PREVIOUS
-            # channel, so the header grows with NN:
-            #     length = 2 (tag) + 2*NN (deltas) + 14 (fixed tail)
-            # ``40 02`` (20 bytes) dominates, but ``40 01`` (18) and
-            # ``40 03`` (22) both occur in production files.  Confirmed
-            # 2026-08-25; the constant ``02 00`` marker moves with NN too
-            # (see :func:`parse_segment_header`).
-            length = 2 * t1 + 16
-        elif is_tagless_segment_header(body, i):
-            # Segment header with no ``40 NN`` tag (NN=0 — the previous channel
-            # needed no continuation deltas).  Emit it as a synthetic ``40 00``
-            # block whose ``data`` is the whole 14-byte record, so the nd=0
-            # offsets in :func:`decode_waveform_v2` line up unchanged.
-            blocks.append(WaveformBlock(
-                offset=i, tag_hi=0x40, tag_lo=0x00,
-                data=bytes(body[i : i + _TAGLESS_HEADER_LEN]),
-                length=_TAGLESS_HEADER_LEN,
-            ))
-            i += _TAGLESS_HEADER_LEN
-            continue
-        else:
-            # Unknown tag; stop.  Caller can inspect ``i`` to see where.
-            break
-
-        if i + length > len(body):
-            break
-
-        data = bytes(body[i + 2 : i + length])
-        blocks.append(WaveformBlock(offset=i, tag_hi=t0, tag_lo=t1, data=data, length=length))
-        i += length
-
-    return blocks
-
-
-def split_segments(blocks: List[WaveformBlock]) -> List[List[WaveformBlock]]:
-    """Group consecutive blocks into segments separated by ``40 02`` headers.
-
-    The first segment is whatever runs before the first ``40 02`` header
-    (typically the "segment 0" preamble data after the body preamble).
-    Subsequent segments start with a ``40 02`` block, then have their
-    own data blocks until the next ``40 02``.
-    """
-    segments: List[List[WaveformBlock]] = []
-    current: List[WaveformBlock] = []
-    for b in blocks:
-        if b.tag_hi == 0x40:
-            if current:
-                segments.append(current)
-            current = [b]
-        else:
-            current.append(b)
-    if current:
-        segments.append(current)
-    return segments
-
-
-def parse_segment_header(block: WaveformBlock) -> Optional[dict]:
-    """Decode the payload of a ``40 NN`` segment header.
-
-    NN (the tag's low byte) is the number of int16 BE continuation deltas
-    the header carries for the PREVIOUS channel, so every field after
-    those deltas shifts by ``2 * NN``.  The payload is ``2 * NN + 14``
-    bytes.  ``40 02`` is the common case; ``40 01`` and ``40 03`` also
-    occur in production files (confirmed 2026-08-25).
-
-    Returns a dict with the labelled fields, or None if *block* is not a
-    segment header or is too short.
-    """
-    if block.tag_hi != 0x40 or block.tag_lo > 0x08:
-        return None
-    nd = block.tag_lo
-    if len(block.data) < 2 * nd + 14:
-        return None
-    p = block.data
-    counter = int.from_bytes(p[2 * nd + 4 : 2 * nd + 8], "little", signed=False)
-    return {
-        "n_prev_deltas": nd,
-        # ``nd`` int16 BE deltas extending the previous channel.
-        "prev_deltas": [
-            int.from_bytes(p[2 * k : 2 * k + 2], "big", signed=True)
-            for k in range(nd)
-        ],
-        "field2": p[2 * nd : 2 * nd + 4],   # 4-byte field, role unconfirmed
-        "counter": counter,                 # legacy: raw uint32 LE of the id field
-        "channel": SEGMENT_CHANNEL_IDS.get(p[2 * nd + 4]),
-        "segment_index": p[2 * nd + 7],
-        "marker": p[2 * nd + 8 : 2 * nd + 10],      # always b"\x02\x00"
-        "anchors": [
-            int.from_bytes(p[2 * nd + 10 : 2 * nd + 12], "big", signed=True),
-            int.from_bytes(p[2 * nd + 12 : 2 * nd + 14], "big", signed=True),
-        ],
-    }
-
-
-def _s4(n: int) -> int:
-    """Sign-extend a 4-bit value to signed int (0..7 → 0..7; 8..F → -8..-1)."""
-    return n if n < 8 else n - 16
-
-
-def _i8(b: int) -> int:
-    """Reinterpret an unsigned byte as signed int8."""
-    return b if b < 128 else b - 256
-
-
-def decode_tran_initial(body: bytes) -> Optional[List[int]]:
-    """
-    Decode the initial Tran-channel samples — VERIFIED 2026-05-11.
-
-    Returns Tran samples in **16-count units** (LSB = 0.005 in/s at Normal
-    range — the same quantization BW uses for its ASCII export).  Returns
-    ``None`` if the body cannot be parsed.
-
-    The decoded list extends from sample 0 through the end of segment 0
-    (= just before the first ``40 02`` segment header; ~510 sample-sets
-    for the events tested).  Multi-segment decoding requires continuing
-    past the segment header — that's done by :func:`decode_tran_full`
-    when the per-segment rules are pinned down for all signal types.
-
-    Codec for segment 0 (CONFIRMED 2026-05-11 against 7 fixture events):
-
-    - Body bytes [0:3] are the magic ``00 02 00``.
-    - Body bytes [3:5] = ``Tran[0]`` as int16 BE in 16-count units.
-    - Body bytes [5:7] = ``Tran[1]`` as int16 BE in 16-count units.
-    - Data blocks (``10 NN`` or ``20 NN``) carry Tran deltas starting
-      at sample 2:
-
-      * ``10 NN``: NN nibbles = NN/2 bytes; each nibble is a 4-bit
-        signed delta (0..7 → 0..+7; 8..F → -8..-1).  High nibble of
-        each byte comes first.
-      * ``20 NN``: NN int8 signed deltas (one delta per byte).
-
-    - ``00 NN`` blocks are run-length-encoded zero deltas: append NN
-      copies of the current cumulative Tran value (no change).
-
-    - ``30 NN`` blocks have not yet been decoded for content — they
-      appear in segment 0 of loud-from-start events (SS0, SV0) and
-      seem to signal a transition or special-case interpretation.
-      The walker steps over them but their data is ignored.
-
-    The walk stops at the first ``40 02`` segment header.
-    """
-    if len(body) < 7 or body[0:3] != b"\x00\x02\x00":
-        return None
-    t0 = int.from_bytes(body[3:5], "big", signed=True)
-    t1 = int.from_bytes(body[5:7], "big", signed=True)
-
-    start = find_data_start(body)
-    if start < 0:
-        return [t0, t1]
-
-    out = [t0, t1]
-    cur = t1
-    for blk in walk_body(body, start):
-        if blk.tag_hi == 0x40:
-            # Segment boundary — stop.  Multi-segment decode is decode_tran_full.
-            break
-        if blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += _s4(nib)
-                    out.append(cur)
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur += _i8(byte)
-                out.append(cur)
-        elif blk.tag_hi == 0x00:
-            # RLE zero deltas: append NN copies of current Tran value.
-            for _ in range(blk.tag_lo):
-                out.append(cur)
-        # 30 NN: unknown content; skip.
-    return out
-
-
-def decode_waveform_legacy(body: bytes) -> Optional[dict]:
-    """
-    SUPERSEDED 2026-08-25 — the tag-dispatch / segment-header model.
-
-    Retained because ``micromate/idf_file.py`` trial-decodes Thor IDFW bodies
-    at many candidate offsets and keeps whichever yields the most samples;
-    the record-chain decoder returns None where this one returned garbage,
-    which shifts that heuristic's winner.  Thor is pinned here until its own
-    body-offset search is reworked.  Do not use for series-3.
-
-    Decode the body into per-channel sample arrays.
-
-    Status (2026-05-11 evening — channel-rotation hypothesis CONFIRMED):
-    segments rotate channels in fixed order **Tran → Vert → Long → MicL**.
-    Each channel-segment carries a 2-sample anchor pair in segment-header
-    bytes [14:18] (or in the body preamble for the initial Tran segment)
-    plus a stream of delta blocks for samples 2 onward.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}``
-    with each channel's decoded samples in 16-count units (LSB = 0.005
-    in/s at Normal range).  Returns ``None`` if the body cannot be
-    parsed.
-    """
-    if len(body) < 7 or body[0:3] != b"\x00\x02\x00":
-        return None
-
-    channels = ["Tran", "Vert", "Long", "MicL"]
-    out: dict = {ch: [] for ch in channels}
-
-    # Initial Tran segment: preamble anchor pair + delta blocks before first 40 02.
-    t0 = int.from_bytes(body[3:5], "big", signed=True)
-    t1 = int.from_bytes(body[5:7], "big", signed=True)
-    out["Tran"].extend([t0, t1])
-
-    start = find_data_start(body)
-    if start < 0:
-        return out
-
-    blocks = walk_body(body, start)
-    seg_idx = [i for i, b in enumerate(blocks) if b.tag_hi == 0x40]
-
-    def apply_blocks(channel: str, anchor: int,
-                     block_start: int, block_end: int) -> int:
-        """Apply delta blocks [block_start, block_end) to *channel*'s sample
-        list, starting from *anchor*.  Returns the final cumulative value."""
-        cur = anchor
-        for bi in range(block_start, block_end):
-            blk = blocks[bi]
-            if (blk.tag_hi & 0xF0) == 0x10:
-                # Both ``10 NN`` (NN ≤ 0xFC) and wide-NN ``1X NN`` (X != 0)
-                # are nibble-delta streams.  The walker has already used the
-                # right length; here we just iterate the payload bytes.
-                for byte in blk.data:
-                    for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                        cur += _s4(nib)
-                        out[channel].append(cur)
-            elif (blk.tag_hi & 0xF0) == 0x20:
-                # ``20 NN`` and wide ``2X NN`` both carry int8 deltas.
-                for byte in blk.data:
-                    cur += _i8(byte)
-                    out[channel].append(cur)
-            elif (blk.tag_hi & 0xF0) == 0x00:
-                # RLE zero-delta run.  Wide form ``0X NN`` carries the high
-                # nibble of a 12-bit NN in the tag byte, same as ``1X``/``2X``.
-                run = ((blk.tag_hi & 0x0F) << 8) | blk.tag_lo
-                for _ in range(run):
-                    out[channel].append(cur)
-            elif blk.tag_hi == 0x30:
-                # 12-bit signed deltas, packed as NN/4 groups of 6 bytes each:
-                #   bytes [0:2] = 16 bits = 4 × 4-bit high nibbles (MSB first)
-                #   bytes [2:6] = 4 × int8 low bytes
-                # Each delta = sign_extend_12((high_nibble << 8) | low_byte).
-                # Confirmed 2026-05-11 against all 14 ``30 NN`` blocks in the
-                # bundled fixtures.
-                n_groups = blk.tag_lo // 4
-                for g in range(n_groups):
-                    grp = blk.data[g * 6 : (g + 1) * 6]
-                    if len(grp) < 6:
-                        break
-                    high_word = (grp[0] << 8) | grp[1]
-                    for k in range(4):
-                        nib = (high_word >> (12 - 4 * k)) & 0xF
-                        v = (nib << 8) | grp[2 + k]
-                        if v >= 0x800:
-                            v -= 0x1000
-                        cur += v
-                        out[channel].append(cur)
-            # 40 02: should not occur in segment data.
-        return cur
-
-    # Initial Tran segment: deltas from start of body up to first 40 02 (or end).
-    first_seg = seg_idx[0] if seg_idx else len(blocks)
-    last_tran_value = apply_blocks("Tran", t1, 0, first_seg)
-
-    # Subsequent segments rotate channels.  Each segment header carries:
-    #   bytes [0:2] and [2:4] = 2 deltas extending the PREVIOUS channel
-    #   bytes [14:16] and [16:18] = anchor pair for THIS segment's channel
-    #
-    # Rotation: V, L, M, T, V, L, M, T, ...  (initial Tran segment is the
-    # implicit T in the cycle.)
-    rotation = ["Vert", "Long", "MicL", "Tran"]
-    # Track each channel's "running cumulative value" so we can apply the
-    # previous-channel extension deltas at every segment boundary.
-    last_value = {"Tran": last_tran_value, "Vert": None, "Long": None, "MicL": None}
-
-    prev_channel = "Tran"
-    for k, hi in enumerate(seg_idx):
-        header = blocks[hi]
-        # Channel comes from the header's own id byte, which is authoritative.
-        # The old rotation-by-position fallback is kept for headers whose id
-        # byte isn't one of the four known values — but a single missed or
-        # extra header would desync rotation and corrupt every later channel,
-        # which is exactly what tagless headers used to cause.
-        _nd = header.tag_lo
-        channel = None
-        if len(header.data) >= 2 * _nd + 8:
-            channel = SEGMENT_CHANNEL_IDS.get(header.data[2 * _nd + 4])
-        if channel is None:
-            channel = rotation[k % 4]
-        # ``40 NN``: NN int16 BE continuation deltas for the previous channel
-        # come first, so every later field shifts by 2*NN.  NN is usually 2
-        # but 1 and 3 both occur (confirmed 2026-08-25).
-        nd = header.tag_lo
-        if len(header.data) < 2 * nd + 14:
-            continue
-        # Validate: real segment headers have the constant `02 00` marker
-        # right after the counter.  Trailer/footer "40 NN" markers contain
-        # ASCII serial bytes or other non-header data there and would
-        # otherwise be mis-read as segment headers, adding spurious tail
-        # samples.
-        if header.data[2 * nd + 8 : 2 * nd + 10] != b"\x02\x00":
-            break
-        # Extend the PREVIOUS channel by NN more samples.
-        if last_value[prev_channel] is not None:
-            v = last_value[prev_channel]
-            for d in range(nd):   # NB: not `k` — that's the segment index
-                v += int.from_bytes(
-                    header.data[2 * d : 2 * d + 2], "big", signed=True
-                )
-                out[prev_channel].append(v)
-            last_value[prev_channel] = v
-        # Anchor pair for THIS segment's channel.
-        c0 = int.from_bytes(
-            header.data[2 * nd + 10 : 2 * nd + 12], "big", signed=True
-        )
-        c1 = int.from_bytes(
-            header.data[2 * nd + 12 : 2 * nd + 14], "big", signed=True
-        )
-        out[channel].extend([c0, c1])
-        # Apply delta blocks for this segment.
-        next_hi = seg_idx[k + 1] if k + 1 < len(seg_idx) else len(blocks)
-        last_value[channel] = apply_blocks(channel, c1, hi + 1, next_hi)
-        prev_channel = channel
-
-    return out
-
-
-# ── ADC-scale conversion helpers ────────────────────────────────────────────
-
-
-# Scaling factor: decode_waveform_v2 produces geo-channel samples in the BW
-# display quantization (16-count units, LSB = 0.005 in/s at Normal range).
-# The legacy consumer pipeline (sfm/event_hdf5.py) expects raw_samples in
-# 1-count ADC units (× full_scale / 32768 → physical).  To plug the new
-# decoder in without rewriting consumers, multiply geo values by 16.
-#
-# Mic samples are already in raw ADC counts (decoded value 1 = 1 mic ADC count
-# = -81.94 dB on the BW display).  Mic values pass through unchanged.
-_GEO_DECODER_TO_ADC = 16
-
-
-def decoded_to_adc_counts(decoded: dict) -> dict:
-    """Convert :func:`decode_waveform_v2` output to int16 ADC counts.
-
-    Geo channels are scaled by ×16 (decoder produces 16-count units,
-    consumer expects 1-count ADC).  Mic is passed through as raw counts.
-    """
-    if not decoded:
-        return {}
-    return {
-        "Tran": [v * _GEO_DECODER_TO_ADC for v in decoded.get("Tran", [])],
-        "Vert": [v * _GEO_DECODER_TO_ADC for v in decoded.get("Vert", [])],
-        "Long": [v * _GEO_DECODER_TO_ADC for v in decoded.get("Long", [])],
-        "MicL": list(decoded.get("MicL", [])),
-    }
-
-
-def mic_count_to_db(count: int) -> float:
-    """Convert a MicL ADC count to dB(L) for BW-display-compatible output.
-
-    Empirical formula (confirmed 2026-05-11 against V70 fixture: count=813
-    → 140.1 dB; count=±1 → ±81.94 dB; count=±24 → ±109.5 dB):
-
-        dB = sign(count) × (81.94 + 20 × log10(|count|))    for |count| ≥ 1
-        dB = 0.0                                            for count == 0
-
-    The constant 81.94 corresponds to 10^(81.94/20) ≈ 12490 mic ADC counts
-    being the dB(L) reference level — almost certainly a calibration
-    constant from the device's mic.
-    """
-    if count == 0:
-        return 0.0
-    sign = 1.0 if count > 0 else -1.0
-    return sign * (81.94 + 20.0 * math.log10(abs(count)))
-
-
-# ── A5-frame entry point ────────────────────────────────────────────────────
-
-
-def decode_a5_frames(a5_frames) -> Optional[dict]:
-    """Decode a list of A5 (BULK_WAVEFORM_STREAM) frames into per-channel
-    int16 ADC samples.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}``
-    with each channel's samples in **1-count ADC units** (the legacy
-    ``event.raw_samples`` convention — multiply by ``full_scale / 32768``
-    to convert to physical units; for mic, use :func:`mic_count_to_db` or
-    a per-count psi factor).
-
-    Returns ``None`` if the frames cannot be parsed.
-
-    This is the wired-up production entry point.  It:
-      1. Reconstructs the BW-binary body bytes from the A5 frames
-         (``blastware_file.extract_body_bytes``).
-      2. Runs the verified codec (``decode_waveform_v2``) on the body.
-      3. Converts to int16 ADC counts via :func:`decoded_to_adc_counts`.
-    """
-    # Local import to avoid a cycle: blastware_file imports models and
-    # ultimately client.py imports waveform_codec.
-    from .blastware_file import extract_body_bytes
-
-    if not a5_frames:
-        return None
-    _strt, body, _footer = extract_body_bytes(a5_frames)
-    if not body:
-        return None
-    decoded = decode_waveform_v2(body)
-    if decoded is None:
-        return None
-    return decoded_to_adc_counts(decoded)
-
-
-# ── Record-chain body model (CONFIRMED 2026-08-25) ──────────────────────────
-#
-# The body is NOT a flat tag-dispatch stream with ``40 NN`` segment headers.
-# It is a chain of self-delimiting per-channel RECORDS:
-#
-#     off+0   field2    uint16     purpose unknown (not a length, not a checksum)
-#     off+2   len       uint16 BE  next_record = off + 2 + len   <- authoritative
-#     off+4   chan_id   0x46 Tran / 0x47 Vert / 0x48 Long / 0x49 MicL
-#                       0x06 = end of waveform stream
-#     off+5   0x00
-#     off+6   0x00
-#     off+7   segment index
-#     off+8   mode      2 bytes, a 3-valued enum (see below)
-#     off+10  anchors   2 x int16 BE, ABSOLUTE — present only when mode is 02 00
-#
-# Mode semantics, all ground-truth verified:
-#     02 00   14-byte header; emit the 2 anchors, then blocks are CUMULATIVE deltas
-#     01 00   10-byte header; no anchors; blocks carry ABSOLUTE sample values
-#     00 03   10-byte header; NO TAGS AT ALL — the data section is raw 12-bit
-#             packed ABSOLUTE samples (6 bytes -> 4 samples)
-#
-# ``40 NN`` is an ordinary int16 BE DATA block (length 2*NN + 2), never a header.
-# The previous model read it as a variable-width segment header of length
-# 2*NN + 16, which is why walks drifted and channels came out unequal.
-#
-# Verified over the 1,388 series-3 waveform binaries in the production
-# snapshot: the length chain terminates on a 0x06 record in 1,387 of them (the
-# exception has an ambiguous footer, handled by the caller), and all four
-# channels come out at identical length in 1,388/1,388 — against 156/1,388
-# under the superseded model.  Against the 75 events with a preserved
-# Blastware ASCII export: sample-count exact 72/75 -> 75/75, fully exact
-# 70/75 -> 73/75.
-
-CHANNEL_IDS = {0x46: "Tran", 0x47: "Vert", 0x48: "Long", 0x49: "MicL"}
-STREAM_END_ID = 0x06
-
-MODE_DELTA    = (0x02, 0x00)
-MODE_ABSOLUTE = (0x01, 0x00)
-MODE_RAW12    = (0x00, 0x03)
-# Raw int16 BE absolute samples, 10-byte header, no tags — the same shape as
-# MODE_RAW12 but two bytes per sample instead of 1.5.  Found on Thor/Micromate
-# segment-0 records (2026-09-10): a `len=1032` record carries exactly
-# (1032 - 8) / 2 = 512 samples and reproduces Thor's own export 512/512
-# exactly.  Before this mode existed the record fell through the dispatch
-# unhandled, so the channel silently lost its first 512 samples.
-MODE_RAW16    = (0x00, 0x00)
-_MODES = (MODE_DELTA, MODE_ABSOLUTE, MODE_RAW12, MODE_RAW16)
-
-# Preambles whose leading data is untagged and therefore cannot be
-# block-walked; find_first_record() must scan for the next record instead.
-_UNTAGGED_MODES = (MODE_RAW12, MODE_RAW16)
-
-
-def _u16(b: bytes, p: int) -> int:
-    return (b[p] << 8) | b[p + 1]
-
-
-def _i16(b: bytes, p: int) -> int:
-    v = _u16(b, p)
-    return v - 0x10000 if v >= 0x8000 else v
-
-
-def data_block_len(body: bytes, p: int) -> Tuple[Optional[int], Optional[int]]:
-    """``(byte_length, n_samples)`` of the data block at *p*, or ``(None, None)``.
-
-    Data-section blocks only — there is no segment-header tag in this model.
-    ``30 NN`` has no trailer-length fallback here; that fallback corrupted
-    records whose ``30 NN`` sat near a record boundary.
-    """
-    if p + 2 > len(body):
-        return None, None
-    t0, t1 = body[p], body[p + 1]
-    hi = t0 & 0xF0
-    nn = ((t0 & 0x0F) << 8) | t1
-    if hi == 0x40:                          # int16 BE data block
-        # NN was capped at 0x08 until 2026-09-11.  That cap had no basis: the
-        # two corpora available at the time only ever used NN in {1,2,3,4,8},
-        # so it was never exercised.  Loud UM12947 events use NN of 12, 16,
-        # 20 ... up to 196, and every value above 8 halted the walk, which
-        # surfaced as silently short channels (walk_body/run stop at the first
-        # unrecognised tag rather than raising).  Verified against Thor's own
-        # exports: 22 length-mismatched files -> 0, and the affected corpus
-        # went to 1,476,242/1,476,249 samples exact.  The real bound is the
-        # buffer; the caller additionally clamps to the record end.
-        if nn == 0 or p + 2 * nn + 2 > len(body):
-            return None, None
-        return 2 * nn + 2, nn
-    if nn == 0 or nn % 4:
-        return None, None
-    if hi == 0x00:
-        return 2, nn                        # RLE hold
-    if hi == 0x10:
-        return nn // 2 + 2, nn              # 4-bit nibble
-    if hi == 0x20:
-        return nn + 2, nn                   # int8
-    if hi == 0x30:
-        return nn * 3 // 2 + 2, nn          # 12-bit packed
-    return None, None
-
-
-def unpack16(data: bytes) -> List[int]:
-    """Raw int16 BE absolute samples (MODE_RAW16)."""
-    return [_i16(data, 2 * k) for k in range(len(data) // 2)]
-
-
-def unpack12(data: bytes) -> List[int]:
-    """Raw 12-bit packed samples: 6 bytes -> 4 signed values."""
-    out: List[int] = []
-    for g in range(len(data) // 6):
-        hi = (data[6 * g] << 8) | data[6 * g + 1]
-        for k in range(4):
-            x = (((hi >> (12 - 4 * k)) & 0xF) << 8) | data[6 * g + 2 + k]
-            out.append(x - 0x1000 if x >= 0x800 else x)
-    return out
-
-
-def is_record(body: bytes, p: int) -> bool:
-    """True if a per-channel record header starts at *p*."""
-    return (p + 10 <= len(body)
-            and body[p + 4] in CHANNEL_IDS
-            and body[p + 5] == 0x00 and body[p + 6] == 0x00
-            and 8 <= _u16(body, p + 2) <= len(body) - p
-            and (body[p + 8], body[p + 9]) in _MODES)
-
-
-def find_first_record(body: bytes) -> Optional[int]:
-    """Offset of the first record, or None.
-
-    Under the normal ``00 02 00`` preamble the leading bytes are segment-0's
-    Tran blocks, so walk them.  Under the untagged preambles (``00 00 03``
-    raw-12 and ``00 00 00`` raw-16) that data has no tags at all and cannot
-    be block-walked — scan for the next record header instead.
-    """
-    if len(body) >= 3 and (body[1], body[2]) in _UNTAGGED_MODES:
-        scan_from = 3
-    else:
-        # Tagged preamble.  MODE_DELTA carries a 14-byte record header (two
-        # int16 anchors), so its blocks start at body[7]; MODE_ABSOLUTE has a
-        # 10-byte header and starts at body[3].
-        i = 3 if (len(body) >= 3 and (body[1], body[2]) == MODE_ABSOLUTE) else 7
-        while i < len(body):
-            if is_record(body, i):
-                nxt = i + 2 + _u16(body, i + 2)
-                if nxt + 5 <= len(body) and (is_record(body, nxt)
-                                             or body[nxt + 4] == STREAM_END_ID):
-                    return i
-            length, _ = data_block_len(body, i)
-            if length is None:
-                return None
-            i += length
-        return None
-    for i in range(scan_from, max(scan_from, len(body) - 10)):
-        if is_record(body, i):
-            nxt = i + 2 + _u16(body, i + 2)
-            if nxt + 5 <= len(body) and (is_record(body, nxt)
-                                         or body[nxt + 4] == STREAM_END_ID):
-                return i
-    return None
-
-
-def walk_records(body: bytes, first: Optional[int] = None) -> List[dict]:
-    """Follow the length chain from *first* to the ``0x06`` terminator."""
-    if first is None:
-        first = find_first_record(body)
-    out: List[dict] = []
-    if first is None:
-        return out
-    p, seen = first, set()
-    while p is not None and p + 10 <= len(body):
-        if p in seen:
-            break
-        seen.add(p)
-        cid = body[p + 4]
-        if cid == STREAM_END_ID or cid not in CHANNEL_IDS:
-            break
-        length = _u16(body, p + 2)
-        if length < 8 or p + 2 + length > len(body):
-            break
-        out.append({"offset": p, "channel": CHANNEL_IDS[cid],
-                    "segment_index": body[p + 7],
-                    "mode": (body[p + 8], body[p + 9]),
-                    "end": p + 2 + length})
-        p += 2 + length
-    return out
-
-
-def decode_waveform_v2(body: bytes) -> Optional[dict]:
-    """Decode a Blastware waveform body into per-channel sample arrays.
-
-    Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}``
-    in 16-count units (LSB = 0.005 in/s at Normal range), or None if *body*
-    is not a decodable waveform body.
-
-    Implements the record-chain model documented above.
-    """
-    if len(body) < 8 or body[0] != 0x00:
-        return None
-    preamble = (body[1], body[2])
-    if preamble not in (MODE_DELTA, MODE_ABSOLUTE, MODE_RAW12, MODE_RAW16):
-        return None
-    first = find_first_record(body)
-    if first is None:
-        return None
-
-    out: dict = {c: [] for c in ("Tran", "Vert", "Long", "MicL")}
-
-    def run(channel: str, start: int, end: int, absolute: bool) -> None:
-        cur = out[channel][-1] if out[channel] else 0
-        i = start
-        while i < end:
-            length, nn = data_block_len(body, i)
-            if length is None or i + length > end:
-                return          # stop this record; the chain resyncs at end
-            hi = body[i] & 0xF0
-            if hi == 0x00:
-                vals = [None] * nn
-            elif hi == 0x10:
-                vals = []
-                for k in range(nn):
-                    byte = body[i + 2 + k // 2]
-                    v = (byte >> 4) if k % 2 == 0 else (byte & 0xF)
-                    vals.append(v - 16 if v >= 8 else v)
-            elif hi == 0x20:
-                vals = [v - 256 if v >= 128 else v
-                        for v in body[i + 2:i + 2 + nn]]
-            elif hi == 0x30:
-                vals = unpack12(body[i + 2:i + length])
-            else:
-                vals = [_i16(body, i + 2 + 2 * k) for k in range(nn)]
-            for v in vals:
-                if v is None:
-                    pass                    # RLE hold, in delta AND absolute modes
-                elif absolute:
-                    cur = v
-                else:
-                    cur += v
-                out[channel].append(cur)
-            i += length
-
-    # Segment 0 is an implicit Tran record carried in the preamble.
-    if preamble == MODE_DELTA:
-        out["Tran"].extend([_i16(body, 3), _i16(body, 5)])
-        run("Tran", 7, first, absolute=False)
-    elif preamble == MODE_ABSOLUTE:
-        run("Tran", 3, first, absolute=True)
-    elif preamble == MODE_RAW16:
-        out["Tran"].extend(unpack16(body[3:first]))
-    else:
-        out["Tran"].extend(unpack12(body[3:first]))
-
-    for rec in walk_records(body, first):
-        ch, off, mode, end = (rec["channel"], rec["offset"],
-                              rec["mode"], rec["end"])
-        if mode == MODE_DELTA:
-            out[ch].extend([_i16(body, off + 10), _i16(body, off + 12)])
-            run(ch, off + 14, end, absolute=False)
-        elif mode == MODE_ABSOLUTE:
-            run(ch, off + 10, end, absolute=True)
-        elif mode == MODE_RAW12:
-            out[ch].extend(unpack12(body[off + 10:end]))
-        elif mode == MODE_RAW16:
-            out[ch].extend(unpack16(body[off + 10:end]))
-    return out

parsers/s3_analyzer.py

@@ -53,9 +53,7 @@ SUB_TABLE: dict[int, tuple[str, str, str]] = {
     0x82: ("TRIGGER_CONFIG_WRITE",      "BW→S3", "0x1C bytes; trigger config block; mirrors SUB 1C"),
     0x83: ("TRIGGER_WRITE_CONFIRM",     "BW→S3", "Short frame; commit step after 0x82"),
     # S3→BW responses
-    0x5A: ("BULK_WAVEFORM_STREAM",       "BW→S3", "Bulk waveform chunk request; response is A5 stream"),
     0xA4: ("POLL_RESPONSE",             "S3→BW", "Response to SUB 5B poll"),
-    0xA5: ("BULK_WAVEFORM_RESPONSE",    "S3→BW", "Response to SUB 5A; waveform chunks + metadata"),
     0xFE: ("FULL_CONFIG_RESPONSE",      "S3→BW", "Response to SUB 01"),
     0xF9: ("CHANNEL_CONFIG_RESPONSE",   "S3→BW", "Response to SUB 06"),
     0xF7: ("EVENT_INDEX_RESPONSE",      "S3→BW", "Response to SUB 08; contains backlight/power-save"),

parsers/s3_parser.py

@@ -33,7 +33,7 @@ STX = 0x02
 ETX = 0x03
 ACK = 0x41
 
-__version__ = "0.2.5"
+__version__ = "0.2.3"
 
 
 @dataclass
@@ -184,9 +184,9 @@ def validate_bw_body_auto(body: bytes) -> Optional[Tuple[bytes, bytes, str]]:
 def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]:
     frames: List[Frame] = []
 
-    IDLE        = 0
+    IDLE = 0
-    IN_FRAME    = 1
+    IN_FRAME = 1
-    IN_FRAME_DLE = 2   # saw DLE inside frame — waiting for next byte
+    AFTER_DLE = 2
 
     state = IDLE
     body = bytearray()
@@ -206,63 +206,66 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]:
                 state = IN_FRAME
                 i += 2
                 continue
-            # ACK bytes, boot strings, garbage — silently ignored
 
         elif state == IN_FRAME:
             if b == DLE:
-                state = IN_FRAME_DLE
+                state = AFTER_DLE
                 i += 1
                 continue
+            body.append(b)
+
+        else:  # AFTER_DLE
+            if b == DLE:
+                body.append(DLE)
+                state = IN_FRAME
+                i += 1
+                continue
+
             if b == ETX:
-                # Bare ETX = real S3 frame terminator (confirmed from S3FrameParser)
                 end_offset = i + 1
                 trailer_start = i + 1
                 trailer_end = trailer_start + trailer_len
                 trailer = blob[trailer_start:trailer_end]
 
-                # S3 checksums are deliberately not validated here.
+                chk_valid = None
-                # Large S3 responses (A5 bulk waveform, E5 compliance) embed
+                chk_type = None
-                # inner DLE+ETX sub-frame terminators whose trailing 0x03 byte
+                chk_hex = None
-                # lands where the parser would expect the SUM8 checksum, causing
+                payload = bytes(body)
-                # false failures.  The live protocol (protocol.py _validate_frame)
+
-                # also skips S3 checksum enforcement for the same reason.
+                if len(body) >= 1:
+                    received_chk = body[-1]
+                    computed_chk = checksum8_sum(bytes(body[:-1]))
+                    if computed_chk == received_chk:
+                        chk_valid = True
+                        chk_type = "SUM8"
+                        chk_hex = f"{received_chk:02x}"
+                        payload = bytes(body[:-1])
+                    else:
+                        chk_valid = False
+
                 frames.append(Frame(
                     index=idx,
                     start_offset=start_offset,
                     end_offset=end_offset,
                     payload_raw=bytes(body),
-                    payload=bytes(body),
+                    payload=payload,
                     trailer=trailer,
-                    checksum_valid=None,
+                    checksum_valid=chk_valid,
-                    checksum_type=None,
+                    checksum_type=chk_type,
-                    checksum_hex=None
+                    checksum_hex=chk_hex
                 ))
 
                 idx += 1
                 state = IDLE
                 i = trailer_end
                 continue
-            body.append(b)
 
-        else:  # IN_FRAME_DLE
-            if b == DLE:
-                # DLE DLE → literal 0x10 in payload
-                body.append(DLE)
-                state = IN_FRAME
-                i += 1
-                continue
-            if b == ETX:
-                # DLE+ETX inside a frame = inner-frame terminator (A4/E5 sub-frames).
-                # Treat as literal data, NOT the outer frame end.
-                body.append(DLE)
-                body.append(ETX)
-                state = IN_FRAME
-                i += 1
-                continue
             # Unexpected DLE + byte → treat as literal data
             body.append(DLE)
             body.append(b)
             state = IN_FRAME
+            i += 1
+            continue
 
         i += 1
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "seismo-relay"
-version = "0.31.0"
+version = "0.12.0"
 description = "Python client and REST server for MiniMate Plus seismographs"
 requires-python = ">=3.10"
 dependencies = [
@@ -12,13 +12,9 @@ dependencies = [
     "uvicorn[standard]>=0.24",
     "pyserial>=3.5",
     "sqlalchemy>=2.0",
-    "python-multipart>=0.0.7",
-    "h5py>=3.10",
-    "numpy>=1.24",
-    "matplotlib>=3.8",
 ]
 
 [tool.setuptools.packages.find]
-# Auto-discovers minimateplus/, micromate/, sfm/, bridges/ as packages
+# Auto-discovers minimateplus/, sfm/, bridges/ as packages
 where = ["."]
-include = ["minimateplus*", "micromate*", "sfm*", "bridges*"]
+include = ["minimateplus*", "sfm*", "bridges*"]

requirements.txt

@@ -2,7 +2,3 @@ fastapi
 uvicorn
 sqlalchemy
 pyserial
-python-multipart
-h5py
-numpy
-matplotlib

scratch/fake_unit.py

@@ -1,33 +0,0 @@
-"""Pretend to be a Micromate on a serial port: log what arrives, reply to POLL.
-
-Proves the modem's return path (serial -> TCP) independently of the real unit.
-"""
-import os, select, sys, termios, time
-
-path, baud = sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else 115200
-B = {9600: termios.B9600, 38400: termios.B38400, 115200: termios.B115200}[baud]
-fd = os.open(path, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
-a = termios.tcgetattr(fd)
-a[0] = a[1] = a[3] = 0
-a[2] = termios.CS8 | termios.CREAD | termios.CLOCAL
-a[4] = a[5] = B
-a[6] = list(a[6]); a[6][termios.VMIN] = 0; a[6][termios.VTIME] = 0
-termios.tcsetattr(fd, termios.TCSANOW, a)
-termios.tcflush(fd, termios.TCIOFLUSH)
-
-# A real POLL probe reply, captured from UM12947 on 2026-09-24.
-REPLY = bytes.fromhex("0200c5a4000000000000300000000000000099") + b"\x03"
-
-print(f"fake unit on {path} @ {baud}; will answer any inbound frame", flush=True)
-while True:
-    r, _, _ = select.select([fd], [], [], 1.0)
-    if not r:
-        continue
-    data = os.read(fd, 4096)
-    if not data:
-        continue
-    ts = time.strftime("%H:%M:%S")
-    print(f"{ts}  IN  {len(data):3} B  {data.hex(' ')}", flush=True)
-    time.sleep(0.02)
-    os.write(fd, REPLY)
-    print(f"{ts}  OUT {len(REPLY):3} B  {REPLY.hex(' ')}  <- canned POLL reply", flush=True)

scratch/mm_frame_parse.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-"""
-mm_frame_parse.py — parse Micromate (Series IV) frames out of a seismo_lab
-raw capture pair.
-
-Why this exists
----------------
-`minimateplus.framing.S3FrameParser` cannot see Micromate traffic.  It locates
-frames by scanning for `DLE STX`, and a Micromate response has **no leading
-DLE** — it starts at a bare `STX`.  It also expects `payload[1] == 0x10`, where
-the Micromate sends `0xC5` (Blastware firmware) or `0x03` (Thor firmware).
-
-The practical consequence, seen on the 9-24-26 setup-push capture: the
-Blastware-side requests parse fine (Thor emits Series III request frames), but
-**every device response is silently dropped or mis-framed** — so a capture that
-actually contains 12 acked writes looks like 12 unanswered requests.
-
-Destuffing
-----------
-One rule covers both directions: after the leading doubled `BW_CMD`, every
-`10 XX` pair on the wire destuffs to `XX`.  That includes `10 03` — Thor
-escapes literal `0x03` bytes in write data so they are not mistaken for ETX,
-exactly as Blastware does.
-
-That rule was chosen by evidence, not assumption: of the four candidates tried
-against the 9-24-26 capture's four data-carrying write frames, it is the only
-one under which all four checksums validate.  See
-`docs/micromate_protocol_reference.md` → *The write path*.
-
-Usage
------
-    python scratch/mm_frame_parse.py <capture-dir>
-    python scratch/mm_frame_parse.py <raw_bw.bin> <raw_s3.bin>
-    python scratch/mm_frame_parse.py <capture-dir> --dump 0x71
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-from pathlib import Path
-
-DLE, STX, ETX, ACK = 0x10, 0x02, 0x03, 0x41
-
-# Request SUB -> short name.  Series III names where they carry over; the
-# Series IV additions are marked.
-SUBNAME = {
-    0x01: "DEVICE_INFO",
-    0x06: "STORAGE_RANGE",
-    0x08: "EVENT_INDEX",
-    0x0A: "WAVEFORM_HDR",
-    0x0C: "WAVEFORM_REC",
-    0x15: "SERIAL",
-    0x1A: "COMPLIANCE_CFG",
-    0x1C: "MONITOR_STATUS",
-    0x1E: "EVENT_HDR",
-    0x2C: "CALLHOME_CFG",
-    0x2E: "TRIGGER_CFG_READ",     # Series IV
-    0x3E: "OPERATOR",
-    0x41: "SETUP_NAME_READ",      # Series IV
-    0x5A: "BULK_DOWNLOAD",
-    0x5B: "POLL",
-    0x68: "EVENT_INDEX_WRITE",
-    0x69: "WAVEFORM_WRITE",
-    0x71: "COMPLIANCE_WRITE",
-    0x72: "CONFIRM_A",
-    0x73: "CONFIRM_B",
-    0x74: "CONFIRM_C",
-    0x82: "TRIGGER_WRITE",
-    0x83: "TRIGGER_CONFIRM",
-    0xDA: "SETUP_FILE_DECL",      # Series IV — names the target .MMB
-    0xFE: "FULL_CFG",
-}
-
-
-def destuff(blob: bytes, start: int, *, is_request: bool) -> tuple[bytes, int, int]:
-    """Destuff one frame starting at `start`.
-
-    Returns (payload, checksum, index_of_terminating_ETX).  `payload` excludes
-    the trailing checksum byte.  A request frame opens `ACK STX 10 10`; a
-    response opens with a bare `STX`.
-    """
-    i = start + (2 if is_request else 1)
-    out = bytearray()
-    if is_request:
-        # The doubled BW_CMD is the one guaranteed stuffed byte.
-        if blob[i : i + 2] != bytes([DLE, DLE]):
-            raise ValueError(f"@0x{start:04x}: request does not open with 10 10")
-        out.append(DLE)
-        i += 2
-    while i < len(blob):
-        b = blob[i]
-        if b == DLE and i + 1 < len(blob):
-            out.append(blob[i + 1])
-            i += 2
-            continue
-        if b == ETX:
-            break
-        out.append(b)
-        i += 1
-    if len(out) < 2:
-        raise ValueError(f"@0x{start:04x}: frame too short")
-    return bytes(out[:-1]), out[-1], i
-
-
-def frames(blob: bytes, *, is_request: bool):
-    """Yield (offset, payload, chk, checksum_kind)."""
-    i, n = 0, len(blob)
-    while i < n:
-        if is_request:
-            if not (blob[i] == ACK and i + 1 < n and blob[i + 1] == STX):
-                i += 1
-                continue
-        elif blob[i] != STX:
-            i += 1
-            continue
-        try:
-            payload, chk, end = destuff(blob, i, is_request=is_request)
-        except ValueError:
-            i += 1
-            continue
-        sum8 = sum(payload) & 0xFF
-        dle_aware = (sum(b for b in payload if b != DLE) & 0xFF)
-        if sum8 == chk:
-            kind = "SUM8"
-        elif dle_aware == chk:
-            kind = "DLE-aware"
-        else:
-            kind = "BAD"
-        yield i, payload, chk, kind
-        i = end + 1
-
-
-def describe(payload: bytes, is_request: bool) -> str:
-    if len(payload) < 3:
-        return "??"
-    sub = payload[2]
-    if is_request:
-        return SUBNAME.get(sub, f"SUB_{sub:02X}")
-    req = 0xFF - sub
-    return "rsp<-" + SUBNAME.get(req, f"SUB_{req:02X}")
-
-
-def report(path: Path, *, is_request: bool, dump_sub: int | None) -> None:
-    blob = path.read_bytes()
-    side = "Thor" if is_request else "unit"
-    print(f"== {side:4}  {path.name}  ({len(blob)} bytes)")
-    n_bad = 0
-    for idx, (off, p, chk, kind) in enumerate(frames(blob, is_request=is_request)):
-        if kind == "BAD":
-            n_bad += 1
-        sub = p[2] if len(p) > 2 else -1
-        flags = p[1] if len(p) > 1 else -1
-        # Requests carry offset at payload[4:6]; responses page at [3:5].
-        word = int.from_bytes(p[4:6] if is_request else p[3:5], "big")
-        data = len(p) - 16 if is_request else max(len(p) - 5, 0)
-        print(
-            f"  [{idx:2}] @0x{off:04x} payload={len(p):5} data={data:5} "
-            f"flags=0x{flags:02x} SUB=0x{sub:02x} {describe(p, is_request):18} "
-            f"{'offset' if is_request else 'page'}=0x{word:04x} chk={kind}"
-        )
-        if dump_sub is not None and sub == dump_sub:
-            body = p[16:] if is_request else p[5:]
-            print(f"       ---- data ({len(body)} bytes) ----")
-            for o in range(0, len(body), 16):
-                chunk = body[o : o + 16]
-                txt = "".join(chr(c) if 32 <= c < 127 else "." for c in chunk)
-                print(f"       {o:06x}  {chunk.hex(' '):<47}  |{txt}|")
-    print(f"  -- {idx + 1} frames, {n_bad} bad checksum\n")
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description=__doc__,
-                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("paths", nargs="+",
-                    help="a capture directory, or raw_bw.bin and raw_s3.bin")
-    ap.add_argument("--dump", default=None,
-                    help="hex-dump the data section of this SUB (e.g. 0x71)")
-    args = ap.parse_args()
-
-    dump_sub = int(args.dump, 0) if args.dump else None
-
-    if len(args.paths) == 1 and Path(args.paths[0]).is_dir():
-        d = Path(args.paths[0])
-        bw = sorted(d.glob("raw_bw_*.bin"))
-        s3 = sorted(d.glob("raw_s3_*.bin"))
-        if not bw or not s3:
-            print(f"{d}: need one raw_bw_*.bin and one raw_s3_*.bin", file=sys.stderr)
-            return 2
-        pairs = [(bw[0], True), (s3[0], False)]
-    elif len(args.paths) == 2:
-        pairs = [(Path(args.paths[0]), True), (Path(args.paths[1]), False)]
-    else:
-        ap.error("pass a capture directory, or exactly two .bin files")
-
-    for path, is_request in pairs:
-        report(path, is_request=is_request, dump_sub=dump_sub)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scratch/next_experiment_skeleton.py

@@ -1,360 +0,0 @@
-"""
-scratch/next_experiment_skeleton.py — segment-channel scoring analyzer.
-
-This is the suggested NEXT EXPERIMENT for cracking the waveform body codec.
-The goal is to figure out what segments 1+ contain, since segment 0 = Tran
-is solved but multi-segment continuation diverges from truth at sample ~512.
-
-────────────────────────────────────────────────────────────────────────────
-The hypothesis to test
-────────────────────────────────────────────────────────────────────────────
-
-Segments rotate through channels:
-
-    segment 0  →  Tran samples 0..509
-    segment 1  →  Vert samples 0..507
-    segment 2  →  Long samples 0..507
-    segment 3  →  Mic  samples 0..507
-    segment 4  →  Tran samples 510..N (continuation)
-    ...
-
-This would explain why segment 0 works perfectly (it's pure Tran) and why
-applying segment 1's blocks as Tran continuation gives wrong values
-(it's actually Vert).
-
-────────────────────────────────────────────────────────────────────────────
-What the analyzer should do
-────────────────────────────────────────────────────────────────────────────
-
-For each segment in each fixture event:
-
-1. Run the segment-0 block-walker + RLE decode (the same algorithm that
-   ``decode_tran_initial`` uses) over the segment's blocks.  Start from
-   some anchor value and produce a cumulative trajectory of length =
-   number-of-deltas-in-segment.
-
-2. For each candidate channel C ∈ {Tran, Vert, Long, MicL}:
-   For each candidate anchor location in the segment-header payload
-   (try [0:2], [2:4], [4:6], [14:16], [16:18] as int16 BE):
-       Compare the decoded trajectory against truth[C] starting from
-       the segment's first sample index.
-       Score = number of matches (or sum of squared errors).
-
-3. Report the best (channel, anchor-location) combination per segment.
-
-If the rotation hypothesis is correct, you'll see:
-    segment 0  →  best score for (Tran, preamble bytes [3:5])    ✓ already known
-    segment 1  →  best score for (Vert, <some-header-byte>)
-    segment 2  →  best score for (Long, <some-header-byte>)
-    segment 3  →  best score for (MicL, <some-header-byte>)
-    segment 4  →  best score for (Tran, continuing from segment 0's end)
-
-If the rotation hypothesis is NOT correct, the scorer will at least narrow
-down what segment 1 actually carries.  Maybe channels interleave at finer
-granularity, or maybe segments alternate by something other than channel.
-
-────────────────────────────────────────────────────────────────────────────
-Why this is a scoring analyzer, not a hand-written decoder
-────────────────────────────────────────────────────────────────────────────
-
-Direct hand-coding ("assume segment 1 is Vert with anchor at byte X") gets
-stuck when the assumption is wrong because the failure mode is silent —
-you get plausible-looking-but-wrong samples and have to manually diff
-against truth to debug.
-
-The scorer is brute-force but cheap: every fixture event × every segment ×
-4 channels × 5 anchor-byte candidates is only ~hundreds of comparisons.
-The winning combination jumps out by score.
-
-────────────────────────────────────────────────────────────────────────────
-Skeleton
-────────────────────────────────────────────────────────────────────────────
-"""
-from __future__ import annotations
-
-import os
-import re
-import sys
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-
-from minimateplus.waveform_codec import walk_body, find_data_start, WaveformBlock
-
-
-# ── Reusable pieces ──────────────────────────────────────────────────────────
-
-
-CHANNELS = ("Tran", "Vert", "Long", "MicL")
-LSB_INV = 200  # 1 in/s / 0.005 in/s/LSB; multiply BW-export floats by this
-               # to get 16-count units (the body's native quantization).
-
-
-@dataclass
-class FixtureEvent:
-    name: str           # e.g. "M529LL1A.SP0"
-    bin_path: str
-    txt_path: str
-    body: bytes
-    truth: dict         # {channel: list of int16-quantized samples}
-    blocks: List[WaveformBlock]
-    segment_starts: List[int]  # block indices of each 40 02 segment header
-    segment_sample_starts: List[int]  # for each segment, the truth sample index it starts at
-
-
-def s4(n: int) -> int:
-    """4-bit signed nibble decode."""
-    return n if n < 8 else n - 16
-
-
-def i8(b: int) -> int:
-    """int8 reinterpret of unsigned byte."""
-    return b if b < 128 else b - 256
-
-
-def load_fixture(name: str) -> FixtureEvent:
-    """Load a fixture event with its truth values and parsed block stream."""
-    # Find the fixture (search both subdirs of tests/fixtures/).
-    base = os.path.join(os.path.dirname(__file__), "..", "tests", "fixtures")
-    candidates = [
-        os.path.join(base, "5-11-26", name),
-        os.path.join(base, "decode-re-5-8-26", "event-a", name),  # not used directly
-    ]
-    bin_path = next((c for c in candidates if os.path.exists(c)), None)
-    if bin_path is None:
-        # Try a glob walk for the 5-8 fixtures (they're in subdirs).
-        for root, _, files in os.walk(base):
-            if name in files:
-                bin_path = os.path.join(root, name)
-                break
-    if bin_path is None:
-        raise FileNotFoundError(name)
-
-    txt_path = bin_path + ".TXT"
-    with open(bin_path, "rb") as f:
-        raw = f.read()
-    body = raw[43:-26]
-    truth = _parse_txt(txt_path)
-    blocks = walk_body(body, find_data_start(body))
-
-    seg_idx = [i for i, b in enumerate(blocks) if b.tag_hi == 0x40]
-    # Segment 0 starts at sample 0; subsequent segments start at the
-    # cumulative sample count from previous segment(s).  Tran's segment 0
-    # is N samples; if rotation hypothesis is correct, segment 1's data
-    # starts at sample 0 for a *different* channel.  The analyzer should
-    # try both "continues from previous segment" and "starts at sample 0
-    # of a different channel."
-    seg_sample_starts = _compute_segment_sample_starts(blocks, seg_idx)
-
-    return FixtureEvent(
-        name=name, bin_path=bin_path, txt_path=txt_path,
-        body=body, truth=truth, blocks=blocks,
-        segment_starts=seg_idx, segment_sample_starts=seg_sample_starts,
-    )
-
-
-def _parse_txt(path: str) -> dict:
-    """Parse BW ASCII TXT export into {channel: [int_samples_in_16_count_units]}."""
-    with open(path, "r", encoding="utf-8", errors="replace") as f:
-        lines = f.read().splitlines()
-    header_idx = next(
-        (i for i, l in enumerate(lines)
-         if all(c in l for c in CHANNELS)),
-        None,
-    )
-    if header_idx is None:
-        return {ch: [] for ch in CHANNELS}
-    out = {ch: [] for ch in CHANNELS}
-    for line in lines[header_idx + 1:]:
-        parts = re.split(r"\s+", line.strip())
-        if len(parts) < 4:
-            continue
-        try:
-            vals = [float(p) for p in parts[:4]]
-        except ValueError:
-            continue
-        for ch, v in zip(CHANNELS, vals):
-            # Multiply by LSB_INV; geo channels are in in/s, MicL is in dB(L)
-            # (which doesn't quantize the same way — leaving raw for MicL is fine,
-            # the scorer should treat MicL specially).
-            out[ch].append(round(v * LSB_INV) if ch != "MicL" else v)
-    return out
-
-
-def _compute_segment_sample_starts(
-    blocks: List[WaveformBlock], seg_idx: List[int]
-) -> List[int]:
-    """Cumulative sample-count up to each segment header (if all blocks treated
-    as Tran continuation).  Useful as one candidate for segment-1-Tran tests.
-
-    The scorer should ALSO try "segment 1 starts at sample 0 of a new channel"
-    as the rotation hypothesis predicts.
-    """
-    starts = []
-    cum = 2  # T[0] + T[1] from preamble
-    for i, b in enumerate(blocks):
-        if i in seg_idx:
-            starts.append(cum)
-        if b.tag_hi == 0x10:
-            cum += b.tag_lo
-        elif b.tag_hi == 0x20:
-            cum += b.tag_lo
-        elif b.tag_hi == 0x00:
-            cum += b.tag_lo
-        # 30 NN and 40 02 don't contribute samples (for this hypothesis)
-    return starts
-
-
-# ── The core algorithm: decode a segment's blocks as deltas ─────────────────
-
-
-def decode_segment_as_channel(
-    blocks: List[WaveformBlock],
-    seg_start_block_idx: int,
-    seg_end_block_idx: int,
-    anchor: int,
-) -> List[int]:
-    """Apply the segment-0 codec rules to a range of blocks, starting from *anchor*.
-
-    Returns a list of cumulative sample values (one per delta).  Does NOT include
-    the anchor itself in the output — the first returned value is anchor + first_delta.
-    """
-    out = []
-    cur = anchor
-    for bi in range(seg_start_block_idx, seg_end_block_idx):
-        blk = blocks[bi]
-        if blk.tag_hi == 0x10:
-            for byte in blk.data:
-                for nib in ((byte >> 4) & 0xF, byte & 0xF):
-                    cur += s4(nib)
-                    out.append(cur)
-        elif blk.tag_hi == 0x20:
-            for byte in blk.data:
-                cur += i8(byte)
-                out.append(cur)
-        elif blk.tag_hi == 0x00:
-            for _ in range(blk.tag_lo):
-                out.append(cur)
-        # 30 NN: skip (content unknown)
-        # 40 02: shouldn't appear in segment data (it's the segment header)
-    return out
-
-
-def score_against_truth(
-    decoded: List[int],
-    truth: List[int],
-    truth_start: int,
-) -> Tuple[int, int]:
-    """Compare *decoded* to truth[truth_start : truth_start + len(decoded)].
-
-    Returns (n_matches, n_compared).
-    """
-    n = min(len(decoded), len(truth) - truth_start)
-    if n <= 0:
-        return (0, 0)
-    matches = sum(1 for i in range(n) if decoded[i] == truth[truth_start + i])
-    return (matches, n)
-
-
-# ── TODO for the next pass ──────────────────────────────────────────────────
-
-
-def score_segment_against_all_channels(
-    event: FixtureEvent,
-    segment_index: int,
-) -> List[Tuple[str, int, int, int]]:
-    """For segment *segment_index* of *event*, find the best (channel, start_sample)
-    fit.
-
-    For each candidate channel C and each candidate starting truth-sample index s,
-    we pick the anchor that makes the FIRST decoded value match truth[C][s], then
-    score the remaining decoded values against truth[C][s+1 : s+N].
-
-    Returns rows of (channel_name, start_sample, n_matches, n_compared)
-    sorted by match-count descending.
-    """
-    # Block range of this segment: from the segment header (inclusive) up to
-    # the next segment header (exclusive), or end-of-blocks.
-    seg_header_idx = event.segment_starts[segment_index]
-    next_header_idx = (
-        event.segment_starts[segment_index + 1]
-        if segment_index + 1 < len(event.segment_starts)
-        else len(event.blocks)
-    )
-
-    # Decode the segment's data blocks (skip the segment-header block itself).
-    # Use anchor=0 — we'll re-anchor when scoring against each channel.
-    deltas_trajectory = decode_segment_as_channel(
-        event.blocks, seg_header_idx + 1, next_header_idx, anchor=0
-    )
-    if not deltas_trajectory:
-        return []
-
-    n = len(deltas_trajectory)
-    results = []
-
-    for ch in ("Tran", "Vert", "Long"):
-        truth = event.truth.get(ch)
-        if not truth or len(truth) < n + 1:
-            continue
-        # For each candidate starting sample s in truth, check if applying
-        # the deltas starting from truth[s] reproduces truth[s+1:s+n+1].
-        best = (0, -1)
-        for s in range(len(truth) - n):
-            anchor = truth[s]
-            offset = anchor - deltas_trajectory[0] + truth[s + 1] - anchor
-            # Recompute: trajectory[i] = anchor + cumulative_delta_through_i
-            # but we already have deltas_trajectory computed from anchor=0,
-            # so trajectory_relative[i] = anchor + deltas_trajectory[i].
-            matches = 0
-            for i in range(n):
-                if truth[s + i + 1] == anchor + deltas_trajectory[i]:
-                    matches += 1
-                # Note: we could break early on first mismatch for "matches start",
-                # but counting total matches gives a more robust score.
-            if matches > best[0]:
-                best = (matches, s)
-        results.append((ch, best[1], best[0], n))
-
-    results.sort(key=lambda r: -r[2])
-    return results
-
-
-# ── Driver ──────────────────────────────────────────────────────────────────
-
-
-def main():
-    """Run the analyzer on all loud-bundle events and print best scores."""
-    events = ["M529LL1A.SP0", "M529LL1A.SS0", "M529LL1A.SV0",
-              "M529LL1L.JQ0", "M529LL1L.V70"]
-    for name in events:
-        try:
-            event = load_fixture(name)
-        except FileNotFoundError:
-            print(f"{name}: fixture not found")
-            continue
-
-        print(f"\n=== {name} ===")
-        print(f"  body bytes: {len(event.body)}")
-        print(f"  blocks: {len(event.blocks)}")
-        print(f"  segments: {len(event.segment_starts)}")
-        print(f"  segment sample-starts (if all blocks are 1 channel):")
-        for si, sample_start in enumerate(event.segment_sample_starts):
-            print(f"    seg {si}: sample {sample_start}")
-
-        for si in range(len(event.segment_starts)):
-            results = score_segment_against_all_channels(event, si)
-            if not results:
-                print(f"  seg {si}: (no scorable data)")
-                continue
-            tag = "✓" if results[0][2] / max(results[0][3], 1) > 0.9 else " "
-            top = results[0]
-            print(f"  seg {si}: best fit {tag} = {top[0]:<5} "
-                  f"starting at sample {top[1]:>5}, {top[2]:>4}/{top[3]:<4} match"
-                  + (f"  (next: {results[1][0]} @{results[1][1]} {results[1][2]}/{results[1][3]})"
-                     if len(results) > 1 else ""))
-
-
-if __name__ == "__main__":
-    main()