From 686ab6e7a61d775b8545c3e6bcb4d8ce7f3f3a50 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 08:11:11 +0000 Subject: [PATCH] fix(codec): geo full scale is 32000 counts; 4 walker framing cases; channel-id from header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent bugs, both found by diffing 75 production events against their preserved Blastware ASCII exports (//_ASCII.TXT). 1. Geo full scale was wrong — every geophone reading was 2.34% low. The codec emits geo samples in 16-count units with a documented LSB of exactly 0.005 in/s, and decoded_to_adc_counts multiplies by 16, so one ADC count is 0.005/16 in/s and 10.000 in/s is 10.0/(0.005/16) = 32000 counts. sfm/event_hdf5.py and minimateplus/event_file_io.py both divided by 32768 (2^15), scaling every sample and derived peak down by 1 - 32000/32768. The error scales with amplitude, so it was invisible on quiet events and worst on the loud ones that matter for compliance. Mic is unaffected (it back-solves its scale from the device peak). 216 per-channel comparisons: 32768 -> 151/216 exact, worst error 0.238 in/s on a 10 in/s event; 32000 -> 216/216 exact, worst 0.005 = 1 LSB. 2. walk_body silently truncated channels on four unhandled framing cases. An unrecognised tag ends the walk and decode_waveform_v2 returns whatever it got, so this surfaced as short channels, never an error: - wide-NN RLE `0X NN` (runs longer than 252 samples) - `30 NN` with NN > 0x10 (the old cap was arbitrary) - variable-width `40 NN` headers: NN counts previous-channel continuation deltas, so the header is 2*NN + 16 bytes; `40 01` and `40 03` occur alongside `40 02` - tagless segment headers: no `40 NN` tag at all, just the 14-byte tail [field2:2][len:2][channel_id:4][marker:2][anchors:4] Also: the header field documented as a "monotonic uint32 LE counter" is really [channel_id][00][00][segment_index], with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. decode_waveform_v2 now takes the channel from that field instead of rotation position, which was fragile: one missed header desynced every channel after it. parse_segment_header now returns n_prev_deltas/prev_deltas/marker/ anchors/channel/segment_index; the old fixed_pattern (02 00 00 01) conflated the 2-byte marker with the first anchor. Ground-truth corpus, end to end through the production path: exact 37 -> 72, truncated 23 -> 3, full-length value errors 15 -> 0. Store-wide, 729 of 1388 series-3 waveform events decode differently and 728 gain samples; the scale fix changes float values on all of them, so stored .h5 files need regenerating. Still open: 3 events truncate at a header variant with a variable-width prefix (2/4/6 bytes) before the channel id and an `01 00` marker. Documented in docs/instantel_protocol_reference.md with byte offsets. +20 tests. No regressions: the byte-exact fixture suite still passes and the full-suite failure list is unchanged from baseline (16 pre-existing failures from gitignored fixtures). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 93 +++++++++++++++ CLAUDE.md | 8 ++ docs/instantel_protocol_reference.md | 157 +++++++++++++++++++++--- docs/waveform_codec_re_status.md | 60 +++++++++- minimateplus/event_file_io.py | 9 +- minimateplus/waveform_codec.py | 171 +++++++++++++++++++++------ sfm/event_hdf5.py | 23 +++- tests/test_event_file_io.py | 9 ++ tests/test_event_hdf5.py | 46 ++++++- tests/test_waveform_codec.py | 137 ++++++++++++++++++++- 10 files changed, 645 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 402dd9f..2f7d1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,99 @@ All notable changes to seismo-relay are documented here. ## [Unreleased] +### Fixed + +- **Geophone full scale is 32000 ADC counts, not 32768 — every geo reading was + 2.3% low.** The verified body codec emits geo samples in 16-count units whose + documented LSB is exactly 0.005 in/s, and `decoded_to_adc_counts` multiplies + by 16, so one ADC count is `0.005/16` in/s and Normal range (10.000 in/s) is + `10.0 / (0.005/16)` = **32000** counts. Both `sfm/event_hdf5.py` and + `minimateplus/event_file_io.py` divided by 32768, scaling every geophone + sample and every derived peak down by `1 - 32000/32768` = **2.34%**. + + Measured against 216 per-channel comparisons with preserved Blastware ASCII + exports: **32768 → 151/216 exact** (worst error 0.238 in/s on a 10 in/s + event); **32000 → 216/216 exact**, worst error 0.005 in/s (exactly 1 LSB — + pure quantization). The error scales with amplitude, so it was invisible on + quiet events and worst on the loud ones that matter for compliance. + + The mic path is unaffected — it back-solves its own per-count factor from the + device-reported peak. + +- **Series-3 waveform codec: four block-framing cases caused silent channel + truncation.** `walk_body` hit its unknown-tag `break` mid-stream and every + channel decoded after that point came out short — typically Vert/Long/MicL, + sometimes at a third of their true length, with no error raised. + + - **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already handled for + `1X NN` / `2X NN` also applies to the `00 NN` RLE tag. Runs longer than + 252 samples must use the wide form (e.g. `01 0c` = 268 repeats). + - **`30 NN` with NN > 0x10** — the `0 < NN <= 0x10` guard was arbitrary; + data-section `30 NN` blocks reach at least NN = 0x18. The length formula + (`NN × 1.5 + 2`) was already correct. + - **Variable-width `40 NN` segment headers** — NN is the *count of + previous-channel continuation deltas*, so the header is `2 × NN + 16` + bytes and every field after the deltas shifts by `2 × NN`. Only `40 02` + (20 bytes) was handled; `40 01` (18) and `40 03` (22) both occur. + - **Tagless segment headers** — a segment header can appear with no + `40 NN` tag at all: just the 14-byte tail + `[field2:2][len:2][channel_id:4][marker:2][anchors:4]`. This is the NN=0 + case (no continuation deltas needed, so no tag and no delta bytes). It is + where the walk stopped in 7 of the 8 events still truncating after the + first three fixes. + +### Changed + +- **Segment channel now comes from the header's own channel-id byte** rather + than from rotation position. The field previously documented as a + "monotonic uint32 LE counter" is really `[channel][00][00][segment_index]` + with `0x46`=Tran `0x47`=Vert `0x48`=Long `0x49`=MicL — verified on + **1697 of 1697** segment headers across the ground-truth corpus with zero + disagreements. Rotation-by-position is kept only as a fallback for unknown + ids; it was fragile because a single missed or extra header (exactly what + tagless headers caused) desynced every channel after it. + +- **`parse_segment_header` return shape** — now `n_prev_deltas`, + `prev_deltas`, `marker`, `anchors`, `channel`, `segment_index` in place of + the fixed-offset `anchor_bytes` / `fixed_pattern` / `tail` keys. The old + `fixed_pattern` (`02 00 00 01`) conflated the 2-byte constant marker with + the first anchor. `counter` is retained as the raw uint32 of the id field. + +### Verification + +Against the 75 ground-truth events (BW binary paired with its preserved +`_ASCII.TXT` export), decoding end-to-end through the production path: + +| | before | after | +|---|---|---| +| exact (full length, within 1 LSB) | 37 | **72** | +| truncated | 23 | **3** | +| full length, value error > 2 LSB | 15 | **0** | + +Worst remaining error among the 72: 0.0050 in/s = exactly 1 LSB. + +No regressions — the byte-exact fixture suite still passes, and the full-suite +failure list is unchanged from baseline (16 pre-existing failures from +gitignored fixtures). + +### Notes + +- **The "DC offset" symptom is _not_ a decode bug.** Events whose geo trace + sits at a constant level instead of oscillating around zero + (dominant-axis `|mean| / peak` >> 0) reproduce *exactly* in Blastware's own + ASCII export — e.g. `BE12599/N599LQD7.8E0W` Tran reads mean +0.345, + min +0.335, max +0.355 in both. It is a known recurring hardware fault (the + operators call it an "offset"): the affected channel's baseline exceeds the + unit's own geo trigger level, so the unit retriggers continuously and floods + the ACH queue with garbage events. Store-wide it affects 2 units of 21 across + 6 episodes; see `scratch/offset_candidates.csv` and the project memory notes. + +- **Still open:** 3 of 75 ground-truth events truncate at a segment-header + variant with a variable-width prefix before the channel-id field (2, 4 or 6 + bytes observed) and an `01 00` marker instead of `02 00`. See the protocol + reference, "Unmapped: variable-prefix segment descriptors". Examples: + `BE12599/N599LPNB.JF0W` at body offset 1155, `BE9558/K558LOF2.820W` at 1485. + --- ## v0.25.0 — 2026-08-25 diff --git a/CLAUDE.md b/CLAUDE.md index 9198786..1e5b2cb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -264,6 +264,14 @@ then `decoded_to_adc_counts()` to scale to int16 ADC counts (geos × 16; mic pass-through). The `.h5` sidecars SFM produces now contain correct samples for any event without walker edge cases. +**Geo full scale is 32000 ADC counts, NOT 32768** (fixed 2026-08-25). +One decoder unit = 16 ADC counts = exactly 0.005 in/s, so +`10.000 in/s / (0.005/16)` = 32000. Consumers must use +`sfm.event_hdf5._GEO_INT16_FS` / `event_file_io._GEO_INT16_FS` (both +32000). Dividing by 32768 reads every geophone sample 2.34% low — +that was a live bug in both modules until 2026-08-25. Mic is +unaffected (it back-solves its scale from the device-reported peak). + The original int16 LE decoder is preserved as `_decode_a5_waveform_LEGACY` for reference but is not called. diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index e10f9da..6c25d27 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -1101,14 +1101,51 @@ Every block starts with a 2-byte tag. Five tag types are confirmed: |-----------|-------------------------------------|-----------------------| | ``10 NN`` | Small-delta data block | NN/2 + 2 bytes | | ``20 NN`` | Literal data block (int8-shaped) | NN + 2 bytes | -| ``00 NN`` | 2-byte marker between data blocks | 2 bytes | +| ``00 NN`` | RLE zero-delta run | 2 bytes | | ``30 NN`` | Trailer summary block | NN × 4 bytes | -| ``40 02`` | Segment header | 20 bytes (fixed) | +| ``40 NN`` | Segment header | 2 × NN + 16 bytes | NN is always a multiple of 4. ``10 NN`` and ``20 NN`` data blocks alternate with ``00 NN`` markers — every ``10/20 NN`` block is followed by a ``00 NN`` marker before the next data block. +###### Wide-NN forms — ``0X NN`` (CONFIRMED 2026-08-25) + +The 12-bit wide-NN encoding already documented for ``1X NN`` / +``2X NN`` (low nibble of the tag byte carries the high nibble of NN, +so effective ``NN = ((tag & 0x0F) << 8) | NN``) **also applies to the +``00 NN`` RLE tag.** A narrow RLE run maxes out at NN = 0xFC, so a +quiet stretch longer than 252 samples must use the wide form. + +Confirmed against six production events, e.g. ``01 0c`` (NN = 268) in +``BE9558/K558LKOF.460W``. Before this was handled, the walker hit its +unknown-tag break at the first long quiet run and silently truncated +every channel decoded after that point. + +###### ``30 NN`` is not capped at NN = 0x10 (CONFIRMED 2026-08-25) + +Data-section ``30 NN`` blocks occur with NN up to at least 0x18 (24), +e.g. ``30 18`` in ``BE18193/T193LQ45.NN0W`` and ``30 14`` in +``BE18193/T193LQ9W.AF0W``. The data-section length formula +(``NN × 1.5 + 2``) holds for these; only the earlier ``NN ≤ 0x10`` +guard was wrong. + +###### ``40 NN`` segment headers are variable width (CONFIRMED 2026-08-25) + +``40 02`` is the common case, but **NN is the count of int16 BE +continuation deltas the header carries for the *previous* channel**, so +the header grows with NN and every field after the deltas shifts by +``2 × NN``: + +``` +length = 2 (tag) + 2 × NN (prev-channel deltas) + 14 (fixed tail) +``` + +``40 01`` (18 bytes) and ``40 03`` (22 bytes) both occur in production +files — see ``BE12599/N599LP1S.UO0W`` and ``BE18438/T438LO30.GA0W``. +In each case the constant ``02 00`` marker sits at ``data[2×NN+8]`` and +the following tag lands exactly on a valid block boundary. + ##### Segments The body is divided into segments separated by ``40 02`` segment headers. @@ -1129,20 +1166,110 @@ fit fewer. Observed first-segment sizes in the bundled fixtures: based on incomplete walks; that figure is wrong. Segments are flash-page-sized in bytes, not sample-count-sized. -The 18-byte ``40 02`` payload structure: +The ``40 NN`` payload structure (offsets shown for the common NN=2 / +18-byte-payload case; add ``2 × (NN − 2)`` to every offset from ``[4:6]`` +onward for other widths): -| Offset | Field | Status | -|-----------|---------------------------------------------|-------------| -| [0:2] | T_delta at first sample of new segment | ✅ confirmed| -| | (int16 BE, in 16-count units) | | -| [2:4] | Likely T_delta at sample seg_start+1 | 🟡 likely | -| [4:6] | Unknown (varies; possibly a checksum) | ❓ open | -| [6:8] | Byte length to next segment header − 2 | ✅ confirmed| -| | (uint16 BE; useful for walker pre-scan) | | -| [8:12] | Monotonic uint32 LE counter | ✅ confirmed| -| | (starts ~0x47, increments by 1 per segment) | | -| [12:14] | Constant ``02 00`` | ✅ confirmed| -| [14:18] | Unknown 4-byte field | ❓ open | +| Offset (NN=2) | Generic | Field | Status | +|---------------|------------------|----------------------------------------|-------------| +| [0:4] | [0 : 2NN] | NN × int16 BE continuation deltas for | ✅ confirmed| +| | | the PREVIOUS channel (16-count units) | | +| [4:6] | [2NN : 2NN+2] | Unknown (varies; possibly a checksum) | ❓ open | +| [6:8] | [2NN+2 : 2NN+4] | Byte length to next segment header − 2 | 🟡 likely | +| | | (uint16 BE; off by ±4 on some files) | | +| [8:12] | [2NN+4 : 2NN+8] | Monotonic uint32 LE counter | ✅ confirmed| +| | | (starts ~0x47, +1 per segment) | | +| [12:14] | [2NN+8 : 2NN+10] | Constant ``02 00`` | ✅ confirmed| +| [14:18] | [2NN+10 : 2NN+14]| THIS channel's 2-sample anchor pair | ✅ confirmed| +| | | (2 × int16 BE) | | + +⚠️ An earlier draft listed ``[14:18]`` as an "unknown 4-byte field" and +``[12:16]`` as a constant ``02 00 00 01``. Both were wrong: the constant +is only the 2-byte ``02 00``, and the four bytes after it are the anchor +pair the decoder needs. Corrected 2026-08-25. + +###### Tagless segment headers (CONFIRMED 2026-08-25) + +A segment header can appear with **no ``40 NN`` tag at all** — just the +14-byte tail: + +``` +[field2:2][len_to_next:2][channel_id:4][marker:2][anchors:4] +``` + +This is the NN=0 case: the previous channel needed no continuation +deltas, so there is no tag and no delta bytes. Detect it by the six +bytes at ``[4:10]`` — a known channel id, two zero bytes, a small +segment index, then the ``01 00`` / ``02 00`` marker. + +It is where the walk stopped in 7 of the 8 events that still truncated +after the wide-RLE / ``30 NN`` / variable-width-``40 NN`` fixes. + +###### The header "counter" is really a channel id (CONFIRMED 2026-08-25) + +The 4-byte field previously documented as a *"monotonic uint32 LE +counter (starts ~0x47, increments by 1 per segment)"* is actually: + +``` +[channel_id:1][00][00][segment_index:1] +``` + +| channel_id | channel | +|---|---| +| ``0x46`` | Tran | +| ``0x47`` | Vert | +| ``0x48`` | Long | +| ``0x49`` | MicL | + +Verified on **1697 of 1697** segment headers across the ground-truth +corpus — every one agrees with the channel the rotation would assign, +zero disagreements, no other id values observed. The old reading was +plausible because the id byte cycles 0x46→0x47→0x48→0x49 and the +segment index increments, which *looks* monotonic in LE. + +Decoders should take the channel from this field rather than from +rotation position: one missed or extra header (exactly what tagless +headers used to cause) desyncs rotation and corrupts every channel +after it. + +###### Geophone full scale is 32000 counts, not 32768 (CONFIRMED 2026-08-25) + +The body codec emits geo samples in 16-count units whose LSB is exactly +**0.005 in/s**. With the consumer-side ``×16`` to ADC counts, one ADC +count is ``0.005 / 16`` in/s, so Normal range (10.000 in/s) is + +``` +10.0 / (0.005 / 16) = 32000 counts +``` + +Dividing by 32768 scales every geophone sample and every derived peak +down by ``1 - 32000/32768`` = **2.34%**. Measured on 216 per-channel +comparisons against preserved Blastware ASCII exports: 32768 gave +151/216 exact (worst error 0.238 in/s on a 10 in/s event); 32000 gives +216/216 exact with a worst error of 0.005 in/s — exactly 1 LSB, i.e. +pure quantization. + +This also explains why Blastware reports geo peaks slightly above +nominal full scale (e.g. 10.14 in/s): the ADC has headroom past 32000. + +###### Unmapped: variable-prefix segment descriptors ❓ OPEN + +Three of 75 ground-truth production events still truncate. In each, +the walk reaches a segment header whose channel-id field is preceded by +a **variable-width prefix** — 2, 4 or 6 bytes have all been observed, +where the standard tagless form always has 4 (``field2`` + ``len``). +These records also carry the ``01 00`` marker rather than ``02 00``, +and appear packed back-to-back with little or no sample data between +them. + +The ``01 00`` marker is *not* simply an anchor count: records carrying +it have been seen with both 2-byte and 4-byte anchor fields in the same +file, so the prefix width and the marker are not yet reconciled. + +The decoder stops cleanly at these rather than emitting garbage. +Examples: ``BE12599/N599LPNB.JF0W`` at body offset 1155 (2-byte +prefix), ``BE12599/N599LPWJ.980W`` at 849 (6-byte prefix), +``BE9558/K558LOF2.820W`` at 1485. Examples from event-c (1 sec single-shot): diff --git a/docs/waveform_codec_re_status.md b/docs/waveform_codec_re_status.md index dd1a6c6..47fbc41 100644 --- a/docs/waveform_codec_re_status.md +++ b/docs/waveform_codec_re_status.md @@ -102,12 +102,24 @@ correct. | | | nibble first; signed 0..7 / 8..F = -8..-1)| | `20 NN` | NN + 2 bytes | int8 signed deltas (1 per byte) | | `00 NN` | 2 bytes | RLE: append NN copies of current value | -| `30 NN` | NN*2 in data section, | Unknown content. Only in loud-from- | -| | NN*4 in trailer | start events. | -| `40 02` | 20 bytes (fixed) | Segment header | +| `30 NN` | NN*1.5 + 2 in data | 12-bit signed deltas (see below). | +| | section, NN*4 trailer | | +| `40 NN` | 2*NN + 16 bytes | Segment header (NN = prev-channel deltas)| NN is always a multiple of 4. +**Wide-NN forms.** `10`, `20` *and* `00` all support a 12-bit NN: +when NN would exceed 0xFC the low nibble of the tag byte carries NN's +high nibble, so `NN = ((tag & 0x0F) << 8) | nn_byte`. Confirmed for +`1X`/`2X` in 2026-05-11 and for `0X` (RLE) in 2026-08-25 — e.g. +`01 0c` = a 268-sample zero-delta run. + +**`40 NN` is variable width.** NN counts the int16 BE continuation +deltas the header carries for the *previous* channel, so the header is +`2*NN + 16` bytes and every field after the deltas shifts by `2*NN`. +`40 01` (18 B) and `40 03` (22 B) both occur alongside the common +`40 02` (20 B). Confirmed 2026-08-25. + Implementation: `walk_body()` in `minimateplus/waveform_codec.py`. ### 7-byte preamble @@ -207,6 +219,48 @@ TL;DR table above are now locked in by pytest regression tests. still bails out partway through. Lower priority since the other 7 events walk cleanly. +4. **Variable-prefix segment descriptors** (found 2026-08-25). + 3 of 75 ground-truth production events still truncate. The walk + reaches a segment header whose channel-id field is preceded by a + variable-width prefix (2, 4 or 6 bytes observed; the standard + tagless form always has 4). These also carry an `01 00` marker + instead of `02 00`. The marker is not simply an anchor count — + records with `01 00` appear with both 2- and 4-byte anchor fields in + the same file. Examples: `BE12599/N599LPNB.JF0W` @1155, + `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. + +## Segment header: channel id and tagless form — 2026-08-25 + +The 4-byte field previously read as a "monotonic uint32 LE counter" is +`[channel_id][00][00][segment_index]`, with `0x46`=Tran `0x47`=Vert +`0x48`=Long `0x49`=MicL. Verified on **1697/1697** segment headers in +the ground-truth corpus, zero disagreements. `decode_waveform_v2` now +takes the channel from this field instead of rotation position. + +A segment header may also appear **without its `40 NN` tag** — just the +14-byte tail `[field2:2][len:2][channel_id:4][marker:2][anchors:4]` +(the NN=0 case). `is_tagless_segment_header()` detects it from the six +bytes at `[4:10]`. + +## Geo scale: full scale is 32000 counts — 2026-08-25 + +One decoder unit (16 ADC counts) is exactly 0.005 in/s, so Normal range +(10.000 in/s) is `10.0 / (0.005/16)` = **32000** ADC counts. Consumers +that divided by 32768 read every geophone sample 2.34% low. Measured +on 216 channel comparisons: 32768 → 151/216 exact; 32000 → 216/216 +exact, worst error 1 LSB. + +## Ground-truth corpus (2026-08-25) + +Beyond the bundled fixtures, the production waveform store keeps each +event's original Blastware ASCII export at +`//_ASCII.TXT`. 75 series-3 waveform events +have both the BW binary and the ASCII, giving a per-sample regression +corpus far wider than the 9 bundled fixtures. Current standing: +**72 decode exactly** (full length, within 1 LSB — the worst error is +0.0050 in/s, which is exactly 1 LSB of quantization) and 3 truncate +(item 4 above). Zero events have full-length value errors. + ## `30 NN` block format — CRACKED 2026-05-11 late The `30 NN` block carries `NN` 12-bit signed deltas, packed as `NN/4` diff --git a/minimateplus/event_file_io.py b/minimateplus/event_file_io.py index 00cb43e..5d3a01f 100644 --- a/minimateplus/event_file_io.py +++ b/minimateplus/event_file_io.py @@ -659,6 +659,11 @@ def file_sha256(path: Union[str, Path], chunk_size: int = 65536) -> str: _GEO_NORMAL_FS_INS = 10.0 _GEO_SENSITIVE_FS_INS = 1.250 _INT16_FS = 32768.0 +# Geophone full-scale count — 32000, not 32768. One decoder unit (16 ADC +# counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Must match +# sfm.event_hdf5._GEO_INT16_FS or sidecar peaks disagree with the plotted +# waveform by 2.3%. Confirmed 2026-08-25 against the BW ASCII corpus. +_GEO_INT16_FS = 32000.0 # Microphone scale factor, psi per ADC count. Approximate — exact factor # depends on the geophone-vs-mic ADC scaling and the firmware reference. @@ -728,7 +733,7 @@ def _peaks_from_samples(samples: dict[str, list[int]]) -> PeakValues: if not ch: return 0.0 m = max(abs(int(v)) for v in ch) - return m / _INT16_FS * _GEO_NORMAL_FS_INS + return m / _GEO_INT16_FS * _GEO_NORMAL_FS_INS tran = _peak_ins(samples.get("Tran", [])) vert = _peak_ins(samples.get("Vert", [])) @@ -742,7 +747,7 @@ def _peaks_from_samples(samples: dict[str, list[int]]) -> PeakValues: pvs = 0.0 n = min(len(samples.get("Tran", [])), len(samples.get("Vert", [])), len(samples.get("Long", []))) if n: - scale = _GEO_NORMAL_FS_INS / _INT16_FS + scale = _GEO_NORMAL_FS_INS / _GEO_INT16_FS T = samples["Tran"]; V = samples["Vert"]; L = samples["Long"] for i in range(n): t = T[i] * scale diff --git a/minimateplus/waveform_codec.py b/minimateplus/waveform_codec.py index c0b40ab..0c96422 100644 --- a/minimateplus/waveform_codec.py +++ b/minimateplus/waveform_codec.py @@ -166,8 +166,14 @@ def find_data_start(body: bytes) -> int: # Try fixed offset 7 first (canonical preamble length). if len(body) >= 9: b, nn = body[7], body[8] - if (b in (0x00, 0x10, 0x20, 0x30) and nn % 4 == 0 and 0 < nn <= 0xFC) \ - or (b == 0x40 and nn == 0x02): + # Accept the same tag vocabulary ``walk_body`` accepts, including the + # wide-NN forms (``0X``/``1X``/``2X``) and the variable-width ``40 NN`` + # segment header. + if ((b & 0xF0) in (0x00, 0x10, 0x20) and nn % 4 == 0 + and ((b & 0x0F) != 0 or 0 < nn <= 0xFC)) \ + or (b == 0x30 and nn % 4 == 0 and 0 < nn <= 0xFC) \ + or (b == 0x40 and 0 < nn <= 0x08) \ + or is_tagless_segment_header(body, 7): return 7 # Fall back to scanning the first 20 bytes. for i in range(min(20, len(body) - 1)): @@ -178,6 +184,31 @@ def find_data_start(body: bytes) -> int: return -1 +# Channel-id byte carried in every segment header. Previously mis-read as a +# "monotonic uint32 LE counter"; it is really ``[channel][00][00][segment]``. +# Verified 2026-08-25 on 1697/1697 segment headers across the ground-truth +# corpus with zero disagreements against the decoded channel rotation. +SEGMENT_CHANNEL_IDS = {0x46: "Tran", 0x47: "Vert", 0x48: "Long", 0x49: "MicL"} + +# A tagless segment header: the 14-byte tail of a ``40 NN`` header with no tag +# and no previous-channel continuation deltas (the NN=0 case). +_TAGLESS_HEADER_LEN = 14 + + +def is_tagless_segment_header(body: bytes, i: int) -> bool: + """True if a bare 14-byte segment header starts at *i*. + + Layout ``[field2:2][len_to_next:2][channel_id:4][marker:2][anchors:4]``. + The discriminator is the 6 bytes at ``[4:10]``: a known channel id, two + zero bytes, a small segment index, and the ``01 00`` / ``02 00`` marker. + """ + if i + _TAGLESS_HEADER_LEN > len(body): + return False + return (body[i + 4] in SEGMENT_CHANNEL_IDS + and body[i + 5] == 0x00 and body[i + 6] == 0x00 + and body[i + 8] in (0x01, 0x02) and body[i + 9] == 0x00) + + def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: """Walk the tagged-block sequence starting at *start* (auto-detected by default). @@ -210,9 +241,15 @@ def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: # Wide-NN int8 block: ``2X NN`` extends NN to 12 bits the same way. wide_nn = ((t0 & 0x0F) << 8) | t1 length = wide_nn + 2 - elif t0 == 0x00 and t1 % 4 == 0: + elif (t0 & 0xF0) == 0x00 and t1 % 4 == 0: + # ``00 NN`` RLE zero-delta run, plus its wide form ``0X NN`` + # (X != 0) which extends NN to 12 bits exactly like ``1X``/``2X``: + # NN = ((t0 & 0x0F) << 8) | t1. A narrow run maxes out at + # NN=0xFC, so quiet stretches longer than 252 samples must use + # the wide form. Confirmed 2026-08-25 against six production + # events (e.g. ``01 0c`` = 268 repeats in K558LKOF.460W). length = 2 - elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0x10: + elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0xFC: # Data-section ``30 NN`` blocks carry NN 12-bit signed deltas packed # as NN/4 groups of (2-byte high-nibble field + 4 × int8 low byte). # Length = NN/4 × 6 + 2 = NN × 1.5 + 2 (= 8 for NN=4, 14 for NN=8, @@ -229,8 +266,28 @@ def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: length = cand_data else: length = cand_trailer - elif t0 == 0x40 and t1 == 0x02: - length = 20 + elif t0 == 0x40 and 0 < t1 <= 0x08: + # ``40 NN`` segment header. NN is the number of int16 BE + # continuation deltas the header carries for the PREVIOUS + # channel, so the header grows with NN: + # length = 2 (tag) + 2*NN (deltas) + 14 (fixed tail) + # ``40 02`` (20 bytes) dominates, but ``40 01`` (18) and + # ``40 03`` (22) both occur in production files. Confirmed + # 2026-08-25; the constant ``02 00`` marker moves with NN too + # (see :func:`parse_segment_header`). + length = 2 * t1 + 16 + elif is_tagless_segment_header(body, i): + # Segment header with no ``40 NN`` tag (NN=0 — the previous channel + # needed no continuation deltas). Emit it as a synthetic ``40 00`` + # block whose ``data`` is the whole 14-byte record, so the nd=0 + # offsets in :func:`decode_waveform_v2` line up unchanged. + blocks.append(WaveformBlock( + offset=i, tag_hi=0x40, tag_lo=0x00, + data=bytes(body[i : i + _TAGLESS_HEADER_LEN]), + length=_TAGLESS_HEADER_LEN, + )) + i += _TAGLESS_HEADER_LEN + continue else: # Unknown tag; stop. Caller can inspect ``i`` to see where. break @@ -256,7 +313,7 @@ def split_segments(blocks: List[WaveformBlock]) -> List[List[WaveformBlock]]: segments: List[List[WaveformBlock]] = [] current: List[WaveformBlock] = [] for b in blocks: - if b.tag_hi == 0x40 and b.tag_lo == 0x02: + if b.tag_hi == 0x40: if current: segments.append(current) current = [b] @@ -268,23 +325,40 @@ def split_segments(blocks: List[WaveformBlock]) -> List[List[WaveformBlock]]: def parse_segment_header(block: WaveformBlock) -> Optional[dict]: - """Decode the 18-byte payload of a ``40 02`` segment header. + """Decode the payload of a ``40 NN`` segment header. - Returns a dict with the labelled fields, or None if *block* is not - a ``40 02`` header. + NN (the tag's low byte) is the number of int16 BE continuation deltas + the header carries for the PREVIOUS channel, so every field after + those deltas shifts by ``2 * NN``. The payload is ``2 * NN + 14`` + bytes. ``40 02`` is the common case; ``40 01`` and ``40 03`` also + occur in production files (confirmed 2026-08-25). + + Returns a dict with the labelled fields, or None if *block* is not a + segment header or is too short. """ - if not (block.tag_hi == 0x40 and block.tag_lo == 0x02): + if block.tag_hi != 0x40 or block.tag_lo > 0x08: return None - if len(block.data) < 18: + nd = block.tag_lo + if len(block.data) < 2 * nd + 14: return None p = block.data - counter = int.from_bytes(p[8:12], "little", signed=False) + counter = int.from_bytes(p[2 * nd + 4 : 2 * nd + 8], "little", signed=False) return { - "anchor_bytes": p[0:4], # 4-byte field, role unconfirmed - "field2": p[4:8], # 4-byte field, role unconfirmed - "counter": counter, # uint32 LE — increments by 1 per segment - "fixed_pattern": p[12:16], # always b"\x02\x00\x00\x01" - "tail": p[16:18], # last 2 bytes + "n_prev_deltas": nd, + # ``nd`` int16 BE deltas extending the previous channel. + "prev_deltas": [ + int.from_bytes(p[2 * k : 2 * k + 2], "big", signed=True) + for k in range(nd) + ], + "field2": p[2 * nd : 2 * nd + 4], # 4-byte field, role unconfirmed + "counter": counter, # legacy: raw uint32 LE of the id field + "channel": SEGMENT_CHANNEL_IDS.get(p[2 * nd + 4]), + "segment_index": p[2 * nd + 7], + "marker": p[2 * nd + 8 : 2 * nd + 10], # always b"\x02\x00" + "anchors": [ + int.from_bytes(p[2 * nd + 10 : 2 * nd + 12], "big", signed=True), + int.from_bytes(p[2 * nd + 12 : 2 * nd + 14], "big", signed=True), + ], } @@ -420,8 +494,11 @@ def decode_waveform_v2(body: bytes) -> Optional[dict]: for byte in blk.data: cur += _i8(byte) out[channel].append(cur) - elif blk.tag_hi == 0x00: - for _ in range(blk.tag_lo): + elif (blk.tag_hi & 0xF0) == 0x00: + # RLE zero-delta run. Wide form ``0X NN`` carries the high + # nibble of a 12-bit NN in the tag byte, same as ``1X``/``2X``. + run = ((blk.tag_hi & 0x0F) << 8) | blk.tag_lo + for _ in range(run): out[channel].append(cur) elif blk.tag_hi == 0x30: # 12-bit signed deltas, packed as NN/4 groups of 6 bytes each: @@ -461,34 +538,54 @@ def decode_waveform_v2(body: bytes) -> Optional[dict]: # previous-channel extension deltas at every segment boundary. last_value = {"Tran": last_tran_value, "Vert": None, "Long": None, "MicL": None} + prev_channel = "Tran" for k, hi in enumerate(seg_idx): - channel = rotation[k % 4] - prev_channel = "Tran" if k == 0 else rotation[(k - 1) % 4] header = blocks[hi] - if len(header.data) < 18: + # Channel comes from the header's own id byte, which is authoritative. + # The old rotation-by-position fallback is kept for headers whose id + # byte isn't one of the four known values — but a single missed or + # extra header would desync rotation and corrupt every later channel, + # which is exactly what tagless headers used to cause. + _nd = header.tag_lo + channel = None + if len(header.data) >= 2 * _nd + 8: + channel = SEGMENT_CHANNEL_IDS.get(header.data[2 * _nd + 4]) + if channel is None: + channel = rotation[k % 4] + # ``40 NN``: NN int16 BE continuation deltas for the previous channel + # come first, so every later field shifts by 2*NN. NN is usually 2 + # but 1 and 3 both occur (confirmed 2026-08-25). + nd = header.tag_lo + if len(header.data) < 2 * nd + 14: continue - # Validate: real segment headers have bytes [12:14] = `02 00`. - # Trailer/footer "40 02" markers contain ASCII serial bytes or other - # non-header data there and would otherwise be mis-interpreted as - # segment headers, adding spurious samples at the tail. - if header.data[12:14] != b"\x02\x00": + # Validate: real segment headers have the constant `02 00` marker + # right after the counter. Trailer/footer "40 NN" markers contain + # ASCII serial bytes or other non-header data there and would + # otherwise be mis-read as segment headers, adding spurious tail + # samples. + if header.data[2 * nd + 8 : 2 * nd + 10] != b"\x02\x00": break - # Extend the PREVIOUS channel by 2 more samples (deltas in bytes [0:4]). - prev_d0 = int.from_bytes(header.data[0:2], "big", signed=True) - prev_d1 = int.from_bytes(header.data[2:4], "big", signed=True) + # Extend the PREVIOUS channel by NN more samples. if last_value[prev_channel] is not None: - v = last_value[prev_channel] + prev_d0 - out[prev_channel].append(v) - v += prev_d1 - out[prev_channel].append(v) + v = last_value[prev_channel] + for d in range(nd): # NB: not `k` — that's the segment index + v += int.from_bytes( + header.data[2 * d : 2 * d + 2], "big", signed=True + ) + out[prev_channel].append(v) last_value[prev_channel] = v # Anchor pair for THIS segment's channel. - c0 = int.from_bytes(header.data[14:16], "big", signed=True) - c1 = int.from_bytes(header.data[16:18], "big", signed=True) + c0 = int.from_bytes( + header.data[2 * nd + 10 : 2 * nd + 12], "big", signed=True + ) + c1 = int.from_bytes( + header.data[2 * nd + 12 : 2 * nd + 14], "big", signed=True + ) out[channel].extend([c0, c1]) # Apply delta blocks for this segment. next_hi = seg_idx[k + 1] if k + 1 < len(seg_idx) else len(blocks) last_value[channel] = apply_blocks(channel, c1, hi + 1, next_hi) + prev_channel = channel return out diff --git a/sfm/event_hdf5.py b/sfm/event_hdf5.py index aa6e6f0..a25b34d 100644 --- a/sfm/event_hdf5.py +++ b/sfm/event_hdf5.py @@ -77,6 +77,20 @@ _GEO_FS_BY_RANGE = { } _INT16_FS = 32768.0 +# Geophone full-scale count. NOT 32768: the verified body codec emits geo +# samples in 16-count units whose documented LSB is exactly 0.005 in/s, and +# ``waveform_codec.decoded_to_adc_counts`` multiplies by 16 — so one ADC count +# is 0.005/16 in/s and Normal range (10.000 in/s) is 10.0 / (0.005/16) = 32000 +# counts. Using 32768 here made every geophone reading 2.3% low +# (1 - 32000/32768 = 0.0234). +# +# Confirmed 2026-08-25 against 216 per-channel comparisons with the preserved +# Blastware ASCII exports: 32000 gives 216/216 exact within 1 LSB (worst error +# 0.005 in/s); 32768 gave 151/216 with a worst error of 0.238 in/s on a +# 10 in/s event. The mic path is unaffected — it back-solves its own scale +# from the device-reported peak (see _mic_scale_factor). +_GEO_INT16_FS = 32000.0 + # Default mic conversion: ADC count → psi. Approximate; exact factor # depends on firmware reference voltage and mic sensitivity, neither of # which is independently confirmed. We try to refine it from the device- @@ -125,15 +139,14 @@ def _samples_to_float( ) -> np.ndarray: """Convert int16 ADC counts → float32 physical units. - Uses _INT16_FS=32768 (not 32767) so that a count of -32768 maps to - exactly -full_scale and +32767 maps to ~+full_scale * 32767/32768. - Matches the device firmware's documented mapping (see CLAUDE.md - geo_hardware_constant rationale). + Uses _GEO_INT16_FS=32000 (see the constant's rationale): one decoder + unit (16 ADC counts) is exactly 0.005 in/s, so full scale is 32000 + counts, not 32768. """ if not samples_int16: return np.array([], dtype=np.float32) arr = np.asarray(samples_int16, dtype=np.int32) # int32 to avoid overflow during scale - return (arr.astype(np.float32) * (full_scale / _INT16_FS)).astype(np.float32) + return (arr.astype(np.float32) * (full_scale / _GEO_INT16_FS)).astype(np.float32) def _mic_scale_factor( diff --git a/tests/test_event_file_io.py b/tests/test_event_file_io.py index 0e043e8..2b0c0d4 100644 --- a/tests/test_event_file_io.py +++ b/tests/test_event_file_io.py @@ -626,3 +626,12 @@ if __name__ == "__main__": failed += 1 print(f"\n{passed} passed, {failed} failed") sys.exit(0 if failed == 0 else 1) + + +def test_peaks_from_samples_uses_32000_full_scale(): + """`_peaks_from_samples` must use the same 32000-count geo full scale as + the .h5 writer, or sidecar peaks disagree with the plotted waveform by + 2.3%. See test_event_hdf5.test_geo_full_scale_count_is_32000.""" + from minimateplus.event_file_io import _peaks_from_samples + pv = _peaks_from_samples({"Tran": [32000], "Vert": [0], "Long": [0], "MicL": []}) + assert abs(pv.tran - 10.0) < 1e-4 diff --git a/tests/test_event_hdf5.py b/tests/test_event_hdf5.py index 86c3336..0560f5d 100644 --- a/tests/test_event_hdf5.py +++ b/tests/test_event_hdf5.py @@ -99,8 +99,14 @@ def test_hdf5_round_trip_preserves_metadata(tmp_path: Path): def test_hdf5_samples_in_physical_units_normal_range(tmp_path: Path): - """Vert hits ADC full-scale (32767) → with Normal range FS=10 in/s, - the HDF5 sample value should be ≈ 10 * 32767/32768 in/s.""" + """Vert hits 32767 ADC counts → with Normal range FS=10 in/s that is + ``10 * 32767/32000`` in/s. + + Geo full scale is 32000 counts, not 32768 (see + test_geo_full_scale_count_is_32000), so 32767 counts sits slightly + ABOVE nominal full scale -- the ADC has headroom past 10.000 in/s, + which is why Blastware reports peaks like 10.14 in/s. This test + previously asserted the 32768 scale and was wrong by 2.3%.""" ev = _make_event_with_samples() h5 = tmp_path / "n.h5" event_hdf5.write_event_hdf5(h5, ev, serial="BE11529", geo_range="normal") @@ -110,7 +116,7 @@ def test_hdf5_samples_in_physical_units_normal_range(tmp_path: Path): assert vert.dtype.name == "float32" assert max(abs(v) for v in vert) > 9.99 # full-scale ≈ 10.0 # The dirac was at n//2 → 32767 ADC counts. - expected_peak = 10.0 * 32767 / 32768 + expected_peak = 10.0 * 32767 / 32000 assert abs(max(vert) - expected_peak) < 1e-3 @@ -122,7 +128,7 @@ def test_hdf5_samples_in_physical_units_sensitive_range(tmp_path: Path): data = event_hdf5.read_event_hdf5(h5) vert = data["samples"]["Vert"] - expected_peak = 1.250 * 32767 / 32768 + expected_peak = 1.250 * 32767 / 32000 assert abs(max(vert) - expected_peak) < 1e-4 @@ -294,3 +300,35 @@ if __name__ == "__main__": failed += 1 print(f"\n{passed} passed, {failed} failed") sys.exit(0 if failed == 0 else 1) + + +# ── Geophone full-scale count ─────────────────────────────────────────────── + +def test_geo_full_scale_count_is_32000(): + """Geo full scale is 32000 ADC counts, not 32768. + + The verified body codec emits geo samples in 16-count units with a + documented LSB of exactly 0.005 in/s, and ``decoded_to_adc_counts`` + multiplies by 16 — so one ADC count is 0.005/16 in/s and Normal range + (10.000 in/s) is 10.0 / (0.005/16) = 32000 counts. + + Using 32768 made every geophone reading 2.3% low (1 - 32000/32768). + Confirmed 2026-08-25 against 216 channel comparisons with preserved + Blastware ASCII exports: 32000 → 216/216 exact within 1 LSB; + 32768 → 151/216, worst error 0.238 in/s on a 10 in/s event. + """ + from sfm.event_hdf5 import _GEO_INT16_FS + assert _GEO_INT16_FS == 32000.0 + + +def test_samples_to_float_lsb_is_exactly_5_milli_ips(): + """One decoder unit (= 16 ADC counts) must be exactly 0.005 in/s.""" + from sfm.event_hdf5 import _samples_to_float + out = _samples_to_float([16], 10.0) + assert abs(float(out[0]) - 0.005) < 1e-9 + + +def test_samples_to_float_full_scale_count_maps_to_full_scale(): + from sfm.event_hdf5 import _samples_to_float + assert abs(float(_samples_to_float([32000], 10.0)[0]) - 10.0) < 1e-4 + assert abs(float(_samples_to_float([32000], 1.25)[0]) - 1.25) < 1e-5 diff --git a/tests/test_waveform_codec.py b/tests/test_waveform_codec.py index ffd84ca..2d3290f 100644 --- a/tests/test_waveform_codec.py +++ b/tests/test_waveform_codec.py @@ -210,9 +210,11 @@ def test_parse_segment_header_decodes_fields(): ) decoded = parse_segment_header(block) assert decoded is not None + assert decoded["n_prev_deltas"] == 2 + assert decoded["prev_deltas"] == [0, 0] assert decoded["counter"] == 0x47 # uint32 LE - assert decoded["fixed_pattern"] == b"\x02\x00\x00\x01" - assert decoded["anchor_bytes"] == b"\x00\x00\x00\x00" + assert decoded["marker"] == b"\x02\x00" + assert decoded["anchors"] == [1, 1] def test_segment_counter_increments(): @@ -516,3 +518,134 @@ def test_decode_a5_frames_empty(): from minimateplus.waveform_codec import decode_a5_frames assert decode_a5_frames([]) is None assert decode_a5_frames(None) is None + + +# ── Wide-NN RLE, wide 30 NN, and variable-width segment headers ────────────── +# +# Three framing cases discovered 2026-08-25 by diffing 75 production events +# against their preserved Blastware ASCII exports. Each caused ``walk_body`` +# to hit its ``else: break`` mid-stream, truncating every channel decoded +# after that point (see CHANGELOG v0.25.1). + +_PREAMBLE = b"\x00\x02\x00\x00\x00\x00\x00" # magic + Tran[0]=0, Tran[1]=0 +_STOP = b"\xff\xff" # unrecognised tag → walker stops + + +def _synth(*chunks: bytes) -> bytes: + return _PREAMBLE + b"".join(chunks) + _STOP + + +def test_walk_body_wide_rle_block(): + """``0X NN`` is a 12-bit-NN RLE run (NN = ((t0 & 0x0F) << 8) | t1). + + Observed as ``01 0c`` (NN=268) in BE9558/K558LKOF.460W and five other + production events. A narrow ``00 NN`` maxes out at NN=0xFC, so runs + longer than 252 samples must use the wide form. + """ + blocks = walk_body(_synth(b"\x01\x0c")) + assert len(blocks) == 1 + assert (blocks[0].tag_hi, blocks[0].tag_lo) == (0x01, 0x0C) + assert blocks[0].length == 2 + + +def test_decode_wide_rle_repeats_full_run(): + """A wide RLE run repeats the running value NN times, not NN & 0xFF.""" + decoded = decode_waveform_v2(_synth(b"\x01\x0c")) + # 2 preamble anchors + 268 repeats + assert len(decoded["Tran"]) == 2 + 268 + assert set(decoded["Tran"]) == {0} + + +def test_walk_body_30_block_nn_above_16(): + """``30 NN`` data blocks are not capped at NN=0x10. + + ``30 18`` (NN=24) appears in BE18193/T193LQ45.NN0W; the old + ``0 < t1 <= 0x10`` guard rejected it and stopped the walk 1033 bytes + into a 4877-byte body. Length is still NN * 1.5 + 2. + """ + payload = bytes(36) # 24 deltas × 1.5 bytes + blocks = walk_body(_synth(b"\x30\x18" + payload, b"\x00\x04")) + assert [b.length for b in blocks] == [38, 2] + + +@pytest.mark.parametrize("nn,hdr_len", [(1, 18), (2, 20), (3, 22)]) +def test_walk_body_segment_header_width_follows_tag_lo(nn, hdr_len): + """``40 NN``: NN is the count of previous-channel continuation deltas. + + Header length = 2 * NN + 16. ``40 02`` (the only form previously + handled) is the NN=2 case at 20 bytes; ``40 01`` (18) and ``40 03`` + (22) both occur in production files. + """ + data = bytearray(hdr_len - 2) + data[2 * nn + 8 : 2 * nn + 10] = b"\x02\x00" # constant marker + blocks = walk_body(_synth(bytes([0x40, nn]) + bytes(data), b"\x00\x04")) + assert [b.length for b in blocks] == [hdr_len, 2] + + +@pytest.mark.parametrize("nn,hdr_len", [(1, 18), (2, 20), (3, 22)]) +def test_segment_header_anchors_track_header_width(nn, hdr_len): + """Anchor pair sits at data[2*NN+10 : 2*NN+14] regardless of width.""" + data = bytearray(hdr_len - 2) + data[2 * nn + 8 : 2 * nn + 10] = b"\x02\x00" + data[2 * nn + 10 : 2 * nn + 12] = (7).to_bytes(2, "big") # anchor 0 + data[2 * nn + 12 : 2 * nn + 14] = (9).to_bytes(2, "big") # anchor 1 + decoded = decode_waveform_v2(_synth(bytes([0x40, nn]) + bytes(data))) + assert decoded["Vert"][:2] == [7, 9] + + +# ── Tagless segment headers ───────────────────────────────────────────────── +# +# A segment header can appear WITHOUT its ``40 NN`` tag: just the 14-byte tail +# ``[field2:2][len:2][channel_id:4][marker:2][anchors:4]``. This is the NN=0 +# case — no continuation deltas for the previous channel, so no tag and no +# delta bytes. Found 2026-08-25: it is where the walk stopped in 7 of the 8 +# remaining truncating production events. +# +# The channel_id field (previously mis-labelled a "monotonic counter") is +# ``[channel][00][00][segment_index]`` with 0x46=Tran 0x47=Vert 0x48=Long +# 0x49=MicL — verified on 1697 of 1697 segment headers across the ground-truth +# corpus, zero disagreements. + +def _tagless(chan_id=0x47, seg=2, marker=b"\x02\x00", a0=0, a1=0): + return (b"\x5d\xee" + b"\x00\xd0" + bytes([chan_id, 0, 0, seg]) + marker + + a0.to_bytes(2, "big", signed=True) + a1.to_bytes(2, "big", signed=True)) + + +def test_walk_body_accepts_tagless_segment_header(): + """A bare 14-byte header is walked as a segment block, not a stop.""" + blocks = walk_body(_synth(b"\x10\x04\x00\x00", _tagless(), b"\x00\x04")) + kinds = [(b.tag_hi, b.tag_lo, b.length) for b in blocks] + assert kinds == [(0x10, 0x04, 4), (0x40, 0x00, 14), (0x00, 0x04, 2)] + + +def test_tagless_header_carries_full_14_bytes_as_data(): + """The synthetic block's data includes the leading bytes (there is no tag + to strip), so decode_waveform_v2's ``2*nd + k`` offsets line up at nd=0.""" + blocks = walk_body(_synth(_tagless())) + hdr = next(b for b in blocks if b.tag_hi == 0x40) + assert len(hdr.data) == 14 + assert hdr.data[8:10] == b"\x02\x00" # marker at 2*0 + 8 + + +def test_tagless_header_anchors_and_channel_id(): + """Anchors decode from data[10:14]; the channel comes from the id byte.""" + decoded = decode_waveform_v2(_synth(_tagless(chan_id=0x48, a0=11, a1=13))) + assert decoded["Long"][:2] == [11, 13] # 0x48 → Long, not rotation + assert decoded["Vert"] == [] + + +@pytest.mark.parametrize("chan_id,name", + [(0x46, "Tran"), (0x47, "Vert"), (0x48, "Long"), (0x49, "MicL")]) +def test_segment_channel_comes_from_id_not_rotation(chan_id, name): + """Channel is taken from the header's id byte. Two headers in a row for + the SAME channel must both land on that channel — rotation-by-position + would put the second one on the next channel and corrupt both.""" + body = _synth(_tagless(chan_id=chan_id, seg=1, a0=5, a1=6), + _tagless(chan_id=chan_id, seg=2, a0=7, a1=8)) + decoded = decode_waveform_v2(body) + # Tran additionally carries the body preamble's 2 anchors (both 0 here). + expected = [0, 0, 5, 6, 7, 8] if name == "Tran" else [5, 6, 7, 8] + assert decoded[name] == expected + for other in ("Tran", "Vert", "Long", "MicL"): + if other != name: + assert decoded[other] == ([0, 0] if other == "Tran" else [])