From 904522a9c5e547586c01630142c31dabb57ef741 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 11 Sep 2026 05:05:46 +0000 Subject: [PATCH] fix(codec): 40 NN int16 blocks are not capped at NN=8 data_block_len() rejected any `40 NN` block with NN > 0x08. That guard had no evidence behind it: every corpus available when it was written used only NN in {1,2,3,4,8}, so it was never exercised. Loud UM12947 events use NN of 12, 16, 20 ... up to 196. Because walk_body/run stop at the first unrecognised tag rather than raising, rejecting those blocks surfaced as silently short channels -- e.g. Tran 1812 / Vert 2132 / Long 2324 on a file whose export carries 2324 for all three. The real bound is the buffer; the caller additionally clamps to the record end. Verified against Thor's own CSV exports for UM12947 (2025-07-14 .. 09-25, 167 waveforms, supplied as CSV.zip): length mismatches 22 -> 0 per-sample exact 1,476,242 / 1,476,249 These are NOT truncated recordings, which was the competing hypothesis -- the exports carry the full sample count. tests/test_waveform_codec.py asserted the cap as intended behaviour. That assertion encoded an assumption, not a verified fact, and is replaced with one pinning the opposite plus the evidence. Across all three ground-truth corpora: 459 waveform files, 3,807,158 / 3,807,165 samples exact. Production IDFW is now 575/575 with zero truncations and zero decode failures (median PPV error -0.0007% across 8 units). Series-3 re-verified unchanged at 14,338/14,338. The 7 residual samples each differ by one 4th-decimal tick and are Thor's own rounding: intersecting the per-sample rounding constraints over that corpus is infeasible (binding pair contradict by 2.3e-11, 7e-5 relative), so no single linear LSB reproduces every printed value. _GEO_LSB_IPS is already pinned to ~1e-11; do not retune it to chase these. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ru8Lg9HkkYvX9VWWo65SmL --- CHANGELOG.md | 38 +++++++++++++++++---- CLAUDE.md | 19 ++++++++--- docs/idf_protocol_reference.md | 61 ++++++++++++++++++++-------------- minimateplus/waveform_codec.py | 13 +++++++- tests/test_idf_binary_codec.py | 29 ++++++++++++++++ tests/test_waveform_codec.py | 23 +++++++++++-- 6 files changed, 143 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12fcf97..167bdf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,13 +87,37 @@ within 2%** (was 66.9% and 56.6%). Combined across both corpora: **292/292 waveform files, 2,330,916/2,330,916 samples exact.** Production IDFW truncations 41 → 22. -**Known open — diagnosed but NOT verified:** 23/575 production IDFW files -(4%), all UM12947 between 2025-07-14 and 2025-09-23, stop the block walker on -tag `40 0c`. `data_block_len()` caps the `40 NN` int16 block at `NN > 0x08`, -but these files use NN up to 196. Both verified corpora only ever use -NN ∈ {1,2,3,4,8}, so the cap is untested there and lifting it leaves both at -100.000% — which is *not* evidence it decodes these correctly. Needs Thor CSV -exports for UM12947 in that date range (the 9-10-26 upload starts 2025-09-25). +### Fixed — `40 NN` int16 blocks with NN > 8 + +`data_block_len()` rejected any `40 NN` block with `NN > 0x08`. The cap had +no evidence behind it: every corpus available when it was written used only +NN ∈ {1,2,3,4,8}, so it was never exercised. Loud UM12947 events use NN of +12, 16, 20 … up to 196, and because the block walker stops at the first +unrecognised tag rather than raising, rejecting them surfaced as **silently +short channels** (e.g. Tran 1812 / Vert 2132 / Long 2324 on a file whose +export has 2324 for all three). The bound is the buffer, not a constant. + +Verified against Thor exports for UM12947 (2025-07-14 … 09-25, 167 +waveforms): length mismatches **22 → 0**, **1,476,242/1,476,249** samples +exact. These are not truncated recordings — the exports carry full sample +counts. + +`tests/test_waveform_codec.py` asserted the cap as intended behaviour; that +assertion was wrong and has been replaced with one pinning the opposite, +carrying the evidence. + +### Result across all three ground-truth corpora + +**459 waveform files, 3,807,158 / 3,807,165 samples exact.** Production +IDFW: **575/575**, zero truncations, zero decode failures, median PPV error +−0.0007% across 8 units. Series-3 re-verified **unchanged at 14,338/14,338** +after every shared-codec change. + +The 7 residual samples each differ by one 4th-decimal tick and are **Thor's +own rounding**: intersecting the per-sample rounding constraints over that +corpus is infeasible (the binding pair contradict by 2.3e-11, 7e-5 relative), +so no single linear LSB reproduces every printed value. `_GEO_LSB_IPS` is +already pinned to ~1e-11 — do not retune it to chase these. --- diff --git a/CLAUDE.md b/CLAUDE.md index a64e60f..b3459e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,11 +47,20 @@ Read this first when picking the project back up. was invisible and Vert came up exactly 512 short), and the histogram interval record is **56 bytes, not 72** — `16 × n_channels + 8`, derived per segment from the cumulative interval counter, never assumed. - **Still open:** 23/575 prod IDFW files (4%), all UM12947 2025-07-14..09-23, - stop the walker on tag `40 0c` — `data_block_len()` caps `40 NN` at NN>8 but - these use up to 196. **Diagnosed, NOT verified** — no Thor export exists for - those dates. Do not ship the cap change without one; see - `docs/idf_protocol_reference.md`. +- **`40 NN` blocks are not capped at NN=8 (2026-09-11).** `data_block_len()` + rejected `NN > 0x08`, a guard with no evidence behind it — the corpora + available when it was written only used NN ∈ {1,2,3,4,8}. Loud UM12947 + events use NN up to 196, and since the walker stops at the first + unrecognised tag rather than raising, this surfaced as silently short + channels. Verified on 167 UM12947 waveforms: length mismatches 22 → 0, + 1,476,242/1,476,249 samples exact. +- **Production IDFW is now 575/575** — zero truncations, zero decode + failures, median PPV error −0.0007% across 8 units (was 41 truncated + 1 + failing, −3.3%). Across all three ground-truth corpora: **459 files, + 3,807,158/3,807,165 samples exact**; the 7 stragglers differ by one + 4th-decimal tick and are Thor's own rounding — no single linear LSB can + reproduce every printed value (the constraints are infeasible by 7e-5 + relative), so do NOT retune `_GEO_LSB_IPS`. - **Open, not blocking:** 14 sensitive-range files show an exact 8x (= 10.0/1.25) units discrepancy; `scripts/backfill_sidecars.py --force` also inserts DB rows for store files that have none (one-time per store) and the diff --git a/docs/idf_protocol_reference.md b/docs/idf_protocol_reference.md index d1bef3c..2fdc035 100644 --- a/docs/idf_protocol_reference.md +++ b/docs/idf_protocol_reference.md @@ -178,6 +178,39 @@ channels come out the same length. `00 00`, so every run of three zero bytes looks like a body start and each costs a full trial decode (~0.5 s/file measured, vs 6 ms/file now). +### `40 NN` is not capped at NN=8 (2026-09-11) + +`data_block_len()` rejected any `40 NN` int16 block with `NN > 0x08`. The cap +had no evidence behind it — every corpus available when it was written used +only NN ∈ {1, 2, 3, 4, 8}, so it was never exercised. Loud events use much +wider blocks: + +| corpus | `40 NN` values | walker stops | +|---|---|---| +| first + 3-channel corpora | 1, 2, 3, 4, 8 | none | +| UM12947 2025-07..09 | 2, 4, 8, **12, 16, 20 … 196** | every value > 8 | + +Because `walk_body`/`run` stop at the first unrecognised tag rather than +raising, this surfaced as **silently short channels** — e.g. Tran 1812 / +Vert 2132 / Long 2324 on a file whose export has 2324 for all three. The +real bound is the buffer (and the caller's record end), not a magic constant. + +Verified against Thor's exports for UM12947 (2025-07-14 … 2025-09-25, 167 +waveforms): length mismatches **22 → 0**, and **1,476,242 / 1,476,249** +samples exact. + +⚠ These events are **not** truncated recordings, which was the competing +hypothesis — the exports carry the full sample count. + +**The 7 residual samples are Thor's rounding, not ours.** Each differs by +exactly one 4th-decimal tick (e.g. decoded 3.3551 vs export 3.3550). +Intersecting the per-sample rounding constraints over this corpus is +**infeasible** — the binding pair (count 2013 → 0.6247, count 4351 → 1.3501) +contradict by 2.3e-11, i.e. 7e-5 relative. No single linear LSB can +reproduce every printed value, so Thor is not doing plain round-half-up on +`count × LSB`. Do not retune `_GEO_LSB_IPS` to chase these; it is already +pinned to ~1e-11. + ### Mic-disabled units are a distinct shape (2026-09-10, second corpus) Some units run with the microphone disabled — **3 channels, not 4** — and that @@ -218,32 +251,10 @@ files that previously decoded no intervals at all. ### What is still open -- **23 of 575 production IDFW files (4%)** still decode with unequal channel - lengths (22) or fail outright (1). All are **UM12947, 2025-07-14 to - 2025-09-23**. The `Tran/Vert/Long 3072 / MicL 0` group was the 3-channel - shape above and is fixed; what remains is ragged, e.g. - `T1812 / V2132 / L2324 / M2324`. +- ~~23 of 575 production IDFW files~~ — **RESOLVED 2026-09-11.** Production + IDFW is now **575/575** with zero truncations and zero decode failures + (median PPV error −0.0007%). See "`40 NN` is not capped at NN=8" above. - **Diagnosed but NOT verified.** These files stop the block walker on tag - `40 0c`, because `data_block_len()` caps the `40 NN` int16 block at - `NN > 0x08`: - - ```python - if hi == 0x40: # int16 BE data block - return (None, None) if (nn == 0 or nn > 0x08) else (2 * nn + 2, nn) - ``` - - The affected files use NN of 12, 16, 20 … up to 196 — every value above 8 - halts the walk, which is why the channel comes up short. Both verified - corpora only ever use NN ∈ {1, 2, 3, 4, 8}, so the cap has never been - exercised, and lifting it leaves both at 100.000%. - - ⚠ **Do not ship the cap change on that evidence.** "Doesn't regress the - known-good corpus" is not "produces correct values here", and a too-loose - guard can accept a false `40 NN` inside data and emit plausible-but-wrong - samples — the exact failure mode this whole effort was about. It needs - Thor CSV exports for UM12947 events between **2025-07-14 and 2025-09-23**; - the `9-10-26-csv-req` upload starts at 2025-09-25 and misses them. - Mic → psi scale is still the rough `2.14e-6` regression, not derived. - Per-channel `int16 field4` in the IDFH interval record (possibly time-of-peak) and the 8-byte tail (PVS data) remain undecoded. diff --git a/minimateplus/waveform_codec.py b/minimateplus/waveform_codec.py index c47ddf5..e8d04dc 100644 --- a/minimateplus/waveform_codec.py +++ b/minimateplus/waveform_codec.py @@ -758,7 +758,18 @@ def data_block_len(body: bytes, p: int) -> Tuple[Optional[int], Optional[int]]: hi = t0 & 0xF0 nn = ((t0 & 0x0F) << 8) | t1 if hi == 0x40: # int16 BE data block - return (None, None) if (nn == 0 or nn > 0x08) else (2 * nn + 2, nn) + # NN was capped at 0x08 until 2026-09-11. That cap had no basis: the + # two corpora available at the time only ever used NN in {1,2,3,4,8}, + # so it was never exercised. Loud UM12947 events use NN of 12, 16, + # 20 ... up to 196, and every value above 8 halted the walk, which + # surfaced as silently short channels (walk_body/run stop at the first + # unrecognised tag rather than raising). Verified against Thor's own + # exports: 22 length-mismatched files -> 0, and the affected corpus + # went to 1,476,242/1,476,249 samples exact. The real bound is the + # buffer; the caller additionally clamps to the record end. + if nn == 0 or p + 2 * nn + 2 > len(body): + return None, None + return 2 * nn + 2, nn if nn == 0 or nn % 4: return None, None if hi == 0x00: diff --git a/tests/test_idf_binary_codec.py b/tests/test_idf_binary_codec.py index 98757d7..06e7c1f 100644 --- a/tests/test_idf_binary_codec.py +++ b/tests/test_idf_binary_codec.py @@ -291,3 +291,32 @@ def test_three_channel_histogram_uses_56_byte_intervals(): assert decoded == pytest.approx( _header_float(header, f"{channel}PPV"), rel=0.02 ) + + +# ─── `40 NN` blocks with NN > 8, verified 2026-09-11 ─────────────────────── + +IDFW_WIDE40 = FIXTURES / "UM12947_20250806134504.IDFW" + + +def test_wide_forty_nn_block_does_not_truncate_channels(): + """Loud events use `40 NN` blocks with NN well above the old cap of 8. + + ``data_block_len()`` rejected NN > 0x08, which halted the block walk + part-way through a record. The walker stops at the first unrecognised + tag instead of raising, so this surfaced as silently short channels — + here Tran 1812 / Vert 2132 / Long 2324 where the export has 2324 for all + three. The affected files use NN of 12, 16, 20 ... up to 196. + """ + rows = _parse_export(IDFW_WIDE40.with_suffix(".IDFW.csv"))[1] + result = read_idf_file(IDFW_WIDE40) + for index, channel in enumerate(GEO_CHANNELS): + decoded = result.samples[channel] + assert len(decoded) == len(rows), ( + f"{channel}: {len(decoded)} samples, export has {len(rows)}" + ) + expected = [row[index] for row in rows] + bad = sum( + 1 for c, v in zip(decoded, expected) + if abs(geo_count_to_ips(c) - v) >= 5e-5 + ) + assert bad == 0, f"{channel}: {bad} samples differ from Thor's export" diff --git a/tests/test_waveform_codec.py b/tests/test_waveform_codec.py index eebcf9d..d2c97b3 100644 --- a/tests/test_waveform_codec.py +++ b/tests/test_waveform_codec.py @@ -712,8 +712,27 @@ def test_forty_nn_is_a_data_block_not_a_segment_header(): """ assert data_block_len(b"\x40\x02\x00\x01\x00\x02", 0) == (6, 2) assert data_block_len(b"\x40\x08" + bytes(16), 0) == (18, 8) - # NN > 8 is not a data block - assert data_block_len(b"\x40\x0c" + bytes(24), 0) == (None, None) + + +def test_forty_nn_is_not_capped_at_eight(): + """NN > 8 is a perfectly ordinary `40 NN` block. + + This test previously asserted the opposite (`40 0c` -> (None, None)), + codifying a guard that had no evidence behind it: the only corpora + available then used NN in {1,2,3,4,8}, so the cap was never exercised. + Loud UM12947 events use NN of 12, 16, 20 ... up to 196, and rejecting + them halted the block walk mid-record — surfacing as silently short + channels, since the walker stops at the first unrecognised tag rather + than raising. Lifting the cap took that corpus from 22 length-mismatched + files to 0, and 1,476,242 of 1,476,249 samples now reproduce Thor's own + CSV export exactly (the 7 stragglers differ by one 4th-decimal tick). + Verified 2026-09-11; see docs/idf_protocol_reference.md. + """ + assert data_block_len(b"\x40\x0c" + bytes(24), 0) == (26, 12) + assert data_block_len(b"\x40\xc4" + bytes(392), 0) == (394, 196) + # The real bound is the buffer: a block that cannot fit is not a block. + assert data_block_len(b"\x40\xc4" + bytes(8), 0) == (None, None) + assert data_block_len(b"\x40\x00" + bytes(8), 0) == (None, None) def test_record_chain_is_followed_by_length_not_by_tag_sniffing():