fix(codec): 40 NN int16 blocks are not capped at NN=8
data_block_len() rejected any `40 NN` block with NN > 0x08. That guard had
no evidence behind it: every corpus available when it was written used only
NN in {1,2,3,4,8}, so it was never exercised. Loud UM12947 events use NN of
12, 16, 20 ... up to 196.
Because walk_body/run stop at the first unrecognised tag rather than
raising, rejecting those blocks surfaced as silently short channels -- e.g.
Tran 1812 / Vert 2132 / Long 2324 on a file whose export carries 2324 for
all three. The real bound is the buffer; the caller additionally clamps to
the record end.
Verified against Thor's own CSV exports for UM12947 (2025-07-14 .. 09-25,
167 waveforms, supplied as CSV.zip):
length mismatches 22 -> 0
per-sample exact 1,476,242 / 1,476,249
These are NOT truncated recordings, which was the competing hypothesis --
the exports carry the full sample count.
tests/test_waveform_codec.py asserted the cap as intended behaviour. That
assertion encoded an assumption, not a verified fact, and is replaced with
one pinning the opposite plus the evidence.
Across all three ground-truth corpora: 459 waveform files,
3,807,158 / 3,807,165 samples exact. Production IDFW is now 575/575 with
zero truncations and zero decode failures (median PPV error -0.0007% across
8 units). Series-3 re-verified unchanged at 14,338/14,338.
The 7 residual samples each differ by one 4th-decimal tick and are Thor's
own rounding: intersecting the per-sample rounding constraints over that
corpus is infeasible (binding pair contradict by 2.3e-11, 7e-5 relative),
so no single linear LSB reproduces every printed value. _GEO_LSB_IPS is
already pinned to ~1e-11; do not retune it to chase these.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ru8Lg9HkkYvX9VWWo65SmL
This commit is contained in:
@@ -291,3 +291,32 @@ def test_three_channel_histogram_uses_56_byte_intervals():
|
||||
assert decoded == pytest.approx(
|
||||
_header_float(header, f"{channel}PPV"), rel=0.02
|
||||
)
|
||||
|
||||
|
||||
# ─── `40 NN` blocks with NN > 8, verified 2026-09-11 ───────────────────────
|
||||
|
||||
IDFW_WIDE40 = FIXTURES / "UM12947_20250806134504.IDFW"
|
||||
|
||||
|
||||
def test_wide_forty_nn_block_does_not_truncate_channels():
|
||||
"""Loud events use `40 NN` blocks with NN well above the old cap of 8.
|
||||
|
||||
``data_block_len()`` rejected NN > 0x08, which halted the block walk
|
||||
part-way through a record. The walker stops at the first unrecognised
|
||||
tag instead of raising, so this surfaced as silently short channels —
|
||||
here Tran 1812 / Vert 2132 / Long 2324 where the export has 2324 for all
|
||||
three. The affected files use NN of 12, 16, 20 ... up to 196.
|
||||
"""
|
||||
rows = _parse_export(IDFW_WIDE40.with_suffix(".IDFW.csv"))[1]
|
||||
result = read_idf_file(IDFW_WIDE40)
|
||||
for index, channel in enumerate(GEO_CHANNELS):
|
||||
decoded = result.samples[channel]
|
||||
assert len(decoded) == len(rows), (
|
||||
f"{channel}: {len(decoded)} samples, export has {len(rows)}"
|
||||
)
|
||||
expected = [row[index] for row in rows]
|
||||
bad = sum(
|
||||
1 for c, v in zip(decoded, expected)
|
||||
if abs(geo_count_to_ips(c) - v) >= 5e-5
|
||||
)
|
||||
assert bad == 0, f"{channel}: {bad} samples differ from Thor's export"
|
||||
|
||||
@@ -712,8 +712,27 @@ def test_forty_nn_is_a_data_block_not_a_segment_header():
|
||||
"""
|
||||
assert data_block_len(b"\x40\x02\x00\x01\x00\x02", 0) == (6, 2)
|
||||
assert data_block_len(b"\x40\x08" + bytes(16), 0) == (18, 8)
|
||||
# NN > 8 is not a data block
|
||||
assert data_block_len(b"\x40\x0c" + bytes(24), 0) == (None, None)
|
||||
|
||||
|
||||
def test_forty_nn_is_not_capped_at_eight():
|
||||
"""NN > 8 is a perfectly ordinary `40 NN` block.
|
||||
|
||||
This test previously asserted the opposite (`40 0c` -> (None, None)),
|
||||
codifying a guard that had no evidence behind it: the only corpora
|
||||
available then used NN in {1,2,3,4,8}, so the cap was never exercised.
|
||||
Loud UM12947 events use NN of 12, 16, 20 ... up to 196, and rejecting
|
||||
them halted the block walk mid-record — surfacing as silently short
|
||||
channels, since the walker stops at the first unrecognised tag rather
|
||||
than raising. Lifting the cap took that corpus from 22 length-mismatched
|
||||
files to 0, and 1,476,242 of 1,476,249 samples now reproduce Thor's own
|
||||
CSV export exactly (the 7 stragglers differ by one 4th-decimal tick).
|
||||
Verified 2026-09-11; see docs/idf_protocol_reference.md.
|
||||
"""
|
||||
assert data_block_len(b"\x40\x0c" + bytes(24), 0) == (26, 12)
|
||||
assert data_block_len(b"\x40\xc4" + bytes(392), 0) == (394, 196)
|
||||
# The real bound is the buffer: a block that cannot fit is not a block.
|
||||
assert data_block_len(b"\x40\xc4" + bytes(8), 0) == (None, None)
|
||||
assert data_block_len(b"\x40\x00" + bytes(8), 0) == (None, None)
|
||||
|
||||
|
||||
def test_record_chain_is_followed_by_length_not_by_tag_sniffing():
|
||||
|
||||
Reference in New Issue
Block a user