fix(histogram): block is big-endian + terminal block tail — 1/1196 to 1211/1211
Two errors in the series-3 histogram block model, both found by diffing against the per-interval data table in the preserved Blastware ASCII exports (1211 files in the prod snapshot — far stronger ground truth than the header PPV used previously). 1. The block is uniformly BIG-ENDIAN. Peaks and half-periods are uint16 BE (T_peak [5:7], T_halfperiod [7:9], V_peak [9:11], V_halfperiod [11:13], L_peak [13:15], L_halfperiod [15:17], M_peak [17:19], M_halfperiod [19:21]); only block_ctr [2:4] is little-endian. The old uint8-peak model silently CLIPPED any peak above 1.275 in/s: the final interval of BE18193/T193LQ9K.OE0H reads 8.270 in/s in BW's export (1654 counts = 0x0676) and decoded as 0x76 = 118 = 0.590. The byte documented as a per-channel "annotation" was never an annotation — it is the half-period's high byte, which is exactly why it was non-zero on the sub-Hz intervals BW renders as "<1.0". The marker is block[4] alone. Testing [4:6] as a uint16 LE marker forced block[5] == 0, which is what capped the peak at one byte. 2. The final block of each stream carries tail 9c 06 00 42 instead of 1e 0a 00 00, and holds arbitrary bytes at [21:23]. Rejecting it dropped the last interval of nearly every histogram — frequently the interval holding the event peak, so the file's PPV read low. Verified end to end through the production path: 1211/1211 histograms decode exactly (interval count + every per-interval peak), plus 842,442 per-interval frequency comparisons with zero mismatches. Previously 1 of 1196 files was fully correct. decode_histogram_body_full records expose `is_terminal` in place of the removed `annotations` tuple. +6 tests. No regressions: full-suite failure list unchanged from baseline. NOTE: stored histogram .h5 files need regenerating to pick this up. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog
This commit is contained in:
@@ -354,23 +354,29 @@ _K558_INTERVAL_12_BLOCK = bytes.fromhex(
|
||||
|
||||
|
||||
def test_extension_byte_does_not_inflate_peak():
|
||||
"""The annotation byte at [7]/[11]/[15]/[19] must NOT contribute to
|
||||
the peak count. Decoded T_peak must be 3 (uint8 byte[6]), NOT
|
||||
53763 (uint16 LE byte[6:8])."""
|
||||
"""The byte after each peak must NOT contribute to the peak count.
|
||||
|
||||
Still true, but for a different reason than originally recorded: the
|
||||
block is uniformly **big-endian**, so T_peak is uint16 BE at [5:7]
|
||||
(= 3 here) and the 0xd2 at [7] is the HIGH BYTE of the big-endian
|
||||
half-period at [7:9], not an "annotation" field. Reading the peak as
|
||||
uint16 LE at [6:8] gave 53763 → 268 in/s, which is what this test was
|
||||
written to prevent.
|
||||
"""
|
||||
body = _K558_INTERVAL_12_BLOCK
|
||||
records = decode_histogram_body_full(body)
|
||||
assert records is not None
|
||||
assert len(records) == 1
|
||||
r = records[0]
|
||||
assert r["t_peak"] == 3, f"T_peak should be 3 (uint8), got {r['t_peak']}"
|
||||
assert r["t_peak"] == 3, f"T_peak should be 3, got {r['t_peak']}"
|
||||
assert r["v_peak"] == 2
|
||||
assert r["l_peak"] == 2
|
||||
assert r["m_peak"] == 16
|
||||
# Half-periods unchanged — still uint16 LE.
|
||||
assert r["t_halfp"] == 0x0045 # 69 → 7.4 Hz
|
||||
# Half-period is uint16 BE — 0xd245 = 53829 samples → 0.0095 Hz, which
|
||||
# is exactly the sub-Hz drift BW rendered as "<1.0" for this interval.
|
||||
assert r["t_halfp"] == 0xd245
|
||||
assert half_period_to_hz(r["t_halfp"]) < 1.0
|
||||
assert r["m_halfp"] == 6 # → 85.3 Hz
|
||||
# Annotation byte is preserved (for future RE) but does not affect peak.
|
||||
assert r["annotations"] == (0xd2, 0x00, 0x00, 0x00)
|
||||
|
||||
|
||||
def test_extension_byte_decoded_to_correct_in_s():
|
||||
@@ -383,3 +389,85 @@ def test_extension_byte_decoded_to_correct_in_s():
|
||||
assert channels["Vert"] == [2]
|
||||
assert channels["Long"] == [2]
|
||||
assert channels["MicL"] == [16]
|
||||
|
||||
|
||||
# ── Big-endian block layout + terminal block (2026-08-25) ───────────────────
|
||||
#
|
||||
# Verified against 1211 production histograms paired with their Blastware
|
||||
# ASCII exports: 1211/1211 decode exactly (interval count + every
|
||||
# per-interval peak), and 842,442 per-interval frequency comparisons match
|
||||
# with zero mismatches.
|
||||
|
||||
def _mk_block(t_peak=3, t_halfp=69, v_peak=2, v_halfp=69, l_peak=2, l_halfp=69,
|
||||
m_peak=16, m_halfp=6, seg=0, ctr=256, tail=b"\x1e\x0a\x00\x00",
|
||||
b22=0x00):
|
||||
"""Build one synthetic 32-byte histogram block, big-endian throughout."""
|
||||
b = bytearray(32)
|
||||
b[0] = 0x00
|
||||
b[1] = seg
|
||||
b[2], b[3] = ctr & 0xFF, (ctr >> 8) & 0xFF # block_ctr is uint16 LE
|
||||
b[4] = 0x0A # marker, uint8
|
||||
for off, val in ((5, t_peak), (7, t_halfp), (9, v_peak), (11, v_halfp),
|
||||
(13, l_peak), (15, l_halfp), (17, m_peak), (19, m_halfp)):
|
||||
b[off], b[off + 1] = (val >> 8) & 0xFF, val & 0xFF # uint16 BE
|
||||
b[22] = b22
|
||||
b[28:32] = tail
|
||||
return bytes(b)
|
||||
|
||||
|
||||
def test_geo_peak_is_uint16_be_not_uint8():
|
||||
"""A peak above the uint8 ceiling (255 counts = 1.275 in/s) must decode.
|
||||
|
||||
Real example: BE18193/T193LQ9K.OE0H's final interval reads 8.270 in/s
|
||||
in the BW export = 1654 counts = 0x0676, which needs both bytes.
|
||||
Reading only byte[6] gave 0x76 = 118 = 0.590 in/s.
|
||||
"""
|
||||
r = decode_histogram_body_full(_mk_block(t_peak=1654))
|
||||
assert r is not None and len(r) == 1
|
||||
assert r[0]["t_peak"] == 1654
|
||||
assert geo_count_to_ins(r[0]["t_peak"]) == pytest.approx(8.27)
|
||||
|
||||
|
||||
def test_half_period_is_uint16_be():
|
||||
"""Half-period spans two bytes big-endian; a large value is sub-Hz."""
|
||||
r = decode_histogram_body_full(_mk_block(t_halfp=53829))
|
||||
assert r[0]["t_halfp"] == 53829
|
||||
assert half_period_to_hz(53829) == pytest.approx(512 / 53829)
|
||||
|
||||
|
||||
def test_terminal_block_tail_is_accepted():
|
||||
"""The LAST block of a histogram stream carries tail `9c 06 00 42`
|
||||
instead of `1e 0a 00 00`. Rejecting it dropped the final interval —
|
||||
which is where the event peak often lives. Observed in 1206 of 1211
|
||||
production histograms, always after every standard-tail block."""
|
||||
body = _mk_block(ctr=256) + _mk_block(ctr=257, t_peak=1654,
|
||||
tail=b"\x9c\x06\x00\x42")
|
||||
ch = decode_histogram_body(body)
|
||||
assert ch is not None
|
||||
assert ch["Tran"] == [3, 1654], "terminal block must not be dropped"
|
||||
|
||||
|
||||
def test_terminal_block_exempt_from_byte22_constraint():
|
||||
"""Terminal blocks carry arbitrary bytes at [22:24]; only standard
|
||||
blocks hold 0x00 there. Requiring it dropped the final interval on
|
||||
files such as BE18438/T438LO30.DC0H."""
|
||||
body = _mk_block(ctr=256) + _mk_block(ctr=257, tail=b"\x9c\x06\x00\x42",
|
||||
b22=0x01)
|
||||
ch = decode_histogram_body(body)
|
||||
assert ch is not None and len(ch["Tran"]) == 2
|
||||
|
||||
|
||||
def test_standard_block_still_requires_byte22_zero():
|
||||
"""The [22] == 0x00 constraint is what keeps trailer content out, so it
|
||||
must still apply to standard-tail blocks."""
|
||||
ch = decode_histogram_body(_mk_block(b22=0x01))
|
||||
assert ch is None
|
||||
|
||||
|
||||
def test_marker_is_single_byte_not_uint16():
|
||||
"""block[4] alone is the 0x0A marker. Treating [4:6] as a uint16 LE
|
||||
marker forced block[5] to zero, which capped every geo peak at 255
|
||||
counts — block[5] is the peak's high byte."""
|
||||
r = decode_histogram_body_full(_mk_block(t_peak=0x0676))
|
||||
assert r is not None, "block[5] != 0 must not disqualify the block"
|
||||
assert r[0]["t_peak"] == 0x0676
|
||||
|
||||
Reference in New Issue
Block a user