feat(histogram): decode multi-interval blocks — recovers 415 files

Sub-minute histogram intervals are packed several to a block so that every block still covers exactly one minute of data: interval intervals/block stride 1 minute 1 32 <- the standard big-endian block 15 s 4 92 2 s 30 612 stride = 12 + n * 20 Block = [00][segment][ctr uint16 LE][0a][00], then n x 20-byte records of 8 x uint16 LITTLE-endian values (T_peak, T_halfp, V_peak, V_halfp, L_peak, L_halfp, M_peak, M_halfp) plus a 2-word tail whose first word is 0000 on every real interval, then a 6-byte block trailer. The standard 32-byte block is BIG-endian; this variant is LITTLE-endian. The tail-word check matters: a session ending mid-block leaves buffer garbage in the remaining interval slots, which decoded as peaks thousands of times the real value. Stride detection also requires at least 2 records, since a 1-record block would have stride 32 and collide with the standard block. Recovers 415 files that decoded to nothing: 216 on BE18193 (2 s intervals) and 199 on BE9440 (15 s). Before decoding to nothing they were being accepted by the WAVEFORM codec, which returned garbage peaking up to 400x the device-reported PPV. Ground truth BE9440/K440L3AQ.T70H (5,710 intervals) matches its Blastware ASCII export exactly: 17,130/17,130 geo peaks, 22,840/22,840 frequencies, 5,710/5,710 mic dB(L). Across all 455 affected files, 1,354/1,365 channel peaks (99.2%) match the device-reported PPV; the 11 that don't are under-reads on BE9440 where the walk stops early. Fixture (binary + ASCII) saved under tests/fixtures/, which is gitignored per repo practice — the ground-truth test skips when absent. Tests: 258 passed, failure list unchanged from baseline. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog
2026-08-26 04:59:00 +00:00
parent 4c58a532de
commit 306104354b
5 changed files with 333 additions and 3 deletions
@@ -471,3 +471,164 @@ def test_marker_is_single_byte_not_uint16():
    r = decode_histogram_body_full(_mk_block(t_peak=0x0676))
    assert r is not None, "block[5] != 0 must not disqualify the block"
    assert r[0]["t_peak"] == 0x0676
+
+
+# ── Multi-interval block variant (2026-08-26) ───────────────────────────────
+#
+# When the histogram interval is SHORTER than one minute the device packs
+# several intervals into one block, so that every block still covers exactly
+# one minute of data:
+#
+#     interval  intervals/block  stride
+#     1 min     1                32   (the standard big-endian block)
+#     15 s      4                92
+#     2 s       30               612
+#
+#     stride = 12 + n_intervals * 20
+#
+# Block layout:
+#     [0]      0x00
+#     [1]      segment_id (256 blocks per segment)
+#     [2:4]    block_ctr  uint16 LE
+#     [4]      0x0a marker
+#     [5]      0x00
+#     [6 ...]  n x 20-byte interval records, each 8 x uint16 LITTLE-endian:
+#                  T_peak, T_halfp, V_peak, V_halfp,
+#                  L_peak, L_halfp, M_peak, M_halfp
+#              followed by 2 words (first is 0x0000)
+#     [-6:]    6-byte block trailer
+#
+# NOTE the endianness flip: the standard 32-byte block is big-endian, this
+# variant is little-endian.
+#
+# Ground truth: BE9440/K440L3AQ.T70H (15 s intervals, 5,710 of them) decodes
+# with 17,130/17,130 geo peak counts, 22,840/22,840 frequencies and
+# 5,710/5,710 mic dB(L) values matching its Blastware ASCII export exactly.
+
+from minimateplus.histogram_codec import (        # noqa: E402
+    detect_multi_interval_stride,
+    walk_multi_interval_blocks,
+)
+
+
+def _mk_multi_block(intervals, seg=0, ctr=256):
+    """Build one multi-interval block from a list of 8-tuples."""
+    b = bytearray()
+    b += bytes([0x00, seg])
+    b += int(ctr).to_bytes(2, "little")
+    b += bytes([0x0A, 0x00])
+    for iv in intervals:
+        for v in iv:
+            b += int(v).to_bytes(2, "little")
+        b += (0).to_bytes(2, "little")
+        b += (5).to_bytes(2, "little")
+    b += bytes(6)
+    assert len(b) == 12 + 20 * len(intervals)
+    return bytes(b)
+
+
+def test_stride_is_twelve_plus_twenty_per_interval():
+    for n in (4, 30):
+        ivs = [(1, 1, 2, 2, 3, 3, 4, 4)] * n
+        body = _mk_multi_block(ivs, ctr=256) + _mk_multi_block(ivs, ctr=257)
+        assert detect_multi_interval_stride(body) == 12 + 20 * n
+
+
+def test_multi_interval_block_decodes_all_four_channels():
+    ivs = [(1, 1, 2, 2, 3, 3, 4, 4), (5, 6, 7, 8, 9, 10, 11, 12)]
+    body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257)
+    recs = walk_multi_interval_blocks(body)
+    assert len(recs) == 4
+    r = recs[0]
+    assert (r["t_peak"], r["v_peak"], r["l_peak"], r["m_peak"]) == (1, 2, 3, 4)
+    assert (r["t_halfp"], r["v_halfp"], r["l_halfp"], r["m_halfp"]) == (1, 2, 3, 4)
+    assert recs[1]["t_peak"] == 5 and recs[1]["m_halfp"] == 12
+
+
+def test_multi_interval_values_are_little_endian():
+    """The standard 32-byte block is big-endian; this variant is not.
+
+    A peak of 0x0100 must decode as 256, not 1.
+    """
+    ivs = [(0x0100, 1, 1, 1, 1, 1, 1, 1), (1, 1, 1, 1, 1, 1, 1, 1)]
+    body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257)
+    assert walk_multi_interval_blocks(body)[0]["t_peak"] == 0x0100
+
+
+def test_decode_histogram_body_falls_back_to_the_variant():
+    ivs = [(1, 1, 2, 2, 3, 3, 4, 4)] * 4
+    body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257)
+    ch = decode_histogram_body(body)
+    assert ch is not None
+    assert len(ch["Tran"]) == 8
+    assert ch["Long"][0] == 3
+
+
+def test_standard_blocks_still_take_precedence():
+    """A body of standard 32-byte blocks must not be re-read as the variant."""
+    std = _mk_block(t_peak=7) + _mk_block(t_peak=9, ctr=257,
+                                          tail=b"\x9c\x06\x00\x42")
+    ch = decode_histogram_body(std)
+    assert ch is not None and ch["Tran"] == [7, 9]
+
+
+# ── Ground truth for the multi-interval variant ─────────────────────────────
+# Fixture is gitignored (like the rest of tests/fixtures); skips when absent.
+
+_MULTI_FIXTURE = os.path.join(
+    os.path.dirname(__file__), "fixtures", "histogram-multi-interval",
+    "K440L3AQ.T70H",
+)
+
+
+@pytest.mark.skipif(not os.path.exists(_MULTI_FIXTURE),
+                    reason="multi-interval fixture not present")
+def test_multi_interval_matches_blastware_ascii_exactly():
+    """BE9440/K440L3AQ.T70H — 5,710 intervals at 15 s, 4 per 92-byte block.
+
+    Every geo peak, every frequency and every mic dB(L) in the file matches
+    the Blastware ASCII export: 17,130 / 22,840 / 5,710 values, zero
+    mismatches.  Before this decoder the file produced nothing at all.
+    """
+    import math
+    import re as _re
+    from minimateplus import blastware_file as _bwf
+
+    raw = open(_MULTI_FIXTURE, "rb").read()
+    bs = _bwf._WAVEFORM_HEADER_SIZE + 21
+    pos, fp = bs, -1
+    while True:
+        pos = raw.find(b"\x0e\x08", pos)
+        if pos < 0 or pos + 26 > len(raw):
+            break
+        if 2015 <= ((raw[pos + 4] << 8) | raw[pos + 5]) <= 2050:
+            fp = pos
+            break
+        pos += 1
+    recs = walk_multi_interval_blocks(raw[bs:fp])
+
+    rows = []
+    for line in open(_MULTI_FIXTURE + "_ASCII.TXT", errors="replace"):
+        p = [x.strip() for x in line.split("\t")]
+        if len(p) >= 11 and _re.match(r"^\d{2}:\d{2}:\d{2}$", p[0]):
+            rows.append(p)
+
+    assert len(recs) == len(rows) == 5710
+
+    def want_count(x):
+        return round(float(x) / 0.005)
+
+    for rec, row in zip(recs, rows):
+        assert rec["t_peak"] == want_count(row[1])
+        assert rec["v_peak"] == want_count(row[3])
+        assert rec["l_peak"] == want_count(row[5])
+        # mic dB(L)
+        assert abs((81.94 + 20 * math.log10(rec["m_peak"])) - float(row[9])) <= 0.06
+        # frequency: half-period <= 5 is BW's ">100 Hz" sentinel
+        for hp, cell in ((rec["t_halfp"], row[2]), (rec["v_halfp"], row[4]),
+                         (rec["l_halfp"], row[6]), (rec["m_halfp"], row[10])):
+            hz = None if hp <= 5 else 512.0 / hp
+            if cell.startswith(">"):
+                assert hz is None
+            elif not cell.startswith("<"):
+                assert hz is not None and abs(hz - float(cell)) <= max(0.55, float(cell) * 0.02)