fix(series4): Thor/Micromate decoder is now per-sample exact
Verified against Thor's own CSV exports, which carry a per-sample four-column block beside every binary (CSV/<name>.IDFW.csv). Those 1,012 paired files were in the corpus all along; the decoder had been pinned to a superseded walker on the stated grounds that "Thor has no ASCII ground truth in the corpus and its geo scaling is separately suspect". Both premises were false. IDFW per-sample exact 39.1% -> 100.000% (1,057,536/1,057,536) IDFW files fully exact 0/153 -> 153/153 IDFW PPV median error -3.32% -> -0.002% IDFH within 2% of Thor PPV 51.1% -> 100.0% (858/858) prod IDFW, 8 units -3.3% -> -0.001% Four independent root causes: - Geo LSB was 0.0003, the 4-dp *display rounding* of the real 0.000310308 mistaken for the LSB, so every series-4 geophone sample read 3.3% low. Pinned to +-6e-11 by intersecting 991,415 rounding constraints; corroborated by the +-full-scale seed (+-32226) left in unwritten IDFH slots. IDFH had a separate, also wrong, 10.0/32768. - IDFH histograms were capped at 250 intervals: the segment validator required the interval counter's high byte to be zero, but the counter is a uint16 cumulative index, so every segment past interval 255 was rejected. Runs over ~4 hours lost their tail, often the peak. 540/858 corpus files affected. - Record mode 00 00 (raw int16, 10-byte header) was unhandled and fell through the dispatch, silently dropping each channel's first 512 samples -- the long-standing "loud events truncate" symptom. MODE_ABSOLUTE is now also accepted as a segment-0 preamble. - The body-offset search matched 00 02 00 *inside* record headers, selecting a candidate part-way down the chain and decoding a rotation-shifted body. It now anchors on record headers and takes the chain head (6 ms/file). Also fixes the separately tracked "UM-series decodes ~1000x low" bug. Series-3 re-verified unchanged at 14,338/14,338 exact after the shared waveform_codec change. Known open: 41/575 prod IDFW files (7%, mostly UM12947/UM20147) decode with unequal channel lengths and also fail metadata extraction -- a different header variant with no Thor export in the store. NOTE: this is a codec change; the Thor store owes a regeneration via scripts/backfill_thor_events.py. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ru8Lg9HkkYvX9VWWo65SmL
This commit is contained in:
Claude Opus 5
+162
-34
@@ -47,19 +47,24 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
# Thor IDFW bodies are pinned to the SUPERSEDED tag-dispatch decoder.
|
||||
# Thor IDFW bodies use the series-3 record-chain decoder.
|
||||
#
|
||||
# _find_waveform_body_offset() trial-decodes every candidate offset and keeps
|
||||
# whichever yields the most samples. The series-3 record-chain decoder
|
||||
# correctly returns None where the legacy walker returned garbage, which
|
||||
# changes that heuristic's winner on 33 of 577 files. The net effect measured
|
||||
# 2026-08-25 was positive (all-channels-equal 8/577 -> 506/577, mean abs PPV
|
||||
# error 0.228 -> 0.173 in/s) but Thor has no ASCII ground truth in the corpus
|
||||
# and its geo scaling is separately suspect, so the switch is deferred until
|
||||
# the body-offset search is reworked to use the record chain directly.
|
||||
from minimateplus.waveform_codec import (
|
||||
decode_waveform_legacy as decode_waveform_v2,
|
||||
)
|
||||
# This was previously pinned to the SUPERSEDED tag-dispatch walker
|
||||
# (`decode_waveform_legacy`) on the stated grounds that "Thor has no ASCII
|
||||
# ground truth in the corpus and its geo scaling is separately suspect".
|
||||
# Both premises were false: Thor writes a per-sample CSV export next to every
|
||||
# binary (see scratch/verify_thor_against_csv.py), and the scaling is now
|
||||
# resolved (see _GEO_LSB_IPS). Measured against that ground truth on
|
||||
# 2026-09-10, the record chain beats the legacy walker outright:
|
||||
#
|
||||
# channel truncation 55/153 files -> 3/153
|
||||
# files exact 98/153 -> 150/153
|
||||
# per-sample exact 99.781% -> 99.854%
|
||||
#
|
||||
# The legacy walker stops at the first unrecognised tag and returns whatever
|
||||
# channels it had, so its failure mode is silent short channels rather than an
|
||||
# error. Do not re-pin it.
|
||||
from minimateplus.waveform_codec import _MODES, decode_waveform_v2, is_record
|
||||
|
||||
from .models import IdfEvent, IdfPeaks, IdfReport
|
||||
|
||||
@@ -91,9 +96,44 @@ _BODY_MAGIC = b"\x00\x02\x00"
|
||||
# 0x0ae2, 0x0d30 in observed events).
|
||||
_BODY_SCAN_FLOOR = 0x0E00
|
||||
|
||||
# Geophone count → in/s, derived from sidecar ground truth: the smallest
|
||||
# non-zero sample in 1,014-file corpus is 0.0003 in/s.
|
||||
_GEO_LSB_IPS = 0.0003
|
||||
# Cap on trial decodes per file. Chain-head detection normally yields one
|
||||
# or two candidates; the cap only bounds the worst case on a corrupt file.
|
||||
_MAX_BODY_CANDIDATES = 16
|
||||
|
||||
# Geophone count → in/s.
|
||||
#
|
||||
# The old value 0.0003 was read off the smallest non-zero sample in the
|
||||
# sidecar corpus, but that sample is Thor's *4-decimal display rounding* of
|
||||
# the true LSB, not the LSB itself. It read every series-4 geophone sample
|
||||
# 3.3% low. The quantisation ladder gives it away: counts 1..6 export as
|
||||
# 0.0003, 0.0006, 0.0009, 0.0012, 0.0016, 0.0019 — an LSB of exactly 0.0003
|
||||
# would end 0.0015, 0.0018.
|
||||
#
|
||||
# The value below maximises exact 4-dp agreement over 1,046,016 paired
|
||||
# samples (454 channel-events, 2 units) at 99.854%, versus 50.7% for 0.0003.
|
||||
# It is a global constant, not a per-unit calibration: all 8 UM units in the
|
||||
# production store independently agree to within ±0.07% on their
|
||||
# device-reported PPV. 1/LSB = 3222.6 counts per in/s.
|
||||
#
|
||||
# The value is pinned, not guessed. Each exported sample constrains the LSB
|
||||
# to the window that rounds to the printed 4-dp figure; intersecting 991,415
|
||||
# such constraints (clean channel-events only) gives
|
||||
#
|
||||
# LSB in [0.000310307933, 0.000310308057] width 1.2e-10
|
||||
#
|
||||
# 0.000310308 sits at the centre of that window. Equivalent full scale is
|
||||
# 10.0 in/s / 0.000310308 = 32226.05 counts.
|
||||
#
|
||||
# Corroboration from the device: an IDFH interval that never recorded keeps
|
||||
# its min/max accumulator at its ±full-scale seed, and that seed is
|
||||
# (min=+32226, max=-32226) — the same magnitude, independently. Note the
|
||||
# tempting closed form 10.0/32226 is very slightly WRONG: it lands 4.5e-10
|
||||
# above the feasible window and loses 78 boundary samples to the literal
|
||||
# value while never winning one. Series-3 uses 32000 counts for the same
|
||||
# 10.0 in/s, so the two generations do NOT share a scale.
|
||||
#
|
||||
# Ground truth + harness: scratch/verify_thor_against_csv.py
|
||||
_GEO_LSB_IPS = 0.000310308
|
||||
|
||||
# Microphone count → psi, derived from sidecar regression on 50 sample
|
||||
# pairs from UM11719_20231219162723.IDFW (mic-heavy event).
|
||||
@@ -104,8 +144,6 @@ _IDFH_INTERVAL_SIZE = 72 # bytes per per-interval record
|
||||
_IDFH_SEGMENT_HEADER = 10 # bytes: [len_be 2B][0a 00 00 00 4B][00 NN 2B][05 3f 2B]
|
||||
_IDFH_SEGMENT_TAIL = 2 # bytes after the interval data block, before next marker
|
||||
_IDFH_HALFP_FREQ_NUM = 512.0 # freq_hz = NUM / halfp; halfp ≤ 5 means ">100 Hz" sentinel
|
||||
_IDFH_GEO_FULL_SCALE = 10.0 # in/s — Normal range
|
||||
_IDFH_INT16_FS = 32768.0
|
||||
_IDFH_CHANNELS = ("Tran", "Vert", "Long", "MicL")
|
||||
|
||||
|
||||
@@ -223,26 +261,64 @@ def _find_waveform_body_offset(buf: bytes) -> Optional[int]:
|
||||
"""
|
||||
if len(buf) < _BODY_SCAN_FLOOR + 8:
|
||||
return None
|
||||
best: Optional[tuple[int, int]] = None # (total_samples, offset)
|
||||
i = _BODY_SCAN_FLOOR
|
||||
while True:
|
||||
j = buf.find(_BODY_MAGIC, i)
|
||||
if j < 0:
|
||||
break
|
||||
i = j + 1
|
||||
|
||||
# 1. Locate every plausible per-channel record header. A header carries
|
||||
# [len 2B][channel_id][00][00] at +2..+6, so anchor the search on the
|
||||
# three-byte ``<cid> 00 00`` signature and validate with is_record().
|
||||
# Scanning candidate *preambles* instead is not viable: MODE_RAW16 is
|
||||
# ``00 00``, so every run of three zero bytes would look like a body
|
||||
# start and each would cost a full trial decode (~0.5 s/file measured).
|
||||
floor = max(0, _BODY_SCAN_FLOOR - 7)
|
||||
starts: list = []
|
||||
for cid in (0x46, 0x47, 0x48, 0x49):
|
||||
sig = bytes((cid, 0x00, 0x00))
|
||||
i = floor
|
||||
while True:
|
||||
j = buf.find(sig, i)
|
||||
if j < 0:
|
||||
break
|
||||
i = j + 1
|
||||
q = j - 4
|
||||
if q >= floor and is_record(buf, q):
|
||||
starts.append(q)
|
||||
if not starts:
|
||||
return None
|
||||
starts.sort()
|
||||
|
||||
# 2. A body begins at the head of a record chain -- a record that no other
|
||||
# record's length field points at. The head's own payload is the
|
||||
# implicit segment-0 Tran record, and the body offset is head + 7 (past
|
||||
# [len 2B][cid][00][00][seg]) so that body[1:3] lands on the mode.
|
||||
ends = {q + 2 + int.from_bytes(buf[q + 2 : q + 4], "big") for q in starts}
|
||||
heads = [q for q in starts if q not in ends] or starts[:1]
|
||||
|
||||
# 3. Trial-decode each head and keep the best. Prefer a candidate where
|
||||
# all four channels come out the same length: scoring on raw sample
|
||||
# count alone picks false positives sitting *inside* a record header,
|
||||
# which decode a plausible-looking but rotation-shifted body that
|
||||
# silently drops each channel's segment 0.
|
||||
best = None
|
||||
best_off = None
|
||||
for head in heads[:_MAX_BODY_CANDIDATES]:
|
||||
j = head + 7
|
||||
if j + 3 > len(buf) or (buf[j + 1], buf[j + 2]) not in _MODES:
|
||||
continue
|
||||
try:
|
||||
decoded = decode_waveform_v2(buf[j:])
|
||||
except Exception:
|
||||
continue
|
||||
if not decoded:
|
||||
continue
|
||||
lengths = [len(v) for v in decoded.values() if v]
|
||||
total = sum(len(v) for v in decoded.values())
|
||||
# A "real" body has more than just the 2-sample preamble.
|
||||
if total <= 2:
|
||||
continue
|
||||
if best is None or total > best[0]:
|
||||
best = (total, j)
|
||||
return best[1] if best else None
|
||||
equal = len(lengths) == 4 and len(set(lengths)) == 1
|
||||
score = (equal, total)
|
||||
if best is None or score > best:
|
||||
best, best_off = score, j
|
||||
return best_off
|
||||
|
||||
|
||||
def _decode_waveform_samples(buf: bytes) -> Optional[dict]:
|
||||
@@ -307,7 +383,11 @@ class IdfhInterval:
|
||||
|
||||
def peak_ips(self, channel: str) -> float:
|
||||
"""Convert peak count to in/s (geo channels only)."""
|
||||
return self.peak_count(channel) / _IDFH_INT16_FS * _IDFH_GEO_FULL_SCALE
|
||||
# Same geo LSB as the waveform path — verified independently against
|
||||
# the IDFH exports: as peak magnitude rises (and 4-dp quantisation
|
||||
# noise falls) the implied LSB converges on 0.0003103, matching
|
||||
# _GEO_LSB_IPS. The old 10.0/32768 read histogram peaks 1.7% low.
|
||||
return self.peak_count(channel) * _GEO_LSB_IPS
|
||||
|
||||
def freq_hz(self, channel: str) -> Optional[float]:
|
||||
halfp = getattr(self, f"{channel.lower()}_halfp")
|
||||
@@ -316,6 +396,33 @@ class IdfhInterval:
|
||||
return _IDFH_HALFP_FREQ_NUM / halfp
|
||||
|
||||
|
||||
def _is_unwritten_interval(interval: "IdfhInterval") -> bool:
|
||||
"""True for an interval slot the device reserved but never wrote.
|
||||
|
||||
Thor seeds each interval's per-channel accumulators at ``min = +full
|
||||
scale`` and ``max = -full scale`` and then narrows them as samples
|
||||
arrive. A slot that never recorded keeps that seed, so ``min > max`` —
|
||||
impossible for real data. Such a record decodes to a full-scale
|
||||
10.0 in/s peak on every channel and, being a max-over-intervals, poisons
|
||||
the whole file's PPV.
|
||||
|
||||
Rare but real: exactly 1 of 497,611 corpus intervals, and it inflated
|
||||
that file's Long PPV from 0.0081 to 10.0 in/s. The inversion is always
|
||||
all-or-nothing across channels (0 partial cases in the corpus), so
|
||||
requiring every channel to be inverted keeps this from ever firing on
|
||||
genuine data.
|
||||
"""
|
||||
return all(
|
||||
mn > mx
|
||||
for mn, mx in (
|
||||
(interval.tran_min, interval.tran_max),
|
||||
(interval.vert_min, interval.vert_max),
|
||||
(interval.long_min, interval.long_max),
|
||||
(interval.micl_min, interval.micl_max),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _decode_idfh_interval(buf72: bytes, offset: int) -> IdfhInterval:
|
||||
"""Decode one 72-byte interval record into per-channel min/max/halfp."""
|
||||
import struct
|
||||
@@ -343,12 +450,22 @@ def decode_idfh_body(buf: bytes) -> list:
|
||||
"""Walk an IDFH file and decode every interval record.
|
||||
|
||||
The body has one or more segments; each segment header is 12 bytes:
|
||||
``[length_be 2B][0a 00 00 00][00 NN_counter][05 3f]`` where ``length``
|
||||
``[length_be 2B][0a 00 00 00][counter_be 2B][05 3f]`` where ``length``
|
||||
is bytes from the magic through the end of the interval block
|
||||
(= 10 + 72 × n_intervals). Segments are separated by a 2-byte tail
|
||||
+ next-segment 2-byte prefix (the bytes before the next length field).
|
||||
Confirmed against the 859-file corpus (181,071 intervals decoded; 1
|
||||
failure is the sig-B BE9439 file).
|
||||
|
||||
``counter`` is a **uint16 BE cumulative interval index** — the 0-based
|
||||
index of the LAST interval in this segment. Segments carry 10
|
||||
intervals each, so it runs 9, 19, 29, ... across the file.
|
||||
|
||||
⚠ This validator used to require ``buf[j + 4] == 0x00``, i.e. that the
|
||||
counter's high byte was zero. That silently capped every histogram at
|
||||
**250 intervals**: the moment the cumulative counter passed 255 the high
|
||||
byte went non-zero and every later segment was rejected, so any
|
||||
monitoring run longer than ~4 hours lost its tail — frequently the part
|
||||
holding the event peak, which is why those files' PPV read low. 540 of
|
||||
858 corpus files were affected. Do not reinstate that check.
|
||||
"""
|
||||
intervals: list = []
|
||||
i = 0
|
||||
@@ -356,8 +473,9 @@ def decode_idfh_body(buf: bytes) -> list:
|
||||
j = buf.find(b"\x0a\x00\x00\x00", i)
|
||||
if j < 0 or j < 2:
|
||||
break
|
||||
# Validate: [length_be][0a 00 00 00][00 NN][05 3f]
|
||||
if buf[j + 4] != 0x00 or buf[j + 6 : j + 8] != b"\x05\x3f":
|
||||
# Validate: [length_be][0a 00 00 00][counter_be][05 3f]. The counter
|
||||
# is deliberately NOT constrained — see the note above.
|
||||
if buf[j + 6 : j + 8] != b"\x05\x3f":
|
||||
i = j + 1
|
||||
continue
|
||||
length = int.from_bytes(buf[j - 2 : j], "big")
|
||||
@@ -366,13 +484,23 @@ def decode_idfh_body(buf: bytes) -> list:
|
||||
i = j + 1
|
||||
continue
|
||||
header_start = j - 2
|
||||
if header_start + length > len(buf):
|
||||
# Truncated / bogus length — not a real segment header.
|
||||
i = j + 1
|
||||
continue
|
||||
interval_start = header_start + _IDFH_SEGMENT_HEADER
|
||||
for k in range(n):
|
||||
off = interval_start + k * _IDFH_INTERVAL_SIZE
|
||||
if off + _IDFH_INTERVAL_SIZE > len(buf):
|
||||
break
|
||||
chunk = buf[off : off + _IDFH_INTERVAL_SIZE]
|
||||
intervals.append(_decode_idfh_interval(chunk, off))
|
||||
interval = _decode_idfh_interval(chunk, off)
|
||||
if _is_unwritten_interval(interval):
|
||||
# Reserved-but-never-recorded slot: the min/max accumulators
|
||||
# still hold their ±full-scale seed. Counting it would
|
||||
# fabricate a 10.0 in/s peak on every channel.
|
||||
continue
|
||||
intervals.append(interval)
|
||||
# Advance past this segment + the 2-byte tail.
|
||||
i = header_start + length + _IDFH_SEGMENT_TAIL
|
||||
return intervals
|
||||
|
||||
Reference in New Issue
Block a user