diff --git a/CHANGELOG.md b/CHANGELOG.md index 179ad17..253e875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,41 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **Series-3 health sweep: 11,603 / 11,603 binaries now clean on every check.** + Swept every series-3 file with the live decoder against five independent + checks — decode exceptions, zero samples, unequal geo channel lengths, peaks + above range full scale, decoded peak vs the device-reported PPV, and waveform + length vs the declared record time. Three real defects surfaced and were + fixed: + + - **`block[22]` is not a constant and must not be tested.** It was documented + as always `0x00` but carries data on loud blocks, and rejecting those threw + away the interval holding the event peak. + `BE18350/T350L7HR.NL0H` block 92 has `block[22]=0x26` and a Tran peak of + `0x0563` = 1379 counts = **6.895 in/s** — exactly the device-reported PPV — + while the file as a whole decoded to 0.015 in/s. `block[0]==0x00`, + `block[4]==0x0A` and the 4-byte tail are six bytes of constraint, which is + what keeps trailer content out. + + - **Block-model dispatch now goes on signature strength, not on whichever + decoder returns first.** A multi-interval body also yields scattered + standard-tail blocks by coincidence; dispatching on "first non-empty" + handed 193 BE18193 files to the standard walker and produced peaks of + 149 in/s against a 10 in/s full scale. + + - **Multi-interval stride detection requires the block counter to increment + by exactly 1.** Without it the detector false-positives on ordinary + standard-block bodies: those carry a header every 32 bytes, and + `192 = 12 + 20×9` and `512 = 12 + 20×25` are both multiples of 32, so a + stride "fits" while actually skipping 6 or 16 real blocks. That misrouted + 9,082 files. + + Partial-block garbage is now trimmed within the final block only, stopping at + the first slot with a non-zero tail word or a geo peak above full scale. + Trimming purely from the end left garbage stranded behind one slot that + happened to have a zero tail word; trimming on the tail word alone truncated + four BE9440 files by up to 2,800 intervals. + - **Sub-minute histogram intervals are packed several to a block — 415 files recovered.** The device always writes one minute of data per block, so a shorter interval just means more intervals in a longer block: diff --git a/minimateplus/histogram_codec.py b/minimateplus/histogram_codec.py index b6746ec..d60f853 100644 --- a/minimateplus/histogram_codec.py +++ b/minimateplus/histogram_codec.py @@ -170,10 +170,14 @@ def _is_data_block(block: bytes) -> bool: return False if block[4] != _BLOCK_MARKER: return False - tail = block[28:32] - if tail == _BLOCK_TAIL: - return block[22] == 0x00 - return tail == _BLOCK_TAIL_TERMINAL + # The 4-byte tail plus block[0]==0 and block[4]==0x0A is already six bytes + # of constraint — enough to keep trailer content out. There is NO extra + # test on block[22]: it was documented as a constant 0x00 but carries data + # on loud blocks, and rejecting those threw away the interval holding the + # event peak. BE18350/T350L7HR.NL0H is the proof: its block 92 has + # block[22]=0x26 and a Tran peak of 0x0563 = 1379 counts = 6.895 in/s, + # exactly the device-reported PPV, while the file decoded to 0.015 in/s. + return block[28:32] in (_BLOCK_TAIL, _BLOCK_TAIL_TERMINAL) def _decode_block(block: bytes) -> Optional[dict]: @@ -248,6 +252,23 @@ def walk_body(body: bytes) -> List[dict]: return records +def _walk_auto(body: bytes) -> List[dict]: + """Pick the block model by signature strength, not by which returns first. + + The multi-interval variant announces itself with consecutive block headers + at an exact ``12 + 20*n`` stride — far stronger evidence than a handful of + scattered standard-tail blocks, which a multi-interval body will also yield + by coincidence. Dispatching on "whichever decoder returns something" + handed 193 BE18193 files to the standard walker and produced peaks of + 149 in/s against a 10 in/s full scale. + """ + if detect_multi_interval_stride(body): + recs = walk_multi_interval_blocks(body) + if recs: + return recs + return walk_body(body) + + def decode_histogram_body(body: bytes) -> Optional[dict]: """Decode a histogram-mode body into per-channel peak-sample arrays. @@ -263,7 +284,7 @@ def decode_histogram_body(body: bytes) -> Optional[dict]: to get 1-count ADC values, then ``count / 32767 * 10.0`` for in/s) - Mic channel: use ``waveform_codec.mic_count_to_db(count)`` """ - records = walk_body(body) or walk_multi_interval_blocks(body) + records = _walk_auto(body) if not records: return None return { @@ -285,7 +306,7 @@ def decode_histogram_body_full(body: bytes) -> Optional[List[dict]]: Returns ``None`` if the body has no valid blocks. """ - records = walk_body(body) or walk_multi_interval_blocks(body) + records = _walk_auto(body) return records if records else None @@ -353,6 +374,10 @@ _MULTI_TRAILER_LEN = 6 _MULTI_MIN_RECORDS = 2 _MULTI_MAX_RECORDS = 64 +# Geo full scale in 16-count units: 10.000 in/s / 0.005 = 2000. A peak above +# this is physically impossible and marks buffer garbage in a partial block. +_GEO_MAX_COUNTS = 2000 + def _is_multi_header(body: bytes, off: int) -> bool: return (off + _MULTI_HEADER_LEN <= len(body) @@ -376,9 +401,24 @@ def detect_multi_interval_stride(body: bytes) -> Optional[int]: continue if not _is_multi_header(body, stride): continue - # confirm on a third block when the body is long enough - if 2 * stride + _MULTI_HEADER_LEN <= len(body) and not _is_multi_header(body, 2 * stride): + # DECISIVE CHECK: consecutive blocks differ by exactly 1 in block_ctr. + # Without it this false-positives on ordinary standard-block bodies: + # those carry a header every 32 bytes, and 192 = 12 + 20*9 and + # 512 = 12 + 20*25 are both multiples of 32, so a stride "fits" while + # actually skipping 6 or 16 real blocks. Sampling a standard body at + # stride 192 handed 9,082 files to the wrong decoder and produced peaks + # of 149 in/s against a 10 in/s full scale. + def _ctr(o: int) -> int: + return body[o + 2] | (body[o + 3] << 8) + + if (_ctr(stride) - _ctr(0)) & 0xFFFF != 1: continue + # confirm on a third block when the body is long enough + if 2 * stride + _MULTI_HEADER_LEN <= len(body): + if not _is_multi_header(body, 2 * stride): + continue + if (_ctr(2 * stride) - _ctr(stride)) & 0xFFFF != 1: + continue return stride return None @@ -403,13 +443,8 @@ def walk_multi_interval_blocks(body: bytes, break # end of the block run; trailer follows for k in range(n_per_block): q = off + _MULTI_HEADER_LEN + _MULTI_RECORD_LEN * k - # The first word of each record's 2-word tail is 0x0000 on every - # real interval. A session ending mid-block leaves the remaining - # slots filled with whatever was in the buffer; emitting those - # produced peaks thousands of times the device-reported PPV. - if u16le(q + 16) != 0: - return out out.append({ + "_tail0": u16le(q + 16), "segment_id": body[off + 1], "block_ctr": u16le(off + 2), "t_peak": u16le(q), "t_halfp": u16le(q + 2), @@ -419,4 +454,26 @@ def walk_multi_interval_blocks(body: bytes, "meta_var": bytes(body[q + 16:q + 20]), "is_terminal": False, }) + # A session ending mid-block leaves the remaining slots of the FINAL block + # filled with whatever was in the buffer. Those decoded as peaks thousands + # of times the device-reported PPV, so they have to go — but only from the + # final block: a non-zero tail word occurs mid-file on real intervals, and + # trimming on that alone truncated four BE9440 files by up to 2,800 + # intervals, while trimming purely from the end left garbage stranded + # behind one slot that happened to have a zero tail word. + # + # Within the final block, stop at the first slot that is not plausibly + # real: a non-zero tail word, or a geo peak above full scale. 16-count + # units put Normal-range full scale (10.000 in/s) at 2000 counts, so + # anything beyond that is physically impossible. + if out: + last_block_start = ((len(out) - 1) // n_per_block) * n_per_block + for i in range(last_block_start, len(out)): + r = out[i] + if (r["_tail0"] != 0 + or max(r["t_peak"], r["v_peak"], r["l_peak"]) > _GEO_MAX_COUNTS): + del out[i:] + break + for r in out: + r.pop("_tail0", None) return out diff --git a/tests/test_histogram_codec.py b/tests/test_histogram_codec.py index 9d24817..558a0c9 100644 --- a/tests/test_histogram_codec.py +++ b/tests/test_histogram_codec.py @@ -457,11 +457,22 @@ def test_terminal_block_exempt_from_byte22_constraint(): assert ch is not None and len(ch["Tran"]) == 2 -def test_standard_block_still_requires_byte22_zero(): - """The [22] == 0x00 constraint is what keeps trailer content out, so it - must still apply to standard-tail blocks.""" - ch = decode_histogram_body(_mk_block(b22=0x01)) - assert ch is None +def test_standard_block_accepts_nonzero_byte22(): + """block[22] is NOT a constant and must not be tested. + + It was documented as always 0x00, but it carries data on loud blocks. + Rejecting those threw away the interval holding the event peak: + BE18350/T350L7HR.NL0H block 92 has block[22]=0x26 and a Tran peak of + 0x0563 = 1379 counts = 6.895 in/s — exactly the device-reported PPV — + while the file as a whole decoded to 0.015 in/s. + + block[0]==0x00, block[4]==0x0A and the 4-byte tail are six bytes of + constraint, which is what keeps trailer content out. + """ + ch = decode_histogram_body(_mk_block(t_peak=1379, b22=0x26)) + assert ch is not None + assert ch["Tran"] == [1379] + assert geo_count_to_ins(ch["Tran"][0]) == pytest.approx(6.895) def test_marker_is_single_byte_not_uint16():