seismo-relay/scratch/mm_frame_parse.py

#!/usr/bin/env python3
"""
mm_frame_parse.py — parse Micromate (Series IV) frames out of a seismo_lab
raw capture pair.

Why this exists
---------------
`minimateplus.framing.S3FrameParser` cannot see Micromate traffic.  It locates
frames by scanning for `DLE STX`, and a Micromate response has **no leading
DLE** — it starts at a bare `STX`.  It also expects `payload[1] == 0x10`, where
the Micromate sends `0xC5` (Blastware firmware) or `0x03` (Thor firmware).

The practical consequence, seen on the 9-24-26 setup-push capture: the
Blastware-side requests parse fine (Thor emits Series III request frames), but
**every device response is silently dropped or mis-framed** — so a capture that
actually contains 12 acked writes looks like 12 unanswered requests.

Destuffing
----------
One rule covers both directions: after the leading doubled `BW_CMD`, every
`10 XX` pair on the wire destuffs to `XX`.  That includes `10 03` — Thor
escapes literal `0x03` bytes in write data so they are not mistaken for ETX,
exactly as Blastware does.

A Micromate escapes exactly four byte values — `0x02 0x03 0x04 0x10` — and
nothing else, which is what makes the uniform rule exact rather than merely
convenient.  Established by re-stuffing all 502 captured frames and comparing
to the wire: 251/251 each direction, where `{0x10}` alone gets 130 and 177.

⚠ Checksum, corrected 2026-09-27
--------------------------------
With uniform destuffing the checksum is **plain SUM8 of the destuffed
payload**.  This script used to try SUM8 *and* a "DLE-aware" variant that
excludes `0x10` bytes, and report whichever matched — which is why it never
flagged a bad frame and why the protocol reference carried the wrong rule for
two days.  The DLE-aware form belongs with *Series III* destuffing, where an
escaped byte survives as two bytes; applying it after uniform destuffing
subtracts the correction twice and disagrees with the wire on 55 of 251
responses.

The lesson generalises: a tool that accepts any of N candidate rules cannot
falsify any of them.  It now validates against SUM8 alone, and reports
`DLE-aware` only to name what a mismatch *would* have been — never as a pass.
See `docs/micromate_protocol_reference.md` → *Checksum*, and
`tests/test_micromate_framing.py`, which pins it.

`micromate/framing.py` is the production implementation; this stays as the
one-shot capture-inspection tool.

Usage
-----
    python scratch/mm_frame_parse.py <capture-dir>
    python scratch/mm_frame_parse.py <raw_bw.bin> <raw_s3.bin>
    python scratch/mm_frame_parse.py <capture-dir> --dump 0x71
"""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

DLE, STX, ETX, ACK = 0x10, 0x02, 0x03, 0x41

# Request SUB -> short name.  Series III names where they carry over; the
# Series IV additions are marked.
SUBNAME = {
    0x01: "DEVICE_INFO",
    0x06: "STORAGE_RANGE",
    0x08: "EVENT_INDEX",
    0x0A: "WAVEFORM_HDR",
    0x0C: "WAVEFORM_REC",
    0x15: "SERIAL",
    0x1A: "COMPLIANCE_CFG",
    0x1C: "MONITOR_STATUS",
    0x1E: "EVENT_HDR",
    0x2C: "CALLHOME_CFG",
    0x2E: "TRIGGER_CFG_READ",     # Series IV
    0x3E: "OPERATOR",
    0x41: "SETUP_NAME_READ",      # Series IV
    0x5A: "BULK_DOWNLOAD",
    0x5B: "POLL",
    0x68: "EVENT_INDEX_WRITE",
    0x69: "WAVEFORM_WRITE",
    0x71: "COMPLIANCE_WRITE",
    0x72: "CONFIRM_A",
    0x73: "CONFIRM_B",
    0x74: "CONFIRM_C",
    0x82: "TRIGGER_WRITE",
    0x83: "TRIGGER_CONFIRM",
    0xDA: "SETUP_FILE_DECL",      # Series IV — names the target .MMB
    0xFE: "FULL_CFG",
}


def destuff(blob: bytes, start: int, *, is_request: bool) -> tuple[bytes, int, int]:
    """Destuff one frame starting at `start`.

    Returns (payload, checksum, index_of_terminating_ETX).  `payload` excludes
    the trailing checksum byte.  A request frame opens `ACK STX 10 10`; a
    response opens with a bare `STX`.
    """
    i = start + (2 if is_request else 1)
    out = bytearray()
    if is_request:
        # The doubled BW_CMD is the one guaranteed stuffed byte.
        if blob[i : i + 2] != bytes([DLE, DLE]):
            raise ValueError(f"@0x{start:04x}: request does not open with 10 10")
        out.append(DLE)
        i += 2
    while i < len(blob):
        b = blob[i]
        if b == DLE and i + 1 < len(blob):
            out.append(blob[i + 1])
            i += 2
            continue
        if b == ETX:
            break
        out.append(b)
        i += 1
    if len(out) < 2:
        raise ValueError(f"@0x{start:04x}: frame too short")
    return bytes(out[:-1]), out[-1], i


def frames(blob: bytes, *, is_request: bool):
    """Yield (offset, payload, chk, checksum_kind)."""
    i, n = 0, len(blob)
    while i < n:
        if is_request:
            if not (blob[i] == ACK and i + 1 < n and blob[i + 1] == STX):
                i += 1
                continue
        elif blob[i] != STX:
            i += 1
            continue
        try:
            payload, chk, end = destuff(blob, i, is_request=is_request)
        except ValueError:
            i += 1
            continue
        # SUM8 of the destuffed payload is THE rule -- 502/502 captured frames.
        # The DLE-aware variant is reported only to name a near-miss; it is
        # never a pass.  See the module docstring.
        if (sum(payload) & 0xFF) == chk:
            kind = "ok"
        elif (sum(b for b in payload if b != DLE) & 0xFF) == chk:
            kind = "BAD(dle-aware)"
        else:
            kind = "BAD"
        yield i, payload, chk, kind
        i = end + 1


def describe(payload: bytes, is_request: bool) -> str:
    if len(payload) < 3:
        return "??"
    sub = payload[2]
    if is_request:
        return SUBNAME.get(sub, f"SUB_{sub:02X}")
    req = 0xFF - sub
    return "rsp<-" + SUBNAME.get(req, f"SUB_{req:02X}")


def report(path: Path, *, is_request: bool, dump_sub: int | None) -> None:
    blob = path.read_bytes()
    side = "Thor" if is_request else "unit"
    print(f"== {side:4}  {path.name}  ({len(blob)} bytes)")
    n_bad = 0
    for idx, (off, p, chk, kind) in enumerate(frames(blob, is_request=is_request)):
        if kind == "BAD":
            n_bad += 1
        sub = p[2] if len(p) > 2 else -1
        flags = p[1] if len(p) > 1 else -1
        # Requests carry offset at payload[4:6]; responses page at [3:5].
        word = int.from_bytes(p[4:6] if is_request else p[3:5], "big")
        data = len(p) - 16 if is_request else max(len(p) - 5, 0)
        print(
            f"  [{idx:2}] @0x{off:04x} payload={len(p):5} data={data:5} "
            f"flags=0x{flags:02x} SUB=0x{sub:02x} {describe(p, is_request):18} "
            f"{'offset' if is_request else 'page'}=0x{word:04x} chk={kind}"
        )
        if dump_sub is not None and sub == dump_sub:
            body = p[16:] if is_request else p[5:]
            print(f"       ---- data ({len(body)} bytes) ----")
            for o in range(0, len(body), 16):
                chunk = body[o : o + 16]
                txt = "".join(chr(c) if 32 <= c < 127 else "." for c in chunk)
                print(f"       {o:06x}  {chunk.hex(' '):<47}  |{txt}|")
    print(f"  -- {idx + 1} frames, {n_bad} bad checksum\n")


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("paths", nargs="+",
                    help="a capture directory, or raw_bw.bin and raw_s3.bin")
    ap.add_argument("--dump", default=None,
                    help="hex-dump the data section of this SUB (e.g. 0x71)")
    args = ap.parse_args()

    dump_sub = int(args.dump, 0) if args.dump else None

    if len(args.paths) == 1 and Path(args.paths[0]).is_dir():
        d = Path(args.paths[0])
        bw = sorted(d.glob("raw_bw_*.bin"))
        s3 = sorted(d.glob("raw_s3_*.bin"))
        if not bw or not s3:
            print(f"{d}: need one raw_bw_*.bin and one raw_s3_*.bin", file=sys.stderr)
            return 2
        pairs = [(bw[0], True), (s3[0], False)]
    elif len(args.paths) == 2:
        pairs = [(Path(args.paths[0]), True), (Path(args.paths[1]), False)]
    else:
        ap.error("pass a capture directory, or exactly two .bin files")

    for path, is_request in pairs:
        report(path, is_request=is_request, dump_sub=dump_sub)
    return 0


if __name__ == "__main__":
    sys.exit(main())