CHANGELOG.md

@@ -121,6 +121,65 @@ All notable changes to seismo-relay are documented here.
 
 ---
 
+## v0.14.0 — 2026-05-02
+
+### Changed (major rewrite)
+
+- **`read_bulk_waveform_stream` — STRT-bounded chunk walk.**  Replaces the
+  earlier `0x0400`-step / `max(key4[2:4], 0x0400)` chunk-counter formula,
+  which over-read ~5× past the actual event end into post-event circular-
+  buffer garbage.  The new walk:
+
+    1. Probe at `counter = start_offset` (event 1: `0x0000`; event N:
+       `cur_key[2:4]`).
+    2. Parse `end_offset` from the STRT record at `data[17]` of the probe
+       response (`end_key[2:4]` field).
+    3. For event 1 only, read the two fixed metadata pages at counter
+       `0x1002` and `0x1004` — these contain the global session-start
+       compliance setup (Project / Client / User Name / Seis Loc /
+       Extended Notes ASCII strings).  Continuation events skip these
+       (BW caches them across the session).
+    4. Walk sample chunks at **`0x0200` increments (NOT `0x0400`)**, bounded
+       by `end_offset` — the loop exits when
+       `next_chunk_counter + 0x0200 > end_offset`.
+    5. Send the proper TERM frame (see new `bulk_waveform_term_v2()`) with
+       `offset_word = end_offset - next_boundary` and
+       `params[2:4] = next_boundary BE`.  The TERM response carries the
+       partial last chunk + 26-byte file footer.
+
+- **New helpers:** `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`
+  and `parse_strt_end_offset(a5_data)` in `minimateplus.framing`.
+
+- **`stop_after_metadata` / `extra_chunks_after_metadata` kwargs are now
+  no-ops** under the v0.14.x walk.  They are retained on the
+  `read_bulk_waveform_stream` signature for backward compatibility but log a
+  DEBUG line when set.  The old "scan for `b'Project:'` and stop one chunk
+  later" workaround is obsolete — the loop is deterministically bounded by
+  the STRT-derived `end_offset`.
+
+- **Project / Client / User Name / Seis Loc string source corrected.**
+  These come from the dedicated metadata pages at counter `0x1002` /
+  `0x1004`, not from "A5 frame 7" of the sample-chunk stream.  The
+  earlier "A5 frame 7" claim was an artifact of the broken `0x0400`-step
+  walk where the bad counter formula coincidentally landed sample-chunk
+  fi=7 on top of the 0x1002 metadata page.
+
+### Verified
+
+- Three independent BW MITM captures (4-27-26 + 5-1-26 + 5-4-26) confirm
+  the new walk matches BW's behaviour event-for-event.
+- `end_offset` values verified across 3 events: `0x1ABE` (4-27-26 2-sec),
+  `0x21F2` (5-1-26 3-sec), `0x417E` (5-1-26 event-2).
+
+### Notes
+
+- Earlier v0.13.0 / v0.13.1 / v0.13.2 entries describe partial steps along
+  the way (some of the file builder fixes, filename bugs, etc.) that were
+  superseded by the full rewrite.  Treat this v0.14.0 entry as the
+  definitive landing point for the corrected SUB 5A protocol.
+
+---
+
 ## v0.14.1 — 2026-05-04
 
 ### Fixed

README.md

@@ -1,4 +1,4 @@
-# seismo-relay  `v0.14.3`
+# seismo-relay  `v0.15.0`
 
 A ground-up replacement for **Blastware** — Instantel's aging Windows-only
 software for managing MiniMate Plus seismographs.
@@ -14,7 +14,11 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
 > byte-perfect against Blastware captures across 2-sec, 3-sec, and 10-sec
 > events.** Generated `.G10` / `.AB0` files open cleanly in Blastware with
 > full Event Reports, frequency analysis, and waveform plots.
-> See [CHANGELOG.md](CHANGELOG.md) for full version history.
+> **v0.15.0 (2026-05-07)** adds layered per-event storage (BW binary +
+> raw 5A pickle + HDF5 + `.sfm.json` sidecar), a plot-ready
+> `sfm.plot.v1` JSON shape with server-side ADC-to-physical-units
+> conversion, and a BW-file importer for ingesting externally-produced
+> events.  See [CHANGELOG.md](CHANGELOG.md) for full version history.
 
 ---
 

minimateplus/blastware_file.py

@@ -639,7 +639,7 @@ def write_blastware_file(
         strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF])
         probe_skip = 7 + 21
 
-    log.warning(
+    log.debug(
         "write_blastware_file: strt_pos_stripped=%d  probe_skip=%d  "
         "probe_data_len=%d  strt_hex=%s",
         strt_pos_stripped if strt_pos_stripped >= 0 else -1,
@@ -708,7 +708,7 @@ def write_blastware_file(
             skip = 12   # sample chunks
 
         contribution = _frame_body_bytes(frame, skip)
-        log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
+        log.debug("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
                   fi, skip, len(frame.data), len(contribution))
         all_bytes.extend(contribution)
 
@@ -717,7 +717,7 @@ def write_blastware_file(
     # one shorter than chunk frames' 5-byte inner header.  Confirmed 2026-04-21.
     if term_frame is not None:
         term_contribution = _frame_body_bytes(term_frame, 11)
-        log.warning(
+        log.debug(
             "write_blastware_file: term_frame data_len=%d  skip=11  "
             "contribution_len=%d  first8=%s",
             len(term_frame.data),
@@ -726,7 +726,7 @@ def write_blastware_file(
         )
         all_bytes.extend(term_contribution)
 
-    log.warning(
+    log.debug(
         "write_blastware_file: all_bytes total=%d  last28=%s",
         len(all_bytes),
         bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(),
@@ -760,7 +760,7 @@ def write_blastware_file(
     if footer_pos >= 0:
         body   = bytes(all_bytes[:footer_pos])
         footer = bytes(all_bytes[footer_pos:footer_pos + 26])
-        log.warning(
+        log.debug(
             "write_blastware_file: real 0e 08 footer at all_bytes[%d]; "
             "truncating %d post-footer bytes",
             footer_pos, len(all_bytes) - footer_pos - 26,

minimateplus/framing.py

@@ -111,14 +111,15 @@ def build_5a_frame(offset_word: int, raw_params: bytes) -> bytes:
     verified against this algorithm on 2026-04-02).
 
     Args:
-        offset_word: 16-bit offset (0x1004 for probe/chunks, 0x005A for term).
+        offset_word: 16-bit offset.  For probe/chunks/metadata pages this is
-        raw_params:  10 or 11 params bytes (from bulk_waveform_params or
+                     `0x1002`.  For the proper TERM frame this is computed by
-                     bulk_waveform_term_params). 0x10 bytes in params are
+                     `bulk_waveform_term_v2()` from the STRT-derived
-                     written RAW — NOT DLE-stuffed. Confirmed 2026-04-06 by
+                     `end_offset`.
-                     comparing wire bytes: BW sends bare `10 04` for chunk 1
+        raw_params:  10, 11, or 12 params bytes (from `bulk_waveform_params`
-                     (counter=0x1004), not stuffed `10 10 04`. Device reads
+                     for probes/samples, `bulk_waveform_term_v2` for TERM, or
-                     params at fixed byte positions; stuffing shifts the bytes
+                     a manually-built 12-byte block for the metadata pages
-                     and corrupts the counter, causing device to ignore the frame.
+                     0x1002 / 0x1004).  See gotcha #3 below — params region
+                     uses partial DLE stuffing of 0x10 bytes.
 
     Returns:
         Complete frame bytes: [ACK][STX][stuffed_section][chk][ETX]
@@ -433,21 +434,26 @@ def bulk_waveform_params(key4: bytes, counter: int, *, is_probe: bool = False) -
 
 def bulk_waveform_term_params(key4: bytes, counter: int) -> bytes:
     """
-    DEPRECATED 2026-05-01 — see bulk_waveform_term_v2().
+    ⛔ DEPRECATED — DO NOT USE IN NEW CODE.
 
-    Build the 10-byte params block for the SUB 5A termination request, OLD layout
+    This is the v1 termination params helper, paired with the broken
-    (used in conjunction with the fixed offset_word=0x005A).  Kept for backward
+    `_BULK_TERM_OFFSET = 0x005A` magic offset_word.  Together they produce a
-    compatibility — produces a tiny ~100-byte device-side terminator response
+    ~100-byte device-side terminator response that does NOT contain the
-    rather than the proper partial-last-chunk + footer payload that BW gets.
+    partial-last-chunk waveform tail or the 26-byte file footer.  Files
+    reconstructed using this terminator are missing their last ~512 bytes of
+    waveform data and have a synthesized footer that disagrees with what BW
+    would have written.
 
-      params[0]   = key4[0]
+    **For new code, use `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`**
-      params[1]   = key4[1]
+    which computes the correct offset_word + params from the STRT-derived
-      params[2]   = (counter >> 8) & 0xFF
+    `end_offset`.  v2 produces wire bytes that match BW exactly across all
-      params[3:]  = zeros
+    tested events (4-27-26 / 5-1-26 / 5-4-26 captures).
 
-    Use bulk_waveform_term_v2() for new code — it computes the verified
+    This function is retained ONLY for the defensive fallback path in
-    offset_word + params from end_offset (extracted from STRT) and the last
+    `read_bulk_waveform_stream()` that triggers when STRT parsing fails or no
-    chunk counter.
+    chunks are fetched (= a malformed event or an unexpected device state).
+    The fallback already logs a WARNING when it activates; if you see that
+    warning, the bug is upstream — STRT should have been parseable.
     """
     if len(key4) != 4:
         raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")

minimateplus/protocol.py

@@ -937,7 +937,7 @@ class MiniMateProtocol:
                 continue
 
             chunk = data_rsp.data[11:]
-            log.warning(
+            log.debug(
                 "read_compliance_config: frame %s  page=0x%04X  data=%d  cfg_chunk=%d  running_total=%d",
                 step_name, data_rsp.page_key, len(data_rsp.data),
                 len(chunk), len(config) + len(chunk),
@@ -957,17 +957,18 @@ class MiniMateProtocol:
         except TimeoutError:
             pass
 
-        log.warning(
+        log.info(
             "read_compliance_config: done — %d cfg bytes total",
             len(config),
         )
 
-        # Hex dump first 128 bytes for field mapping
+        # Hex dump first 128 bytes — useful only for field-mapping work, not normal operation.
+        if log.isEnabledFor(logging.DEBUG):
             for row in range(0, min(len(config), 128), 16):
                 row_bytes = bytes(config[row:row + 16])
                 hex_part = ' '.join(f'{b:02x}' for b in row_bytes)
                 asc_part = ''.join(chr(b) if 32 <= b < 127 else '.' for b in row_bytes)
-            log.warning("  cfg[%04x]: %-48s  %s", row, hex_part, asc_part)
+                log.debug("  cfg[%04x]: %-48s  %s", row, hex_part, asc_part)
 
         return bytes(config)
 

tests/test_5a_protocol.py

@@ -0,0 +1,252 @@
+"""
+test_5a_protocol.py — Regression test for the v0.14.x SUB 5A protocol fixes.
+
+Verifies that SFM's framing helpers reproduce Blastware's exact wire bytes
+for every 5A request frame in the 5-1-26 "bwcap3sec" capture, AND that the
+file builder produces a byte-identical file when fed the BW capture's A5
+responses.
+
+Together these two tests protect all four v0.14.x fixes:
+
+  v0.14.0 — STRT-bounded chunk walk (probe @ 0, metadata pages @ 0x1002 +
+            0x1004, samples @ 0x0600..0x1E00 step 0x0200, TERM at residual)
+  v0.14.1 — event-N probe counter is `start_offset`, not `start_offset+0x46`
+            (covered by the multi-event captures, not this 3-sec event-1
+            capture — but the helpers are the same code path)
+  v0.14.2 — file body assembly is contiguous concatenation, no de-duplication
+  v0.14.3 — partial DLE stuffing of `0x10` bytes in 5A params (counter=0x1000
+            wire bytes are `10 10 00`, not `10 00`)
+
+If any of these fixes regresses, this test fails immediately with a clear
+byte-level diff.
+
+Run:
+    python -m pytest tests/test_5a_protocol.py -v
+or:
+    python tests/test_5a_protocol.py
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import pytest
+
+# Allow running from the project root without installation
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from minimateplus.framing import (
+    S3FrameParser,
+    build_5a_frame,
+    bulk_waveform_params,
+    bulk_waveform_term_v2,
+)
+
+
+# ── Capture loading ────────────────────────────────────────────────────────────
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Reference BW MITM capture: BW saving a 3-sec event 0 (start_key=01110000,
+# end_offset=0x21F2). 17 5A frames: probe + 2 metadata pages + 13 samples + TERM.
+BW_TX_PATH = os.path.join(
+    ROOT,
+    "bridges/captures/5-1-26/comcheck/bwcap3sec/"
+    "raw_bw_20260501_165723_copy_3sec_waveform_to_disk.bin",
+)
+BW_S3_PATH = os.path.join(
+    ROOT,
+    "bridges/captures/5-1-26/comcheck/bwcap3sec/"
+    "raw_s3_20260501_165723_copy_3sec_waveform_to_disk.bin",
+)
+
+# BW's saved Blastware file for the same event (used for file-builder verification).
+BW_SAVED_FILE = os.path.join(
+    ROOT, "example-events/decode_test/5-1-26/bw/M529LKIQ.G10",
+)
+
+
+def _split_bw_frames(data: bytes) -> list[bytes]:
+    """Split BW TX bytes into individual frames (ACK STX … bare ETX)."""
+    frames: list[bytes] = []
+    i = 0
+    while i < len(data):
+        if data[i] != 0x41 or i + 1 >= len(data) or data[i + 1] != 0x02:
+            i += 1
+            continue
+        j = i + 2
+        while j < len(data):
+            if data[j] == 0x03:
+                break
+            if data[j] == 0x10 and j + 1 < len(data):
+                j += 2
+                continue
+            j += 1
+        if j >= len(data):
+            break
+        frames.append(data[i : j + 1])
+        i = j + 1
+    return frames
+
+
+@pytest.fixture(scope="module")
+def bw_5a_frames() -> list[bytes]:
+    """All 5A frames from the BW TX capture, in wire order."""
+    if not os.path.exists(BW_TX_PATH):
+        pytest.skip(f"BW capture not found: {BW_TX_PATH}")
+    raw = open(BW_TX_PATH, "rb").read()
+    frames = [
+        f for f in _split_bw_frames(raw)
+        if len(f) >= 6 and f[5] == 0x5A   # body[3] == 0x5A (SUB)
+    ]
+    assert len(frames) == 17, f"expected 17 5A frames in capture, got {len(frames)}"
+    return frames
+
+
+@pytest.fixture(scope="module")
+def bw_a5_frames():
+    """All A5 (response) frames from the matching S3 capture."""
+    if not os.path.exists(BW_S3_PATH):
+        pytest.skip(f"BW S3 capture not found: {BW_S3_PATH}")
+    raw = open(BW_S3_PATH, "rb").read()
+    p = S3FrameParser()
+    p.feed(raw)
+    a5 = [f for f in p.frames if f.sub == 0xA5]
+    assert len(a5) == 17, f"expected 17 A5 frames in capture, got {len(a5)}"
+    return a5
+
+
+# ── 5A request frame byte-perfect verification ────────────────────────────────
+
+KEY4 = bytes.fromhex("01110000")   # start_key for the 3-sec event 0
+END_OFFSET = 0x21F2                # parsed from STRT in the BW capture
+LAST_CHUNK_COUNTER = 0x1E00        # last full 0x0200-byte chunk before TERM
+
+SAMPLE_COUNTERS = (
+    0x0600, 0x0800, 0x0A00, 0x0C00, 0x0E00,
+    0x1000, 0x1200, 0x1400, 0x1600, 0x1800,
+    0x1A00, 0x1C00, 0x1E00,
+)
+
+
+def _meta_params(key: bytes, counter: int) -> bytes:
+    """Build the 12-byte metadata-page params block (matches BW for 0x1002 / 0x1004)."""
+    return bytes(
+        [
+            0x00, key[0], key[1],
+            (counter >> 8) & 0xFF, counter & 0xFF,
+            0, 0, 0, 0, 0, 0, 0,
+        ]
+    )
+
+
+def test_probe_frame_byte_perfect(bw_5a_frames):
+    """Probe @ counter=0x0000 (frame 0)."""
+    sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, 0, is_probe=True))
+    assert sfm == bw_5a_frames[0], (
+        f"\nSFM:    {sfm.hex()}\nBW:     {bw_5a_frames[0].hex()}"
+    )
+
+
+@pytest.mark.parametrize("idx,counter", [(1, 0x1002), (2, 0x1004)])
+def test_metadata_page_frames_byte_perfect(bw_5a_frames, idx, counter):
+    """Metadata pages @ counter=0x1002 and 0x1004 (frames 1 and 2)."""
+    sfm = build_5a_frame(0x1002, _meta_params(KEY4, counter))
+    assert sfm == bw_5a_frames[idx], (
+        f"\nSFM:    {sfm.hex()}\nBW:     {bw_5a_frames[idx].hex()}"
+    )
+
+
+@pytest.mark.parametrize("i,counter", list(enumerate(SAMPLE_COUNTERS)))
+def test_sample_chunk_frames_byte_perfect(bw_5a_frames, i, counter):
+    """
+    Sample chunks @ counter=0x0600..0x1E00, step 0x0200 (frames 3..15).
+
+    Critically, frame 8 (counter=0x1000) requires the v0.14.3 partial DLE
+    stuffing fix — wire params include `10 10 00` for the counter, not `10 00`.
+    """
+    sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, counter))
+    bw_idx = 3 + i
+    assert sfm == bw_5a_frames[bw_idx], (
+        f"\ncounter=0x{counter:04X}"
+        f"\nSFM:    {sfm.hex()}"
+        f"\nBW:     {bw_5a_frames[bw_idx].hex()}"
+    )
+
+
+def test_term_frame_byte_perfect(bw_5a_frames):
+    """TERM frame at residual (frame 16)."""
+    offset_word, params = bulk_waveform_term_v2(KEY4, END_OFFSET, LAST_CHUNK_COUNTER)
+    sfm = build_5a_frame(offset_word, params)
+    assert sfm == bw_5a_frames[16], (
+        f"\nSFM:    {sfm.hex()}\nBW:     {bw_5a_frames[16].hex()}"
+    )
+
+
+def test_strt_end_offset_parsing(bw_a5_frames):
+    """The probe response (A5[0]) carries STRT at byte 17 with end_offset=0x21F2."""
+    from minimateplus.framing import parse_strt_end_offset
+
+    end_offset = parse_strt_end_offset(bw_a5_frames[0].data)
+    assert end_offset == END_OFFSET, (
+        f"expected end_offset=0x{END_OFFSET:04X}, got "
+        f"{f'0x{end_offset:04X}' if end_offset is not None else 'None'}"
+    )
+
+
+# ── File builder byte-perfect verification ────────────────────────────────────
+
+def test_blastware_file_builder_byte_perfect(bw_a5_frames):
+    """
+    Feed the BW capture's A5 frames into write_blastware_file() and verify the
+    output is byte-identical to BW's saved M529LKIQ.G10 reference file.
+
+    This protects the v0.14.2 strip-removal fix and the file-builder skip
+    values (probe=38, meta=13, samples=12, TERM=11).
+    """
+    if not os.path.exists(BW_SAVED_FILE):
+        pytest.skip(f"BW saved file not found: {BW_SAVED_FILE}")
+
+    import tempfile
+
+    from minimateplus.blastware_file import write_blastware_file
+    from minimateplus.models import Event
+
+    ev = Event(index=0)
+    ev._waveform_key = KEY4
+    ev.rectime_seconds = 3
+    ev.timestamp = None   # let the builder pull the footer from the TERM frame
+
+    with tempfile.NamedTemporaryFile(suffix=".G10", delete=False) as tf:
+        tmp_path = tf.name
+    try:
+        write_blastware_file(ev, bw_a5_frames, tmp_path)
+        sfm_bytes = open(tmp_path, "rb").read()
+    finally:
+        os.unlink(tmp_path)
+
+    bw_bytes = open(BW_SAVED_FILE, "rb").read()
+
+    assert len(sfm_bytes) == len(bw_bytes), (
+        f"file size mismatch: SFM={len(sfm_bytes)} BW={len(bw_bytes)}"
+    )
+
+    if sfm_bytes != bw_bytes:
+        # Find first diff for actionable error message
+        for i in range(len(bw_bytes)):
+            if bw_bytes[i] != sfm_bytes[i]:
+                ctx_start = max(0, i - 8)
+                ctx_end = min(len(bw_bytes), i + 16)
+                pytest.fail(
+                    f"file diverges at byte 0x{i:04X}\n"
+                    f"  BW :  {bw_bytes[ctx_start:ctx_end].hex()}\n"
+                    f"  SFM:  {sfm_bytes[ctx_start:ctx_end].hex()}\n"
+                    f"          {'  ' * (i - ctx_start)}^^"
+                )
+
+
+# ── Standalone runner ─────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))