From 9400f59167044e5d785c01b8e08be3c6956ee8d1 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 8 May 2026 19:06:26 +0000 Subject: [PATCH 1/2] doc: update readme to 0.15.0 --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 27dda6c..9590923 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# seismo-relay `v0.14.3` +# seismo-relay `v0.15.0` A ground-up replacement for **Blastware** — Instantel's aging Windows-only software for managing MiniMate Plus seismographs. @@ -14,7 +14,11 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). > byte-perfect against Blastware captures across 2-sec, 3-sec, and 10-sec > events.** Generated `.G10` / `.AB0` files open cleanly in Blastware with > full Event Reports, frequency analysis, and waveform plots. -> See [CHANGELOG.md](CHANGELOG.md) for full version history. +> **v0.15.0 (2026-05-07)** adds layered per-event storage (BW binary + +> raw 5A pickle + HDF5 + `.sfm.json` sidecar), a plot-ready +> `sfm.plot.v1` JSON shape with server-side ADC-to-physical-units +> conversion, and a BW-file importer for ingesting externally-produced +> events. See [CHANGELOG.md](CHANGELOG.md) for full version history. --- From 9123269b1f68b321b7c40a49ed47bcc1d0cbf9b7 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Wed, 6 May 2026 14:18:31 -0400 Subject: [PATCH 2/2] feat(protocol): implement v0.14.0 SUB 5A protocol rewrite with enhanced chunk handling and new helpers test: add regression tests for v0.14.x SUB 5A protocol fixes refactor(logging): change warning logs to debug for less verbosity in write_blastware_file --- CHANGELOG.md | 59 ++++++++ minimateplus/blastware_file.py | 12 +- minimateplus/framing.py | 46 +++--- minimateplus/protocol.py | 17 +-- tests/test_5a_protocol.py | 252 +++++++++++++++++++++++++++++++++ 5 files changed, 352 insertions(+), 34 deletions(-) create mode 100644 tests/test_5a_protocol.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5206b2a..85936fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,65 @@ All notable changes to seismo-relay are documented here. --- +## v0.14.0 — 2026-05-02 + +### Changed (major rewrite) + +- **`read_bulk_waveform_stream` — STRT-bounded chunk walk.** Replaces the + earlier `0x0400`-step / `max(key4[2:4], 0x0400)` chunk-counter formula, + which over-read ~5× past the actual event end into post-event circular- + buffer garbage. The new walk: + + 1. Probe at `counter = start_offset` (event 1: `0x0000`; event N: + `cur_key[2:4]`). + 2. Parse `end_offset` from the STRT record at `data[17]` of the probe + response (`end_key[2:4]` field). + 3. For event 1 only, read the two fixed metadata pages at counter + `0x1002` and `0x1004` — these contain the global session-start + compliance setup (Project / Client / User Name / Seis Loc / + Extended Notes ASCII strings). Continuation events skip these + (BW caches them across the session). + 4. Walk sample chunks at **`0x0200` increments (NOT `0x0400`)**, bounded + by `end_offset` — the loop exits when + `next_chunk_counter + 0x0200 > end_offset`. + 5. Send the proper TERM frame (see new `bulk_waveform_term_v2()`) with + `offset_word = end_offset - next_boundary` and + `params[2:4] = next_boundary BE`. The TERM response carries the + partial last chunk + 26-byte file footer. + +- **New helpers:** `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)` + and `parse_strt_end_offset(a5_data)` in `minimateplus.framing`. + +- **`stop_after_metadata` / `extra_chunks_after_metadata` kwargs are now + no-ops** under the v0.14.x walk. They are retained on the + `read_bulk_waveform_stream` signature for backward compatibility but log a + DEBUG line when set. The old "scan for `b'Project:'` and stop one chunk + later" workaround is obsolete — the loop is deterministically bounded by + the STRT-derived `end_offset`. + +- **Project / Client / User Name / Seis Loc string source corrected.** + These come from the dedicated metadata pages at counter `0x1002` / + `0x1004`, not from "A5 frame 7" of the sample-chunk stream. The + earlier "A5 frame 7" claim was an artifact of the broken `0x0400`-step + walk where the bad counter formula coincidentally landed sample-chunk + fi=7 on top of the 0x1002 metadata page. + +### Verified + +- Three independent BW MITM captures (4-27-26 + 5-1-26 + 5-4-26) confirm + the new walk matches BW's behaviour event-for-event. +- `end_offset` values verified across 3 events: `0x1ABE` (4-27-26 2-sec), + `0x21F2` (5-1-26 3-sec), `0x417E` (5-1-26 event-2). + +### Notes + +- Earlier v0.13.0 / v0.13.1 / v0.13.2 entries describe partial steps along + the way (some of the file builder fixes, filename bugs, etc.) that were + superseded by the full rewrite. Treat this v0.14.0 entry as the + definitive landing point for the corrected SUB 5A protocol. + +--- + ## v0.14.1 — 2026-05-04 ### Fixed diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index ee73390..f99a44b 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -639,7 +639,7 @@ def write_blastware_file( strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF]) probe_skip = 7 + 21 - log.warning( + log.debug( "write_blastware_file: strt_pos_stripped=%d probe_skip=%d " "probe_data_len=%d strt_hex=%s", strt_pos_stripped if strt_pos_stripped >= 0 else -1, @@ -708,8 +708,8 @@ def write_blastware_file( skip = 12 # sample chunks contribution = _frame_body_bytes(frame, skip) - log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", - fi, skip, len(frame.data), len(contribution)) + log.debug("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", + fi, skip, len(frame.data), len(contribution)) all_bytes.extend(contribution) # Terminator contributes its content, which ends with the 26-byte footer. @@ -717,7 +717,7 @@ def write_blastware_file( # one shorter than chunk frames' 5-byte inner header. Confirmed 2026-04-21. if term_frame is not None: term_contribution = _frame_body_bytes(term_frame, 11) - log.warning( + log.debug( "write_blastware_file: term_frame data_len=%d skip=11 " "contribution_len=%d first8=%s", len(term_frame.data), @@ -726,7 +726,7 @@ def write_blastware_file( ) all_bytes.extend(term_contribution) - log.warning( + log.debug( "write_blastware_file: all_bytes total=%d last28=%s", len(all_bytes), bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(), @@ -760,7 +760,7 @@ def write_blastware_file( if footer_pos >= 0: body = bytes(all_bytes[:footer_pos]) footer = bytes(all_bytes[footer_pos:footer_pos + 26]) - log.warning( + log.debug( "write_blastware_file: real 0e 08 footer at all_bytes[%d]; " "truncating %d post-footer bytes", footer_pos, len(all_bytes) - footer_pos - 26, diff --git a/minimateplus/framing.py b/minimateplus/framing.py index e26e0f0..e513e23 100644 --- a/minimateplus/framing.py +++ b/minimateplus/framing.py @@ -111,14 +111,15 @@ def build_5a_frame(offset_word: int, raw_params: bytes) -> bytes: verified against this algorithm on 2026-04-02). Args: - offset_word: 16-bit offset (0x1004 for probe/chunks, 0x005A for term). - raw_params: 10 or 11 params bytes (from bulk_waveform_params or - bulk_waveform_term_params). 0x10 bytes in params are - written RAW — NOT DLE-stuffed. Confirmed 2026-04-06 by - comparing wire bytes: BW sends bare `10 04` for chunk 1 - (counter=0x1004), not stuffed `10 10 04`. Device reads - params at fixed byte positions; stuffing shifts the bytes - and corrupts the counter, causing device to ignore the frame. + offset_word: 16-bit offset. For probe/chunks/metadata pages this is + `0x1002`. For the proper TERM frame this is computed by + `bulk_waveform_term_v2()` from the STRT-derived + `end_offset`. + raw_params: 10, 11, or 12 params bytes (from `bulk_waveform_params` + for probes/samples, `bulk_waveform_term_v2` for TERM, or + a manually-built 12-byte block for the metadata pages + 0x1002 / 0x1004). See gotcha #3 below — params region + uses partial DLE stuffing of 0x10 bytes. Returns: Complete frame bytes: [ACK][STX][stuffed_section][chk][ETX] @@ -433,21 +434,26 @@ def bulk_waveform_params(key4: bytes, counter: int, *, is_probe: bool = False) - def bulk_waveform_term_params(key4: bytes, counter: int) -> bytes: """ - DEPRECATED 2026-05-01 — see bulk_waveform_term_v2(). + ⛔ DEPRECATED — DO NOT USE IN NEW CODE. - Build the 10-byte params block for the SUB 5A termination request, OLD layout - (used in conjunction with the fixed offset_word=0x005A). Kept for backward - compatibility — produces a tiny ~100-byte device-side terminator response - rather than the proper partial-last-chunk + footer payload that BW gets. + This is the v1 termination params helper, paired with the broken + `_BULK_TERM_OFFSET = 0x005A` magic offset_word. Together they produce a + ~100-byte device-side terminator response that does NOT contain the + partial-last-chunk waveform tail or the 26-byte file footer. Files + reconstructed using this terminator are missing their last ~512 bytes of + waveform data and have a synthesized footer that disagrees with what BW + would have written. - params[0] = key4[0] - params[1] = key4[1] - params[2] = (counter >> 8) & 0xFF - params[3:] = zeros + **For new code, use `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`** + which computes the correct offset_word + params from the STRT-derived + `end_offset`. v2 produces wire bytes that match BW exactly across all + tested events (4-27-26 / 5-1-26 / 5-4-26 captures). - Use bulk_waveform_term_v2() for new code — it computes the verified - offset_word + params from end_offset (extracted from STRT) and the last - chunk counter. + This function is retained ONLY for the defensive fallback path in + `read_bulk_waveform_stream()` that triggers when STRT parsing fails or no + chunks are fetched (= a malformed event or an unexpected device state). + The fallback already logs a WARNING when it activates; if you see that + warning, the bug is upstream — STRT should have been parseable. """ if len(key4) != 4: raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}") diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 3fb3b05..1c0bdea 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -937,7 +937,7 @@ class MiniMateProtocol: continue chunk = data_rsp.data[11:] - log.warning( + log.debug( "read_compliance_config: frame %s page=0x%04X data=%d cfg_chunk=%d running_total=%d", step_name, data_rsp.page_key, len(data_rsp.data), len(chunk), len(config) + len(chunk), @@ -957,17 +957,18 @@ class MiniMateProtocol: except TimeoutError: pass - log.warning( + log.info( "read_compliance_config: done — %d cfg bytes total", len(config), ) - # Hex dump first 128 bytes for field mapping - for row in range(0, min(len(config), 128), 16): - row_bytes = bytes(config[row:row + 16]) - hex_part = ' '.join(f'{b:02x}' for b in row_bytes) - asc_part = ''.join(chr(b) if 32 <= b < 127 else '.' for b in row_bytes) - log.warning(" cfg[%04x]: %-48s %s", row, hex_part, asc_part) + # Hex dump first 128 bytes — useful only for field-mapping work, not normal operation. + if log.isEnabledFor(logging.DEBUG): + for row in range(0, min(len(config), 128), 16): + row_bytes = bytes(config[row:row + 16]) + hex_part = ' '.join(f'{b:02x}' for b in row_bytes) + asc_part = ''.join(chr(b) if 32 <= b < 127 else '.' for b in row_bytes) + log.debug(" cfg[%04x]: %-48s %s", row, hex_part, asc_part) return bytes(config) diff --git a/tests/test_5a_protocol.py b/tests/test_5a_protocol.py new file mode 100644 index 0000000..bc8beff --- /dev/null +++ b/tests/test_5a_protocol.py @@ -0,0 +1,252 @@ +""" +test_5a_protocol.py — Regression test for the v0.14.x SUB 5A protocol fixes. + +Verifies that SFM's framing helpers reproduce Blastware's exact wire bytes +for every 5A request frame in the 5-1-26 "bwcap3sec" capture, AND that the +file builder produces a byte-identical file when fed the BW capture's A5 +responses. + +Together these two tests protect all four v0.14.x fixes: + + v0.14.0 — STRT-bounded chunk walk (probe @ 0, metadata pages @ 0x1002 + + 0x1004, samples @ 0x0600..0x1E00 step 0x0200, TERM at residual) + v0.14.1 — event-N probe counter is `start_offset`, not `start_offset+0x46` + (covered by the multi-event captures, not this 3-sec event-1 + capture — but the helpers are the same code path) + v0.14.2 — file body assembly is contiguous concatenation, no de-duplication + v0.14.3 — partial DLE stuffing of `0x10` bytes in 5A params (counter=0x1000 + wire bytes are `10 10 00`, not `10 00`) + +If any of these fixes regresses, this test fails immediately with a clear +byte-level diff. + +Run: + python -m pytest tests/test_5a_protocol.py -v +or: + python tests/test_5a_protocol.py +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +# Allow running from the project root without installation +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from minimateplus.framing import ( + S3FrameParser, + build_5a_frame, + bulk_waveform_params, + bulk_waveform_term_v2, +) + + +# ── Capture loading ──────────────────────────────────────────────────────────── + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Reference BW MITM capture: BW saving a 3-sec event 0 (start_key=01110000, +# end_offset=0x21F2). 17 5A frames: probe + 2 metadata pages + 13 samples + TERM. +BW_TX_PATH = os.path.join( + ROOT, + "bridges/captures/5-1-26/comcheck/bwcap3sec/" + "raw_bw_20260501_165723_copy_3sec_waveform_to_disk.bin", +) +BW_S3_PATH = os.path.join( + ROOT, + "bridges/captures/5-1-26/comcheck/bwcap3sec/" + "raw_s3_20260501_165723_copy_3sec_waveform_to_disk.bin", +) + +# BW's saved Blastware file for the same event (used for file-builder verification). +BW_SAVED_FILE = os.path.join( + ROOT, "example-events/decode_test/5-1-26/bw/M529LKIQ.G10", +) + + +def _split_bw_frames(data: bytes) -> list[bytes]: + """Split BW TX bytes into individual frames (ACK STX … bare ETX).""" + frames: list[bytes] = [] + i = 0 + while i < len(data): + if data[i] != 0x41 or i + 1 >= len(data) or data[i + 1] != 0x02: + i += 1 + continue + j = i + 2 + while j < len(data): + if data[j] == 0x03: + break + if data[j] == 0x10 and j + 1 < len(data): + j += 2 + continue + j += 1 + if j >= len(data): + break + frames.append(data[i : j + 1]) + i = j + 1 + return frames + + +@pytest.fixture(scope="module") +def bw_5a_frames() -> list[bytes]: + """All 5A frames from the BW TX capture, in wire order.""" + if not os.path.exists(BW_TX_PATH): + pytest.skip(f"BW capture not found: {BW_TX_PATH}") + raw = open(BW_TX_PATH, "rb").read() + frames = [ + f for f in _split_bw_frames(raw) + if len(f) >= 6 and f[5] == 0x5A # body[3] == 0x5A (SUB) + ] + assert len(frames) == 17, f"expected 17 5A frames in capture, got {len(frames)}" + return frames + + +@pytest.fixture(scope="module") +def bw_a5_frames(): + """All A5 (response) frames from the matching S3 capture.""" + if not os.path.exists(BW_S3_PATH): + pytest.skip(f"BW S3 capture not found: {BW_S3_PATH}") + raw = open(BW_S3_PATH, "rb").read() + p = S3FrameParser() + p.feed(raw) + a5 = [f for f in p.frames if f.sub == 0xA5] + assert len(a5) == 17, f"expected 17 A5 frames in capture, got {len(a5)}" + return a5 + + +# ── 5A request frame byte-perfect verification ──────────────────────────────── + +KEY4 = bytes.fromhex("01110000") # start_key for the 3-sec event 0 +END_OFFSET = 0x21F2 # parsed from STRT in the BW capture +LAST_CHUNK_COUNTER = 0x1E00 # last full 0x0200-byte chunk before TERM + +SAMPLE_COUNTERS = ( + 0x0600, 0x0800, 0x0A00, 0x0C00, 0x0E00, + 0x1000, 0x1200, 0x1400, 0x1600, 0x1800, + 0x1A00, 0x1C00, 0x1E00, +) + + +def _meta_params(key: bytes, counter: int) -> bytes: + """Build the 12-byte metadata-page params block (matches BW for 0x1002 / 0x1004).""" + return bytes( + [ + 0x00, key[0], key[1], + (counter >> 8) & 0xFF, counter & 0xFF, + 0, 0, 0, 0, 0, 0, 0, + ] + ) + + +def test_probe_frame_byte_perfect(bw_5a_frames): + """Probe @ counter=0x0000 (frame 0).""" + sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, 0, is_probe=True)) + assert sfm == bw_5a_frames[0], ( + f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[0].hex()}" + ) + + +@pytest.mark.parametrize("idx,counter", [(1, 0x1002), (2, 0x1004)]) +def test_metadata_page_frames_byte_perfect(bw_5a_frames, idx, counter): + """Metadata pages @ counter=0x1002 and 0x1004 (frames 1 and 2).""" + sfm = build_5a_frame(0x1002, _meta_params(KEY4, counter)) + assert sfm == bw_5a_frames[idx], ( + f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[idx].hex()}" + ) + + +@pytest.mark.parametrize("i,counter", list(enumerate(SAMPLE_COUNTERS))) +def test_sample_chunk_frames_byte_perfect(bw_5a_frames, i, counter): + """ + Sample chunks @ counter=0x0600..0x1E00, step 0x0200 (frames 3..15). + + Critically, frame 8 (counter=0x1000) requires the v0.14.3 partial DLE + stuffing fix — wire params include `10 10 00` for the counter, not `10 00`. + """ + sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, counter)) + bw_idx = 3 + i + assert sfm == bw_5a_frames[bw_idx], ( + f"\ncounter=0x{counter:04X}" + f"\nSFM: {sfm.hex()}" + f"\nBW: {bw_5a_frames[bw_idx].hex()}" + ) + + +def test_term_frame_byte_perfect(bw_5a_frames): + """TERM frame at residual (frame 16).""" + offset_word, params = bulk_waveform_term_v2(KEY4, END_OFFSET, LAST_CHUNK_COUNTER) + sfm = build_5a_frame(offset_word, params) + assert sfm == bw_5a_frames[16], ( + f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[16].hex()}" + ) + + +def test_strt_end_offset_parsing(bw_a5_frames): + """The probe response (A5[0]) carries STRT at byte 17 with end_offset=0x21F2.""" + from minimateplus.framing import parse_strt_end_offset + + end_offset = parse_strt_end_offset(bw_a5_frames[0].data) + assert end_offset == END_OFFSET, ( + f"expected end_offset=0x{END_OFFSET:04X}, got " + f"{f'0x{end_offset:04X}' if end_offset is not None else 'None'}" + ) + + +# ── File builder byte-perfect verification ──────────────────────────────────── + +def test_blastware_file_builder_byte_perfect(bw_a5_frames): + """ + Feed the BW capture's A5 frames into write_blastware_file() and verify the + output is byte-identical to BW's saved M529LKIQ.G10 reference file. + + This protects the v0.14.2 strip-removal fix and the file-builder skip + values (probe=38, meta=13, samples=12, TERM=11). + """ + if not os.path.exists(BW_SAVED_FILE): + pytest.skip(f"BW saved file not found: {BW_SAVED_FILE}") + + import tempfile + + from minimateplus.blastware_file import write_blastware_file + from minimateplus.models import Event + + ev = Event(index=0) + ev._waveform_key = KEY4 + ev.rectime_seconds = 3 + ev.timestamp = None # let the builder pull the footer from the TERM frame + + with tempfile.NamedTemporaryFile(suffix=".G10", delete=False) as tf: + tmp_path = tf.name + try: + write_blastware_file(ev, bw_a5_frames, tmp_path) + sfm_bytes = open(tmp_path, "rb").read() + finally: + os.unlink(tmp_path) + + bw_bytes = open(BW_SAVED_FILE, "rb").read() + + assert len(sfm_bytes) == len(bw_bytes), ( + f"file size mismatch: SFM={len(sfm_bytes)} BW={len(bw_bytes)}" + ) + + if sfm_bytes != bw_bytes: + # Find first diff for actionable error message + for i in range(len(bw_bytes)): + if bw_bytes[i] != sfm_bytes[i]: + ctx_start = max(0, i - 8) + ctx_end = min(len(bw_bytes), i + 16) + pytest.fail( + f"file diverges at byte 0x{i:04X}\n" + f" BW : {bw_bytes[ctx_start:ctx_end].hex()}\n" + f" SFM: {sfm_bytes[ctx_start:ctx_end].hex()}\n" + f" {' ' * (i - ctx_start)}^^" + ) + + +# ── Standalone runner ───────────────────────────────────────────────────────── + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))