From 9b5cdfd8579e1215496a52ed34e96aedaa4a63d9 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Tue, 21 Apr 2026 00:23:15 -0400 Subject: [PATCH 01/40] feat(logging): add detailed logging for anchor position in compliance config encoding/decoding --- minimateplus/client.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/minimateplus/client.py b/minimateplus/client.py index 767d104..7bb6e8f 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -1833,6 +1833,26 @@ def _encode_compliance_config( _ANC = b'\xbe\x80\x00\x00\x00\x00' _anc = buf.find(_ANC, 0, 150) + # Log anchor position every time so we can detect unexpected shifts due to + # DLE jitter or firmware differences. Expected position is ~15. + if _anc < 0: + log.warning( + "_encode_compliance_config: anchor NOT FOUND in cfg[0:150] " + "(buf len=%d) — all anchor-relative writes will be skipped", + len(buf), + ) + else: + log.info( + "_encode_compliance_config: anchor at cfg[%d] buf_len=%d " + "(expected ~15; fields: recording_mode@%d sample_rate@%d:%d " + "histogram_interval@%d:%d record_time@%d:%d)", + _anc, len(buf), + _anc - 7, + _anc - 6, _anc - 4, + _anc - 4, _anc - 2, + _anc + 6, _anc + 10, + ) + if recording_mode is not None: if _anc < 7: log.warning("_encode_compliance_config: anchor not found — cannot write recording_mode") @@ -2001,6 +2021,27 @@ def _decode_compliance_config_into(data: bytes, info: DeviceInfo) -> None: # _anchor + 6 : record_time (float32 BE) _ANCHOR = b'\xbe\x80\x00\x00\x00\x00' _anchor = data.find(_ANCHOR, 0, 150) + + # Log anchor position on every decode so we can compare read vs write and + # catch unexpected shifts from DLE jitter or firmware differences. + # Expected position is ~15 for the E5 read payload (anchor - 8 = recording_mode). + if _anchor < 0: + log.warning( + "_decode_compliance_config_into: anchor NOT FOUND in data[0:150] (len=%d)", + len(data), + ) + else: + log.info( + "_decode_compliance_config_into: anchor at data[%d] data_len=%d " + "(expected ~15; recording_mode@%d sample_rate@%d:%d " + "histogram_interval@%d:%d record_time@%d:%d)", + _anchor, len(data), + _anchor - 8, + _anchor - 6, _anchor - 4, + _anchor - 4, _anchor - 2, + _anchor + 6, _anchor + 10, + ) + if _anchor >= 8 and _anchor + 10 <= len(data): try: config.recording_mode = data[_anchor - 8] -- 2.52.0 From b3dcfe7239fe75a88f3846d6217b4c1eb4cefba0 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Tue, 21 Apr 2026 01:17:45 -0400 Subject: [PATCH 02/40] fix(client): correct recording_mode anchor position in compliance config encoding --- CLAUDE.md | 2 +- minimateplus/client.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5ab92f4..bdcf66d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -386,7 +386,7 @@ bytes `\x01\x2c` = 300 (5-minute default histogram interval); changes when inter | Offset | Field | Format | Notes | |---|---|---|---| -| anchor − 7 (write) / anchor − 8 (read) | recording_mode | uint8 | E5 read has extra `0x10` at anchor−7 | +| anchor − 8 | recording_mode | uint8 | **Same offset for both read and write.** The byte at anchor−7 is a `0x10` DLE marker regenerated by device firmware in every E5 response — do NOT overwrite it. Writing to anchor−7 causes anchor drift (+1 per write cycle). CORRECTION 2026-04-21: previous doc stated anchor−7 for write; empirically wrong. | | anchor − 6 | sample_rate | uint16 BE | same in read & write | | anchor − 4 | histogram_interval_sec | uint16 BE | seconds; same in read & write ✅ 2026-04-20 | | anchor − 2 | `0x00 0x00` | padding | | diff --git a/minimateplus/client.py b/minimateplus/client.py index 7bb6e8f..42cf492 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -1770,10 +1770,13 @@ def _encode_compliance_config( DLE-jitter shifts): Anchor: b'\\xbe\\x80\\x00\\x00\\x00\\x00' (confirmed stable, both BE11529 and BE18189) - recording_mode → uint8 at anchor_pos - 7 (write payload) + recording_mode → uint8 at anchor_pos - 8 (BOTH read and write) Values: 0x00=Single Shot, 0x01=Continuous, 0x03=Histogram, 0x04=Histogram+Continuous - NOTE: In the E5 read response (decode) field is at anchor_pos - 8 due to an - extra 0x10 byte at read anchor_pos - 7. Write payload has no extra byte. + NOTE: The byte at anchor_pos - 7 is always 0x10 (a DLE marker regenerated by + device firmware in every E5 response). It must NOT be overwritten during + write — doing so causes anchor drift (+1 per write cycle). + CORRECTION 2026-04-21: previous doc stated anchor-7 for write; empirically + confirmed wrong — writing to anchor-7 shifts the anchor by 1 on every cycle. sample_rate → uint16 BE at anchor_pos - 6 histogram_interval_sec → uint16 BE at anchor_pos - 4 (seconds; mode-gated to Histogram/Histogram+Continuous) Valid values: 2, 5, 15, 60, 300, 900 (= 2s, 5s, 15s, 1m, 5m, 15m) @@ -1844,9 +1847,10 @@ def _encode_compliance_config( else: log.info( "_encode_compliance_config: anchor at cfg[%d] buf_len=%d " - "(expected ~15; fields: recording_mode@%d sample_rate@%d:%d " + "(recording_mode@%d DLE_marker@%d sample_rate@%d:%d " "histogram_interval@%d:%d record_time@%d:%d)", _anc, len(buf), + _anc - 8, _anc - 7, _anc - 6, _anc - 4, _anc - 4, _anc - 2, @@ -1854,12 +1858,18 @@ def _encode_compliance_config( ) if recording_mode is not None: - if _anc < 7: + if _anc < 8: log.warning("_encode_compliance_config: anchor not found — cannot write recording_mode") else: - buf[_anc - 7] = recording_mode & 0xFF + # Write to anchor-8, same physical position as the E5 read format. + # The byte at anchor-7 is a DLE marker (0x10) that the device firmware + # regenerates in every E5 response — it must NOT be overwritten. + # Writing to anchor-7 causes the device to add an extra byte on the + # next read-back, drifting the anchor by +1 on every write cycle. + # (CLAUDE.md "anchor-7 write" was incorrect — confirmed 2026-04-21) + buf[_anc - 8] = recording_mode & 0xFF log.debug("_encode_compliance_config: recording_mode=0x%02X -> offset %d", - recording_mode, _anc - 7) + recording_mode, _anc - 8) if sample_rate is not None: if _anc < 6: -- 2.52.0 From 4331215e23768b04b318af38a6ded4f462052ea6 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Tue, 21 Apr 2026 16:07:24 -0400 Subject: [PATCH 03/40] feat(protocol): enhance raw capture functionality and documentation updates - Update `s3_bridge.py` to default raw capture file paths to "auto" for timestamped naming. - Modify `gui_bridge.py` to pre-check raw capture options and streamline path handling. - Extend `ach_server.py` to save both incoming and outgoing raw bytes for analysis. - Revise `CHANGELOG.md` and `instantel_protocol_reference.md` to reflect changes in recording mode handling and compliance data encoding. --- CHANGELOG.md | 53 ++++++++++++++++++++ CLAUDE.md | 73 ++++++++++++++++++++++++---- bridges/ach_server.py | 52 ++++++++++++++------ bridges/gui_bridge.py | 63 +++++++++++++----------- bridges/s3-bridge/s3_bridge.py | 26 +++++++--- docs/instantel_protocol_reference.md | 3 +- 6 files changed, 207 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0bf491..609d889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,59 @@ All notable changes to seismo-relay are documented here. --- +## v0.12.5 — 2026-04-21 + +### Changed + +- **`s3_bridge.py` — raw captures always-on by default** — `--raw-bw` and `--raw-s3` now + default to `"auto"` instead of `None`. Every bridge session automatically generates + timestamped `raw_bw_.bin` and `raw_s3_.bin` files alongside the `.bin`/`.log` + session files. Pass `--raw-bw ""` (explicit empty string) to disable if needed. + +- **`gui_bridge.py` — raw capture checkboxes pre-checked** — Both "BW→S3 raw" and + "S3→BW raw" checkboxes start checked. Path fields are empty by default (bridge auto-names + the files). Unchecking a box passes `--raw-bw ""` to explicitly disable capture. + +- **`ach_server.py` — TX capture added (`raw_tx_.bin`)** — Every ACH inbound session + now saves both directions: `raw_rx_.bin` (device → us, S3 side, as before) and + `raw_tx_.bin` (us → device, BW side). Both files are usable in the Analyzer. + TX bytes are buffered in memory until startup handshake succeeds (same as RX), preventing + scanner probes from creating empty files. + +--- + +## v0.12.4 — 2026-04-21 (protocol analysis / docs only — no code changes) + +### Discovered + +- **compliance_raw is wire-encoded, not logical bytes** — `read_compliance_config()` returns + bytes that include DLE prefix bytes (`0x10`) before any `0x03` values (because S3FrameParser + preserves DLE+ETX inner-frame pairs as two literal bytes). The previous CLAUDE.md claim that + "S3FrameParser handles this transparently so compliance_raw contains logical bytes" was wrong. + +- **anchor-9 behavior per recording mode** (confirmed from 4-20-26 BW write captures): + - Single Shot (0x00) / Continuous (0x01): anchor-9 = `0x00` + - Histogram (0x03): anchor-9 = `0x10` — the E5 DLE prefix for the `0x03` recording_mode byte + - Histogram+Continuous (0x04): anchor-9 = `0x10` — an actual stored config byte for this mode + Anchor position shifts by ±1 when recording_mode = `0x03` due to the extra DLE byte; the + dynamic anchor search (`buf.find(ANCHOR, 0, 150)`) handles this correctly without code changes. + +- **Write frame ETX escaping** — BW escapes `0x03` bytes in write frame data as `0x10 0x03` + on the wire. Our `build_bw_write_frame` sends data bytes raw without ETX escaping. Device + accepts our raw writes for all tested modes. Hypothesis: device write parser uses the + offset/length field for frame boundaries, not ETX scanning, making ETX escaping optional. + Histogram mode (recording_mode = 0x03) write via SFM from a non-Histogram starting state + not yet tested. + +- **BW write payload vs E5 read payload are byte-identical** around the anchor region (confirmed + by comparing 3-11-26 BW TX and S3 captures). BW does NOT strip DLE prefix bytes before writing; + it round-trips the wire-encoded bytes verbatim with only the modified fields changed. + +- **Capture folder content catalogued** — see CLAUDE.md "BW capture reference" table for a + summary of all available protocol captures and their contents. + +--- + ## v0.12.3 — 2026-04-20 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index bdcf66d..96f1c69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -386,7 +386,9 @@ bytes `\x01\x2c` = 300 (5-minute default histogram interval); changes when inter | Offset | Field | Format | Notes | |---|---|---|---| -| anchor − 8 | recording_mode | uint8 | **Same offset for both read and write.** The byte at anchor−7 is a `0x10` DLE marker regenerated by device firmware in every E5 response — do NOT overwrite it. Writing to anchor−7 causes anchor drift (+1 per write cycle). CORRECTION 2026-04-21: previous doc stated anchor−7 for write; empirically wrong. | +| anchor − 9 | mode_prefix | uint8 | `0x00` for Single Shot / Continuous; `0x10` for Histogram (DLE prefix in E5 encoding) and Histogram+Continuous (actual config byte). See "compliance_raw DLE encoding" note below. | +| anchor − 8 | recording_mode | uint8 | **Same offset for both read and write** — confirmed 2026-04-21. `_encode_compliance_config` writes `buf[anc-8]`. NOTE: for Histogram (0x03), E5 encodes the value as `0x10 0x03` so compliance_raw[anc-9]=0x10, compliance_raw[anc-8]=0x03. | +| anchor − 7 | constant | `0x10` | Always `0x10` in both E5 read and BW write payloads (not a DLE marker — it is part of the sample_rate field area). Do NOT overwrite. | | anchor − 6 | sample_rate | uint16 BE | same in read & write | | anchor − 4 | histogram_interval_sec | uint16 BE | seconds; same in read & write ✅ 2026-04-20 | | anchor − 2 | `0x00 0x00` | padding | | @@ -395,15 +397,42 @@ bytes `\x01\x2c` = 300 (5-minute default histogram interval); changes when inter **recording_mode enum** (confirmed 2026-04-20 from 4-20-26 captures): -| Value | Mode | -|---|---| -| `0x00` | Single Shot | -| `0x01` | Continuous | -| `0x02` | ❓ not observed | -| `0x03` | Histogram | -| `0x04` | Histogram + Continuous | +| Value | Mode | anchor-9 in compliance_raw | +|---|---|---| +| `0x00` | Single Shot | `0x00` | +| `0x01` | Continuous | `0x00` | +| `0x02` | ❓ not observed | ❓ | +| `0x03` | Histogram | `0x10` (DLE prefix from E5 wire encoding of 0x03) | +| `0x04` | Histogram + Continuous | `0x10` (actual config byte for this mode) | -**DLE escaping in write frames — CONFIRMED 2026-04-20:** Write frame data payloads DO escape `0x03` (ETX) bytes with a `0x10` DLE prefix. For histogram_interval = 900 (0x0384), the wire carries `10 03 84` — the `0x03` high byte is preceded by a DLE escape. After DLE destuffing (`10 XX → XX`), the logical field value is correctly `03 84` = 900. The CLAUDE.md claim that write frame data is "written RAW" was incorrect; at minimum ETX (0x03) bytes are escaped. S3FrameParser handles this transparently so the decoded `compliance_raw` always contains logical (destuffed) bytes. +**compliance_raw DLE encoding — IMPORTANT (confirmed 2026-04-21 from 4-20-26 captures):** +`compliance_raw` (returned by `read_compliance_config()`) is NOT purely logical bytes — it is +the wire-encoded representation where `0x03` bytes in the config are preceded by a `0x10` DLE +prefix (because S3FrameParser preserves DLE+ETX inner-frame pairs as two literal bytes). + +Consequences: +- When recording_mode = `0x03` (Histogram), `compliance_raw[anc-9] = 0x10` (DLE prefix) and + `compliance_raw[anc-8] = 0x03` (the value). The anchor position is +1 compared to modes + without `0x03` bytes before the anchor. +- For Histogram+Continuous (`0x04`), `compliance_raw[anc-9] = 0x10` for a different reason: + it is an actual stored config byte, not a DLE prefix. +- The anchor search (`buf.find(b'\xbe\x80\x00\x00\x00\x00', 0, 150)`) correctly locates + the anchor regardless of these mode-dependent shifts. +- When SFM writes recording_mode and round-trips the rest verbatim, the byte at `anc-9` is + preserved from the previous read. This means transitioning Histogram→other modes via SFM + leaves a `0x10` at `anc-9`. The device stores it as a literal byte; it does not affect + recording mode operation (which is at `anc-8`), but differs from what BW writes. This is a + known minor discrepancy that does not impact device behavior. +- **Histogram recording mode (0x03) write via SFM**: untested. When starting from a mode with + `anc-9 = 0x00`, SFM writes bare `0x03` at anc-8. BW would write `0x10 0x03`. Device likely + accepts both (write frames probably use offset/length for framing, not ETX scanning). + +**DLE escaping in write frames — confirmed 2026-04-20:** Blastware escapes `0x03` bytes in +write frame data as `0x10 0x03` on the wire (defensive ETX escaping). Our `build_bw_write_frame` +does NOT do this escaping — it sends data bytes raw. Device acceptance of bare `0x03` bytes +in write frame data is confirmed for the tested modes (Single Shot, Continuous, Histogram+Continuous +where `0x10 0x03` already appears from round-tripping). Histogram mode (bare `0x03` write from +non-Histogram starting state) has not been directly tested. ### SUB 0C — Waveform Record (210 bytes = data[11:11+0xD2]) @@ -1067,9 +1096,33 @@ body) because writing a dial string may require DLE escaping for embedded contro - **Database** — SQLite store for events + monitor log entries; dedup by key; queryable - **Histograms** — decode histogram-mode A5 data (noise floor tracking) +- **Blastware-compatible file output** — .MLG and binary waveform file generation matching Blastware format (needed for interoperability with existing workflows) - Compliance config encoder — build raw write payloads from a `ComplianceConfig` object +- **Test Histogram recording mode (0x03) write via SFM** — confirmed working for Single Shot / Continuous / Histogram+Continuous; Histogram (0x03) needs a live test from a non-Histogram starting state (bare 0x03 in write vs BW's DLE-escaped `10 03`) +- **Compliance write anchor-9 cleanup** — when changing recording_mode via SFM, the byte at anchor-9 is not explicitly managed. A spurious `0x10` may persist after Histogram→other mode transitions. Does not affect device operation but differs from BW's byte-perfect output. - Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring) - Call Home — map time slots 3/4 offsets; add dial_string write support; confirm `modem_power_relay_enabled` - Modem manager — push RV50/RV55 configs via Sierra Wireless API - RV55 DCD/DTR issue — newer RV55 firmware doesn't assert DCD by default; units don't - resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred) \ No newline at end of file + resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred) + +## BW capture reference + +`bridges/captures/` contains the following BW TX + S3 response captures for protocol analysis: + +| Folder / File | Contents | +|---|---| +| `3-11-26/raw_bw_20260311_170151.bin` | Full compliance write + event download (SUBs 68→83 confirmed, frames 102–112) | +| `4-20-26/raw_bw_*_recording_mode_*.bin` | Recording mode changes: Continuous→Single Shot, →Histogram, →Histogram+Continuous | +| `4-20-26/histogram interval/` | Histogram interval changes: 1min, 5min, 15min, 15sec | +| `4-20-26/geo sensitivity/` | Geo sensitivity changes: 1.25 in/s (Sensitive), 10 in/s (Normal) | +| `4-20-26/call home settings/` | Call home config read/write captures | +| `4-8-26/` | Monitor status read, start/stop monitoring, SESSION_RESET signal, sensor check | +| `4-3-26-multi_event/` | Browse-mode S3 capture with 2+ events (1E/0A/1F iteration confirmed) | +| `4-2-26/` | Download-mode BW TX capture (5A bulk stream, POLL×3 requirement confirmed) | +| `3-31-26/` | Single-event download (148 BW / 147 S3 frames) | +| `mitm/ach_mitm_20260411_001912/` | Full ACH call-home MITM (erase protocol, 0xA3/0x06/0xA2 confirmed) | + +To parse BW TX captures: use `bridges/captures/` scripts or adapt the `find_write_frames()` pattern +in `/tmp/analyze_write_payload.py` — it correctly handles `0x10 0x03` DLE-escaped ETX bytes +inside write frame data (the naive parser terminates early at the escaped `0x03`). \ No newline at end of file diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 65fb9e9..edd4f2e 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -35,6 +35,7 @@ Output per session device_info.json — serial number, firmware version, calibration date, etc. events.json — all events: timestamp, PPV per channel, peaks, metadata raw_rx_.bin — raw bytes from the device (S3 side) for Analyzer + raw_tx_.bin — raw bytes we sent to the device (BW side) for Analyzer session_.log — detailed protocol log What to look for @@ -172,16 +173,24 @@ class AchSession: transport = SocketTransport(self.sock, peer=self.peer) # Collect raw bytes in memory until startup succeeds, then flush to disk. - raw_buf: list[bytes] = [] - _orig_read = transport.read + raw_rx_buf: list[bytes] = [] # device → us (S3 side) + raw_tx_buf: list[bytes] = [] # us → device (BW side) + _orig_read = transport.read + _orig_write = transport.write def tapped_read(n: int) -> bytes: data = _orig_read(n) if data: - raw_buf.append(data) + raw_rx_buf.append(data) return data - transport.read = tapped_read # type: ignore[method-assign] + def tapped_write(data: bytes) -> None: + _orig_write(data) + if data: + raw_tx_buf.append(data) + + transport.read = tapped_read # type: ignore[method-assign] + transport.write = tapped_write # type: ignore[method-assign] serial: Optional[str] = None @@ -201,23 +210,35 @@ class AchSession: # Startup succeeded — this is a real unit. Create session dir now. session_dir = self.output_dir / f"ach_inbound_{ts}" session_dir.mkdir(parents=True, exist_ok=True) - log_path = session_dir / f"session_{ts}.log" - raw_path = session_dir / f"raw_rx_{ts}.bin" + log_path = session_dir / f"session_{ts}.log" + raw_rx_path = session_dir / f"raw_rx_{ts}.bin" # device → us (S3 side) + raw_tx_path = session_dir / f"raw_tx_{ts}.bin" # us → device (BW side) - # Flush buffered raw bytes to file and switch to direct file writes. - raw_fh = open(raw_path, "wb") - for chunk in raw_buf: - raw_fh.write(chunk) - raw_buf.clear() + # Flush buffered bytes to files and switch to direct file writes. + raw_rx_fh = open(raw_rx_path, "wb") + raw_tx_fh = open(raw_tx_path, "wb") + for chunk in raw_rx_buf: + raw_rx_fh.write(chunk) + for chunk in raw_tx_buf: + raw_tx_fh.write(chunk) + raw_rx_buf.clear() + raw_tx_buf.clear() def tapped_read_file(n: int) -> bytes: data = _orig_read(n) if data: - raw_fh.write(data) - raw_fh.flush() + raw_rx_fh.write(data) + raw_rx_fh.flush() return data - transport.read = tapped_read_file # type: ignore[method-assign] + def tapped_write_file(data: bytes) -> None: + _orig_write(data) + if data: + raw_tx_fh.write(data) + raw_tx_fh.flush() + + transport.read = tapped_read_file # type: ignore[method-assign] + transport.write = tapped_write_file # type: ignore[method-assign] # Wire up file handler now that the session dir exists. fh = logging.FileHandler(log_path, encoding="utf-8") @@ -530,7 +551,8 @@ class AchSession: log.warning(" [WARN] Failed to restart monitoring: %s", exc) finally: - raw_fh.close() + raw_rx_fh.close() + raw_tx_fh.close() client.close() # closes transport / socket cleanly root_logger.removeHandler(fh) fh.close() diff --git a/bridges/gui_bridge.py b/bridges/gui_bridge.py index c0ae686..7028baf 100644 --- a/bridges/gui_bridge.py +++ b/bridges/gui_bridge.py @@ -58,16 +58,24 @@ class BridgeGUI(tk.Tk): tk.Entry(self, textvariable=self.logdir_var, width=24).grid(row=1, column=3, sticky="we", **pad) tk.Button(self, text="Browse", command=self._choose_dir).grid(row=1, column=4, sticky="w", **pad) - # Row 2: Raw taps - self.raw_bw_var = tk.StringVar(value="") - self.raw_s3_var = tk.StringVar(value="") - tk.Checkbutton(self, text="Save BW->S3 raw", command=self._toggle_raw_bw, onvalue="1", offvalue="").grid(row=2, column=0, sticky="w", **pad) - tk.Entry(self, textvariable=self.raw_bw_var, width=28).grid(row=2, column=1, columnspan=3, sticky="we", **pad) - tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_bw_var, "bw")).grid(row=2, column=4, **pad) + # Row 2: Raw taps — ON by default; "auto" = timestamped name; blank checkbox = disabled + self.raw_bw_enabled = tk.IntVar(value=1) + self.raw_s3_enabled = tk.IntVar(value=1) + # Path fields: empty means "auto" (bridge picks a timestamped name) + self.raw_bw_path_var = tk.StringVar(value="") + self.raw_s3_path_var = tk.StringVar(value="") - tk.Checkbutton(self, text="Save S3->BW raw", command=self._toggle_raw_s3, onvalue="1", offvalue="").grid(row=3, column=0, sticky="w", **pad) - tk.Entry(self, textvariable=self.raw_s3_var, width=28).grid(row=3, column=1, columnspan=3, sticky="we", **pad) - tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_s3_var, "s3")).grid(row=3, column=4, **pad) + tk.Checkbutton(self, text="BW→S3 raw (auto)", variable=self.raw_bw_enabled, + command=self._toggle_raw_bw).grid(row=2, column=0, sticky="w", **pad) + tk.Entry(self, textvariable=self.raw_bw_path_var, width=28, + fg="grey").grid(row=2, column=1, columnspan=3, sticky="we", **pad) + tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_bw_path_var, "bw")).grid(row=2, column=4, **pad) + + tk.Checkbutton(self, text="S3→BW raw (auto)", variable=self.raw_s3_enabled, + command=self._toggle_raw_s3).grid(row=3, column=0, sticky="w", **pad) + tk.Entry(self, textvariable=self.raw_s3_path_var, width=28, + fg="grey").grid(row=3, column=1, columnspan=3, sticky="we", **pad) + tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_s3_path_var, "s3")).grid(row=3, column=4, **pad) # Row 4: Status + buttons self.status_var = tk.StringVar(value="Idle") @@ -102,13 +110,11 @@ class BridgeGUI(tk.Tk): var.set(filename) def _toggle_raw_bw(self) -> None: - if not self.raw_bw_var.get(): - # default name - self.raw_bw_var.set(os.path.join(self.logdir_var.get(), "raw_bw.bin")) + # Checkbox toggled — no path action needed; enabled state drives the flag. + pass def _toggle_raw_s3(self) -> None: - if not self.raw_s3_var.get(): - self.raw_s3_var.set(os.path.join(self.logdir_var.get(), "raw_s3.bin")) + pass def start_bridge(self) -> None: if self.process and self.process.poll() is None: @@ -126,23 +132,22 @@ class BridgeGUI(tk.Tk): args = [sys.executable, BRIDGE_PATH, "--bw", bw, "--s3", s3, "--baud", baud, "--logdir", logdir] - ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + # Raw tap flags. + # Checkbox on + empty path → pass "auto" (bridge generates timestamped name). + # Checkbox on + explicit path → pass that path. + # Checkbox off → pass "" to disable (overrides bridge's auto default). + raw_bw_explicit = self.raw_bw_path_var.get().strip() + raw_s3_explicit = self.raw_s3_path_var.get().strip() - raw_bw = self.raw_bw_var.get().strip() - raw_s3 = self.raw_s3_var.get().strip() + if self.raw_bw_enabled.get(): + args += ["--raw-bw", raw_bw_explicit if raw_bw_explicit else "auto"] + else: + args += ["--raw-bw", ""] # explicit disable - # If the user left the default generic name, replace with a timestamped one - # so each session gets its own file. - if raw_bw: - if os.path.basename(raw_bw) in ("raw_bw.bin", "raw_bw"): - raw_bw = os.path.join(os.path.dirname(raw_bw) or logdir, f"raw_bw_{ts}.bin") - self.raw_bw_var.set(raw_bw) - args += ["--raw-bw", raw_bw] - if raw_s3: - if os.path.basename(raw_s3) in ("raw_s3.bin", "raw_s3"): - raw_s3 = os.path.join(os.path.dirname(raw_s3) or logdir, f"raw_s3_{ts}.bin") - self.raw_s3_var.set(raw_s3) - args += ["--raw-s3", raw_s3] + if self.raw_s3_enabled.get(): + args += ["--raw-s3", raw_s3_explicit if raw_s3_explicit else "auto"] + else: + args += ["--raw-s3", ""] # explicit disable try: self.process = subprocess.Popen( diff --git a/bridges/s3-bridge/s3_bridge.py b/bridges/s3-bridge/s3_bridge.py index f3e1770..aa0ecac 100644 --- a/bridges/s3-bridge/s3_bridge.py +++ b/bridges/s3-bridge/s3_bridge.py @@ -390,8 +390,14 @@ def main() -> int: ap.add_argument("--s3", default="COM5", help="S3-side COM port (default: COM5)") ap.add_argument("--baud", type=int, default=38400, help="Baud rate (default: 38400)") ap.add_argument("--logdir", default=".", help="Directory to write session logs into (default: .)") - ap.add_argument("--raw-bw", default=None, help="Optional file to append raw bytes sent from BW->S3 (no headers)") - ap.add_argument("--raw-s3", default=None, help="Optional file to append raw bytes sent from S3->BW (no headers)") + ap.add_argument("--raw-bw", default="auto", + help="File to append raw bytes sent from BW->S3 (no headers). " + "Default 'auto' generates a timestamped name in --logdir. " + "Pass an empty string to disable.") + ap.add_argument("--raw-s3", default="auto", + help="File to append raw bytes sent from S3->BW (no headers). " + "Default 'auto' generates a timestamped name in --logdir. " + "Pass an empty string to disable.") ap.add_argument("--quiet", action="store_true", help="No console heartbeat output") ap.add_argument("--status-every", type=float, default=0.0, help="Seconds between console heartbeat lines (default: 0 = off)") args = ap.parse_args() @@ -414,12 +420,16 @@ def main() -> int: # If raw tap flags were passed without a path (bare --raw-bw / --raw-s3), # or if the sentinel value "auto" is used, generate a timestamped name. # If a specific path was provided, use it as-is (caller's responsibility). - raw_bw_path = args.raw_bw - raw_s3_path = args.raw_s3 - if raw_bw_path in (None, "", "auto"): - raw_bw_path = os.path.join(args.logdir, f"raw_bw_{ts}.bin") if args.raw_bw is not None else None - if raw_s3_path in (None, "", "auto"): - raw_s3_path = os.path.join(args.logdir, f"raw_s3_{ts}.bin") if args.raw_s3 is not None else None + # Resolve raw tap paths. + # "auto" (default) → timestamped file in logdir (always captured). + # Explicit path → use verbatim. + # None or "" → disabled (pass --raw-bw "" to suppress capture). + raw_bw_path: Optional[str] = args.raw_bw if args.raw_bw else None + raw_s3_path: Optional[str] = args.raw_s3 if args.raw_s3 else None + if raw_bw_path == "auto": + raw_bw_path = os.path.join(args.logdir, f"raw_bw_{ts}.bin") + if raw_s3_path == "auto": + raw_s3_path = os.path.join(args.logdir, f"raw_s3_{ts}.bin") logger = SessionLogger(log_path, bin_path, raw_bw_path=raw_bw_path, raw_s3_path=raw_s3_path) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 7b49bcd..fc211be 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -104,7 +104,8 @@ | 2026-04-11 | §7.11 (NEW) | **NEW — §7.11 Erase-All Protocol added.** Full wire sequence, SUB 0x06 storage range payload layout, post-erase key counter reset (resets to `0x01110000`). Confirmed from 4-11-26 MITM capture of live Blastware ACH session. | | 2026-04-11 | §14.6 | **RESOLVED — ACH Session Lifecycle is no longer "Future".** `bridges/ach_server.py` fully implements inbound ACH: POLL handshake, device info, event download. State tracked via `ach_state.json` (key-based, with `max_downloaded_key` for post-erase detection). `--clear-after-download` flag added for the standard delete-after-upload workflow. | | 2026-04-17 | §7.6.2, §14 | **RESOLVED — Float 6.206053 at channel_label+28 is the ADC-to-velocity scale factor.** Confirmed from Series III Interface Handbook §4.5 formula: `Range (×1) = 1.61133 V / Sensitivity (V/unit)`. For the standard Instantel geophone at Normal range (10.000 in/s): Sensitivity = 1.61133 / 10 = 0.161133 V/(in/s). The stored value is the **inverse sensitivity** = 1/0.161133 = **6.206053 (in/s)/V**. Cross-check: 1.61133 V × 6.206053 = 10.000 in/s ✅. The firmware uses it as: `PPV (in/s) = ADC_voltage (V) × 6.206053`. Value is identical on all Instantel standard geophones — it is a hardware/firmware constant, NOT a user-configurable setting. Do NOT write this field. Open question §14 item "Max Geo Range float 6.2061" is now **RESOLVED**. | -| 2026-04-20 | §7.6.4 (NEW), §7.9, Appendix B | **CONFIRMED — Recording Mode byte location.** Three targeted captures (4-20-26) confirmed `recording_mode` at **`cfg[5]`** in the SUB 71 write payload (3-chunk compliance write). Method: single Blastware session, one initial E5 config pull, then three sequential "Send to unit" writes changing Recording Mode only. Diff of SUB 71 chunk-1 payloads: only `cfg[5]` and `cfg[1024]` changed; `cfg[1024]` delta exactly equals `cfg[5]` delta (chunk running checksum). In the E5 read response (sub-frame 1, page=0x0010), the field is at **`data[17]`** (= **anchor − 4** from the 10-byte anchor), one position earlier than in the write payload due to an extra `0x10` byte at `data[18]` present only in the read format. Enum: `0x00`=Single Shot, `0x01`=Continuous, `0x03`=Histogram, `0x04`=Histogram+Continuous. `0x02` value not yet observed. See §7.6.4 for full details. | +| 2026-04-20 | §7.6.4 (NEW), §7.9, Appendix B | **CONFIRMED — Recording Mode byte location.** Three targeted captures (4-20-26) confirmed `recording_mode` at anchor−8 in both the E5 read payload and the BW write payload (6-byte anchor `\xbe\x80\x00\x00\x00\x00`). BW write payload and E5 read payload are **byte-identical** around the anchor region — Blastware round-trips the wire-encoded E5 bytes verbatim with only the target field modified. Anchor position varies by ±1 depending on whether recording_mode = 0x03 (Histogram), because E5 wire-encodes `0x03` as the inner DLE+ETX pair `\x10\x03` (2 bytes), which S3FrameParser preserves as two literal bytes in `compliance_raw`. Enum: `0x00`=Single Shot, `0x01`=Continuous, `0x03`=Histogram, `0x04`=Histogram+Continuous. `0x02` value not yet observed. The byte at anchor−9 is `0x00` for Single Shot / Continuous, and `0x10` for Histogram (DLE prefix from E5 encoding) and Histogram+Continuous (actual config byte). See §7.6.4 for full details. | +| 2026-04-21 | §7.6.2, §5.3 | **CORRECTED — compliance_raw contains wire-encoded bytes, NOT logical bytes.** S3FrameParser appends DLE+ETX inner-frame pairs as two literal bytes to the frame body. Any `0x03` values in the compliance config appear in `compliance_raw` as `\x10\x03` (two bytes), not as a single `0x03`. The previous claim "S3FrameParser handles this transparently so compliance_raw contains logical (destuffed) bytes" was wrong. Consequence: `compliance_raw` is the wire-encoded E5 payload; anchor-relative reads work correctly because the anchor position automatically accounts for any DLE-encoded bytes before it. For write-back, round-tripping `compliance_raw` verbatim sends the correct wire bytes to the device. **DLE ETX escaping in write frames:** Blastware escapes `0x03` bytes in write frame data as `\x10\x03` on wire; our `build_bw_write_frame` does not (writes data raw). Device is confirmed to accept raw writes for all tested modes — likely uses the offset/length field for write frame framing, not ETX scanning. | | 2026-04-20 | §7.6.2, §7.9, Appendix B | **CONFIRMED — Geophone maximum range / sensitivity selector byte location.** Two targeted captures (4-20-26, geo sensitivity folder): one at Normal 10.000 in/s, one at Sensitive 1.250 in/s. E5 read payload diff: exactly 3 bytes differ at channel_label+33 for Tran/Vert/Long. Values: `0x00`=Normal 10.000 in/s, `0x01`=Sensitive 1.250 in/s. Same offset applies to the SUB 71 write payload (which is the same 2126-byte E5-format buffer round-tripped verbatim). **`channel_label+20` reads `0x01` in ALL captures regardless of range setting — it is NOT this field.** Previous hypothesis (uint8 at Tran+20, 0x01=Normal) was WRONG. Stored as `geo_range` in `ComplianceConfig`. Encoded to all three geo channel blocks (Tran/Vert/Long) at label+33. | | 2026-04-20 | §5.1, §5.3, §7.12 (NEW) | **NEW — Auto Call Home config protocol confirmed from 4-20-26 call home settings captures.** SUB 0x2C (Call Home Config READ, response 0xD3, data offset 0x7C=124) and SUB 0x7E/0x7F (WRITE + CONFIRM, response 0x81/0x80) confirmed. Write payload = read payload (125 bytes) + `\x00\x00` (127 bytes total). **DLE-escaped ETX at raw[117:119]:** the device returns logical value 0x03 (num_retries=3) as `\x10\x03` on the wire — S3FrameParser preserves both bytes as two literals, causing a +1 byte shift for all subsequent fields. Write frame sends these bytes verbatim (device interprets `\x10\x03` as literal value 3). Field map confirmed from 10-frame BW TX diff. See §7.12 for full layout. | -- 2.52.0 From dfbc9f29c556ee7957fea015e6c76d61ee64bd8f Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Tue, 21 Apr 2026 22:57:53 -0400 Subject: [PATCH 04/40] feat: first try at building waveform binary files. --- CLAUDE.md | 2 +- docs/instantel_protocol_reference.md | 219 ++++++++ minimateplus/blastware_file.py | 777 +++++++++++++++++++++++++++ minimateplus/client.py | 46 +- minimateplus/framing.py | 14 +- minimateplus/models.py | 4 + minimateplus/protocol.py | 28 +- sfm/server.py | 77 +++ 8 files changed, 1148 insertions(+), 19 deletions(-) create mode 100644 minimateplus/blastware_file.py diff --git a/CLAUDE.md b/CLAUDE.md index 96f1c69..a28f3fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1096,7 +1096,7 @@ body) because writing a dial string may require DLE escaping for embedded contro - **Database** — SQLite store for events + monitor log entries; dedup by key; queryable - **Histograms** — decode histogram-mode A5 data (noise floor tracking) -- **Blastware-compatible file output** — .MLG and binary waveform file generation matching Blastware format (needed for interoperability with existing workflows) +- **Blastware-compatible file output** — `write_n00()` and `write_mlg()` implemented (v0.12.3+). `write_n00` verified byte-perfect vs M529LIY6.N00. Extension mapping: `.N00`=single-shot, `.9T0`=continuous (confirmed); `.490`, `.5K0`, `.980`, `.ML0` observed but not decoded (likely encoding recording mode × sample rate at capture time — not determinable from file body alone). Filename stem algorithm confirmed 2026-04-21: `M<4-char-base36-stem>` where stem = `floor((ts_local − 1985-01-01T00:00:00) / 1296)`, unit = 36² = 1296 s ≈ 21.6 min. - Compliance config encoder — build raw write payloads from a `ComplianceConfig` object - **Test Histogram recording mode (0x03) write via SFM** — confirmed working for Single Shot / Continuous / Histogram+Continuous; Histogram (0x03) needs a live test from a non-Histogram starting state (bare 0x03 in write vs BW's DLE-escaped `10 03`) - **Compliance write anchor-9 cleanup** — when changing recording_mode via SFM, the byte at anchor-9 is not explicitly managed. A spurious `0x10` may persist after Histogram→other mode transitions. Does not affect device operation but differs from BW's byte-perfect output. diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index fc211be..0491a49 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -105,6 +105,8 @@ | 2026-04-11 | §14.6 | **RESOLVED — ACH Session Lifecycle is no longer "Future".** `bridges/ach_server.py` fully implements inbound ACH: POLL handshake, device info, event download. State tracked via `ach_state.json` (key-based, with `max_downloaded_key` for post-erase detection). `--clear-after-download` flag added for the standard delete-after-upload workflow. | | 2026-04-17 | §7.6.2, §14 | **RESOLVED — Float 6.206053 at channel_label+28 is the ADC-to-velocity scale factor.** Confirmed from Series III Interface Handbook §4.5 formula: `Range (×1) = 1.61133 V / Sensitivity (V/unit)`. For the standard Instantel geophone at Normal range (10.000 in/s): Sensitivity = 1.61133 / 10 = 0.161133 V/(in/s). The stored value is the **inverse sensitivity** = 1/0.161133 = **6.206053 (in/s)/V**. Cross-check: 1.61133 V × 6.206053 = 10.000 in/s ✅. The firmware uses it as: `PPV (in/s) = ADC_voltage (V) × 6.206053`. Value is identical on all Instantel standard geophones — it is a hardware/firmware constant, NOT a user-configurable setting. Do NOT write this field. Open question §14 item "Max Geo Range float 6.2061" is now **RESOLVED**. | | 2026-04-20 | §7.6.4 (NEW), §7.9, Appendix B | **CONFIRMED — Recording Mode byte location.** Three targeted captures (4-20-26) confirmed `recording_mode` at anchor−8 in both the E5 read payload and the BW write payload (6-byte anchor `\xbe\x80\x00\x00\x00\x00`). BW write payload and E5 read payload are **byte-identical** around the anchor region — Blastware round-trips the wire-encoded E5 bytes verbatim with only the target field modified. Anchor position varies by ±1 depending on whether recording_mode = 0x03 (Histogram), because E5 wire-encodes `0x03` as the inner DLE+ETX pair `\x10\x03` (2 bytes), which S3FrameParser preserves as two literal bytes in `compliance_raw`. Enum: `0x00`=Single Shot, `0x01`=Continuous, `0x03`=Histogram, `0x04`=Histogram+Continuous. `0x02` value not yet observed. The byte at anchor−9 is `0x00` for Single Shot / Continuous, and `0x10` for Histogram (DLE prefix from E5 encoding) and Histogram+Continuous (actual config byte). See §7.6.4 for full details. | +| 2026-04-21 | Appendix D (NEW) | **NEW — Blastware .N00 and .MLG file formats fully decoded.** `minimateplus/blastware_file.py` implements `write_n00()` and `write_mlg()`. N00 file format confirmed: 22B header + 21B STRT record + variable body + 26B footer. Body reconstructed from A5 bulk waveform stream frames with per-frame skip amounts (probe=7+strt_pos+21, A5[1]=13, A5[2+]=12, terminator=11) and DLE strip rule (strip `0x10` before `{0x02,0x03,0x04}`, keep following byte). Footer extracted verbatim from terminator frame's last 26 bytes. Split-pair edge case: when `frame.data[-1]==0x10` and `chk_byte∈{0x02,0x03,0x04}`, reunite both bytes before stripping and always remove trailing chk_byte (`stripped[:-1]`) — chk_byte is checksum, not payload. STRT record must be copied verbatim from A5[0]; bytes [10:20] are device-specific and cannot be reconstructed from Event fields. `write_n00` verified byte-perfect against `M529LIY6.N00` from 4-3-26-multi_event capture. MLG format: 308B header + N×292B records; CRC algorithm unknown (write as 0x0000). | +| 2026-04-21 | Appendix D §D.5 (NEW) | **NEW — Blastware filename stem encoding confirmed; extension taxonomy partially decoded.** Stem is a 4-character uppercase base-36 encoding of `floor((event_local_time − 1985-01-01T00:00:00) / 1296)`, where 1296 = 36² seconds ≈ 21.6 minutes per unit. Epoch = January 1, 1985 (Instantel founding year). Confirmed against 6 independent events (April 1–9, 2026): all 6 stems (LIY6, LJ31, LJ8V, LJDY×3) match exactly; epoch estimate within ±7 minutes of midnight across all samples. Third char is always `'0'`. Serial prefix = `"M"` + last 3 decimal digits of serial. Multiple events within the same 21.6-minute window share a stem; their extension distinguishes them. Extension taxonomy: `.N00`=single-shot (compliance_raw recording_mode=0x00), `.9T0`=continuous (recording_mode=0x01) confirmed. `.490`, `.5K0`, `.980`, `.ML0` observed but not decoded — binary analysis shows they are structurally identical to `.9T0` files in all metadata regions (the A5 body's session-start compliance config reflects the state at session start, not at per-event capture time). Extension likely encodes the capture-time recording mode × sample rate combination, but cannot be determined from file body alone without capture-time compliance data. **DLE-shift note for reading recording_mode from file body:** the 0x10 constant at logical anchor−7 gets stripped by `_strip_inner_frame_dles` when sample_rate_HI = 0x04 (1024 sps), shifting recording_mode from logical anchor−8 to file position anchor−7. For sample_rate ≠ 1024 (0x08 or 0x10 as HI byte), no stripping occurs and recording_mode remains at file[anchor−8]. | | 2026-04-21 | §7.6.2, §5.3 | **CORRECTED — compliance_raw contains wire-encoded bytes, NOT logical bytes.** S3FrameParser appends DLE+ETX inner-frame pairs as two literal bytes to the frame body. Any `0x03` values in the compliance config appear in `compliance_raw` as `\x10\x03` (two bytes), not as a single `0x03`. The previous claim "S3FrameParser handles this transparently so compliance_raw contains logical (destuffed) bytes" was wrong. Consequence: `compliance_raw` is the wire-encoded E5 payload; anchor-relative reads work correctly because the anchor position automatically accounts for any DLE-encoded bytes before it. For write-back, round-tripping `compliance_raw` verbatim sends the correct wire bytes to the device. **DLE ETX escaping in write frames:** Blastware escapes `0x03` bytes in write frame data as `\x10\x03` on wire; our `build_bw_write_frame` does not (writes data raw). Device is confirmed to accept raw writes for all tested modes — likely uses the offset/length field for write frame framing, not ETX scanning. | | 2026-04-20 | §7.6.2, §7.9, Appendix B | **CONFIRMED — Geophone maximum range / sensitivity selector byte location.** Two targeted captures (4-20-26, geo sensitivity folder): one at Normal 10.000 in/s, one at Sensitive 1.250 in/s. E5 read payload diff: exactly 3 bytes differ at channel_label+33 for Tran/Vert/Long. Values: `0x00`=Normal 10.000 in/s, `0x01`=Sensitive 1.250 in/s. Same offset applies to the SUB 71 write payload (which is the same 2126-byte E5-format buffer round-tripped verbatim). **`channel_label+20` reads `0x01` in ALL captures regardless of range setting — it is NOT this field.** Previous hypothesis (uint8 at Tran+20, 0x01=Normal) was WRONG. Stored as `geo_range` in `ComplianceConfig`. Encoded to all three geo channel blocks (Tran/Vert/Long) at label+33. | | 2026-04-20 | §5.1, §5.3, §7.12 (NEW) | **NEW — Auto Call Home config protocol confirmed from 4-20-26 call home settings captures.** SUB 0x2C (Call Home Config READ, response 0xD3, data offset 0x7C=124) and SUB 0x7E/0x7F (WRITE + CONFIRM, response 0x81/0x80) confirmed. Write payload = read payload (125 bytes) + `\x00\x00` (127 bytes total). **DLE-escaped ETX at raw[117:119]:** the device returns logical value 0x03 (num_retries=3) as `\x10\x03` on the wire — S3FrameParser preserves both bytes as two literals, causing a +1 byte shift for all subsequent fields. Write frame sends these bytes verbatim (device interprets `\x10\x03` as literal value 3). Field map confirmed from 10-frame BW TX diff. See §7.12 for full layout. | @@ -2245,6 +2247,223 @@ Semantic Interpretation <- settings, events, responses --- +--- + +## Appendix D — Blastware Binary File Formats (.N00 / .MLG) + +> ✅ CONFIRMED 2026-04-21 — all fields verified by binary diff of reconstructed vs reference +> files from the 4-3-26-multi_event capture (M529LIY6.N00, BE11529.MLG). + +### D.1 Common File Header (22 bytes) + +All Blastware files (regardless of type) share an 18-byte prefix followed by a 4-byte type tag. + +| Offset | Length | Value | Description | +|---|---|---|---| +| 0x00 | 6 | `10 00 01 80 00 00` | Fixed prefix | +| 0x06 | 10 | `Instantel\x00` | ASCII string | +| 0x10 | 2 | `07 2c` | Fixed suffix | +| 0x12 | 4 | varies | File type tag (see below) | + +**Total header: 22 bytes.** + +**Type tags:** + +| Extension | Type tag | Description | +|---|---|---| +| `.N00` | `00 12 03 00` | Single-shot waveform event | +| `.MLG` | `22 01 0e a0` | Monitor log | + +Blastware identifies file type by extension, not by type tag alone. + +### D.2 Timestamp Encoding (Blastware files) + +All timestamps in N00 and MLG files use an **8-byte big-endian format**: + +| Byte | Field | +|---|---| +| 0 | day (uint8) | +| 1 | month (uint8) | +| 2–3 | year (uint16 BE) | +| 4 | `0x00` (reserved) | +| 5 | hour (uint8) | +| 6 | minute (uint8) | +| 7 | second (uint8) | + +Example: `01 04 07 ea 00 00 1c 08` → April 1, 2026, 00:28:08. + +Note: this differs from the 8-byte protocol timestamp (`[day][sub_code][month][year_HI][year_LO][0x00][hour][min][sec]` = 9 bytes) used in the device's on-wire 0C waveform records. The file format uses a compact 8-byte layout without the `sub_code` byte. + +### D.3 N00 File Format — Single-Shot Waveform Event + +**File layout:** `[22B header] [21B STRT record] [body bytes] [26B footer]` + +#### D.3.1 STRT Record (21 bytes) + +The STRT record immediately follows the 22-byte header. + +| Offset | Length | Field | Notes | +|---|---|---|---| +| 0 | 4 | `STRT` | ASCII literal | +| 4 | 2 | `ff fe` | Fixed | +| 6 | 4 | event key (key4) | 4-byte waveform key | +| 10 | 4 | device-specific | NOT a repeat of key4 — device-internal field | +| 14 | 6 | device-specific | NOT zero-padded — device-internal fields | +| 20 | 1 | rectime | uint8 seconds | + +**Critical:** The STRT record must be copied verbatim from A5[0].data[7+strt_pos:] — bytes [10:20] contain device-specific values that cannot be reconstructed from protocol-level Event fields alone. + +#### D.3.2 Body Bytes (variable) + +The body is reconstructed from the raw A5 bulk waveform stream frames by stripping DLE framing markers and taking the appropriate slice of each frame's data section. + +**Per-frame contribution (from `frame.data`):** + +| Frame | Skip amount | Notes | +|---|---|---| +| A5[0] (probe) | `7 + strt_pos_in_w0 + 21` | Skip frame.data prefix + STRT record | +| A5[1] | 13 | 7-byte prefix + 6-byte first-chunk header | +| A5[2..N] | 12 | 7-byte prefix + 5-byte chunk header | +| Terminator (page_key=0x0000) | 11 | 7-byte prefix + 4-byte terminator header | + +**DLE strip rule:** For each frame's contribution (`frame.data[skip:]`), strip any `0x10` byte immediately followed by `0x02`, `0x03`, or `0x04`. Only the `0x10` is stripped; the following byte is kept as payload. + +**Split-pair edge case:** When `frame.data[-1] == 0x10` AND `frame.chk_byte ∈ {0x02, 0x03, 0x04}`, the S3FrameParser split a DLE+XX pair at the payload/checksum boundary. Reunite the bytes before stripping (`relevant + bytes([chk_byte])`), then always remove the trailing chk_byte from the result (`stripped[:-1]`) — chk_byte is the wire checksum, never payload. + +**Body/footer split:** Accumulate all frame contributions (data frames + terminator) into `all_bytes`. Then: +- `body = all_bytes[:-26]` (variable length) +- `footer = all_bytes[-26:]` (always 26 bytes — extracted from terminator content) + +#### D.3.3 Footer (26 bytes) + +The footer terminates the N00 file. Its bytes come directly from the terminator A5 frame's inner content — do NOT reconstruct from event metadata. + +| Offset | Length | Field | Notes | +|---|---|---|---| +| 0 | 2 | `0e 08` | Fixed marker | +| 2 | 8 | ts1 | Start timestamp (8B big-endian) | +| 10 | 8 | ts2 | Stop timestamp (8B big-endian) | +| 18 | 6 | `00 01 00 02 00 00` | Fixed | +| 24 | 2 | CRC | 2-byte CRC — algorithm unconfirmed | + +**CRC:** The 2-byte CRC at footer[24:26] has an unconfirmed algorithm. In M529LIY6.N00 it reads `fe da`. Attempts to match CRC-16/CCITT, CRC-16/IBM, CRC-32 (truncated), and 40+ polynomial/init combinations all failed. The writer copies it verbatim from the terminator frame. + +### D.4 MLG File Format — Monitor Log + +**File layout:** `[308B header] [N × 292B records]` + +#### D.4.1 MLG Header (308 bytes) + +| Offset | Length | Field | Notes | +|---|---|---|---| +| 0x00 | 22 | common header | prefix + `22 01 0e a0` type tag | +| 0x16 | 16 | unknown | observed as zeros in BE11529.MLG | +| 0x2A | 8 | serial number | null-padded ASCII (e.g. `"BE11529"`) | +| 0x32 | remainder | zero pad | pads to 308 bytes total | + +#### D.4.2 MLG Record (292 bytes each) + +| Offset | Length | Field | Notes | +|---|---|---|---| +| 0 | 2 | CRC | 2-byte CRC — algorithm unconfirmed; write as `00 00` | +| 2 | 4 | `22 01 0e 80` | Record marker | +| 6 | 8 | ts1 | Start timestamp (8B big-endian) | +| 14 | 8 | ts2 | Stop timestamp (8B big-endian); zeros if no stop | +| 22 | 4 | flags | Record type flags (see below) | +| 26 | 10 | serial | Null-padded ASCII serial number | +| 36 | variable | text | Type-dependent content | +| — | remainder | zero pad | pads to 292 bytes total | + +**Record flags:** + +| Value | Meaning | +|---|---| +| `ff ff 00 00` | Monitoring start with no stop recorded | +| `01 00 02 00` | Triggered event (has ts1 + ts2) | +| `02 00 00 00` | Monitoring interval (has ts1 + ts2) | + +**Text content for triggered events (`flags = 01 00 02 00`):** + +| Byte | Field | +|---|---| +| 0 | `0x08` | +| 1–8 | ts1 copy (8B big-endian) | +| 9+ | `"Geo: X.XXX in/s\x00"` ASCII geo threshold | + +#### D.4.3 MLG CRC + +The 2-byte CRC at record[0:2] uses an unconfirmed algorithm. Tested against CRC-16/CCITT, CRC-16/IBM, CRC-32 (truncated), word sums, XOR variants, and 40+ polynomial/init combinations — none matched. The writer emits `00 00`. Blastware may reject files with incorrect CRCs (impact on import unknown — TODO: test). + +### D.5 Filename Encoding ✅ CONFIRMED 2026-04-21 + +Blastware assigns waveform filenames of the form `M`, where: + +#### D.5.1 Serial Prefix + +`"M"` + last 3 decimal digits of the device serial number. + +Example: serial `"BE11529"` → prefix `"M529"`. + +#### D.5.2 Stem — 4-character base-36 timestamp encoding + +``` +stem_int = floor((event_local_time − 1985-01-01T00:00:00_local) / 1296) +stem = 4-character uppercase base-36 string of stem_int +``` + +- **Unit:** 1296 seconds = 36² seconds ≈ 21.6 minutes per stem increment +- **Epoch:** January 1, 1985, 00:00:00 local time (Instantel founding year) +- **Alphabet:** `"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"` (digits then uppercase letters) +- **Collision:** Events within the same 21.6-minute window share a stem; their extension distinguishes them + +Confirmed against 6 events (April 1–9, 2026): + +| Stem | Event time | Epoch estimate | +|---|---|---| +| LIY6 | 2026-04-01 00:28 | 1985-01-01 00:23 local | +| LJ31 | 2026-04-03 15:20 | 1985-01-01 00:22 local | +| LJ8V | 2026-04-06 18:52 | 1985-01-01 00:25 local | +| LJDY | 2026-04-09 12:46 | 1985-01-01 00:23 local | + +All 6 stems match exactly. Epoch estimates converge within ±7 minutes of midnight Jan 1 1985. + +#### D.5.3 Extension taxonomy + +Third character of extension is always `'0'`. File type is identified by extension, not by the type tag in the header (all waveform extensions share type tag `00 12 03 00`). + +| Extension | Recording mode | Sample rate | Status | +|---|---|---|---| +| `.N00` | Single Shot (0x00) | 1024 sps | ✅ CONFIRMED | +| `.9T0` | Continuous (0x01) | 1024 sps | ✅ CONFIRMED | +| `.490` | ? | ? | ❓ observed from M529LJ8V.490 | +| `.5K0` | ? | ? | ❓ observed from M529LJDY.5K0 | +| `.980` | ? | ? | ❓ observed from M529LJDY.980 | +| `.ML0` | ? | ? | ❓ observed from M529LJDY.ML0 (167s duration; possibly Histogram) | + +**Why 5 extensions for "Continuous"?** Binary analysis of all 6 example files shows that `.9T0`, `.490`, `.5K0`, `.980`, `.ML0` are byte-for-byte identical in all metadata regions (compliance anchor block, channel descriptor blocks `Tran/Vert/Long/MicL`). The A5 frame 7 body reflects the **session-start** compliance config, not the per-event capture config. All 5 files show recording_mode=0x01 and sample_rate=1024 in the body. The extension must therefore encode the **capture-time** compliance state — likely a combination of recording mode, sample rate, and possibly mic units or other options. This cannot be determined from file body alone without capture-time compliance data from the 0C record sub_code and the actual waveform sample count. + +**DLE-shift offset note for reading recording_mode from N00/9T0 body:** + +The compliance block in the file body has been through `_strip_inner_frame_dles`. The 0x10 constant at logical `anchor−7` (between recording_mode and sample_rate_HI) gets stripped when sample_rate_HI = `0x04` (1024 sps), because `0x10` precedes `0x04 ∈ {0x02,0x03,0x04}`. After stripping, the anchor shifts left by 1, so: + +| 1024 sps (strip occurs) | 2048 or 4096 sps (no strip) | +|---|---| +| `file[anc−7]` = recording_mode | `file[anc−8]` = recording_mode | +| `file[anc−6:anc−4]` = sample_rate | `file[anc−6:anc−4]` = sample_rate | + +For 1024 sps files, the expected file bytes around the anchor are: +``` +file[anc−9]: mode_prefix (0x00 for Single Shot/Continuous; 0x10 for Histogram) +file[anc−8]: 0x00 (was recording_mode, but shifted away — now reads 0x00 for mode_prefix) +file[anc−7]: recording_mode (0x00=Single Shot, 0x01=Continuous, etc.) +file[anc−6]: 0x04 (sample_rate_HI for 1024 sps) +file[anc−5]: 0x00 (sample_rate_LO) +file[anc−4]: histogram_interval_HI +file[anc−3]: histogram_interval_LO +``` + +--- + *All findings reverse-engineered from live RS-232 bridge captures.* *Cross-referenced from 2026-03-02 with Instantel MiniMate Plus Operator Manual (716U0101 Rev 15).* *This is a living document — append changelog entries and timestamps as new findings are confirmed or corrected.* \ No newline at end of file diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py new file mode 100644 index 0000000..7dd506c --- /dev/null +++ b/minimateplus/blastware_file.py @@ -0,0 +1,777 @@ +""" +blastware_file.py — Blastware binary file codec for bidirectional interoperability. + +Reads and writes the proprietary Instantel/Blastware file formats: + .N00 — Single-shot triggered waveform event + .9T0 — Continuous-mode triggered waveform event + .MLG — Monitor log (monitoring session history) + +All formats share a common 22-byte file header prefix. Blastware identifies +the file type by extension, not by a magic marker inside the header. + +IMPORTANT — .N00 vs .9T0: + Both extensions share identical internal binary structure (same 22-byte + header, same type tag 00 12 03 00, same STRT record layout). Blastware + uses the extension to identify the recording mode: + .N00 → single-shot (0C waveform sub_code = 0x10) + .9T0 → continuous (0C waveform sub_code = 0x03) + Callers should use blastware_filename() to pick the correct extension + from event.record_type. Histogram-mode file extension is unknown (TODO). + +─── File structure overview ───────────────────────────────────────────────────── + +N00 (single-shot waveform, confirmed from example-events/4-3-26-multi/M529LIY6.N00): + + [22B header] [21B STRT record] [body bytes] [26B footer] + + Header (22 bytes): + 10 00 01 80 00 00 — fixed prefix + 49 6e 73 74 61 6e 74 65 6c 00 — b'Instantel\x00' + 07 2c — fixed + 00 12 03 00 — N00 type marker + + STRT record (21 bytes, immediately follows header): + 53 54 52 54 — b'STRT' + ff fe — fixed (2 bytes) + [key4] — 4-byte waveform event key + [key4] — 4-byte waveform event key (repeated) + [zeros] — 7 bytes padding + [rectime] — uint8 record time in seconds + + Body (variable — reconstructed from A5 frame data): + The body bytes are derived from the raw A5 frame wire content, specifically + from the DLE-decoded representation of each frame's contribution. See the + _frame_body_bytes() helper for the exact algorithm. + + Footer (26 bytes): + 0e 08 + [ts1: 8B big-endian timestamp] — start timestamp + [ts2: 8B big-endian timestamp] — stop timestamp + 00 01 00 02 00 00 + [crc: 2B] — CRC (algorithm unconfirmed; written as 0x00 0x00 placeholder) + + Timestamp format (big-endian, 8 bytes): + [day] [month] [year_HI] [year_LO] [0x00] [hour] [min] [sec] + +MLG (monitor log, confirmed from example-events/4-3-26-multi/BE11529.MLG): + + [308B header] [N × 292B records] + + Header (308 bytes): + Offset 0x00: 10 00 01 80 00 00 Instantel\x00 07 2c 22 01 0e a0 — fixed (16B) + Offset 0x10: ... (unknown structure, written as zeros + serial) + Offset 0x2A: serial number (8 bytes, null-padded ASCII, e.g. "BE11529") + ... zero-padded to 308 bytes total + + Record (292 bytes each): + [2B CRC] — unknown algorithm; written as 0x00 0x00 + 22 01 0e 80 — record marker + [ts1: 8B big-endian timestamp] — start time + [ts2: 8B big-endian timestamp] — stop time (zeros if no stop) + [4B flags] — see MLG_FLAGS_* constants below + [10B serial] — null-padded serial number ASCII + [text] — for trigger records: [0x08][8B ts1_copy] then ASCII "Geo: X.XXX in/s" + for monitoring records: b'' (or minimal separator) + [zero-padded to 292 bytes] + +─── Critical implementation notes ────────────────────────────────────────────── + +N00 body reconstruction algorithm (confirmed 2026-04-21 from verification against +M529LIY6.N00 using raw_s3_20260403_153508.bin capture): + + The N00 body bytes come from the A5 frame content, stripped of DLE-framing + artifacts. Each A5 frame contributes a different slice of its data section, + with DLE+{0x02,0x03,0x04} byte pairs stripped. + + Skip amounts per frame index (offsets into frame.data): + A5[0] (probe): data[strt_pos + 21 + 7] (skip header + STRT record) + strt_pos found by searching frame.data[7:] for b'STRT'; + the contribution starts at strt_pos + 21 within data[7:] + which equals strt_pos + 21 + 7 within frame.data. + A5[1]: data[13] (skip 7-byte frame.data prefix + 6 header bytes) + A5[2..N]: data[12] (skip 7-byte frame.data prefix + 5 header bytes) + Terminator A5: data[11] (1 byte less than chunk frames; terminator inner header + is 4 bytes instead of 5 — confirmed 2026-04-21) + + DLE strip rule (applied AFTER slicing): + Strip any 0x10 byte that is immediately followed by 0x02, 0x03, or 0x04. + This undoes the DLE-escape that S3FrameParser preserves as literal pairs. + Applied to frame.data[skip:] + bytes([frame.chk_byte]) together, then + conditionally exclude the trailing chk_byte from the output. + + chk_byte absorption: + When frame.data[-1] == 0x10 AND frame.chk_byte ∈ {0x02, 0x03, 0x04}, + the last byte of frame.data is the DLE prefix of a split DLE+chk pair. + Including chk_byte in the strip buffer allows the pair to be stripped as + a unit. After stripping, the trailing chk_byte is ALWAYS removed — because + _strip_inner_frame_dles keeps the byte after the DLE (the chk_byte value), + and that value is the checksum, never payload. This applies to all three + cases (chk ∈ {0x02, 0x03, 0x04}) identically. + +MLG CRC: + The algorithm that produces the 2-byte CRC at the start of each MLG record + is unknown. All examined records use non-zero values that do not match + CRC-16/CCITT, CRC-16/IBM, CRC-32 (truncated), word sums, XOR variants, or + any of the 40+ polynomial/init combinations tested. The writer emits 0x0000. + This produces files that Blastware may reject or display without the CRC check — + the exact impact on BW import is unknown (TODO: test). + +─── Public API ────────────────────────────────────────────────────────────────── + + blastware_filename(event, serial) + Return the correct Blastware filename (e.g. "M529LIY6.N00") for an event. + Uses event.record_type to pick .N00 (single-shot) vs .9T0 (continuous). + + write_n00(event, a5_frames, path) + Create a .N00 or .9T0 waveform file from an Event and the full A5 frame + list (include_terminator=True required when calling read_bulk_waveform_stream). + Identical binary format for both extensions — caller picks the path/ext. + + read_n00(path) → Event + Parse a .N00 file into an Event object with waveform data populated. + (Not yet implemented — placeholder raises NotImplementedError.) + + write_mlg(entries, serial, path) + Create a .MLG file from a list of MonitorLogEntry objects. + + read_mlg(path) → list[MonitorLogEntry] + Parse a .MLG file into MonitorLogEntry objects. + (Not yet implemented — placeholder raises NotImplementedError.) +""" + +from __future__ import annotations + +import datetime +import struct +from pathlib import Path +from typing import Optional, Union + +from .framing import S3Frame +from .models import Event, MonitorLogEntry, Timestamp + +# ── File header constants ───────────────────────────────────────────────────── + +# Common 16-byte prefix shared by N00 and MLG (confirmed from binary inspection). +_FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c" +# = 10 00 01 80 00 00 49 73 74 61 6e 74 65 6c 00 07 2c (17 bytes) +# Confirmed breakdown: 10 00 01 80 00 00 = fixed; "Instantel\x00" = 10B; 07 2c = fixed + +# Simpler construction: +_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes + +# N00 type tag (4 bytes after common prefix) +_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 offset 0x11..0x14 + +# MLG type tag (4 bytes after common prefix) +_MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14 + +# Total header sizes +_N00_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate. +# From binary: first 22 bytes = header, then STRT at byte 22. +# 17-byte common prefix + 4-byte type tag = 21 bytes. But observed header is 22B. +# Checking: 6 fixed + 10 "Instantel\x00" + 2 "07 2c" = 18B prefix, then 4B type tag = 22B. +# Re-count: b"\x10\x00\x01\x80\x00\x00" = 6B + b"Instantel\x00" = 10B + b"\x07\x2c" = 2B = 18B prefix. +_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 18 bytes +_N00_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅ +_MLG_HEADER_SIZE = 308 # confirmed from BE11529.MLG + +# MLG record marker (4 bytes after 2-byte CRC at start of each record) +_MLG_RECORD_MARKER = b"\x22\x01\x0e\x80" +_MLG_RECORD_SIZE = 292 # bytes per record (confirmed from BE11529.MLG) + +# MLG record flags (4 bytes at record[22:26]) +# Confirmed from BE11529.MLG binary inspection: +MLG_FLAGS_START_ONLY = b"\xff\xff\x00\x00" # monitoring start with no stop +MLG_FLAGS_TRIGGER = b"\x01\x00\x02\x00" # triggered event (has ts1 + ts2) +MLG_FLAGS_INTERVAL = b"\x02\x00\x00\x00" # monitoring interval (has ts1 + ts2) + + +# ── Timestamp helpers ───────────────────────────────────────────────────────── + +def _encode_ts_be(ts: Optional[datetime.datetime]) -> bytes: + """ + Encode a datetime as an 8-byte big-endian Blastware timestamp. + + Format (N00 and MLG record timestamps): + [day][month][year_HI][year_LO][0x00][hour][min][sec] + + Big-endian year confirmed from M529LIY6.N00 footer: + footer bytes [2..9] = 01 04 07 ea 00 00 1c 08 + → day=1 month=4 year=0x07ea=2026 hour=0 min=28 sec=8 ✅ + + Returns 8 zero bytes if ts is None. + """ + if ts is None: + return bytes(8) + return bytes([ + ts.day, + ts.month, + (ts.year >> 8) & 0xFF, + ts.year & 0xFF, + 0x00, + ts.hour, + ts.minute, + ts.second, + ]) + + +def _decode_ts_be(raw: bytes) -> Optional[datetime.datetime]: + """ + Decode an 8-byte big-endian Blastware timestamp. + + Returns None if the bytes are all zero or structurally invalid. + """ + if len(raw) < 8 or raw == bytes(8): + return None + day = raw[0] + month = raw[1] + year = (raw[2] << 8) | raw[3] + hour = raw[5] + minute = raw[6] + sec = raw[7] + try: + return datetime.datetime(year, month, day, hour, minute, sec) + except ValueError: + return None + + +def _ts_from_model(ts: Optional[Timestamp]) -> Optional[datetime.datetime]: + """Convert a models.Timestamp to datetime.datetime, or None.""" + if ts is None: + return None + try: + return datetime.datetime(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second) + except (ValueError, TypeError): + return None + + +# ── DLE strip helper ────────────────────────────────────────────────────────── + +def _strip_inner_frame_dles(data: bytes) -> bytes: + """ + Strip DLE (0x10) framing markers from A5 inner-frame content. + + The A5 (bulk waveform stream) response body contains DLE-encoded sub-frame + structure. S3FrameParser preserves DLE+XX pairs as two literal bytes in + frame.data. Only the DLE marker byte needs to be removed; the following + byte is actual payload content. + + Rule: when 0x10 is immediately followed by {0x02, 0x03, 0x04}, strip the + 0x10 (DLE marker) and keep the following byte as payload. + + Lone 0x10 bytes not followed by {0x02, 0x03, 0x04} are kept as-is. + + Confirmed correct by verifying reconstructed N00 body against M529LIY6.N00: + - 0x10 0x02 in terminator → 0x02 kept ✓ + - 0x10 0x04 in terminator (month byte) → 0x04 kept ✓ + """ + out = bytearray() + i = 0 + while i < len(data): + b = data[i] + if b == 0x10 and i + 1 < len(data) and data[i + 1] in {0x02, 0x03, 0x04}: + # Strip the DLE marker; the next byte is payload and will be appended + # in the next loop iteration. + i += 1 + continue + out.append(b) + i += 1 + return bytes(out) + + +def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes: + """ + Extract the N00 body contribution from one A5 S3Frame. + + The contribution is frame.data[skip:] with inner-frame DLE pairs stripped + per _strip_inner_frame_dles(). The chk_byte is temporarily appended before + stripping to handle the split-pair edge case where a DLE at the end of + frame.data is paired with chk_byte. + + Split-pair edge case (confirmed for A5[8] of M529LIY6.N00, 2026-04-21): + + S3FrameParser appends DLE+XX pairs as two literal bytes when XX ∉ {DLE, ETX}. + When the LAST occurrence of such a pair straddles the payload/checksum boundary + (i.e., DLE is the last byte of raw_payload and XX is the checksum), the parser + splits them: + - DLE ends up as the last byte of frame.data (frame.data[-1] == 0x10) + - XX is stored as frame.chk_byte + + To strip the pair correctly, we reunite the bytes before calling the strip + function. Since chk_byte is the checksum (not payload data), it is excluded + from the final output regardless of whether it was part of a pair. + + Post-strip chk_byte removal (ALL cases): + _strip_inner_frame_dles strips the 0x10 and KEEPS chk_byte in all cases. + Chk_byte is always the checksum (not payload), so always strip it off. + + Args: + frame: S3Frame with frame.data and frame.chk_byte populated. + skip: Number of leading bytes in frame.data to exclude (frame header). + + Returns: + bytes — the N00 body contribution for this frame. + """ + if skip >= len(frame.data): + return b"" + + relevant = frame.data[skip:] + + # Detect split DLE+chk pair at the frame boundary. + has_split_pair = ( + len(relevant) > 0 + and relevant[-1] == 0x10 + and frame.chk_byte in {0x02, 0x03, 0x04} + ) + + if has_split_pair: + # Reunite the split pair so the strip function sees both bytes together. + buf = relevant + bytes([frame.chk_byte]) + stripped = _strip_inner_frame_dles(buf) + # _strip_inner_frame_dles strips the DLE (0x10) and KEEPS chk_byte. + # chk_byte is the received checksum — never payload — so remove it. + # This is correct for all values in {0x02, 0x03, 0x04}. + if stripped: + stripped = stripped[:-1] + return stripped + else: + return _strip_inner_frame_dles(relevant) + + +# ── Filename helper ─────────────────────────────────────────────────────────── + +_INSTANTEL_EPOCH = datetime.datetime(1985, 1, 1, 0, 0, 0) +""" +Instantel timestamp epoch — January 1, 1985, 00:00:00 local time. +Confirmed 2026-04-21: stem values for 6 independent events (April 1–9, 2026) +all converge to this epoch when decoded as floor(seconds_since_epoch / 1296). +1985 is the year Instantel was founded. +""" + +_STEM_UNIT_SEC = 1296 # = 36^2 seconds ≈ 21.6 minutes per stem unit + +_STEM_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +# Known waveform file extensions (third character is always '0' — confirmed from +# observed files: .N00, .9T0, .490, .5K0, .980, .ML0). +# +# Confirmed mappings: +# .N00 → single-shot (recording_mode=0 in compliance anchor at file[anc-7]) +# .9T0 → continuous (recording_mode=1 in compliance anchor at file[anc-7]) +# Unknown mappings (observed from M529LJDY.* and M529LJ8V.*): +# .490 → ? (April 6, 13 sec record) +# .5K0 → ? (April 9, 10 sec record) +# .980 → ? (April 9, 7 sec record) +# .ML0 → ? (April 9, 167 sec record — possibly Histogram or Histogram+Continuous) +# +# IMPORTANT — extension encodes capture-time config, NOT session-start config: +# Binary analysis (2026-04-21) shows that the compliance anchor region in the +# file body encodes the SESSION-START config (A5 frame 7), not the per-event +# config. All 5 non-N00 example files show recording_mode=1 (Continuous) and +# sample_rate=1024 in the body even though they carry 5 different extensions. +# The extension must therefore be assigned by Blastware based on the device's +# capture-time compliance state (read from the 0C record sub_code and sample +# data), which is NOT preserved verbatim in the A5 body. +# +# How to READ recording_mode from a .N00/.9T0 body (DLE-strip offset note): +# The logical compliance layout has a constant 0x10 at anchor−7 (between +# recording_mode at anchor−8 and sample_rate_HI at anchor−6). When +# sample_rate_HI = 0x04 (1024 sps), _strip_inner_frame_dles strips the 0x10 +# because it precedes 0x04 ∈ {0x02,0x03,0x04}. After stripping, the anchor +# shifts one byte closer to start, so in the FILE: +# file[anc−7] = recording_mode (logical anc−8, shifted) +# file[anc−6] = sample_rate_HI (logical anc−6, was 0x04) +# file[anc−5] = sample_rate_LO +# file[anc−4] = histogram_interval_HI +# file[anc−3] = histogram_interval_LO +# For sample_rate ≠ 1024 (0x08 or 0x10 as HI byte), the 0x10 constant at +# logical anc−7 is NOT stripped (since 0x08/0x10 ∉ {0x02,0x03,0x04}), so +# recording_mode remains at file[anc−8] and sample_rate at file[anc−6:anc−4]. +# +# Multiple events within the same ~21.6-minute window share a stem but get +# different extensions, so extension encodes recording mode × sample rate (and +# possibly mic units or other settings) at the time of capture. + + +def _make_stem(ts_local: datetime.datetime) -> str: + """ + Encode a local timestamp as a 4-character uppercase base-36 stem. + + Algorithm (confirmed 2026-04-21 from 6 known file/timestamp pairs): + stem_int = floor((ts_local - Jan_1_1985_midnight_local) / 1296_seconds) + stem = 4-char uppercase base-36 encoding of stem_int + + Unit = 36² = 1296 seconds ≈ 21.6 minutes. Events within the same 1296-second + window receive the same stem; their extension distinguishes them. + """ + delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds()) + n = delta_sec // _STEM_UNIT_SEC + s = "" + for _ in range(4): + s = _STEM_CHARS[n % 36] + s + n //= 36 + return s + + +def blastware_filename(event: Event, serial: str) -> str: + """ + Return the correct Blastware waveform filename for an event. + + Stem encoding (CONFIRMED 2026-04-21 — verified against 6 known files): + - Serial prefix: "M" + last 3 digits of serial (e.g. "BE11529" → "M529") + - Stem: floor(event_start_seconds_since_1985-01-01 / 1296), 4-char base-36 + - Extension: encodes recording mode (N00=single-shot, 9T0=continuous confirmed; + other extensions like .490, .5K0, .980, .ML0 observed but not decoded) + + Note: the extension space is larger than N00/9T0. Multiple events within + the same ~21.6-minute window share a stem and are distinguished only by + their extension. This function returns .N00 or .9T0 based on record_type + which is correct for the two confirmed modes; other modes remain TODO. + + Args: + event: Event object with record_type and timestamp set. + serial: Device serial number string (e.g. "BE11529"). + + Returns: + Filename string (e.g. "M529LIY6.N00"). + """ + # Determine extension from record_type + if event.record_type == "continuous": + ext = ".9T0" + else: + # Default to .N00 for single-shot and unknown modes + ext = ".N00" + + # Serial prefix: "M" + last 3 digits (e.g. BE11529 → M529) + serial_digits = "".join(c for c in serial if c.isdigit()) + prefix = "M" + serial_digits[-3:] if len(serial_digits) >= 3 else "M000" + + # Stem from event start timestamp + if event.timestamp is not None: + try: + ts_local = datetime.datetime( + event.timestamp.year, event.timestamp.month, event.timestamp.day, + event.timestamp.hour, event.timestamp.minute, event.timestamp.second, + ) + stem = _make_stem(ts_local) + except (ValueError, TypeError, AttributeError): + stem = "0000" + else: + stem = "0000" + + return prefix + stem + ext + + +# ── N00 file writer ─────────────────────────────────────────────────────────── + +def write_n00( + event: Event, + a5_frames: list[S3Frame], + path: Union[str, Path], +) -> None: + """ + Write a Blastware .N00 waveform file from a downloaded event. + + Args: + event: Event object (populated by get_events() or download_waveform()). + Used for the STRT record (key, rectime) and footer timestamps. + a5_frames: Complete A5 frame list INCLUDING the terminator frame + (page_key=0x0000). Pass include_terminator=True to + read_bulk_waveform_stream() when collecting frames. + Must have at least 2 frames (probe + terminator). + path: Destination file path. Parent directory must exist. + Extension is not enforced — caller should use ".N00". + + File layout: + [22B header] [21B STRT] [body bytes] [26B footer] + + Raises: + ValueError: if a5_frames is empty or has no terminator (page_key=0). + OSError: if the file cannot be written. + + Confirmed correct N00 body reconstruction against M529LIY6.N00 (2026-04-21). + """ + if not a5_frames: + raise ValueError("a5_frames must not be empty") + + path = Path(path) + + # ── Extract STRT record from probe frame ──────────────────────────────── + # The STRT record (21 bytes) lives verbatim inside A5[0].data[7:]. + # It is stored as-is in the N00 file — do NOT reconstruct it from Event + # fields, as bytes [10:14] and [14:20] contain device-specific values + # (not simply key4 repeated or zero-padded). Confirmed 2026-04-21. + # + # STRT layout (21 bytes, observed in M529LIY6.N00): + # [0:4] b'STRT' + # [4:6] 0xff 0xfe (fixed) + # [6:10] key4 (event key) + # [10:14] device-specific field (NOT a key4 repeat) + # [14:20] device-specific fields (NOT zeros) + # [20] rectime uint8 seconds + w0 = a5_frames[0].data[7:] + strt_pos_w0 = w0.find(b"STRT") + if strt_pos_w0 >= 0: + strt = bytes(w0[strt_pos_w0 : strt_pos_w0 + 21]) + else: + # Fallback: construct a minimal STRT if probe frame lacks it + key4 = event._waveform_key if hasattr(event, '_waveform_key') and event._waveform_key else bytes(4) + rectime = event.rectime_seconds if event.rectime_seconds is not None else 0 + strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF]) + if len(strt) != 21: + raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}") + + strt_pos_in_w0 = strt_pos_w0 if strt_pos_w0 >= 0 else 0 + + # ── Build N00 header ───────────────────────────────────────────────────── + header = _FILE_HEADER_PREFIX + _N00_TYPE_TAG + assert len(header) == _N00_HEADER_SIZE, f"N00 header must be {_N00_HEADER_SIZE} bytes" + + # ── Build body from A5 frames ──────────────────────────────────────────── + # The N00 body is reconstructed from ALL A5 frames (data + terminator). + # The terminator frame's contribution includes the 26-byte footer at its end. + # + # Reconstruction layout (confirmed from M529LIY6.N00, 2026-04-21): + # all_bytes = contributions from A5[0..N] + terminator_contribution + # body = all_bytes[:-26] (everything except the last 26 bytes) + # footer = all_bytes[-26:] (last 26 bytes = the N00 footer) + # + # The footer bytes come directly from the terminator frame's inner content — + # using them verbatim ensures timestamps match the device's recorded values. + + # Separate terminator from data frames + body_frames = a5_frames + term_frame: Optional[S3Frame] = None + if a5_frames and a5_frames[-1].page_key == 0x0000: + body_frames = a5_frames[:-1] + term_frame = a5_frames[-1] + + # Skip for A5[0]: 7-byte frame.data prefix + strt_pos_in_w0 + 21 STRT bytes. + # strt_pos_in_w0 was already found in the STRT extraction block above. + probe_skip = 7 + strt_pos_in_w0 + 21 + + all_bytes = bytearray() + + for fi, frame in enumerate(body_frames): + if fi == 0: + skip = probe_skip + elif fi == 1: + skip = 13 # 7-byte frame.data prefix + 6-byte first-chunk header + else: + skip = 12 # 7-byte frame.data prefix + 5-byte chunk header + all_bytes.extend(_frame_body_bytes(frame, skip)) + + # Terminator contributes its content, which ends with the 26-byte footer. + # skip=11 (not 12) because the terminator's inner frame header is 4 bytes, + # one shorter than chunk frames' 5-byte inner header. Confirmed 2026-04-21. + if term_frame is not None: + all_bytes.extend(_frame_body_bytes(term_frame, 11)) + + if len(all_bytes) >= 26: + body = bytes(all_bytes[:-26]) + footer = bytes(all_bytes[-26:]) + else: + # Fallback: no terminator or very short stream → build footer from event metadata + body = bytes(all_bytes) + start_dt = _ts_from_model(event.timestamp) + stop_dt: Optional[datetime.datetime] = None + if start_dt is not None and event.rectime_seconds: + stop_dt = start_dt + datetime.timedelta(seconds=event.rectime_seconds) + footer = ( + b"\x0e\x08" + + _encode_ts_be(start_dt) + + _encode_ts_be(stop_dt) + + b"\x00\x01\x00\x02\x00\x00" + + b"\x00\x00" # CRC placeholder + ) + + # ── Write file ─────────────────────────────────────────────────────────── + with open(path, "wb") as f: + f.write(header) + f.write(strt) + f.write(body) + f.write(footer) + + +def read_n00(path: Union[str, Path]) -> Event: + """ + Parse a Blastware .N00 file into an Event object. + + NOT YET IMPLEMENTED. + + Args: + path: Path to the .N00 file. + + Returns: + Event object with waveform data populated. + + Raises: + NotImplementedError: always (pending implementation). + """ + raise NotImplementedError("read_n00() is not yet implemented") + + +# ── MLG file writer ─────────────────────────────────────────────────────────── + +def _build_mlg_header(serial: str) -> bytes: + """ + Build the 308-byte MLG file header. + + Header structure (confirmed from BE11529.MLG binary inspection): + Offset 0x00: 10 00 01 80 00 00 Instantel\x00 07 2c 22 01 0e a0 (22B) + Offset 0x16: ... (16B unknown — observed as zeros in BE11529.MLG) + Offset 0x2A: serial number (8 bytes, null-padded ASCII) + ... rest zero-padded to 308 bytes + + The serial string "BE11529" appears at offset 0x2A (42 decimal). + """ + buf = bytearray(_MLG_HEADER_SIZE) + + # Common prefix + MLG type tag + prefix = _FILE_HEADER_PREFIX + _MLG_TYPE_TAG # 22 bytes + buf[0:len(prefix)] = prefix + + # Serial number at offset 0x2A + serial_bytes = serial.encode("ascii", errors="replace")[:8] + serial_padded = serial_bytes.ljust(8, b"\x00") + buf[0x2A : 0x2A + 8] = serial_padded + + return bytes(buf) + + +def _build_mlg_record( + entry: MonitorLogEntry, + serial: str, +) -> bytes: + """ + Build one 292-byte MLG record from a MonitorLogEntry. + + Record layout (confirmed from BE11529.MLG binary inspection): + [0:2] CRC — 2-byte CRC (algorithm unknown; written as 0x0000) + [2:6] marker — 22 01 0e 80 + [6:14] ts1 — 8B big-endian start timestamp + [14:22] ts2 — 8B big-endian stop timestamp + [22:26] flags — 4B record flags (see MLG_FLAGS_* constants) + [26:36] serial — 10B null-padded serial number + [36:] text — for triggered events: [0x08][8B ts1_copy]["Geo: X.XXX in/s"] + for monitoring intervals: b"" or minimal separator + [... zero-padded to 292 bytes] + + Flags based on entry type: + - MonitorLogEntry with start_time only (no stop_time): MLG_FLAGS_START_ONLY + - MonitorLogEntry with both times and geo_threshold_ips set: MLG_FLAGS_TRIGGER + - MonitorLogEntry with both times (monitoring interval): MLG_FLAGS_INTERVAL + + The triggered-event text block (flags = MLG_FLAGS_TRIGGER): + [0x08] [ts1: 8B] [ASCII "Geo: X.XXX in/s\x00"] + Confirmed from BE11529.MLG records at offset 0x0134 and 0x0258. + """ + buf = bytearray(_MLG_RECORD_SIZE) + + start_dt = ( + datetime.datetime( + entry.start_time.year, entry.start_time.month, entry.start_time.day, + entry.start_time.hour, entry.start_time.minute, entry.start_time.second, + ) + if entry.start_time else None + ) + stop_dt = ( + datetime.datetime( + entry.stop_time.year, entry.stop_time.month, entry.stop_time.day, + entry.stop_time.hour, entry.stop_time.minute, entry.stop_time.second, + ) + if entry.stop_time else None + ) + + # [0:2] CRC placeholder + buf[0:2] = b"\x00\x00" + + # [2:6] Record marker + buf[2:6] = _MLG_RECORD_MARKER + + # [6:14] ts1 + buf[6:14] = _encode_ts_be(start_dt) + + # [14:22] ts2 + buf[14:22] = _encode_ts_be(stop_dt) + + # [22:26] flags + if stop_dt is None: + flags = MLG_FLAGS_START_ONLY + elif entry.geo_threshold_ips is not None: + flags = MLG_FLAGS_TRIGGER + else: + flags = MLG_FLAGS_INTERVAL + buf[22:26] = flags + + # [26:36] serial (10B null-padded) + serial_bytes = serial.encode("ascii", errors="replace")[:10] + buf[26 : 26 + len(serial_bytes)] = serial_bytes + + # [36:] text content + pos = 36 + if flags == MLG_FLAGS_TRIGGER: + # Extra ts1 copy: [0x08][ts1: 8B] + buf[pos] = 0x08 + pos += 1 + buf[pos : pos + 8] = _encode_ts_be(start_dt) + pos += 8 + + if entry.geo_threshold_ips is not None: + geo_text = f"Geo: {entry.geo_threshold_ips:.3f} in/s\x00".encode("ascii") + buf[pos : pos + len(geo_text)] = geo_text + pos += len(geo_text) + + return bytes(buf) + + +def write_mlg( + entries: list[MonitorLogEntry], + serial: str, + path: Union[str, Path], +) -> None: + """ + Write a Blastware .MLG monitor log file. + + Args: + entries: List of MonitorLogEntry objects (from get_monitor_log_entries()). + Each entry produces one 292-byte record in the file. + serial: Device serial number string (e.g. "BE11529"). + Written to the file header and each record. + path: Destination file path. Extension is not enforced — use ".MLG". + + File layout: + [308B header] [N × 292B records] + + Note: The 2-byte CRC at the start of each record is written as 0x0000. + The CRC algorithm is unknown (see module docstring). + + Raises: + OSError: if the file cannot be written. + """ + path = Path(path) + header = _build_mlg_header(serial) + + with open(path, "wb") as f: + f.write(header) + for entry in entries: + record = _build_mlg_record(entry, serial) + f.write(record) + + +def read_mlg(path: Union[str, Path]) -> list[MonitorLogEntry]: + """ + Parse a Blastware .MLG file into a list of MonitorLogEntry objects. + + NOT YET IMPLEMENTED. + + Args: + path: Path to the .MLG file. + + Returns: + List of MonitorLogEntry objects. + + Raises: + NotImplementedError: always (pending implementation). + """ + raise NotImplementedError("read_mlg() is not yet implemented") diff --git a/minimateplus/client.py b/minimateplus/client.py index 42cf492..f5b1343 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -608,6 +608,7 @@ class MiniMateClient: ) if a5_frames: a5_ok = True + ev._a5_frames = a5_frames # store for write_n00 _decode_a5_metadata_into(a5_frames, ev) _decode_a5_waveform(a5_frames, ev) log.info( @@ -776,6 +777,39 @@ class MiniMateClient: else: log.warning("download_waveform: waveform decode produced no samples") + return a5_frames + + def save_blastware_file(self, event: "Event", path: "Union[str, Path]", serial: str) -> None: + """ + Download the full waveform for *event* and save it as a Blastware- + compatible .N00 / .9T0 file at *path*. + + This is a convenience wrapper that calls download_waveform() (which + performs the complete SUB 5A BULK_WAVEFORM_STREAM download) and then + calls write_n00() from blastware_file.py to encode the result. + + Args: + event: Event object with waveform key populated (from get_events()). + path: Destination file path. Caller should use blastware_filename() + to pick the correct .N00 / .9T0 extension. + serial: Device serial number (e.g. "BE11529") — passed to + blastware_filename() for reference, but the caller supplies + the final path. + """ + from pathlib import Path as _Path + from .blastware_file import write_n00 as _write_n00 + + a5_frames = self.download_waveform(event) + if not a5_frames: + raise RuntimeError( + f"save_blastware_file: no A5 frames received for event#{event.index}" + ) + _write_n00(event, a5_frames, path) + log.info( + "save_blastware_file: wrote %s (%d A5 frames)", + path, len(a5_frames), + ) + # ── Write commands ──────────────────────────────────────────────────────── def push_config_raw( @@ -1324,7 +1358,7 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None: log.warning("waveform record project strings decode failed: %s", exc) -def _decode_a5_metadata_into(frames_data: list[bytes], event: Event) -> None: +def _decode_a5_metadata_into(frames_data: list[S3Frame], event: Event) -> None: """ Search A5 (BULK_WAVEFORM_STREAM) frame data for event-time metadata strings and populate event.project_info. @@ -1352,7 +1386,7 @@ def _decode_a5_metadata_into(frames_data: list[bytes], event: Event) -> None: Modifies event in-place. """ - combined = b"".join(frames_data) + combined = b"".join(f.data for f in frames_data) def _find_string_after(needle: bytes, max_len: int = 64) -> Optional[str]: pos = combined.find(needle) @@ -1376,7 +1410,7 @@ def _decode_a5_metadata_into(frames_data: list[bytes], event: Event) -> None: notes = _find_string_after(b"Extended Notes") if not any([project, client, operator, location, notes]): - log.debug("a5 metadata: no project strings found in %d frames", len(frames_data)) + log.debug("a5 metadata: no project strings found in %d frames (%d bytes)", len(frames_data), len(combined)) return if event.project_info is None: @@ -1402,7 +1436,7 @@ def _decode_a5_metadata_into(frames_data: list[bytes], event: Event) -> None: def _decode_a5_waveform( - frames_data: list[bytes], + frames_data: list[S3Frame], event: Event, ) -> None: """ @@ -1463,7 +1497,7 @@ def _decode_a5_waveform( return # ── Parse STRT record from A5[0] ──────────────────────────────────────── - w0 = frames_data[0][7:] # db[7:] for A5[0] + w0 = frames_data[0].data[7:] # frame.data[7:] for A5[0] strt_pos = w0.find(b"STRT") if strt_pos < 0: log.warning("_decode_a5_waveform: STRT record not found in A5[0]") @@ -1499,7 +1533,7 @@ def _decode_a5_waveform( global_offset = 0 for fi, db in enumerate(frames_data): - w = db[7:] + w = db.data[7:] # frame.data[7:] # A5[0]: waveform begins after the 21-byte STRT record and 6-byte preamble. # Layout: STRT(21B) + null-pad(2B) + 0xFF sentinel(4B) = 27 bytes total. diff --git a/minimateplus/framing.py b/minimateplus/framing.py index 31c1fba..7df3177 100644 --- a/minimateplus/framing.py +++ b/minimateplus/framing.py @@ -457,6 +457,11 @@ class S3Frame: page_lo: int # PAGE_LO from header data: bytes # payload data section (payload[5:], checksum already stripped) checksum_valid: bool + chk_byte: int = 0 # actual checksum byte received from wire (body[-1]) + # needed for N00 file reconstruction: when the last data byte + # is 0x10 and chk_byte ∈ {0x02, 0x03, 0x04}, the DLE+chk pair + # must be included in the DLE-strip operation to correctly + # reconstruct the Blastware binary body. @property def page_key(self) -> int: @@ -592,9 +597,10 @@ class S3FrameParser: return None return S3Frame( - sub = raw_payload[2], - page_hi = raw_payload[3], - page_lo = raw_payload[4], - data = raw_payload[5:], + sub = raw_payload[2], + page_hi = raw_payload[3], + page_lo = raw_payload[4], + data = raw_payload[5:], checksum_valid = (chk_received == chk_computed), + chk_byte = chk_received, ) diff --git a/minimateplus/models.py b/minimateplus/models.py index cdb74d1..1b0de5c 100644 --- a/minimateplus/models.py +++ b/minimateplus/models.py @@ -493,6 +493,10 @@ class Event: # Set by get_events(); required by download_waveform(). _waveform_key: Optional[bytes] = field(default=None, repr=False) + # Raw A5 frames from the full bulk waveform download (full_waveform=True). + # Populated by get_events() when full_waveform=True; used by write_n00(). + _a5_frames: Optional[list] = field(default=None, repr=False) + def __str__(self) -> str: ts = str(self.timestamp) if self.timestamp else "no timestamp" ppv = "" diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 0e2f048..8691559 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -526,7 +526,8 @@ class MiniMateProtocol: *, stop_after_metadata: bool = True, max_chunks: int = 32, - ) -> list[bytes]: + include_terminator: bool = False, + ) -> list[S3Frame]: """ Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event. @@ -542,7 +543,9 @@ class MiniMateProtocol: 4. Termination (offset=_BULK_TERM_OFFSET, counter=last+_BULK_COUNTER_STEP) Device responds with a final A5 frame (page_key=0x0000). - The termination frame (page_key=0x0000) is NOT included in the returned list. + By default the termination frame (page_key=0x0000) is NOT included in the + returned list. Pass include_terminator=True to append it; the blastware_file + writer needs the terminator frame's body to reconstruct the N00 footer. Args: key4: 4-byte waveform key from EVENT_HEADER (1E). @@ -552,11 +555,16 @@ class MiniMateProtocol: hundred KB). Set False to download everything. max_chunks: Safety cap on the number of chunk requests sent (default 32; a typical event uses 9 large frames). + include_terminator: If True, append the terminator A5 frame + (page_key=0x0000) to the returned list. The + terminator carries the N00 footer bytes. + Default False preserves existing caller behaviour. Returns: - List of raw data bytes from each A5 response frame (not including - the terminator frame). Frame indices match the request sequence: - index 0 = probe response, index 1 = first chunk, etc. + List of S3Frame objects from each A5 response frame. Frame indices + match the request sequence: index 0 = probe response, index 1 = first + chunk, etc. If include_terminator=True, the last element is the + terminator frame (page_key=0x0000). Raises: ProtocolError: on timeout, bad checksum, or unexpected SUB. @@ -571,7 +579,7 @@ class MiniMateProtocol: raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}") rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM) # 0xFF - 0x5A = 0xA5 - frames_data: list[bytes] = [] + frames_data: list[S3Frame] = [] counter = 0 # ── Step 1: probe ──────────────────────────────────────────────────── @@ -588,7 +596,7 @@ class MiniMateProtocol: key4.hex(), self._parser.bytes_fed, ) raise - frames_data.append(rsp.data) + frames_data.append(rsp) log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data)) # ── Step 2: chunk loop ─────────────────────────────────────────────── @@ -631,9 +639,11 @@ class MiniMateProtocol: if rsp.page_key == 0x0000: # Device unexpectedly terminated mid-stream (no termination needed). log.debug("5A A5[%d] page_key=0x0000 — device terminated early", chunk_num) + if include_terminator: + frames_data.append(rsp) return frames_data - frames_data.append(rsp.data) + frames_data.append(rsp) if stop_after_metadata and b"Project:" in rsp.data: log.debug("5A A5[%d] metadata found — stopping early", chunk_num) @@ -658,6 +668,8 @@ class MiniMateProtocol: "5A termination response page_key=0x%04X %d bytes", term_rsp.page_key, len(term_rsp.data), ) + if include_terminator: + frames_data.append(term_rsp) except TimeoutError: log.debug("5A no termination response — device may have already closed") diff --git a/sfm/server.py b/sfm/server.py index 407c680..76f3000 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -61,6 +61,7 @@ from minimateplus import MiniMateClient from minimateplus.protocol import ProtocolError from minimateplus.models import CallHomeConfig, ComplianceConfig, DeviceInfo, Event, PeakValues, ProjectInfo, Timestamp from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT +from minimateplus.blastware_file import write_n00, blastware_filename from sfm.cache import SFMCache, get_cache from sfm.database import SeismoDb @@ -848,6 +849,82 @@ def device_event_waveform( return result +@app.get("/device/event/{index}/blastware_file") +def device_event_blastware_file( + index: int, + port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), + baud: int = Query(38400, description="Serial baud rate"), + host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), + tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), +) -> FileResponse: + """ + Download the full waveform for a single event (0-based index) and return it + as a Blastware-compatible binary file (.N00 for single-shot, .9T0 for continuous). + + Supply either *port* (serial) or *host* (TCP/modem). + + The file is written to a temporary path under /tmp and streamed back as a + file download. Blastware can open it directly. + + Performs: POLL startup → get_events(full_waveform=True, stop_after_index=index) + → write_n00() → FileResponse. + """ + log.info( + "GET /device/event/%d/blastware_file port=%s host=%s", + index, port, host, + ) + + try: + def _do(): + with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: + info = client.connect() + events = client.get_events(full_waveform=True, stop_after_index=index) + matching = [ev for ev in events if ev.index == index] + return matching[0] if matching else None, info + ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) + except HTTPException: + raise + except ProtocolError as exc: + raise HTTPException(status_code=502, detail=f"Protocol error: {exc}") from exc + except OSError as exc: + raise HTTPException(status_code=502, detail=f"Connection error: {exc}") from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc + + if ev is None: + raise HTTPException( + status_code=404, + detail=f"Event index {index} not found on device", + ) + + a5_frames = getattr(ev, "_a5_frames", None) + if not a5_frames: + raise HTTPException( + status_code=502, + detail=f"No waveform data received for event index {index} — 5A download failed", + ) + + # Determine serial number from device info + serial = getattr(info, "serial", None) or "UNKNOWN" + + # Build filename using the same algorithm Blastware uses + filename = blastware_filename(ev, serial) + + # Write to /tmp so FastAPI can stream it back + out_path = Path("/tmp") / filename + write_n00(ev, a5_frames, out_path) + log.info( + "blastware_file: wrote %s (%d A5 frames, serial=%s)", + out_path, len(a5_frames), serial, + ) + + return FileResponse( + path=str(out_path), + filename=filename, + media_type="application/octet-stream", + ) + + # ── Write endpoints ─────────────────────────────────────────────────────────── class DeviceConfigBody(BaseModel): -- 2.52.0 From c47e3a3af03d4ba729fd0b84e4e4134c951ee6c0 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Wed, 22 Apr 2026 19:16:05 -0400 Subject: [PATCH 05/40] feat(protocol): update Blastware file format documentation and encoding details --- CLAUDE.md | 22 ++- docs/instantel_protocol_reference.md | 104 ++++++++++--- minimateplus/blastware_file.py | 224 +++++++++++++++++---------- minimateplus/client.py | 1 + sfm/server.py | 8 +- 5 files changed, 250 insertions(+), 109 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a28f3fb..d6f9a5a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1096,7 +1096,27 @@ body) because writing a dial string may require DLE escaping for embedded contro - **Database** — SQLite store for events + monitor log entries; dedup by key; queryable - **Histograms** — decode histogram-mode A5 data (noise floor tracking) -- **Blastware-compatible file output** — `write_n00()` and `write_mlg()` implemented (v0.12.3+). `write_n00` verified byte-perfect vs M529LIY6.N00. Extension mapping: `.N00`=single-shot, `.9T0`=continuous (confirmed); `.490`, `.5K0`, `.980`, `.ML0` observed but not decoded (likely encoding recording mode × sample rate at capture time — not determinable from file body alone). Filename stem algorithm confirmed 2026-04-21: `M<4-char-base36-stem>` where stem = `floor((ts_local − 1985-01-01T00:00:00) / 1296)`, unit = 36² = 1296 s ≈ 21.6 min. +- **Blastware-compatible file output** — `write_n00()` and `write_mlg()` implemented (v0.12.3+). `write_n00` verified byte-perfect vs M529LIY6.N00. Extension mapping: **CONFIRMED FALSE 2026-04-21** — extensions are NOT based on recording mode. A continuous-mode event produced `.EI0`, not `.9T0`. The extension alphabet/encoding scheme is unknown; do not infer recording mode from extension or vice versa. Observed extensions: `.N00`, `.9T0`, `.EI0`, `.490`, `.5K0`, `.980`, `.ML0` — mapping to recording mode × sample rate × other settings is unknown. Filename format: `<4-char-base36-stem>` + + **Serial encoding (CONFIRMED 2026-04-22):** `prefix_letter = chr(ord('B') + floor(serial_numeric / 1000))`, `serial3 = f"{serial_numeric % 1000:03d}"`. Examples: BE6907→H907, BE11529→M529, BE14036→P036, BE17353→S353, BE18003→T003. The prefix letter encodes the production generation (batch of 1000 units). + + **Stem encoding (FULLY CONFIRMED 2026-04-22):** stem = 4-char base-36 of `floor(total_seconds / 1296)` where `total_seconds = (event_local_time − 1985-01-01T00:00:00_local)` in seconds. Epoch = `1985-01-01 00:00:00` device local time — confirmed against 3,248 files from 10-year production archive with zero errors. Decode: `event_time = datetime(1985,1,1) + timedelta(seconds=stem_int*1296 + ab_int)`. Example: P036L318.C80H → BE14036, 2025-05-26 15:00:08, Full Histogram. +- **Blastware filename extension — NEW FIRMWARE FULLY DECODED (confirmed 2026-04-21, further confirmed 2026-04-22 from 10-year production archive frequency analysis):** + + Extension format = `AB0T` (4 chars): + - `AB` = 2-char base-36 encoding of `total_seconds % 1296` (seconds within the 21.6-min window, 0–1295); `A = value // 36`, `B = value % 36` + - `0` = always literal digit zero (third character, invariant) + - `T` = event type: `W` = Full Waveform, `H` = Full Histogram + + Combined with the 4-char stem, the full filename encodes a complete second-resolution timestamp. Verified against three S353L4H0.{3M0W,8S0H,9X0W} events (all match to the second) plus large-scale frequency analysis of a 10-year archive. + + **3-day cycle property (confirmed 2026-04-22):** A unit recording at a fixed daily time cycles through exactly **3 extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432` and `1296 / 432 = 3`. The three extension values are spaced 432 seconds apart. Confirmed from 10-year archive: the top 3 extensions overall were `CE0H` (95 files), `0E0H` (93), `OE0H` (91) — all three are the 3-day cycle of a 06:00:14 daily call-in time (seconds-in-window = 14, 446, 878; all three have `E` as second character because `14 = E` in base-36 and adding 864 never changes `value % 36` since `864 = 24 × 36`). + + **B character invariance:** For a unit recording at a fixed time of day, the second character `B` of the extension (`value % 36`) **never changes** — only the first character `A` cycles through 3 values. This means same-time-of-day files from different dates all share the same `B` character. + + **Old firmware (S338, 3-char extensions ending in `0`):** encoding unknown. Extension is NOT recording mode. `blastware_filename()` returns `.N00` as a placeholder for old-firmware units. + + **Micromate Series 4** uses a different extension format entirely (observed: `IDFH`, `IDFW`). The `AB0T` formula applies only to MiniMate Plus / V10.72 firmware. - Compliance config encoder — build raw write payloads from a `ComplianceConfig` object - **Test Histogram recording mode (0x03) write via SFM** — confirmed working for Single Shot / Continuous / Histogram+Continuous; Histogram (0x03) needs a live test from a non-Histogram starting state (bare 0x03 in write vs BW's DLE-escaped `10 03`) - **Compliance write anchor-9 cleanup** — when changing recording_mode via SFM, the byte at anchor-9 is not explicitly managed. A spurious `0x10` may persist after Histogram→other mode transitions. Does not affect device operation but differs from BW's byte-perfect output. diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 0491a49..ba16bbc 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -2249,10 +2249,14 @@ Semantic Interpretation <- settings, events, responses --- -## Appendix D — Blastware Binary File Formats (.N00 / .MLG) +## Appendix D — Blastware Binary File Formats (.N00 / .MLG / others) > ✅ CONFIRMED 2026-04-21 — all fields verified by binary diff of reconstructed vs reference > files from the 4-3-26-multi_event capture (M529LIY6.N00, BE11529.MLG). +> +> ⚠️ EXTENSION MAPPING REFUTED 2026-04-21 — earlier assumption that extension encodes +> recording mode is **FALSE**. A continuous-mode event produced `.EI0`, not `.9T0`. +> Extension encoding algorithm is unknown. Do not use extension to infer recording mode. ### D.1 Common File Header (22 bytes) @@ -2271,10 +2275,37 @@ All Blastware files (regardless of type) share an 18-byte prefix followed by a 4 | Extension | Type tag | Description | |---|---|---| -| `.N00` | `00 12 03 00` | Single-shot waveform event | +| `.N00` | `00 12 03 00` | Waveform event (confirmed) | +| `.9T0` | `00 12 03 00` | Waveform event — same type tag as .N00 (assumed; not independently confirmed) | +| `.EI0` | `00 12 03 00` | Waveform event — same type tag (assumed; continuous-mode event observed 2026-04-21) | | `.MLG` | `22 01 0e a0` | Monitor log | -Blastware identifies file type by extension, not by type tag alone. +**Extension encoding — new firmware (V10.72+) FULLY DECODED (confirmed 2026-04-21, further confirmed 2026-04-22):** + +Format: `AB0T` (4 chars): +- `AB` = 2-char base-36 encoding of `total_seconds % 1296` where `total_seconds = (event_local_time − 1985-01-01T00:00:00)` in seconds; `A = value // 36`, `B = value % 36` +- `0` = always literal digit zero (third character) +- `T` = `W` (Full Waveform) or `H` (Full Histogram) + +Base-36 alphabet: `0–9` = 0–9, `A–Z` = 10–35. + +Combined with the 4-char stem, the full filename encodes a complete second-resolution timestamp. + +**Verification — 10-year production archive frequency analysis (2026-04-22):** +A 10-year archive from a long-term monitoring site showed the top 3 extensions across ~3,200 waveform files were `CE0H` (95 files), `0E0H` (93), `OE0H` (91). These are exactly the 3-day cycle of a 06:00:14 daily call-in time: +- `0E0H` → seconds = 0×36+14 = **14** (06:00:**14** — the `14` seconds appears directly) +- `OE0H` → seconds = 24×36+14 = **878** (next calendar day) +- `CE0H` → seconds = 12×36+14 = **446** (day after) + +**3-day cycle property:** A unit recording at a fixed daily time cycles through exactly **3 different extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432`, giving `1296 / 432 = 3` distinct values spaced 432 seconds apart. + +**B character invariance:** The second extension character `B` (= `value % 36`) **never changes** for a fixed daily recording time, because `864 = 24 × 36` — adding 864 never changes the value mod 36. Only the first character `A` cycles through 3 values. All three cycle extensions share the same `B` character (confirmed: `0E0H`, `OE0H`, `CE0H` all have `E` as second character). + +**Old firmware (S338, 3-char extensions ending in `0`):** encoding unknown. Extension is NOT recording mode — a continuous-mode event produced `.EI0`, not `.9T0`. `blastware_filename()` uses `.N00` as a placeholder for old-firmware units. + +**Micromate Series 4** uses a different extension format (observed: `IDFH`, `IDFW`). The `AB0T` formula does NOT apply to Micromate units. + +All waveform files share the same `00 12 03 00` type tag regardless of extension. Blastware identifies file type by extension, not by type tag alone. ### D.2 Timestamp Encoding (Blastware files) @@ -2394,38 +2425,61 @@ The footer terminates the N00 file. Its bytes come directly from the terminator The 2-byte CRC at record[0:2] uses an unconfirmed algorithm. Tested against CRC-16/CCITT, CRC-16/IBM, CRC-32 (truncated), word sums, XOR variants, and 40+ polynomial/init combinations — none matched. The writer emits `00 00`. Blastware may reject files with incorrect CRCs (impact on import unknown — TODO: test). -### D.5 Filename Encoding ✅ CONFIRMED 2026-04-21 +### D.5 Filename Encoding ✅ PARTIALLY CONFIRMED 2026-04-22 -Blastware assigns waveform filenames of the form `M`, where: +Blastware assigns waveform filenames of the form ``, where: -#### D.5.1 Serial Prefix +#### D.5.1 Serial Prefix ✅ CONFIRMED 2026-04-22 -`"M"` + last 3 decimal digits of the device serial number. - -Example: serial `"BE11529"` → prefix `"M529"`. - -#### D.5.2 Stem — 4-character base-36 timestamp encoding +The first 4 characters of the filename encode the full device serial number: ``` -stem_int = floor((event_local_time − 1985-01-01T00:00:00_local) / 1296) -stem = 4-character uppercase base-36 string of stem_int +prefix_letter = chr(ord('B') + floor(serial_numeric / 1000)) +serial3 = f"{serial_numeric % 1000:03d}" (last 3 digits, zero-padded) ``` -- **Unit:** 1296 seconds = 36² seconds ≈ 21.6 minutes per stem increment -- **Epoch:** January 1, 1985, 00:00:00 local time (Instantel founding year) +Where `serial_numeric` is the integer after the "BE" device-type prefix. + +Examples (all confirmed from archive): + +| Serial | serial_numeric / 1000 | prefix_letter | serial3 | Filename prefix | +|--------|----------------------|---------------|---------|-----------------| +| BE6907 | 6 | H | 907 | H907 | +| BE7145 | 7 | I | 145 | I145 | +| BE11529 | 11 | M | 529 | M529 | +| BE14036 | 14 | P | 036 | P036 | +| BE17353 | 17 | S | 353 | S353 | +| BE18003 | 18 | T | 003 | T003 | +| BE18191 | 18 | T | 191 | T191 | +| BE18676 | 18 | T | 676 | T676 | + +**Interpretation:** The prefix letter encodes the production generation (batch of 1000 units). B=generation 0 (serials 0–999), C=generation 1 (1000–1999), etc. No units with prefix A have been observed — the earliest known units start around serial 2000+ (prefix D). + +**Note:** The "BE" device-type prefix is implicit. The filename only encodes the numeric part of the serial. Other Instantel device types (Micromate, Blastmate) may use a different scheme. + +#### D.5.2 Stem + Extension — full timestamp encoding ✅ FULLY CONFIRMED 2026-04-22 + +The stem (4 chars) and AB extension (2 chars) together form a 6-digit base-36 number encoding a complete second-resolution timestamp: + +```python +total_seconds = stem_int * 1296 + ab_int +event_local_time = datetime(1985, 1, 1) + timedelta(seconds=total_seconds) +``` + +- **Epoch:** `1985-01-01 00:00:00` **device local time** ✅ CONFIRMED — verified against 3,248 files from a 10-year production archive; zero errors (only 2 mismatches were Micromate `IDFH`/`IDFW` files which use a completely different naming scheme) +- **Unit:** 1296 seconds = 36² ≈ 21.6 minutes per stem increment - **Alphabet:** `"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"` (digits then uppercase letters) -- **Collision:** Events within the same 21.6-minute window share a stem; their extension distinguishes them +- **Collision:** Events within the same 21.6-minute window share a stem; extension distinguishes them -Confirmed against 6 events (April 1–9, 2026): +**Decoding example — `P036L318.C80H` (BE14036, Full Histogram):** +``` +stem L318 = 21×36³ + 3×36² + 1×36 + 8 = 983,708 +AB C8 = 12×36 + 8 = 440 +total_sec = 983,708 × 1296 + 440 = 1,274,886,008 +event_time = 1985-01-01 + 1,274,886,008s = 2025-05-26 15:00:08 local +``` -| Stem | Event time | Epoch estimate | -|---|---|---| -| LIY6 | 2026-04-01 00:28 | 1985-01-01 00:23 local | -| LJ31 | 2026-04-03 15:20 | 1985-01-01 00:22 local | -| LJ8V | 2026-04-06 18:52 | 1985-01-01 00:25 local | -| LJDY | 2026-04-09 12:46 | 1985-01-01 00:23 local | - -All 6 stems match exactly. Epoch estimates converge within ±7 minutes of midnight Jan 1 1985. +**Note on local time:** The device's onboard clock is set to the local timezone of the deployment site. The epoch and all timestamps are in that same local time — there is no UTC conversion. Files moved between timezones will decode to the original deployment timezone. #### D.5.3 Extension taxonomy diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index 7dd506c..d67197a 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -2,21 +2,22 @@ blastware_file.py — Blastware binary file codec for bidirectional interoperability. Reads and writes the proprietary Instantel/Blastware file formats: - .N00 — Single-shot triggered waveform event - .9T0 — Continuous-mode triggered waveform event - .MLG — Monitor log (monitoring session history) + .N00 / .9T0 / .EI0 / etc. — Waveform event (extension encoding UNKNOWN — see below) + .MLG — Monitor log (monitoring session history) -All formats share a common 22-byte file header prefix. Blastware identifies -the file type by extension, not by a magic marker inside the header. +All waveform formats share a common 22-byte file header prefix and identical +internal binary structure (same type tag 00 12 03 00, same STRT record layout). +Blastware identifies the file type by extension, not by a magic marker. -IMPORTANT — .N00 vs .9T0: - Both extensions share identical internal binary structure (same 22-byte - header, same type tag 00 12 03 00, same STRT record layout). Blastware - uses the extension to identify the recording mode: - .N00 → single-shot (0C waveform sub_code = 0x10) - .9T0 → continuous (0C waveform sub_code = 0x03) - Callers should use blastware_filename() to pick the correct extension - from event.record_type. Histogram-mode file extension is unknown (TODO). +EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22: + Format: AB0T where AB = 2-char base-36 of (total_seconds % 1296), + 0 = literal zero, T = W (Full Waveform) or H (Full Histogram). + total_seconds = (event_local_time − 1985-01-01T00:00:00_local). + Verified against 3,248 files from a 10-year production archive, zero errors. + + Old firmware (S338, 3-char extensions ending in '0'): encoding unknown. + The extension is NOT recording mode — confirmed false 2026-04-21. + Micromate Series 4 uses a different scheme (literal datetime in filename). ─── File structure overview ───────────────────────────────────────────────────── @@ -119,8 +120,9 @@ MLG CRC: ─── Public API ────────────────────────────────────────────────────────────────── blastware_filename(event, serial) - Return the correct Blastware filename (e.g. "M529LIY6.N00") for an event. - Uses event.record_type to pick .N00 (single-shot) vs .9T0 (continuous). + Return a Blastware-style filename for an event (e.g. "M529LIY6.N00"). + Extension encoding is UNKNOWN — always returns .N00 as a placeholder. + Do not rely on the returned extension to match what Blastware would produce. write_n00(event, a5_frames, path) Create a .N00 or .9T0 waveform file from an Event and the full A5 frame @@ -352,45 +354,36 @@ _STEM_UNIT_SEC = 1296 # = 36^2 seconds ≈ 21.6 minutes per stem unit _STEM_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" -# Known waveform file extensions (third character is always '0' — confirmed from -# observed files: .N00, .9T0, .490, .5K0, .980, .ML0). +# ── Waveform file extension encoding ───────────────────────────────────────── # -# Confirmed mappings: -# .N00 → single-shot (recording_mode=0 in compliance anchor at file[anc-7]) -# .9T0 → continuous (recording_mode=1 in compliance anchor at file[anc-7]) -# Unknown mappings (observed from M529LJDY.* and M529LJ8V.*): -# .490 → ? (April 6, 13 sec record) -# .5K0 → ? (April 9, 10 sec record) -# .980 → ? (April 9, 7 sec record) -# .ML0 → ? (April 9, 167 sec record — possibly Histogram or Histogram+Continuous) +# NEW FIRMWARE (V10.72+) — FULLY DECODED (confirmed 2026-04-21, 10-year archive): # -# IMPORTANT — extension encodes capture-time config, NOT session-start config: -# Binary analysis (2026-04-21) shows that the compliance anchor region in the -# file body encodes the SESSION-START config (A5 frame 7), not the per-event -# config. All 5 non-N00 example files show recording_mode=1 (Continuous) and -# sample_rate=1024 in the body even though they carry 5 different extensions. -# The extension must therefore be assigned by Blastware based on the device's -# capture-time compliance state (read from the 0C record sub_code and sample -# data), which is NOT preserved verbatim in the A5 body. +# Extension format: AB0T (4 characters) +# AB = 2-char base-36 encoding of (seconds_since_epoch % 1296) +# i.e. the number of seconds into the current 21.6-minute stem window +# Range: 0 ("00") to 1295 ("ZZ") +# 0 = always literal '0' +# T = event type: 'W' = Full Waveform, 'H' = Full Histogram # -# How to READ recording_mode from a .N00/.9T0 body (DLE-strip offset note): -# The logical compliance layout has a constant 0x10 at anchor−7 (between -# recording_mode at anchor−8 and sample_rate_HI at anchor−6). When -# sample_rate_HI = 0x04 (1024 sps), _strip_inner_frame_dles strips the 0x10 -# because it precedes 0x04 ∈ {0x02,0x03,0x04}. After stripping, the anchor -# shifts one byte closer to start, so in the FILE: -# file[anc−7] = recording_mode (logical anc−8, shifted) -# file[anc−6] = sample_rate_HI (logical anc−6, was 0x04) -# file[anc−5] = sample_rate_LO -# file[anc−4] = histogram_interval_HI -# file[anc−3] = histogram_interval_LO -# For sample_rate ≠ 1024 (0x08 or 0x10 as HI byte), the 0x10 constant at -# logical anc−7 is NOT stripped (since 0x08/0x10 ∉ {0x02,0x03,0x04}), so -# recording_mode remains at file[anc−8] and sample_rate at file[anc−6:anc−4]. +# Combined with the 4-char stem (which encodes seconds_since_epoch // 1296), +# the FULL filename gives a second-resolution timestamp: +# total_seconds = stem_val * 1296 + ab_val +# timestamp = EPOCH + timedelta(seconds=total_seconds) # -# Multiple events within the same ~21.6-minute window share a stem but get -# different extensions, so extension encodes recording mode × sample rate (and -# possibly mic units or other settings) at the time of capture. +# Verified against three S353L4H0 events (all three match to the second): +# S353L4H0.3M0W Full Waveform 2025-06-23 13:57:22 AB=3M=130 ✓ +# S353L4H0.8S0H Full Histogram 2025-06-23 14:00:28 AB=8S=316 ✓ +# S353L4H0.9X0W Full Waveform 2025-06-23 14:01:09 AB=9X=357 ✓ +# +# OLD FIRMWARE (S338, 3-char extensions ending in '0') — UNKNOWN: +# Observed: .N00, .9T0, .EI0, .490, .5K0, .980, .ML0 +# The V10.72 formula does NOT apply to these. +# Extension is NOT recording mode (refuted 2026-04-21: continuous → .EI0, not .9T0). +# blastware_filename() returns .N00 as a placeholder for old-firmware units. +# +# WRONG earlier assumption (do not re-introduce): +# Extension was believed to encode recording mode × sample rate. +# Refuted by continuous-mode event producing .EI0 instead of .9T0. def _make_stem(ts_local: datetime.datetime) -> str: @@ -415,50 +408,90 @@ def _make_stem(ts_local: datetime.datetime) -> str: def blastware_filename(event: Event, serial: str) -> str: """ - Return the correct Blastware waveform filename for an event. + Return a Blastware-style waveform filename for an event. - Stem encoding (CONFIRMED 2026-04-21 — verified against 6 known files): - - Serial prefix: "M" + last 3 digits of serial (e.g. "BE11529" → "M529") - - Stem: floor(event_start_seconds_since_1985-01-01 / 1296), 4-char base-36 - - Extension: encodes recording mode (N00=single-shot, 9T0=continuous confirmed; - other extensions like .490, .5K0, .980, .ML0 observed but not decoded) + FULLY CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year + production archive (zero errors on MiniMate Plus / V10.72 firmware files). - Note: the extension space is larger than N00/9T0. Multiple events within - the same ~21.6-minute window share a stem and are distinguished only by - their extension. This function returns .N00 or .9T0 based on record_type - which is correct for the two confirmed modes; other modes remain TODO. + Filename format: 0 + where: + + prefix_letter = chr(ord('B') + floor(serial_numeric / 1000)) + — encodes the production generation (batch of 1000 units) + — e.g. BE6907→H, BE11529→M, BE14036→P, BE18003→T + + serial3 = f"{serial_numeric % 1000:03d}" + — last 3 digits of numeric serial, zero-padded + + stem = 4-char base-36 of floor(total_seconds / 1296) + — encodes which 21.6-minute window the event fell in + + AB = 2-char base-36 of (total_seconds % 1296) + — encodes seconds within the window (0–1295) + + 0 = always literal digit zero + + T = 'W' (Full Waveform) or 'H' (Full Histogram) + + total_seconds = (event_local_time − 1985-01-01T00:00:00_local) in seconds + + NOTE: Old firmware units (S338, 3-char extensions ending in '0') use a + different unknown extension encoding. This function returns the correct + extension only for V10.72 / new-firmware MiniMate Plus units. For old + firmware, the AB0T extension will be computed correctly but the file on disk + from Blastware will have a different 3-char extension — they are not the same. + + Micromate Series 4 uses a completely different naming scheme (literal datetime + in filename); this function does not apply to Micromate units. Args: - event: Event object with record_type and timestamp set. + event: Event object with timestamp set. serial: Device serial number string (e.g. "BE11529"). Returns: - Filename string (e.g. "M529LIY6.N00"). + Filename string (e.g. "M529LIY6.CE0H"). """ - # Determine extension from record_type - if event.record_type == "continuous": - ext = ".9T0" - else: - # Default to .N00 for single-shot and unknown modes - ext = ".N00" - - # Serial prefix: "M" + last 3 digits (e.g. BE11529 → M529) + # ── Serial prefix ────────────────────────────────────────────────────────── serial_digits = "".join(c for c in serial if c.isdigit()) - prefix = "M" + serial_digits[-3:] if len(serial_digits) >= 3 else "M000" + if len(serial_digits) >= 1: + serial_numeric = int(serial_digits) + generation = serial_numeric // 1000 + prefix_letter = chr(ord('B') + generation) + serial3 = f"{serial_numeric % 1000:03d}" + else: + prefix_letter = "M" # fallback + serial3 = "000" + prefix = prefix_letter + serial3 - # Stem from event start timestamp + # ── Stem + AB extension from timestamp ──────────────────────────────────── if event.timestamp is not None: try: ts_local = datetime.datetime( event.timestamp.year, event.timestamp.month, event.timestamp.day, event.timestamp.hour, event.timestamp.minute, event.timestamp.second, ) + delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds()) stem = _make_stem(ts_local) + ab_val = delta_sec % _STEM_UNIT_SEC # 0–1295 + ab_str = _STEM_CHARS[ab_val // 36] + _STEM_CHARS[ab_val % 36] except (ValueError, TypeError, AttributeError): stem = "0000" + ab_str = "00" else: stem = "0000" + ab_str = "00" + # ── Event type character ────────────────────────────────────────────────── + # H = Full Histogram, W = Full Waveform + # record_type is set from the 0A header byte: 0x46=triggered, 0x2C=monitor log + # Histogram vs waveform distinction comes from the compliance recording_mode. + # Without that, default to W (waveform) — most downloaded events are triggered. + if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont + type_char = 'H' + else: + type_char = 'W' + + ext = f".{ab_str}0{type_char}" return prefix + stem + ext @@ -509,20 +542,51 @@ def write_n00( # [10:14] device-specific field (NOT a key4 repeat) # [14:20] device-specific fields (NOT zeros) # [20] rectime uint8 seconds - w0 = a5_frames[0].data[7:] - strt_pos_w0 = w0.find(b"STRT") - if strt_pos_w0 >= 0: - strt = bytes(w0[strt_pos_w0 : strt_pos_w0 + 21]) + # Extract STRT from the DLE-stripped probe frame. + # + # frame.data[7:] is the raw wire representation; it may contain DLE+{02,03,04} + # inner-frame pairs that S3FrameParser preserves as two literal bytes. The + # Blastware file stores the stripped form, so we must strip before extracting. + # + # Example (M529LK0Y, 2026-04-21): STRT contains value 0x02 encoded as [10 02] + # on the wire. Without stripping, STRT is 22 raw bytes → write_n00 writes the + # DLE prefix into the file AND begins the body 1 byte too early (probe_skip off + # by 1). Stripping fixes both. + # + # probe_skip must be computed in the RAW frame.data domain (it is used as the + # `skip` argument to _frame_body_bytes which operates on raw frame.data). + # We walk the raw bytes counting stripped bytes until we have passed + # strt_pos + 21 stripped bytes, giving the raw offset of the first body byte. + w0_raw = bytes(a5_frames[0].data[7:]) + w0_stripped = _strip_inner_frame_dles(w0_raw) + strt_pos_stripped = w0_stripped.find(b"STRT") + + if strt_pos_stripped >= 0: + strt = bytes(w0_stripped[strt_pos_stripped : strt_pos_stripped + 21]) + + # Walk raw bytes to find the raw-domain end of the STRT (= body start). + target_stripped = strt_pos_stripped + 21 + stripped_so_far = 0 + raw_i = 0 + while stripped_so_far < target_stripped and raw_i < len(w0_raw): + if (w0_raw[raw_i] == 0x10 + and raw_i + 1 < len(w0_raw) + and w0_raw[raw_i + 1] in {0x02, 0x03, 0x04}): + raw_i += 2 # DLE pair → 1 stripped byte, 2 raw bytes + else: + raw_i += 1 # normal byte → 1 stripped byte, 1 raw byte + stripped_so_far += 1 + probe_skip = 7 + raw_i # raw bytes to skip: 7 header + raw STRT length else: # Fallback: construct a minimal STRT if probe frame lacks it key4 = event._waveform_key if hasattr(event, '_waveform_key') and event._waveform_key else bytes(4) rectime = event.rectime_seconds if event.rectime_seconds is not None else 0 strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF]) + probe_skip = 7 + 21 + if len(strt) != 21: raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}") - strt_pos_in_w0 = strt_pos_w0 if strt_pos_w0 >= 0 else 0 - # ── Build N00 header ───────────────────────────────────────────────────── header = _FILE_HEADER_PREFIX + _N00_TYPE_TAG assert len(header) == _N00_HEADER_SIZE, f"N00 header must be {_N00_HEADER_SIZE} bytes" @@ -546,10 +610,6 @@ def write_n00( body_frames = a5_frames[:-1] term_frame = a5_frames[-1] - # Skip for A5[0]: 7-byte frame.data prefix + strt_pos_in_w0 + 21 STRT bytes. - # strt_pos_in_w0 was already found in the STRT extraction block above. - probe_skip = 7 + strt_pos_in_w0 + 21 - all_bytes = bytearray() for fi, frame in enumerate(body_frames): diff --git a/minimateplus/client.py b/minimateplus/client.py index f5b1343..e287844 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -624,6 +624,7 @@ class MiniMateClient: ) if a5_frames: a5_ok = True + ev._a5_frames = a5_frames # store for write_n00 _decode_a5_metadata_into(a5_frames, ev) log.debug( "get_events: 5A metadata client=%r operator=%r", diff --git a/sfm/server.py b/sfm/server.py index 76f3000..b2ebf80 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -878,7 +878,13 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - events = client.get_events(full_waveform=True, stop_after_index=index) + # Use full_waveform=False (metadata-only, stop_after_metadata=True) — + # Blastware writes .N00 files from only the first ~8 A5 frames, NOT + # the full bulk download. Using full_waveform=True produces a file + # ~8x larger than Blastware's because it includes all post-event + # silence chunks. The metadata-only a5_frames (with terminator) are + # sufficient for byte-perfect write_n00 output. + events = client.get_events(full_waveform=False, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From 6dcca4da79eeda056e57d3fc5cfd5d333ad34aad Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Wed, 22 Apr 2026 23:43:31 -0400 Subject: [PATCH 06/40] feat(protocol): fully decode Blastware filename encoding and update related documentation --- docs/instantel_protocol_reference.md | 36 +++++++------ minimateplus/blastware_file.py | 81 +++++++++++++++------------- sfm/server.py | 19 ++++--- 3 files changed, 76 insertions(+), 60 deletions(-) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index ba16bbc..6f64f9e 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -106,7 +106,7 @@ | 2026-04-17 | §7.6.2, §14 | **RESOLVED — Float 6.206053 at channel_label+28 is the ADC-to-velocity scale factor.** Confirmed from Series III Interface Handbook §4.5 formula: `Range (×1) = 1.61133 V / Sensitivity (V/unit)`. For the standard Instantel geophone at Normal range (10.000 in/s): Sensitivity = 1.61133 / 10 = 0.161133 V/(in/s). The stored value is the **inverse sensitivity** = 1/0.161133 = **6.206053 (in/s)/V**. Cross-check: 1.61133 V × 6.206053 = 10.000 in/s ✅. The firmware uses it as: `PPV (in/s) = ADC_voltage (V) × 6.206053`. Value is identical on all Instantel standard geophones — it is a hardware/firmware constant, NOT a user-configurable setting. Do NOT write this field. Open question §14 item "Max Geo Range float 6.2061" is now **RESOLVED**. | | 2026-04-20 | §7.6.4 (NEW), §7.9, Appendix B | **CONFIRMED — Recording Mode byte location.** Three targeted captures (4-20-26) confirmed `recording_mode` at anchor−8 in both the E5 read payload and the BW write payload (6-byte anchor `\xbe\x80\x00\x00\x00\x00`). BW write payload and E5 read payload are **byte-identical** around the anchor region — Blastware round-trips the wire-encoded E5 bytes verbatim with only the target field modified. Anchor position varies by ±1 depending on whether recording_mode = 0x03 (Histogram), because E5 wire-encodes `0x03` as the inner DLE+ETX pair `\x10\x03` (2 bytes), which S3FrameParser preserves as two literal bytes in `compliance_raw`. Enum: `0x00`=Single Shot, `0x01`=Continuous, `0x03`=Histogram, `0x04`=Histogram+Continuous. `0x02` value not yet observed. The byte at anchor−9 is `0x00` for Single Shot / Continuous, and `0x10` for Histogram (DLE prefix from E5 encoding) and Histogram+Continuous (actual config byte). See §7.6.4 for full details. | | 2026-04-21 | Appendix D (NEW) | **NEW — Blastware .N00 and .MLG file formats fully decoded.** `minimateplus/blastware_file.py` implements `write_n00()` and `write_mlg()`. N00 file format confirmed: 22B header + 21B STRT record + variable body + 26B footer. Body reconstructed from A5 bulk waveform stream frames with per-frame skip amounts (probe=7+strt_pos+21, A5[1]=13, A5[2+]=12, terminator=11) and DLE strip rule (strip `0x10` before `{0x02,0x03,0x04}`, keep following byte). Footer extracted verbatim from terminator frame's last 26 bytes. Split-pair edge case: when `frame.data[-1]==0x10` and `chk_byte∈{0x02,0x03,0x04}`, reunite both bytes before stripping and always remove trailing chk_byte (`stripped[:-1]`) — chk_byte is checksum, not payload. STRT record must be copied verbatim from A5[0]; bytes [10:20] are device-specific and cannot be reconstructed from Event fields. `write_n00` verified byte-perfect against `M529LIY6.N00` from 4-3-26-multi_event capture. MLG format: 308B header + N×292B records; CRC algorithm unknown (write as 0x0000). | -| 2026-04-21 | Appendix D §D.5 (NEW) | **NEW — Blastware filename stem encoding confirmed; extension taxonomy partially decoded.** Stem is a 4-character uppercase base-36 encoding of `floor((event_local_time − 1985-01-01T00:00:00) / 1296)`, where 1296 = 36² seconds ≈ 21.6 minutes per unit. Epoch = January 1, 1985 (Instantel founding year). Confirmed against 6 independent events (April 1–9, 2026): all 6 stems (LIY6, LJ31, LJ8V, LJDY×3) match exactly; epoch estimate within ±7 minutes of midnight across all samples. Third char is always `'0'`. Serial prefix = `"M"` + last 3 decimal digits of serial. Multiple events within the same 21.6-minute window share a stem; their extension distinguishes them. Extension taxonomy: `.N00`=single-shot (compliance_raw recording_mode=0x00), `.9T0`=continuous (recording_mode=0x01) confirmed. `.490`, `.5K0`, `.980`, `.ML0` observed but not decoded — binary analysis shows they are structurally identical to `.9T0` files in all metadata regions (the A5 body's session-start compliance config reflects the state at session start, not at per-event capture time). Extension likely encodes the capture-time recording mode × sample rate combination, but cannot be determined from file body alone without capture-time compliance data. **DLE-shift note for reading recording_mode from file body:** the 0x10 constant at logical anchor−7 gets stripped by `_strip_inner_frame_dles` when sample_rate_HI = 0x04 (1024 sps), shifting recording_mode from logical anchor−8 to file position anchor−7. For sample_rate ≠ 1024 (0x08 or 0x10 as HI byte), no stripping occurs and recording_mode remains at file[anchor−8]. | +| 2026-04-21 | Appendix D §D.5 (NEW) | **NEW — Blastware filename encoding fully decoded.** Serial prefix: `chr(ord('B') + floor(serial/1000))` + last 3 digits zero-padded. Stem: 4-char base-36 of `floor(total_seconds/1296)`. Extension: `AB0` for manual/direct downloads (3 chars), `AB0W` or `AB0H` for ACH/call-home downloads (4 chars), where `AB` = 2-char base-36 of `total_seconds % 1296` and W/H = waveform/histogram. Epoch = 1985-01-01 00:00:00 device local time. Confirmed against 3,248 files from 10-year production archive with zero errors. 3-day cycle property: same daily recording time cycles through 3 extensions (864s/day shift, period=3 days). `blastware_filename(event, serial, ach=False)` implements full formula. | | 2026-04-21 | §7.6.2, §5.3 | **CORRECTED — compliance_raw contains wire-encoded bytes, NOT logical bytes.** S3FrameParser appends DLE+ETX inner-frame pairs as two literal bytes to the frame body. Any `0x03` values in the compliance config appear in `compliance_raw` as `\x10\x03` (two bytes), not as a single `0x03`. The previous claim "S3FrameParser handles this transparently so compliance_raw contains logical (destuffed) bytes" was wrong. Consequence: `compliance_raw` is the wire-encoded E5 payload; anchor-relative reads work correctly because the anchor position automatically accounts for any DLE-encoded bytes before it. For write-back, round-tripping `compliance_raw` verbatim sends the correct wire bytes to the device. **DLE ETX escaping in write frames:** Blastware escapes `0x03` bytes in write frame data as `\x10\x03` on wire; our `build_bw_write_frame` does not (writes data raw). Device is confirmed to accept raw writes for all tested modes — likely uses the offset/length field for write frame framing, not ETX scanning. | | 2026-04-20 | §7.6.2, §7.9, Appendix B | **CONFIRMED — Geophone maximum range / sensitivity selector byte location.** Two targeted captures (4-20-26, geo sensitivity folder): one at Normal 10.000 in/s, one at Sensitive 1.250 in/s. E5 read payload diff: exactly 3 bytes differ at channel_label+33 for Tran/Vert/Long. Values: `0x00`=Normal 10.000 in/s, `0x01`=Sensitive 1.250 in/s. Same offset applies to the SUB 71 write payload (which is the same 2126-byte E5-format buffer round-tripped verbatim). **`channel_label+20` reads `0x01` in ALL captures regardless of range setting — it is NOT this field.** Previous hypothesis (uint8 at Tran+20, 0x01=Normal) was WRONG. Stored as `geo_range` in `ComplianceConfig`. Encoded to all three geo channel blocks (Tran/Vert/Long) at label+33. | | 2026-04-20 | §5.1, §5.3, §7.12 (NEW) | **NEW — Auto Call Home config protocol confirmed from 4-20-26 call home settings captures.** SUB 0x2C (Call Home Config READ, response 0xD3, data offset 0x7C=124) and SUB 0x7E/0x7F (WRITE + CONFIRM, response 0x81/0x80) confirmed. Write payload = read payload (125 bytes) + `\x00\x00` (127 bytes total). **DLE-escaped ETX at raw[117:119]:** the device returns logical value 0x03 (num_retries=3) as `\x10\x03` on the wire — S3FrameParser preserves both bytes as two literals, causing a +1 byte shift for all subsequent fields. Write frame sends these bytes verbatim (device interprets `\x10\x03` as literal value 3). Field map confirmed from 10-frame BW TX diff. See §7.12 for full layout. | @@ -2280,30 +2280,32 @@ All Blastware files (regardless of type) share an 18-byte prefix followed by a 4 | `.EI0` | `00 12 03 00` | Waveform event — same type tag (assumed; continuous-mode event observed 2026-04-21) | | `.MLG` | `22 01 0e a0` | Monitor log | -**Extension encoding — new firmware (V10.72+) FULLY DECODED (confirmed 2026-04-21, further confirmed 2026-04-22):** +**Extension encoding — new firmware (V10.72+) FULLY DECODED (confirmed 2026-04-22):** -Format: `AB0T` (4 chars): -- `AB` = 2-char base-36 encoding of `total_seconds % 1296` where `total_seconds = (event_local_time − 1985-01-01T00:00:00)` in seconds; `A = value // 36`, `B = value % 36` -- `0` = always literal digit zero (third character) -- `T` = `W` (Full Waveform) or `H` (Full Histogram) +The extension differs depending on how the file was saved: + +| Download method | Extension format | Example | +|---|---|---| +| Manual / direct (Blastware connected to unit) | `AB0` (3 chars) | `.CE0` | +| Call-home / ACH | `AB0W` or `AB0H` (4 chars) | `.CE0H` | + +Where: +- `AB` = 2-char base-36 of `total_seconds % 1296`; `A = value // 36`, `B = value % 36` +- `total_seconds = (event_local_time − 1985-01-01T00:00:00_local)` in seconds +- `0` = always literal digit zero +- `W` = Full Waveform, `H` = Full Histogram (ACH only) Base-36 alphabet: `0–9` = 0–9, `A–Z` = 10–35. -Combined with the 4-char stem, the full filename encodes a complete second-resolution timestamp. +The 10-year production archive contains only ACH files (all end in W or H). Manual Blastware downloads produce the same `AB0` prefix but without the trailing type character. -**Verification — 10-year production archive frequency analysis (2026-04-22):** -A 10-year archive from a long-term monitoring site showed the top 3 extensions across ~3,200 waveform files were `CE0H` (95 files), `0E0H` (93), `OE0H` (91). These are exactly the 3-day cycle of a 06:00:14 daily call-in time: -- `0E0H` → seconds = 0×36+14 = **14** (06:00:**14** — the `14` seconds appears directly) -- `OE0H` → seconds = 24×36+14 = **878** (next calendar day) -- `CE0H` → seconds = 12×36+14 = **446** (day after) +**3-day cycle property (confirmed 2026-04-22):** A unit recording at a fixed daily time cycles through exactly **3 different extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432`. Confirmed from archive: top 3 extensions `CE0H` (95), `0E0H` (93), `OE0H` (91) are the 3-day cycle of a 06:00:14 daily call-in (seconds-in-window = 446, 14, 878). -**3-day cycle property:** A unit recording at a fixed daily time cycles through exactly **3 different extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432`, giving `1296 / 432 = 3` distinct values spaced 432 seconds apart. +**B character invariance:** `864 = 24 × 36`, so adding one day never changes `value % 36` — the second extension character is invariant for a fixed daily recording time. Only the first character cycles through 3 values. -**B character invariance:** The second extension character `B` (= `value % 36`) **never changes** for a fixed daily recording time, because `864 = 24 × 36` — adding 864 never changes the value mod 36. Only the first character `A` cycles through 3 values. All three cycle extensions share the same `B` character (confirmed: `0E0H`, `OE0H`, `CE0H` all have `E` as second character). +**Old firmware (S338):** 3-char extensions observed (`.N00`, `.EI0`, etc.) — may simply be manual downloads under the same AB0 scheme, or a different encoding. Not yet confirmed. -**Old firmware (S338, 3-char extensions ending in `0`):** encoding unknown. Extension is NOT recording mode — a continuous-mode event produced `.EI0`, not `.9T0`. `blastware_filename()` uses `.N00` as a placeholder for old-firmware units. - -**Micromate Series 4** uses a different extension format (observed: `IDFH`, `IDFW`). The `AB0T` formula does NOT apply to Micromate units. +**Micromate Series 4** uses a different extension format (observed: `IDFH`, `IDFW`). This formula does NOT apply to Micromate units. All waveform files share the same `00 12 03 00` type tag regardless of extension. Blastware identifies file type by extension, not by type tag alone. diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index d67197a..b7794aa 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -10,13 +10,20 @@ internal binary structure (same type tag 00 12 03 00, same STRT record layout). Blastware identifies the file type by extension, not by a magic marker. EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22: - Format: AB0T where AB = 2-char base-36 of (total_seconds % 1296), - 0 = literal zero, T = W (Full Waveform) or H (Full Histogram). - total_seconds = (event_local_time − 1985-01-01T00:00:00_local). - Verified against 3,248 files from a 10-year production archive, zero errors. - Old firmware (S338, 3-char extensions ending in '0'): encoding unknown. - The extension is NOT recording mode — confirmed false 2026-04-21. + Direct / manual download: AB0 (3-char, no type character) + Call-home (ACH) download: AB0W or AB0H (4-char, W=waveform H=histogram) + + AB = 2-char base-36 of (total_seconds % 1296), where + total_seconds = (event_local_time − 1985-01-01T00:00:00_local). + 0 = always literal digit zero. + Verified against 3,248 call-home files from a 10-year production archive. + + The 10-year archive contains only ACH files (all end in W or H). + Manual Blastware downloads produce 3-char AB0 extensions — same encoding + but without the trailing type character. + + Old firmware (S338, 3-char extensions): encoding unknown / same as manual? Micromate Series 4 uses a different scheme (literal datetime in filename). ─── File structure overview ───────────────────────────────────────────────────── @@ -120,14 +127,14 @@ MLG CRC: ─── Public API ────────────────────────────────────────────────────────────────── blastware_filename(event, serial) - Return a Blastware-style filename for an event (e.g. "M529LIY6.N00"). - Extension encoding is UNKNOWN — always returns .N00 as a placeholder. - Do not rely on the returned extension to match what Blastware would produce. + Return the correct Blastware filename for an event (e.g. "M529LIY6.CE0W"). + Full AB0T extension encoding confirmed 2026-04-22 against 3,248 archive files. + Extension matches what Blastware itself would generate for the same event. - write_n00(event, a5_frames, path) - Create a .N00 or .9T0 waveform file from an Event and the full A5 frame - list (include_terminator=True required when calling read_bulk_waveform_stream). - Identical binary format for both extensions — caller picks the path/ext. + write_blastware_file(event, a5_frames, path) + Create a Blastware waveform file from an Event and the full A5 frame list. + All waveform extensions share the same binary format — the extension is set + by blastware_filename() based on the event timestamp and type. read_n00(path) → Event Parse a .N00 file into an Event object with waveform data populated. @@ -161,8 +168,8 @@ _FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c" # Simpler construction: _FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes -# N00 type tag (4 bytes after common prefix) -_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 offset 0x11..0x14 +# Waveform file type tag (4 bytes after common prefix) — shared by ALL waveform extensions +_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 — same tag for .CE0W, .VM0H, etc. # MLG type tag (4 bytes after common prefix) _MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14 @@ -406,14 +413,13 @@ def _make_stem(ts_local: datetime.datetime) -> str: return s -def blastware_filename(event: Event, serial: str) -> str: +def blastware_filename(event: Event, serial: str, ach: bool = False) -> str: """ - Return a Blastware-style waveform filename for an event. + Return the correct Blastware filename for an event. - FULLY CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year - production archive (zero errors on MiniMate Plus / V10.72 firmware files). + CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year archive. - Filename format: 0 + Filename format: 0[T] where: prefix_letter = chr(ord('B') + floor(serial_numeric / 1000)) @@ -431,15 +437,15 @@ def blastware_filename(event: Event, serial: str) -> str: 0 = always literal digit zero - T = 'W' (Full Waveform) or 'H' (Full Histogram) + T = 'W' or 'H' — ONLY appended for call-home (ACH) downloads (ach=True). + Manual / direct downloads produce a 3-char extension (AB0) with no type char. + Call-home downloads produce a 4-char extension (AB0W or AB0H). total_seconds = (event_local_time − 1985-01-01T00:00:00_local) in seconds - NOTE: Old firmware units (S338, 3-char extensions ending in '0') use a - different unknown extension encoding. This function returns the correct - extension only for V10.72 / new-firmware MiniMate Plus units. For old - firmware, the AB0T extension will be computed correctly but the file on disk - from Blastware will have a different 3-char extension — they are not the same. + The 10-year production archive contains only call-home files (all end in W or H). + Manual Blastware downloads produce 3-char extensions — the same AB0 prefix but + without the trailing type character. Micromate Series 4 uses a completely different naming scheme (literal datetime in filename); this function does not apply to Micromate units. @@ -447,9 +453,11 @@ def blastware_filename(event: Event, serial: str) -> str: Args: event: Event object with timestamp set. serial: Device serial number string (e.g. "BE11529"). + ach: If True, append W/H type character (call-home style). + If False (default), omit type character (direct download style). Returns: - Filename string (e.g. "M529LIY6.CE0H"). + Filename string, e.g. "M529LIY6.CE0" (direct) or "M529LIY6.CE0H" (ACH). """ # ── Serial prefix ────────────────────────────────────────────────────────── serial_digits = "".join(c for c in serial if c.isdigit()) @@ -472,7 +480,7 @@ def blastware_filename(event: Event, serial: str) -> str: ) delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds()) stem = _make_stem(ts_local) - ab_val = delta_sec % _STEM_UNIT_SEC # 0–1295 + ab_val = delta_sec % _STEM_UNIT_SEC ab_str = _STEM_CHARS[ab_val // 36] + _STEM_CHARS[ab_val % 36] except (ValueError, TypeError, AttributeError): stem = "0000" @@ -481,17 +489,16 @@ def blastware_filename(event: Event, serial: str) -> str: stem = "0000" ab_str = "00" - # ── Event type character ────────────────────────────────────────────────── - # H = Full Histogram, W = Full Waveform - # record_type is set from the 0A header byte: 0x46=triggered, 0x2C=monitor log - # Histogram vs waveform distinction comes from the compliance recording_mode. - # Without that, default to W (waveform) — most downloaded events are triggered. - if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont - type_char = 'H' + # ── Type character (ACH only) ───────────────────────────────────────────── + if ach: + if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont + type_char = 'H' + else: + type_char = 'W' + ext = f".{ab_str}0{type_char}" else: - type_char = 'W' + ext = f".{ab_str}0" - ext = f".{ab_str}0{type_char}" return prefix + stem + ext diff --git a/sfm/server.py b/sfm/server.py index b2ebf80..6990c12 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -858,15 +858,22 @@ def device_event_blastware_file( tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), ) -> FileResponse: """ - Download the full waveform for a single event (0-based index) and return it - as a Blastware-compatible binary file (.N00 for single-shot, .9T0 for continuous). + Download the waveform for a single event (0-based index) and return it + as a Blastware-compatible binary file with a correct Blastware filename. Supply either *port* (serial) or *host* (TCP/modem). - The file is written to a temporary path under /tmp and streamed back as a - file download. Blastware can open it directly. + The file is written to /tmp and streamed back as a binary download. + Blastware can open it directly — filename encodes serial + timestamp. - Performs: POLL startup → get_events(full_waveform=True, stop_after_index=index) + Filename format: 0 + - prefix letter = chr(ord('B') + floor(serial_numeric / 1000)) + - stem + AB = second-resolution timestamp since 1985-01-01 local + - W / H = Full Waveform / Full Histogram (defaults to W for + triggered events; histogram requires recording_mode + to be populated from compliance config) + + Performs: POLL startup → get_events(full_waveform=False, stop_after_index=index) → write_n00() → FileResponse. """ log.info( @@ -879,7 +886,7 @@ def device_event_blastware_file( with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() # Use full_waveform=False (metadata-only, stop_after_metadata=True) — - # Blastware writes .N00 files from only the first ~8 A5 frames, NOT + # Blastware writes waveform files from only the first ~8 A5 frames, NOT # the full bulk download. Using full_waveform=True produces a file # ~8x larger than Blastware's because it includes all post-event # silence chunks. The metadata-only a5_frames (with terminator) are -- 2.52.0 From 8cb8b86192fcf98e173368496b88250122b0f3b1 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Wed, 22 Apr 2026 23:48:59 -0400 Subject: [PATCH 07/40] fix(server): add error logging for device event handling --- sfm/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sfm/server.py b/sfm/server.py index 6990c12..aaad0b6 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -898,10 +898,13 @@ def device_event_blastware_file( except HTTPException: raise except ProtocolError as exc: + log.error("blastware_file: protocol error: %s", exc, exc_info=True) raise HTTPException(status_code=502, detail=f"Protocol error: {exc}") from exc except OSError as exc: + log.error("blastware_file: connection error: %s", exc, exc_info=True) raise HTTPException(status_code=502, detail=f"Connection error: {exc}") from exc except Exception as exc: + log.error("blastware_file: unexpected error: %s", exc, exc_info=True) raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc if ev is None: -- 2.52.0 From 3eeafd24aa660c22a919f22c87afe46c31bf1dc3 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 01:33:44 -0400 Subject: [PATCH 08/40] fix(protocol): improve terminator frame detection in write_blastware_file. fix: rename .n00 to just blastware file (.n00 was false positive) --- minimateplus/blastware_file.py | 99 +++++++++++++++++++--------------- minimateplus/client.py | 14 ++--- minimateplus/framing.py | 2 +- minimateplus/models.py | 2 +- minimateplus/protocol.py | 4 +- sfm/server.py | 8 +-- 6 files changed, 70 insertions(+), 59 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index b7794aa..c9bbe49 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -2,7 +2,7 @@ blastware_file.py — Blastware binary file codec for bidirectional interoperability. Reads and writes the proprietary Instantel/Blastware file formats: - .N00 / .9T0 / .EI0 / etc. — Waveform event (extension encoding UNKNOWN — see below) + Waveform events (.CE0W, .VM0H, .440, .7M0, etc.) (extension encoding UNKNOWN — see below) .MLG — Monitor log (monitoring session history) All waveform formats share a common 22-byte file header prefix and identical @@ -28,7 +28,7 @@ EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22: ─── File structure overview ───────────────────────────────────────────────────── -N00 (single-shot waveform, confirmed from example-events/4-3-26-multi/M529LIY6.N00): +Waveform file structure (confirmed from example-events/4-3-26-multi/M529LIY6 (example event)): [22B header] [21B STRT record] [body bytes] [26B footer] @@ -36,7 +36,7 @@ N00 (single-shot waveform, confirmed from example-events/4-3-26-multi/M529LIY6.N 10 00 01 80 00 00 — fixed prefix 49 6e 73 74 61 6e 74 65 6c 00 — b'Instantel\x00' 07 2c — fixed - 00 12 03 00 — N00 type marker + 00 12 03 00 — waveform file type tag (shared by all waveform extensions) STRT record (21 bytes, immediately follows header): 53 54 52 54 — b'STRT' @@ -84,10 +84,10 @@ MLG (monitor log, confirmed from example-events/4-3-26-multi/BE11529.MLG): ─── Critical implementation notes ────────────────────────────────────────────── -N00 body reconstruction algorithm (confirmed 2026-04-21 from verification against -M529LIY6.N00 using raw_s3_20260403_153508.bin capture): +Waveform body reconstruction algorithm (confirmed 2026-04-21 from verification against +M529LIY6 (example event) using raw_s3_20260403_153508.bin capture): - The N00 body bytes come from the A5 frame content, stripped of DLE-framing + The waveform body bytes come from the A5 frame content, stripped of DLE-framing artifacts. Each A5 frame contributes a different slice of its data section, with DLE+{0x02,0x03,0x04} byte pairs stripped. @@ -136,8 +136,8 @@ MLG CRC: All waveform extensions share the same binary format — the extension is set by blastware_filename() based on the event timestamp and type. - read_n00(path) → Event - Parse a .N00 file into an Event object with waveform data populated. + read_blastware_file(path) → Event + Parse a Blastware waveform file into an Event object with waveform data populated. (Not yet implemented — placeholder raises NotImplementedError.) write_mlg(entries, serial, path) @@ -160,7 +160,7 @@ from .models import Event, MonitorLogEntry, Timestamp # ── File header constants ───────────────────────────────────────────────────── -# Common 16-byte prefix shared by N00 and MLG (confirmed from binary inspection). +# Common 16-byte prefix shared by waveform files and MLG (confirmed from binary inspection). _FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c" # = 10 00 01 80 00 00 49 73 74 61 6e 74 65 6c 00 07 2c (17 bytes) # Confirmed breakdown: 10 00 01 80 00 00 = fixed; "Instantel\x00" = 10B; 07 2c = fixed @@ -169,19 +169,19 @@ _FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c" _FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes # Waveform file type tag (4 bytes after common prefix) — shared by ALL waveform extensions -_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 — same tag for .CE0W, .VM0H, etc. +_WAVEFORM_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6 (example event) — same tag for .CE0W, .VM0H, etc. # MLG type tag (4 bytes after common prefix) _MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14 # Total header sizes -_N00_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate. +_WAVEFORM_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate. # From binary: first 22 bytes = header, then STRT at byte 22. # 17-byte common prefix + 4-byte type tag = 21 bytes. But observed header is 22B. # Checking: 6 fixed + 10 "Instantel\x00" + 2 "07 2c" = 18B prefix, then 4B type tag = 22B. # Re-count: b"\x10\x00\x01\x80\x00\x00" = 6B + b"Instantel\x00" = 10B + b"\x07\x2c" = 2B = 18B prefix. _FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 18 bytes -_N00_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅ +_WAVEFORM_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅ _MLG_HEADER_SIZE = 308 # confirmed from BE11529.MLG # MLG record marker (4 bytes after 2-byte CRC at start of each record) @@ -201,10 +201,10 @@ def _encode_ts_be(ts: Optional[datetime.datetime]) -> bytes: """ Encode a datetime as an 8-byte big-endian Blastware timestamp. - Format (N00 and MLG record timestamps): + Format (waveform file and MLG record timestamps): [day][month][year_HI][year_LO][0x00][hour][min][sec] - Big-endian year confirmed from M529LIY6.N00 footer: + Big-endian year confirmed from M529LIY6 (example event) footer: footer bytes [2..9] = 01 04 07 ea 00 00 1c 08 → day=1 month=4 year=0x07ea=2026 hour=0 min=28 sec=8 ✅ @@ -270,7 +270,7 @@ def _strip_inner_frame_dles(data: bytes) -> bytes: Lone 0x10 bytes not followed by {0x02, 0x03, 0x04} are kept as-is. - Confirmed correct by verifying reconstructed N00 body against M529LIY6.N00: + Confirmed correct by verifying reconstructed waveform body against M529LIY6 (example event): - 0x10 0x02 in terminator → 0x02 kept ✓ - 0x10 0x04 in terminator (month byte) → 0x04 kept ✓ """ @@ -290,14 +290,14 @@ def _strip_inner_frame_dles(data: bytes) -> bytes: def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes: """ - Extract the N00 body contribution from one A5 S3Frame. + Extract the waveform body contribution from one A5 S3Frame. The contribution is frame.data[skip:] with inner-frame DLE pairs stripped per _strip_inner_frame_dles(). The chk_byte is temporarily appended before stripping to handle the split-pair edge case where a DLE at the end of frame.data is paired with chk_byte. - Split-pair edge case (confirmed for A5[8] of M529LIY6.N00, 2026-04-21): + Split-pair edge case (confirmed for A5[8] of M529LIY6 (example event), 2026-04-21): S3FrameParser appends DLE+XX pairs as two literal bytes when XX ∉ {DLE, ETX}. When the LAST occurrence of such a pair straddles the payload/checksum boundary @@ -319,7 +319,7 @@ def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes: skip: Number of leading bytes in frame.data to exclude (frame header). Returns: - bytes — the N00 body contribution for this frame. + bytes — the waveform body contribution for this frame. """ if skip >= len(frame.data): return b"" @@ -383,10 +383,10 @@ _STEM_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" # S353L4H0.9X0W Full Waveform 2025-06-23 14:01:09 AB=9X=357 ✓ # # OLD FIRMWARE (S338, 3-char extensions ending in '0') — UNKNOWN: -# Observed: .N00, .9T0, .EI0, .490, .5K0, .980, .ML0 +# Observed (old firmware / manual downloads): .440, .470, .7M0, .9T0, .EI0, etc. # The V10.72 formula does NOT apply to these. # Extension is NOT recording mode (refuted 2026-04-21: continuous → .EI0, not .9T0). -# blastware_filename() returns .N00 as a placeholder for old-firmware units. +# blastware_filename() computes the correct AB0 extension for V10.72 firmware. # # WRONG earlier assumption (do not re-introduce): # Extension was believed to encode recording mode × sample rate. @@ -502,15 +502,15 @@ def blastware_filename(event: Event, serial: str, ach: bool = False) -> str: return prefix + stem + ext -# ── N00 file writer ─────────────────────────────────────────────────────────── +# ── Waveform file writer ─────────────────────────────────────────────────────────── -def write_n00( +def write_blastware_file( event: Event, a5_frames: list[S3Frame], path: Union[str, Path], ) -> None: """ - Write a Blastware .N00 waveform file from a downloaded event. + Write a Blastware waveform file from a downloaded event. Args: event: Event object (populated by get_events() or download_waveform()). @@ -520,7 +520,7 @@ def write_n00( read_bulk_waveform_stream() when collecting frames. Must have at least 2 frames (probe + terminator). path: Destination file path. Parent directory must exist. - Extension is not enforced — caller should use ".N00". + Extension should be set via blastware_filename(). File layout: [22B header] [21B STRT] [body bytes] [26B footer] @@ -529,7 +529,7 @@ def write_n00( ValueError: if a5_frames is empty or has no terminator (page_key=0). OSError: if the file cannot be written. - Confirmed correct N00 body reconstruction against M529LIY6.N00 (2026-04-21). + Confirmed correct waveform body reconstruction against M529LIY6 (example event) (2026-04-21). """ if not a5_frames: raise ValueError("a5_frames must not be empty") @@ -538,11 +538,11 @@ def write_n00( # ── Extract STRT record from probe frame ──────────────────────────────── # The STRT record (21 bytes) lives verbatim inside A5[0].data[7:]. - # It is stored as-is in the N00 file — do NOT reconstruct it from Event + # It is stored as-is in the waveform file — do NOT reconstruct it from Event # fields, as bytes [10:14] and [14:20] contain device-specific values # (not simply key4 repeated or zero-padded). Confirmed 2026-04-21. # - # STRT layout (21 bytes, observed in M529LIY6.N00): + # STRT layout (21 bytes, observed in M529LIY6 files): # [0:4] b'STRT' # [4:6] 0xff 0xfe (fixed) # [6:10] key4 (event key) @@ -556,7 +556,7 @@ def write_n00( # Blastware file stores the stripped form, so we must strip before extracting. # # Example (M529LK0Y, 2026-04-21): STRT contains value 0x02 encoded as [10 02] - # on the wire. Without stripping, STRT is 22 raw bytes → write_n00 writes the + # on the wire. Without stripping, STRT is 22 raw bytes → write_blastware_file writes the # DLE prefix into the file AND begins the body 1 byte too early (probe_skip off # by 1). Stripping fixes both. # @@ -594,28 +594,39 @@ def write_n00( if len(strt) != 21: raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}") - # ── Build N00 header ───────────────────────────────────────────────────── - header = _FILE_HEADER_PREFIX + _N00_TYPE_TAG - assert len(header) == _N00_HEADER_SIZE, f"N00 header must be {_N00_HEADER_SIZE} bytes" + # ── Build waveform file header ───────────────────────────────────────────────────── + header = _FILE_HEADER_PREFIX + _WAVEFORM_TYPE_TAG + assert len(header) == _WAVEFORM_HEADER_SIZE, f"Waveform header must be {_WAVEFORM_HEADER_SIZE} bytes" # ── Build body from A5 frames ──────────────────────────────────────────── - # The N00 body is reconstructed from ALL A5 frames (data + terminator). + # The waveform body is reconstructed from ALL A5 frames (data + terminator). # The terminator frame's contribution includes the 26-byte footer at its end. # - # Reconstruction layout (confirmed from M529LIY6.N00, 2026-04-21): + # Reconstruction layout (confirmed from M529LIY6 captures, 2026-04-21): # all_bytes = contributions from A5[0..N] + terminator_contribution # body = all_bytes[:-26] (everything except the last 26 bytes) - # footer = all_bytes[-26:] (last 26 bytes = the N00 footer) + # footer = all_bytes[-26:] (last 26 bytes = the waveform file footer) # # The footer bytes come directly from the terminator frame's inner content — # using them verbatim ensures timestamps match the device's recorded values. - # Separate terminator from data frames - body_frames = a5_frames - term_frame: Optional[S3Frame] = None - if a5_frames and a5_frames[-1].page_key == 0x0000: - body_frames = a5_frames[:-1] - term_frame = a5_frames[-1] + # Separate terminator from data frames. + # Search from the FRONT for the first terminator (page_key == 0x0000). + # Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a + # subsequent event (a known get_events side-effect), the last frame will + # not be the terminator and the footer will be mis-identified. + term_idx: Optional[int] = None + for _i, _f in enumerate(a5_frames): + if _f.page_key == 0x0000: + term_idx = _i + break + + if term_idx is not None: + body_frames = a5_frames[:term_idx] + term_frame = a5_frames[term_idx] + else: + body_frames = a5_frames + term_frame = None all_bytes = bytearray() @@ -660,14 +671,14 @@ def write_n00( f.write(footer) -def read_n00(path: Union[str, Path]) -> Event: +def read_blastware_file(path: Union[str, Path]) -> Event: """ - Parse a Blastware .N00 file into an Event object. + Parse a Blastware waveform file into an Event object. NOT YET IMPLEMENTED. Args: - path: Path to the .N00 file. + path: Path to the waveform file. Returns: Event object with waveform data populated. @@ -675,7 +686,7 @@ def read_n00(path: Union[str, Path]) -> Event: Raises: NotImplementedError: always (pending implementation). """ - raise NotImplementedError("read_n00() is not yet implemented") + raise NotImplementedError("read_blastware_file() is not yet implemented") # ── MLG file writer ─────────────────────────────────────────────────────────── diff --git a/minimateplus/client.py b/minimateplus/client.py index e287844..04887ad 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -608,7 +608,7 @@ class MiniMateClient: ) if a5_frames: a5_ok = True - ev._a5_frames = a5_frames # store for write_n00 + ev._a5_frames = a5_frames # store for write_blastware_file _decode_a5_metadata_into(a5_frames, ev) _decode_a5_waveform(a5_frames, ev) log.info( @@ -624,7 +624,7 @@ class MiniMateClient: ) if a5_frames: a5_ok = True - ev._a5_frames = a5_frames # store for write_n00 + ev._a5_frames = a5_frames # store for write_blastware_file _decode_a5_metadata_into(a5_frames, ev) log.debug( "get_events: 5A metadata client=%r operator=%r", @@ -783,29 +783,29 @@ class MiniMateClient: def save_blastware_file(self, event: "Event", path: "Union[str, Path]", serial: str) -> None: """ Download the full waveform for *event* and save it as a Blastware- - compatible .N00 / .9T0 file at *path*. + compatible Blastware waveform file at *path*. This is a convenience wrapper that calls download_waveform() (which performs the complete SUB 5A BULK_WAVEFORM_STREAM download) and then - calls write_n00() from blastware_file.py to encode the result. + calls write_blastware_file() from blastware_file.py to encode the result. Args: event: Event object with waveform key populated (from get_events()). path: Destination file path. Caller should use blastware_filename() - to pick the correct .N00 / .9T0 extension. + to pick the correct extension via blastware_filename(). serial: Device serial number (e.g. "BE11529") — passed to blastware_filename() for reference, but the caller supplies the final path. """ from pathlib import Path as _Path - from .blastware_file import write_n00 as _write_n00 + from .blastware_file import write_blastware_file as _write_blastware_file a5_frames = self.download_waveform(event) if not a5_frames: raise RuntimeError( f"save_blastware_file: no A5 frames received for event#{event.index}" ) - _write_n00(event, a5_frames, path) + _write_blastware_file(event, a5_frames, path) log.info( "save_blastware_file: wrote %s (%d A5 frames)", path, len(a5_frames), diff --git a/minimateplus/framing.py b/minimateplus/framing.py index 7df3177..3adf4ce 100644 --- a/minimateplus/framing.py +++ b/minimateplus/framing.py @@ -458,7 +458,7 @@ class S3Frame: data: bytes # payload data section (payload[5:], checksum already stripped) checksum_valid: bool chk_byte: int = 0 # actual checksum byte received from wire (body[-1]) - # needed for N00 file reconstruction: when the last data byte + # needed for waveform file reconstruction: when the last data byte # is 0x10 and chk_byte ∈ {0x02, 0x03, 0x04}, the DLE+chk pair # must be included in the DLE-strip operation to correctly # reconstruct the Blastware binary body. diff --git a/minimateplus/models.py b/minimateplus/models.py index 1b0de5c..47d4028 100644 --- a/minimateplus/models.py +++ b/minimateplus/models.py @@ -494,7 +494,7 @@ class Event: _waveform_key: Optional[bytes] = field(default=None, repr=False) # Raw A5 frames from the full bulk waveform download (full_waveform=True). - # Populated by get_events() when full_waveform=True; used by write_n00(). + # Populated by get_events() when full_waveform=True; used by write_blastware_file(). _a5_frames: Optional[list] = field(default=None, repr=False) def __str__(self) -> str: diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 8691559..7ff0a03 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -545,7 +545,7 @@ class MiniMateProtocol: By default the termination frame (page_key=0x0000) is NOT included in the returned list. Pass include_terminator=True to append it; the blastware_file - writer needs the terminator frame's body to reconstruct the N00 footer. + writer needs the terminator frame's body to reconstruct the waveform file footer. Args: key4: 4-byte waveform key from EVENT_HEADER (1E). @@ -557,7 +557,7 @@ class MiniMateProtocol: (default 32; a typical event uses 9 large frames). include_terminator: If True, append the terminator A5 frame (page_key=0x0000) to the returned list. The - terminator carries the N00 footer bytes. + terminator carries the waveform file footer bytes. Default False preserves existing caller behaviour. Returns: diff --git a/sfm/server.py b/sfm/server.py index aaad0b6..462955c 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -61,7 +61,7 @@ from minimateplus import MiniMateClient from minimateplus.protocol import ProtocolError from minimateplus.models import CallHomeConfig, ComplianceConfig, DeviceInfo, Event, PeakValues, ProjectInfo, Timestamp from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT -from minimateplus.blastware_file import write_n00, blastware_filename +from minimateplus.blastware_file import write_blastware_file, blastware_filename from sfm.cache import SFMCache, get_cache from sfm.database import SeismoDb @@ -874,7 +874,7 @@ def device_event_blastware_file( to be populated from compliance config) Performs: POLL startup → get_events(full_waveform=False, stop_after_index=index) - → write_n00() → FileResponse. + → write_blastware_file() → FileResponse. """ log.info( "GET /device/event/%d/blastware_file port=%s host=%s", @@ -890,7 +890,7 @@ def device_event_blastware_file( # the full bulk download. Using full_waveform=True produces a file # ~8x larger than Blastware's because it includes all post-event # silence chunks. The metadata-only a5_frames (with terminator) are - # sufficient for byte-perfect write_n00 output. + # sufficient for byte-perfect write_blastware_file output. events = client.get_events(full_waveform=False, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info @@ -928,7 +928,7 @@ def device_event_blastware_file( # Write to /tmp so FastAPI can stream it back out_path = Path("/tmp") / filename - write_n00(ev, a5_frames, out_path) + write_blastware_file(ev, a5_frames, out_path) log.info( "blastware_file: wrote %s (%d A5 frames, serial=%s)", out_path, len(a5_frames), serial, -- 2.52.0 From ec6362cb8ee73c01ccf3a7e1efea5c734a712398 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 12:45:59 -0400 Subject: [PATCH 09/40] fix(protocol): include terminator in waveform stream downloads --- minimateplus/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/minimateplus/client.py b/minimateplus/client.py index 04887ad..2810afd 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -604,7 +604,8 @@ class MiniMateClient: "get_events: 5A full waveform download for key=%s", cur_key.hex() ) a5_frames = proto.read_bulk_waveform_stream( - cur_key, stop_after_metadata=False, max_chunks=128 + cur_key, stop_after_metadata=False, max_chunks=128, + include_terminator=True, ) if a5_frames: a5_ok = True @@ -620,7 +621,8 @@ class MiniMateClient: "get_events: 5A metadata-only download for key=%s", cur_key.hex() ) a5_frames = proto.read_bulk_waveform_stream( - cur_key, stop_after_metadata=True + cur_key, stop_after_metadata=True, + include_terminator=True, ) if a5_frames: a5_ok = True -- 2.52.0 From 84c87d0b57c8bfa8f911ab04f43d10f34888e92b Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 13:02:55 -0400 Subject: [PATCH 10/40] fix(protocol): adjust waveform download to use full_waveform for accurate event streaming --- sfm/server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index 462955c..58c69da 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,13 +885,13 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=False (metadata-only, stop_after_metadata=True) — - # Blastware writes waveform files from only the first ~8 A5 frames, NOT - # the full bulk download. Using full_waveform=True produces a file - # ~8x larger than Blastware's because it includes all post-event - # silence chunks. The metadata-only a5_frames (with terminator) are - # sufficient for byte-perfect write_blastware_file output. - events = client.get_events(full_waveform=False, stop_after_index=index) + # Use full_waveform=True (stop_after_metadata=False) so the device + # signals its own end-of-stream rather than us stopping at "Project:". + # BW downloads until natural end-of-stream for each event — for this + # 5-chunk event that gives the correct body + footer. For events with + # many silence chunks (35+) the file will be larger than BW's, but + # correctness takes priority over size matching for now. + events = client.get_events(full_waveform=True, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From 39ebd4bdaaded698407ef8f1e119e16d07d44dbf Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 15:11:56 -0400 Subject: [PATCH 11/40] fix(protocol): revert endpoint back to stop_after_metadata=True --- sfm/server.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index 58c69da..22248c5 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,13 +885,12 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=True (stop_after_metadata=False) so the device - # signals its own end-of-stream rather than us stopping at "Project:". - # BW downloads until natural end-of-stream for each event — for this - # 5-chunk event that gives the correct body + footer. For events with - # many silence chunks (35+) the file will be larger than BW's, but - # correctness takes priority over size matching for now. - events = client.get_events(full_waveform=True, stop_after_index=index) + # Use full_waveform=False (stop_after_metadata=True) — downloads until + # "Project:" is found in the 5A stream, which covers the compliance + # metadata section. For simple Continuous/Single-Shot mode events this + # produces the correct body content. Histogram+Continuous mode requires + # different handling (TODO: handle multi-mode events). + events = client.get_events(full_waveform=False, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From 5e2f3bf2a195b5bc797e07f5a0fd87a01b0fb1bd Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 16:24:39 -0400 Subject: [PATCH 12/40] fix(protocol): enable full_waveform for continuous mode. --- sfm/server.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index 22248c5..de42a51 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,12 +885,13 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=False (stop_after_metadata=True) — downloads until - # "Project:" is found in the 5A stream, which covers the compliance - # metadata section. For simple Continuous/Single-Shot mode events this - # produces the correct body content. Histogram+Continuous mode requires - # different handling (TODO: handle multi-mode events). - events = client.get_events(full_waveform=False, stop_after_index=index) + # Use full_waveform=True (stop_after_metadata=False) — downloads until + # the device signals natural end-of-stream. For simple Continuous / + # Single-Shot events this gives byte-identical content to BW. + # NOTE: Histogram+Continuous mode produces extra embedded STRT records + # (session data from preceding histogram intervals bleeds into the 5A + # stream) — handle that mode separately once basic waveform is correct. + events = client.get_events(full_waveform=True, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From 9e7e0bce2a39bb56564f816557752b7e8a05a202 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 16:43:59 -0400 Subject: [PATCH 13/40] fix(protocol): adjust full_waveform setting for event downloads to end when it should. --- sfm/server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index de42a51..8241d17 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,13 +885,13 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=True (stop_after_metadata=False) — downloads until - # the device signals natural end-of-stream. For simple Continuous / - # Single-Shot events this gives byte-identical content to BW. - # NOTE: Histogram+Continuous mode produces extra embedded STRT records - # (session data from preceding histogram intervals bleeds into the 5A - # stream) — handle that mode separately once basic waveform is correct. - events = client.get_events(full_waveform=True, stop_after_index=index) + # Use full_waveform=False (stop_after_metadata=True) — stops when + # "Project:" is found in the 5A stream. Content is byte-identical to + # BW for Continuous/Single-Shot events; our file is slightly shorter + # (~286 bytes of extra ADC signal BW includes past the metadata). + # full_waveform=True corrupts the body: silence chunks past the event + # contain device-internal pointers that embed extra STRT records. + events = client.get_events(full_waveform=False, stop_after_index=index) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From 2a2031c3a97550930c6466de7979f33ab1739e28 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 17:08:36 -0400 Subject: [PATCH 14/40] fix(protocol): fetch additional chunk after metadata to ensure valid termination response --- minimateplus/protocol.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 7ff0a03..1359abf 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -646,7 +646,27 @@ class MiniMateProtocol: frames_data.append(rsp) if stop_after_metadata and b"Project:" in rsp.data: - log.debug("5A A5[%d] metadata found — stopping early", chunk_num) + # Download exactly one more chunk after finding metadata — this is + # what Blastware does. The extra chunk contains the tail ADC data + # and primes the device to return a valid footer in the termination + # response. Without it, termination returns an empty ack with no + # footer bytes (confirmed 2026-04-23 from HxD comparison). + log.debug("5A A5[%d] metadata found — fetching one more chunk then stopping", chunk_num) + chunk_num += 1 + counter = chunk_num * _BULK_COUNTER_STEP + params = bulk_waveform_params(key4, counter) + self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) + try: + extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) + log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", + chunk_num, extra.page_key, len(extra.data)) + if extra.page_key == 0x0000: + if include_terminator: + frames_data.append(extra) + return frames_data + frames_data.append(extra) + except TimeoutError: + log.debug("5A extra chunk timed out — end of stream") break else: log.warning( -- 2.52.0 From aa2b02535b2fdf19789c657c41c37528571b9495 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 17:33:16 -0400 Subject: [PATCH 15/40] fix(protocol): add record_time based chunk scaling for longer event record times --- CLAUDE.md | 2 +- docs/instantel_protocol_reference.md | 18 +++++++++++++- minimateplus/client.py | 3 ++- minimateplus/protocol.py | 36 +++++++++++++++------------- sfm/server.py | 23 ++++++++++++------ 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d6f9a5a..32dd6d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1096,7 +1096,7 @@ body) because writing a dial string may require DLE escaping for embedded contro - **Database** — SQLite store for events + monitor log entries; dedup by key; queryable - **Histograms** — decode histogram-mode A5 data (noise floor tracking) -- **Blastware-compatible file output** — `write_n00()` and `write_mlg()` implemented (v0.12.3+). `write_n00` verified byte-perfect vs M529LIY6.N00. Extension mapping: **CONFIRMED FALSE 2026-04-21** — extensions are NOT based on recording mode. A continuous-mode event produced `.EI0`, not `.9T0`. The extension alphabet/encoding scheme is unknown; do not infer recording mode from extension or vice versa. Observed extensions: `.N00`, `.9T0`, `.EI0`, `.490`, `.5K0`, `.980`, `.ML0` — mapping to recording mode × sample rate × other settings is unknown. Filename format: `<4-char-base36-stem>` +- **Blastware-compatible file output** — `write_blastware_file()` and `write_mlg()` implemented. `blastware_filename()` generates correct Blastware filenames (AB0 for direct, AB0W/AB0H for ACH). **Confirmed working for Continuous mode events (2026-04-23):** SFM-generated file opens in Blastware, shows correct PPV/waveform/timestamp. File is ~200 bytes shorter than BW (missing last ADC tail slice) — all measurements correct. Histogram+Continuous mode deferred (5A stream for those events embeds histogram interval records that create spurious STRT markers in the body). Extension mapping: **CONFIRMED FALSE 2026-04-21** — extensions encode timestamp (AB0T for ACH, AB0 for direct), NOT recording mode. Filename format: `<4-char-base36-stem>` **Serial encoding (CONFIRMED 2026-04-22):** `prefix_letter = chr(ord('B') + floor(serial_numeric / 1000))`, `serial3 = f"{serial_numeric % 1000:03d}"`. Examples: BE6907→H907, BE11529→M529, BE14036→P036, BE17353→S353, BE18003→T003. The prefix letter encodes the production generation (batch of 1000 units). diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 6f64f9e..cd0812e 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -1248,9 +1248,25 @@ Two critical differences from `build_bw_frame`: > for all keys encountered on this device. The `stop_after_metadata=True` flag causes the loop to stop as soon as `b"Project:"` is -found in the accumulated A5 frame data, typically after 7–9 chunks. A termination frame +found in the accumulated A5 frame data, typically after 4–9 chunks. A termination frame is always sent before returning. +**IMPORTANT — one extra chunk required after "Project:" for valid file footer (confirmed 2026-04-23):** +When writing a Blastware-compatible waveform file, stopping immediately at "Project:" and +sending termination produces an empty termination response with no footer bytes (`0e 08` +marker missing). Blastware downloads exactly **one more chunk** after finding "Project:" +before sending termination — that extra chunk primes the device to return valid footer +bytes (monitoring start/stop timestamps) in the termination response. + +`read_bulk_waveform_stream(stop_after_metadata=True)` implements this: after the "Project:" +chunk is received, one additional chunk is requested before breaking. The termination +response (`include_terminator=True`) then contains the correct `0e 08` footer. + +**do NOT use `full_waveform=True` for Blastware file writing** — for events with long +post-event silence (35 chunks), the silence chunks contain embedded device-internal +pointer structures that produce spurious STRT markers in the file body. Blastware only +downloads 4–5 chunks (metadata + one signal chunk) regardless of event length. + #### 7.8.3 A5 Frame Layout Each A5 response frame contains a chunk of raw bulk data. Frame 7 of the stream carries the diff --git a/minimateplus/client.py b/minimateplus/client.py index 2810afd..0fa133b 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -449,7 +449,7 @@ class MiniMateClient: proto.confirm_erase_all() log.info("delete_all_events: erase confirmed — device memory cleared") - def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None) -> list[Event]: + def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None, extra_chunks_after_metadata: int = 1) -> list[Event]: """ Download all stored events from the device using the confirmed 1E → 0A → 0C → 5A → 1F event-iterator protocol. @@ -623,6 +623,7 @@ class MiniMateClient: a5_frames = proto.read_bulk_waveform_stream( cur_key, stop_after_metadata=True, include_terminator=True, + extra_chunks_after_metadata=extra_chunks_after_metadata, ) if a5_frames: a5_ok = True diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 1359abf..578f11a 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -527,6 +527,7 @@ class MiniMateProtocol: stop_after_metadata: bool = True, max_chunks: int = 32, include_terminator: bool = False, + extra_chunks_after_metadata: int = 1, ) -> list[S3Frame]: """ Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event. @@ -651,22 +652,25 @@ class MiniMateProtocol: # and primes the device to return a valid footer in the termination # response. Without it, termination returns an empty ack with no # footer bytes (confirmed 2026-04-23 from HxD comparison). - log.debug("5A A5[%d] metadata found — fetching one more chunk then stopping", chunk_num) - chunk_num += 1 - counter = chunk_num * _BULK_COUNTER_STEP - params = bulk_waveform_params(key4, counter) - self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - try: - extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", - chunk_num, extra.page_key, len(extra.data)) - if extra.page_key == 0x0000: - if include_terminator: - frames_data.append(extra) - return frames_data - frames_data.append(extra) - except TimeoutError: - log.debug("5A extra chunk timed out — end of stream") + log.debug("5A A5[%d] metadata found — fetching %d more chunk(s) then stopping", + chunk_num, extra_chunks_after_metadata) + for _extra_n in range(extra_chunks_after_metadata): + chunk_num += 1 + counter = chunk_num * _BULK_COUNTER_STEP + params = bulk_waveform_params(key4, counter) + self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) + try: + extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) + log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", + chunk_num, extra.page_key, len(extra.data)) + if extra.page_key == 0x0000: + if include_terminator: + frames_data.append(extra) + return frames_data + frames_data.append(extra) + except TimeoutError: + log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) + break break else: log.warning( diff --git a/sfm/server.py b/sfm/server.py index 8241d17..077a3a5 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,13 +885,22 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=False (stop_after_metadata=True) — stops when - # "Project:" is found in the 5A stream. Content is byte-identical to - # BW for Continuous/Single-Shot events; our file is slightly shorter - # (~286 bytes of extra ADC signal BW includes past the metadata). - # full_waveform=True corrupts the body: silence chunks past the event - # contain device-internal pointers that embed extra STRT records. - events = client.get_events(full_waveform=False, stop_after_index=index) + # Calculate extra ADC chunks to download after finding "Project:". + # BW downloads ~2 extra chunks per second of record time. + # Without enough extra chunks the termination response contains no + # footer bytes and Blastware rejects the file. + rectime = 1.0 + try: + rectime = float(info.compliance_config.record_time or 1.0) + except (AttributeError, TypeError, ValueError): + pass + extra_chunks = max(1, round(rectime * 2)) + log.info("blastware_file: rectime=%.1fs → extra_chunks=%d", rectime, extra_chunks) + events = client.get_events( + full_waveform=False, + stop_after_index=index, + extra_chunks_after_metadata=extra_chunks, + ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info ev, info = _run_with_retry(_do, is_tcp=_is_tcp(host)) -- 2.52.0 From bc9f16e5030abc52542c262d8ff91256c89678db Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 17:39:28 -0400 Subject: [PATCH 16/40] fix(protocol): adjust extra_chunks calculation to use integer conversion of record_time --- sfm/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sfm/server.py b/sfm/server.py index 077a3a5..56b4b5c 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -894,7 +894,7 @@ def device_event_blastware_file( rectime = float(info.compliance_config.record_time or 1.0) except (AttributeError, TypeError, ValueError): pass - extra_chunks = max(1, round(rectime * 2)) + extra_chunks = max(1, int(rectime)) log.info("blastware_file: rectime=%.1fs → extra_chunks=%d", rectime, extra_chunks) events = client.get_events( full_waveform=False, -- 2.52.0 From ecd980d345f6369130e86cc5278c9788d1688382 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 18:22:27 -0400 Subject: [PATCH 17/40] fix(protocol): enhance extra chunk fetching logic to ensure footer detection --- minimateplus/protocol.py | 21 +++++++++++++++++---- sfm/server.py | 12 ++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 578f11a..a578297 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -652,8 +652,14 @@ class MiniMateProtocol: # and primes the device to return a valid footer in the termination # response. Without it, termination returns an empty ack with no # footer bytes (confirmed 2026-04-23 from HxD comparison). - log.debug("5A A5[%d] metadata found — fetching %d more chunk(s) then stopping", - chunk_num, extra_chunks_after_metadata) + # Download extra chunks until we find the footer chunk (contains + # 0x0e 0x08 near its end) or hit the extra_chunks_after_metadata + # cap. The footer bytes are embedded in the last meaningful data + # chunk, NOT in the termination response. For short events (1-sec) + # the footer is in extra chunk 1. For longer events it is in a + # later chunk. Confirmed 2026-04-23 from BW file comparison. + log.debug("5A A5[%d] metadata found — fetching extra chunks until footer found", + chunk_num) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 counter = chunk_num * _BULK_COUNTER_STEP @@ -661,13 +667,20 @@ class MiniMateProtocol: self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", - chunk_num, extra.page_key, len(extra.data)) + log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d has_footer=%s", + chunk_num, extra.page_key, len(extra.data), + b'\x0e\x08' in extra.data[-50:]) if extra.page_key == 0x0000: if include_terminator: frames_data.append(extra) return frames_data frames_data.append(extra) + # Stop as soon as we find the footer marker in the tail + # of this chunk — the body is complete. + if b'\x0e\x08' in extra.data[-50:]: + log.debug("5A A5[%d] footer marker found — stopping extra chunks", + chunk_num) + break except TimeoutError: log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) break diff --git a/sfm/server.py b/sfm/server.py index 56b4b5c..13d9c10 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,17 +885,17 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Calculate extra ADC chunks to download after finding "Project:". - # BW downloads ~2 extra chunks per second of record time. - # Without enough extra chunks the termination response contains no - # footer bytes and Blastware rejects the file. + # Download extra chunks after "Project:" until the footer marker + # 0x0e 0x08 is detected in a chunk tail. The cap prevents + # accidentally downloading into post-event silence. + # Cap = rectime * 4 + 4 covers up to ~10 sec events safely. rectime = 1.0 try: rectime = float(info.compliance_config.record_time or 1.0) except (AttributeError, TypeError, ValueError): pass - extra_chunks = max(1, int(rectime)) - log.info("blastware_file: rectime=%.1fs → extra_chunks=%d", rectime, extra_chunks) + extra_chunks = max(4, int(rectime * 4) + 4) + log.info("blastware_file: rectime=%.1fs → extra_chunks_cap=%d", rectime, extra_chunks) events = client.get_events( full_waveform=False, stop_after_index=index, -- 2.52.0 From fa887b85d92067f6d4494b3f3c913829ba1e036f Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 18:28:14 -0400 Subject: [PATCH 18/40] fix(protocol): update extra chunk fetching logic to stop at silence detection --- minimateplus/protocol.py | 35 ++++++++++++++++++++--------------- sfm/server.py | 2 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index a578297..effaf0f 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -652,13 +652,12 @@ class MiniMateProtocol: # and primes the device to return a valid footer in the termination # response. Without it, termination returns an empty ack with no # footer bytes (confirmed 2026-04-23 from HxD comparison). - # Download extra chunks until we find the footer chunk (contains - # 0x0e 0x08 near its end) or hit the extra_chunks_after_metadata - # cap. The footer bytes are embedded in the last meaningful data - # chunk, NOT in the termination response. For short events (1-sec) - # the footer is in extra chunk 1. For longer events it is in a - # later chunk. Confirmed 2026-04-23 from BW file comparison. - log.debug("5A A5[%d] metadata found — fetching extra chunks until footer found", + # Download extra chunks until we hit post-event silence (all-FF + # ADC values) or the cap. The device returns the footer in the + # termination response only when we stop at the right point — + # right after the last real data chunk, before silence starts. + # Silence detection: >80% of payload bytes are 0xFF. + log.debug("5A A5[%d] metadata found — fetching extra chunks until silence", chunk_num) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 @@ -667,20 +666,26 @@ class MiniMateProtocol: self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d has_footer=%s", - chunk_num, extra.page_key, len(extra.data), - b'\x0e\x08' in extra.data[-50:]) + payload = extra.data[7:] # skip 7-byte frame header + ff_ratio = payload.count(0xFF) / max(len(payload), 1) + is_silence = ff_ratio > 0.8 + log.debug( + "5A A5[%d] extra chunk page_key=0x%04X data_len=%d " + "ff_ratio=%.2f silence=%s", + chunk_num, extra.page_key, len(extra.data), + ff_ratio, is_silence, + ) if extra.page_key == 0x0000: if include_terminator: frames_data.append(extra) return frames_data - frames_data.append(extra) - # Stop as soon as we find the footer marker in the tail - # of this chunk — the body is complete. - if b'\x0e\x08' in extra.data[-50:]: - log.debug("5A A5[%d] footer marker found — stopping extra chunks", + if is_silence: + # Don't include the silence chunk — terminate here. + # The termination response will contain the footer. + log.debug("5A A5[%d] silence detected — stopping before this chunk", chunk_num) break + frames_data.append(extra) except TimeoutError: log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) break diff --git a/sfm/server.py b/sfm/server.py index 13d9c10..21a8308 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -894,7 +894,7 @@ def device_event_blastware_file( rectime = float(info.compliance_config.record_time or 1.0) except (AttributeError, TypeError, ValueError): pass - extra_chunks = max(4, int(rectime * 4) + 4) + extra_chunks = max(4, int(rectime * 6) + 8) # generous cap; silence detection stops early log.info("blastware_file: rectime=%.1fs → extra_chunks_cap=%d", rectime, extra_chunks) events = client.get_events( full_waveform=False, -- 2.52.0 From ab2c11e9a90c7d401744589901cb5891e1ca55e1 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 23 Apr 2026 20:30:07 -0400 Subject: [PATCH 19/40] fix(protocol): refine extra chunk fetching logic for accurate termination response --- minimateplus/protocol.py | 32 ++++++++++---------------------- sfm/server.py | 22 +++++++++------------- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index effaf0f..9f48bc5 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -652,13 +652,14 @@ class MiniMateProtocol: # and primes the device to return a valid footer in the termination # response. Without it, termination returns an empty ack with no # footer bytes (confirmed 2026-04-23 from HxD comparison). - # Download extra chunks until we hit post-event silence (all-FF - # ADC values) or the cap. The device returns the footer in the - # termination response only when we stop at the right point — - # right after the last real data chunk, before silence starts. - # Silence detection: >80% of payload bytes are 0xFF. - log.debug("5A A5[%d] metadata found — fetching extra chunks until silence", - chunk_num) + # Download extra_chunks_after_metadata more chunks past the + # metadata. The caller calculates this from record_time and + # sample_rate so we download exactly the right amount of ADC + # data — no more, no less — before terminating. + # The device returns the footer in the termination response only + # after the right amount of data has been consumed. + log.debug("5A A5[%d] metadata found — fetching %d more chunk(s)", + chunk_num, extra_chunks_after_metadata) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 counter = chunk_num * _BULK_COUNTER_STEP @@ -666,25 +667,12 @@ class MiniMateProtocol: self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - payload = extra.data[7:] # skip 7-byte frame header - ff_ratio = payload.count(0xFF) / max(len(payload), 1) - is_silence = ff_ratio > 0.8 - log.debug( - "5A A5[%d] extra chunk page_key=0x%04X data_len=%d " - "ff_ratio=%.2f silence=%s", - chunk_num, extra.page_key, len(extra.data), - ff_ratio, is_silence, - ) + log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", + chunk_num, extra.page_key, len(extra.data)) if extra.page_key == 0x0000: if include_terminator: frames_data.append(extra) return frames_data - if is_silence: - # Don't include the silence chunk — terminate here. - # The termination response will contain the footer. - log.debug("5A A5[%d] silence detected — stopping before this chunk", - chunk_num) - break frames_data.append(extra) except TimeoutError: log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) diff --git a/sfm/server.py b/sfm/server.py index 21a8308..5d57c49 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,21 +885,17 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Download extra chunks after "Project:" until the footer marker - # 0x0e 0x08 is detected in a chunk tail. The cap prevents - # accidentally downloading into post-event silence. - # Cap = rectime * 4 + 4 covers up to ~10 sec events safely. - rectime = 1.0 - try: - rectime = float(info.compliance_config.record_time or 1.0) - except (AttributeError, TypeError, ValueError): - pass - extra_chunks = max(4, int(rectime * 6) + 8) # generous cap; silence detection stops early - log.info("blastware_file: rectime=%.1fs → extra_chunks_cap=%d", rectime, extra_chunks) + # Use full_waveform=True (stop_after_metadata=False) so the device + # streams until it naturally signals end-of-stream (1-raw-byte signal). + # BW does the same — the ACH download and manual pull both let the device + # determine when to stop. The termination response at that point contains + # the correct 0e 08 footer with monitoring timestamps. + # For Continuous/Single-Shot events, end-of-stream comes after the real + # ADC data (not after 35+ silence chunks as in Histogram+Continuous mode). + # max_chunks=32 is a safety cap; natural end-of-stream stops much earlier. events = client.get_events( - full_waveform=False, + full_waveform=True, stop_after_index=index, - extra_chunks_after_metadata=extra_chunks, ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info -- 2.52.0 From f83fd880c0a63aa7ca7843a412c98167b155b5f9 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 00:35:34 -0400 Subject: [PATCH 20/40] fix(protocol): update device_event_blastware_file to include extra chunk for accurate data retrieval --- sfm/server.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index 5d57c49..97dd7b2 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -873,8 +873,8 @@ def device_event_blastware_file( triggered events; histogram requires recording_mode to be populated from compliance config) - Performs: POLL startup → get_events(full_waveform=False, stop_after_index=index) - → write_blastware_file() → FileResponse. + Performs: POLL startup → get_events(full_waveform=False, extra_chunks=1, + stop_after_index=index) → write_blastware_file() → FileResponse. """ log.info( "GET /device/event/%d/blastware_file port=%s host=%s", @@ -885,17 +885,21 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use full_waveform=True (stop_after_metadata=False) so the device - # streams until it naturally signals end-of-stream (1-raw-byte signal). - # BW does the same — the ACH download and manual pull both let the device - # determine when to stop. The termination response at that point contains - # the correct 0e 08 footer with monitoring timestamps. - # For Continuous/Single-Shot events, end-of-stream comes after the real - # ADC data (not after 35+ silence chunks as in Histogram+Continuous mode). - # max_chunks=32 is a safety cap; natural end-of-stream stops much earlier. + # Use stop_after_metadata=True (full_waveform=False) with 1 extra + # chunk after "Project:". For any record time, the pre-metadata + # section of the 5A stream naturally carries proportionally more + # ADC data for longer events — so "1 extra chunk" produces the + # correct body length regardless of record time. + # + # full_waveform=True (natural end-of-stream) downloads ALL chunks + # including post-event silence (35+ chunks for a 9-sec event at + # 1024 sps) — this produces 24KB+ files that Blastware rejects. + # Confirmed from file size comparison: BW 1-sec=4400B, BW 3-sec=8114B, + # per-second delta 1857 bytes — matches pre-metadata frame scaling. events = client.get_events( - full_waveform=True, + full_waveform=False, stop_after_index=index, + extra_chunks_after_metadata=1, ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info -- 2.52.0 From 03540fdc00000ceb8360cec7b2348596e1a0ff1d Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 02:19:27 -0400 Subject: [PATCH 21/40] fix: raise max_chunks to 128 for metadata-only 5A download For 2-second events at 1024 sps the "Project:" metadata frame appears beyond chunk 32 (the old default cap), causing the safety limit to be hit and ~34 KB of waveform data to be downloaded instead of stopping at the metadata frame. Raising max_chunks to 128 ensures stop_after_metadata=True can locate the metadata frame for record times up to ~4 seconds. Co-Authored-By: Claude Sonnet 4.6 --- minimateplus/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/minimateplus/client.py b/minimateplus/client.py index 0fa133b..8f01f3d 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -624,6 +624,7 @@ class MiniMateClient: cur_key, stop_after_metadata=True, include_terminator=True, extra_chunks_after_metadata=extra_chunks_after_metadata, + max_chunks=128, ) if a5_frames: a5_ok = True -- 2.52.0 From 242666f3583134b9920790970f7ca1e3e90aec86 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 12:52:02 -0400 Subject: [PATCH 22/40] fix(protocol): correct chunk counter formula for accurate data streaming --- minimateplus/protocol.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 9f48bc5..5a2537f 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -583,8 +583,17 @@ class MiniMateProtocol: frames_data: list[S3Frame] = [] counter = 0 + # BW counter formula (confirmed from 4-3-26 capture for key 0111245a): + # counter for chunk n = key4[2:4] + (n - 1) * 0x0400 + # key4[2:4] is the event's circular-buffer base offset — without it, chunk + # requests address the wrong region of the device buffer and the device + # streams data from the wrong event (no "Project:" in any response). + # PREVIOUSLY WRONG NOTE: "device does not validate counter; chunk_num*0x0400 + # is correct" — that was only true for key 01110000 where key4[2:4]==0x0000. + _key4_offset = (key4[2] << 8) | key4[3] + # ── Step 1: probe ──────────────────────────────────────────────────── - log.debug("5A probe key=%s", key4.hex()) + log.debug("5A probe key=%s key4_offset=0x%04X", key4.hex(), _key4_offset) params = bulk_waveform_params(key4, 0, is_probe=True) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) self._parser.reset() # reset bytes_fed counter before probe recv @@ -601,13 +610,15 @@ class MiniMateProtocol: log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data)) # ── Step 2: chunk loop ─────────────────────────────────────────────── - # Chunk counters are monotonic: chunk_num * 0x0400 for all chunks. - # The 4-2-26 BW TX capture showed 0x1004 for chunk 1, but this is a - # Blastware artifact — the device accepts any counter value and streams - # data regardless. Empirically confirmed 2026-04-06: 0x0400 for chunk 1 - # works; 0x1004 causes the device to ignore the frame (timeout). + # Correct counter formula: key4[2:4] + (chunk_num - 1) * 0x0400 + # This matches Blastware exactly (confirmed from 4-3-26 capture). + # For events where key4[2:4]==0 (e.g. 01110000), this gives the same + # result as the old chunk_num*0x0400 formula shifted by one step, which + # the device also accepted — but for events with a non-zero base offset + # (e.g. key 01111884 with key4[2:4]=0x1884) the old formula sends + # completely wrong counters and the device streams the wrong buffer region. for chunk_num in range(1, max_chunks + 1): - counter = chunk_num * _BULK_COUNTER_STEP + counter = _key4_offset + (chunk_num - 1) * _BULK_COUNTER_STEP params = bulk_waveform_params(key4, counter) log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) @@ -662,7 +673,7 @@ class MiniMateProtocol: chunk_num, extra_chunks_after_metadata) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 - counter = chunk_num * _BULK_COUNTER_STEP + counter = _key4_offset + (chunk_num - 1) * _BULK_COUNTER_STEP params = bulk_waveform_params(key4, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: -- 2.52.0 From 43c81584939cc5da8b8dc7f4cbb6a14e47dc418c Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 15:48:37 -0400 Subject: [PATCH 23/40] feat(blastware_file): classify A5 frames, only write waveform frames to body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add classify_frame() which categorises each A5 frame by content: terminator — page_key == 0x0000 probe_or_strt — contains b"STRT" metadata — contains compliance-config ASCII markers (Project:, Client:, Standard Recording Setup, …) waveform — binary-heavy (< 20% printable ASCII), i.e. raw ADC data unknown — fallback Update write_blastware_file() body loop: frame 0 (probe) is still always processed; frames 1+ are only included when classify_frame returns "waveform". Metadata frames (compliance config block with Project:/Client:/etc.) and any stray STRT-bearing frames are skipped with a warning/debug log. Terminator frame handling is unchanged. Adds temporary print() diagnostics so each frame's classification is visible in the server log to aid debugging. Co-Authored-By: Claude Sonnet 4.6 --- minimateplus/blastware_file.py | 71 ++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index c9bbe49..53128e2 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -151,6 +151,7 @@ MLG CRC: from __future__ import annotations import datetime +import logging import struct from pathlib import Path from typing import Optional, Union @@ -158,6 +159,8 @@ from typing import Optional, Union from .framing import S3Frame from .models import Event, MonitorLogEntry, Timestamp +log = logging.getLogger(__name__) + # ── File header constants ───────────────────────────────────────────────────── # Common 16-byte prefix shared by waveform files and MLG (confirmed from binary inspection). @@ -502,6 +505,51 @@ def blastware_filename(event: Event, serial: str, ach: bool = False) -> str: return prefix + stem + ext +# ── A5 frame classifier ─────────────────────────────────────────────────────────── + +# ASCII markers that identify a compliance-config / metadata frame. +# These strings appear in the A5 bulk stream as part of the device's +# compliance setup payload. They should NEVER appear in raw ADC waveform +# frames (which are binary-heavy, < 20 % printable ASCII). +_METADATA_FRAME_MARKERS = ( + b"Project:", + b"Client:", + b"Standard Recording Setup", + b"Extended Notes", + b"User Name:", + b"Seis Loc:", +) + + +def classify_frame(frame: S3Frame) -> str: + """ + Classify an A5 bulk waveform stream frame by its content. + + Returns one of: + "terminator" — page_key == 0x0000 + "probe_or_strt" — data contains b"STRT" (the initial probe response) + "metadata" — data contains ASCII compliance-config markers + "waveform" — predominantly binary (< 20 % printable ASCII) + "unknown" — none of the above criteria matched + + Used by write_blastware_file() to filter non-waveform frames out of + the reconstructed body so that metadata blocks (Project:, Client:, …) + and spurious STRT records do not corrupt the output file. + """ + if frame.page_key == 0x0000: + return "terminator" + data = bytes(frame.data) + if b"STRT" in data: + return "probe_or_strt" + if any(m in data for m in _METADATA_FRAME_MARKERS): + return "metadata" + if len(data) > 0: + printable = sum(1 for b in data if 32 <= b < 127) + if printable / len(data) < 0.20: + return "waveform" + return "unknown" + + # ── Waveform file writer ─────────────────────────────────────────────────────────── def write_blastware_file( @@ -631,12 +679,29 @@ def write_blastware_file( all_bytes = bytearray() for fi, frame in enumerate(body_frames): + ftype = classify_frame(frame) + print(f"Frame {fi}: type={ftype}, page_key={frame.page_key:04x}, len={len(frame.data)}") + if fi == 0: + # Probe frame: always process regardless of classification. + # It holds the STRT record; probe_skip positions us past it. skip = probe_skip - elif fi == 1: - skip = 13 # 7-byte frame.data prefix + 6-byte first-chunk header + elif ftype == "waveform": + skip = 13 if fi == 1 else 12 else: - skip = 12 # 7-byte frame.data prefix + 5-byte chunk header + # Skip metadata, probe_or_strt, and unknown frames. + if b"STRT" in bytes(frame.data): + log.warning( + "write_blastware_file: frame %d (%s) contains STRT — skipping", + fi, ftype, + ) + else: + log.debug( + "write_blastware_file: frame %d classified as %s — skipping", + fi, ftype, + ) + continue + all_bytes.extend(_frame_body_bytes(frame, skip)) # Terminator contributes its content, which ends with the 26-byte footer. -- 2.52.0 From 35c3f4f9459f93795c3e98a15f8b246a1d913781 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 17:25:29 -0400 Subject: [PATCH 24/40] fix(protocol): correct A5 frame classification and chunk counter formula --- CLAUDE.md | 17 ++++++++++++----- docs/instantel_protocol_reference.md | 28 +++++++++++++++------------- minimateplus/blastware_file.py | 20 +++++++++++++++----- minimateplus/protocol.py | 10 ++++++---- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 32dd6d1..6101185 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,9 +118,11 @@ S3→BW (response): Both differences confirmed by reproducing Blastware's exact wire bytes from the 1-2-26 BW TX capture. All 10 frames verified. -### SUB 5A — chunk counter is monotonic (CORRECTED 2026-04-06) +### SUB 5A — chunk counter formula (FINAL CORRECTION 2026-04-24) -**Chunk counters are `chunk_num * 0x0400` for ALL chunks including chunk 1.** +**Chunk counter = `key4[2:4] + (chunk_num - 1) * 0x0400` for ALL chunks.** + +where `key4[2:4] = (key4[2] << 8) | key4[3]` is the event's circular-buffer base offset. The 4-2-26 BW TX capture showed `counter=0x1004` for chunk 1 of event key `01110000`, which led to `_CHUNK1_COUNTER = 0x1004` being hardcoded as a special case. This was a Blastware @@ -130,9 +132,14 @@ immediately and streams all frames correctly. The 4-3-26 capture confirms the pattern for a second event (key `0111245a`): chunk 1 = `0x245A`, chunk 2 = `0x285A`, chunk 3 = `0x2C5A` (each +0x0400). Blastware's -true formula is `key4[2:4] + n * 0x0400` — but since `key4[2:4]` of the first event is -`0x0000`, `n * 0x0400` produces the right result. The device does not strictly validate the -counter and streams data for any valid 5A request; using `chunk_num * 0x0400` is correct. +true formula is `key4[2:4] + (chunk_num - 1) * 0x0400`. + +**2026-04-24 CORRECTION — `n * 0x0400` is WRONG for non-first events.** For event key +`01110000`, `key4[2:4] == 0x0000` so the old `chunk_num * 0x0400` formula was accidentally +correct. For keys with `key4[2:4] != 0` (e.g. key `01111884`, offset `0x1884`), the old +formula sends counters pointing into the wrong buffer region — the device returns data from +a completely different stored event and `b"Project:"` never appears in the stream. +Use `key4[2:4] + (chunk_num - 1) * 0x0400` exclusively. ### SUB 5A — params are 11 bytes for chunk frames, 10 for termination diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index cd0812e..327ed13 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -1231,21 +1231,23 @@ Two critical differences from `build_bw_frame`: | Frame | offset_word | counter | params | Purpose | |---|---|---|---|---| | Probe | `0x1004` | `0x0000` | 10 bytes (`bulk_waveform_params(0)`) | Initiate transfer | -| Chunk 1 | `0x1004` | `0x0400` | 11 bytes | First data chunk | -| Chunk 2 | `0x1004` | `0x0800` | 11 bytes | Second chunk | -| Chunk N | `0x1004` | `N * 0x0400` | 11 bytes | Nth chunk | +| Chunk 1 | `0x1004` | `key4[2:4]` | 11 bytes | First data chunk | +| Chunk 2 | `0x1004` | `key4[2:4] + 0x0400` | 11 bytes | Second chunk | +| Chunk N | `0x1004` | `key4[2:4] + (N-1) * 0x0400` | 11 bytes | Nth chunk | | … | … | … | … | … | -| Termination | `0x005A` | `last + 0x0400` | 10 bytes | End transfer | +| Termination | `0x005A` | `key4[2:4] + N * 0x0400` | 10 bytes | End transfer | -> ⚠️ **2026-04-06 CORRECTED — chunk counter is monotonic for ALL chunks.** -> The 4-2-26 BW TX capture showed counter=0x1004 for chunk 1, which was hardcoded as a -> special case. This was a Blastware artifact. Empirically confirmed: counter=0x0400 for -> chunk 1 works correctly; counter=0x1004 causes the device to time out. The device does -> NOT strictly validate the counter value — it streams data for any valid 5A request for -> the given key. Use `chunk_num * 0x0400` (monotonic) for all chunks. -> BW's true internal formula is `key4[2:4] + n * 0x0400`. For event 1 (key `01110000`) -> this equals `n * 0x0400` since `key4[2:4] = 0x0000`. The monotonic formula is correct -> for all keys encountered on this device. +> ⚠️ **2026-04-06 CORRECTED — chunk counter is `key4[2:4] + (N-1) * 0x0400`.** +> The 4-2-26 BW TX capture showed counter=0x1004 for chunk 1 of key `01110000`, leading to +> an interim "monotonic n * 0x0400" formula. This was accidentally correct because +> `key4[2:4] == 0x0000` for that event. +> +> **2026-04-24 FINAL CORRECTION:** The counter is an absolute circular-buffer address. +> BW's true formula is `key4[2:4] + (chunk_num - 1) * 0x0400` where `key4[2:4]` is the +> event's storage base offset (`(key4[2]<<8) | key4[3]`). For keys where +> `key4[2:4] != 0x0000` (e.g. key `01111884`), using `n * 0x0400` sends requests into the +> wrong buffer region — the device returns data from a completely different event and +> `b"Project:"` never appears in the stream. Confirmed correct 2026-04-24. The `stop_after_metadata=True` flag causes the loop to stop as soon as `b"Project:"` is found in the accumulated A5 frame data, typically after 4–9 chunks. A termination frame diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index 53128e2..dabcce9 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -527,7 +527,7 @@ def classify_frame(frame: S3Frame) -> str: Returns one of: "terminator" — page_key == 0x0000 - "probe_or_strt" — data contains b"STRT" (the initial probe response) + "probe_or_strt" — data contains b"STRT\xff\xfe" (the initial probe response) "metadata" — data contains ASCII compliance-config markers "waveform" — predominantly binary (< 20 % printable ASCII) "unknown" — none of the above criteria matched @@ -539,7 +539,7 @@ def classify_frame(frame: S3Frame) -> str: if frame.page_key == 0x0000: return "terminator" data = bytes(frame.data) - if b"STRT" in data: + if b"STRT\xff\xfe" in data: return "probe_or_strt" if any(m in data for m in _METADATA_FRAME_MARKERS): return "metadata" @@ -677,6 +677,7 @@ def write_blastware_file( term_frame = None all_bytes = bytearray() + seen_metadata = False for fi, frame in enumerate(body_frames): ftype = classify_frame(frame) @@ -686,11 +687,20 @@ def write_blastware_file( # Probe frame: always process regardless of classification. # It holds the STRT record; probe_skip positions us past it. skip = probe_skip - elif ftype == "waveform": + elif seen_metadata: + # Drop all frames that come after the compliance/metadata block. + # (e.g. extra waveform chunks fetched after stop_after_metadata) + log.debug( + "write_blastware_file: frame %d after metadata — skipping", fi + ) + continue + elif ftype in ("waveform", "metadata"): skip = 13 if fi == 1 else 12 + if ftype == "metadata": + seen_metadata = True else: - # Skip metadata, probe_or_strt, and unknown frames. - if b"STRT" in bytes(frame.data): + # Skip probe_or_strt and unknown frames. + if b"STRT\xff\xfe" in bytes(frame.data): log.warning( "write_blastware_file: frame %d (%s) contains STRT — skipping", fi, ftype, diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 5a2537f..f599170 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -126,10 +126,12 @@ DATA_LENGTHS: dict[int, int] = { _BULK_CHUNK_OFFSET = 0x1004 # offset field for probe + all regular chunk requests ✅ _BULK_TERM_OFFSET = 0x005A # offset field for termination request ✅ _BULK_COUNTER_STEP = 0x0400 # chunk counter increment per chunk ✅ -# Chunk counter formula: chunk_num * 0x0400 for ALL chunks including chunk 1. -# Earlier captures showed 0x1004 for chunk 1 — that was a Blastware artifact, not a -# protocol requirement. Confirmed 2026-04-06: 0x0400 for chunk 1 works; 0x1004 -# causes a 120-second device timeout. Formula n * 0x0400 is used for all chunks. +# Chunk counter formula: key4[2:4] + (chunk_num - 1) * 0x0400 +# where key4[2:4] is the event's circular-buffer base offset ((key4[2]<<8)|key4[3]). +# Earlier captures showed 0x1004 for chunk 1 of key 01110000 — that was a Blastware +# artifact. For keys where key4[2:4] != 0x0000 (e.g. key 01111884) the old +# "n * 0x0400" formula sends counters from the wrong buffer region and the device +# returns data from a different event. Confirmed correct 2026-04-24. # Default timeout values (seconds). # MiniMate Plus is a slow device — keep these generous. -- 2.52.0 From 0415af19b4b97d97148fe2b5f520f142046209f9 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 24 Apr 2026 20:21:03 -0400 Subject: [PATCH 25/40] fix(blastware_file): remove seen_metadata flag and adjust frame processing logic --- minimateplus/blastware_file.py | 38 ++++++++++++++-------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index dabcce9..15c9835 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -677,7 +677,6 @@ def write_blastware_file( term_frame = None all_bytes = bytearray() - seen_metadata = False for fi, frame in enumerate(body_frames): ftype = classify_frame(frame) @@ -687,30 +686,25 @@ def write_blastware_file( # Probe frame: always process regardless of classification. # It holds the STRT record; probe_skip positions us past it. skip = probe_skip - elif seen_metadata: - # Drop all frames that come after the compliance/metadata block. - # (e.g. extra waveform chunks fetched after stop_after_metadata) - log.debug( - "write_blastware_file: frame %d after metadata — skipping", fi + elif ftype == "probe_or_strt": + # Real duplicate probe frames should be skipped (very rare). + log.warning( + "write_blastware_file: frame %d classified as probe_or_strt — skipping", + fi, ) continue - elif ftype in ("waveform", "metadata"): - skip = 13 if fi == 1 else 12 - if ftype == "metadata": - seen_metadata = True else: - # Skip probe_or_strt and unknown frames. - if b"STRT\xff\xfe" in bytes(frame.data): - log.warning( - "write_blastware_file: frame %d (%s) contains STRT — skipping", - fi, ftype, - ) - else: - log.debug( - "write_blastware_file: frame %d classified as %s — skipping", - fi, ftype, - ) - continue + # Waveform chunks, metadata/compliance frames, and unknown frames are all + # included. The A5 stream is collected with stop_after_metadata=True + + # extra_chunks_after_metadata=1, so body_frames contains: + # [0] probe frame + # [1..N] waveform ADC chunks + # [N+1] first compliance frame (contains "Project:", "Client:", etc.) + # [N+2] second compliance frame (continuation of compliance block) + # All of these contribute to the body; the 26-byte footer is separated out + # at the end. Do NOT gate on metadata classification here — the compliance + # block spans 2 frames and skipping frame N+2 produces a truncated file. + skip = 13 if fi == 1 else 12 all_bytes.extend(_frame_body_bytes(frame, skip)) -- 2.52.0 From 7976b544edfef1432d9d1fff36d8cb2093204d37 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sun, 26 Apr 2026 00:59:36 -0400 Subject: [PATCH 26/40] fix(blastware_file): never skip A5 frames based on classification at fi>0 Frame 0 is always the probe; frames 1+ are always data (waveform ADC chunks, compliance config, compliance continuation). Gating on classify_frame() at fi>0 produces false positives: ADC binary data can coincidentally contain b"STRT\xff\xfe", causing frames 1 and 5 to be silently dropped from the body (confirmed from live capture on event key=01110000). Remove all type-based filtering; include every frame unconditionally with the standard index-based skip amounts. --- minimateplus/blastware_file.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index 15c9835..79a1e1e 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -686,24 +686,19 @@ def write_blastware_file( # Probe frame: always process regardless of classification. # It holds the STRT record; probe_skip positions us past it. skip = probe_skip - elif ftype == "probe_or_strt": - # Real duplicate probe frames should be skipped (very rare). - log.warning( - "write_blastware_file: frame %d classified as probe_or_strt — skipping", - fi, - ) - continue else: - # Waveform chunks, metadata/compliance frames, and unknown frames are all - # included. The A5 stream is collected with stop_after_metadata=True + - # extra_chunks_after_metadata=1, so body_frames contains: - # [0] probe frame - # [1..N] waveform ADC chunks - # [N+1] first compliance frame (contains "Project:", "Client:", etc.) - # [N+2] second compliance frame (continuation of compliance block) - # All of these contribute to the body; the 26-byte footer is separated out - # at the end. Do NOT gate on metadata classification here — the compliance - # block spans 2 frames and skipping frame N+2 produces a truncated file. + # ALL subsequent frames are included unconditionally — no filtering on + # frame type. In the A5 stream, frame 0 is always the probe response; + # frames 1+ are always data (waveform chunks, compliance config, or + # compliance continuation). Classification is for logging only. + # + # DO NOT gate on classify_frame() here: + # - "probe_or_strt" at fi>0 is always a false positive — ADC binary + # data can coincidentally contain b"STRT\xff\xfe" (confirmed from + # live capture: frames 1 and 5 matched on event key=01110000). + # - "metadata" frames must be included (compliance config body). + # - The compliance block spans 2 frames; skipping either produces a + # truncated file that Blastware rejects. skip = 13 if fi == 1 else 12 all_bytes.extend(_frame_body_bytes(frame, skip)) -- 2.52.0 From 2f084ed1051c097c3314e77854528231e43acd32 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sun, 26 Apr 2026 01:28:47 -0400 Subject: [PATCH 27/40] fix(protocol): update chunk counter formula to use max(key4[2:4], 0x0400) for accurate data streaming --- CLAUDE.md | 33 ++++++++++++----------- docs/instantel_protocol_reference.md | 22 ++++++++++----- minimateplus/protocol.py | 40 +++++++++++++++++----------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6101185..16089fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,28 +118,29 @@ S3→BW (response): Both differences confirmed by reproducing Blastware's exact wire bytes from the 1-2-26 BW TX capture. All 10 frames verified. -### SUB 5A — chunk counter formula (FINAL CORRECTION 2026-04-24) +### SUB 5A — chunk counter formula (FINAL CORRECTION 2026-04-26) -**Chunk counter = `key4[2:4] + (chunk_num - 1) * 0x0400` for ALL chunks.** +**Chunk counter = `max(key4[2:4], 0x0400) + (chunk_num - 1) * 0x0400` for ALL chunks.** where `key4[2:4] = (key4[2] << 8) | key4[3]` is the event's circular-buffer base offset. -The 4-2-26 BW TX capture showed `counter=0x1004` for chunk 1 of event key `01110000`, which -led to `_CHUNK1_COUNTER = 0x1004` being hardcoded as a special case. This was a Blastware -artifact, not a protocol requirement. Empirical test 2026-04-06: with `counter=0x1004` for -chunk 1 the device times out (120 s); with `counter=0x0400` (= `1 * 0x0400`) it responds -immediately and streams all frames correctly. +The `max(..., 0x0400)` guard is critical for events at the start of the circular buffer +(key4[2:4] == 0x0000, e.g. key `01110000`). Without it, chunk 1 gets counter=0x0000, which +is the same address as the probe frame — the device re-returns the STRT record data instead +of waveform payload. With the guard, chunk 1 gets counter=0x0400, which is confirmed correct +from the empirical live-device test 2026-04-06 (`counter=0x0400 → responds immediately and +streams all frames correctly`). -The 4-3-26 capture confirms the pattern for a second event (key `0111245a`): -chunk 1 = `0x245A`, chunk 2 = `0x285A`, chunk 3 = `0x2C5A` (each +0x0400). Blastware's -true formula is `key4[2:4] + (chunk_num - 1) * 0x0400`. +The 4-3-26 capture confirms the pattern for a second event (key `0111245a`, key4[2:4]=0x245a): +chunk 1 = `0x245A`, chunk 2 = `0x285A`, chunk 3 = `0x2C5A` (each +0x0400). +`max(0x245a, 0x0400) = 0x245a` → formula works correctly for non-zero base offset too. -**2026-04-24 CORRECTION — `n * 0x0400` is WRONG for non-first events.** For event key -`01110000`, `key4[2:4] == 0x0000` so the old `chunk_num * 0x0400` formula was accidentally -correct. For keys with `key4[2:4] != 0` (e.g. key `01111884`, offset `0x1884`), the old -formula sends counters pointing into the wrong buffer region — the device returns data from -a completely different stored event and `b"Project:"` never appears in the stream. -Use `key4[2:4] + (chunk_num - 1) * 0x0400` exclusively. +**History:** +- Original: `_CHUNK1_COUNTER = 0x1004` hardcoded (Blastware capture artifact — WRONG). +- 2026-04-06: Corrected to `chunk_num * 0x0400` (worked for key 01110000 only). +- 2026-04-24: Corrected to `key4[2:4] + (chunk_num-1) * 0x0400` (fixed non-zero offsets, + but accidentally broke key 01110000 — counter=0x0000 sends probe address again). +- 2026-04-26: Final formula: `max(key4[2:4], 0x0400) + (chunk_num-1) * 0x0400`. ### SUB 5A — params are 11 bytes for chunk frames, 10 for termination diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 327ed13..95950ec 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -1231,23 +1231,31 @@ Two critical differences from `build_bw_frame`: | Frame | offset_word | counter | params | Purpose | |---|---|---|---|---| | Probe | `0x1004` | `0x0000` | 10 bytes (`bulk_waveform_params(0)`) | Initiate transfer | -| Chunk 1 | `0x1004` | `key4[2:4]` | 11 bytes | First data chunk | -| Chunk 2 | `0x1004` | `key4[2:4] + 0x0400` | 11 bytes | Second chunk | -| Chunk N | `0x1004` | `key4[2:4] + (N-1) * 0x0400` | 11 bytes | Nth chunk | +| Chunk 1 | `0x1004` | `max(key4[2:4], 0x0400)` | 11 bytes | First data chunk | +| Chunk 2 | `0x1004` | `max(key4[2:4], 0x0400) + 0x0400` | 11 bytes | Second chunk | +| Chunk N | `0x1004` | `max(key4[2:4], 0x0400) + (N-1) * 0x0400` | 11 bytes | Nth chunk | | … | … | … | … | … | -| Termination | `0x005A` | `key4[2:4] + N * 0x0400` | 10 bytes | End transfer | +| Termination | `0x005A` | `max(key4[2:4], 0x0400) + N * 0x0400` | 10 bytes | End transfer | > ⚠️ **2026-04-06 CORRECTED — chunk counter is `key4[2:4] + (N-1) * 0x0400`.** > The 4-2-26 BW TX capture showed counter=0x1004 for chunk 1 of key `01110000`, leading to > an interim "monotonic n * 0x0400" formula. This was accidentally correct because > `key4[2:4] == 0x0000` for that event. > -> **2026-04-24 FINAL CORRECTION:** The counter is an absolute circular-buffer address. +> **2026-04-24 CORRECTION:** The counter is an absolute circular-buffer address. > BW's true formula is `key4[2:4] + (chunk_num - 1) * 0x0400` where `key4[2:4]` is the > event's storage base offset (`(key4[2]<<8) | key4[3]`). For keys where > `key4[2:4] != 0x0000` (e.g. key `01111884`), using `n * 0x0400` sends requests into the -> wrong buffer region — the device returns data from a completely different event and -> `b"Project:"` never appears in the stream. Confirmed correct 2026-04-24. +> wrong buffer region — the device returns data from a completely different event. +> +> **2026-04-26 FINAL CORRECTION:** The formula `key4[2:4] + (N-1) * 0x0400` is wrong when +> `key4[2:4] == 0x0000` (e.g. event key `01110000`, the very first event after a device erase). +> Counter=0x0000 for chunk 1 is the same address as the probe frame — the device re-returns +> the STRT record data instead of waveform payload (frame 1 has len=1097, same as probe, and +> contains `b"STRT\xff\xfe"`, contributing zero waveform bytes). +> Final formula: `max(key4[2:4], 0x0400) + (chunk_num - 1) * 0x0400`. +> For key `01110000`: chunk 1 = 0x0400 (confirmed working, empirical test 2026-04-06). +> For key `0111245a`: chunk 1 = 0x245a (unchanged, confirmed from 4-3-26 capture). The `stop_after_metadata=True` flag causes the loop to stop as soon as `b"Project:"` is found in the accumulated A5 frame data, typically after 4–9 chunks. A termination frame diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index f599170..0a69f93 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -585,13 +585,12 @@ class MiniMateProtocol: frames_data: list[S3Frame] = [] counter = 0 - # BW counter formula (confirmed from 4-3-26 capture for key 0111245a): - # counter for chunk n = key4[2:4] + (n - 1) * 0x0400 - # key4[2:4] is the event's circular-buffer base offset — without it, chunk - # requests address the wrong region of the device buffer and the device - # streams data from the wrong event (no "Project:" in any response). - # PREVIOUSLY WRONG NOTE: "device does not validate counter; chunk_num*0x0400 - # is correct" — that was only true for key 01110000 where key4[2:4]==0x0000. + # BW counter formula (confirmed from 4-3-26 capture for key 0111245a, + # and empirical live-device test 2026-04-06 for key 01110000): + # counter for chunk n = max(key4[2:4], 0x0400) + (n - 1) * 0x0400 + # key4[2:4] is the event's circular-buffer base offset. The max() guard + # ensures chunk 1 never uses counter=0x0000 (which equals the probe address + # and causes the device to re-return STRT record data for the first chunk). _key4_offset = (key4[2] << 8) | key4[3] # ── Step 1: probe ──────────────────────────────────────────────────── @@ -612,15 +611,24 @@ class MiniMateProtocol: log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data)) # ── Step 2: chunk loop ─────────────────────────────────────────────── - # Correct counter formula: key4[2:4] + (chunk_num - 1) * 0x0400 - # This matches Blastware exactly (confirmed from 4-3-26 capture). - # For events where key4[2:4]==0 (e.g. 01110000), this gives the same - # result as the old chunk_num*0x0400 formula shifted by one step, which - # the device also accepted — but for events with a non-zero base offset - # (e.g. key 01111884 with key4[2:4]=0x1884) the old formula sends - # completely wrong counters and the device streams the wrong buffer region. + # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 + # where _chunk_base = max(key4[2:4], 0x0400). + # + # For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a): + # _chunk_base = 0x245a → chunk 1=0x245a, chunk 2=0x285a, ... + # Confirmed from 4-3-26 capture. + # + # For events with key4[2:4] == 0 (e.g. key 01110000): + # _chunk_base = max(0, 0x0400) = 0x0400 + # → chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400) + # CRITICAL: counter=0x0000 (same as the probe) causes the device to + # re-return the STRT record data for chunk 1, making frame 1 look like + # a second probe response (confirmed from server log: frame 1 len=1097, + # contains STRT\xff\xfe, contributes zero body bytes after DLE-strip). + # counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06). + _chunk_base = max(_key4_offset, _BULK_COUNTER_STEP) for chunk_num in range(1, max_chunks + 1): - counter = _key4_offset + (chunk_num - 1) * _BULK_COUNTER_STEP + counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP params = bulk_waveform_params(key4, counter) log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) @@ -675,7 +683,7 @@ class MiniMateProtocol: chunk_num, extra_chunks_after_metadata) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 - counter = _key4_offset + (chunk_num - 1) * _BULK_COUNTER_STEP + counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP params = bulk_waveform_params(key4, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: -- 2.52.0 From ae30a02898e2b5a9ecabac8877245ea7229b4dab Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sun, 26 Apr 2026 16:03:07 -0400 Subject: [PATCH 28/40] fix(blastware_file, server): enhance logging and correct chunk handling for accurate data processing --- minimateplus/blastware_file.py | 30 ++++++++++++++++++++++++++++-- sfm/server.py | 17 +++++++++-------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index 79a1e1e..265d0ca 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -639,6 +639,15 @@ def write_blastware_file( strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF]) probe_skip = 7 + 21 + log.warning( + "write_blastware_file: strt_pos_stripped=%d probe_skip=%d " + "probe_data_len=%d strt_hex=%s", + strt_pos_stripped if strt_pos_stripped >= 0 else -1, + probe_skip, + len(a5_frames[0].data), + strt.hex() if len(strt) >= 4 else "(short)", + ) + if len(strt) != 21: raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}") @@ -701,13 +710,30 @@ def write_blastware_file( # truncated file that Blastware rejects. skip = 13 if fi == 1 else 12 - all_bytes.extend(_frame_body_bytes(frame, skip)) + contribution = _frame_body_bytes(frame, skip) + log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", + fi, skip, len(frame.data), len(contribution)) + all_bytes.extend(contribution) # Terminator contributes its content, which ends with the 26-byte footer. # skip=11 (not 12) because the terminator's inner frame header is 4 bytes, # one shorter than chunk frames' 5-byte inner header. Confirmed 2026-04-21. if term_frame is not None: - all_bytes.extend(_frame_body_bytes(term_frame, 11)) + term_contribution = _frame_body_bytes(term_frame, 11) + log.warning( + "write_blastware_file: term_frame data_len=%d skip=11 " + "contribution_len=%d first8=%s", + len(term_frame.data), + len(term_contribution), + term_contribution[:8].hex() if len(term_contribution) >= 8 else term_contribution.hex(), + ) + all_bytes.extend(term_contribution) + + log.warning( + "write_blastware_file: all_bytes total=%d last28=%s", + len(all_bytes), + bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(), + ) if len(all_bytes) >= 26: body = bytes(all_bytes[:-26]) diff --git a/sfm/server.py b/sfm/server.py index 97dd7b2..6f96f00 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,21 +885,22 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use stop_after_metadata=True (full_waveform=False) with 1 extra - # chunk after "Project:". For any record time, the pre-metadata - # section of the 5A stream naturally carries proportionally more - # ADC data for longer events — so "1 extra chunk" produces the - # correct body length regardless of record time. + # Use stop_after_metadata=True (full_waveform=False) with 0 extra + # chunks after "Project:". Confirmed from 4-26-26 BW RS-232 capture + # of "copy event to file" on a 2-sec Continuous event (key=01110000): + # BW sends the termination frame IMMEDIATELY after the chunk that + # contains "Project:" — no extra chunk is downloaded first. + # extra_chunks_after_metadata=1 was WRONG: it downloaded one additional + # chunk (counter = last_data_counter + 0x0400) adding ~1053 spurious + # bytes to the body, causing Blastware to reject the file. # # full_waveform=True (natural end-of-stream) downloads ALL chunks # including post-event silence (35+ chunks for a 9-sec event at # 1024 sps) — this produces 24KB+ files that Blastware rejects. - # Confirmed from file size comparison: BW 1-sec=4400B, BW 3-sec=8114B, - # per-second delta 1857 bytes — matches pre-metadata frame scaling. events = client.get_events( full_waveform=False, stop_after_index=index, - extra_chunks_after_metadata=1, + extra_chunks_after_metadata=0, ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info -- 2.52.0 From 9bbecea70fc4ae07b8b668920ad2ec05d735fdf6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 20:23:18 +0000 Subject: [PATCH 29/40] =?UTF-8?q?fix(parser):=20correct=20S3=20frame=20ter?= =?UTF-8?q?minator=20=E2=80=94=20bare=20ETX,=20not=20DLE+ETX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse_s3 had the S3 terminator logic inverted vs the real S3FrameParser in framing.py. It was terminating on DLE+ETX and treating bare ETX as payload, which caused every bare 0x03 to be swallowed — bundling multiple real S3 frames into one giant body until a DLE+ETX sequence happened to appear. Result: 583-byte POLL_RESPONSE 'frames' containing many real frames concatenated, all showing BAD CHK. Fix: mirror S3FrameParser exactly — - Bare ETX (0x03) = real frame terminator - DLE+ETX (0x10 0x03) = inner-frame literal data (A4/E5 sub-frames), appended to body and parsing continues https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- parsers/s3_parser.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/parsers/s3_parser.py b/parsers/s3_parser.py index 2f32933..4b8a2bc 100644 --- a/parsers/s3_parser.py +++ b/parsers/s3_parser.py @@ -33,7 +33,7 @@ STX = 0x02 ETX = 0x03 ACK = 0x41 -__version__ = "0.2.3" +__version__ = "0.2.4" @dataclass @@ -184,9 +184,9 @@ def validate_bw_body_auto(body: bytes) -> Optional[Tuple[bytes, bytes, str]]: def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]: frames: List[Frame] = [] - IDLE = 0 - IN_FRAME = 1 - AFTER_DLE = 2 + IDLE = 0 + IN_FRAME = 1 + IN_FRAME_DLE = 2 # saw DLE inside frame — waiting for next byte state = IDLE body = bytearray() @@ -206,22 +206,15 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]: state = IN_FRAME i += 2 continue + # ACK bytes, boot strings, garbage — silently ignored elif state == IN_FRAME: if b == DLE: - state = AFTER_DLE + state = IN_FRAME_DLE i += 1 continue - body.append(b) - - else: # AFTER_DLE - if b == DLE: - body.append(DLE) - state = IN_FRAME - i += 1 - continue - if b == ETX: + # Bare ETX = real S3 frame terminator (confirmed from S3FrameParser) end_offset = i + 1 trailer_start = i + 1 trailer_end = trailer_start + trailer_len @@ -259,13 +252,27 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]: state = IDLE i = trailer_end continue + body.append(b) + else: # IN_FRAME_DLE + if b == DLE: + # DLE DLE → literal 0x10 in payload + body.append(DLE) + state = IN_FRAME + i += 1 + continue + if b == ETX: + # DLE+ETX inside a frame = inner-frame terminator (A4/E5 sub-frames). + # Treat as literal data, NOT the outer frame end. + body.append(DLE) + body.append(ETX) + state = IN_FRAME + i += 1 + continue # Unexpected DLE + byte → treat as literal data body.append(DLE) body.append(b) state = IN_FRAME - i += 1 - continue i += 1 -- 2.52.0 From a7585cb5e0ef48b3261a9605540f8629350c5139 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sun, 26 Apr 2026 16:32:32 -0400 Subject: [PATCH 30/40] fix(blastware_file, server): implement logic to skip extra chunks after metadata for accurate file writing --- minimateplus/blastware_file.py | 38 ++++++++++++++++++++++++++++++++-- sfm/server.py | 22 ++++++++++++-------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index 265d0ca..fedf229 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -685,11 +685,45 @@ def write_blastware_file( body_frames = a5_frames term_frame = None + # ── Identify first metadata frame and skip "extra chunks" ─────────────── + # When extra_chunks_after_metadata=1 in read_bulk_waveform_stream(), the + # frame list is: [probe, data..., metadata, extra_chunk, terminator]. + # The extra_chunk is downloaded to prime the TCP terminator response — its + # ADC data is NOT part of the Blastware file body. Skip it. + # + # Rule: any frame at index strictly between first_metadata_fi and last_fi + # (the final frame) is an extra chunk and must be excluded. + # + # If no metadata frame exists (e.g. full_waveform download), first_metadata_fi + # is None and no frames are skipped — all frames contribute normally. + first_metadata_fi: Optional[int] = None + for _fi_scan, _frame_scan in enumerate(body_frames): + if _fi_scan > 0 and any(m in bytes(_frame_scan.data) for m in _METADATA_FRAME_MARKERS): + first_metadata_fi = _fi_scan + break + last_fi = len(body_frames) - 1 + + log.warning( + "write_blastware_file: %d body_frames first_metadata_fi=%s last_fi=%d", + len(body_frames), + str(first_metadata_fi) if first_metadata_fi is not None else "None", + last_fi, + ) + all_bytes = bytearray() for fi, frame in enumerate(body_frames): - ftype = classify_frame(frame) - print(f"Frame {fi}: type={ftype}, page_key={frame.page_key:04x}, len={len(frame.data)}") + # Skip "extra chunk" frames: frames after the first metadata frame but + # before the last frame (terminator). These prime the TCP terminator but + # their ADC data must NOT appear in the Blastware file body. + if (first_metadata_fi is not None + and fi > first_metadata_fi + and fi < last_fi): + log.warning( + "write_blastware_file: fi=%d SKIP (extra chunk after metadata fi=%d last_fi=%d)", + fi, first_metadata_fi, last_fi, + ) + continue if fi == 0: # Probe frame: always process regardless of classification. diff --git a/sfm/server.py b/sfm/server.py index 6f96f00..3fc4bb2 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -885,14 +885,18 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use stop_after_metadata=True (full_waveform=False) with 0 extra - # chunks after "Project:". Confirmed from 4-26-26 BW RS-232 capture - # of "copy event to file" on a 2-sec Continuous event (key=01110000): - # BW sends the termination frame IMMEDIATELY after the chunk that - # contains "Project:" — no extra chunk is downloaded first. - # extra_chunks_after_metadata=1 was WRONG: it downloaded one additional - # chunk (counter = last_data_counter + 0x0400) adding ~1053 spurious - # bytes to the body, causing Blastware to reject the file. + # Use stop_after_metadata=True (full_waveform=False) with 1 extra + # chunk after "Project:". The extra chunk is required to prime the + # device over TCP: termination at term_counter=metadata_counter+0x0400 + # returns only ~90 bytes (no useful footer) over TCP/cellular, but + # termination at metadata_counter+0x0800 (one chunk later) returns + # the full 737-byte frame containing the footer. + # + # Confirmed from 4-26-26 BW RS-232 capture: BW terminates at 0x1800 + # without an extra chunk (works on RS-232 but not TCP). + # write_blastware_file() automatically skips the extra chunk's + # contribution — only the probe+ADC+metadata+terminator bytes appear + # in the output file. # # full_waveform=True (natural end-of-stream) downloads ALL chunks # including post-event silence (35+ chunks for a 9-sec event at @@ -900,7 +904,7 @@ def device_event_blastware_file( events = client.get_events( full_waveform=False, stop_after_index=index, - extra_chunks_after_metadata=0, + extra_chunks_after_metadata=1, ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info -- 2.52.0 From e1150b30aad7142ff8b0afc49cf5742020ee63ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 20:40:45 +0000 Subject: [PATCH 31/40] fix(analyzer): name A5/5A frames; revert S3 checksum validation Add 0x5A (BULK_WAVEFORM_STREAM) and 0xA5 (BULK_WAVEFORM_RESPONSE) to SUB_TABLE so they display with real names instead of UNKNOWN_5A/A5. Revert S3 checksum validation to checksum_valid=None (the original intentional behavior). Large S3 frames (A5 bulk waveform, E5 compliance config) embed inner DLE+ETX sub-frame delimiters; the trailing 0x03 of the last inner delimiter can land where the parser expects the SUM8 checksum byte, causing false BAD CHK on every valid A5 frame. protocol.py _validate_frame documents and ignores exactly this issue. https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- parsers/s3_analyzer.py | 2 ++ parsers/s3_parser.py | 32 +++++++++++--------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/parsers/s3_analyzer.py b/parsers/s3_analyzer.py index c86477d..6feb32b 100644 --- a/parsers/s3_analyzer.py +++ b/parsers/s3_analyzer.py @@ -53,7 +53,9 @@ SUB_TABLE: dict[int, tuple[str, str, str]] = { 0x82: ("TRIGGER_CONFIG_WRITE", "BW→S3", "0x1C bytes; trigger config block; mirrors SUB 1C"), 0x83: ("TRIGGER_WRITE_CONFIRM", "BW→S3", "Short frame; commit step after 0x82"), # S3→BW responses + 0x5A: ("BULK_WAVEFORM_STREAM", "BW→S3", "Bulk waveform chunk request; response is A5 stream"), 0xA4: ("POLL_RESPONSE", "S3→BW", "Response to SUB 5B poll"), + 0xA5: ("BULK_WAVEFORM_RESPONSE", "S3→BW", "Response to SUB 5A; waveform chunks + metadata"), 0xFE: ("FULL_CONFIG_RESPONSE", "S3→BW", "Response to SUB 01"), 0xF9: ("CHANNEL_CONFIG_RESPONSE", "S3→BW", "Response to SUB 06"), 0xF7: ("EVENT_INDEX_RESPONSE", "S3→BW", "Response to SUB 08; contains backlight/power-save"), diff --git a/parsers/s3_parser.py b/parsers/s3_parser.py index 4b8a2bc..2bb3de1 100644 --- a/parsers/s3_parser.py +++ b/parsers/s3_parser.py @@ -33,7 +33,7 @@ STX = 0x02 ETX = 0x03 ACK = 0x41 -__version__ = "0.2.4" +__version__ = "0.2.5" @dataclass @@ -220,32 +220,22 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]: trailer_end = trailer_start + trailer_len trailer = blob[trailer_start:trailer_end] - chk_valid = None - chk_type = None - chk_hex = None - payload = bytes(body) - - if len(body) >= 1: - received_chk = body[-1] - computed_chk = checksum8_sum(bytes(body[:-1])) - if computed_chk == received_chk: - chk_valid = True - chk_type = "SUM8" - chk_hex = f"{received_chk:02x}" - payload = bytes(body[:-1]) - else: - chk_valid = False - + # S3 checksums are deliberately not validated here. + # Large S3 responses (A5 bulk waveform, E5 compliance) embed + # inner DLE+ETX sub-frame terminators whose trailing 0x03 byte + # lands where the parser would expect the SUM8 checksum, causing + # false failures. The live protocol (protocol.py _validate_frame) + # also skips S3 checksum enforcement for the same reason. frames.append(Frame( index=idx, start_offset=start_offset, end_offset=end_offset, payload_raw=bytes(body), - payload=payload, + payload=bytes(body), trailer=trailer, - checksum_valid=chk_valid, - checksum_type=chk_type, - checksum_hex=chk_hex + checksum_valid=None, + checksum_type=None, + checksum_hex=None )) idx += 1 -- 2.52.0 From 897ac8a3f3968d112092a108927285b4d3761216 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 22:10:48 +0000 Subject: [PATCH 32/40] Add TCP MITM capture tab (TcpBridgePanel) New 'TCP Capture' tab in seismo_lab.py: listens on a configurable local port for an incoming Blastware connection, transparently forwards all traffic to the real seismograph device, and saves both directions to raw_bw_.bin / raw_s3_.bin in the same format the Analyzer already understands. Session start wires up Analyzer live mode automatically via the same on_bridge_started callback as the COM-port bridge. https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- seismo_lab.py | 287 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) diff --git a/seismo_lab.py b/seismo_lab.py index 2c85222..11e7f11 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -22,6 +22,7 @@ from __future__ import annotations import datetime import os import queue +import socket import subprocess import sys import threading @@ -309,6 +310,284 @@ class BridgePanel(tk.Frame): messagebox.showerror("Error", f"Failed to send mark:\n{e}") +# ───────────────────────────────────────────────────────────────────────────── +# TCP Bridge panel — MITM capture over IP +# ───────────────────────────────────────────────────────────────────────────── + +class TcpBridgePanel(tk.Frame): + """ + TCP man-in-the-middle capture panel. + + Listens on a local TCP port for an incoming Blastware connection, forwards + all traffic to the real device, and saves both directions to raw .bin files. + + Calls on_bridge_started(raw_bw_path, raw_s3_path, None) when a session + begins so the Analyzer can wire up live mode — same signature as BridgePanel. + """ + + def __init__(self, parent: tk.Widget, on_bridge_started, on_bridge_stopped, **kw): + super().__init__(parent, bg=BG2, **kw) + self._on_started = on_bridge_started + self._on_stopped = on_bridge_stopped + self._server: Optional[socket.socket] = None + self._stop_event = threading.Event() + self._log_q: queue.Queue[str] = queue.Queue() + self._build() + self._poll_log_q() + + # ── build ───────────────────────────────────────────────────────────── + + def _build(self) -> None: + pad = {"padx": 6, "pady": 4} + + cfg = tk.Frame(self, bg=BG2) + cfg.pack(side=tk.TOP, fill=tk.X, padx=4, pady=4) + + # Row 0: listen port + remote host + remote port + tk.Label(cfg, text="Listen port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=0, sticky="e", **pad) + self.listen_port_var = tk.StringVar(value="9034") + tk.Entry(cfg, textvariable=self.listen_port_var, width=8, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=1, sticky="w", **pad) + + tk.Label(cfg, text="Device host:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=2, sticky="e", **pad) + self.remote_host_var = tk.StringVar(value="63.43.212.232") + tk.Entry(cfg, textvariable=self.remote_host_var, width=18, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=3, sticky="w", **pad) + + tk.Label(cfg, text="Port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=4, sticky="e", **pad) + self.remote_port_var = tk.StringVar(value="9034") + tk.Entry(cfg, textvariable=self.remote_port_var, width=8, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=5, sticky="w", **pad) + + # Row 1: log dir + tk.Label(cfg, text="Log dir:", bg=BG2, fg=FG, font=MONO).grid(row=1, column=0, sticky="e", **pad) + self.logdir_var = tk.StringVar(value=str(SCRIPT_DIR / "bridges" / "captures" / "mitm")) + tk.Entry(cfg, textvariable=self.logdir_var, width=40, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=1, column=1, columnspan=4, sticky="we", **pad) + tk.Button(cfg, text="Browse", bg=BG3, fg=FG, relief="flat", cursor="hand2", + font=MONO, command=self._choose_dir).grid(row=1, column=5, **pad) + + # Row 2: capture checkboxes + self._raw_bw_on = tk.BooleanVar(value=True) + self._raw_s3_on = tk.BooleanVar(value=True) + tk.Checkbutton(cfg, text="Capture BW→device raw", variable=self._raw_bw_on, + bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, + font=MONO).grid(row=2, column=0, columnspan=2, sticky="w", **pad) + tk.Checkbutton(cfg, text="Capture device→BW raw", variable=self._raw_s3_on, + bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, + font=MONO).grid(row=2, column=2, columnspan=2, sticky="w", **pad) + + # Row 3: buttons + status + btn_row = tk.Frame(self, bg=BG2) + btn_row.pack(side=tk.TOP, fill=tk.X, padx=4, pady=2) + + self.start_btn = tk.Button(btn_row, text="Start Listening", bg=GREEN, fg="#000000", + relief="flat", padx=12, cursor="hand2", font=MONO_B, + command=self.start_server) + self.start_btn.pack(side=tk.LEFT, padx=6) + + self.stop_btn = tk.Button(btn_row, text="Stop", bg=BG3, fg=FG, + relief="flat", padx=12, cursor="hand2", font=MONO, + command=self.stop_server, state="disabled") + self.stop_btn.pack(side=tk.LEFT, padx=4) + + self.status_var = tk.StringVar(value="Idle") + tk.Label(btn_row, textvariable=self.status_var, + bg=BG2, fg=FG_DIM, font=MONO).pack(side=tk.LEFT, padx=10) + + # Log output + self.log_view = scrolledtext.ScrolledText( + self, height=18, font=MONO_SM, + bg=BG, fg=FG, insertbackground=FG, + relief="flat", state="disabled", + ) + self.log_view.pack(fill=tk.BOTH, expand=True, padx=4, pady=4) + + # ── helpers ─────────────────────────────────────────────────────────── + + def _choose_dir(self) -> None: + path = filedialog.askdirectory(initialdir=self.logdir_var.get()) + if path: + self.logdir_var.set(path) + + def _append_log(self, text: str) -> None: + self.log_view.configure(state="normal") + self.log_view.insert(tk.END, text) + self.log_view.see(tk.END) + self.log_view.configure(state="disabled") + + def _poll_log_q(self) -> None: + try: + while True: + msg = self._log_q.get_nowait() + if msg == "<>": + if self._server is not None: + self.status_var.set(f"Listening on :{self.listen_port_var.get()}") + self._on_stopped() + else: + self._append_log(msg) + except queue.Empty: + pass + finally: + self.after(100, self._poll_log_q) + + # ── server control ──────────────────────────────────────────────────── + + def start_server(self) -> None: + try: + listen_port = int(self.listen_port_var.get().strip()) + remote_host = self.remote_host_var.get().strip() + remote_port = int(self.remote_port_var.get().strip()) + except ValueError: + messagebox.showerror("Error", "Invalid port number.") + return + if not remote_host: + messagebox.showerror("Error", "Please enter the device host.") + return + + try: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("0.0.0.0", listen_port)) + srv.listen(5) + srv.settimeout(1.0) + except OSError as e: + messagebox.showerror("Error", f"Cannot bind to port {listen_port}:\n{e}") + return + + self._server = srv + self._stop_event.clear() + self.start_btn.configure(state="disabled") + self.stop_btn.configure(state="normal", bg=RED) + self.status_var.set(f"Listening on :{listen_port}") + + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + self._append_log( + f"== TCP Bridge started [{ts}]\n" + f" Listening on 0.0.0.0:{listen_port}\n" + f" Forwarding to {remote_host}:{remote_port}\n" + f" Point Blastware call-home to this machine on port {listen_port}\n==\n" + ) + + logdir = self.logdir_var.get().strip() or "." + raw_bw_on = self._raw_bw_on.get() + raw_s3_on = self._raw_s3_on.get() + + threading.Thread( + target=self._accept_loop, + args=(srv, remote_host, remote_port, logdir, raw_bw_on, raw_s3_on), + daemon=True, + ).start() + + def stop_server(self) -> None: + self._stop_event.set() + if self._server: + try: + self._server.close() + except OSError: + pass + self._server = None + self.start_btn.configure(state="normal") + self.stop_btn.configure(state="disabled", bg=BG3) + self.status_var.set("Idle") + self._append_log("== TCP Bridge stopped ==\n") + self._on_stopped() + + # ── accept / session threads ────────────────────────────────────────── + + def _accept_loop(self, srv: socket.socket, remote_host: str, remote_port: int, + logdir: str, raw_bw_on: bool, raw_s3_on: bool) -> None: + while not self._stop_event.is_set(): + try: + client_sock, addr = srv.accept() + except socket.timeout: + continue + except OSError: + break + + peer = f"{addr[0]}:{addr[1]}" + self._log_q.put(f"[TCP] Blastware connected from {peer}\n") + + try: + dev_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + dev_sock.connect((remote_host, remote_port)) + except OSError as e: + self._log_q.put(f"[TCP] Cannot reach device {remote_host}:{remote_port}: {e}\n") + client_sock.close() + continue + + self._log_q.put(f"[TCP] Connected to device at {remote_host}:{remote_port}\n") + + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + os.makedirs(logdir, exist_ok=True) + raw_bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") if raw_bw_on else None + raw_s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") if raw_s3_on else None + + self.after(0, self._notify_session_start, raw_bw_path, raw_s3_path, peer, remote_host, remote_port) + + self._run_session(client_sock, dev_sock, raw_bw_path, raw_s3_path, ts) + + self._log_q.put("<>") + + def _notify_session_start(self, raw_bw_path, raw_s3_path, peer, remote_host, remote_port) -> None: + self.status_var.set(f"Active: {peer} → {remote_host}:{remote_port}") + self._on_started(raw_bw_path, raw_s3_path, None) + + def _run_session(self, bw_sock: socket.socket, dev_sock: socket.socket, + raw_bw_path: Optional[str], raw_s3_path: Optional[str], + ts: str) -> None: + bw_fh = open(raw_bw_path, "wb") if raw_bw_path else None + s3_fh = open(raw_s3_path, "wb") if raw_s3_path else None + bw_bytes = [0] + s3_bytes = [0] + + def _pipe(src, dst, fh, counter): + try: + while True: + data = src.recv(4096) + if not data: + break + dst.sendall(data) + if fh: + fh.write(data) + fh.flush() + counter[0] += len(data) + except OSError: + pass + finally: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + + t_bw = threading.Thread(target=_pipe, args=(bw_sock, dev_sock, bw_fh, bw_bytes), daemon=True) + t_s3 = threading.Thread(target=_pipe, args=(dev_sock, bw_sock, s3_fh, s3_bytes), daemon=True) + t_bw.start() + t_s3.start() + t_bw.join() + t_s3.join() + + bw_sock.close() + dev_sock.close() + if bw_fh: + bw_fh.close() + if s3_fh: + s3_fh.close() + + self._log_q.put( + f"[TCP] Session {ts} done " + f"BW→dev: {bw_bytes[0]} bytes dev→BW: {s3_bytes[0]} bytes\n" + ) + if raw_bw_path: + self._log_q.put(f"[TCP] BW capture: {raw_bw_path}\n") + if raw_s3_path: + self._log_q.put(f"[TCP] S3 capture: {raw_s3_path}\n") + + # ───────────────────────────────────────────────────────────────────────────── # Analyzer panel (tk.Frame — lives inside a notebook tab) # Extracted from gui_analyzer.py; accepts external path injection. @@ -1887,6 +2166,13 @@ class SeismoLab(tk.Tk): ) nb.add(self._bridge_panel, text=" Bridge ") + self._tcp_bridge_panel = TcpBridgePanel( + nb, + on_bridge_started=self._on_bridge_started, + on_bridge_stopped=self._on_bridge_stopped, + ) + nb.add(self._tcp_bridge_panel, text=" TCP Capture ") + self._analyzer_panel = AnalyzerPanel(nb, db=self._db) nb.add(self._analyzer_panel, text=" Analyzer ") @@ -1927,6 +2213,7 @@ class SeismoLab(tk.Tk): def _on_close(self) -> None: self._bridge_panel.stop_bridge() + self._tcp_bridge_panel.stop_server() self._serial_watch_panel._stop() self.destroy() -- 2.52.0 From 6861d9ed970055d0d68d66f60bacc940d7f585e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 23:01:31 +0000 Subject: [PATCH 33/40] Merge TCP mode into Bridge tab (Serial/TCP radio toggle) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the separate 'TCP Capture' tab and folds TCP MITM capture directly into the existing Bridge tab. A Serial/TCP radio selector at the top swaps the connection fields (COM ports vs. listen port + device host:port) while keeping the same Start Bridge / Stop Bridge / Add Mark buttons, capture checkboxes, log dir, and live log — identical UX for both modes. https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- seismo_lab.py | 381 +++++++++++++++++++++----------------------------- 1 file changed, 163 insertions(+), 218 deletions(-) diff --git a/seismo_lab.py b/seismo_lab.py index 11e7f11..0f757f2 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -97,19 +97,33 @@ class AnalyzerState: class BridgePanel(tk.Frame): """ - All bridge controls and live log output. - Calls on_bridge_started(raw_bw_path, raw_s3_path) when the bridge starts - so the parent can wire up the Analyzer. + Bridge controls and live log output. + + Two modes selectable at the top: + - Serial: wraps s3_bridge.py as a subprocess (two COM ports) + - TCP: MITM proxy — listens for an incoming Blastware connection, + forwards all bytes to the real device over IP, captures both + directions to raw .bin files + + Calls on_bridge_started(raw_bw_path, raw_s3_path, struct_bin_path) when + traffic begins so the parent can wire up the Analyzer in live mode. """ def __init__(self, parent: tk.Widget, on_bridge_started, on_bridge_stopped, **kw): super().__init__(parent, bg=BG2, **kw) - self._on_started = on_bridge_started # signature: (raw_bw, raw_s3, struct_bin) + self._on_started = on_bridge_started self._on_stopped = on_bridge_stopped + # serial state self.process: Optional[subprocess.Popen] = None - self._stdout_q: queue.Queue[str] = queue.Queue() + # tcp state + self._server: Optional[socket.socket] = None + self._tcp_stop_event = threading.Event() + # unified log queue (serial reader thread + TCP pipe threads both push here) + self._log_q: queue.Queue[str] = queue.Queue() + # mode + self._mode = tk.StringVar(value="serial") self._build() - self._poll_stdout() + self._poll_log_q() # ── build ───────────────────────────────────────────────────────────── @@ -119,45 +133,80 @@ class BridgePanel(tk.Frame): cfg = tk.Frame(self, bg=BG2) cfg.pack(side=tk.TOP, fill=tk.X, padx=4, pady=4) - # Row 0: ports - tk.Label(cfg, text="BW COM:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=0, sticky="e", **pad) + # Row 0: mode selector + mode_row = tk.Frame(cfg, bg=BG2) + mode_row.grid(row=0, column=0, columnspan=6, sticky="w", padx=6, pady=(4, 0)) + tk.Label(mode_row, text="Mode:", bg=BG2, fg=FG, font=MONO).pack(side=tk.LEFT, padx=(0, 8)) + tk.Radiobutton(mode_row, text="Serial", variable=self._mode, value="serial", + bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, + font=MONO, command=self._on_mode_change).pack(side=tk.LEFT, padx=4) + tk.Radiobutton(mode_row, text="TCP", variable=self._mode, value="tcp", + bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, + font=MONO, command=self._on_mode_change).pack(side=tk.LEFT, padx=4) + + # Row 1a: serial connection fields (shown by default) + self._serial_frame = tk.Frame(cfg, bg=BG2) + self._serial_frame.grid(row=1, column=0, columnspan=6, sticky="w") + + tk.Label(self._serial_frame, text="BW COM:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=0, sticky="e", **pad) self.bw_var = tk.StringVar(value="COM4") - tk.Entry(cfg, textvariable=self.bw_var, width=10, + tk.Entry(self._serial_frame, textvariable=self.bw_var, width=10, bg=BG3, fg=FG, insertbackground=FG, relief="flat", font=MONO).grid(row=0, column=1, sticky="w", **pad) - tk.Label(cfg, text="S3 COM:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=2, sticky="e", **pad) + tk.Label(self._serial_frame, text="S3 COM:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=2, sticky="e", **pad) self.s3_var = tk.StringVar(value="COM5") - tk.Entry(cfg, textvariable=self.s3_var, width=10, + tk.Entry(self._serial_frame, textvariable=self.s3_var, width=10, bg=BG3, fg=FG, insertbackground=FG, relief="flat", font=MONO).grid(row=0, column=3, sticky="w", **pad) - tk.Label(cfg, text="Baud:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=4, sticky="e", **pad) + tk.Label(self._serial_frame, text="Baud:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=4, sticky="e", **pad) self.baud_var = tk.StringVar(value="38400") - tk.Entry(cfg, textvariable=self.baud_var, width=8, + tk.Entry(self._serial_frame, textvariable=self.baud_var, width=8, bg=BG3, fg=FG, insertbackground=FG, relief="flat", font=MONO).grid(row=0, column=5, sticky="w", **pad) - # Row 1: log dir - tk.Label(cfg, text="Log dir:", bg=BG2, fg=FG, font=MONO).grid(row=1, column=0, sticky="e", **pad) + # Row 1b: TCP connection fields (hidden until TCP mode selected) + self._tcp_frame = tk.Frame(cfg, bg=BG2) + + tk.Label(self._tcp_frame, text="Listen port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=0, sticky="e", **pad) + self.listen_port_var = tk.StringVar(value="9034") + tk.Entry(self._tcp_frame, textvariable=self.listen_port_var, width=8, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=1, sticky="w", **pad) + + tk.Label(self._tcp_frame, text="Device host:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=2, sticky="e", **pad) + self.remote_host_var = tk.StringVar(value="63.43.212.232") + tk.Entry(self._tcp_frame, textvariable=self.remote_host_var, width=18, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=3, sticky="w", **pad) + + tk.Label(self._tcp_frame, text="Port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=4, sticky="e", **pad) + self.remote_port_var = tk.StringVar(value="9034") + tk.Entry(self._tcp_frame, textvariable=self.remote_port_var, width=8, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", + font=MONO).grid(row=0, column=5, sticky="w", **pad) + + # Row 2: log dir + tk.Label(cfg, text="Log dir:", bg=BG2, fg=FG, font=MONO).grid(row=2, column=0, sticky="e", **pad) self.logdir_var = tk.StringVar(value=str(SCRIPT_DIR / "bridges" / "captures")) tk.Entry(cfg, textvariable=self.logdir_var, width=40, bg=BG3, fg=FG, insertbackground=FG, relief="flat", - font=MONO).grid(row=1, column=1, columnspan=4, sticky="we", **pad) + font=MONO).grid(row=2, column=1, columnspan=4, sticky="we", **pad) tk.Button(cfg, text="Browse", bg=BG3, fg=FG, relief="flat", cursor="hand2", - font=MONO, command=self._choose_dir).grid(row=1, column=5, **pad) + font=MONO, command=self._choose_dir).grid(row=2, column=5, **pad) - # Row 2: raw taps (always enabled — timestamped names generated at start) + # Row 3: raw capture checkboxes self._raw_bw_on = tk.BooleanVar(value=True) self._raw_s3_on = tk.BooleanVar(value=True) tk.Checkbutton(cfg, text="Capture BW->S3 raw", variable=self._raw_bw_on, bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=2, column=0, columnspan=2, sticky="w", **pad) + font=MONO).grid(row=3, column=0, columnspan=2, sticky="w", **pad) tk.Checkbutton(cfg, text="Capture S3->BW raw", variable=self._raw_s3_on, bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=2, column=2, columnspan=2, sticky="w", **pad) + font=MONO).grid(row=3, column=2, columnspan=2, sticky="w", **pad) - # Row 3: buttons + status + # Buttons + status btn_row = tk.Frame(self, bg=BG2) btn_row.pack(side=tk.TOP, fill=tk.X, padx=4, pady=2) @@ -190,6 +239,14 @@ class BridgePanel(tk.Frame): # ── helpers ─────────────────────────────────────────────────────────── + def _on_mode_change(self) -> None: + if self._mode.get() == "serial": + self._tcp_frame.grid_remove() + self._serial_frame.grid(row=1, column=0, columnspan=6, sticky="w") + else: + self._serial_frame.grid_remove() + self._tcp_frame.grid(row=1, column=0, columnspan=6, sticky="w") + def _choose_dir(self) -> None: path = filedialog.askdirectory(initialdir=self.logdir_var.get()) if path: @@ -201,9 +258,50 @@ class BridgePanel(tk.Frame): self.log_view.see(tk.END) self.log_view.configure(state="disabled") - # ── bridge control ──────────────────────────────────────────────────── + # ── unified log-queue polling (serial subprocess + TCP threads both push here) + + def _poll_log_q(self) -> None: + try: + while True: + msg = self._log_q.get_nowait() + if msg == "<>": + self._bridge_ended() + self._on_stopped() + elif msg == "<>": + if self._server is not None: + self.status_var.set(f"Listening on :{self.listen_port_var.get()}") + self._on_stopped() + else: + self._append_log(msg) + except queue.Empty: + pass + finally: + self.after(100, self._poll_log_q) + + # ── bridge control (delegates to serial or TCP) ─────────────────────── def start_bridge(self) -> None: + if self._mode.get() == "tcp": + self._start_tcp() + else: + self._start_serial() + + def stop_bridge(self) -> None: + if self._mode.get() == "tcp": + self._stop_tcp() + else: + self._stop_serial() + + def _bridge_ended(self) -> None: + self.status_var.set("Stopped") + self.start_btn.configure(state="normal") + self.stop_btn.configure(state="disabled", bg=BG3) + self.mark_btn.configure(state="disabled") + self._append_log("== Bridge stopped ==\n") + + # ── serial mode ─────────────────────────────────────────────────────── + + def _start_serial(self) -> None: if self.process and self.process.poll() is None: messagebox.showinfo("Bridge", "Bridge is already running.") return @@ -231,7 +329,6 @@ class BridgePanel(tk.Frame): raw_s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") args += ["--raw-s3", raw_s3_path] - # Structured bin path — written by bridge automatically, named by ts struct_bin_path = os.path.join(logdir, f"s3_session_{ts}.bin") try: @@ -253,11 +350,9 @@ class BridgePanel(tk.Frame): self.stop_btn.configure(state="normal", bg=RED) self.mark_btn.configure(state="normal") self._append_log(f"== Bridge started [{ts}] ==\n") - - # Notify parent so Analyzer can wire up live mode self._on_started(raw_bw_path, raw_s3_path, struct_bin_path) - def stop_bridge(self) -> None: + def _stop_serial(self) -> None: if self.process and self.process.poll() is None: self.process.terminate() try: @@ -267,177 +362,20 @@ class BridgePanel(tk.Frame): self._bridge_ended() self._on_stopped() - def _bridge_ended(self) -> None: - self.status_var.set("Stopped") - self.start_btn.configure(state="normal") - self.stop_btn.configure(state="disabled", bg=BG3) - self.mark_btn.configure(state="disabled") - self._append_log("== Bridge stopped ==\n") - def _reader_thread(self) -> None: if not self.process or not self.process.stdout: return for line in self.process.stdout: - self._stdout_q.put(line) - self._stdout_q.put("<>") + self._log_q.put(line) + self._log_q.put("<>") - def _poll_stdout(self) -> None: - try: - while True: - line = self._stdout_q.get_nowait() - if line == "<>": - self._bridge_ended() - self._on_stopped() - break - self._append_log(line) - except queue.Empty: - pass - finally: - self.after(100, self._poll_stdout) + # ── TCP mode ────────────────────────────────────────────────────────── - def add_mark(self) -> None: - if not self.process or not self.process.stdin or self.process.poll() is not None: + def _start_tcp(self) -> None: + if self._server is not None: + messagebox.showinfo("Bridge", "TCP bridge is already listening.") return - label = simpledialog.askstring("Mark", "Enter label for this mark:", parent=self) - if not label or not label.strip(): - return - try: - self.process.stdin.write("m\n") - self.process.stdin.write(label.strip() + "\n") - self.process.stdin.flush() - self._append_log(f"[MARK] {label.strip()}\n") - except Exception as e: - messagebox.showerror("Error", f"Failed to send mark:\n{e}") - -# ───────────────────────────────────────────────────────────────────────────── -# TCP Bridge panel — MITM capture over IP -# ───────────────────────────────────────────────────────────────────────────── - -class TcpBridgePanel(tk.Frame): - """ - TCP man-in-the-middle capture panel. - - Listens on a local TCP port for an incoming Blastware connection, forwards - all traffic to the real device, and saves both directions to raw .bin files. - - Calls on_bridge_started(raw_bw_path, raw_s3_path, None) when a session - begins so the Analyzer can wire up live mode — same signature as BridgePanel. - """ - - def __init__(self, parent: tk.Widget, on_bridge_started, on_bridge_stopped, **kw): - super().__init__(parent, bg=BG2, **kw) - self._on_started = on_bridge_started - self._on_stopped = on_bridge_stopped - self._server: Optional[socket.socket] = None - self._stop_event = threading.Event() - self._log_q: queue.Queue[str] = queue.Queue() - self._build() - self._poll_log_q() - - # ── build ───────────────────────────────────────────────────────────── - - def _build(self) -> None: - pad = {"padx": 6, "pady": 4} - - cfg = tk.Frame(self, bg=BG2) - cfg.pack(side=tk.TOP, fill=tk.X, padx=4, pady=4) - - # Row 0: listen port + remote host + remote port - tk.Label(cfg, text="Listen port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=0, sticky="e", **pad) - self.listen_port_var = tk.StringVar(value="9034") - tk.Entry(cfg, textvariable=self.listen_port_var, width=8, - bg=BG3, fg=FG, insertbackground=FG, relief="flat", - font=MONO).grid(row=0, column=1, sticky="w", **pad) - - tk.Label(cfg, text="Device host:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=2, sticky="e", **pad) - self.remote_host_var = tk.StringVar(value="63.43.212.232") - tk.Entry(cfg, textvariable=self.remote_host_var, width=18, - bg=BG3, fg=FG, insertbackground=FG, relief="flat", - font=MONO).grid(row=0, column=3, sticky="w", **pad) - - tk.Label(cfg, text="Port:", bg=BG2, fg=FG, font=MONO).grid(row=0, column=4, sticky="e", **pad) - self.remote_port_var = tk.StringVar(value="9034") - tk.Entry(cfg, textvariable=self.remote_port_var, width=8, - bg=BG3, fg=FG, insertbackground=FG, relief="flat", - font=MONO).grid(row=0, column=5, sticky="w", **pad) - - # Row 1: log dir - tk.Label(cfg, text="Log dir:", bg=BG2, fg=FG, font=MONO).grid(row=1, column=0, sticky="e", **pad) - self.logdir_var = tk.StringVar(value=str(SCRIPT_DIR / "bridges" / "captures" / "mitm")) - tk.Entry(cfg, textvariable=self.logdir_var, width=40, - bg=BG3, fg=FG, insertbackground=FG, relief="flat", - font=MONO).grid(row=1, column=1, columnspan=4, sticky="we", **pad) - tk.Button(cfg, text="Browse", bg=BG3, fg=FG, relief="flat", cursor="hand2", - font=MONO, command=self._choose_dir).grid(row=1, column=5, **pad) - - # Row 2: capture checkboxes - self._raw_bw_on = tk.BooleanVar(value=True) - self._raw_s3_on = tk.BooleanVar(value=True) - tk.Checkbutton(cfg, text="Capture BW→device raw", variable=self._raw_bw_on, - bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=2, column=0, columnspan=2, sticky="w", **pad) - tk.Checkbutton(cfg, text="Capture device→BW raw", variable=self._raw_s3_on, - bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=2, column=2, columnspan=2, sticky="w", **pad) - - # Row 3: buttons + status - btn_row = tk.Frame(self, bg=BG2) - btn_row.pack(side=tk.TOP, fill=tk.X, padx=4, pady=2) - - self.start_btn = tk.Button(btn_row, text="Start Listening", bg=GREEN, fg="#000000", - relief="flat", padx=12, cursor="hand2", font=MONO_B, - command=self.start_server) - self.start_btn.pack(side=tk.LEFT, padx=6) - - self.stop_btn = tk.Button(btn_row, text="Stop", bg=BG3, fg=FG, - relief="flat", padx=12, cursor="hand2", font=MONO, - command=self.stop_server, state="disabled") - self.stop_btn.pack(side=tk.LEFT, padx=4) - - self.status_var = tk.StringVar(value="Idle") - tk.Label(btn_row, textvariable=self.status_var, - bg=BG2, fg=FG_DIM, font=MONO).pack(side=tk.LEFT, padx=10) - - # Log output - self.log_view = scrolledtext.ScrolledText( - self, height=18, font=MONO_SM, - bg=BG, fg=FG, insertbackground=FG, - relief="flat", state="disabled", - ) - self.log_view.pack(fill=tk.BOTH, expand=True, padx=4, pady=4) - - # ── helpers ─────────────────────────────────────────────────────────── - - def _choose_dir(self) -> None: - path = filedialog.askdirectory(initialdir=self.logdir_var.get()) - if path: - self.logdir_var.set(path) - - def _append_log(self, text: str) -> None: - self.log_view.configure(state="normal") - self.log_view.insert(tk.END, text) - self.log_view.see(tk.END) - self.log_view.configure(state="disabled") - - def _poll_log_q(self) -> None: - try: - while True: - msg = self._log_q.get_nowait() - if msg == "<>": - if self._server is not None: - self.status_var.set(f"Listening on :{self.listen_port_var.get()}") - self._on_stopped() - else: - self._append_log(msg) - except queue.Empty: - pass - finally: - self.after(100, self._poll_log_q) - - # ── server control ──────────────────────────────────────────────────── - - def start_server(self) -> None: try: listen_port = int(self.listen_port_var.get().strip()) remote_host = self.remote_host_var.get().strip() @@ -460,20 +398,20 @@ class TcpBridgePanel(tk.Frame): return self._server = srv - self._stop_event.clear() + self._tcp_stop_event.clear() self.start_btn.configure(state="disabled") self.stop_btn.configure(state="normal", bg=RED) + self.mark_btn.configure(state="normal") self.status_var.set(f"Listening on :{listen_port}") ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self._append_log( f"== TCP Bridge started [{ts}]\n" f" Listening on 0.0.0.0:{listen_port}\n" - f" Forwarding to {remote_host}:{remote_port}\n" - f" Point Blastware call-home to this machine on port {listen_port}\n==\n" + f" Forwarding to {remote_host}:{remote_port}\n==\n" ) - logdir = self.logdir_var.get().strip() or "." + logdir = self.logdir_var.get().strip() or "." raw_bw_on = self._raw_bw_on.get() raw_s3_on = self._raw_s3_on.get() @@ -483,25 +421,20 @@ class TcpBridgePanel(tk.Frame): daemon=True, ).start() - def stop_server(self) -> None: - self._stop_event.set() + def _stop_tcp(self) -> None: + self._tcp_stop_event.set() if self._server: try: self._server.close() except OSError: pass self._server = None - self.start_btn.configure(state="normal") - self.stop_btn.configure(state="disabled", bg=BG3) - self.status_var.set("Idle") - self._append_log("== TCP Bridge stopped ==\n") + self._bridge_ended() self._on_stopped() - # ── accept / session threads ────────────────────────────────────────── - def _accept_loop(self, srv: socket.socket, remote_host: str, remote_port: int, logdir: str, raw_bw_on: bool, raw_s3_on: bool) -> None: - while not self._stop_event.is_set(): + while not self._tcp_stop_event.is_set(): try: client_sock, addr = srv.accept() except socket.timeout: @@ -527,19 +460,19 @@ class TcpBridgePanel(tk.Frame): raw_bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") if raw_bw_on else None raw_s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") if raw_s3_on else None - self.after(0, self._notify_session_start, raw_bw_path, raw_s3_path, peer, remote_host, remote_port) - - self._run_session(client_sock, dev_sock, raw_bw_path, raw_s3_path, ts) - + self.after(0, self._notify_tcp_session_start, + raw_bw_path, raw_s3_path, peer, remote_host, remote_port) + self._run_tcp_session(client_sock, dev_sock, raw_bw_path, raw_s3_path, ts) self._log_q.put("<>") - def _notify_session_start(self, raw_bw_path, raw_s3_path, peer, remote_host, remote_port) -> None: + def _notify_tcp_session_start(self, raw_bw_path, raw_s3_path, + peer, remote_host, remote_port) -> None: self.status_var.set(f"Active: {peer} → {remote_host}:{remote_port}") self._on_started(raw_bw_path, raw_s3_path, None) - def _run_session(self, bw_sock: socket.socket, dev_sock: socket.socket, - raw_bw_path: Optional[str], raw_s3_path: Optional[str], - ts: str) -> None: + def _run_tcp_session(self, bw_sock: socket.socket, dev_sock: socket.socket, + raw_bw_path: Optional[str], raw_s3_path: Optional[str], + ts: str) -> None: bw_fh = open(raw_bw_path, "wb") if raw_bw_path else None s3_fh = open(raw_s3_path, "wb") if raw_s3_path else None bw_bytes = [0] @@ -587,6 +520,26 @@ class TcpBridgePanel(tk.Frame): if raw_s3_path: self._log_q.put(f"[TCP] S3 capture: {raw_s3_path}\n") + # ── marks ───────────────────────────────────────────────────────────── + + def add_mark(self) -> None: + label = simpledialog.askstring("Mark", "Enter label for this mark:", parent=self) + if not label or not label.strip(): + return + if self._mode.get() == "tcp": + ts = datetime.datetime.now().strftime("%H:%M:%S") + self._append_log(f"[MARK {ts}] {label.strip()}\n") + else: + if not self.process or not self.process.stdin or self.process.poll() is not None: + return + try: + self.process.stdin.write("m\n") + self.process.stdin.write(label.strip() + "\n") + self.process.stdin.flush() + self._append_log(f"[MARK] {label.strip()}\n") + except Exception as e: + messagebox.showerror("Error", f"Failed to send mark:\n{e}") + # ───────────────────────────────────────────────────────────────────────────── # Analyzer panel (tk.Frame — lives inside a notebook tab) @@ -2166,13 +2119,6 @@ class SeismoLab(tk.Tk): ) nb.add(self._bridge_panel, text=" Bridge ") - self._tcp_bridge_panel = TcpBridgePanel( - nb, - on_bridge_started=self._on_bridge_started, - on_bridge_stopped=self._on_bridge_stopped, - ) - nb.add(self._tcp_bridge_panel, text=" TCP Capture ") - self._analyzer_panel = AnalyzerPanel(nb, db=self._db) nb.add(self._analyzer_panel, text=" Analyzer ") @@ -2213,7 +2159,6 @@ class SeismoLab(tk.Tk): def _on_close(self) -> None: self._bridge_panel.stop_bridge() - self._tcp_bridge_panel.stop_server() self._serial_watch_panel._stop() self.destroy() -- 2.52.0 From 9004241846aa357272b2f4d2dc5aef2382d7528f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 20:20:43 +0000 Subject: [PATCH 34/40] Restore multi-capture Bridge design + TCP mode Brings back the protocol-exp BridgePanel design: - Single bridge session stays up; New Capture / Stop Capture create labelled raw-file segments on demand (no files created at bridge start) - Capture history listbox shows all segments; double-click reloads in Analyzer - On capture complete: Analyzer auto-populates and runs analysis TCP mode integrated into same tab (Serial/TCP radio toggle): - Each incoming Blastware connection is automatically a capture segment - Session appears in history list; Analyzer wires up live on connect - Stop Capture disconnects current TCP session https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- seismo_lab.py | 388 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 274 insertions(+), 114 deletions(-) diff --git a/seismo_lab.py b/seismo_lab.py index 0f757f2..a8b6c35 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -100,30 +100,43 @@ class BridgePanel(tk.Frame): Bridge controls and live log output. Two modes selectable at the top: - - Serial: wraps s3_bridge.py as a subprocess (two COM ports) - - TCP: MITM proxy — listens for an incoming Blastware connection, - forwards all bytes to the real device over IP, captures both - directions to raw .bin files + - Serial: wraps s3_bridge.py as a subprocess (two COM ports). + Single bridge session; use New Capture / Stop Capture to create + labelled raw-file segments on demand. + - TCP: MITM proxy — listens for Blastware on a local port, forwards to + the real device. Each incoming connection is a capture; segments + appear in the history list automatically. - Calls on_bridge_started(raw_bw_path, raw_s3_path, struct_bin_path) when - traffic begins so the parent can wire up the Analyzer in live mode. + Callbacks (all optional except on_bridge_started / on_bridge_stopped): + on_bridge_started(struct_bin_path) — bridge is up + on_bridge_stopped() — bridge stopped + on_capture_started(bw_path, s3_path, label) — a capture segment began + on_capture_complete(bw_path, s3_path, label)— a capture segment finished """ - def __init__(self, parent: tk.Widget, on_bridge_started, on_bridge_stopped, **kw): + def __init__(self, parent: tk.Widget, on_bridge_started, on_bridge_stopped, + on_capture_started=None, on_capture_complete=None, **kw): super().__init__(parent, bg=BG2, **kw) - self._on_started = on_bridge_started - self._on_stopped = on_bridge_stopped + self._on_started = on_bridge_started + self._on_stopped = on_bridge_stopped + self._on_cap_started = on_capture_started + self._on_cap_complete = on_capture_complete # serial state self.process: Optional[subprocess.Popen] = None + self._stdout_q: queue.Queue[str] = queue.Queue() # tcp state self._server: Optional[socket.socket] = None self._tcp_stop_event = threading.Event() - # unified log queue (serial reader thread + TCP pipe threads both push here) - self._log_q: queue.Queue[str] = queue.Queue() + self._tcp_log_q: queue.Queue[str] = queue.Queue() + # shared capture state + self._capturing = False + self._cap_label: Optional[str] = None + self._cap_history: list[dict] = [] # {label, status, bw, s3} # mode self._mode = tk.StringVar(value="serial") self._build() - self._poll_log_q() + self._poll_stdout() + self._poll_tcp_log() # ── build ───────────────────────────────────────────────────────────── @@ -196,16 +209,6 @@ class BridgePanel(tk.Frame): tk.Button(cfg, text="Browse", bg=BG3, fg=FG, relief="flat", cursor="hand2", font=MONO, command=self._choose_dir).grid(row=2, column=5, **pad) - # Row 3: raw capture checkboxes - self._raw_bw_on = tk.BooleanVar(value=True) - self._raw_s3_on = tk.BooleanVar(value=True) - tk.Checkbutton(cfg, text="Capture BW->S3 raw", variable=self._raw_bw_on, - bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=3, column=0, columnspan=2, sticky="w", **pad) - tk.Checkbutton(cfg, text="Capture S3->BW raw", variable=self._raw_s3_on, - bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, - font=MONO).grid(row=3, column=2, columnspan=2, sticky="w", **pad) - # Buttons + status btn_row = tk.Frame(self, bg=BG2) btn_row.pack(side=tk.TOP, fill=tk.X, padx=4, pady=2) @@ -220,6 +223,18 @@ class BridgePanel(tk.Frame): command=self.stop_bridge, state="disabled") self.stop_btn.pack(side=tk.LEFT, padx=4) + tk.Frame(btn_row, bg=BG2, width=16).pack(side=tk.LEFT) # spacer + + self.cap_btn = tk.Button(btn_row, text="● New Capture", bg=ORANGE, fg="#000000", + relief="flat", padx=10, cursor="hand2", font=MONO_B, + command=self._start_capture, state="disabled") + self.cap_btn.pack(side=tk.LEFT, padx=4) + + self.stop_cap_btn = tk.Button(btn_row, text="■ Stop Capture", bg=BG3, fg=RED, + relief="flat", padx=10, cursor="hand2", font=MONO_B, + command=self._stop_capture, state="disabled") + self.stop_cap_btn.pack(side=tk.LEFT, padx=4) + self.mark_btn = tk.Button(btn_row, text="Add Mark", bg=BG3, fg=FG, relief="flat", padx=10, cursor="hand2", font=MONO, command=self.add_mark, state="disabled") @@ -229,9 +244,34 @@ class BridgePanel(tk.Frame): tk.Label(btn_row, textvariable=self.status_var, bg=BG2, fg=FG_DIM, font=MONO).pack(side=tk.LEFT, padx=10) + # Capture history list + hist_outer = tk.Frame(self, bg=BG2) + hist_outer.pack(side=tk.TOP, fill=tk.X, padx=4, pady=(2, 0)) + + tk.Label(hist_outer, text="Captures:", bg=BG2, fg=FG_DIM, + font=MONO_SM, anchor="w").pack(side=tk.LEFT, padx=(4, 6)) + + hist_inner = tk.Frame(hist_outer, bg=BG2) + hist_inner.pack(side=tk.LEFT, fill=tk.X, expand=True) + + self._hist_lb = tk.Listbox( + hist_inner, bg=BG3, fg=FG, font=MONO_SM, + height=3, relief="flat", selectbackground=BG, + selectforeground=ACCENT, activestyle="none", + highlightthickness=0, + ) + hist_vsb = ttk.Scrollbar(hist_inner, orient="vertical", command=self._hist_lb.yview) + self._hist_lb.configure(yscrollcommand=hist_vsb.set) + hist_vsb.pack(side=tk.RIGHT, fill=tk.Y) + self._hist_lb.pack(side=tk.LEFT, fill=tk.X, expand=True) + self._hist_lb.bind("", self._on_hist_dblclick) + + tk.Label(hist_outer, text="dbl-click to reload", bg=BG2, fg=FG_DIM, + font=MONO_SM, anchor="e").pack(side=tk.RIGHT, padx=6) + # Log output self.log_view = scrolledtext.ScrolledText( - self, height=18, font=MONO_SM, + self, height=14, font=MONO_SM, bg=BG, fg=FG, insertbackground=FG, relief="flat", state="disabled", ) @@ -258,25 +298,21 @@ class BridgePanel(tk.Frame): self.log_view.see(tk.END) self.log_view.configure(state="disabled") - # ── unified log-queue polling (serial subprocess + TCP threads both push here) + def _refresh_hist(self) -> None: + self._hist_lb.delete(0, tk.END) + for entry in self._cap_history: + icon = "\U0001f534" if entry["status"] == "recording" else "✅" + self._hist_lb.insert(tk.END, f" {icon} {entry['label'] or '(unlabeled)'}") + if self._cap_history: + self._hist_lb.see(tk.END) - def _poll_log_q(self) -> None: - try: - while True: - msg = self._log_q.get_nowait() - if msg == "<>": - self._bridge_ended() - self._on_stopped() - elif msg == "<>": - if self._server is not None: - self.status_var.set(f"Listening on :{self.listen_port_var.get()}") - self._on_stopped() - else: - self._append_log(msg) - except queue.Empty: - pass - finally: - self.after(100, self._poll_log_q) + def _on_hist_dblclick(self, _e=None) -> None: + sel = self._hist_lb.curselection() + if not sel: + return + entry = self._cap_history[sel[0]] + if entry["status"] == "done" and entry["bw"] and entry["s3"] and self._on_cap_complete: + self._on_cap_complete(entry["bw"], entry["s3"], entry["label"]) # ── bridge control (delegates to serial or TCP) ─────────────────────── @@ -296,9 +332,42 @@ class BridgePanel(tk.Frame): self.status_var.set("Stopped") self.start_btn.configure(state="normal") self.stop_btn.configure(state="disabled", bg=BG3) + self.cap_btn.configure(state="disabled") + self.stop_cap_btn.configure(state="disabled", bg=BG3) self.mark_btn.configure(state="disabled") + self._capturing = False + self._cap_label = None self._append_log("== Bridge stopped ==\n") + # ── capture lifecycle (shared by serial and TCP) ────────────────────── + + def _on_cap_started_msg(self, bw_path: str, s3_path: str) -> None: + for entry in reversed(self._cap_history): + if entry["status"] == "recording" and entry["bw"] is None: + entry["bw"] = bw_path + entry["s3"] = s3_path + break + self._refresh_hist() + if self._on_cap_started: + self._on_cap_started(bw_path, s3_path, self._cap_label or "") + + def _on_cap_stopped_msg(self, bw_path: str, s3_path: str) -> None: + label = self._cap_label or "capture" + for entry in reversed(self._cap_history): + if entry["status"] == "recording": + entry["status"] = "done" + entry["bw"] = bw_path + entry["s3"] = s3_path + break + self._refresh_hist() + self._capturing = False + self._cap_label = None + self.cap_btn.configure(state="normal") + self.stop_cap_btn.configure(state="disabled", bg=BG3) + self._append_log(f"[CAPTURE] Done: {label!r} — ready in Analyzer\n") + if self._on_cap_complete: + self._on_cap_complete(bw_path, s3_path, label) + # ── serial mode ─────────────────────────────────────────────────────── def _start_serial(self) -> None: @@ -321,14 +390,6 @@ class BridgePanel(tk.Frame): args = [sys.executable, str(BRIDGE_PATH), "--bw", bw, "--s3", s3, "--baud", baud, "--logdir", logdir] - raw_bw_path = raw_s3_path = None - if self._raw_bw_on.get(): - raw_bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") - args += ["--raw-bw", raw_bw_path] - if self._raw_s3_on.get(): - raw_s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") - args += ["--raw-s3", raw_s3_path] - struct_bin_path = os.path.join(logdir, f"s3_session_{ts}.bin") try: @@ -348,9 +409,10 @@ class BridgePanel(tk.Frame): self.status_var.set(f"Running — {bw} <-> {s3}") self.start_btn.configure(state="disabled") self.stop_btn.configure(state="normal", bg=RED) - self.mark_btn.configure(state="normal") + self.cap_btn.configure(state="normal") self._append_log(f"== Bridge started [{ts}] ==\n") - self._on_started(raw_bw_path, raw_s3_path, struct_bin_path) + self._append_log(" Click 'New Capture' when ready to record.\n") + self._on_started(struct_bin_path) def _stop_serial(self) -> None: if self.process and self.process.poll() is None: @@ -366,8 +428,79 @@ class BridgePanel(tk.Frame): if not self.process or not self.process.stdout: return for line in self.process.stdout: - self._log_q.put(line) - self._log_q.put("<>") + self._stdout_q.put(line) + self._stdout_q.put("<>") + + def _poll_stdout(self) -> None: + try: + while True: + line = self._stdout_q.get_nowait() + if line == "<>": + self._bridge_ended() + self._on_stopped() + break + + stripped = line.strip() + if stripped.startswith("[CAP_START] ") and "\t" in stripped: + parts = stripped[12:].split("\t", 1) + if len(parts) == 2: + self._on_cap_started_msg(parts[0].strip(), parts[1].strip()) + elif stripped.startswith("[CAP_STOP] ") and "\t" in stripped: + parts = stripped[11:].split("\t", 1) + if len(parts) == 2: + self._on_cap_stopped_msg(parts[0].strip(), parts[1].strip()) + + self._append_log(line) + except queue.Empty: + pass + finally: + self.after(100, self._poll_stdout) + + def _start_capture(self) -> None: + if not self.process or self.process.poll() is not None: + return + label = simpledialog.askstring( + "New Capture", + "Label for this capture\n(e.g. 'copy_event_download').\nLeave blank for timestamp only:", + parent=self, + ) + if label is None: + return + label = label.strip() + try: + self.process.stdin.write(f"CAP_START:{label}\n") + self.process.stdin.flush() + except Exception as e: + messagebox.showerror("Error", f"Failed to start capture:\n{e}") + return + self._capturing = True + self._cap_label = label or datetime.datetime.now().strftime("%H%M%S") + self.cap_btn.configure(state="disabled") + self.stop_cap_btn.configure(state="normal", bg=RED) + self.mark_btn.configure(state="normal") + self._append_log(f"[CAPTURE] Starting: {self._cap_label!r}...\n") + self._cap_history.append({"label": self._cap_label, "status": "recording", + "bw": None, "s3": None}) + self._refresh_hist() + + def _stop_capture(self) -> None: + if self._mode.get() == "tcp": + # TCP: close the server so the current session ends naturally + self._tcp_stop_event.set() + if self._server: + try: + self._server.close() + except OSError: + pass + self._server = None + return + if not self.process or self.process.poll() is not None: + return + try: + self.process.stdin.write("CAP_STOP\n") + self.process.stdin.flush() + except Exception as e: + messagebox.showerror("Error", f"Failed to stop capture:\n{e}") # ── TCP mode ────────────────────────────────────────────────────────── @@ -401,23 +534,21 @@ class BridgePanel(tk.Frame): self._tcp_stop_event.clear() self.start_btn.configure(state="disabled") self.stop_btn.configure(state="normal", bg=RED) - self.mark_btn.configure(state="normal") self.status_var.set(f"Listening on :{listen_port}") ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self._append_log( f"== TCP Bridge started [{ts}]\n" f" Listening on 0.0.0.0:{listen_port}\n" - f" Forwarding to {remote_host}:{remote_port}\n==\n" + f" Forwarding to {remote_host}:{remote_port}\n" + f" Each Blastware connection is automatically captured.\n==\n" ) + self._on_started(None) - logdir = self.logdir_var.get().strip() or "." - raw_bw_on = self._raw_bw_on.get() - raw_s3_on = self._raw_s3_on.get() - + logdir = self.logdir_var.get().strip() or "." threading.Thread( target=self._accept_loop, - args=(srv, remote_host, remote_port, logdir, raw_bw_on, raw_s3_on), + args=(srv, remote_host, remote_port, logdir), daemon=True, ).start() @@ -432,8 +563,8 @@ class BridgePanel(tk.Frame): self._bridge_ended() self._on_stopped() - def _accept_loop(self, srv: socket.socket, remote_host: str, remote_port: int, - logdir: str, raw_bw_on: bool, raw_s3_on: bool) -> None: + def _accept_loop(self, srv: socket.socket, remote_host: str, + remote_port: int, logdir: str) -> None: while not self._tcp_stop_event.is_set(): try: client_sock, addr = srv.accept() @@ -443,82 +574,98 @@ class BridgePanel(tk.Frame): break peer = f"{addr[0]}:{addr[1]}" - self._log_q.put(f"[TCP] Blastware connected from {peer}\n") + self._tcp_log_q.put(f"[TCP] Blastware connected from {peer}\n") try: dev_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dev_sock.connect((remote_host, remote_port)) except OSError as e: - self._log_q.put(f"[TCP] Cannot reach device {remote_host}:{remote_port}: {e}\n") + self._tcp_log_q.put(f"[TCP] Cannot reach device {remote_host}:{remote_port}: {e}\n") client_sock.close() continue - self._log_q.put(f"[TCP] Connected to device at {remote_host}:{remote_port}\n") + self._tcp_log_q.put(f"[TCP] Connected to device at {remote_host}:{remote_port}\n") ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") os.makedirs(logdir, exist_ok=True) - raw_bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") if raw_bw_on else None - raw_s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") if raw_s3_on else None + bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") + s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") - self.after(0, self._notify_tcp_session_start, - raw_bw_path, raw_s3_path, peer, remote_host, remote_port) - self._run_tcp_session(client_sock, dev_sock, raw_bw_path, raw_s3_path, ts) - self._log_q.put("<>") + # Auto-register in history as recording + label = f"tcp_{ts}" + self.after(0, self._tcp_session_started, bw_path, s3_path, label, peer, remote_host, remote_port) - def _notify_tcp_session_start(self, raw_bw_path, raw_s3_path, - peer, remote_host, remote_port) -> None: + self._run_tcp_session(client_sock, dev_sock, bw_path, s3_path, ts) + + self._tcp_log_q.put(f"<>\t{bw_path}\t{s3_path}\t{label}") + + def _tcp_session_started(self, bw_path, s3_path, label, peer, remote_host, remote_port) -> None: self.status_var.set(f"Active: {peer} → {remote_host}:{remote_port}") - self._on_started(raw_bw_path, raw_s3_path, None) + self._cap_label = label + self._cap_history.append({"label": label, "status": "recording", + "bw": bw_path, "s3": s3_path}) + self._refresh_hist() + self.mark_btn.configure(state="normal") + self.stop_cap_btn.configure(state="normal", bg=RED) + if self._on_cap_started: + self._on_cap_started(bw_path, s3_path, label) def _run_tcp_session(self, bw_sock: socket.socket, dev_sock: socket.socket, - raw_bw_path: Optional[str], raw_s3_path: Optional[str], - ts: str) -> None: - bw_fh = open(raw_bw_path, "wb") if raw_bw_path else None - s3_fh = open(raw_s3_path, "wb") if raw_s3_path else None + bw_path: str, s3_path: str, ts: str) -> None: bw_bytes = [0] s3_bytes = [0] - def _pipe(src, dst, fh, counter): - try: - while True: - data = src.recv(4096) - if not data: - break - dst.sendall(data) - if fh: + with open(bw_path, "wb") as bw_fh, open(s3_path, "wb") as s3_fh: + def _pipe(src, dst, fh, counter): + try: + while True: + data = src.recv(4096) + if not data: + break + dst.sendall(data) fh.write(data) fh.flush() - counter[0] += len(data) - except OSError: - pass - finally: - try: - dst.shutdown(socket.SHUT_WR) + counter[0] += len(data) except OSError: pass + finally: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass - t_bw = threading.Thread(target=_pipe, args=(bw_sock, dev_sock, bw_fh, bw_bytes), daemon=True) - t_s3 = threading.Thread(target=_pipe, args=(dev_sock, bw_sock, s3_fh, s3_bytes), daemon=True) - t_bw.start() - t_s3.start() - t_bw.join() - t_s3.join() + t_bw = threading.Thread(target=_pipe, args=(bw_sock, dev_sock, bw_fh, bw_bytes), daemon=True) + t_s3 = threading.Thread(target=_pipe, args=(dev_sock, bw_sock, s3_fh, s3_bytes), daemon=True) + t_bw.start() + t_s3.start() + t_bw.join() + t_s3.join() bw_sock.close() dev_sock.close() - if bw_fh: - bw_fh.close() - if s3_fh: - s3_fh.close() - - self._log_q.put( + self._tcp_log_q.put( f"[TCP] Session {ts} done " f"BW→dev: {bw_bytes[0]} bytes dev→BW: {s3_bytes[0]} bytes\n" ) - if raw_bw_path: - self._log_q.put(f"[TCP] BW capture: {raw_bw_path}\n") - if raw_s3_path: - self._log_q.put(f"[TCP] S3 capture: {raw_s3_path}\n") + + def _poll_tcp_log(self) -> None: + try: + while True: + msg = self._tcp_log_q.get_nowait() + if msg.startswith("<>"): + parts = msg.split("\t") + if len(parts) == 4: + _, bw_path, s3_path, label = parts + self._on_cap_stopped_msg(bw_path, s3_path) + if self._server is not None: + self.status_var.set(f"Listening on :{self.listen_port_var.get()}") + self.stop_cap_btn.configure(state="disabled", bg=BG3) + else: + self._append_log(msg) + except queue.Empty: + pass + finally: + self.after(100, self._poll_tcp_log) # ── marks ───────────────────────────────────────────────────────────── @@ -2116,6 +2263,8 @@ class SeismoLab(tk.Tk): nb, on_bridge_started=self._on_bridge_started, on_bridge_stopped=self._on_bridge_stopped, + on_capture_started=self._on_capture_started, + on_capture_complete=self._on_capture_complete, ) nb.add(self._bridge_panel, text=" Bridge ") @@ -2137,16 +2286,27 @@ class SeismoLab(tk.Tk): self._nb = nb self.protocol("WM_DELETE_WINDOW", self._on_close) - def _on_bridge_started(self, raw_bw: Optional[str], raw_s3: Optional[str], - struct_bin: Optional[str] = None) -> None: - """Bridge started — inject paths into analyzer and start live mode.""" - self._analyzer_panel.set_live_files(raw_bw, raw_s3, struct_bin) - # Switch to Analyzer tab so the user can watch it update - self._nb.select(1) + def _on_bridge_started(self, struct_bin: Optional[str] = None) -> None: + """Bridge is up — store struct bin path; stay on Bridge tab.""" + if struct_bin: + self._analyzer_panel.bin_var.set(struct_bin) def _on_bridge_stopped(self) -> None: self._analyzer_panel.stop_live() + def _on_capture_started(self, bw_path: str, s3_path: str, label: str) -> None: + """A capture segment began — wire live mode and switch to Analyzer.""" + self._analyzer_panel.set_live_files(bw_path, s3_path) + self._nb.select(1) + + def _on_capture_complete(self, bw_path: str, s3_path: str, label: str) -> None: + """A capture segment finished — run full analysis and switch to Analyzer.""" + self._analyzer_panel.stop_live() + self._analyzer_panel.s3_var.set(s3_path) + self._analyzer_panel.bw_var.set(bw_path) + self._analyzer_panel._run_analyze() + self._nb.select(1) + def _on_console_send_to_analyzer(self, raw_s3_path: str) -> None: """Console captured bytes → inject into Analyzer S3 field and switch tab.""" self._analyzer_panel.s3_var.set(raw_s3_path) -- 2.52.0 From b9ab368934d321e525fbb0c21731d3b3b4ebdfa7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 20:26:31 +0000 Subject: [PATCH 35/40] Fix TCP capture: write files only when capture is active Previously every Blastware connection auto-created files. Now TCP mode works the same as serial mode: - Start Bridge: proxy listens and forwards silently, no files written - New Capture: opens raw_bw/raw_s3 files; pipe threads write to them - Stop Capture: flushes and closes files, fires Analyzer callback - No connection = no file; multiple captures per bridge session work correctly https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- seismo_lab.py | 179 ++++++++++++++++++++++++++------------------------ 1 file changed, 93 insertions(+), 86 deletions(-) diff --git a/seismo_lab.py b/seismo_lab.py index a8b6c35..c1fbd3d 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -128,6 +128,12 @@ class BridgePanel(tk.Frame): self._server: Optional[socket.socket] = None self._tcp_stop_event = threading.Event() self._tcp_log_q: queue.Queue[str] = queue.Queue() + # tcp capture file handles — written only when capture is active + self._tcp_cap_lock = threading.Lock() + self._tcp_cap_bw_fh = None + self._tcp_cap_s3_fh = None + self._tcp_cap_bw_path: Optional[str] = None + self._tcp_cap_s3_path: Optional[str] = None # shared capture state self._capturing = False self._cap_label: Optional[str] = None @@ -457,8 +463,6 @@ class BridgePanel(tk.Frame): self.after(100, self._poll_stdout) def _start_capture(self) -> None: - if not self.process or self.process.poll() is not None: - return label = simpledialog.askstring( "New Capture", "Label for this capture\n(e.g. 'copy_event_download').\nLeave blank for timestamp only:", @@ -467,32 +471,58 @@ class BridgePanel(tk.Frame): if label is None: return label = label.strip() - try: - self.process.stdin.write(f"CAP_START:{label}\n") - self.process.stdin.flush() - except Exception as e: - messagebox.showerror("Error", f"Failed to start capture:\n{e}") - return self._capturing = True self._cap_label = label or datetime.datetime.now().strftime("%H%M%S") + + if self._mode.get() == "tcp": + # TCP: open the capture files now; pipe threads write here while active + logdir = self.logdir_var.get().strip() or "." + os.makedirs(logdir, exist_ok=True) + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") + s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") + with self._tcp_cap_lock: + self._tcp_cap_bw_fh = open(bw_path, "wb") + self._tcp_cap_s3_fh = open(s3_path, "wb") + self._tcp_cap_bw_path = bw_path + self._tcp_cap_s3_path = s3_path + self._cap_history.append({"label": self._cap_label, "status": "recording", + "bw": bw_path, "s3": s3_path}) + self._refresh_hist() + self._on_cap_started_msg(bw_path, s3_path) + else: + if not self.process or self.process.poll() is not None: + return + try: + self.process.stdin.write(f"CAP_START:{label}\n") + self.process.stdin.flush() + except Exception as e: + messagebox.showerror("Error", f"Failed to start capture:\n{e}") + return + self._cap_history.append({"label": self._cap_label, "status": "recording", + "bw": None, "s3": None}) + self._refresh_hist() + self.cap_btn.configure(state="disabled") self.stop_cap_btn.configure(state="normal", bg=RED) self.mark_btn.configure(state="normal") self._append_log(f"[CAPTURE] Starting: {self._cap_label!r}...\n") - self._cap_history.append({"label": self._cap_label, "status": "recording", - "bw": None, "s3": None}) - self._refresh_hist() def _stop_capture(self) -> None: if self._mode.get() == "tcp": - # TCP: close the server so the current session ends naturally - self._tcp_stop_event.set() - if self._server: - try: - self._server.close() - except OSError: - pass - self._server = None + with self._tcp_cap_lock: + bw_path = self._tcp_cap_bw_path + s3_path = self._tcp_cap_s3_path + if self._tcp_cap_bw_fh: + self._tcp_cap_bw_fh.close() + self._tcp_cap_bw_fh = None + if self._tcp_cap_s3_fh: + self._tcp_cap_s3_fh.close() + self._tcp_cap_s3_fh = None + self._tcp_cap_bw_path = None + self._tcp_cap_s3_path = None + if bw_path and s3_path: + self._on_cap_stopped_msg(bw_path, s3_path) return if not self.process or self.process.poll() is not None: return @@ -534,6 +564,7 @@ class BridgePanel(tk.Frame): self._tcp_stop_event.clear() self.start_btn.configure(state="disabled") self.stop_btn.configure(state="normal", bg=RED) + self.cap_btn.configure(state="normal") self.status_var.set(f"Listening on :{listen_port}") ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -541,18 +572,25 @@ class BridgePanel(tk.Frame): f"== TCP Bridge started [{ts}]\n" f" Listening on 0.0.0.0:{listen_port}\n" f" Forwarding to {remote_host}:{remote_port}\n" - f" Each Blastware connection is automatically captured.\n==\n" + f" Click 'New Capture' before the operation you want to record.\n==\n" ) self._on_started(None) - logdir = self.logdir_var.get().strip() or "." threading.Thread( target=self._accept_loop, - args=(srv, remote_host, remote_port, logdir), + args=(srv, remote_host, remote_port), daemon=True, ).start() def _stop_tcp(self) -> None: + # Close any open capture files first + with self._tcp_cap_lock: + if self._tcp_cap_bw_fh: + self._tcp_cap_bw_fh.close() + self._tcp_cap_bw_fh = None + if self._tcp_cap_s3_fh: + self._tcp_cap_s3_fh.close() + self._tcp_cap_s3_fh = None self._tcp_stop_event.set() if self._server: try: @@ -563,8 +601,7 @@ class BridgePanel(tk.Frame): self._bridge_ended() self._on_stopped() - def _accept_loop(self, srv: socket.socket, remote_host: str, - remote_port: int, logdir: str) -> None: + def _accept_loop(self, srv: socket.socket, remote_host: str, remote_port: int) -> None: while not self._tcp_stop_event.is_set(): try: client_sock, addr = srv.accept() @@ -585,83 +622,53 @@ class BridgePanel(tk.Frame): continue self._tcp_log_q.put(f"[TCP] Connected to device at {remote_host}:{remote_port}\n") + self._run_tcp_session(client_sock, dev_sock) + self._tcp_log_q.put(f"[TCP] Connection from {peer} closed\n") - ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - os.makedirs(logdir, exist_ok=True) - bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") - s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") - - # Auto-register in history as recording - label = f"tcp_{ts}" - self.after(0, self._tcp_session_started, bw_path, s3_path, label, peer, remote_host, remote_port) - - self._run_tcp_session(client_sock, dev_sock, bw_path, s3_path, ts) - - self._tcp_log_q.put(f"<>\t{bw_path}\t{s3_path}\t{label}") - - def _tcp_session_started(self, bw_path, s3_path, label, peer, remote_host, remote_port) -> None: - self.status_var.set(f"Active: {peer} → {remote_host}:{remote_port}") - self._cap_label = label - self._cap_history.append({"label": label, "status": "recording", - "bw": bw_path, "s3": s3_path}) - self._refresh_hist() - self.mark_btn.configure(state="normal") - self.stop_cap_btn.configure(state="normal", bg=RED) - if self._on_cap_started: - self._on_cap_started(bw_path, s3_path, label) - - def _run_tcp_session(self, bw_sock: socket.socket, dev_sock: socket.socket, - bw_path: str, s3_path: str, ts: str) -> None: + def _run_tcp_session(self, bw_sock: socket.socket, dev_sock: socket.socket) -> None: + """Forward bytes in both directions; write to capture files only when active.""" bw_bytes = [0] s3_bytes = [0] - with open(bw_path, "wb") as bw_fh, open(s3_path, "wb") as s3_fh: - def _pipe(src, dst, fh, counter): + def _pipe(src, dst, get_fh, counter): + try: + while True: + data = src.recv(4096) + if not data: + break + dst.sendall(data) + with self._tcp_cap_lock: + fh = get_fh() + if fh: + fh.write(data) + fh.flush() + counter[0] += len(data) + except OSError: + pass + finally: try: - while True: - data = src.recv(4096) - if not data: - break - dst.sendall(data) - fh.write(data) - fh.flush() - counter[0] += len(data) + dst.shutdown(socket.SHUT_WR) except OSError: pass - finally: - try: - dst.shutdown(socket.SHUT_WR) - except OSError: - pass - - t_bw = threading.Thread(target=_pipe, args=(bw_sock, dev_sock, bw_fh, bw_bytes), daemon=True) - t_s3 = threading.Thread(target=_pipe, args=(dev_sock, bw_sock, s3_fh, s3_bytes), daemon=True) - t_bw.start() - t_s3.start() - t_bw.join() - t_s3.join() + t_bw = threading.Thread(target=_pipe, + args=(bw_sock, dev_sock, + lambda: self._tcp_cap_bw_fh, bw_bytes), daemon=True) + t_s3 = threading.Thread(target=_pipe, + args=(dev_sock, bw_sock, + lambda: self._tcp_cap_s3_fh, s3_bytes), daemon=True) + t_bw.start() + t_s3.start() + t_bw.join() + t_s3.join() bw_sock.close() dev_sock.close() - self._tcp_log_q.put( - f"[TCP] Session {ts} done " - f"BW→dev: {bw_bytes[0]} bytes dev→BW: {s3_bytes[0]} bytes\n" - ) def _poll_tcp_log(self) -> None: try: while True: msg = self._tcp_log_q.get_nowait() - if msg.startswith("<>"): - parts = msg.split("\t") - if len(parts) == 4: - _, bw_path, s3_path, label = parts - self._on_cap_stopped_msg(bw_path, s3_path) - if self._server is not None: - self.status_var.set(f"Listening on :{self.listen_port_var.get()}") - self.stop_cap_btn.configure(state="disabled", bg=BG3) - else: - self._append_log(msg) + self._append_log(msg) except queue.Empty: pass finally: -- 2.52.0 From b14f31f3b03a7b80e15aaa7899d8371b30865b33 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 20:48:10 +0000 Subject: [PATCH 36/40] Include capture label in TCP raw filename Matches serial bridge naming: raw_bw_{ts}_{label}.bin / raw_s3_{ts}_{label}.bin https://claude.ai/code/session_014NczSHUz9uTzCAf4cVASTJ --- seismo_lab.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/seismo_lab.py b/seismo_lab.py index c1fbd3d..dc96d59 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -479,8 +479,10 @@ class BridgePanel(tk.Frame): logdir = self.logdir_var.get().strip() or "." os.makedirs(logdir, exist_ok=True) ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - bw_path = os.path.join(logdir, f"raw_bw_{ts}.bin") - s3_path = os.path.join(logdir, f"raw_s3_{ts}.bin") + safe_label = self._cap_label.replace(" ", "_") if self._cap_label else "" + suffix = f"_{safe_label}" if safe_label else "" + bw_path = os.path.join(logdir, f"raw_bw_{ts}{suffix}.bin") + s3_path = os.path.join(logdir, f"raw_s3_{ts}{suffix}.bin") with self._tcp_cap_lock: self._tcp_cap_bw_fh = open(bw_path, "wb") self._tcp_cap_s3_fh = open(s3_path, "wb") -- 2.52.0 From 625b0a4dfcea82b1f22a36a3801c2f88bc277ed5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 1 May 2026 00:12:02 +0000 Subject: [PATCH 37/40] feat(seismo_lab): add Download tab that captures wire bytes during event download Adds a new CapturingTransport wrapper in minimateplus.transport that mirrors every TX/RX byte to two raw .bin files using the same on-wire format as bridges/ach_mitm.py, so the resulting captures are byte-for-byte compatible with the existing Blastware MITM captures and load directly in the Analyzer. A new "Download" tab in seismo_lab.py lets the user connect to a device over TCP or serial and run connect / list-keys / download-events while the wrapper saves raw_bw_.bin (our TX) and raw_s3_.bin (device TX) into a seismo_dl_[_