serversdown/seismo-relay
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add full event download pipeline

Browse Source
This commit is contained in:
Brian Harrison
2026-03-31 20:48:03 -04:00
parent 6a0422a6fc
commit 9f52745bb4
4 changed files with 595 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
minimateplus/client.py
View File

@@ -44,9 +44,6 @@ from .protocol import MiniMateProtocol, ProtocolError
from .protocol import (
SUB_SERIAL_NUMBER,
SUB_FULL_CONFIG,
SUB_EVENT_INDEX,
SUB_EVENT_HEADER,
SUB_WAVEFORM_RECORD,
)
from .transport import SerialTransport, BaseTransport
@@ -150,39 +147,94 @@ class MiniMateClient:
def get_events(self, include_waveforms: bool = True) -> list[Event]:
"""
Download all stored events from the device.
Download all stored events from the device using the confirmed
1E → 0A → 0C → 1F event-iterator protocol.
For each event in the index:
1. SUB 1E — event header (timestamp, sample rate)
2. SUB 0C — full waveform record (peak values, project strings)
Sequence (confirmed from 3-31-26 Blastware capture):
1. SUB 1E — get first waveform key
2. For each key until b'\\x00\\x00\\x00\\x00':
a. SUB 0A — waveform header (first event only, to confirm full record)
b. SUB 0C — full waveform record (peak values, project strings)
c. SUB 1F — advance to next key (token=0xFE skips partial bins)
Subsequent keys returned by 1F (token=0xFE) are guaranteed to be full
records, so 0A is only called for the first event. This exactly
matches Blastware's observed behaviour.
Raw ADC waveform samples (SUB 5A bulk stream) are NOT downloaded
here — they can be large. Pass include_waveforms=True to also
download them (not yet implemented, reserved for a future call).
Args:
include_waveforms: Reserved. Currently ignored.
here — they are large (several MB per event) and fetched separately.
include_waveforms is reserved for a future call.
Returns:
List of Event objects, one per stored record on the device.
List of Event objects, one per stored waveform record.
Raises:
ProtocolError: on any communication failure.
ProtocolError: on unrecoverable communication failure.
"""
proto = self._require_proto()
log.info("get_events: reading event index (SUB 08)")
index_data = proto.read(SUB_EVENT_INDEX)
event_count = _decode_event_count(index_data)
log.info("get_events: %d event(s) found", event_count)
log.info("get_events: requesting first event (SUB 1E)")
try:
key4, _event_data8 = proto.read_event_first()
except ProtocolError as exc:
raise ProtocolError(f"get_events: 1E failed: {exc}") from exc
if key4 == b"\x00\x00\x00\x00":
log.info("get_events: device reports no stored events")
return []
events: list[Event] = []
for i in range(event_count):
log.info("get_events: downloading event %d/%d", i + 1, event_count)
ev = self._download_event(proto, i)
if ev:
events.append(ev)
idx = 0
is_first = True
while key4 != b"\x00\x00\x00\x00":
log.info(
"get_events: record %d key=%s", idx, key4.hex()
)
ev = Event(index=idx)
# First event: call 0A to verify it's a full record (0x30 length).
# Subsequent keys come from 1F(0xFE) which guarantees full records,
# so we skip 0A for those — exactly matching Blastware behaviour.
proceed = True
if is_first:
try:
_hdr, rec_len = proto.read_waveform_header(key4)
if rec_len < 0x30:
log.warning(
"get_events: first key=%s is partial (len=0x%02X) — skipping",
key4.hex(), rec_len,
)
proceed = False
except ProtocolError as exc:
log.warning(
"get_events: 0A failed for key=%s: %s — skipping 0C",
key4.hex(), exc,
)
proceed = False
is_first = False
if proceed:
# SUB 0C — full waveform record (peak values, project strings)
try:
record = proto.read_waveform_record(key4)
_decode_waveform_record_into(record, ev)
except ProtocolError as exc:
log.warning(
"get_events: 0C failed for key=%s: %s", key4.hex(), exc
)
events.append(ev)
idx += 1
# SUB 1F — advance to the next full waveform record key
try:
key4 = proto.advance_event()
except ProtocolError as exc:
log.warning("get_events: 1F failed: %s — stopping iteration", exc)
break
log.info("get_events: downloaded %d event(s)", len(events))
return events
# ── Internal helpers ──────────────────────────────────────────────────────
@@ -192,48 +244,6 @@ class MiniMateClient:
raise RuntimeError("MiniMateClient is not connected. Call open() first.")
return self._proto
def _download_event(
self, proto: MiniMateProtocol, index: int
) -> Optional[Event]:
"""Download header + waveform record for one event by index."""
ev = Event(index=index)
# SUB 1E — event header (timestamp, sample rate).
#
# The two-step event-header read passes the event index at payload[5]
# of the data-request frame (consistent with all other reads).
# This limits addressing to events 0–255 without a multi-byte scheme;
# the MiniMate Plus stores up to ~1000 events, so high indices may need
# a revised approach once we have captured event-download frames.
try:
from .framing import build_bw_frame
from .protocol import _expected_rsp_sub, SUB_EVENT_HEADER
# Step 1 — probe (offset=0)
probe_frame = build_bw_frame(SUB_EVENT_HEADER, 0)
proto._send(probe_frame)
_probe_rsp = proto._recv_one(expected_sub=_expected_rsp_sub(SUB_EVENT_HEADER))
# Step 2 — data request (offset = event index, clamped to 0xFF)
event_offset = min(index, 0xFF)
data_frame = build_bw_frame(SUB_EVENT_HEADER, event_offset)
proto._send(data_frame)
data_rsp = proto._recv_one(expected_sub=_expected_rsp_sub(SUB_EVENT_HEADER))
_decode_event_header_into(data_rsp.data, ev)
except ProtocolError as exc:
log.warning("event %d: header read failed: %s", index, exc)
return ev # Return partial event rather than losing it entirely
# SUB 0C — full waveform record (peak values, project strings).
try:
wf_data = proto.read(SUB_WAVEFORM_RECORD)
_decode_waveform_record_into(wf_data, ev)
except ProtocolError as exc:
log.warning("event %d: waveform record read failed: %s", index, exc)
return ev
# ── Decoder functions ─────────────────────────────────────────────────────────
#
@@ -353,37 +363,49 @@ def _decode_event_count(data: bytes) -> int:
def _decode_event_header_into(data: bytes, event: Event) -> None:
"""
Decode SUB E1 (EVENT_HEADER_RESPONSE) into an existing Event.
Decode SUB E1 (EVENT_HEADER_RESPONSE) raw data section into an Event.
The 6-byte timestamp is at the start of the data payload.
Sample rate location is not yet confirmed — left as None for now.
The waveform key is at data[11:15] (extracted separately in
MiniMateProtocol.read_event_first). The remaining 4 bytes at
data[15:19] are not yet decoded (❓ — possibly sample rate or flags).
Date information (year/month/day) lives in the waveform record (SUB 0C),
not in the 1E response. This function is a placeholder for any future
metadata we decode from the 8-byte 1E data block.
Modifies event in-place.
"""
if len(data) < 6:
log.warning("event header payload too short (%d bytes)", len(data))
return
try:
event.timestamp = Timestamp.from_bytes(data[:6])
except ValueError as exc:
log.warning("event header timestamp decode failed: %s", exc)
# Nothing confirmed yet from the 8-byte data block beyond the key at [0:4].
# Leave event.timestamp as None — it will be populated from the 0C record.
pass
def _decode_waveform_record_into(data: bytes, event: Event) -> None:
"""
Decode SUB F3 (FULL_WAVEFORM_RECORD) data into an existing Event.
Decode a 210-byte SUB F3 (FULL_WAVEFORM_RECORD) record into an Event.
Peak values are stored as IEEE 754 big-endian floats. Confirmed
positions per §7.5 (search for the known float bytes in the payload).
The *data* argument is the raw record bytes returned by
MiniMateProtocol.read_waveform_record() — i.e. data_rsp.data[11:11+0xD2].
This decoder is intentionally conservative — it searches for the
canonical 4×float32 pattern rather than relying on a fixed offset,
since the exact field layout is only partially confirmed.
Extracts:
- record_type: "Histogram" or "Waveform" (string search) 🔶
- peak_values: label-based float32 lookup (confirmed ✅)
- project_info: "Project:", "Client:", etc. string search ✅
Timestamp in the waveform record:
7-byte format: [0x09][year:2 BE][0x00][hour][minute][second]
Month and day come from a separate source (not yet fully mapped ❓).
For now we leave event.timestamp as None.
Modifies event in-place.
"""
# Attempt to extract four consecutive IEEE 754 BE floats from the
# known region of the payload (offsets are 🔶 INFERRED from captured data)
# ── Record type ───────────────────────────────────────────────────────────
try:
event.record_type = _extract_record_type(data)
except Exception as exc:
log.warning("waveform record type decode failed: %s", exc)
# ── Peak values ───────────────────────────────────────────────────────────
try:
peak_values = _extract_peak_floats(data)
if peak_values:
@@ -391,7 +413,7 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
except Exception as exc:
log.warning("waveform record peak decode failed: %s", exc)
# Project strings — search for known ASCII labels
# ── Project strings ───────────────────────────────────────────────────────
try:
project_info = _extract_project_strings(data)
if project_info:
@@ -400,41 +422,69 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
log.warning("waveform record project strings decode failed: %s", exc)
def _extract_record_type(data: bytes) -> Optional[str]:
"""
Search the waveform record for a record-type indicator string.
Confirmed types from 3-31-26 capture: "Histogram", "Waveform".
Returns the first match, or None if neither is found.
"""
for rtype in (b"Histogram", b"Waveform"):
if data.find(rtype) >= 0:
return rtype.decode()
return None
def _extract_peak_floats(data: bytes) -> Optional[PeakValues]:
"""
Scan the waveform record payload for four sequential float32 BE values
corresponding to Tran, Vert, Long, MicL peak values.
Locate per-channel peak particle velocity values in the 210-byte
waveform record by searching for the embedded channel label strings
("Tran", "Vert", "Long", "MicL") and reading the IEEE 754 BE float
at label_offset + 6.
The exact offset is not confirmed (🔶), so we do a heuristic scan:
look for four consecutive 4-byte groups where each decodes as a
plausible PPV value (0 < v < 100 in/s or psi).
The floats are NOT 4-byte aligned in the record (confirmed from
3-31-26 capture), so the previous step-4 scan missed Tran, Long, and
MicL entirely. Label-based lookup is the correct approach.
Returns PeakValues if a plausible group is found, else None.
Channel labels are separated by inner-frame bytes (0x10 0x03 = DLE ETX),
which the S3FrameParser preserves as literal data. Searching for the
4-byte ASCII label strings is robust to this structure.
Returns PeakValues if at least one channel label is found, else None.
"""
# Require at least 16 bytes for 4 floats
if len(data) < 16:
return None
# (label_bytes, field_name)
channels = (
(b"Tran", "tran"),
(b"Vert", "vert"),
(b"Long", "long_"),
(b"MicL", "micl"),
)
vals: dict[str, float] = {}
for start in range(0, len(data) - 15, 4):
for label_bytes, field in channels:
pos = data.find(label_bytes)
if pos < 0:
continue
float_off = pos + 6
if float_off + 4 > len(data):
log.debug("peak float: label %s at %d but float runs past end", label_bytes, pos)
continue
try:
vals = struct.unpack_from(">4f", data, start)
val = struct.unpack_from(">f", data, float_off)[0]
except struct.error:
continue
log.debug("peak float: %s at label+6 (%d) = %.6f", label_bytes.decode(), float_off, val)
vals[field] = val
# All four values should be non-negative and within plausible PPV range
if all(0.0 <= v < 100.0 for v in vals):
tran, vert, long_, micl = vals
# MicL (psi) is typically much smaller than geo values
# Simple sanity: at least two non-zero values
if sum(v > 0 for v in vals) >= 2:
log.debug(
"peak floats at offset %d: T=%.4f V=%.4f L=%.4f M=%.6f",
start, tran, vert, long_, micl
)
return PeakValues(
tran=tran, vert=vert, long=long_, micl=micl
)
return None
if not vals:
return None
return PeakValues(
tran=vals.get("tran"),
vert=vals.get("vert"),
long=vals.get("long_"),
micl=vals.get("micl"),
)
def _extract_project_strings(data: bytes) -> Optional[ProjectInfo]:

63
minimateplus/framing.py
View File

@@ -90,33 +90,90 @@ def checksum(payload: bytes) -> int:
# ── BW→S3 frame builder ───────────────────────────────────────────────────────
def build_bw_frame(sub: int, offset: int = 0) -> bytes:
def build_bw_frame(sub: int, offset: int = 0, params: bytes = bytes(10)) -> bytes:
"""
Build a BW→S3 read-command frame.
The payload is always 16 de-stuffed bytes:
[BW_CMD, 0x00, sub, 0x00, 0x00, offset, 0x00 × 10]
[BW_CMD, 0x00, sub, 0x00, 0x00, offset] + params(10 bytes)
Confirmed from BW capture analysis: payload[3] and payload[4] are always
0x00 across all observed read commands. The two-step offset lives at
payload[5]: 0x00 for the length-probe step, DATA_LEN for the data-fetch step.
The 10 params bytes (payload[6..15]) are zero for standard reads. For
keyed reads (SUBs 0A, 0C) the 4-byte waveform key lives at params[4..7]
(= payload[10..13]). For token-based reads (SUBs 1E, 1F) a single token
byte lives at params[6] (= payload[12]). Use waveform_key_params() and
token_params() helpers to build these safely.
Wire output: [ACK] [STX] dle_stuff(payload + checksum) [ETX]
Args:
sub: SUB command byte (e.g. 0x01 = FULL_CONFIG_READ)
offset: Value placed at payload[5].
Pass 0 for the probe step; pass DATA_LENGTHS[sub] for the data step.
params: 10 bytes placed at payload[6..15]. Default: all zeros.
Returns:
Complete frame bytes ready to write to the serial port / socket.
"""
payload = bytes([BW_CMD, 0x00, sub, 0x00, 0x00, offset]) + bytes(_BW_PAYLOAD_SIZE - 6)
if len(params) != 10:
raise ValueError(f"params must be exactly 10 bytes, got {len(params)}")
payload = bytes([BW_CMD, 0x00, sub, 0x00, 0x00, offset]) + params
chk = checksum(payload)
wire = bytes([ACK, STX]) + dle_stuff(payload + bytes([chk])) + bytes([ETX])
return wire
def waveform_key_params(key4: bytes) -> bytes:
"""
Build the 10-byte params block that carries a 4-byte waveform key.
Used for SUBs 0A (WAVEFORM_HEADER) and 0C (WAVEFORM_RECORD).
The key goes at params[4..7], which maps to payload[10..13].
Confirmed from 3-31-26 capture: 0A and 0C request frames carry the
4-byte record address at payload[10..13]. Probe and data-fetch steps
carry the same key in both frames.
Args:
key4: exactly 4 bytes — the opaque waveform record address returned
by the EVENT_HEADER (1E) or EVENT_ADVANCE (1F) response.
Returns:
10-byte params block with key embedded at positions [4..7].
"""
if len(key4) != 4:
raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
p = bytearray(10)
p[4:8] = key4
return bytes(p)
def token_params(token: int = 0) -> bytes:
"""
Build the 10-byte params block that carries a single token byte.
Used for SUBs 1E (EVENT_HEADER) and 1F (EVENT_ADVANCE).
The token goes at params[6], which maps to payload[12].
Confirmed from 3-31-26 capture:
- token=0x00: first-event read / browse mode (no download marking)
- token=0xfe: download mode (causes 1F to skip partial bins and
advance to the next full record)
Args:
token: single byte to place at params[6] / payload[12].
Returns:
10-byte params block with token at position [6].
"""
p = bytearray(10)
p[6] = token
return bytes(p)
# ── Pre-built POLL frames ─────────────────────────────────────────────────────
#
# POLL (SUB 0x5B) uses the same two-step pattern as all other reads — the

168
minimateplus/protocol.py
View File

@@ -29,6 +29,8 @@ from .framing import (
S3Frame,
S3FrameParser,
build_bw_frame,
waveform_key_params,
token_params,
POLL_PROBE,
POLL_DATA,
)
@@ -53,6 +55,7 @@ SUB_EVENT_INDEX = 0x08
SUB_CHANNEL_CONFIG = 0x06
SUB_TRIGGER_CONFIG = 0x1C
SUB_EVENT_HEADER = 0x1E
SUB_EVENT_ADVANCE = 0x1F
SUB_WAVEFORM_HEADER = 0x0A
SUB_WAVEFORM_RECORD = 0x0C
SUB_BULK_WAVEFORM = 0x5A
@@ -74,6 +77,11 @@ DATA_LENGTHS: dict[int, int] = {
SUB_FULL_CONFIG: 0x98, # 152-byte full config block ✅
SUB_EVENT_INDEX: 0x58, # 88-byte event index ✅
SUB_TRIGGER_CONFIG: 0x2C, # 44-byte trigger config 🔶
SUB_EVENT_HEADER: 0x08, # 8-byte event header (waveform key + event data) ✅
SUB_EVENT_ADVANCE: 0x08, # 8-byte next-key response ✅
# SUB_WAVEFORM_HEADER (0x0A) is VARIABLE — length read from probe response
# data[4]. Do NOT add it here; use read_waveform_header() instead. ✅
SUB_WAVEFORM_RECORD: 0xD2, # 210-byte waveform/histogram record ✅
SUB_UNKNOWN_2E: 0x1A, # 26 bytes, purpose TBD 🔶
0x09: 0xCA, # 202 bytes, purpose TBD 🔶
# SUB_COMPLIANCE (0x1A) uses a multi-step sequence with a 2090-byte total;
@@ -227,6 +235,166 @@ class MiniMateProtocol:
"""
self._send(POLL_PROBE)
# ── Event download API ────────────────────────────────────────────────────
def read_event_first(self) -> tuple[bytes, bytes]:
"""
Send the SUB 1E (EVENT_HEADER) two-step read and return the first
waveform key and accompanying 8-byte event data block.
This always uses all-zero params — the device returns the first stored
event's waveform key unconditionally.
Returns:
(key4, event_data8) where:
key4 — 4-byte opaque waveform record address (data[11:15])
event_data8 — full 8-byte data section (data[11:19])
Raises:
ProtocolError: on timeout, bad checksum, or wrong response SUB.
Confirmed from 3-31-26 capture: 1E request uses all-zero params;
response data section layout is:
[LENGTH_ECHO:1][00×4][KEY_ECHO:4][00×2][KEY4:4][EXTRA:4] …
Actual data starts at data[11]; first 4 bytes are the waveform key.
"""
rsp_sub = _expected_rsp_sub(SUB_EVENT_HEADER)
length = DATA_LENGTHS[SUB_EVENT_HEADER] # 0x08
log.debug("read_event_first: 1E probe")
self._send(build_bw_frame(SUB_EVENT_HEADER, 0))
self._recv_one(expected_sub=rsp_sub)
log.debug("read_event_first: 1E data request offset=0x%02X", length)
self._send(build_bw_frame(SUB_EVENT_HEADER, length))
data_rsp = self._recv_one(expected_sub=rsp_sub)
event_data8 = data_rsp.data[11:19]
key4 = data_rsp.data[11:15]
log.debug("read_event_first: key=%s", key4.hex())
return key4, event_data8
def read_waveform_header(self, key4: bytes) -> tuple[bytes, int]:
"""
Send the SUB 0A (WAVEFORM_HEADER) two-step read for *key4*.
The data length for 0A is VARIABLE and must be read from the probe
response at data[4]. Two known values:
0x30 — full histogram bin (has a waveform record to follow)
0x26 — partial histogram bin (no waveform record)
Args:
key4: 4-byte waveform record address from 1E or 1F.
Returns:
(header_bytes, record_length) where:
header_bytes — raw data section starting at data[11]
record_length — DATA_LENGTH read from probe (0x30 or 0x26)
Raises:
ProtocolError: on timeout, bad checksum, or wrong response SUB.
Confirmed from 3-31-26 capture: 0A probe response data[4] carries
the variable length; data-request uses that length as the offset byte.
"""
rsp_sub = _expected_rsp_sub(SUB_WAVEFORM_HEADER)
params = waveform_key_params(key4)
log.debug("read_waveform_header: 0A probe key=%s", key4.hex())
self._send(build_bw_frame(SUB_WAVEFORM_HEADER, 0, params))
probe_rsp = self._recv_one(expected_sub=rsp_sub)
# Variable length — read from probe response data[4]
length = probe_rsp.data[4] if len(probe_rsp.data) > 4 else 0x30
log.debug("read_waveform_header: 0A data request offset=0x%02X", length)
if length == 0:
return b"", 0
self._send(build_bw_frame(SUB_WAVEFORM_HEADER, length, params))
data_rsp = self._recv_one(expected_sub=rsp_sub)
header_bytes = data_rsp.data[11:11 + length]
log.debug(
"read_waveform_header: key=%s length=0x%02X is_full=%s",
key4.hex(), length, length == 0x30,
)
return header_bytes, length
def read_waveform_record(self, key4: bytes) -> bytes:
"""
Send the SUB 0C (WAVEFORM_RECORD / FULL_WAVEFORM_RECORD) two-step read.
Returns the 210-byte waveform/histogram record containing:
- Record type string ("Histogram" or "Waveform") at a variable offset
- Per-channel labels ("Tran", "Vert", "Long", "MicL") with PPV floats
at label_offset + 6
Args:
key4: 4-byte waveform record address.
Returns:
210-byte record bytes (data[11:11+0xD2]).
Raises:
ProtocolError: on timeout, bad checksum, or wrong response SUB.
Confirmed from 3-31-26 capture: 0C always uses offset=0xD2 (210 bytes).
"""
rsp_sub = _expected_rsp_sub(SUB_WAVEFORM_RECORD)
length = DATA_LENGTHS[SUB_WAVEFORM_RECORD] # 0xD2
params = waveform_key_params(key4)
log.debug("read_waveform_record: 0C probe key=%s", key4.hex())
self._send(build_bw_frame(SUB_WAVEFORM_RECORD, 0, params))
self._recv_one(expected_sub=rsp_sub)
log.debug("read_waveform_record: 0C data request offset=0x%02X", length)
self._send(build_bw_frame(SUB_WAVEFORM_RECORD, length, params))
data_rsp = self._recv_one(expected_sub=rsp_sub)
record = data_rsp.data[11:11 + length]
log.debug("read_waveform_record: received %d record bytes", len(record))
return record
def advance_event(self) -> bytes:
"""
Send the SUB 1F (EVENT_ADVANCE) two-step read with download-mode token
(0xFE) and return the next waveform key.
In download mode (token=0xFE), the device skips partial histogram bins
and returns the key of the next FULL record directly. This is the
Blastware-observed behaviour for iterating through all stored events.
Returns:
key4 — 4-byte next waveform key from data[11:15].
Returns b'\\x00\\x00\\x00\\x00' when there are no more events.
Raises:
ProtocolError: on timeout, bad checksum, or wrong response SUB.
Confirmed from 3-31-26 capture: 1F uses token=0xFE at params[6];
loop termination is key4 == b'\\x00\\x00\\x00\\x00'.
"""
rsp_sub = _expected_rsp_sub(SUB_EVENT_ADVANCE)
length = DATA_LENGTHS[SUB_EVENT_ADVANCE] # 0x08
params = token_params(0xFE)
log.debug("advance_event: 1F probe")
self._send(build_bw_frame(SUB_EVENT_ADVANCE, 0, params))
self._recv_one(expected_sub=rsp_sub)
log.debug("advance_event: 1F data request offset=0x%02X", length)
self._send(build_bw_frame(SUB_EVENT_ADVANCE, length, params))
data_rsp = self._recv_one(expected_sub=rsp_sub)
key4 = data_rsp.data[11:15]
log.debug(
"advance_event: next key=%s done=%s",
key4.hex(), key4 == b"\x00\x00\x00\x00",
)
return key4
# ── Internal helpers ──────────────────────────────────────────────────────
def _send(self, frame: bytes) -> None: