serversdown/seismo-relay
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add full event download pipeline

Browse Source
This commit is contained in:
Brian Harrison
2026-03-31 20:48:03 -04:00
parent 6a0422a6fc
commit 9f52745bb4
4 changed files with 595 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
minimateplus/client.py
View File

@@ -44,9 +44,6 @@ from .protocol import MiniMateProtocol, ProtocolError
from .protocol import (
SUB_SERIAL_NUMBER,
SUB_FULL_CONFIG,
SUB_EVENT_INDEX,
SUB_EVENT_HEADER,
SUB_WAVEFORM_RECORD,
)
from .transport import SerialTransport, BaseTransport
@@ -150,39 +147,94 @@ class MiniMateClient:
def get_events(self, include_waveforms: bool = True) -> list[Event]:
"""
Download all stored events from the device.
Download all stored events from the device using the confirmed
1E → 0A → 0C → 1F event-iterator protocol.
For each event in the index:
1. SUB 1E — event header (timestamp, sample rate)
2. SUB 0C — full waveform record (peak values, project strings)
Sequence (confirmed from 3-31-26 Blastware capture):
1. SUB 1E — get first waveform key
2. For each key until b'\\x00\\x00\\x00\\x00':
a. SUB 0A — waveform header (first event only, to confirm full record)
b. SUB 0C — full waveform record (peak values, project strings)
c. SUB 1F — advance to next key (token=0xFE skips partial bins)
Subsequent keys returned by 1F (token=0xFE) are guaranteed to be full
records, so 0A is only called for the first event. This exactly
matches Blastware's observed behaviour.
Raw ADC waveform samples (SUB 5A bulk stream) are NOT downloaded
here — they can be large. Pass include_waveforms=True to also
download them (not yet implemented, reserved for a future call).
Args:
include_waveforms: Reserved. Currently ignored.
here — they are large (several MB per event) and fetched separately.
include_waveforms is reserved for a future call.
Returns:
List of Event objects, one per stored record on the device.
List of Event objects, one per stored waveform record.
Raises:
ProtocolError: on any communication failure.
ProtocolError: on unrecoverable communication failure.
"""
proto = self._require_proto()
log.info("get_events: reading event index (SUB 08)")
index_data = proto.read(SUB_EVENT_INDEX)
event_count = _decode_event_count(index_data)
log.info("get_events: %d event(s) found", event_count)
log.info("get_events: requesting first event (SUB 1E)")
try:
key4, _event_data8 = proto.read_event_first()
except ProtocolError as exc:
raise ProtocolError(f"get_events: 1E failed: {exc}") from exc
if key4 == b"\x00\x00\x00\x00":
log.info("get_events: device reports no stored events")
return []
events: list[Event] = []
for i in range(event_count):
log.info("get_events: downloading event %d/%d", i + 1, event_count)
ev = self._download_event(proto, i)
if ev:
events.append(ev)
idx = 0
is_first = True
while key4 != b"\x00\x00\x00\x00":
log.info(
"get_events: record %d key=%s", idx, key4.hex()
)
ev = Event(index=idx)
# First event: call 0A to verify it's a full record (0x30 length).
# Subsequent keys come from 1F(0xFE) which guarantees full records,
# so we skip 0A for those — exactly matching Blastware behaviour.
proceed = True
if is_first:
try:
_hdr, rec_len = proto.read_waveform_header(key4)
if rec_len < 0x30:
log.warning(
"get_events: first key=%s is partial (len=0x%02X) — skipping",
key4.hex(), rec_len,
)
proceed = False
except ProtocolError as exc:
log.warning(
"get_events: 0A failed for key=%s: %s — skipping 0C",
key4.hex(), exc,
)
proceed = False
is_first = False
if proceed:
# SUB 0C — full waveform record (peak values, project strings)
try:
record = proto.read_waveform_record(key4)
_decode_waveform_record_into(record, ev)
except ProtocolError as exc:
log.warning(
"get_events: 0C failed for key=%s: %s", key4.hex(), exc
)
events.append(ev)
idx += 1
# SUB 1F — advance to the next full waveform record key
try:
key4 = proto.advance_event()
except ProtocolError as exc:
log.warning("get_events: 1F failed: %s — stopping iteration", exc)
break
log.info("get_events: downloaded %d event(s)", len(events))
return events
# ── Internal helpers ──────────────────────────────────────────────────────
@@ -192,48 +244,6 @@ class MiniMateClient:
raise RuntimeError("MiniMateClient is not connected. Call open() first.")
return self._proto
def _download_event(
self, proto: MiniMateProtocol, index: int
) -> Optional[Event]:
"""Download header + waveform record for one event by index."""
ev = Event(index=index)
# SUB 1E — event header (timestamp, sample rate).
#
# The two-step event-header read passes the event index at payload[5]
# of the data-request frame (consistent with all other reads).
# This limits addressing to events 0–255 without a multi-byte scheme;
# the MiniMate Plus stores up to ~1000 events, so high indices may need
# a revised approach once we have captured event-download frames.
try:
from .framing import build_bw_frame
from .protocol import _expected_rsp_sub, SUB_EVENT_HEADER
# Step 1 — probe (offset=0)
probe_frame = build_bw_frame(SUB_EVENT_HEADER, 0)
proto._send(probe_frame)
_probe_rsp = proto._recv_one(expected_sub=_expected_rsp_sub(SUB_EVENT_HEADER))
# Step 2 — data request (offset = event index, clamped to 0xFF)
event_offset = min(index, 0xFF)
data_frame = build_bw_frame(SUB_EVENT_HEADER, event_offset)
proto._send(data_frame)
data_rsp = proto._recv_one(expected_sub=_expected_rsp_sub(SUB_EVENT_HEADER))
_decode_event_header_into(data_rsp.data, ev)
except ProtocolError as exc:
log.warning("event %d: header read failed: %s", index, exc)
return ev # Return partial event rather than losing it entirely
# SUB 0C — full waveform record (peak values, project strings).
try:
wf_data = proto.read(SUB_WAVEFORM_RECORD)
_decode_waveform_record_into(wf_data, ev)
except ProtocolError as exc:
log.warning("event %d: waveform record read failed: %s", index, exc)
return ev
# ── Decoder functions ─────────────────────────────────────────────────────────
#
@@ -353,37 +363,49 @@ def _decode_event_count(data: bytes) -> int:
def _decode_event_header_into(data: bytes, event: Event) -> None:
"""
Decode SUB E1 (EVENT_HEADER_RESPONSE) into an existing Event.
Decode SUB E1 (EVENT_HEADER_RESPONSE) raw data section into an Event.
The 6-byte timestamp is at the start of the data payload.
Sample rate location is not yet confirmed — left as None for now.
The waveform key is at data[11:15] (extracted separately in
MiniMateProtocol.read_event_first). The remaining 4 bytes at
data[15:19] are not yet decoded (❓ — possibly sample rate or flags).
Date information (year/month/day) lives in the waveform record (SUB 0C),
not in the 1E response. This function is a placeholder for any future
metadata we decode from the 8-byte 1E data block.
Modifies event in-place.
"""
if len(data) < 6:
log.warning("event header payload too short (%d bytes)", len(data))
return
try:
event.timestamp = Timestamp.from_bytes(data[:6])
except ValueError as exc:
log.warning("event header timestamp decode failed: %s", exc)
# Nothing confirmed yet from the 8-byte data block beyond the key at [0:4].
# Leave event.timestamp as None — it will be populated from the 0C record.
pass
def _decode_waveform_record_into(data: bytes, event: Event) -> None:
"""
Decode SUB F3 (FULL_WAVEFORM_RECORD) data into an existing Event.
Decode a 210-byte SUB F3 (FULL_WAVEFORM_RECORD) record into an Event.
Peak values are stored as IEEE 754 big-endian floats. Confirmed
positions per §7.5 (search for the known float bytes in the payload).
The *data* argument is the raw record bytes returned by
MiniMateProtocol.read_waveform_record() — i.e. data_rsp.data[11:11+0xD2].
This decoder is intentionally conservative — it searches for the
canonical 4×float32 pattern rather than relying on a fixed offset,
since the exact field layout is only partially confirmed.
Extracts:
- record_type: "Histogram" or "Waveform" (string search) 🔶
- peak_values: label-based float32 lookup (confirmed ✅)
- project_info: "Project:", "Client:", etc. string search ✅
Timestamp in the waveform record:
7-byte format: [0x09][year:2 BE][0x00][hour][minute][second]
Month and day come from a separate source (not yet fully mapped ❓).
For now we leave event.timestamp as None.
Modifies event in-place.
"""
# Attempt to extract four consecutive IEEE 754 BE floats from the
# known region of the payload (offsets are 🔶 INFERRED from captured data)
# ── Record type ───────────────────────────────────────────────────────────
try:
event.record_type = _extract_record_type(data)
except Exception as exc:
log.warning("waveform record type decode failed: %s", exc)
# ── Peak values ───────────────────────────────────────────────────────────
try:
peak_values = _extract_peak_floats(data)
if peak_values:
@@ -391,7 +413,7 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
except Exception as exc:
log.warning("waveform record peak decode failed: %s", exc)
# Project strings — search for known ASCII labels
# ── Project strings ───────────────────────────────────────────────────────
try:
project_info = _extract_project_strings(data)
if project_info:
@@ -400,41 +422,69 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
log.warning("waveform record project strings decode failed: %s", exc)
def _extract_record_type(data: bytes) -> Optional[str]:
"""
Search the waveform record for a record-type indicator string.
Confirmed types from 3-31-26 capture: "Histogram", "Waveform".
Returns the first match, or None if neither is found.
"""
for rtype in (b"Histogram", b"Waveform"):
if data.find(rtype) >= 0:
return rtype.decode()
return None
def _extract_peak_floats(data: bytes) -> Optional[PeakValues]:
"""
Scan the waveform record payload for four sequential float32 BE values
corresponding to Tran, Vert, Long, MicL peak values.
Locate per-channel peak particle velocity values in the 210-byte
waveform record by searching for the embedded channel label strings
("Tran", "Vert", "Long", "MicL") and reading the IEEE 754 BE float
at label_offset + 6.
The exact offset is not confirmed (🔶), so we do a heuristic scan:
look for four consecutive 4-byte groups where each decodes as a
plausible PPV value (0 < v < 100 in/s or psi).
The floats are NOT 4-byte aligned in the record (confirmed from
3-31-26 capture), so the previous step-4 scan missed Tran, Long, and
MicL entirely. Label-based lookup is the correct approach.
Returns PeakValues if a plausible group is found, else None.
Channel labels are separated by inner-frame bytes (0x10 0x03 = DLE ETX),
which the S3FrameParser preserves as literal data. Searching for the
4-byte ASCII label strings is robust to this structure.
Returns PeakValues if at least one channel label is found, else None.
"""
# Require at least 16 bytes for 4 floats
if len(data) < 16:
return None
# (label_bytes, field_name)
channels = (
(b"Tran", "tran"),
(b"Vert", "vert"),
(b"Long", "long_"),
(b"MicL", "micl"),
)
vals: dict[str, float] = {}
for start in range(0, len(data) - 15, 4):
for label_bytes, field in channels:
pos = data.find(label_bytes)
if pos < 0:
continue
float_off = pos + 6
if float_off + 4 > len(data):
log.debug("peak float: label %s at %d but float runs past end", label_bytes, pos)
continue
try:
vals = struct.unpack_from(">4f", data, start)
val = struct.unpack_from(">f", data, float_off)[0]
except struct.error:
continue
log.debug("peak float: %s at label+6 (%d) = %.6f", label_bytes.decode(), float_off, val)
vals[field] = val
# All four values should be non-negative and within plausible PPV range
if all(0.0 <= v < 100.0 for v in vals):
tran, vert, long_, micl = vals
# MicL (psi) is typically much smaller than geo values
# Simple sanity: at least two non-zero values
if sum(v > 0 for v in vals) >= 2:
log.debug(
"peak floats at offset %d: T=%.4f V=%.4f L=%.4f M=%.6f",
start, tran, vert, long_, micl
)
return PeakValues(
tran=tran, vert=vert, long=long_, micl=micl
)
return None
if not vals:
return None
return PeakValues(
tran=vals.get("tran"),
vert=vals.get("vert"),
long=vals.get("long_"),
micl=vals.get("micl"),
)
def _extract_project_strings(data: bytes) -> Optional[ProjectInfo]: