v0.14.3 - Full waveform DL pipeline tested and working. #15

Merged
serversdown merged 12 commits from protocol-fix into main 2026-05-05 20:49:48 -04:00
7 changed files with 409 additions and 278 deletions
Showing only changes of commit 45e61fbcaf - Show all commits
+31
View File
@@ -4,6 +4,34 @@ All notable changes to seismo-relay are documented here.
--- ---
## v0.13.2 — 2026-05-01
### Fixed
- **`_extract_record_type` — third 0C-record header format ("short", 8 bytes).**
A live SFM download against BE11529 produced files named `M5290000.000`
(zero-stamped) because the 0C waveform record's first bytes were
`01 05 07 ea ...` — neither the 9-byte single-shot layout (`0x10` at byte 1)
nor the 10-byte continuous layout (`0x10` at bytes 0 and 2). Investigation
showed this is a third format observed in the wild: an 8-byte header with no
marker bytes at all (`[day][month][year_BE:2][unknown][hour][min][sec]`).
The detection logic now scans the year (uint16 BE) at byte 2 / byte 3 / byte
4 and picks whichever offset returns a sensible year (20152050) — each
format has the year at a unique position so this disambiguates cleanly.
- New format → `event.record_type = "Waveform (Short)"`,
`Timestamp.from_short_record()`.
- Existing single-shot and continuous parsers unchanged.
- The user's event from May 1, 2026 13:21:37 now correctly resolves to a
filename like `M529LKIQ.G10` instead of `M5290000.000`.
### Added
- `Timestamp.from_short_record(data)` — decodes the 8-byte header.
- `_detect_record_format(data)` — internal helper returning
`"single_shot" / "continuous" / "short" / None` via year-position scan.
---
## v0.13.1 — 2026-05-01 ## v0.13.1 — 2026-05-01
### Fixed ### Fixed
@@ -23,6 +51,9 @@ All notable changes to seismo-relay are documented here.
it's the 10-byte continuous header; else if byte 1 is `0x10`, it's the it's the 10-byte continuous header; else if byte 1 is `0x10`, it's the
9-byte single-shot header. Day-of-month no longer matters. 9-byte single-shot header. Day-of-month no longer matters.
*Superseded by v0.13.2 — the user's actual record uses a third 8-byte format
with no `0x10` markers, which v0.13.1 still misclassified.*
--- ---
## v0.13.0 — 2026-05-01 ## v0.13.0 — 2026-05-01
+1 -1
View File
@@ -2,7 +2,7 @@
Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for
managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem
(Sierra Wireless RV50 / RV55). Current version: **v0.13.1**. (Sierra Wireless RV50 / RV55). Current version: **v0.13.2**.
When new information about the protocol is discovered, please update the instantel_protocol_reference.md with the findings in addition to this document When new information about the protocol is discovered, please update the instantel_protocol_reference.md with the findings in addition to this document
+47 -54
View File
@@ -672,11 +672,13 @@ def write_blastware_file(
# Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a # Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a
# subsequent event (a known get_events side-effect), the last frame will # subsequent event (a known get_events side-effect), the last frame will
# not be the terminator and the footer will be mis-identified. # not be the terminator and the footer will be mis-identified.
# TERM detection (v0.14.0):
# Legacy walk: TERM has page_key == 0x0000.
# BW-exact walk: TERM has page_key != 0x0010 (e.g. 0x0001).
# The TERM is always the LAST frame in the list (include_terminator=True).
term_idx: Optional[int] = None term_idx: Optional[int] = None
for _i, _f in enumerate(a5_frames): if a5_frames and a5_frames[-1].page_key != 0x0010:
if _f.page_key == 0x0000: term_idx = len(a5_frames) - 1
term_idx = _i
break
if term_idx is not None: if term_idx is not None:
body_frames = a5_frames[:term_idx] body_frames = a5_frames[:term_idx]
@@ -685,64 +687,33 @@ def write_blastware_file(
body_frames = a5_frames body_frames = a5_frames
term_frame = None term_frame = None
# ── Identify first metadata frame and skip "extra chunks" ─────────────── # Frame contribution loop (v0.14.0 BW-exact walk).
# When extra_chunks_after_metadata=1 in read_bulk_waveform_stream(), the # Frame structure:
# frame list is: [probe, data..., metadata, extra_chunk, terminator]. # Event 1: [probe] [meta@0x1002] [meta@0x1004] [samples ...] [TERM-not-in-body]
# The extra_chunk is downloaded to prime the TCP terminator response — its # Event N: [probe@start+0x46] [samples ...] [TERM-not-in-body]
# ADC data is NOT part of the Blastware file body. Skip it.
# #
# Rule: any frame at index strictly between first_metadata_fi and last_fi # Skip values per frame (confirmed from byte-diff vs BW-saved file
# (the final frame) is an extra chunk and must be excluded. # M529LKIQ.G10):
# # probe (fi=0): probe_skip (depends on STRT position)
# If no metadata frame exists (e.g. full_waveform download), first_metadata_fi # meta@0x1002 (fi=1): 13 (6-byte inner header)
# is None and no frames are skipped — all frames contribute normally. # meta@0x1004 (fi=2): 13 (6-byte inner header)
first_metadata_fi: Optional[int] = None # sample chunks (fi=3+): 12 (5-byte inner header)
for _fi_scan, _frame_scan in enumerate(body_frames):
if _fi_scan > 0 and any(m in bytes(_frame_scan.data) for m in _METADATA_FRAME_MARKERS):
first_metadata_fi = _fi_scan
break
last_fi = len(body_frames) - 1 last_fi = len(body_frames) - 1
log.warning( log.debug(
"write_blastware_file: %d body_frames first_metadata_fi=%s last_fi=%d", "write_blastware_file: %d body_frames last_fi=%d",
len(body_frames), len(body_frames), last_fi,
str(first_metadata_fi) if first_metadata_fi is not None else "None",
last_fi,
) )
all_bytes = bytearray() all_bytes = bytearray()
for fi, frame in enumerate(body_frames): for fi, frame in enumerate(body_frames):
# Skip "extra chunk" frames: frames after the first metadata frame but
# before the last frame (terminator). These prime the TCP terminator but
# their ADC data must NOT appear in the Blastware file body.
if (first_metadata_fi is not None
and fi > first_metadata_fi
and fi < last_fi):
log.warning(
"write_blastware_file: fi=%d SKIP (extra chunk after metadata fi=%d last_fi=%d)",
fi, first_metadata_fi, last_fi,
)
continue
if fi == 0: if fi == 0:
# Probe frame: always process regardless of classification.
# It holds the STRT record; probe_skip positions us past it.
skip = probe_skip skip = probe_skip
elif fi in (1, 2):
skip = 13 # metadata pages
else: else:
# ALL subsequent frames are included unconditionally — no filtering on skip = 12 # sample chunks
# frame type. In the A5 stream, frame 0 is always the probe response;
# frames 1+ are always data (waveform chunks, compliance config, or
# compliance continuation). Classification is for logging only.
#
# DO NOT gate on classify_frame() here:
# - "probe_or_strt" at fi>0 is always a false positive — ADC binary
# data can coincidentally contain b"STRT\xff\xfe" (confirmed from
# live capture: frames 1 and 5 matched on event key=01110000).
# - "metadata" frames must be included (compliance config body).
# - The compliance block spans 2 frames; skipping either produces a
# truncated file that Blastware rejects.
skip = 13 if fi == 1 else 12
contribution = _frame_body_bytes(frame, skip) contribution = _frame_body_bytes(frame, skip)
log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
@@ -769,11 +740,33 @@ def write_blastware_file(
bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(), bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(),
) )
if len(all_bytes) >= 26: # Find the first valid 0e 08 footer marker (v0.14.0). The device's
# TERM response contains the real Blastware footer; older walks
# accidentally fetched data past the footer. Validate by checking the
# year field (uint16 BE at offset+4) is in 2015..2050.
footer_pos = -1
pos = 0
while True:
pos = bytes(all_bytes).find(b"\x0e\x08", pos)
if pos < 0 or pos + 26 > len(all_bytes):
break
yr = (all_bytes[pos + 4] << 8) | all_bytes[pos + 5]
if 2015 <= yr <= 2050:
footer_pos = pos
break
pos += 1
if footer_pos >= 0:
body = bytes(all_bytes[:footer_pos])
footer = bytes(all_bytes[footer_pos:footer_pos + 26])
log.warning(
"write_blastware_file: real 0e 08 footer at all_bytes[%d]; "
"truncating %d post-footer bytes",
footer_pos, len(all_bytes) - footer_pos - 26,
)
elif len(all_bytes) >= 26:
body = bytes(all_bytes[:-26]) body = bytes(all_bytes[:-26])
footer = bytes(all_bytes[-26:]) footer = bytes(all_bytes[-26:])
else: else:
# Fallback: no terminator or very short stream → build footer from event metadata
body = bytes(all_bytes) body = bytes(all_bytes)
start_dt = _ts_from_model(event.timestamp) start_dt = _ts_from_model(event.timestamp)
stop_dt: Optional[datetime.datetime] = None stop_dt: Optional[datetime.datetime] = None
@@ -784,7 +777,7 @@ def write_blastware_file(
+ _encode_ts_be(start_dt) + _encode_ts_be(start_dt)
+ _encode_ts_be(stop_dt) + _encode_ts_be(stop_dt)
+ b"\x00\x01\x00\x02\x00\x00" + b"\x00\x01\x00\x02\x00\x00"
+ b"\x00\x00" # CRC placeholder + b"\x00\x00"
) )
# ── Write file ─────────────────────────────────────────────────────────── # ── Write file ───────────────────────────────────────────────────────────
+62 -35
View File
@@ -1345,6 +1345,11 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None:
event.timestamp = Timestamp.from_continuous_record(data) event.timestamp = Timestamp.from_continuous_record(data)
except Exception as exc: except Exception as exc:
log.warning("continuous record timestamp decode failed: %s", exc) log.warning("continuous record timestamp decode failed: %s", exc)
elif event.record_type == "Waveform (Short)":
try:
event.timestamp = Timestamp.from_short_record(data)
except Exception as exc:
log.warning("short record timestamp decode failed: %s", exc)
# ── Peak values (per-channel PPV + Peak Vector Sum) ─────────────────────── # ── Peak values (per-channel PPV + Peak Vector Sum) ───────────────────────
try: try:
@@ -1636,51 +1641,73 @@ def _decode_a5_waveform(
} }
def _detect_record_format(data: bytes) -> Optional[str]:
"""
Detect which timestamp-header format a 210-byte 0C waveform record uses.
THREE formats observed on BE11529 firmware S338.17:
"single_shot" — 9-byte header:
[day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec]
sub_code=0x10 at byte [1]. Year at [3:5].
"continuous" — 10-byte header:
[0x10] [day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec]
marker 0x10 at byte [0] AND byte [2]. Year at [4:6].
"short" — 8-byte header (NEW 2026-05-01):
[day] [month] [year_BE:2] [unknown] [hour] [min] [sec]
No marker bytes. Year at [2:4].
Each format has the year (uint16 BE) at a UNIQUE byte position, so we can
disambiguate by scanning each candidate position and picking the one
where the year falls in a sane range (2015..2050).
Returns "single_shot" / "continuous" / "short" or None if no format matches.
"""
if len(data) < 8:
return None
def _sane_year(hi: int, lo: int) -> bool:
y = (hi << 8) | lo
return 2015 <= y <= 2050
# Order matters: prefer formats with stronger marker-byte evidence first.
if data[1] == 0x10 and len(data) >= 9 and _sane_year(data[3], data[4]):
return "single_shot"
if (data[0] == 0x10 and data[2] == 0x10
and len(data) >= 10 and _sane_year(data[4], data[5])):
return "continuous"
if _sane_year(data[2], data[3]):
return "short"
return None
def _extract_record_type(data: bytes) -> Optional[str]: def _extract_record_type(data: bytes) -> Optional[str]:
""" """
Detect the waveform record format by inspecting the first 3 bytes of the Return a human-readable name for the waveform record format detected
210-byte record returned by SUB 0C. in the first bytes of a 210-byte 0C record.
Two formats exist (confirmed from BE11529 captures and CLAUDE.md docs): Maps to the format codes returned by _detect_record_format():
"single_shot""Waveform"
Single-shot mode — 9-byte header: "continuous""Waveform (Continuous)"
data[0] = day "short""Waveform (Short)"
data[1] = 0x10 ← sub_code marker None → "Unknown(XX.YY.ZZ)"
data[2] = month
data[3:5] = year (BE)
...
Continuous mode — 10-byte header:
data[0] = 0x10 ← marker A
data[1] = day ← variable (NOT 0x10)
data[2] = 0x10 ← marker B
data[3] = month
data[4:6] = year (BE)
...
Disambiguate by checking BOTH data[0] and data[2]:
- data[0]==0x10 AND data[2]==0x10 → Continuous (10-byte header)
- data[1]==0x10 → Single-shot (9-byte header)
- otherwise → Unknown
Previous logic only checked data[1] and so mis-classified continuous-mode
records as "Unknown(0xXX)" wherever day != 0x10 — see filename
M5290000.000 regression report (2026-05-01 SFM log).
""" """
if len(data) < 3: fmt = _detect_record_format(data)
return None if fmt == "single_shot":
# 10-byte continuous format: 0x10 markers at byte 0 AND byte 2
if data[0] == 0x10 and data[2] == 0x10:
return "Waveform (Continuous)"
# 9-byte single-shot format: 0x10 sub_code marker at byte 1
if data[1] == 0x10:
return "Waveform" return "Waveform"
if fmt == "continuous":
return "Waveform (Continuous)"
if fmt == "short":
return "Waveform (Short)"
if len(data) >= 3:
log.warning( log.warning(
"_extract_record_type: unrecognized header: data[0:3]=%02X %02X %02X", "_extract_record_type: unrecognized header: data[0:3]=%02X %02X %02X",
data[0], data[1], data[2], data[0], data[1], data[2],
) )
return f"Unknown({data[0]:02X}.{data[1]:02X}.{data[2]:02X})" return f"Unknown({data[0]:02X}.{data[1]:02X}.{data[2]:02X})"
return None
def _extract_peak_floats(data: bytes) -> Optional[PeakValues]: def _extract_peak_floats(data: bytes) -> Optional[PeakValues]:
""" """
+52
View File
@@ -201,6 +201,58 @@ class Timestamp:
second=second, second=second,
) )
@classmethod
def from_short_record(cls, data: bytes) -> "Timestamp":
"""
Decode an 8-byte timestamp header from a 210-byte waveform record.
Wire layout (✅ CONFIRMED 2026-05-01 against live SFM run on BE11529 in
Continuous mode, day-of-month = 1 May, raw: 01 05 07 ea 00 0d 15 25):
byte[0]: day (uint8)
byte[1]: month (uint8)
bytes[2-3]: year (big-endian uint16)
byte[4]: unknown (0x00 in observed sample)
byte[5]: hour (uint8)
byte[6]: minute (uint8)
byte[7]: second (uint8)
This is a third format observed in the wild — distinct from the 9-byte
(single-shot, sub_code=0x10 at [1]) and 10-byte (continuous, 0x10 at
[0] AND [2]) layouts. No marker bytes; disambiguated by where the
year lands when scanned at byte 2/3/4.
Args:
data: at least 8 bytes; only the first 8 are consumed.
Returns:
Decoded Timestamp.
Raises:
ValueError: if data is fewer than 8 bytes.
"""
if len(data) < 8:
raise ValueError(
f"Short record timestamp requires at least 8 bytes, got {len(data)}"
)
day = data[0]
month = data[1]
year = struct.unpack_from(">H", data, 2)[0]
unknown_byte = data[4]
hour = data[5]
minute = data[6]
second = data[7]
return cls(
raw=bytes(data[:8]),
flag=0,
year=year,
unknown_byte=unknown_byte,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
)
@property @property
def clock_set(self) -> bool: def clock_set(self) -> bool:
"""False when year == 1995 (factory default / battery-lost state).""" """False when year == 1995 (factory default / battery-lost state)."""
+190 -160
View File
@@ -136,9 +136,10 @@ DATA_LENGTHS: dict[int, int] = {
# existing blastware_file.py builder relies on the 0x0400-step frame structure # existing blastware_file.py builder relies on the 0x0400-step frame structure
# to produce valid files. Switching to BW's 0x0200 step is a separate task # to produce valid files. Switching to BW's 0x0200 step is a separate task
# that also requires updating the file builder. # that also requires updating the file builder.
_BULK_CHUNK_OFFSET = 0x1004 # offset_word for probe + all chunk requests # BW-exact protocol values (v0.14.0). Verified against 4-27-26 + 5-1-26 captures.
_BULK_TERM_OFFSET = 0x005A # offset_word for the legacy terminator _BULK_CHUNK_OFFSET = 0x1002 # offset_word for probe + all chunk requests
_BULK_COUNTER_STEP = 0x0400 # chunk counter increment _BULK_TERM_OFFSET = 0x005A # offset_word for the legacy terminator (fallback only)
_BULK_COUNTER_STEP = 0x0200 # chunk counter increment (matches chunk payload size)
# Default timeout values (seconds). # Default timeout values (seconds).
# MiniMate Plus is a slow device — keep these generous. # MiniMate Plus is a slow device — keep these generous.
@@ -533,231 +534,260 @@ class MiniMateProtocol:
self, self,
key4: bytes, key4: bytes,
*, *,
stop_after_metadata: bool = True, stop_after_metadata: bool = True, # DEPRECATED — no-op under BW-exact walk
max_chunks: int = 32, max_chunks: int = 256, # safety cap only; loop is bounded by end_offset
include_terminator: bool = False, include_terminator: bool = False,
extra_chunks_after_metadata: int = 1, extra_chunks_after_metadata: int = 1, # DEPRECATED — no-op
) -> list[S3Frame]: ) -> list[S3Frame]:
""" """
Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event. Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event using
Blastware's exact protocol. REWRITTEN 2026-05-02 (v0.14.0).
The bulk waveform stream carries both raw ADC samples (large) and Algorithm (matches BW captures across 2-sec / 3-sec / event-2):
event-time metadata strings ("Project:", "Client:", "User Name:",
"Seis Loc:", "Extended Notes") embedded in one of the middle frames
(confirmed: A5[7] of 9 for 1-2-26 capture).
Protocol is request-per-chunk, NOT a continuous stream: 1. Probe
1. Probe (offset=_BULK_CHUNK_OFFSET, is_probe=True, counter=0x0000) - For events at start_key[2:4] = 0x0000 (first event after erase
2. Chunks (offset=_BULK_CHUNK_OFFSET, is_probe=False, counter+=0x0400) / wrap): probe at counter=0x0000 with full key in params.
3. Loop until metadata found (stop_after_metadata=True) or max_chunks - For continuation events (start_key[2:4] != 0): first chunk at
4. Termination (offset=_BULK_TERM_OFFSET, counter=last+_BULK_COUNTER_STEP) counter = start_key[2:4] + 0x0046; acts as both probe and
Device responds with a final A5 frame (page_key=0x0000). first sample chunk; response carries STRT.
By default the termination frame (page_key=0x0000) is NOT included in the 2. Parse end_offset from STRT record at data[23:27] of the probe response.
returned list. Pass include_terminator=True to append it; the blastware_file
writer needs the terminator frame's body to reconstruct the waveform file footer.
Args: 3. Read two fixed metadata pages at counter=0x1002 and counter=0x1004
key4: 4-byte waveform key from EVENT_HEADER (1E). — global session metadata (Project / Client / User Name / Seis Loc
stop_after_metadata: If True (default), send termination as soon as / Extended Notes ASCII strings). Event 1 only; continuation
b"Project:" is found in a frame's data — avoids events skip these (BW caches them across the session).
downloading the full ADC waveform payload (several
hundred KB). Set False to download everything. 4. Walk sample chunks at 0x0200 increments, starting from 0x0600 for
max_chunks: Safety cap on the number of chunk requests sent event 1 or `start + 0x0046 + 0x0200` for continuation events.
(default 32; a typical event uses 9 large frames). Stop when `next_chunk + 0x0200 > end_offset`.
include_terminator: If True, append the terminator A5 frame
(page_key=0x0000) to the returned list. The 5. Send TERM frame with offset_word and params computed by
terminator carries the waveform file footer bytes. `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`.
Default False preserves existing caller behaviour. The TERM response contains the partial last chunk (residual =
end_offset - next_boundary) including the 26-byte 0e 08 file
footer.
Returns: Returns:
List of S3Frame objects from each A5 response frame. Frame indices List of S3Frame objects from each A5 response (probe, metadata
match the request sequence: index 0 = probe response, index 1 = first pages, sample chunks, optional TERM response). Caller passes
chunk, etc. If include_terminator=True, the last element is the `include_terminator=True` (e.g. write_blastware_file) to keep the
terminator frame (page_key=0x0000). TERM response in the list — it's required to reconstruct the
file footer.
Deprecated kwargs:
stop_after_metadata: legacy "Project:"-string-based stop condition.
No-op under the BW-exact walk; the loop is
deterministically bounded by end_offset from
STRT. Accepted for backward compat.
extra_chunks_after_metadata: same.
Raises: Raises:
ProtocolError: on timeout, bad checksum, or unexpected SUB. ProtocolError: on timeout / bad checksum / unexpected SUB.
Confirmed from 1-2-26 BW TX/RX captures (2026-04-02):
- probe + 8 regular chunks + 1 termination = 10 TX frames
- 9 large A5 responses + 1 terminator A5 = 10 RX frames
- page_key=0x0010 on large frames; page_key=0x0000 on terminator ✅
- "Project:" metadata at A5[7].data[626] ✅
""" """
if len(key4) != 4: if len(key4) != 4:
raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}") raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM) # 0xFF - 0x5A = 0xA5 # Quietly accept and warn on deprecated kwargs.
if not stop_after_metadata:
log.debug("5A: stop_after_metadata=False is no-op under BW-exact walk")
if extra_chunks_after_metadata not in (0, 1):
log.debug("5A: extra_chunks_after_metadata=%d is no-op under BW-exact walk",
extra_chunks_after_metadata)
rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM) # 0xA5
frames_data: list[S3Frame] = [] frames_data: list[S3Frame] = []
counter = 0
# BW counter formula (confirmed from 4-3-26 capture for key 0111245a, start_offset = (key4[2] << 8) | key4[3]
# and empirical live-device test 2026-04-06 for key 01110000): is_event_1 = (start_offset == 0)
# counter for chunk n = max(key4[2:4], 0x0400) + (n - 1) * 0x0400
# key4[2:4] is the event's circular-buffer base offset. The max() guard
# ensures chunk 1 never uses counter=0x0000 (which equals the probe address
# and causes the device to re-return STRT record data for the first chunk).
_key4_offset = (key4[2] << 8) | key4[3]
# ── Step 1: probe ──────────────────────────────────────────────────── # ── Step 1: probe / first chunk ──────────────────────────────────────
log.debug("5A probe key=%s key4_offset=0x%04X", key4.hex(), _key4_offset) if is_event_1:
params = bulk_waveform_params(key4, 0, is_probe=True) probe_counter = 0
self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) probe_params = bulk_waveform_params(key4, 0, is_probe=True)
self._parser.reset() # reset bytes_fed counter before probe recv log.debug("5A probe (event-1) key=%s counter=0x0000", key4.hex())
else:
# Continuation events: first 5A request lands at start+0x0046,
# acting as both probe and first sample chunk. Confirmed from
# 5-1-26 "copy 2nd address event" capture.
probe_counter = start_offset + 0x0046
probe_params = bulk_waveform_params(key4, probe_counter)
log.debug(
"5A probe (event-N) key=%s counter=0x%04X (start+0x46)",
key4.hex(), probe_counter,
)
self._send(build_5a_frame(_BULK_CHUNK_OFFSET, probe_params))
self._parser.reset()
try: try:
rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False) rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False)
except TimeoutError: except TimeoutError:
log.warning( log.warning(
"5A probe TIMED OUT for key=%s" "5A probe TIMED OUT for key=%s%d raw bytes received",
"%d raw bytes received (no complete A5 frame assembled)",
key4.hex(), self._parser.bytes_fed, key4.hex(), self._parser.bytes_fed,
) )
raise raise
frames_data.append(rsp)
log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data))
# ── Parse STRT end_offset from probe response (NEW 2026-05-01) ──────── frames_data.append(rsp)
# The first A5 response contains a STRT record at data[17:]. The log.debug("5A A5[0] (probe) page_key=0x%04X %d bytes",
# bytes at data[23:27] are the event's end-key, whose low 16 bits rsp.page_key, len(rsp.data))
# are the absolute device-buffer address where the event ends. Use
# this to bound the chunk loop and stop the over-read past event end. # ── Step 2: parse STRT end_offset from probe response ────────────────
# See docs/instantel_protocol_reference.md §7.8.5 and CLAUDE.md end_offset = parse_strt_end_offset(rsp.data)
# "SUB 5A — STRT record encodes end_offset". if end_offset is None:
_end_offset = parse_strt_end_offset(rsp.data) log.warning(
if _end_offset is None: "5A probe response did not contain a STRT record; "
# Defensive fallback — let max_chunks cap the walk. "cannot bound chunk loop — falling back to max_chunks=%d cap",
log.warning("5A: STRT not found in probe; cannot bound chunk loop") max_chunks,
_end_offset = 0xFFFF )
end_offset = 0xFFFF # impossible value → loop runs to max_chunks
else: else:
log.debug( log.info(
"5A STRT start_offset=0x%04X end_offset=0x%04X size=0x%04X", "5A STRT start_offset=0x%04X end_offset=0x%04X size=0x%04X",
_key4_offset, _end_offset, _end_offset - _key4_offset, start_offset, end_offset, end_offset - start_offset,
) )
# ── Step 2: chunk loop ─────────────────────────────────────────────── # ── Step 3: metadata pages 0x1002 + 0x1004 (event 1 only) ────────────
# Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 # Confirmed from BW captures: BW reads these two fixed device-buffer
# where _chunk_base = max(key4[2:4], 0x0400). # pages immediately after the probe for events at start_key[2:4]=0.
# # Continuation events skip them (BW caches across the session).
# For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a): # Their content is global compliance-setup metadata: Project, Client,
# _chunk_base = 0x245a → chunk 1=0x245a, chunk 2=0x285a, ... # User Name, Seis Loc, Extended Notes.
# Confirmed from 4-3-26 capture. if is_event_1:
# for meta_counter in (0x1002, 0x1004):
# For events with key4[2:4] == 0 (e.g. key 01110000): # Metadata page params have an extra trailing 0x00 byte
# _chunk_base = max(0, 0x0400) = 0x0400 # (12-byte params instead of 11) — empirical from BW captures.
# → chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400) # Checksum-neutral but matches BW byte-for-byte.
# CRITICAL: counter=0x0000 (same as the probe) causes the device to meta_params = bytes([
# re-return the STRT record data for chunk 1, making frame 1 look like 0x00,
# a second probe response (confirmed from server log: frame 1 len=1097, key4[0], key4[1],
# contains STRT\xff\xfe, contributes zero body bytes after DLE-strip). (meta_counter >> 8) & 0xFF,
# counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06). meta_counter & 0xFF,
_chunk_base = max(_key4_offset, _BULK_COUNTER_STEP) 0, 0, 0, 0, 0, 0, 0,
for chunk_num in range(1, max_chunks + 1): ])
counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP log.debug("5A metadata page counter=0x%04X", meta_counter)
# Stop when we'd step past the event end (NEW 2026-05-01). Without self._send(build_5a_frame(_BULK_CHUNK_OFFSET, meta_params))
# this, the device returns post-event circular-buffer data which self._parser.reset()
# corrupts the reconstructed file for events ≥ 2 sec. try:
if counter >= _end_offset: meta_rsp = self._recv_one(
expected_sub=rsp_sub, reset_parser=False, timeout=10.0,
)
except TimeoutError:
log.warning(
"5A metadata page 0x%04X TIMED OUT — continuing",
meta_counter,
)
continue
frames_data.append(meta_rsp)
log.debug(
"5A meta@0x%04X page_key=0x%04X %d bytes",
meta_counter, meta_rsp.page_key, len(meta_rsp.data),
)
# ── Step 4: sample chunk loop, bounded by end_offset ─────────────────
# Sample chunks start at:
# event 1: counter = 0x0600
# event N (>0): counter = probe_counter + 0x0200
# (probe was the first sample chunk)
if is_event_1:
counter = 0x0600
else:
counter = probe_counter + _BULK_COUNTER_STEP
last_chunk_counter: Optional[int] = (
probe_counter if not is_event_1 else None
)
chunks_fetched = 0
while chunks_fetched < max_chunks:
# Stop when next chunk would straddle the event end.
if counter + _BULK_COUNTER_STEP > end_offset:
log.debug( log.debug(
"5A chunk loop done at counter=0x%04X (end=0x%04X); " "5A chunk loop done at counter=0x%04X (end=0x%04X); "
"%d chunks fetched", "%d chunks fetched",
counter, _end_offset, len(frames_data), counter, end_offset, chunks_fetched,
) )
break break
params = bulk_waveform_params(key4, counter) params = bulk_waveform_params(key4, counter)
log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) log.debug("5A chunk #%d counter=0x%04X", chunks_fetched + 1, counter)
self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
self._parser.reset() # reset bytes_fed for accurate per-chunk count self._parser.reset()
try: try:
rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False, timeout=10.0) rsp = self._recv_one(
expected_sub=rsp_sub, reset_parser=False, timeout=10.0,
)
except TimeoutError: except TimeoutError:
raw = self._parser.bytes_fed raw = self._parser.bytes_fed
log.warning( log.warning(
"5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d", "5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d",
chunk_num, counter, raw, chunks_fetched + 1, counter, raw,
) )
if raw > 0 and frames_data: if raw > 0 and frames_data:
# Device sent a partial byte (likely a bare DLE/ETX end-of-stream
# signal) but never completed a full frame. Treat as graceful
# stream end and fall through to the termination step.
log.warning( log.warning(
"5A end-of-stream detected at chunk=%d (raw_bytes=%d, " "5A unexpected end-of-stream — proceeding to TERM",
"frames_collected=%d) — proceeding to termination",
chunk_num, raw, len(frames_data),
) )
break break
raise raise
log.warning( log.debug(
"5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", "5A RX chunk=%d page_key=0x%04X data_len=%d",
chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, chunks_fetched + 1, rsp.page_key, len(rsp.data),
) )
if rsp.page_key == 0x0000: if rsp.page_key == 0x0000:
# Device unexpectedly terminated mid-stream (no termination needed). # Device terminated mid-stream unexpectedly.
log.debug("5A A5[%d] page_key=0x0000 — device terminated early", chunk_num) log.warning(
"5A unexpected page_key=0x0000 mid-stream at counter=0x%04X",
counter,
)
if include_terminator: if include_terminator:
frames_data.append(rsp) frames_data.append(rsp)
return frames_data return frames_data
frames_data.append(rsp) frames_data.append(rsp)
last_chunk_counter = counter
if stop_after_metadata and b"Project:" in rsp.data: counter += _BULK_COUNTER_STEP
# Download exactly one more chunk after finding metadata — this is chunks_fetched += 1
# what Blastware does. The extra chunk contains the tail ADC data
# and primes the device to return a valid footer in the termination
# response. Without it, termination returns an empty ack with no
# footer bytes (confirmed 2026-04-23 from HxD comparison).
# Download extra_chunks_after_metadata more chunks past the
# metadata. The caller calculates this from record_time and
# sample_rate so we download exactly the right amount of ADC
# data — no more, no less — before terminating.
# The device returns the footer in the termination response only
# after the right amount of data has been consumed.
log.debug("5A A5[%d] metadata found — fetching %d more chunk(s)",
chunk_num, extra_chunks_after_metadata)
for _extra_n in range(extra_chunks_after_metadata):
chunk_num += 1
counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP
params = bulk_waveform_params(key4, counter)
self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
try:
extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0)
log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d",
chunk_num, extra.page_key, len(extra.data))
if extra.page_key == 0x0000:
if include_terminator:
frames_data.append(extra)
return frames_data
frames_data.append(extra)
except TimeoutError:
log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1)
break
break
else: else:
log.warning( log.warning(
"5A reached max_chunks=%d without end-of-stream; sending termination", "5A reached max_chunks=%d at counter=0x%04X (end=0x%04X)",
max_chunks, max_chunks, counter, end_offset,
) )
# ── Step 3: termination ────────────────────────────────────────────── # ── Step 5: TERM with proper end_offset-derived formula ──────────────
term_counter = counter + _BULK_COUNTER_STEP if last_chunk_counter is None or end_offset == 0xFFFF:
term_params = bulk_waveform_term_params(key4, term_counter) # No STRT or no chunks fetched — fall back to legacy TERM.
log.debug( log.warning(
"5A termination term_counter=0x%04X offset=0x%04X", "5A using legacy TERM (offset_word=0x005A); "
term_counter, _BULK_TERM_OFFSET, "end_offset unavailable or no chunks fetched",
)
legacy_counter = (last_chunk_counter or probe_counter) + _BULK_COUNTER_STEP
term_offset_word = _BULK_TERM_OFFSET # 0x005A
term_params = bulk_waveform_term_params(key4, legacy_counter)
else:
term_offset_word, term_params = bulk_waveform_term_v2(
key4, end_offset, last_chunk_counter,
) )
self._send(build_5a_frame(_BULK_TERM_OFFSET, term_params))
try:
term_rsp = self._recv_one(expected_sub=rsp_sub)
log.debug( log.debug(
"5A termination response page_key=0x%04X %d bytes", "5A TERM offset_word=0x%04X params[2:4]=%s end=0x%04X "
"last_chunk=0x%04X",
term_offset_word, term_params[2:4].hex(),
end_offset, last_chunk_counter,
)
self._send(build_5a_frame(term_offset_word, term_params))
try:
term_rsp = self._recv_one(expected_sub=rsp_sub, timeout=10.0)
log.info(
"5A TERM response page_key=0x%04X %d bytes",
term_rsp.page_key, len(term_rsp.data), term_rsp.page_key, len(term_rsp.data),
) )
if include_terminator: if include_terminator:
frames_data.append(term_rsp) frames_data.append(term_rsp)
except TimeoutError: except TimeoutError:
log.debug("5A no termination response — device may have already closed") log.warning("5A no TERM response (timeout)")
return frames_data return frames_data
+19 -21
View File
@@ -37,6 +37,7 @@ from __future__ import annotations
import datetime import datetime
import logging import logging
import sys import sys
import tempfile
import threading import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -863,8 +864,8 @@ def device_event_blastware_file(
Supply either *port* (serial) or *host* (TCP/modem). Supply either *port* (serial) or *host* (TCP/modem).
The file is written to /tmp and streamed back as a binary download. The file is written to the OS temp directory and streamed back as a binary
Blastware can open it directly — filename encodes serial + timestamp. download. Blastware can open it directly — filename encodes serial + timestamp.
Filename format: <prefix><serial3><stem><AB>0<W|H> Filename format: <prefix><serial3><stem><AB>0<W|H>
- prefix letter = chr(ord('B') + floor(serial_numeric / 1000)) - prefix letter = chr(ord('B') + floor(serial_numeric / 1000))
@@ -885,26 +886,13 @@ def device_event_blastware_file(
def _do(): def _do():
with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: with _build_client(port, baud, host, tcp_port, timeout=120.0) as client:
info = client.connect() info = client.connect()
# Use stop_after_metadata=True (full_waveform=False) with 1 extra # Under v0.14.0 BW-exact 5A walk, the chunk loop is bounded by
# chunk after "Project:". The extra chunk is required to prime the # the event end_offset extracted from STRT. No more
# device over TCP: termination at term_counter=metadata_counter+0x0400 # stop_after_metadata / extra_chunks gymnastics — these
# returns only ~90 bytes (no useful footer) over TCP/cellular, but # kwargs are now no-ops.
# termination at metadata_counter+0x0800 (one chunk later) returns
# the full 737-byte frame containing the footer.
#
# Confirmed from 4-26-26 BW RS-232 capture: BW terminates at 0x1800
# without an extra chunk (works on RS-232 but not TCP).
# write_blastware_file() automatically skips the extra chunk's
# contribution — only the probe+ADC+metadata+terminator bytes appear
# in the output file.
#
# full_waveform=True (natural end-of-stream) downloads ALL chunks
# including post-event silence (35+ chunks for a 9-sec event at
# 1024 sps) — this produces 24KB+ files that Blastware rejects.
events = client.get_events( events = client.get_events(
full_waveform=False, full_waveform=False,
stop_after_index=index, stop_after_index=index,
extra_chunks_after_metadata=1,
) )
matching = [ev for ev in events if ev.index == index] matching = [ev for ev in events if ev.index == index]
return matching[0] if matching else None, info return matching[0] if matching else None, info
@@ -940,8 +928,18 @@ def device_event_blastware_file(
# Build filename using the same algorithm Blastware uses # Build filename using the same algorithm Blastware uses
filename = blastware_filename(ev, serial) filename = blastware_filename(ev, serial)
# Write to /tmp so FastAPI can stream it back # Write to OS temp dir (cross-platform: /tmp on Linux/macOS,
out_path = Path("/tmp") / filename # %TEMP% on Windows) so FastAPI can stream it back via FileResponse.
out_path = Path(tempfile.gettempdir()) / filename
# Delete any stale file at this path before writing. On Windows we have
# observed the new (smaller) file getting trailing zero-bytes from the
# previous (larger) file when filesystem semantics around open(...,"wb")
# don't truncate cleanly (e.g. through a synced folder). Explicit unlink
# eliminates that ambiguity.
try:
out_path.unlink()
except FileNotFoundError:
pass
write_blastware_file(ev, a5_frames, out_path) write_blastware_file(ev, a5_frames, out_path)
log.info( log.info(
"blastware_file: wrote %s (%d A5 frames, serial=%s)", "blastware_file: wrote %s (%d A5 frames, serial=%s)",