diff --git a/CHANGELOG.md b/CHANGELOG.md index a72537f..e98b2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to seismo-relay are documented here. --- +## v0.13.2 — 2026-05-01 + +### Fixed + +- **`_extract_record_type` — third 0C-record header format ("short", 8 bytes).** + A live SFM download against BE11529 produced files named `M5290000.000` + (zero-stamped) because the 0C waveform record's first bytes were + `01 05 07 ea ...` — neither the 9-byte single-shot layout (`0x10` at byte 1) + nor the 10-byte continuous layout (`0x10` at bytes 0 and 2). Investigation + showed this is a third format observed in the wild: an 8-byte header with no + marker bytes at all (`[day][month][year_BE:2][unknown][hour][min][sec]`). + The detection logic now scans the year (uint16 BE) at byte 2 / byte 3 / byte + 4 and picks whichever offset returns a sensible year (2015–2050) — each + format has the year at a unique position so this disambiguates cleanly. + - New format → `event.record_type = "Waveform (Short)"`, + `Timestamp.from_short_record()`. + - Existing single-shot and continuous parsers unchanged. + - The user's event from May 1, 2026 13:21:37 now correctly resolves to a + filename like `M529LKIQ.G10` instead of `M5290000.000`. + +### Added + +- `Timestamp.from_short_record(data)` — decodes the 8-byte header. +- `_detect_record_format(data)` — internal helper returning + `"single_shot" / "continuous" / "short" / None` via year-position scan. + +--- + ## v0.13.1 — 2026-05-01 ### Fixed @@ -23,6 +51,9 @@ All notable changes to seismo-relay are documented here. it's the 10-byte continuous header; else if byte 1 is `0x10`, it's the 9-byte single-shot header. Day-of-month no longer matters. + *Superseded by v0.13.2 — the user's actual record uses a third 8-byte format + with no `0x10` markers, which v0.13.1 still misclassified.* + --- ## v0.13.0 — 2026-05-01 diff --git a/CLAUDE.md b/CLAUDE.md index f908461..f06d849 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.13.1**. +(Sierra Wireless RV50 / RV55). Current version: **v0.13.2**. When new information about the protocol is discovered, please update the instantel_protocol_reference.md with the findings in addition to this document diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index fedf229..774f2b1 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -672,11 +672,13 @@ def write_blastware_file( # Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a # subsequent event (a known get_events side-effect), the last frame will # not be the terminator and the footer will be mis-identified. + # TERM detection (v0.14.0): + # Legacy walk: TERM has page_key == 0x0000. + # BW-exact walk: TERM has page_key != 0x0010 (e.g. 0x0001). + # The TERM is always the LAST frame in the list (include_terminator=True). term_idx: Optional[int] = None - for _i, _f in enumerate(a5_frames): - if _f.page_key == 0x0000: - term_idx = _i - break + if a5_frames and a5_frames[-1].page_key != 0x0010: + term_idx = len(a5_frames) - 1 if term_idx is not None: body_frames = a5_frames[:term_idx] @@ -685,64 +687,33 @@ def write_blastware_file( body_frames = a5_frames term_frame = None - # ── Identify first metadata frame and skip "extra chunks" ─────────────── - # When extra_chunks_after_metadata=1 in read_bulk_waveform_stream(), the - # frame list is: [probe, data..., metadata, extra_chunk, terminator]. - # The extra_chunk is downloaded to prime the TCP terminator response — its - # ADC data is NOT part of the Blastware file body. Skip it. + # Frame contribution loop (v0.14.0 BW-exact walk). + # Frame structure: + # Event 1: [probe] [meta@0x1002] [meta@0x1004] [samples ...] [TERM-not-in-body] + # Event N: [probe@start+0x46] [samples ...] [TERM-not-in-body] # - # Rule: any frame at index strictly between first_metadata_fi and last_fi - # (the final frame) is an extra chunk and must be excluded. - # - # If no metadata frame exists (e.g. full_waveform download), first_metadata_fi - # is None and no frames are skipped — all frames contribute normally. - first_metadata_fi: Optional[int] = None - for _fi_scan, _frame_scan in enumerate(body_frames): - if _fi_scan > 0 and any(m in bytes(_frame_scan.data) for m in _METADATA_FRAME_MARKERS): - first_metadata_fi = _fi_scan - break + # Skip values per frame (confirmed from byte-diff vs BW-saved file + # M529LKIQ.G10): + # probe (fi=0): probe_skip (depends on STRT position) + # meta@0x1002 (fi=1): 13 (6-byte inner header) + # meta@0x1004 (fi=2): 13 (6-byte inner header) + # sample chunks (fi=3+): 12 (5-byte inner header) last_fi = len(body_frames) - 1 - log.warning( - "write_blastware_file: %d body_frames first_metadata_fi=%s last_fi=%d", - len(body_frames), - str(first_metadata_fi) if first_metadata_fi is not None else "None", - last_fi, + log.debug( + "write_blastware_file: %d body_frames last_fi=%d", + len(body_frames), last_fi, ) all_bytes = bytearray() for fi, frame in enumerate(body_frames): - # Skip "extra chunk" frames: frames after the first metadata frame but - # before the last frame (terminator). These prime the TCP terminator but - # their ADC data must NOT appear in the Blastware file body. - if (first_metadata_fi is not None - and fi > first_metadata_fi - and fi < last_fi): - log.warning( - "write_blastware_file: fi=%d SKIP (extra chunk after metadata fi=%d last_fi=%d)", - fi, first_metadata_fi, last_fi, - ) - continue - if fi == 0: - # Probe frame: always process regardless of classification. - # It holds the STRT record; probe_skip positions us past it. skip = probe_skip + elif fi in (1, 2): + skip = 13 # metadata pages else: - # ALL subsequent frames are included unconditionally — no filtering on - # frame type. In the A5 stream, frame 0 is always the probe response; - # frames 1+ are always data (waveform chunks, compliance config, or - # compliance continuation). Classification is for logging only. - # - # DO NOT gate on classify_frame() here: - # - "probe_or_strt" at fi>0 is always a false positive — ADC binary - # data can coincidentally contain b"STRT\xff\xfe" (confirmed from - # live capture: frames 1 and 5 matched on event key=01110000). - # - "metadata" frames must be included (compliance config body). - # - The compliance block spans 2 frames; skipping either produces a - # truncated file that Blastware rejects. - skip = 13 if fi == 1 else 12 + skip = 12 # sample chunks contribution = _frame_body_bytes(frame, skip) log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", @@ -769,11 +740,33 @@ def write_blastware_file( bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(), ) - if len(all_bytes) >= 26: + # Find the first valid 0e 08 footer marker (v0.14.0). The device's + # TERM response contains the real Blastware footer; older walks + # accidentally fetched data past the footer. Validate by checking the + # year field (uint16 BE at offset+4) is in 2015..2050. + footer_pos = -1 + pos = 0 + while True: + pos = bytes(all_bytes).find(b"\x0e\x08", pos) + if pos < 0 or pos + 26 > len(all_bytes): + break + yr = (all_bytes[pos + 4] << 8) | all_bytes[pos + 5] + if 2015 <= yr <= 2050: + footer_pos = pos + break + pos += 1 + if footer_pos >= 0: + body = bytes(all_bytes[:footer_pos]) + footer = bytes(all_bytes[footer_pos:footer_pos + 26]) + log.warning( + "write_blastware_file: real 0e 08 footer at all_bytes[%d]; " + "truncating %d post-footer bytes", + footer_pos, len(all_bytes) - footer_pos - 26, + ) + elif len(all_bytes) >= 26: body = bytes(all_bytes[:-26]) footer = bytes(all_bytes[-26:]) else: - # Fallback: no terminator or very short stream → build footer from event metadata body = bytes(all_bytes) start_dt = _ts_from_model(event.timestamp) stop_dt: Optional[datetime.datetime] = None @@ -784,7 +777,7 @@ def write_blastware_file( + _encode_ts_be(start_dt) + _encode_ts_be(stop_dt) + b"\x00\x01\x00\x02\x00\x00" - + b"\x00\x00" # CRC placeholder + + b"\x00\x00" ) # ── Write file ─────────────────────────────────────────────────────────── diff --git a/minimateplus/client.py b/minimateplus/client.py index 12664c0..7b1f9eb 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -1345,6 +1345,11 @@ def _decode_waveform_record_into(data: bytes, event: Event) -> None: event.timestamp = Timestamp.from_continuous_record(data) except Exception as exc: log.warning("continuous record timestamp decode failed: %s", exc) + elif event.record_type == "Waveform (Short)": + try: + event.timestamp = Timestamp.from_short_record(data) + except Exception as exc: + log.warning("short record timestamp decode failed: %s", exc) # ── Peak values (per-channel PPV + Peak Vector Sum) ─────────────────────── try: @@ -1636,51 +1641,73 @@ def _decode_a5_waveform( } +def _detect_record_format(data: bytes) -> Optional[str]: + """ + Detect which timestamp-header format a 210-byte 0C waveform record uses. + + THREE formats observed on BE11529 firmware S338.17: + + "single_shot" — 9-byte header: + [day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec] + sub_code=0x10 at byte [1]. Year at [3:5]. + + "continuous" — 10-byte header: + [0x10] [day] [0x10] [month] [year_BE:2] [unknown] [hour] [min] [sec] + marker 0x10 at byte [0] AND byte [2]. Year at [4:6]. + + "short" — 8-byte header (NEW 2026-05-01): + [day] [month] [year_BE:2] [unknown] [hour] [min] [sec] + No marker bytes. Year at [2:4]. + + Each format has the year (uint16 BE) at a UNIQUE byte position, so we can + disambiguate by scanning each candidate position and picking the one + where the year falls in a sane range (2015..2050). + + Returns "single_shot" / "continuous" / "short" or None if no format matches. + """ + if len(data) < 8: + return None + + def _sane_year(hi: int, lo: int) -> bool: + y = (hi << 8) | lo + return 2015 <= y <= 2050 + + # Order matters: prefer formats with stronger marker-byte evidence first. + if data[1] == 0x10 and len(data) >= 9 and _sane_year(data[3], data[4]): + return "single_shot" + if (data[0] == 0x10 and data[2] == 0x10 + and len(data) >= 10 and _sane_year(data[4], data[5])): + return "continuous" + if _sane_year(data[2], data[3]): + return "short" + return None + + def _extract_record_type(data: bytes) -> Optional[str]: """ - Detect the waveform record format by inspecting the first 3 bytes of the - 210-byte record returned by SUB 0C. + Return a human-readable name for the waveform record format detected + in the first bytes of a 210-byte 0C record. - Two formats exist (confirmed from BE11529 captures and CLAUDE.md docs): - - Single-shot mode — 9-byte header: - data[0] = day - data[1] = 0x10 ← sub_code marker - data[2] = month - data[3:5] = year (BE) - ... - - Continuous mode — 10-byte header: - data[0] = 0x10 ← marker A - data[1] = day ← variable (NOT 0x10) - data[2] = 0x10 ← marker B - data[3] = month - data[4:6] = year (BE) - ... - - Disambiguate by checking BOTH data[0] and data[2]: - - data[0]==0x10 AND data[2]==0x10 → Continuous (10-byte header) - - data[1]==0x10 → Single-shot (9-byte header) - - otherwise → Unknown - - Previous logic only checked data[1] and so mis-classified continuous-mode - records as "Unknown(0xXX)" wherever day != 0x10 — see filename - M5290000.000 regression report (2026-05-01 SFM log). + Maps to the format codes returned by _detect_record_format(): + "single_shot" → "Waveform" + "continuous" → "Waveform (Continuous)" + "short" → "Waveform (Short)" + None → "Unknown(XX.YY.ZZ)" """ - if len(data) < 3: - return None - # 10-byte continuous format: 0x10 markers at byte 0 AND byte 2 - if data[0] == 0x10 and data[2] == 0x10: - return "Waveform (Continuous)" - # 9-byte single-shot format: 0x10 sub_code marker at byte 1 - if data[1] == 0x10: + fmt = _detect_record_format(data) + if fmt == "single_shot": return "Waveform" - log.warning( - "_extract_record_type: unrecognized header: data[0:3]=%02X %02X %02X", - data[0], data[1], data[2], - ) - return f"Unknown({data[0]:02X}.{data[1]:02X}.{data[2]:02X})" - + if fmt == "continuous": + return "Waveform (Continuous)" + if fmt == "short": + return "Waveform (Short)" + if len(data) >= 3: + log.warning( + "_extract_record_type: unrecognized header: data[0:3]=%02X %02X %02X", + data[0], data[1], data[2], + ) + return f"Unknown({data[0]:02X}.{data[1]:02X}.{data[2]:02X})" + return None def _extract_peak_floats(data: bytes) -> Optional[PeakValues]: """ diff --git a/minimateplus/models.py b/minimateplus/models.py index 47d4028..91d6344 100644 --- a/minimateplus/models.py +++ b/minimateplus/models.py @@ -201,6 +201,58 @@ class Timestamp: second=second, ) + @classmethod + def from_short_record(cls, data: bytes) -> "Timestamp": + """ + Decode an 8-byte timestamp header from a 210-byte waveform record. + + Wire layout (✅ CONFIRMED 2026-05-01 against live SFM run on BE11529 in + Continuous mode, day-of-month = 1 May, raw: 01 05 07 ea 00 0d 15 25): + byte[0]: day (uint8) + byte[1]: month (uint8) + bytes[2-3]: year (big-endian uint16) + byte[4]: unknown (0x00 in observed sample) + byte[5]: hour (uint8) + byte[6]: minute (uint8) + byte[7]: second (uint8) + + This is a third format observed in the wild — distinct from the 9-byte + (single-shot, sub_code=0x10 at [1]) and 10-byte (continuous, 0x10 at + [0] AND [2]) layouts. No marker bytes; disambiguated by where the + year lands when scanned at byte 2/3/4. + + Args: + data: at least 8 bytes; only the first 8 are consumed. + + Returns: + Decoded Timestamp. + + Raises: + ValueError: if data is fewer than 8 bytes. + """ + if len(data) < 8: + raise ValueError( + f"Short record timestamp requires at least 8 bytes, got {len(data)}" + ) + day = data[0] + month = data[1] + year = struct.unpack_from(">H", data, 2)[0] + unknown_byte = data[4] + hour = data[5] + minute = data[6] + second = data[7] + return cls( + raw=bytes(data[:8]), + flag=0, + year=year, + unknown_byte=unknown_byte, + month=month, + day=day, + hour=hour, + minute=minute, + second=second, + ) + @property def clock_set(self) -> bool: """False when year == 1995 (factory default / battery-lost state).""" diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 2abeb1a..48fbfbe 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -136,9 +136,10 @@ DATA_LENGTHS: dict[int, int] = { # existing blastware_file.py builder relies on the 0x0400-step frame structure # to produce valid files. Switching to BW's 0x0200 step is a separate task # that also requires updating the file builder. -_BULK_CHUNK_OFFSET = 0x1004 # offset_word for probe + all chunk requests -_BULK_TERM_OFFSET = 0x005A # offset_word for the legacy terminator -_BULK_COUNTER_STEP = 0x0400 # chunk counter increment +# BW-exact protocol values (v0.14.0). Verified against 4-27-26 + 5-1-26 captures. +_BULK_CHUNK_OFFSET = 0x1002 # offset_word for probe + all chunk requests +_BULK_TERM_OFFSET = 0x005A # offset_word for the legacy terminator (fallback only) +_BULK_COUNTER_STEP = 0x0200 # chunk counter increment (matches chunk payload size) # Default timeout values (seconds). # MiniMate Plus is a slow device — keep these generous. @@ -533,231 +534,260 @@ class MiniMateProtocol: self, key4: bytes, *, - stop_after_metadata: bool = True, - max_chunks: int = 32, + stop_after_metadata: bool = True, # DEPRECATED — no-op under BW-exact walk + max_chunks: int = 256, # safety cap only; loop is bounded by end_offset include_terminator: bool = False, - extra_chunks_after_metadata: int = 1, + extra_chunks_after_metadata: int = 1, # DEPRECATED — no-op ) -> list[S3Frame]: """ - Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event. + Download the SUB 5A (BULK_WAVEFORM_STREAM) A5 frames for one event using + Blastware's exact protocol. REWRITTEN 2026-05-02 (v0.14.0). - The bulk waveform stream carries both raw ADC samples (large) and - event-time metadata strings ("Project:", "Client:", "User Name:", - "Seis Loc:", "Extended Notes") embedded in one of the middle frames - (confirmed: A5[7] of 9 for 1-2-26 capture). + Algorithm (matches BW captures across 2-sec / 3-sec / event-2): - Protocol is request-per-chunk, NOT a continuous stream: - 1. Probe (offset=_BULK_CHUNK_OFFSET, is_probe=True, counter=0x0000) - 2. Chunks (offset=_BULK_CHUNK_OFFSET, is_probe=False, counter+=0x0400) - 3. Loop until metadata found (stop_after_metadata=True) or max_chunks - 4. Termination (offset=_BULK_TERM_OFFSET, counter=last+_BULK_COUNTER_STEP) - Device responds with a final A5 frame (page_key=0x0000). + 1. Probe + - For events at start_key[2:4] = 0x0000 (first event after erase + / wrap): probe at counter=0x0000 with full key in params. + - For continuation events (start_key[2:4] != 0): first chunk at + counter = start_key[2:4] + 0x0046; acts as both probe and + first sample chunk; response carries STRT. - By default the termination frame (page_key=0x0000) is NOT included in the - returned list. Pass include_terminator=True to append it; the blastware_file - writer needs the terminator frame's body to reconstruct the waveform file footer. + 2. Parse end_offset from STRT record at data[23:27] of the probe response. - Args: - key4: 4-byte waveform key from EVENT_HEADER (1E). - stop_after_metadata: If True (default), send termination as soon as - b"Project:" is found in a frame's data — avoids - downloading the full ADC waveform payload (several - hundred KB). Set False to download everything. - max_chunks: Safety cap on the number of chunk requests sent - (default 32; a typical event uses 9 large frames). - include_terminator: If True, append the terminator A5 frame - (page_key=0x0000) to the returned list. The - terminator carries the waveform file footer bytes. - Default False preserves existing caller behaviour. + 3. Read two fixed metadata pages at counter=0x1002 and counter=0x1004 + — global session metadata (Project / Client / User Name / Seis Loc + / Extended Notes ASCII strings). Event 1 only; continuation + events skip these (BW caches them across the session). + + 4. Walk sample chunks at 0x0200 increments, starting from 0x0600 for + event 1 or `start + 0x0046 + 0x0200` for continuation events. + Stop when `next_chunk + 0x0200 > end_offset`. + + 5. Send TERM frame with offset_word and params computed by + `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`. + The TERM response contains the partial last chunk (residual = + end_offset - next_boundary) including the 26-byte 0e 08 file + footer. Returns: - List of S3Frame objects from each A5 response frame. Frame indices - match the request sequence: index 0 = probe response, index 1 = first - chunk, etc. If include_terminator=True, the last element is the - terminator frame (page_key=0x0000). + List of S3Frame objects from each A5 response (probe, metadata + pages, sample chunks, optional TERM response). Caller passes + `include_terminator=True` (e.g. write_blastware_file) to keep the + TERM response in the list — it's required to reconstruct the + file footer. + + Deprecated kwargs: + stop_after_metadata: legacy "Project:"-string-based stop condition. + No-op under the BW-exact walk; the loop is + deterministically bounded by end_offset from + STRT. Accepted for backward compat. + extra_chunks_after_metadata: same. Raises: - ProtocolError: on timeout, bad checksum, or unexpected SUB. - - Confirmed from 1-2-26 BW TX/RX captures (2026-04-02): - - probe + 8 regular chunks + 1 termination = 10 TX frames - - 9 large A5 responses + 1 terminator A5 = 10 RX frames - - page_key=0x0010 on large frames; page_key=0x0000 on terminator ✅ - - "Project:" metadata at A5[7].data[626] ✅ + ProtocolError: on timeout / bad checksum / unexpected SUB. """ if len(key4) != 4: raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}") - rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM) # 0xFF - 0x5A = 0xA5 + # Quietly accept and warn on deprecated kwargs. + if not stop_after_metadata: + log.debug("5A: stop_after_metadata=False is no-op under BW-exact walk") + if extra_chunks_after_metadata not in (0, 1): + log.debug("5A: extra_chunks_after_metadata=%d is no-op under BW-exact walk", + extra_chunks_after_metadata) + + rsp_sub = _expected_rsp_sub(SUB_BULK_WAVEFORM) # 0xA5 frames_data: list[S3Frame] = [] - counter = 0 - # BW counter formula (confirmed from 4-3-26 capture for key 0111245a, - # and empirical live-device test 2026-04-06 for key 01110000): - # counter for chunk n = max(key4[2:4], 0x0400) + (n - 1) * 0x0400 - # key4[2:4] is the event's circular-buffer base offset. The max() guard - # ensures chunk 1 never uses counter=0x0000 (which equals the probe address - # and causes the device to re-return STRT record data for the first chunk). - _key4_offset = (key4[2] << 8) | key4[3] + start_offset = (key4[2] << 8) | key4[3] + is_event_1 = (start_offset == 0) - # ── Step 1: probe ──────────────────────────────────────────────────── - log.debug("5A probe key=%s key4_offset=0x%04X", key4.hex(), _key4_offset) - params = bulk_waveform_params(key4, 0, is_probe=True) - self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - self._parser.reset() # reset bytes_fed counter before probe recv + # ── Step 1: probe / first chunk ────────────────────────────────────── + if is_event_1: + probe_counter = 0 + probe_params = bulk_waveform_params(key4, 0, is_probe=True) + log.debug("5A probe (event-1) key=%s counter=0x0000", key4.hex()) + else: + # Continuation events: first 5A request lands at start+0x0046, + # acting as both probe and first sample chunk. Confirmed from + # 5-1-26 "copy 2nd address event" capture. + probe_counter = start_offset + 0x0046 + probe_params = bulk_waveform_params(key4, probe_counter) + log.debug( + "5A probe (event-N) key=%s counter=0x%04X (start+0x46)", + key4.hex(), probe_counter, + ) + + self._send(build_5a_frame(_BULK_CHUNK_OFFSET, probe_params)) + self._parser.reset() try: rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False) except TimeoutError: log.warning( - "5A probe TIMED OUT for key=%s — " - "%d raw bytes received (no complete A5 frame assembled)", + "5A probe TIMED OUT for key=%s — %d raw bytes received", key4.hex(), self._parser.bytes_fed, ) raise - frames_data.append(rsp) - log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data)) - # ── Parse STRT end_offset from probe response (NEW 2026-05-01) ──────── - # The first A5 response contains a STRT record at data[17:]. The - # bytes at data[23:27] are the event's end-key, whose low 16 bits - # are the absolute device-buffer address where the event ends. Use - # this to bound the chunk loop and stop the over-read past event end. - # See docs/instantel_protocol_reference.md §7.8.5 and CLAUDE.md - # "SUB 5A — STRT record encodes end_offset". - _end_offset = parse_strt_end_offset(rsp.data) - if _end_offset is None: - # Defensive fallback — let max_chunks cap the walk. - log.warning("5A: STRT not found in probe; cannot bound chunk loop") - _end_offset = 0xFFFF + frames_data.append(rsp) + log.debug("5A A5[0] (probe) page_key=0x%04X %d bytes", + rsp.page_key, len(rsp.data)) + + # ── Step 2: parse STRT end_offset from probe response ──────────────── + end_offset = parse_strt_end_offset(rsp.data) + if end_offset is None: + log.warning( + "5A probe response did not contain a STRT record; " + "cannot bound chunk loop — falling back to max_chunks=%d cap", + max_chunks, + ) + end_offset = 0xFFFF # impossible value → loop runs to max_chunks else: - log.debug( + log.info( "5A STRT start_offset=0x%04X end_offset=0x%04X size=0x%04X", - _key4_offset, _end_offset, _end_offset - _key4_offset, + start_offset, end_offset, end_offset - start_offset, ) - # ── Step 2: chunk loop ─────────────────────────────────────────────── - # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 - # where _chunk_base = max(key4[2:4], 0x0400). - # - # For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a): - # _chunk_base = 0x245a → chunk 1=0x245a, chunk 2=0x285a, ... - # Confirmed from 4-3-26 capture. - # - # For events with key4[2:4] == 0 (e.g. key 01110000): - # _chunk_base = max(0, 0x0400) = 0x0400 - # → chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400) - # CRITICAL: counter=0x0000 (same as the probe) causes the device to - # re-return the STRT record data for chunk 1, making frame 1 look like - # a second probe response (confirmed from server log: frame 1 len=1097, - # contains STRT\xff\xfe, contributes zero body bytes after DLE-strip). - # counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06). - _chunk_base = max(_key4_offset, _BULK_COUNTER_STEP) - for chunk_num in range(1, max_chunks + 1): - counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP - # Stop when we'd step past the event end (NEW 2026-05-01). Without - # this, the device returns post-event circular-buffer data which - # corrupts the reconstructed file for events ≥ 2 sec. - if counter >= _end_offset: + # ── Step 3: metadata pages 0x1002 + 0x1004 (event 1 only) ──────────── + # Confirmed from BW captures: BW reads these two fixed device-buffer + # pages immediately after the probe for events at start_key[2:4]=0. + # Continuation events skip them (BW caches across the session). + # Their content is global compliance-setup metadata: Project, Client, + # User Name, Seis Loc, Extended Notes. + if is_event_1: + for meta_counter in (0x1002, 0x1004): + # Metadata page params have an extra trailing 0x00 byte + # (12-byte params instead of 11) — empirical from BW captures. + # Checksum-neutral but matches BW byte-for-byte. + meta_params = bytes([ + 0x00, + key4[0], key4[1], + (meta_counter >> 8) & 0xFF, + meta_counter & 0xFF, + 0, 0, 0, 0, 0, 0, 0, + ]) + log.debug("5A metadata page counter=0x%04X", meta_counter) + self._send(build_5a_frame(_BULK_CHUNK_OFFSET, meta_params)) + self._parser.reset() + try: + meta_rsp = self._recv_one( + expected_sub=rsp_sub, reset_parser=False, timeout=10.0, + ) + except TimeoutError: + log.warning( + "5A metadata page 0x%04X TIMED OUT — continuing", + meta_counter, + ) + continue + frames_data.append(meta_rsp) + log.debug( + "5A meta@0x%04X page_key=0x%04X %d bytes", + meta_counter, meta_rsp.page_key, len(meta_rsp.data), + ) + + # ── Step 4: sample chunk loop, bounded by end_offset ───────────────── + # Sample chunks start at: + # event 1: counter = 0x0600 + # event N (>0): counter = probe_counter + 0x0200 + # (probe was the first sample chunk) + if is_event_1: + counter = 0x0600 + else: + counter = probe_counter + _BULK_COUNTER_STEP + + last_chunk_counter: Optional[int] = ( + probe_counter if not is_event_1 else None + ) + chunks_fetched = 0 + + while chunks_fetched < max_chunks: + # Stop when next chunk would straddle the event end. + if counter + _BULK_COUNTER_STEP > end_offset: log.debug( "5A chunk loop done at counter=0x%04X (end=0x%04X); " "%d chunks fetched", - counter, _end_offset, len(frames_data), + counter, end_offset, chunks_fetched, ) break - params = bulk_waveform_params(key4, counter) - log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) + + params = bulk_waveform_params(key4, counter) + log.debug("5A chunk #%d counter=0x%04X", chunks_fetched + 1, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - self._parser.reset() # reset bytes_fed for accurate per-chunk count + self._parser.reset() try: - rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False, timeout=10.0) + rsp = self._recv_one( + expected_sub=rsp_sub, reset_parser=False, timeout=10.0, + ) except TimeoutError: raw = self._parser.bytes_fed log.warning( "5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d", - chunk_num, counter, raw, + chunks_fetched + 1, counter, raw, ) if raw > 0 and frames_data: - # Device sent a partial byte (likely a bare DLE/ETX end-of-stream - # signal) but never completed a full frame. Treat as graceful - # stream end and fall through to the termination step. log.warning( - "5A end-of-stream detected at chunk=%d (raw_bytes=%d, " - "frames_collected=%d) — proceeding to termination", - chunk_num, raw, len(frames_data), + "5A unexpected end-of-stream — proceeding to TERM", ) break raise - log.warning( - "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", - chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, + log.debug( + "5A RX chunk=%d page_key=0x%04X data_len=%d", + chunks_fetched + 1, rsp.page_key, len(rsp.data), ) if rsp.page_key == 0x0000: - # Device unexpectedly terminated mid-stream (no termination needed). - log.debug("5A A5[%d] page_key=0x0000 — device terminated early", chunk_num) + # Device terminated mid-stream unexpectedly. + log.warning( + "5A unexpected page_key=0x0000 mid-stream at counter=0x%04X", + counter, + ) if include_terminator: frames_data.append(rsp) return frames_data frames_data.append(rsp) - - if stop_after_metadata and b"Project:" in rsp.data: - # Download exactly one more chunk after finding metadata — this is - # what Blastware does. The extra chunk contains the tail ADC data - # and primes the device to return a valid footer in the termination - # response. Without it, termination returns an empty ack with no - # footer bytes (confirmed 2026-04-23 from HxD comparison). - # Download extra_chunks_after_metadata more chunks past the - # metadata. The caller calculates this from record_time and - # sample_rate so we download exactly the right amount of ADC - # data — no more, no less — before terminating. - # The device returns the footer in the termination response only - # after the right amount of data has been consumed. - log.debug("5A A5[%d] metadata found — fetching %d more chunk(s)", - chunk_num, extra_chunks_after_metadata) - for _extra_n in range(extra_chunks_after_metadata): - chunk_num += 1 - counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP - params = bulk_waveform_params(key4, counter) - self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - try: - extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", - chunk_num, extra.page_key, len(extra.data)) - if extra.page_key == 0x0000: - if include_terminator: - frames_data.append(extra) - return frames_data - frames_data.append(extra) - except TimeoutError: - log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) - break - break + last_chunk_counter = counter + counter += _BULK_COUNTER_STEP + chunks_fetched += 1 else: log.warning( - "5A reached max_chunks=%d without end-of-stream; sending termination", - max_chunks, + "5A reached max_chunks=%d at counter=0x%04X (end=0x%04X)", + max_chunks, counter, end_offset, ) - # ── Step 3: termination ────────────────────────────────────────────── - term_counter = counter + _BULK_COUNTER_STEP - term_params = bulk_waveform_term_params(key4, term_counter) - log.debug( - "5A termination term_counter=0x%04X offset=0x%04X", - term_counter, _BULK_TERM_OFFSET, - ) - self._send(build_5a_frame(_BULK_TERM_OFFSET, term_params)) - try: - term_rsp = self._recv_one(expected_sub=rsp_sub) + # ── Step 5: TERM with proper end_offset-derived formula ────────────── + if last_chunk_counter is None or end_offset == 0xFFFF: + # No STRT or no chunks fetched — fall back to legacy TERM. + log.warning( + "5A using legacy TERM (offset_word=0x005A); " + "end_offset unavailable or no chunks fetched", + ) + legacy_counter = (last_chunk_counter or probe_counter) + _BULK_COUNTER_STEP + term_offset_word = _BULK_TERM_OFFSET # 0x005A + term_params = bulk_waveform_term_params(key4, legacy_counter) + else: + term_offset_word, term_params = bulk_waveform_term_v2( + key4, end_offset, last_chunk_counter, + ) log.debug( - "5A termination response page_key=0x%04X %d bytes", + "5A TERM offset_word=0x%04X params[2:4]=%s end=0x%04X " + "last_chunk=0x%04X", + term_offset_word, term_params[2:4].hex(), + end_offset, last_chunk_counter, + ) + + self._send(build_5a_frame(term_offset_word, term_params)) + try: + term_rsp = self._recv_one(expected_sub=rsp_sub, timeout=10.0) + log.info( + "5A TERM response page_key=0x%04X %d bytes", term_rsp.page_key, len(term_rsp.data), ) if include_terminator: frames_data.append(term_rsp) except TimeoutError: - log.debug("5A no termination response — device may have already closed") + log.warning("5A no TERM response (timeout)") return frames_data diff --git a/sfm/server.py b/sfm/server.py index 3fc4bb2..9683254 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -37,6 +37,7 @@ from __future__ import annotations import datetime import logging import sys +import tempfile import threading import time from pathlib import Path @@ -863,8 +864,8 @@ def device_event_blastware_file( Supply either *port* (serial) or *host* (TCP/modem). - The file is written to /tmp and streamed back as a binary download. - Blastware can open it directly — filename encodes serial + timestamp. + The file is written to the OS temp directory and streamed back as a binary + download. Blastware can open it directly — filename encodes serial + timestamp. Filename format: 0 - prefix letter = chr(ord('B') + floor(serial_numeric / 1000)) @@ -885,26 +886,13 @@ def device_event_blastware_file( def _do(): with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() - # Use stop_after_metadata=True (full_waveform=False) with 1 extra - # chunk after "Project:". The extra chunk is required to prime the - # device over TCP: termination at term_counter=metadata_counter+0x0400 - # returns only ~90 bytes (no useful footer) over TCP/cellular, but - # termination at metadata_counter+0x0800 (one chunk later) returns - # the full 737-byte frame containing the footer. - # - # Confirmed from 4-26-26 BW RS-232 capture: BW terminates at 0x1800 - # without an extra chunk (works on RS-232 but not TCP). - # write_blastware_file() automatically skips the extra chunk's - # contribution — only the probe+ADC+metadata+terminator bytes appear - # in the output file. - # - # full_waveform=True (natural end-of-stream) downloads ALL chunks - # including post-event silence (35+ chunks for a 9-sec event at - # 1024 sps) — this produces 24KB+ files that Blastware rejects. + # Under v0.14.0 BW-exact 5A walk, the chunk loop is bounded by + # the event end_offset extracted from STRT. No more + # stop_after_metadata / extra_chunks gymnastics — these + # kwargs are now no-ops. events = client.get_events( full_waveform=False, stop_after_index=index, - extra_chunks_after_metadata=1, ) matching = [ev for ev in events if ev.index == index] return matching[0] if matching else None, info @@ -940,8 +928,18 @@ def device_event_blastware_file( # Build filename using the same algorithm Blastware uses filename = blastware_filename(ev, serial) - # Write to /tmp so FastAPI can stream it back - out_path = Path("/tmp") / filename + # Write to OS temp dir (cross-platform: /tmp on Linux/macOS, + # %TEMP% on Windows) so FastAPI can stream it back via FileResponse. + out_path = Path(tempfile.gettempdir()) / filename + # Delete any stale file at this path before writing. On Windows we have + # observed the new (smaller) file getting trailing zero-bytes from the + # previous (larger) file when filesystem semantics around open(...,"wb") + # don't truncate cleanly (e.g. through a synced folder). Explicit unlink + # eliminates that ambiguity. + try: + out_path.unlink() + except FileNotFoundError: + pass write_blastware_file(ev, a5_frames, out_path) log.info( "blastware_file: wrote %s (%d A5 frames, serial=%s)",