diff --git a/CLAUDE.md b/CLAUDE.md index 16089fe..4196b5f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -347,6 +347,24 @@ Do NOT use fixed absolute offsets for sample_rate or record_time. Quiet Mode enabled. Parser handles this — do not strip it manually before feeding to `S3FrameParser`. +**SUB 5A (bulk waveform) TCP frame splitting — confirmed 2026-04-27:** + +Over TCP via cellular modem, each 5A chunk request that produces a single ~1100-byte +A5 response over direct RS-232 arrives as **two separate, complete S3 frames** of +~550 bytes each. This is because the device sends its RS-232 response as multiple +frames, and the modem's Data Forwarding Timeout (~100-150 ms) delivers them to us +as separate TCP segments, each parsed as a complete S3 frame. + +Example for a 2-second Continuous event (BE11529, key=01110000) via TCP: +- 1 probe frame (554 B) + 5 chunks × 2 frames (556-573 B) + 1 extra chunk × 2 frames + 1 terminator (208 B) = **14 A5 frames** +- All 14 frames contribute body data; using all of them gives the correct 6864-byte file. + +**Fix (confirmed 2026-04-27):** `_recv_5a_batch()` in `protocol.py` collects ALL +A5 frames per chunk request before the next request is sent, using a 0.5 s batch +timeout after the first frame to catch the ~150 ms delayed second frame. `write_blastware_file()` +includes ALL body frames without skipping — the extra chunk's frames are part of the +body data, NOT padding to be discarded. + ### Required ACEmanager settings (Sierra Wireless RV50/RV55) | Setting | Value | Why | diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index fedf229..32f3c99 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -685,64 +685,34 @@ def write_blastware_file( body_frames = a5_frames term_frame = None - # ── Identify first metadata frame and skip "extra chunks" ─────────────── - # When extra_chunks_after_metadata=1 in read_bulk_waveform_stream(), the - # frame list is: [probe, data..., metadata, extra_chunk, terminator]. - # The extra_chunk is downloaded to prime the TCP terminator response — its - # ADC data is NOT part of the Blastware file body. Skip it. - # - # Rule: any frame at index strictly between first_metadata_fi and last_fi - # (the final frame) is an extra chunk and must be excluded. - # - # If no metadata frame exists (e.g. full_waveform download), first_metadata_fi - # is None and no frames are skipped — all frames contribute normally. - first_metadata_fi: Optional[int] = None - for _fi_scan, _frame_scan in enumerate(body_frames): - if _fi_scan > 0 and any(m in bytes(_frame_scan.data) for m in _METADATA_FRAME_MARKERS): - first_metadata_fi = _fi_scan - break - last_fi = len(body_frames) - 1 - log.warning( - "write_blastware_file: %d body_frames first_metadata_fi=%s last_fi=%d", + "write_blastware_file: %d body_frames term_idx=%s", len(body_frames), - str(first_metadata_fi) if first_metadata_fi is not None else "None", - last_fi, + str(term_idx) if term_idx is not None else "None", ) all_bytes = bytearray() for fi, frame in enumerate(body_frames): - # Skip "extra chunk" frames: frames after the first metadata frame but - # before the last frame (terminator). These prime the TCP terminator but - # their ADC data must NOT appear in the Blastware file body. - if (first_metadata_fi is not None - and fi > first_metadata_fi - and fi < last_fi): - log.warning( - "write_blastware_file: fi=%d SKIP (extra chunk after metadata fi=%d last_fi=%d)", - fi, first_metadata_fi, last_fi, - ) - continue - + # All body frames contribute to the waveform body — no frames are skipped. + # + # Over TCP via cellular modem, _recv_5a_batch() correctly collects all + # A5 frames per chunk request (the device's ~1100-byte RS-232 response + # is forwarded as ~2 TCP segments of ~550 bytes each, each parsed as a + # separate S3 frame). ALL of these frames contain ADC body data and + # must be included in the file — confirmed from 4-27-26 TCP capture + # analysis: contributions from all 14 frames → 6821 bytes → file 6864 bytes. + # + # Skip amounts (offsets into frame.data): + # fi=0 (probe): probe_skip — skips the type_tag header + STRT record + # fi=1: 13 — 7-byte frame.data prefix + 6 inner header bytes + # fi>=2: 12 — 7-byte frame.data prefix + 5 inner header bytes if fi == 0: - # Probe frame: always process regardless of classification. - # It holds the STRT record; probe_skip positions us past it. skip = probe_skip + elif fi == 1: + skip = 13 else: - # ALL subsequent frames are included unconditionally — no filtering on - # frame type. In the A5 stream, frame 0 is always the probe response; - # frames 1+ are always data (waveform chunks, compliance config, or - # compliance continuation). Classification is for logging only. - # - # DO NOT gate on classify_frame() here: - # - "probe_or_strt" at fi>0 is always a false positive — ADC binary - # data can coincidentally contain b"STRT\xff\xfe" (confirmed from - # live capture: frames 1 and 5 matched on event key=01110000). - # - "metadata" frames must be included (compliance config body). - # - The compliance block spans 2 frames; skipping either produces a - # truncated file that Blastware rejects. - skip = 13 if fi == 1 else 12 + skip = 12 contribution = _frame_body_bytes(frame, skip) log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d", diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 0a69f93..f85b977 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -599,7 +599,7 @@ class MiniMateProtocol: self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) self._parser.reset() # reset bytes_fed counter before probe recv try: - rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False) + probe_batch = self._recv_5a_batch(rsp_sub) except TimeoutError: log.warning( "5A probe TIMED OUT for key=%s — " @@ -607,8 +607,12 @@ class MiniMateProtocol: key4.hex(), self._parser.bytes_fed, ) raise - frames_data.append(rsp) - log.debug("5A A5[0] page_key=0x%04X %d bytes", rsp.page_key, len(rsp.data)) + frames_data.extend(probe_batch) + log.debug( + "5A probe: %d frame(s) page_keys=%s", + len(probe_batch), + [f"0x{f.page_key:04X}" for f in probe_batch], + ) # ── Step 2: chunk loop ─────────────────────────────────────────────── # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 @@ -634,7 +638,12 @@ class MiniMateProtocol: self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) self._parser.reset() # reset bytes_fed for accurate per-chunk count try: - rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False, timeout=10.0) + # Collect ALL frames from this chunk response. + # Over TCP via modem, a single large A5 device response (~1100 bytes + # RS-232) is split across ~2 TCP segments, each parsed as its own + # complete S3 frame. _recv_5a_batch gathers all of them so that + # every subsequent chunk request is paired with the correct response. + batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) except TimeoutError: raw = self._parser.bytes_fed log.warning( @@ -653,48 +662,48 @@ class MiniMateProtocol: break raise - log.warning( - "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", - chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, - ) + # Process all frames from this batch. + metadata_found = False + for rsp in batch: + log.warning( + "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", + chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, + ) + if rsp.page_key == 0x0000: + # Device unexpectedly terminated mid-stream. + log.debug("5A page_key=0x0000 — device terminated early") + if include_terminator: + frames_data.append(rsp) + return frames_data + frames_data.append(rsp) + if stop_after_metadata and b"Project:" in rsp.data: + metadata_found = True - if rsp.page_key == 0x0000: - # Device unexpectedly terminated mid-stream (no termination needed). - log.debug("5A A5[%d] page_key=0x0000 — device terminated early", chunk_num) - if include_terminator: - frames_data.append(rsp) - return frames_data - - frames_data.append(rsp) - - if stop_after_metadata and b"Project:" in rsp.data: - # Download exactly one more chunk after finding metadata — this is - # what Blastware does. The extra chunk contains the tail ADC data - # and primes the device to return a valid footer in the termination - # response. Without it, termination returns an empty ack with no - # footer bytes (confirmed 2026-04-23 from HxD comparison). - # Download extra_chunks_after_metadata more chunks past the - # metadata. The caller calculates this from record_time and - # sample_rate so we download exactly the right amount of ADC - # data — no more, no less — before terminating. - # The device returns the footer in the termination response only - # after the right amount of data has been consumed. - log.debug("5A A5[%d] metadata found — fetching %d more chunk(s)", - chunk_num, extra_chunks_after_metadata) + if metadata_found: + # Download extra_chunks_after_metadata more chunks after metadata. + # This primes the device to return the valid waveform footer in the + # termination response — without it the terminator carries too few bytes + # (confirmed 2026-04-23). The extra chunk data also belongs in the + # file body (confirmed from TCP capture analysis 2026-04-27). + log.debug("5A metadata found — fetching %d more chunk(s)", + extra_chunks_after_metadata) for _extra_n in range(extra_chunks_after_metadata): chunk_num += 1 counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP params = bulk_waveform_params(key4, counter) self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) try: - extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0) - log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d", - chunk_num, extra.page_key, len(extra.data)) - if extra.page_key == 0x0000: - if include_terminator: - frames_data.append(extra) - return frames_data - frames_data.append(extra) + extra_batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) + for ef in extra_batch: + log.debug( + "5A extra chunk page_key=0x%04X data_len=%d", + ef.page_key, len(ef.data), + ) + if ef.page_key == 0x0000: + if include_terminator: + frames_data.append(ef) + return frames_data + frames_data.append(ef) except TimeoutError: log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) break @@ -1383,6 +1392,63 @@ class MiniMateProtocol: log.debug("TX %d bytes: %s", len(frame), frame.hex()) self._transport.write(frame) + def _recv_5a_batch( + self, + expected_sub: int, + first_timeout: float = 10.0, + batch_timeout: float = 0.5, + ) -> list[S3Frame]: + """ + Collect all S3 frames that arrive as part of one device response. + + Over TCP via cellular modem, a single device A5 response (~1100 bytes of + RS-232 data) is forwarded in multiple TCP segments due to the modem's + data-forwarding timeout (~100-150 ms per segment). Each TCP segment + contains a complete, valid S3 frame (~550 bytes). Calling _recv_one() + once returns only the first segment's frame and misses the rest, causing + the chunk request/response pairing to cascade out of alignment. + + This helper collects ALL frames before returning, by trying additional + short-timeout receives after the first frame arrives. + + The caller must call self._parser.reset() before this method to ensure + bytes_fed is accurate; this method always uses reset_parser=False. + + Args: + expected_sub: Expected SUB byte for validation. + first_timeout: Timeout for the mandatory first frame. Should be + generous (default 10 s) since the device may be slow. + batch_timeout: Short timeout for subsequent frames. Default 0.5 s + — comfortably longer than the modem forwarding gap + (~150 ms) but short enough to avoid stalling when + only one frame is expected (probe, terminator). + + Returns: + List of S3Frame objects in arrival order (at least one). + + Raises: + TimeoutError: If no frame arrives within first_timeout. + UnexpectedResponse: If any frame has the wrong SUB byte. + """ + frames: list[S3Frame] = [] + first = self._recv_one( + expected_sub=expected_sub, + reset_parser=False, + timeout=first_timeout, + ) + frames.append(first) + while True: + try: + extra = self._recv_one( + expected_sub=expected_sub, + reset_parser=False, + timeout=batch_timeout, + ) + frames.append(extra) + except TimeoutError: + break + return frames + def _recv_one( self, expected_sub: Optional[int] = None, diff --git a/sfm/server.py b/sfm/server.py index 3fc4bb2..5e10349 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -886,17 +886,14 @@ def device_event_blastware_file( with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() # Use stop_after_metadata=True (full_waveform=False) with 1 extra - # chunk after "Project:". The extra chunk is required to prime the - # device over TCP: termination at term_counter=metadata_counter+0x0400 - # returns only ~90 bytes (no useful footer) over TCP/cellular, but - # termination at metadata_counter+0x0800 (one chunk later) returns - # the full 737-byte frame containing the footer. + # chunk after "Project:". The extra chunk primes the device so that + # the termination response carries the full waveform footer bytes. + # Without it the terminator returns only ~90 bytes (no useful footer). # - # Confirmed from 4-26-26 BW RS-232 capture: BW terminates at 0x1800 - # without an extra chunk (works on RS-232 but not TCP). - # write_blastware_file() automatically skips the extra chunk's - # contribution — only the probe+ADC+metadata+terminator bytes appear - # in the output file. + # The extra chunk's ADC data IS part of the Blastware file body — + # confirmed from 4-27-26 TCP capture: all 14 A5 frames (including the + # extra chunk's 2 TCP sub-frames) contribute to the correct 6864-byte + # output. write_blastware_file() includes all frames unconditionally. # # full_waveform=True (natural end-of-stream) downloads ALL chunks # including post-event silence (35+ chunks for a 9-sec event at