fix: add new helper (_recv_5a_batch()) that helps with assembling chunks over TCP

CLAUDE.md

@@ -347,6 +347,24 @@ Do NOT use fixed absolute offsets for sample_rate or record_time.
   Quiet Mode enabled. Parser handles this — do not strip it manually before feeding to
   `S3FrameParser`.
 
+**SUB 5A (bulk waveform) TCP frame splitting — confirmed 2026-04-27:**
+
+Over TCP via cellular modem, each 5A chunk request that produces a single ~1100-byte
+A5 response over direct RS-232 arrives as **two separate, complete S3 frames** of
+~550 bytes each.  This is because the device sends its RS-232 response as multiple
+frames, and the modem's Data Forwarding Timeout (~100-150 ms) delivers them to us
+as separate TCP segments, each parsed as a complete S3 frame.
+
+Example for a 2-second Continuous event (BE11529, key=01110000) via TCP:
+- 1 probe frame (554 B) + 5 chunks × 2 frames (556-573 B) + 1 extra chunk × 2 frames + 1 terminator (208 B) = **14 A5 frames**
+- All 14 frames contribute body data; using all of them gives the correct 6864-byte file.
+
+**Fix (confirmed 2026-04-27):**  `_recv_5a_batch()` in `protocol.py` collects ALL
+A5 frames per chunk request before the next request is sent, using a 0.5 s batch
+timeout after the first frame to catch the ~150 ms delayed second frame.  `write_blastware_file()`
+includes ALL body frames without skipping — the extra chunk's frames are part of the
+body data, NOT padding to be discarded.
+
 ### Required ACEmanager settings (Sierra Wireless RV50/RV55)
 
 | Setting | Value | Why |

minimateplus/blastware_file.py

@@ -685,64 +685,34 @@ def write_blastware_file(
         body_frames = a5_frames
         term_frame  = None
 
-    # ── Identify first metadata frame and skip "extra chunks" ───────────────
-    # When extra_chunks_after_metadata=1 in read_bulk_waveform_stream(), the
-    # frame list is: [probe, data..., metadata, extra_chunk, terminator].
-    # The extra_chunk is downloaded to prime the TCP terminator response — its
-    # ADC data is NOT part of the Blastware file body.  Skip it.
-    #
-    # Rule: any frame at index strictly between first_metadata_fi and last_fi
-    #       (the final frame) is an extra chunk and must be excluded.
-    #
-    # If no metadata frame exists (e.g. full_waveform download), first_metadata_fi
-    # is None and no frames are skipped — all frames contribute normally.
-    first_metadata_fi: Optional[int] = None
-    for _fi_scan, _frame_scan in enumerate(body_frames):
-        if _fi_scan > 0 and any(m in bytes(_frame_scan.data) for m in _METADATA_FRAME_MARKERS):
-            first_metadata_fi = _fi_scan
-            break
-    last_fi = len(body_frames) - 1
-
     log.warning(
-        "write_blastware_file: %d body_frames  first_metadata_fi=%s  last_fi=%d",
+        "write_blastware_file: %d body_frames  term_idx=%s",
         len(body_frames),
-        str(first_metadata_fi) if first_metadata_fi is not None else "None",
+        str(term_idx) if term_idx is not None else "None",
-        last_fi,
     )
 
     all_bytes = bytearray()
 
     for fi, frame in enumerate(body_frames):
-        # Skip "extra chunk" frames: frames after the first metadata frame but
+        # All body frames contribute to the waveform body — no frames are skipped.
-        # before the last frame (terminator).  These prime the TCP terminator but
-        # their ADC data must NOT appear in the Blastware file body.
-        if (first_metadata_fi is not None
-                and fi > first_metadata_fi
-                and fi < last_fi):
-            log.warning(
-                "write_blastware_file: fi=%d SKIP (extra chunk after metadata fi=%d last_fi=%d)",
-                fi, first_metadata_fi, last_fi,
-            )
-            continue
-
-        if fi == 0:
-            # Probe frame: always process regardless of classification.
-            # It holds the STRT record; probe_skip positions us past it.
-            skip = probe_skip
-        else:
-            # ALL subsequent frames are included unconditionally — no filtering on
-            # frame type.  In the A5 stream, frame 0 is always the probe response;
-            # frames 1+ are always data (waveform chunks, compliance config, or
-            # compliance continuation).  Classification is for logging only.
         #
-            # DO NOT gate on classify_frame() here:
+        # Over TCP via cellular modem, _recv_5a_batch() correctly collects all
-            #   - "probe_or_strt" at fi>0 is always a false positive — ADC binary
+        # A5 frames per chunk request (the device's ~1100-byte RS-232 response
-            #     data can coincidentally contain b"STRT\xff\xfe" (confirmed from
+        # is forwarded as ~2 TCP segments of ~550 bytes each, each parsed as a
-            #     live capture: frames 1 and 5 matched on event key=01110000).
+        # separate S3 frame).  ALL of these frames contain ADC body data and
-            #   - "metadata" frames must be included (compliance config body).
+        # must be included in the file — confirmed from 4-27-26 TCP capture
-            #   - The compliance block spans 2 frames; skipping either produces a
+        # analysis: contributions from all 14 frames → 6821 bytes → file 6864 bytes.
-            #     truncated file that Blastware rejects.
+        #
-            skip = 13 if fi == 1 else 12
+        # Skip amounts (offsets into frame.data):
+        #   fi=0 (probe): probe_skip — skips the type_tag header + STRT record
+        #   fi=1:         13         — 7-byte frame.data prefix + 6 inner header bytes
+        #   fi>=2:        12         — 7-byte frame.data prefix + 5 inner header bytes
+        if fi == 0:
+            skip = probe_skip
+        elif fi == 1:
+            skip = 13
+        else:
+            skip = 12
 
         contribution = _frame_body_bytes(frame, skip)
         log.warning("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",

minimateplus/protocol.py

@@ -599,7 +599,7 @@ class MiniMateProtocol:
         self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
         self._parser.reset()        # reset bytes_fed counter before probe recv
         try:
-            rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False)
+            probe_batch = self._recv_5a_batch(rsp_sub)
         except TimeoutError:
             log.warning(
                 "5A probe TIMED OUT for key=%s — "
@@ -607,8 +607,12 @@ class MiniMateProtocol:
                 key4.hex(), self._parser.bytes_fed,
             )
             raise
-        frames_data.append(rsp)
+        frames_data.extend(probe_batch)
-        log.debug("5A A5[0]  page_key=0x%04X  %d bytes", rsp.page_key, len(rsp.data))
+        log.debug(
+            "5A probe: %d frame(s)  page_keys=%s",
+            len(probe_batch),
+            [f"0x{f.page_key:04X}" for f in probe_batch],
+        )
 
         # ── Step 2: chunk loop ───────────────────────────────────────────────
         # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400
@@ -634,7 +638,12 @@ class MiniMateProtocol:
             self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
             self._parser.reset()        # reset bytes_fed for accurate per-chunk count
             try:
-                rsp = self._recv_one(expected_sub=rsp_sub, reset_parser=False, timeout=10.0)
+                # Collect ALL frames from this chunk response.
+                # Over TCP via modem, a single large A5 device response (~1100 bytes
+                # RS-232) is split across ~2 TCP segments, each parsed as its own
+                # complete S3 frame.  _recv_5a_batch gathers all of them so that
+                # every subsequent chunk request is paired with the correct response.
+                batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0)
             except TimeoutError:
                 raw = self._parser.bytes_fed
                 log.warning(
@@ -653,48 +662,48 @@ class MiniMateProtocol:
                     break
                 raise
 
+            # Process all frames from this batch.
+            metadata_found = False
+            for rsp in batch:
                 log.warning(
                     "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s",
                     chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data,
                 )
-
                 if rsp.page_key == 0x0000:
-                # Device unexpectedly terminated mid-stream (no termination needed).
+                    # Device unexpectedly terminated mid-stream.
-                log.debug("5A A5[%d] page_key=0x0000 — device terminated early", chunk_num)
+                    log.debug("5A page_key=0x0000 — device terminated early")
                     if include_terminator:
                         frames_data.append(rsp)
                     return frames_data
-
                 frames_data.append(rsp)
-
                 if stop_after_metadata and b"Project:" in rsp.data:
-                # Download exactly one more chunk after finding metadata — this is
+                    metadata_found = True
-                # what Blastware does.  The extra chunk contains the tail ADC data
+
-                # and primes the device to return a valid footer in the termination
+            if metadata_found:
-                # response.  Without it, termination returns an empty ack with no
+                # Download extra_chunks_after_metadata more chunks after metadata.
-                # footer bytes (confirmed 2026-04-23 from HxD comparison).
+                # This primes the device to return the valid waveform footer in the
-                # Download extra_chunks_after_metadata more chunks past the
+                # termination response — without it the terminator carries too few bytes
-                # metadata.  The caller calculates this from record_time and
+                # (confirmed 2026-04-23).  The extra chunk data also belongs in the
-                # sample_rate so we download exactly the right amount of ADC
+                # file body (confirmed from TCP capture analysis 2026-04-27).
-                # data — no more, no less — before terminating.
+                log.debug("5A metadata found — fetching %d more chunk(s)",
-                # The device returns the footer in the termination response only
+                          extra_chunks_after_metadata)
-                # after the right amount of data has been consumed.
-                log.debug("5A A5[%d] metadata found — fetching %d more chunk(s)",
-                          chunk_num, extra_chunks_after_metadata)
                 for _extra_n in range(extra_chunks_after_metadata):
                     chunk_num += 1
                     counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP
                     params = bulk_waveform_params(key4, counter)
                     self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params))
                     try:
-                        extra = self._recv_one(expected_sub=rsp_sub, timeout=10.0)
+                        extra_batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0)
-                        log.debug("5A A5[%d] extra chunk page_key=0x%04X data_len=%d",
+                        for ef in extra_batch:
-                                  chunk_num, extra.page_key, len(extra.data))
+                            log.debug(
-                        if extra.page_key == 0x0000:
+                                "5A extra chunk page_key=0x%04X data_len=%d",
+                                ef.page_key, len(ef.data),
+                            )
+                            if ef.page_key == 0x0000:
                                 if include_terminator:
-                                frames_data.append(extra)
+                                    frames_data.append(ef)
                                 return frames_data
-                        frames_data.append(extra)
+                            frames_data.append(ef)
                     except TimeoutError:
                         log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1)
                         break
@@ -1383,6 +1392,63 @@ class MiniMateProtocol:
         log.debug("TX %d bytes: %s", len(frame), frame.hex())
         self._transport.write(frame)
 
+    def _recv_5a_batch(
+        self,
+        expected_sub: int,
+        first_timeout: float = 10.0,
+        batch_timeout: float = 0.5,
+    ) -> list[S3Frame]:
+        """
+        Collect all S3 frames that arrive as part of one device response.
+
+        Over TCP via cellular modem, a single device A5 response (~1100 bytes of
+        RS-232 data) is forwarded in multiple TCP segments due to the modem's
+        data-forwarding timeout (~100-150 ms per segment).  Each TCP segment
+        contains a complete, valid S3 frame (~550 bytes).  Calling _recv_one()
+        once returns only the first segment's frame and misses the rest, causing
+        the chunk request/response pairing to cascade out of alignment.
+
+        This helper collects ALL frames before returning, by trying additional
+        short-timeout receives after the first frame arrives.
+
+        The caller must call self._parser.reset() before this method to ensure
+        bytes_fed is accurate; this method always uses reset_parser=False.
+
+        Args:
+            expected_sub:   Expected SUB byte for validation.
+            first_timeout:  Timeout for the mandatory first frame.  Should be
+                            generous (default 10 s) since the device may be slow.
+            batch_timeout:  Short timeout for subsequent frames.  Default 0.5 s
+                            — comfortably longer than the modem forwarding gap
+                            (~150 ms) but short enough to avoid stalling when
+                            only one frame is expected (probe, terminator).
+
+        Returns:
+            List of S3Frame objects in arrival order (at least one).
+
+        Raises:
+            TimeoutError:       If no frame arrives within first_timeout.
+            UnexpectedResponse: If any frame has the wrong SUB byte.
+        """
+        frames: list[S3Frame] = []
+        first = self._recv_one(
+            expected_sub=expected_sub,
+            reset_parser=False,
+            timeout=first_timeout,
+        )
+        frames.append(first)
+        while True:
+            try:
+                extra = self._recv_one(
+                    expected_sub=expected_sub,
+                    reset_parser=False,
+                    timeout=batch_timeout,
+                )
+                frames.append(extra)
+            except TimeoutError:
+                break
+        return frames
+
     def _recv_one(
         self,
         expected_sub: Optional[int] = None,

sfm/server.py

@@ -886,17 +886,14 @@ def device_event_blastware_file(
             with _build_client(port, baud, host, tcp_port, timeout=120.0) as client:
                 info = client.connect()
                 # Use stop_after_metadata=True (full_waveform=False) with 1 extra
-                # chunk after "Project:".  The extra chunk is required to prime the
+                # chunk after "Project:".  The extra chunk primes the device so that
-                # device over TCP: termination at term_counter=metadata_counter+0x0400
+                # the termination response carries the full waveform footer bytes.
-                # returns only ~90 bytes (no useful footer) over TCP/cellular, but
+                # Without it the terminator returns only ~90 bytes (no useful footer).
-                # termination at metadata_counter+0x0800 (one chunk later) returns
-                # the full 737-byte frame containing the footer.
                 #
-                # Confirmed from 4-26-26 BW RS-232 capture: BW terminates at 0x1800
+                # The extra chunk's ADC data IS part of the Blastware file body —
-                # without an extra chunk (works on RS-232 but not TCP).
+                # confirmed from 4-27-26 TCP capture: all 14 A5 frames (including the
-                # write_blastware_file() automatically skips the extra chunk's
+                # extra chunk's 2 TCP sub-frames) contribute to the correct 6864-byte
-                # contribution — only the probe+ADC+metadata+terminator bytes appear
+                # output.  write_blastware_file() includes all frames unconditionally.
-                # in the output file.
                 #
                 # full_waveform=True (natural end-of-stream) downloads ALL chunks
                 # including post-event silence (35+ chunks for a 9-sec event at