diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 6f2e874..b09b639 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -526,7 +526,7 @@ class MiniMateProtocol: self, key4: bytes, *, - stop_after_metadata: bool = True, + stop_after_metadata: bool = False, max_chunks: int = 32, include_terminator: bool = False, extra_chunks_after_metadata: int = 1, @@ -625,106 +625,166 @@ class MiniMateProtocol: _effective_extra_chunks, ) - # ── Step 2: chunk loop ─────────────────────────────────────────────── - # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 - # where _chunk_base = max(key4[2:4], 0x0400). - # - # For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a): - # _chunk_base = 0x245a → chunk 1=0x245a, chunk 2=0x285a, ... - # Confirmed from 4-3-26 capture. - # - # For events with key4[2:4] == 0 (e.g. key 01110000): - # _chunk_base = max(0, 0x0400) = 0x0400 - # → chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400) - # CRITICAL: counter=0x0000 (same as the probe) causes the device to - # re-return the STRT record data for chunk 1, making frame 1 look like - # a second probe response (confirmed from server log: frame 1 len=1097, - # contains STRT\xff\xfe, contributes zero body bytes after DLE-strip). - # counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06). + # # ── Step 2: chunk loop ─────────────────────────────────────────────── + # # Counter formula: _chunk_base + (chunk_num - 1) * 0x0400 + # # where _chunk_base = max(key4[2:4], 0x0400). + # # + # # For events with key4[2:4] != 0 (e.g. key 0111245a, offset 0x245a): + # # _chunk_base = 0x245a → chunk 1=0x245a, chunk 2=0x285a, ... + # # Confirmed from 4-3-26 capture. + # # + # # For events with key4[2:4] == 0 (e.g. key 01110000): + # # _chunk_base = max(0, 0x0400) = 0x0400 + # # → chunk 1=0x0400, chunk 2=0x0800, ... (= old chunk_num*0x0400) + # # CRITICAL: counter=0x0000 (same as the probe) causes the device to + # # re-return the STRT record data for chunk 1, making frame 1 look like + # # a second probe response (confirmed from server log: frame 1 len=1097, + # # contains STRT\xff\xfe, contributes zero body bytes after DLE-strip). + # # counter=0x0400 for chunk 1 confirmed working (empirical test 2026-04-06). + # _chunk_base = max(_key4_offset, _BULK_COUNTER_STEP) + # for chunk_num in range(1, max_chunks + 1): + # counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP + # params = bulk_waveform_params(key4, counter) + # log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) + # self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) + # self._parser.reset() # reset bytes_fed for accurate per-chunk count + # try: + # # Collect ALL frames from this chunk response. + # # Over TCP via modem, a single large A5 device response (~1100 bytes + # # RS-232) is split across ~2 TCP segments, each parsed as its own + # # complete S3 frame. _recv_5a_batch gathers all of them so that + # # every subsequent chunk request is paired with the correct response. + # batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) + # except TimeoutError: + # raw = self._parser.bytes_fed + # log.warning( + # "5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d", + # chunk_num, counter, raw, + # ) + # if raw > 0 and frames_data: + # # Device sent a partial byte (likely a bare DLE/ETX end-of-stream + # # signal) but never completed a full frame. Treat as graceful + # # stream end and fall through to the termination step. + # log.warning( + # "5A end-of-stream detected at chunk=%d (raw_bytes=%d, " + # "frames_collected=%d) — proceeding to termination", + # chunk_num, raw, len(frames_data), + # ) + # break + # raise + + # # Process all frames from this batch. + # metadata_found = False + # for rsp in batch: + # log.warning( + # "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", + # chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, + # ) + # if rsp.page_key == 0x0000: + # # Device unexpectedly terminated mid-stream. + # log.debug("5A page_key=0x0000 — device terminated early") + # if include_terminator: + # frames_data.append(rsp) + # return frames_data + # frames_data.append(rsp) + # if stop_after_metadata and b"Project:" in rsp.data: + # metadata_found = True + + # if metadata_found: + # # Download extra_chunks_after_metadata more chunks after metadata. + # # This primes the device to return the valid waveform footer in the + # # termination response — without it the terminator carries too few bytes + # # (confirmed 2026-04-23). The extra chunk data also belongs in the + # # file body (confirmed from TCP capture analysis 2026-04-27). + # log.debug("5A metadata found — fetching %d more chunk(s)", + # _effective_extra_chunks) + # for _extra_n in range(_effective_extra_chunks): + # chunk_num += 1 + # counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP + # params = bulk_waveform_params(key4, counter) + # self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) + # try: + # extra_batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) + # for ef in extra_batch: + # log.debug( + # "5A extra chunk page_key=0x%04X data_len=%d", + # ef.page_key, len(ef.data), + # ) + # if ef.page_key == 0x0000: + # if include_terminator: + # frames_data.append(ef) + # return frames_data + # frames_data.append(ef) + # except TimeoutError: + # log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) + # break + # break + # else: + # log.warning( + # "5A reached max_chunks=%d without end-of-stream; sending termination", + # max_chunks, + # ) + # ── Step 2: chunk loop ─────────────────────────────────────────────── + # TEST MODE: + # Do NOT stop on "Project:". + # Project appears inside the 5A data stream and is not an EOF marker. _chunk_base = max(_key4_offset, _BULK_COUNTER_STEP) + for chunk_num in range(1, max_chunks + 1): counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP - params = bulk_waveform_params(key4, counter) - log.debug("5A chunk %d counter=0x%04X", chunk_num, counter) + params = bulk_waveform_params(key4, counter) + + log.warning( + "5A chunk request chunk=%d counter=0x%04X", + chunk_num, + counter, + ) + self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - self._parser.reset() # reset bytes_fed for accurate per-chunk count + self._parser.reset() + try: - # Collect ALL frames from this chunk response. - # Over TCP via modem, a single large A5 device response (~1100 bytes - # RS-232) is split across ~2 TCP segments, each parsed as its own - # complete S3 frame. _recv_5a_batch gathers all of them so that - # every subsequent chunk request is paired with the correct response. batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) except TimeoutError: raw = self._parser.bytes_fed log.warning( "5A TIMEOUT chunk=%d counter=0x%04X raw_bytes=%d", - chunk_num, counter, raw, + chunk_num, + counter, + raw, ) + if raw > 0 and frames_data: - # Device sent a partial byte (likely a bare DLE/ETX end-of-stream - # signal) but never completed a full frame. Treat as graceful - # stream end and fall through to the termination step. log.warning( - "5A end-of-stream detected at chunk=%d (raw_bytes=%d, " - "frames_collected=%d) — proceeding to termination", - chunk_num, raw, len(frames_data), + "5A probable end-of-stream at chunk=%d; proceeding to termination", + chunk_num, ) break + raise - # Process all frames from this batch. - metadata_found = False for rsp in batch: + contains_project = b"Project:" in rsp.data + log.warning( "5A RX chunk=%d page_key=0x%04X data_len=%d contains_Project=%s", - chunk_num, rsp.page_key, len(rsp.data), b"Project:" in rsp.data, + chunk_num, + rsp.page_key, + len(rsp.data), + contains_project, ) + if rsp.page_key == 0x0000: - # Device unexpectedly terminated mid-stream. - log.debug("5A page_key=0x0000 — device terminated early") + log.warning("5A page_key=0x0000 — device terminated stream") if include_terminator: frames_data.append(rsp) return frames_data + frames_data.append(rsp) - if stop_after_metadata and b"Project:" in rsp.data: - metadata_found = True - - if metadata_found: - # Download extra_chunks_after_metadata more chunks after metadata. - # This primes the device to return the valid waveform footer in the - # termination response — without it the terminator carries too few bytes - # (confirmed 2026-04-23). The extra chunk data also belongs in the - # file body (confirmed from TCP capture analysis 2026-04-27). - log.debug("5A metadata found — fetching %d more chunk(s)", - _effective_extra_chunks) - for _extra_n in range(_effective_extra_chunks): - chunk_num += 1 - counter = _chunk_base + (chunk_num - 1) * _BULK_COUNTER_STEP - params = bulk_waveform_params(key4, counter) - self._send(build_5a_frame(_BULK_CHUNK_OFFSET, params)) - try: - extra_batch = self._recv_5a_batch(rsp_sub, first_timeout=10.0) - for ef in extra_batch: - log.debug( - "5A extra chunk page_key=0x%04X data_len=%d", - ef.page_key, len(ef.data), - ) - if ef.page_key == 0x0000: - if include_terminator: - frames_data.append(ef) - return frames_data - frames_data.append(ef) - except TimeoutError: - log.debug("5A extra chunk %d timed out — end of stream", _extra_n + 1) - break - break - else: - log.warning( - "5A reached max_chunks=%d without end-of-stream; sending termination", - max_chunks, - ) + # IMPORTANT: + # Do not stop on Project. It is metadata inside the stream, + # not a stream terminator. # ── Step 3: termination ────────────────────────────────────────────── term_counter = counter + _BULK_COUNTER_STEP term_params = bulk_waveform_term_params(key4, term_counter)