feat: add waveform store handling #16

Merged
serversdown merged 5 commits from sfm-waveform-store into main 2026-05-08 15:03:33 -04:00
2 changed files with 79 additions and 34 deletions
Showing only changes of commit bbed85f7e2 - Show all commits
+78 -33
View File
@@ -1572,59 +1572,106 @@ def _decode_a5_waveform(
# STRT byte layout (21 bytes; verified against M529LIY6 reference files # STRT byte layout (21 bytes; verified against M529LIY6 reference files
# and re-confirmed against live BE11529 captures, 2026-05-08): # and re-confirmed against live BE11529 captures, 2026-05-08):
# [0:4] b'STRT' # [0:4] b'STRT'
# [4:6] 0xff 0xfe fixed # [4:6] 0xff 0xfe sentinel
# [6:10] end_key (4-byte device flash address where event ends) # [6:10] end_key 4-byte BE flash address where event ends
# [10:14] start_key (4-byte device flash address where event starts) # [10:14] start_key 4-byte BE flash address where event starts
# [14:18] device-specific (4 bytes; semantics not pinned) # [14:18] device-specific (semantics not pinned; values vary across events
# [18] 0x46 record-type marker (= 70 in decimal — NOT rectime!) # and don't hold authoritative total_samples / pretrig)
# [18] 0x46 record-type marker (NOT rectime)
# [19] device-specific # [19] device-specific
# [20] rectime (uint8 seconds, user-set Record Time) # [20] sometimes rectime, sometimes 0 — not reliable
# #
# The earlier reading of `rectime_seconds = strt[18]` always returned # AUTHORITATIVE values must come from compliance_config (sample_rate,
# 70 for a real waveform event because it was reading the 0x46 marker. # record_time) and from end_offset - start_offset arithmetic (event size).
# Caller should prefer compliance_config.record_time when available # Earlier code claimed STRT[8:10]=total_samples and STRT[16:18]=pretrig;
# (that's the authoritative user-set value) and fall back to this. # those positions actually overlap end_key low-word and dev-specific bytes
total_samples = struct.unpack_from(">H", strt, 8)[0] # respectively. We surface the address-derived event size so consumers
pretrig_samples = struct.unpack_from(">H", strt, 16)[0] # can sanity-check chunk-loop bounds, but `total_samples` per channel must
rectime_seconds = strt[20] # be derived externally (sample_rate × record_time, or computed from the
# decoded sample count below).
end_key = strt[6:10]
start_key = strt[10:14]
end_offset_in_strt = (end_key[2] << 8) | end_key[3]
start_offset_in_strt = (start_key[2] << 8) | start_key[3]
is_event_1 = (start_offset_in_strt == 0x0000)
event.total_samples = total_samples # Don't trust STRT for these — leave them as None so the caller can
event.pretrig_samples = pretrig_samples # backfill from compliance_config (the authoritative source).
event.rectime_seconds = rectime_seconds event.total_samples = None
event.pretrig_samples = None
event.rectime_seconds = None
log.debug( log.debug(
"_decode_a5_waveform: STRT total_samples=%d pretrig=%d rectime=%ds " "_decode_a5_waveform: STRT start_key=%s end_key=%s "
"(strt[18]=0x%02X record-type marker, strt[20]=0x%02X rectime)", "start_off=0x%04X end_off=0x%04X is_event_1=%s "
total_samples, pretrig_samples, rectime_seconds, strt[18], strt[20], "dev-specific[14:18]=%s strt[20]=0x%02X",
start_key.hex(), end_key.hex(),
start_offset_in_strt, end_offset_in_strt, is_event_1,
strt[14:18].hex(), strt[20],
) )
# ── Collect per-frame waveform bytes with global offset tracking ───────── # ── Collect per-frame waveform bytes with global offset tracking ─────────
# global_offset is the cumulative byte count across all frames, used to # global_offset is the cumulative byte count across all frames, used to
# compute the channel alignment at each frame boundary. # compute the channel alignment at each frame boundary.
#
# Frame layout under the v0.14.0+ walk:
# frames_data[0] = probe response (page_addr 0x0000;
# contains STRT + post-STRT data)
# frames_data[1..2] = (event 1 only) metadata pages
# page_addr = 0x1002 / 0x1004
# frames_data[mid] = sample chunks at flash addresses
# 0x0600, 0x0800, … (page_addr in
# {0x0600..0x1FFE})
# frames_data[last] = TERM response (page_key=0x0000)
#
# We identify metadata pages by their PAGE ADDRESS at db.data[4:6] (the
# 2-byte counter the device echoes back), NOT by content scan. An earlier
# needle-based detection (b"Project:", b"Client:", etc.) was the wrong
# layer of abstraction:
# • The actual metadata pages 0x1002 / 0x1004 do NOT contain ASCII
# project strings on this firmware (S338.17 / BE11529).
# • The strings physically live at flash address 0x1600 — which falls
# inside the sample-chunk address range. Skipping that frame would
# drop a real sample chunk.
# BW handles the "samples region happens to contain string bytes" case
# by just rendering the bytes verbatim; we do the same.
_METADATA_PAGES = (b"\x10\x02", b"\x10\x04")
chunks: list[tuple[int, bytes]] = [] # (frame_idx, waveform_bytes) chunks: list[tuple[int, bytes]] = [] # (frame_idx, waveform_bytes)
global_offset = 0 global_offset = 0
for fi, db in enumerate(frames_data): for fi, db in enumerate(frames_data):
page_addr = db.data[4:6] if len(db.data) >= 6 else b""
w = db.data[7:] # frame.data[7:] w = db.data[7:] # frame.data[7:]
# A5[0]: waveform begins after the 21-byte STRT record and 6-byte preamble. # A5[0]: probe response. Two cases:
# Layout: STRT(21B) + null-pad(2B) + 0xFF sentinel(4B) = 27 bytes total. # - Event 1 (start_offset_in_strt == 0x0000): the bytes after STRT
# are the device's *pre-event reserved area* (flash 0x0046 to
# 0x0600), NOT samples. We must skip them; samples begin at
# the first dedicated chunk frame at counter=0x0600.
# - Event N (continuation, start_offset != 0x0000): the bytes after
# the STRT record ARE the first slice of real samples for the
# event (BW's chunk loop addresses the probe as a sample chunk).
if fi == 0: if fi == 0:
sp = w.find(b"STRT") sp = w.find(b"STRT")
if sp < 0: if sp < 0:
continue continue
if is_event_1:
# No usable samples in the probe — pre-event reserved bytes.
continue
# Layout: STRT(21B) + null-pad(2B) + 0xFF sentinel(4B) = 27 bytes total.
wave = w[sp + 27 :] wave = w[sp + 27 :]
# Frame 7 carries event-time metadata strings ("Project:", "Client:", …) # Skip the dedicated metadata pages (event 1 only): page_addr 0x1002 / 0x1004.
# and no waveform ADC data. elif page_addr in _METADATA_PAGES:
elif fi == 7: log.debug(
"_decode_a5_waveform: skipping metadata page fi=%d page_addr=%s",
fi, page_addr.hex(),
)
continue continue
# Terminator frames have page_key=0x0000 and are excluded upstream # Sample chunk (or TERM): strip the 8-byte per-frame header.
# (read_bulk_waveform_stream returns early on page_key==0).
# No hardcoded frame-index skip here — all non-metadata frames are data.
else: else:
# Strip the 8-byte per-frame header (ctr + 6 zero bytes)
if len(w) < 8: if len(w) < 8:
continue continue
wave = w[8:] wave = w[8:]
@@ -1638,10 +1685,8 @@ def _decode_a5_waveform(
total_bytes = global_offset total_bytes = global_offset
n_sets = total_bytes // 8 n_sets = total_bytes // 8
log.debug( log.debug(
"_decode_a5_waveform: %d chunks, %dB total → %d complete sample-sets " "_decode_a5_waveform: %d chunks, %dB total → %d complete sample-sets",
"(%d of %d expected; %.0f%%)", len(chunks), total_bytes, n_sets,
len(chunks), total_bytes, n_sets, n_sets, total_samples,
100.0 * n_sets / total_samples if total_samples else 0,
) )
if n_sets == 0: if n_sets == 0:
@@ -1699,7 +1744,7 @@ def _decode_a5_waveform(
"Tran": tran, "Tran": tran,
"Vert": vert, "Vert": vert,
"Long": long_, "Long": long_,
"Mic": mic, "MicL": mic,
} }
+1 -1
View File
@@ -690,7 +690,7 @@ def device_event_waveform(
if the device is not storing all frames yet, or the capture was partial) if the device is not storing all frames yet, or the capture was partial)
- **sample_rate**: samples per second (from compliance config) - **sample_rate**: samples per second (from compliance config)
- **channels**: dict of channel name → list of signed int16 ADC counts - **channels**: dict of channel name → list of signed int16 ADC counts
(keys: "Tran", "Vert", "Long", "Mic") (keys: "Tran", "Vert", "Long", "MicL")
**Caching**: full waveforms are cached permanently after the first download — **Caching**: full waveforms are cached permanently after the first download —
they are immutable once recorded on the device. Subsequent requests for the they are immutable once recorded on the device. Subsequent requests for the