feat(cache): implement integrity checks for cached events and waveforms
- Added `waveform_key` and `event_timestamp` columns to `CachedEvent` and `CachedWaveform` for integrity verification. - Implemented logic to flush the cache when a mismatch in (waveform_key, event_timestamp) is detected during event and waveform updates. - Enhanced `set_events` and `set_waveform` methods to check for mismatches and trigger cache eviction as necessary. - Introduced a new `LiveCache` class to manage in-memory caching of live device data, separating it from the server logic for better testability. - Added tests to verify the correctness of cache invalidation logic, particularly for post-erase key reuse scenarios. - Updated web application to include a "Force refresh" toggle, allowing users to bypass the cache and re-fetch data from the device.
This commit is contained in:
+124
-85
@@ -449,7 +449,7 @@ class MiniMateClient:
|
||||
proto.confirm_erase_all()
|
||||
log.info("delete_all_events: erase confirmed — device memory cleared")
|
||||
|
||||
def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None, extra_chunks_after_metadata: int = 1) -> list[Event]:
|
||||
def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None, skip_waveform_for_events: Optional[dict] = None, extra_chunks_after_metadata: int = 1) -> list[Event]:
|
||||
"""
|
||||
Download all stored events from the device using the confirmed
|
||||
1E → 0A → 0C → 5A → 1F event-iterator protocol.
|
||||
@@ -497,37 +497,24 @@ class MiniMateClient:
|
||||
events: list[Event] = []
|
||||
idx = 0
|
||||
|
||||
# Legacy bare-key skip set is deprecated: the device's key counter
|
||||
# resets to 0x01110000 after every memory erase, so a key in this set
|
||||
# cannot be trusted to identify the same physical event across erases.
|
||||
# If a caller still passes it, log a warning and ignore — full
|
||||
# downloads will run for every event so the bug never silently bites.
|
||||
if skip_waveform_for_keys:
|
||||
log.warning(
|
||||
"get_events: skip_waveform_for_keys is deprecated and unsafe "
|
||||
"(post-erase key reuse); ignoring %d entries. Use "
|
||||
"skip_waveform_for_events={key: timestamp_iso} instead.",
|
||||
len(skip_waveform_for_keys),
|
||||
)
|
||||
skip_evts: dict[str, str] = dict(skip_waveform_for_events or {})
|
||||
|
||||
while data8[4:8] != b"\x00\x00\x00\x00":
|
||||
cur_key = key4 # key for this event's 0A/1E-arm/0C/5A calls
|
||||
log.info("get_events: record %d key=%s", idx, cur_key.hex())
|
||||
|
||||
# Fast-advance path: if this key is already downloaded, skip
|
||||
# 1E-arm/0C/POLL/5A entirely. Only 0A + 1F(browse) are needed
|
||||
# to advance the device's internal pointer to the next event.
|
||||
# This is identical to the browse-mode walk in count_events().
|
||||
if skip_waveform_for_keys and cur_key.hex() in skip_waveform_for_keys:
|
||||
log.debug("get_events: key=%s already seen -- fast-advance only", cur_key.hex())
|
||||
try:
|
||||
proto.read_waveform_header(cur_key)
|
||||
except ProtocolError as exc:
|
||||
log.warning(
|
||||
"get_events: 0A failed for key=%s (skip path): %s -- stopping",
|
||||
cur_key.hex(), exc,
|
||||
)
|
||||
break
|
||||
try:
|
||||
key4, data8 = proto.advance_event(browse=True)
|
||||
except ProtocolError as exc:
|
||||
log.warning(
|
||||
"get_events: 1F failed for key=%s (skip path): %s -- stopping",
|
||||
cur_key.hex(), exc,
|
||||
)
|
||||
break
|
||||
idx += 1
|
||||
if stop_after_index is not None and idx > stop_after_index:
|
||||
break
|
||||
continue
|
||||
|
||||
ev = Event(index=idx)
|
||||
ev._waveform_key = cur_key
|
||||
|
||||
@@ -574,72 +561,96 @@ class MiniMateClient:
|
||||
"get_events: 0C failed for key=%s: %s", cur_key.hex(), exc
|
||||
)
|
||||
|
||||
# SUB 1F (download-arm) — send token=0xFE BEFORE POLL+5A to arm the
|
||||
# device's bulk stream state machine. Cache the returned key as a
|
||||
# fallback for loop iteration when 5A fails (see iteration block below).
|
||||
# Confirmed from 4-2-26 capture frames 66-67 (1F before frames 68-73 POLL).
|
||||
arm_key4: Optional[bytes] = None
|
||||
try:
|
||||
arm_key4, _ = proto.advance_event(browse=False) # arm 5A
|
||||
log.info("get_events: 1F(download) — 5A armed, arm_key=%s", arm_key4.hex())
|
||||
except ProtocolError as exc:
|
||||
log.warning("get_events: 1F(download) arm failed: %s", exc)
|
||||
# ── Skip-5A decision based on (key, timestamp) match ──────
|
||||
# If skip_waveform_for_events maps cur_key.hex() to a non-empty
|
||||
# ISO timestamp matching what we just read from 0C, this is
|
||||
# the same physical event we already have on disk — bypass
|
||||
# the 1F(arm)+POLL+5A bulk download. Otherwise (no entry, or
|
||||
# timestamp mismatch indicating post-erase reuse) fall through
|
||||
# to the full download.
|
||||
expected_ts = skip_evts.get(cur_key.hex(), "")
|
||||
actual_ts = _event_timestamp_iso(ev)
|
||||
skip_5a = bool(expected_ts and actual_ts and expected_ts == actual_ts)
|
||||
if skip_5a:
|
||||
log.info(
|
||||
"get_events: key=%s (key, ts=%s) match — skipping 5A bulk download",
|
||||
cur_key.hex(), actual_ts,
|
||||
)
|
||||
|
||||
# POLL × 3 — BW sends 3 full POLL cycles between 1F and 5A.
|
||||
# Confirmed from 4-2-26 BW TX capture (frames 68-73 before 5A at 74).
|
||||
log.info("get_events: POLL × 3 before 5A")
|
||||
for _p in range(3):
|
||||
arm_key4: Optional[bytes] = None
|
||||
a5_ok = False
|
||||
|
||||
if not skip_5a:
|
||||
# SUB 1F (download-arm) — send token=0xFE BEFORE POLL+5A to arm the
|
||||
# device's bulk stream state machine. Cache the returned key as a
|
||||
# fallback for loop iteration when 5A fails (see iteration block below).
|
||||
# Confirmed from 4-2-26 capture frames 66-67 (1F before frames 68-73 POLL).
|
||||
try:
|
||||
proto.poll()
|
||||
arm_key4, _ = proto.advance_event(browse=False) # arm 5A
|
||||
log.info("get_events: 1F(download) — 5A armed, arm_key=%s", arm_key4.hex())
|
||||
except ProtocolError as exc:
|
||||
log.warning("get_events: POLL %d failed: %s", _p, exc)
|
||||
log.warning("get_events: 1F(download) arm failed: %s", exc)
|
||||
|
||||
# POLL × 3 — BW sends 3 full POLL cycles between 1F and 5A.
|
||||
# Confirmed from 4-2-26 BW TX capture (frames 68-73 before 5A at 74).
|
||||
log.info("get_events: POLL × 3 before 5A")
|
||||
for _p in range(3):
|
||||
try:
|
||||
proto.poll()
|
||||
except ProtocolError as exc:
|
||||
log.warning("get_events: POLL %d failed: %s", _p, exc)
|
||||
|
||||
# SUB 5A — bulk waveform stream (uses cur_key, the event set up by 0A+1E+0C).
|
||||
# By default (full_waveform=False): stop after frame 7 for metadata only.
|
||||
# When full_waveform=True: fetch all chunks and decode raw ADC samples.
|
||||
a5_ok = False
|
||||
try:
|
||||
if full_waveform:
|
||||
log.info(
|
||||
"get_events: 5A full waveform download for key=%s", cur_key.hex()
|
||||
)
|
||||
a5_frames = proto.read_bulk_waveform_stream(
|
||||
cur_key, stop_after_metadata=False, max_chunks=128,
|
||||
include_terminator=True,
|
||||
)
|
||||
if a5_frames:
|
||||
a5_ok = True
|
||||
ev._a5_frames = a5_frames # store for write_blastware_file
|
||||
_decode_a5_metadata_into(a5_frames, ev)
|
||||
_decode_a5_waveform(a5_frames, ev)
|
||||
#
|
||||
# Bypassed when skip_5a is True — the event is left with
|
||||
# _a5_frames=None, which signals to the caller (e.g.
|
||||
# ach_server.py) that this event was matched by (key, ts) and
|
||||
# already has a stored .file in the persistent waveform store.
|
||||
if not skip_5a:
|
||||
try:
|
||||
if full_waveform:
|
||||
log.info(
|
||||
"get_events: 5A decoded %d sample-sets",
|
||||
len((ev.raw_samples or {}).get("Tran", [])),
|
||||
"get_events: 5A full waveform download for key=%s", cur_key.hex()
|
||||
)
|
||||
else:
|
||||
log.info(
|
||||
"get_events: 5A metadata-only download for key=%s", cur_key.hex()
|
||||
)
|
||||
a5_frames = proto.read_bulk_waveform_stream(
|
||||
cur_key, stop_after_metadata=True,
|
||||
include_terminator=True,
|
||||
extra_chunks_after_metadata=extra_chunks_after_metadata,
|
||||
max_chunks=128,
|
||||
)
|
||||
if a5_frames:
|
||||
a5_ok = True
|
||||
ev._a5_frames = a5_frames # store for write_blastware_file
|
||||
_decode_a5_metadata_into(a5_frames, ev)
|
||||
log.debug(
|
||||
"get_events: 5A metadata client=%r operator=%r",
|
||||
ev.project_info.client if ev.project_info else None,
|
||||
ev.project_info.operator if ev.project_info else None,
|
||||
a5_frames = proto.read_bulk_waveform_stream(
|
||||
cur_key, stop_after_metadata=False, max_chunks=128,
|
||||
include_terminator=True,
|
||||
)
|
||||
except ProtocolError as exc:
|
||||
log.warning(
|
||||
"get_events: 5A failed for key=%s: %s — metadata unavailable",
|
||||
cur_key.hex(), exc,
|
||||
)
|
||||
if a5_frames:
|
||||
a5_ok = True
|
||||
ev._a5_frames = a5_frames # store for write_blastware_file
|
||||
_decode_a5_metadata_into(a5_frames, ev)
|
||||
_decode_a5_waveform(a5_frames, ev)
|
||||
log.info(
|
||||
"get_events: 5A decoded %d sample-sets",
|
||||
len((ev.raw_samples or {}).get("Tran", [])),
|
||||
)
|
||||
else:
|
||||
log.info(
|
||||
"get_events: 5A metadata-only download for key=%s", cur_key.hex()
|
||||
)
|
||||
a5_frames = proto.read_bulk_waveform_stream(
|
||||
cur_key, stop_after_metadata=True,
|
||||
include_terminator=True,
|
||||
extra_chunks_after_metadata=extra_chunks_after_metadata,
|
||||
max_chunks=128,
|
||||
)
|
||||
if a5_frames:
|
||||
a5_ok = True
|
||||
ev._a5_frames = a5_frames # store for write_blastware_file
|
||||
_decode_a5_metadata_into(a5_frames, ev)
|
||||
log.debug(
|
||||
"get_events: 5A metadata client=%r operator=%r",
|
||||
ev.project_info.client if ev.project_info else None,
|
||||
ev.project_info.operator if ev.project_info else None,
|
||||
)
|
||||
except ProtocolError as exc:
|
||||
log.warning(
|
||||
"get_events: 5A failed for key=%s: %s — metadata unavailable",
|
||||
cur_key.hex(), exc,
|
||||
)
|
||||
|
||||
# SUB 1F — loop iteration.
|
||||
#
|
||||
@@ -652,7 +663,14 @@ class MiniMateClient:
|
||||
# Confirmed from 4-3-26 browse-mode captures: browse=True params
|
||||
# are correct for multi-event iteration. Conditional logic added
|
||||
# 2026-04-06 to avoid post-failure state disruption.
|
||||
if a5_ok:
|
||||
#
|
||||
# NEW 2026-05-06: when skip_5a=True we never entered the 5A
|
||||
# state at all (we read 0A+1E(arm)+0C and chose to bypass).
|
||||
# 1F(browse) is safe in this scenario — the device's iteration
|
||||
# pointer is independent of the bulk-stream state machine, and
|
||||
# we never put it into the half-attempted 5A state that the
|
||||
# earlier "post-failure 1F disruption" warning is about.
|
||||
if skip_5a or a5_ok:
|
||||
# 5A succeeded — use browse 1F for reliable key advancement.
|
||||
try:
|
||||
key4, data8 = proto.advance_event(browse=True)
|
||||
@@ -1174,6 +1192,27 @@ class MiniMateClient:
|
||||
# Pure functions: bytes → model field population.
|
||||
# Kept here (not in models.py) to isolate protocol knowledge from data shapes.
|
||||
|
||||
def _event_timestamp_iso(event: Event) -> str:
|
||||
"""
|
||||
Return a stable ISO-8601 string for the event's 0C-derived timestamp,
|
||||
or "" if the event has no timestamp populated.
|
||||
|
||||
The format intentionally matches what `bridges/ach_server.py` writes
|
||||
into `ach_state.json:downloaded_events[*]` so the (key, ts) compare
|
||||
in get_events()'s skip path is a simple string equality.
|
||||
"""
|
||||
ts = getattr(event, "timestamp", None)
|
||||
if ts is None:
|
||||
return ""
|
||||
try:
|
||||
return datetime.datetime(
|
||||
ts.year, ts.month, ts.day,
|
||||
ts.hour or 0, ts.minute or 0, ts.second or 0,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
return str(ts)
|
||||
|
||||
|
||||
def _decode_serial_number(data: bytes) -> DeviceInfo:
|
||||
"""
|
||||
Decode SUB EA (SERIAL_NUMBER_RESPONSE) payload into a new DeviceInfo.
|
||||
|
||||
Reference in New Issue
Block a user