serversdown/seismo-relay
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cache): implement integrity checks for cached events and waveforms

Browse Source 
- Added `waveform_key` and `event_timestamp` columns to `CachedEvent` and `CachedWaveform` for integrity verification.
- Implemented logic to flush the cache when a mismatch in (waveform_key, event_timestamp) is detected during event and waveform updates.
- Enhanced `set_events` and `set_waveform` methods to check for mismatches and trigger cache eviction as necessary.
- Introduced a new `LiveCache` class to manage in-memory caching of live device data, separating it from the server logic for better testability.
- Added tests to verify the correctness of cache invalidation logic, particularly for post-erase key reuse scenarios.
- Updated web application to include a "Force refresh" toggle, allowing users to bypass the cache and re-fetch data from the device.
This commit is contained in:
serversdown
2026-05-07 04:42:00 +00:00
parent 0484680c89
commit 9afa3484f4
7 changed files with 890 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sfm/cache.py
+140 -11
View File
@@ -83,13 +83,24 @@ class CachedEvent(Base):
Events are immutable once recorded on the device; once we have an event in
the cache it never needs to be re-downloaded unless explicitly requested.
The two extra columns `waveform_key` and `event_timestamp` are an
integrity stamp: when set_event() / set_waveform() are called with a
different (waveform_key, event_timestamp) for the same (conn_key, index),
we know the device was erased and re-recorded — the cached row no longer
refers to the same physical event and the entire device's cache is
flushed before the new entry is written. This catches the post-erase
key-reuse bug where the device's first new event (key 01110000) collides
with the first event we previously downloaded.
"""
__tablename__ = "cached_events"
conn_key = sa.Column(sa.String, primary_key=True)
index = sa.Column(sa.Integer, primary_key=True)
event_json = sa.Column(sa.Text, nullable=False) # serialised Event dict
cached_at = sa.Column(sa.Float, nullable=False) # Unix timestamp
conn_key = sa.Column(sa.String, primary_key=True)
index = sa.Column(sa.Integer, primary_key=True)
event_json = sa.Column(sa.Text, nullable=False) # serialised Event dict
cached_at = sa.Column(sa.Float, nullable=False) # Unix timestamp
waveform_key = sa.Column(sa.String, nullable=True) # 8-hex device key
event_timestamp = sa.Column(sa.String, nullable=True) # ISO-8601 from 0C
class CachedWaveform(Base):
@@ -97,14 +108,18 @@ class CachedWaveform(Base):
Full raw ADC waveform for a single event (SUB 5A full download).
These are large (up to several MB) and expensive to fetch over cellular.
Once downloaded they are immutable and cached permanently.
Once downloaded they are immutable and cached permanently — but the
cache row is invalidated when the device is erased and a new event lands
at the same index (see CachedEvent docstring).
"""
__tablename__ = "cached_waveforms"
conn_key = sa.Column(sa.String, primary_key=True)
index = sa.Column(sa.Integer, primary_key=True)
waveform_json = sa.Column(sa.Text, nullable=False) # full /device/event/{idx}/waveform response JSON
cached_at = sa.Column(sa.Float, nullable=False)
conn_key = sa.Column(sa.String, primary_key=True)
index = sa.Column(sa.Integer, primary_key=True)
waveform_json = sa.Column(sa.Text, nullable=False) # full /device/event/{idx}/waveform response JSON
cached_at = sa.Column(sa.Float, nullable=False)
waveform_key = sa.Column(sa.String, nullable=True) # 8-hex device key
event_timestamp = sa.Column(sa.String, nullable=True) # ISO-8601 from 0C
class CachedMonitorStatus(Base):
@@ -149,6 +164,23 @@ class SFMCache:
engine = sa.create_engine(url, connect_args={"check_same_thread": False})
Base.metadata.create_all(engine)
self._Session = orm.sessionmaker(bind=engine)
# In-place schema migration: add the (waveform_key, event_timestamp)
# integrity-stamp columns to legacy cache DBs that predate the
# post-erase eviction logic. ALTER TABLE ADD COLUMN is idempotent
# via the column-presence check below.
with engine.begin() as conn:
for table in ("cached_events", "cached_waveforms"):
cols = {
r[1]
for r in conn.exec_driver_sql(f"PRAGMA table_info({table})").fetchall()
}
for new_col, ddl in (
("waveform_key", "TEXT"),
("event_timestamp", "TEXT"),
):
if new_col not in cols:
log.info("cache schema: %s ADD COLUMN %s %s", table, new_col, ddl)
conn.exec_driver_sql(f"ALTER TABLE {table} ADD COLUMN {new_col} {ddl}")
log.info("SFM cache opened: %s", db_path)
# ── Connection key ────────────────────────────────────────────────────────
@@ -242,15 +274,91 @@ class SFMCache:
row = s.get(CachedEvent, (conn_key, index))
return json.loads(row.event_json) if row else None
@staticmethod
def _event_signature(ev: dict) -> tuple[Optional[str], Optional[str]]:
"""
Extract the (waveform_key_hex, timestamp_iso) integrity stamp from
a serialised event dict. Either field may be None if the source
Event was missing it; the comparison logic in set_events/set_waveform
treats "both sides have a value AND they differ" as the only
eviction trigger, so partial data never spuriously flushes cache.
"""
key = ev.get("waveform_key") or ev.get("_waveform_key")
if isinstance(key, (bytes, bytearray)):
key = bytes(key).hex()
ts = ev.get("timestamp")
if isinstance(ts, dict):
# _serialise_timestamp returns a dict like {"iso": "...", ...}
ts = ts.get("iso") or ts.get("string") or None
return (key if isinstance(key, str) else None,
ts if isinstance(ts, str) else None)
def _maybe_flush_on_mismatch(
self,
s,
conn_key: str,
index: int,
new_key: Optional[str],
new_ts: Optional[str],
) -> bool:
"""
Check whether the cached entry at (conn_key, index) has a different
(waveform_key, timestamp) than the incoming one. If so, treat it as
a post-erase key-reuse signal and flush ALL cached events/waveforms
for this device, then return True.
Returns False when no flush was needed.
"""
if not new_key and not new_ts:
return False # nothing to compare against
existing = s.get(CachedEvent, (conn_key, index))
if existing is None:
existing = s.get(CachedWaveform, (conn_key, index))
if existing is None:
return False
old_key = existing.waveform_key
old_ts = existing.event_timestamp
# Only flush when both sides have populated values and they differ.
differs = (
(new_key and old_key and new_key != old_key)
or (new_ts and old_ts and new_ts != old_ts)
)
if not differs:
return False
log.warning(
"cache: device %s — index %d (key=%s, ts=%s) replaces (key=%s, ts=%s); "
"flushing all cached events/waveforms for this device "
"(post-erase key reuse detected)",
conn_key, index, new_key, new_ts, old_key, old_ts,
)
s.query(CachedEvent).filter_by(conn_key=conn_key).delete()
s.query(CachedWaveform).filter_by(conn_key=conn_key).delete()
return True
def set_events(self, conn_key: str, events: list[dict]) -> None:
"""
Upsert a list of event dicts. Existing rows are updated; new rows are
inserted. This is used to add newly-discovered events to the cache.
Eviction: if any incoming event has a different (waveform_key,
timestamp) than the row currently cached at the same index, we flush
the entire device's cache before inserting the new entries. Catches
post-erase key reuse where index 0 silently switches identity.
"""
now = time.time()
with self._Session() as s:
# Eviction check: scan incoming events for any (index, key, ts)
# that conflicts with a cached row. A single conflict triggers
# a full device-wide flush so we don't end up with a mixed-era
# cache.
for ev in events:
key, ts = self._event_signature(ev)
if self._maybe_flush_on_mismatch(s, conn_key, ev["index"], key, ts):
s.commit()
break # cache is now empty for this device; carry on
for ev in events:
idx = ev["index"]
key, ts = self._event_signature(ev)
row = s.get(CachedEvent, (conn_key, idx))
if row is None:
row = CachedEvent(
@@ -258,12 +366,18 @@ class SFMCache:
index=idx,
event_json=json.dumps(ev),
cached_at=now,
waveform_key=key,
event_timestamp=ts,
)
s.add(row)
log.debug("cached new event %d for %s", idx, conn_key)
else:
# Refresh in case project_info was backfilled after initial store
row.event_json = json.dumps(ev)
if key:
row.waveform_key = key
if ts:
row.event_timestamp = ts
s.commit()
# ── Waveforms ─────────────────────────────────────────────────────────────
@@ -278,8 +392,16 @@ class SFMCache:
return json.loads(row.waveform_json)
def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None:
"""Store a full waveform response dict permanently."""
"""
Store a full waveform response dict permanently.
Like set_events, this checks the (waveform_key, timestamp) signature
of the incoming entry against what's currently cached at the same
index. A mismatch flushes the entire device's cache before insert.
"""
key, ts = self._event_signature(waveform)
with self._Session() as s:
self._maybe_flush_on_mismatch(s, conn_key, index, key, ts)
row = s.get(CachedWaveform, (conn_key, index))
if row is None:
row = CachedWaveform(
@@ -287,13 +409,20 @@ class SFMCache:
index=index,
waveform_json=json.dumps(waveform),
cached_at=time.time(),
waveform_key=key,
event_timestamp=ts,
)
s.add(row)
else:
row.waveform_json = json.dumps(waveform)
row.cached_at = time.time()
if key:
row.waveform_key = key
if ts:
row.event_timestamp = ts
s.commit()
log.debug("cached waveform for %s event %d", conn_key, index)
log.debug("cached waveform for %s event %d (key=%s, ts=%s)",
conn_key, index, key, ts)
# ── Monitor status ────────────────────────────────────────────────────────
sfm/live_cache.py
+189
View File
@@ -0,0 +1,189 @@
"""
sfm/live_cache.py — Thread-safe in-memory cache for live SFM device data.
Extracted from sfm/server.py so the cache logic is importable and testable
without pulling in fastapi/uvicorn.
Caching strategy
----------------
Keyed by `conn_key` ("tcp:host:port" or "serial:port:baud"). Does NOT
persist across server restarts.
device_info cached until POST /device/config marks it dirty
events cached by (conn_key, device_event_count); re-fetched when
a quick count_events() probe shows new events on the device
monitor_status 30-second TTL (changes frequently during monitoring)
waveforms permanent within a process — but auto-evicted at the device
level when a (waveform_key, timestamp) mismatch is detected
at the same index (post-erase key reuse — the device's
event-key counter resets to 0x01110000 after every erase,
so the same `(conn_key, index)` slot can refer to a
brand-new physical event).
All endpoints accept ?force=true to bypass the cache and re-read.
"""
from __future__ import annotations
import threading
import time
from typing import Optional
_MONITOR_STATUS_TTL = 30.0 # seconds
class LiveCache:
"""
Thread-safe in-memory cache for live SFM device data.
One singleton per server process.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._device_info: dict[str, dict] = {}
self._events: dict[str, tuple[int, list]] = {}
self._monitor_status: dict[str, tuple[float, dict]] = {}
self._config_dirty: dict[str, bool] = {}
self._waveforms: dict[tuple, dict] = {}
# ── Connection key ────────────────────────────────────────────────────────
@staticmethod
def make_conn_key(
host: Optional[str],
tcp_port: int,
port: Optional[str],
baud: int,
) -> str:
if host:
return f"tcp:{host}:{tcp_port}"
return f"serial:{port}:{baud}"
# ── Eviction signature ────────────────────────────────────────────────────
@staticmethod
def _event_signature(ev: dict) -> tuple[Optional[str], Optional[str]]:
"""Return (waveform_key_hex, timestamp_iso) from a serialised event."""
key = ev.get("waveform_key") or ev.get("_waveform_key")
if isinstance(key, (bytes, bytearray)):
key = bytes(key).hex()
ts = ev.get("timestamp")
if isinstance(ts, dict):
ts = ts.get("iso") or ts.get("string") or None
return (key if isinstance(key, str) else None,
ts if isinstance(ts, str) else None)
def _flush_device(self, conn_key: str) -> None:
"""Drop all cached events + waveforms for one device. Caller holds lock."""
self._events.pop(conn_key, None)
stale_wf_keys = [k for k in self._waveforms if k[0] == conn_key]
for k in stale_wf_keys:
self._waveforms.pop(k, None)
# ── Device info ───────────────────────────────────────────────────────────
def get_device_info(self, conn_key: str) -> Optional[dict]:
with self._lock:
if self._config_dirty.get(conn_key):
return None
return self._device_info.get(conn_key)
def set_device_info(self, conn_key: str, info: dict) -> None:
with self._lock:
self._device_info[conn_key] = info
self._config_dirty[conn_key] = False
# ── Events ────────────────────────────────────────────────────────────────
def get_events(self, conn_key: str, device_count: int) -> Optional[list]:
with self._lock:
if self._config_dirty.get(conn_key):
return None
entry = self._events.get(conn_key)
if entry is None:
return None
cached_count, events = entry
return events if cached_count == device_count else None
def set_events(self, conn_key: str, device_count: int, events: list) -> None:
"""
Replace the cached events list for `conn_key`. If any incoming event
has a different (waveform_key, timestamp) than the cached entry at
the same index, flush the entire conn_key's event + waveform cache
first. Catches post-erase key reuse.
"""
with self._lock:
cached_entry = self._events.get(conn_key)
cached_events = cached_entry[1] if cached_entry else []
cached_by_index = {e.get("index"): e for e in cached_events}
evict = False
for ev in events:
idx = ev.get("index")
if idx is None:
continue
cached = cached_by_index.get(idx)
if cached is None:
continue
new_key, new_ts = self._event_signature(ev)
old_key, old_ts = self._event_signature(cached)
if (new_key and old_key and new_key != old_key) or \
(new_ts and old_ts and new_ts != old_ts):
evict = True
break
if evict:
self._flush_device(conn_key)
self._events[conn_key] = (device_count, events)
# ── Monitor status ────────────────────────────────────────────────────────
def get_monitor_status(self, conn_key: str) -> Optional[dict]:
with self._lock:
entry = self._monitor_status.get(conn_key)
if entry is None:
return None
fetched_at, status = entry
if time.time() - fetched_at > _MONITOR_STATUS_TTL:
return None
return status
def set_monitor_status(self, conn_key: str, status: dict) -> None:
with self._lock:
self._monitor_status[conn_key] = (time.time(), status)
def invalidate_monitor_status(self, conn_key: str) -> None:
with self._lock:
self._monitor_status.pop(conn_key, None)
# ── Config dirty flag ─────────────────────────────────────────────────────
def mark_config_dirty(self, conn_key: str) -> None:
with self._lock:
self._config_dirty[conn_key] = True
self._events.pop(conn_key, None)
# ── Waveforms (permanent cache, evicted on (key,ts) mismatch) ─────────────
def get_waveform(self, conn_key: str, index: int) -> Optional[dict]:
with self._lock:
return self._waveforms.get((conn_key, index))
def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None:
"""
Cache a waveform. Evicts the device's whole cache when the existing
entry at the same index has a different (waveform_key, timestamp).
"""
with self._lock:
existing = self._waveforms.get((conn_key, index))
if existing is not None:
new_key, new_ts = self._event_signature(waveform)
old_key, old_ts = self._event_signature(existing)
differs = (
(new_key and old_key and new_key != old_key)
or (new_ts and old_ts and new_ts != old_ts)
)
if differs:
self._flush_device(conn_key)
self._waveforms[(conn_key, index)] = waveform
sfm/server.py
+8 -112
View File
@@ -66,6 +66,7 @@ from minimateplus.blastware_file import write_blastware_file, blastware_filename
from minimateplus.client import _decode_a5_metadata_into, _decode_a5_waveform
from sfm.cache import SFMCache, get_cache
from sfm.database import SeismoDb
from sfm.live_cache import LiveCache as _LiveCache
from sfm.waveform_store import WaveformStore
logging.basicConfig(
@@ -142,116 +143,6 @@ def _get_store() -> WaveformStore:
#
# All endpoints accept ?force=true to bypass the cache and re-read from device.
_MONITOR_STATUS_TTL = 30.0 # seconds
class _LiveCache:
"""
Thread-safe in-memory cache for live SFM device data.
One singleton per server process.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
# conn_key → serialised device info dict
self._device_info: dict[str, dict] = {}
# conn_key → (device_event_count_when_cached, [event dicts])
self._events: dict[str, tuple[int, list]] = {}
# conn_key → (fetched_at_unix, status_dict)
self._monitor_status: dict[str, tuple[float, dict]] = {}
# conn_key → bool (True = re-read device on next /device/info)
self._config_dirty: dict[str, bool] = {}
# (conn_key, event_index) → waveform dict (permanent)
self._waveforms: dict[tuple, dict] = {}
# ── Connection key ────────────────────────────────────────────────────────
@staticmethod
def make_conn_key(
host: Optional[str],
tcp_port: int,
port: Optional[str],
baud: int,
) -> str:
if host:
return f"tcp:{host}:{tcp_port}"
return f"serial:{port}:{baud}"
# ── Device info ───────────────────────────────────────────────────────────
def get_device_info(self, conn_key: str) -> Optional[dict]:
with self._lock:
if self._config_dirty.get(conn_key):
return None
return self._device_info.get(conn_key)
def set_device_info(self, conn_key: str, info: dict) -> None:
with self._lock:
self._device_info[conn_key] = info
self._config_dirty[conn_key] = False
# ── Events ────────────────────────────────────────────────────────────────
def get_events(self, conn_key: str, device_count: int) -> Optional[list]:
"""
Return cached events if the device's current event count matches what
we had when we last fetched. Returns None (cache miss) otherwise.
"""
with self._lock:
if self._config_dirty.get(conn_key):
return None
entry = self._events.get(conn_key)
if entry is None:
return None
cached_count, events = entry
return events if cached_count == device_count else None
def set_events(self, conn_key: str, device_count: int, events: list) -> None:
with self._lock:
self._events[conn_key] = (device_count, events)
# ── Monitor status ────────────────────────────────────────────────────────
def get_monitor_status(self, conn_key: str) -> Optional[dict]:
with self._lock:
entry = self._monitor_status.get(conn_key)
if entry is None:
return None
fetched_at, status = entry
if time.time() - fetched_at > _MONITOR_STATUS_TTL:
return None
return status
def set_monitor_status(self, conn_key: str, status: dict) -> None:
with self._lock:
self._monitor_status[conn_key] = (time.time(), status)
def invalidate_monitor_status(self, conn_key: str) -> None:
with self._lock:
self._monitor_status.pop(conn_key, None)
# ── Config dirty flag ─────────────────────────────────────────────────────
def mark_config_dirty(self, conn_key: str) -> None:
"""
Called after a successful POST /device/config write.
Forces next /device/info and /device/events to re-read from the device.
"""
with self._lock:
self._config_dirty[conn_key] = True
self._events.pop(conn_key, None)
# ── Waveforms (permanent cache) ───────────────────────────────────────────
def get_waveform(self, conn_key: str, index: int) -> Optional[dict]:
with self._lock:
return self._waveforms.get((conn_key, index))
def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None:
with self._lock:
self._waveforms[(conn_key, index)] = waveform
_live_cache = _LiveCache()
@@ -872,6 +763,7 @@ def device_event_blastware_file(
baud: int = Query(38400, description="Serial baud rate"),
host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"),
tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"),
force: bool = Query(False, description="Bypass any cached/dedup'd state and re-download from device"),
) -> FileResponse:
"""
Download the waveform for a single event (0-based index) and return it
@@ -893,9 +785,13 @@ def device_event_blastware_file(
stop_after_index=index) → write_blastware_file() → FileResponse.
"""
log.info(
"GET /device/event/%d/blastware_file port=%s host=%s",
index, port, host,
"GET /device/event/%d/blastware_file port=%s host=%s force=%s",
index, port, host, force,
)
# `force` always re-downloads from the device. This endpoint already
# never short-circuits via cache, so `force` is reserved for parity with
# the other live endpoints and to suppress the post-download persist
# (see end of handler) when the caller wants a fetch-only escape hatch.
try:
def _do():
sfm/sfm_webapp.html
+51 -1
View File
@@ -609,6 +609,36 @@
.section-btn:hover { color: var(--text); }
.section-btn.active { background: var(--blue); color: #fff; }
/* ── Force-refresh toggle ── */
.force-toggle {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 10px;
border: 1px solid var(--border);
border-radius: 6px;
background: var(--bg);
cursor: pointer;
font-size: 11px;
font-weight: 600;
color: var(--text-dim);
user-select: none;
white-space: nowrap;
transition: background 0.12s, color 0.12s, border-color 0.12s;
}
.force-toggle input { margin: 0; cursor: pointer; }
.force-toggle:hover { color: var(--text); }
.force-toggle.active {
background: rgba(248, 81, 73, 0.18);
border-color: #f85149;
color: #ff7b72;
}
.force-toggle .ft-dot {
width: 6px; height: 6px; border-radius: 50%;
background: var(--text-mute);
}
.force-toggle.active .ft-dot { background: #f85149; box-shadow: 0 0 6px #f85149; }
/* ── Section containers ── */
#section-live, #section-db {
display: flex;
@@ -654,6 +684,13 @@
<button class="section-btn active" onclick="switchSection('live')">Live Device</button>
<button class="section-btn" onclick="switchSection('db')">Database</button>
</div>
<div class="hdr-sep"></div>
<label class="force-toggle" id="force-toggle"
title="Bypass server cache and dedup. Forces a fresh download from the device on every live request — useful when the device has been erased and the cache is showing stale events.">
<input type="checkbox" id="force-cb" onchange="onForceToggle()">
<span class="ft-dot"></span>
<span>Force refresh</span>
</label>
</header>
<!-- ════════════════════════════════════════════════════════════════
@@ -1214,8 +1251,21 @@ function setCfgStatus(msg, cls = '') {
el.className = cls;
}
// "Force refresh" override — when enabled, every live-device request is
// sent with ?force=true so the server bypasses its in-memory + persistent
// caches and re-reads from the device. Manual escape hatch for cases where
// the cache has gone stale (e.g. post-erase key reuse — see ach_server.py
// and sfm/cache.py for the eviction logic).
let forceRefresh = false;
function onForceToggle() {
forceRefresh = document.getElementById('force-cb').checked;
document.getElementById('force-toggle').classList.toggle('active', forceRefresh);
}
function deviceParams() {
return `host=${encodeURIComponent(devHost())}&tcp_port=${devPort()}`;
const base = `host=${encodeURIComponent(devHost())}&tcp_port=${devPort()}`;
return forceRefresh ? `${base}&force=true` : base;
}
// ── Section switching ─────────────────────────────────────────────────────────