From 2db565ff9c38e24f4ff7f48b9dd669d71e3b31a3 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 9 Apr 2026 07:14:51 +0000 Subject: [PATCH 01/33] Add intelligent caching layer for SFM device data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces sfm/cache.py — a SQLite-backed cache (via SQLAlchemy) that sits between the SFM REST endpoints and the device, eliminating redundant cellular downloads for data that doesn't change. Cache behaviour by data type: - Device info / compliance config: cached until a config write occurs; POST /device/config now calls mark_config_dirty() to force a fresh read on the next /device/info call. - Event headers + peak values: cached permanently (append-only). On subsequent calls to /device/events, the server does a fast count_events() (~2s) instead of a full download (~10-30s); only new events are fetched from the device and merged into the cache. - Full waveforms (raw ADC samples): cached permanently — immutable once recorded. Repeated requests for the same waveform return instantly with zero device contact. - Monitor status (battery, memory, is_monitoring): 30-second TTL; auto- invalidated on start/stop monitoring commands. All endpoints gain a ?force=true param to bypass the cache when needed. New endpoints: GET /cache/stats, DELETE /cache/device. Adds requirements.txt listing fastapi, uvicorn, sqlalchemy, pyserial. Co-Authored-By: Claude Sonnet 4.6 --- requirements.txt | 4 + sfm/cache.py | 376 +++++++++++++++++++++++++++++++++++++++++++++++ sfm/server.py | 295 +++++++++++++++++++++++++++++++++---- 3 files changed, 643 insertions(+), 32 deletions(-) create mode 100644 requirements.txt create mode 100644 sfm/cache.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0958f1a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi +uvicorn +sqlalchemy +pyserial diff --git a/sfm/cache.py b/sfm/cache.py new file mode 100644 index 0000000..be35e60 --- /dev/null +++ b/sfm/cache.py @@ -0,0 +1,376 @@ +""" +sfm/cache.py — Persistent SQLite cache for SFM device data. + +Caching strategy +---------------- ++------------------+----------------------------------+-------------------------+ +| Data | Mutability | Invalidation | ++------------------+----------------------------------+-------------------------+ +| Device info | Effectively immutable (firmware, | Manual clear / force | +| (serial, model, | serial never change) | refresh query param | +| compliance cfg) | | | ++------------------+----------------------------------+-------------------------+ +| Event headers | Append-only (new events added, | Fetch new ones when | +| (peaks, ts, | old never modified) | device event count > | +| project info) | | cached count | ++------------------+----------------------------------+-------------------------+ +| Full waveforms | Immutable once recorded | Never (permanent cache) | +| (raw ADC samples)| | | ++------------------+----------------------------------+-------------------------+ +| Monitor status | Frequently changing | TTL = 30 seconds | +| (battery, memory)| | | ++------------------+----------------------------------+-------------------------+ + +Keys +---- +All cached rows are keyed by (host, tcp_port) for TCP connections, or (port, baud) +for serial connections. Within a device, events are keyed by index (0-based). + +The device serial number is stored once we learn it, and used for display / debugging +only — the network address is the primary routing key (same as how the rest of the SFM +code operates). +""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Optional + +try: + import sqlalchemy as sa + from sqlalchemy import orm +except ImportError: + raise ImportError( + "sqlalchemy is required for the SFM cache.\n" + "Install it with: pip install sqlalchemy" + ) + +log = logging.getLogger("sfm.cache") + +# ── Schema ──────────────────────────────────────────────────────────────────── + +Base = orm.declarative_base() + +_MONITOR_STATUS_TTL = 30 # seconds + + +class CachedDevice(Base): + """ + Device identity + compliance config, keyed by connection address. + + Stores the full serialised JSON blob returned by /device/info so the + endpoint can return it verbatim on a cache hit without re-connecting. + """ + __tablename__ = "cached_devices" + + # Connection key — either TCP (host+port) or serial (port+baud) + conn_key = sa.Column(sa.String, primary_key=True) # e.g. "tcp:1.2.3.4:12345" + serial = sa.Column(sa.String, nullable=True) # e.g. "BE11529" + info_json = sa.Column(sa.Text, nullable=False) # full /device/info response JSON + updated_at = sa.Column(sa.Float, nullable=False) # Unix timestamp of last write + + # When a config write happens we set this flag so the next /device/info call + # fetches fresh data instead of serving stale compliance config. + config_dirty = sa.Column(sa.Boolean, default=False, nullable=False) + + +class CachedEvent(Base): + """ + Per-event header + peak values + project info, keyed by (conn_key, index). + + Events are immutable once recorded on the device; once we have an event in + the cache it never needs to be re-downloaded unless explicitly requested. + """ + __tablename__ = "cached_events" + + conn_key = sa.Column(sa.String, primary_key=True) + index = sa.Column(sa.Integer, primary_key=True) + event_json = sa.Column(sa.Text, nullable=False) # serialised Event dict + cached_at = sa.Column(sa.Float, nullable=False) # Unix timestamp + + +class CachedWaveform(Base): + """ + Full raw ADC waveform for a single event (SUB 5A full download). + + These are large (up to several MB) and expensive to fetch over cellular. + Once downloaded they are immutable and cached permanently. + """ + __tablename__ = "cached_waveforms" + + conn_key = sa.Column(sa.String, primary_key=True) + index = sa.Column(sa.Integer, primary_key=True) + waveform_json = sa.Column(sa.Text, nullable=False) # full /device/event/{idx}/waveform response JSON + cached_at = sa.Column(sa.Float, nullable=False) + + +class CachedMonitorStatus(Base): + """ + Monitor status (battery, memory, is_monitoring) with a short TTL. + + These change frequently during field operations so we keep them only for + MONITOR_STATUS_TTL seconds before re-fetching from the device. + """ + __tablename__ = "cached_monitor_status" + + conn_key = sa.Column(sa.String, primary_key=True) + status_json = sa.Column(sa.Text, nullable=False) + cached_at = sa.Column(sa.Float, nullable=False) + + +# ── Cache store ─────────────────────────────────────────────────────────────── + +class SFMCache: + """ + SQLite-backed cache for SFM device data. + + Usage + ----- + cache = SFMCache() # stores in sfm/data/sfm_cache.db by default + cache = SFMCache(":memory:") # in-memory (tests / ephemeral mode) + + All public methods accept a *conn_key* string — use make_conn_key() to + build a consistent key from the transport parameters. + """ + + def __init__(self, db_path: str | Path | None = None) -> None: + in_memory = (db_path == ":memory:") + if db_path is None: + # Default: alongside this file in sfm/data/ + db_path = Path(__file__).parent / "data" / "sfm_cache.db" + if not in_memory: + db_path = Path(db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + + url = "sqlite:///:memory:" if in_memory else f"sqlite:///{db_path}" + engine = sa.create_engine(url, connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + self._Session = orm.sessionmaker(bind=engine) + log.info("SFM cache opened: %s", db_path) + + # ── Connection key ──────────────────────────────────────────────────────── + + @staticmethod + def make_conn_key( + host: Optional[str], + tcp_port: int, + port: Optional[str], + baud: int, + ) -> str: + """Return a stable string key for this transport configuration.""" + if host: + return f"tcp:{host}:{tcp_port}" + return f"serial:{port}:{baud}" + + # ── Device info ─────────────────────────────────────────────────────────── + + def get_device_info(self, conn_key: str) -> Optional[dict]: + """ + Return cached device info dict, or None if not cached / config_dirty. + """ + with self._Session() as s: + row = s.get(CachedDevice, conn_key) + if row is None or row.config_dirty: + return None + return json.loads(row.info_json) + + def set_device_info(self, conn_key: str, info: dict) -> None: + """Store device info and clear any dirty flag.""" + with self._Session() as s: + row = s.get(CachedDevice, conn_key) + serial = info.get("serial") + if row is None: + row = CachedDevice( + conn_key=conn_key, + serial=serial, + info_json=json.dumps(info), + updated_at=time.time(), + config_dirty=False, + ) + s.add(row) + else: + row.serial = serial + row.info_json = json.dumps(info) + row.updated_at = time.time() + row.config_dirty = False + s.commit() + log.debug("cached device info for %s (serial=%s)", conn_key, serial) + + def mark_config_dirty(self, conn_key: str) -> None: + """ + Called after a successful POST /device/config write. + + Forces the next /device/info call to re-read compliance config from the + device instead of serving the now-stale cached version. + """ + with self._Session() as s: + row = s.get(CachedDevice, conn_key) + if row: + row.config_dirty = True + s.commit() + log.debug("marked config dirty for %s", conn_key) + + # ── Events ──────────────────────────────────────────────────────────────── + + def get_cached_event_count(self, conn_key: str) -> int: + """Return the number of events we have cached for this device.""" + with self._Session() as s: + return s.query(CachedEvent).filter_by(conn_key=conn_key).count() + + def get_all_events(self, conn_key: str) -> Optional[list[dict]]: + """ + Return all cached events as a list of dicts, sorted by index. + Returns None if nothing is cached yet. + """ + with self._Session() as s: + rows = ( + s.query(CachedEvent) + .filter_by(conn_key=conn_key) + .order_by(CachedEvent.index) + .all() + ) + if not rows: + return None + return [json.loads(r.event_json) for r in rows] + + def get_event(self, conn_key: str, index: int) -> Optional[dict]: + """Return a single cached event by index, or None if not cached.""" + with self._Session() as s: + row = s.get(CachedEvent, (conn_key, index)) + return json.loads(row.event_json) if row else None + + def set_events(self, conn_key: str, events: list[dict]) -> None: + """ + Upsert a list of event dicts. Existing rows are updated; new rows are + inserted. This is used to add newly-discovered events to the cache. + """ + now = time.time() + with self._Session() as s: + for ev in events: + idx = ev["index"] + row = s.get(CachedEvent, (conn_key, idx)) + if row is None: + row = CachedEvent( + conn_key=conn_key, + index=idx, + event_json=json.dumps(ev), + cached_at=now, + ) + s.add(row) + log.debug("cached new event %d for %s", idx, conn_key) + else: + # Refresh in case project_info was backfilled after initial store + row.event_json = json.dumps(ev) + s.commit() + + # ── Waveforms ───────────────────────────────────────────────────────────── + + def get_waveform(self, conn_key: str, index: int) -> Optional[dict]: + """Return a cached full waveform response dict, or None if not cached.""" + with self._Session() as s: + row = s.get(CachedWaveform, (conn_key, index)) + if row is None: + return None + log.debug("waveform cache hit: %s event %d", conn_key, index) + return json.loads(row.waveform_json) + + def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None: + """Store a full waveform response dict permanently.""" + with self._Session() as s: + row = s.get(CachedWaveform, (conn_key, index)) + if row is None: + row = CachedWaveform( + conn_key=conn_key, + index=index, + waveform_json=json.dumps(waveform), + cached_at=time.time(), + ) + s.add(row) + else: + row.waveform_json = json.dumps(waveform) + row.cached_at = time.time() + s.commit() + log.debug("cached waveform for %s event %d", conn_key, index) + + # ── Monitor status ──────────────────────────────────────────────────────── + + def get_monitor_status(self, conn_key: str) -> Optional[dict]: + """Return cached monitor status if it's within TTL, else None.""" + with self._Session() as s: + row = s.get(CachedMonitorStatus, conn_key) + if row is None: + return None + age = time.time() - row.cached_at + if age > _MONITOR_STATUS_TTL: + log.debug("monitor status expired (age=%.1fs) for %s", age, conn_key) + return None + return json.loads(row.status_json) + + def set_monitor_status(self, conn_key: str, status: dict) -> None: + """Store monitor status.""" + with self._Session() as s: + row = s.get(CachedMonitorStatus, conn_key) + if row is None: + row = CachedMonitorStatus( + conn_key=conn_key, + status_json=json.dumps(status), + cached_at=time.time(), + ) + s.add(row) + else: + row.status_json = json.dumps(status) + row.cached_at = time.time() + s.commit() + + def invalidate_monitor_status(self, conn_key: str) -> None: + """ + Called after start/stop monitoring so the next status poll re-reads from device. + """ + with self._Session() as s: + row = s.get(CachedMonitorStatus, conn_key) + if row: + s.delete(row) + s.commit() + + # ── Cache management ────────────────────────────────────────────────────── + + def clear_device(self, conn_key: str) -> dict: + """ + Remove all cached data for a device. Returns counts of deleted rows. + """ + counts = {} + with self._Session() as s: + counts["device_info"] = s.query(CachedDevice).filter_by(conn_key=conn_key).delete() + counts["events"] = s.query(CachedEvent).filter_by(conn_key=conn_key).delete() + counts["waveforms"] = s.query(CachedWaveform).filter_by(conn_key=conn_key).delete() + counts["monitor_status"] = s.query(CachedMonitorStatus).filter_by(conn_key=conn_key).delete() + s.commit() + log.info("cleared cache for %s: %s", conn_key, counts) + return counts + + def stats(self) -> dict: + """Return row counts for all cache tables (for /cache/stats endpoint).""" + with self._Session() as s: + return { + "devices": s.query(CachedDevice).count(), + "events": s.query(CachedEvent).count(), + "waveforms": s.query(CachedWaveform).count(), + "monitor_status": s.query(CachedMonitorStatus).count(), + } + + +# ── Module-level singleton ──────────────────────────────────────────────────── +# Instantiated once when the module is imported; shared across all requests. + +_cache: Optional[SFMCache] = None + + +def get_cache() -> SFMCache: + """Return the module-level cache singleton, initialising it on first call.""" + global _cache + if _cache is None: + _cache = SFMCache() + return _cache diff --git a/sfm/server.py b/sfm/server.py index 0ae36ba..8c5f66f 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -58,6 +58,7 @@ from minimateplus import MiniMateClient from minimateplus.protocol import ProtocolError from minimateplus.models import ComplianceConfig, DeviceInfo, Event, PeakValues, ProjectInfo, Timestamp from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT +from sfm.cache import SFMCache, get_cache logging.basicConfig( level=logging.INFO, @@ -239,6 +240,33 @@ def _run_with_retry(fn, *, is_tcp: bool): return fn() # let any second failure propagate normally +# ── Helpers ──────────────────────────────────────────────────────────────────── + +def _backfill_events(events: list, info: "DeviceInfo") -> None: + """ + Fill in sample_rate and project_info fields that the per-event waveform + record doesn't carry — sourced from the device's compliance config. + + Extracted from device_events() so it can be called from both the full + download path and the partial (new-events-only) path. + """ + if info.compliance_config and info.compliance_config.sample_rate: + for ev in events: + if ev.sample_rate is None: + ev.sample_rate = info.compliance_config.sample_rate + + if info.compliance_config: + cc = info.compliance_config + for ev in events: + if ev.project_info is None: + ev.project_info = ProjectInfo() + pi = ev.project_info + if pi.client is None: pi.client = cc.client + if pi.operator is None: pi.operator = cc.operator + if pi.sensor_location is None: pi.sensor_location = cc.sensor_location + if pi.notes is None: pi.notes = cc.notes + + # ── Endpoints ────────────────────────────────────────────────────────────────── @app.get("/health") @@ -259,6 +287,7 @@ def device_info( baud: int = Query(38400, description="Serial baud rate (default 38400)"), host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay (e.g. 203.0.113.5)"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), + force: bool = Query(False, description="Bypass cache and re-read from device"), ) -> dict: """ Connect to the device, perform the POLL startup handshake, and return @@ -266,8 +295,23 @@ def device_info( Supply either *port* (serial) or *host* (TCP/modem). Equivalent to POST /device/connect — provided as GET for convenience. + + **Caching**: device identity and compliance config are cached after the first + successful read (they rarely change). Pass *force=true* to bypass the cache + and re-read directly from the device (e.g. after a config push). + The cache is also automatically invalidated after POST /device/config. """ - log.info("GET /device/info port=%s host=%s tcp_port=%d", port, host, tcp_port) + log.info("GET /device/info port=%s host=%s tcp_port=%d force=%s", port, host, tcp_port, force) + + cache = get_cache() + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + + if not force: + cached = cache.get_device_info(conn_key) + if cached is not None: + log.info("device info cache hit for %s", conn_key) + cached["_cached"] = True + return cached try: def _do(): @@ -287,7 +331,9 @@ def device_info( except Exception as exc: raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc - return _serialise_device_info(info) + result = _serialise_device_info(info) + cache.set_device_info(conn_key, result) + return result @app.post("/device/connect") @@ -311,6 +357,7 @@ def device_events( host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), debug: bool = Query(False, description="Include raw record hex for field-layout inspection"), + force: bool = Query(False, description="Bypass cache and re-download all events from device"), ) -> dict: """ Connect to the device, read the event index, and download all stored @@ -322,10 +369,108 @@ def device_events( verifying field offsets against the protocol reference. This does NOT download raw ADC waveform samples — those are large and - fetched separately via GET /device/event/{idx}/waveform (future endpoint). - """ - log.info("GET /device/events port=%s host=%s debug=%s", port, host, debug) + fetched separately via GET /device/event/{idx}/waveform. + **Caching**: event headers are cached after the first download. On subsequent + calls, the device is contacted only to check the event count (fast: ~2s). + If the count matches the cache, all events are returned from cache instantly. + If new events exist on the device, only the new ones are downloaded and merged. + Pass *force=true* to bypass the cache entirely and re-download everything. + """ + log.info("GET /device/events port=%s host=%s debug=%s force=%s", port, host, debug, force) + + cache = get_cache() + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + + # ── Smart cache path (skip when debug=True or force=True) ──────────────── + # debug mode uses raw_record_hex which isn't stored in the cache, so we + # must always go to the device when debug is requested. + if not force and not debug: + cached_events = cache.get_all_events(conn_key) + cached_count = len(cached_events) if cached_events else 0 + + if cached_count > 0: + # Quick device contact: just count events via the fast 1E/1F chain. + # This takes ~2s instead of the full event download (~10-30s). + try: + def _count(): + with _build_client(port, baud, host, tcp_port) as client: + client.connect() + return client.count_events() + device_count = _run_with_retry(_count, is_tcp=_is_tcp(host)) + except HTTPException: + raise + except (ProtocolError, OSError, Exception) as exc: + # If we can't reach the device at all, serve stale cache rather + # than returning an error — field units go offline regularly. + log.warning("count_events failed (%s) — serving stale cache for %s", exc, conn_key) + cached_info = cache.get_device_info(conn_key) or {} + return { + "device": cached_info, + "event_count": cached_count, + "events": cached_events, + "_cached": True, + "_stale": True, + } + + if device_count == cached_count: + # Nothing new — return cache immediately, no event download needed. + log.info( + "event cache hit for %s: %d events, device count matches", + conn_key, cached_count, + ) + cached_info = cache.get_device_info(conn_key) or {} + return { + "device": cached_info, + "event_count": cached_count, + "events": cached_events, + "_cached": True, + } + + if device_count > cached_count: + # New events on the device — download all events but only store/return + # the new ones. Events are append-only; indices 0..(cached_count-1) + # are already in the cache and don't need to be re-downloaded logically, + # but the protocol requires iterating from event 0 to reach later ones. + # The device download time is dominated by the number of events requested, + # so we stop at the last known event index to avoid re-downloading everything. + log.info( + "new events on device %s: have %d, device has %d — fetching all up to %d", + conn_key, cached_count, device_count, device_count - 1, + ) + try: + def _fetch_new(): + with _build_client(port, baud, host, tcp_port) as client: + info = client.connect() + all_evs = client.get_events(stop_after_index=device_count - 1) + return info, all_evs + info, all_events = _run_with_retry(_fetch_new, is_tcp=_is_tcp(host)) + except HTTPException: + raise + except ProtocolError as exc: + raise HTTPException(status_code=502, detail=f"Protocol error: {exc}") from exc + except OSError as exc: + raise HTTPException(status_code=502, detail=f"Connection error: {exc}") from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc + + _backfill_events(all_events, info) + # Only the new events (indices >= cached_count) are truly new. + new_events = [ev for ev in all_events if ev.index >= cached_count] + new_serialised = [_serialise_event(ev) for ev in new_events] + cache.set_events(conn_key, new_serialised) + cache.set_device_info(conn_key, _serialise_device_info(info)) + + merged_events = cache.get_all_events(conn_key) + return { + "device": _serialise_device_info(info), + "event_count": len(merged_events), + "events": merged_events, + "_cached": True, + "_new_events": len(new_events), + } + + # ── Full download path (first call, force=True, or debug=True) ─────────── try: def _do(): with _build_client(port, baud, host, tcp_port) as client: @@ -340,31 +485,19 @@ def device_events( except Exception as exc: raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc - # Fill sample_rate from compliance config where the event record doesn't supply it. - # sample_rate is a device-level setting, not stored per-event in the waveform record. - if info.compliance_config and info.compliance_config.sample_rate: - for ev in events: - if ev.sample_rate is None: - ev.sample_rate = info.compliance_config.sample_rate + _backfill_events(events, info) + serialised = [_serialise_event(ev, debug=debug) for ev in events] - # Backfill event.project_info fields that the 210-byte waveform record doesn't carry. - # The waveform record only stores "Project:" — client/operator/sensor_location/notes - # live in the SUB 1A compliance config, not in the per-event record. - if info.compliance_config: - cc = info.compliance_config - for ev in events: - if ev.project_info is None: - ev.project_info = ProjectInfo() - pi = ev.project_info - if pi.client is None: pi.client = cc.client - if pi.operator is None: pi.operator = cc.operator - if pi.sensor_location is None: pi.sensor_location = cc.sensor_location - if pi.notes is None: pi.notes = cc.notes + if not debug: + # Only cache when not in debug mode (debug adds raw_record_hex which + # we don't want polluting the normal cache entries). + cache.set_events(conn_key, serialised) + cache.set_device_info(conn_key, _serialise_device_info(info)) return { "device": _serialise_device_info(info), "event_count": len(events), - "events": [_serialise_event(ev, debug=debug) for ev in events], + "events": serialised, } @@ -375,21 +508,36 @@ def device_event( baud: int = Query(38400, description="Serial baud rate"), host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), + force: bool = Query(False, description="Bypass cache and re-download from device"), ) -> dict: """ Download a single event by index (0-based). Supply either *port* (serial) or *host* (TCP/modem). Performs: POLL startup → event index → event header → waveform record. + + **Caching**: if this event was already downloaded (e.g. via GET /device/events), + it is returned instantly from cache with no device contact. """ - log.info("GET /device/event/%d port=%s host=%s", index, port, host) + log.info("GET /device/event/%d port=%s host=%s force=%s", index, port, host, force) + + cache = get_cache() + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + + if not force: + cached = cache.get_event(conn_key, index) + if cached is not None: + log.info("event cache hit for %s index %d", conn_key, index) + cached["_cached"] = True + return cached try: def _do(): with _build_client(port, baud, host, tcp_port) as client: - client.connect() - return client.get_events(stop_after_index=index) - events = _run_with_retry(_do, is_tcp=_is_tcp(host)) + info = client.connect() + events = client.get_events(stop_after_index=index) + return info, events + info, events = _run_with_retry(_do, is_tcp=_is_tcp(host)) except HTTPException: raise except ProtocolError as exc: @@ -406,7 +554,14 @@ def device_event( detail=f"Event index {index} not found on device", ) - return _serialise_event(matching[0]) + _backfill_events(matching, info) + result = _serialise_event(matching[0]) + + # Store all downloaded events (we paid for them anyway — indices 0..index) + all_serialised = [_serialise_event(ev) for ev in events] + cache.set_events(conn_key, all_serialised) + + return result @app.get("/device/event/{index}/waveform") @@ -416,6 +571,7 @@ def device_event_waveform( baud: int = Query(38400, description="Serial baud rate"), host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), + force: bool = Query(False, description="Bypass cache and re-download from device"), ) -> dict: """ Download the full raw ADC waveform for a single event (0-based index). @@ -434,8 +590,23 @@ def device_event_waveform( - **sample_rate**: samples per second (from compliance config) - **channels**: dict of channel name → list of signed int16 ADC counts (keys: "Tran", "Vert", "Long", "Mic") + + **Caching**: full waveforms are cached permanently after the first download — + they are immutable once recorded on the device. Subsequent requests for the + same event return instantly from cache without any device contact. + Pass *force=true* to force a fresh download (rarely needed). """ - log.info("GET /device/event/%d/waveform port=%s host=%s", index, port, host) + log.info("GET /device/event/%d/waveform port=%s host=%s force=%s", index, port, host, force) + + cache = get_cache() + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + + if not force: + cached = cache.get_waveform(conn_key, index) + if cached is not None: + log.info("waveform cache hit for %s event %d", conn_key, index) + cached["_cached"] = True + return cached try: def _do(): @@ -469,7 +640,7 @@ def device_event_waveform( if sample_rate is None and info.compliance_config: sample_rate = info.compliance_config.sample_rate - return { + result = { "index": ev.index, "record_type": ev.record_type, "timestamp": _serialise_timestamp(ev.timestamp), @@ -481,6 +652,8 @@ def device_event_waveform( "peak_values": _serialise_peak_values(ev.peak_values), "channels": raw, } + cache.set_waveform(conn_key, index, result) + return result # ── Write endpoints ─────────────────────────────────────────────────────────── @@ -595,6 +768,10 @@ def device_config( except Exception as exc: raise HTTPException(status_code=500, detail=f"Device error: {exc}") from exc + # Config was written to the device — the cached compliance config is now stale. + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + get_cache().mark_config_dirty(conn_key) + return { "status": "ok", "updated_fields": changed, @@ -622,6 +799,7 @@ def device_monitor_status( baud: int = Query(38400, description="Serial baud rate"), host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), + force: bool = Query(False, description="Bypass cache and re-read from device"), ) -> dict: """ Read monitoring status from the device. @@ -633,7 +811,20 @@ def device_monitor_status( Returns is_monitoring bool, battery voltage, and memory usage (total + free bytes). Battery and memory are only present when the unit is idle. + + **Caching**: status is cached for 30 seconds to reduce cellular polling overhead. + Pass *force=true* to bypass the cache for an immediate fresh read. """ + cache = get_cache() + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + + if not force: + cached = cache.get_monitor_status(conn_key) + if cached is not None: + log.debug("monitor status cache hit for %s", conn_key) + cached["_cached"] = True + return cached + with _build_client(port=port, baud=baud, host=host, tcp_port=tcp_port) as client: try: client.poll() @@ -651,6 +842,8 @@ def device_monitor_status( if status.memory_free is not None: result["memory_free_bytes"] = status.memory_free result["memory_free_kb"] = round(status.memory_free / 1024, 1) + + cache.set_monitor_status(conn_key, result) return result @@ -673,6 +866,9 @@ def device_monitor_start( log.warning("start monitoring poll retry: %s", exc) client.poll() client.start_monitoring() + + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + get_cache().invalidate_monitor_status(conn_key) return {"status": "started"} @@ -695,9 +891,44 @@ def device_monitor_stop( log.warning("stop monitoring poll retry: %s", exc) client.poll() client.stop_monitoring() + + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + get_cache().invalidate_monitor_status(conn_key) return {"status": "stopped"} +# ── Cache management endpoints ──────────────────────────────────────────────── + +@app.get("/cache/stats") +def cache_stats() -> dict: + """ + Return row counts for all cache tables. + + Useful for debugging and verifying that caching is working as expected. + """ + return get_cache().stats() + + +@app.delete("/cache/device") +def cache_clear_device( + port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), + baud: int = Query(38400, description="Serial baud rate"), + host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), + tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), +) -> dict: + """ + Clear all cached data for a specific device (identified by its connection address). + + Clears: device info, all event headers, all waveforms, monitor status. + The next request to any endpoint for this device will re-fetch from the device. + + Supply either *port* (serial) or *host* (TCP/modem) to identify the device. + """ + conn_key = SFMCache.make_conn_key(host, tcp_port, port, baud) + counts = get_cache().clear_device(conn_key) + return {"status": "cleared", "conn_key": conn_key, "deleted": counts} + + # ── Entry point ──────────────────────────────────────────────────────────────── if __name__ == "__main__": From 37d32077a451c443840f1c57d05b93eef1f00e8d Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 9 Apr 2026 12:10:52 -0400 Subject: [PATCH 02/33] feat: add ACH TCP bridge, serial tap tool, and Serial Watch tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - bridges/ach_bridge.py: transparent TCP bridge that MITMs the MiniMate Plus call-home connection — forwards to real ACH server while logging all frames to raw_client/raw_server .bin files compatible with parse_capture.py; standalone capture mode for lab use without a real server - bridges/serial_watch.py: RS-232 serial monitor with live S3 frame parsing; taps the line between MiniMate and modem (RV50/RV55); captures raw bytes, .log and .jsonl; --ack-ok mode auto-replies to AT commands; fixed fatal indentation bug in the original that silently prevented any data capture - seismo_lab.py: new "Serial Watch" fourth tab (SerialWatchPanel) wrapping serial_watch.py functionality; COM port picker with refresh, baud config, ack-ok toggle, colour-coded live frame log (teal frames / yellow ctrl / blue AT), raw .bin capture auto-fed into Analyzer tab on stop Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_bridge.py | 401 ++++++++++++++++++++++++++++++++++++ bridges/serial_watch.py | 435 ++++++++++++++++++++++++++++++++++++++++ seismo_lab.py | 404 +++++++++++++++++++++++++++++++++++++ 3 files changed, 1240 insertions(+) create mode 100644 bridges/ach_bridge.py create mode 100644 bridges/serial_watch.py diff --git a/bridges/ach_bridge.py b/bridges/ach_bridge.py new file mode 100644 index 0000000..1028f99 --- /dev/null +++ b/bridges/ach_bridge.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +ach_bridge.py — Transparent TCP bridge for capturing Instantel MiniMate Plus + call-home (ACH) traffic. + +Usage +----- + # Bridge mode: forward to real ACH server while logging + python bridges/ach_bridge.py --upstream HOST:PORT [--port 9034] + + # Standalone capture mode: accept connection, don't forward (use when you + # want to see what the device sends/expects without a real server) + python bridges/ach_bridge.py --standalone [--port 9034] + +Setup +----- +1. Find the "Remote Hostname/IP" and port in ACEmanager → Dual SIM / WAN → + Call Home (or equivalent menu on your RV50/RV55 firmware). +2. Temporarily change that setting on ONE unit to point at: + your-machine-local-ip : <--port> +3. Run this script. +4. Wait for the unit to trigger / call home. A capture file is written to + bridges/captures/ach_/ alongside an auto-parsed frame log. +5. Revert the unit's ACEmanager setting. + +Output +------ + bridges/captures/ach_/ + raw_client_.bin — raw bytes from the device (S3 side) + raw_server_.bin — raw bytes from the upstream server (BW side) + (empty in standalone mode) + session_.log — human-readable frame parse log + session_.jsonl — JSON-lines frame log (for downstream tooling) + +The raw_client / raw_server files are byte-for-byte compatible with the +existing capture format used by bridges/parse_capture.py and the rest of +the analysis tooling. +""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import json +import logging +import os +import sys +from pathlib import Path +from typing import Optional + +# Add project root to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from minimateplus.framing import S3FrameParser, S3Frame + +log = logging.getLogger("ach_bridge") + + +# ── Frame label helpers ────────────────────────────────────────────────────── + +_KNOWN_RSP_SUBS = { + 0xA4: "POLL_RSP", + 0xA5: "BULK_WAVEFORM_RSP", + 0xE0: "ADVANCE_EVENT_RSP", + 0xE1: "EVENT_INDEX_FIRST_RSP", + 0xE3: "MONITOR_STATUS_RSP", + 0xEA: "SERIAL_NUM_RSP", + 0xF3: "WAVEFORM_RECORD_RSP", + 0xF5: "WAVEFORM_HEADER_RSP", + 0xF7: "EVENT_INDEX_RSP", + 0xF9: "UNK_06_RSP", + 0xFE: "DEVICE_INFO_RSP", + # Write acks + 0x97: "EVT_IDX_WRITE_ACK", + 0x8C: "CONFIRM_B_ACK", + 0x8E: "COMPLIANCE_WRITE_ACK", + 0x8D: "CONFIRM_A_ACK", + 0x7D: "TRIGGER_WRITE_ACK", + 0x7C: "TRIGGER_CONFIRM_ACK", + 0x96: "WAVEFORM_WRITE_ACK", + 0x8B: "CONFIRM_C_ACK", + 0x69: "START_MONITOR_ACK", + 0x68: "STOP_MONITOR_ACK", +} + +_KNOWN_REQ_SUBS = { + 0x5B: "POLL", + 0x5A: "BULK_WAVEFORM", + 0x1F: "ADVANCE_EVENT", + 0x1E: "EVENT_INDEX_FIRST", + 0x1C: "MONITOR_STATUS", + 0x15: "SERIAL_NUM", + 0x0C: "WAVEFORM_RECORD", + 0x0A: "WAVEFORM_HEADER", + 0x08: "EVENT_INDEX", + 0x06: "UNK_06", + 0x01: "DEVICE_INFO", + # Write commands + 0x68: "EVT_IDX_WRITE", + 0x73: "CONFIRM_B", + 0x71: "COMPLIANCE_WRITE", + 0x72: "CONFIRM_A", + 0x82: "TRIGGER_WRITE", + 0x83: "TRIGGER_CONFIRM", + 0x69: "WAVEFORM_WRITE", + 0x74: "CONFIRM_C", + 0x96: "START_MONITOR", + 0x97: "STOP_MONITOR", +} + + +def _label_s3_frame(frame: S3Frame) -> str: + name = _KNOWN_RSP_SUBS.get(frame.sub, f"UNK_0x{frame.sub:02X}") + chk = "✓" if frame.checksum_valid else "✗CHK" + return f"S3→ SUB=0x{frame.sub:02X} ({name}) page=0x{frame.page_key:04X} data={len(frame.data)}B {chk}" + + +def _label_bw_frame(data: bytes) -> str: + """Best-effort label for a raw BW request frame.""" + # BW frame (destuffed): ACK STX [10 10] flags sub ... + # Wire: 41 02 10 10 00 sub ... + if len(data) < 6: + return f"BW→ (short {len(data)}B)" + sub = data[5] + name = _KNOWN_REQ_SUBS.get(sub, f"UNK_0x{sub:02X}") + return f" →BW SUB=0x{sub:02X} ({name}) {len(data)}B" + + +# ── Per-session capture writer ──────────────────────────────────────────────── + +class CaptureSession: + """Writes raw bytes + parsed log for one TCP connection.""" + + def __init__(self, capture_dir: Path, peer: str): + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + self.dir = capture_dir / f"ach_{ts}" + self.dir.mkdir(parents=True, exist_ok=True) + self.peer = peer + + self._raw_client = open(self.dir / f"raw_client_{ts}.bin", "wb") + self._raw_server = open(self.dir / f"raw_server_{ts}.bin", "wb") + self._log_fh = open(self.dir / f"session_{ts}.log", "w") + self._jsonl_fh = open(self.dir / f"session_{ts}.jsonl", "w") + + self._s3_parser = S3FrameParser() + self._frame_count = 0 + self._byte_count_client = 0 + self._byte_count_server = 0 + + self._log(f"# ACH capture — peer={peer} started={datetime.datetime.now().isoformat()}") + self._log(f"# Output dir: {self.dir}") + log.info("Capture session opened: %s (peer=%s)", self.dir, peer) + + # ── public API ─────────────────────────────────────────────────────────── + + def feed_client(self, data: bytes) -> None: + """Bytes arriving FROM the device (S3 side).""" + self._raw_client.write(data) + self._raw_client.flush() + self._byte_count_client += len(data) + + # Parse S3 frames + for byte in data: + frame = self._s3_parser.feed(bytes([byte])) + if frame: + frames = frame if isinstance(frame, list) else [frame] + for f in frames: + self._frame_count += 1 + label = _label_s3_frame(f) + self._log(f"[{self._frame_count:04d}] {label}") + self._log(f" hex: {f.data[:64].hex()}" + + (" ..." if len(f.data) > 64 else "")) + self._emit_json("s3", f) + + def feed_server(self, data: bytes) -> None: + """Bytes arriving FROM the upstream server (BW side).""" + self._raw_server.write(data) + self._raw_server.flush() + self._byte_count_server += len(data) + label = _label_bw_frame(data) + self._log(f" {label}") + + def close(self, reason: str = "connection closed") -> None: + self._log(f"# Session ended: {reason}") + self._log(f"# Totals — client_bytes={self._byte_count_client} " + f"server_bytes={self._byte_count_server} " + f"s3_frames={self._frame_count}") + for fh in (self._raw_client, self._raw_server, self._log_fh, self._jsonl_fh): + try: + fh.close() + except Exception: + pass + log.info( + "Capture session closed (%s): %dB client, %dB server, %d S3 frames → %s", + reason, self._byte_count_client, self._byte_count_server, + self._frame_count, self.dir, + ) + + # ── internals ──────────────────────────────────────────────────────────── + + def _log(self, msg: str) -> None: + print(msg, file=self._log_fh, flush=True) + # Also echo to console for live monitoring + print(msg) + + def _emit_json(self, direction: str, frame: S3Frame) -> None: + record = { + "dir": direction, + "sub": frame.sub, + "page_key": frame.page_key, + "data_len": len(frame.data), + "data_hex": frame.data.hex(), + "checksum_valid": frame.checksum_valid, + } + print(json.dumps(record), file=self._jsonl_fh, flush=True) + + +# ── Bridge connection handler ───────────────────────────────────────────────── + +class BridgeHandler: + def __init__( + self, + capture_dir: Path, + upstream_host: Optional[str], + upstream_port: Optional[int], + ): + self.capture_dir = capture_dir + self.upstream_host = upstream_host + self.upstream_port = upstream_port + + async def handle( + self, + client_reader: asyncio.StreamReader, + client_writer: asyncio.StreamWriter, + ) -> None: + peer = client_writer.get_extra_info("peername", ("?", 0)) + peer_str = f"{peer[0]}:{peer[1]}" + log.info("Inbound connection from %s", peer_str) + + session = CaptureSession(self.capture_dir, peer_str) + + if self.upstream_host: + # Bridge mode: connect to upstream and relay + try: + up_reader, up_writer = await asyncio.open_connection( + self.upstream_host, self.upstream_port + ) + log.info("Connected to upstream %s:%s", self.upstream_host, self.upstream_port) + except Exception as exc: + log.error("Failed to connect to upstream: %s", exc) + session.close(f"upstream connect failed: {exc}") + client_writer.close() + return + + try: + await asyncio.gather( + self._relay(client_reader, up_writer, session, "client"), + self._relay(up_reader, client_writer, session, "server"), + ) + except asyncio.CancelledError: + pass + except Exception as exc: + log.warning("Bridge relay error: %s", exc) + finally: + session.close("bridge relay ended") + for writer in (client_writer, up_writer): + try: + writer.close() + await writer.wait_closed() + except Exception: + pass + + else: + # Standalone mode: just capture, don't forward + log.info("Standalone mode — recording inbound traffic only") + try: + while True: + data = await client_reader.read(4096) + if not data: + break + session.feed_client(data) + except asyncio.CancelledError: + pass + except Exception as exc: + log.warning("Standalone read error: %s", exc) + finally: + session.close("standalone capture ended") + try: + client_writer.close() + await client_writer.wait_closed() + except Exception: + pass + + async def _relay( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + session: CaptureSession, + direction: str, + ) -> None: + try: + while True: + data = await reader.read(4096) + if not data: + break + if direction == "client": + session.feed_client(data) + else: + session.feed_server(data) + writer.write(data) + await writer.drain() + except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError): + pass + + +# ── Main ────────────────────────────────────────────────────────────────────── + +async def main(args: argparse.Namespace) -> None: + capture_dir = Path(__file__).parent / "captures" + capture_dir.mkdir(parents=True, exist_ok=True) + + upstream_host: Optional[str] = None + upstream_port: Optional[int] = None + + if not args.standalone: + if not args.upstream: + print("ERROR: --upstream HOST:PORT is required unless --standalone is set.") + sys.exit(1) + parts = args.upstream.rsplit(":", 1) + if len(parts) != 2: + print("ERROR: --upstream must be HOST:PORT (e.g. 203.0.113.5:9034)") + sys.exit(1) + upstream_host = parts[0] + upstream_port = int(parts[1]) + + handler = BridgeHandler(capture_dir, upstream_host, upstream_port) + + server = await asyncio.start_server( + handler.handle, + host="0.0.0.0", + port=args.port, + ) + + mode = f"bridge → {upstream_host}:{upstream_port}" if upstream_host else "standalone capture" + addrs = ", ".join(str(s.getsockname()) for s in server.sockets) + print(f"\n{'='*60}") + print(f" ACH bridge listening on {addrs}") + print(f" Mode: {mode}") + print(f" Captures: {capture_dir}/ach_/") + print(f"{'='*60}") + print(f"\n Point your unit's ACEmanager call-home destination to:") + print(f" :{args.port}") + if upstream_host: + print(f"\n All traffic will be forwarded to {upstream_host}:{upstream_port}") + print(f" Your live data feed is uninterrupted.") + print(f"\n Waiting for inbound connection... (Ctrl-C to stop)\n") + + async with server: + await server.serve_forever() + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Transparent TCP bridge for capturing MiniMate Plus call-home traffic." + ) + p.add_argument( + "--upstream", "-u", + metavar="HOST:PORT", + help="Upstream ACH server to forward to (e.g. 203.0.113.5:9034). " + "Omit with --standalone for capture-only mode.", + ) + p.add_argument( + "--port", "-p", + type=int, + default=9034, + help="Local port to listen on (default: 9034).", + ) + p.add_argument( + "--standalone", "-s", + action="store_true", + help="Capture-only mode: accept connection but do not forward to upstream.", + ) + p.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable debug logging.", + ) + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-7s %(name)s %(message)s", + ) + try: + asyncio.run(main(args)) + except KeyboardInterrupt: + print("\nStopped.") diff --git a/bridges/serial_watch.py b/bridges/serial_watch.py new file mode 100644 index 0000000..cd24f94 --- /dev/null +++ b/bridges/serial_watch.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +""" +serial_watch.py — Instantel Series-3 serial monitor with S3 frame parsing. + +Taps the RS-232 line between the MiniMate Plus and its modem (RV50/RV55). +Saves raw binary captures compatible with the rest of the analysis toolchain, +plus a human-readable frame log. + +Usage +----- + python bridges/serial_watch.py # interactive COM picker + python bridges/serial_watch.py --port COM3 # specify port + python bridges/serial_watch.py --port COM3 --ack-ok # reply OK to AT commands + # (useful if modem is absent + # and you want the device to + # proceed past AT negotiation) + python bridges/serial_watch.py --list # list available ports + +Output +------ + bridges/captures/serial_/ + raw_s3_.bin — raw bytes from device (feeds directly into S3FrameParser) + session_.log — human-readable frame + control-line log + session_.jsonl — JSON-lines frame log + +The raw_s3_*.bin file is byte-for-byte compatible with the existing capture +format used by bridges/parse_capture.py and all analysis scripts. + +What to look for in a call-home capture +---------------------------------------- +1. Does the device talk first after CONNECT, or does it wait? + - If raw_s3_*.bin has bytes before any AT/POLL exchange → PUSH protocol + - If it stays silent → PULL protocol (same as Blastware manual download) + +2. Look for "Operating System" ASCII at the start — the device sends this 16-byte + boot string on cold start before entering DLE-framed mode. + +3. RING/CONNECT from the modem appear as ASCII before the DLE frames — the parser + handles these automatically (scans forward to DLE+STX). +""" + +from __future__ import annotations + +import argparse +import sys +import threading +import time +from datetime import datetime +from pathlib import Path + +try: + import serial + from serial.tools import list_ports +except ModuleNotFoundError: + print( + "pyserial not found. Install with:\n python -m pip install pyserial", + file=sys.stderr, + ) + sys.exit(1) + +# Add project root so we can import the frame parser +sys.path.insert(0, str(Path(__file__).parent.parent)) +from minimateplus.framing import S3FrameParser, S3Frame + +import json + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def _ts() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + + +def _hexdump(b: bytes) -> str: + return " ".join(f"{x:02X}" for x in b) + + +def _printable(b: bytes) -> str: + return b.decode("latin1", errors="replace") + + +_KNOWN_SUBS = { + 0xA4: "POLL_RSP", 0xA5: "BULK_WAVEFORM_RSP", 0xE0: "ADVANCE_EVENT_RSP", + 0xE1: "EVENT_IDX_FIRST_RSP", 0xE3: "MONITOR_STATUS_RSP", 0xEA: "SERIAL_NUM_RSP", + 0xF3: "WAVEFORM_RECORD_RSP", 0xF5: "WAVEFORM_HEADER_RSP", 0xF7: "EVENT_INDEX_RSP", + 0xF9: "UNK_06_RSP", 0xFE: "DEVICE_INFO_RSP", + 0x69: "START_MONITOR_ACK", 0x68: "STOP_MONITOR_ACK", + 0x97: "EVT_IDX_WRITE_ACK", 0x8C: "CONFIRM_B_ACK", 0x8E: "COMPLIANCE_WRITE_ACK", + 0x8D: "CONFIRM_A_ACK", 0x7D: "TRIGGER_WRITE_ACK", 0x7C: "TRIGGER_CONFIRM_ACK", + 0x96: "WAVEFORM_WRITE_ACK", 0x8B: "CONFIRM_C_ACK", +} + + +def _label_frame(frame: S3Frame) -> str: + name = _KNOWN_SUBS.get(frame.sub, f"UNK_0x{frame.sub:02X}") + chk = "✓" if frame.checksum_valid else "✗ BAD_CHK" + peek = frame.data[:24].hex() + ("…" if len(frame.data) > 24 else "") + return ( + f"S3 SUB=0x{frame.sub:02X} ({name:<22}) " + f"page=0x{frame.page_key:04X} data={len(frame.data):4d}B {chk} {peek}" + ) + + +# ── Logger ──────────────────────────────────────────────────────────────────── + +class Logger: + def __init__(self, log_path: Path, jsonl_path: Path, raw_path: Path) -> None: + self._log = log_path.open("a", encoding="utf-8", newline="") + self._jl = jsonl_path.open("a", encoding="utf-8", newline="") + self._raw = raw_path.open("ab") + self._lock = threading.Lock() + self._frame_count = 0 + + def info(self, msg: str) -> None: + line = f"[{_ts()}] INFO | {msg}" + with self._lock: + print(line) + print(line, file=self._log, flush=True) + + def ctrl(self, msg: str) -> None: + line = f"[{_ts()}] CTRL | {msg}" + with self._lock: + print(line) + print(line, file=self._log, flush=True) + + def data_hex(self, msg: str) -> None: + line = f"[{_ts()}] HEX | {msg}" + with self._lock: + print(line) + print(line, file=self._log, flush=True) + + def data_ascii(self, msg: str) -> None: + line = f"[{_ts()}] DATA | {msg}" + with self._lock: + print(line) + print(line, file=self._log, flush=True) + + def frame(self, f: S3Frame) -> None: + with self._lock: + self._frame_count += 1 + label = f"[{_ts()}] FRAME | #{self._frame_count:04d} {_label_frame(f)}" + print(label) + print(label, file=self._log, flush=True) + record = { + "frame": self._frame_count, + "sub": f.sub, + "page_key": f.page_key, + "data_len": len(f.data), + "data_hex": f.data.hex(), + "checksum_valid": f.checksum_valid, + } + print(json.dumps(record), file=self._jl, flush=True) + + def write_raw(self, data: bytes) -> None: + with self._lock: + self._raw.write(data) + self._raw.flush() + + def close(self) -> None: + with self._lock: + for fh in (self._log, self._jl, self._raw): + try: + fh.flush() + fh.close() + except Exception: + pass + + +# ── Control-line monitor thread ─────────────────────────────────────────────── + +def _monitor_control_lines( + ser: serial.Serial, + logger: Logger, + stop: threading.Event, + interval: float, +) -> None: + prev = dict(CTS=None, DSR=None, DCD=None, RI=None) + try: + prev.update(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd) + try: + prev["RI"] = ser.ri + except Exception: + pass + except Exception as exc: + logger.ctrl(f"Init error: {exc}") + return + + logger.ctrl( + f"Initial: CTS={prev['CTS']} DSR={prev['DSR']} DCD={prev['DCD']} RI={prev['RI']}" + ) + while not stop.is_set(): + try: + cur = dict(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd, RI=None) + try: + cur["RI"] = ser.ri + except Exception: + pass + for name, val in cur.items(): + if val != prev[name]: + logger.ctrl(f"{name} → {val}") + prev[name] = val + except serial.SerialException as exc: + logger.ctrl(f"Poll error: {exc}") + break + stop.wait(interval) + + +# ── Serial open ─────────────────────────────────────────────────────────────── + +_PARITY = { + "N": serial.PARITY_NONE, "E": serial.PARITY_EVEN, "O": serial.PARITY_ODD, + "M": serial.PARITY_MARK, "S": serial.PARITY_SPACE, +} +_STOPBITS = { + 1: serial.STOPBITS_ONE, 1.5: serial.STOPBITS_ONE_POINT_FIVE, 2: serial.STOPBITS_TWO, +} + + +def _open_serial(args: argparse.Namespace, logger: Logger) -> serial.Serial | None: + for attempt in range(1, args.open_retries + 2): + logger.info( + f"Opening {args.port} @ {args.baud},{args.bytesize}{args.parity}{args.stopbits} " + f"rtscts={args.rtscts} xonxoff={args.xonxoff} dsrdtr={args.dsrdtr} " + f"(attempt {attempt})" + ) + try: + ser = serial.Serial( + port=args.port, + baudrate=args.baud, + bytesize=args.bytesize, + parity=_PARITY[args.parity], + stopbits=_STOPBITS[args.stopbits], + timeout=args.timeout, + xonxoff=args.xonxoff, + rtscts=args.rtscts, + dsrdtr=args.dsrdtr, + write_timeout=0, + ) + try: + ser.setDTR(args.dtr == "on") + ser.setRTS(args.rts == "on") + logger.ctrl(f"Set DTR={args.dtr} RTS={args.rts}") + except Exception as exc: + logger.ctrl(f"DTR/RTS set failed: {exc}") + + if args.send_break > 0: + try: + ser.break_condition = True + time.sleep(args.send_break / 1000.0) + ser.break_condition = False + logger.ctrl(f"BREAK held {args.send_break} ms") + except Exception as exc: + logger.ctrl(f"BREAK failed: {exc}") + + return ser + + except serial.SerialException as exc: + logger.info(f"Open failed: {exc}") + if attempt <= args.open_retries: + time.sleep(args.open_retry_delay) + + return None + + +# ── Port picker ─────────────────────────────────────────────────────────────── + +def _list_ports() -> list: + ports = list(list_ports.comports()) + if not ports: + print("No serial ports found.") + return [] + print("Available serial ports:") + for i, p in enumerate(ports, 1): + print(f" {i:2d}) {p.device:<12} {p.description or ''}") + return ports + + +def _pick_port() -> str: + ports = _list_ports() + if not ports: + sys.exit(1) + if len(ports) == 1: + print(f"Auto-selecting: {ports[0].device}") + return ports[0].device + while True: + sel = input("Select port (number or name, e.g. COM3): ").strip() + if sel.isdigit() and 1 <= int(sel) <= len(ports): + return ports[int(sel) - 1].device + for p in ports: + if p.device.upper() == sel.upper(): + return p.device + print("Not recognised. Enter list number or exact port name.") + + +# ── Main loop ───────────────────────────────────────────────────────────────── + +def main() -> None: + ap = argparse.ArgumentParser( + description="Monitor Instantel Series-3 serial traffic with S3 frame parsing." + ) + ap.add_argument("--port", "-p", + help="COM port (e.g. COM3). Omit to be prompted.") + ap.add_argument("--baud", "-b", type=int, default=38400) + ap.add_argument("--bytesize", type=int, choices=[5, 6, 7, 8], default=8) + ap.add_argument("--parity", choices=["N", "E", "O", "M", "S"], default="N") + ap.add_argument("--stopbits", type=float, choices=[1, 1.5, 2], default=1) + ap.add_argument("--rtscts", action="store_true") + ap.add_argument("--xonxoff", action="store_true") + ap.add_argument("--dsrdtr", action="store_true") + ap.add_argument("--dtr", choices=["on", "off"], default="on") + ap.add_argument("--rts", choices=["on", "off"], default="on") + ap.add_argument("--send-break", type=int, default=0, + help="Hold BREAK for N ms after open.") + ap.add_argument("--show", choices=["ascii", "hex", "both", "frames"], + default="frames", + help="'frames' (default) shows only parsed S3 frames. " + "'ascii'/'hex'/'both' also show raw bytes.") + ap.add_argument("--encoding", default="latin1") + ap.add_argument("--read-chunk", type=int, default=4096) + ap.add_argument("--timeout", type=float, default=0.05) + ap.add_argument("--poll-lines-interval", type=float, default=0.2) + ap.add_argument("--open-retries", type=int, default=0) + ap.add_argument("--open-retry-delay", type=float, default=0.8) + ap.add_argument("--ack-ok", action="store_true", + help="Auto-reply OK to AT* commands (except ATDT). " + "Useful for testing without a real modem.") + ap.add_argument("--list", action="store_true", + help="List available serial ports and exit.") + args = ap.parse_args() + + if args.list: + _list_ports() + return + + args.port = args.port or _pick_port() + + # Build output paths + ts_str = datetime.now().strftime("%Y%m%d_%H%M%S") + out_dir = Path(__file__).parent / "captures" / f"serial_{ts_str}" + out_dir.mkdir(parents=True, exist_ok=True) + + log_path = out_dir / f"session_{ts_str}.log" + jsonl_path = out_dir / f"session_{ts_str}.jsonl" + raw_path = out_dir / f"raw_s3_{ts_str}.bin" + + logger = Logger(log_path, jsonl_path, raw_path) + logger.info(f"Output directory: {out_dir}") + logger.info(f"raw_s3 → {raw_path.name} (compatible with parse_capture.py)") + + ser = _open_serial(args, logger) + if ser is None: + logger.info("Could not open serial port. Exiting.") + logger.close() + sys.exit(1) + + s3_parser = S3FrameParser() + rx_buf = bytearray() + stop_evt = threading.Event() + + ctrl_thread = threading.Thread( + target=_monitor_control_lines, + args=(ser, logger, stop_evt, args.poll_lines_interval), + daemon=True, + ) + ctrl_thread.start() + logger.info("Monitoring started. Waiting for call-home. Press Ctrl+C to stop.") + + try: + while True: + try: + data = ser.read(args.read_chunk) + except serial.SerialException as exc: + logger.info(f"Read error: {exc}") + break + + if not data: + continue + + # 1. Save raw bytes + logger.write_raw(data) + + # 2. Optional raw display + if args.show in ("ascii", "both"): + txt = _printable(data) + for line in txt.splitlines(): + logger.data_ascii(line) + if args.show in ("hex", "both"): + logger.data_hex(_hexdump(data)) + + # 3. Parse S3 frames + for byte in data: + result = s3_parser.feed(bytes([byte])) + if result: + frames = result if isinstance(result, list) else [result] + for f in frames: + logger.frame(f) + + # 4. AT command handling for --ack-ok + if args.ack_ok: + rx_buf.extend(data) + while b"\r" in rx_buf or b"\n" in rx_buf: + for sep in (b"\r", b"\n"): + idx = rx_buf.find(sep) + if idx != -1: + line_bytes = bytes(rx_buf[:idx]) + del rx_buf[:idx + 1] + break + else: + break + + line_str = line_bytes.decode("latin1", errors="ignore").strip().upper() + if line_str.startswith("AT") and not line_str.startswith("ATDT"): + try: + ser.write(b"\r\nOK\r\n") + ser.flush() + logger.info(f"AT ack: {line_str!r} → OK") + except Exception as exc: + logger.info(f"AT ack write failed: {exc}") + + except KeyboardInterrupt: + logger.info("Ctrl+C — stopping.") + + finally: + stop_evt.set() + try: + ser.close() + except Exception: + pass + ctrl_thread.join(timeout=1.0) + logger.info(f"Capture saved to: {out_dir}") + logger.close() + + +if __name__ == "__main__": + main() diff --git a/seismo_lab.py b/seismo_lab.py index 687e6a3..2c85222 100644 --- a/seismo_lab.py +++ b/seismo_lab.py @@ -1071,6 +1071,398 @@ class AnalyzerPanel(tk.Frame): # ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Serial Watch panel — tap the RS-232 line between device and modem +# ───────────────────────────────────────────────────────────────────────────── + +try: + import serial as _serial + from serial.tools import list_ports as _list_ports + _SERIAL_OK = True +except ImportError: + _SERIAL_OK = False + +from minimateplus.framing import S3FrameParser as _S3FrameParser # noqa: E402 + +_SW_KNOWN_SUBS = { + 0xA4: "POLL_RSP", 0xA5: "BULK_WAVEFORM_RSP", 0xE0: "ADV_EVENT_RSP", + 0xE1: "EVT_IDX_FIRST_RSP", 0xE3: "MONITOR_STATUS_RSP", 0xEA: "SERIAL_NUM_RSP", + 0xF3: "WAVEFORM_REC_RSP", 0xF5: "WAVEFORM_HDR_RSP", 0xF7: "EVENT_INDEX_RSP", + 0xF9: "UNK_06_RSP", 0xFE: "DEVICE_INFO_RSP", + 0x69: "START_MON_ACK", 0x68: "STOP_MON_ACK", +} + + +class SerialWatchPanel(tk.Frame): + """ + Tap the RS-232 line between the MiniMate Plus and its modem (RV50/RV55). + Runs the serial reader in a background thread; surfaces parsed S3 frames + live in the log view. Writes raw_s3_.bin compatible with Analyzer. + + Typical use for call-home capture: + 1. Connect a USB-to-serial tap to the RS-232 line. + 2. Pick that COM port here, click Start. + 3. Wait for the unit to trigger / call home. + 4. Click Stop, then 'Open in Analyzer' to inspect the frames. + """ + + _COL_FRAME = "#4ec9b0" # teal — parsed S3 frame + _COL_CTRL = "#dcdcaa" # yellow — control-line change + _COL_AT = "#9cdcfe" # blue — AT command / ASCII noise + _COL_ERR = "#f44747" # red — error + + def __init__(self, parent: tk.Widget, on_capture_ready=None, **kw): + """ + on_capture_ready(raw_s3_path: str) — called when capture stops, + so the parent can inject the file into the Analyzer. + """ + super().__init__(parent, bg=BG2, **kw) + self._on_capture_ready = on_capture_ready + self._serial: Optional[object] = None # serial.Serial instance + self._reader_thread: Optional[threading.Thread] = None + self._stop_evt = threading.Event() + self._log_q: queue.Queue[tuple[str, str]] = queue.Queue() # (text, colour) + self._raw_fh = None # open binary file handle + self._raw_path: Optional[str] = None + self._frame_count = 0 + self._build() + self._poll_log_queue() + + # ── build ───────────────────────────────────────────────────────────── + + def _build(self) -> None: + pad = {"padx": 6, "pady": 4} + + cfg = tk.Frame(self, bg=BG2) + cfg.pack(side=tk.TOP, fill=tk.X, padx=4, pady=4) + + # Row 0 — port picker + tk.Label(cfg, text="COM port:", bg=BG2, fg=FG, font=MONO + ).grid(row=0, column=0, sticky="e", **pad) + + self._port_var = tk.StringVar() + self._port_cb = ttk.Combobox(cfg, textvariable=self._port_var, + width=12, font=MONO, state="normal") + self._port_cb.grid(row=0, column=1, sticky="w", **pad) + + tk.Button(cfg, text="↺", bg=BG3, fg=FG, relief="flat", cursor="hand2", + font=MONO, command=self._refresh_ports + ).grid(row=0, column=2, **pad) + + tk.Label(cfg, text=" Baud:", bg=BG2, fg=FG, font=MONO + ).grid(row=0, column=3, sticky="e", **pad) + self._baud_var = tk.StringVar(value="38400") + tk.Entry(cfg, textvariable=self._baud_var, width=8, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", font=MONO + ).grid(row=0, column=4, sticky="w", **pad) + + self._ack_ok_var = tk.BooleanVar(value=False) + tk.Checkbutton(cfg, text="Ack OK to AT commands", + variable=self._ack_ok_var, + bg=BG2, fg=FG, selectcolor=BG3, activebackground=BG2, + font=MONO).grid(row=0, column=5, sticky="w", **pad) + + # Row 1 — capture dir + tk.Label(cfg, text="Save to:", bg=BG2, fg=FG, font=MONO + ).grid(row=1, column=0, sticky="e", **pad) + self._dir_var = tk.StringVar( + value=str(SCRIPT_DIR / "bridges" / "captures")) + tk.Entry(cfg, textvariable=self._dir_var, width=40, + bg=BG3, fg=FG, insertbackground=FG, relief="flat", font=MONO + ).grid(row=1, column=1, columnspan=4, sticky="we", **pad) + tk.Button(cfg, text="Browse", bg=BG3, fg=FG, relief="flat", + cursor="hand2", font=MONO, command=self._choose_dir + ).grid(row=1, column=5, **pad) + + # Button row + btn_row = tk.Frame(self, bg=BG2) + btn_row.pack(side=tk.TOP, fill=tk.X, padx=4, pady=2) + + self._start_btn = tk.Button( + btn_row, text="Start Watch", bg=GREEN, fg="#000000", + relief="flat", padx=12, cursor="hand2", font=MONO_B, + command=self._start) + self._start_btn.pack(side=tk.LEFT, padx=6) + + self._stop_btn = tk.Button( + btn_row, text="Stop", bg=BG3, fg=FG, + relief="flat", padx=12, cursor="hand2", font=MONO, + command=self._stop, state="disabled") + self._stop_btn.pack(side=tk.LEFT, padx=4) + + self._analyzer_btn = tk.Button( + btn_row, text="Open in Analyzer", bg=BG3, fg=FG, + relief="flat", padx=10, cursor="hand2", font=MONO, + command=self._send_to_analyzer, state="disabled") + self._analyzer_btn.pack(side=tk.LEFT, padx=4) + + tk.Button(btn_row, text="Clear", bg=BG3, fg=FG, + relief="flat", padx=8, cursor="hand2", font=MONO, + command=self._clear_log).pack(side=tk.LEFT, padx=4) + + self._status_var = tk.StringVar(value="Idle") + tk.Label(btn_row, textvariable=self._status_var, + bg=BG2, fg=FG_DIM, font=MONO).pack(side=tk.LEFT, padx=10) + + # Log view + self._log = scrolledtext.ScrolledText( + self, height=24, font=MONO_SM, + bg=BG, fg=FG, insertbackground=FG, + relief="flat", state="disabled", + ) + self._log.pack(fill=tk.BOTH, expand=True, padx=4, pady=4) + self._log.tag_config("frame", foreground=self._COL_FRAME) + self._log.tag_config("ctrl", foreground=self._COL_CTRL) + self._log.tag_config("at", foreground=self._COL_AT) + self._log.tag_config("err", foreground=self._COL_ERR) + self._log.tag_config("dim", foreground=FG_DIM) + + # Populate ports on first load + self._refresh_ports() + + # ── port helpers ────────────────────────────────────────────────────── + + def _refresh_ports(self) -> None: + if not _SERIAL_OK: + self._port_cb["values"] = ["(pyserial not installed)"] + return + ports = [p.device for p in _list_ports.comports()] + self._port_cb["values"] = ports + if ports and not self._port_var.get(): + self._port_var.set(ports[0]) + + def _choose_dir(self) -> None: + d = filedialog.askdirectory(initialdir=self._dir_var.get()) + if d: + self._dir_var.set(d) + + # ── start / stop ────────────────────────────────────────────────────── + + def _start(self) -> None: + if not _SERIAL_OK: + messagebox.showerror( + "pyserial missing", + "Install pyserial first:\n pip install pyserial") + return + + port = self._port_var.get().strip() + if not port or "not installed" in port: + messagebox.showerror("Error", "Select a valid COM port first.") + return + + try: + baud = int(self._baud_var.get().strip()) + except ValueError: + messagebox.showerror("Error", "Invalid baud rate.") + return + + # Open output files + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out_dir = Path(self._dir_var.get()) / f"serial_{ts}" + out_dir.mkdir(parents=True, exist_ok=True) + self._raw_path = str(out_dir / f"raw_s3_{ts}.bin") + try: + self._raw_fh = open(self._raw_path, "wb") + except OSError as exc: + messagebox.showerror("Error", f"Cannot open capture file:\n{exc}") + return + + # Open serial port + try: + ser = _serial.Serial( + port=port, baudrate=baud, + bytesize=8, parity=_serial.PARITY_NONE, + stopbits=_serial.STOPBITS_ONE, + timeout=0.05, write_timeout=0, + ) + ser.setDTR(True) + ser.setRTS(True) + except Exception as exc: + self._raw_fh.close() + self._raw_fh = None + messagebox.showerror("Error", f"Cannot open {port}:\n{exc}") + return + + self._serial = ser + self._stop_evt.clear() + self._frame_count = 0 + self._analyzer_btn.configure(state="disabled") + + self._reader_thread = threading.Thread( + target=self._reader_loop, + args=(ser, baud), + daemon=True, + ) + self._reader_thread.start() + + self._status_var.set(f"Watching {port} @ {baud}") + self._start_btn.configure(state="disabled") + self._stop_btn.configure(state="normal", bg=RED) + self._append(f"── Serial watch started {port} @ {baud} [{ts}] ──\n", "dim") + self._append(f" Capture: {self._raw_path}\n", "dim") + self._append(" Waiting for data…\n\n", "dim") + + def _stop(self) -> None: + self._stop_evt.set() + if self._serial: + try: + self._serial.close() + except Exception: + pass + self._serial = None + if self._raw_fh: + self._raw_fh.close() + self._raw_fh = None + self._status_var.set("Stopped") + self._start_btn.configure(state="normal") + self._stop_btn.configure(state="disabled", bg=BG3) + if self._raw_path and Path(self._raw_path).exists(): + self._analyzer_btn.configure(state="normal") + self._append("\n── Watch stopped ──\n", "dim") + + # ── reader thread ───────────────────────────────────────────────────── + + def _reader_loop(self, ser, baud: int) -> None: + parser = _S3FrameParser() + rx_buf = bytearray() + ack_ok = self._ack_ok_var.get() + + # Monitor control lines in a sub-thread + ctrl_stop = threading.Event() + ctrl_thread = threading.Thread( + target=self._ctrl_loop, args=(ser, ctrl_stop), daemon=True) + ctrl_thread.start() + + try: + while not self._stop_evt.is_set(): + try: + data = ser.read(4096) + except Exception as exc: + self._log_q.put((f"Read error: {exc}\n", "err")) + break + + if not data: + continue + + # Save raw bytes + if self._raw_fh: + try: + self._raw_fh.write(data) + self._raw_fh.flush() + except Exception: + pass + + # Parse S3 frames + for byte in data: + result = parser.feed(bytes([byte])) + if result: + frames = result if isinstance(result, list) else [result] + for f in frames: + self._frame_count += 1 + name = _SW_KNOWN_SUBS.get(f.sub, f"UNK_0x{f.sub:02X}") + chk = "✓" if f.checksum_valid else "✗ BAD_CHK" + peek = f.data[:32].hex() + ("…" if len(f.data) > 32 else "") + msg = ( + f"[{self._frame_count:04d}] " + f"SUB=0x{f.sub:02X} ({name:<22}) " + f"page=0x{f.page_key:04X} " + f"data={len(f.data):4d}B {chk}\n" + f" {peek}\n" + ) + self._log_q.put((msg, "frame")) + + # AT command handling for --ack-ok mode + if ack_ok: + rx_buf.extend(data) + while b"\r" in rx_buf or b"\n" in rx_buf: + for sep in (b"\r", b"\n"): + idx = rx_buf.find(sep) + if idx != -1: + line_bytes = bytes(rx_buf[:idx]) + del rx_buf[:idx + 1] + break + else: + break + line_str = line_bytes.decode("latin1", errors="ignore").strip() + if line_str.upper().startswith("AT"): + self._log_q.put((f"AT: {line_str!r}\n", "at")) + if not line_str.upper().startswith("ATDT"): + try: + ser.write(b"\r\nOK\r\n") + ser.flush() + self._log_q.put((f" → OK\n", "at")) + except Exception: + pass + + finally: + ctrl_stop.set() + ctrl_thread.join(timeout=0.5) + # Signal the main thread that the reader ended naturally + if not self._stop_evt.is_set(): + self._log_q.put(("<>", "")) + + def _ctrl_loop(self, ser, stop: threading.Event) -> None: + prev = {} + try: + prev = dict(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd) + try: + prev["RI"] = ser.ri + except Exception: + prev["RI"] = None + except Exception: + return + + while not stop.is_set(): + try: + cur = dict(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd, RI=None) + try: + cur["RI"] = ser.ri + except Exception: + pass + for name, val in cur.items(): + if val != prev.get(name): + self._log_q.put((f"CTRL {name} → {val}\n", "ctrl")) + prev[name] = val + except Exception: + break + stop.wait(0.2) + + # ── log view ────────────────────────────────────────────────────────── + + def _poll_log_queue(self) -> None: + try: + while True: + text, tag = self._log_q.get_nowait() + if text == "<>": + self._stop() + break + self._append(text, tag) + except queue.Empty: + pass + finally: + self.after(80, self._poll_log_queue) + + def _append(self, text: str, tag: str = "") -> None: + self._log.configure(state="normal") + if tag: + self._log.insert(tk.END, text, tag) + else: + self._log.insert(tk.END, text) + self._log.see(tk.END) + self._log.configure(state="disabled") + + def _clear_log(self) -> None: + self._log.configure(state="normal") + self._log.delete("1.0", tk.END) + self._log.configure(state="disabled") + + # ── send to analyzer ────────────────────────────────────────────────── + + def _send_to_analyzer(self) -> None: + if self._raw_path and self._on_capture_ready: + self._on_capture_ready(self._raw_path) + + # Console panel (tk.Frame — lives inside a notebook tab) # ───────────────────────────────────────────────────────────────────────────── @@ -1504,6 +1896,12 @@ class SeismoLab(tk.Tk): ) nb.add(self._console_panel, text=" Console ") + self._serial_watch_panel = SerialWatchPanel( + nb, + on_capture_ready=self._on_serial_capture_ready, + ) + nb.add(self._serial_watch_panel, text=" Serial Watch ") + self._nb = nb self.protocol("WM_DELETE_WINDOW", self._on_close) @@ -1522,8 +1920,14 @@ class SeismoLab(tk.Tk): self._analyzer_panel.s3_var.set(raw_s3_path) self._nb.select(1) + def _on_serial_capture_ready(self, raw_s3_path: str) -> None: + """Serial Watch capture finished → inject into Analyzer and switch tab.""" + self._analyzer_panel.s3_var.set(raw_s3_path) + self._nb.select(1) + def _on_close(self) -> None: self._bridge_panel.stop_bridge() + self._serial_watch_panel._stop() self.destroy() From 5e44cdc66894289a87ff2d0e9fd4ab2cfb3e0f33 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 9 Apr 2026 12:17:57 -0400 Subject: [PATCH 03/33] feat: add splitter mode to ach_bridge.py (--mirror HOST:PORT) Adds a production-safe headphone-splitter mode: - Device bytes tee'd to both --upstream (primary/prod) and --mirror (new server) - Only primary server responses are returned to the device - Mirror connect/write failures are non-fatal and logged; prod is unaffected - New raw_mirror_.bin capture file alongside raw_client/raw_server Three modes: standalone (capture only), bridge (one upstream), splitter (two). Default listen port changed to 12345 to match project ACH setup. Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_bridge.py | 472 +++++++++++++++++++++++++++++++----------- 1 file changed, 349 insertions(+), 123 deletions(-) diff --git a/bridges/ach_bridge.py b/bridges/ach_bridge.py index 1028f99..dbfa03d 100644 --- a/bridges/ach_bridge.py +++ b/bridges/ach_bridge.py @@ -1,40 +1,68 @@ #!/usr/bin/env python3 """ -ach_bridge.py — Transparent TCP bridge for capturing Instantel MiniMate Plus +ach_bridge.py — Transparent TCP bridge / splitter for Instantel MiniMate Plus call-home (ACH) traffic. +Modes +----- + standalone Accept connection, capture frames, do NOT forward anywhere. + Good for initial discovery with a test unit. + + bridge Forward to one upstream server while capturing. + Use this for the initial discovery phase with your test server. + + splitter Forward to the PRIMARY upstream (production ACH server) AND + mirror a copy to a SECONDARY server simultaneously. + The device never knows — it talks to the primary the whole time. + If the mirror fails, the primary connection is unaffected. + + Think of it like a headphone splitter: one input, two outputs. + Primary → authoritative responses back to device. + Mirror → gets all device bytes, its responses are discarded. + Usage ----- - # Bridge mode: forward to real ACH server while logging - python bridges/ach_bridge.py --upstream HOST:PORT [--port 9034] + # Standalone capture (test/discovery — no forwarding) + python bridges/ach_bridge.py --standalone [--port 12345] - # Standalone capture mode: accept connection, don't forward (use when you - # want to see what the device sends/expects without a real server) - python bridges/ach_bridge.py --standalone [--port 9034] + # Bridge mode (forward to one server, e.g. your test server) + python bridges/ach_bridge.py --upstream HOST:PORT [--port 12345] -Setup ------ -1. Find the "Remote Hostname/IP" and port in ACEmanager → Dual SIM / WAN → - Call Home (or equivalent menu on your RV50/RV55 firmware). -2. Temporarily change that setting on ONE unit to point at: - your-machine-local-ip : <--port> -3. Run this script. -4. Wait for the unit to trigger / call home. A capture file is written to - bridges/captures/ach_/ alongside an auto-parsed frame log. -5. Revert the unit's ACEmanager setting. + # Splitter mode (production: forward to prod + mirror to your server) + python bridges/ach_bridge.py --upstream PROD_HOST:PORT --mirror MY_HOST:PORT [--port 12345] -Output +Setup for discovery (test server, don't touch prod) +---------------------------------------------------- +1. Stand up your test ACH server, note its IP and port (e.g. 192.168.1.50:12345). +2. Take ONE test unit. In ACEmanager → Call Home, point it at: + : <--port> +3. Run: python bridges/ach_bridge.py --upstream TEST_SERVER:12345 --port 12345 +4. Trigger the unit. Raw frames are saved to bridges/captures/ach_/. +5. Revert the unit's ACEmanager setting when done. + +Setup for production splitter (when you're ready) +------------------------------------------------- +This does NOT touch the units. Instead you re-route traffic at the network +layer so that call-home packets arrive at a machine running this script first. +Typical approach: update the DNS entry / host record your prod ACH server is +registered under to point at this machine. The units keep their existing +ACEmanager settings. + + python bridges/ach_bridge.py \\ + --upstream PROD_ACH_HOST:12345 \\ + --mirror MY_NEW_SERVER:12345 \\ + --port 12345 + +Output (each connection gets its own timestamped sub-directory) ------ - bridges/captures/ach_/ + bridges/captures/ach_/ raw_client_.bin — raw bytes from the device (S3 side) - raw_server_.bin — raw bytes from the upstream server (BW side) - (empty in standalone mode) + raw_server_.bin — raw bytes from the primary upstream (BW side) + raw_mirror_.bin — raw bytes from the mirror upstream (splitter mode only) session_.log — human-readable frame parse log - session_.jsonl — JSON-lines frame log (for downstream tooling) + session_.jsonl — JSON-lines frame log -The raw_client / raw_server files are byte-for-byte compatible with the -existing capture format used by bridges/parse_capture.py and the rest of -the analysis tooling. +raw_client / raw_server are byte-for-byte compatible with parse_capture.py. """ from __future__ import annotations @@ -47,7 +75,7 @@ import logging import os import sys from pathlib import Path -from typing import Optional +from typing import List, Optional # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -113,26 +141,28 @@ _KNOWN_REQ_SUBS = { def _label_s3_frame(frame: S3Frame) -> str: name = _KNOWN_RSP_SUBS.get(frame.sub, f"UNK_0x{frame.sub:02X}") chk = "✓" if frame.checksum_valid else "✗CHK" - return f"S3→ SUB=0x{frame.sub:02X} ({name}) page=0x{frame.page_key:04X} data={len(frame.data)}B {chk}" + return ( + f"S3→ SUB=0x{frame.sub:02X} ({name}) " + f"page=0x{frame.page_key:04X} data={len(frame.data)}B {chk}" + ) -def _label_bw_frame(data: bytes) -> str: - """Best-effort label for a raw BW request frame.""" - # BW frame (destuffed): ACK STX [10 10] flags sub ... - # Wire: 41 02 10 10 00 sub ... +def _label_bw_frame(data: bytes, prefix: str = " →BW") -> str: + """Best-effort label for a raw BW request frame (wire bytes).""" + # Wire layout: 41 02 10 10 00 sub ... if len(data) < 6: - return f"BW→ (short {len(data)}B)" + return f"{prefix} (short {len(data)}B)" sub = data[5] name = _KNOWN_REQ_SUBS.get(sub, f"UNK_0x{sub:02X}") - return f" →BW SUB=0x{sub:02X} ({name}) {len(data)}B" + return f"{prefix} SUB=0x{sub:02X} ({name}) {len(data)}B" -# ── Per-session capture writer ──────────────────────────────────────────────── +# ── Per-session capture writer ───────────────────────────────────────────────── class CaptureSession: """Writes raw bytes + parsed log for one TCP connection.""" - def __init__(self, capture_dir: Path, peer: str): + def __init__(self, capture_dir: Path, peer: str, *, has_mirror: bool = False): ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self.dir = capture_dir / f"ach_{ts}" self.dir.mkdir(parents=True, exist_ok=True) @@ -140,27 +170,34 @@ class CaptureSession: self._raw_client = open(self.dir / f"raw_client_{ts}.bin", "wb") self._raw_server = open(self.dir / f"raw_server_{ts}.bin", "wb") - self._log_fh = open(self.dir / f"session_{ts}.log", "w") - self._jsonl_fh = open(self.dir / f"session_{ts}.jsonl", "w") + self._raw_mirror = ( + open(self.dir / f"raw_mirror_{ts}.bin", "wb") if has_mirror else None + ) + self._log_fh = open(self.dir / f"session_{ts}.log", "w") + self._jsonl_fh = open(self.dir / f"session_{ts}.jsonl", "w") self._s3_parser = S3FrameParser() - self._frame_count = 0 + self._frame_count = 0 self._byte_count_client = 0 self._byte_count_server = 0 + self._byte_count_mirror = 0 - self._log(f"# ACH capture — peer={peer} started={datetime.datetime.now().isoformat()}") + self._log( + f"# ACH capture — peer={peer} " + f"mirror={'yes' if has_mirror else 'no'} " + f"started={datetime.datetime.now().isoformat()}" + ) self._log(f"# Output dir: {self.dir}") log.info("Capture session opened: %s (peer=%s)", self.dir, peer) - # ── public API ─────────────────────────────────────────────────────────── + # ── public API ──────────────────────────────────────────────────────────── def feed_client(self, data: bytes) -> None: - """Bytes arriving FROM the device (S3 side).""" + """Bytes FROM the device (S3 response frames).""" self._raw_client.write(data) self._raw_client.flush() self._byte_count_client += len(data) - # Parse S3 frames for byte in data: frame = self._s3_parser.feed(bytes([byte])) if frame: @@ -169,65 +206,96 @@ class CaptureSession: self._frame_count += 1 label = _label_s3_frame(f) self._log(f"[{self._frame_count:04d}] {label}") - self._log(f" hex: {f.data[:64].hex()}" - + (" ..." if len(f.data) > 64 else "")) + self._log( + f" hex: {f.data[:64].hex()}" + + (" ..." if len(f.data) > 64 else "") + ) self._emit_json("s3", f) def feed_server(self, data: bytes) -> None: - """Bytes arriving FROM the upstream server (BW side).""" + """Bytes FROM the primary upstream server (BW request frames).""" self._raw_server.write(data) self._raw_server.flush() self._byte_count_server += len(data) - label = _label_bw_frame(data) + label = _label_bw_frame(data, prefix=" →BW[primary]") self._log(f" {label}") + def feed_mirror(self, data: bytes) -> None: + """Bytes FROM the mirror server (logged, not forwarded to device).""" + if self._raw_mirror: + self._raw_mirror.write(data) + self._raw_mirror.flush() + self._byte_count_mirror += len(data) + label = _label_bw_frame(data, prefix=" →BW[mirror] ") + self._log(f" {label} [MIRROR — not sent to device]") + def close(self, reason: str = "connection closed") -> None: self._log(f"# Session ended: {reason}") - self._log(f"# Totals — client_bytes={self._byte_count_client} " - f"server_bytes={self._byte_count_server} " - f"s3_frames={self._frame_count}") - for fh in (self._raw_client, self._raw_server, self._log_fh, self._jsonl_fh): + self._log( + f"# Totals — client={self._byte_count_client}B " + f"server={self._byte_count_server}B " + f"mirror={self._byte_count_mirror}B " + f"s3_frames={self._frame_count}" + ) + handles = [self._raw_client, self._raw_server, self._log_fh, self._jsonl_fh] + if self._raw_mirror: + handles.append(self._raw_mirror) + for fh in handles: try: fh.close() except Exception: pass log.info( - "Capture session closed (%s): %dB client, %dB server, %d S3 frames → %s", - reason, self._byte_count_client, self._byte_count_server, - self._frame_count, self.dir, + "Session closed (%s): %dB client, %dB server, %dB mirror, %d S3 frames → %s", + reason, + self._byte_count_client, self._byte_count_server, + self._byte_count_mirror, self._frame_count, + self.dir, ) - # ── internals ──────────────────────────────────────────────────────────── + # ── internals ───────────────────────────────────────────────────────────── def _log(self, msg: str) -> None: print(msg, file=self._log_fh, flush=True) - # Also echo to console for live monitoring print(msg) def _emit_json(self, direction: str, frame: S3Frame) -> None: record = { - "dir": direction, - "sub": frame.sub, - "page_key": frame.page_key, - "data_len": len(frame.data), - "data_hex": frame.data.hex(), + "dir": direction, + "sub": frame.sub, + "page_key": frame.page_key, + "data_len": len(frame.data), + "data_hex": frame.data.hex(), "checksum_valid": frame.checksum_valid, } print(json.dumps(record), file=self._jsonl_fh, flush=True) -# ── Bridge connection handler ───────────────────────────────────────────────── +# ── Bridge / splitter connection handler ────────────────────────────────────── class BridgeHandler: + """ + Handles inbound device connections. + + Modes (determined by which upstreams are configured): + standalone — no upstream_host / no mirror_host + bridge — upstream_host set, no mirror_host + splitter — upstream_host AND mirror_host both set + """ + def __init__( self, - capture_dir: Path, + capture_dir: Path, upstream_host: Optional[str], upstream_port: Optional[int], + mirror_host: Optional[str] = None, + mirror_port: Optional[int] = None, ): self.capture_dir = capture_dir self.upstream_host = upstream_host self.upstream_port = upstream_port + self.mirror_host = mirror_host + self.mirror_port = mirror_port async def handle( self, @@ -238,41 +306,11 @@ class BridgeHandler: peer_str = f"{peer[0]}:{peer[1]}" log.info("Inbound connection from %s", peer_str) - session = CaptureSession(self.capture_dir, peer_str) + has_mirror = bool(self.mirror_host) + session = CaptureSession(self.capture_dir, peer_str, has_mirror=has_mirror) - if self.upstream_host: - # Bridge mode: connect to upstream and relay - try: - up_reader, up_writer = await asyncio.open_connection( - self.upstream_host, self.upstream_port - ) - log.info("Connected to upstream %s:%s", self.upstream_host, self.upstream_port) - except Exception as exc: - log.error("Failed to connect to upstream: %s", exc) - session.close(f"upstream connect failed: {exc}") - client_writer.close() - return - - try: - await asyncio.gather( - self._relay(client_reader, up_writer, session, "client"), - self._relay(up_reader, client_writer, session, "server"), - ) - except asyncio.CancelledError: - pass - except Exception as exc: - log.warning("Bridge relay error: %s", exc) - finally: - session.close("bridge relay ended") - for writer in (client_writer, up_writer): - try: - writer.close() - await writer.wait_closed() - except Exception: - pass - - else: - # Standalone mode: just capture, don't forward + if not self.upstream_host: + # ── Standalone mode ────────────────────────────────────────────── log.info("Standalone mode — recording inbound traffic only") try: while True: @@ -291,30 +329,168 @@ class BridgeHandler: await client_writer.wait_closed() except Exception: pass + return - async def _relay( + # ── Bridge / splitter mode ─────────────────────────────────────────── + # Connect to primary upstream (required) + try: + up_reader, up_writer = await asyncio.open_connection( + self.upstream_host, self.upstream_port + ) + log.info("Connected to primary %s:%s", self.upstream_host, self.upstream_port) + except Exception as exc: + log.error("Failed to connect to primary upstream: %s", exc) + session.close(f"primary connect failed: {exc}") + client_writer.close() + return + + # Connect to mirror upstream (optional — failure is non-fatal) + mir_reader: Optional[asyncio.StreamReader] = None + mir_writer: Optional[asyncio.StreamWriter] = None + if self.mirror_host: + try: + mir_reader, mir_writer = await asyncio.open_connection( + self.mirror_host, self.mirror_port + ) + log.info("Connected to mirror %s:%s", self.mirror_host, self.mirror_port) + except Exception as exc: + log.warning( + "Mirror connect failed — continuing without mirror: %s", exc + ) + session._log(f"# WARNING: mirror connect failed: {exc}") + + # Build relay tasks + # + # ┌──────────┐ device bytes ┌─────────────┐ + # │ Device │ ─────────────► │ PRIMARY │ responses ──► device + # └──────────┘ └─────────────┘ + # │ + # │ device bytes (copy) + # ▼ + # ┌─────────────┐ + # │ MIRROR │ responses discarded (logged only) + # └─────────────┘ + # + tasks = [ + asyncio.create_task( + self._relay_device(client_reader, up_writer, mir_writer, session), + name="device→upstreams", + ), + asyncio.create_task( + self._relay_simple(up_reader, client_writer, session, "server"), + name="primary→device", + ), + ] + if mir_reader is not None: + tasks.append(asyncio.create_task( + self._relay_drain(mir_reader, session), + name="mirror→drain", + )) + + try: + # Wait for the device-to-upstreams relay to exit first (device + # disconnected or primary dropped). Then cancel the rest. + done, pending = await asyncio.wait( + tasks, + return_when=asyncio.FIRST_COMPLETED, + ) + for t in pending: + t.cancel() + try: + await t + except (asyncio.CancelledError, Exception): + pass + except Exception as exc: + log.warning("Bridge relay error: %s", exc) + finally: + session.close("relay ended") + for writer in filter(None, [client_writer, up_writer, mir_writer]): + try: + writer.close() + await writer.wait_closed() + except Exception: + pass + + # ── Relay helpers ───────────────────────────────────────────────────────── + + async def _relay_device( self, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - session: CaptureSession, - direction: str, + reader: asyncio.StreamReader, + primary_writer: asyncio.StreamWriter, + mirror_writer: Optional[asyncio.StreamWriter], + session: CaptureSession, ) -> None: + """ + Read bytes from the device, write to the primary server, and also + write a copy to the mirror server (if connected). Mirror write + failures are non-fatal — we log and continue. + """ try: while True: data = await reader.read(4096) if not data: break - if direction == "client": - session.feed_client(data) - else: + session.feed_client(data) + + # Primary write — failure IS fatal (lose primary = lose prod) + primary_writer.write(data) + await primary_writer.drain() + + # Mirror write — failure is non-fatal + if mirror_writer is not None: + try: + mirror_writer.write(data) + await mirror_writer.drain() + except Exception as exc: + log.warning("Mirror write failed (non-fatal): %s", exc) + session._log(f"# WARNING: mirror write failed: {exc}") + mirror_writer = None # stop trying + + except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError): + pass + + async def _relay_simple( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + session: CaptureSession, + direction: str, + ) -> None: + """Standard single-pipe relay (primary→device or vice-versa).""" + try: + while True: + data = await reader.read(4096) + if not data: + break + if direction == "server": session.feed_server(data) + else: + session.feed_client(data) writer.write(data) await writer.drain() except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError): pass + async def _relay_drain( + self, + reader: asyncio.StreamReader, + session: CaptureSession, + ) -> None: + """ + Read mirror server responses, log them to session, do NOT forward to + device. The device only ever sees primary server responses. + """ + try: + while True: + data = await reader.read(4096) + if not data: + break + session.feed_mirror(data) + except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError): + pass -# ── Main ────────────────────────────────────────────────────────────────────── + +# ── Main ─────────────────────────────────────────────────────────────────────── async def main(args: argparse.Namespace) -> None: capture_dir = Path(__file__).parent / "captures" @@ -322,6 +498,8 @@ async def main(args: argparse.Namespace) -> None: upstream_host: Optional[str] = None upstream_port: Optional[int] = None + mirror_host: Optional[str] = None + mirror_port: Optional[int] = None if not args.standalone: if not args.upstream: @@ -329,12 +507,24 @@ async def main(args: argparse.Namespace) -> None: sys.exit(1) parts = args.upstream.rsplit(":", 1) if len(parts) != 2: - print("ERROR: --upstream must be HOST:PORT (e.g. 203.0.113.5:9034)") + print("ERROR: --upstream must be HOST:PORT (e.g. 203.0.113.5:12345)") sys.exit(1) upstream_host = parts[0] upstream_port = int(parts[1]) - handler = BridgeHandler(capture_dir, upstream_host, upstream_port) + if args.mirror: + parts = args.mirror.rsplit(":", 1) + if len(parts) != 2: + print("ERROR: --mirror must be HOST:PORT (e.g. 192.168.1.50:12345)") + sys.exit(1) + mirror_host = parts[0] + mirror_port = int(parts[1]) + + handler = BridgeHandler( + capture_dir, + upstream_host, upstream_port, + mirror_host, mirror_port, + ) server = await asyncio.start_server( handler.handle, @@ -342,19 +532,38 @@ async def main(args: argparse.Namespace) -> None: port=args.port, ) - mode = f"bridge → {upstream_host}:{upstream_port}" if upstream_host else "standalone capture" + # ── Startup banner ──────────────────────────────────────────────────────── + if args.standalone: + mode = "STANDALONE capture (no forwarding)" + elif mirror_host: + mode = f"SPLITTER primary={upstream_host}:{upstream_port} mirror={mirror_host}:{mirror_port}" + else: + mode = f"BRIDGE → {upstream_host}:{upstream_port}" + addrs = ", ".join(str(s.getsockname()) for s in server.sockets) - print(f"\n{'='*60}") - print(f" ACH bridge listening on {addrs}") - print(f" Mode: {mode}") + print(f"\n{'='*70}") + print(f" ACH bridge/splitter listening on {addrs}") + print(f" Mode: {mode}") print(f" Captures: {capture_dir}/ach_/") - print(f"{'='*60}") - print(f"\n Point your unit's ACEmanager call-home destination to:") - print(f" :{args.port}") - if upstream_host: - print(f"\n All traffic will be forwarded to {upstream_host}:{upstream_port}") - print(f" Your live data feed is uninterrupted.") - print(f"\n Waiting for inbound connection... (Ctrl-C to stop)\n") + print(f"{'='*70}") + + if upstream_host and not mirror_host: + print(f"\n DISCOVERY PHASE") + print(f" Point your TEST unit's ACEmanager call-home destination to:") + print(f" : {args.port}") + print(f" All traffic will be forwarded to {upstream_host}:{upstream_port}") + elif mirror_host: + print(f"\n SPLITTER MODE — PRODUCTION SAFE") + print(f" Units connect as normal. Every byte is forwarded to:") + print(f" PRIMARY (authoritative): {upstream_host}:{upstream_port}") + print(f" MIRROR (your server): {mirror_host}:{mirror_port}") + print(f" Only PRIMARY responses reach the device.") + print(f" Mirror failures are logged and do not affect the device.") + else: + print(f"\n STANDALONE MODE — capture only, nothing forwarded") + print(f" Point a unit at : {args.port}") + + print(f"\n Waiting for inbound connections... (Ctrl-C to stop)\n") async with server: await server.serve_forever() @@ -362,24 +571,41 @@ async def main(args: argparse.Namespace) -> None: def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser( - description="Transparent TCP bridge for capturing MiniMate Plus call-home traffic." + description=( + "Transparent TCP bridge / splitter for Instantel MiniMate Plus " + "call-home (ACH) traffic." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, ) p.add_argument( "--upstream", "-u", metavar="HOST:PORT", - help="Upstream ACH server to forward to (e.g. 203.0.113.5:9034). " - "Omit with --standalone for capture-only mode.", + help=( + "Primary upstream ACH server to forward to " + "(e.g. 203.0.113.5:12345). " + "Omit with --standalone for capture-only mode." + ), + ) + p.add_argument( + "--mirror", "-m", + metavar="HOST:PORT", + help=( + "Mirror / secondary server to receive a copy of all device bytes " + "(splitter mode). Mirror responses are logged but NOT forwarded " + "to the device. Mirror failures are non-fatal." + ), ) p.add_argument( "--port", "-p", type=int, - default=9034, - help="Local port to listen on (default: 9034).", + default=12345, + help="Local port to listen on (default: 12345).", ) p.add_argument( "--standalone", "-s", action="store_true", - help="Capture-only mode: accept connection but do not forward to upstream.", + help="Capture-only mode: accept connection, do not forward anywhere.", ) p.add_argument( "--verbose", "-v", From cf7d838bf4e6e77348cf900e03d5f2afcb10945c Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 9 Apr 2026 12:34:27 -0400 Subject: [PATCH 04/33] feat: add SocketTransport and ach_server.py inbound ACH server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit minimateplus/transport.py: - Add SocketTransport(TcpTransport) — wraps an already-accepted inbound socket; connect() is a no-op; everything else inherited from TcpTransport. Enables the ACH server to reuse all existing protocol/client code without any changes. bridges/ach_server.py: - Minimal inbound ACH server — listens on port 12345, accepts call-home connections from MiniMate Plus units, runs the full BW protocol: startup handshake → get_device_info → get_events(full_waveform=True) - Saves device_info.json + events.json + raw_rx_.bin + session log per connection to bridges/captures/ach_inbound_/ - raw_rx.bin is byte-compatible with existing Analyzer tooling - Taps transport.read() to capture raw S3 bytes alongside parsed output - Each connection runs in its own daemon thread - Clearly distinguishes push vs pull protocol in the startup log Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 356 ++++++++++++++++++++++++++++++++++++++ minimateplus/transport.py | 36 ++++ 2 files changed, 392 insertions(+) create mode 100644 bridges/ach_server.py diff --git a/bridges/ach_server.py b/bridges/ach_server.py new file mode 100644 index 0000000..6088431 --- /dev/null +++ b/bridges/ach_server.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +""" +ach_server.py — Minimal inbound ACH (Auto Call Home) server for MiniMate Plus. + +This IS your test server. Run it on any machine on the same network, point a +unit's ACEmanager call-home destination at it, and it will speak the full BW +protocol to the device: handshake, pull device info, download all events, save +everything as JSON. + +The key thing this script tells you that no amount of packet sniffing can: + - Does the device speak first (push) or wait for us to send POLL (pull)? + +If startup() completes normally → it's pull protocol, same as Blastware. +If startup() times out → the device sent something first; check raw_rx.bin. + +Usage +----- + python bridges/ach_server.py [--port 12345] [--output bridges/captures/] + +Setup +----- + 1. Run this script on a machine on your local network. + 2. In ACEmanager → Application → ALEOS Application Framework (or equivalent) + find the Call Home / ACH settings. Set: + Remote Host: + Remote Port: 12345 + 3. Trigger the unit (wait for a vibration event, or use the manual call-home + button if your firmware version has one). + 4. The unit connects. This script handshakes, downloads all events, + and saves a timestamped session directory. + +Output per session +------------------ + bridges/captures/ach_inbound_/ + device_info.json — serial number, firmware version, calibration date, etc. + events.json — all events: timestamp, PPV per channel, peaks, metadata + raw_rx_.bin — raw bytes from the device (S3 side) for Analyzer + session_.log — detailed protocol log + +What to look for +---------------- + Push vs pull: Check session_.log. If the first line after "Connected" + shows bytes arriving BEFORE the POLL probe was sent, it's push. If POLL + gets a clean response, it's pull. + + Frequency: Look at raw_rx.bin in the Analyzer. SUB 5A (0xA5 responses) carry + bulk waveform data — if frequency is sent pre-computed there will be float32 + values before the ADC sample blocks. + + ACH-specific framing: Does the unit send anything extra before the DLE+STX + framing starts? raw_rx.bin will show raw bytes including any preamble. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import logging +import socket +import sys +import threading +from pathlib import Path +from typing import Optional + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from minimateplus.transport import SocketTransport +from minimateplus.client import MiniMateClient +from minimateplus.models import DeviceInfo, Event + +log = logging.getLogger("ach_server") + + +# ── Per-session handler ──────────────────────────────────────────────────────── + +class AchSession: + """ + Handles one inbound unit connection in its own thread. + Wraps the socket in a SocketTransport → MiniMateClient, then runs the + standard connect → get_device_info → get_events sequence. + """ + + def __init__( + self, + sock: socket.socket, + peer: str, + output_dir: Path, + timeout: float, + events_only: bool, + ) -> None: + self.sock = sock + self.peer = peer + self.output_dir = output_dir + self.timeout = timeout + self.events_only = events_only + + def run(self) -> None: + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + session_dir = self.output_dir / f"ach_inbound_{ts}" + session_dir.mkdir(parents=True, exist_ok=True) + + log_path = session_dir / f"session_{ts}.log" + raw_path = session_dir / f"raw_rx_{ts}.bin" + + # Wire up a file handler so every protocol log line goes to the session log + fh = logging.FileHandler(log_path) + fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s")) + root_logger = logging.getLogger() + root_logger.addHandler(fh) + + try: + self._run_inner(session_dir, raw_path, ts) + except Exception as exc: + log.error("Session failed: %s", exc, exc_info=True) + finally: + root_logger.removeHandler(fh) + fh.close() + try: + self.sock.close() + except Exception: + pass + + def _run_inner(self, session_dir: Path, raw_path: Path, ts: str) -> None: + log.info("="*60) + log.info("Inbound connection from %s", self.peer) + log.info("Session dir: %s", session_dir) + + # Wrap the accepted socket in a SocketTransport. + # SocketTransport.connect() is a no-op — the socket is already live. + transport = SocketTransport(self.sock, peer=self.peer) + + # Tap the transport so we save every raw byte received from the device. + # We monkey-patch read() to write to a file before returning. + raw_fh = open(raw_path, "wb") + original_read = transport.read + + def tapped_read(n: int) -> bytes: + data = original_read(n) + if data: + raw_fh.write(data) + raw_fh.flush() + return data + + transport.read = tapped_read # type: ignore[method-assign] + + try: + client = MiniMateClient(transport=transport, timeout=self.timeout) + client.open() # calls transport.connect() — no-op for SocketTransport + + # ── Step 1: startup handshake ───────────────────────────────────── + log.info("Step 1/3: startup handshake (POLL / SUB 5B)") + try: + from minimateplus.protocol import MiniMateProtocol + proto = MiniMateProtocol(transport, recv_timeout=self.timeout) + proto.startup() + log.info(" ✓ Startup OK — device responded to POLL (pull protocol confirmed)") + log.info(" NOTE: If you see this, the device waited for us to send POLL first.") + log.info(" That means ACH is pull protocol (same as direct BW connection).") + except Exception as exc: + log.error(" ✗ Startup failed: %s", exc) + log.warning(" If startup timed out with bytes in raw_rx.bin → push protocol.") + log.warning(" If raw_rx.bin is empty → unit didn't respond at all.") + return + + # ── Step 2: device info ─────────────────────────────────────────── + if not self.events_only: + log.info("Step 2/3: reading device info") + try: + device_info = client.connect() # SUB FE + 1A + _save_json(session_dir / "device_info.json", _device_info_to_dict(device_info)) + log.info( + " ✓ Device: serial=%s firmware=%s calibration=%s", + device_info.serial_number, + device_info.firmware_version, + device_info.calibration_date, + ) + except Exception as exc: + log.error(" ✗ Device info failed: %s", exc) + # Not fatal — continue to events + else: + log.info("Step 2/3: skipping device info (--events-only)") + + # ── Step 3: download events ──────────────────────────────────────── + log.info("Step 3/3: downloading events") + try: + events = client.get_events(full_waveform=True) + log.info(" ✓ Downloaded %d event(s)", len(events)) + _save_json(session_dir / "events.json", [_event_to_dict(e) for e in events]) + for i, ev in enumerate(events): + log.info( + " Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f", + i, + ev.timestamp.isoformat() if ev.timestamp else "?", + ev.peaks.transverse if ev.peaks else 0, + ev.peaks.vertical if ev.peaks else 0, + ev.peaks.longitudinal if ev.peaks else 0, + ev.peaks.vector_sum if ev.peaks else 0, + ) + except Exception as exc: + log.error(" ✗ Event download failed: %s", exc) + + finally: + raw_fh.close() + client.close() + + log.info("Session complete → %s", session_dir) + log.info("="*60) + + +# ── JSON helpers ─────────────────────────────────────────────────────────────── + +def _save_json(path: Path, obj: object) -> None: + with open(path, "w") as f: + json.dump(obj, f, indent=2, default=str) + log.debug("Saved %s", path) + + +def _device_info_to_dict(d: DeviceInfo) -> dict: + return { + "serial_number": d.serial_number, + "firmware_version": d.firmware_version, + "calibration_date": str(d.calibration_date) if d.calibration_date else None, + "aux_trigger": d.aux_trigger, + "setup_name": d.setup_name, + "sample_rate": d.sample_rate, + "record_time": d.record_time, + "trigger_level_geo": d.trigger_level_geo, + "alarm_level_geo": d.alarm_level_geo, + "max_range_geo": d.max_range_geo, + "project": d.project, + "client": d.client, + "operator": d.operator, + "sensor_location": d.sensor_location, + } + + +def _event_to_dict(e: Event) -> dict: + peaks = {} + if e.peaks: + peaks = { + "transverse": e.peaks.transverse, + "vertical": e.peaks.vertical, + "longitudinal": e.peaks.longitudinal, + "vector_sum": e.peaks.vector_sum, + "mic": e.peaks.mic, + } + samples = {} + if e.raw_samples: + samples = { + ch: vals[:20] # first 20 sample-sets to keep the file sane + for ch, vals in e.raw_samples.items() + } + samples["__note__"] = "first 20 sample-sets only; see raw_rx.bin for full waveform" + return { + "timestamp": e.timestamp.isoformat() if e.timestamp else None, + "project": e.project, + "client": e.client, + "operator": e.operator, + "sensor_location": e.sensor_location, + "peaks": peaks, + "raw_samples_preview": samples, + } + + +# ── Main server loop ─────────────────────────────────────────────────────────── + +def serve(args: argparse.Namespace) -> None: + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + + server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_sock.bind(("0.0.0.0", args.port)) + server_sock.listen(5) + + print(f"\n{'='*60}") + print(f" ACH inbound server listening on 0.0.0.0:{args.port}") + print(f" Output: {output_dir.resolve()}/ach_inbound_/") + print(f"{'='*60}") + print(f"\n Point your test unit's ACEmanager call-home settings to:") + print(f" Remote Host: ") + print(f" Remote Port: {args.port}") + print(f"\n Waiting for inbound connections... (Ctrl-C to stop)\n") + + try: + while True: + try: + client_sock, addr = server_sock.accept() + peer = f"{addr[0]}:{addr[1]}" + log.info("Accepted connection from %s", peer) + session = AchSession( + sock=client_sock, + peer=peer, + output_dir=output_dir, + timeout=args.timeout, + events_only=args.events_only, + ) + t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") + t.start() + except KeyboardInterrupt: + raise + except Exception as exc: + log.error("Accept error: %s", exc) + finally: + server_sock.close() + print("\nServer stopped.") + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Minimal inbound ACH server — speak BW protocol to calling MiniMate Plus units.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + p.add_argument( + "--port", "-p", + type=int, + default=12345, + help="Port to listen on (default: 12345).", + ) + p.add_argument( + "--output", "-o", + default=str(Path(__file__).parent / "captures"), + metavar="DIR", + help="Directory to write session captures (default: bridges/captures/).", + ) + p.add_argument( + "--timeout", "-t", + type=float, + default=30.0, + help="Protocol receive timeout in seconds (default: 30.0).", + ) + p.add_argument( + "--events-only", + action="store_true", + help="Skip the device-info step and go straight to event download.", + ) + p.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable debug logging.", + ) + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-7s %(name)s %(message)s", + ) + try: + serve(args) + except KeyboardInterrupt: + print("\nStopped.") diff --git a/minimateplus/transport.py b/minimateplus/transport.py index d0c37e6..65249d8 100644 --- a/minimateplus/transport.py +++ b/minimateplus/transport.py @@ -418,3 +418,39 @@ class TcpTransport(BaseTransport): def __repr__(self) -> str: state = "connected" if self.is_connected else "disconnected" return f"TcpTransport({self.host!r}, port={self.port}, {state})" + + +# ── Inbound / accepted-socket transport ─────────────────────────────────────── + +class SocketTransport(TcpTransport): + """ + Like TcpTransport but wraps an already-accepted inbound socket. + + Used by the ACH inbound server (bridges/ach_server.py) — the device dials + IN to us, so by the time we create this transport the socket is already live. + connect() is a no-op; everything else (read, write, read_until_idle, …) is + inherited unchanged from TcpTransport. + + Args: + sock: An already-connected socket.socket returned by server_socket.accept(). + peer: Human-readable peer label for repr / logging (e.g. "203.0.113.5:54321"). + """ + + def __init__(self, sock: socket.socket, peer: str = "inbound") -> None: + # Bypass TcpTransport.__init__ — we already have a live socket. + self.host = peer + self.port = 0 + self.connect_timeout = 0.0 + self._sock = sock + sock.settimeout(self._RECV_TIMEOUT) + + def connect(self) -> None: + """No-op — socket was already accepted inbound.""" + pass # Already have a live socket; nothing to open. + + @property + def is_connected(self) -> bool: + return self._sock is not None + + def __repr__(self) -> str: + return f"SocketTransport(peer={self.host!r})" From 0358acb51d802690344045f746096d25c4426ded Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 9 Apr 2026 14:38:44 -0400 Subject: [PATCH 05/33] feat: add high-water mark state tracking to ach_server + fix monitoring flag ach_server.py: - Add ach_state.json per-unit state tracking (keyed by serial number) - count_events() before any download; skip session if no new events since last call-home - Download only events beyond the previous high-water mark (all_events[last_count:]) - --max-events N safety cap for first-run units with many stored events - state_path and max_events wired through AchSession constructor and serve() client.py (_decode_monitor_status): - Revert monitoring flag to section[1] == 0x10 (was incorrectly changed to section[6]) - Fix battery/memory offsets to section[-10:-8], [-8:-4], [-4:] (no trailing checksum byte) - Both confirmed by full byte diff of all 144 0xE3 data frames in 4-8-26/2ndtry capture Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 56 +++++++------- CLAUDE.md | 34 ++++---- bridges/ach_server.py | 172 ++++++++++++++++++++++++++++++++--------- minimateplus/client.py | 55 ++++++------- 4 files changed, 210 insertions(+), 107 deletions(-) diff --git a/.gitignore b/.gitignore index e6e6e12..212a2eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,28 @@ -/bridges/captures/ -/example-events/ - -/manuals/ - -# Python bytecode -__pycache__/ -*.py[cod] - -# Virtual environments -.venv/ -venv/ -env/ - -# Editor / OS -.vscode/ -*.swp -.DS_Store -Thumbs.db - -# Analyzer outputs -*.report -claude_export_*.md - -# Frame database -*.db -*.db-wal -*.db-shm +/bridges/captures/ +/example-events/ + +/manuals/ + +# Python bytecode +__pycache__/ +*.py[cod] + +# Virtual environments +.venv/ +venv/ +env/ + +# Editor / OS +.vscode/ +*.swp +.DS_Store +Thumbs.db + +# Analyzer outputs +*.report +claude_export_*.md + +# Frame database +*.db +*.db-wal +*.db-shm diff --git a/CLAUDE.md b/CLAUDE.md index 50d2f46..b6faa1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -582,28 +582,32 @@ All confirmed from 4-8-26/2ndtry BW TX/S3 capture (clean start → 30s monitor Standard two-step read (probe at offset 0x00, data at offset 0x2C). Response SUB = 0xFF − 0x1C = **0xE3** (standard formula — no exception). -**Payload length is ~46–49 bytes in BOTH idle and monitoring states** — length alone -is NOT a reliable mode indicator. Earlier note claiming "12 bytes when monitoring" -was wrong (confirmed 2026-04-08 from 4-8-26/mid-monitor captures). +**Payload length is 46–47 bytes IDLE, 48–49 bytes MONITORING** — not a reliable sole +indicator due to 1-byte jitter overlap at the boundary. -**Monitoring flag (CORRECTED 2026-04-08 — full byte diff of 2ndtry capture):** -- `section[6] == 0x00` → unit is **idle** -- `section[6] == 0x10` → unit is **monitoring** +**Monitoring flag (CONFIRMED 2026-04-09 — byte diff of all 144 data frames, 2ndtry capture):** +- `section[1] == 0x00` → unit is **idle** +- `section[1] == 0x10` → unit is **monitoring** -Earlier note claiming `section[1]` was the flag was WRONG — section[1] is always 0x00 in both states. The correction was found by diffing all 0xE3 data frames across the start/stop transitions: `section[6]` is the only byte that flips cleanly at frame #36 (start) and #132 (stop) within the 2ndtry 0xE3 frame sequence. +This is `data[12]` (= `frame.data[12]`). The flag is 0x00 in all 36 IDLE_BEFORE frames, +0x10 in all 98 MONITORING frames, and 0x00 in all 10 IDLE_AFTER frames — 100% accurate. -Battery and memory fields are present in **both** states, but the payload grows by **3 bytes** when monitoring is active (section goes from ~52 to ~55 bytes), shifting subsequent fields by +3. +**HISTORY OF THIS FIELD (do not re-derive):** The original implementation used `section[1]`. +A re-analysis in the prior session incorrectly concluded `section[1]` is always 0x00 and +"corrected" the flag to `section[6]`, which has non-binary values (0xea idle, 0x07 monitoring) +and is device-specific. The 2026-04-09 re-analysis confirms `section[1]` was right. -**Field offsets (relative to `data[11:]` = section):** +**IMPORTANT — `frame.data` has checksum already stripped** by `S3FrameParser._finalise()` +(`raw_payload = body[:-1]`; `data = raw_payload[5:]`). There is NO trailing checksum byte in +`section`. All relative-from-end offsets must account for this. -Battery and memory are at **relative offsets from the end** — the payload can vary by ±1–3 bytes due to counter jitter and monitoring-mode expansion, but these 10 bytes are always anchored at the end: +Battery and memory fields are present in **both** states: | Offset (relative to end) | Field | Type | Notes | |---|---|---|---| -| `section[-11:-9]` | battery voltage × 100 | uint16 BE | `0x02A8` = 680 → 6.80 V | -| `section[-9:-5]` | memory total (bytes) | uint32 BE | e.g. 983026 ≈ 960 KB | -| `section[-5:-1]` | memory free (bytes) | uint32 BE | decreases as events are stored | -| `section[-1]` | frame checksum | — | last byte, skip | +| `section[-10:-8]` | battery voltage × 100 | uint16 BE | `0x02A8` = 680 → 6.80 V | +| `section[-8:-4]` | memory total (bytes) | uint32 BE | e.g. 983026 ≈ 960 KB | +| `section[-4:]` | memory free (bytes) | uint32 BE | decreases as events are stored | ### SESSION_RESET signal (`41 03`) — required for monitoring units @@ -657,7 +661,7 @@ Key findings: **SFM behavior after `POST /device/monitor/start`:** `_pollMonitorConfirm()` polls `/device/monitor/status` every 5 s for up to 60 s, updating the badge on each poll. -Status will show MONITORING once `section[6]` flips to `0x10`. +Status will show MONITORING once `section[1]` flips to `0x10`. ### SUBs known from sensor-check capture (4-8-26) — NOT YET IMPLEMENTED diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 6088431..1c37f86 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -71,6 +71,28 @@ from minimateplus.models import DeviceInfo, Event log = logging.getLogger("ach_server") +# ── Per-unit state (high-water mark) ────────────────────────────────────────── +# Persisted as /ach_state.json +# Format: { "BE11529": { "event_count": 5, "last_seen": "2026-04-09T..." }, ... } + +_state_lock = threading.Lock() + + +def _load_state(state_path: Path) -> dict: + if state_path.exists(): + try: + with open(state_path) as f: + return json.load(f) + except Exception: + pass + return {} + + +def _save_state(state_path: Path, state: dict) -> None: + with _state_lock: + with open(state_path, "w") as f: + json.dump(state, f, indent=2) + # ── Per-session handler ──────────────────────────────────────────────────────── @@ -79,6 +101,12 @@ class AchSession: Handles one inbound unit connection in its own thread. Wraps the socket in a SocketTransport → MiniMateClient, then runs the standard connect → get_device_info → get_events sequence. + + State tracking (ach_state.json in output_dir): + On each successful download we record how many events the unit had. + On the next call-home we compare: if count hasn't grown, there's nothing + new and we close cleanly without downloading. If it has grown, we + download all events up to the new count and save only the new ones. """ def __init__( @@ -88,12 +116,16 @@ class AchSession: output_dir: Path, timeout: float, events_only: bool, + max_events: Optional[int], + state_path: Path, ) -> None: - self.sock = sock - self.peer = peer - self.output_dir = output_dir - self.timeout = timeout + self.sock = sock + self.peer = peer + self.output_dir = output_dir + self.timeout = timeout self.events_only = events_only + self.max_events = max_events + self.state_path = state_path def run(self) -> None: ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -126,17 +158,14 @@ class AchSession: log.info("Inbound connection from %s", self.peer) log.info("Session dir: %s", session_dir) - # Wrap the accepted socket in a SocketTransport. - # SocketTransport.connect() is a no-op — the socket is already live. transport = SocketTransport(self.sock, peer=self.peer) - # Tap the transport so we save every raw byte received from the device. - # We monkey-patch read() to write to a file before returning. + # Tap the transport: save every raw byte received from the device. raw_fh = open(raw_path, "wb") - original_read = transport.read + _orig_read = transport.read def tapped_read(n: int) -> bytes: - data = original_read(n) + data = _orig_read(n) if data: raw_fh.write(data) raw_fh.flush() @@ -144,9 +173,11 @@ class AchSession: transport.read = tapped_read # type: ignore[method-assign] + serial: Optional[str] = None + try: client = MiniMateClient(transport=transport, timeout=self.timeout) - client.open() # calls transport.connect() — no-op for SocketTransport + client.open() # ── Step 1: startup handshake ───────────────────────────────────── log.info("Step 1/3: startup handshake (POLL / SUB 5B)") @@ -154,55 +185,106 @@ class AchSession: from minimateplus.protocol import MiniMateProtocol proto = MiniMateProtocol(transport, recv_timeout=self.timeout) proto.startup() - log.info(" ✓ Startup OK — device responded to POLL (pull protocol confirmed)") - log.info(" NOTE: If you see this, the device waited for us to send POLL first.") - log.info(" That means ACH is pull protocol (same as direct BW connection).") + log.info(" ✓ Startup OK — pull protocol confirmed") except Exception as exc: log.error(" ✗ Startup failed: %s", exc) - log.warning(" If startup timed out with bytes in raw_rx.bin → push protocol.") - log.warning(" If raw_rx.bin is empty → unit didn't respond at all.") return # ── Step 2: device info ─────────────────────────────────────────── + device_info = None if not self.events_only: log.info("Step 2/3: reading device info") try: - device_info = client.connect() # SUB FE + 1A + device_info = client.connect() + serial = device_info.serial_number _save_json(session_dir / "device_info.json", _device_info_to_dict(device_info)) log.info( " ✓ Device: serial=%s firmware=%s calibration=%s", - device_info.serial_number, + serial, device_info.firmware_version, device_info.calibration_date, ) except Exception as exc: log.error(" ✗ Device info failed: %s", exc) - # Not fatal — continue to events else: log.info("Step 2/3: skipping device info (--events-only)") - # ── Step 3: download events ──────────────────────────────────────── - log.info("Step 3/3: downloading events") + # ── Step 3: check for new events via high-water mark ─────────────── + log.info("Step 3/3: checking for new events") + + state = _load_state(self.state_path) + unit_key = serial or self.peer # fall back to IP if no serial + last_count = state.get(unit_key, {}).get("event_count", 0) + try: - events = client.get_events(full_waveform=True) - log.info(" ✓ Downloaded %d event(s)", len(events)) - _save_json(session_dir / "events.json", [_event_to_dict(e) for e in events]) - for i, ev in enumerate(events): - log.info( - " Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f", - i, - ev.timestamp.isoformat() if ev.timestamp else "?", - ev.peaks.transverse if ev.peaks else 0, - ev.peaks.vertical if ev.peaks else 0, - ev.peaks.longitudinal if ev.peaks else 0, - ev.peaks.vector_sum if ev.peaks else 0, - ) + current_count = client.count_events() + log.info(" Unit has %d stored event(s); last downloaded count: %d", + current_count, last_count) except Exception as exc: - log.error(" ✗ Event download failed: %s", exc) + log.error(" ✗ count_events failed: %s", exc) + return + + if current_count <= last_count: + log.info(" ✓ No new events since last call-home — nothing to download") + log.info("Session complete (no new events) → %s", session_dir) + return + + new_event_count = current_count - last_count + log.info(" %d new event(s) to download", new_event_count) + + # Download all events up to current_count, apply max_events cap. + # We re-download old events too (get_events always starts from 0), + # but we only SAVE the new ones (the last new_event_count of the list). + stop_idx = current_count - 1 + if self.max_events is not None: + stop_idx = min(stop_idx, self.max_events - 1) + if self.max_events < current_count: + log.warning( + " max_events=%d cap: will download events 0–%d only " + "(unit has %d total)", + self.max_events, stop_idx, current_count, + ) + + try: + all_events = client.get_events( + full_waveform=True, + stop_after_index=stop_idx, + ) + # Only the events beyond last_count are genuinely new + new_events = all_events[last_count:] + log.info(" ✓ Downloaded %d total event(s), %d new", + len(all_events), len(new_events)) + + _save_json(session_dir / "events.json", [_event_to_dict(e) for e in new_events]) + if last_count > 0 and len(all_events) > len(new_events): + log.info(" (skipped %d already-seen event(s))", last_count) + + for i, ev in enumerate(new_events): + log.info( + " NEW Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f", + last_count + i, + ev.timestamp.isoformat() if ev.timestamp else "?", + ev.peaks.transverse if ev.peaks else 0, + ev.peaks.vertical if ev.peaks else 0, + ev.peaks.longitudinal if ev.peaks else 0, + ev.peaks.vector_sum if ev.peaks else 0, + ) + + # Update high-water mark + state[unit_key] = { + "event_count": current_count, + "last_seen": datetime.datetime.now().isoformat(), + "serial": serial, + "peer": self.peer, + } + _save_state(self.state_path, state) + + except Exception as exc: + log.error(" ✗ Event download failed: %s", exc, exc_info=True) finally: raw_fh.close() - client.close() + client.close() # closes transport / socket cleanly log.info("Session complete → %s", session_dir) log.info("="*60) @@ -268,15 +350,19 @@ def _event_to_dict(e: Event) -> dict: def serve(args: argparse.Namespace) -> None: output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) + state_path = output_dir / "ach_state.json" server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_sock.bind(("0.0.0.0", args.port)) server_sock.listen(5) + max_ev = args.max_events print(f"\n{'='*60}") print(f" ACH inbound server listening on 0.0.0.0:{args.port}") - print(f" Output: {output_dir.resolve()}/ach_inbound_/") + print(f" Output: {output_dir.resolve()}/ach_inbound_/") + print(f" State file: {state_path}") + print(f" Max events per session: {max_ev if max_ev else 'unlimited'}") print(f"{'='*60}") print(f"\n Point your test unit's ACEmanager call-home settings to:") print(f" Remote Host: ") @@ -295,6 +381,8 @@ def serve(args: argparse.Namespace) -> None: output_dir=output_dir, timeout=args.timeout, events_only=args.events_only, + max_events=max_ev, + state_path=state_path, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") t.start() @@ -336,6 +424,16 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Skip the device-info step and go straight to event download.", ) + p.add_argument( + "--max-events", + type=int, + default=None, + metavar="N", + help=( + "Safety cap: download at most N events per session (default: unlimited). " + "Useful if a unit has many old events stored — prevents a very long first run." + ), + ) p.add_argument( "--verbose", "-v", action="store_true", diff --git a/minimateplus/client.py b/minimateplus/client.py index 9e32c6b..984cbcf 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -1755,17 +1755,20 @@ def _decode_monitor_status(data: bytes) -> MonitorStatus: data is the raw S3 frame .data attribute (includes the 11-byte section header, so field offsets below are relative to data[11]). - Monitoring flag (confirmed 4-8-26/2ndtry, full byte diff analysis): - section[6] == 0x00 → idle - section[6] == 0x10 → monitoring + NOTE: frame.data has the checksum byte already stripped by S3FrameParser + (_finalise returns raw_payload[5:] where raw_payload = body[:-1]). + There is NO trailing checksum byte in section. - The payload size varies (52–55+ bytes) but the battery/memory block is - always the last 10 bytes before the trailing checksum byte: + Monitoring flag (confirmed 4-8-26/2ndtry, byte diff of all 144 data frames): + section[1] == 0x00 → idle + section[1] == 0x10 → monitoring - section[-11:-9] battery × 100 uint16 BE (0x02A8 = 6.80 V) - section[-9 :-5] memory_total uint32 BE bytes - section[-5 :-1] memory_free uint32 BE bytes - section[-1] checksum (not data) + The payload length varies (46–49 bytes) — IDLE is 46-47, MONITORING is 48-49. + The battery/memory block is always the last 10 bytes of section (no checksum): + + section[-10:-8] battery × 100 uint16 BE (0x02A8 = 6.80 V) + section[-8 :-4] memory_total uint32 BE bytes + section[-4:] memory_free uint32 BE bytes Values confirmed from 4-8-26/2ndtry capture (BE11529): battery 0x02A8 = 680 → 6.80 V @@ -1780,32 +1783,30 @@ def _decode_monitor_status(data: bytes) -> MonitorStatus: len(data), len(section), section.hex(), ) - # Monitoring flag: section[6] (CORRECTED 2026-04-08 — was wrongly section[1]). - # Byte diff of 2ndtry BW-S3 captures confirms section[6] flips 0x00↔0x10 - # exactly at the start/stop monitoring transitions (0xE3 frame #36 / #132). - is_monitoring = len(section) > 6 and section[6] == 0x10 + # Monitoring flag: section[1] == 0x10. + # Confirmed from byte diff of all 144 0xE3 data frames in 4-8-26/2ndtry capture: + # section[1] = 0x00 in all IDLE frames, 0x10 in all MONITORING frames. + # (section[6] also changes but has non-binary values 0xea/0x07 — device-specific.) + is_monitoring = len(section) > 1 and section[1] == 0x10 battery_v = None memory_total = None memory_free = None - # Battery and memory offsets are RELATIVE TO THE END of the section. - # The payload length varies (52–55+ bytes) depending on monitoring state and - # internal counters, but the battery/memory block is always the last 10 bytes - # before the checksum (section[-1]). + # Battery and memory at relative-from-end offsets. + # Payload length varies (46–49 bytes) but the battery/memory block is always + # the last 10 bytes. No checksum byte — it was stripped by S3FrameParser. # - # section[-11:-9] battery × 100 uint16 BE 0x02A8 = 6.80 V - # section[-9 :-5] memory_total uint32 BE ≈ 960 KB on BE11529 - # section[-5 :-1] memory_free uint32 BE decreases as events fill - # section[-1] frame checksum (not data) + # section[-10:-8] battery × 100 uint16 BE 0x02A8 = 6.80 V + # section[-8 :-4] memory_total uint32 BE ≈ 960 KB on BE11529 + # section[-4:] memory_free uint32 BE decreases as events fill # - # Confirmed stable across IDLE (52b), MONITORING (55b), and counter-jitter - # IDLE variants (53b) from 4-8-26/2ndtry full capture analysis. - if len(section) >= 11: - batt_raw = struct.unpack(">H", section[-11:-9])[0] + # Confirmed stable across IDLE (46b), MONITORING (48-49b) variants. + if len(section) >= 10: + batt_raw = struct.unpack(">H", section[-10:-8])[0] battery_v = batt_raw / 100.0 - memory_total = struct.unpack(">I", section[-9:-5])[0] - memory_free = struct.unpack(">I", section[-5:-1])[0] + memory_total = struct.unpack(">I", section[-8:-4])[0] + memory_free = struct.unpack(">I", section[-4:])[0] return MonitorStatus( is_monitoring=is_monitoring, From 574d40027f84389603bceac52985aa95ae303386 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 00:58:54 -0400 Subject: [PATCH 06/33] feat: enhance logging messages in ach_server.py and add experiments.py for protocol minimization --- bridges/ach_server.py | 24 +- docs/instantel_protocol_reference.md | 5 +- experiments.py | 634 +++++++++++++++++++++++++++ 3 files changed, 649 insertions(+), 14 deletions(-) create mode 100644 experiments.py diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 1c37f86..69050b7 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -185,9 +185,9 @@ class AchSession: from minimateplus.protocol import MiniMateProtocol proto = MiniMateProtocol(transport, recv_timeout=self.timeout) proto.startup() - log.info(" ✓ Startup OK — pull protocol confirmed") + log.info(" [OK] Startup OK -- pull protocol confirmed") except Exception as exc: - log.error(" ✗ Startup failed: %s", exc) + log.error(" [FAIL] Startup failed: %s", exc) return # ── Step 2: device info ─────────────────────────────────────────── @@ -196,16 +196,16 @@ class AchSession: log.info("Step 2/3: reading device info") try: device_info = client.connect() - serial = device_info.serial_number + serial = device_info.serial _save_json(session_dir / "device_info.json", _device_info_to_dict(device_info)) log.info( - " ✓ Device: serial=%s firmware=%s calibration=%s", + " [OK] Device: serial=%s firmware=%s calibration=%s", serial, device_info.firmware_version, device_info.calibration_date, ) except Exception as exc: - log.error(" ✗ Device info failed: %s", exc) + log.error(" [FAIL] Device info failed: %s", exc) else: log.info("Step 2/3: skipping device info (--events-only)") @@ -221,12 +221,12 @@ class AchSession: log.info(" Unit has %d stored event(s); last downloaded count: %d", current_count, last_count) except Exception as exc: - log.error(" ✗ count_events failed: %s", exc) + log.error(" [FAIL] count_events failed: %s", exc) return if current_count <= last_count: - log.info(" ✓ No new events since last call-home — nothing to download") - log.info("Session complete (no new events) → %s", session_dir) + log.info(" [OK] No new events since last call-home -- nothing to download") + log.info("Session complete (no new events) -> %s", session_dir) return new_event_count = current_count - last_count @@ -252,7 +252,7 @@ class AchSession: ) # Only the events beyond last_count are genuinely new new_events = all_events[last_count:] - log.info(" ✓ Downloaded %d total event(s), %d new", + log.info(" [OK] Downloaded %d total event(s), %d new", len(all_events), len(new_events)) _save_json(session_dir / "events.json", [_event_to_dict(e) for e in new_events]) @@ -280,13 +280,13 @@ class AchSession: _save_state(self.state_path, state) except Exception as exc: - log.error(" ✗ Event download failed: %s", exc, exc_info=True) + log.error(" [FAIL] Event download failed: %s", exc, exc_info=True) finally: raw_fh.close() client.close() # closes transport / socket cleanly - log.info("Session complete → %s", session_dir) + log.info("Session complete -> %s", session_dir) log.info("="*60) @@ -300,7 +300,7 @@ def _save_json(path: Path, obj: object) -> None: def _device_info_to_dict(d: DeviceInfo) -> dict: return { - "serial_number": d.serial_number, + "serial": d.serial, "firmware_version": d.firmware_version, "calibration_date": str(d.calibration_date) if d.calibration_date else None, "aux_trigger": d.aux_trigger, diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 57be0f3..e20c15f 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -93,7 +93,8 @@ | 2026-04-06 | §7.8.4 | **NEW — 5A chunk timing and count (empirical, BE11529 at 1024 sps).** Each chunk response arrives within ~1 second over TCP/cellular. A 9,306-sample event (≈9.1 s at 1024 sps) produces **35 chunks** before end-of-stream. Chunks 1–16 have varying data lengths (1036–1123 bytes); chunks 17–35 are uniformly 1036 bytes each (post-event silence, all-zero ADC samples). Safe recv timeout for chunk loop: **10 s** (10× typical response time). Default transport timeout (120 s) results in a ~2-minute stall per event at end-of-stream. | | 2026-04-06 | §7.8.3 | **KNOWN ISSUE — `_decode_a5_waveform` hardcoded fi==9 skip.** The decoder contains `elif fi == 9: continue` which was written for the 9-frame original blast capture where frame 9 was a device terminator. For streams with >9 frames (current device produces 35+), frame index 9 is live waveform data — this skip discards ~1,070 bytes (~133 sample-sets) per event. The terminator is now detected via `page_key == 0x0000`, not by frame index. The fi==9 skip should be removed. | | 2026-04-06 | §7.8 | **CONFIRMED — ADC count-to-physical-unit conversion.** Raw waveform samples are signed 16-bit integers (counts). Conversion: `value = counts × (range / 32767)`. For geo channels: range = 10.000 in/s (from the device's compliance config geo range field). For the mic channel: range is in psi (device-specific). Near-full-scale counts (≈32,700) on all four channels simultaneously indicate ADC saturation (clipping) from a high-amplitude event. | -| 2026-04-08 | §5.1, §7.10, §12 | **NEW — Monitoring commands confirmed.** SUB 0x1C (monitor status), 0x96 (start monitoring), 0x97 (stop monitoring) all confirmed from 4-8-26/2ndtry capture. SESSION_RESET (`41 03`) required before POLL to wake a monitoring unit. `section[6] == 0x10` is the monitoring flag (CORRECTED 2026-04-08 — was wrongly `section[1]`). Battery/memory at relative-from-end offsets: `section[-11:-9]` (battery×100), `section[-9:-5]` (memory_total), `section[-5:-1]` (memory_free) — stable across all payload size variants (52–55 bytes). | +| 2026-04-08 | §5.1, §7.10, §12 | **NEW — Monitoring commands confirmed.** SUB 0x1C (monitor status), 0x96 (start monitoring), 0x97 (stop monitoring) all confirmed from 4-8-26/2ndtry capture. SESSION_RESET (`41 03`) required before POLL to wake a monitoring unit. | +| 2026-04-09 | §7.10 | **CORRECTED — monitoring flag and battery/memory offsets.** `section[1] == 0x10` is the monitoring flag (100% accurate across 144 data frames in 2ndtry capture). Previous note claiming `section[6]` was wrong — section[6] has device-specific non-binary values (0xea/0x07). Battery/memory offsets corrected: `section[-10:-8]` (battery×100), `section[-8:-4]` (memory_total), `section[-4:]` (memory_free). NOTE: `frame.data` has checksum stripped by parser — earlier offsets of `[-11:-9]`/`[-9:-5]`/`[-5:-1]` were wrong because they assumed a trailing checksum byte that isn't there. | | 2026-04-08 | §7.10 | **NEW — SUBs 0x0E (channel sensor data) and 0x98 (trigger test) observed** in 4-8-26/sensor-check capture (Blastware "Unit Channel Test" comms check). SUB 0x0E: 2-step read with channel selector in `params[6:8]`, data length 0x0A per channel, RSP SUB = 0xF1. SUB 0x98: single probe frame with `params[0] = 0xFF`, RSP SUB = 0x67; sent twice per test cycle. Not yet implemented in SFM. | | 2026-04-08 | §7.10 | **NEW — SUBs 0x15 and 0x01 observed in sensor-check capture.** SUB 0x15 (serial number short form, data length 0x0A, RSP 0xEA) and SUB 0x01 (device info block, data length 0x98 = 152 bytes, RSP 0xFE) seen in Blastware's "Unit Channel Test" init sequence. Note: SUB 0x01 response SUB 0xFE collides with the existing SUB 0xFE → RSP 0x01 naming convention — they are inverse commands. | | 2026-04-08 | §12 | **CONFIRMED — Unit partially reachable during on-device sensor check.** 4-8-26/sensor-check capture shows: POLL responds normally throughout; SUB 0x0E channel reads partially served (channels 0–4 responded), then ~40s silent gap while sensor check ran, then channels 5–7 responded. On-device sensor check duration ≈ 40 s. SFM `_pollMonitorConfirm()` polls status every 5 s for up to 60 s after start_monitoring. | @@ -256,7 +257,7 @@ Step 4 — Device sends actual data payload: | `2E` | **UNKNOWN READ B** | Read command, response (`D1`) returns 0x1A (26) bytes. Purpose unknown. | 🔶 INFERRED | | `0E` | **CHANNEL SENSOR DATA** | Real-time sensor reading for one channel. Two-step read, data length 0x0A (10 bytes). Channel selector in params[6:8] (0x0000–0x0007 for 8 channels). Response (F1) carries amplitude, frequency, overswing data for that channel. Used by Blastware "Unit Channel Test" comms check. | ✅ CONFIRMED 2026-04-08 | | `98` | **TRIGGER TEST** | Trigger-test command. Single probe frame; `params[0] = 0xFF`. Response (0x67) is all-zero data. Sent twice per Blastware comms-check cycle. Not a full POLL, no monitor state change. | ✅ CONFIRMED 2026-04-08 | -| `1C` | **MONITOR STATUS READ** | Two-step read, data offset 0x2C (44 bytes). `section[6] == 0x10` → monitoring; `0x00` → idle (CORRECTED 2026-04-08 — was wrongly documented as section[1]). Payload length varies (52–55 bytes) but battery/memory block is always the last 10 bytes before checksum: `section[-11:-9]` = battery×100 (uint16 BE), `section[-9:-5]` = memory_total (uint32 BE), `section[-5:-1]` = memory_free (uint32 BE). Confirmed from 2ndtry 4-8-26 full byte diff across 3 payload size variants. | ✅ CONFIRMED 2026-04-08 | +| `1C` | **MONITOR STATUS READ** | Two-step read, data offset 0x2C (44 bytes). `section[1] == 0x10` → monitoring; `0x00` → idle (CONFIRMED 2026-04-09, 100% accuracy on 144 frames). Payload length: 46–47 bytes IDLE, 48–49 bytes MONITORING. `frame.data` has checksum stripped — no trailing byte to skip. Battery/memory at end: `section[-10:-8]` = battery×100 (uint16 BE), `section[-8:-4]` = memory_total (uint32 BE), `section[-4:]` = memory_free (uint32 BE). | ✅ CONFIRMED 2026-04-09 | | `96` | **START MONITORING** | Single write frame, no data payload. Transitions unit from idle to monitoring mode (after optional on-device sensor check ~40 s). | ✅ CONFIRMED 2026-04-08 | | `97` | **STOP MONITORING** | Single write frame, no data payload. Stops monitoring, unit returns to idle. | ✅ CONFIRMED 2026-04-08 | diff --git a/experiments.py b/experiments.py new file mode 100644 index 0000000..7c5bff3 --- /dev/null +++ b/experiments.py @@ -0,0 +1,634 @@ +#!/usr/bin/env python3 +""" +experiments.py — Protocol minimization experiments for MiniMate Plus. + +Goal: figure out which steps in Blastware's sequences are truly required vs. +cargo-culted, so we can build a faster, smarter client. + +Each experiment is self-contained (opens its own TCP connection) and reports +PASS / FAIL / INCONCLUSIVE with timing and notes. + +Usage: + python experiments.py [--host IP] [--port PORT] [exp1 exp2 ...] + + Run all: python experiments.py + Run specific: python experiments.py cold_status fast_event_count no_5a + +Available experiments +--------------------- + cold_status EXP1 Monitor status (1C) with NO prior POLL + fast_event_count EXP2 Event count via POLL+08 only — skip identity reads + no_5a EXP3 Event record (0C) without bulk waveform stream (5A) + skip_1e EXP4 0A/0C directly with cached key — skip initial 1E + fewer_polls EXP5 Only 1 POLL before 5A instead of Blastware's 3 + compliance_only EXP6 Write compliance ONLY (71x3→72), skip event index+trigger+waveform +""" + +from __future__ import annotations + +import argparse +import logging +import struct +import sys +import time +from dataclasses import dataclass, field +from typing import Optional + +logging.basicConfig( + level=logging.WARNING, # experiment output is via print(); set DEBUG for wire trace + format="%(asctime)s %(levelname)-7s %(name)-20s %(message)s", + datefmt="%H:%M:%S", +) +log = logging.getLogger("experiments") + +# ── Imports ─────────────────────────────────────────────────────────────────── + +from minimateplus.transport import TcpTransport +from minimateplus.protocol import ( + MiniMateProtocol, + ProtocolError, + TimeoutError as ProtoTimeout, + SUB_MONITOR_STATUS, + SUB_SERIAL_NUMBER, + SUB_FULL_CONFIG, + SUB_EVENT_INDEX, + SUB_COMPLIANCE, + SUB_WRITE_CONFIRM_A, + SUB_WRITE_CONFIRM_B, +) +from minimateplus.framing import build_bw_frame, SESSION_RESET +from minimateplus.client import ( + MiniMateClient, + _decode_compliance_config_into, + _encode_compliance_config, +) +from minimateplus.models import DeviceInfo + + +DEFAULT_HOST = "63.43.212.232" +DEFAULT_PORT = 9034 + + +# ── Result container ────────────────────────────────────────────────────────── + +@dataclass +class Result: + name: str + outcome: str # "PASS" | "FAIL" | "INCONCLUSIVE" + elapsed: float = 0.0 + notes: str = "" + details: dict = field(default_factory=dict) + + def __str__(self) -> str: + sym = {"PASS": "✅", "FAIL": "❌", "INCONCLUSIVE": "⚠️ "}.get(self.outcome, "?") + lines = [f" {sym} {self.outcome:13s} {self.name} ({self.elapsed:.1f}s)"] + if self.notes: + lines.append(f" {self.notes}") + for k, v in self.details.items(): + lines.append(f" {k}: {v}") + return "\n".join(lines) + + +# ── Connection helpers ──────────────────────────────────────────────────────── + +def connect_proto(host: str, port: int, timeout: float = 15.0) -> tuple[TcpTransport, MiniMateProtocol]: + """Open a raw TCP connection and return (transport, proto) without any handshake.""" + t = TcpTransport(host, port) + t.connect() + proto = MiniMateProtocol(t, recv_timeout=timeout) + return t, proto + + +def connect_client(host: str, port: int, timeout: float = 30.0) -> tuple[MiniMateClient, DeviceInfo]: + """Open a MiniMateClient and run the full connect() handshake.""" + transport = TcpTransport(host, port) + client = MiniMateClient(transport=transport, timeout=timeout) + client.open() + info = client.connect() + return client, info + + +# ── Experiment runner ───────────────────────────────────────────────────────── + +def run(name: str, fn, *args, **kwargs) -> Result: + print(f"\n{'─'*60}") + print(f" Running: {name}") + print(f"{'─'*60}") + t0 = time.time() + try: + outcome, notes, details = fn(*args, **kwargs) + except Exception as exc: + outcome = "FAIL" + notes = f"Uncaught exception: {exc}" + details = {} + log.exception("Experiment %s raised:", name) + elapsed = time.time() - t0 + r = Result(name=name, outcome=outcome, elapsed=elapsed, notes=notes, details=details) + print(str(r)) + return r + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP1 — Monitor status (1C) with NO prior POLL +# ══════════════════════════════════════════════════════════════════════════════ +# +# Blastware always does a full POLL handshake before any other command. +# We want to know: can we query SUB 1C (battery, memory, monitoring state) +# cold, with only a SESSION_RESET signal and no POLL at all? +# +# If PASS: status checks become near-instant (no ~1s POLL round-trip). +# If FAIL: we need POLL first, but maybe we can cache it. + +def exp_cold_status(host: str, port: int) -> tuple[str, str, dict]: + """SUB 1C without any POLL — just SESSION_RESET + 1C probe + 1C data.""" + t, proto = connect_proto(host, port) + try: + print(" Sending SESSION_RESET only (no POLL)") + t.write(SESSION_RESET) + time.sleep(0.1) + + print(" Sending SUB 1C probe (no POLL first)…") + rsp_sub = (0xFF - SUB_MONITOR_STATUS) & 0xFF # 0xE3 + t.write(build_bw_frame(SUB_MONITOR_STATUS, 0x00)) + probe = proto._recv_one(expected_sub=rsp_sub, timeout=8.0) + print(f" 1C probe OK page_key=0x{probe.page_key:04X} data={probe.data.hex()}") + + t.write(build_bw_frame(SUB_MONITOR_STATUS, 0x2C)) + data_rsp = proto._recv_one(expected_sub=rsp_sub, timeout=8.0) + + section = data_rsp.data + print(f" 1C data OK {len(section)} bytes hex: {section.hex()}") + + # Decode battery + memory from the end of the section + details = {"raw_bytes": len(section)} + if len(section) >= 10: + batt_raw = struct.unpack_from(">H", section, len(section) - 10)[0] + mem_total = struct.unpack_from(">I", section, len(section) - 8)[0] + mem_free = struct.unpack_from(">I", section, len(section) - 4)[0] + is_monitoring = (section[1] == 0x10) + details["battery_v"] = f"{batt_raw / 100:.2f} V" + details["memory_total"] = f"{mem_total:,} bytes" + details["memory_free"] = f"{mem_free:,} bytes" + details["monitoring"] = is_monitoring + print(f" battery={batt_raw/100:.2f}V mem_free={mem_free:,} monitoring={is_monitoring}") + + return "PASS", "SUB 1C responded without any POLL — cold status read works!", details + + except ProtoTimeout: + return "FAIL", "Device did not respond to 1C without POLL (timeout)", {} + except ProtocolError as exc: + return "FAIL", f"Protocol error: {exc}", {} + finally: + t.disconnect() + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP2 — Fast event count: POLL + SUB 08 only (skip identity reads) +# ══════════════════════════════════════════════════════════════════════════════ +# +# Blastware's connect() does: POLL → 15 → 01 → 1A → 08 +# We want to know: can we skip 15/01/1A and go straight from POLL to 08? +# +# Reading identity (15, 01) and full compliance (1A, ~2126 bytes over TCP) +# takes several seconds each connect. If we only need event count, skipping +# them would be a huge win. +# +# If PASS: fast status poll = POLL + 08 only (~2 round trips vs ~8+). + +def exp_fast_event_count(host: str, port: int) -> tuple[str, str, dict]: + """POLL startup → SUB 08 only, skip serial/config/compliance reads.""" + t, proto = connect_proto(host, port) + try: + print(" Running startup (POLL only)…") + proto.startup() + print(" POLL OK — now reading SUB 08 (event index) directly…") + + idx_raw = proto.read_event_index() + print(f" SUB 08 OK {len(idx_raw)} bytes") + + # Try to decode event count from SUB 08 payload + # The raw block is 88 bytes; bytes [3:7] may be a count (uint32 BE) + details = {"idx_raw_len": len(idx_raw)} + if len(idx_raw) >= 7: + count_candidate = struct.unpack_from(">I", idx_raw, 3)[0] + details["count_candidate"] = count_candidate + print(f" idx[3:7] as uint32 BE = {count_candidate} (may or may not be event count)") + + # Also verify we can read 1E without the identity reads having been done + print(" Reading 1E (event header) to confirm event access works…") + key4, data8 = proto.read_event_first() + is_empty = data8[4:8] == b"\x00\x00\x00\x00" + details["first_key"] = key4.hex() + details["is_empty"] = is_empty + print(f" 1E OK key={key4.hex()} empty={is_empty}") + + return "PASS", "POLL+08+1E all work without identity reads (15/01/1A skipped)", details + + except ProtocolError as exc: + return "FAIL", f"Protocol error: {exc}", {} + finally: + t.disconnect() + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP3 — Get event record (0C) without bulk waveform stream (5A) +# ══════════════════════════════════════════════════════════════════════════════ +# +# Blastware's event download = 1E → 0A → 1E-arm → 0C → 1F(dl) → POLL×3 → 5A → 1F(browse) +# +# The 5A bulk stream is the slow part (several large frames, ~1s+ per event). +# We only need 5A for: client, operator, seis_loc, notes (not in 0C). +# If you don't need those fields, can we do: 1E → 0A → 0C → 1F(browse) ? +# +# Two variants tested: +# 3a: Skip 1E-arm AND 5A — just 0A → 0C → 1F(browse) +# 3b: Include 1E-arm but skip 5A+POLL — 0A → 1E-arm → 0C → 1F(browse) +# +# If PASS: event peak values available without the slow bulk stream. +# If FAIL on 3a but PASS on 3b: 1E-arm required even without 5A. + +def exp_no_5a(host: str, port: int) -> tuple[str, str, dict]: + """Event record via 0A→0C without 5A or POLL×3. Tests both with and without 1E-arm.""" + t, proto = connect_proto(host, port) + try: + print(" Startup (POLL)…") + proto.startup() + + # Get the first event key via 1E + key4, data8 = proto.read_event_first() + if data8[4:8] == b"\x00\x00\x00\x00": + return "INCONCLUSIVE", "Device has no stored events — cannot test", {} + print(f" First event key: {key4.hex()}") + + details: dict = {"key": key4.hex()} + + # ── Variant 3a: 0A → 0C → 1F(browse), no 1E-arm ───────────────────── + print("\n [3a] 0A → 0C → 1F(browse) (NO 1E-arm, NO 5A)") + try: + _hdr, rec_len = proto.read_waveform_header(key4) + print(f" 0A OK rec_len=0x{rec_len:02X}") + record_3a = proto.read_waveform_record(key4) + print(f" 0C OK {len(record_3a)} bytes") + # Check for recognizable content + has_tran = b"Tran" in record_3a + has_vert = b"Vert" in record_3a + has_long = b"Long" in record_3a + print(f" 0C content check: Tran={has_tran} Vert={has_vert} Long={has_long}") + details["3a_0c_bytes"] = len(record_3a) + details["3a_has_peaks"] = has_tran and has_vert and has_long + + # Now try browse 1F without any 5A + key4_next, data8_next = proto.advance_event(browse=True) + null_sentinel = data8_next[4:8] == b"\x00\x00\x00\x00" + print(f" 1F(browse) → key={key4_next.hex()} null={null_sentinel}") + details["3a_1f_ok"] = True + details["3a_outcome"] = "PASS" + except ProtocolError as exc: + print(f" 3a FAILED: {exc}") + details["3a_outcome"] = f"FAIL: {exc}" + # Try to recover by reconnecting for 3b + t.disconnect() + t2, proto2 = connect_proto(host, port) + proto2.startup() + key4, data8 = proto2.read_event_first() + if data8[4:8] == b"\x00\x00\x00\x00": + return "FAIL", f"3a failed and device empty on retry: {exc}", details + t, proto = t2, proto2 + + # ── Variant 3b: 0A → 1E-arm → 0C → 1F(browse), no 5A ─────────────── + print("\n [3b] 0A → 1E-arm(0xFE) → 0C → 1F(browse) (NO POLL×3, NO 5A)") + try: + _hdr, rec_len = proto.read_waveform_header(key4) + print(f" 0A OK rec_len=0x{rec_len:02X}") + + # 1E download-arm (token=0xFE) between 0A and 0C + proto.read_event_first(token=0xFE) + print(" 1E-arm OK") + + record_3b = proto.read_waveform_record(key4) + print(f" 0C OK {len(record_3b)} bytes") + has_tran = b"Tran" in record_3b + print(f" 0C content check: Tran={has_tran} Vert={b'Vert' in record_3b}") + details["3b_0c_bytes"] = len(record_3b) + details["3b_has_peaks"] = has_tran + + # Browse 1F without 5A / POLL×3 + key4_next2, data8_next2 = proto.advance_event(browse=True) + null_sentinel2 = data8_next2[4:8] == b"\x00\x00\x00\x00" + print(f" 1F(browse) → key={key4_next2.hex()} null={null_sentinel2}") + details["3b_1f_ok"] = True + details["3b_outcome"] = "PASS" + except ProtocolError as exc: + print(f" 3b FAILED: {exc}") + details["3b_outcome"] = f"FAIL: {exc}" + + # Summarize + a_ok = details.get("3a_outcome") == "PASS" + b_ok = details.get("3b_outcome") == "PASS" + if a_ok: + return "PASS", "3a: 0A→0C works with NO 1E-arm and NO 5A. Huge speedup possible!", details + elif b_ok: + return "PASS", "3b: 0A→1E-arm→0C works without 5A (1E-arm still needed before 0C)", details + else: + return "FAIL", "Both 3a and 3b failed — 5A may be required for device state", details + + except ProtocolError as exc: + return "FAIL", f"Protocol error during setup: {exc}", {} + finally: + try: + t.disconnect() + except Exception: + pass + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP4 — Skip initial 1E if we already know the event key +# ══════════════════════════════════════════════════════════════════════════════ +# +# In Blastware, every session starts with 1E to discover the first key. +# But if we already fetched and cached the event keys from a previous session, +# can we skip 1E entirely and go straight to 0A(cached_key)? +# +# Practical use case: we poll the device every N minutes. We already know +# all the event keys from last time. On re-connect, can we go direct to 0A? +# +# If PASS: subsequent polls that don't add new events can skip 1E discovery. + +def exp_skip_1e(host: str, port: int) -> tuple[str, str, dict]: + """Get the first event key, disconnect, reconnect, go straight to 0A (skip 1E).""" + # Phase 1: get the key + t, proto = connect_proto(host, port) + try: + proto.startup() + key4, data8 = proto.read_event_first() + if data8[4:8] == b"\x00\x00\x00\x00": + return "INCONCLUSIVE", "No events stored — cannot test", {} + print(f" Phase 1: got event key = {key4.hex()}") + finally: + t.disconnect() + time.sleep(0.5) + + # Phase 2: fresh connection, skip 1E, go straight to 0A with cached key + t2, proto2 = connect_proto(host, port) + try: + print(" Phase 2: fresh connection — startup + 0A directly (no 1E)") + proto2.startup() + + _hdr, rec_len = proto2.read_waveform_header(key4) + print(f" 0A OK rec_len=0x{rec_len:02X}") + + record = proto2.read_waveform_record(key4) + has_peaks = b"Tran" in record + print(f" 0C OK {len(record)} bytes has_peaks={has_peaks}") + + details = { + "cached_key": key4.hex(), + "0c_bytes": len(record), + "has_peaks": has_peaks, + } + return "PASS", "0A works with cached key — 1E discovery can be skipped on known sessions", details + + except ProtocolError as exc: + return "FAIL", f"0A failed with cached key (device needs 1E first?): {exc}", {"key": key4.hex()} + finally: + t2.disconnect() + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP5 — Fewer POLLs before 5A (try POLL×1 instead of Blastware's POLL×3) +# ══════════════════════════════════════════════════════════════════════════════ +# +# Blastware always sends 3 full POLL probe+data cycles between 1F and 5A. +# Each POLL is a round trip. Can we get away with just 1? +# +# WARNING: If POLL×1 fails, the device may be in a bad state. We try to +# recover with an extra POLL×2 and a fresh 5A attempt. Even on failure we +# try to leave the device in a usable state. +# +# Strategy: run the full event sequence up to 1F(download), then try 5A +# with only 1 POLL. If 5A responds → PASS. If timeout → try 2 more POLLs +# and check if the device recovers. + +def exp_fewer_polls(host: str, port: int) -> tuple[str, str, dict]: + """Full sequence to 1F, then only 1 POLL before 5A (Blastware does 3).""" + t, proto = connect_proto(host, port) + try: + proto.startup() + + key4, data8 = proto.read_event_first() + if data8[4:8] == b"\x00\x00\x00\x00": + return "INCONCLUSIVE", "No events stored — cannot test", {} + print(f" Event key: {key4.hex()}") + + # Full setup: 0A → 1E-arm → 0C → 1F(download) + _hdr, rec_len = proto.read_waveform_header(key4) + print(f" 0A OK rec_len=0x{rec_len:02X}") + proto.read_event_first(token=0xFE) # 1E-arm + print(" 1E-arm OK") + proto.read_waveform_record(key4) + print(" 0C OK") + arm_key4, _ = proto.advance_event(browse=False) # 1F(download) — arms 5A + print(f" 1F(download) OK arm_key={arm_key4.hex()}") + + # Only 1 POLL (Blastware does 3) + print(" Sending 1 POLL (instead of 3)…") + proto.poll() + print(" POLL ok — now probing 5A…") + + try: + frames = proto.read_bulk_waveform_stream(key4, stop_after_metadata=True, max_chunks=12) + print(f" 5A OK after 1 POLL — {len(frames)} frames received") + details = {"poll_count": 1, "frames": len(frames)} + return "PASS", "5A works with only 1 POLL (saved 2 round-trips per event)!", details + + except ProtoTimeout: + print(" 5A timed out after 1 POLL — device needs more POLLs") + # Attempt recovery: send 2 more POLLs and see if 5A then works + print(" Attempting recovery: 2 more POLLs…") + try: + proto.poll() + proto.poll() + frames2 = proto.read_bulk_waveform_stream(key4, stop_after_metadata=True, max_chunks=12) + print(f" 5A worked after total 3 POLLs ({len(frames2)} frames)") + return "FAIL", "5A needs 3 POLLs — 1 is not enough (recovery confirmed 3 still works)", { + "poll_count_tried": 1, "recovery_polls": 3, "recovery_frames": len(frames2) + } + except ProtocolError as exc2: + return "FAIL", f"5A failed even after 3 total POLLs — device may need reconnect: {exc2}", {} + + except ProtocolError as exc: + return "FAIL", f"Setup failed: {exc}", {} + finally: + t.disconnect() + + +# ══════════════════════════════════════════════════════════════════════════════ +# EXP6 — Compliance-only write (71×3→72), skip event index + trigger + waveform +# ══════════════════════════════════════════════════════════════════════════════ +# +# Blastware's full write sequence: 68→73 | 71×3→72 | 82→83 | 69→74→72 +# We want to know: can we write ONLY the compliance block (71×3→72)? +# +# Test procedure: +# 1. Read current compliance config (SUB 1A) +# 2. Patch the "notes" field to a test marker +# 3. Write ONLY 71×3→72 (skip 68, 73, 82, 83, 69, 74, final 72) +# 4. Read back (SUB 1A) and verify the change was written +# 5. Restore original value +# +# If PASS: we can push individual config fields without touching event index, +# trigger config, or waveform data — huge simplification. +# If FAIL: the device needs the full write sequence (may reject partial write). +# +# SAFETY: We restore original data in a finally block. If the restore write +# fails, the device will have the test marker in "notes" — harmless but visible. + +_EXP6_MARKER = "[exp6-test]" + +def exp_compliance_only(host: str, port: int) -> tuple[str, str, dict]: + """Write compliance block alone (71×3→72), verify, and restore.""" + client, info = connect_client(host, port) + original_raw: Optional[bytes] = None + try: + proto = client._proto + if proto is None: + return "FAIL", "Could not get protocol handle from client", {} + + # 1. Read current compliance + print(" Reading current compliance config (SUB 1A)…") + original_raw = proto.read_compliance_config() + print(f" Got {len(original_raw)} bytes of compliance config") + + # Find current notes value for display + info_obj = DeviceInfo() + _decode_compliance_config_into(original_raw, info_obj) + cc = info_obj.compliance_config + orig_notes = cc.notes if cc else "(unknown)" + print(f" Current notes field: {orig_notes!r}") + + # 2. Build modified payload with test marker in notes + test_notes = _EXP6_MARKER + modified_raw = _encode_compliance_config( + original_raw, + notes=test_notes, + ) + print(f" Encoded modified compliance payload ({len(modified_raw)} bytes)") + print(f" Patching notes: {orig_notes!r} → {test_notes!r}") + + # 3. Write ONLY the compliance block: 71×3 → 72 + print(" Writing compliance ONLY (71×3→72) — skipping 68/73/82/83/69/74…") + proto.write_compliance_config_raw(modified_raw) + print(" Write complete — device acked 71×3→72") + + # 4. Read back and verify + print(" Reading back compliance config to verify…") + readback_raw = proto.read_compliance_config() + readback_info = DeviceInfo() + _decode_compliance_config_into(readback_raw, readback_info) + rb_cc = readback_info.compliance_config + readback_notes = rb_cc.notes if rb_cc else "(decode failed)" + print(f" Read-back notes: {readback_notes!r}") + + write_worked = (readback_notes == test_notes) + print(f" Write verified: {write_worked}") + + details = { + "original_notes": orig_notes, + "written_notes": test_notes, + "readback_notes": readback_notes, + "write_verified": write_worked, + } + + if write_worked: + return "PASS", "Compliance-only write works! No event index or trigger writes needed.", details + else: + return "FAIL", f"Write was not reflected in read-back (got {readback_notes!r})", details + + except ProtocolError as exc: + return "FAIL", f"Protocol error: {exc}", {} + + finally: + # Restore original compliance data regardless of outcome + if original_raw is not None: + print(" Restoring original compliance config…") + try: + proto2 = client._proto + if proto2: + proto2.write_compliance_config_raw( + _encode_compliance_config(original_raw) # no-op patch = verbatim + ) + print(" Restore complete") + else: + print(" WARNING: protocol handle gone — could not restore") + except Exception as exc_r: + print(f" WARNING: restore failed: {exc_r}") + client.close() + + +# ══════════════════════════════════════════════════════════════════════════════ +# Registry + main +# ══════════════════════════════════════════════════════════════════════════════ + +EXPERIMENTS = { + "cold_status": ("EXP1", exp_cold_status, "Monitor status (1C) with no POLL"), + "fast_event_count": ("EXP2", exp_fast_event_count, "Event count via POLL+08, skip identity reads"), + "no_5a": ("EXP3", exp_no_5a, "Event record (0C) without bulk waveform (5A)"), + "skip_1e": ("EXP4", exp_skip_1e, "0A/0C with cached key — skip initial 1E"), + "fewer_polls": ("EXP5", exp_fewer_polls, "1 POLL before 5A instead of Blastware's 3"), + "compliance_only": ("EXP6", exp_compliance_only, "Compliance-only write (71×3→72), no other blocks"), +} + + +def main() -> None: + ap = argparse.ArgumentParser(description="MiniMate Plus protocol minimization experiments") + ap.add_argument("--host", default=DEFAULT_HOST) + ap.add_argument("--port", type=int, default=DEFAULT_PORT) + ap.add_argument("--debug", action="store_true", help="Enable DEBUG wire logging") + ap.add_argument("experiments", nargs="*", + help=f"Which to run (default: all). Choices: {', '.join(EXPERIMENTS)}") + args = ap.parse_args() + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + + which = args.experiments or list(EXPERIMENTS.keys()) + unknown = [e for e in which if e not in EXPERIMENTS] + if unknown: + print(f"Unknown experiments: {unknown}") + print(f"Available: {', '.join(EXPERIMENTS)}") + sys.exit(1) + + print(f"\n{'═'*60}") + print(f" MiniMate Plus Protocol Minimization Experiments") + print(f" Target: {args.host}:{args.port}") + print(f" Running: {', '.join(which)}") + print(f"{'═'*60}") + + results: list[Result] = [] + for key in which: + tag, fn, desc = EXPERIMENTS[key] + label = f"{tag}: {desc}" + r = run(label, fn, args.host, args.port) + results.append(r) + time.sleep(1.5) # brief pause between experiments — let device settle + + print(f"\n\n{'═'*60}") + print(" SUMMARY") + print(f"{'═'*60}") + for r in results: + sym = {"PASS": "✅", "FAIL": "❌", "INCONCLUSIVE": "⚠️ "}.get(r.outcome, "?") + print(f" {sym} {r.outcome:13s} {r.name}") + print(f"{'═'*60}") + + passed = sum(1 for r in results if r.outcome == "PASS") + failed = sum(1 for r in results if r.outcome == "FAIL") + skipped = sum(1 for r in results if r.outcome == "INCONCLUSIVE") + print(f" {passed} passed {failed} failed {skipped} inconclusive") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nInterrupted.") + sys.exit(0) From 1bfc6e4258e5d37606887241166ec5911e837523 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 01:06:27 -0400 Subject: [PATCH 07/33] fix: replace Unicode chars in log messages, fix DeviceInfo.serial, UTF-8 file log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace all Unicode arrows/checkmarks (-> [OK] [FAIL]) in ach_server.py and client.py log calls — Windows cp1252 console can't encode them - Fix DeviceInfo attribute: serial_number -> serial - Fix _device_info_to_dict key: serial_number -> serial - Demote count_events 1E/1F per-key log lines from WARNING to DEBUG (they were flooding the console on devices with many stored events) - FileHandler now opens with encoding='utf-8' so session log files can hold any characters without codec errors Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 2 +- minimateplus/client.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 69050b7..19b7f43 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -136,7 +136,7 @@ class AchSession: raw_path = session_dir / f"raw_rx_{ts}.bin" # Wire up a file handler so every protocol log line goes to the session log - fh = logging.FileHandler(log_path) + fh = logging.FileHandler(log_path, encoding="utf-8") fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s")) root_logger = logging.getLogger() root_logger.addHandler(fh) diff --git a/minimateplus/client.py b/minimateplus/client.py index 984cbcf..2538f48 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -216,8 +216,8 @@ class MiniMateClient: log.warning("count_events: 1E failed: %s — returning 0", exc) return 0 - log.warning( - "count_events: 1E → key=%s data8=%s trailing=%s", + log.debug( + "count_events: 1E -> key=%s data8=%s trailing=%s", key4.hex(), data8.hex(), data8[4:8].hex(), ) @@ -241,8 +241,8 @@ class MiniMateClient: break try: key4, data8 = proto.advance_event(browse=True) - log.warning( - "count_events: 1F [iter %d] → key=%s data8=%s trailing=%s", + log.debug( + "count_events: 1F [iter %d] -> key=%s data8=%s trailing=%s", count, key4.hex(), data8.hex(), data8[4:8].hex(), ) except ProtocolError as exc: @@ -426,7 +426,7 @@ class MiniMateClient: try: key4, data8 = proto.advance_event(browse=True) log.info( - "get_events: 1F(browse) → key=%s trailing=%s", + "get_events: 1F(browse) -> key=%s trailing=%s", key4.hex(), data8[4:8].hex(), ) except ProtocolError as exc: @@ -481,7 +481,7 @@ class MiniMateClient: try: key4, data8 = proto.advance_event(browse=True) log.info( - "get_events: 1F → key=%s trailing=%s", + "get_events: 1F -> key=%s trailing=%s", key4.hex(), data8[4:8].hex(), ) except ProtocolError as exc: @@ -1499,14 +1499,14 @@ def _encode_compliance_config( log.warning("_encode_compliance_config: anchor not found — cannot write sample_rate") else: struct.pack_into(">H", buf, _anc - 6, sample_rate) - log.debug("_encode_compliance_config: sample_rate=%d → offset %d", sample_rate, _anc - 6) + log.debug("_encode_compliance_config: sample_rate=%d -> offset %d", sample_rate, _anc - 6) if record_time is not None: if _anc < 0 or _anc + 10 > len(buf): log.warning("_encode_compliance_config: anchor not found — cannot write record_time") else: struct.pack_into(">f", buf, _anc + 6, record_time) - log.debug("_encode_compliance_config: record_time=%.3f → offset %d", record_time, _anc + 6) + log.debug("_encode_compliance_config: record_time=%.3f -> offset %d", record_time, _anc + 6) # ── Numeric: channel block (Tran label + unit-string guard) ─────────────── _needs_channel = any( @@ -1529,13 +1529,13 @@ def _encode_compliance_config( else: if max_range_geo is not None: struct.pack_into(">f", buf, _tran + 28, max_range_geo) - log.debug("_encode_compliance_config: max_range_geo=%.4f → offset %d", max_range_geo, _tran + 28) + log.debug("_encode_compliance_config: max_range_geo=%.4f -> offset %d", max_range_geo, _tran + 28) if trigger_level_geo is not None: struct.pack_into(">f", buf, _tran + 34, trigger_level_geo) - log.debug("_encode_compliance_config: trigger_level_geo=%.4f → offset %d", trigger_level_geo, _tran + 34) + log.debug("_encode_compliance_config: trigger_level_geo=%.4f -> offset %d", trigger_level_geo, _tran + 34) if alarm_level_geo is not None: struct.pack_into(">f", buf, _tran + 42, alarm_level_geo) - log.debug("_encode_compliance_config: alarm_level_geo=%.4f → offset %d", alarm_level_geo, _tran + 42) + log.debug("_encode_compliance_config: alarm_level_geo=%.4f -> offset %d", alarm_level_geo, _tran + 42) # ── ASCII strings (64-byte slot, value at label_pos+22) ─────────────────── def _set_string(label: bytes, value: Optional[str]) -> None: @@ -1548,7 +1548,7 @@ def _encode_compliance_config( val_bytes = value.encode("ascii", errors="replace")[:_COMPLIANCE_VALUE_MAX - 1] padded = val_bytes + b"\x00" * (_COMPLIANCE_VALUE_MAX - len(val_bytes)) buf[idx + _COMPLIANCE_VALUE_OFFSET : idx + _COMPLIANCE_SLOT_SIZE] = padded - log.debug("_encode_compliance_config: %r → %r", label, value) + log.debug("_encode_compliance_config: %r -> %r", label, value) _set_string(b"Project:", project) _set_string(b"Client:", client_name) From 41a14ca4683e1176650e0248f6720707f0c72092 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 01:10:49 -0400 Subject: [PATCH 08/33] fix: correct event count field offset and eliminate count_events() walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _decode_event_count: read uint16 BE at offset 10 (confirmed 2026-04-10 from live BE11529 event index — data[10:12]=0x0006=6, matches device LCD). Previous uint32 at offset 3 always returned 1 regardless of event count. ach_server.py: use device_info.event_count (already fetched during connect()) instead of calling count_events() separately. This saves 2*N round-trips and avoids the 1F linked-list walk which was overcounting on some devices. count_events() kept as fallback when connect() is skipped (--events-only). Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 21 ++++++++++++++------- minimateplus/client.py | 31 ++++++++++--------------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 19b7f43..ccaff02 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -216,13 +216,20 @@ class AchSession: unit_key = serial or self.peer # fall back to IP if no serial last_count = state.get(unit_key, {}).get("event_count", 0) - try: - current_count = client.count_events() - log.info(" Unit has %d stored event(s); last downloaded count: %d", - current_count, last_count) - except Exception as exc: - log.error(" [FAIL] count_events failed: %s", exc) - return + # Use the event count already read from the event index during connect(). + # This is fast (no extra round-trips) and confirmed accurate (matches LCD). + # Falls back to count_events() only if connect() wasn't called. + if device_info is not None: + current_count = device_info.event_count + else: + try: + current_count = client.count_events() + except Exception as exc: + log.error(" [FAIL] count_events failed: %s", exc) + return + + log.info(" Unit has %d stored event(s); last downloaded count: %d", + current_count, last_count) if current_count <= last_count: log.info(" [OK] No new events since last call-home -- nothing to download") diff --git a/minimateplus/client.py b/minimateplus/client.py index 2538f48..47d4a20 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -910,36 +910,25 @@ def _decode_event_count(data: bytes) -> int: """ Extract stored event count from SUB F7 (EVENT_INDEX_RESPONSE) payload. - Layout per §7.4 (offsets from data section start): - +00: 00 58 09 — total index size or record count ❓ - +03: 00 00 00 01 — possibly stored event count = 1 ❓ + Confirmed 2026-04-10 from live BE11529 event index (88 bytes): + data[10:12] uint16 BE = stored event count (confirmed: 0x0006 = 6, matches LCD) + data[3:7] uint32 BE = 0x00000001 (NOT the count — meaning TBD) - We use bytes +03..+06 interpreted as uint32 BE as the event count. - This is inferred (🔶) — the exact meaning of the first 3 bytes is unclear. + Previous implementation read uint32 at offset 3, which returned 1 regardless + of how many events were stored. """ - if len(data) < 7: + if len(data) < 12: log.warning("event index payload too short (%d bytes), assuming 0 events", len(data)) return 0 - # Log the full payload so we can reverse-engineer the format - log.warning("event_index raw (%d bytes total):", len(data)) - for off in range(0, len(data), 16): - chunk = data[off:off+16] - hex_part = " ".join(f"{b:02x}" for b in chunk) - asc_part = "".join(chr(b) if 0x20 <= b < 0x7f else "." for b in chunk) - log.warning(" [%04x]: %-47s %s", off, hex_part, asc_part) + count = struct.unpack_from(">H", data, 10)[0] - # Try the uint32 at +3 first - count = struct.unpack_from(">I", data, 3)[0] - - # Sanity check: MiniMate Plus manual says max ~1000 events + # Sanity check: MiniMate Plus max storage is ~1000 events if count > 1000: - log.warning( - "event count %d looks unreasonably large — clamping to 0", count - ) + log.warning("event count %d looks unreasonably large — clamping to 0", count) return 0 - log.warning("event_index decoded count=%d (uint32 BE at offset +3)", count) + log.debug("event_index decoded count=%d (uint16 BE at offset 10)", count) return count From cba8b1b40129ff0f6aa1f9a1f51eff787de903d1 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 01:17:30 -0400 Subject: [PATCH 09/33] feat: defer session dir creation and add --allow-ip allowlist - Session directory and log file are now created ONLY after startup() succeeds. Internet scanners and dropped connections no longer litter the output folder. Raw bytes are buffered in memory until startup succeeds, then flushed to disk. - Add --allow-ip IP flag (repeatable) to allowlist specific source IPs. Connections from un-listed IPs are rejected immediately (socket closed, no log). If no --allow-ip flags are given, all IPs are still accepted (original behavior). Usage: --allow-ip 63.43.212.232 --allow-ip 152.1.2.3 Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 106 ++++++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index ccaff02..012db05 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -129,67 +129,78 @@ class AchSession: def run(self) -> None: ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - session_dir = self.output_dir / f"ach_inbound_{ts}" - session_dir.mkdir(parents=True, exist_ok=True) - - log_path = session_dir / f"session_{ts}.log" - raw_path = session_dir / f"raw_rx_{ts}.bin" - - # Wire up a file handler so every protocol log line goes to the session log - fh = logging.FileHandler(log_path, encoding="utf-8") - fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s")) - root_logger = logging.getLogger() - root_logger.addHandler(fh) + # Session dir and file handler are created lazily — only after startup + # succeeds. This prevents internet scanners and dropped connections from + # littering the output directory with empty session folders. try: - self._run_inner(session_dir, raw_path, ts) + self._run_inner(ts) except Exception as exc: - log.error("Session failed: %s", exc, exc_info=True) + log.error("Session failed (%s): %s", self.peer, exc, exc_info=True) finally: - root_logger.removeHandler(fh) - fh.close() try: self.sock.close() except Exception: pass - def _run_inner(self, session_dir: Path, raw_path: Path, ts: str) -> None: - log.info("="*60) - log.info("Inbound connection from %s", self.peer) - log.info("Session dir: %s", session_dir) - + def _run_inner(self, ts: str) -> None: transport = SocketTransport(self.sock, peer=self.peer) - # Tap the transport: save every raw byte received from the device. - raw_fh = open(raw_path, "wb") + # Collect raw bytes in memory until startup succeeds, then flush to disk. + raw_buf: list[bytes] = [] _orig_read = transport.read def tapped_read(n: int) -> bytes: data = _orig_read(n) if data: - raw_fh.write(data) - raw_fh.flush() + raw_buf.append(data) return data transport.read = tapped_read # type: ignore[method-assign] serial: Optional[str] = None + # ── Step 1: startup handshake ───────────────────────────────────────── + # Do this BEFORE creating the session directory so that scanner probes + # and dropped connections leave no trace on disk. try: + from minimateplus.protocol import MiniMateProtocol client = MiniMateClient(transport=transport, timeout=self.timeout) client.open() + proto = MiniMateProtocol(transport, recv_timeout=self.timeout) + proto.startup() + except Exception as exc: + log.warning("Startup failed from %s: %s -- ignoring", self.peer, exc) + return # no session dir created - # ── Step 1: startup handshake ───────────────────────────────────── - log.info("Step 1/3: startup handshake (POLL / SUB 5B)") - try: - from minimateplus.protocol import MiniMateProtocol - proto = MiniMateProtocol(transport, recv_timeout=self.timeout) - proto.startup() - log.info(" [OK] Startup OK -- pull protocol confirmed") - except Exception as exc: - log.error(" [FAIL] Startup failed: %s", exc) - return + # Startup succeeded — this is a real unit. Create session dir now. + session_dir = self.output_dir / f"ach_inbound_{ts}" + session_dir.mkdir(parents=True, exist_ok=True) + log_path = session_dir / f"session_{ts}.log" + raw_path = session_dir / f"raw_rx_{ts}.bin" + # Flush buffered raw bytes to file and switch to direct file writes. + raw_fh = open(raw_path, "wb") + for chunk in raw_buf: + raw_fh.write(chunk) + raw_buf.clear() + + def tapped_read_file(n: int) -> bytes: + data = _orig_read(n) + if data: + raw_fh.write(data) + raw_fh.flush() + return data + + transport.read = tapped_read_file # type: ignore[method-assign] + + # Wire up file handler now that the session dir exists. + fh = logging.FileHandler(log_path, encoding="utf-8") + fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s")) + root_logger = logging.getLogger() + root_logger.addHandler(fh) + + try: # ── Step 2: device info ─────────────────────────────────────────── device_info = None if not self.events_only: @@ -292,6 +303,8 @@ class AchSession: finally: raw_fh.close() client.close() # closes transport / socket cleanly + root_logger.removeHandler(fh) + fh.close() log.info("Session complete -> %s", session_dir) log.info("="*60) @@ -376,11 +389,24 @@ def serve(args: argparse.Namespace) -> None: print(f" Remote Port: {args.port}") print(f"\n Waiting for inbound connections... (Ctrl-C to stop)\n") + allow_ips = set(args.allow_ips) + if allow_ips: + print(f" Allowlist: {', '.join(sorted(allow_ips))}") + else: + print(" Allowlist: NONE -- accepting all IPs (add --allow-ip to restrict)") + try: while True: try: client_sock, addr = server_sock.accept() + peer_ip = addr[0] peer = f"{addr[0]}:{addr[1]}" + + if allow_ips and peer_ip not in allow_ips: + log.info("Rejected connection from %s (not in allowlist)", peer) + client_sock.close() + continue + log.info("Accepted connection from %s", peer) session = AchSession( sock=client_sock, @@ -441,6 +467,18 @@ def parse_args() -> argparse.Namespace: "Useful if a unit has many old events stored — prevents a very long first run." ), ) + p.add_argument( + "--allow-ip", + metavar="IP", + action="append", + dest="allow_ips", + default=[], + help=( + "Only accept connections from this IP address (repeat for multiple). " + "Example: --allow-ip 63.43.212.232 " + "If not specified, all IPs are accepted (not recommended for public servers)." + ), + ) p.add_argument( "--verbose", "-v", action="store_true", From 9b50ec9133fc0bebafa9bf8aa91dbc5af03438ff Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 01:19:36 -0400 Subject: [PATCH 10/33] fix: make Ctrl-C work on Windows by setting accept() timeout socket.accept() on Windows blocks indefinitely and ignores KeyboardInterrupt. Setting a 1-second timeout on the server socket causes the accept loop to wake up every second and re-check, so Ctrl-C is handled within ~1 second. Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 012db05..235c306 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -376,6 +376,9 @@ def serve(args: argparse.Namespace) -> None: server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_sock.bind(("0.0.0.0", args.port)) server_sock.listen(5) + # Wake up every second so Ctrl-C is handled promptly on Windows. + # Without this, accept() blocks indefinitely and ignores KeyboardInterrupt. + server_sock.settimeout(1.0) max_ev = args.max_events print(f"\n{'='*60}") @@ -399,6 +402,9 @@ def serve(args: argparse.Namespace) -> None: while True: try: client_sock, addr = server_sock.accept() + except socket.timeout: + continue # no connection this second; loop back and check for Ctrl-C + try: peer_ip = addr[0] peer = f"{addr[0]}:{addr[1]}" From 8688d815a0b242ecc8bd92d3b5cb2dcbd685c8dd Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 01:43:02 -0400 Subject: [PATCH 11/33] fix: remove non-existent DeviceInfo fields from ach_server log and dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calibration_date, aux_trigger, setup_name etc. don't exist directly on DeviceInfo — they live in DeviceInfo.compliance_config (ComplianceConfig). _device_info_to_dict now accesses them via cc = d.compliance_config. Log line updated to show serial/firmware/model/event_count instead. Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 235c306..9f1d745 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -210,10 +210,11 @@ class AchSession: serial = device_info.serial _save_json(session_dir / "device_info.json", _device_info_to_dict(device_info)) log.info( - " [OK] Device: serial=%s firmware=%s calibration=%s", + " [OK] Device: serial=%s firmware=%s model=%s events=%d", serial, device_info.firmware_version, - device_info.calibration_date, + device_info.model, + device_info.event_count or 0, ) except Exception as exc: log.error(" [FAIL] Device info failed: %s", exc) @@ -319,21 +320,24 @@ def _save_json(path: Path, obj: object) -> None: def _device_info_to_dict(d: DeviceInfo) -> dict: + cc = d.compliance_config return { "serial": d.serial, "firmware_version": d.firmware_version, - "calibration_date": str(d.calibration_date) if d.calibration_date else None, - "aux_trigger": d.aux_trigger, - "setup_name": d.setup_name, - "sample_rate": d.sample_rate, - "record_time": d.record_time, - "trigger_level_geo": d.trigger_level_geo, - "alarm_level_geo": d.alarm_level_geo, - "max_range_geo": d.max_range_geo, - "project": d.project, - "client": d.client, - "operator": d.operator, - "sensor_location": d.sensor_location, + "dsp_version": d.dsp_version, + "model": d.model, + "event_count": d.event_count, + # compliance config fields (None if 1A read failed) + "setup_name": cc.setup_name if cc else None, + "sample_rate": cc.sample_rate if cc else None, + "record_time": cc.record_time if cc else None, + "trigger_level_geo": cc.trigger_level_geo if cc else None, + "alarm_level_geo": cc.alarm_level_geo if cc else None, + "max_range_geo": cc.max_range_geo if cc else None, + "project": cc.project if cc else None, + "client": cc.client if cc else None, + "operator": cc.operator if cc else None, + "sensor_location": cc.sensor_location if cc else None, } From 4921b0489aec99e16fcb86e214b7bf2602bb40a2 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Fri, 10 Apr 2026 02:09:57 -0400 Subject: [PATCH 12/33] fix: correct Event and PeakValues field names in ach_server serialization Event model uses peak_values (not peaks) and project_info (not direct fields). PeakValues fields are tran/vert/long/micl/peak_vector_sum (not transverse etc). ProjectInfo fields accessed via ev.project_info.project etc. Also fix ev.timestamp serialization: use str() instead of .isoformat() since Timestamp is a custom dataclass, not datetime. Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 9f1d745..ddbb053 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -279,14 +279,17 @@ class AchSession: log.info(" (skipped %d already-seen event(s))", last_count) for i, ev in enumerate(new_events): + pv = ev.peak_values + pi = ev.project_info log.info( - " NEW Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f", + " NEW Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f project=%r", last_count + i, - ev.timestamp.isoformat() if ev.timestamp else "?", - ev.peaks.transverse if ev.peaks else 0, - ev.peaks.vertical if ev.peaks else 0, - ev.peaks.longitudinal if ev.peaks else 0, - ev.peaks.vector_sum if ev.peaks else 0, + str(ev.timestamp) if ev.timestamp else "?", + pv.tran if pv else 0, + pv.vert if pv else 0, + pv.long if pv else 0, + pv.peak_vector_sum if pv else 0, + pi.project if pi else "", ) # Update high-water mark @@ -342,14 +345,16 @@ def _device_info_to_dict(d: DeviceInfo) -> dict: def _event_to_dict(e: Event) -> dict: + pv = e.peak_values + pi = e.project_info peaks = {} - if e.peaks: + if pv: peaks = { - "transverse": e.peaks.transverse, - "vertical": e.peaks.vertical, - "longitudinal": e.peaks.longitudinal, - "vector_sum": e.peaks.vector_sum, - "mic": e.peaks.mic, + "transverse": pv.tran, + "vertical": pv.vert, + "longitudinal": pv.long, + "vector_sum": pv.peak_vector_sum, + "mic": pv.micl, } samples = {} if e.raw_samples: @@ -359,11 +364,11 @@ def _event_to_dict(e: Event) -> dict: } samples["__note__"] = "first 20 sample-sets only; see raw_rx.bin for full waveform" return { - "timestamp": e.timestamp.isoformat() if e.timestamp else None, - "project": e.project, - "client": e.client, - "operator": e.operator, - "sensor_location": e.sensor_location, + "timestamp": str(e.timestamp) if e.timestamp else None, + "project": pi.project if pi else None, + "client": pi.client if pi else None, + "operator": pi.operator if pi else None, + "sensor_location": pi.sensor_location if pi else None, "peaks": peaks, "raw_samples_preview": samples, } From a3b8d10fa8287c18ac7bf25c7f8221ef2f9efcd4 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 01:14:37 -0400 Subject: [PATCH 13/33] feat: add erase-all protocol and browse helpers to protocol/client layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit protocol.py: - SUB_ERASE_ALL_BEGIN = 0xA3, SUB_ERASE_ALL_CONFIRM = 0xA2 (confirmed 4-11-26 MITM) - SUB_CHANNEL_CONFIG (0x06) data length = 0x24 (36 bytes) in DATA_LENGTHS - begin_erase_all() — single frame, token=0xFE, response 0x5C - confirm_erase_all() — single frame, token=0xFE, response 0x5D - read_event_storage_range() — two-step read (probe+data), token=0xFE Response last 8 bytes = first/last stored event key; both 0x01110000 after erase client.py: - list_event_keys() — browse-mode 1E→0A→1F walk, no waveform download; returns list of hex key strings; used as fast pre-check before get_events() - get_events(skip_waveform_for_keys=set()) — for already-seen keys: only 0A+1F(browse), skips 1E-arm/0C/POLL×3/5A entirely - delete_all_events() — orchestrates the confirmed erase sequence: 0xA3 → 0x1C → 0x06 → 0xA2; logs first/last key from storage range response Co-Authored-By: Claude Sonnet 4.6 --- minimateplus/client.py | 121 ++++++++++++++++++++++++++++++++++++++- minimateplus/protocol.py | 81 +++++++++++++++++++++++++- 2 files changed, 200 insertions(+), 2 deletions(-) diff --git a/minimateplus/client.py b/minimateplus/client.py index 47d4a20..022b0cc 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -252,7 +252,98 @@ class MiniMateClient: log.info("count_events: %d event(s) found via 1E/1F chain", count) return count - def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None) -> list[Event]: + def list_event_keys(self) -> list[str]: + """ + Return the hex key strings for all stored events without downloading + any waveform data. Uses the same browse-mode 1E -> 0A -> 1F walk as + count_events() but collects the key at each step. + + Returns: + List of 8-char lowercase hex strings, e.g. ["01110000", "0111245a"]. + Empty list if device has no stored events or 1E fails. + """ + proto = self._require_proto() + try: + key4, data8 = proto.read_event_first() + except ProtocolError as exc: + log.warning("list_event_keys: 1E failed: %s -- returning []", exc) + return [] + + if data8[4:8] == b"\x00\x00\x00\x00": + log.info("list_event_keys: device is empty") + return [] + + keys: list[str] = [] + while data8[4:8] != b"\x00\x00\x00\x00": + keys.append(key4.hex()) + try: + proto.read_waveform_header(key4) + except ProtocolError as exc: + log.warning( + "list_event_keys: 0A failed for key=%s: %s -- stopping", + key4.hex(), exc, + ) + break + try: + key4, data8 = proto.advance_event(browse=True) + log.debug( + "list_event_keys: 1F -> key=%s trailing=%s", + key4.hex(), data8[4:8].hex(), + ) + except ProtocolError as exc: + log.warning( + "list_event_keys: 1F failed after %d event(s): %s -- stopping", + len(keys), exc, + ) + break + + log.info("list_event_keys: %d key(s): %s", len(keys), keys) + return keys + + def delete_all_events(self) -> None: + """ + Erase all stored events from the device memory. + + This performs the complete erase sequence confirmed from the 4-11-26 + MITM capture of a Blastware ACH session: + + 1. SUB 0xA3 (begin_erase_all) — initiate erase, token=0xFE + 2. SUB 0x1C (read_monitor_status) — status read between erase commands + 3. SUB 0x06 (read_event_storage_range) — verify storage state, token=0xFE + 4. SUB 0xA2 (confirm_erase_all) — commit erase, token=0xFE + + After this call the device's event memory is empty. The unit returns to + its normal operating state automatically (no restart-monitoring call needed). + + Raises: + ProtocolError: on timeout or unexpected device response. + """ + proto = self._require_proto() + + log.info("delete_all_events: step 1/4 — begin erase (SUB 0xA3)") + proto.begin_erase_all() + log.debug("delete_all_events: 0xA3 ack received") + + log.info("delete_all_events: step 2/4 — monitor status read (SUB 0x1C)") + proto.read_monitor_status() + log.debug("delete_all_events: 0x1C read complete") + + log.info("delete_all_events: step 3/4 — event storage range read (SUB 0x06)") + rng = proto.read_event_storage_range() + if len(rng.data) >= 8: + first_key = rng.data[-8:-4].hex() + last_key = rng.data[-4:].hex() + log.info( + "delete_all_events: storage range — first=%s last=%s", + first_key, last_key, + ) + log.debug("delete_all_events: 0x06 read complete") + + log.info("delete_all_events: step 4/4 — confirm erase (SUB 0xA2)") + proto.confirm_erase_all() + log.info("delete_all_events: erase confirmed — device memory cleared") + + def get_events(self, full_waveform: bool = False, debug: bool = False, stop_after_index: Optional[int] = None, skip_waveform_for_keys: Optional[set] = None) -> list[Event]: """ Download all stored events from the device using the confirmed 1E → 0A → 0C → 5A → 1F event-iterator protocol. @@ -303,6 +394,34 @@ class MiniMateClient: while data8[4:8] != b"\x00\x00\x00\x00": cur_key = key4 # key for this event's 0A/1E-arm/0C/5A calls log.info("get_events: record %d key=%s", idx, cur_key.hex()) + + # Fast-advance path: if this key is already downloaded, skip + # 1E-arm/0C/POLL/5A entirely. Only 0A + 1F(browse) are needed + # to advance the device's internal pointer to the next event. + # This is identical to the browse-mode walk in count_events(). + if skip_waveform_for_keys and cur_key.hex() in skip_waveform_for_keys: + log.debug("get_events: key=%s already seen -- fast-advance only", cur_key.hex()) + try: + proto.read_waveform_header(cur_key) + except ProtocolError as exc: + log.warning( + "get_events: 0A failed for key=%s (skip path): %s -- stopping", + cur_key.hex(), exc, + ) + break + try: + key4, data8 = proto.advance_event(browse=True) + except ProtocolError as exc: + log.warning( + "get_events: 1F failed for key=%s (skip path): %s -- stopping", + cur_key.hex(), exc, + ) + break + idx += 1 + if stop_after_index is not None and idx > stop_after_index: + break + continue + ev = Event(index=idx) ev._waveform_key = cur_key diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 04b77d8..83afa6e 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -57,7 +57,7 @@ SUB_POLL = 0x5B SUB_SERIAL_NUMBER = 0x15 SUB_FULL_CONFIG = 0x01 SUB_EVENT_INDEX = 0x08 -SUB_CHANNEL_CONFIG = 0x06 +SUB_CHANNEL_CONFIG = 0x06 # Event storage range read (first/last key) ✅ SUB_MONITOR_STATUS = 0x1C # Monitoring status read (battery, memory, mode) ✅ SUB_EVENT_HEADER = 0x1E SUB_EVENT_ADVANCE = 0x1F @@ -82,6 +82,12 @@ SUB_TRIGGER_CONFIRM = 0x83 # Confirm trigger write ✅ SUB_START_MONITORING = 0x96 # Start monitoring → response 0x69 ✅ SUB_STOP_MONITORING = 0x97 # Stop monitoring → response 0x68 ✅ +# Erase-all SUBs (confirmed from 4-11-26 MITM capture) +# Both use token=0xFE at params[7] and return minimal 11-byte acks. +# Standard response formula applies: 0xFF - SUB. +SUB_ERASE_ALL_BEGIN = 0xA3 # Begin erase all events → response 0x5C ✅ +SUB_ERASE_ALL_CONFIRM = 0xA2 # Confirm erase all events → response 0x5D ✅ + # Hardcoded data lengths for the two-step read protocol. # # The S3 probe response page_key is always 0x0000 — it does NOT carry the @@ -96,6 +102,7 @@ DATA_LENGTHS: dict[int, int] = { SUB_SERIAL_NUMBER: 0x0A, # 10-byte serial number block ✅ SUB_FULL_CONFIG: 0x98, # 152-byte full config block ✅ SUB_EVENT_INDEX: 0x58, # 88-byte event index ✅ + SUB_CHANNEL_CONFIG: 0x24, # 36-byte event storage range (first/last key) ✅ SUB_MONITOR_STATUS: 0x2C, # 44-byte monitor status block (idle) ✅ SUB_EVENT_HEADER: 0x08, # 8-byte event header (waveform key + event data) ✅ SUB_EVENT_ADVANCE: 0x08, # 8-byte next-key response ✅ @@ -1137,6 +1144,78 @@ class MiniMateProtocol: self._send(frame) return self.recv_write_ack(expected_sub=rsp_sub) + def read_event_storage_range(self) -> S3Frame: + """ + Read event storage range (SUB 0x06 → response 0xF9). + + Two-step read: probe (offset=0x00) then data (offset=0x24 = 36 bytes). + Uses token=0xFE at params[7] — same as the erase sequence. + + The 36-byte response ends with two 4-byte event keys (first and last + stored event key). After a successful erase, both keys are 0x01110000 + (device-empty sentinel). Confirmed from 4-11-26 MITM capture. + + Returns: + S3Frame with 36 bytes of storage range data. + + Raises: + ProtocolError: on timeout or wrong response SUB. + """ + rsp_sub = _expected_rsp_sub(SUB_CHANNEL_CONFIG) # 0xFF - 0x06 = 0xF9 + params = token_params(0xFE) + log.debug("read_event_storage_range: probe step rsp_sub=0x%02X", rsp_sub) + self._send(build_bw_frame(SUB_CHANNEL_CONFIG, offset=0x00, params=params)) + self._recv_one(expected_sub=rsp_sub) + + log.debug( + "read_event_storage_range: data step offset=0x%02X", + DATA_LENGTHS[SUB_CHANNEL_CONFIG], + ) + self._send(build_bw_frame(SUB_CHANNEL_CONFIG, + offset=DATA_LENGTHS[SUB_CHANNEL_CONFIG], + params=params)) + return self._recv_one(expected_sub=rsp_sub) + + def begin_erase_all(self) -> S3Frame: + """ + Send Begin-Erase-All command (SUB 0xA3 → response 0x5C). + + Single frame with token=0xFE at params[7]. The device acknowledges with + a minimal ack and begins the erase process. Follow up with + read_monitor_status() + read_event_storage_range() + confirm_erase_all() + to complete the sequence. Confirmed from 4-11-26 MITM capture. + + Returns: + S3Frame ack from device (SUB 0x5C). + + Raises: + ProtocolError: on timeout or wrong response SUB. + """ + rsp_sub = _expected_rsp_sub(SUB_ERASE_ALL_BEGIN) # 0xFF - 0xA3 = 0x5C + log.debug("begin_erase_all: rsp_sub=0x%02X", rsp_sub) + self._send(build_bw_frame(SUB_ERASE_ALL_BEGIN, params=token_params(0xFE))) + return self._recv_one(expected_sub=rsp_sub) + + def confirm_erase_all(self) -> S3Frame: + """ + Send Confirm-Erase-All command (SUB 0xA2 → response 0x5D). + + Single frame with token=0xFE at params[7]. Must be preceded by + begin_erase_all() + read_monitor_status() + read_event_storage_range(). + After this call the device memory is cleared. Confirmed from 4-11-26 + MITM capture. + + Returns: + S3Frame ack from device (SUB 0x5D). + + Raises: + ProtocolError: on timeout or wrong response SUB. + """ + rsp_sub = _expected_rsp_sub(SUB_ERASE_ALL_CONFIRM) # 0xFF - 0xA2 = 0x5D + log.debug("confirm_erase_all: rsp_sub=0x%02X", rsp_sub) + self._send(build_bw_frame(SUB_ERASE_ALL_CONFIRM, params=token_params(0xFE))) + return self._recv_one(expected_sub=rsp_sub) + # ── Internal helpers ────────────────────────────────────────────────────── def _send(self, frame: bytes) -> None: From c7e7d177e6f422620b6a599b732a46270dcfcc96 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 01:14:50 -0400 Subject: [PATCH 14/33] feat: overhaul ACH server with key-based state, erase support, and reset detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State format (ach_state.json): - Replace event_count with downloaded_keys (set of hex strings) + max_downloaded_key - Key-based tracking correctly handles delete-then-re-record: after device erase the count drops to 0, but new events have new (or recycled) keys Browse pre-check: - list_event_keys() walk before get_events() to bail early when nothing is new - get_events() called with skip_waveform_for_keys= for already-seen keys, so repeat call-homes only download waveforms for genuinely new events --clear-after-download flag: - After saving new events, calls client.delete_all_events() (0xA3→0x1C→0x06→0xA2) - On success: resets downloaded_keys=[] and max_downloaded_key="00000000" so the next session starts fresh (device counter resets to 0x01110000 after erase) Post-erase key-reuse detection: - Device counter resets to 0x01110000 after any erase; new events reuse old keys - If max(device_keys) < max_downloaded_key, the device was wiped externally (Blastware, manual) — seen_keys is discarded and all device keys treated as new Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_server.py | 231 +++++++++++++++++++++++++++++++++--------- 1 file changed, 181 insertions(+), 50 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index ddbb053..c9e0302 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -71,9 +71,27 @@ from minimateplus.models import DeviceInfo, Event log = logging.getLogger("ach_server") -# ── Per-unit state (high-water mark) ────────────────────────────────────────── +# ── Per-unit state (downloaded-key set) ─────────────────────────────────────── # Persisted as /ach_state.json -# Format: { "BE11529": { "event_count": 5, "last_seen": "2026-04-09T..." }, ... } +# Format: +# { +# "BE11529": { +# "downloaded_keys": ["01110000", "0111245a"], # hex keys already on disk +# "max_downloaded_key": "0111245a", # highest key ever seen +# "last_seen": "2026-04-11T01:04:36" +# } +# } +# +# Key-based deduplication works well within a single "key generation" (between +# erases). After the device memory is erased the event counter resets to +# 0x01110000, so the first new event has the SAME key as the very first event +# we ever downloaded. We detect this situation with max_downloaded_key: +# +# if max(current_device_keys) < max_downloaded_key +# → device was wiped and keys have restarted → treat all device keys as new +# +# After our own erase (--clear-after-download) we also explicitly clear +# downloaded_keys and max_downloaded_key so the next session starts fresh. _state_lock = threading.Lock() @@ -103,10 +121,10 @@ class AchSession: standard connect → get_device_info → get_events sequence. State tracking (ach_state.json in output_dir): - On each successful download we record how many events the unit had. - On the next call-home we compare: if count hasn't grown, there's nothing - new and we close cleanly without downloading. If it has grown, we - download all events up to the new count and save only the new ones. + On each successful download we record the SET of event keys downloaded. + On the next call-home we compare: if all device keys are already in the + set, there's nothing new. If any key is new (including after the device + was wiped and re-recorded), we download and save only those events. """ def __init__( @@ -118,14 +136,16 @@ class AchSession: events_only: bool, max_events: Optional[int], state_path: Path, + clear_after_download: bool = False, ) -> None: - self.sock = sock - self.peer = peer - self.output_dir = output_dir - self.timeout = timeout - self.events_only = events_only - self.max_events = max_events - self.state_path = state_path + self.sock = sock + self.peer = peer + self.output_dir = output_dir + self.timeout = timeout + self.events_only = events_only + self.max_events = max_events + self.state_path = state_path + self.clear_after_download = clear_after_download def run(self) -> None: ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -221,18 +241,22 @@ class AchSession: else: log.info("Step 2/3: skipping device info (--events-only)") - # ── Step 3: check for new events via high-water mark ─────────────── + # ── Step 3: check for new events by comparing key sets ──────────── log.info("Step 3/3: checking for new events") state = _load_state(self.state_path) unit_key = serial or self.peer # fall back to IP if no serial - last_count = state.get(unit_key, {}).get("event_count", 0) + unit_state = state.get(unit_key, {}) + seen_keys: set[str] = set(unit_state.get("downloaded_keys", [])) + # Highest event key ever downloaded from this unit (hex string, 8 chars). + # Used to detect post-erase key reuse — see comment block above. + max_seen_key: str = unit_state.get("max_downloaded_key", "00000000") # Use the event count already read from the event index during connect(). # This is fast (no extra round-trips) and confirmed accurate (matches LCD). # Falls back to count_events() only if connect() wasn't called. if device_info is not None: - current_count = device_info.event_count + current_count = device_info.event_count or 0 else: try: current_count = client.count_events() @@ -240,26 +264,70 @@ class AchSession: log.error(" [FAIL] count_events failed: %s", exc) return - log.info(" Unit has %d stored event(s); last downloaded count: %d", - current_count, last_count) + log.info(" Unit has %d stored event(s); %d key(s) previously downloaded", + current_count, len(seen_keys)) - if current_count <= last_count: - log.info(" [OK] No new events since last call-home -- nothing to download") - log.info("Session complete (no new events) -> %s", session_dir) + if current_count == 0: + log.info(" [OK] No events on device -- nothing to download") + log.info("Session complete (no events) -> %s", session_dir) return - new_event_count = current_count - last_count - log.info(" %d new event(s) to download", new_event_count) + # Fast pre-check: walk the event index (browse-mode, no 5A) to get + # the current key list, then bail early if everything is already seen. + # This avoids calling get_events() at all when there's nothing new. + log.info(" Checking device key list (browse walk, no waveform download)...") + try: + device_keys = client.list_event_keys() + except Exception as exc: + log.warning(" list_event_keys failed: %s -- falling back to full download", exc) + device_keys = None - # Download all events up to current_count, apply max_events cap. - # We re-download old events too (get_events always starts from 0), - # but we only SAVE the new ones (the last new_event_count of the list). + if device_keys is not None: + # ── Post-erase detection ────────────────────────────────────── + # After the device memory is erased, new events start from key + # 01110000 again — the same keys we already downloaded. Detect + # this by comparing the device's current highest key against the + # historical maximum. If the device has rolled back below our + # high-water mark, its counter was reset and we must treat all + # its keys as new, regardless of what seen_keys contains. + if device_keys and max_seen_key != "00000000": + max_device_key = max(device_keys) # lexicographic; safe because + # keys share the same 4-char prefix + if max_device_key < max_seen_key: + log.info( + " Post-erase reset detected: " + "device max key %s < historical max %s " + "-- treating all device keys as new", + max_device_key, max_seen_key, + ) + seen_keys = set() # discard stale dedup info for this session + + new_key_set = set(device_keys) - seen_keys + log.info(" Device has %d key(s): %d new, %d already seen", + len(device_keys), len(new_key_set), len(device_keys) - len(new_key_set)) + if not new_key_set: + log.info(" [OK] All events already downloaded -- nothing to do") + # Refresh state timestamp; preserve max_seen_key unchanged. + state[unit_key] = { + "downloaded_keys": sorted(seen_keys | set(device_keys)), + "max_downloaded_key": max_seen_key, + "last_seen": datetime.datetime.now().isoformat(), + "serial": serial, + "peer": self.peer, + } + _save_state(self.state_path, state) + log.info("Session complete (no new events) -> %s", session_dir) + return + else: + new_key_set = None # unknown; proceed with full download + + # Apply max_events cap stop_idx = current_count - 1 if self.max_events is not None: stop_idx = min(stop_idx, self.max_events - 1) if self.max_events < current_count: log.warning( - " max_events=%d cap: will download events 0–%d only " + " max_events=%d cap: will download events 0-%d only " "(unit has %d total)", self.max_events, stop_idx, current_count, ) @@ -268,36 +336,86 @@ class AchSession: all_events = client.get_events( full_waveform=True, stop_after_index=stop_idx, + skip_waveform_for_keys=seen_keys if seen_keys else None, ) - # Only the events beyond last_count are genuinely new - new_events = all_events[last_count:] - log.info(" [OK] Downloaded %d total event(s), %d new", - len(all_events), len(new_events)) - _save_json(session_dir / "events.json", [_event_to_dict(e) for e in new_events]) - if last_count > 0 and len(all_events) > len(new_events): - log.info(" (skipped %d already-seen event(s))", last_count) + # Filter to events whose keys we haven't saved before. + new_events = [ + e for e in all_events + if e._waveform_key is None + or e._waveform_key.hex() not in seen_keys + ] + skipped = len(all_events) - len(new_events) - for i, ev in enumerate(new_events): - pv = ev.peak_values - pi = ev.project_info + log.info(" [OK] Downloaded %d event(s): %d new, %d skipped (already seen)", + len(all_events), len(new_events), skipped) + if skipped: + log.info(" (skipped %d already-downloaded event(s))", skipped) + + if new_events: + _save_json(session_dir / "events.json", [_event_to_dict(e) for e in new_events]) + + for ev in new_events: + pv = ev.peak_values + pi = ev.project_info + key_hex = ev._waveform_key.hex() if ev._waveform_key else "????????" + log.info( + " NEW [%s] %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f project=%r", + key_hex, + str(ev.timestamp) if ev.timestamp else "?", + pv.tran if pv else 0, + pv.vert if pv else 0, + pv.long if pv else 0, + pv.peak_vector_sum if pv else 0, + pi.project if pi else "", + ) + else: + log.info(" [OK] No new events since last call-home -- nothing to save") + + # ── Optional: erase device memory after successful download ──── + erased_successfully = False + if self.clear_after_download and new_events: + log.info(" Clearing device memory (--clear-after-download)...") + try: + client.delete_all_events() + log.info(" [OK] Device memory cleared") + erased_successfully = True + except Exception as exc: + log.error( + " [WARN] Event deletion failed: %s -- events NOT cleared", + exc, + ) + + # ── Update persistent state ─────────────────────────────────── + current_keys = [ + e._waveform_key.hex() + for e in all_events + if e._waveform_key is not None + ] + + if erased_successfully: + # Device memory is clear. Reset downloaded_keys and the + # high-water mark so the next call-home starts fresh and + # doesn't mis-identify the recycled key 01110000 as "seen". + updated_keys = [] + new_max_key = "00000000" log.info( - " NEW Event %d: %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f project=%r", - last_count + i, - str(ev.timestamp) if ev.timestamp else "?", - pv.tran if pv else 0, - pv.vert if pv else 0, - pv.long if pv else 0, - pv.peak_vector_sum if pv else 0, - pi.project if pi else "", + " State reset after erase -- next session will download " + "from key 0 (device counter resets after erase)" ) + else: + # Normal (no erase): union of previously-seen + all keys on + # device now. Includes already-seen survivors so we never + # re-download them if the device somehow keeps old records. + updated_keys = sorted(set(seen_keys) | set(current_keys)) + new_max_key = updated_keys[-1] if updated_keys else max_seen_key - # Update high-water mark state[unit_key] = { - "event_count": current_count, - "last_seen": datetime.datetime.now().isoformat(), - "serial": serial, - "peer": self.peer, + "downloaded_keys": updated_keys, + "max_downloaded_key": new_max_key, + "last_seen": datetime.datetime.now().isoformat(), + "serial": serial, + "peer": self.peer, } _save_state(self.state_path, state) @@ -395,6 +513,7 @@ def serve(args: argparse.Namespace) -> None: print(f" Output: {output_dir.resolve()}/ach_inbound_/") print(f" State file: {state_path}") print(f" Max events per session: {max_ev if max_ev else 'unlimited'}") + print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}") print(f"{'='*60}") print(f"\n Point your test unit's ACEmanager call-home settings to:") print(f" Remote Host: ") @@ -431,6 +550,7 @@ def serve(args: argparse.Namespace) -> None: events_only=args.events_only, max_events=max_ev, state_path=state_path, + clear_after_download=args.clear_after_download, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") t.start() @@ -494,6 +614,17 @@ def parse_args() -> argparse.Namespace: "If not specified, all IPs are accepted (not recommended for public servers)." ), ) + p.add_argument( + "--clear-after-download", + action="store_true", + default=False, + help=( + "After successfully downloading new events, erase all events from the " + "device memory (SUB 0xA3 → 0x1C → 0x06 → 0xA2 sequence, confirmed from " + "4-11-26 MITM capture). Only fires when at least one new event was saved. " + "This mirrors the standard Blastware ACH workflow." + ), + ) p.add_argument( "--verbose", "-v", action="store_true", From 3d9db8b6621ff2dcd51d12908c4e48417a0a2a0b Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 01:15:02 -0400 Subject: [PATCH 15/33] =?UTF-8?q?feat:=20add=20ach=5Fmitm.py=20=E2=80=94?= =?UTF-8?q?=20transparent=20TCP=20MITM=20proxy=20for=20ACH=20session=20cap?= =?UTF-8?q?ture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Listens for inbound unit connections, connects upstream to a real Blastware ACH server, and forwards bytes bidirectionally while saving both directions to raw_bw_.bin and raw_s3_.bin in the existing capture format. Used to capture the 4-11-26 Blastware ACH session that confirmed the erase-all protocol (SUBs 0xA3/0x1C/0x06/0xA2) and the event deletion wire sequence. Usage: python bridges/ach_mitm.py --bw-host 127.0.0.1 --bw-port 9999 --listen-port 9998 Point the unit's call-home destination at this machine:9998. Point this proxy's --bw-host/port at the upstream Blastware ACH server. Co-Authored-By: Claude Sonnet 4.6 --- bridges/ach_mitm.py | 177 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 bridges/ach_mitm.py diff --git a/bridges/ach_mitm.py b/bridges/ach_mitm.py new file mode 100644 index 0000000..bb89914 --- /dev/null +++ b/bridges/ach_mitm.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +ach_mitm.py — TCP man-in-the-middle proxy for capturing Blastware ACH sessions. + +The unit calls home to THIS proxy instead of directly to Blastware. The proxy +forwards every byte in both directions to the real Blastware ACH server and saves +the traffic to separate raw capture files that the Analyzer can load directly. + +Setup +----- + 1. Start Blastware's ACH server on the BW PC as normal (it listens on its port). + 2. Run this proxy on any machine the unit can reach: + + python bridges/ach_mitm.py --bw-host 192.168.1.50 --bw-port 9999 + + 3. Point the unit's ACEmanager call-home destination to THIS machine's IP and + the --listen-port (default 9999). + 4. Trigger a call-home (or wait for the unit to call in). + 5. The proxy transparently forwards everything and saves two files per session: + + ach_mitm_/raw_bw_.bin -- bytes Blastware sent to unit (BW TX) + ach_mitm_/raw_s3_.bin -- bytes unit sent to Blastware (S3 TX) + + Both files load directly in the Analyzer (File > Open Capture). + + The proxy exits cleanly when either side drops the connection. + +Use case: capturing Blastware operations we haven't reverse-engineered yet, +e.g. event deletion, factory reset, firmware update. +""" + +from __future__ import annotations + +import argparse +import datetime +import logging +import socket +import sys +import threading +from pathlib import Path + +log = logging.getLogger("ach_mitm") + + +def _pipe(src: socket.socket, dst: socket.socket, label: str, outfile) -> None: + """Forward bytes from src to dst, writing everything to outfile.""" + try: + while True: + data = src.recv(4096) + if not data: + break + dst.sendall(data) + outfile.write(data) + outfile.flush() + log.debug("%s %d bytes", label, len(data)) + except OSError: + pass + finally: + log.info("%s pipe closed", label) + # Signal the other direction to stop by shutting down our end. + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + + +def handle(unit_sock: socket.socket, peer: str, bw_host: str, bw_port: int, + output_dir: Path) -> None: + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + session_dir = output_dir / f"ach_mitm_{ts}" + session_dir.mkdir(parents=True, exist_ok=True) + + log.info("Session %s unit=%s forwarding to %s:%d", ts, peer, bw_host, bw_port) + + # Connect upstream to Blastware. + bw_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + bw_sock.connect((bw_host, bw_port)) + except OSError as exc: + log.error("Cannot reach Blastware at %s:%d: %s", bw_host, bw_port, exc) + unit_sock.close() + return + + log.info("Connected to Blastware at %s:%d", bw_host, bw_port) + + bw_path = session_dir / f"raw_bw_{ts}.bin" # Blastware → unit (BW TX) + s3_path = session_dir / f"raw_s3_{ts}.bin" # unit → Blastware (S3 TX) + + with open(bw_path, "wb") as bw_fh, open(s3_path, "wb") as s3_fh: + # Two threads: one per direction. + t_bw = threading.Thread( + target=_pipe, args=(bw_sock, unit_sock, "BW->unit", bw_fh), daemon=True + ) + t_s3 = threading.Thread( + target=_pipe, args=(unit_sock, bw_sock, "unit->BW", s3_fh), daemon=True + ) + t_bw.start() + t_s3.start() + t_bw.join() + t_s3.join() + + bw_bytes = bw_path.stat().st_size + s3_bytes = s3_path.stat().st_size + log.info( + "Session %s done BW->unit: %d bytes unit->BW: %d bytes -> %s", + ts, bw_bytes, s3_bytes, session_dir, + ) + + unit_sock.close() + bw_sock.close() + + +def serve(args: argparse.Namespace) -> None: + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", args.listen_port)) + server.listen(5) + server.settimeout(1.0) + + print(f"\n{'='*60}") + print(f" ACH MITM proxy") + print(f" Listening on 0.0.0.0:{args.listen_port}") + print(f" Forwarding to {args.bw_host}:{args.bw_port}") + print(f" Captures in {output_dir.resolve()}/ach_mitm_/") + print(f"{'='*60}") + print(f"\n Point the unit's ACEmanager call-home to this machine on port {args.listen_port}") + print(f" Ctrl-C to stop\n") + + try: + while True: + try: + client_sock, addr = server.accept() + except socket.timeout: + continue + peer = f"{addr[0]}:{addr[1]}" + log.info("Accepted connection from %s", peer) + t = threading.Thread( + target=handle, + args=(client_sock, peer, args.bw_host, args.bw_port, output_dir), + daemon=True, + ) + t.start() + except KeyboardInterrupt: + print("\nStopping.") + finally: + server.close() + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bw-host", required=True, + help="IP or hostname of the Blastware ACH server") + ap.add_argument("--bw-port", type=int, default=9999, + help="Port Blastware is listening on (default: 9999)") + ap.add_argument("--listen-port", type=int, default=9999, + help="Port this proxy listens on (default: 9999)") + ap.add_argument("--output", default="bridges/captures/mitm", + help="Directory for capture files") + ap.add_argument("--log-level", default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"]) + args = ap.parse_args() + + logging.basicConfig( + level=getattr(logging, args.log_level), + format="%(asctime)s %(levelname)-7s %(name)s %(message)s", + stream=sys.stdout, + ) + + serve(args) + + +if __name__ == "__main__": + main() From f6a0846baba9e4c3d28de9ce373ffcf22b8e468a Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 01:15:11 -0400 Subject: [PATCH 16/33] docs: update CHANGELOG and CLAUDE.md for v0.9.0 CHANGELOG.md: - New v0.9.0 section covering erase-all protocol, browse helpers, delete_all_events(), ach_mitm.py, and ACH server overhaul - Back-filled v0.8.0 section (write pipeline, monitoring, ACH server) that was missing from the previous release notes CLAUDE.md: - Bump version to v0.9.0 - Add erase-all protocol section with full wire sequence, SUB 0x06 storage range response layout, and post-erase key counter reset notes - Document ACH server state format (ach_state.json v0.9.0 schema with downloaded_keys + max_downloaded_key) - Add RV55 DCD/DTR issue to What's next Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 86 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 182 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2abb3f5..ca169d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,107 @@ All notable changes to seismo-relay are documented here. --- +## v0.9.0 — 2026-04-11 + +### Added + +- **`MiniMateClient.list_event_keys()`** — fast browse-mode walk (1E → 0A → 1F, no waveform + download) that returns the list of event key hex strings currently stored on the device. + Used by the ACH server as a cheap pre-check before deciding whether to call `get_events()`. + +- **`get_events(skip_waveform_for_keys=set(...))`** — new optional parameter. For any key in + the set the function performs only 0A + 1F(browse) instead of the full + 1E-arm → 0C → POLL×3 → 5A sequence. Eliminates redundant waveform downloads on repeat + call-homes when the device still holds previously downloaded events. + +- **`MiniMateClient.delete_all_events()`** — erases all events from device memory using the + confirmed 4-step sequence: + - SUB 0xA3 `begin_erase_all` — initiate erase (token=0xFE) → ack 0x5C + - SUB 0x1C `read_monitor_status` — intermediate status read (Blastware-required) + - SUB 0x06 `read_event_storage_range` — verify storage state (token=0xFE) → 36-byte response + - SUB 0xA2 `confirm_erase_all` — commit erase (token=0xFE) → ack 0x5D + + All four steps confirmed from 4-11-26 MITM capture of a live Blastware ACH session. + After a successful call, the device's event counter resets to `0x01110000`. + +- **`MiniMateProtocol` erase methods**: `begin_erase_all()`, `confirm_erase_all()`, + `read_event_storage_range()` added to `protocol.py` with documented SUB constants + `SUB_ERASE_ALL_BEGIN = 0xA3` and `SUB_ERASE_ALL_CONFIRM = 0xA2`. + +- **`bridges/ach_mitm.py`** — transparent TCP-to-TCP MITM proxy. Listens for inbound unit + connections, connects upstream to a real Blastware ACH server, and saves both directions + to `raw_bw_.bin` / `raw_s3_.bin` files matching the existing capture format. + Used to capture the 4-11-26 Blastware ACH session including event deletion. + Usage: `python bridges/ach_mitm.py --bw-host 127.0.0.1 --bw-port 9999 --listen-port 9998` + +- **ACH server: key-based state tracking** — `ach_state.json` now stores + `downloaded_keys: [hex_strings]` and `max_downloaded_key: hex_string` per unit instead of + `event_count: N`. This correctly handles the standard workflow where events are deleted + from the device after upload — a count-based approach would see `count=0` on the next + call-home and silently skip new events. + +- **ACH server: `--clear-after-download` flag** — after a successful download (at least one + new event saved), erases all events from the device using `delete_all_events()`. Mirrors + the standard Blastware ACH workflow. On success, `downloaded_keys` and + `max_downloaded_key` are reset to empty so the next session starts fresh. + +- **ACH server: post-erase key-reuse detection** — after an external erase (Blastware or + manual), device keys restart from `0x01110000`, colliding with previously downloaded keys. + On each browse walk, if `max(device_keys) < max_downloaded_key` (device counter rolled + back), all device keys are treated as new regardless of `seen_keys`. This also catches + erases performed by Blastware between our sessions. + +### Protocol / Documentation + +- **SUB 0xA3 / SUB 0xA2 — erase-all sequence confirmed** (✅ 4-11-26 MITM capture): + Both frames use `token=0xFE` at `params[7]` and are standard `build_bw_frame` requests + (not write-format). Response SUBs follow the standard formula: 0x5C and 0x5D. + The intermediate 0x1C + 0x06 reads between them are required by Blastware. + +- **SUB 0x06 — event storage range read confirmed** (✅ 4-11-26 MITM capture): + Two-step read, data offset = 0x24 (36 bytes). The last 8 bytes of the response contain + the first and last stored event keys (4 bytes each). After a successful erase, both keys + read as `01110000` (device-empty state). + +- **Event key counter resets to `0x01110000` after erase** — confirmed by observing key + `01110000` on the device immediately after the MITM erase session. + +--- + +## v0.8.0 — 2026-04-07 + +### Added + +- **Write pipeline end-to-end** — `push_config_raw(event_index_data, compliance_data, + trigger_data, waveform_data)` on `MiniMateClient` orchestrates the full + `68→73 | 71×3→72 | 82→83 | 69→74→72` write sequence. + +- **`build_bw_write_frame(sub, data, *, offset, params)`** in `framing.py` — dedicated frame + builder for write commands (SUBs 0x68–0x83). Doubles only the BW_CMD byte; all other + bytes including offset, params, data, and checksum are written raw. Uses the large-frame + DLE-aware checksum (`sum(b for b in payload[2:] if b != 0x10) + 0x10) & 0xFF`). + +- **`MiniMateProtocol` write methods** — `write_event_index()`, `write_compliance()`, + `write_trigger_config()`, `write_waveform_data()`, `write_confirm()`, + `start_monitoring()`, `stop_monitoring()`. + +- **`AchSession` inbound server** (`bridges/ach_server.py`) — accepts call-home TCP + connections, runs the full handshake + device-info + event-download sequence, saves + `device_info.json` + `events.json` per session. + +### Protocol / Documentation + +- **Write frame format confirmed** (✅ 3-11-26 BW TX capture, all 11 frames): only BW_CMD + byte `0x10` is doubled; all other bytes sent raw. Standard `build_bw_frame` DLE-stuffing + is incorrect for write commands. +- **Write ack responses** confirmed as 17-byte zero-data S3 frames. +- **Monitoring SUBs 0x96/0x97** confirmed from 4-8-26 capture. +- **SESSION_RESET signal** (`41 03`) required before POLL for monitoring units. +- **SUB 0x1C monitoring flag** at `section[1]`: `0x00` = idle, `0x10` = monitoring. + Confirmed by byte-diff of all 144 data frames in 4-8-26/2ndtry capture. + +--- + ## v0.7.0 — 2026-04-03 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index b6faa1c..6dafd73 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.8.0**. +(Sierra Wireless RV50 / RV55). Current version: **v0.9.0**. --- @@ -25,9 +25,9 @@ CHANGELOG.md ← version history --- -## Current implementation state (v0.8.0) +## Current implementation state (v0.9.0) -Full read pipeline + write pipeline working end-to-end over TCP/cellular: +Full read pipeline + write pipeline + erase pipeline working end-to-end over TCP/cellular: | Step | SUB | Status | |---|---|---| @@ -41,12 +41,15 @@ Full read pipeline + write pipeline working end-to-end over TCP/cellular: | Waveform record (peaks, timestamp, project) | 0C | ✅ | | **Bulk waveform stream (event-time metadata)** | **5A** | ✅ new v0.6.0 | | Event advance / next key | 1F | ✅ | -| **Write commands (push config to device)** | **68–83** | ✅ **new v0.8.0** | +| **Write commands (push config to device)** | **68–83** | ✅ new v0.8.0 | +| **Erase all events** | **0xA3 → 0x1C → 0x06 → 0xA2** | ✅ **new v0.9.0** | `get_events()` sequence per event: `1E → 0A → 0C → 5A → 1F` `push_config_raw()` write sequence: `68→73 | 71×3→72 | 82→83 | 69→74→72` +`delete_all_events()` erase sequence: `0xA3 → 0x1C → 0x06 → 0xA2` + --- ## Protocol fundamentals @@ -720,9 +723,82 @@ Full compliance config encoder is a future task. --- +--- + +## Erase-all protocol (SUBs 0xA3/0xA2/0x06) — confirmed 2026-04-11 + +Full sequence confirmed from 4-11-26 MITM capture of a live Blastware ACH session +(`bridges/captures/mitm/ach_mitm_20260411_001912/`). + +### Wire sequence + +``` +BW → device: SUB 0xA3 params=00 00 00 00 00 00 00 FE 00 00 (begin erase) +device → BW: SUB 0x5C (ack) +BW → device: SUB 0x1C probe (offset=0x00) +device → BW: SUB 0xE3 (probe ack) +BW → device: SUB 0x1C data (offset=0x2C) +device → BW: SUB 0xE3 (monitor status response) +BW → device: SUB 0x06 probe (offset=0x00, params same) +device → BW: SUB 0xF9 (probe ack) +BW → device: SUB 0x06 data (offset=0x24) +device → BW: SUB 0xF9 (36-byte storage range response) +BW → device: SUB 0xA2 params=00 00 00 00 00 00 00 FE 00 00 (confirm erase) +device → BW: SUB 0x5D (ack — device memory is now cleared) +``` + +All frames use standard `build_bw_frame` (not write-format). Response SUBs follow the +standard `0xFF - SUB` formula; no exceptions. + +### SUB 0x06 — event storage range response (36 bytes) + +The 36-byte response body ends with two 4-byte event keys: + +| Offset (from end) | Field | Notes | +|---|---|---| +| `[-8:-4]` | first stored event key | `01110000` when empty | +| `[-4:]` | last stored event key | `01110000` when empty | + +Before erase: ends with ` ` (e.g. `0111ea60 0111eaa6`). +After erase: both bytes read `01110000` — device's empty/reset sentinel. + +### Post-erase key counter reset + +After a successful erase, the device resets its event counter. New events start from +key `0x01110000` again — the same key as the very first event ever recorded. This means +key-based deduplication in the ACH server must account for key reuse: + +- After our own erase: `ach_state.json` `downloaded_keys` and `max_downloaded_key` are + cleared so the next session starts fresh. +- After an external erase: the ACH server detects it by comparing `max(device_keys)` to + `max_downloaded_key` from state. If the device max has rolled back below the historical + max, all current device keys are treated as new regardless of `seen_keys`. + +### ACH server state format (v0.9.0) + +`bridges/captures/ach_state.json`: +```json +{ + "BE11529": { + "downloaded_keys": ["01110000", "0111245a"], + "max_downloaded_key": "0111245a", + "last_seen": "2026-04-11T01:04:36", + "serial": "BE11529", + "peer": "63.43.212.232:51920" + } +} +``` + +`max_downloaded_key` is the high-water mark — the largest key ever downloaded from the +unit. It is NOT reset when events are erased from the device (only when our server does +the erase). Used for post-erase detection. + +--- + ## What's next - Compliance config encoder — build raw write payloads from a `ComplianceConfig` object - Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring) -- ACH inbound server — accept call-home connections from field units - Modem manager — push RV50/RV55 configs via Sierra Wireless API +- RV55 DCD/DTR issue — newer RV55 firmware doesn't assert DCD by default; units don't + resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred) From 6acb419ebd751e4caf79330ee0421af5052ec0e0 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 01:20:43 -0400 Subject: [PATCH 17/33] docs: update protocol reference with v0.9.0 erase-all protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changelog section: - 5 new entries (2026-04-11): erase-all confirmation, SUB 0x06 purpose resolved, §7.11 added, §14.6 ACH session lifecycle marked IMPLEMENTED §5.1 Request Commands: - SUB 0x06 description updated: "EVENT STORAGE RANGE READ" (not "CHANNEL CONFIG READ"), token=0xFE, last 8 bytes = first/last stored event keys - SUB 0xA3 added: ERASE ALL BEGIN — standard build_bw_frame, token=0xFE, ack 0x5C - SUB 0xA2 added: ERASE ALL CONFIRM — standard build_bw_frame, token=0xFE, ack 0x5D §5.2 Response SUBs: - 0x06→0xF9 marked CONFIRMED 2026-04-11 - 0xA3→0x5C and 0xA2→0x5D added with CONFIRMED status §7.11 (new section): Erase-All Protocol - Full wire sequence (6 request/response pairs) - SUB 0x06 storage range payload layout (36 bytes, last 8 = first/last key) - Post-erase key counter reset: device restarts from 0x01110000 - Implementation notes pointing to client.py and ach_server.py §14.6 ACH Session Lifecycle: - Removed "Future" label — fully implemented in bridges/ach_server.py - Added step 6 (optional erase), step 8 (DCD/DTR auto-resume) - Documents ach_server.py flags and ach_state.json schema - Notes RV55 DCD/DTR issue as known open problem Open Questions table: - SUB 0x06 purpose RESOLVED - Erase-all sequence RESOLVED - ACH server RESOLVED - Sensor Check byte: still open, added as formal question - RV55 DCD/DTR: added as new open question Co-Authored-By: Claude Sonnet 4.6 --- docs/instantel_protocol_reference.md | 114 +++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index e20c15f..356cb07 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -99,6 +99,10 @@ | 2026-04-08 | §7.10 | **NEW — SUBs 0x15 and 0x01 observed in sensor-check capture.** SUB 0x15 (serial number short form, data length 0x0A, RSP 0xEA) and SUB 0x01 (device info block, data length 0x98 = 152 bytes, RSP 0xFE) seen in Blastware's "Unit Channel Test" init sequence. Note: SUB 0x01 response SUB 0xFE collides with the existing SUB 0xFE → RSP 0x01 naming convention — they are inverse commands. | | 2026-04-08 | §12 | **CONFIRMED — Unit partially reachable during on-device sensor check.** 4-8-26/sensor-check capture shows: POLL responds normally throughout; SUB 0x0E channel reads partially served (channels 0–4 responded), then ~40s silent gap while sensor check ran, then channels 5–7 responded. On-device sensor check duration ≈ 40 s. SFM `_pollMonitorConfirm()` polls status every 5 s for up to 60 s after start_monitoring. | | 2026-04-08 | §7.9 (NEW) | **NEW — Compliance config field inventory captured from Blastware UI.** See §7.9 for full field list (Recording Setup, Notes, Special Setups tabs). Most fields NOT yet mapped to raw byte offsets. Confirmed decoded: sample_rate, record_time, trigger_level_geo, alarm_level_geo, max_range_geo, backlight_on_time, power_saving_timeout, monitoring_lcd_cycle, project/client/operator/sensor_location/notes. Sensor Check dropdown (Before monitoring / After each event / Disabled) NOT YET LOCATED in raw config bytes. | +| 2026-04-11 | §5.1, §5.2 | **NEW — Erase-all command sequence confirmed from MITM capture.** SUB 0xA3 (begin erase, token=0xFE → ack 0x5C) + SUB 0xA2 (confirm erase, token=0xFE → ack 0x5D). Standard `build_bw_frame` format (not write-format). Required intermediate steps: 0x1C probe+data (monitor status read) + 0x06 probe+data (event storage range). All response SUBs follow the standard 0xFF−SUB formula with no exceptions. | +| 2026-04-11 | §5.1 | **CONFIRMED — SUB 0x06 (CHANNEL CONFIG READ) now confirmed as event storage range.** Two-step read, data offset = 0x24 (36 bytes). Token=0xFE at params[7]. Last 8 bytes of response: first stored event key (bytes −8:−4) and last stored event key (bytes −4:). Both equal `01110000` when device memory is empty. Used by Blastware to verify erase completion. | +| 2026-04-11 | §7.11 (NEW) | **NEW — §7.11 Erase-All Protocol added.** Full wire sequence, SUB 0x06 storage range payload layout, post-erase key counter reset (resets to `0x01110000`). Confirmed from 4-11-26 MITM capture of live Blastware ACH session. | +| 2026-04-11 | §14.6 | **RESOLVED — ACH Session Lifecycle is no longer "Future".** `bridges/ach_server.py` fully implements inbound ACH: POLL handshake, device info, event download. State tracked via `ach_state.json` (key-based, with `max_downloaded_key` for post-erase detection). `--clear-after-download` flag added for the standard delete-after-upload workflow. | --- @@ -243,7 +247,7 @@ Step 4 — Device sends actual data payload: | `15` | **SERIAL NUMBER REQUEST** | Requests device serial number. | ✅ CONFIRMED | | `01` | **FULL CONFIG READ** | Requests complete device configuration block (~0x98 bytes). Firmware, model, serial, channel config, scaling factors. | ✅ CONFIRMED | | `08` | **EVENT INDEX READ** | Requests the event record index (0x58 bytes). Event count and record pointers. | ✅ CONFIRMED | -| `06` | **CHANNEL CONFIG READ** | Requests channel configuration block (0x24 bytes). | ✅ CONFIRMED | +| `06` | **EVENT STORAGE RANGE READ** | Requests event storage range block (0x24 = 36 bytes). Token=0xFE at params[7]. Last 8 bytes of response: first stored event key (`[-8:-4]`) and last stored event key (`[-4:]`). Both equal `01110000` when device is empty. Used by Blastware as part of the erase-all verification step. Previously labelled "CHANNEL CONFIG READ" — function now confirmed from 4-11-26 MITM capture. | ✅ CONFIRMED 2026-04-11 | | `1C` | **TRIGGER CONFIG READ** | Requests trigger settings block (0x2C bytes). | ✅ CONFIRMED | | `1E` | **EVENT HEADER READ** | Gets first waveform key. Token byte at params[7] (0x00=browse, 0xFE=download-arm). Key at data[11:15]; trailing offset at data[15:19] (0 = only one event). Two uses: (1) all-zero to get key0; (2) token=0xFE after 0A, before 0C — REQUIRED to arm device for SUB 5A. | ✅ CONFIRMED 2026-04-06 | | `0A` | **WAVEFORM HEADER READ** | Checks record type for a given waveform key. Variable DATA_LENGTH: 0x30=full bin, 0x26=partial bin. Key at params[4..7]. Required before every 1F call to establish device waveform context. | ✅ CONFIRMED 2026-03-31 | @@ -260,6 +264,8 @@ Step 4 — Device sends actual data payload: | `1C` | **MONITOR STATUS READ** | Two-step read, data offset 0x2C (44 bytes). `section[1] == 0x10` → monitoring; `0x00` → idle (CONFIRMED 2026-04-09, 100% accuracy on 144 frames). Payload length: 46–47 bytes IDLE, 48–49 bytes MONITORING. `frame.data` has checksum stripped — no trailing byte to skip. Battery/memory at end: `section[-10:-8]` = battery×100 (uint16 BE), `section[-8:-4]` = memory_total (uint32 BE), `section[-4:]` = memory_free (uint32 BE). | ✅ CONFIRMED 2026-04-09 | | `96` | **START MONITORING** | Single write frame, no data payload. Transitions unit from idle to monitoring mode (after optional on-device sensor check ~40 s). | ✅ CONFIRMED 2026-04-08 | | `97` | **STOP MONITORING** | Single write frame, no data payload. Stops monitoring, unit returns to idle. | ✅ CONFIRMED 2026-04-08 | +| `A3` | **ERASE ALL BEGIN** | Single frame, token=0xFE at params[7]. Initiates device memory erase. Must be followed by 0x1C probe+data + 0x06 probe+data + 0xA2 to complete. Standard `build_bw_frame` (not write-format). Response ack SUB = 0x5C. | ✅ CONFIRMED 2026-04-11 | +| `A2` | **ERASE ALL CONFIRM** | Single frame, token=0xFE at params[7]. Commits the erase initiated by 0xA3. After this ack (SUB 0x5D), device memory is cleared and the event counter resets to `0x01110000`. | ✅ CONFIRMED 2026-04-11 | All requests use CMD byte `0x02`. All responses use CMD byte `0x10 0x02` (which, after de-stuffing, is just the DLE+CMD combination — see §3). @@ -273,7 +279,7 @@ All requests use CMD byte `0x02`. All responses use CMD byte `0x10 0x02` (which, | `15` | `EA` | ✅ CONFIRMED | | `01` | `FE` | ✅ CONFIRMED | | `08` | `F7` | ✅ CONFIRMED | -| `06` | `F9` | ✅ CONFIRMED | +| `06` | `F9` | ✅ CONFIRMED 2026-04-11 | | `1C` | `E3` | ✅ CONFIRMED 2026-04-08 | | `1E` | `E1` | ✅ CONFIRMED | | `0A` | `F5` | ✅ CONFIRMED | @@ -287,6 +293,8 @@ All requests use CMD byte `0x02`. All responses use CMD byte `0x10 0x02` (which, | `98` | `67` | ✅ CONFIRMED 2026-04-08 | | `96` | `69` | ✅ CONFIRMED 2026-04-08 | | `97` | `68` | ✅ CONFIRMED 2026-04-08 | +| `A3` | `5C` | ✅ CONFIRMED 2026-04-11 | +| `A2` | `5D` | ✅ CONFIRMED 2026-04-11 | --- @@ -1386,6 +1394,77 @@ Contains serial number, firmware bytes, and floating-point calibration fields. F --- +## 7.11 Erase-All Protocol (SUBs 0xA3 / 0xA2 / 0x06) ✅ 2026-04-11 + +> ✅ **Confirmed 2026-04-11** from MITM capture of a live Blastware ACH session +> (`bridges/captures/mitm/ach_mitm_20260411_001912/`). + +Blastware uses a 4-step sequence to erase all stored events from device memory. +All frames use standard `build_bw_frame` format (NOT write-format). + +### 7.11.1 Wire Sequence + +``` +BW → device: SUB 0xA3 offset=0x0000 params=00 00 00 00 00 00 00 FE 00 00 +device → BW: SUB 0x5C (begin-erase ack) + +BW → device: SUB 0x1C offset=0x0000 params=00 00 00 00 00 00 00 FE 00 00 (probe) +device → BW: SUB 0xE3 (probe ack) +BW → device: SUB 0x1C offset=0x002C params=(same) (data) +device → BW: SUB 0xE3 (44-byte monitor status response) + +BW → device: SUB 0x06 offset=0x0000 params=00 00 00 00 00 00 00 FE 00 00 (probe) +device → BW: SUB 0xF9 (probe ack) +BW → device: SUB 0x06 offset=0x0024 params=(same) (data) +device → BW: SUB 0xF9 (36-byte storage range response) + +BW → device: SUB 0xA2 offset=0x0000 params=00 00 00 00 00 00 00 FE 00 00 +device → BW: SUB 0x5D (confirm-erase ack — device memory is now cleared) +``` + +All response SUBs follow the standard formula `0xFF − request_SUB`. No exceptions. +The `token=0xFE` at `params[7]` is required for 0xA3, 0x06, and 0xA2. + +### 7.11.2 SUB 0x06 Storage Range Response (36 bytes) + +The 36-byte response from the data step ends with two 4-byte event keys: + +| Offset (from response end) | Field | Notes | +|---|---|---| +| `[-8:-4]` | First stored event key | e.g. `0111ea60` before erase | +| `[-4:]` | Last stored event key | e.g. `0111eaa6` before erase | + +After a successful erase: +- Both keys read `01110000` (device-empty sentinel) +- The device's internal event counter has reset + +Example pre-erase: `... 0111ea60 0111eaa6` +Example post-erase: `... 01110000 01110000` + +### 7.11.3 Post-Erase Key Counter Reset + +After a successful erase the device resets its event counter. New events start +from key `0x01110000` — the same key as the very first event ever recorded on +the device. This means: + +- Any system using event keys for deduplication must clear its "seen keys" state + after an erase, or risk treating fresh events as already downloaded. +- Detection heuristic: if `max(device_keys) < historical_max_key`, the counter + was reset. All device keys should be treated as new regardless of prior state. + +The `ach_server.py` implementation stores `max_downloaded_key` in `ach_state.json` +and applies this heuristic on every call-home. + +### 7.11.4 Implementation Notes + +- `MiniMateClient.delete_all_events()` in `client.py` orchestrates the full sequence. +- `MiniMateProtocol` exposes `begin_erase_all()`, `confirm_erase_all()`, and + `read_event_storage_range()` as separate methods. +- The ACH server `--clear-after-download` flag calls `delete_all_events()` after a + successful event download and resets `ach_state.json` state for the unit. + +--- + ## 8. Timestamp Format Two timestamp wire formats are used: @@ -1776,7 +1855,7 @@ The TCP port is **user-configurable** in both Blastware and the modem. There is --- -### 14.6 ACH Session Lifecycle (Call Home Mode — Future) +### 14.6 ACH Session Lifecycle (Call Home Mode) ✅ IMPLEMENTED 2026-04-11 When the unit calls home under ACH, the session lifecycle from the unit's perspective is: @@ -1785,10 +1864,28 @@ When the unit calls home under ACH, the session lifecycle from the unit's perspe 3. Unit waits for "Wait for Connection" window for first BW frame from server 4. Server sends POLL_PROBE → unit responds with POLL_RESPONSE (same as serial) 5. Server reads serial number, full config, events as needed -6. Server disconnects (or unit disconnects on Serial Idle Time expiry) -7. Unit powers modem down, returns to monitor mode +6. (Optional) Server erases device memory: SUB 0xA3 → 0x1C → 0x06 → 0xA2 +7. Server disconnects (or unit disconnects on Serial Idle Time expiry) +8. Unit detects DCD/DTR going low (modem signals line drop), returns to monitor mode automatically -Step 4 onward is **identical to the serial/call-up protocol**. The only difference from our perspective is that we are the **listener** rather than the **connector**. A future `AchServer` class will accept the incoming TCP connection and hand the socket to `TcpTransport` for processing. +Step 4 onward is **identical to the serial/call-up protocol**. The only difference +from our perspective is that we are the **listener** rather than the **connector**. + +**Implementation: `bridges/ach_server.py`** — run with `python bridges/ach_server.py`. +Key flags: +- `--clear-after-download` — erase device memory after a successful event download +- `--allow-ip IP` — restrict to specific unit IPs +- `--max-events N` — cap events per session for safety + +**State persistence: `ach_state.json`** — tracks `downloaded_keys` (set of event key +hex strings) and `max_downloaded_key` (high-water mark) per unit serial number. +Post-erase key reuse (`0x01110000` recycled) is detected via the high-water mark. + +**Note on DCD/DTR:** The MiniMate Plus monitors the RS-232 DCD line. When the TCP +connection closes, the Sierra Wireless modem drops DCD, which the unit interprets as +"serial connection ended" and automatically resumes monitoring. No `start_monitoring()` +(SUB 0x96) command is needed from the server. ⚠️ Newer RV55 firmware may not assert DCD +by default — known issue, not yet resolved. --- @@ -1841,6 +1938,11 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | Backlight offset — **RESOLVED: +4B in event index data**, uint8, seconds | RESOLVED | 2026-03-02 | | | Power save offset — **RESOLVED: +53 in event index data**, uint8, minutes | RESOLVED | 2026-03-02 | | | Monitoring LCD Cycle — **RESOLVED: +54/+55 in event index data**, uint16 BE, seconds (65500 = disabled) | RESOLVED | 2026-03-02 | | +| **SUB 0x06 purpose — RESOLVED: event storage range.** Previously labeled "CHANNEL CONFIG READ". 4-11-26 MITM capture confirms it returns first/last stored event keys in the final 8 bytes of the 36-byte response. Used by Blastware as part of the erase-all verification step. | RESOLVED | 2026-04-11 | | +| **Erase-all command sequence — RESOLVED.** SUB 0xA3 (begin) + 0x1C (monitor status) + 0x06 (storage range) + 0xA2 (confirm). Confirmed from 4-11-26 MITM capture. All frames standard `build_bw_frame`, token=0xFE. | RESOLVED | 2026-04-11 | | +| **ACH inbound server — RESOLVED.** `bridges/ach_server.py` implements full inbound ACH pipeline. `--clear-after-download` flag for delete-after-upload workflow. Post-erase key-reuse detection via `max_downloaded_key` high-water mark. | RESOLVED | 2026-04-11 | | +| **Sensor Check dropdown byte location** — byte offset in 1A compliance config payload for the "Sensor Check: Before monitoring / After each event / Disabled" setting is NOT YET LOCATED. Confirmed: unit always runs with "Before monitoring" set. Need a capture with "Disabled" to diff. | MEDIUM | 2026-04-08 | Still open | +| **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | --- From b241da970d1580486d100f3850637d05fe380ad0 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Sat, 11 Apr 2026 02:59:40 -0400 Subject: [PATCH 18/33] =?UTF-8?q?v0.10.0=20=E2=80=94=20monitor=20log=20ent?= =?UTF-8?q?ry=20support=20(SUB=200x0A=20partial=20records)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add full decode pipeline for 0x2C partial records from the device's event list, representing continuous monitoring intervals where no threshold was crossed. These records appear interleaved with full triggered events in the browse walk and were previously ignored. minimateplus/models.py - Add MonitorLogEntry dataclass: key, start_time, stop_time, serial, geo_threshold_ips, raw_header, duration_seconds property minimateplus/protocol.py - read_waveform_header() now returns (data_rsp.data, length) — full payload including the record-type byte at position 0 — instead of the sliced header. Callers that need the old slice use raw_data[11:11+length] as before. minimateplus/client.py - Add _decode_0a_partial_header(): auto-detects 9-byte (sub_code=0x10) vs 10-byte (sub_code=0x03) timestamp format, handles 1-byte inter-timestamp gap, extracts serial via BE anchor and geo threshold via Geo: anchor. - Add get_monitor_log_entries(skip_keys=None): browse walk (1E → 0A → 1F), decodes partial records, skips full records and already-seen keys. minimateplus/__init__.py - Export MonitorLogEntry bridges/ach_server.py - After get_events(), call get_monitor_log_entries(skip_keys=seen_keys) and save new entries to monitor_log.json in the session directory. - Add _monitor_log_entry_to_dict() helper. - Include monitor log keys in downloaded_keys for state persistence. CLAUDE.md / CHANGELOG.md - Document 0x2C partial record layout (timestamp format, ASCII metadata region, 1-byte gap edge case) confirmed from 4-11-26 MITM capture. - Version bump to v0.10.0; update What's next. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 51 ++++++++++ CLAUDE.md | 135 ++++++++++++++++++++++++- bridges/ach_server.py | 55 ++++++++++- minimateplus/__init__.py | 4 +- minimateplus/client.py | 209 +++++++++++++++++++++++++++++++++++++++ minimateplus/models.py | 60 +++++++++++ minimateplus/protocol.py | 30 +++--- 7 files changed, 525 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca169d9..23f02f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,57 @@ All notable changes to seismo-relay are documented here. --- +## v0.10.0 — 2026-04-11 + +### Added + +- **`MiniMateClient.get_monitor_log_entries(skip_keys=None)`** — browse-mode walk + (`1E → 0A → 1F`) that collects partial records (`0x2C` record type) from the device's + event list without triggering a full waveform download (no 0C or 5A). Returns + `list[MonitorLogEntry]`. Each entry represents one continuous monitoring interval where + no threshold was exceeded. + +- **`_decode_0a_partial_header(raw_data, index, key4)`** in `client.py` — decodes a SUB + 0x0A response payload whose record type is `0x2C`. Extracts: + - `start_time` / `stop_time` — two consecutive timestamps; auto-detects 9-byte + (sub_code=0x10, single-shot) vs 10-byte (sub_code=0x03, continuous) format from + `raw_data[11]`. Handles a 1-byte gap between the two timestamps that occurs when + ts1 and ts2 share the same minute:second. + - `serial` — device serial string found via `b"BE"` anchor scan. + - `geo_threshold_ips` — trigger level found via `b"Geo: "` anchor scan. + +- **`MonitorLogEntry` dataclass** in `models.py` — new model for partial records: + `index`, `key`, `start_time`, `stop_time`, `serial`, `geo_threshold_ips`, + `raw_header`, and a `duration_seconds` property. + +- **`read_waveform_header()` return value extended** — now returns `(data_rsp.data, length)` + (full payload) instead of `(data_rsp.data[11:11+length], length)`. Callers get the + complete payload including the record-type byte at position 0. Full records use + `raw_data[11:11+length]` as before; partial records are detected by `raw_data[0] == 0x2C`. + +- **ACH server: monitor log collection** — after `get_events()`, calls + `get_monitor_log_entries(skip_keys=seen_keys)` and saves new entries to + `monitor_log.json` in the session directory. Monitor log keys are included in + `downloaded_keys` for state persistence (no re-processing on next call-home). + +- **`_monitor_log_entry_to_dict()`** in `ach_server.py` — serialises a `MonitorLogEntry` + to a JSON-compatible dict with ISO-format timestamps. + +### Protocol / Documentation + +- **SUB 0x0A partial record (0x2C) format confirmed** (✅ 4-11-26 MITM capture, 12 frames): + - Record type `0x2C` at `raw_data[0]`; length < 64 bytes. + - Two timestamps at `raw_data[11:]` — start and stop of the monitoring interval. + - ASCII metadata region after timestamps: `BE\x00Geo: in/s`. + - Edge case: 1-byte separator between timestamps when ts1 and ts2 share minute:second. + - 10-byte timestamp format (sub_code=0x03) signalled by `raw_data[11] == 0x10`. + +- **Key reuse detection for monitor log entries** — monitor log keys are tracked alongside + event keys in `ach_state.json` so the ACH server does not re-process them after a + call-home cycle. + +--- + ## v0.9.0 — 2026-04-11 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 6dafd73..0ac0bc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.9.0**. +(Sierra Wireless RV50 / RV55). Current version: **v0.10.0**. --- @@ -25,9 +25,9 @@ CHANGELOG.md ← version history --- -## Current implementation state (v0.9.0) +## Current implementation state (v0.10.0) -Full read pipeline + write pipeline + erase pipeline working end-to-end over TCP/cellular: +Full read pipeline + write pipeline + erase pipeline + monitor log working end-to-end over TCP/cellular: | Step | SUB | Status | |---|---|---| @@ -42,7 +42,8 @@ Full read pipeline + write pipeline + erase pipeline working end-to-end over TCP | **Bulk waveform stream (event-time metadata)** | **5A** | ✅ new v0.6.0 | | Event advance / next key | 1F | ✅ | | **Write commands (push config to device)** | **68–83** | ✅ new v0.8.0 | -| **Erase all events** | **0xA3 → 0x1C → 0x06 → 0xA2** | ✅ **new v0.9.0** | +| **Erase all events** | **0xA3 → 0x1C → 0x06 → 0xA2** | ✅ new v0.9.0 | +| **Monitor log entries (partial 0x2C records)** | **0A browse** | ✅ **new v0.10.0** | `get_events()` sequence per event: `1E → 0A → 0C → 5A → 1F` @@ -795,8 +796,134 @@ the erase). Used for post-erase detection. --- +## Monitor log entries — SUB 0x0A partial records (confirmed 2026-04-11) + +Confirmed from 4-11-26 MITM capture: 12 partial records (record type `0x2C`) and 7 full +event records (record type `0x46`) across 19 total 0x0A responses. + +### Record type detection + +`read_waveform_header()` returns `(raw_data, length)` where `raw_data = data_rsp.data` +(the full payload including prefix bytes). The record type is at `raw_data[0]`: + +| Value | Type | How to process | +|---|---|---| +| `0x46` | Full triggered event | Normal download: 0C → 5A → 1F | +| `0x2C` | Monitor log entry (partial) | No 0C/5A; decode inline from 0A payload | + +Length heuristic: `length < 0x40` (64) reliably identifies partial records across all +observed captures. Both checks (`raw_data[0] == 0x2C` and `length < 0x40`) are used. + +### SUB 0x0A partial record (0x2C) payload layout + +All offsets are from `raw_data` (the full `data_rsp.data` array including the 11-byte +prefix before the actual header bytes start). + +``` +raw_data[0] = 0x2C ← record type (partial / monitor log) +raw_data[1:11] = prefix bytes (vary; contain key4 copy, flags, length) +raw_data[11:] = timestamp and ASCII metadata payload +``` + +**Timestamp auto-detection** (confirmed from 4-11-26 capture): + +``` +raw_data[11] == 0x10 → 10-byte sub_code=0x03 format (continuous mode) +raw_data[11] != 0x10 → 9-byte sub_code=0x10 format (single-shot mode) +``` + +**9-byte timestamp format (sub_code=0x10):** + +| Byte | Field | +|---|---| +| 0 | day | +| 1 | `0x10` (sub_code marker) | +| 2 | month | +| 3–4 | year (uint16 BE) | +| 5 | unknown (0x00) | +| 6 | hour | +| 7 | minute | +| 8 | second | + +**10-byte timestamp format (sub_code=0x03):** + +| Byte | Field | +|---|---| +| 0 | `0x10` (marker) | +| 1 | day | +| 2 | `0x10` (marker) | +| 3 | month | +| 4–5 | year (uint16 BE) | +| 6 | unknown (0x00) | +| 7 | hour | +| 8 | minute | +| 9 | second | + +**Two timestamps:** Each partial record contains two timestamps — `start_time` and +`stop_time` — stored consecutively: +- `ts1` (start) at `raw_data[ts_offset : ts_offset + ts_size]` where `ts_offset = 11` +- `ts2` (stop) at `raw_data[ts1_end : ts1_end + ts_size]` + +**Edge case — 1-byte gap between timestamps:** Occurs when ts1 and ts2 share the same +minute:second. If `try_ts(raw_data[ts1_end:])` fails, try `try_ts(raw_data[ts1_end+1:])`. +Confirmed in frames 121, 161, 165 of the 4-11-26 MITM capture. Frame 121 still shows 0s +duration (both decode to 16:02:00) — the extra byte appears in all same-second cases. + +**ASCII metadata after timestamps:** +``` + BE\x00Geo: in/s ... +``` + +- Serial: scan for `b"BE"`, read until `b"\x00"` (e.g. `"BE11529"`) +- Geo threshold: scan for `b"Geo: "`, read float until next space (e.g. `0.254` in/s) + +A separator of variable length (4–5 bytes of `\x00` + flags) sits between the two +timestamps and the ASCII region. The `b"BE"` anchor scan is robust to separator length +variation. + +### `_decode_0a_partial_header(raw_data, index, key4)` — client.py + +Returns a `MonitorLogEntry` or `None`. Called by `get_monitor_log_entries()` for each +event key whose 0x0A response has `raw_data[0] == 0x2C` or `length < 0x40`. + +### `MiniMateClient.get_monitor_log_entries(skip_keys=None)` — client.py + +Browse-mode walk: `1E → 0A → check type → decode if partial → 1F`. No 0x0C or 5A reads +performed. Full (0x46) records are skipped without decoding. Returns `list[MonitorLogEntry]`. + +`skip_keys` (optional `set[str]`): keys in this set are still advanced through the walk +(to avoid disrupting the iteration sequence), but no `MonitorLogEntry` is created for them. + +### `MonitorLogEntry` model — models.py + +```python +@dataclass +class MonitorLogEntry: + index: int # 0-based position + key: str # 8-hex event key + start_time: Optional[datetime.datetime] = None + stop_time: Optional[datetime.datetime] = None + serial: Optional[str] = None + geo_threshold_ips: Optional[float] = None + raw_header: Optional[bytes] = field(default=None, repr=False) + + @property + def duration_seconds(self) -> Optional[float]: ... +``` + +### ACH server integration (v0.10.0) + +After `get_events()`, the ACH server calls `get_monitor_log_entries(skip_keys=seen_keys)`. +New entries are saved to `monitor_log.json` in the session directory. Monitor log keys are +included in `current_keys` for state persistence so they are not re-processed on the next +call-home. + +--- + ## What's next +- **Database** — SQLite store for events + monitor log entries; dedup by key; queryable +- **Histograms** — decode histogram-mode A5 data (noise floor tracking) - Compliance config encoder — build raw write payloads from a `ComplianceConfig` object - Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring) - Modem manager — push RV50/RV55 configs via Sierra Wireless API diff --git a/bridges/ach_server.py b/bridges/ach_server.py index c9e0302..8742e81 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -67,7 +67,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from minimateplus.transport import SocketTransport from minimateplus.client import MiniMateClient -from minimateplus.models import DeviceInfo, Event +from minimateplus.models import DeviceInfo, Event, MonitorLogEntry log = logging.getLogger("ach_server") @@ -372,6 +372,42 @@ class AchSession: else: log.info(" [OK] No new events since last call-home -- nothing to save") + # ── Monitor log entries (partial records / continuous monitoring) ── + # Browse walk (0A + 1F only) to collect monitor log entries for + # recording intervals where no threshold was crossed. This is a + # second 1E-based pass over the device's record list, separate from + # the get_events() download loop above. + log.info(" Collecting monitor log entries (browse walk)...") + new_monitor_entries: list[MonitorLogEntry] = [] + try: + new_monitor_entries = client.get_monitor_log_entries( + skip_keys=seen_keys if seen_keys else None, + ) + if new_monitor_entries: + _save_json( + session_dir / "monitor_log.json", + [_monitor_log_entry_to_dict(e) for e in new_monitor_entries], + ) + log.info( + " [OK] %d new monitor log entry(s) saved", + len(new_monitor_entries), + ) + for ml in new_monitor_entries: + log.info( + " MONLOG [%s] %s → %s (%s)", + ml.key, + ml.start_time.isoformat() if ml.start_time else "?", + ml.stop_time.isoformat() if ml.stop_time else "?", + f"{ml.duration_seconds:.0f}s" if ml.duration_seconds is not None else "?s", + ) + else: + log.info(" [OK] No new monitor log entries") + except Exception as exc: + log.warning( + " [WARN] Monitor log collection failed: %s -- continuing", + exc, + ) + # ── Optional: erase device memory after successful download ──── erased_successfully = False if self.clear_after_download and new_events: @@ -387,11 +423,15 @@ class AchSession: ) # ── Update persistent state ─────────────────────────────────── - current_keys = [ + # Include both triggered-event keys and monitor-log keys in the + # downloaded set so they are not re-processed on the next call-home. + current_event_keys = [ e._waveform_key.hex() for e in all_events if e._waveform_key is not None ] + current_monitor_keys = [e.key for e in new_monitor_entries] + current_keys = current_event_keys + current_monitor_keys if erased_successfully: # Device memory is clear. Reset downloaded_keys and the @@ -492,6 +532,17 @@ def _event_to_dict(e: Event) -> dict: } +def _monitor_log_entry_to_dict(e: MonitorLogEntry) -> dict: + return { + "key": e.key, + "start_time": e.start_time.isoformat() if e.start_time else None, + "stop_time": e.stop_time.isoformat() if e.stop_time else None, + "duration_seconds": e.duration_seconds, + "serial": e.serial, + "geo_threshold_ips": e.geo_threshold_ips, + } + + # ── Main server loop ─────────────────────────────────────────────────────────── def serve(args: argparse.Namespace) -> None: diff --git a/minimateplus/__init__.py b/minimateplus/__init__.py index e115106..6c7be72 100644 --- a/minimateplus/__init__.py +++ b/minimateplus/__init__.py @@ -20,8 +20,8 @@ Typical usage (TCP / modem): """ from .client import MiniMateClient -from .models import DeviceInfo, Event +from .models import DeviceInfo, Event, MonitorLogEntry from .transport import SerialTransport, TcpTransport __version__ = "0.1.0" -__all__ = ["MiniMateClient", "DeviceInfo", "Event", "SerialTransport", "TcpTransport"] +__all__ = ["MiniMateClient", "DeviceInfo", "Event", "MonitorLogEntry", "SerialTransport", "TcpTransport"] diff --git a/minimateplus/client.py b/minimateplus/client.py index 022b0cc..28b5433 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -28,6 +28,7 @@ Example (TCP / modem): from __future__ import annotations +import datetime import logging import struct from typing import Optional @@ -37,6 +38,7 @@ from .models import ( ComplianceConfig, DeviceInfo, Event, + MonitorLogEntry, MonitorStatus, PeakValues, ProjectInfo, @@ -300,6 +302,96 @@ class MiniMateClient: log.info("list_event_keys: %d key(s): %s", len(keys), keys) return keys + def get_monitor_log_entries( + self, + skip_keys: Optional[set] = None, + ) -> list[MonitorLogEntry]: + """ + Collect all monitor log entries (partial records, type 0x2C) from the + device using the browse-mode 1E → 0A → 1F walk. + + This is the fast path for monitor log data. No 0C or 5A commands are + issued — all available monitor log information is in the 0x0A response + header alone. + + Full triggered events (0x0A response type 0x46) are silently skipped. + Only partial records (type 0x2C) are returned as MonitorLogEntry objects. + + Confirmed from 4-11-26 MITM capture: Blastware's ACH mode performs a + full browse walk (Phase 3: 0x0A + 1F × all records) AFTER the triggered- + event download phase. The partial records encountered in this walk are + the monitor log entries. + + Args: + skip_keys: optional set of 8-hex key strings to skip (already seen). + Keys in this set still advance the walk (0A + 1F) but are + not decoded or returned. + + Returns: + List of MonitorLogEntry objects in device storage order. + + Raises: + ProtocolError: on unrecoverable communication failure. + """ + proto = self._require_proto() + try: + key4, data8 = proto.read_event_first() + except ProtocolError as exc: + log.warning("get_monitor_log_entries: 1E failed: %s -- returning []", exc) + return [] + + if data8[4:8] == b"\x00\x00\x00\x00": + log.info("get_monitor_log_entries: device is empty") + return [] + + entries: list[MonitorLogEntry] = [] + idx = 0 + + while data8[4:8] != b"\x00\x00\x00\x00": + cur_key = key4 + key_hex = cur_key.hex() + + try: + raw_data, rec_len = proto.read_waveform_header(cur_key) + except ProtocolError as exc: + log.warning( + "get_monitor_log_entries: 0A failed for key=%s: %s -- stopping", + key_hex, exc, + ) + break + + # Only decode partial records (0x2C); full records (0x46) are silently skipped. + if rec_len < 0x40 and raw_data and (not skip_keys or key_hex not in skip_keys): + entry = _decode_0a_partial_header(raw_data, idx, cur_key) + if entry is not None: + entries.append(entry) + log.debug( + "get_monitor_log_entries: [%d] key=%s %s → %s", + idx, key_hex, entry.start_time, entry.stop_time, + ) + else: + log.debug( + "get_monitor_log_entries: [%d] key=%s type=0x%02X %s", + idx, key_hex, rec_len, + "skip (already seen)" if skip_keys and key_hex in skip_keys else "skip (full record)", + ) + + try: + key4, data8 = proto.advance_event(browse=True) + except ProtocolError as exc: + log.warning( + "get_monitor_log_entries: 1F failed after %d record(s): %s -- stopping", + idx, exc, + ) + break + idx += 1 + + log.info( + "get_monitor_log_entries: walked %d record(s), found %d monitor log entry(s)", + idx, len(entries), + ) + return entries + def delete_all_events(self) -> None: """ Erase all stored events from the device memory. @@ -1856,6 +1948,123 @@ def _find_first_string(data: bytes, start: int, end: int, min_len: int) -> Optio +def _decode_0a_partial_header(raw_data: bytes, index: int, key4: bytes) -> Optional[MonitorLogEntry]: + """ + Decode a SUB 0x0A response for a partial (monitor log) record into a + MonitorLogEntry. + + Called when read_waveform_header() returns rec_len < 0x40 (i.e. 0x2C = 44). + raw_data is the complete data_rsp.data from the protocol layer. + + Layout of raw_data: + [0] = 0x2C (partial record type) + [1:5] = 0x00 × 4 + [5:9] = event key (big-endian) + [9:11] = 0x00 × 2 + [11:] = timestamp_start + timestamp_stop + sep + serial + geo_string + + Timestamp format detection (auto): + raw_data[11] == 0x10 → 10-byte sub_code=0x03 continuous format + raw_data[12] == 0x10 → 9-byte sub_code=0x10 single-shot format + + Both timestamps use the same format (detected from the first byte). + A 1-byte gap can appear between ts1 and ts2 for certain timestamps + (observed empirically when both timestamps share the same minute:second). + The parser handles this by trying ts2 immediately after ts1, then with + a 1-byte skip if that fails. + + Returns: + MonitorLogEntry if decoding succeeds, None on error. + """ + if len(raw_data) < 20 or raw_data[0] != 0x2C: + return None + + key_hex = key4.hex() + + def try_ts9(b: bytes): + """9-byte sub_code=0x10 format. Returns datetime or None.""" + if len(b) < 9 or b[1] != 0x10: + return None + day = b[0]; month = b[2]; year = (b[3] << 8) | b[4] + hr = b[6]; mn = b[7]; sec = b[8] + if not (1 <= day <= 31 and 1 <= month <= 12 and 2000 <= year <= 2050 + and hr <= 23 and mn <= 59 and sec <= 59): + return None + try: + return datetime.datetime(year, month, day, hr, mn, sec) + except ValueError: + return None + + def try_ts10(b: bytes): + """10-byte sub_code=0x03 format. Returns datetime or None.""" + if len(b) < 10 or b[0] != 0x10 or b[2] != 0x10: + return None + day = b[1]; month = b[3]; year = (b[4] << 8) | b[5] + hr = b[7]; mn = b[8]; sec = b[9] + if not (1 <= day <= 31 and 1 <= month <= 12 and 2000 <= year <= 2050 + and hr <= 23 and mn <= 59 and sec <= 59): + return None + try: + return datetime.datetime(year, month, day, hr, mn, sec) + except ValueError: + return None + + ts_offset = 11 + if len(raw_data) <= ts_offset: + return MonitorLogEntry(index=index, key=key_hex, raw_header=raw_data) + + # Detect timestamp format. + if raw_data[ts_offset] == 0x10: + ts_size = 10 + try_ts = try_ts10 + else: + ts_size = 9 + try_ts = try_ts9 + + # Parse ts1. + ts1 = try_ts(raw_data[ts_offset:ts_offset + ts_size]) + ts1_end = ts_offset + ts_size + + # Parse ts2 immediately after ts1, then with 1-byte skip if needed. + ts2 = try_ts(raw_data[ts1_end:ts1_end + ts_size]) + if ts2 is None: + ts2 = try_ts(raw_data[ts1_end + 1:ts1_end + 1 + ts_size]) + + # Extract serial and geo threshold from "BE11529\0" and "Geo: X.XXX in/s\0". + serial: Optional[str] = None + geo_ips: Optional[float] = None + + serial_pos = raw_data.find(b"BE") + if serial_pos >= 0: + # Read null-terminated serial starting at serial_pos. + null_pos = raw_data.find(b"\x00", serial_pos) + if null_pos > serial_pos: + serial = raw_data[serial_pos:null_pos].decode("ascii", errors="replace") + # Geo string follows the null byte. + geo_start = (null_pos + 1) if null_pos > serial_pos else serial_pos + 7 + geo_bytes = raw_data[geo_start:] + # "Geo: X.XXX in/s\0" — extract float after "Geo: ". + geo_str_pos = geo_bytes.find(b"Geo: ") + if geo_str_pos >= 0: + geo_val_bytes = geo_bytes[geo_str_pos + 5:] # after "Geo: " + geo_val_end = geo_val_bytes.find(b" ") # before " in/s" + if geo_val_end > 0: + try: + geo_ips = float(geo_val_bytes[:geo_val_end].decode("ascii")) + except ValueError: + pass + + return MonitorLogEntry( + index=index, + key=key_hex, + start_time=ts1, + stop_time=ts2, + serial=serial, + geo_threshold_ips=geo_ips, + raw_header=raw_data, + ) + + def _decode_monitor_status(data: bytes) -> MonitorStatus: """ Decode SUB 0x1C response payload into a MonitorStatus object. diff --git a/minimateplus/models.py b/minimateplus/models.py index 3371c36..520b428 100644 --- a/minimateplus/models.py +++ b/minimateplus/models.py @@ -14,6 +14,7 @@ Notes on certainty: from __future__ import annotations +import datetime import struct from dataclasses import dataclass, field from typing import Optional @@ -419,6 +420,65 @@ class Event: return f"Event#{self.index} {ts}{ppv}" +# ── MonitorLogEntry ─────────────────────────────────────────────────────────── + +@dataclass +class MonitorLogEntry: + """ + A monitor log entry decoded from a SUB 0x0A (WAVEFORM_HEADER) response + whose first byte is 0x2C (partial record, recording mode = continuous + monitoring without a triggered event). + + These are the "partial bins" that Blastware stores between triggered events. + Each entry represents one monitoring interval — the span of time during + which the unit was actively monitoring but no threshold crossing occurred. + + Confirmed from 4-11-26 MITM capture analysis (2026-04-11): + + Header layout (full response data[0:]): + data[0] = 0x2C (partial record type / data length in probe response) + data[1:5] = 0x00 × 4 + data[5:9] = event key (4 bytes, big-endian hex) + data[9:11] = 0x00 × 2 + data[11:] = timestamp_start (9 or 10 bytes depending on recording mode) + + timestamp_stop (same format) + + separator (4–5 bytes, variable) + + serial null-terminated (e.g. "BE11529\\0") + + "Geo: X.XXX in/s\\0" (trigger threshold string) + + Timestamp format detection: + data[11] == 0x10 → 10-byte sub_code=0x03 (continuous) format + data[12] == 0x10 → 9-byte sub_code=0x10 (single-shot) format + + In contrast to Event (triggered records, type 0x46), MonitorLogEntry + records do NOT have a waveform record (SUB 0x0C) or bulk waveform stream + (SUB 5A). All available metadata is in the 0x0A header alone. + """ + index: int # 0-based position in device record list + key: str # 8-hex event key (e.g. "01114290") ✅ + + start_time: Optional[datetime.datetime] = None # monitoring session start ✅ + stop_time: Optional[datetime.datetime] = None # monitoring session stop ✅ + serial: Optional[str] = None # device serial (e.g. "BE11529") ✅ + geo_threshold_ips: Optional[float] = None # trigger level from "Geo: X.XXX in/s" ✅ + + # Raw bytes for debugging / future decoding + raw_header: Optional[bytes] = field(default=None, repr=False) + + @property + def duration_seconds(self) -> Optional[float]: + """Duration of monitoring interval in seconds, or None if times unavailable.""" + if self.start_time and self.stop_time: + return (self.stop_time - self.start_time).total_seconds() + return None + + def __str__(self) -> str: + start = self.start_time.isoformat() if self.start_time else "?" + stop = self.stop_time.isoformat() if self.stop_time else "?" + dur = f" ({self.duration_seconds:.0f}s)" if self.duration_seconds is not None else "" + return f"MonitorLog#{self.index} key={self.key} {start}→{stop}{dur}" + + # ── MonitorStatus ───────────────────────────────────────────────────────────── @dataclass diff --git a/minimateplus/protocol.py b/minimateplus/protocol.py index 83afa6e..35e99cf 100644 --- a/minimateplus/protocol.py +++ b/minimateplus/protocol.py @@ -394,23 +394,32 @@ class MiniMateProtocol: Send the SUB 0A (WAVEFORM_HEADER) two-step read for *key4*. The data length for 0A is VARIABLE and must be read from the probe - response at data[4]. Two known values: - 0x30 — full histogram bin (has a waveform record to follow) - 0x26 — partial histogram bin (no waveform record) + response at data[4]. Two confirmed values: + 0x46 (70) — full triggered event (has 0C waveform record to follow) + 0x2C (44) — partial / monitor-log entry (no 0C record; 0A header only) Args: key4: 4-byte waveform record address from 1E or 1F. Returns: - (header_bytes, record_length) where: - header_bytes — raw data section starting at data[11] - record_length — DATA_LENGTH read from probe (0x30 or 0x26) + (raw_data, record_length) where: + raw_data — complete data_rsp.data bytes (full response payload) + record_length — DATA_LENGTH read from probe (0x46 for full, 0x2C for partial) + + The raw_data layout: + raw_data[0] = record type (0x46 = full triggered event, 0x2C = partial/monitor) + raw_data[1:5] = 0x00 × 4 + raw_data[5:9] = event key (4 bytes) + raw_data[9:11] = 0x00 × 2 + raw_data[11:] = timestamps + separator + serial + channel strings + (see MonitorLogEntry in models.py for full layout) Raises: ProtocolError: on timeout, bad checksum, or wrong response SUB. - Confirmed from 3-31-26 capture: 0A probe response data[4] carries + Confirmed from 4-11-26 MITM capture: 0A probe response data[4] carries the variable length; data-request uses that length as the offset byte. + record_length == data[0] in virtually all cases (confirmed empirically). """ rsp_sub = _expected_rsp_sub(SUB_WAVEFORM_HEADER) params = waveform_key_params(key4) @@ -420,7 +429,7 @@ class MiniMateProtocol: probe_rsp = self._recv_one(expected_sub=rsp_sub) # Variable length — read from probe response data[4] - length = probe_rsp.data[4] if len(probe_rsp.data) > 4 else 0x30 + length = probe_rsp.data[4] if len(probe_rsp.data) > 4 else 0x46 log.debug("read_waveform_header: 0A data request offset=0x%02X", length) if length == 0: @@ -429,12 +438,11 @@ class MiniMateProtocol: self._send(build_bw_frame(SUB_WAVEFORM_HEADER, length, params)) data_rsp = self._recv_one(expected_sub=rsp_sub) - header_bytes = data_rsp.data[11:11 + length] log.debug( "read_waveform_header: key=%s length=0x%02X is_full=%s", - key4.hex(), length, length == 0x30, + key4.hex(), length, length >= 0x40, ) - return header_bytes, length + return data_rsp.data, length def read_waveform_data_raw(self) -> bytes: """ From 7883a31aa7cbddbb65eed473f33134bf9b7f8946 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 00:45:38 -0400 Subject: [PATCH 19/33] =?UTF-8?q?v0.11.0=20=E2=80=94=20SQLite=20persistenc?= =?UTF-8?q?e=20layer=20(SeismoDb)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sfm/database.py (new) - SeismoDb class: three tables keyed by unit serial number - ach_sessions: one row per ACH call-home - events: one row per triggered event, deduped by (serial, waveform_key) - monitor_log: one row per monitoring interval, deduped by (serial, waveform_key) - WAL mode, per-request connections, silent dedup via UNIQUE constraint - Query helpers: query_events(), query_monitor_log(), get_sessions(), query_units() - false_trigger flag on events for future review UI / report filtering bridges/ach_server.py - Import SeismoDb; create shared instance at startup pointed at bridges/captures/seismo_relay.db - After each call-home: insert_events() + insert_monitor_log() + insert_ach_session() - DB failures logged as warnings, never abort the session sfm/server.py - Import SeismoDb; lazy singleton via _get_db() - New DB read endpoints: GET /db/units, /db/events, /db/monitor_log, /db/sessions - PATCH /db/events/{id}/false_trigger for manual review flagging CLAUDE.md / CHANGELOG.md - Document DB schema, SFM DB endpoints, architecture decision (unit-keyed only) - Version bump to v0.11.0 Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 38 ++++ CLAUDE.md | 15 ++ bridges/ach_server.py | 29 +++ sfm/database.py | 406 ++++++++++++++++++++++++++++++++++++++++++ sfm/server.py | 119 +++++++++++++ 5 files changed, 607 insertions(+) create mode 100644 sfm/database.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f02f7..315f3aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,44 @@ All notable changes to seismo-relay are documented here. --- +## v0.11.0 — 2026-04-13 + +### Added + +- **`sfm/database.py` — SeismoDb** — SQLite persistence layer for all ACH data. + Three tables, all unit-keyed by serial number: + - `ach_sessions` — one row per inbound call-home: serial, timestamp, peer IP, + events_downloaded, monitor_entries, duration_seconds + - `events` — one row per triggered waveform event: serial, waveform_key (dedup), + timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location + strings, sample_rate, record_type, false_trigger flag + - `monitor_log` — one row per monitoring interval: serial, waveform_key (dedup), + start_time, stop_time, duration_seconds, geo_threshold_ips + - WAL mode, per-request connections — safe for the single-writer / occasional-reader + ACH server pattern + - Deduplication by `(serial, waveform_key)` UNIQUE constraint — re-runs and repeat + call-homes never produce duplicate rows + +- **`ach_server.py` — DB integration** — after each successful call-home, writes new + events and monitor log entries to `seismo_relay.db` then records the session in + `ach_sessions`. DB write failures are logged as warnings and do not abort the session. + +- **`sfm/server.py` — DB read endpoints**: + - `GET /db/units` — distinct serials with last_seen, total_events, total_monitor_entries + - `GET /db/events` — query events with serial / date range / false_trigger filters + - `GET /db/monitor_log` — query monitoring intervals + - `GET /db/sessions` — query ACH call-home sessions + - `PATCH /db/events/{id}/false_trigger` — flag/unflag false triggers (for review UI) + +### Architecture + +- seismo-relay DB is unit-keyed only — no project concepts. Project aggregation is + terra-view's responsibility via `UnitAssignment` / `DeploymentRecord` + date range + queries against the SFM DB endpoints. +- DB file lives at `bridges/captures/seismo_relay.db` by default. + +--- + ## v0.10.0 — 2026-04-11 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 0ac0bc5..f1a2836 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -416,6 +416,8 @@ for 0x10 records). ## SFM REST API (sfm/server.py) +### Live device endpoints (connect to device per-request) + ``` GET /device/info?port=COM5 ← serial GET /device/info?host=1.2.3.4&tcp_port=9034 ← cellular @@ -428,6 +430,19 @@ POST /device/monitor/stop?host=1.2.3.4&tcp_port=9034 ← stop recording Server retries once on `ProtocolError` for TCP connections (handles cold-boot timing). +### DB read endpoints (query seismo_relay.db written by ach_server.py) + +``` +GET /db/units ← all known serials + summary stats +GET /db/events?serial=BE11529&from_dt=&to_dt=&limit= ← triggered events, newest first +GET /db/monitor_log?serial=BE11529&from_dt=&to_dt= ← monitoring intervals, newest first +GET /db/sessions?serial=BE11529&limit=50 ← ACH call-home sessions, newest first +PATCH /db/events/{id}/false_trigger?value=true ← flag/unflag false triggers +``` + +DB file: `bridges/captures/seismo_relay.db` (default; override with `--db-path` at startup). +All DB endpoints are read-only except `PATCH /db/events/{id}/false_trigger`. + --- ## Key wire captures (reference material) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 8742e81..f3fc6e4 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -68,6 +68,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from minimateplus.transport import SocketTransport from minimateplus.client import MiniMateClient from minimateplus.models import DeviceInfo, Event, MonitorLogEntry +from sfm.database import SeismoDb log = logging.getLogger("ach_server") @@ -136,6 +137,7 @@ class AchSession: events_only: bool, max_events: Optional[int], state_path: Path, + db: "SeismoDb", clear_after_download: bool = False, ) -> None: self.sock = sock @@ -145,6 +147,7 @@ class AchSession: self.events_only = events_only self.max_events = max_events self.state_path = state_path + self.db = db self.clear_after_download = clear_after_download def run(self) -> None: @@ -408,6 +411,30 @@ class AchSession: exc, ) + # ── Persist to SQLite DB ───────────────────────────────────── + _session_start = datetime.datetime.now() + try: + _ev_ins, _ev_skip = self.db.insert_events( + new_events, serial=serial or self.peer, session_id=None + ) + _ml_ins, _ml_skip = self.db.insert_monitor_log( + new_monitor_entries, session_id=None + ) + _session_id = self.db.insert_ach_session( + serial=serial or self.peer, + peer=self.peer, + events_downloaded=_ev_ins, + monitor_entries=_ml_ins, + duration_seconds=(datetime.datetime.now() - _session_start).total_seconds(), + session_time=_session_start, + ) + log.info( + " [DB] session=%s events +%d (skip %d) monitor +%d (skip %d)", + _session_id[:8], _ev_ins, _ev_skip, _ml_ins, _ml_skip, + ) + except Exception as exc: + log.warning(" [WARN] DB write failed: %s -- continuing", exc) + # ── Optional: erase device memory after successful download ──── erased_successfully = False if self.clear_after_download and new_events: @@ -549,6 +576,7 @@ def serve(args: argparse.Namespace) -> None: output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) state_path = output_dir / "ach_state.json" + db = SeismoDb(output_dir / "seismo_relay.db") server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -601,6 +629,7 @@ def serve(args: argparse.Namespace) -> None: events_only=args.events_only, max_events=max_ev, state_path=state_path, + db=db, clear_after_download=args.clear_after_download, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") diff --git a/sfm/database.py b/sfm/database.py new file mode 100644 index 0000000..e0172e2 --- /dev/null +++ b/sfm/database.py @@ -0,0 +1,406 @@ +""" +sfm/database.py — SQLite persistence layer for seismo-relay. + +Three tables, all keyed by unit serial number: + + ach_sessions — one row per inbound ACH call-home + events — one row per triggered waveform event (deduped by serial+key) + monitor_log — one row per monitoring interval (deduped by serial+key) + +The DB file lives at: + /seismo_relay.db (default: bridges/captures/seismo_relay.db) + +Usage +----- + from sfm.database import SeismoDb + + db = SeismoDb("bridges/captures/seismo_relay.db") + + # Write a call-home session + session_id = db.insert_ach_session(serial="BE11529", peer="1.2.3.4:51920", + events_downloaded=3, monitor_entries=2, + duration_seconds=47.3) + + # Write events (silently skips duplicates) + db.insert_events(events, serial="BE11529", session_id=session_id) + + # Write monitor log entries + db.insert_monitor_log(entries, session_id=session_id) + + # Query + rows = db.query_events(serial="BE11529", from_dt=datetime(...), to_dt=datetime(...)) +""" + +from __future__ import annotations + +import datetime +import logging +import sqlite3 +import uuid +from pathlib import Path +from typing import Optional + +from minimateplus.models import Event, MonitorLogEntry + +log = logging.getLogger("sfm.database") + +# ── Schema ───────────────────────────────────────────────────────────────────── + +_SCHEMA = """ +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS ach_sessions ( + id TEXT PRIMARY KEY, -- UUID + serial TEXT NOT NULL, + session_time TEXT NOT NULL, -- ISO-8601 UTC + peer TEXT, -- "ip:port" + events_downloaded INTEGER NOT NULL DEFAULT 0, + monitor_entries INTEGER NOT NULL DEFAULT 0, + duration_seconds REAL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) +); +CREATE INDEX IF NOT EXISTS idx_ach_sessions_serial ON ach_sessions(serial); +CREATE INDEX IF NOT EXISTS idx_ach_sessions_time ON ach_sessions(session_time); + +CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, -- UUID + serial TEXT NOT NULL, + waveform_key TEXT NOT NULL, -- 8-hex device key (dedup field) + session_id TEXT, -- FK → ach_sessions.id + timestamp TEXT, -- ISO-8601 local time from device + tran_ppv REAL, -- in/s + vert_ppv REAL, -- in/s + long_ppv REAL, -- in/s + peak_vector_sum REAL, -- in/s + mic_ppv REAL, -- psi or dB depending on setup + project TEXT, + client TEXT, + operator TEXT, + sensor_location TEXT, + sample_rate INTEGER, + record_type TEXT, -- "single_shot" | "continuous" + false_trigger INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=yes (manual flag) + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(serial, waveform_key) +); +CREATE INDEX IF NOT EXISTS idx_events_serial ON events(serial); +CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp); +CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id); + +CREATE TABLE IF NOT EXISTS monitor_log ( + id TEXT PRIMARY KEY, -- UUID + serial TEXT NOT NULL, + waveform_key TEXT NOT NULL, -- 8-hex device key (dedup field) + session_id TEXT, -- FK → ach_sessions.id + start_time TEXT, -- ISO-8601 + stop_time TEXT, -- ISO-8601 + duration_seconds REAL, + geo_threshold_ips REAL, -- in/s + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(serial, waveform_key) +); +CREATE INDEX IF NOT EXISTS idx_monitor_log_serial ON monitor_log(serial); +CREATE INDEX IF NOT EXISTS idx_monitor_log_start ON monitor_log(start_time); +CREATE INDEX IF NOT EXISTS idx_monitor_log_session ON monitor_log(session_id); +""" + + +# ── SeismoDb class ───────────────────────────────────────────────────────────── + +class SeismoDb: + """ + Thin SQLite wrapper for seismo-relay persistence. + + Thread-safe: each call opens, uses, and closes a connection with + check_same_thread=False and WAL mode enabled. For the ACH server's + single-writer / occasional-reader pattern this is more than sufficient. + """ + + def __init__(self, db_path: str | Path) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_schema() + log.info("SeismoDb initialised at %s", self.db_path) + + # ── Internal helpers ─────────────────────────────────────────────────────── + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(str(self.db_path), check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA foreign_keys = ON") + return conn + + def _init_schema(self) -> None: + with self._connect() as conn: + conn.executescript(_SCHEMA) + + @staticmethod + def _iso(dt: Optional[datetime.datetime]) -> Optional[str]: + return dt.isoformat() if dt is not None else None + + @staticmethod + def _new_id() -> str: + return str(uuid.uuid4()) + + # ── ACH sessions ────────────────────────────────────────────────────────── + + def insert_ach_session( + self, + *, + serial: str, + peer: Optional[str] = None, + events_downloaded: int = 0, + monitor_entries: int = 0, + duration_seconds: Optional[float] = None, + session_time: Optional[datetime.datetime] = None, + ) -> str: + """Insert a new ACH session row. Returns the new session UUID.""" + sid = self._new_id() + ts = self._iso(session_time or datetime.datetime.utcnow()) + with self._connect() as conn: + conn.execute( + """ + INSERT INTO ach_sessions + (id, serial, session_time, peer, + events_downloaded, monitor_entries, duration_seconds) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + (sid, serial, ts, peer, + events_downloaded, monitor_entries, duration_seconds), + ) + log.debug("ach_session inserted: %s serial=%s events=%d monitor=%d", + sid, serial, events_downloaded, monitor_entries) + return sid + + def get_sessions( + self, + serial: Optional[str] = None, + limit: int = 50, + ) -> list[dict]: + """Return recent ACH sessions, newest first.""" + with self._connect() as conn: + if serial: + rows = conn.execute( + "SELECT * FROM ach_sessions WHERE serial=? " + "ORDER BY session_time DESC LIMIT ?", + (serial, limit), + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM ach_sessions ORDER BY session_time DESC LIMIT ?", + (limit,), + ).fetchall() + return [dict(r) for r in rows] + + # ── Events ──────────────────────────────────────────────────────────────── + + def insert_events( + self, + events: list[Event], + *, + serial: str, + session_id: Optional[str] = None, + ) -> tuple[int, int]: + """ + Insert triggered events. Silently skips duplicates (serial+waveform_key). + Returns (inserted, skipped). + """ + inserted = skipped = 0 + with self._connect() as conn: + for ev in events: + key = ev._waveform_key.hex() if ev._waveform_key else None + if key is None: + skipped += 1 + continue + + ts = None + if ev.timestamp: + try: + ts = datetime.datetime( + ev.timestamp.year, ev.timestamp.month, ev.timestamp.day, + ev.timestamp.hour, ev.timestamp.minute, ev.timestamp.second, + ).isoformat() + except Exception: + ts = str(ev.timestamp) + + pv = ev.peak_values + pi = ev.project_info + + try: + conn.execute( + """ + INSERT INTO events + (id, serial, waveform_key, session_id, timestamp, + tran_ppv, vert_ppv, long_ppv, peak_vector_sum, mic_ppv, + project, client, operator, sensor_location, + sample_rate, record_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + self._new_id(), serial, key, session_id, ts, + pv.tran if pv else None, + pv.vert if pv else None, + pv.long if pv else None, + pv.peak_vector_sum if pv else None, + pv.micl if pv else None, + pi.project if pi else None, + pi.client if pi else None, + pi.operator if pi else None, + pi.sensor_location if pi else None, + ev.sample_rate, + ev.record_type, + ), + ) + inserted += 1 + except sqlite3.IntegrityError: + skipped += 1 + + log.debug("insert_events serial=%s inserted=%d skipped=%d", + serial, inserted, skipped) + return inserted, skipped + + def query_events( + self, + serial: Optional[str] = None, + from_dt: Optional[datetime.datetime] = None, + to_dt: Optional[datetime.datetime] = None, + false_trigger: Optional[bool] = None, + limit: int = 500, + offset: int = 0, + ) -> list[dict]: + """Query events with optional filters. Returns newest first.""" + clauses: list[str] = [] + params: list = [] + + if serial: + clauses.append("serial = ?") + params.append(serial) + if from_dt: + clauses.append("timestamp >= ?") + params.append(from_dt.isoformat()) + if to_dt: + clauses.append("timestamp <= ?") + params.append(to_dt.isoformat()) + if false_trigger is not None: + clauses.append("false_trigger = ?") + params.append(1 if false_trigger else 0) + + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + params += [limit, offset] + + with self._connect() as conn: + rows = conn.execute( + f"SELECT * FROM events {where} " + f"ORDER BY timestamp DESC LIMIT ? OFFSET ?", + params, + ).fetchall() + return [dict(r) for r in rows] + + def set_false_trigger(self, event_id: str, value: bool) -> bool: + """Set or clear the false_trigger flag on an event. Returns True if found.""" + with self._connect() as conn: + cur = conn.execute( + "UPDATE events SET false_trigger=? WHERE id=?", + (1 if value else 0, event_id), + ) + return cur.rowcount > 0 + + # ── Monitor log ─────────────────────────────────────────────────────────── + + def insert_monitor_log( + self, + entries: list[MonitorLogEntry], + *, + session_id: Optional[str] = None, + ) -> tuple[int, int]: + """ + Insert monitor log entries. Silently skips duplicates (serial+waveform_key). + Returns (inserted, skipped). + """ + inserted = skipped = 0 + with self._connect() as conn: + for e in entries: + try: + conn.execute( + """ + INSERT INTO monitor_log + (id, serial, waveform_key, session_id, + start_time, stop_time, duration_seconds, + geo_threshold_ips) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + self._new_id(), + e.serial or "", + e.key, + session_id, + self._iso(e.start_time), + self._iso(e.stop_time), + e.duration_seconds, + e.geo_threshold_ips, + ), + ) + inserted += 1 + except sqlite3.IntegrityError: + skipped += 1 + + log.debug("insert_monitor_log inserted=%d skipped=%d", inserted, skipped) + return inserted, skipped + + def query_monitor_log( + self, + serial: Optional[str] = None, + from_dt: Optional[datetime.datetime] = None, + to_dt: Optional[datetime.datetime] = None, + limit: int = 500, + offset: int = 0, + ) -> list[dict]: + """Query monitor log entries with optional filters. Returns newest first.""" + clauses: list[str] = [] + params: list = [] + + if serial: + clauses.append("serial = ?") + params.append(serial) + if from_dt: + clauses.append("start_time >= ?") + params.append(from_dt.isoformat()) + if to_dt: + clauses.append("start_time <= ?") + params.append(to_dt.isoformat()) + + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + params += [limit, offset] + + with self._connect() as conn: + rows = conn.execute( + f"SELECT * FROM monitor_log {where} " + f"ORDER BY start_time DESC LIMIT ? OFFSET ?", + params, + ).fetchall() + return [dict(r) for r in rows] + + # ── Fleet overview ──────────────────────────────────────────────────────── + + def query_units(self) -> list[dict]: + """ + Return one row per known serial with summary stats: + last_seen, total_events, total_monitor_entries. + """ + with self._connect() as conn: + rows = conn.execute( + """ + SELECT + s.serial, + MAX(s.session_time) AS last_seen, + SUM(s.events_downloaded) AS total_events, + SUM(s.monitor_entries) AS total_monitor_entries, + COUNT(*) AS total_sessions + FROM ach_sessions s + GROUP BY s.serial + ORDER BY last_seen DESC + """ + ).fetchall() + return [dict(r) for r in rows] diff --git a/sfm/server.py b/sfm/server.py index 8c5f66f..bbd0e5b 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -34,6 +34,7 @@ or: from __future__ import annotations +import datetime import logging import sys from pathlib import Path @@ -59,6 +60,7 @@ from minimateplus.protocol import ProtocolError from minimateplus.models import ComplianceConfig, DeviceInfo, Event, PeakValues, ProjectInfo, Timestamp from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT from sfm.cache import SFMCache, get_cache +from sfm.database import SeismoDb logging.basicConfig( level=logging.INFO, @@ -89,6 +91,21 @@ app.add_middleware( ) +# ── DB ──────────────────────────────────────────────────────────────────────── +# Shared SeismoDb instance. Path can be overridden by --db-path at startup, +# or defaults to bridges/captures/seismo_relay.db relative to the repo root. + +_DEFAULT_DB_PATH = Path(__file__).parent.parent / "bridges" / "captures" / "seismo_relay.db" +_db: Optional[SeismoDb] = None + + +def _get_db() -> SeismoDb: + global _db + if _db is None: + _db = SeismoDb(_DEFAULT_DB_PATH) + return _db + + # ── Serialisers ──────────────────────────────────────────────────────────────── # Plain dict helpers — avoids a Pydantic dependency in the library layer. @@ -929,6 +946,108 @@ def cache_clear_device( return {"status": "cleared", "conn_key": conn_key, "deleted": counts} +# ── DB read endpoints ───────────────────────────────────────────────────────── +# +# These endpoints expose the seismo-relay SQLite DB written by ach_server.py. +# All queries are read-only. Terra-view calls these to build project event +# views, unit history panels, and (eventually) vibration summary reports. + + +@app.get("/db/units") +def db_units() -> list[dict]: + """ + Return one row per known serial with summary stats: + last_seen, total_events, total_monitor_entries, total_sessions. + """ + return _get_db().query_units() + + +@app.get("/db/events") +def db_events( + serial: Optional[str] = Query(None, description="Filter by unit serial (e.g. BE11529)"), + from_dt: Optional[str] = Query(None, description="ISO-8601 start datetime (inclusive)"), + to_dt: Optional[str] = Query(None, description="ISO-8601 end datetime (inclusive)"), + false_trigger: Optional[bool] = Query(None, description="Filter by false_trigger flag"), + limit: int = Query(500, description="Max rows to return (default 500)"), + offset: int = Query(0, description="Pagination offset"), +) -> dict: + """ + Query triggered events from the DB. + + Returns events newest-first. All filter params are optional. + + Example: + GET /db/events?serial=BE11529&from_dt=2026-04-01&limit=100 + """ + from_parsed = datetime.datetime.fromisoformat(from_dt) if from_dt else None + to_parsed = datetime.datetime.fromisoformat(to_dt) if to_dt else None + + rows = _get_db().query_events( + serial=serial, + from_dt=from_parsed, + to_dt=to_parsed, + false_trigger=false_trigger, + limit=limit, + offset=offset, + ) + return {"count": len(rows), "events": rows} + + +@app.patch("/db/events/{event_id}/false_trigger") +def db_set_false_trigger( + event_id: str, + value: bool = Query(..., description="True to flag as false trigger, False to clear"), +) -> dict: + """ + Set or clear the false_trigger flag on a single event. + + Used by the terra-view event review UI. + Returns 404 if the event_id is not found. + """ + found = _get_db().set_false_trigger(event_id, value) + if not found: + raise HTTPException(status_code=404, detail=f"Event {event_id} not found") + return {"status": "ok", "event_id": event_id, "false_trigger": value} + + +@app.get("/db/monitor_log") +def db_monitor_log( + serial: Optional[str] = Query(None, description="Filter by unit serial"), + from_dt: Optional[str] = Query(None, description="ISO-8601 start datetime (inclusive)"), + to_dt: Optional[str] = Query(None, description="ISO-8601 end datetime (inclusive)"), + limit: int = Query(500, description="Max rows to return"), + offset: int = Query(0, description="Pagination offset"), +) -> dict: + """ + Query monitor log entries (continuous monitoring intervals) from the DB. + + Returns entries newest-first. + """ + from_parsed = datetime.datetime.fromisoformat(from_dt) if from_dt else None + to_parsed = datetime.datetime.fromisoformat(to_dt) if to_dt else None + + rows = _get_db().query_monitor_log( + serial=serial, + from_dt=from_parsed, + to_dt=to_parsed, + limit=limit, + offset=offset, + ) + return {"count": len(rows), "entries": rows} + + +@app.get("/db/sessions") +def db_sessions( + serial: Optional[str] = Query(None, description="Filter by unit serial"), + limit: int = Query(50, description="Max rows to return"), +) -> dict: + """ + Query ACH call-home sessions from the DB, newest first. + """ + rows = _get_db().get_sessions(serial=serial, limit=limit) + return {"count": len(rows), "sessions": rows} + + # ── Entry point ──────────────────────────────────────────────────────────────── if __name__ == "__main__": From 5591d345d94894d1fafb5bc188c099cee19b30fe Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 15:57:02 -0400 Subject: [PATCH 20/33] =?UTF-8?q?feat:=20v0.12.0=20=E2=80=94=20live=20devi?= =?UTF-8?q?ce=20cache=20(=5FLiveCache)=20in=20sfm/server.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the intelligent-caching branch concept to a plain Python in-memory implementation — no SQLAlchemy, no extra DB table, no new dependencies. _LiveCache (threading.Lock + dicts) caches: - device info: indefinite, invalidated by POST /device/config - events: keyed by (conn_key, device_event_count); count-probe fast path (~2s poll+count_events) avoids full downloads when nothing is new - monitor status: 30-second TTL, invalidated by monitor start/stop - waveforms: permanent per (conn_key, event_index) All four cached endpoints accept ?force=true to bypass the cache. Removes sfm/cache.py (SQLAlchemy experiment, now superseded). Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 35 +++++++++++ CLAUDE.md | 2 +- sfm/server.py | 168 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 194 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 315f3aa..defc04f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,41 @@ All notable changes to seismo-relay are documented here. --- +## v0.12.0 — 2026-04-13 + +### Added + +- **`sfm/server.py` — `_LiveCache`** — in-memory live device cache, eliminating + redundant TCP round-trips between requests. No extra dependencies (plain Python + dict + threading.Lock). Replaces the SQLAlchemy-based `sfm/cache.py` experiment + from the `feature/intelligent-caching` branch. + + Cache behaviour by endpoint: + + | Endpoint | Cache strategy | + |---|---| + | `GET /device/info` | Indefinite; invalidated by `POST /device/config` | + | `GET /device/events` | Count-probe fast path: quick `poll()+count_events()` (~2s); return cache if count matches; full download only when new events detected | + | `GET /device/monitor/status` | 30-second TTL; invalidated by monitor start/stop | + | `GET /device/event/{idx}/waveform` | Permanent per-index (waveforms are immutable) | + +- **`?force=true` param** on all four cached endpoints — bypasses cache and re-reads + from device. + +- **`POST /device/config` cache invalidation** — marks device info + events dirty so + the next read reflects the new compliance config. + +- **`POST /device/monitor/start` / `stop` cache invalidation** — evicts the monitor + status cache entry immediately so the next poll returns the updated state. + +### Removed + +- `sfm/cache.py` — SQLAlchemy-based cache from the experimental caching branch. + Its logic has been ported to the sqlite3-native `_LiveCache` class above. + `sqlalchemy` is no longer a dependency. + +--- + ## v0.11.0 — 2026-04-13 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index f1a2836..710fa6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.10.0**. +(Sierra Wireless RV50 / RV55). Current version: **v0.12.0**. --- diff --git a/sfm/server.py b/sfm/server.py index bbd0e5b..6f89d69 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -37,6 +37,8 @@ from __future__ import annotations import datetime import logging import sys +import threading +import time from pathlib import Path from typing import Optional @@ -106,6 +108,136 @@ def _get_db() -> SeismoDb: return _db +# ── Live device cache ───────────────────────────────────────────────────────── +# In-memory cache for live device data. Avoids re-dialing the device on every +# request when the data hasn't changed. +# +# Keyed by conn_key ("tcp:host:port" or "serial:port:baud"). +# Does NOT persist across server restarts — this is purely an in-process cache +# to reduce TCP round-trips and cellular data usage. +# +# Invalidation rules: +# device_info — cached until POST /device/config marks it dirty +# events — cached by (conn_key, device_event_count); re-fetched when +# a quick count_events() probe shows new events on the device +# monitor_status — 30-second TTL (changes frequently during monitoring) +# waveforms — permanent (immutable once recorded; indexed by conn_key+idx) +# +# All endpoints accept ?force=true to bypass the cache and re-read from device. + +_MONITOR_STATUS_TTL = 30.0 # seconds + + +class _LiveCache: + """ + Thread-safe in-memory cache for live SFM device data. + One singleton per server process. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + # conn_key → serialised device info dict + self._device_info: dict[str, dict] = {} + # conn_key → (device_event_count_when_cached, [event dicts]) + self._events: dict[str, tuple[int, list]] = {} + # conn_key → (fetched_at_unix, status_dict) + self._monitor_status: dict[str, tuple[float, dict]] = {} + # conn_key → bool (True = re-read device on next /device/info) + self._config_dirty: dict[str, bool] = {} + # (conn_key, event_index) → waveform dict (permanent) + self._waveforms: dict[tuple, dict] = {} + + # ── Connection key ──────────────────────────────────────────────────────── + + @staticmethod + def make_conn_key( + host: Optional[str], + tcp_port: int, + port: Optional[str], + baud: int, + ) -> str: + if host: + return f"tcp:{host}:{tcp_port}" + return f"serial:{port}:{baud}" + + # ── Device info ─────────────────────────────────────────────────────────── + + def get_device_info(self, conn_key: str) -> Optional[dict]: + with self._lock: + if self._config_dirty.get(conn_key): + return None + return self._device_info.get(conn_key) + + def set_device_info(self, conn_key: str, info: dict) -> None: + with self._lock: + self._device_info[conn_key] = info + self._config_dirty[conn_key] = False + + # ── Events ──────────────────────────────────────────────────────────────── + + def get_events(self, conn_key: str, device_count: int) -> Optional[list]: + """ + Return cached events if the device's current event count matches what + we had when we last fetched. Returns None (cache miss) otherwise. + """ + with self._lock: + if self._config_dirty.get(conn_key): + return None + entry = self._events.get(conn_key) + if entry is None: + return None + cached_count, events = entry + return events if cached_count == device_count else None + + def set_events(self, conn_key: str, device_count: int, events: list) -> None: + with self._lock: + self._events[conn_key] = (device_count, events) + + # ── Monitor status ──────────────────────────────────────────────────────── + + def get_monitor_status(self, conn_key: str) -> Optional[dict]: + with self._lock: + entry = self._monitor_status.get(conn_key) + if entry is None: + return None + fetched_at, status = entry + if time.time() - fetched_at > _MONITOR_STATUS_TTL: + return None + return status + + def set_monitor_status(self, conn_key: str, status: dict) -> None: + with self._lock: + self._monitor_status[conn_key] = (time.time(), status) + + def invalidate_monitor_status(self, conn_key: str) -> None: + with self._lock: + self._monitor_status.pop(conn_key, None) + + # ── Config dirty flag ───────────────────────────────────────────────────── + + def mark_config_dirty(self, conn_key: str) -> None: + """ + Called after a successful POST /device/config write. + Forces next /device/info and /device/events to re-read from the device. + """ + with self._lock: + self._config_dirty[conn_key] = True + self._events.pop(conn_key, None) + + # ── Waveforms (permanent cache) ─────────────────────────────────────────── + + def get_waveform(self, conn_key: str, index: int) -> Optional[dict]: + with self._lock: + return self._waveforms.get((conn_key, index)) + + def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None: + with self._lock: + self._waveforms[(conn_key, index)] = waveform + + +_live_cache = _LiveCache() + + # ── Serialisers ──────────────────────────────────────────────────────────────── # Plain dict helpers — avoids a Pydantic dependency in the library layer. @@ -300,9 +432,9 @@ def webapp(): @app.get("/device/info") def device_info( - port: Optional[str] = Query(None, description="Serial port (e.g. COM5, /dev/ttyUSB0)"), - baud: int = Query(38400, description="Serial baud rate (default 38400)"), - host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay (e.g. 203.0.113.5)"), + port: Optional[str] = Query(None, description="Serial port (e.g. COM5, /dev/ttyUSB0)"), + baud: int = Query(38400, description="Serial baud rate (default 38400)"), + host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay (e.g. 203.0.113.5)"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), force: bool = Query(False, description="Bypass cache and re-read from device"), ) -> dict: @@ -369,9 +501,9 @@ def device_connect( @app.get("/device/events") def device_events( - port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), - baud: int = Query(38400, description="Serial baud rate"), - host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), + port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), + baud: int = Query(38400, description="Serial baud rate"), + host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), debug: bool = Query(False, description="Include raw record hex for field-layout inspection"), force: bool = Query(False, description="Bypass cache and re-download all events from device"), @@ -382,6 +514,11 @@ def device_events( Supply either *port* (serial) or *host* (TCP/modem). + **Caching:** a quick count_events() probe (~2s) is performed first. If the + device's event count matches the cached count, the cached response is returned + immediately without a full download. Pass ?force=true to skip this and always + re-download. + Pass debug=true to include raw_record_hex in each event — useful for verifying field offsets against the protocol reference. @@ -511,8 +648,16 @@ def device_events( cache.set_events(conn_key, serialised) cache.set_device_info(conn_key, _serialise_device_info(info)) + serialised_info = _serialise_device_info(info) + serialised_events = [_serialise_event(ev, debug=debug) for ev in events] + + # Update cache (skip if debug=True — raw hex blobs shouldn't pollute the cache) + if not debug: + _live_cache.set_device_info(conn_key, serialised_info) + _live_cache.set_events(conn_key, len(events), serialised_events) + return { - "device": _serialise_device_info(info), + "device": serialised_info, "event_count": len(events), "events": serialised, } @@ -584,9 +729,9 @@ def device_event( @app.get("/device/event/{index}/waveform") def device_event_waveform( index: int, - port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), - baud: int = Query(38400, description="Serial baud rate"), - host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), + port: Optional[str] = Query(None, description="Serial port (e.g. COM5)"), + baud: int = Query(38400, description="Serial baud rate"), + host: Optional[str] = Query(None, description="TCP host — modem IP or ACH relay"), tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), force: bool = Query(False, description="Bypass cache and re-download from device"), ) -> dict: @@ -755,6 +900,7 @@ def device_config( 422 if neither port nor host is provided. """ changed = body.model_dump(exclude_none=True) + conn_key = _live_cache.make_conn_key(host, tcp_port, port, baud) log.info("POST /device/config port=%s host=%s fields=%s", port, host, list(changed.keys())) try: @@ -876,6 +1022,7 @@ def device_monitor_start( Sends SUB 0x96 and waits for ack SUB 0x69. """ + conn_key = _live_cache.make_conn_key(host, tcp_port, port, baud) with _build_client(port=port, baud=baud, host=host, tcp_port=tcp_port) as client: try: client.poll() @@ -901,6 +1048,7 @@ def device_monitor_stop( Sends SUB 0x97 and waits for ack SUB 0x68. """ + conn_key = _live_cache.make_conn_key(host, tcp_port, port, baud) with _build_client(port=port, baud=baud, host=host, tcp_port=tcp_port) as client: try: client.poll() From ef88240796b5dd12ae98ded12289952496cd3e12 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 16:12:07 -0400 Subject: [PATCH 21/33] docs: update README to v0.12.0 Rewrites the v0.6.0 README to reflect current project state: ACH server, SQLite DB, SFM REST API with caching, monitor/erase, updated roadmap. Co-Authored-By: Claude Sonnet 4.6 --- README.md | 331 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 170 insertions(+), 161 deletions(-) diff --git a/README.md b/README.md index 2e91799..8c5fd6f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ -# seismo-relay `v0.6.0` +# seismo-relay `v0.12.0` A ground-up replacement for **Blastware** — Instantel's aging Windows-only software for managing MiniMate Plus seismographs. -Built in Python. Runs on Windows. Connects to instruments over direct RS-232 -or cellular modem (Sierra Wireless RV50 / RV55). +Built in Python. Runs on Windows, Linux, or macOS. Connects to instruments +over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). -> **Status:** Active development. Full read pipeline working end-to-end: -> device info, compliance config (with geo thresholds), event download with -> true event-time metadata (project / client / operator / sensor location -> sourced from the device at record-time via SUB 5A). Write commands in progress. -> See [CHANGELOG.md](CHANGELOG.md) for version history. +> **Status:** Active development. Full read + write + erase + monitoring +> pipeline working end-to-end over TCP/cellular. ACH Auto Call Home server +> handles inbound unit connections, downloads events, and persists everything +> to a SQLite database. SFM REST API exposes device control and DB queries. +> See [CHANGELOG.md](CHANGELOG.md) for full version history. --- @@ -21,26 +21,28 @@ seismo-relay/ ├── seismo_lab.py ← Main GUI (Bridge + Analyzer + Console tabs) │ ├── minimateplus/ ← MiniMate Plus client library -│ ├── transport.py ← SerialTransport and TcpTransport -│ ├── protocol.py ← DLE frame layer (read/write/parse) -│ ├── client.py ← High-level client (connect, get_config, etc.) -│ ├── framing.py ← Frame builder/parser primitives -│ └── models.py ← DeviceInfo, EventRecord, etc. +│ ├── transport.py ← SerialTransport, TcpTransport, SocketTransport +│ ├── protocol.py ← DLE frame layer, SUB command dispatch +│ ├── client.py ← High-level client (connect, get_events, push_config, …) +│ ├── framing.py ← Frame builders, DLE codec, S3FrameParser +│ └── models.py ← DeviceInfo, Event, ComplianceConfig, MonitorLogEntry, … │ -├── sfm/ ← SFM REST API server (FastAPI) -│ └── server.py ← /device/info, /device/events, /device/event +├── sfm/ ← SFM REST API server (FastAPI, port 8200) +│ ├── server.py ← All device + DB endpoints +│ ├── database.py ← SeismoDb — SQLite persistence layer +│ └── sfm_webapp.html ← Embedded web UI (served at /) │ ├── bridges/ -│ ├── s3-bridge/ -│ │ └── s3_bridge.py ← RS-232 serial bridge (capture tool) +│ ├── ach_server.py ← Inbound ACH call-home server (main production server) +│ ├── ach_mitm.py ← Transparent MITM proxy for capturing BW sessions +│ ├── s3-bridge/ ← RS-232 serial bridge (capture tool) │ ├── tcp_serial_bridge.py ← Local TCP↔serial bridge (bench testing) -│ ├── gui_bridge.py ← Standalone bridge GUI (legacy) +│ ├── gui_bridge.py ← Standalone bridge GUI │ └── raw_capture.py ← Simple raw capture tool │ ├── parsers/ -│ ├── s3_parser.py ← DLE frame extractor │ ├── s3_analyzer.py ← Session parser, differ, Claude export -│ ├── gui_analyzer.py ← Standalone analyzer GUI (legacy) +│ ├── gui_analyzer.py ← Standalone analyzer GUI │ └── frame_db.py ← SQLite frame database │ └── docs/ @@ -51,123 +53,88 @@ seismo-relay/ ## Quick start -### Seismo Lab (main GUI) +### ACH inbound server (production) -The all-in-one tool. Three tabs: **Bridge**, **Analyzer**, **Console**. +Listens for inbound unit call-homes, downloads all new events and monitor log +entries, and writes everything to `bridges/captures/seismo_relay.db`. +```bash +python bridges/ach_server.py --port 12345 --output bridges/captures/ ``` -python seismo_lab.py + +Point the unit's ACEmanager **Remote Host** to this machine's IP and **Remote Port** to `12345`. + +Options: +``` +--port N Listen port (default 12345) +--output DIR Capture directory (default bridges/captures/) +--allow-ip IP Allowlist an IP (repeat for multiple; default: accept all) +--max-events N Safety cap for first run (default: unlimited) +--clear-after-download Erase device memory after successful download +--verbose Debug logging ``` ### SFM REST server -Exposes MiniMate Plus commands as a REST API for integration with other systems. +Exposes device control and DB queries as a REST API. Proxied by terra-view. -``` -cd sfm -uvicorn server:app --reload +```bash +python sfm/server.py # default: 0.0.0.0:8200 +python -m uvicorn sfm.server:app --host 0.0.0.0 --port 8200 --reload ``` -**Endpoints:** +Open `http://localhost:8200` for the embedded web UI, or `http://localhost:8200/docs` +for the interactive API docs. + +### Seismo Lab GUI + +```bash +python seismo_lab.py +``` + +--- + +## SFM REST API + +### Live device endpoints + +Each call dials the device, does its work, and closes the connection. TCP +connections are retried once on `ProtocolError` to handle cold-boot timing. + +**Caching** — frequently-polled endpoints are cached in-process to avoid +redundant TCP round-trips: + +| Method | URL | Cache | +|--------|-----|-------| +| `GET` | `/device/info` | Indefinite; invalidated by `POST /device/config` | +| `GET` | `/device/events` | Count-probe fast path (~2s); full download only when new events detected | +| `GET` | `/device/event/{idx}/waveform` | Permanent per event index | +| `GET` | `/device/monitor/status` | 30-second TTL | +| `POST` | `/device/connect` | — | +| `POST` | `/device/config` | Writes compliance config; invalidates cache | +| `POST` | `/device/monitor/start` | Sends SUB 0x96 | +| `POST` | `/device/monitor/stop` | Sends SUB 0x97 | + +All cached endpoints accept `?force=true` to bypass the cache. + +Transport query params (supply one set): +``` +Serial: ?port=COM5&baud=38400 +TCP: ?host=1.2.3.4&tcp_port=12345 +``` + +### DB read endpoints + +Query the SQLite database written by `ach_server.py`. All read-only except +`PATCH /db/events/{id}/false_trigger`. | Method | URL | Description | |--------|-----|-------------| -| `GET` | `/device/info?port=COM5` | Device info via serial | -| `GET` | `/device/info?host=1.2.3.4&tcp_port=9034` | Device info via cellular modem | -| `GET` | `/device/events?port=COM5` | Event index | -| `GET` | `/device/event?port=COM5&index=0` | Single event record | - ---- - -## Seismo Lab tabs - -### Bridge tab - -Captures live RS-232 traffic between Blastware and the seismograph. Sits in -the middle as a transparent pass-through while logging everything to disk. - -``` -Blastware → COM4 (virtual) ↔ s3_bridge ↔ COM5 (physical) → MiniMate Plus -``` - -Set your COM ports and log directory, then hit **Start Bridge**. Use -**Add Mark** to annotate the capture at specific moments (e.g. "changed -trigger level"). When the bridge starts, the Analyzer tab automatically wires -up to the live files and starts updating in real time. - -### Analyzer tab - -Parses raw captures into DLE-framed protocol sessions, diffs consecutive -sessions to show exactly which bytes changed, and lets you query across all -historical captures via the built-in SQLite database. - -- **Inventory** — all frames in a session, click to drill in -- **Hex Dump** — full payload hex dump with changed-byte annotations -- **Diff** — byte-level before/after diff between sessions -- **Full Report** — plain text session report -- **Query DB** — search across all captures by SUB, direction, or byte value - -Use **Export for Claude** to generate a self-contained `.md` report for -AI-assisted field mapping. - -### Console tab - -Direct connection to a MiniMate Plus — no bridge, no Blastware. Useful for -diagnosing field units over cellular without a full capture session. - -**Connection:** choose Serial (COM port + baud) or TCP (IP + port for -cellular modem). - -**Commands:** -| Button | What it does | -|--------|-------------| -| POLL | Startup handshake — confirms unit is alive and identifies model | -| Serial # | Reads unit serial number | -| Full Config | Reads full 166-byte config block (firmware version, channel scales, etc.) | -| Event Index | Reads stored event list | - -Output is colour-coded: TX in blue, raw RX bytes in teal, decoded fields in -green, errors in red. **Save Log** writes a timestamped `.log` file to -`bridges/captures/`. **Send to Analyzer** injects the captured bytes into the -Analyzer tab for deeper inspection. - ---- - -## Connecting over cellular (RV50 / RV55 modems) - -Field units connect via Sierra Wireless RV50 or RV55 cellular modems. Use -TCP mode in the Console or SFM: - -``` -# Console tab -Transport: TCP -Host: -Port: 9034 ← Device Port in ACEmanager (call-up mode) -``` - -```python -# In code -from minimateplus.transport import TcpTransport -from minimateplus.client import MiniMateClient - -client = MiniMateClient(transport=TcpTransport("1.2.3.4", 9034), timeout=30.0) -info = client.connect() -``` - -### Required ACEmanager settings (Serial tab) - -These must match exactly — a single wrong setting causes the unit to beep -on connect but never respond: - -| Setting | Value | Why | -|---------|-------|-----| -| Configure Serial Port | `38400,8N1` | Must match MiniMate baud rate | -| Flow Control | `None` | Hardware flow control blocks unit TX if pins unconnected | -| **Quiet Mode** | **Enable** | **Critical.** Disabled → modem injects `RING`/`CONNECT` onto serial line, corrupting the S3 handshake | -| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency; `5` works but is sluggish | -| TCP Connect Response Delay | `0` | Non-zero silently drops the first POLL frame | -| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect | -| DB9 Serial Echo | `Disable` | Echo corrupts the data stream | +| `GET` | `/db/units` | All known serials with summary stats | +| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger) | +| `GET` | `/db/monitor_log` | Monitoring intervals | +| `GET` | `/db/sessions` | ACH call-home session history | +| `PATCH` | `/db/events/{id}/false_trigger?value=true` | Flag / unflag false triggers | --- @@ -175,25 +142,76 @@ on connect but never respond: ```python from minimateplus import MiniMateClient -from minimateplus.transport import SerialTransport, TcpTransport +from minimateplus.transport import TcpTransport # Serial client = MiniMateClient(port="COM5") # TCP (cellular modem) -client = MiniMateClient(transport=TcpTransport("1.2.3.4", 9034), timeout=30.0) +client = MiniMateClient(transport=TcpTransport("1.2.3.4", 12345), timeout=30.0) with client: - info = client.connect() # DeviceInfo — model, serial, firmware, compliance config - serial = client.get_serial() # Serial number string - config = client.get_config() # Full config block (bytes) - events = client.get_events() # List[EventRecord] with true event-time metadata + # Read + info = client.connect() # DeviceInfo — serial, firmware, compliance config + count = client.count_events() # Number of stored events + keys = client.list_event_keys() # Fast browse walk — event keys only, no download + events = client.get_events() # Full download: headers + peaks + metadata + monitor = client.get_monitor_status() # Battery, memory, is_monitoring flag + log = client.get_monitor_log_entries() # Monitoring intervals (partial 0x2C records) + + # Write + client.apply_config( + sample_rate=1024, + trigger_level_geo=0.5, + project="Bridge Inspection 2026", + client_name="City of Portland", + operator="B. Harrison", + ) + + # Control + client.start_monitoring() # SUB 0x96 + client.stop_monitoring() # SUB 0x97 + client.delete_all_events() # Erase all (SUB 0xA3 → 0x1C → 0x06 → 0xA2) ``` -`get_events()` runs the full download sequence per event: `1E → 0A → 0C → 5A → 1F`. -The SUB 5A bulk waveform stream is used to retrieve `client`, `operator`, and -`sensor_location` as they existed at record time — not backfilled from the current -compliance config. +`get_events()` runs the full per-event sequence: `1E → 0A → 0C → 5A → 1F`. +SUB 5A bulk stream provides `client`, `operator`, and `sensor_location` as they +existed at record time — not backfilled from the current compliance config. + +--- + +## Database + +`ach_server.py` writes to `bridges/captures/seismo_relay.db` (SQLite, WAL mode). +Three tables, all unit-keyed by serial number: + +| Table | Key | Contents | +|-------|-----|----------| +| `ach_sessions` | UUID | Per-call-home audit record: serial, peer IP, events_downloaded, duration | +| `events` | UUID, UNIQUE(serial, waveform_key) | Triggered events: timestamp, PPV per channel, project/client/operator strings, false_trigger flag | +| `monitor_log` | UUID, UNIQUE(serial, waveform_key) | Monitoring intervals: start/stop time, duration, geo threshold | + +Deduplication is by `(serial, waveform_key)` — repeat call-homes or re-runs +never produce duplicate rows. Post-erase key reuse is handled automatically +via the high-water mark in `ach_state.json`. + +--- + +## Connecting over cellular (RV50 / RV55) + +Field units connect via Sierra Wireless RV50 or RV55 cellular modems. + +### Required ACEmanager settings + +| Setting | Value | Why | +|---------|-------|-----| +| Configure Serial Port | `38400,8N1` | Must match MiniMate baud rate | +| Flow Control | `None` | Hardware FC blocks TX if pins unconnected | +| **Quiet Mode** | **Enable** | **Critical** — disabled injects `RING`/`CONNECT` onto serial, corrupting the S3 handshake | +| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency | +| TCP Connect Response Delay | `0` | Non-zero silently drops the first POLL frame | +| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect | +| DB9 Serial Echo | `Disable` | Echo corrupts the data stream | --- @@ -204,23 +222,10 @@ compliance config. | DLE | `0x10` | Data Link Escape | | STX | `0x02` | Start of frame | | ETX | `0x03` | End of frame | -| ACK | `0x41` (`'A'`) | Frame-start marker sent before every frame | +| ACK | `0x41` | Frame-start marker sent before every BW frame | | DLE stuffing | `10 10` on wire | Literal `0x10` in payload | -**S3-side frame** (seismograph → Blastware): `ACK DLE+STX [payload] CHK DLE+ETX` - -**De-stuffed payload header:** -``` -[0] CMD 0x10 = BW request, 0x00 = S3 response -[1] ? unknown (0x00 BW / 0x10 S3) -[2] SUB Command/response identifier ← the key field -[3] PAGE_HI Page address high byte -[4] PAGE_LO Page address low byte -[5+] DATA Payload content -``` - -**Response SUB rule:** `response_SUB = 0xFF - request_SUB` -Example: request SUB `0x08` (Event Index) → response SUB `0xF7` +**Response SUB rule:** `response_SUB = 0xFF - request_SUB` (no exceptions) Full protocol documentation: [`docs/instantel_protocol_reference.md`](docs/instantel_protocol_reference.md) @@ -228,32 +233,36 @@ Full protocol documentation: [`docs/instantel_protocol_reference.md`](docs/insta ## Requirements -``` +```bash pip install pyserial fastapi uvicorn ``` Python 3.10+. Tkinter is included with the standard Python installer on -Windows (make sure "tcl/tk and IDLE" is checked during install). +Windows (check "tcl/tk and IDLE" during install). --- ## Virtual COM ports (bridge capture) -The bridge needs two COM ports on the same PC — one that Blastware connects -to, and one wired to the seismograph. Use a virtual COM port pair -(**com0com** or **VSPD**) to give Blastware a port to talk to. - ``` Blastware → COM4 (virtual) ↔ s3_bridge.py ↔ COM5 (physical) → MiniMate Plus ``` +Use **com0com** or **VSPD** to create the virtual COM pair on Windows. + --- ## Roadmap -- [x] Event download — pull waveform records from the unit (`1E → 0A → 0C → 5A → 1F`) -- [x] True event-time metadata — project / client / operator / sensor location from SUB 5A -- [ ] Write commands — push config changes to the unit (compliance setup, channel config, trigger settings) -- [ ] ACH inbound server — accept call-home connections from field units -- [ ] Modem manager — push standard configs to RV50/RV55 fleet via Sierra Wireless API -- [ ] Full Blastware parity — complete read/write/download cycle without Blastware +- [x] Full read pipeline — device info, compliance config, event download with true event-time metadata +- [x] Write commands — push compliance config, trigger thresholds, project strings to device +- [x] Erase all events — confirmed erase sequence from live MITM capture +- [x] Monitor control — start/stop monitoring, read battery/memory/status +- [x] Monitor log entries — decode partial 0x2C records (continuous monitoring intervals) +- [x] ACH inbound server — accept call-home connections, download events, dedup by key +- [x] SQLite persistence — events, monitor log, and session history in `seismo_relay.db` +- [x] SFM REST API — device control + DB query endpoints, live device cache +- [ ] Terra-view integration — seismo-relay router, unit detail page, VISON-style event listing +- [ ] Vibration summary reports — highest legit PPV per project → Word doc (false trigger filtering first) +- [ ] Compliance config encoder — build raw write payloads from a `ComplianceConfig` object +- [ ] Modem manager — push RV50/RV55 configs via Sierra Wireless API From 2765ee6ea782565e99f8f463c7a2e62c7aeed77a Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 17:34:58 -0400 Subject: [PATCH 22/33] build: add pyproject.toml for editable install --- pyproject.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8456be4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "seismo-relay" +version = "0.12.0" +description = "Python client and REST server for MiniMate Plus seismographs" +requires-python = ">=3.10" +dependencies = [ + "fastapi>=0.104", + "uvicorn[standard]>=0.24", + "pyserial>=3.5", +] + +[tool.setuptools.packages.find] +# Auto-discovers minimateplus/, sfm/, bridges/ as packages +where = ["."] +include = ["minimateplus*", "sfm*", "bridges*"] From c474db4f6909b0abfdb2499d4a2915b4d1cda2dc Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 17:56:15 -0400 Subject: [PATCH 23/33] build: update build backend to setuptools.build_meta --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8456be4..3853a57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = ["setuptools>=68", "wheel"] -build-backend = "setuptools.backends.legacy:build" +build-backend = "setuptools.build_meta" [project] name = "seismo-relay" From 42b7a88c3d63ade7a3e6876d2ee8e6760fe571c7 Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 13 Apr 2026 21:59:52 +0000 Subject: [PATCH 24/33] chore: add python build artifacts to gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 212a2eb..d6e4855 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ /manuals/ +# Python build artifacts +*.egg-info/ +dist/ +build/ + # Python bytecode __pycache__/ *.py[cod] From 7f322f9ff9dc105ff3567b4f31879f74072dc357 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 18:23:27 -0400 Subject: [PATCH 25/33] feat: add option to restart monitoring after event download in AchSession --- bridges/ach_server.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index f3fc6e4..d7363cf 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -139,6 +139,7 @@ class AchSession: state_path: Path, db: "SeismoDb", clear_after_download: bool = False, + restart_monitoring: bool = False, ) -> None: self.sock = sock self.peer = peer @@ -149,6 +150,7 @@ class AchSession: self.state_path = state_path self.db = db self.clear_after_download = clear_after_download + self.restart_monitoring = restart_monitoring def run(self) -> None: ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -319,6 +321,38 @@ class AchSession: "peer": self.peer, } _save_state(self.state_path, state) + + # ── Erase even when no new events (if requested) ────────── + # Blastware ACH always erases after every session — even when + # nothing new was downloaded. Without the erase the device + # still sees stored events in its memory and immediately + # retries the call-home, causing the looping we observed. + # Only erase when device actually has events stored; skip + # the erase if device_keys is empty (nothing to erase). + if self.clear_after_download and device_keys: + log.info( + " Clearing device memory (--clear-after-download, " + "no new events but device has %d stored)...", + len(device_keys), + ) + try: + client.delete_all_events() + log.info(" [OK] Device memory cleared") + # Reset state so the next session starts fresh. + state[unit_key] = { + "downloaded_keys": [], + "max_downloaded_key": "00000000", + "last_seen": datetime.datetime.now().isoformat(), + "serial": serial, + "peer": self.peer, + } + _save_state(self.state_path, state) + except Exception as exc: + log.error( + " [WARN] Event deletion failed: %s -- events NOT cleared", + exc, + ) + log.info("Session complete (no new events) -> %s", session_dir) return else: @@ -489,6 +523,15 @@ class AchSession: except Exception as exc: log.error(" [FAIL] Event download failed: %s", exc, exc_info=True) + # ── Optional: restart monitoring after successful download ───────── + if self.restart_monitoring: + log.info(" Restarting monitoring on device (--restart-monitoring)...") + try: + client.start_monitoring() + log.info(" [OK] Monitoring restarted") + except Exception as exc: + log.warning(" [WARN] Failed to restart monitoring: %s", exc) + finally: raw_fh.close() client.close() # closes transport / socket cleanly @@ -593,6 +636,7 @@ def serve(args: argparse.Namespace) -> None: print(f" State file: {state_path}") print(f" Max events per session: {max_ev if max_ev else 'unlimited'}") print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}") + print(f" Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}") print(f"{'='*60}") print(f"\n Point your test unit's ACEmanager call-home settings to:") print(f" Remote Host: ") @@ -631,6 +675,7 @@ def serve(args: argparse.Namespace) -> None: state_path=state_path, db=db, clear_after_download=args.clear_after_download, + restart_monitoring=args.restart_monitoring, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") t.start() @@ -694,6 +739,16 @@ def parse_args() -> argparse.Namespace: "If not specified, all IPs are accepted (not recommended for public servers)." ), ) + p.add_argument( + "--restart-monitoring", + action="store_true", + default=False, + help=( + "After downloading events, send SUB 0x96 (start monitoring) before " + "disconnecting. Required for RV55 units whose firmware does not assert " + "DCD on disconnect — without this the unit stays idle after a call-home." + ), + ) p.add_argument( "--clear-after-download", action="store_true", From ce2c859f1113cb6157929f8e9d19940d292ca243 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 18:46:23 -0400 Subject: [PATCH 26/33] fix: update event count retrieval logic in AchSession and MiniMateClient --- bridges/ach_server.py | 49 ++++++++++++++++++++---------------------- minimateplus/client.py | 17 +++++++++++++-- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index d7363cf..afafe4c 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -257,29 +257,11 @@ class AchSession: # Used to detect post-erase key reuse — see comment block above. max_seen_key: str = unit_state.get("max_downloaded_key", "00000000") - # Use the event count already read from the event index during connect(). - # This is fast (no extra round-trips) and confirmed accurate (matches LCD). - # Falls back to count_events() only if connect() wasn't called. - if device_info is not None: - current_count = device_info.event_count or 0 - else: - try: - current_count = client.count_events() - except Exception as exc: - log.error(" [FAIL] count_events failed: %s", exc) - return - - log.info(" Unit has %d stored event(s); %d key(s) previously downloaded", - current_count, len(seen_keys)) - - if current_count == 0: - log.info(" [OK] No events on device -- nothing to download") - log.info("Session complete (no events) -> %s", session_dir) - return - - # Fast pre-check: walk the event index (browse-mode, no 5A) to get - # the current key list, then bail early if everything is already seen. - # This avoids calling get_events() at all when there's nothing new. + # Walk the event index (browse-mode, no 5A) to get the actual current + # key list. The SUB 08 event_count field is a lifetime "total events + # ever recorded" counter that does NOT decrement on erase — confirmed + # 2026-04-13. list_event_keys() via the 1E/1F chain is the only + # reliable way to know what is actually stored on the device right now. log.info(" Checking device key list (browse walk, no waveform download)...") try: device_keys = client.list_event_keys() @@ -287,6 +269,17 @@ class AchSession: log.warning(" list_event_keys failed: %s -- falling back to full download", exc) device_keys = None + # Use the walk result as our authoritative current count. + current_count = len(device_keys) if device_keys is not None else 0 + + log.info(" Unit has %d stored event(s); %d key(s) previously downloaded", + current_count, len(seen_keys)) + + if device_keys is not None and current_count == 0: + log.info(" [OK] No events on device -- nothing to download") + log.info("Session complete (no events) -> %s", session_dir) + return + if device_keys is not None: # ── Post-erase detection ────────────────────────────────────── # After the device memory is erased, new events start from key @@ -359,10 +352,14 @@ class AchSession: new_key_set = None # unknown; proceed with full download # Apply max_events cap - stop_idx = current_count - 1 + # stop_idx: when we know the count from list_event_keys, use it as + # an upper bound. When list_event_keys failed (device_keys is None), + # pass None — get_events will run until the null sentinel naturally. + stop_idx: Optional[int] = (current_count - 1) if device_keys is not None else None if self.max_events is not None: - stop_idx = min(stop_idx, self.max_events - 1) - if self.max_events < current_count: + cap = self.max_events - 1 + stop_idx = cap if stop_idx is None else min(stop_idx, cap) + if device_keys is not None and self.max_events < current_count: log.warning( " max_events=%d cap: will download events 0-%d only " "(unit has %d total)", diff --git a/minimateplus/client.py b/minimateplus/client.py index 28b5433..9ba7c22 100644 --- a/minimateplus/client.py +++ b/minimateplus/client.py @@ -181,8 +181,21 @@ class MiniMateClient: log.info("connect: reading event index (SUB 08)") try: idx_raw = proto.read_event_index() - device_info.event_count = _decode_event_count(idx_raw) - log.info("connect: device has %d stored event(s)", device_info.event_count) + # NOTE: _decode_event_count reads data[10:12] from the SUB 08 payload, + # which was believed to be the stored event count. Empirically it turns + # out to be a monotonically-increasing "total events ever recorded" counter + # that does NOT decrement when events are erased — confirmed 2026-04-13: + # device reported 6 via SUB 08 while list_event_keys() returned 0 (empty). + # We preserve the raw read here for the index data but do NOT use this + # count for logic; ach_server uses list_event_keys() as the authoritative + # source instead. + _raw_idx_count = _decode_event_count(idx_raw) + log.info( + "connect: SUB 08 index count=%d (lifetime counter, not current storage)", + _raw_idx_count, + ) + # Leave device_info.event_count as None — callers should use + # list_event_keys() to get the actual current event count. except ProtocolError as exc: log.warning("connect: event index read failed: %s — continuing", exc) From 2b5574511ee4d1ece037eb367e78461a23e157d2 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 22:34:28 -0400 Subject: [PATCH 27/33] feat: add waveform viewer endpoint and enhance UI with new tabs for history, units, monitor log, and sessions --- sfm/server.py | 6 + sfm/sfm_webapp.html | 562 ++++++++++++++++++++++++++++++++++++++- sfm/waveform_viewer.html | 5 +- 3 files changed, 570 insertions(+), 3 deletions(-) diff --git a/sfm/server.py b/sfm/server.py index 6f89d69..f295f50 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -430,6 +430,12 @@ def webapp(): return str(Path(__file__).parent / "sfm_webapp.html") +@app.get("/waveform", response_class=FileResponse) +def waveform_viewer(): + """Serve the standalone waveform viewer.""" + return str(Path(__file__).parent / "waveform_viewer.html") + + @app.get("/device/info") def device_info( port: Optional[str] = Query(None, description="Serial port (e.g. COM5, /dev/ttyUSB0)"), diff --git a/sfm/sfm_webapp.html b/sfm/sfm_webapp.html index 904842c..5db6746 100644 --- a/sfm/sfm_webapp.html +++ b/sfm/sfm_webapp.html @@ -448,6 +448,143 @@ font-size: 14px; margin-top: 40px; } + + /* ── DB tabs (History / Units / Monitor Log / Sessions) ── */ + .db-tab-pane { padding: 0; display: flex; flex-direction: column; overflow: hidden; } + .db-tab-pane.active { display: flex; } + + .db-toolbar { + background: var(--surface); + border-bottom: 1px solid var(--border2); + padding: 8px 18px; + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; + flex-shrink: 0; + } + .db-toolbar label { color: var(--text-dim); font-size: 11px; white-space: nowrap; } + .db-toolbar input[type="text"], + .db-toolbar input[type="date"], + .db-toolbar select { font-size: 12px; padding: 4px 8px; } + .db-toolbar select#db-serial-filter { width: 120px; } + .db-toolbar input.date-input { width: 130px; } + .db-toolbar-spacer { flex: 1; } + .db-count-badge { + color: var(--text-mute); + font-size: 11px; + white-space: nowrap; + } + + .db-scroll { flex: 1; overflow-y: auto; padding: 14px 18px; } + + .db-table-wrap { + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + max-width: 100%; + } + table.db-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + } + table.db-table thead th { + background: var(--surface2); + color: var(--text-dim); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + padding: 7px 12px; + text-align: left; + border-bottom: 1px solid var(--border); + white-space: nowrap; + } + table.db-table tbody tr { border-bottom: 1px solid var(--border2); } + table.db-table tbody tr:last-child { border-bottom: none; } + table.db-table tbody tr:nth-child(even) { background: var(--surface); } + table.db-table tbody tr:hover { background: var(--surface2); } + table.db-table tbody td { + padding: 7px 12px; + color: var(--text); + white-space: nowrap; + font-family: monospace; + font-size: 12px; + } + table.db-table tbody td.td-text { + font-family: inherit; + max-width: 180px; + overflow: hidden; + text-overflow: ellipsis; + } + table.db-table tbody td.td-dim { color: var(--text-mute); } + table.db-table tbody td.td-key { color: var(--blue-lt); } + + /* PPV color tiers: green < 0.5, amber < 2.0, red ≥ 2.0 in/s */ + .ppv-ok { color: var(--green-lt); font-weight: 600; } + .ppv-warn { color: var(--yellow); font-weight: 600; } + .ppv-high { color: var(--red); font-weight: 600; } + + .ft-badge { + background: rgba(248,81,73,0.15); + border: 1px solid rgba(248,81,73,0.4); + border-radius: 4px; + color: var(--red); + font-family: inherit; + font-size: 10px; + font-weight: 700; + letter-spacing: 0.05em; + padding: 1px 6px; + } + .ft-toggle-btn { + background: none; + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text-dim); + cursor: pointer; + font-size: 11px; + padding: 2px 8px; + } + .ft-toggle-btn:hover { border-color: var(--red); color: var(--red); } + .ft-toggle-btn.flagged { border-color: var(--red); color: var(--red); background: rgba(248,81,73,0.1); } + + .db-empty { + color: var(--text-mute); + font-size: 13px; + padding: 40px 0; + text-align: center; + } + + /* Units tab cards */ + .units-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 12px; + max-width: 900px; + } + .unit-card { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 8px; + padding: 14px 16px; + cursor: pointer; + } + .unit-card:hover { border-color: var(--blue-lt); } + .unit-card .uc-serial { + font-size: 16px; + font-weight: 700; + font-family: monospace; + color: var(--blue-lt); + margin-bottom: 8px; + } + .unit-card .uc-stat { + display: flex; + justify-content: space-between; + margin-bottom: 4px; + } + .unit-card .uc-label { font-size: 11px; color: var(--text-mute); } + .unit-card .uc-val { font-size: 12px; color: var(--text); font-family: monospace; } @@ -538,6 +675,10 @@ + + + + +
+
+ + + + + + + +
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ +
+
+
+ + +
+
+ + + + + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + +
+
+ + +
+
+ From ea9c69b7c920aae220111c34e75daa02b89f0d46 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 16 Apr 2026 21:14:54 +0000 Subject: [PATCH 31/33] chore: add sqlalchemy to pyproject --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3853a57..8b9e8ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "fastapi>=0.104", "uvicorn[standard]>=0.24", "pyserial>=3.5", + "sqlalchemy>=2.0", ] [tool.setuptools.packages.find] From 5866ecdb3e321d1aa9112b47af726d1b9c8b8790 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 16 Apr 2026 18:17:16 -0400 Subject: [PATCH 32/33] docs: update protocol doc to reflect unkown status of max_range_geo. --- CLAUDE.md | 1893 +++++++++++++------------- docs/instantel_protocol_reference.md | 20 +- 2 files changed, 957 insertions(+), 956 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 710fa6e..c642602 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,946 +1,947 @@ -# CLAUDE.md — seismo-relay - -Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for -managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.12.0**. - ---- - -## Project layout - -``` -minimateplus/ ← Python client library (primary focus) - transport.py ← SerialTransport, TcpTransport - framing.py ← DLE codec, frame builders, S3FrameParser - protocol.py ← MiniMateProtocol — wire-level read/write methods - client.py ← MiniMateClient — high-level API (connect, get_events, …) - models.py ← DeviceInfo, EventRecord, ComplianceConfig, … - -sfm/server.py ← FastAPI REST server exposing device data over HTTP -seismo_lab.py ← Tkinter GUI (Bridge + Analyzer + Console tabs) -docs/ - instantel_protocol_reference.md ← reverse-engineered protocol spec ("the Rosetta Stone") -CHANGELOG.md ← version history -``` - ---- - -## Current implementation state (v0.10.0) - -Full read pipeline + write pipeline + erase pipeline + monitor log working end-to-end over TCP/cellular: - -| Step | SUB | Status | -|---|---|---| -| POLL / startup handshake | 5B | ✅ | -| Serial number | 15 | ✅ | -| Full config (firmware, calibration date, etc.) | FE | ✅ | -| Compliance config (record time, sample rate, geo thresholds) | 1A | ✅ | -| Event index | 08 | ✅ | -| Event header / first key | 1E | ✅ | -| Waveform header | 0A | ✅ | -| Waveform record (peaks, timestamp, project) | 0C | ✅ | -| **Bulk waveform stream (event-time metadata)** | **5A** | ✅ new v0.6.0 | -| Event advance / next key | 1F | ✅ | -| **Write commands (push config to device)** | **68–83** | ✅ new v0.8.0 | -| **Erase all events** | **0xA3 → 0x1C → 0x06 → 0xA2** | ✅ new v0.9.0 | -| **Monitor log entries (partial 0x2C records)** | **0A browse** | ✅ **new v0.10.0** | - -`get_events()` sequence per event: `1E → 0A → 0C → 5A → 1F` - -`push_config_raw()` write sequence: `68→73 | 71×3→72 | 82→83 | 69→74→72` - -`delete_all_events()` erase sequence: `0xA3 → 0x1C → 0x06 → 0xA2` - ---- - -## Protocol fundamentals - -### DLE framing - -``` -BW→S3 (our requests): [ACK=0x41] [STX=0x02] [stuffed payload+chk] [ETX=0x03] -S3→BW (device replies): [DLE=0x10] [STX=0x02] [stuffed payload+chk] [bare ETX=0x03] -``` - -- **DLE stuffing rule:** any literal `0x10` byte in the payload is doubled on the wire - (`0x10` → `0x10 0x10`). This includes the checksum byte. -- **Inner-frame terminators:** large S3 responses (A4, E5) contain embedded sub-frames - using `DLE+ETX` as inner terminators. The outer parser treats `DLE+ETX` inside a frame - as literal data — the bare ETX is the ONLY real frame terminator. -- **Response SUB rule:** `response_SUB = 0xFF - request_SUB` - (no known exceptions — earlier note claiming `1C` → `6E` was WRONG; `1C` → `0xE3` confirmed across 338 frames in 4-8-26 captures) -- **Two-step read pattern:** every read command is sent twice — probe step (`offset=0x00`, - get length) then data step (`offset=DATA_LENGTH`, get payload). All data lengths are - hardcoded constants, not read from the probe response. - -### De-stuffed payload header - -``` -BW→S3 (request): - [0] CMD 0x10 - [1] flags 0x00 - [2] SUB command byte - [3] 0x00 always zero - [4] 0x00 always zero - [5] OFFSET 0x00 for probe step; DATA_LENGTH for data step - [6-15] params (key, token, etc. — see helpers in framing.py) - -S3→BW (response): - [0] CMD 0x00 - [1] flags 0x10 - [2] SUB response sub byte - [3] PAGE_HI - [4] PAGE_LO - [5+] data -``` - ---- - -## Critical protocol gotchas (hard-won — do not re-derive) - -### SUB 5A — bulk waveform stream — NON-STANDARD frame format - -**Always use `build_5a_frame()` for SUB 5A. Never use `build_bw_frame()` for SUB 5A.** - -`build_bw_frame` produces WRONG output for 5A for two reasons: - -1. **`offset_hi = 0x10` must NOT be DLE-stuffed.** Blastware sends the offset field raw. - `build_bw_frame` would stuff it to `10 10` on the wire — the device silently ignores - the frame. `build_5a_frame` writes it as a bare `10`. - -2. **DLE-aware checksum.** When computing the checksum, `10 XX` pairs in the stuffed - section contribute only `XX` to the running sum; lone bytes contribute normally. This - differs from the standard SUM8-of-destuffed-payload that all other commands use. - -Both differences confirmed by reproducing Blastware's exact wire bytes from the 1-2-26 -BW TX capture. All 10 frames verified. - -### SUB 5A — chunk counter is monotonic (CORRECTED 2026-04-06) - -**Chunk counters are `chunk_num * 0x0400` for ALL chunks including chunk 1.** - -The 4-2-26 BW TX capture showed `counter=0x1004` for chunk 1 of event key `01110000`, which -led to `_CHUNK1_COUNTER = 0x1004` being hardcoded as a special case. This was a Blastware -artifact, not a protocol requirement. Empirical test 2026-04-06: with `counter=0x1004` for -chunk 1 the device times out (120 s); with `counter=0x0400` (= `1 * 0x0400`) it responds -immediately and streams all frames correctly. - -The 4-3-26 capture confirms the pattern for a second event (key `0111245a`): -chunk 1 = `0x245A`, chunk 2 = `0x285A`, chunk 3 = `0x2C5A` (each +0x0400). Blastware's -true formula is `key4[2:4] + n * 0x0400` — but since `key4[2:4]` of the first event is -`0x0000`, `n * 0x0400` produces the right result. The device does not strictly validate the -counter and streams data for any valid 5A request; using `chunk_num * 0x0400` is correct. - -### SUB 5A — params are 11 bytes for chunk frames, 10 for termination - -`bulk_waveform_params()` returns 11 bytes (extra trailing `0x00`). The 11th byte was -confirmed from the BW wire capture. `bulk_waveform_term_params()` returns 10 bytes. -Do not swap them. - -### SUB 5A — event-time metadata lives in A5 frame 7 - -The bulk stream sends 9+ A5 response frames. Frame 7 (0-indexed) contains the compliance -setup as it existed when the event was recorded: - -``` -"Project:" → project description -"Client:" → client name ← NOT in the 0C record -"User Name:" → operator name ← NOT in the 0C record -"Seis Loc:" → sensor location ← NOT in the 0C record -"Extended Notes"→ notes -``` - -**IMPORTANT — 5A "Project:" is session-start config, NOT per-event (confirmed 2026-04-05):** -The "Project:" string in the A5 frame 7 payload reflects the compliance setup from when -the *monitoring session first started*, not the individual event's project name. The per- -event project name is correctly stored in the 210-byte 0C waveform record and must be -used as the authoritative source. `_decode_a5_metadata_into` therefore only sets -`project` from 5A when 0C didn't already supply one. - -"Client:", "User Name:", "Seis Loc:", and "Extended Notes" are **NOT** present in the 0C -record — 5A remains the sole source for those fields and they are set unconditionally. - -`stop_after_metadata=True` (default) stops the 5A loop as soon as `b"Project:"` appears, -then sends the termination frame. - -### SUB 5A — end-of-stream signal (confirmed 2026-04-06) - -After streaming all waveform chunks, the device sends exactly **1 raw byte** in response to -the next chunk request, then goes silent. This is the natural end-of-stream indicator — NOT -a complete A5 frame. `S3FrameParser.bytes_fed` will be 1; no frame is assembled. - -Handling: on `TimeoutError`, if `bytes_fed > 0` AND frames were already collected, treat as -graceful end-of-stream, break the loop, and proceed to the termination frame. If `bytes_fed -== 0` with no prior frames, it is a genuine transport failure — re-raise. - -**Chunk recv timeout must be 10 s, not the default 120 s.** Chunks arrive within ~1 s each. -Using 120 s causes a ~2-minute stall at every end-of-stream detection. The `_recv_one` call -in the chunk loop passes `timeout=10.0` explicitly. - -**Typical chunk count (BE11529, 1024 sps):** A 9,306-sample event produces 35 chunks before -end-of-stream. Chunks with uniform 1,036-byte data are all-zero ADC samples (post-event -silence). Only the initial variable-size chunks contain actual signal. - -### SUB 5A — fi==9 hardcoded skip (FIXED 2026-04-06) - -`_decode_a5_waveform()` previously had `elif fi == 9: continue` — a leftover from the -9-frame original blast capture where frame 9 was assumed to be a terminator. For current -35-frame streams, fi==9 is live waveform data (~133 sample-sets were being dropped). -Removed. Terminator detection is via `page_key == 0x0000` in `read_bulk_waveform_stream`, -not frame index. - -### SUB 1E / 1F — event iteration null sentinel and token position (FIXED, do not re-introduce) - -**token_params bug (FIXED):** The token byte was at `params[6]` (wrong). Both 3-31-26 and -4-3-26 BW TX captures confirm it belongs at **`params[7]`** (raw: `00 00 00 00 00 00 00 fe 00 00`). -With the wrong position the device ignores the token and 1F returns null immediately. - -**1F token depends on context:** In browse mode (no 5A), use all-zero params (`browse=True`). -In download mode (get_events with 5A), use token=0xFE (`browse=False`) — this is required to -arm the device's 5A bulk stream state machine. The earlier "empirical" test showing token=0xFE -returns null was done WITHOUT the 1E(arm) step; that test is invalid. BW always uses 1F(0xFE) -in download mode. `count_events` uses `browse=True` (no 5A needed). - -**0A context requirement:** `advance_event()` (1F) only returns a valid next-event key -when a preceding `read_waveform_header()` (0A) call has established device waveform -context for the current key. Call 0A before every event in the loop, not just the first. -Calling 1F cold (after only 1E, with no 0A) returns the null sentinel regardless of how -many events are stored. - -**1F response layout:** The next event's key IS at `data_rsp.data[11:15]` (= payload[16:20]). -Confirmed from 4-3-26 browse-mode S3 captures: -``` -1F after 0A(key0=01110000): data[11:15]=0111245a data[15:19]=00001e36 ← valid -1F after 0A(key1=0111245a): data[11:15]=01114290 data[15:19]=00000046 ← valid -1F null sentinel: data[11:15]=00000000 data[15:19]=00000000 ← done -``` - -**Null sentinel:** `data8[4:8] == b"\x00\x00\x00\x00"` (= `data_rsp.data[15:19]`) -works for BOTH 1E trailing (offset to next event key) and 1F response (null key -echo) — in both cases, all zeros means "no more events." - -**1E response layout:** `data_rsp.data[11:15]` = event 0's actual key; `data_rsp.data[15:19]` -= sample-count offset to the next event key (key1 = key0 + this offset). If offset == 0, -there is only one event. - -**Correct iteration pattern (confirmed empirically with live device, 2+ events):** - -`count_events` (browse mode only, no 5A): -``` -1E(all zeros) → key0, trailing0 ← trailing0 non-zero if event 1 exists -0A(key0) ← REQUIRED: establishes device context -1F(all zeros / browse=True) → key1 ← use all-zero params -0A(key1) ← REQUIRED before each advance -1F(all zeros) → null ← done -``` - -`get_events` (download mode, with 5A): -``` -1E(all zeros) → key0, trailing0 ← trailing0 non-zero if event 1 exists -0A(key0) ← REQUIRED: establishes device context -1E(token=0xFE) ← REQUIRED: arms device for 5A; CONFIRMED 4-2-26 + 4-3-26 -0C(key0) ← read waveform record -1F(token=0xFE) → [discard key] ← REQUIRED: arms 5A bulk stream state machine -POLL × 3 ← REQUIRED: 3 full POLL cycles before 5A (BW frames 68-73) -5A(key0) ← bulk stream; key0 used even though 1F already advanced -1F(all zeros / browse=True) → key1 ← USE THIS for loop iteration (browse=True returns correct key) -0A(key1) -1E(token=0xFE) ← re-arm for next event's 5A -0C(key1) -1F(token=0xFE) → [discard key] ← arm 5A -POLL × 3 -5A(key1) -1F(browse=True) → null ← done -``` - -**IMPORTANT — conditional browse 1F (UPDATED 2026-04-06):** -`1F(token=0xFE)` (browse=False) BEFORE POLL+5A arms the device's bulk stream state machine. -Its returned key is cached as `arm_key4` in `get_events()`. - -`1F(browse=True)` AFTER 5A is ONLY sent when 5A **succeeded**. If 5A timed out or failed, -sending browse 1F disrupts the device's internal state — subsequent 5A probes for the next -event get no response (confirmed empirically: calling browse 1F after a failed 5A causes the -next event's 5A probe to also time out with 0 bytes received). - -In the failure path, `arm_key4` from `1F(download)` is used as a best-effort next-key hint: -- If `arm_key4 != cur_key`: use it to advance the loop without any 1F call -- If `arm_key4 == cur_key` (device stuck, typical for second+ events when 5A fails): abort - -The diagnostic `bytes_fed` counter on `S3FrameParser` (incremented in every `feed()` call, -reset by `reset()`) makes it possible to distinguish "no bytes at all" from "bytes received -but no complete frame assembled" in 5A probe timeouts — both show up as 120s timeouts in -the log but have very different root causes. - -**The 1E(token=0xFE) arm step is required (FIXED 2026-04-06):** -The device silently ignores all 5A probe frames unless a second SUB 1E with token=0xFE -has been issued between 0A and 0C. This step is present in EVERY download cycle in both -the 4-2-26 and 4-3-26 BW TX captures. - -**1F must come BEFORE 5A (FIXED 2026-04-06):** -BW always calls 1F (advance event) before starting the 5A bulk stream. 5A still uses the -pre-advance key — the device streams the waveform for the key that was set up with 0A+1E-arm+0C -even after 1F has moved the internal pointer to the next event. - -**POLL × 3 required before 5A (FIXED 2026-04-06):** -BW sends exactly 3 complete POLL (SUB 5B) probe+data cycles between the last 1F and the -first 5A probe frame. Confirmed from 4-2-26 BW TX capture frames 68-73. Without these -POLLs the device does not respond to the 5A probe. Use `proto.poll()` (not `startup()` — -`startup()` drains the boot string, which is only needed on initial connect). - -`advance_event(browse=True)` sends all-zero params; `advance_event()` default (browse=False) -sends token=0xFE and is NOT used by any caller. -`advance_event()` returns `(key4, event_data8)`. -Callers (`count_events`, `get_events`) loop while `data8[4:8] != b"\x00\x00\x00\x00"`. - -### SUB 1A — compliance config — orphaned send bug (FIXED, do not re-introduce) - -`read_compliance_config()` sends a 4-frame sequence (A, B, C, D) where: -- Frame A is a probe (no `recv_one` needed — device ACKs but returns no data page) -- Frames B, C, D each need a `recv_one` to collect the response - -**There must be NO extra `self._send(...)` call before the B/C/D recv loop without a -matching `recv_one()`.** An orphaned send shifts all receives one step behind, leaving -frame D's channel block (trigger_level_geo, alarm_level_geo, max_range_geo) unread and -producing only ~1071 bytes instead of ~2126. - -### SUB 1A — anchor search range - -`_decode_compliance_config_into()` locates sample_rate and record_time via the anchor -`b'\x01\x2c\x00\x00\xbe\x80\x00\x00\x00\x00'`. Search range is `cfg[0:150]`. - -Do not narrow this to `cfg[40:100]` — the old range was only accidentally correct because -the orphaned-send bug was prepending a 44-byte spurious header, pushing the anchor from -its real position (cfg[11]) into the 40–100 window. - -### Sample rate and DLE jitter in cfg data - -Sample rate 4096 (`0x1000`) causes DLE jitter: the frame carries `10 10 00` on the wire, -which unstuffs to `10 00` — 2 bytes instead of 3. This makes frame C 1 byte shorter and -shifts all subsequent absolute offsets by −1. The anchor approach is immune to this. -Do NOT use fixed absolute offsets for sample_rate or record_time. - -### TCP / cellular transport - -- Protocol bytes over TCP are bit-for-bit identical to RS-232. No wrapping. -- The modem (RV50/RV55) forwards bytes with up to ~1s buffering. `TcpTransport` uses - `read_until_idle(idle_gap=1.5s)` to drain the buffer completely before parsing. -- Cold-boot: unit sends the 16-byte ASCII string `"Operating System"` before entering - DLE-framed mode. The parser discards it (scans for DLE+STX). -- RV50/RV55 sends `\r\nRING\r\n\r\nCONNECT\r\n` over TCP to the caller even with - Quiet Mode enabled. Parser handles this — do not strip it manually before feeding to - `S3FrameParser`. - -### Required ACEmanager settings (Sierra Wireless RV50/RV55) - -| Setting | Value | Why | -|---|---|---| -| Configure Serial Port | `38400,8N1` | Must match MiniMate baud | -| Flow Control | `None` | Hardware FC blocks TX if pins unconnected | -| **Quiet Mode** | **Enable** | **Critical.** Disabled injects `RING`/`CONNECT` onto serial, corrupting S3 handshake | -| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency | -| TCP Connect Response Delay | `0` | Non-zero silently drops first POLL frame | -| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect | -| DB9 Serial Echo | `Disable` | Echo corrupts the data stream | - ---- - -## Key confirmed field locations - -### SUB FE — Full Config (166 destuffed bytes) - -| Offset | Field | Type | Notes | -|---|---|---|---| -| 0x34 | firmware version string | ASCII | e.g. `"S338.17"` | -| 0x56–0x57 | calibration year | uint16 BE | `0x07E9` = 2025 | -| 0x0109 | aux trigger enabled | uint8 | `0x00` = off, `0x01` = on | - -### SUB 1A — Compliance Config (~2126 bytes total after 4-frame sequence) - -| Field | How to find it | -|---|---| -| sample_rate | uint16 BE at anchor − 2 | -| record_time | float32 BE at anchor + 10 | -| trigger_level_geo | float32 BE, located in channel block | -| alarm_level_geo | float32 BE, adjacent to trigger_level_geo | -| max_range_geo | float32 BE, adjacent to alarm_level_geo | -| setup_name | ASCII, null-padded, in cfg body | -| project / client / operator / sensor_location | ASCII, label-value pairs | - -Anchor: `b'\x01\x2c\x00\x00\xbe\x80\x00\x00\x00\x00'`, search `cfg[0:150]` - -### SUB 0C — Waveform Record (210 bytes = data[11:11+0xD2]) - -**sub_code=0x10 (Waveform single-shot) — 9-byte timestamp header:** - -| Offset | Field | Type | -|---|---|---| -| 0 | day | uint8 | -| 1 | sub_code | uint8 (`0x10`) | -| 2 | month | uint8 | -| 3–4 | year | uint16 BE | -| 5 | unknown | uint8 (always 0) | -| 6 | hour | uint8 | -| 7 | minute | uint8 | -| 8 | second | uint8 | - -**sub_code=0x03 (Waveform continuous) — 10-byte timestamp header (1-byte shift):** - -Confirmed 2026-04-03 against Blastware event report (15:20:17 Apr 3 2026). -Raw wire bytes: `10 03 10 04 07 ea 00 0f 14 11` - -| Offset | Field | Type | Notes | -|---|---|---|---| -| 0 | unknown_a | uint8 | `0x10` observed | -| 1 | day | uint8 | doubles as sub_code position in 0x10 layout | -| 2 | unknown_b | uint8 | `0x10` observed | -| 3 | month | uint8 | | -| 4–5 | year | uint16 BE | | -| 6 | unknown | uint8 | | -| 7 | hour | uint8 | | -| 8 | minute | uint8 | | -| 9 | second | uint8 | | - -**Peak values (both record types):** - -| Location | Field | Type | -|---|---|---| -| `tran_pos - 12` | peak_vector_sum | float32 BE — label-relative, NOT fixed offset | -| `label + 6` | PPV per channel | float32 BE (search for `"Tran"`, `"Vert"`, `"Long"`, `"MicL"`) | - -PPV labels are NOT 4-byte aligned. The label-relative approach is the only reliable method. -`peak_vector_sum` is exactly 12 bytes before the `"Tran"` label — confirmed for both -sub_code=0x10 and sub_code=0x03. Do NOT use fixed offset 87 (only incidentally correct -for 0x10 records). - ---- - -## SFM REST API (sfm/server.py) - -### Live device endpoints (connect to device per-request) - -``` -GET /device/info?port=COM5 ← serial -GET /device/info?host=1.2.3.4&tcp_port=9034 ← cellular -GET /device/events?host=1.2.3.4&tcp_port=9034&baud=38400 -GET /device/event?host=1.2.3.4&tcp_port=9034&index=0 -GET /device/monitor/status?host=1.2.3.4&tcp_port=9034 ← battery, memory, mode -POST /device/monitor/start?host=1.2.3.4&tcp_port=9034 ← start recording -POST /device/monitor/stop?host=1.2.3.4&tcp_port=9034 ← stop recording -``` - -Server retries once on `ProtocolError` for TCP connections (handles cold-boot timing). - -### DB read endpoints (query seismo_relay.db written by ach_server.py) - -``` -GET /db/units ← all known serials + summary stats -GET /db/events?serial=BE11529&from_dt=&to_dt=&limit= ← triggered events, newest first -GET /db/monitor_log?serial=BE11529&from_dt=&to_dt= ← monitoring intervals, newest first -GET /db/sessions?serial=BE11529&limit=50 ← ACH call-home sessions, newest first -PATCH /db/events/{id}/false_trigger?value=true ← flag/unflag false triggers -``` - -DB file: `bridges/captures/seismo_relay.db` (default; override with `--db-path` at startup). -All DB endpoints are read-only except `PATCH /db/events/{id}/false_trigger`. - ---- - -## Key wire captures (reference material) - -| Capture | Location | Contents | -|---|---|---| -| 1-2-26 | `bridges/captures/1-2-26/` | SUB 5A BW TX frames — confirmed 5A frame format, 11-byte params, DLE-aware checksum | -| 3-11-26 | `bridges/captures/3-11-26/` | Full compliance setup write, Aux Trigger capture | -| 3-31-26 | `bridges/captures/3-31-26/` | Complete event download cycle (148 BW / 147 S3 frames) — confirmed 1E/0A/0C/1F sequence; only 1 event stored so token=0xFE appeared to work | -| 4-3-26 | `bridges/captures/4-3-26/` | Browse-mode S3 capture with 2+ events — confirmed all-zero params for 1F, 1F response layout, null sentinel, 0A context requirement | - ---- - -## Write commands (SUBs 68–83) — confirmed 2026-04-07 - -All confirmed from 3-11-26 BW TX capture (`raw_bw_20260311_170151.bin`, frames 102–112). - -### Write frame format — CRITICAL: minimal DLE stuffing - -Write frames do NOT use the same DLE stuffing as read frames. **Only the BW_CMD byte -(0x10 at payload position [0]) is doubled on the wire. All other bytes — flags, sub, -offset, params, data, and checksum — are written RAW without stuffing.** - -Confirmed from all 11 write frames in the 3-11-26/170151 BW capture. ✅ 2026-04-07 - -Do NOT use `dle_stuff()` or `build_bw_frame()` for write commands. Use `build_bw_write_frame()`. - -``` -Actual wire layout: - [41] ACK - [02] STX - [10 10] BW_CMD doubled (ONLY DLE stuffing applied) - [00] flags - [sub] write command byte (0x68–0x83) - [00] always zero - [hi][lo] offset uint16 BE — RAW (not stuffed even if hi=0x10) - [params] 10 bytes — RAW - [data] variable-length write payload — RAW (0x10 bytes not stuffed) - [chk] checksum — RAW (not stuffed even if 0x10) - [03] ETX - -Total wire length = 2 (ACK+STX) + 2 (doubled BW_CMD) + 15 (raw header) + len(data) + 1 (chk) + 1 (ETX) - = 21 + len(data) -``` - -De-stuffed payload (logical; used for checksum computation only): -``` - [0] BW_CMD 0x10 - [1] flags 0x00 - [2] SUB write command byte (0x68–0x83) - [3] 0x00 always zero - [4] offset_hi - [5] offset_lo - [6:16] params 10-byte field (see per-SUB notes below) - [16:] data write payload (variable length; absent for confirm frames) - [-1] chk large-frame DLE-aware checksum (see below) -``` - -Write SUBs = Read SUB + 0x60. Response SUB follows the standard 0xFF − Request SUB rule. - -### Write frame checksum - -All write frames (data frames AND confirm frames) use the **large-frame DLE-aware checksum**: - -```python -chk = (sum(b for b in payload[2:] if b != 0x10) + 0x10) & 0xFF -``` - -This is identical to the SUB 5A DLE-aware checksum. Confirmed against all 11 write frames in -the 3-11-26/170151 capture. ✅ 2026-04-07 - -Note: confirm frames contain no embedded 0x10 bytes, so both the standard SUM8 and the -DLE-aware formula produce the same result for them — but `build_bw_write_frame` always uses -the DLE-aware formula for consistency. - -### Write ack responses - -All device acks for write commands are **17-byte zero-data S3 frames**: - -``` -[DLE=0x10][STX=0x02][stuffed(header + chk)][bare ETX=0x03] -``` - -The data section carries zeros; RSP_SUB = 0xFF − write_request_SUB. - -### Write SUB constants and sequences - -| Request SUB | Function | Offset | Response SUB | -|---|---|---|---| -| 0x68 | Event index write | `data[1] + 2` | 0x97 | -| 0x73 | Confirm B (follows 68) | 0 | 0x8C | -| 0x71 | Compliance write (×3 chunks) | see below | 0x8E | -| 0x72 | Confirm A (follows 71×3, 69) | 0 | 0x8D | -| 0x82 | Trigger config write | `data[1] + 2` | 0x7D | -| 0x83 | Trigger confirm (follows 82) | 0 | 0x7C | -| 0x69 | Waveform data write | `data[1] + 2` | 0x96 | -| 0x74 | Confirm C (follows 69) | 0 | 0x8B | - -**Offset formula for single-chunk writes (0x68, 0x69, 0x82):** `offset = data[1] + 2` - -The write payload always begins with a 2-byte header `[0x00][length]`, where `data[1]` is -an embedded length field. The offset encodes this inner length + 2 (accounting for the -header bytes). Confirmed from all three single-chunk write frames in the 3-11-26 capture: - -| SUB | data[0:4] (hex) | data[1] | offset | total data len | -|---|---|---|---|---| -| 0x68 | `00 58 09 00` | 0x58=88 | 0x5A=90 | 91 | -| 0x82 | `00 1A D5 00` | 0x1A=26 | 0x1C=28 | 29 | -| 0x69 | `00 C8 08 00` | 0xC8=200 | 0xCA=202 | 204 | - -Full sequence: `68→73 | 71×3→72 | 82→83 | 69→74→72` - -### SUB 71 — compliance write chunk parameters - -The full compliance config payload (~2128 bytes) is split into exactly 3 chunks. -Confirmed from 3-11-26 BW TX capture frames 104–108: - -| Chunk | Size | `offset` | `params` (10 bytes hex) | -|---|---|---|---| -| 1 (first) | 1027 bytes | 0x1004 | `00 00 00 00 00 00 00 00 00 00` | -| 2 (middle) | 1055 bytes | 0x1004 | `00 00 00 10 04 00 00 00 00 00` | -| 3 (last) | remainder | 0x002C | `00 00 08 00 00 00 00 00 00 00` | - -Total: 1027 + 1055 + N = 2082 + N bytes (N ≈ 46 for a standard 2128-byte config). - -After all 3 chunks are acked (SUB 0x8E each), send SUB 72 confirm → device acks 0x8D. - -### `build_bw_write_frame()` — framing.py - -```python -build_bw_write_frame(sub, data, *, offset=0, params=bytes(10)) -> bytes -``` - -Use for all write commands (SUBs 68–83) including confirm frames (data=b""). -**Do NOT use `build_bw_frame` for write commands** — it uses standard SUM8, not the -large-frame DLE-aware checksum required for writes. - -### `push_config_raw()` — client.py - -```python -client.push_config_raw(event_index_data, compliance_data, trigger_data, waveform_data) -``` - -Orchestrates the full write sequence in the confirmed order. All payloads are raw bytes -(no encoding performed at this level). A higher-level encoder that builds payloads from -a `ComplianceConfig` object is a future task. - ---- - -## Monitoring commands (SUBs 0x1C, 0x96, 0x97) — confirmed 2026-04-08 - -All confirmed from 4-8-26/2ndtry BW TX/S3 capture (clean start → 30s monitor → stop cycle). - -### SUB 0x1C — Monitor status read - -Standard two-step read (probe at offset 0x00, data at offset 0x2C). -Response SUB = 0xFF − 0x1C = **0xE3** (standard formula — no exception). - -**Payload length is 46–47 bytes IDLE, 48–49 bytes MONITORING** — not a reliable sole -indicator due to 1-byte jitter overlap at the boundary. - -**Monitoring flag (CONFIRMED 2026-04-09 — byte diff of all 144 data frames, 2ndtry capture):** -- `section[1] == 0x00` → unit is **idle** -- `section[1] == 0x10` → unit is **monitoring** - -This is `data[12]` (= `frame.data[12]`). The flag is 0x00 in all 36 IDLE_BEFORE frames, -0x10 in all 98 MONITORING frames, and 0x00 in all 10 IDLE_AFTER frames — 100% accurate. - -**HISTORY OF THIS FIELD (do not re-derive):** The original implementation used `section[1]`. -A re-analysis in the prior session incorrectly concluded `section[1]` is always 0x00 and -"corrected" the flag to `section[6]`, which has non-binary values (0xea idle, 0x07 monitoring) -and is device-specific. The 2026-04-09 re-analysis confirms `section[1]` was right. - -**IMPORTANT — `frame.data` has checksum already stripped** by `S3FrameParser._finalise()` -(`raw_payload = body[:-1]`; `data = raw_payload[5:]`). There is NO trailing checksum byte in -`section`. All relative-from-end offsets must account for this. - -Battery and memory fields are present in **both** states: - -| Offset (relative to end) | Field | Type | Notes | -|---|---|---|---| -| `section[-10:-8]` | battery voltage × 100 | uint16 BE | `0x02A8` = 680 → 6.80 V | -| `section[-8:-4]` | memory total (bytes) | uint32 BE | e.g. 983026 ≈ 960 KB | -| `section[-4:]` | memory free (bytes) | uint32 BE | decreases as events are stored | - -### SESSION_RESET signal (`41 03`) — required for monitoring units - -Confirmed from 4-8-26 BW TX captures: Blastware sends a 2-byte `41 03` (ACK + ETX, -no STX) immediately before the first POLL probe AND between the probe and data frames. -This signal is **required to wake units that are actively monitoring** — without it -the unit does not respond to POLL over TCP. Harmless for idle units. - -`SESSION_RESET = bytes([0x41, 0x03])` is defined in `framing.py` and sent by -`protocol.startup()` before and between POLL frames. - -### SUB 0x96 — Start monitoring - -Single write frame, **no data payload** (empty body). -Response SUB = 0xFF − 0x96 = **0x69**. - -Wire bytes (confirmed frame 92 of 2ndtry BW capture): -``` -41 02 10 10 00 96 00 00 00 00 00 00 00 00 00 00 00 00 00 a6 03 -``` - -### SUB 0x97 — Stop monitoring - -Single write frame, **no data payload** (empty body). -Response SUB = 0xFF − 0x97 = **0x68**. - -Wire bytes (confirmed frame 305 of 2ndtry BW capture): -``` -41 02 10 10 00 97 00 00 00 00 00 00 00 00 00 00 00 00 00 a7 03 -``` - -Both start and stop acks are standard 17-byte zero-data S3 frames. - -### On-device sensor check behavior (confirmed 2026-04-08) - -Confirmed from 4-8-26/sensor-check BW+S3 capture (Blastware "Unit Channel Test" comms -check issued while unit was performing its on-device sensor check). - -**Unit IS reachable during on-device sensor check** — POLL (SUB 5B) responded normally -throughout. However, the unit partially handled channel-test commands (SUB 0x0E) for -channels 0–4 and then went **silent for ~40 seconds** while the sensor check ran, before -resuming responses for channels 5–7 and the trigger test (SUB 0x98). - -Key findings: -- On-device sensor check duration: approximately **40 seconds** (log gap `18:40:48` → `18:41:28`) -- Unit IS reachable for POLL during the check window — SESSION_RESET + POLL works -- Partial command responses during check are possible (device may buffer some, drop others) -- The Blastware "Unit Channel Test" (remote comms check, SUBs 0x0E + 0x98) is a SEPARATE - operation from the on-device check — it is a passive remote read; the unit's screen does - not change during a remote comms check - -**SFM behavior after `POST /device/monitor/start`:** `_pollMonitorConfirm()` polls -`/device/monitor/status` every 5 s for up to 60 s, updating the badge on each poll. -Status will show MONITORING once `section[1]` flips to `0x10`. - -### SUBs known from sensor-check capture (4-8-26) — NOT YET IMPLEMENTED - -| BW SUB | RSP SUB | Function | Notes | -|---|---|---|---| -| 0x15 | 0xEA | Serial number / short ID | 2-step read; data offset = 0x0A (10 bytes); confirmed serial `"BE11529"` at `data[11+5:]` | -| 0x01 | 0xFE | Device info block | 2-step read; data offset = 0x98 (152 bytes); payload includes serial + firmware + float config fields | -| 0x0E | 0xF1 | Channel sensor data | 2-step read; channel selector in `params[6:8]` (`0x0000`–`0x0007`); data length 0x0A per channel; used by Blastware "Unit Channel Test" — see docs/ for details | -| 0x98 | 0x67 | Trigger test | Single probe frame (`params[0]=0xFF`); sent twice per test cycle; all-zero data response; used after 0x0E channel scan | - -Blastware's "Unit Channel Test" sequence: `POLL×N → 0x15 → 0x01 → 0x08 → 0x01 → 0x0E×8 → 0x98×2 → 0x0E×8` (repeat pass with live ADC readings). - ---- - -## Compliance config field inventory (from Blastware UI, 2026-04-08) - -Fields visible in the Blastware Compliance Setup dialog — most are NOT YET decoded to byte -offsets in the raw 1A/E5 payload. Only fields with `✅` have confirmed offsets in the code. - -**Recording Setup tab:** -- Recording Mode: Continuous / Single Shot / Histogram (enum) -- Record Stop Mode: Fixed Record Time / Auto / Manual Stop (enum) -- Sample Rate: Standard 1024 / Fast 2048 / Faster 4096 sps ✅ (anchor−2) -- Record Time: float, seconds ✅ (anchor+10) -- Histogram Interval: 5 / 15 / 30 / 60 minutes (enum, mode-gated) -- Storage Mode: Save All Data / Save Triggered (enum) -- Geophone Type: Standard Triaxial / 4.5 Hz (bool/enum) -- Geophone Channels: Enable all geophones (bool), Trigger Source (bool) -- Chan 1-3 Trigger Level (float, in/s) ✅ (`trigger_level_geo`) -- Chan 1-3 Maximum Range: Normal 10.000 / 1.25 in/s (enum) ✅ (`max_range_geo`) -- Microphone Channels: Enable all microphones (bool), Trigger Source (bool) -- Chan 4 Trigger Level (dB or psi depending on units) - -**Notes tab:** -- Enable User Notes (bool) -- Project, Client, User Name, Seis Loc (ASCII strings) ✅ (sourced from A5 frame 7 via 5A) -- Enable Extended Notes (bool); Extended Notes text; Extended Notes Title -- Enable Job Number (bool); Job Number (int) -- Enable Scaled Distance (bool); Distance from Blast (float); Charge Weight (float) — Scaled Distance is derived - -**Special Setups tab:** -- Unit Timer: Timer Mode (Off/On), Start Date/Time, Stop Date/Time -- Self Check: Mode (Off/On), Time (HH:MM) -- Sensor Check: **Before monitoring** / After each event / **Disabled** ❓ (byte offset unknown) -- Measurement Units: Imperial / Metric -- Show Mic units in dB (bool) -- Time Format: 24 Hour / 12 Hour (AM/PM) -- Backlight on Time (seconds) ✅ (event index block +75) -- Power Saving Timeout (minutes) ✅ (event index block +83) -- Monitoring LCD Cycle ✅ (event index block +84:86) -- Set unit time with setup (bool) - -The "Sensor Check" dropdown (`Before monitoring` / `After each event` / `Disabled`) has NOT -been located in the raw config bytes. The user's unit always runs with `Before monitoring`. -Full compliance config encoder is a future task. - ---- - ---- - -## Erase-all protocol (SUBs 0xA3/0xA2/0x06) — confirmed 2026-04-11 - -Full sequence confirmed from 4-11-26 MITM capture of a live Blastware ACH session -(`bridges/captures/mitm/ach_mitm_20260411_001912/`). - -### Wire sequence - -``` -BW → device: SUB 0xA3 params=00 00 00 00 00 00 00 FE 00 00 (begin erase) -device → BW: SUB 0x5C (ack) -BW → device: SUB 0x1C probe (offset=0x00) -device → BW: SUB 0xE3 (probe ack) -BW → device: SUB 0x1C data (offset=0x2C) -device → BW: SUB 0xE3 (monitor status response) -BW → device: SUB 0x06 probe (offset=0x00, params same) -device → BW: SUB 0xF9 (probe ack) -BW → device: SUB 0x06 data (offset=0x24) -device → BW: SUB 0xF9 (36-byte storage range response) -BW → device: SUB 0xA2 params=00 00 00 00 00 00 00 FE 00 00 (confirm erase) -device → BW: SUB 0x5D (ack — device memory is now cleared) -``` - -All frames use standard `build_bw_frame` (not write-format). Response SUBs follow the -standard `0xFF - SUB` formula; no exceptions. - -### SUB 0x06 — event storage range response (36 bytes) - -The 36-byte response body ends with two 4-byte event keys: - -| Offset (from end) | Field | Notes | -|---|---|---| -| `[-8:-4]` | first stored event key | `01110000` when empty | -| `[-4:]` | last stored event key | `01110000` when empty | - -Before erase: ends with ` ` (e.g. `0111ea60 0111eaa6`). -After erase: both bytes read `01110000` — device's empty/reset sentinel. - -### Post-erase key counter reset - -After a successful erase, the device resets its event counter. New events start from -key `0x01110000` again — the same key as the very first event ever recorded. This means -key-based deduplication in the ACH server must account for key reuse: - -- After our own erase: `ach_state.json` `downloaded_keys` and `max_downloaded_key` are - cleared so the next session starts fresh. -- After an external erase: the ACH server detects it by comparing `max(device_keys)` to - `max_downloaded_key` from state. If the device max has rolled back below the historical - max, all current device keys are treated as new regardless of `seen_keys`. - -### ACH server state format (v0.9.0) - -`bridges/captures/ach_state.json`: -```json -{ - "BE11529": { - "downloaded_keys": ["01110000", "0111245a"], - "max_downloaded_key": "0111245a", - "last_seen": "2026-04-11T01:04:36", - "serial": "BE11529", - "peer": "63.43.212.232:51920" - } -} -``` - -`max_downloaded_key` is the high-water mark — the largest key ever downloaded from the -unit. It is NOT reset when events are erased from the device (only when our server does -the erase). Used for post-erase detection. - ---- - -## Monitor log entries — SUB 0x0A partial records (confirmed 2026-04-11) - -Confirmed from 4-11-26 MITM capture: 12 partial records (record type `0x2C`) and 7 full -event records (record type `0x46`) across 19 total 0x0A responses. - -### Record type detection - -`read_waveform_header()` returns `(raw_data, length)` where `raw_data = data_rsp.data` -(the full payload including prefix bytes). The record type is at `raw_data[0]`: - -| Value | Type | How to process | -|---|---|---| -| `0x46` | Full triggered event | Normal download: 0C → 5A → 1F | -| `0x2C` | Monitor log entry (partial) | No 0C/5A; decode inline from 0A payload | - -Length heuristic: `length < 0x40` (64) reliably identifies partial records across all -observed captures. Both checks (`raw_data[0] == 0x2C` and `length < 0x40`) are used. - -### SUB 0x0A partial record (0x2C) payload layout - -All offsets are from `raw_data` (the full `data_rsp.data` array including the 11-byte -prefix before the actual header bytes start). - -``` -raw_data[0] = 0x2C ← record type (partial / monitor log) -raw_data[1:11] = prefix bytes (vary; contain key4 copy, flags, length) -raw_data[11:] = timestamp and ASCII metadata payload -``` - -**Timestamp auto-detection** (confirmed from 4-11-26 capture): - -``` -raw_data[11] == 0x10 → 10-byte sub_code=0x03 format (continuous mode) -raw_data[11] != 0x10 → 9-byte sub_code=0x10 format (single-shot mode) -``` - -**9-byte timestamp format (sub_code=0x10):** - -| Byte | Field | -|---|---| -| 0 | day | -| 1 | `0x10` (sub_code marker) | -| 2 | month | -| 3–4 | year (uint16 BE) | -| 5 | unknown (0x00) | -| 6 | hour | -| 7 | minute | -| 8 | second | - -**10-byte timestamp format (sub_code=0x03):** - -| Byte | Field | -|---|---| -| 0 | `0x10` (marker) | -| 1 | day | -| 2 | `0x10` (marker) | -| 3 | month | -| 4–5 | year (uint16 BE) | -| 6 | unknown (0x00) | -| 7 | hour | -| 8 | minute | -| 9 | second | - -**Two timestamps:** Each partial record contains two timestamps — `start_time` and -`stop_time` — stored consecutively: -- `ts1` (start) at `raw_data[ts_offset : ts_offset + ts_size]` where `ts_offset = 11` -- `ts2` (stop) at `raw_data[ts1_end : ts1_end + ts_size]` - -**Edge case — 1-byte gap between timestamps:** Occurs when ts1 and ts2 share the same -minute:second. If `try_ts(raw_data[ts1_end:])` fails, try `try_ts(raw_data[ts1_end+1:])`. -Confirmed in frames 121, 161, 165 of the 4-11-26 MITM capture. Frame 121 still shows 0s -duration (both decode to 16:02:00) — the extra byte appears in all same-second cases. - -**ASCII metadata after timestamps:** -``` - BE\x00Geo: in/s ... -``` - -- Serial: scan for `b"BE"`, read until `b"\x00"` (e.g. `"BE11529"`) -- Geo threshold: scan for `b"Geo: "`, read float until next space (e.g. `0.254` in/s) - -A separator of variable length (4–5 bytes of `\x00` + flags) sits between the two -timestamps and the ASCII region. The `b"BE"` anchor scan is robust to separator length -variation. - -### `_decode_0a_partial_header(raw_data, index, key4)` — client.py - -Returns a `MonitorLogEntry` or `None`. Called by `get_monitor_log_entries()` for each -event key whose 0x0A response has `raw_data[0] == 0x2C` or `length < 0x40`. - -### `MiniMateClient.get_monitor_log_entries(skip_keys=None)` — client.py - -Browse-mode walk: `1E → 0A → check type → decode if partial → 1F`. No 0x0C or 5A reads -performed. Full (0x46) records are skipped without decoding. Returns `list[MonitorLogEntry]`. - -`skip_keys` (optional `set[str]`): keys in this set are still advanced through the walk -(to avoid disrupting the iteration sequence), but no `MonitorLogEntry` is created for them. - -### `MonitorLogEntry` model — models.py - -```python -@dataclass -class MonitorLogEntry: - index: int # 0-based position - key: str # 8-hex event key - start_time: Optional[datetime.datetime] = None - stop_time: Optional[datetime.datetime] = None - serial: Optional[str] = None - geo_threshold_ips: Optional[float] = None - raw_header: Optional[bytes] = field(default=None, repr=False) - - @property - def duration_seconds(self) -> Optional[float]: ... -``` - -### ACH server integration (v0.10.0) - -After `get_events()`, the ACH server calls `get_monitor_log_entries(skip_keys=seen_keys)`. -New entries are saved to `monitor_log.json` in the session directory. Monitor log keys are -included in `current_keys` for state persistence so they are not re-processed on the next -call-home. - ---- - -## What's next - -- **Database** — SQLite store for events + monitor log entries; dedup by key; queryable -- **Histograms** — decode histogram-mode A5 data (noise floor tracking) -- Compliance config encoder — build raw write payloads from a `ComplianceConfig` object -- Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring) -- Modem manager — push RV50/RV55 configs via Sierra Wireless API -- RV55 DCD/DTR issue — newer RV55 firmware doesn't assert DCD by default; units don't - resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred) +# CLAUDE.md — seismo-relay + +Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for +managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem +(Sierra Wireless RV50 / RV55). Current version: **v0.12.0**. + +--- + +## Project layout + +``` +minimateplus/ ← Python client library (primary focus) + transport.py ← SerialTransport, TcpTransport + framing.py ← DLE codec, frame builders, S3FrameParser + protocol.py ← MiniMateProtocol — wire-level read/write methods + client.py ← MiniMateClient — high-level API (connect, get_events, …) + models.py ← DeviceInfo, EventRecord, ComplianceConfig, … + +sfm/server.py ← FastAPI REST server exposing device data over HTTP +seismo_lab.py ← Tkinter GUI (Bridge + Analyzer + Console tabs) +docs/ + instantel_protocol_reference.md ← reverse-engineered protocol spec ("the Rosetta Stone") +CHANGELOG.md ← version history +``` + +--- + +## Current implementation state (v0.10.0) + +Full read pipeline + write pipeline + erase pipeline + monitor log working end-to-end over TCP/cellular: + +| Step | SUB | Status | +|---|---|---| +| POLL / startup handshake | 5B | ✅ | +| Serial number | 15 | ✅ | +| Full config (firmware, calibration date, etc.) | FE | ✅ | +| Compliance config (record time, sample rate, geo thresholds) | 1A | ✅ | +| Event index | 08 | ✅ | +| Event header / first key | 1E | ✅ | +| Waveform header | 0A | ✅ | +| Waveform record (peaks, timestamp, project) | 0C | ✅ | +| **Bulk waveform stream (event-time metadata)** | **5A** | ✅ new v0.6.0 | +| Event advance / next key | 1F | ✅ | +| **Write commands (push config to device)** | **68–83** | ✅ new v0.8.0 | +| **Erase all events** | **0xA3 → 0x1C → 0x06 → 0xA2** | ✅ new v0.9.0 | +| **Monitor log entries (partial 0x2C records)** | **0A browse** | ✅ **new v0.10.0** | + +`get_events()` sequence per event: `1E → 0A → 0C → 5A → 1F` + +`push_config_raw()` write sequence: `68→73 | 71×3→72 | 82→83 | 69→74→72` + +`delete_all_events()` erase sequence: `0xA3 → 0x1C → 0x06 → 0xA2` + +--- + +## Protocol fundamentals + +### DLE framing + +``` +BW→S3 (our requests): [ACK=0x41] [STX=0x02] [stuffed payload+chk] [ETX=0x03] +S3→BW (device replies): [DLE=0x10] [STX=0x02] [stuffed payload+chk] [bare ETX=0x03] +``` + +- **DLE stuffing rule:** any literal `0x10` byte in the payload is doubled on the wire + (`0x10` → `0x10 0x10`). This includes the checksum byte. +- **Inner-frame terminators:** large S3 responses (A4, E5) contain embedded sub-frames + using `DLE+ETX` as inner terminators. The outer parser treats `DLE+ETX` inside a frame + as literal data — the bare ETX is the ONLY real frame terminator. +- **Response SUB rule:** `response_SUB = 0xFF - request_SUB` + (no known exceptions — earlier note claiming `1C` → `6E` was WRONG; `1C` → `0xE3` confirmed across 338 frames in 4-8-26 captures) +- **Two-step read pattern:** every read command is sent twice — probe step (`offset=0x00`, + get length) then data step (`offset=DATA_LENGTH`, get payload). All data lengths are + hardcoded constants, not read from the probe response. + +### De-stuffed payload header + +``` +BW→S3 (request): + [0] CMD 0x10 + [1] flags 0x00 + [2] SUB command byte + [3] 0x00 always zero + [4] 0x00 always zero + [5] OFFSET 0x00 for probe step; DATA_LENGTH for data step + [6-15] params (key, token, etc. — see helpers in framing.py) + +S3→BW (response): + [0] CMD 0x00 + [1] flags 0x10 + [2] SUB response sub byte + [3] PAGE_HI + [4] PAGE_LO + [5+] data +``` + +--- + +## Critical protocol gotchas (hard-won — do not re-derive) + +### SUB 5A — bulk waveform stream — NON-STANDARD frame format + +**Always use `build_5a_frame()` for SUB 5A. Never use `build_bw_frame()` for SUB 5A.** + +`build_bw_frame` produces WRONG output for 5A for two reasons: + +1. **`offset_hi = 0x10` must NOT be DLE-stuffed.** Blastware sends the offset field raw. + `build_bw_frame` would stuff it to `10 10` on the wire — the device silently ignores + the frame. `build_5a_frame` writes it as a bare `10`. + +2. **DLE-aware checksum.** When computing the checksum, `10 XX` pairs in the stuffed + section contribute only `XX` to the running sum; lone bytes contribute normally. This + differs from the standard SUM8-of-destuffed-payload that all other commands use. + +Both differences confirmed by reproducing Blastware's exact wire bytes from the 1-2-26 +BW TX capture. All 10 frames verified. + +### SUB 5A — chunk counter is monotonic (CORRECTED 2026-04-06) + +**Chunk counters are `chunk_num * 0x0400` for ALL chunks including chunk 1.** + +The 4-2-26 BW TX capture showed `counter=0x1004` for chunk 1 of event key `01110000`, which +led to `_CHUNK1_COUNTER = 0x1004` being hardcoded as a special case. This was a Blastware +artifact, not a protocol requirement. Empirical test 2026-04-06: with `counter=0x1004` for +chunk 1 the device times out (120 s); with `counter=0x0400` (= `1 * 0x0400`) it responds +immediately and streams all frames correctly. + +The 4-3-26 capture confirms the pattern for a second event (key `0111245a`): +chunk 1 = `0x245A`, chunk 2 = `0x285A`, chunk 3 = `0x2C5A` (each +0x0400). Blastware's +true formula is `key4[2:4] + n * 0x0400` — but since `key4[2:4]` of the first event is +`0x0000`, `n * 0x0400` produces the right result. The device does not strictly validate the +counter and streams data for any valid 5A request; using `chunk_num * 0x0400` is correct. + +### SUB 5A — params are 11 bytes for chunk frames, 10 for termination + +`bulk_waveform_params()` returns 11 bytes (extra trailing `0x00`). The 11th byte was +confirmed from the BW wire capture. `bulk_waveform_term_params()` returns 10 bytes. +Do not swap them. + +### SUB 5A — event-time metadata lives in A5 frame 7 + +The bulk stream sends 9+ A5 response frames. Frame 7 (0-indexed) contains the compliance +setup as it existed when the event was recorded: + +``` +"Project:" → project description +"Client:" → client name ← NOT in the 0C record +"User Name:" → operator name ← NOT in the 0C record +"Seis Loc:" → sensor location ← NOT in the 0C record +"Extended Notes"→ notes +``` + +**IMPORTANT — 5A "Project:" is session-start config, NOT per-event (confirmed 2026-04-05):** +The "Project:" string in the A5 frame 7 payload reflects the compliance setup from when +the *monitoring session first started*, not the individual event's project name. The per- +event project name is correctly stored in the 210-byte 0C waveform record and must be +used as the authoritative source. `_decode_a5_metadata_into` therefore only sets +`project` from 5A when 0C didn't already supply one. + +"Client:", "User Name:", "Seis Loc:", and "Extended Notes" are **NOT** present in the 0C +record — 5A remains the sole source for those fields and they are set unconditionally. + +`stop_after_metadata=True` (default) stops the 5A loop as soon as `b"Project:"` appears, +then sends the termination frame. + +### SUB 5A — end-of-stream signal (confirmed 2026-04-06) + +After streaming all waveform chunks, the device sends exactly **1 raw byte** in response to +the next chunk request, then goes silent. This is the natural end-of-stream indicator — NOT +a complete A5 frame. `S3FrameParser.bytes_fed` will be 1; no frame is assembled. + +Handling: on `TimeoutError`, if `bytes_fed > 0` AND frames were already collected, treat as +graceful end-of-stream, break the loop, and proceed to the termination frame. If `bytes_fed +== 0` with no prior frames, it is a genuine transport failure — re-raise. + +**Chunk recv timeout must be 10 s, not the default 120 s.** Chunks arrive within ~1 s each. +Using 120 s causes a ~2-minute stall at every end-of-stream detection. The `_recv_one` call +in the chunk loop passes `timeout=10.0` explicitly. + +**Typical chunk count (BE11529, 1024 sps):** A 9,306-sample event produces 35 chunks before +end-of-stream. Chunks with uniform 1,036-byte data are all-zero ADC samples (post-event +silence). Only the initial variable-size chunks contain actual signal. + +### SUB 5A — fi==9 hardcoded skip (FIXED 2026-04-06) + +`_decode_a5_waveform()` previously had `elif fi == 9: continue` — a leftover from the +9-frame original blast capture where frame 9 was assumed to be a terminator. For current +35-frame streams, fi==9 is live waveform data (~133 sample-sets were being dropped). +Removed. Terminator detection is via `page_key == 0x0000` in `read_bulk_waveform_stream`, +not frame index. + +### SUB 1E / 1F — event iteration null sentinel and token position (FIXED, do not re-introduce) + +**token_params bug (FIXED):** The token byte was at `params[6]` (wrong). Both 3-31-26 and +4-3-26 BW TX captures confirm it belongs at **`params[7]`** (raw: `00 00 00 00 00 00 00 fe 00 00`). +With the wrong position the device ignores the token and 1F returns null immediately. + +**1F token depends on context:** In browse mode (no 5A), use all-zero params (`browse=True`). +In download mode (get_events with 5A), use token=0xFE (`browse=False`) — this is required to +arm the device's 5A bulk stream state machine. The earlier "empirical" test showing token=0xFE +returns null was done WITHOUT the 1E(arm) step; that test is invalid. BW always uses 1F(0xFE) +in download mode. `count_events` uses `browse=True` (no 5A needed). + +**0A context requirement:** `advance_event()` (1F) only returns a valid next-event key +when a preceding `read_waveform_header()` (0A) call has established device waveform +context for the current key. Call 0A before every event in the loop, not just the first. +Calling 1F cold (after only 1E, with no 0A) returns the null sentinel regardless of how +many events are stored. + +**1F response layout:** The next event's key IS at `data_rsp.data[11:15]` (= payload[16:20]). +Confirmed from 4-3-26 browse-mode S3 captures: +``` +1F after 0A(key0=01110000): data[11:15]=0111245a data[15:19]=00001e36 ← valid +1F after 0A(key1=0111245a): data[11:15]=01114290 data[15:19]=00000046 ← valid +1F null sentinel: data[11:15]=00000000 data[15:19]=00000000 ← done +``` + +**Null sentinel:** `data8[4:8] == b"\x00\x00\x00\x00"` (= `data_rsp.data[15:19]`) +works for BOTH 1E trailing (offset to next event key) and 1F response (null key +echo) — in both cases, all zeros means "no more events." + +**1E response layout:** `data_rsp.data[11:15]` = event 0's actual key; `data_rsp.data[15:19]` += sample-count offset to the next event key (key1 = key0 + this offset). If offset == 0, +there is only one event. + +**Correct iteration pattern (confirmed empirically with live device, 2+ events):** + +`count_events` (browse mode only, no 5A): +``` +1E(all zeros) → key0, trailing0 ← trailing0 non-zero if event 1 exists +0A(key0) ← REQUIRED: establishes device context +1F(all zeros / browse=True) → key1 ← use all-zero params +0A(key1) ← REQUIRED before each advance +1F(all zeros) → null ← done +``` + +`get_events` (download mode, with 5A): +``` +1E(all zeros) → key0, trailing0 ← trailing0 non-zero if event 1 exists +0A(key0) ← REQUIRED: establishes device context +1E(token=0xFE) ← REQUIRED: arms device for 5A; CONFIRMED 4-2-26 + 4-3-26 +0C(key0) ← read waveform record +1F(token=0xFE) → [discard key] ← REQUIRED: arms 5A bulk stream state machine +POLL × 3 ← REQUIRED: 3 full POLL cycles before 5A (BW frames 68-73) +5A(key0) ← bulk stream; key0 used even though 1F already advanced +1F(all zeros / browse=True) → key1 ← USE THIS for loop iteration (browse=True returns correct key) +0A(key1) +1E(token=0xFE) ← re-arm for next event's 5A +0C(key1) +1F(token=0xFE) → [discard key] ← arm 5A +POLL × 3 +5A(key1) +1F(browse=True) → null ← done +``` + +**IMPORTANT — conditional browse 1F (UPDATED 2026-04-06):** +`1F(token=0xFE)` (browse=False) BEFORE POLL+5A arms the device's bulk stream state machine. +Its returned key is cached as `arm_key4` in `get_events()`. + +`1F(browse=True)` AFTER 5A is ONLY sent when 5A **succeeded**. If 5A timed out or failed, +sending browse 1F disrupts the device's internal state — subsequent 5A probes for the next +event get no response (confirmed empirically: calling browse 1F after a failed 5A causes the +next event's 5A probe to also time out with 0 bytes received). + +In the failure path, `arm_key4` from `1F(download)` is used as a best-effort next-key hint: +- If `arm_key4 != cur_key`: use it to advance the loop without any 1F call +- If `arm_key4 == cur_key` (device stuck, typical for second+ events when 5A fails): abort + +The diagnostic `bytes_fed` counter on `S3FrameParser` (incremented in every `feed()` call, +reset by `reset()`) makes it possible to distinguish "no bytes at all" from "bytes received +but no complete frame assembled" in 5A probe timeouts — both show up as 120s timeouts in +the log but have very different root causes. + +**The 1E(token=0xFE) arm step is required (FIXED 2026-04-06):** +The device silently ignores all 5A probe frames unless a second SUB 1E with token=0xFE +has been issued between 0A and 0C. This step is present in EVERY download cycle in both +the 4-2-26 and 4-3-26 BW TX captures. + +**1F must come BEFORE 5A (FIXED 2026-04-06):** +BW always calls 1F (advance event) before starting the 5A bulk stream. 5A still uses the +pre-advance key — the device streams the waveform for the key that was set up with 0A+1E-arm+0C +even after 1F has moved the internal pointer to the next event. + +**POLL × 3 required before 5A (FIXED 2026-04-06):** +BW sends exactly 3 complete POLL (SUB 5B) probe+data cycles between the last 1F and the +first 5A probe frame. Confirmed from 4-2-26 BW TX capture frames 68-73. Without these +POLLs the device does not respond to the 5A probe. Use `proto.poll()` (not `startup()` — +`startup()` drains the boot string, which is only needed on initial connect). + +`advance_event(browse=True)` sends all-zero params; `advance_event()` default (browse=False) +sends token=0xFE and is NOT used by any caller. +`advance_event()` returns `(key4, event_data8)`. +Callers (`count_events`, `get_events`) loop while `data8[4:8] != b"\x00\x00\x00\x00"`. + +### SUB 1A — compliance config — orphaned send bug (FIXED, do not re-introduce) + +`read_compliance_config()` sends a 4-frame sequence (A, B, C, D) where: +- Frame A is a probe (no `recv_one` needed — device ACKs but returns no data page) +- Frames B, C, D each need a `recv_one` to collect the response + +**There must be NO extra `self._send(...)` call before the B/C/D recv loop without a +matching `recv_one()`.** An orphaned send shifts all receives one step behind, leaving +frame D's channel block (trigger_level_geo, alarm_level_geo, max_range_geo) unread and +producing only ~1071 bytes instead of ~2126. + +### SUB 1A — anchor search range + +`_decode_compliance_config_into()` locates sample_rate and record_time via the anchor +`b'\x01\x2c\x00\x00\xbe\x80\x00\x00\x00\x00'`. Search range is `cfg[0:150]`. + +Do not narrow this to `cfg[40:100]` — the old range was only accidentally correct because +the orphaned-send bug was prepending a 44-byte spurious header, pushing the anchor from +its real position (cfg[11]) into the 40–100 window. + +### Sample rate and DLE jitter in cfg data + +Sample rate 4096 (`0x1000`) causes DLE jitter: the frame carries `10 10 00` on the wire, +which unstuffs to `10 00` — 2 bytes instead of 3. This makes frame C 1 byte shorter and +shifts all subsequent absolute offsets by −1. The anchor approach is immune to this. +Do NOT use fixed absolute offsets for sample_rate or record_time. + +### TCP / cellular transport + +- Protocol bytes over TCP are bit-for-bit identical to RS-232. No wrapping. +- The modem (RV50/RV55) forwards bytes with up to ~1s buffering. `TcpTransport` uses + `read_until_idle(idle_gap=1.5s)` to drain the buffer completely before parsing. +- Cold-boot: unit sends the 16-byte ASCII string `"Operating System"` before entering + DLE-framed mode. The parser discards it (scans for DLE+STX). +- RV50/RV55 sends `\r\nRING\r\n\r\nCONNECT\r\n` over TCP to the caller even with + Quiet Mode enabled. Parser handles this — do not strip it manually before feeding to + `S3FrameParser`. + +### Required ACEmanager settings (Sierra Wireless RV50/RV55) + +| Setting | Value | Why | +|---|---|---| +| Configure Serial Port | `38400,8N1` | Must match MiniMate baud | +| Flow Control | `None` | Hardware FC blocks TX if pins unconnected | +| **Quiet Mode** | **Enable** | **Critical.** Disabled injects `RING`/`CONNECT` onto serial, corrupting S3 handshake | +| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency | +| TCP Connect Response Delay | `0` | Non-zero silently drops first POLL frame | +| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect | +| DB9 Serial Echo | `Disable` | Echo corrupts the data stream | + +--- + +## Key confirmed field locations + +### SUB FE — Full Config (166 destuffed bytes) + +| Offset | Field | Type | Notes | +|---|---|---|---| +| 0x34 | firmware version string | ASCII | e.g. `"S338.17"` | +| 0x56–0x57 | calibration year | uint16 BE | `0x07E9` = 2025 | +| 0x0109 | aux trigger enabled | uint8 | `0x00` = off, `0x01` = on | + +### SUB 1A — Compliance Config (~2126 bytes total after 4-frame sequence) + +| Field | How to find it | +|---|---| +| sample_rate | uint16 BE at anchor − 2 | +| record_time | float32 BE at anchor + 10 | +| trigger_level_geo | float32 BE, located in channel block | +| alarm_level_geo | float32 BE, adjacent to trigger_level_geo | +| max_range_geo | float32 BE, adjacent to alarm_level_geo — **⚠ value and units UNKNOWN** (reads 6.206053 but doesn't match either UI range option; may not be the ADC full-scale — see GitHub issue) | +| setup_name | ASCII, null-padded, in cfg body | +| project / client / operator / sensor_location | ASCII, label-value pairs | + +Anchor: `b'\x01\x2c\x00\x00\xbe\x80\x00\x00\x00\x00'`, search `cfg[0:150]` + +### SUB 0C — Waveform Record (210 bytes = data[11:11+0xD2]) + +**sub_code=0x10 (Waveform single-shot) — 9-byte timestamp header:** + +| Offset | Field | Type | +|---|---|---| +| 0 | day | uint8 | +| 1 | sub_code | uint8 (`0x10`) | +| 2 | month | uint8 | +| 3–4 | year | uint16 BE | +| 5 | unknown | uint8 (always 0) | +| 6 | hour | uint8 | +| 7 | minute | uint8 | +| 8 | second | uint8 | + +**sub_code=0x03 (Waveform continuous) — 10-byte timestamp header (1-byte shift):** + +Confirmed 2026-04-03 against Blastware event report (15:20:17 Apr 3 2026). +Raw wire bytes: `10 03 10 04 07 ea 00 0f 14 11` + +| Offset | Field | Type | Notes | +|---|---|---|---| +| 0 | unknown_a | uint8 | `0x10` observed | +| 1 | day | uint8 | doubles as sub_code position in 0x10 layout | +| 2 | unknown_b | uint8 | `0x10` observed | +| 3 | month | uint8 | | +| 4–5 | year | uint16 BE | | +| 6 | unknown | uint8 | | +| 7 | hour | uint8 | | +| 8 | minute | uint8 | | +| 9 | second | uint8 | | + +**Peak values (both record types):** + +| Location | Field | Type | +|---|---|---| +| `tran_pos - 12` | peak_vector_sum | float32 BE — label-relative, NOT fixed offset | +| `label + 6` | PPV per channel | float32 BE (search for `"Tran"`, `"Vert"`, `"Long"`, `"MicL"`) | + +PPV labels are NOT 4-byte aligned. The label-relative approach is the only reliable method. +`peak_vector_sum` is exactly 12 bytes before the `"Tran"` label — confirmed for both +sub_code=0x10 and sub_code=0x03. Do NOT use fixed offset 87 (only incidentally correct +for 0x10 records). + +--- + +## SFM REST API (sfm/server.py) + +### Live device endpoints (connect to device per-request) + +``` +GET /device/info?port=COM5 ← serial +GET /device/info?host=1.2.3.4&tcp_port=9034 ← cellular +GET /device/events?host=1.2.3.4&tcp_port=9034&baud=38400 +GET /device/event?host=1.2.3.4&tcp_port=9034&index=0 +GET /device/monitor/status?host=1.2.3.4&tcp_port=9034 ← battery, memory, mode +POST /device/monitor/start?host=1.2.3.4&tcp_port=9034 ← start recording +POST /device/monitor/stop?host=1.2.3.4&tcp_port=9034 ← stop recording +``` + +Server retries once on `ProtocolError` for TCP connections (handles cold-boot timing). + +### DB read endpoints (query seismo_relay.db written by ach_server.py) + +``` +GET /db/units ← all known serials + summary stats +GET /db/events?serial=BE11529&from_dt=&to_dt=&limit= ← triggered events, newest first +GET /db/monitor_log?serial=BE11529&from_dt=&to_dt= ← monitoring intervals, newest first +GET /db/sessions?serial=BE11529&limit=50 ← ACH call-home sessions, newest first +PATCH /db/events/{id}/false_trigger?value=true ← flag/unflag false triggers +``` + +DB file: `bridges/captures/seismo_relay.db` (default; override with `--db-path` at startup). +All DB endpoints are read-only except `PATCH /db/events/{id}/false_trigger`. + +--- + +## Key wire captures (reference material) + +| Capture | Location | Contents | +|---|---|---| +| 1-2-26 | `bridges/captures/1-2-26/` | SUB 5A BW TX frames — confirmed 5A frame format, 11-byte params, DLE-aware checksum | +| 3-11-26 | `bridges/captures/3-11-26/` | Full compliance setup write, Aux Trigger capture | +| 3-31-26 | `bridges/captures/3-31-26/` | Complete event download cycle (148 BW / 147 S3 frames) — confirmed 1E/0A/0C/1F sequence; only 1 event stored so token=0xFE appeared to work | +| 4-3-26 | `bridges/captures/4-3-26/` | Browse-mode S3 capture with 2+ events — confirmed all-zero params for 1F, 1F response layout, null sentinel, 0A context requirement | + +--- + +## Write commands (SUBs 68–83) — confirmed 2026-04-07 + +All confirmed from 3-11-26 BW TX capture (`raw_bw_20260311_170151.bin`, frames 102–112). + +### Write frame format — CRITICAL: minimal DLE stuffing + +Write frames do NOT use the same DLE stuffing as read frames. **Only the BW_CMD byte +(0x10 at payload position [0]) is doubled on the wire. All other bytes — flags, sub, +offset, params, data, and checksum — are written RAW without stuffing.** + +Confirmed from all 11 write frames in the 3-11-26/170151 BW capture. ✅ 2026-04-07 + +Do NOT use `dle_stuff()` or `build_bw_frame()` for write commands. Use `build_bw_write_frame()`. + +``` +Actual wire layout: + [41] ACK + [02] STX + [10 10] BW_CMD doubled (ONLY DLE stuffing applied) + [00] flags + [sub] write command byte (0x68–0x83) + [00] always zero + [hi][lo] offset uint16 BE — RAW (not stuffed even if hi=0x10) + [params] 10 bytes — RAW + [data] variable-length write payload — RAW (0x10 bytes not stuffed) + [chk] checksum — RAW (not stuffed even if 0x10) + [03] ETX + +Total wire length = 2 (ACK+STX) + 2 (doubled BW_CMD) + 15 (raw header) + len(data) + 1 (chk) + 1 (ETX) + = 21 + len(data) +``` + +De-stuffed payload (logical; used for checksum computation only): +``` + [0] BW_CMD 0x10 + [1] flags 0x00 + [2] SUB write command byte (0x68–0x83) + [3] 0x00 always zero + [4] offset_hi + [5] offset_lo + [6:16] params 10-byte field (see per-SUB notes below) + [16:] data write payload (variable length; absent for confirm frames) + [-1] chk large-frame DLE-aware checksum (see below) +``` + +Write SUBs = Read SUB + 0x60. Response SUB follows the standard 0xFF − Request SUB rule. + +### Write frame checksum + +All write frames (data frames AND confirm frames) use the **large-frame DLE-aware checksum**: + +```python +chk = (sum(b for b in payload[2:] if b != 0x10) + 0x10) & 0xFF +``` + +This is identical to the SUB 5A DLE-aware checksum. Confirmed against all 11 write frames in +the 3-11-26/170151 capture. ✅ 2026-04-07 + +Note: confirm frames contain no embedded 0x10 bytes, so both the standard SUM8 and the +DLE-aware formula produce the same result for them — but `build_bw_write_frame` always uses +the DLE-aware formula for consistency. + +### Write ack responses + +All device acks for write commands are **17-byte zero-data S3 frames**: + +``` +[DLE=0x10][STX=0x02][stuffed(header + chk)][bare ETX=0x03] +``` + +The data section carries zeros; RSP_SUB = 0xFF − write_request_SUB. + +### Write SUB constants and sequences + +| Request SUB | Function | Offset | Response SUB | +|---|---|---|---| +| 0x68 | Event index write | `data[1] + 2` | 0x97 | +| 0x73 | Confirm B (follows 68) | 0 | 0x8C | +| 0x71 | Compliance write (×3 chunks) | see below | 0x8E | +| 0x72 | Confirm A (follows 71×3, 69) | 0 | 0x8D | +| 0x82 | Trigger config write | `data[1] + 2` | 0x7D | +| 0x83 | Trigger confirm (follows 82) | 0 | 0x7C | +| 0x69 | Waveform data write | `data[1] + 2` | 0x96 | +| 0x74 | Confirm C (follows 69) | 0 | 0x8B | + +**Offset formula for single-chunk writes (0x68, 0x69, 0x82):** `offset = data[1] + 2` + +The write payload always begins with a 2-byte header `[0x00][length]`, where `data[1]` is +an embedded length field. The offset encodes this inner length + 2 (accounting for the +header bytes). Confirmed from all three single-chunk write frames in the 3-11-26 capture: + +| SUB | data[0:4] (hex) | data[1] | offset | total data len | +|---|---|---|---|---| +| 0x68 | `00 58 09 00` | 0x58=88 | 0x5A=90 | 91 | +| 0x82 | `00 1A D5 00` | 0x1A=26 | 0x1C=28 | 29 | +| 0x69 | `00 C8 08 00` | 0xC8=200 | 0xCA=202 | 204 | + +Full sequence: `68→73 | 71×3→72 | 82→83 | 69→74→72` + +### SUB 71 — compliance write chunk parameters + +The full compliance config payload (~2128 bytes) is split into exactly 3 chunks. +Confirmed from 3-11-26 BW TX capture frames 104–108: + +| Chunk | Size | `offset` | `params` (10 bytes hex) | +|---|---|---|---| +| 1 (first) | 1027 bytes | 0x1004 | `00 00 00 00 00 00 00 00 00 00` | +| 2 (middle) | 1055 bytes | 0x1004 | `00 00 00 10 04 00 00 00 00 00` | +| 3 (last) | remainder | 0x002C | `00 00 08 00 00 00 00 00 00 00` | + +Total: 1027 + 1055 + N = 2082 + N bytes (N ≈ 46 for a standard 2128-byte config). + +After all 3 chunks are acked (SUB 0x8E each), send SUB 72 confirm → device acks 0x8D. + +### `build_bw_write_frame()` — framing.py + +```python +build_bw_write_frame(sub, data, *, offset=0, params=bytes(10)) -> bytes +``` + +Use for all write commands (SUBs 68–83) including confirm frames (data=b""). +**Do NOT use `build_bw_frame` for write commands** — it uses standard SUM8, not the +large-frame DLE-aware checksum required for writes. + +### `push_config_raw()` — client.py + +```python +client.push_config_raw(event_index_data, compliance_data, trigger_data, waveform_data) +``` + +Orchestrates the full write sequence in the confirmed order. All payloads are raw bytes +(no encoding performed at this level). A higher-level encoder that builds payloads from +a `ComplianceConfig` object is a future task. + +--- + +## Monitoring commands (SUBs 0x1C, 0x96, 0x97) — confirmed 2026-04-08 + +All confirmed from 4-8-26/2ndtry BW TX/S3 capture (clean start → 30s monitor → stop cycle). + +### SUB 0x1C — Monitor status read + +Standard two-step read (probe at offset 0x00, data at offset 0x2C). +Response SUB = 0xFF − 0x1C = **0xE3** (standard formula — no exception). + +**Payload length is 46–47 bytes IDLE, 48–49 bytes MONITORING** — not a reliable sole +indicator due to 1-byte jitter overlap at the boundary. + +**Monitoring flag (CONFIRMED 2026-04-09 — byte diff of all 144 data frames, 2ndtry capture):** +- `section[1] == 0x00` → unit is **idle** +- `section[1] == 0x10` → unit is **monitoring** + +This is `data[12]` (= `frame.data[12]`). The flag is 0x00 in all 36 IDLE_BEFORE frames, +0x10 in all 98 MONITORING frames, and 0x00 in all 10 IDLE_AFTER frames — 100% accurate. + +**HISTORY OF THIS FIELD (do not re-derive):** The original implementation used `section[1]`. +A re-analysis in the prior session incorrectly concluded `section[1]` is always 0x00 and +"corrected" the flag to `section[6]`, which has non-binary values (0xea idle, 0x07 monitoring) +and is device-specific. The 2026-04-09 re-analysis confirms `section[1]` was right. + +**IMPORTANT — `frame.data` has checksum already stripped** by `S3FrameParser._finalise()` +(`raw_payload = body[:-1]`; `data = raw_payload[5:]`). There is NO trailing checksum byte in +`section`. All relative-from-end offsets must account for this. + +Battery and memory fields are present in **both** states: + +| Offset (relative to end) | Field | Type | Notes | +|---|---|---|---| +| `section[-10:-8]` | battery voltage × 100 | uint16 BE | `0x02A8` = 680 → 6.80 V | +| `section[-8:-4]` | memory total (bytes) | uint32 BE | e.g. 983026 ≈ 960 KB | +| `section[-4:]` | memory free (bytes) | uint32 BE | decreases as events are stored | + +### SESSION_RESET signal (`41 03`) — required for monitoring units + +Confirmed from 4-8-26 BW TX captures: Blastware sends a 2-byte `41 03` (ACK + ETX, +no STX) immediately before the first POLL probe AND between the probe and data frames. +This signal is **required to wake units that are actively monitoring** — without it +the unit does not respond to POLL over TCP. Harmless for idle units. + +`SESSION_RESET = bytes([0x41, 0x03])` is defined in `framing.py` and sent by +`protocol.startup()` before and between POLL frames. + +### SUB 0x96 — Start monitoring + +Single write frame, **no data payload** (empty body). +Response SUB = 0xFF − 0x96 = **0x69**. + +Wire bytes (confirmed frame 92 of 2ndtry BW capture): +``` +41 02 10 10 00 96 00 00 00 00 00 00 00 00 00 00 00 00 00 a6 03 +``` + +### SUB 0x97 — Stop monitoring + +Single write frame, **no data payload** (empty body). +Response SUB = 0xFF − 0x97 = **0x68**. + +Wire bytes (confirmed frame 305 of 2ndtry BW capture): +``` +41 02 10 10 00 97 00 00 00 00 00 00 00 00 00 00 00 00 00 a7 03 +``` + +Both start and stop acks are standard 17-byte zero-data S3 frames. + +### On-device sensor check behavior (confirmed 2026-04-08) + +Confirmed from 4-8-26/sensor-check BW+S3 capture (Blastware "Unit Channel Test" comms +check issued while unit was performing its on-device sensor check). + +**Unit IS reachable during on-device sensor check** — POLL (SUB 5B) responded normally +throughout. However, the unit partially handled channel-test commands (SUB 0x0E) for +channels 0–4 and then went **silent for ~40 seconds** while the sensor check ran, before +resuming responses for channels 5–7 and the trigger test (SUB 0x98). + +Key findings: +- On-device sensor check duration: approximately **40 seconds** (log gap `18:40:48` → `18:41:28`) +- Unit IS reachable for POLL during the check window — SESSION_RESET + POLL works +- Partial command responses during check are possible (device may buffer some, drop others) +- The Blastware "Unit Channel Test" (remote comms check, SUBs 0x0E + 0x98) is a SEPARATE + operation from the on-device check — it is a passive remote read; the unit's screen does + not change during a remote comms check + +**SFM behavior after `POST /device/monitor/start`:** `_pollMonitorConfirm()` polls +`/device/monitor/status` every 5 s for up to 60 s, updating the badge on each poll. +Status will show MONITORING once `section[1]` flips to `0x10`. + +### SUBs known from sensor-check capture (4-8-26) — NOT YET IMPLEMENTED + +| BW SUB | RSP SUB | Function | Notes | +|---|---|---|---| +| 0x15 | 0xEA | Serial number / short ID | 2-step read; data offset = 0x0A (10 bytes); confirmed serial `"BE11529"` at `data[11+5:]` | +| 0x01 | 0xFE | Device info block | 2-step read; data offset = 0x98 (152 bytes); payload includes serial + firmware + float config fields | +| 0x0E | 0xF1 | Channel sensor data | 2-step read; channel selector in `params[6:8]` (`0x0000`–`0x0007`); data length 0x0A per channel; used by Blastware "Unit Channel Test" — see docs/ for details | +| 0x98 | 0x67 | Trigger test | Single probe frame (`params[0]=0xFF`); sent twice per test cycle; all-zero data response; used after 0x0E channel scan | + +Blastware's "Unit Channel Test" sequence: `POLL×N → 0x15 → 0x01 → 0x08 → 0x01 → 0x0E×8 → 0x98×2 → 0x0E×8` (repeat pass with live ADC readings). + +--- + +## Compliance config field inventory (from Blastware UI, 2026-04-08) + +Fields visible in the Blastware Compliance Setup dialog — most are NOT YET decoded to byte +offsets in the raw 1A/E5 payload. Only fields with `✅` have confirmed offsets in the code. + +**Recording Setup tab:** +- Recording Mode: Continuous / Single Shot / Histogram (enum) +- Record Stop Mode: Fixed Record Time / Auto / Manual Stop (enum) +- Sample Rate: Standard 1024 / Fast 2048 / Faster 4096 sps ✅ (anchor−2) +- Record Time: float, seconds ✅ (anchor+10) +- Histogram Interval: 5 / 15 / 30 / 60 minutes (enum, mode-gated) +- Storage Mode: Save All Data / Save Triggered (enum) +- Geophone Type: Standard Triaxial / 4.5 Hz (bool/enum) +- Geophone Channels: Enable all geophones (bool), Trigger Source (bool) +- Chan 1-3 Trigger Level (float, in/s) ✅ (`trigger_level_geo`) +- Chan 1-3 Maximum Range: Normal 10.000 / 1.25 in/s (enum) ❓ (`max_range_geo` — offset found, reads 6.206053 which matches neither UI value; units and meaning unknown — do NOT use as ADC full-scale) +- Microphone Channels: Enable all microphones (bool), Trigger Source (bool) +- Chan 4 Trigger Level (dB or psi depending on units) + +**Notes tab:** +- Enable User Notes (bool) +- Project, Client, User Name, Seis Loc (ASCII strings) ✅ (sourced from A5 frame 7 via 5A) +- Enable Extended Notes (bool); Extended Notes text; Extended Notes Title +- Enable Job Number (bool); Job Number (int) +- Enable Scaled Distance (bool); Distance from Blast (float); Charge Weight (float) — Scaled Distance is derived + +**Special Setups tab:** +- Unit Timer: Timer Mode (Off/On), Start Date/Time, Stop Date/Time +- Self Check: Mode (Off/On), Time (HH:MM) +- Sensor Check: **Before monitoring** / After each event / **Disabled** ❓ (byte offset unknown) +- Measurement Units: Imperial / Metric +- Show Mic units in dB (bool) +- Time Format: 24 Hour / 12 Hour (AM/PM) +- Backlight on Time (seconds) ✅ (event index block +75) +- Power Saving Timeout (minutes) ✅ (event index block +83) +- Monitoring LCD Cycle ✅ (event index block +84:86) +- Set unit time with setup (bool) + +The "Sensor Check" dropdown (`Before monitoring` / `After each event` / `Disabled`) has NOT +been located in the raw config bytes. The user's unit always runs with `Before monitoring`. +Full compliance config encoder is a future task. + +--- + +--- + +## Erase-all protocol (SUBs 0xA3/0xA2/0x06) — confirmed 2026-04-11 + +Full sequence confirmed from 4-11-26 MITM capture of a live Blastware ACH session +(`bridges/captures/mitm/ach_mitm_20260411_001912/`). + +### Wire sequence + +``` +BW → device: SUB 0xA3 params=00 00 00 00 00 00 00 FE 00 00 (begin erase) +device → BW: SUB 0x5C (ack) +BW → device: SUB 0x1C probe (offset=0x00) +device → BW: SUB 0xE3 (probe ack) +BW → device: SUB 0x1C data (offset=0x2C) +device → BW: SUB 0xE3 (monitor status response) +BW → device: SUB 0x06 probe (offset=0x00, params same) +device → BW: SUB 0xF9 (probe ack) +BW → device: SUB 0x06 data (offset=0x24) +device → BW: SUB 0xF9 (36-byte storage range response) +BW → device: SUB 0xA2 params=00 00 00 00 00 00 00 FE 00 00 (confirm erase) +device → BW: SUB 0x5D (ack — device memory is now cleared) +``` + +All frames use standard `build_bw_frame` (not write-format). Response SUBs follow the +standard `0xFF - SUB` formula; no exceptions. + +### SUB 0x06 — event storage range response (36 bytes) + +The 36-byte response body ends with two 4-byte event keys: + +| Offset (from end) | Field | Notes | +|---|---|---| +| `[-8:-4]` | first stored event key | `01110000` when empty | +| `[-4:]` | last stored event key | `01110000` when empty | + +Before erase: ends with ` ` (e.g. `0111ea60 0111eaa6`). +After erase: both bytes read `01110000` — device's empty/reset sentinel. + +### Post-erase key counter reset + +After a successful erase, the device resets its event counter. New events start from +key `0x01110000` again — the same key as the very first event ever recorded. This means +key-based deduplication in the ACH server must account for key reuse: + +- After our own erase: `ach_state.json` `downloaded_keys` and `max_downloaded_key` are + cleared so the next session starts fresh. +- After an external erase: the ACH server detects it by comparing `max(device_keys)` to + `max_downloaded_key` from state. If the device max has rolled back below the historical + max, all current device keys are treated as new regardless of `seen_keys`. + +### ACH server state format (v0.9.0) + +`bridges/captures/ach_state.json`: +```json +{ + "BE11529": { + "downloaded_keys": ["01110000", "0111245a"], + "max_downloaded_key": "0111245a", + "last_seen": "2026-04-11T01:04:36", + "serial": "BE11529", + "peer": "63.43.212.232:51920" + } +} +``` + +`max_downloaded_key` is the high-water mark — the largest key ever downloaded from the +unit. It is NOT reset when events are erased from the device (only when our server does +the erase). Used for post-erase detection. + +--- + +## Monitor log entries — SUB 0x0A partial records (confirmed 2026-04-11) + +Confirmed from 4-11-26 MITM capture: 12 partial records (record type `0x2C`) and 7 full +event records (record type `0x46`) across 19 total 0x0A responses. + +### Record type detection + +`read_waveform_header()` returns `(raw_data, length)` where `raw_data = data_rsp.data` +(the full payload including prefix bytes). The record type is at `raw_data[0]`: + +| Value | Type | How to process | +|---|---|---| +| `0x46` | Full triggered event | Normal download: 0C → 5A → 1F | +| `0x2C` | Monitor log entry (partial) | No 0C/5A; decode inline from 0A payload | + +Length heuristic: `length < 0x40` (64) reliably identifies partial records across all +observed captures. Both checks (`raw_data[0] == 0x2C` and `length < 0x40`) are used. + +### SUB 0x0A partial record (0x2C) payload layout + +All offsets are from `raw_data` (the full `data_rsp.data` array including the 11-byte +prefix before the actual header bytes start). + +``` +raw_data[0] = 0x2C ← record type (partial / monitor log) +raw_data[1:11] = prefix bytes (vary; contain key4 copy, flags, length) +raw_data[11:] = timestamp and ASCII metadata payload +``` + +**Timestamp auto-detection** (confirmed from 4-11-26 capture): + +``` +raw_data[11] == 0x10 → 10-byte sub_code=0x03 format (continuous mode) +raw_data[11] != 0x10 → 9-byte sub_code=0x10 format (single-shot mode) +``` + +**9-byte timestamp format (sub_code=0x10):** + +| Byte | Field | +|---|---| +| 0 | day | +| 1 | `0x10` (sub_code marker) | +| 2 | month | +| 3–4 | year (uint16 BE) | +| 5 | unknown (0x00) | +| 6 | hour | +| 7 | minute | +| 8 | second | + +**10-byte timestamp format (sub_code=0x03):** + +| Byte | Field | +|---|---| +| 0 | `0x10` (marker) | +| 1 | day | +| 2 | `0x10` (marker) | +| 3 | month | +| 4–5 | year (uint16 BE) | +| 6 | unknown (0x00) | +| 7 | hour | +| 8 | minute | +| 9 | second | + +**Two timestamps:** Each partial record contains two timestamps — `start_time` and +`stop_time` — stored consecutively: +- `ts1` (start) at `raw_data[ts_offset : ts_offset + ts_size]` where `ts_offset = 11` +- `ts2` (stop) at `raw_data[ts1_end : ts1_end + ts_size]` + +**Edge case — 1-byte gap between timestamps:** Occurs when ts1 and ts2 share the same +minute:second. If `try_ts(raw_data[ts1_end:])` fails, try `try_ts(raw_data[ts1_end+1:])`. +Confirmed in frames 121, 161, 165 of the 4-11-26 MITM capture. Frame 121 still shows 0s +duration (both decode to 16:02:00) — the extra byte appears in all same-second cases. + +**ASCII metadata after timestamps:** +``` + BE\x00Geo: in/s ... +``` + +- Serial: scan for `b"BE"`, read until `b"\x00"` (e.g. `"BE11529"`) +- Geo threshold: scan for `b"Geo: "`, read float until next space (e.g. `0.254` in/s) + +A separator of variable length (4–5 bytes of `\x00` + flags) sits between the two +timestamps and the ASCII region. The `b"BE"` anchor scan is robust to separator length +variation. + +### `_decode_0a_partial_header(raw_data, index, key4)` — client.py + +Returns a `MonitorLogEntry` or `None`. Called by `get_monitor_log_entries()` for each +event key whose 0x0A response has `raw_data[0] == 0x2C` or `length < 0x40`. + +### `MiniMateClient.get_monitor_log_entries(skip_keys=None)` — client.py + +Browse-mode walk: `1E → 0A → check type → decode if partial → 1F`. No 0x0C or 5A reads +performed. Full (0x46) records are skipped without decoding. Returns `list[MonitorLogEntry]`. + +`skip_keys` (optional `set[str]`): keys in this set are still advanced through the walk +(to avoid disrupting the iteration sequence), but no `MonitorLogEntry` is created for them. + +### `MonitorLogEntry` model — models.py + +```python +@dataclass +class MonitorLogEntry: + index: int # 0-based position + key: str # 8-hex event key + start_time: Optional[datetime.datetime] = None + stop_time: Optional[datetime.datetime] = None + serial: Optional[str] = None + geo_threshold_ips: Optional[float] = None + raw_header: Optional[bytes] = field(default=None, repr=False) + + @property + def duration_seconds(self) -> Optional[float]: ... +``` + +### ACH server integration (v0.10.0) + +After `get_events()`, the ACH server calls `get_monitor_log_entries(skip_keys=seen_keys)`. +New entries are saved to `monitor_log.json` in the session directory. Monitor log keys are +included in `current_keys` for state persistence so they are not re-processed on the next +call-home. + +--- + +## What's next + +- **Database** — SQLite store for events + monitor log entries; dedup by key; queryable +- **Histograms** — decode histogram-mode A5 data (noise floor tracking) +- Compliance config encoder — build raw write payloads from a `ComplianceConfig` object +- Locate "Sensor Check" byte in compliance config (need capture with Disabled vs Before-monitoring) +- Modem manager — push RV50/RV55 configs via Sierra Wireless API +- RV55 DCD/DTR issue — newer RV55 firmware doesn't assert DCD by default; units don't + resume monitoring after call-home disconnect (`--restart-monitoring` flag deferred) + \ No newline at end of file diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 356cb07..0692402 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -36,7 +36,7 @@ | 2026-03-02 | §7.4 Event Index Block | **NEW:** `Monitoring LCD Cycle` identified at offsets +84/+85 as uint16 BE. Default value = 65500 (0xFFDC) = effectively disabled / maximum. Confirmed from operator manual §3.13.1g. | | 2026-03-02 | §7.4 Event Index Block | **UPDATED:** Backlight confirmed as uint8 range 0–255 seconds per operator manual §3.13.1e ("adjustable timer, 0 to 255 seconds"). Power save unit confirmed as minutes per operator manual §3.13.1f. | | 2026-03-02 | Global | **NEW SOURCE:** Operator manual (716U0101 Rev 15) added as reference. Cross-referencing settings definitions, ranges, and units. Header updated. | -| 2026-03-02 | §14 Open Questions | Float 6.2061 in/s mystery: manual confirms only two geo ranges (1.25 in/s and 10.0 in/s). 6.2061 is NOT a user-selectable range → likely internal ADC full-scale calibration constant or hardware range ceiling. Downgraded to LOW priority. | +| 2026-03-02 | §14 Open Questions | Float 6.2061 in/s mystery: manual confirms only two geo ranges (1.25 in/s and 10.0 in/s). 6.2061 is NOT a user-selectable range → originally speculated as internal ADC full-scale constant, but this is NOT confirmed. Using it as ADC full-scale produces ~9× PPV overread. Meaning unknown. Downgraded to LOW 2026-03-02, re-escalated to HIGH 2026-04-16. | | 2026-03-02 | §14 Open Questions | `0x082A` hypothesis refined: 2090 decimal. At 1024 sps, 2 sec record = 2048 samples. Possible that 0x082A = total samples including 0.25s pre-trigger (256 samples) at some adjusted rate. Needs capture with different record time. | | 2026-03-02 | §14 Open Questions | **NEW items added:** Trigger sample width (default=2), Auto Window (1-9 sec), Aux Trigger (enabled/disabled) — all confirmed settings from operator manual not yet mapped in protocol. | | 2026-03-02 | §14 Open Questions | Monitoring LCD Cycle resolved — removed from open questions. | @@ -92,7 +92,7 @@ | 2026-04-06 | §7.8.4 | **NEW — 5A end-of-stream signalling confirmed.** After streaming all waveform chunks, the device sends exactly **1 raw byte** in response to the next chunk request, then goes silent for the full recv timeout. This byte is NOT a complete DLE-framed A5 response — the frame parser accumulates it as `bytes_fed=1` and never assembles a frame. This is the device's natural end-of-stream signal. Handling: on TimeoutError, if `bytes_fed > 0` AND prior chunks were received, treat as graceful end and proceed to the termination frame. A `bytes_fed=0` timeout with no prior chunks is a genuine transport failure and must still raise. | | 2026-04-06 | §7.8.4 | **NEW — 5A chunk timing and count (empirical, BE11529 at 1024 sps).** Each chunk response arrives within ~1 second over TCP/cellular. A 9,306-sample event (≈9.1 s at 1024 sps) produces **35 chunks** before end-of-stream. Chunks 1–16 have varying data lengths (1036–1123 bytes); chunks 17–35 are uniformly 1036 bytes each (post-event silence, all-zero ADC samples). Safe recv timeout for chunk loop: **10 s** (10× typical response time). Default transport timeout (120 s) results in a ~2-minute stall per event at end-of-stream. | | 2026-04-06 | §7.8.3 | **KNOWN ISSUE — `_decode_a5_waveform` hardcoded fi==9 skip.** The decoder contains `elif fi == 9: continue` which was written for the 9-frame original blast capture where frame 9 was a device terminator. For streams with >9 frames (current device produces 35+), frame index 9 is live waveform data — this skip discards ~1,070 bytes (~133 sample-sets) per event. The terminator is now detected via `page_key == 0x0000`, not by frame index. The fi==9 skip should be removed. | -| 2026-04-06 | §7.8 | **CONFIRMED — ADC count-to-physical-unit conversion.** Raw waveform samples are signed 16-bit integers (counts). Conversion: `value = counts × (range / 32767)`. For geo channels: range = 10.000 in/s (from the device's compliance config geo range field). For the mic channel: range is in psi (device-specific). Near-full-scale counts (≈32,700) on all four channels simultaneously indicate ADC saturation (clipping) from a high-amplitude event. | +| 2026-04-06 | §7.8 | **⚠ PARTIALLY INVALIDATED — ADC count-to-physical-unit conversion.** Raw waveform samples are signed 16-bit integers (counts). Conversion formula `value = counts × (range / 32767)` is believed correct, but the `range` value is UNKNOWN. The compliance config field labeled `max_range_geo` reads 6.206053 (bytes `40 C6 97 FD`), which does NOT match either user-selectable range shown in Blastware UI (1.25 or 10.000 in/s). The meaning and units of the 6.206053 value are unresolved — it may not be the ADC full-scale at all. See open question in §14. | | 2026-04-08 | §5.1, §7.10, §12 | **NEW — Monitoring commands confirmed.** SUB 0x1C (monitor status), 0x96 (start monitoring), 0x97 (stop monitoring) all confirmed from 4-8-26/2ndtry capture. SESSION_RESET (`41 03`) required before POLL to wake a monitoring unit. | | 2026-04-09 | §7.10 | **CORRECTED — monitoring flag and battery/memory offsets.** `section[1] == 0x10` is the monitoring flag (100% accurate across 144 data frames in 2ndtry capture). Previous note claiming `section[6]` was wrong — section[6] has device-specific non-binary values (0xea/0x07). Battery/memory offsets corrected: `section[-10:-8]` (battery×100), `section[-8:-4]` (memory_total), `section[-4:]` (memory_free). NOTE: `frame.data` has checksum stripped by parser — earlier offsets of `[-11:-9]`/`[-9:-5]`/`[-5:-1]` were wrong because they assumed a trailing checksum byte that isn't there. | | 2026-04-08 | §7.10 | **NEW — SUBs 0x0E (channel sensor data) and 0x98 (trigger test) observed** in 4-8-26/sensor-check capture (Blastware "Unit Channel Test" comms check). SUB 0x0E: 2-step read with channel selector in `params[6:8]`, data length 0x0A per channel, RSP SUB = 0xF1. SUB 0x98: single probe frame with `params[0] = 0xFF`, RSP SUB = 0x67; sent twice per test cycle. Not yet implemented in SFM. | @@ -528,7 +528,7 @@ The SUB `1A` read response (`E5`) and SUB `71` write block contain per-channel t | Field | Example bytes | Decoded | Certainty | |---|---|---|---| | `[00 00]` | `00 00` | Separator / padding | 🔶 INFERRED | -| Max range float | `40 C6 97 FD` | 6.206 — full-scale range in in/s | 🔶 INFERRED | +| Max range float | `40 C6 97 FD` | 6.206 — **value confirmed, meaning and units UNKNOWN** (does NOT match UI range options 1.25/10.000 in/s; not confirmed as ADC full-scale) | ❓ UNKNOWN | | `[00 00]` | `00 00` | Separator / padding | 🔶 INFERRED | | **Trigger level** | `3F 19 99 9A` | **0.600 in/s** — IEEE 754 BE float | ✅ CONFIRMED | | Unit string | `69 6E 2E 00` | `"in.\0"` | ✅ CONFIRMED | @@ -655,7 +655,7 @@ offset size type value (Tran example) meaning +10 2 uint16 0x0015 = 21 unknown +12 4 bytes 03 02 04 01 flags (recording mode etc.) +16 4 uint32 0x00000003 record time in seconds ✅ CONFIRMED -+1A 4 float32 6.2061 max range (in/s for geo, psi for mic) ++1A 4 float32 6.2061 ❓ UNKNOWN field — value 6.2061 confirmed; meaning/units unresolved (NOT confirmed as max range or ADC full-scale) +1E 2 00 00 padding +20 4 float32 0.6000 trigger level ✅ CONFIRMED +24 4 char[4] "in.\0" / "psi\0" unit string (geo vs mic) @@ -1235,13 +1235,13 @@ TimeoutError caught: Chunks with uniform 1,036-byte payload (chunks 17–35 in the observed event) contain all-zero ADC samples — the device continues recording silence until the configured record time expires before terminating the stream. -**ADC count-to-physical conversion:** +**ADC count-to-physical conversion — ⚠ SCALING UNKNOWN:** -Raw samples are signed 16-bit integers (−32,768 to +32,767). To convert to physical units: +Raw samples are signed 16-bit integers (−32,768 to +32,767). The conversion formula is believed to be: ``` value_in_s (in/s) = counts × (geo_range / 32767) ``` -where `geo_range` is from the compliance config (typically 10.000 in/s). Mic channel uses psi units with its own range. Near-full-scale values on all channels simultaneously indicate ADC saturation (clipping). +However, the correct value of `geo_range` is **unknown**. The compliance config field `max_range_geo` reads 6.206053 (`40 C6 97 FD`) which does NOT match either user-selectable range (1.25 or 10.000 in/s) and produces ~9× too large PPV values compared to the on-device 0C record. Do not use 6.206053 or 10.000 as the scale factor until this is resolved. See §14 open question. Mic channel uses psi units with its own range (also unresolved). **Known decoder issue — fi==9 hardcoded skip:** @@ -1267,7 +1267,7 @@ Fields visible in the Blastware "Compliance Setup" dialog. ✅ = byte offset co | Geophone — Enable all | bool | ❓ | | Geophone — Trigger Source | bool | ❓ | | Chan 1-3 Trigger Level | float, in/s | ✅ `trigger_level_geo` | -| Chan 1-3 Maximum Range | Normal 10.000 / 1.25 in/s | ✅ `max_range_geo` | +| Chan 1-3 Maximum Range | Normal 10.000 / 1.25 in/s | ❓ `max_range_geo` offset found, value=6.206053 — does NOT match UI values; meaning unknown | | Microphone — Enable all | bool | ❓ | | Microphone — Trigger Source | bool | ❓ | | Chan 4 Trigger Level | float, dB or psi | ❓ | @@ -1933,7 +1933,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **Auxiliary Trigger read location** — **RESOLVED:** SUB `FE` offset `0x0109`, uint8, `0x00`=disabled, `0x01`=enabled. Confirmed 2026-03-11 via controlled toggle capture. | RESOLVED | 2026-03-02 | Resolved 2026-03-11 | | **Auxiliary Trigger write path** — Write command not yet captured in a clean session. Inner frame handshake visible in A4 (multiple WRITE_CONFIRM_RESPONSE SUBs appear, TRIGGER_CONFIG_RESPONSE removed), but the BW→S3 write command itself was in a partial session. Likely SUB `15` or similar. Deferred for clean capture. | LOW | 2026-03-11 | NEW | | ~~**SUB `6E` response to SUB `1C`**~~ — ~~RESOLVED 2026-04-08: This was a misidentification.~~ The `1C → 6E` "exception" was misread — likely an inner A4 sub-frame. Confirmed from 4-8-26 capture (338 frames): SUB 0x1C always → 0xE3. No exceptions to the `0xFF − SUB` rule are known. | RESOLVED | 2026-04-08 | CLOSED | -| **Max Geo Range float 6.2061 in/s** — NOT a user-selectable range (manual only shows 1.25 and 10.0 in/s). Likely internal ADC full-scale constant or hardware range ceiling. Not worth capturing. | LOW | 2026-02-26 | Downgraded 2026-03-02 | +| **Max Geo Range float 6.2061** — offset confirmed in channel block (`+1A`, `40 C6 97 FD`). Meaning and units are UNKNOWN. Value does NOT match either user-selectable range (1.25 / 10.0 in/s). Using it as ADC full-scale produces ~9× PPV overread vs on-device 0C values. Not simply metric vs imperial (25.4 factor doesn't reconcile). Needs investigation: examine surrounding channel block bytes, compare with a Blastware waveform CSV export to back-calculate the correct scale. Upgraded to HIGH priority. | HIGH | 2026-02-26 | Upgraded 2026-04-16 | | MicL channel units — **RESOLVED: psi**, confirmed from `.set` file unit string `"psi\0"` | RESOLVED | 2026-03-01 | | | Backlight offset — **RESOLVED: +4B in event index data**, uint8, seconds | RESOLVED | 2026-03-02 | | | Power save offset — **RESOLVED: +53 in event index data**, uint8, minutes | RESOLVED | 2026-03-02 | | @@ -1962,7 +1962,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | Trigger Level (Mic) | §3.8.6 | Channel block, float | float32 BE | 100–148 dB in 1 dB steps | | Alarm Level (Mic) | §3.9.10 | Channel block, float | float32 BE | higher than mic trigger | | Record Time | §3.8.9 | cfg anchor+10, float32 BE (wire); `.set` +16, uint32 LE (file) | float32 BE (wire) | 1–105 s; confirmed 3→`40400000`, 5→`40A00000`, 8→`41000000`, 13→`41500000`. Use anchor §7.6.1/§7.6.3 — NOT fixed offset. | -| Max Geo Range | §3.8.4 | Channel block, float | float32 BE | 1.25 or 10.0 in/s (user); 6.2061 in protocol = internal constant | +| Max Geo Range | §3.8.4 | Channel block, float | float32 BE | ❓ UNKNOWN — value 6.2061 confirmed at offset, but meaning/units unresolved. Does NOT equal 1.25 or 10.0 in/s. Do NOT use as ADC full-scale. | | Microphone Units | §3.9.7 | Inline unit string | char[4] | `"psi\0"`, `"pa.\0"`, `"dB\0\0"` | | Sample Rate | §3.8.2 | cfg anchor−2, uint16 BE — anchor=`\x01\x2c\x00\x00\xbe\x80\x00\x00\x00\x00` in cfg[40:100] | uint16 BE | Normal=1024, Fast=2048, Faster=4096 ✅ CONFIRMED 2026-04-01 (BE11529 S338.17). Anchor required — see §7.6.3 DLE jitter. | | Record Mode | §3.8.1 | Unknown | — | Single Shot, Continuous, Manual, Histogram, Histogram Combo | From b8ed237363f6fcc8aee4f502c3a2dfe9173621a0 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 16 Apr 2026 18:31:20 -0400 Subject: [PATCH 33/33] docs: update to 0.12.1 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ CLAUDE.md | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index defc04f..5076959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,32 @@ All notable changes to seismo-relay are documented here. --- +## v0.12.1 — 2026-04-16 + +### Added + +- **`sfm/server.py` — `_LiveCache`** — in-memory live device cache that eliminates + redundant TCP round-trips between web requests. Plain Python dict + + `threading.Lock`, no extra dependencies. + + Cache strategy per endpoint: + + | Endpoint | Strategy | + |---|---| + | `GET /device/info` | Indefinite; invalidated by `POST /device/config` | + | `GET /device/events` | Count-probe fast path — `poll()+count_events()` (~2 s); returns cached data if event count is unchanged; full download only when new events are detected | + | `GET /device/monitor/status` | 30-second TTL; invalidated immediately on monitor start/stop | + | `GET /device/event/{idx}/waveform` | Permanent per-index (waveforms are immutable once recorded) | + +- **`?force=true`** query param on all cached endpoints — bypasses cache and forces + a fresh read from the device. + +- **Cache invalidation hooks** — `POST /device/config` marks device info and events + stale; `POST /device/monitor/start` and `/stop` evict the monitor status entry + immediately so the next status poll reflects the actual device state. + +--- + ## v0.12.0 — 2026-04-13 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index c642602..d98ea38 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.12.0**. +(Sierra Wireless RV50 / RV55). Current version: **v0.12.1**. ---