From 3e7de848bc1c8efb0d7f239025bac430a10c8f3d Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 22:45:58 -0400 Subject: [PATCH] fix: update unique constraints in events and monitor_log tables to use timestamp and serial number. Can't use event keys because minimates resuse them after clearing memory. --- sfm/database.py | 92 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/sfm/database.py b/sfm/database.py index e0172e2..110ba7a 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -4,8 +4,8 @@ sfm/database.py — SQLite persistence layer for seismo-relay. Three tables, all keyed by unit serial number: ach_sessions — one row per inbound ACH call-home - events — one row per triggered waveform event (deduped by serial+key) - monitor_log — one row per monitoring interval (deduped by serial+key) + events — one row per triggered waveform event (deduped by serial+timestamp) + monitor_log — one row per monitoring interval (deduped by serial+start_time) The DB file lives at: /seismo_relay.db (default: bridges/captures/seismo_relay.db) @@ -82,7 +82,7 @@ CREATE TABLE IF NOT EXISTS events ( record_type TEXT, -- "single_shot" | "continuous" false_trigger INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=yes (manual flag) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), - UNIQUE(serial, waveform_key) + UNIQUE(serial, timestamp) ); CREATE INDEX IF NOT EXISTS idx_events_serial ON events(serial); CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp); @@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS monitor_log ( duration_seconds REAL, geo_threshold_ips REAL, -- in/s created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), - UNIQUE(serial, waveform_key) + UNIQUE(serial, start_time) ); CREATE INDEX IF NOT EXISTS idx_monitor_log_serial ON monitor_log(serial); CREATE INDEX IF NOT EXISTS idx_monitor_log_start ON monitor_log(start_time); @@ -135,6 +135,86 @@ class SeismoDb: def _init_schema(self) -> None: with self._connect() as conn: conn.executescript(_SCHEMA) + self._migrate(conn) + + def _migrate(self, conn: sqlite3.Connection) -> None: + """Apply in-place schema migrations for existing databases.""" + + # Migration 1: change events UNIQUE from (serial, waveform_key) [or any + # waveform_key-based variant] to (serial, timestamp). + # Rationale: device key counter resets to 01110000 after every erase, so + # waveform_key is not a stable dedup field across erase cycles. The event + # timestamp (from the device clock) is the correct natural key. + row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='events'" + ).fetchone() + if row and "UNIQUE(serial, timestamp)" not in row[0]: + log.info("_migrate: rebuilding events table — UNIQUE(serial, timestamp)") + conn.executescript(""" + ALTER TABLE events RENAME TO events_old; + + CREATE TABLE events ( + id TEXT PRIMARY KEY, + serial TEXT NOT NULL, + waveform_key TEXT NOT NULL, + session_id TEXT, + timestamp TEXT, + tran_ppv REAL, + vert_ppv REAL, + long_ppv REAL, + peak_vector_sum REAL, + mic_ppv REAL, + project TEXT, + client TEXT, + operator TEXT, + sensor_location TEXT, + sample_rate INTEGER, + record_type TEXT, + false_trigger INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(serial, timestamp) + ); + + INSERT OR IGNORE INTO events SELECT * FROM events_old; + DROP TABLE events_old; + + CREATE INDEX IF NOT EXISTS idx_events_serial ON events(serial); + CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp); + CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id); + """) + log.info("_migrate: events table rebuilt OK") + + # Migration 2: change monitor_log UNIQUE from (serial, waveform_key) to + # (serial, start_time) — same reasoning as events. + row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='monitor_log'" + ).fetchone() + if row and "UNIQUE(serial, start_time)" not in row[0]: + log.info("_migrate: rebuilding monitor_log table — UNIQUE(serial, start_time)") + conn.executescript(""" + ALTER TABLE monitor_log RENAME TO monitor_log_old; + + CREATE TABLE monitor_log ( + id TEXT PRIMARY KEY, + serial TEXT NOT NULL, + waveform_key TEXT NOT NULL, + session_id TEXT, + start_time TEXT, + stop_time TEXT, + duration_seconds REAL, + geo_threshold_ips REAL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(serial, start_time) + ); + + INSERT OR IGNORE INTO monitor_log SELECT * FROM monitor_log_old; + DROP TABLE monitor_log_old; + + CREATE INDEX IF NOT EXISTS idx_monitor_log_serial ON monitor_log(serial); + CREATE INDEX IF NOT EXISTS idx_monitor_log_start ON monitor_log(start_time); + CREATE INDEX IF NOT EXISTS idx_monitor_log_session ON monitor_log(session_id); + """) + log.info("_migrate: monitor_log table rebuilt OK") @staticmethod def _iso(dt: Optional[datetime.datetime]) -> Optional[str]: @@ -204,7 +284,7 @@ class SeismoDb: session_id: Optional[str] = None, ) -> tuple[int, int]: """ - Insert triggered events. Silently skips duplicates (serial+waveform_key). + Insert triggered events. Silently skips duplicates (serial+timestamp). Returns (inserted, skipped). """ inserted = skipped = 0 @@ -316,7 +396,7 @@ class SeismoDb: session_id: Optional[str] = None, ) -> tuple[int, int]: """ - Insert monitor log entries. Silently skips duplicates (serial+waveform_key). + Insert monitor log entries. Silently skips duplicates (serial+start_time). Returns (inserted, skipped). """ inserted = skipped = 0