serversdown/seismo-relay
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request 'v0.23.0 - ZC freq in events store' (#31) from dev into main

Browse Source 
Reviewed-on: #31
This commit was merged in pull request #31.
This commit is contained in:
serversdown
2026-08-06 12:12:19 -04:00
parent 483762607e aebb5644bd
commit 5b65718b72
8 changed files with 387 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+34
View File
@@ -8,6 +8,40 @@ All notable changes to seismo-relay are documented here.
---
## v0.23.0 — 2026-08-04
**Per-channel ZC frequency in the events store.** The `events` table and the
`/db/events` feed now carry per-channel zero-crossing frequency, so consumers
(Terra-View's Vibration Summary) get frequency in bulk instead of one sidecar
fetch per event.
### Added
- **`events` columns** `tran_zc_freq`, `vert_zc_freq`, `long_zc_freq`, `mic_zc_freq`
(Hz) + `*_zc_above_range` flags (the Blastware `>N Hz` device-ceiling case). Added
via the existing incremental `_migrate` ADD COLUMN pass — **auto-migrates on
`SeismoDb()` construction, no manual migration**. `query_events` / `get_event`
(and thus `/db/events`) return them automatically (`SELECT *`).
- **Populated at ingest** — `PeakValues` gains the ZC-freq fields; both
`apply_report_to_event` and `apply_bw_report_dict_to_event` copy per-channel
`zc_freq_hz` / `zc_freq_above_range` from the parsed report onto them, and
`insert_events` persists them on INSERT and UPSERT.
- **Backfill** `scripts/backfill_event_zc_freq.py` — fills the columns for existing
events from their `.sfm.json` sidecars (idempotent, UPDATE-only).
### Fixed
- `sfm/server.py` app-version string was stale at `0.17.0`; now tracks the real version.
### Upgrade Notes
Run the backfill once after deploying, against the events DB + waveform store:
`python3 scripts/backfill_event_zc_freq.py --db-path <seismo_relay.db> --store-root <waveforms/>`
Events whose sidecar carries no frequency (or that have no sidecar) stay NULL and
render "—" downstream.
---
## v0.22.0 — 2026-07-03
Full-snapshot bundle support (SFM side). Adds the DB-snapshot, recent-waveform,
minimateplus/event_file_io.py
+24
View File
@@ -270,6 +270,19 @@ def apply_report_to_event(event: Event, report: BwAsciiReport) -> None:
if report.mic.pspl_dbl is not None and report.mic.pspl_dbl > 0:
pv.micl = _dbl_to_psi(report.mic.pspl_dbl)
if (t := ch.get("Tran")):
pv.tran_zc_freq = t.zc_freq_hz
pv.tran_zc_above_range = bool(getattr(t, "zc_freq_above_range", False))
if (v := ch.get("Vert")):
pv.vert_zc_freq = v.zc_freq_hz
pv.vert_zc_above_range = bool(getattr(v, "zc_freq_above_range", False))
if (l := ch.get("Long")):
pv.long_zc_freq = l.zc_freq_hz
pv.long_zc_above_range = bool(getattr(l, "zc_freq_above_range", False))
if report.mic is not None:
pv.mic_zc_freq = report.mic.zc_freq_hz
pv.mic_zc_above_range = bool(getattr(report.mic, "zc_freq_above_range", False))
if event.project_info is None:
event.project_info = ProjectInfo()
pi = event.project_info
@@ -329,6 +342,17 @@ def apply_bw_report_dict_to_event(event: Event, bw_report: dict) -> None:
if pspl is not None and pspl > 0:
pv.micl = _dbl_to_psi(pspl)
for axis, freq_attr, above_attr in (
("tran", "tran_zc_freq", "tran_zc_above_range"),
("vert", "vert_zc_freq", "vert_zc_above_range"),
("long", "long_zc_freq", "long_zc_above_range"),
):
chd = peaks.get(axis) or {}
setattr(pv, freq_attr, chd.get("zc_freq_hz"))
setattr(pv, above_attr, bool(chd.get("zc_freq_above_range", False)))
pv.mic_zc_freq = mic.get("zc_freq_hz")
pv.mic_zc_above_range = bool(mic.get("zc_freq_above_range", False))
rec = bw_report.get("recording") or {}
sr = rec.get("sample_rate_sps")
if sr:
minimateplus/models.py
+8
View File
@@ -352,6 +352,14 @@ class PeakValues:
long: Optional[float] = None # Longitudinal PPV (in/s) ✅
micl: Optional[float] = None # Air overpressure (psi) 🔶 (units uncertain)
peak_vector_sum: Optional[float] = None # Scalar geo PVS (in/s) ✅
tran_zc_freq: Optional[float] = None
vert_zc_freq: Optional[float] = None
long_zc_freq: Optional[float] = None
mic_zc_freq: Optional[float] = None
tran_zc_above_range: bool = False
vert_zc_above_range: bool = False
long_zc_above_range: bool = False
mic_zc_above_range: bool = False
# ── Project / operator metadata ───────────────────────────────────────────────
pyproject.toml
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "seismo-relay"
version = "0.22.0"
version = "0.23.0"
description = "Python client and REST server for MiniMate Plus seismographs"
requires-python = ">=3.10"
dependencies = [
scripts/backfill_event_zc_freq.py
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Backfill events.*_zc_freq / *_zc_above_range from .sfm.json sidecars.
Idempotent. Reads each event's sidecar bw_report and writes the per-channel
ZC frequency into the DB columns via a direct UPDATE. Events with no sidecar
(or no bw_report freq) are left untouched. Mirrors scripts/backfill_record_type.py.
"""
from __future__ import annotations
import argparse
import logging
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from sfm.database import SeismoDb
from sfm.waveform_store import WaveformStore
log = logging.getLogger("backfill_event_zc_freq")
def _freqs_from_sidecar(sidecar: dict) -> dict | None:
bw = (sidecar or {}).get("bw_report") or {}
peaks = bw.get("peaks") or {}
mic = bw.get("mic") or {}
out = {}
for axis in ("tran", "vert", "long"):
chd = peaks.get(axis) or {}
out[f"{axis}_zc_freq"] = chd.get("zc_freq_hz")
out[f"{axis}_zc_above_range"] = 1 if chd.get("zc_freq_above_range") else 0
out["mic_zc_freq"] = mic.get("zc_freq_hz")
out["mic_zc_above_range"] = 1 if mic.get("zc_freq_above_range") else 0
# None if there's literally no frequency anywhere.
if all(out[f"{a}_zc_freq"] is None for a in ("tran", "vert", "long", "mic")):
return None
return out
def backfill_zc_freq(db: SeismoDb, store, *, dry_run: bool = False) -> dict:
counts = {"updated": 0, "skipped_no_sidecar": 0, "skipped_no_freq": 0}
# Pull every event (paginate generously; adjust if a store has >100k events).
rows = db.query_events(limit=1_000_000)
for row in rows:
serial = row.get("serial")
filename = row.get("blastware_filename")
if not serial or not filename:
counts["skipped_no_sidecar"] += 1
continue
sidecar = store.load_sidecar(serial, filename)
if sidecar is None:
counts["skipped_no_sidecar"] += 1
continue
freqs = _freqs_from_sidecar(sidecar)
if freqs is None:
counts["skipped_no_freq"] += 1
continue
if dry_run:
counts["updated"] += 1
continue
with db._connect() as conn:
conn.execute(
"""UPDATE events SET
tran_zc_freq=?, vert_zc_freq=?, long_zc_freq=?, mic_zc_freq=?,
tran_zc_above_range=?, vert_zc_above_range=?,
long_zc_above_range=?, mic_zc_above_range=?
WHERE id=?""",
(freqs["tran_zc_freq"], freqs["vert_zc_freq"], freqs["long_zc_freq"],
freqs["mic_zc_freq"], freqs["tran_zc_above_range"],
freqs["vert_zc_above_range"], freqs["long_zc_above_range"],
freqs["mic_zc_above_range"], row["id"]),
)
counts["updated"] += 1
return counts
def main(argv=None) -> int:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--db-path", default=str(
Path(__file__).resolve().parent.parent / "bridges" / "captures" / "seismo_relay.db"))
p.add_argument("--store-root", default=None)
p.add_argument("--dry-run", action="store_true")
p.add_argument("-v", "--verbose", action="store_true")
args = p.parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
db_path = Path(args.db_path).expanduser().resolve()
store_root = (
Path(args.store_root).expanduser().resolve()
if args.store_root else db_path.parent / "waveforms"
)
db = SeismoDb(db_path)
store = WaveformStore(store_root)
counts = backfill_zc_freq(db, store, dry_run=args.dry_run)
log.info("backfill_zc_freq: %s", counts)
return 0
if __name__ == "__main__":
raise SystemExit(main())
sfm/database.py
+47 -3
View File
@@ -86,6 +86,14 @@ CREATE TABLE IF NOT EXISTS events (
a5_pickle_filename TEXT, -- "<filename>.a5.pkl" sidecar
sidecar_filename TEXT, -- "<filename>.sfm.json" review/metadata sidecar
device_family TEXT, -- "series3" (MiniMate Plus / BW) | "series4" (Micromate / Thor) — drives per-family UI rendering (units, labels)
tran_zc_freq REAL, -- Transverse ZC frequency (Hz)
vert_zc_freq REAL, -- Vertical ZC frequency (Hz)
long_zc_freq REAL, -- Longitudinal ZC frequency (Hz)
mic_zc_freq REAL, -- Microphone ZC frequency (Hz)
tran_zc_above_range INTEGER, -- 1 = value is device ceiling, render ">N Hz"
vert_zc_above_range INTEGER,
long_zc_above_range INTEGER,
mic_zc_above_range INTEGER,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
UNIQUE(serial, timestamp)
);
@@ -200,6 +208,14 @@ class SeismoDb:
("a5_pickle_filename", "TEXT"),
("sidecar_filename", "TEXT"),
("device_family", "TEXT"),
("tran_zc_freq", "REAL"),
("vert_zc_freq", "REAL"),
("long_zc_freq", "REAL"),
("mic_zc_freq", "REAL"),
("tran_zc_above_range", "INTEGER"),
("vert_zc_above_range", "INTEGER"),
("long_zc_above_range", "INTEGER"),
("mic_zc_above_range", "INTEGER"),
):
if col not in existing_cols:
log.info("_migrate: events ADD COLUMN %s %s", col, ddl)
@@ -399,8 +415,12 @@ class SeismoDb:
sample_rate, record_type,
blastware_filename, blastware_filesize,
a5_pickle_filename, sidecar_filename,
device_family)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
device_family,
tran_zc_freq, vert_zc_freq, long_zc_freq, mic_zc_freq,
tran_zc_above_range, vert_zc_above_range,
long_zc_above_range, mic_zc_above_range)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?)
""",
(
self._new_id(), serial, key, session_id, ts,
@@ -420,6 +440,14 @@ class SeismoDb:
rec.get("a5_pickle_filename"),
rec.get("sidecar_filename"),
device_family,
pv.tran_zc_freq if pv else None,
pv.vert_zc_freq if pv else None,
pv.long_zc_freq if pv else None,
pv.mic_zc_freq if pv else None,
(1 if (pv and pv.tran_zc_above_range) else 0),
(1 if (pv and pv.vert_zc_above_range) else 0),
(1 if (pv and pv.long_zc_above_range) else 0),
(1 if (pv and pv.mic_zc_above_range) else 0),
),
)
inserted += 1
@@ -461,7 +489,15 @@ class SeismoDb:
blastware_filesize = ?,
a5_pickle_filename = ?,
sidecar_filename = ?,
device_family = COALESCE(?, device_family)
device_family = COALESCE(?, device_family),
tran_zc_freq = ?,
vert_zc_freq = ?,
long_zc_freq = ?,
mic_zc_freq = ?,
tran_zc_above_range = ?,
vert_zc_above_range = ?,
long_zc_above_range = ?,
mic_zc_above_range = ?
WHERE serial = ? AND timestamp = ?
""",
(
@@ -481,6 +517,14 @@ class SeismoDb:
rec.get("a5_pickle_filename") if rec else None,
rec.get("sidecar_filename") if rec else None,
device_family,
pv.tran_zc_freq if pv else None,
pv.vert_zc_freq if pv else None,
pv.long_zc_freq if pv else None,
pv.mic_zc_freq if pv else None,
(1 if (pv and pv.tran_zc_above_range) else 0),
(1 if (pv and pv.vert_zc_above_range) else 0),
(1 if (pv and pv.long_zc_above_range) else 0),
(1 if (pv and pv.mic_zc_above_range) else 0),
serial,
ts,
),
sfm/server.py
+1 -1
View File
@@ -90,7 +90,7 @@ app = FastAPI(
"Implements the minimateplus RS-232 protocol library.\n"
"Proxied by terra-view at /api/sfm/*."
),
version="0.17.0",
version="0.23.0",
)
# Allow requests from the waveform viewer opened as a local file (file://)
tests/test_zc_freq_columns.py
+174
View File
@@ -0,0 +1,174 @@
from __future__ import annotations
import os, sys, sqlite3
from pathlib import Path
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sfm.database import SeismoDb
ZC_COLS = ["tran_zc_freq", "vert_zc_freq", "long_zc_freq", "mic_zc_freq",
"tran_zc_above_range", "vert_zc_above_range",
"long_zc_above_range", "mic_zc_above_range"]
def test_fresh_db_has_zc_freq_columns(tmp_path: Path):
db = SeismoDb(tmp_path / "seismo_relay.db")
with sqlite3.connect(str(tmp_path / "seismo_relay.db")) as conn:
cols = {r[1] for r in conn.execute("PRAGMA table_info(events)").fetchall()}
for c in ZC_COLS:
assert c in cols, f"missing column {c}"
def test_migration_adds_zc_freq_columns_to_legacy_db(tmp_path: Path):
db_path = tmp_path / "old.db"
# A "modern-minus-zc" events table (has blastware_* but not the zc columns).
with sqlite3.connect(str(db_path)) as conn:
conn.executescript("""
CREATE TABLE events (
id TEXT PRIMARY KEY, serial TEXT NOT NULL, waveform_key TEXT NOT NULL,
session_id TEXT, timestamp TEXT,
tran_ppv REAL, vert_ppv REAL, long_ppv REAL, peak_vector_sum REAL, mic_ppv REAL,
project TEXT, client TEXT, operator TEXT, sensor_location TEXT,
sample_rate INTEGER, record_type TEXT,
false_trigger INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
UNIQUE(serial, timestamp)
);
INSERT INTO events (id, serial, waveform_key, timestamp)
VALUES ('legacy-id', 'BE11529', '01110000', '2026-04-01T12:00:00');
""")
db = SeismoDb(db_path) # triggers _migrate
rows = db.query_events(serial="BE11529")
assert len(rows) == 1
for c in ZC_COLS:
assert c in rows[0] # present
assert rows[0][c] is None # NULL for the legacy row
from minimateplus.models import PeakValues, Event
from minimateplus import event_file_io
from minimateplus.bw_ascii_report import BwAsciiReport, ChannelStats, MicStats
def _report_with_freqs():
r = BwAsciiReport()
r.channels["Tran"] = ChannelStats(ppv_ips=0.075, zc_freq_hz=100.0, zc_freq_above_range=True)
r.channels["Vert"] = ChannelStats(ppv_ips=0.220, zc_freq_hz=85.0)
r.channels["Long"] = ChannelStats(ppv_ips=0.045, zc_freq_hz=73.0)
r.mic = MicStats(zc_freq_hz=51.0)
return r
def test_apply_report_copies_zc_freq_onto_peakvalues():
ev = Event(index=0)
event_file_io.apply_report_to_event(ev, _report_with_freqs())
pv = ev.peak_values
assert pv.vert_zc_freq == 85.0
assert pv.tran_zc_freq == 100.0
assert pv.tran_zc_above_range is True
assert pv.vert_zc_above_range is False
assert pv.mic_zc_freq == 51.0
def test_apply_bw_report_dict_copies_zc_freq():
ev = Event(index=0)
bw = {
"peaks": {
"tran": {"ppv_ips": 0.075, "zc_freq_hz": 100.0, "zc_freq_above_range": True},
"vert": {"ppv_ips": 0.220, "zc_freq_hz": 85.0},
"long": {"ppv_ips": 0.045, "zc_freq_hz": 73.0},
},
"mic": {"zc_freq_hz": 51.0, "zc_freq_above_range": False},
}
event_file_io.apply_bw_report_dict_to_event(ev, bw)
pv = ev.peak_values
assert pv.vert_zc_freq == 85.0
assert pv.tran_zc_above_range is True
assert pv.mic_zc_freq == 51.0
def _event_with_zc(waveform_key="01110000"):
from minimateplus.models import Event, Timestamp
ev = Event(index=0)
ev._waveform_key = bytes.fromhex(waveform_key)
ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0,
month=6, day=25, hour=8, minute=50, second=0)
ev.record_type = "Waveform"
ev.peak_values = PeakValues(
tran=0.075, vert=0.220, long=0.045, peak_vector_sum=0.231, micl=0.01,
tran_zc_freq=100.0, vert_zc_freq=85.0, long_zc_freq=73.0, mic_zc_freq=51.0,
tran_zc_above_range=True,
)
return ev
def test_insert_events_persists_zc_freq(tmp_path: Path):
db = SeismoDb(tmp_path / "seismo_relay.db")
ins, _ = db.insert_events([_event_with_zc()], serial="BE11529")
assert ins == 1
row = db.query_events(serial="BE11529")[0]
assert row["vert_zc_freq"] == 85.0
assert row["tran_zc_freq"] == 100.0
assert row["tran_zc_above_range"] == 1
assert row["vert_zc_above_range"] == 0
assert row["mic_zc_freq"] == 51.0
def test_upsert_updates_zc_freq(tmp_path: Path):
db = SeismoDb(tmp_path / "seismo_relay.db")
db.insert_events([_event_with_zc()], serial="BE11529")
ev2 = _event_with_zc()
ev2.peak_values.vert_zc_freq = 91.0 # same (serial, timestamp) → UPSERT
db.insert_events([ev2], serial="BE11529")
row = db.query_events(serial="BE11529")[0]
assert row["vert_zc_freq"] == 91.0
def test_backfill_zc_freq_from_sidecar(tmp_path: Path):
import importlib
mod = importlib.import_module("scripts.backfill_event_zc_freq")
db = SeismoDb(tmp_path / "seismo_relay.db")
# Insert an event WITHOUT freq (simulates a pre-feature row), with a filename.
from minimateplus.models import Event, Timestamp
ev = Event(index=0)
ev._waveform_key = bytes.fromhex("01110000")
ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0,
month=6, day=25, hour=8, minute=50, second=0)
ev.record_type = "Waveform"
db.insert_events([ev], serial="BE11529",
waveform_records={"01110000": {"filename": "M529AAAA.AB0T"}})
class FakeStore:
def load_sidecar(self, serial, filename):
return {"bw_report": {
"peaks": {"vert": {"zc_freq_hz": 85.0},
"tran": {"zc_freq_hz": 100.0, "zc_freq_above_range": True}},
"mic": {"zc_freq_hz": 51.0}}}
counts = mod.backfill_zc_freq(db, FakeStore())
assert counts["updated"] == 1
row = db.query_events(serial="BE11529")[0]
assert row["vert_zc_freq"] == 85.0
assert row["tran_zc_above_range"] == 1
assert row["mic_zc_freq"] == 51.0
def test_backfill_skips_event_without_sidecar(tmp_path: Path):
import importlib
mod = importlib.import_module("scripts.backfill_event_zc_freq")
db = SeismoDb(tmp_path / "seismo_relay.db")
from minimateplus.models import Event, Timestamp
ev = Event(index=0)
ev._waveform_key = bytes.fromhex("01110000")
ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0,
month=6, day=25, hour=8, minute=50, second=0)
db.insert_events([ev], serial="BE11529",
waveform_records={"01110000": {"filename": "M529AAAA.AB0T"}})
class NoneStore:
def load_sidecar(self, serial, filename): return None
counts = mod.backfill_zc_freq(db, NoneStore())
assert counts["updated"] == 0
assert counts["skipped_no_sidecar"] == 1