From ddf6a73292a17f97f8a5e8c534c3e11fba8614aa Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 29 Jul 2026 17:32:54 +0000 Subject: [PATCH] feat(scripts): backfill event ZC freq columns from sidecars --- scripts/backfill_event_zc_freq.py | 98 +++++++++++++++++++++++++++++++ tests/test_zc_freq_columns.py | 50 ++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 scripts/backfill_event_zc_freq.py diff --git a/scripts/backfill_event_zc_freq.py b/scripts/backfill_event_zc_freq.py new file mode 100644 index 0000000..15526fd --- /dev/null +++ b/scripts/backfill_event_zc_freq.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Backfill events.*_zc_freq / *_zc_above_range from .sfm.json sidecars. + +Idempotent. Reads each event's sidecar bw_report and writes the per-channel +ZC frequency into the DB columns via a direct UPDATE. Events with no sidecar +(or no bw_report freq) are left untouched. Mirrors scripts/backfill_record_type.py. +""" +from __future__ import annotations +import argparse +import logging +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from sfm.database import SeismoDb +from sfm.waveform_store import WaveformStore + +log = logging.getLogger("backfill_event_zc_freq") + + +def _freqs_from_sidecar(sidecar: dict) -> dict | None: + bw = (sidecar or {}).get("bw_report") or {} + peaks = bw.get("peaks") or {} + mic = bw.get("mic") or {} + out = {} + for axis in ("tran", "vert", "long"): + chd = peaks.get(axis) or {} + out[f"{axis}_zc_freq"] = chd.get("zc_freq_hz") + out[f"{axis}_zc_above_range"] = 1 if chd.get("zc_freq_above_range") else 0 + out["mic_zc_freq"] = mic.get("zc_freq_hz") + out["mic_zc_above_range"] = 1 if mic.get("zc_freq_above_range") else 0 + # None if there's literally no frequency anywhere. + if all(out[f"{a}_zc_freq"] is None for a in ("tran", "vert", "long", "mic")): + return None + return out + + +def backfill_zc_freq(db: SeismoDb, store, *, dry_run: bool = False) -> dict: + counts = {"updated": 0, "skipped_no_sidecar": 0, "skipped_no_freq": 0} + # Pull every event (paginate generously; adjust if a store has >100k events). + rows = db.query_events(limit=1_000_000) + for row in rows: + serial = row.get("serial") + filename = row.get("blastware_filename") + if not serial or not filename: + counts["skipped_no_sidecar"] += 1 + continue + sidecar = store.load_sidecar(serial, filename) + if sidecar is None: + counts["skipped_no_sidecar"] += 1 + continue + freqs = _freqs_from_sidecar(sidecar) + if freqs is None: + counts["skipped_no_freq"] += 1 + continue + if dry_run: + counts["updated"] += 1 + continue + with db._connect() as conn: + conn.execute( + """UPDATE events SET + tran_zc_freq=?, vert_zc_freq=?, long_zc_freq=?, mic_zc_freq=?, + tran_zc_above_range=?, vert_zc_above_range=?, + long_zc_above_range=?, mic_zc_above_range=? + WHERE id=?""", + (freqs["tran_zc_freq"], freqs["vert_zc_freq"], freqs["long_zc_freq"], + freqs["mic_zc_freq"], freqs["tran_zc_above_range"], + freqs["vert_zc_above_range"], freqs["long_zc_above_range"], + freqs["mic_zc_above_range"], row["id"]), + ) + counts["updated"] += 1 + return counts + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--db-path", default=str( + Path(__file__).resolve().parent.parent / "bridges" / "captures" / "seismo_relay.db")) + p.add_argument("--store-root", default=None) + p.add_argument("--dry-run", action="store_true") + p.add_argument("-v", "--verbose", action="store_true") + args = p.parse_args(argv) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + db_path = Path(args.db_path).expanduser().resolve() + store_root = ( + Path(args.store_root).expanduser().resolve() + if args.store_root else db_path.parent / "waveforms" + ) + db = SeismoDb(db_path) + store = WaveformStore(store_root) + counts = backfill_zc_freq(db, store, dry_run=args.dry_run) + log.info("backfill_zc_freq: %s", counts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_zc_freq_columns.py b/tests/test_zc_freq_columns.py index 1b9a456..8142a02 100644 --- a/tests/test_zc_freq_columns.py +++ b/tests/test_zc_freq_columns.py @@ -122,3 +122,53 @@ def test_upsert_updates_zc_freq(tmp_path: Path): db.insert_events([ev2], serial="BE11529") row = db.query_events(serial="BE11529")[0] assert row["vert_zc_freq"] == 91.0 + + +def test_backfill_zc_freq_from_sidecar(tmp_path: Path): + import importlib + mod = importlib.import_module("scripts.backfill_event_zc_freq") + + db = SeismoDb(tmp_path / "seismo_relay.db") + # Insert an event WITHOUT freq (simulates a pre-feature row), with a filename. + from minimateplus.models import Event, Timestamp + ev = Event(index=0) + ev._waveform_key = bytes.fromhex("01110000") + ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, + month=6, day=25, hour=8, minute=50, second=0) + ev.record_type = "Waveform" + db.insert_events([ev], serial="BE11529", + waveform_records={"01110000": {"filename": "M529AAAA.AB0T"}}) + + class FakeStore: + def load_sidecar(self, serial, filename): + return {"bw_report": { + "peaks": {"vert": {"zc_freq_hz": 85.0}, + "tran": {"zc_freq_hz": 100.0, "zc_freq_above_range": True}}, + "mic": {"zc_freq_hz": 51.0}}} + + counts = mod.backfill_zc_freq(db, FakeStore()) + assert counts["updated"] == 1 + row = db.query_events(serial="BE11529")[0] + assert row["vert_zc_freq"] == 85.0 + assert row["tran_zc_above_range"] == 1 + assert row["mic_zc_freq"] == 51.0 + + +def test_backfill_skips_event_without_sidecar(tmp_path: Path): + import importlib + mod = importlib.import_module("scripts.backfill_event_zc_freq") + db = SeismoDb(tmp_path / "seismo_relay.db") + from minimateplus.models import Event, Timestamp + ev = Event(index=0) + ev._waveform_key = bytes.fromhex("01110000") + ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, + month=6, day=25, hour=8, minute=50, second=0) + db.insert_events([ev], serial="BE11529", + waveform_records={"01110000": {"filename": "M529AAAA.AB0T"}}) + + class NoneStore: + def load_sidecar(self, serial, filename): return None + + counts = mod.backfill_zc_freq(db, NoneStore()) + assert counts["updated"] == 0 + assert counts["skipped_no_sidecar"] == 1