serversdownandClaude Opus 5
260bf0bc67
backfill_event_shape.py skipped rows whose .h5 produced no shape and left the previously stored value in place. A stale shape outlives the decode it came from and silently feeds the false-trigger detector. Found while re-running the backfill after the histogram codec fix: 493 rows in the prod snapshot were carrying shape metrics that no longer matched their .h5 — e.g. BE17353/S353LDOK.XZ0H held crest_factor from a 223-sample decode while its .h5 holds a single interval. These predate today's work (present in the pre-32000 snapshot), so this is pre-existing behaviour rather than fallout from the codec fixes. Now NULLs shape_crest_factor / shape_near_peak_count / shape_sample_count / shape_axis in that case and reports a `cleared_stale` count. Verified on the snapshot: 493 cleared, 0 stale rows remaining, 11570 rows matching their .h5 exactly. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog
64 lines
2.9 KiB
Python
64 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Backfill events.shape_* from each event's .h5 waveform samples. Idempotent."""
|
|
from __future__ import annotations
|
|
import argparse, logging, sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from sfm.database import SeismoDb
|
|
from sfm.waveform_store import WaveformStore
|
|
from sfm.shape_metrics import shape_from_h5
|
|
|
|
log = logging.getLogger("backfill_event_shape")
|
|
|
|
def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict:
|
|
counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0,
|
|
"cleared_stale": 0}
|
|
for row in db.query_events(limit=1_000_000):
|
|
serial, filename = row.get("serial"), row.get("blastware_filename")
|
|
if not serial or not filename:
|
|
counts["skipped_no_h5"] += 1; continue
|
|
h5_path = store.hdf5_path_for(serial, filename)
|
|
if not h5_path.exists():
|
|
counts["skipped_no_h5"] += 1; continue
|
|
shape = shape_from_h5(h5_path)
|
|
if shape is None:
|
|
# The .h5 can no longer yield a shape (fewer than 2 samples, or a
|
|
# flat trace). Clear any previously stored value rather than
|
|
# leaving it behind — a stale shape outlives the decode it came
|
|
# from and silently feeds the false-trigger detector. Seen after
|
|
# a decoder fix shrinks an event: 493 rows in the prod snapshot
|
|
# were carrying metrics from a superseded decode (2026-08-25).
|
|
if row.get("shape_crest_factor") is not None:
|
|
if not dry_run:
|
|
with db._connect() as conn:
|
|
conn.execute(
|
|
"UPDATE events SET shape_crest_factor=NULL, "
|
|
"shape_near_peak_count=NULL, shape_sample_count=NULL, "
|
|
"shape_axis=NULL WHERE id=?", (row["id"],))
|
|
counts["cleared_stale"] += 1
|
|
counts["skipped_no_samples"] += 1; continue
|
|
if not dry_run:
|
|
with db._connect() as conn:
|
|
conn.execute(
|
|
"UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, "
|
|
"shape_sample_count=?, shape_axis=? WHERE id=?",
|
|
(shape["crest_factor"], shape["near_peak_count"],
|
|
shape["sample_count"], shape["axis"], row["id"]))
|
|
counts["updated"] += 1
|
|
log.info("backfill_shape: %s", counts)
|
|
return counts
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--db-path", required=True)
|
|
ap.add_argument("--store-root", required=True)
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
logging.basicConfig(level=logging.INFO)
|
|
counts = backfill_shape(SeismoDb(a.db_path), WaveformStore(a.store_root), dry_run=a.dry_run)
|
|
print(counts)
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|