From 260bf0bc67f1089373e31bfa5f869c4bc1490f99 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 19:11:39 +0000 Subject: [PATCH] fix(backfill): clear stale shape_* when the .h5 can no longer yield a shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backfill_event_shape.py skipped rows whose .h5 produced no shape and left the previously stored value in place. A stale shape outlives the decode it came from and silently feeds the false-trigger detector. Found while re-running the backfill after the histogram codec fix: 493 rows in the prod snapshot were carrying shape metrics that no longer matched their .h5 — e.g. BE17353/S353LDOK.XZ0H held crest_factor from a 223-sample decode while its .h5 holds a single interval. These predate today's work (present in the pre-32000 snapshot), so this is pre-existing behaviour rather than fallout from the codec fixes. Now NULLs shape_crest_factor / shape_near_peak_count / shape_sample_count / shape_axis in that case and reports a `cleared_stale` count. Verified on the snapshot: 493 cleared, 0 stale rows remaining, 11570 rows matching their .h5 exactly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- scripts/backfill_event_shape.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/backfill_event_shape.py b/scripts/backfill_event_shape.py index 9de2f85..975c716 100644 --- a/scripts/backfill_event_shape.py +++ b/scripts/backfill_event_shape.py @@ -11,7 +11,8 @@ from sfm.shape_metrics import shape_from_h5 log = logging.getLogger("backfill_event_shape") def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict: - counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0} + counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0, + "cleared_stale": 0} for row in db.query_events(limit=1_000_000): serial, filename = row.get("serial"), row.get("blastware_filename") if not serial or not filename: @@ -21,6 +22,20 @@ def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) counts["skipped_no_h5"] += 1; continue shape = shape_from_h5(h5_path) if shape is None: + # The .h5 can no longer yield a shape (fewer than 2 samples, or a + # flat trace). Clear any previously stored value rather than + # leaving it behind — a stale shape outlives the decode it came + # from and silently feeds the false-trigger detector. Seen after + # a decoder fix shrinks an event: 493 rows in the prod snapshot + # were carrying metrics from a superseded decode (2026-08-25). + if row.get("shape_crest_factor") is not None: + if not dry_run: + with db._connect() as conn: + conn.execute( + "UPDATE events SET shape_crest_factor=NULL, " + "shape_near_peak_count=NULL, shape_sample_count=NULL, " + "shape_axis=NULL WHERE id=?", (row["id"],)) + counts["cleared_stale"] += 1 counts["skipped_no_samples"] += 1; continue if not dry_run: with db._connect() as conn: