serversdownandClaude Opus 4.8
3554d00583
Productionizes the validated scratch/offset_scan3.py: a DC offset (baseline shifted off zero — sensor bumped/settled/drifted) is |median(pre-trigger)| >= 5 counts (0.025 in/s) AND flat across pre/mid/end thirds (spread <= 0.02); a transient moves one third and is rejected by the spread test. - shape_metrics: offset_from_samples / offset_from_h5 (reads .h5 samples + pretrig_samples attr; range-aware via the .h5's in/s float samples) - events schema: shape_offset / _axis / _pre / _spread (via _SCHEMA + the _migrate ADD COLUMN loop only; NOT the Migration-1 rebuild), threaded through insert + upsert mirroring shape_* - ingest: computed at all three waveform_store save paths alongside shape - backfill_event_shape: also computes + stores (and stale-clears) offset - exposed via /db/events automatically (SELECT *) Gating to waveforms is done downstream in terra-view ft_suspicion (mirrors how shape is ignored for histograms), not at the SFM call sites. 13 new tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf
75 lines
3.6 KiB
Python
75 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Backfill events.shape_* and shape_offset_* from each event's .h5 samples. Idempotent."""
|
|
from __future__ import annotations
|
|
import argparse, logging, sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from sfm.database import SeismoDb
|
|
from sfm.waveform_store import WaveformStore
|
|
from sfm.shape_metrics import shape_from_h5, offset_from_h5
|
|
|
|
log = logging.getLogger("backfill_event_shape")
|
|
|
|
def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict:
|
|
counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0,
|
|
"cleared_stale": 0}
|
|
for row in db.query_events(limit=1_000_000):
|
|
serial, filename = row.get("serial"), row.get("blastware_filename")
|
|
if not serial or not filename:
|
|
counts["skipped_no_h5"] += 1; continue
|
|
h5_path = store.hdf5_path_for(serial, filename)
|
|
if not h5_path.exists():
|
|
counts["skipped_no_h5"] += 1; continue
|
|
shape = shape_from_h5(h5_path)
|
|
offset = offset_from_h5(h5_path)
|
|
if shape is None:
|
|
# The .h5 can no longer yield a shape (fewer than 2 samples, or a
|
|
# flat trace). Clear any previously stored value rather than
|
|
# leaving it behind — a stale shape outlives the decode it came
|
|
# from and silently feeds the false-trigger detector. Seen after
|
|
# a decoder fix shrinks an event: 493 rows in the prod snapshot
|
|
# were carrying metrics from a superseded decode (2026-08-25).
|
|
if (row.get("shape_crest_factor") is not None
|
|
or row.get("shape_offset") is not None):
|
|
if not dry_run:
|
|
with db._connect() as conn:
|
|
conn.execute(
|
|
"UPDATE events SET shape_crest_factor=NULL, "
|
|
"shape_near_peak_count=NULL, shape_sample_count=NULL, "
|
|
"shape_axis=NULL, shape_offset=NULL, shape_offset_axis=NULL, "
|
|
"shape_offset_pre=NULL, shape_offset_spread=NULL WHERE id=?",
|
|
(row["id"],))
|
|
counts["cleared_stale"] += 1
|
|
counts["skipped_no_samples"] += 1; continue
|
|
if not dry_run:
|
|
with db._connect() as conn:
|
|
conn.execute(
|
|
"UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, "
|
|
"shape_sample_count=?, shape_axis=?, shape_offset=?, "
|
|
"shape_offset_axis=?, shape_offset_pre=?, shape_offset_spread=? "
|
|
"WHERE id=?",
|
|
(shape["crest_factor"], shape["near_peak_count"],
|
|
shape["sample_count"], shape["axis"],
|
|
(1 if offset["offset"] else 0) if offset else None,
|
|
offset["axis"] if offset else None,
|
|
offset["pre"] if offset else None,
|
|
offset["spread"] if offset else None,
|
|
row["id"]))
|
|
counts["updated"] += 1
|
|
log.info("backfill_shape: %s", counts)
|
|
return counts
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--db-path", required=True)
|
|
ap.add_argument("--store-root", required=True)
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
logging.basicConfig(level=logging.INFO)
|
|
counts = backfill_shape(SeismoDb(a.db_path), WaveformStore(a.store_root), dry_run=a.dry_run)
|
|
print(counts)
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|