feat(offset): DC-offset detector productionized into the shape pipeline
Productionizes the validated scratch/offset_scan3.py: a DC offset (baseline shifted off zero — sensor bumped/settled/drifted) is |median(pre-trigger)| >= 5 counts (0.025 in/s) AND flat across pre/mid/end thirds (spread <= 0.02); a transient moves one third and is rejected by the spread test. - shape_metrics: offset_from_samples / offset_from_h5 (reads .h5 samples + pretrig_samples attr; range-aware via the .h5's in/s float samples) - events schema: shape_offset / _axis / _pre / _spread (via _SCHEMA + the _migrate ADD COLUMN loop only; NOT the Migration-1 rebuild), threaded through insert + upsert mirroring shape_* - ingest: computed at all three waveform_store save paths alongside shape - backfill_event_shape: also computes + stores (and stale-clears) offset - exposed via /db/events automatically (SELECT *) Gating to waveforms is done downstream in terra-view ft_suspicion (mirrors how shape is ignored for histograms), not at the SFM call sites. 13 new tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill events.shape_* from each event's .h5 waveform samples. Idempotent."""
|
||||
"""Backfill events.shape_* and shape_offset_* from each event's .h5 samples. Idempotent."""
|
||||
from __future__ import annotations
|
||||
import argparse, logging, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from sfm.database import SeismoDb
|
||||
from sfm.waveform_store import WaveformStore
|
||||
from sfm.shape_metrics import shape_from_h5
|
||||
from sfm.shape_metrics import shape_from_h5, offset_from_h5
|
||||
|
||||
log = logging.getLogger("backfill_event_shape")
|
||||
|
||||
@@ -21,6 +21,7 @@ def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False)
|
||||
if not h5_path.exists():
|
||||
counts["skipped_no_h5"] += 1; continue
|
||||
shape = shape_from_h5(h5_path)
|
||||
offset = offset_from_h5(h5_path)
|
||||
if shape is None:
|
||||
# The .h5 can no longer yield a shape (fewer than 2 samples, or a
|
||||
# flat trace). Clear any previously stored value rather than
|
||||
@@ -28,22 +29,32 @@ def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False)
|
||||
# from and silently feeds the false-trigger detector. Seen after
|
||||
# a decoder fix shrinks an event: 493 rows in the prod snapshot
|
||||
# were carrying metrics from a superseded decode (2026-08-25).
|
||||
if row.get("shape_crest_factor") is not None:
|
||||
if (row.get("shape_crest_factor") is not None
|
||||
or row.get("shape_offset") is not None):
|
||||
if not dry_run:
|
||||
with db._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE events SET shape_crest_factor=NULL, "
|
||||
"shape_near_peak_count=NULL, shape_sample_count=NULL, "
|
||||
"shape_axis=NULL WHERE id=?", (row["id"],))
|
||||
"shape_axis=NULL, shape_offset=NULL, shape_offset_axis=NULL, "
|
||||
"shape_offset_pre=NULL, shape_offset_spread=NULL WHERE id=?",
|
||||
(row["id"],))
|
||||
counts["cleared_stale"] += 1
|
||||
counts["skipped_no_samples"] += 1; continue
|
||||
if not dry_run:
|
||||
with db._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, "
|
||||
"shape_sample_count=?, shape_axis=? WHERE id=?",
|
||||
"shape_sample_count=?, shape_axis=?, shape_offset=?, "
|
||||
"shape_offset_axis=?, shape_offset_pre=?, shape_offset_spread=? "
|
||||
"WHERE id=?",
|
||||
(shape["crest_factor"], shape["near_peak_count"],
|
||||
shape["sample_count"], shape["axis"], row["id"]))
|
||||
shape["sample_count"], shape["axis"],
|
||||
(1 if offset["offset"] else 0) if offset else None,
|
||||
offset["axis"] if offset else None,
|
||||
offset["pre"] if offset else None,
|
||||
offset["spread"] if offset else None,
|
||||
row["id"]))
|
||||
counts["updated"] += 1
|
||||
log.info("backfill_shape: %s", counts)
|
||||
return counts
|
||||
|
||||
Reference in New Issue
Block a user