serversdown/seismo-relay
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(offset): DC-offset detector productionized into the shape pipeline

Browse Source 
Productionizes the validated scratch/offset_scan3.py: a DC offset (baseline
shifted off zero — sensor bumped/settled/drifted) is |median(pre-trigger)| >= 5
counts (0.025 in/s) AND flat across pre/mid/end thirds (spread <= 0.02); a
transient moves one third and is rejected by the spread test.

- shape_metrics: offset_from_samples / offset_from_h5 (reads .h5 samples +
  pretrig_samples attr; range-aware via the .h5's in/s float samples)
- events schema: shape_offset / _axis / _pre / _spread (via _SCHEMA + the
  _migrate ADD COLUMN loop only; NOT the Migration-1 rebuild), threaded through
  insert + upsert mirroring shape_*
- ingest: computed at all three waveform_store save paths alongside shape
- backfill_event_shape: also computes + stores (and stale-clears) offset
- exposed via /db/events automatically (SELECT *)

Gating to waveforms is done downstream in terra-view ft_suspicion (mirrors how
shape is ignored for histograms), not at the SFM call sites. 13 new tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf
This commit is contained in:
serversdown
2026-09-02 04:47:01 +00:00
co-authored by Claude Opus 4.8
parent b29ca50b35
commit 3554d00583
6 changed files with 294 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sfm/database.py
+25 -3
View File
@@ -99,6 +99,10 @@ CREATE TABLE IF NOT EXISTS events (
shape_near_peak_count INTEGER, -- samples >= 0.5 * peak (FT: few; real: many)
shape_sample_count INTEGER, -- total samples (to normalize near_peak_count)
shape_axis TEXT, -- geophone channel measured ("Tran"/"Vert"/"Long")
shape_offset INTEGER, -- 1 = DC-offset false trigger (pre-trigger baseline off zero + flat). Meaningful for waveforms only.
shape_offset_axis TEXT, -- geo channel the offset was measured on
shape_offset_pre REAL, -- pre-trigger baseline median (in/s)
shape_offset_spread REAL, -- max(pre,mid,end) - min(...) in in/s; small = constant/DC
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
UNIQUE(serial, timestamp)
);
@@ -225,6 +229,10 @@ class SeismoDb:
("shape_near_peak_count", "INTEGER"),
("shape_sample_count", "INTEGER"),
("shape_axis", "TEXT"),
("shape_offset", "INTEGER"),
("shape_offset_axis", "TEXT"),
("shape_offset_pre", "REAL"),
("shape_offset_spread", "REAL"),
("reviewed_real", "INTEGER NOT NULL DEFAULT 0"),
):
if col not in existing_cols:
@@ -430,9 +438,11 @@ class SeismoDb:
tran_zc_above_range, vert_zc_above_range,
long_zc_above_range, mic_zc_above_range,
shape_crest_factor, shape_near_peak_count,
shape_sample_count, shape_axis)
shape_sample_count, shape_axis,
shape_offset, shape_offset_axis,
shape_offset_pre, shape_offset_spread)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
self._new_id(), serial, key, session_id, ts,
@@ -464,6 +474,10 @@ class SeismoDb:
rec.get("shape_near_peak_count"),
rec.get("shape_sample_count"),
rec.get("shape_axis"),
rec.get("shape_offset"),
rec.get("shape_offset_axis"),
rec.get("shape_offset_pre"),
rec.get("shape_offset_spread"),
),
)
inserted += 1
@@ -517,7 +531,11 @@ class SeismoDb:
shape_crest_factor = COALESCE(?, shape_crest_factor),
shape_near_peak_count = COALESCE(?, shape_near_peak_count),
shape_sample_count = COALESCE(?, shape_sample_count),
shape_axis = COALESCE(?, shape_axis)
shape_axis = COALESCE(?, shape_axis),
shape_offset = COALESCE(?, shape_offset),
shape_offset_axis = COALESCE(?, shape_offset_axis),
shape_offset_pre = COALESCE(?, shape_offset_pre),
shape_offset_spread = COALESCE(?, shape_offset_spread)
WHERE serial = ? AND timestamp = ?
""",
(
@@ -549,6 +567,10 @@ class SeismoDb:
rec.get("shape_near_peak_count") if rec else None,
rec.get("shape_sample_count") if rec else None,
rec.get("shape_axis") if rec else None,
rec.get("shape_offset") if rec else None,
rec.get("shape_offset_axis") if rec else None,
rec.get("shape_offset_pre") if rec else None,
rec.get("shape_offset_spread") if rec else None,
serial,
ts,
),
sfm/shape_metrics.py
+69
View File
@@ -47,6 +47,60 @@ def shape_from_samples(chans: dict) -> dict | None:
return s
# ── Offset (DC-baseline) detection ────────────────────────────────────────────
# A DC offset is a false trigger where the geophone baseline sits at a constant
# non-zero floor (sensor bumped / settled / drifted) instead of oscillating
# around zero. Brian's method (validated in scratch/offset_scan3.py): the
# pre-trigger window is definitionally quiet, so a true offset shows |pre| off
# zero AND stays flat across the record (pre ≈ mid ≈ end). A transient moves one
# third relative to the others and is rejected by the spread test.
# Thresholds are in in/s (the .h5 samples are already range-scaled); validated at
# Normal range (10 in/s) — the only range in the fleet.
OFFSET_FLOOR = 0.025 # |pre| at/above this reads as an off-zero baseline (5 A/D counts)
OFFSET_MAX_SPREAD = 0.02 # max(pre,mid,end) - min(...) at/below this reads as flat/constant
def _channel_offset(x, pretrig_n):
"""Return (pre, spread, is_offset) for one channel, or None if unusable."""
x = np.asarray(x, dtype=float)
n = x.size
if n < 3:
return None
t = n // 3
pre = x[:pretrig_n] if (pretrig_n and 0 < pretrig_n < n) else x[:t]
mid, end = x[t:2 * t], x[2 * t:]
if pre.size == 0 or mid.size == 0 or end.size == 0:
return None
vals = [float(np.median(seg)) for seg in (pre, mid, end)]
spread = max(vals) - min(vals)
is_offset = abs(vals[0]) >= OFFSET_FLOOR and spread <= OFFSET_MAX_SPREAD
return vals[0], spread, is_offset
def offset_from_samples(chans: dict, pretrig_n) -> dict | None:
"""Detect a DC-offset false trigger across the geophone channels.
An event is offset if ANY geo channel's pre-trigger baseline is off zero and
flat across the record. Reports the tripping axis (or, if none trips, the
most-offset-like axis) with its ``pre``/``spread`` for transparency + tuning.
Returns None when no geo channel is usable.
"""
results = []
for ax in _GEO_CHANNELS:
x = chans.get(ax)
if x is None:
continue
r = _channel_offset(x, pretrig_n)
if r is not None:
results.append((ax, r[0], r[1], r[2]))
if not results:
return None
offenders = [r for r in results if r[3]]
ax, pre, spread, _ = max(offenders or results, key=lambda r: abs(r[1]))
return {"offset": bool(offenders), "axis": ax,
"pre": round(pre, 6), "spread": round(spread, 6)}
def shape_from_h5(path) -> dict | None:
import h5py
try:
@@ -56,3 +110,18 @@ def shape_from_h5(path) -> dict | None:
except Exception:
return None
return shape_from_samples(chans)
def offset_from_h5(path) -> dict | None:
"""offset_from_samples fed from an event's .h5 (float32 in/s geo samples +
the pretrig_samples attribute)."""
import h5py
try:
with h5py.File(path, "r") as f:
chans = {ax: f[f"samples/{ax}"][:] for ax in _GEO_CHANNELS
if f"samples/{ax}" in f}
pretrig_n = f.attrs.get("pretrig_samples")
except Exception:
return None
pretrig_n = int(pretrig_n) if pretrig_n is not None else 0
return offset_from_samples(chans, pretrig_n)
sfm/waveform_store.py
+25 -1
View File
@@ -41,7 +41,7 @@ from minimateplus.blastware_file import blastware_filename, write_blastware_file
from minimateplus.framing import S3Frame
from minimateplus.models import Event
from sfm import event_hdf5
from sfm.shape_metrics import shape_from_h5
from sfm.shape_metrics import shape_from_h5, offset_from_h5
log = logging.getLogger("sfm.waveform_store")
@@ -270,6 +270,13 @@ class WaveformStore:
"shape_sample_count": _shape["sample_count"],
"shape_axis": _shape["axis"],
} if _shape else {}
_offset = offset_from_h5(hdf5_path) if hdf5_filename else None
_offset_rec = {
"shape_offset": 1 if _offset["offset"] else 0,
"shape_offset_axis": _offset["axis"],
"shape_offset_pre": _offset["pre"],
"shape_offset_spread": _offset["spread"],
} if _offset else {}
return {
"filename": filename,
"filesize": filesize,
@@ -278,6 +285,7 @@ class WaveformStore:
"hdf5_filename": hdf5_filename,
"sidecar_filename": sidecar_path.name,
**_shape_rec,
**_offset_rec,
}
def save_imported_bw(
@@ -461,6 +469,13 @@ class WaveformStore:
"shape_sample_count": _shape["sample_count"],
"shape_axis": _shape["axis"],
} if _shape else {}
_offset = offset_from_h5(hdf5_path) if hdf5_filename else None
_offset_rec = {
"shape_offset": 1 if _offset["offset"] else 0,
"shape_offset_axis": _offset["axis"],
"shape_offset_pre": _offset["pre"],
"shape_offset_spread": _offset["spread"],
} if _offset else {}
return ev, {
"filename": filename,
"filesize": filesize,
@@ -470,6 +485,7 @@ class WaveformStore:
"sidecar_filename": sidecar_path.name,
"serial": serial,
**_shape_rec,
**_offset_rec,
}
def save_imported_idf(
@@ -751,6 +767,13 @@ class WaveformStore:
"shape_sample_count": _shape["sample_count"],
"shape_axis": _shape["axis"],
} if _shape else {}
_offset = offset_from_h5(hdf5_path) if hdf5_filename else None
_offset_rec = {
"shape_offset": 1 if _offset["offset"] else 0,
"shape_offset_axis": _offset["axis"],
"shape_offset_pre": _offset["pre"],
"shape_offset_spread": _offset["spread"],
} if _offset else {}
return ev, {
"filename": filename,
"filesize": filesize,
@@ -760,6 +783,7 @@ class WaveformStore:
"sidecar_filename": sidecar_path.name,
"serial": serial,
**_shape_rec,
**_offset_rec,
}
def load_a5(self, serial: str, filename: str) -> Optional[list[S3Frame]]: