fix(twins): interval-based histogram/waveform matching in find_twins (#102 sub-task 2)
A real trigger is recorded twice — as a triggered waveform (stamped at the trigger instant) and inside the scheduled histogram whose interval contains it (stamped at the 7am/7pm interval start). The two twins routinely differ by HOURS, so the old ±5-minute window in find_twins silently missed them — which broke review propagation (flagging one twin left its twin unflagged). Twins are now matched by: same serial + identical peak_vector_sum + OPPOSITE record type + the waveform's timestamp falling within the histogram's interval (bounded by the next same-serial histogram). Matching keys off record timestamps (not call-in/received times, which drift with field connectivity). window_seconds is retained but ignored. Rewrote test_find_twins + test_twin_propagation for the new contract (incl. the 75-min-apart UM12947 case, cross-type exclusion, containing-interval selection, open-ended latest interval). Full suite: 264 passed; the 16 failures are pre-existing (missing gitignored fixtures + a v0.26.0 codec case), unchanged from baseline. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf
This commit is contained in:
Claude Opus 4.8
+57
-19
@@ -1,33 +1,71 @@
|
||||
import datetime
|
||||
import sqlite3
|
||||
from sfm.database import SeismoDb
|
||||
from minimateplus.models import Event, Timestamp
|
||||
|
||||
|
||||
def _ins(db, key, serial, pvs, ts):
|
||||
def _ins(db, key, serial, pvs, ts, record_type="Waveform"):
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = bytes.fromhex(key)
|
||||
ev.timestamp = ts
|
||||
# peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly
|
||||
db.insert_events([ev], serial=serial)
|
||||
row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0]
|
||||
import sqlite3
|
||||
with sqlite3.connect(db.db_path) as c:
|
||||
c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"]))
|
||||
c.execute("UPDATE events SET peak_vector_sum=?, record_type=? WHERE id=?",
|
||||
(pvs, record_type, row["id"]))
|
||||
return row["id"]
|
||||
|
||||
|
||||
def test_find_twins_matches_same_serial_pvs_near_time(tmp_path):
|
||||
def _ts(hour, minute, second=0, day=25):
|
||||
return Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=day,
|
||||
hour=hour, minute=minute, second=second)
|
||||
|
||||
|
||||
def test_histogram_and_waveform_twin_across_hours(tmp_path):
|
||||
# The real UM12947 case: histogram stamped at its 7pm interval start, the
|
||||
# triggered waveform 75 min later — same serial + identical PVS. The old
|
||||
# ±5-min window missed this; interval matching catches it, both directions.
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5)
|
||||
twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45)
|
||||
far = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=21, minute=0, second=0)
|
||||
# d needs a timestamp distinct from `twin` (UNIQUE(serial, timestamp) would
|
||||
# otherwise collide with b and UPSERT onto its row instead of inserting a
|
||||
# new one) while staying near `base` in time.
|
||||
near = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=44)
|
||||
a = _ins(db, "01110001", "BE1", 0.4763, base)
|
||||
b = _ins(db, "01110002", "BE1", 0.4763, twin) # twin: same serial+pvs, 40s apart
|
||||
c = _ins(db, "01110003", "BE1", 0.4763, far) # same pvs but >window away
|
||||
d = _ins(db, "01110004", "BE1", 0.9999, near) # near time but different pvs
|
||||
ids = {r["id"] for r in db.find_twins(a, window_seconds=300)}
|
||||
assert ids == {b}
|
||||
hist_pm = _ins(db, "01110001", "BE1", 0.4763, _ts(19, 31, 17), "Histogram")
|
||||
wave = _ins(db, "01110002", "BE1", 0.4763, _ts(20, 46, 44), "Waveform")
|
||||
_ins(db, "01110003", "BE1", 0.0100, _ts(7, 0, 0, day=26), "Histogram") # bounds the interval
|
||||
assert {r["id"] for r in db.find_twins(hist_pm)} == {wave}
|
||||
assert {r["id"] for r in db.find_twins(wave)} == {hist_pm}
|
||||
|
||||
|
||||
def test_same_type_not_twinned(tmp_path):
|
||||
# Two waveforms, same serial + PVS, seconds apart → NOT twins (cross-type only).
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
a = _ins(db, "01110001", "BE1", 0.4763, _ts(20, 19, 5), "Waveform")
|
||||
_ins(db, "01110002", "BE1", 0.4763, _ts(20, 19, 45), "Waveform")
|
||||
assert db.find_twins(a) == []
|
||||
|
||||
|
||||
def test_waveform_matches_only_the_containing_interval(tmp_path):
|
||||
# Two overnight intervals with the same PVS; a waveform in the SECOND interval
|
||||
# must twin with that histogram, never the first — even though PVS matches both.
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
h1 = _ins(db, "01110001", "BE1", 0.4763, _ts(19, 0, 0, day=25), "Histogram")
|
||||
h2 = _ins(db, "01110002", "BE1", 0.4763, _ts(7, 0, 0, day=26), "Histogram")
|
||||
w = _ins(db, "01110003", "BE1", 0.4763, _ts(8, 0, 0, day=26), "Waveform")
|
||||
assert {r["id"] for r in db.find_twins(w)} == {h2}
|
||||
assert w not in {r["id"] for r in db.find_twins(h1)}
|
||||
|
||||
|
||||
def test_different_pvs_not_twinned(tmp_path):
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
h = _ins(db, "01110001", "BE1", 0.4763, _ts(19, 0, 0), "Histogram")
|
||||
_ins(db, "01110002", "BE1", 0.9999, _ts(20, 0, 0), "Waveform") # different PVS
|
||||
assert db.find_twins(h) == []
|
||||
|
||||
|
||||
def test_open_ended_latest_interval(tmp_path):
|
||||
# A waveform after the latest histogram (nothing bounds the interval) still twins.
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
h = _ins(db, "01110001", "BE1", 0.4763, _ts(19, 0, 0), "Histogram")
|
||||
w = _ins(db, "01110002", "BE1", 0.4763, _ts(23, 30, 0), "Waveform")
|
||||
assert {r["id"] for r in db.find_twins(h)} == {w}
|
||||
|
||||
|
||||
def test_missing_fields_returns_empty(tmp_path):
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
assert db.find_twins("nonexistent-id") == []
|
||||
|
||||
@@ -3,31 +3,34 @@ from sfm.database import SeismoDb
|
||||
from minimateplus.models import Event, Timestamp
|
||||
|
||||
|
||||
def _ins(db, key, serial, pvs, ts):
|
||||
def _ins(db, key, serial, pvs, ts, record_type="Waveform"):
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = bytes.fromhex(key)
|
||||
ev.timestamp = ts
|
||||
# peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly
|
||||
db.insert_events([ev], serial=serial)
|
||||
row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0]
|
||||
with sqlite3.connect(db.db_path) as c:
|
||||
c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"]))
|
||||
c.execute("UPDATE events SET peak_vector_sum=?, record_type=? WHERE id=?",
|
||||
(pvs, record_type, row["id"]))
|
||||
return row["id"]
|
||||
|
||||
|
||||
def test_propagate_copies_flags_to_twins(tmp_path):
|
||||
def _ts(hour, minute, second=0, day=25):
|
||||
return Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=day,
|
||||
hour=hour, minute=minute, second=second)
|
||||
|
||||
|
||||
def test_propagate_copies_flags_across_hours_apart_twins(tmp_path):
|
||||
# Flagging the waveform FT propagates to its histogram twin 75 min earlier
|
||||
# (the interval matcher pairs them; the old ±5-min window would have missed it).
|
||||
db = SeismoDb(tmp_path / "s.db")
|
||||
base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5)
|
||||
twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45)
|
||||
other = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=44)
|
||||
hist = _ins(db, "01110001", "BE1", 0.4763, _ts(19, 31, 17), "Histogram")
|
||||
wave = _ins(db, "01110002", "BE1", 0.4763, _ts(20, 46, 44), "Waveform") # twin, 75 min later
|
||||
other = _ins(db, "01110003", "BE1", 0.9999, _ts(20, 20, 0), "Waveform") # different pvs
|
||||
|
||||
primary_id = _ins(db, "01110001", "BE1", 0.4763, base)
|
||||
twin_id = _ins(db, "01110002", "BE1", 0.4763, twin) # twin: same serial+pvs, 40s apart
|
||||
non_twin_id = _ins(db, "01110003", "BE1", 0.9999, other) # near time but different pvs
|
||||
db.update_event_review(wave, {"false_trigger": True})
|
||||
moved = db.propagate_review_to_twins(wave)
|
||||
|
||||
db.update_event_review(primary_id, {"false_trigger": True})
|
||||
moved = db.propagate_review_to_twins(primary_id)
|
||||
|
||||
assert twin_id in moved
|
||||
assert db.get_event(twin_id)["false_trigger"] == 1
|
||||
assert db.get_event(non_twin_id)["false_trigger"] == 0
|
||||
assert hist in moved
|
||||
assert db.get_event(hist)["false_trigger"] == 1
|
||||
assert db.get_event(other)["false_trigger"] == 0
|
||||
|
||||
Reference in New Issue
Block a user