402 lines
15 KiB
Python
402 lines
15 KiB
Python
"""
|
|
test_event_file_io.py — sidecar write/read/patch round-trips,
|
|
WaveformStore sidecar integration, and the BW-import path.
|
|
|
|
Run:
|
|
python tests/test_event_file_io.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import pytest
|
|
except ImportError:
|
|
pytest = None # type: ignore
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from minimateplus import event_file_io
|
|
from minimateplus.framing import S3Frame
|
|
from minimateplus.models import Event, Timestamp
|
|
|
|
|
|
# ── Fixtures shared with test_waveform_store.py ───────────────────────────────
|
|
|
|
|
|
def _make_synthetic_event() -> tuple[Event, list[S3Frame]]:
|
|
"""Same shape as tests/test_waveform_store.py — minimum viable Event +
|
|
A5 stream that makes write_blastware_file emit a parseable file.
|
|
|
|
STRT is exactly 21 bytes; rectime_seconds lands at byte 18 to match
|
|
`_decode_a5_waveform`'s expected layout (which is also what
|
|
`read_blastware_file()` reads back)."""
|
|
key4 = bytes.fromhex("01110000")
|
|
rectime = 3
|
|
strt = bytearray(21)
|
|
strt[0:4] = b"STRT"
|
|
strt[4:6] = b"\xff\xfe"
|
|
strt[6:10] = key4 # end_key (per data[23:27] in CLAUDE.md)
|
|
strt[10:14] = key4 # start_key (per data[27:31])
|
|
strt[18] = rectime
|
|
strt = bytes(strt)
|
|
|
|
probe_data = bytes(7) + strt + bytes(32)
|
|
probe = S3Frame(sub=0xA5, page_hi=0x10, page_lo=0x00, data=probe_data,
|
|
checksum_valid=True, chk_byte=0x00)
|
|
|
|
sample = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x10,
|
|
data=bytes(7) + bytes(0x0200), checksum_valid=True,
|
|
chk_byte=0x00)
|
|
|
|
# Build a valid 26-byte footer (0e 08 + ts1 + ts2 + 6 const + 2 crc)
|
|
# and embed it at the END of the terminator's contribution so
|
|
# write_blastware_file finds the real `0e 08` marker rather than
|
|
# falling back to slicing the last 26 bytes of zero garbage.
|
|
# ts byte order: [day][month][year_HI][year_LO][0x00][hour][min][sec]
|
|
footer = (
|
|
b"\x0e\x08"
|
|
+ bytes([6, 5, 0x07, 0xea, 0, 12, 34, 56]) # ts1 = 2026-05-06 12:34:56
|
|
+ bytes([6, 5, 0x07, 0xea, 0, 12, 35, 6]) # ts2 = ts1 + ~10s
|
|
+ b"\x00\x01\x00\x02\x00\x00"
|
|
+ b"\x00\x00"
|
|
)
|
|
assert len(footer) == 26
|
|
term_data = bytes(11) + bytes(38) + footer # 11 prefix + 38 pad + 26 footer = 75
|
|
term = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x00,
|
|
data=term_data, checksum_valid=True, chk_byte=0x00)
|
|
|
|
ev = Event(index=0)
|
|
ev._waveform_key = key4
|
|
ev.timestamp = Timestamp(
|
|
raw=b"", flag=0x10, year=2026, unknown_byte=0,
|
|
month=5, day=6, hour=12, minute=34, second=56,
|
|
)
|
|
ev.rectime_seconds = rectime
|
|
ev.record_type = "Waveform"
|
|
ev._a5_frames = [probe, sample, term]
|
|
return ev, [probe, sample, term]
|
|
|
|
|
|
# ── Sidecar write/read round-trip ─────────────────────────────────────────────
|
|
|
|
|
|
def test_event_to_sidecar_dict_shape():
|
|
ev, _ = _make_synthetic_event()
|
|
d = event_file_io.event_to_sidecar_dict(
|
|
ev,
|
|
serial="BE11529",
|
|
blastware_filename="M529LKIQ.7M0W",
|
|
blastware_filesize=1024,
|
|
blastware_sha256="abcd" * 16,
|
|
source_kind="sfm-live",
|
|
a5_pickle_filename="M529LKIQ.7M0W.a5.pkl",
|
|
)
|
|
|
|
assert d["schema_version"] == event_file_io.SCHEMA_VERSION
|
|
assert d["kind"] == event_file_io.SIDECAR_KIND
|
|
assert d["event"]["serial"] == "BE11529"
|
|
assert d["event"]["timestamp"] == "2026-05-06T12:34:56"
|
|
assert d["event"]["waveform_key"] == "01110000"
|
|
assert d["blastware"]["sha256"] == "abcd" * 16
|
|
assert d["source"]["kind"] == "sfm-live"
|
|
assert d["review"] == {
|
|
"false_trigger": False, "reviewer": None,
|
|
"reviewed_at": None, "notes": "",
|
|
}
|
|
assert d["extensions"] == {}
|
|
|
|
|
|
def test_sidecar_write_and_read_round_trip(tmp_path: Path):
|
|
ev, _ = _make_synthetic_event()
|
|
path = tmp_path / "M529LKIQ.7M0W.sfm.json"
|
|
src = event_file_io.event_to_sidecar_dict(
|
|
ev, serial="BE11529",
|
|
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
|
blastware_sha256="x" * 64, source_kind="sfm-ach",
|
|
)
|
|
event_file_io.write_sidecar(path, src)
|
|
loaded = event_file_io.read_sidecar(path)
|
|
assert loaded["event"] == src["event"]
|
|
assert loaded["blastware"] == src["blastware"]
|
|
assert loaded["source"]["kind"] == "sfm-ach"
|
|
|
|
|
|
def test_sidecar_persists_raw_0c_record_in_extensions(tmp_path: Path):
|
|
"""An Event with _raw_record populated should land its 210 bytes
|
|
base64-encoded in extensions.raw_records.waveform_record_b64, so
|
|
later analysis (e.g. mapping Peak Acceleration / Time of Peak / ZC
|
|
Freq byte offsets) can run offline against the saved sidecar."""
|
|
import base64
|
|
|
|
ev, _ = _make_synthetic_event()
|
|
# Synthesize a 210-byte 0C record with embedded label needles so
|
|
# the dump tool's anchor scan has something to find.
|
|
raw = bytearray(210)
|
|
raw[10:14] = b"Tran"
|
|
raw[60:64] = b"Vert"
|
|
raw[110:114] = b"Long"
|
|
raw[160:164] = b"MicL"
|
|
ev._raw_record = bytes(raw)
|
|
|
|
d = event_file_io.event_to_sidecar_dict(
|
|
ev, serial="BE11529",
|
|
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
|
blastware_sha256="x" * 64, source_kind="sfm-live",
|
|
)
|
|
|
|
rr = d["extensions"]["raw_records"]
|
|
assert rr["waveform_record_len"] == 210
|
|
decoded = base64.b64decode(rr["waveform_record_b64"])
|
|
assert decoded == ev._raw_record
|
|
|
|
# Round-trip through write/read
|
|
path = tmp_path / "raw0c.sfm.json"
|
|
event_file_io.write_sidecar(path, d)
|
|
loaded = event_file_io.read_sidecar(path)
|
|
assert (
|
|
base64.b64decode(loaded["extensions"]["raw_records"]["waveform_record_b64"])
|
|
== ev._raw_record
|
|
)
|
|
|
|
|
|
def test_sidecar_omits_raw_records_when_event_has_no_0c(tmp_path: Path):
|
|
"""Events without a _raw_record (e.g. constructed by importers that
|
|
never see 0C) should NOT add an empty raw_records block — keep the
|
|
sidecar clean for those flows."""
|
|
ev, _ = _make_synthetic_event()
|
|
assert ev._raw_record is None
|
|
|
|
d = event_file_io.event_to_sidecar_dict(
|
|
ev, serial="BE11529",
|
|
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
|
blastware_sha256="x" * 64, source_kind="bw-import",
|
|
)
|
|
assert d["extensions"] == {}
|
|
|
|
|
|
def test_sidecar_rejects_unsupported_schema_version(tmp_path: Path):
|
|
path = tmp_path / "future.sfm.json"
|
|
path.write_text(json.dumps({
|
|
"schema_version": event_file_io.SCHEMA_VERSION + 1,
|
|
"kind": event_file_io.SIDECAR_KIND,
|
|
}))
|
|
try:
|
|
event_file_io.read_sidecar(path)
|
|
except ValueError as exc:
|
|
assert "schema_version" in str(exc)
|
|
return
|
|
raise AssertionError("read_sidecar should have rejected unsupported version")
|
|
|
|
|
|
def test_sidecar_extensions_survive_round_trip(tmp_path: Path):
|
|
"""Forward-compat: unknown keys inside `extensions` survive a r/w cycle."""
|
|
ev, _ = _make_synthetic_event()
|
|
path = tmp_path / "x.sfm.json"
|
|
d = event_file_io.event_to_sidecar_dict(
|
|
ev, serial="BE11529",
|
|
blastware_filename="X", blastware_filesize=0, blastware_sha256="",
|
|
source_kind="sfm-live",
|
|
extensions={"vendor.acme.gps": {"lat": 40.7, "lon": -74.0}},
|
|
)
|
|
event_file_io.write_sidecar(path, d)
|
|
back = event_file_io.read_sidecar(path)
|
|
assert back["extensions"]["vendor.acme.gps"]["lat"] == 40.7
|
|
|
|
|
|
def test_sidecar_patch_review_stamps_reviewed_at(tmp_path: Path):
|
|
ev, _ = _make_synthetic_event()
|
|
path = tmp_path / "patch.sfm.json"
|
|
event_file_io.write_sidecar(
|
|
path,
|
|
event_file_io.event_to_sidecar_dict(
|
|
ev, serial="BE11529",
|
|
blastware_filename="X", blastware_filesize=0, blastware_sha256="",
|
|
source_kind="sfm-live",
|
|
),
|
|
)
|
|
new = event_file_io.patch_sidecar(
|
|
path,
|
|
review={"false_trigger": True, "notes": "truck thump", "reviewer": "brian"},
|
|
)
|
|
assert new["review"]["false_trigger"] is True
|
|
assert new["review"]["notes"] == "truck thump"
|
|
assert new["review"]["reviewer"] == "brian"
|
|
assert new["review"]["reviewed_at"], "reviewed_at must be auto-stamped"
|
|
|
|
on_disk = event_file_io.read_sidecar(path)
|
|
assert on_disk["review"]["false_trigger"] is True
|
|
|
|
|
|
# ── WaveformStore integration ─────────────────────────────────────────────────
|
|
|
|
|
|
def test_waveform_store_save_writes_sidecar(tmp_path: Path):
|
|
from sfm.waveform_store import WaveformStore
|
|
|
|
store = WaveformStore(tmp_path / "waveforms")
|
|
ev, frames = _make_synthetic_event()
|
|
rec = store.save(ev, serial="BE11529", a5_frames=frames, source_kind="sfm-live")
|
|
|
|
assert rec["sidecar_filename"].endswith(".sfm.json")
|
|
assert rec["sha256"] and len(rec["sha256"]) == 64
|
|
|
|
sc = store.load_sidecar("BE11529", rec["filename"])
|
|
assert sc is not None
|
|
assert sc["blastware"]["filename"] == rec["filename"]
|
|
assert sc["blastware"]["sha256"] == rec["sha256"]
|
|
assert sc["source"]["kind"] == "sfm-live"
|
|
# The .a5.pkl reference should match the actual filename on disk.
|
|
assert sc["source"]["a5_pickle_filename"] == rec["a5_pickle_filename"]
|
|
|
|
|
|
def test_waveform_store_save_preserves_review_across_resave(tmp_path: Path):
|
|
"""Re-saving the same event must preserve a user's prior review edits."""
|
|
from sfm.waveform_store import WaveformStore
|
|
|
|
store = WaveformStore(tmp_path / "waveforms")
|
|
ev, frames = _make_synthetic_event()
|
|
rec = store.save(ev, serial="BE11529", a5_frames=frames)
|
|
|
|
# User flips false_trigger and adds a note.
|
|
store.patch_sidecar(
|
|
"BE11529", rec["filename"],
|
|
review={"false_trigger": True, "notes": "hello"},
|
|
)
|
|
|
|
# A second save (e.g. Force refresh re-download) must keep those edits.
|
|
store.save(ev, serial="BE11529", a5_frames=frames)
|
|
sc = store.load_sidecar("BE11529", rec["filename"])
|
|
assert sc["review"]["false_trigger"] is True
|
|
assert sc["review"]["notes"] == "hello"
|
|
|
|
|
|
def test_waveform_store_patch_sidecar_returns_none_when_missing(tmp_path: Path):
|
|
from sfm.waveform_store import WaveformStore
|
|
|
|
store = WaveformStore(tmp_path / "waveforms")
|
|
out = store.patch_sidecar("BE99999", "no.such.W", review={"notes": "x"})
|
|
assert out is None
|
|
|
|
|
|
# ── DB integration: sidecar_filename column + update_event_review ─────────────
|
|
|
|
|
|
def test_seismodb_persists_sidecar_filename_and_review_sync(tmp_path: Path):
|
|
from sfm.database import SeismoDb
|
|
|
|
db = SeismoDb(tmp_path / "seismo_relay.db")
|
|
ev, _ = _make_synthetic_event()
|
|
|
|
rec = {
|
|
"filename": "M529LKIQ.7M0W",
|
|
"filesize": 8708,
|
|
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
|
"sidecar_filename": "M529LKIQ.7M0W.sfm.json",
|
|
}
|
|
inserted, _ = db.insert_events(
|
|
[ev], serial="BE11529",
|
|
waveform_records={ev._waveform_key.hex(): rec},
|
|
)
|
|
assert inserted == 1
|
|
|
|
rows = db.query_events(serial="BE11529")
|
|
row = rows[0]
|
|
assert row["sidecar_filename"] == rec["sidecar_filename"]
|
|
|
|
# update_event_review keeps false_trigger column in sync with sidecar.
|
|
assert db.update_event_review(row["id"], {"false_trigger": True}) is True
|
|
again = db.get_event(row["id"])
|
|
assert again["false_trigger"] == 1
|
|
|
|
# Empty review block (no false_trigger key) → no-op but row exists.
|
|
assert db.update_event_review(row["id"], {"notes": "x"}) is True
|
|
|
|
|
|
# ── BW-file reader (read_blastware_file) ─────────────────────────────────────
|
|
|
|
|
|
def test_read_blastware_file_round_trip(tmp_path: Path):
|
|
"""write → read → key/timestamp/rectime survive."""
|
|
from minimateplus.blastware_file import write_blastware_file, blastware_filename
|
|
|
|
ev, frames = _make_synthetic_event()
|
|
bw_path = tmp_path / blastware_filename(ev, "BE11529")
|
|
write_blastware_file(ev, frames, bw_path)
|
|
|
|
parsed = event_file_io.read_blastware_file(bw_path)
|
|
assert parsed._waveform_key == ev._waveform_key
|
|
assert parsed.rectime_seconds == ev.rectime_seconds
|
|
# Timestamp lands via the footer; year/month/day/hour/min/sec all survive.
|
|
assert parsed.timestamp is not None
|
|
assert parsed.timestamp.year == ev.timestamp.year
|
|
assert parsed.timestamp.month == ev.timestamp.month
|
|
assert parsed.timestamp.day == ev.timestamp.day
|
|
assert parsed.timestamp.hour == ev.timestamp.hour
|
|
assert parsed.timestamp.minute == ev.timestamp.minute
|
|
assert parsed.timestamp.second == ev.timestamp.second
|
|
# No A5 source recoverable.
|
|
assert parsed._a5_frames is None
|
|
# Peaks computed from samples (synthetic = zero samples → zero peaks).
|
|
assert parsed.peak_values is not None
|
|
assert parsed.peak_values.peak_vector_sum == 0.0
|
|
|
|
|
|
def test_save_imported_bw_round_trip(tmp_path: Path):
|
|
"""save_imported_bw stores a copy + sidecar with source.kind = bw-import."""
|
|
from minimateplus.blastware_file import write_blastware_file, blastware_filename
|
|
from sfm.waveform_store import WaveformStore
|
|
|
|
# Produce a BW file outside the store.
|
|
ev, frames = _make_synthetic_event()
|
|
fname = blastware_filename(ev, "BE11529")
|
|
src = tmp_path / fname
|
|
write_blastware_file(ev, frames, src)
|
|
|
|
store = WaveformStore(tmp_path / "waveforms")
|
|
parsed_ev, rec = store.save_imported_bw(src.read_bytes(), source_path=src)
|
|
|
|
assert rec["filename"] == fname
|
|
assert rec["a5_pickle_filename"] is None # no A5 source for BW imports
|
|
sc = store.load_sidecar("BE11529", fname)
|
|
assert sc is not None
|
|
assert sc["source"]["kind"] == "bw-import"
|
|
assert sc["source"]["a5_pickle_filename"] is None
|
|
# The stored binary should match the source byte-for-byte (we just copied).
|
|
stored_path = store.open_blastware("BE11529", fname)
|
|
assert stored_path is not None
|
|
assert stored_path.read_bytes() == src.read_bytes()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if pytest is not None:
|
|
pytest.main([__file__, "-v"])
|
|
else:
|
|
import inspect
|
|
import traceback as _tb
|
|
|
|
passed = failed = 0
|
|
for _name, _fn in sorted(globals().items()):
|
|
if not _name.startswith("test_") or not callable(_fn):
|
|
continue
|
|
try:
|
|
_sig = inspect.signature(_fn)
|
|
if "tmp_path" in _sig.parameters:
|
|
with tempfile.TemporaryDirectory() as _td:
|
|
_fn(Path(_td))
|
|
else:
|
|
_fn()
|
|
print(f"PASS {_name}")
|
|
passed += 1
|
|
except Exception:
|
|
print(f"FAIL {_name}")
|
|
_tb.print_exc()
|
|
failed += 1
|
|
print(f"\n{passed} passed, {failed} failed")
|
|
sys.exit(0 if failed == 0 else 1)
|