seismo-relay v0.19.0 — device-family separation + micromate/ package
Tighten the Series III / Series IV boundary so UI and storage dispatch
on a clean signal instead of sniffing filenames or applying magnitude
heuristics.
Phase 1 — events.device_family column ("series3" | "series4"):
self-applying migration with filename-based backfill of existing rows
(1,132 backfilled on prod 2026-05-20); plumbed through every import
path (BW endpoint, IDF endpoint, ACH server, BW CLI, sidecar
backfill); UPSERT preserves via COALESCE; UI dispatches on it.
Phase 2 — extract micromate/ package alongside minimateplus/:
native IdfEvent / IdfReport / IdfPeaks / IdfProjectInfo /
IdfSensorCheck (mic in dB(L), not pseudo-psi); moved
idf_ascii_report.py from sfm/ to micromate/; refactored
save_imported_idf to use IdfEvent and bridge to minimateplus.Event at
the SQL-insert boundary; idf_file.py stub for the future binary codec.
Phase 3 prep — docs/idf_protocol_reference.md captures the two
observed Thor binary header signatures (1,012 newer-firmware files vs
2 old files whose layout is byte-for-byte BW-STRT-compatible), file-size
hints suggesting int8 sample encoding, open questions in dependency
order, and a concrete first-session plan for cracking the codec.
Also rolled in the v0.18.1 hotfixes that motivated this work:
- idf_ascii_report parser now handles "<0.005 in/s" (below-threshold)
and "N/A" markers without leaving raw strings in numeric DB columns.
- sfm_webapp.html: defensive _ppvFmt / mic formatter so future
data-shape drift can't kill the whole events table render.
All 1,014 example-data sidecars round-trip through the new package.
See CHANGELOG.md for full notes.
This commit is contained in:
+32
-89
@@ -426,99 +426,48 @@ class WaveformStore:
|
||||
`.IDFH`) produced by Thor's TXT exporter.
|
||||
|
||||
Thor binaries are stored as opaque bytes — seismo-relay doesn't
|
||||
decode the proprietary IDF binary format. Device-authoritative
|
||||
metadata comes from the paired `.IDFW.txt` / `.IDFH.txt` sidecar
|
||||
when supplied; we parse that text and surface its fields onto
|
||||
the returned Event so the SFM database row has real PPV/project
|
||||
values instead of NULLs.
|
||||
yet decode the proprietary IDF binary format (codec slot lives
|
||||
at ``micromate/idf_file.py``). Device-authoritative metadata
|
||||
comes from the paired ``.IDFW.txt`` / ``.IDFH.txt`` sidecar
|
||||
when supplied.
|
||||
|
||||
Workflow:
|
||||
1. Parse the paired TXT report (when supplied) via
|
||||
`sfm.idf_ascii_report.parse_idf_report`.
|
||||
2. Build a minimal `Event` populated from the report fields
|
||||
(timestamp, peaks, project info, sample_rate, record_type).
|
||||
3. Resolve serial from filename prefix or `serial_hint`.
|
||||
4. Copy bytes verbatim into <root>/<serial>/<filename>.
|
||||
5. Write the `.sfm.json` sidecar with source.kind = "idf-import".
|
||||
``micromate.parse_idf_report`` → dict.
|
||||
2. Wrap parsed dict + filename into a typed ``micromate.IdfEvent``.
|
||||
3. Copy bytes verbatim into ``<root>/<serial>/<filename>``.
|
||||
4. Bridge IdfEvent → ``minimateplus.Event`` (for the existing
|
||||
sidecar / DB insert machinery) via
|
||||
``IdfEvent.to_minimateplus_event(waveform_key)``.
|
||||
5. Write the ``.sfm.json`` sidecar with
|
||||
``source.kind = "idf-import"`` and the full raw IDF report
|
||||
under ``extensions.idf_report``.
|
||||
|
||||
Returns (event, record_dict) so the endpoint can both insert
|
||||
Returns ``(event, record_dict)`` so the endpoint can both insert
|
||||
into SeismoDb and surface the parsed event.
|
||||
"""
|
||||
from sfm.idf_ascii_report import (
|
||||
parse_idf_report,
|
||||
parse_event_filename,
|
||||
serial_from_filename as _idf_serial_from_filename,
|
||||
)
|
||||
from minimateplus.models import (
|
||||
Event, PeakValues, ProjectInfo, Timestamp,
|
||||
)
|
||||
from micromate import IdfEvent, parse_idf_report
|
||||
|
||||
# Parse the .txt sidecar (best-effort; non-fatal on failure).
|
||||
report: dict = {}
|
||||
report_dict: dict = {}
|
||||
if idf_report_text is not None:
|
||||
try:
|
||||
report = parse_idf_report(idf_report_text)
|
||||
report_dict = parse_idf_report(idf_report_text)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save_imported_idf: report parse failed: %s — continuing without it",
|
||||
exc,
|
||||
)
|
||||
|
||||
# Resolve serial: prefer the explicit hint, fall back to filename prefix.
|
||||
serial = (
|
||||
serial_hint
|
||||
or report.get("serial_number")
|
||||
or _idf_serial_from_filename(source_path.name)
|
||||
or "UNKNOWN"
|
||||
)
|
||||
# Build the typed IdfEvent. Filename is authoritative for
|
||||
# (serial, timestamp, kind); the report's event_datetime takes
|
||||
# precedence over the filename timestamp inside from_report().
|
||||
idf_event = IdfEvent.from_report(report_dict, source_path.name)
|
||||
|
||||
# Resolve event timestamp + kind from the filename (always present).
|
||||
parsed_name = parse_event_filename(source_path.name)
|
||||
kind = "Waveform"
|
||||
ts_dt: Optional[datetime.datetime] = None
|
||||
if parsed_name is not None:
|
||||
_, ts_dt, kind_token = parsed_name
|
||||
kind = "Histogram" if kind_token == "IDFH" else "Waveform"
|
||||
# Report's event_datetime is the device-authoritative value; prefer it.
|
||||
if "event_datetime" in report:
|
||||
try:
|
||||
ts_dt = datetime.datetime.fromisoformat(report["event_datetime"])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
ts_obj: Optional[Timestamp] = None
|
||||
if ts_dt is not None:
|
||||
ts_obj = Timestamp(
|
||||
raw=bytes(9),
|
||||
flag=0,
|
||||
year=ts_dt.year,
|
||||
unknown_byte=0,
|
||||
month=ts_dt.month,
|
||||
day=ts_dt.day,
|
||||
hour=ts_dt.hour,
|
||||
minute=ts_dt.minute,
|
||||
second=ts_dt.second,
|
||||
)
|
||||
|
||||
# Build PeakValues from the report (fields are None when absent).
|
||||
pv = PeakValues(
|
||||
tran=report.get("tran_ppv"),
|
||||
vert=report.get("vert_ppv"),
|
||||
long=report.get("long_ppv"),
|
||||
micl=report.get("mic_ppv"),
|
||||
peak_vector_sum=report.get("peak_vector_sum"),
|
||||
)
|
||||
|
||||
# Build ProjectInfo. See idf_ascii_report — Thor's title strings
|
||||
# carry project / client / company / notes in TitleString1..4.
|
||||
pi = ProjectInfo(
|
||||
setup_name=report.get("setup"),
|
||||
project=report.get("project"),
|
||||
client=report.get("client"),
|
||||
operator=report.get("operator"),
|
||||
sensor_location=None, # Thor folds location into TitleString1 = project
|
||||
notes=report.get("notes"),
|
||||
)
|
||||
# Operator-supplied serial_hint wins over the binary's filename
|
||||
# prefix when both are present (e.g. callers passing a known-good
|
||||
# serial that overrides a misnamed export).
|
||||
serial = serial_hint or idf_event.serial or "UNKNOWN"
|
||||
|
||||
# Filesystem write.
|
||||
filename = source_path.name
|
||||
@@ -532,16 +481,10 @@ class WaveformStore:
|
||||
# surrogate — every distinct binary maps to a distinct row.
|
||||
waveform_key = bytes.fromhex(sha256)[:16]
|
||||
|
||||
ev = Event(
|
||||
index=0,
|
||||
timestamp=ts_obj,
|
||||
sample_rate=report.get("sample_rate"),
|
||||
peak_values=pv,
|
||||
project_info=pi,
|
||||
record_type=kind,
|
||||
rectime_seconds=report.get("record_time_sec"),
|
||||
)
|
||||
ev._waveform_key = waveform_key
|
||||
# Bridge to minimateplus.Event for the existing sidecar / DB
|
||||
# insert paths. See IdfEvent.to_minimateplus_event() for the
|
||||
# caveats of this bridge (mic units, missing fields → sidecar).
|
||||
ev = idf_event.to_minimateplus_event(waveform_key)
|
||||
|
||||
# Write the sidecar. Source kind "idf-import" was added to the
|
||||
# allow-list in event_file_io.event_to_sidecar_dict for this.
|
||||
@@ -567,14 +510,14 @@ class WaveformStore:
|
||||
# consumers can recover the rich derived fields that don't fit
|
||||
# the BW-shaped event model (Peak Acceleration / Displacement,
|
||||
# Time of Peak, sensor self-check, calibration, firmware).
|
||||
if report:
|
||||
sidecar["extensions"]["idf_report"] = report
|
||||
if report_dict:
|
||||
sidecar["extensions"]["idf_report"] = report_dict
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
log.info(
|
||||
"WaveformStore.save_imported_idf serial=%s filename=%s filesize=%d "
|
||||
"report_attached=%s",
|
||||
serial, filename, filesize, bool(report),
|
||||
serial, filename, filesize, bool(report_dict),
|
||||
)
|
||||
return ev, {
|
||||
"filename": filename,
|
||||
|
||||
Reference in New Issue
Block a user