From 350f81f8b510d6217fc144c9c8bf946a8ab02ec0 Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 20 May 2026 05:22:28 +0000 Subject: [PATCH 1/4] fix: add thor specific ascii parser. --- sfm/idf_ascii_report.py | 70 +++++++++++++++++++++++++++-------------- sfm/sfm_webapp.html | 17 +++++++--- 2 files changed, 59 insertions(+), 28 deletions(-) diff --git a/sfm/idf_ascii_report.py b/sfm/idf_ascii_report.py index ea26293..9775b42 100644 --- a/sfm/idf_ascii_report.py +++ b/sfm/idf_ascii_report.py @@ -65,9 +65,17 @@ def _normalize_key(raw: str) -> str: def _strip_unit_suffix(value: str) -> str: - """Return the numeric part of values like "0.2119 in/s" → "0.2119".""" + """Return the numeric part of values like "0.2119 in/s" → "0.2119". + + Also strips Thor's below/above-threshold prefixes: + "<0.005 in/s" → "0.005" (below-noise-floor reading) + ">100 Hz" → "100" (above-measurement-range reading) + """ parts = value.strip().split() - return parts[0] if parts else value.strip() + token = parts[0] if parts else value.strip() + if token.startswith("<") or token.startswith(">"): + token = token[1:] + return token def _parse_float(value: str) -> Optional[float]: @@ -178,38 +186,54 @@ def parse_idf_report(text: Union[str, bytes]) -> Dict[str, Any]: except ValueError: pass - # Numeric scalars - for key in ("sample_rate",): + # Numeric scalars. For every field we typify here, we MUST drop the + # raw string copy from `out` when parsing fails — Thor writes things + # like "<0.005 in/s" (below threshold) and "N/A" (not measured) that + # would otherwise linger in `out` as strings, sneak into SQLite REAL + # columns via permissive type affinity, and then crash the JS + # frontend on `.toFixed(...)`. + int_fields = ("sample_rate",) + for key in int_fields: v = raw.get(key) - if v is not None: - iv = _parse_int(v) - if iv is not None: - out[key] = iv + if v is None: + continue + iv = _parse_int(v) + if iv is not None: + out[key] = iv + else: + out.pop(key, None) - for key in ("tran_ppv", "vert_ppv", "long_ppv", "peak_vector_sum", - "tran_zc_freq", "vert_zc_freq", "long_zc_freq", - "tran_peak_acceleration", "vert_peak_acceleration", - "long_peak_acceleration", - "tran_peak_displacement", "vert_peak_displacement", - "long_peak_displacement", - "tran_time_of_peak", "vert_time_of_peak", "long_time_of_peak", - "mic_time_of_peak", "mic_zc_freq"): + float_fields = ( + "tran_ppv", "vert_ppv", "long_ppv", "peak_vector_sum", + "tran_zc_freq", "vert_zc_freq", "long_zc_freq", + "tran_peak_acceleration", "vert_peak_acceleration", + "long_peak_acceleration", + "tran_peak_displacement", "vert_peak_displacement", + "long_peak_displacement", + "tran_time_of_peak", "vert_time_of_peak", "long_time_of_peak", + "mic_time_of_peak", "mic_zc_freq", + ) + for key in float_fields: v = raw.get(key) - if v is not None: - fv = _parse_float(v) - if fv is not None: - out[key] = fv + if v is None: + continue + fv = _parse_float(v) + if fv is not None: + out[key] = fv + else: + out.pop(key, None) # Microphone — Thor reports MicPSPL (dB(L)) which is the closest - # analogue to BW's mic_ppv. Stored as a float; units are in the - # original raw field (`mic_pspl` raw entry preserves "99.4 dB(L)"). + # analogue to BW's mic_ppv. The raw "99.4 dB(L)" string stays in + # `out` under the original `mic_pspl` key for display; the parsed + # float goes in `mic_ppv`. mic = raw.get("mic_pspl") if mic is not None: fv = _parse_float(mic) if fv is not None: out["mic_ppv"] = fv - # Record / pre-trigger duration + # Record / pre-trigger duration — same drop-on-failure discipline. rt = raw.get("record_time") if rt is not None: fv = _parse_float(rt) diff --git a/sfm/sfm_webapp.html b/sfm/sfm_webapp.html index 17474b4..7925c83 100644 --- a/sfm/sfm_webapp.html +++ b/sfm/sfm_webapp.html @@ -2285,13 +2285,16 @@ let sessLoaded = false; const _unitSerials = new Set(); function _ppvClass(v) { - if (v == null) return ''; - if (v >= 2.0) return 'ppv-high'; - if (v >= 0.5) return 'ppv-warn'; + const n = (v == null) ? null : Number(v); + if (n == null || !isFinite(n)) return ''; + if (n >= 2.0) return 'ppv-high'; + if (n >= 0.5) return 'ppv-warn'; return 'ppv-ok'; } function _ppvFmt(v) { - return v != null ? v.toFixed(5) : '—'; + if (v == null) return '—'; + const n = typeof v === 'number' ? v : Number(v); + return isFinite(n) ? n.toFixed(5) : String(v); } function _fmtTs(ts) { if (!ts) return '—'; @@ -2386,7 +2389,11 @@ async function loadHistory() { ${_ppvFmt(ev.vert_ppv)} ${_ppvFmt(ev.long_ppv)} ${_ppvFmt(pvs)} - ${ev.mic_ppv != null && ev.mic_ppv > 0 ? (20 * Math.log10(ev.mic_ppv / DBL_REF)).toFixed(1) + ' dBL' : '—'} + ${(() => { + const m = ev.mic_ppv == null ? null : Number(ev.mic_ppv); + if (m == null || !isFinite(m) || m <= 0) return '—'; + return (20 * Math.log10(m / DBL_REF)).toFixed(1) + ' dBL'; + })()} ${ev.project ?? '—'} ${ev.client ?? '—'} ${ev.record_type ?? '—'} -- 2.52.0 From 3265ad6fa34a148405f9bf9cdac1aacb04311421 Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 20 May 2026 05:43:52 +0000 Subject: [PATCH 2/4] fix: apply psi dbL conversion rule --- sfm/sfm_webapp.html | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sfm/sfm_webapp.html b/sfm/sfm_webapp.html index 7925c83..019f900 100644 --- a/sfm/sfm_webapp.html +++ b/sfm/sfm_webapp.html @@ -2392,6 +2392,11 @@ async function loadHistory() { ${(() => { const m = ev.mic_ppv == null ? null : Number(ev.mic_ppv); if (m == null || !isFinite(m) || m <= 0) return '—'; + // BW (.AB0*/.N00) stores mic_ppv as psi → convert to dBL. + // Thor IDF (.IDFH/.IDFW) already stores dBL → display directly. + const fn = (ev.blastware_filename || '').toUpperCase(); + const isThor = fn.endsWith('.IDFH') || fn.endsWith('.IDFW'); + if (isThor) return m.toFixed(1) + ' dBL'; return (20 * Math.log10(m / DBL_REF)).toFixed(1) + ' dBL'; })()} ${ev.project ?? '—'} @@ -2454,11 +2459,20 @@ function _renderSidecar(data) { document.getElementById('sc-title').textContent = `Event — ${bw.filename || ev.waveform_key || 'unknown'}`; - const fmtPpv = v => (v == null ? '—' : Number(v).toFixed(5) + ' in/s'); + const fmtPpv = v => { + if (v == null) return '—'; + const n = Number(v); + return isFinite(n) ? n.toFixed(5) + ' in/s' : String(v); + }; const fmtMic = v => { - if (v == null || v <= 0) return '—'; - const dbl = 20 * Math.log10(v / DBL_REF); - return `${dbl.toFixed(1)} dBL (${v.toExponential(2)} psi)`; + if (v == null) return '—'; + const n = Number(v); + if (!isFinite(n) || n <= 0) return '—'; + // Source-aware: Thor IDF imports store mic as dB(L); BW imports store + // it as psi. `src` is in the outer scope from _renderSidecar(). + if ((src.kind || '') === 'idf-import') return `${n.toFixed(1)} dBL`; + const dbl = 20 * Math.log10(n / DBL_REF); + return `${dbl.toFixed(1)} dBL (${n.toExponential(2)} psi)`; }; document.getElementById('sc-f-serial').textContent = ev.serial || '—'; -- 2.52.0 From e95ac692ee4713e41171bcaa17c64532de5ec57c Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 20 May 2026 06:15:50 +0000 Subject: [PATCH 3/4] feat: add device family to separate s3 and s4 events. --- bridges/ach_server.py | 1 + scripts/backfill_sidecars.py | 1 + sfm/database.py | 59 ++++++++++++++++++++++++++++++++++-- sfm/import_bw.py | 1 + sfm/server.py | 3 ++ sfm/sfm_webapp.html | 25 ++++++++------- 6 files changed, 76 insertions(+), 14 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index c048d4c..0bfd8af 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -516,6 +516,7 @@ class AchSession: serial=serial or self.peer, session_id=None, waveform_records=waveform_records, + device_family="series3", ) _ml_ins, _ml_skip = self.db.insert_monitor_log( new_monitor_entries, session_id=None diff --git a/scripts/backfill_sidecars.py b/scripts/backfill_sidecars.py index 6b7cb82..b937e8c 100644 --- a/scripts/backfill_sidecars.py +++ b/scripts/backfill_sidecars.py @@ -326,6 +326,7 @@ def main(argv=None) -> int: }} if ev._waveform_key else None ), + device_family="series3", ) except Exception as exc: log.warning("DB upsert failed for %s: %s", path.name, exc) diff --git a/sfm/database.py b/sfm/database.py index 8228d6c..1792f20 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -85,6 +85,7 @@ CREATE TABLE IF NOT EXISTS events ( blastware_filesize INTEGER, -- bytes; NULL if no event file saved a5_pickle_filename TEXT, -- ".a5.pkl" sidecar sidecar_filename TEXT, -- ".sfm.json" review/metadata sidecar + device_family TEXT, -- "series3" (MiniMate Plus / BW) | "series4" (Micromate / Thor) — drives per-family UI rendering (units, labels) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), UNIQUE(serial, timestamp) ); @@ -198,11 +199,53 @@ class SeismoDb: ("blastware_filesize", "INTEGER"), ("a5_pickle_filename", "TEXT"), ("sidecar_filename", "TEXT"), + ("device_family", "TEXT"), ): if col not in existing_cols: log.info("_migrate: events ADD COLUMN %s %s", col, ddl) conn.execute(f"ALTER TABLE events ADD COLUMN {col} {ddl}") + # Migration 1c: backfill device_family for existing rows by sniffing + # the device-native binary filename's extension. Thor (Micromate + # Series IV) writes `.IDFH` / `.IDFW`; MiniMate Plus (Series III) + # writes `.AB0*` / `.N00` / `.` Blastware extensions. We do + # this here rather than from sidecars so the migration is fully + # self-contained (doesn't need the waveform-store root) and runs at + # DB-init time. Only fills NULL device_family so re-runs are no-ops. + rebackfill = conn.execute( + "SELECT COUNT(*) FROM events WHERE device_family IS NULL" + ).fetchone() + if rebackfill and rebackfill[0] > 0: + log.info("_migrate: backfilling device_family for %d events", rebackfill[0]) + # Series IV (Thor IDF) — extension is exactly .IDFH or .IDFW + conn.execute( + """ + UPDATE events + SET device_family = 'series4' + WHERE device_family IS NULL + AND ( + UPPER(blastware_filename) LIKE '%.IDFH' + OR UPPER(blastware_filename) LIKE '%.IDFW' + ) + """ + ) + # Everything else with a filename → Series III (Blastware family) + conn.execute( + """ + UPDATE events + SET device_family = 'series3' + WHERE device_family IS NULL + AND blastware_filename IS NOT NULL + """ + ) + # Rows with no filename (e.g. older monitor_log-derived events) + # stay NULL — UI handles NULL as "unknown family". + remaining = conn.execute( + "SELECT COUNT(*) FROM events WHERE device_family IS NULL" + ).fetchone()[0] + log.info("_migrate: device_family backfill complete (remaining NULL=%d)", + remaining) + # Migration 2: change monitor_log UNIQUE from (serial, waveform_key) to # (serial, start_time) — same reasoning as events. row = conn.execute( @@ -302,6 +345,7 @@ class SeismoDb: serial: str, session_id: Optional[str] = None, waveform_records: Optional[dict[str, dict]] = None, + device_family: Optional[str] = None, ) -> tuple[int, int]: """ Insert triggered events. Silently skips duplicates (serial+timestamp). @@ -316,6 +360,11 @@ class SeismoDb: (dedup hit), the matching waveform record is upserted onto the existing row so a re-download via the live endpoint refreshes the file metadata. + + ``device_family`` (optional): "series3" (MiniMate Plus / Blastware) or + "series4" (Micromate / Thor). Drives per-family UI rendering — most + importantly the mic-unit convention (psi vs dB(L)). Set on every + insert and overwritten on every UPSERT so the latest writer wins. """ inserted = skipped = 0 wave_recs = waveform_records or {} @@ -349,8 +398,9 @@ class SeismoDb: project, client, operator, sensor_location, sample_rate, record_type, blastware_filename, blastware_filesize, - a5_pickle_filename, sidecar_filename) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + a5_pickle_filename, sidecar_filename, + device_family) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( self._new_id(), serial, key, session_id, ts, @@ -369,6 +419,7 @@ class SeismoDb: rec.get("filesize"), rec.get("a5_pickle_filename"), rec.get("sidecar_filename"), + device_family, ), ) inserted += 1 @@ -409,7 +460,8 @@ class SeismoDb: blastware_filename = ?, blastware_filesize = ?, a5_pickle_filename = ?, - sidecar_filename = ? + sidecar_filename = ?, + device_family = COALESCE(?, device_family) WHERE serial = ? AND timestamp = ? """, ( @@ -428,6 +480,7 @@ class SeismoDb: rec.get("filesize") if rec else None, rec.get("a5_pickle_filename") if rec else None, rec.get("sidecar_filename") if rec else None, + device_family, serial, ts, ), diff --git a/sfm/import_bw.py b/sfm/import_bw.py index f49097b..5d31bdd 100644 --- a/sfm/import_bw.py +++ b/sfm/import_bw.py @@ -166,6 +166,7 @@ def main(argv: list[str] | None = None) -> int: {ev._waveform_key.hex(): rec} if ev._waveform_key else None ), + device_family="series3", ) tag = "OK " if ins else ("SKIP" if sk else "OK ") print(f" [{tag}] {path.name} → {rec['filename']} " diff --git a/sfm/server.py b/sfm/server.py index 2378ab9..078fb96 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -918,6 +918,7 @@ def device_event_blastware_file( [ev], serial=serial, waveform_records={ev._waveform_key.hex(): rec}, + device_family="series3", ) log.info( "blastware_file: persisted to store (%s, %d bytes)", @@ -2434,6 +2435,7 @@ async def db_import_blastware_file( ev._waveform_key.hex(): rec if ev._waveform_key else None } if ev._waveform_key else None, + device_family="series3", ) results.append({ "filename": filename, @@ -2558,6 +2560,7 @@ async def db_import_idf_file( waveform_records={ ev._waveform_key.hex(): rec } if ev._waveform_key else None, + device_family="series4", ) results.append({ "filename": filename, diff --git a/sfm/sfm_webapp.html b/sfm/sfm_webapp.html index 019f900..576ae94 100644 --- a/sfm/sfm_webapp.html +++ b/sfm/sfm_webapp.html @@ -2392,11 +2392,9 @@ async function loadHistory() { ${(() => { const m = ev.mic_ppv == null ? null : Number(ev.mic_ppv); if (m == null || !isFinite(m) || m <= 0) return '—'; - // BW (.AB0*/.N00) stores mic_ppv as psi → convert to dBL. - // Thor IDF (.IDFH/.IDFW) already stores dBL → display directly. - const fn = (ev.blastware_filename || '').toUpperCase(); - const isThor = fn.endsWith('.IDFH') || fn.endsWith('.IDFW'); - if (isThor) return m.toFixed(1) + ' dBL'; + // Series III (MiniMate Plus / BW) stores mic_ppv as psi → convert. + // Series IV (Micromate / Thor) already stores dB(L) → display direct. + if (ev.device_family === 'series4') return m.toFixed(1) + ' dBL'; return (20 * Math.log10(m / DBL_REF)).toFixed(1) + ' dBL'; })()} ${ev.project ?? '—'} @@ -2464,13 +2462,18 @@ function _renderSidecar(data) { const n = Number(v); return isFinite(n) ? n.toFixed(5) + ' in/s' : String(v); }; + // Map sidecar source.kind → device family (Series IV ingest path is + // "idf-import"; everything else is Series III today). The events-list + // table uses ev.device_family from the DB row, but sidecars don't carry + // that column — source.kind is the equivalent signal here. + const family = ((src.kind || '') === 'idf-import') ? 'series4' : 'series3'; const fmtMic = v => { if (v == null) return '—'; const n = Number(v); if (!isFinite(n) || n <= 0) return '—'; - // Source-aware: Thor IDF imports store mic as dB(L); BW imports store - // it as psi. `src` is in the outer scope from _renderSidecar(). - if ((src.kind || '') === 'idf-import') return `${n.toFixed(1)} dBL`; + // Series IV (Micromate / Thor) stores mic as dB(L); Series III (BW) + // stores it as psi and we render both for cross-reference. + if (family === 'series4') return `${n.toFixed(1)} dBL`; const dbl = 20 * Math.log10(n / DBL_REF); return `${dbl.toFixed(1)} dBL (${n.toExponential(2)} psi)`; }; @@ -2767,9 +2770,9 @@ document.getElementById('api-base').value = window.location.origin;

Source / files

-
BW filename
—
-
BW filesize
—
-
BW sha256
—
+
Event file
—
+
File size
—
+
File sha256
—
Source kind
—
Captured at
—
-- 2.52.0 From ecc935482bf61d913b8114b6cab38caecb809f47 Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 20 May 2026 15:19:49 +0000 Subject: [PATCH 4/4] =?UTF-8?q?seismo-relay=20v0.19.0=20=E2=80=94=20device?= =?UTF-8?q?-family=20separation=20+=20micromate/=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tighten the Series III / Series IV boundary so UI and storage dispatch on a clean signal instead of sniffing filenames or applying magnitude heuristics. Phase 1 — events.device_family column ("series3" | "series4"): self-applying migration with filename-based backfill of existing rows (1,132 backfilled on prod 2026-05-20); plumbed through every import path (BW endpoint, IDF endpoint, ACH server, BW CLI, sidecar backfill); UPSERT preserves via COALESCE; UI dispatches on it. Phase 2 — extract micromate/ package alongside minimateplus/: native IdfEvent / IdfReport / IdfPeaks / IdfProjectInfo / IdfSensorCheck (mic in dB(L), not pseudo-psi); moved idf_ascii_report.py from sfm/ to micromate/; refactored save_imported_idf to use IdfEvent and bridge to minimateplus.Event at the SQL-insert boundary; idf_file.py stub for the future binary codec. Phase 3 prep — docs/idf_protocol_reference.md captures the two observed Thor binary header signatures (1,012 newer-firmware files vs 2 old files whose layout is byte-for-byte BW-STRT-compatible), file-size hints suggesting int8 sample encoding, open questions in dependency order, and a concrete first-session plan for cracking the codec. Also rolled in the v0.18.1 hotfixes that motivated this work: - idf_ascii_report parser now handles "<0.005 in/s" (below-threshold) and "N/A" markers without leaving raw strings in numeric DB columns. - sfm_webapp.html: defensive _ppvFmt / mic formatter so future data-shape drift can't kill the whole events table render. All 1,014 example-data sidecars round-trip through the new package. See CHANGELOG.md for full notes. --- CHANGELOG.md | 27 ++ README.md | 152 ++++++++-- docs/idf_protocol_reference.md | 284 +++++++++++++++++++ micromate/__init__.py | 48 ++++ {sfm => micromate}/idf_ascii_report.py | 2 +- micromate/idf_file.py | 64 +++++ micromate/models.py | 377 +++++++++++++++++++++++++ pyproject.toml | 6 +- sfm/server.py | 2 +- sfm/waveform_store.py | 121 +++----- tests/test_idf_ascii_report.py | 2 +- 11 files changed, 966 insertions(+), 119 deletions(-) create mode 100644 docs/idf_protocol_reference.md create mode 100644 micromate/__init__.py rename {sfm => micromate}/idf_ascii_report.py (99%) create mode 100644 micromate/idf_file.py create mode 100644 micromate/models.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e7ceae..f2d4f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ All notable changes to seismo-relay are documented here. --- +## v0.19.0 — 2026-05-20 + +The "device-family separation" release. Tightens the boundary between Series III (MiniMate Plus / Blastware) and Series IV (Micromate / Thor) so the UI and storage layer dispatch deterministically by family instead of sniffing filename extensions or magnitude heuristics. + +### Added — Phase 1: `device_family` column on `events` + +- **`events.device_family TEXT`** — new column carrying `"series3"` or `"series4"`. Populated by every import path (`/db/import/blastware_file`, `/db/import/idf_file`, ACH server, BW CLI, sidecar backfill script). Returned through `/db/events` since `query_events` uses `SELECT *`. +- **Self-applying migration** — on startup, `ALTER TABLE ... ADD COLUMN` lands the new column; a follow-on `UPDATE` backfills existing rows from the binary filename extension (`.IDFH`/`.IDFW` → `series4`, everything else → `series3`). No manual SQL needed. +- **UPSERT preserves family** — re-imports without an explicit family don't blank existing rows (`COALESCE(?, device_family)`). +- **UI dispatches on the column** — `sfm_webapp.html` events-table mic formatter now branches on `ev.device_family === 'series4'` (Thor stores native dB(L); BW stores psi). Modal uses `source.kind === 'idf-import'` from the sidecar (sidecars don't carry the DB column). Source-files section labels changed from "BW filename / BW filesize / BW sha256" to format-neutral "Event file / File size / File sha256". + +### Added — Phase 2: `micromate/` package alongside `minimateplus/` + +- **`micromate/`** — new sibling package for the Thor / Micromate Series IV device. Currently scoped to offline-file ingest; live-device support (TCP transport, framing, protocol, client) will land here when reverse-engineering happens. + - `micromate/idf_ascii_report.py` — moved from `sfm/idf_ascii_report.py`. No behaviour change. + - `micromate/models.py` — typed `IdfReport`, `IdfEvent`, `IdfPeaks`, `IdfProjectInfo`, `IdfSensorCheck`. Stores mic in native `mic_pspl_dbl` (dB(L)) instead of the pseudo-psi shoehorn that the BW-shaped model uses. `IdfEvent.from_report()` constructs from a parsed dict + filename; `IdfEvent.to_minimateplus_event(waveform_key)` bridges to the existing sidecar / DB-insert machinery. + - `micromate/idf_file.py` — placeholder for the binary codec (`.IDFH` / `.IDFW`). Stubbed `read_idf_file()` raises `NotImplementedError`; documents the planned reverse-engineering path. +- **`WaveformStore.save_imported_idf`** refactored to use the native `IdfEvent` and bridge at the SQL-insert boundary. Cleaner separation of "parse a Thor event" (in `micromate/`) from "store it on disk + write a sidecar" (in `sfm/waveform_store.py`). +- **Tests** — `tests/test_idf_ascii_report.py` imports updated to `micromate.idf_ascii_report`. All 1,014 example-data sidecars round-trip through `IdfEvent.from_report()` without errors. + +### Companion releases + +- **thor-watcher** unaffected — it talks to the relay over HTTP only. No version bump needed. +- **terra-view** unaffected today; can use `device_family` in its event-detail rendering when convenient. + +--- + ## v0.18.0 — 2026-05-19 The "Thor / Series IV ingest adapter" release. Seismo-relay can now accept event files from Instantel Micromate Series IV (Thor) units alongside the existing MiniMate Plus (Series III) Blastware pipeline. diff --git a/README.md b/README.md index 79534c8..c057f68 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ -# seismo-relay `v0.17.0` +# seismo-relay `v0.19.0` A ground-up replacement for **Blastware** — Instantel's aging Windows-only -software for managing MiniMate Plus seismographs. +software for managing seismographs. Supports both the **MiniMate Plus +(Series III)** and the **Micromate (Series IV / "Thor")** families: +Series III via the live RS-232 / TCP wire protocol *and* Blastware ACH file +ingest; Series IV currently via Thor TXT-paired IDF file ingest, with the +binary codec on the roadmap. Built in Python. Runs on Windows, Linux, or macOS. Connects to instruments over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). @@ -19,6 +23,18 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). > every Blastware ACH event lands in SeismoDb with device-authoritative > peaks, project metadata, sensor self-check, and ZC/Time-of-Peak data, > without depending on the still-undecoded waveform body codec. +> **v0.18.0 (2026-05-19)** adds Thor / Micromate Series IV ingest at +> `/db/import/idf_file` — paired with **thor-watcher v0.3.0**, every +> `.IDFH` / `.IDFW` event file (plus its `.txt` sidecar) lands in +> SeismoDb the same way BW events do. See +> [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md) for +> the IDF format reference and reverse-engineering plan. +> **v0.19.0 (2026-05-20)** separates Series III and Series IV at the +> code level: new `micromate/` package alongside `minimateplus/`, new +> `events.device_family` DB column ("series3" / "series4") so the UI +> and storage layer dispatch deterministically instead of sniffing +> filenames. Self-applying migration backfills existing rows from the +> binary filename extension. > See [CHANGELOG.md](CHANGELOG.md) for full version history. --- @@ -29,17 +45,25 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). seismo-relay/ ├── seismo_lab.py ← Main GUI (Bridge + Analyzer + Download + Console tabs) │ -├── minimateplus/ ← MiniMate Plus client library +├── minimateplus/ ← Series III (MiniMate Plus) client library │ ├── transport.py ← SerialTransport, TcpTransport, SocketTransport │ ├── protocol.py ← DLE frame layer, SUB command dispatch │ ├── client.py ← High-level client (connect, get_events, delete_all_events, push_config, get_call_home_config, …) │ ├── framing.py ← Frame builders, DLE codec, S3FrameParser │ ├── models.py ← DeviceInfo, Event, ComplianceConfig, MonitorLogEntry, CallHomeConfig, … +│ ├── bw_ascii_report.py ← Parse BW per-event ASCII reports (.TXT sidecars) +│ ├── event_file_io.py ← Read BW binaries, write .sfm.json sidecars │ └── blastware_file.py ← Write events to Blastware-compatible .AB0 files │ +├── micromate/ ← Series IV (Micromate / Thor) client library (NEW v0.19) +│ ├── models.py ← IdfEvent, IdfReport, IdfPeaks, IdfProjectInfo, IdfSensorCheck (mic in native dB(L)) +│ ├── idf_ascii_report.py ← Parse Thor .IDFW.txt / .IDFH.txt event sidecars +│ └── idf_file.py ← Stub for the .IDFW / .IDFH binary codec (reverse-engineering pending) +│ ├── sfm/ ← SFM REST API server (FastAPI, port 8200) -│ ├── server.py ← Live device endpoints + DB query endpoints + caching -│ ├── database.py ← SeismoDb — SQLite persistence (events, monitor_log, ach_sessions, sessions table) +│ ├── server.py ← Live device endpoints + DB query + ingest endpoints + caching +│ ├── database.py ← SeismoDb — SQLite persistence (events, monitor_log, ach_sessions) +│ ├── waveform_store.py ← On-disk store for BW + IDF event binaries + .sfm.json sidecars │ └── sfm_webapp.html ← Embedded web UI with Call Home config tab │ ├── bridges/ @@ -56,7 +80,8 @@ seismo-relay/ │ └── frame_db.py ← SQLite frame database │ └── docs/ - └── instantel_protocol_reference.md ← Reverse-engineered protocol spec + ├── instantel_protocol_reference.md ← Series III protocol spec (the Rosetta Stone) + └── idf_protocol_reference.md ← Series IV (Thor IDF) format reference + codec RE plan ``` --- @@ -148,11 +173,23 @@ Query the SQLite database written by `ach_server.py`. All read-only except | Method | URL | Description | |--------|-----|-------------| | `GET` | `/db/units` | All known serials with summary stats | -| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger) | +| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger). Response rows include `device_family` ("series3" / "series4") so clients dispatch on unit type without sniffing filenames. | | `GET` | `/db/monitor_log` | Monitoring intervals | | `GET` | `/db/sessions` | ACH call-home session history | | `PATCH` | `/db/events/{id}/false_trigger?value=true` | Flag / unflag false triggers | +### File ingest endpoints + +Used by watcher daemons to push field-collected event files into the SFM DB ++ waveform store. Both accept multipart uploads of binary event files +optionally paired with their ASCII sidecar reports; both dedup by +`(serial, timestamp)` and UPSERT device-authoritative fields on re-import. + +| Method | URL | Description | +|--------|-----|-------------| +| `POST` | `/db/import/blastware_file` | Series III: `.AB0*` / `.N00` binaries + paired `_ASCII.TXT`. Source: `series3-watcher`. | +| `POST` | `/db/import/idf_file` | Series IV: `.IDFH` / `.IDFW` binaries + paired `.IDFW.txt` / `.IDFH.txt`. Source: `thor-watcher`. | + --- ## minimateplus library @@ -214,22 +251,77 @@ not per individual event). --- +## micromate library + +Series IV / Thor support, sibling to `minimateplus`. Currently scoped to +offline-file ingest from Thor's TXT exporter; live-device protocol is +deferred until the binary codec is cracked. + +```python +from micromate import IdfEvent, parse_idf_report + +# Parse a .IDFW.txt / .IDFH.txt sidecar (1014 example files round-trip cleanly) +text = open("UM11719_20231219162723.IDFW.txt").read() +report_dict = parse_idf_report(text) # permissive dict + +# Wrap into a typed event using the device-native binary filename +event = IdfEvent.from_report(report_dict, "UM11719_20231219162723.IDFW") + +event.serial # "UM11719" +event.kind # "Waveform" or "Histogram" +event.peaks.transverse_ips # 0.0251 (in/s, native unit) +event.peaks.mic_pspl_dbl # 99.4 (dB(L), Thor's native mic unit — NOT psi) +event.project_info.project # "UPMC Presby-Loc 3-Level1-1R Elevator Rm" +event.sensor_check.tran # True (passed self-check) +event.firmware_version # "Micromate ISEE 11.0AK" +event.calibration_text # "November 22, 2023 by Instantel" + +# Bridge to the existing minimateplus.Event shape for the DB / sidecar paths +# (waveform_key is a 16-byte sha256 prefix when ingesting from a binary file) +bridged_event = event.to_minimateplus_event(waveform_key=b"\x00" * 16) +``` + +The binary codec (`.IDFW` / `.IDFH` event files themselves) is on the +roadmap — see [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md) +for everything known so far, the two observed file signatures, and the +reverse-engineering plan. The `micromate/idf_file.py` stub is where +`read_idf_file()` will land. + +--- + ## Database -`ach_server.py` writes to `bridges/captures/seismo_relay.db` (SQLite, WAL mode) using the -`SeismoDb` persistence layer. Four tables, all unit-keyed by serial number: +`ach_server.py` and the file-ingest endpoints write to +`bridges/captures/seismo_relay.db` (SQLite, WAL mode) via the `SeismoDb` +persistence layer. Three tables, all unit-keyed by serial number: | Table | Key | Contents | |-------|-----|----------| | `ach_sessions` | UUID | Per-call-home audit record: serial, timestamp, peer IP, events_downloaded, monitor_entries, duration_seconds | -| `events` | UUID, UNIQUE(serial, waveform_key) | Triggered events: timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location strings, sample_rate, record_type, false_trigger flag | -| `monitor_log` | UUID, UNIQUE(serial, waveform_key) | Monitoring intervals: serial, waveform_key, start_time, stop_time, duration_seconds, geo_threshold_ips | -| `events.false_trigger` | Boolean flag | PATCH endpoint to mark/unmark false triggers for review | +| `events` | UUID, UNIQUE(serial, timestamp) | Triggered events: timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location strings, sample_rate, record_type, false_trigger flag, **`device_family`** ("series3" / "series4"), `blastware_filename` (binary at-rest in `waveforms/`), sidecar references | +| `monitor_log` | UUID, UNIQUE(serial, start_time) | Monitoring intervals: serial, waveform_key, start_time, stop_time, duration_seconds, geo_threshold_ips | -Deduplication is by `(serial, waveform_key)` — repeat call-homes or re-runs never -produce duplicate rows. Post-erase key reuse is handled automatically via the -high-water mark in `ach_state.json`. Key-based state tracking allows correct -handling of device erasures (external or post-download). +**Deduplication is by `(serial, timestamp)`** — the device clock is the +stable natural key. Repeat call-homes or re-runs UPSERT the row in place, +refreshing every device-authoritative field (peaks, project strings, +sample_rate, file references) so the latest writer wins. `false_trigger` +and `device_family` are preserved across UPSERTs. Earlier versions used +`(serial, waveform_key)` for dedup, but the device's event-key counter +resets to `0x01110000` after every erase, so timestamps are the correct +dedup field. Migration handles the transition transparently on first +startup. + +**`device_family` (added v0.19.0)** discriminates Series III from Series +IV at the SQL level. Set by every import path; the UI dispatches on it +to render mic units correctly (Series III: psi → dBL conversion; Series +IV: native dBL passthrough). Existing rows are backfilled at first +startup of v0.19.0+ by sniffing the binary filename extension. + +The on-disk waveform store lives at `bridges/captures/waveforms//` +and holds the original event binaries (BW `.AB0*` / `.N00` for Series III, +`.IDFH` / `.IDFW` for Series IV) plus their `.sfm.json` review/metadata +sidecars. Series III events also produce `.a5.pkl` source-frame pickles +and `.h5` clean-waveform exports; Series IV doesn't yet (pending codec). --- @@ -311,18 +403,27 @@ Use **com0com** or **VSPD** to create the virtual COM pair on Windows. ## Key Features -**Device support:** -- [x] Full read/write/erase pipelines +**Series III (MiniMate Plus) device support:** +- [x] Full read/write/erase pipelines over RS-232 or TCP/cellular - [x] Compliance config (recording mode, sample rate, histogram interval, geo sensitivity, project strings) - [x] Auto Call Home config (read/write ACH settings, dial string, time slots, retries) - [x] Monitor control (start/stop, status polling, battery/memory) - [x] Monitor log entries (continuous monitoring intervals without full waveform download) +- [x] Blastware file ingest at `/db/import/blastware_file` (paired with `series3-watcher`) + +**Series IV (Micromate / Thor) device support:** +- [x] Thor IDF file ingest at `/db/import/idf_file` (paired with `thor-watcher`, v0.18.0+) +- [x] Native `IdfEvent` / `IdfReport` typed models — mic in dB(L), full title strings, sensor self-check, calibration, firmware version +- [x] Parser verified against 1,014 paired `.txt` sidecars in `thor-watcher/example-data/` +- [ ] Binary `.IDFW` / `.IDFH` codec — pending (see Roadmap + [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md)) +- [ ] Live-device protocol — pending codec **Data persistence:** -- [x] SQLite database (`seismo_relay.db`) with 4 tables: ach_sessions, events, monitor_log, plus false_trigger flag -- [x] Deduplication by waveform key (handles re-runs and repeat call-homes) -- [x] Post-erase key-reuse detection (tracks high-water mark) -- [x] Session state (`ach_state.json`) with downloaded keys and max key +- [x] SQLite database (`seismo_relay.db`) with `events`, `monitor_log`, `ach_sessions` tables +- [x] Per-row `device_family` column ("series3" / "series4") for clean UI / unit-of-measurement dispatch (v0.19.0+) +- [x] Deduplication by `(serial, timestamp)` — natural key handles post-erase counter resets +- [x] UPSERT on re-import refreshes every device-authoritative field (peaks, project, sample_rate); preserves operator review state (`false_trigger`) +- [x] Post-erase key-reuse detection (tracks high-water mark in `ach_state.json`) **REST API:** - [x] Live device endpoints with in-memory caching (`_LiveCache`) @@ -330,6 +431,7 @@ Use **com0com** or **VSPD** to create the virtual COM pair on Windows. - [x] DB query endpoints (units, events, monitor_log, sessions, false_trigger PATCH) - [x] Call Home config read/write endpoints - [x] Blastware file download endpoint (`/device/event/{index}/blastware_file`) +- [x] Import endpoints for both device families (`/db/import/blastware_file`, `/db/import/idf_file`) **File output (v0.7+, byte-perfect as of v0.14.3):** - [x] Blastware-compatible `.AB0` / `.G10` file generation (waveform + metadata) @@ -359,8 +461,10 @@ Use **com0com** or **VSPD** to create the virtual COM pair on Windows. ### High-impact (unblocks product features) -- [ ] **Waveform body codec reverse-engineering.** The 5A bulk-stream body is some kind of compressed/encoded format (not raw int16 LE as previously assumed — see §7.6.1 retraction in `docs/instantel_protocol_reference.md`). Structural framing is ~50% decoded on branch `claude/codec-re-cBGNe` (tagged-block walker, segment counters); per-byte sample mapping is still open. Until this lands, the in-app waveform viewer renders garbage and BW-import peak values fall back to `_peaks_from_samples()` saturation noise. Workaround: pair every BW-imported event with its `_ASCII.TXT` so the device-authoritative peaks land in the DB regardless of codec. -- [ ] **In-app waveform viewer accuracy.** Depends on codec decode. Plot.v1 JSON pipeline + viewer skeleton already exist; will start showing real waveforms automatically once `_decode_a5_waveform` produces correct samples. +- [ ] **Series III waveform body codec reverse-engineering.** The 5A bulk-stream body is some kind of compressed/encoded format (not raw int16 LE as previously assumed — see §7.6.1 retraction in `docs/instantel_protocol_reference.md`). Structural framing is ~50% decoded on branch `claude/codec-re-cBGNe` (tagged-block walker, segment counters); per-byte sample mapping is still open. Until this lands, the in-app waveform viewer renders garbage and BW-import peak values fall back to `_peaks_from_samples()` saturation noise. Workaround: pair every BW-imported event with its `_ASCII.TXT` so the device-authoritative peaks land in the DB regardless of codec. +- [ ] **Series IV (Thor IDF) binary codec reverse-engineering.** `.IDFH` / `.IDFW` files are currently stored opaquely by `WaveformStore.save_imported_idf`, with all metadata sourced from the paired `.txt` sidecar. This works because thor-watcher forwards both files together, but operators who haven't enabled Thor's TXT exporter get rows with NULL peaks. Cracking the binary closes that gap and unlocks waveform display. Starting-point reference at [`docs/idf_protocol_reference.md`](docs/idf_protocol_reference.md) — two observed file signatures (1,012 newer-firmware files + 2 old files whose layout matches the Series III STRT-record format), suggested first-session plan (~2-4 hrs), 1,014 paired binary+txt files available as ground truth in `thor-watcher/example-data/`. Code seam ready at `micromate/idf_file.py`. +- [ ] **In-app waveform viewer accuracy.** Depends on Series III codec decode. Plot.v1 JSON pipeline + viewer skeleton already exist; will start showing real waveforms automatically once `_decode_a5_waveform` produces correct samples. Series IV waveforms come online when the IDF codec lands. +- [ ] **Series IV live-device support.** Once the IDF binary is decoded, extend `micromate/` with `transport.py` / `framing.py` / `protocol.py` / `client.py` mirroring the `minimateplus/` package layout — depends on capturing Thor's wire protocol (TCP / RS-232 captures TBD). - [ ] **Terra-view integration** — seismo-relay router, unit detail page, VISON-style event listing. - [ ] **Vibration summary reports** — highest legit PPV per project → Word doc (false-trigger filtering first). diff --git a/docs/idf_protocol_reference.md b/docs/idf_protocol_reference.md new file mode 100644 index 0000000..643de53 --- /dev/null +++ b/docs/idf_protocol_reference.md @@ -0,0 +1,284 @@ +# IDF Protocol Reference — Thor / Micromate Series IV + +Starting-point reference for reverse-engineering Instantel's Micromate +Series IV event-file format. Sibling to +[instantel_protocol_reference.md](instantel_protocol_reference.md) (the +Series III "Rosetta Stone") — this doc holds what we know so far and +the open questions still to crack. + +**Status (2026-05-20):** ASCII text sidecar fully decoded (1,014 +sample files round-trip). Binary `.IDFH` / `.IDFW` codec +**not yet implemented** — binaries are stored opaquely by +`WaveformStore.save_imported_idf`, with metadata sourced from the +paired `.txt` sidecar. + +--- + +## File model + +### Filename convention + +``` +_. +``` + +- **SERIAL** — literal device serial, two-letter prefix + numeric + suffix. Examples seen: `UM11719`, `UM13981`, `UM20147`, `BE9439`. + Unlike Series III BW filenames (`M529LK44.AB0`, base-36 stem), + Series IV filenames carry the serial in plain text. +- **YYYYMMDDHHMMSS** — 14-char ASCII timestamp in **device local + time** (no timezone marker). +- **KIND** — `IDFH` for histograms, `IDFW` for waveforms. + +The `.IDFH.txt` / `.IDFW.txt` ASCII sidecar lives in a `TXT/` +**subfolder** of the unit's directory, not alongside the binary. +This pairing convention is encoded in +`event_forwarder.idf_report_path()`. + +### Directory layout + +``` +C:\THORDATA\ +└── \ + └── \ ← unit serial dir + ├── UM12345_20260520100000.MLG ← monitor log (not events) + ├── UM12345_20260520100000.IDFH ← histogram event (binary) + ├── UM12345_20260520100000.IDFW ← waveform event (binary) + ├── UM12345_20260520100000.IDFW.CDB ← cache-DB variant (skip) + ├── TXT\ + │ ├── UM12345_20260520100000.IDFH.txt ← histogram ASCII sidecar + │ └── UM12345_20260520100000.IDFW.txt ← waveform ASCII sidecar + ├── CSV\, HTML\, PDF\, XML\ ← operator-facing derived exports + └── ... +``` + +The `.IDFW.CDB` files share the binary's basename but appear to be a +separate cache/database variant. Their first 8 bytes match the +**old**-firmware Thor signature (see below) regardless of which +signature the paired `.IDFW` uses. Purpose unknown; sizes vary +wildly (observed 123 B → 40,491 B). Thor-watcher's forwarder +deliberately skips them. + +### Sample corpus + +The `thor-watcher/example-data/THORDATA_example/` tree carries +**1,014 paired .IDFW / .IDFH + .txt files** spanning 2020–2023 +across nine units (UM11719, UM13981, UM20147, …, plus BE9439 from +2020). This is the reverse-engineering ground truth. + +--- + +## ASCII sidecar (`.IDFW.txt` / `.IDFH.txt`) — fully decoded + +Shape: plain text, one `"Key : Value"` line per metadata field, +followed for waveforms by a tab-separated sample table headed by +the literal line `Waveform Data Channels`. Parsed by +[`micromate/idf_ascii_report.py`](../micromate/idf_ascii_report.py). +See [`micromate/models.py`](../micromate/models.py) for the typed +`IdfReport` shape. + +### Notable conventions + +- **Units are native to Thor** — geophone in **in/s**, microphone in + **dB(L)** (not psi like Series III BW reports), frequency in Hz, + acceleration in g, displacement in in. +- **Below-threshold readings** appear as the literal string + `<0.005 in/s` (155 occurrences in the sample corpus) — the parser + strips the `<` and treats the numeric remainder as the value. +- **Out-of-range / not-measured** values appear as `N/A` — parser + drops the field rather than letting the string leak into a numeric + column. +- **Firmware string** observed: `Micromate ISEE 11.0AK`. +- **TitleString1..4** are operator-defined free-text slots; Thor's + default labels map them to Location / Client / Company / Notes, + which the parser surfaces as `project` / `client` / `operator` / + `notes`. +- **Histogram sidecars** use `HistogramStartDate` / `HistogramStartTime` + in place of waveform's `EventDate` / `EventTime`. Parser falls + through to either. +- **Histogram tabular block** lacks the `Waveform Data Channels` + marker; instead it's a multi-line column header followed by + per-interval rows (`