Compare commits
208 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a18712442f | |||
| 8aea46b8a0 | |||
| 9123269b1f | |||
| 9400f59167 | |||
| bbed85f7e2 | |||
| c641d5fc10 | |||
| 9afa3484f4 | |||
| 0484680c89 | |||
| 3711b11bda | |||
| 52c6e7b618 | |||
| 29ebc75656 | |||
| ebfe9877fa | |||
| c914a15e12 | |||
| a27693242d | |||
| eefec0bd64 | |||
| 7444738883 | |||
| 6b76934a04 | |||
| 7b62c790a9 | |||
| b66cc9d075 | |||
| 4ab604eff1 | |||
| e15f1567ef | |||
| bb33ad3837 | |||
| 45e61fbcaf | |||
| d758825c67 | |||
| 0fbb39c21a | |||
| 1ef55521b1 | |||
| 738b39f3cb | |||
| 625b0a4dfc | |||
| b14f31f3b0 | |||
| b9ab368934 | |||
| 9004241846 | |||
| 6861d9ed97 | |||
| 5cd5652560 | |||
| 897ac8a3f3 | |||
| 310fc5986c | |||
| e1150b30aa | |||
| a7585cb5e0 | |||
| 9bbecea70f | |||
| ae30a02898 | |||
| 2f084ed105 | |||
| 7976b544ed | |||
| 0415af19b4 | |||
| 35c3f4f945 | |||
| 43c8158493 | |||
| 242666f358 | |||
| 03540fdc00 | |||
| f83fd880c0 | |||
| ab2c11e9a9 | |||
| fa887b85d9 | |||
| ecd980d345 | |||
| bc9f16e503 | |||
| aa2b02535b | |||
| 2a2031c3a9 | |||
| 9e7e0bce2a | |||
| 5e2f3bf2a1 | |||
| 39ebd4bdaa | |||
| 84c87d0b57 | |||
| ec6362cb8e | |||
| 3eeafd24aa | |||
| 8cb8b86192 | |||
| 6dcca4da79 | |||
| c47e3a3af0 | |||
| dfbc9f29c5 | |||
| 4331215e23 | |||
| b3dcfe7239 | |||
| 9b5cdfd857 | |||
| 4a0c9b6da5 | |||
| 7129aae279 | |||
| 2186bc238b | |||
| 3fb24e1895 | |||
| 7bdd7c92f2 | |||
| b6ffdcfa87 | |||
| a7aec31915 | |||
| 34df9ec5fa | |||
| eec6c3dc6a | |||
| 702e06873e | |||
| 94767f5a9d | |||
| e04114fd6c | |||
| f10c5c1b86 | |||
| aa28495a43 | |||
| b23cf4bb50 | |||
| 969010b983 | |||
| 5fba9bcff8 | |||
| ec7be4d784 | |||
| b8ed237363 | |||
| 5866ecdb3e | |||
| ea9c69b7c9 | |||
| 71bcf71cf7 | |||
| 3e7de848bc | |||
| 72a4209cfd | |||
| 2b5574511e | |||
| ce2c859f11 | |||
| 7f322f9ff9 | |||
| 42b7a88c3d | |||
| c474db4f69 | |||
| 2765ee6ea7 | |||
| ef88240796 | |||
| 5591d345d9 | |||
| 7883a31aa7 | |||
| b241da970d | |||
| 6acb419ebd | |||
| f6a0846bab | |||
| 3d9db8b662 | |||
| c7e7d177e6 | |||
| a3b8d10fa8 | |||
| 4921b0489a | |||
| 8688d815a0 | |||
| 9b50ec9133 | |||
| cba8b1b401 | |||
| 41a14ca468 | |||
| 1bfc6e4258 | |||
| 574d40027f | |||
| 0358acb51d | |||
| cf7d838bf4 | |||
| 5e44cdc668 | |||
| 37d32077a4 | |||
| b384ba66d1 | |||
| 27d9823cc1 | |||
| 70c9528611 | |||
| e8bef1ac7c | |||
| 27db663579 | |||
| e5ea17388a | |||
| c0a5131c7d | |||
| 4ec2f33308 | |||
| 6282eacf8b | |||
| 034b3f044d | |||
| 48d7e94c02 | |||
| 03d224ccc3 | |||
| ef2c38e7db | |||
| b9a8e50b3c | |||
| 77d9c17680 | |||
| 8a1bd34551 | |||
| 09788b931a | |||
| e712d68505 | |||
| 8f5da918b5 | |||
| a03c77af09 | |||
| 87fa9c954f | |||
| 3f7b5c07b5 | |||
| 3d2ebfc057 | |||
| 9d9c14af79 | |||
| ab14328c8b | |||
| 0baf343bf5 | |||
| 05421764a5 | |||
| 74233d7e31 | |||
| 46a86939b7 | |||
| 2db565ff9c | |||
| 990cb8850e | |||
| dda5683572 | |||
| 16e072698b | |||
| c8c57e950c | |||
| a41e7a9e1a | |||
| 8545daac04 | |||
| 1a9dcc04b4 | |||
| a7ab6eaf7c | |||
| 7005ae766d | |||
| bcc044655a | |||
| c2ab94f20c | |||
| b5828de534 | |||
| 9bef430451 | |||
| 781d21f132 | |||
| e3a5c6f07d | |||
| 1397f8486f | |||
| 5b3e8af1e3 | |||
| ad1c9e48b0 | |||
| 227c481022 | |||
| 33de4239f4 | |||
| d0d5a18d5c | |||
| 41090a9346 | |||
| d87e02fab2 | |||
| 57e7225a62 | |||
| 5d43acd827 | |||
| dfa09d2a4f | |||
| ecb1147216 | |||
| 1c570b083a | |||
| 2286d2ccf8 | |||
| 755050b347 | |||
| 6adf8b6078 | |||
| 4fb1bbfe35 | |||
| 3effa1aab5 | |||
| 95f2becf21 | |||
| 2cb95cd45e | |||
| 7cd8fda5e8 | |||
| f495b91d8a | |||
| e4730376ad | |||
| 23e4febba6 | |||
| 8941dd0aef | |||
| dfb974d658 | |||
| 790e442a7a | |||
| 5d0f0855f2 | |||
| 0f5aa7a3fc | |||
| 3b04d4683b | |||
| 0363425d83 | |||
| 66967e036c | |||
| 9bf20803c2 | |||
| 9b1ed1f3a8 | |||
| 501b5080e9 | |||
| 5948c833bd | |||
| c4a5da893c | |||
| 638e60532c | |||
| 6eecd0c1d1 | |||
| 870a10365e | |||
| b2d10fd689 | |||
| ce44852383 | |||
| 6a42facf02 | |||
| 4b703811d9 | |||
| ea4475c9ad | |||
| df51fe0668 | |||
| 114cbb4679 |
+33
-28
@@ -1,28 +1,33 @@
|
||||
/bridges/captures/
|
||||
/example-events/
|
||||
|
||||
/manuals/
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Editor / OS
|
||||
.vscode/
|
||||
*.swp
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Analyzer outputs
|
||||
*.report
|
||||
claude_export_*.md
|
||||
|
||||
# Frame database
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
/bridges/captures/
|
||||
/example-events/
|
||||
|
||||
/manuals/
|
||||
|
||||
# Python build artifacts
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Editor / OS
|
||||
.vscode/
|
||||
*.swp
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Analyzer outputs
|
||||
*.report
|
||||
claude_export_*.md
|
||||
|
||||
# Frame database
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
|
||||
+783
@@ -4,6 +4,789 @@ All notable changes to seismo-relay are documented here.
|
||||
|
||||
---
|
||||
|
||||
## v0.15.0 — 2026-05-07
|
||||
|
||||
### Added
|
||||
|
||||
- **Layered event storage architecture.** Each event now lands as four
|
||||
files in the per-serial waveform store, each with a clear role:
|
||||
|
||||
- `<filename>` — the Blastware-readable binary (BW file). Untouched.
|
||||
- `<filename>.a5.pkl` — the raw 5A frames (regenerative source).
|
||||
- `<filename>.h5` — clean per-channel waveform arrays in physical
|
||||
units (in/s for geo, psi for mic) plus event metadata (HDF5 with
|
||||
gzip compression). This is the canonical format for downstream
|
||||
analysis tools.
|
||||
- `<filename>.sfm.json` — the modern review/metadata sidecar (peaks,
|
||||
project, source provenance, review state, extensions).
|
||||
|
||||
SQLite (`seismo_relay.db`) is the searchable index over all four.
|
||||
|
||||
- **Plot-ready waveform JSON (`sfm.plot.v1`).** The `/device/event/{idx}/waveform`
|
||||
and `/db/events/{id}/waveform.json` endpoints now return samples in
|
||||
physical units with explicit time-axis metadata, peak markers, and
|
||||
per-channel unit hints — no more guessing the ADC-to-velocity scale
|
||||
client-side. The webapp waveform viewer was rewritten to consume
|
||||
this shape.
|
||||
|
||||
- **In-app waveform viewer accuracy fix.** The standalone SFM webapp
|
||||
viewer was scaling geophone amplitudes by `geoAdcScale / 32767`
|
||||
(≈ 6.206 / 32767), where `geoAdcScale = 6.206053` is the device's
|
||||
*in/s per V* hardware constant — not the ADC-counts-to-velocity
|
||||
factor. This silently scaled every plot ~38% too low for Normal-range
|
||||
geophones (the correct full-scale is 10.0 in/s, or 1.25 in/s for
|
||||
Sensitive). Conversion is now done server-side using the geo_range
|
||||
from compliance config; the client just plots.
|
||||
|
||||
- New `sfm/event_hdf5.py` module: `write_event_hdf5()`,
|
||||
`read_event_hdf5()`, plus a plot-JSON helper.
|
||||
- Backfill script extended to also emit `.h5` for existing events.
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Added `h5py>=3.10` and `numpy>=1.24` for the HDF5 storage layer.
|
||||
- Added `python-multipart>=0.0.7` (required by FastAPI for the
|
||||
`/db/import/blastware_file` endpoint introduced in this release).
|
||||
|
||||
---
|
||||
|
||||
## v0.14.3 — 2026-05-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`build_5a_frame` — DLE-stuffing rule for 0x10 bytes in params (the
|
||||
long-standing >1-sec event 0 "won't open in BW" bug).**
|
||||
|
||||
Previously `build_5a_frame` wrote params bytes RAW with no DLE stuffing,
|
||||
based on the incorrect assumption that the device handled all `0x10`
|
||||
bytes in params literally. It does not. The device's actual de-stuffing
|
||||
rule for the params region is:
|
||||
|
||||
- `10 10` → de-stuffs to `10`
|
||||
- `10 02/03/04` → kept literal (inner-frame markers)
|
||||
- `10 X` for other X → de-stuffs to just `X` (drops the `0x10`)
|
||||
|
||||
When the counter passed in params has `0x10` in the high byte (e.g.
|
||||
counter=`0x1000` produces params bytes `... 10 00 ...`), the device
|
||||
silently corrupts the request to counter=`0x__00` and responds with
|
||||
whatever lives at that wrong address. For counter=0x1000 the wrong
|
||||
address was 0x0000, so the response was a copy of the file header +
|
||||
STRT record. That STRT block then got embedded in the assembled body
|
||||
at file offset `0x1016`, and Blastware refused to open the file
|
||||
(interprets the second STRT as a malformed multi-event file).
|
||||
|
||||
This explains the entire >1-sec event-0 failure pattern:
|
||||
|
||||
- 1-sec events have `end_offset < 0x1000`, so the chunk walk never
|
||||
requests counter `0x10__` and the bug never triggers.
|
||||
- 2-sec / 3-sec / longer events all need a chunk at counter `0x1000`
|
||||
(and longer events also need `0x1200`, `0x1400`, etc., none of which
|
||||
have `0x10` in the high byte except `0x1000`). Just one corrupted
|
||||
response is enough to embed STRT in the body and break the file.
|
||||
|
||||
Verified against BW 5-1-26 "copy 3sec" capture: all 17 5A request
|
||||
frames (probe + 2 metadata pages + 13 sample chunks + TERM) now match
|
||||
BW's wire output **byte-for-byte**, including the doubled `10 10 00`
|
||||
for counter=0x1000.
|
||||
|
||||
### Notes
|
||||
|
||||
- `0x10` bytes in `offset_hi` (the standalone offset field at body[5])
|
||||
are still written RAW — confirmed correct per the 1-2-26 capture.
|
||||
- BW's actual encoding of `10 02` / `10 04` for meta pages 0x1002 /
|
||||
0x1004 is *not* doubled — it relies on the device keeping `10 02`
|
||||
and `10 04` as literal pairs. This is preserved by the fix.
|
||||
|
||||
---
|
||||
|
||||
## v0.14.2 — 2026-05-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`blastware_file.py` — removed harmful "duplicate header+STRT" strip.**
|
||||
The v0.13.x strip logic was matching the byte sequence `00 12 03 00 STRT`
|
||||
in legitimate waveform data — sample chunks at counter `0x1000` and
|
||||
beyond often contain those bytes coincidentally — and zeroing 25 bytes
|
||||
of valid samples per match. This is why event 0 (event-1 case in the
|
||||
protocol) downloads of >1-sec recordings always failed in BW: the strip
|
||||
destroyed real data at body offset `0x1012..0x102B` and propagated
|
||||
alignment differences through the rest of the body. Sub-1-sec events
|
||||
worked because their `end_offset` was below `0x1002`, so no sample
|
||||
chunks landed in the metadata-page region and the strip's needle never
|
||||
matched. Verified fix by re-feeding the BW 5-1-26 "copy 3sec" capture's
|
||||
A5 frames into the file builder: output is now byte-identical to BW's
|
||||
saved `M529LKIQ.G10` reference (8708 bytes, 0 differences).
|
||||
- BW already concatenates frame contributions in stream order without
|
||||
any de-duplication; SFM now does the same.
|
||||
|
||||
---
|
||||
|
||||
## v0.14.0 — 2026-05-02
|
||||
|
||||
### Changed (major rewrite)
|
||||
|
||||
- **`read_bulk_waveform_stream` — STRT-bounded chunk walk.** Replaces the
|
||||
earlier `0x0400`-step / `max(key4[2:4], 0x0400)` chunk-counter formula,
|
||||
which over-read ~5× past the actual event end into post-event circular-
|
||||
buffer garbage. The new walk:
|
||||
|
||||
1. Probe at `counter = start_offset` (event 1: `0x0000`; event N:
|
||||
`cur_key[2:4]`).
|
||||
2. Parse `end_offset` from the STRT record at `data[17]` of the probe
|
||||
response (`end_key[2:4]` field).
|
||||
3. For event 1 only, read the two fixed metadata pages at counter
|
||||
`0x1002` and `0x1004` — these contain the global session-start
|
||||
compliance setup (Project / Client / User Name / Seis Loc /
|
||||
Extended Notes ASCII strings). Continuation events skip these
|
||||
(BW caches them across the session).
|
||||
4. Walk sample chunks at **`0x0200` increments (NOT `0x0400`)**, bounded
|
||||
by `end_offset` — the loop exits when
|
||||
`next_chunk_counter + 0x0200 > end_offset`.
|
||||
5. Send the proper TERM frame (see new `bulk_waveform_term_v2()`) with
|
||||
`offset_word = end_offset - next_boundary` and
|
||||
`params[2:4] = next_boundary BE`. The TERM response carries the
|
||||
partial last chunk + 26-byte file footer.
|
||||
|
||||
- **New helpers:** `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`
|
||||
and `parse_strt_end_offset(a5_data)` in `minimateplus.framing`.
|
||||
|
||||
- **`stop_after_metadata` / `extra_chunks_after_metadata` kwargs are now
|
||||
no-ops** under the v0.14.x walk. They are retained on the
|
||||
`read_bulk_waveform_stream` signature for backward compatibility but log a
|
||||
DEBUG line when set. The old "scan for `b'Project:'` and stop one chunk
|
||||
later" workaround is obsolete — the loop is deterministically bounded by
|
||||
the STRT-derived `end_offset`.
|
||||
|
||||
- **Project / Client / User Name / Seis Loc string source corrected.**
|
||||
These come from the dedicated metadata pages at counter `0x1002` /
|
||||
`0x1004`, not from "A5 frame 7" of the sample-chunk stream. The
|
||||
earlier "A5 frame 7" claim was an artifact of the broken `0x0400`-step
|
||||
walk where the bad counter formula coincidentally landed sample-chunk
|
||||
fi=7 on top of the 0x1002 metadata page.
|
||||
|
||||
### Verified
|
||||
|
||||
- Three independent BW MITM captures (4-27-26 + 5-1-26 + 5-4-26) confirm
|
||||
the new walk matches BW's behaviour event-for-event.
|
||||
- `end_offset` values verified across 3 events: `0x1ABE` (4-27-26 2-sec),
|
||||
`0x21F2` (5-1-26 3-sec), `0x417E` (5-1-26 event-2).
|
||||
|
||||
### Notes
|
||||
|
||||
- Earlier v0.13.0 / v0.13.1 / v0.13.2 entries describe partial steps along
|
||||
the way (some of the file builder fixes, filename bugs, etc.) that were
|
||||
superseded by the full rewrite. Treat this v0.14.0 entry as the
|
||||
definitive landing point for the corrected SUB 5A protocol.
|
||||
|
||||
---
|
||||
|
||||
## v0.14.1 — 2026-05-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`read_bulk_waveform_stream` — event-N probe counter off-by-`0x46`.**
|
||||
Continuation events (start_key[2:4] != 0) were being probed at counter
|
||||
`start_offset + 0x0046` instead of just `start_offset`. In the iteration
|
||||
walk, `cur_key` from 1F is already the off=0x46 WAVEHDR record key, so the
|
||||
earlier formula effectively double-counted the WAVEHDR offset. The probe
|
||||
landed one WAVEHDR past the actual event start, the response no longer
|
||||
contained the STRT record at byte 17, `parse_strt_end_offset` returned
|
||||
`None`, and the chunk loop fell back to the `max_chunks=128` cap — walking
|
||||
~110 chunks of post-event circular-buffer garbage. Verified against the
|
||||
5-1-26 "copy 2nd address" and 5-4-26 BW 2-sec event captures: BW probes
|
||||
counter=`0x2238` with key=`01112238` and STRT is present at byte 17 of
|
||||
the response (end_offset=`0x417E`).
|
||||
- **CLAUDE.md / docs/instantel_protocol_reference.md** — corrected the
|
||||
event-N section to clarify that `start_key` in those formulas is the
|
||||
off=0x46 key, not the off=0x2C boundary key, and removed the spurious
|
||||
`+0x46` from the chunk-walk pseudocode.
|
||||
|
||||
---
|
||||
|
||||
## v0.13.2 — 2026-05-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`_extract_record_type` — third 0C-record header format ("short", 8 bytes).**
|
||||
A live SFM download against BE11529 produced files named `M5290000.000`
|
||||
(zero-stamped) because the 0C waveform record's first bytes were
|
||||
`01 05 07 ea ...` — neither the 9-byte single-shot layout (`0x10` at byte 1)
|
||||
nor the 10-byte continuous layout (`0x10` at bytes 0 and 2). Investigation
|
||||
showed this is a third format observed in the wild: an 8-byte header with no
|
||||
marker bytes at all (`[day][month][year_BE:2][unknown][hour][min][sec]`).
|
||||
The detection logic now scans the year (uint16 BE) at byte 2 / byte 3 / byte
|
||||
4 and picks whichever offset returns a sensible year (2015–2050) — each
|
||||
format has the year at a unique position so this disambiguates cleanly.
|
||||
- New format → `event.record_type = "Waveform (Short)"`,
|
||||
`Timestamp.from_short_record()`.
|
||||
- Existing single-shot and continuous parsers unchanged.
|
||||
- The user's event from May 1, 2026 13:21:37 now correctly resolves to a
|
||||
filename like `M529LKIQ.G10` instead of `M5290000.000`.
|
||||
|
||||
### Added
|
||||
|
||||
- `Timestamp.from_short_record(data)` — decodes the 8-byte header.
|
||||
- `_detect_record_format(data)` — internal helper returning
|
||||
`"single_shot" / "continuous" / "short" / None` via year-position scan.
|
||||
|
||||
---
|
||||
|
||||
## v0.13.1 — 2026-05-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`_extract_record_type` — Continuous-mode record headers misclassified as Unknown.**
|
||||
In single-shot mode the 0C waveform record's 9-byte header puts the sub_code
|
||||
marker `0x10` at byte 1, with the day at byte 0. In Continuous mode the
|
||||
header is 10 bytes with the marker at byte 0 *and* byte 2, and the day at
|
||||
byte 1. Previous logic only inspected byte 1 and treated any value other
|
||||
than `0x10` / `0x03` as `"Unknown"`, which prevented `event.timestamp` from
|
||||
being populated for any continuous-mode event whose day-of-month wasn't
|
||||
exactly 3 or 16. As a downstream effect, `blastware_filename()` saw
|
||||
`event.timestamp == None`, fell back to `stem="0000"` / `ab="00"`, and
|
||||
produced filenames like `M5290000.000`. Discovered from a live SFM run on
|
||||
BE11529 in continuous mode (day-of-month = 5).
|
||||
Now disambiguates by checking BOTH byte 0 and byte 2: if both are `0x10`,
|
||||
it's the 10-byte continuous header; else if byte 1 is `0x10`, it's the
|
||||
9-byte single-shot header. Day-of-month no longer matters.
|
||||
|
||||
*Superseded by v0.13.2 — the user's actual record uses a third 8-byte format
|
||||
with no `0x10` markers, which v0.13.1 still misclassified.*
|
||||
|
||||
---
|
||||
|
||||
## v0.13.0 — 2026-05-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- **SUB 5A bulk waveform stream — over-read bug for events ≥ 2 sec.**
|
||||
`read_bulk_waveform_stream` was walking the chunk counter past the actual
|
||||
end of the event, picking up post-event circular-buffer garbage that
|
||||
corrupted reconstructed Blastware files for any waveform > ~1 sec. The
|
||||
loop now extracts the event's `end_offset` from the STRT record at
|
||||
`data[23:27]` of the probe response and stops the chunk walk when the next
|
||||
counter would step past it. Verified against three BW MITM captures
|
||||
(4-27-26 + 5-1-26): 2-sec event drops from 37 over-read chunks to 7
|
||||
bounded chunks; 3-sec drops to 9; non-zero-start "event 2" drops to 9.
|
||||
|
||||
### Added
|
||||
|
||||
- `framing.bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)` —
|
||||
computes the corrected SUB 5A TERM frame's `(offset_word, params)` per the
|
||||
formula confirmed across all 3 BW captures. Not yet wired into
|
||||
`read_bulk_waveform_stream` (the legacy TERM is still used to preserve the
|
||||
existing `blastware_file.write_blastware_file` frame-structure expectations);
|
||||
available for the next iteration that switches to BW's 0x0200 chunk step.
|
||||
- `framing.parse_strt_end_offset(a5_data)` — extracts the event-end pointer
|
||||
from the STRT record in an A5 response payload.
|
||||
|
||||
### Documentation
|
||||
|
||||
- **CLAUDE.md and `docs/instantel_protocol_reference.md` extensively
|
||||
rewritten** to reflect the corrected SUB 5A protocol. See:
|
||||
- CLAUDE.md "SUB 5A — chunk counter formula (REWRITTEN 2026-05-01)"
|
||||
- CLAUDE.md "SUB 5A — STRT record encodes end_offset"
|
||||
- CLAUDE.md "SUB 5A — TERM frame formula"
|
||||
- CLAUDE.md "SUB 5A — fixed metadata pages 0x1002 and 0x1004"
|
||||
- CLAUDE.md "SUB 0A — WAVEHDR response length distinguishes events from
|
||||
boundaries" (0x46 = real event, 0x2C = boundary marker)
|
||||
- protocol reference §7.8.5 / §7.8.6 / §7.8.7 / §7.8.8
|
||||
- The previous chunk-counter formula (`max(key4[2:4], 0x0400) + (chunk-1) *
|
||||
0x0400`) is now marked DEPRECATED and explicitly tagged WRONG with
|
||||
pointers to the new sections, so future work doesn't re-derive it.
|
||||
|
||||
### Known minor diffs vs Blastware (deferred to a follow-up)
|
||||
|
||||
- We still use the OLD 0x0400 chunk step rather than BW's 0x0200; switching
|
||||
also requires updating `blastware_file.write_blastware_file`'s skip values
|
||||
and "extra chunk after metadata" logic, which depends on a fresh capture
|
||||
to verify.
|
||||
- We still use the legacy fixed `offset_word=0x005A` TERM frame rather than
|
||||
BW's `end_offset - next_boundary` formula, for the same reason.
|
||||
- Two fixed metadata pages at counter `0x1002` and `0x1004` are not yet
|
||||
read explicitly; under the current 0x0400 walk their content is reachable
|
||||
via the sample chunk that covers buffer addresses `[0x1000, 0x1400)`.
|
||||
|
||||
---
|
||||
|
||||
## v0.12.6 — 2026-05-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`blastware_file.py` — waveform frame classification** — A5 frame classification for
|
||||
waveform-only vs header-only frames now uses `frame.record_type` instead of frame index.
|
||||
Only waveform frames (0x46) are written to the file body; metadata frames are skipped.
|
||||
Fixes spurious data corruption from incorrectly classified frames.
|
||||
|
||||
- **`s3_analyzer.py` — A5/5A frame naming** — Bulk waveform stream frames (SUB 5A response)
|
||||
are now correctly labeled "A5" in analyzer output instead of being conflated with other
|
||||
multi-frame responses (SUB A4, E5, etc.).
|
||||
|
||||
- **`S3FrameParser` — frame terminator detection** — Corrected the bare ETX terminator
|
||||
detection. Frame termination is now correctly identified by a standalone `ETX=0x03` byte,
|
||||
not by the `DLE+ETX` sequence (which is part of the payload when it appears within a frame).
|
||||
|
||||
---
|
||||
|
||||
## v0.12.5 — 2026-04-21
|
||||
|
||||
### Added
|
||||
|
||||
- **`seismo_lab.py` — Download tab** — New fourth tab for live wire-byte capture during event
|
||||
downloads. Captures both BW→device and device→S3 frames in real time, allowing inspection
|
||||
of the 5A bulk stream chunk sequence and frame-by-frame analysis without needing a bridge
|
||||
or MITM proxy. Files are saved with user-specified labels for easy tracking.
|
||||
|
||||
### Changed
|
||||
|
||||
- **`s3_bridge.py` — raw captures always-on by default** — `--raw-bw` and `--raw-s3` now
|
||||
default to `"auto"` instead of `None`. Every bridge session automatically generates
|
||||
timestamped `raw_bw_<ts>.bin` and `raw_s3_<ts>.bin` files alongside the `.bin`/`.log`
|
||||
session files. Pass `--raw-bw ""` (explicit empty string) to disable if needed.
|
||||
|
||||
- **`gui_bridge.py` — raw capture checkboxes pre-checked** — Both "BW→S3 raw" and
|
||||
"S3→BW raw" checkboxes start checked. Path fields are empty by default (bridge auto-names
|
||||
the files). Unchecking a box passes `--raw-bw ""` to explicitly disable capture.
|
||||
|
||||
- **`Bridge tab` — TCP mode added** — Serial/TCP radio toggle allows connection via cellular
|
||||
modem (RV50/RV55) instead of direct RS-232. Supports multi-capture design (simultaneous
|
||||
Bridge + Analyzer + Download sessions).
|
||||
|
||||
- **`ach_server.py` — TX capture added (`raw_tx_<ts>.bin`)** — Every ACH inbound session
|
||||
now saves both directions: `raw_rx_<ts>.bin` (device → us, S3 side, as before) and
|
||||
`raw_tx_<ts>.bin` (us → device, BW side). Both files are usable in the Analyzer.
|
||||
TX bytes are buffered in memory until startup handshake succeeds (same as RX), preventing
|
||||
scanner probes from creating empty files.
|
||||
|
||||
---
|
||||
|
||||
## v0.12.4 — 2026-04-21 (protocol analysis / docs only — no code changes)
|
||||
|
||||
### Discovered
|
||||
|
||||
- **compliance_raw is wire-encoded, not logical bytes** — `read_compliance_config()` returns
|
||||
bytes that include DLE prefix bytes (`0x10`) before any `0x03` values (because S3FrameParser
|
||||
preserves DLE+ETX inner-frame pairs as two literal bytes). The previous CLAUDE.md claim that
|
||||
"S3FrameParser handles this transparently so compliance_raw contains logical bytes" was wrong.
|
||||
|
||||
- **anchor-9 behavior per recording mode** (confirmed from 4-20-26 BW write captures):
|
||||
- Single Shot (0x00) / Continuous (0x01): anchor-9 = `0x00`
|
||||
- Histogram (0x03): anchor-9 = `0x10` — the E5 DLE prefix for the `0x03` recording_mode byte
|
||||
- Histogram+Continuous (0x04): anchor-9 = `0x10` — an actual stored config byte for this mode
|
||||
Anchor position shifts by ±1 when recording_mode = `0x03` due to the extra DLE byte; the
|
||||
dynamic anchor search (`buf.find(ANCHOR, 0, 150)`) handles this correctly without code changes.
|
||||
|
||||
- **Write frame ETX escaping** — BW escapes `0x03` bytes in write frame data as `0x10 0x03`
|
||||
on the wire. Our `build_bw_write_frame` sends data bytes raw without ETX escaping. Device
|
||||
accepts our raw writes for all tested modes. Hypothesis: device write parser uses the
|
||||
offset/length field for frame boundaries, not ETX scanning, making ETX escaping optional.
|
||||
Histogram mode (recording_mode = 0x03) write via SFM from a non-Histogram starting state
|
||||
not yet tested.
|
||||
|
||||
- **BW write payload vs E5 read payload are byte-identical** around the anchor region (confirmed
|
||||
by comparing 3-11-26 BW TX and S3 captures). BW does NOT strip DLE prefix bytes before writing;
|
||||
it round-trips the wire-encoded bytes verbatim with only the modified fields changed.
|
||||
|
||||
- **Capture folder content catalogued** — see CLAUDE.md "BW capture reference" table for a
|
||||
summary of all available protocol captures and their contents.
|
||||
|
||||
---
|
||||
|
||||
## v0.12.3 — 2026-04-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Auto Call Home config protocol** — Full read/write/decode/encode pipeline for the
|
||||
device's Remote Access → Setup Unit ACH settings, confirmed from 4-20-26 call home
|
||||
settings captures.
|
||||
|
||||
**Protocol (new):**
|
||||
- `SUB 0x2C` — Call Home Config READ (response `0xD3`); two-step read; data offset
|
||||
`0x7C` = 124; raw payload 125 bytes (1-byte longer than DATA_LENGTH due to DLE-escaped
|
||||
`\x10\x03` at raw[117:119] representing num_retries = 3)
|
||||
- `SUB 0x7E` — Call Home Config WRITE (response `0x81`); 127-byte payload (125-byte read
|
||||
payload + `\x00\x00`); offset = `data[1]+2 = 0x7E`; write format (DLE-aware checksum)
|
||||
- `SUB 0x7F` — Call Home WRITE CONFIRM (response `0x80`); no data
|
||||
|
||||
**Field map (confirmed from 10-frame BW TX diff):**
|
||||
- `raw[5]` — auto_call_home_enabled (bool)
|
||||
- `raw[6:46]` — dial_string (40-byte null-padded ASCII)
|
||||
- `raw[87]` — after_event_recorded (bool)
|
||||
- `raw[91]` — at_specified_times (bool)
|
||||
- `raw[93]` — time1_enabled / `raw[101]` — time1_hour / `raw[102]` — time1_min
|
||||
- `raw[95]` — time2_enabled / `raw[105]` — time2_hour / `raw[106]` — time2_min
|
||||
- `raw[117:119]` — `\x10\x03` (DLE-escaped 0x03 = num_retries value 3)
|
||||
- `raw[120]` — time_between_retries_sec / `raw[122]` — wait_for_connection_sec / `raw[124]` — warm_up_time_sec
|
||||
|
||||
**Library (`minimateplus/`):**
|
||||
- `models.py` — `CallHomeConfig` dataclass (14 fields; `raw` bytes preserved for
|
||||
round-trip writes)
|
||||
- `protocol.py` — `SUB_CALL_HOME = 0x2C`, `SUB_CALL_HOME_WRITE = 0x7E`,
|
||||
`SUB_CALL_HOME_CONFIRM = 0x7F`; `read_call_home_config()`, `write_call_home_config()`
|
||||
- `client.py` — `get_call_home_config()`, `set_call_home_config()`,
|
||||
`_decode_call_home_config()` (handles DLE prefix at raw[117]),
|
||||
`_encode_call_home_config()` (patches in-place; raises `ValueError` if hour/min = 3)
|
||||
|
||||
**REST API (`sfm/server.py`):**
|
||||
- `GET /device/call_home` — reads and decodes call home config from device
|
||||
- `POST /device/call_home` — reads, patches specified fields, writes back to device
|
||||
- `CallHomeConfigBody` Pydantic model with 9 optional writable fields
|
||||
|
||||
**Web UI (`sfm/sfm_webapp.html`):**
|
||||
- New "Call Home" tab with enable flag, dial string (read-only), after-event trigger,
|
||||
at-specified-times flag, two time slots (enable + HH:MM each), and read-only retry
|
||||
settings (num_retries, time_between_retries_sec, wait_for_connection_sec,
|
||||
warm_up_time_sec)
|
||||
- "Read from Device", "Write to Device", "Clear Form" action buttons
|
||||
- Client-side guard: rejects hour or minute value equal to 3 with a clear message
|
||||
explaining the DLE-encoding limitation
|
||||
|
||||
---
|
||||
|
||||
## v0.12.2 — 2026-04-20
|
||||
|
||||
### Added / Fixed
|
||||
|
||||
- **Geophone sensitivity / maximum range field confirmed** — 4-20-26 geo sensitivity
|
||||
captures (1.25 in/s vs 10 in/s) diffed across all three SUB 71 write chunks and both
|
||||
E5 read payloads. The `geo_range` uint8 field per channel is now fully confirmed:
|
||||
- E5 read offset: `channel_label + 33`; SUB 71 write offset: `channel_label + 29`
|
||||
- `0x00` = Normal 10.000 in/s (standard gain); `0x01` = Sensitive 1.250 in/s (high gain)
|
||||
- **Correction:** previous hypothesis (`channel_label+20`, `0x01`=Normal) was wrong.
|
||||
`channel_label+20` reads `0x01` on ALL captures regardless of range — not this field.
|
||||
- `_decode_compliance_config_into`: read offset corrected from `tran_pos+20` → `tran_pos+33`
|
||||
- `_encode_compliance_config`: added `geo_range` parameter; writes to Tran/Vert/Long at `+29`
|
||||
- `apply_config`: added `geo_range` parameter
|
||||
- `POST /device/config`: added `geo_range` to `DeviceConfigBody`
|
||||
- Web UI Config tab: added "Maximum Range — Geo" select (Normal / Sensitive)
|
||||
- Web UI Device tab: added "Max Range (geo)" row to compliance table
|
||||
|
||||
- **`recording_mode` + `histogram_interval_sec` confirmed and implemented** (4-20-26 captures)
|
||||
- `recording_mode`: uint8 at anchor−8 (E5 read) / anchor−7 (write); enum: 0x00=Single Shot,
|
||||
0x01=Continuous, 0x03=Histogram, 0x04=Histogram+Continuous
|
||||
- `histogram_interval_sec`: uint16 BE seconds at anchor−4; same offset in read & write;
|
||||
valid: 2, 5, 15, 60, 300, 900 (matching Blastware dropdown: 2s, 5s, 15s, 1m, 5m, 15m)
|
||||
- Both fields added to `ComplianceConfig`, `_decode_compliance_config_into`,
|
||||
`_encode_compliance_config`, `apply_config`, REST API body, and web UI
|
||||
|
||||
---
|
||||
|
||||
## v0.12.1 — 2026-04-16
|
||||
|
||||
### Added
|
||||
|
||||
- **`sfm/server.py` — `_LiveCache`** — in-memory live device cache that eliminates
|
||||
redundant TCP round-trips between web requests. Plain Python dict +
|
||||
`threading.Lock`, no extra dependencies.
|
||||
|
||||
Cache strategy per endpoint:
|
||||
|
||||
| Endpoint | Strategy |
|
||||
|---|---|
|
||||
| `GET /device/info` | Indefinite; invalidated by `POST /device/config` |
|
||||
| `GET /device/events` | Count-probe fast path — `poll()+count_events()` (~2 s); returns cached data if event count is unchanged; full download only when new events are detected |
|
||||
| `GET /device/monitor/status` | 30-second TTL; invalidated immediately on monitor start/stop |
|
||||
| `GET /device/event/{idx}/waveform` | Permanent per-index (waveforms are immutable once recorded) |
|
||||
|
||||
- **`?force=true`** query param on all cached endpoints — bypasses cache and forces
|
||||
a fresh read from the device.
|
||||
|
||||
- **Cache invalidation hooks** — `POST /device/config` marks device info and events
|
||||
stale; `POST /device/monitor/start` and `/stop` evict the monitor status entry
|
||||
immediately so the next status poll reflects the actual device state.
|
||||
|
||||
---
|
||||
|
||||
## v0.12.0 — 2026-04-13
|
||||
|
||||
### Added
|
||||
|
||||
- **`sfm/server.py` — `_LiveCache`** — in-memory live device cache, eliminating
|
||||
redundant TCP round-trips between requests. No extra dependencies (plain Python
|
||||
dict + threading.Lock). Replaces the SQLAlchemy-based `sfm/cache.py` experiment
|
||||
from the `feature/intelligent-caching` branch.
|
||||
|
||||
Cache behaviour by endpoint:
|
||||
|
||||
| Endpoint | Cache strategy |
|
||||
|---|---|
|
||||
| `GET /device/info` | Indefinite; invalidated by `POST /device/config` |
|
||||
| `GET /device/events` | Count-probe fast path: quick `poll()+count_events()` (~2s); return cache if count matches; full download only when new events detected |
|
||||
| `GET /device/monitor/status` | 30-second TTL; invalidated by monitor start/stop |
|
||||
| `GET /device/event/{idx}/waveform` | Permanent per-index (waveforms are immutable) |
|
||||
|
||||
- **`?force=true` param** on all four cached endpoints — bypasses cache and re-reads
|
||||
from device.
|
||||
|
||||
- **`POST /device/config` cache invalidation** — marks device info + events dirty so
|
||||
the next read reflects the new compliance config.
|
||||
|
||||
- **`POST /device/monitor/start` / `stop` cache invalidation** — evicts the monitor
|
||||
status cache entry immediately so the next poll returns the updated state.
|
||||
|
||||
### Removed
|
||||
|
||||
- `sfm/cache.py` — SQLAlchemy-based cache from the experimental caching branch.
|
||||
Its logic has been ported to the sqlite3-native `_LiveCache` class above.
|
||||
`sqlalchemy` is no longer a dependency.
|
||||
|
||||
---
|
||||
|
||||
## v0.11.0 — 2026-04-13
|
||||
|
||||
### Added
|
||||
|
||||
- **`sfm/database.py` — SeismoDb** — SQLite persistence layer for all ACH data.
|
||||
Three tables, all unit-keyed by serial number:
|
||||
- `ach_sessions` — one row per inbound call-home: serial, timestamp, peer IP,
|
||||
events_downloaded, monitor_entries, duration_seconds
|
||||
- `events` — one row per triggered waveform event: serial, waveform_key (dedup),
|
||||
timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location
|
||||
strings, sample_rate, record_type, false_trigger flag
|
||||
- `monitor_log` — one row per monitoring interval: serial, waveform_key (dedup),
|
||||
start_time, stop_time, duration_seconds, geo_threshold_ips
|
||||
- WAL mode, per-request connections — safe for the single-writer / occasional-reader
|
||||
ACH server pattern
|
||||
- Deduplication by `(serial, waveform_key)` UNIQUE constraint — re-runs and repeat
|
||||
call-homes never produce duplicate rows
|
||||
|
||||
- **`ach_server.py` — DB integration** — after each successful call-home, writes new
|
||||
events and monitor log entries to `seismo_relay.db` then records the session in
|
||||
`ach_sessions`. DB write failures are logged as warnings and do not abort the session.
|
||||
|
||||
- **`sfm/server.py` — DB read endpoints**:
|
||||
- `GET /db/units` — distinct serials with last_seen, total_events, total_monitor_entries
|
||||
- `GET /db/events` — query events with serial / date range / false_trigger filters
|
||||
- `GET /db/monitor_log` — query monitoring intervals
|
||||
- `GET /db/sessions` — query ACH call-home sessions
|
||||
- `PATCH /db/events/{id}/false_trigger` — flag/unflag false triggers (for review UI)
|
||||
|
||||
### Architecture
|
||||
|
||||
- seismo-relay DB is unit-keyed only — no project concepts. Project aggregation is
|
||||
terra-view's responsibility via `UnitAssignment` / `DeploymentRecord` + date range
|
||||
queries against the SFM DB endpoints.
|
||||
- DB file lives at `bridges/captures/seismo_relay.db` by default.
|
||||
|
||||
---
|
||||
|
||||
## v0.10.0 — 2026-04-11
|
||||
|
||||
### Added
|
||||
|
||||
- **`MiniMateClient.get_monitor_log_entries(skip_keys=None)`** — browse-mode walk
|
||||
(`1E → 0A → 1F`) that collects partial records (`0x2C` record type) from the device's
|
||||
event list without triggering a full waveform download (no 0C or 5A). Returns
|
||||
`list[MonitorLogEntry]`. Each entry represents one continuous monitoring interval where
|
||||
no threshold was exceeded.
|
||||
|
||||
- **`_decode_0a_partial_header(raw_data, index, key4)`** in `client.py` — decodes a SUB
|
||||
0x0A response payload whose record type is `0x2C`. Extracts:
|
||||
- `start_time` / `stop_time` — two consecutive timestamps; auto-detects 9-byte
|
||||
(sub_code=0x10, single-shot) vs 10-byte (sub_code=0x03, continuous) format from
|
||||
`raw_data[11]`. Handles a 1-byte gap between the two timestamps that occurs when
|
||||
ts1 and ts2 share the same minute:second.
|
||||
- `serial` — device serial string found via `b"BE"` anchor scan.
|
||||
- `geo_threshold_ips` — trigger level found via `b"Geo: "` anchor scan.
|
||||
|
||||
- **`MonitorLogEntry` dataclass** in `models.py` — new model for partial records:
|
||||
`index`, `key`, `start_time`, `stop_time`, `serial`, `geo_threshold_ips`,
|
||||
`raw_header`, and a `duration_seconds` property.
|
||||
|
||||
- **`read_waveform_header()` return value extended** — now returns `(data_rsp.data, length)`
|
||||
(full payload) instead of `(data_rsp.data[11:11+length], length)`. Callers get the
|
||||
complete payload including the record-type byte at position 0. Full records use
|
||||
`raw_data[11:11+length]` as before; partial records are detected by `raw_data[0] == 0x2C`.
|
||||
|
||||
- **ACH server: monitor log collection** — after `get_events()`, calls
|
||||
`get_monitor_log_entries(skip_keys=seen_keys)` and saves new entries to
|
||||
`monitor_log.json` in the session directory. Monitor log keys are included in
|
||||
`downloaded_keys` for state persistence (no re-processing on next call-home).
|
||||
|
||||
- **`_monitor_log_entry_to_dict()`** in `ach_server.py` — serialises a `MonitorLogEntry`
|
||||
to a JSON-compatible dict with ISO-format timestamps.
|
||||
|
||||
### Protocol / Documentation
|
||||
|
||||
- **SUB 0x0A partial record (0x2C) format confirmed** (✅ 4-11-26 MITM capture, 12 frames):
|
||||
- Record type `0x2C` at `raw_data[0]`; length < 64 bytes.
|
||||
- Two timestamps at `raw_data[11:]` — start and stop of the monitoring interval.
|
||||
- ASCII metadata region after timestamps: `BE<serial>\x00Geo: <float> in/s`.
|
||||
- Edge case: 1-byte separator between timestamps when ts1 and ts2 share minute:second.
|
||||
- 10-byte timestamp format (sub_code=0x03) signalled by `raw_data[11] == 0x10`.
|
||||
|
||||
- **Key reuse detection for monitor log entries** — monitor log keys are tracked alongside
|
||||
event keys in `ach_state.json` so the ACH server does not re-process them after a
|
||||
call-home cycle.
|
||||
|
||||
---
|
||||
|
||||
## v0.9.0 — 2026-04-11
|
||||
|
||||
### Added
|
||||
|
||||
- **`MiniMateClient.list_event_keys()`** — fast browse-mode walk (1E → 0A → 1F, no waveform
|
||||
download) that returns the list of event key hex strings currently stored on the device.
|
||||
Used by the ACH server as a cheap pre-check before deciding whether to call `get_events()`.
|
||||
|
||||
- **`get_events(skip_waveform_for_keys=set(...))`** — new optional parameter. For any key in
|
||||
the set the function performs only 0A + 1F(browse) instead of the full
|
||||
1E-arm → 0C → POLL×3 → 5A sequence. Eliminates redundant waveform downloads on repeat
|
||||
call-homes when the device still holds previously downloaded events.
|
||||
|
||||
- **`MiniMateClient.delete_all_events()`** — erases all events from device memory using the
|
||||
confirmed 4-step sequence:
|
||||
- SUB 0xA3 `begin_erase_all` — initiate erase (token=0xFE) → ack 0x5C
|
||||
- SUB 0x1C `read_monitor_status` — intermediate status read (Blastware-required)
|
||||
- SUB 0x06 `read_event_storage_range` — verify storage state (token=0xFE) → 36-byte response
|
||||
- SUB 0xA2 `confirm_erase_all` — commit erase (token=0xFE) → ack 0x5D
|
||||
|
||||
All four steps confirmed from 4-11-26 MITM capture of a live Blastware ACH session.
|
||||
After a successful call, the device's event counter resets to `0x01110000`.
|
||||
|
||||
- **`MiniMateProtocol` erase methods**: `begin_erase_all()`, `confirm_erase_all()`,
|
||||
`read_event_storage_range()` added to `protocol.py` with documented SUB constants
|
||||
`SUB_ERASE_ALL_BEGIN = 0xA3` and `SUB_ERASE_ALL_CONFIRM = 0xA2`.
|
||||
|
||||
- **`bridges/ach_mitm.py`** — transparent TCP-to-TCP MITM proxy. Listens for inbound unit
|
||||
connections, connects upstream to a real Blastware ACH server, and saves both directions
|
||||
to `raw_bw_<ts>.bin` / `raw_s3_<ts>.bin` files matching the existing capture format.
|
||||
Used to capture the 4-11-26 Blastware ACH session including event deletion.
|
||||
Usage: `python bridges/ach_mitm.py --bw-host 127.0.0.1 --bw-port 9999 --listen-port 9998`
|
||||
|
||||
- **ACH server: key-based state tracking** — `ach_state.json` now stores
|
||||
`downloaded_keys: [hex_strings]` and `max_downloaded_key: hex_string` per unit instead of
|
||||
`event_count: N`. This correctly handles the standard workflow where events are deleted
|
||||
from the device after upload — a count-based approach would see `count=0` on the next
|
||||
call-home and silently skip new events.
|
||||
|
||||
- **ACH server: `--clear-after-download` flag** — after a successful download (at least one
|
||||
new event saved), erases all events from the device using `delete_all_events()`. Mirrors
|
||||
the standard Blastware ACH workflow. On success, `downloaded_keys` and
|
||||
`max_downloaded_key` are reset to empty so the next session starts fresh.
|
||||
|
||||
- **ACH server: post-erase key-reuse detection** — after an external erase (Blastware or
|
||||
manual), device keys restart from `0x01110000`, colliding with previously downloaded keys.
|
||||
On each browse walk, if `max(device_keys) < max_downloaded_key` (device counter rolled
|
||||
back), all device keys are treated as new regardless of `seen_keys`. This also catches
|
||||
erases performed by Blastware between our sessions.
|
||||
|
||||
### Protocol / Documentation
|
||||
|
||||
- **SUB 0xA3 / SUB 0xA2 — erase-all sequence confirmed** (✅ 4-11-26 MITM capture):
|
||||
Both frames use `token=0xFE` at `params[7]` and are standard `build_bw_frame` requests
|
||||
(not write-format). Response SUBs follow the standard formula: 0x5C and 0x5D.
|
||||
The intermediate 0x1C + 0x06 reads between them are required by Blastware.
|
||||
|
||||
- **SUB 0x06 — event storage range read confirmed** (✅ 4-11-26 MITM capture):
|
||||
Two-step read, data offset = 0x24 (36 bytes). The last 8 bytes of the response contain
|
||||
the first and last stored event keys (4 bytes each). After a successful erase, both keys
|
||||
read as `01110000` (device-empty state).
|
||||
|
||||
- **Event key counter resets to `0x01110000` after erase** — confirmed by observing key
|
||||
`01110000` on the device immediately after the MITM erase session.
|
||||
|
||||
---
|
||||
|
||||
## v0.8.0 — 2026-04-07
|
||||
|
||||
### Added
|
||||
|
||||
- **Write pipeline end-to-end** — `push_config_raw(event_index_data, compliance_data,
|
||||
trigger_data, waveform_data)` on `MiniMateClient` orchestrates the full
|
||||
`68→73 | 71×3→72 | 82→83 | 69→74→72` write sequence.
|
||||
|
||||
- **`build_bw_write_frame(sub, data, *, offset, params)`** in `framing.py` — dedicated frame
|
||||
builder for write commands (SUBs 0x68–0x83). Doubles only the BW_CMD byte; all other
|
||||
bytes including offset, params, data, and checksum are written raw. Uses the large-frame
|
||||
DLE-aware checksum (`sum(b for b in payload[2:] if b != 0x10) + 0x10) & 0xFF`).
|
||||
|
||||
- **`MiniMateProtocol` write methods** — `write_event_index()`, `write_compliance()`,
|
||||
`write_trigger_config()`, `write_waveform_data()`, `write_confirm()`,
|
||||
`start_monitoring()`, `stop_monitoring()`.
|
||||
|
||||
- **`AchSession` inbound server** (`bridges/ach_server.py`) — accepts call-home TCP
|
||||
connections, runs the full handshake + device-info + event-download sequence, saves
|
||||
`device_info.json` + `events.json` per session.
|
||||
|
||||
### Protocol / Documentation
|
||||
|
||||
- **Write frame format confirmed** (✅ 3-11-26 BW TX capture, all 11 frames): only BW_CMD
|
||||
byte `0x10` is doubled; all other bytes sent raw. Standard `build_bw_frame` DLE-stuffing
|
||||
is incorrect for write commands.
|
||||
- **Write ack responses** confirmed as 17-byte zero-data S3 frames.
|
||||
- **Monitoring SUBs 0x96/0x97** confirmed from 4-8-26 capture.
|
||||
- **SESSION_RESET signal** (`41 03`) required before POLL for monitoring units.
|
||||
- **SUB 0x1C monitoring flag** at `section[1]`: `0x00` = idle, `0x10` = monitoring.
|
||||
Confirmed by byte-diff of all 144 data frames in 4-8-26/2ndtry capture.
|
||||
|
||||
---
|
||||
|
||||
## v0.7.0 — 2026-04-03
|
||||
|
||||
### Added
|
||||
- **Raw ADC waveform decode — `_decode_a5_waveform(frames_data, event)`** in `client.py`.
|
||||
Parses the complete set of SUB 5A A5 response frames into per-channel time-series:
|
||||
- Reads the STRT record from A5[0] (bytes 7+): extracts `total_samples` (BE uint16 at +8),
|
||||
`pretrig_samples` (BE uint16 at +16), and `rectime_seconds` (uint8 at +18) into
|
||||
`event.total_samples / pretrig_samples / rectime_seconds`.
|
||||
- Skips the 6-byte preamble (`00 00 ff ff ff ff`) that follows the 21-byte STRT header;
|
||||
waveform data begins at `strt_pos + 27`.
|
||||
- Strips the 8-byte per-frame counter header from A5[1–6, 8] before appending waveform bytes.
|
||||
- Skips A5[7] (metadata-only) and A5[9] (terminator).
|
||||
- **Cross-frame alignment correction**: accumulates `running_offset % 8` across all frames
|
||||
and discards `(8 − align) % 8` leading bytes per frame to re-align to a T/V/L/M boundary.
|
||||
Required because individual frame waveform payloads are not always multiples of 8 bytes.
|
||||
- Decodes as 4-channel interleaved signed 16-bit LE at 8 bytes per sample-set:
|
||||
bytes 0–1 = Tran, 2–3 = Vert, 4–5 = Long, 6–7 = Mic.
|
||||
- Stores result in `event.raw_samples = {"Tran": [...], "Vert": [...], "Long": [...], "Mic": [...]}`.
|
||||
- **`download_waveform(event)` public method** on `MiniMateClient`.
|
||||
Issues a full SUB 5A stream with `stop_after_metadata=False`, then calls
|
||||
`_decode_a5_waveform()` to populate `event.raw_samples` and `event.total_samples /
|
||||
pretrig_samples / rectime_seconds`. Previously only metadata frames were fetched during
|
||||
`get_events()`; raw waveform data is now available on demand.
|
||||
- **`Event` model new fields** (`models.py`): `total_samples`, `pretrig_samples`,
|
||||
`rectime_seconds` (from STRT record), and `_waveform_key` (4-byte key stored during
|
||||
`get_events()` for later use by `download_waveform()`).
|
||||
|
||||
### Protocol / Documentation
|
||||
- **SUB 5A A5[0] STRT record layout confirmed** (✅ 2026-04-03, 4-2-26 blast capture):
|
||||
- STRT header is 21 bytes: `b"STRT"` + length fields + `total_samples` (BE uint16 at +8) +
|
||||
`pretrig_samples` (BE uint16 at +16) + `rectime_seconds` (uint8 at +18).
|
||||
- Followed by 6-byte preamble: `00 00 ff ff ff ff`. Waveform begins at `strt_pos + 27`.
|
||||
- Confirmed: 4-2-26 blast → `total_samples=9306`, `pretrig_samples=298`, `rectime_seconds=70`.
|
||||
- **Blast/waveform mode A5 format confirmed** (✅ 2026-04-03, 4-2-26 blast capture):
|
||||
4-channel interleaved int16 LE at 8 bytes per sample-set; cross-frame alignment correction
|
||||
required. 948 of 9306 total sample-sets captured via `stop_after_metadata=True` (10 frames).
|
||||
- **Noise/histogram mode A5 format — endianness corrected** (✅ 2026-04-03, 3-31-26 capture):
|
||||
32-byte block samples are signed 16-bit **little-endian** (previously documented as BE).
|
||||
`0a 00` → LE int16 = 10 (correct noise floor); BE would give 2560 (wrong).
|
||||
- Protocol reference §7.6 rewritten — split into §7.6.1 (Blast/Waveform mode) and §7.6.2
|
||||
(Noise/Histogram mode), each with confirmed field layouts and open questions noted.
|
||||
|
||||
---
|
||||
|
||||
## v0.6.0 — 2026-04-02
|
||||
|
||||
### Added
|
||||
- **True event-time metadata via SUB 5A bulk waveform stream** — `get_events()` now issues a SUB 5A request after each SUB 0C download, reads the A5 response frames, and extracts the `Client:`, `User Name:`, and `Seis Loc:` fields as they existed at the moment the event was recorded. Previously these fields were backfilled from the current compliance config (SUB 1A), which reflects today's setup, not the setup active when the event triggered.
|
||||
- `build_5a_frame(offset_word, raw_params)` in `framing.py` — reproduces Blastware's exact wire format for SUB 5A requests: raw (non-DLE-stuffed) `offset_hi`, DLE-stuffed params, and a DLE-aware checksum where `10 XX` pairs count only `XX`.
|
||||
- `bulk_waveform_params()` returns 11 bytes (extra trailing `0x00` confirmed from 1-2-26 BW wire capture).
|
||||
- `read_bulk_waveform_stream(key4, *, stop_after_metadata=True, max_chunks=32)` in `protocol.py` — loops sending chunk requests (counter increments `0x0400` per chunk), stops early when `b"Project:"` is found, then sends a termination frame.
|
||||
- `_decode_a5_metadata_into(frames_data, event)` in `client.py` — needle-searches A5 frame data for `Project:`, `Client:`, `User Name:`, `Seis Loc:`, `Extended Notes` and overwrites `event.project_info`.
|
||||
- **`get_events()` sequence extended** — now `1E → 0A → 0C → 5A → 1F` per event.
|
||||
|
||||
### Fixed
|
||||
- **Compliance config (SUB 1A) channel block missing** — orphaned `self._send(build_bw_frame(SUB_COMPLIANCE, 0x2A, _DATA_PARAMS))` before the B/C/D receive loop had no corresponding `recv_one()`, shifting all subsequent receives one step behind and leaving frame D's channel-block data (trigger_level_geo, alarm_level_geo, max_range_geo) unread. Removed the orphaned send. Total config bytes received now correctly ~2126 (was ~1071).
|
||||
- **Compliance config anchor search range** — `_decode_compliance_config_into()` searched `cfg[40:100]` for the sample-rate/record-time anchor. With the orphaned-send bug fixed the 44-byte padding it had been adding is gone, and the anchor now appears at `cfg[11]`. Search widened to `cfg[0:150]` to be robust to future layout shifts.
|
||||
- Removed byte-content deduplication from `read_compliance_config()` — was masking the real receive-ordering bug.
|
||||
|
||||
### Protocol / Documentation
|
||||
- **SUB 5A frame format confirmed** — `offset_hi` byte (`0x10`) must be sent raw (not DLE-stuffed); checksum is DLE-aware (only the second byte of a `10 XX` pair is summed). Standard `build_bw_frame` DLE-stuffs `0x10` incorrectly for 5A — a dedicated `build_5a_frame` is required.
|
||||
- **Event-time metadata source confirmed** — `Client:`, `User Name:`, and `Seis Loc:` strings are present in A5 frame 7 of the bulk waveform stream (SUB 5A), not in the 210-byte SUB 0C waveform record. They reflect the compliance setup as it was when the event was stored on the device.
|
||||
|
||||
---
|
||||
|
||||
## v0.5.0 — 2026-03-31
|
||||
|
||||
### Added
|
||||
|
||||
@@ -1,14 +1,24 @@
|
||||
# seismo-relay `v0.5.0`
|
||||
# seismo-relay `v0.15.0`
|
||||
|
||||
A ground-up replacement for **Blastware** — Instantel's aging Windows-only
|
||||
software for managing MiniMate Plus seismographs.
|
||||
|
||||
Built in Python. Runs on Windows. Connects to instruments over direct RS-232
|
||||
or cellular modem (Sierra Wireless RV50 / RV55).
|
||||
Built in Python. Runs on Windows, Linux, or macOS. Connects to instruments
|
||||
over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
|
||||
|
||||
> **Status:** Active development. Core read pipeline working (device info,
|
||||
> config, event index). Event download and write commands in progress.
|
||||
> See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
> **Status:** Active development. Full read + write + erase + monitoring
|
||||
> pipeline working end-to-end over TCP/cellular. ACH Auto Call Home server
|
||||
> handles inbound unit connections, downloads events, and persists everything
|
||||
> to a SQLite database. SFM REST API exposes device control and DB queries.
|
||||
> **As of v0.14.3 (2026-05-05): SUB 5A bulk waveform protocol is verified
|
||||
> byte-perfect against Blastware captures across 2-sec, 3-sec, and 10-sec
|
||||
> events.** Generated `.G10` / `.AB0` files open cleanly in Blastware with
|
||||
> full Event Reports, frequency analysis, and waveform plots.
|
||||
> **v0.15.0 (2026-05-07)** adds layered per-event storage (BW binary +
|
||||
> raw 5A pickle + HDF5 + `.sfm.json` sidecar), a plot-ready
|
||||
> `sfm.plot.v1` JSON shape with server-side ADC-to-physical-units
|
||||
> conversion, and a BW-file importer for ingesting externally-produced
|
||||
> events. See [CHANGELOG.md](CHANGELOG.md) for full version history.
|
||||
|
||||
---
|
||||
|
||||
@@ -16,29 +26,32 @@ or cellular modem (Sierra Wireless RV50 / RV55).
|
||||
|
||||
```
|
||||
seismo-relay/
|
||||
├── seismo_lab.py ← Main GUI (Bridge + Analyzer + Console tabs)
|
||||
├── seismo_lab.py ← Main GUI (Bridge + Analyzer + Download + Console tabs)
|
||||
│
|
||||
├── minimateplus/ ← MiniMate Plus client library
|
||||
│ ├── transport.py ← SerialTransport and TcpTransport
|
||||
│ ├── protocol.py ← DLE frame layer (read/write/parse)
|
||||
│ ├── client.py ← High-level client (connect, get_config, etc.)
|
||||
│ ├── framing.py ← Frame builder/parser primitives
|
||||
│ └── models.py ← DeviceInfo, EventRecord, etc.
|
||||
│ ├── transport.py ← SerialTransport, TcpTransport, SocketTransport
|
||||
│ ├── protocol.py ← DLE frame layer, SUB command dispatch
|
||||
│ ├── client.py ← High-level client (connect, get_events, delete_all_events, push_config, get_call_home_config, …)
|
||||
│ ├── framing.py ← Frame builders, DLE codec, S3FrameParser
|
||||
│ ├── models.py ← DeviceInfo, Event, ComplianceConfig, MonitorLogEntry, CallHomeConfig, …
|
||||
│ └── blastware_file.py ← Write events to Blastware-compatible .AB0 files
|
||||
│
|
||||
├── sfm/ ← SFM REST API server (FastAPI)
|
||||
│ └── server.py ← /device/info, /device/events, /device/event
|
||||
├── sfm/ ← SFM REST API server (FastAPI, port 8200)
|
||||
│ ├── server.py ← Live device endpoints + DB query endpoints + caching
|
||||
│ ├── database.py ← SeismoDb — SQLite persistence (events, monitor_log, ach_sessions, sessions table)
|
||||
│ └── sfm_webapp.html ← Embedded web UI with Call Home config tab
|
||||
│
|
||||
├── bridges/
|
||||
│ ├── s3-bridge/
|
||||
│ │ └── s3_bridge.py ← RS-232 serial bridge (capture tool)
|
||||
│ ├── ach_server.py ← Inbound ACH call-home server (main production server)
|
||||
│ ├── ach_mitm.py ← Transparent MITM proxy for capturing BW sessions
|
||||
│ ├── s3-bridge/ ← RS-232 serial bridge (capture tool)
|
||||
│ ├── tcp_serial_bridge.py ← Local TCP↔serial bridge (bench testing)
|
||||
│ ├── gui_bridge.py ← Standalone bridge GUI (legacy)
|
||||
│ ├── gui_bridge.py ← Standalone bridge GUI with raw capture checkboxes
|
||||
│ └── raw_capture.py ← Simple raw capture tool
|
||||
│
|
||||
├── parsers/
|
||||
│ ├── s3_parser.py ← DLE frame extractor
|
||||
│ ├── s3_analyzer.py ← Session parser, differ, Claude export
|
||||
│ ├── gui_analyzer.py ← Standalone analyzer GUI (legacy)
|
||||
│ ├── gui_analyzer.py ← Standalone analyzer GUI
|
||||
│ └── frame_db.py ← SQLite frame database
|
||||
│
|
||||
└── docs/
|
||||
@@ -49,123 +62,95 @@ seismo-relay/
|
||||
|
||||
## Quick start
|
||||
|
||||
### Seismo Lab (main GUI)
|
||||
### ACH inbound server (production)
|
||||
|
||||
The all-in-one tool. Three tabs: **Bridge**, **Analyzer**, **Console**.
|
||||
Listens for inbound unit call-homes, downloads all new events and monitor log
|
||||
entries, and writes everything to `bridges/captures/seismo_relay.db`.
|
||||
|
||||
```bash
|
||||
python bridges/ach_server.py --port 12345 --output bridges/captures/
|
||||
```
|
||||
python seismo_lab.py
|
||||
|
||||
Point the unit's ACEmanager **Remote Host** to this machine's IP and **Remote Port** to `12345`.
|
||||
|
||||
Options:
|
||||
```
|
||||
--port N Listen port (default 12345)
|
||||
--output DIR Capture directory (default bridges/captures/)
|
||||
--allow-ip IP Allowlist an IP (repeat for multiple; default: accept all)
|
||||
--max-events N Safety cap for first run (default: unlimited)
|
||||
--clear-after-download Erase device memory after successful download
|
||||
--verbose Debug logging
|
||||
```
|
||||
|
||||
### SFM REST server
|
||||
|
||||
Exposes MiniMate Plus commands as a REST API for integration with other systems.
|
||||
Exposes device control and DB queries as a REST API. Proxied by terra-view.
|
||||
|
||||
```
|
||||
cd sfm
|
||||
uvicorn server:app --reload
|
||||
```bash
|
||||
python sfm/server.py # default: 0.0.0.0:8200
|
||||
python -m uvicorn sfm.server:app --host 0.0.0.0 --port 8200 --reload
|
||||
```
|
||||
|
||||
**Endpoints:**
|
||||
Open `http://localhost:8200` for the embedded web UI, or `http://localhost:8200/docs`
|
||||
for the interactive API docs.
|
||||
|
||||
### Seismo Lab GUI
|
||||
|
||||
```bash
|
||||
python seismo_lab.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SFM REST API
|
||||
|
||||
### Live device endpoints
|
||||
|
||||
Each call dials the device, does its work, and closes the connection. TCP
|
||||
connections are retried once on `ProtocolError` to handle cold-boot timing.
|
||||
|
||||
**In-memory caching** — frequently-polled endpoints avoid redundant TCP round-trips
|
||||
via a thread-safe `_LiveCache` (plain Python dict + `threading.Lock`):
|
||||
|
||||
| Method | URL | Cache Strategy |
|
||||
|--------|-----|---|
|
||||
| `GET` | `/device/info` | Indefinite; invalidated by `POST /device/config` |
|
||||
| `GET` | `/device/events` | Count-probe fast path (~2s); full download only when new events detected |
|
||||
| `GET` | `/device/event/{idx}/waveform` | Permanent per event index |
|
||||
| `GET` | `/device/monitor/status` | 30-second TTL; invalidated by monitor start/stop |
|
||||
| `GET` | `/device/call_home` | Fresh read from device (not cached) |
|
||||
| `POST` | `/device/connect` | — |
|
||||
| `POST` | `/device/config` | Writes compliance config; invalidates info + events cache |
|
||||
| `POST` | `/device/config/project` | Patches project/client/operator/sensor_location strings |
|
||||
| `POST` | `/device/monitor/start` | Sends SUB 0x96; immediately evicts status cache |
|
||||
| `POST` | `/device/monitor/stop` | Sends SUB 0x97; immediately evicts status cache |
|
||||
| `POST` | `/device/call_home` | Reads, patches specified fields, writes back to device |
|
||||
|
||||
**Cache bypass** — All cached endpoints accept `?force=true` to skip the cache and
|
||||
force a fresh read from the device.
|
||||
|
||||
**Cache stats** — `GET /cache/stats` returns hit/miss counts and TTL info; `DELETE /cache/device`
|
||||
clears the device cache immediately.
|
||||
|
||||
Transport query params (supply one set):
|
||||
```
|
||||
Serial: ?port=COM5&baud=38400
|
||||
TCP: ?host=1.2.3.4&tcp_port=12345
|
||||
```
|
||||
|
||||
### DB read endpoints
|
||||
|
||||
Query the SQLite database written by `ach_server.py`. All read-only except
|
||||
`PATCH /db/events/{id}/false_trigger`.
|
||||
|
||||
| Method | URL | Description |
|
||||
|--------|-----|-------------|
|
||||
| `GET` | `/device/info?port=COM5` | Device info via serial |
|
||||
| `GET` | `/device/info?host=1.2.3.4&tcp_port=9034` | Device info via cellular modem |
|
||||
| `GET` | `/device/events?port=COM5` | Event index |
|
||||
| `GET` | `/device/event?port=COM5&index=0` | Single event record |
|
||||
|
||||
---
|
||||
|
||||
## Seismo Lab tabs
|
||||
|
||||
### Bridge tab
|
||||
|
||||
Captures live RS-232 traffic between Blastware and the seismograph. Sits in
|
||||
the middle as a transparent pass-through while logging everything to disk.
|
||||
|
||||
```
|
||||
Blastware → COM4 (virtual) ↔ s3_bridge ↔ COM5 (physical) → MiniMate Plus
|
||||
```
|
||||
|
||||
Set your COM ports and log directory, then hit **Start Bridge**. Use
|
||||
**Add Mark** to annotate the capture at specific moments (e.g. "changed
|
||||
trigger level"). When the bridge starts, the Analyzer tab automatically wires
|
||||
up to the live files and starts updating in real time.
|
||||
|
||||
### Analyzer tab
|
||||
|
||||
Parses raw captures into DLE-framed protocol sessions, diffs consecutive
|
||||
sessions to show exactly which bytes changed, and lets you query across all
|
||||
historical captures via the built-in SQLite database.
|
||||
|
||||
- **Inventory** — all frames in a session, click to drill in
|
||||
- **Hex Dump** — full payload hex dump with changed-byte annotations
|
||||
- **Diff** — byte-level before/after diff between sessions
|
||||
- **Full Report** — plain text session report
|
||||
- **Query DB** — search across all captures by SUB, direction, or byte value
|
||||
|
||||
Use **Export for Claude** to generate a self-contained `.md` report for
|
||||
AI-assisted field mapping.
|
||||
|
||||
### Console tab
|
||||
|
||||
Direct connection to a MiniMate Plus — no bridge, no Blastware. Useful for
|
||||
diagnosing field units over cellular without a full capture session.
|
||||
|
||||
**Connection:** choose Serial (COM port + baud) or TCP (IP + port for
|
||||
cellular modem).
|
||||
|
||||
**Commands:**
|
||||
| Button | What it does |
|
||||
|--------|-------------|
|
||||
| POLL | Startup handshake — confirms unit is alive and identifies model |
|
||||
| Serial # | Reads unit serial number |
|
||||
| Full Config | Reads full 166-byte config block (firmware version, channel scales, etc.) |
|
||||
| Event Index | Reads stored event list |
|
||||
|
||||
Output is colour-coded: TX in blue, raw RX bytes in teal, decoded fields in
|
||||
green, errors in red. **Save Log** writes a timestamped `.log` file to
|
||||
`bridges/captures/`. **Send to Analyzer** injects the captured bytes into the
|
||||
Analyzer tab for deeper inspection.
|
||||
|
||||
---
|
||||
|
||||
## Connecting over cellular (RV50 / RV55 modems)
|
||||
|
||||
Field units connect via Sierra Wireless RV50 or RV55 cellular modems. Use
|
||||
TCP mode in the Console or SFM:
|
||||
|
||||
```
|
||||
# Console tab
|
||||
Transport: TCP
|
||||
Host: <modem public IP>
|
||||
Port: 9034 ← Device Port in ACEmanager (call-up mode)
|
||||
```
|
||||
|
||||
```python
|
||||
# In code
|
||||
from minimateplus.transport import TcpTransport
|
||||
from minimateplus.client import MiniMateClient
|
||||
|
||||
client = MiniMateClient(transport=TcpTransport("1.2.3.4", 9034))
|
||||
info = client.connect()
|
||||
```
|
||||
|
||||
### Required ACEmanager settings (Serial tab)
|
||||
|
||||
These must match exactly — a single wrong setting causes the unit to beep
|
||||
on connect but never respond:
|
||||
|
||||
| Setting | Value | Why |
|
||||
|---------|-------|-----|
|
||||
| Configure Serial Port | `38400,8N1` | Must match MiniMate baud rate |
|
||||
| Flow Control | `None` | Hardware flow control blocks unit TX if pins unconnected |
|
||||
| **Quiet Mode** | **Enable** | **Critical.** Disabled → modem injects `RING`/`CONNECT` onto serial line, corrupting the S3 handshake |
|
||||
| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency; `5` works but is sluggish |
|
||||
| TCP Connect Response Delay | `0` | Non-zero silently drops the first POLL frame |
|
||||
| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect |
|
||||
| DB9 Serial Echo | `Disable` | Echo corrupts the data stream |
|
||||
| `GET` | `/db/units` | All known serials with summary stats |
|
||||
| `GET` | `/db/events` | Triggered events (filter by serial, date range, false_trigger) |
|
||||
| `GET` | `/db/monitor_log` | Monitoring intervals |
|
||||
| `GET` | `/db/sessions` | ACH call-home session history |
|
||||
| `PATCH` | `/db/events/{id}/false_trigger?value=true` | Flag / unflag false triggers |
|
||||
|
||||
---
|
||||
|
||||
@@ -173,21 +158,96 @@ on connect but never respond:
|
||||
|
||||
```python
|
||||
from minimateplus import MiniMateClient
|
||||
from minimateplus.transport import SerialTransport, TcpTransport
|
||||
from minimateplus.transport import TcpTransport
|
||||
|
||||
# Serial
|
||||
client = MiniMateClient(port="COM5")
|
||||
|
||||
# TCP (cellular modem)
|
||||
client = MiniMateClient(transport=TcpTransport("1.2.3.4", 9034), timeout=30.0)
|
||||
client = MiniMateClient(transport=TcpTransport("1.2.3.4", 12345), timeout=30.0)
|
||||
|
||||
with client:
|
||||
info = client.connect() # DeviceInfo — model, serial, firmware
|
||||
serial = client.get_serial() # Serial number string
|
||||
config = client.get_config() # Full config block (bytes)
|
||||
events = client.get_events() # Event index
|
||||
# Read
|
||||
info = client.connect() # DeviceInfo — serial, firmware, compliance config
|
||||
count = client.count_events() # Number of stored events
|
||||
keys = client.list_event_keys() # Fast browse walk — event keys only, no download
|
||||
events = client.get_events() # Full download: headers + peaks + metadata
|
||||
monitor = client.get_monitor_status() # Battery, memory, is_monitoring flag
|
||||
log = client.get_monitor_log_entries() # Monitoring intervals (partial 0x2C records)
|
||||
ach_cfg = client.get_call_home_config() # Auto Call Home settings (SUB 0x2C)
|
||||
|
||||
# Write
|
||||
client.apply_config(
|
||||
sample_rate=1024,
|
||||
recording_mode="Continuous", # Single Shot / Continuous / Histogram / Histogram+Continuous
|
||||
histogram_interval_sec=15, # 2, 5, 15, 60, 300, 900
|
||||
trigger_level_geo=0.5,
|
||||
geo_range="Normal", # Normal (10.000 in/s) / Sensitive (1.25 in/s)
|
||||
project="Bridge Inspection 2026",
|
||||
client_name="City of Portland",
|
||||
operator="B. Harrison",
|
||||
)
|
||||
|
||||
client.set_call_home_config(
|
||||
auto_call_home_enabled=True,
|
||||
after_event_recorded=True,
|
||||
at_specified_times=True,
|
||||
time1_hour=18, time1_min=30, # 6:30 PM
|
||||
time2_hour=6, time2_min=0, # 6:00 AM
|
||||
)
|
||||
|
||||
# Control
|
||||
client.start_monitoring() # SUB 0x96
|
||||
client.stop_monitoring() # SUB 0x97
|
||||
client.delete_all_events() # Erase all (SUB 0xA3 → 0x1C → 0x06 → 0xA2)
|
||||
```
|
||||
|
||||
`get_events()` runs the full per-event sequence:
|
||||
`1E → 0A → 1E(arm token=0xFE) → 0C → 1F(arm) → POLL×3 → 5A → 1F(browse)`.
|
||||
SUB 5A bulk stream walks chunks bounded by the `end_offset` extracted from
|
||||
the STRT record at byte 17 of the probe response — no over-reading, no
|
||||
chunk-count cap. Project / client / operator / sensor location strings come
|
||||
from the dedicated metadata pages at counter `0x1002` and `0x1004`,
|
||||
read once per session (they reflect the compliance setup at session start,
|
||||
not per individual event).
|
||||
|
||||
---
|
||||
|
||||
## Database
|
||||
|
||||
`ach_server.py` writes to `bridges/captures/seismo_relay.db` (SQLite, WAL mode) using the
|
||||
`SeismoDb` persistence layer. Four tables, all unit-keyed by serial number:
|
||||
|
||||
| Table | Key | Contents |
|
||||
|-------|-----|----------|
|
||||
| `ach_sessions` | UUID | Per-call-home audit record: serial, timestamp, peer IP, events_downloaded, monitor_entries, duration_seconds |
|
||||
| `events` | UUID, UNIQUE(serial, waveform_key) | Triggered events: timestamp, Tran/Vert/Long/VectorSum/Mic PPV, project/client/operator/sensor_location strings, sample_rate, record_type, false_trigger flag |
|
||||
| `monitor_log` | UUID, UNIQUE(serial, waveform_key) | Monitoring intervals: serial, waveform_key, start_time, stop_time, duration_seconds, geo_threshold_ips |
|
||||
| `events.false_trigger` | Boolean flag | PATCH endpoint to mark/unmark false triggers for review |
|
||||
|
||||
Deduplication is by `(serial, waveform_key)` — repeat call-homes or re-runs never
|
||||
produce duplicate rows. Post-erase key reuse is handled automatically via the
|
||||
high-water mark in `ach_state.json`. Key-based state tracking allows correct
|
||||
handling of device erasures (external or post-download).
|
||||
|
||||
---
|
||||
|
||||
## Connecting over cellular (RV50 / RV55)
|
||||
|
||||
Field units connect via Sierra Wireless RV50 or RV55 cellular modems.
|
||||
|
||||
### Required ACEmanager settings
|
||||
|
||||
| Setting | Value | Why |
|
||||
|---------|-------|-----|
|
||||
| Configure Serial Port | `38400,8N1` | Must match MiniMate baud rate |
|
||||
| Flow Control | `None` | Hardware FC blocks TX if pins unconnected |
|
||||
| **Quiet Mode** | **Enable** | **Critical** — disabled injects `RING`/`CONNECT` onto serial, corrupting the S3 handshake |
|
||||
| Data Forwarding Timeout | `1` (= 0.1 s) | Lower latency |
|
||||
| TCP Connect Response Delay | `0` | Non-zero silently drops the first POLL frame |
|
||||
| TCP Idle Timeout | `2` (minutes) | Prevents premature disconnect |
|
||||
| DB9 Serial Echo | `Disable` | Echo corrupts the data stream |
|
||||
|
||||
---
|
||||
|
||||
## Protocol quick-reference
|
||||
@@ -197,55 +257,109 @@ with client:
|
||||
| DLE | `0x10` | Data Link Escape |
|
||||
| STX | `0x02` | Start of frame |
|
||||
| ETX | `0x03` | End of frame |
|
||||
| ACK | `0x41` (`'A'`) | Frame-start marker sent before every frame |
|
||||
| ACK | `0x41` | Frame-start marker sent before every BW frame |
|
||||
| DLE stuffing | `10 10` on wire | Literal `0x10` in payload |
|
||||
|
||||
**S3-side frame** (seismograph → Blastware): `ACK DLE+STX [payload] CHK DLE+ETX`
|
||||
|
||||
**De-stuffed payload header:**
|
||||
```
|
||||
[0] CMD 0x10 = BW request, 0x00 = S3 response
|
||||
[1] ? unknown (0x00 BW / 0x10 S3)
|
||||
[2] SUB Command/response identifier ← the key field
|
||||
[3] PAGE_HI Page address high byte
|
||||
[4] PAGE_LO Page address low byte
|
||||
[5+] DATA Payload content
|
||||
```
|
||||
|
||||
**Response SUB rule:** `response_SUB = 0xFF - request_SUB`
|
||||
Example: request SUB `0x08` (Event Index) → response SUB `0xF7`
|
||||
**Response SUB rule:** `response_SUB = 0xFF - request_SUB` (no exceptions)
|
||||
|
||||
Full protocol documentation: [`docs/instantel_protocol_reference.md`](docs/instantel_protocol_reference.md)
|
||||
|
||||
---
|
||||
|
||||
## Compliance Config Features
|
||||
|
||||
The REST API and web UI expose full control over device compliance settings:
|
||||
|
||||
- **Recording Mode** (Single Shot / Continuous / Histogram / Histogram+Continuous)
|
||||
- **Sample Rate** (1024 / 2048 / 4096 sps)
|
||||
- **Record Time** (float, seconds)
|
||||
- **Histogram Interval** (2s, 5s, 15s, 1m, 5m, 15m) — when recording mode includes histogram
|
||||
- **Geo Trigger Levels** (float, in/s per channel)
|
||||
- **Geo Maximum Range** (Normal 10.000 in/s / Sensitive 1.250 in/s per channel)
|
||||
- **Project / Client / Operator / Sensor Location** (ASCII strings)
|
||||
|
||||
Auto Call Home config:
|
||||
- **Auto Call Home Enable** (bool)
|
||||
- **Dial String** (read-only; 40-byte ASCII)
|
||||
- **Trigger on Event** (bool)
|
||||
- **Scheduled Call-Ins** (two time slots with HH:MM each)
|
||||
- **Retry Settings** (count, delay, connection timeout, warm-up time)
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
```
|
||||
```bash
|
||||
pip install pyserial fastapi uvicorn
|
||||
```
|
||||
|
||||
Python 3.10+. Tkinter is included with the standard Python installer on
|
||||
Windows (make sure "tcl/tk and IDLE" is checked during install).
|
||||
Windows (check "tcl/tk and IDLE" during install).
|
||||
|
||||
---
|
||||
|
||||
## Virtual COM ports (bridge capture)
|
||||
|
||||
The bridge needs two COM ports on the same PC — one that Blastware connects
|
||||
to, and one wired to the seismograph. Use a virtual COM port pair
|
||||
(**com0com** or **VSPD**) to give Blastware a port to talk to.
|
||||
|
||||
```
|
||||
Blastware → COM4 (virtual) ↔ s3_bridge.py ↔ COM5 (physical) → MiniMate Plus
|
||||
```
|
||||
|
||||
Use **com0com** or **VSPD** to create the virtual COM pair on Windows.
|
||||
|
||||
---
|
||||
|
||||
## Roadmap
|
||||
## Key Features
|
||||
|
||||
- [ ] Event download — pull waveform records from the unit (SUBs `1E` → `0A` → `0C` → `5A`)
|
||||
- [ ] Write commands — push config changes to the unit (compliance setup, channel config, trigger settings)
|
||||
- [ ] ACH inbound server — accept call-home connections from field units
|
||||
- [ ] Modem manager — push standard configs to RV50/RV55 fleet via Sierra Wireless API
|
||||
- [ ] Full Blastware parity — complete read/write/download cycle without Blastware
|
||||
**Device support:**
|
||||
- [x] Full read/write/erase pipelines
|
||||
- [x] Compliance config (recording mode, sample rate, histogram interval, geo sensitivity, project strings)
|
||||
- [x] Auto Call Home config (read/write ACH settings, dial string, time slots, retries)
|
||||
- [x] Monitor control (start/stop, status polling, battery/memory)
|
||||
- [x] Monitor log entries (continuous monitoring intervals without full waveform download)
|
||||
|
||||
**Data persistence:**
|
||||
- [x] SQLite database (`seismo_relay.db`) with 4 tables: ach_sessions, events, monitor_log, plus false_trigger flag
|
||||
- [x] Deduplication by waveform key (handles re-runs and repeat call-homes)
|
||||
- [x] Post-erase key-reuse detection (tracks high-water mark)
|
||||
- [x] Session state (`ach_state.json`) with downloaded keys and max key
|
||||
|
||||
**REST API:**
|
||||
- [x] Live device endpoints with in-memory caching (`_LiveCache`)
|
||||
- [x] Cache statistics (`/cache/stats`) and manual invalidation (`/cache/device`)
|
||||
- [x] DB query endpoints (units, events, monitor_log, sessions, false_trigger PATCH)
|
||||
- [x] Call Home config read/write endpoints
|
||||
- [x] Blastware file download endpoint (`/device/event/{index}/blastware_file`)
|
||||
|
||||
**File output (v0.7+, byte-perfect as of v0.14.3):**
|
||||
- [x] Blastware-compatible `.AB0` / `.G10` file generation (waveform + metadata)
|
||||
- [x] Multi-channel waveform decode from SUB 5A bulk stream
|
||||
- [x] Second-resolution timestamp encoding in Blastware filename
|
||||
- [x] **Byte-perfect against BW reference captures** (verified across 2-sec / 3-sec / 10-sec event durations, both event 0 and event N continuation events)
|
||||
- [x] STRT-bounded chunk walk + correct event-N probe counter + partial DLE stuffing of `0x10` in 5A params (the four fixes that landed in v0.14.0–v0.14.3)
|
||||
|
||||
**Capture tools:**
|
||||
- [x] Serial-to-TCP bridge with raw BW/S3 capture (s3_bridge.py, defaults to auto-capture)
|
||||
- [x] GUI bridge with raw capture checkboxes (gui_bridge.py)
|
||||
- [x] ACH inbound server with bidirectional capture (ach_server.py saves raw_tx + raw_rx)
|
||||
- [x] Transparent TCP MITM proxy for live BW session capture (ach_mitm.py)
|
||||
|
||||
**Analysis tools:**
|
||||
- [x] s3_analyzer.py — session parser, frame differ, Claude export
|
||||
- [x] gui_analyzer.py — standalone analyzer GUI
|
||||
- [x] frame_db.py — SQLite frame database for capture analysis
|
||||
|
||||
**seismo_lab.py GUI:**
|
||||
- [x] Bridge tab — Serial/TCP mode selector with raw capture options
|
||||
- [x] Analyzer tab — BW/S3 capture playback and differencing
|
||||
- [x] Download tab — Live wire-byte capture during event download
|
||||
- [x] Console tab — Logging and diagnostics
|
||||
|
||||
## Roadmap (Future)
|
||||
|
||||
- [ ] Verify 30-sec event download — body may exceed `0xFFFF` and force the device into a different `end_key` encoding (none of 2/3/10-sec test cases hit this boundary)
|
||||
- [ ] Terra-view integration — seismo-relay router, unit detail page, VISON-style event listing
|
||||
- [ ] Vibration summary reports — highest legit PPV per project → Word doc (false trigger filtering first)
|
||||
- [ ] Compliance config encoder — build raw write payloads from a `ComplianceConfig` object
|
||||
- [ ] Modem manager — push RV50/RV55 configs via Sierra Wireless API
|
||||
- [ ] Histogram mode recording support (5A stream analysis for mode 0x03)
|
||||
- [ ] Call Home dial_string write support (requires DLE escaping for embedded control characters)
|
||||
|
||||
@@ -0,0 +1,627 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ach_bridge.py — Transparent TCP bridge / splitter for Instantel MiniMate Plus
|
||||
call-home (ACH) traffic.
|
||||
|
||||
Modes
|
||||
-----
|
||||
standalone Accept connection, capture frames, do NOT forward anywhere.
|
||||
Good for initial discovery with a test unit.
|
||||
|
||||
bridge Forward to one upstream server while capturing.
|
||||
Use this for the initial discovery phase with your test server.
|
||||
|
||||
splitter Forward to the PRIMARY upstream (production ACH server) AND
|
||||
mirror a copy to a SECONDARY server simultaneously.
|
||||
The device never knows — it talks to the primary the whole time.
|
||||
If the mirror fails, the primary connection is unaffected.
|
||||
|
||||
Think of it like a headphone splitter: one input, two outputs.
|
||||
Primary → authoritative responses back to device.
|
||||
Mirror → gets all device bytes, its responses are discarded.
|
||||
|
||||
Usage
|
||||
-----
|
||||
# Standalone capture (test/discovery — no forwarding)
|
||||
python bridges/ach_bridge.py --standalone [--port 12345]
|
||||
|
||||
# Bridge mode (forward to one server, e.g. your test server)
|
||||
python bridges/ach_bridge.py --upstream HOST:PORT [--port 12345]
|
||||
|
||||
# Splitter mode (production: forward to prod + mirror to your server)
|
||||
python bridges/ach_bridge.py --upstream PROD_HOST:PORT --mirror MY_HOST:PORT [--port 12345]
|
||||
|
||||
Setup for discovery (test server, don't touch prod)
|
||||
----------------------------------------------------
|
||||
1. Stand up your test ACH server, note its IP and port (e.g. 192.168.1.50:12345).
|
||||
2. Take ONE test unit. In ACEmanager → Call Home, point it at:
|
||||
<this machine's LAN IP> : <--port>
|
||||
3. Run: python bridges/ach_bridge.py --upstream TEST_SERVER:12345 --port 12345
|
||||
4. Trigger the unit. Raw frames are saved to bridges/captures/ach_<ts>/.
|
||||
5. Revert the unit's ACEmanager setting when done.
|
||||
|
||||
Setup for production splitter (when you're ready)
|
||||
-------------------------------------------------
|
||||
This does NOT touch the units. Instead you re-route traffic at the network
|
||||
layer so that call-home packets arrive at a machine running this script first.
|
||||
Typical approach: update the DNS entry / host record your prod ACH server is
|
||||
registered under to point at this machine. The units keep their existing
|
||||
ACEmanager settings.
|
||||
|
||||
python bridges/ach_bridge.py \\
|
||||
--upstream PROD_ACH_HOST:12345 \\
|
||||
--mirror MY_NEW_SERVER:12345 \\
|
||||
--port 12345
|
||||
|
||||
Output (each connection gets its own timestamped sub-directory)
|
||||
------
|
||||
bridges/captures/ach_<ts>/
|
||||
raw_client_<ts>.bin — raw bytes from the device (S3 side)
|
||||
raw_server_<ts>.bin — raw bytes from the primary upstream (BW side)
|
||||
raw_mirror_<ts>.bin — raw bytes from the mirror upstream (splitter mode only)
|
||||
session_<ts>.log — human-readable frame parse log
|
||||
session_<ts>.jsonl — JSON-lines frame log
|
||||
|
||||
raw_client / raw_server are byte-for-byte compatible with parse_capture.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from minimateplus.framing import S3FrameParser, S3Frame
|
||||
|
||||
log = logging.getLogger("ach_bridge")
|
||||
|
||||
|
||||
# ── Frame label helpers ──────────────────────────────────────────────────────
|
||||
|
||||
_KNOWN_RSP_SUBS = {
|
||||
0xA4: "POLL_RSP",
|
||||
0xA5: "BULK_WAVEFORM_RSP",
|
||||
0xE0: "ADVANCE_EVENT_RSP",
|
||||
0xE1: "EVENT_INDEX_FIRST_RSP",
|
||||
0xE3: "MONITOR_STATUS_RSP",
|
||||
0xEA: "SERIAL_NUM_RSP",
|
||||
0xF3: "WAVEFORM_RECORD_RSP",
|
||||
0xF5: "WAVEFORM_HEADER_RSP",
|
||||
0xF7: "EVENT_INDEX_RSP",
|
||||
0xF9: "UNK_06_RSP",
|
||||
0xFE: "DEVICE_INFO_RSP",
|
||||
# Write acks
|
||||
0x97: "EVT_IDX_WRITE_ACK",
|
||||
0x8C: "CONFIRM_B_ACK",
|
||||
0x8E: "COMPLIANCE_WRITE_ACK",
|
||||
0x8D: "CONFIRM_A_ACK",
|
||||
0x7D: "TRIGGER_WRITE_ACK",
|
||||
0x7C: "TRIGGER_CONFIRM_ACK",
|
||||
0x96: "WAVEFORM_WRITE_ACK",
|
||||
0x8B: "CONFIRM_C_ACK",
|
||||
0x69: "START_MONITOR_ACK",
|
||||
0x68: "STOP_MONITOR_ACK",
|
||||
}
|
||||
|
||||
_KNOWN_REQ_SUBS = {
|
||||
0x5B: "POLL",
|
||||
0x5A: "BULK_WAVEFORM",
|
||||
0x1F: "ADVANCE_EVENT",
|
||||
0x1E: "EVENT_INDEX_FIRST",
|
||||
0x1C: "MONITOR_STATUS",
|
||||
0x15: "SERIAL_NUM",
|
||||
0x0C: "WAVEFORM_RECORD",
|
||||
0x0A: "WAVEFORM_HEADER",
|
||||
0x08: "EVENT_INDEX",
|
||||
0x06: "UNK_06",
|
||||
0x01: "DEVICE_INFO",
|
||||
# Write commands
|
||||
0x68: "EVT_IDX_WRITE",
|
||||
0x73: "CONFIRM_B",
|
||||
0x71: "COMPLIANCE_WRITE",
|
||||
0x72: "CONFIRM_A",
|
||||
0x82: "TRIGGER_WRITE",
|
||||
0x83: "TRIGGER_CONFIRM",
|
||||
0x69: "WAVEFORM_WRITE",
|
||||
0x74: "CONFIRM_C",
|
||||
0x96: "START_MONITOR",
|
||||
0x97: "STOP_MONITOR",
|
||||
}
|
||||
|
||||
|
||||
def _label_s3_frame(frame: S3Frame) -> str:
|
||||
name = _KNOWN_RSP_SUBS.get(frame.sub, f"UNK_0x{frame.sub:02X}")
|
||||
chk = "✓" if frame.checksum_valid else "✗CHK"
|
||||
return (
|
||||
f"S3→ SUB=0x{frame.sub:02X} ({name}) "
|
||||
f"page=0x{frame.page_key:04X} data={len(frame.data)}B {chk}"
|
||||
)
|
||||
|
||||
|
||||
def _label_bw_frame(data: bytes, prefix: str = " →BW") -> str:
|
||||
"""Best-effort label for a raw BW request frame (wire bytes)."""
|
||||
# Wire layout: 41 02 10 10 00 sub ...
|
||||
if len(data) < 6:
|
||||
return f"{prefix} (short {len(data)}B)"
|
||||
sub = data[5]
|
||||
name = _KNOWN_REQ_SUBS.get(sub, f"UNK_0x{sub:02X}")
|
||||
return f"{prefix} SUB=0x{sub:02X} ({name}) {len(data)}B"
|
||||
|
||||
|
||||
# ── Per-session capture writer ─────────────────────────────────────────────────
|
||||
|
||||
class CaptureSession:
|
||||
"""Writes raw bytes + parsed log for one TCP connection."""
|
||||
|
||||
def __init__(self, capture_dir: Path, peer: str, *, has_mirror: bool = False):
|
||||
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
self.dir = capture_dir / f"ach_{ts}"
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
self.peer = peer
|
||||
|
||||
self._raw_client = open(self.dir / f"raw_client_{ts}.bin", "wb")
|
||||
self._raw_server = open(self.dir / f"raw_server_{ts}.bin", "wb")
|
||||
self._raw_mirror = (
|
||||
open(self.dir / f"raw_mirror_{ts}.bin", "wb") if has_mirror else None
|
||||
)
|
||||
self._log_fh = open(self.dir / f"session_{ts}.log", "w")
|
||||
self._jsonl_fh = open(self.dir / f"session_{ts}.jsonl", "w")
|
||||
|
||||
self._s3_parser = S3FrameParser()
|
||||
self._frame_count = 0
|
||||
self._byte_count_client = 0
|
||||
self._byte_count_server = 0
|
||||
self._byte_count_mirror = 0
|
||||
|
||||
self._log(
|
||||
f"# ACH capture — peer={peer} "
|
||||
f"mirror={'yes' if has_mirror else 'no'} "
|
||||
f"started={datetime.datetime.now().isoformat()}"
|
||||
)
|
||||
self._log(f"# Output dir: {self.dir}")
|
||||
log.info("Capture session opened: %s (peer=%s)", self.dir, peer)
|
||||
|
||||
# ── public API ────────────────────────────────────────────────────────────
|
||||
|
||||
def feed_client(self, data: bytes) -> None:
|
||||
"""Bytes FROM the device (S3 response frames)."""
|
||||
self._raw_client.write(data)
|
||||
self._raw_client.flush()
|
||||
self._byte_count_client += len(data)
|
||||
|
||||
for byte in data:
|
||||
frame = self._s3_parser.feed(bytes([byte]))
|
||||
if frame:
|
||||
frames = frame if isinstance(frame, list) else [frame]
|
||||
for f in frames:
|
||||
self._frame_count += 1
|
||||
label = _label_s3_frame(f)
|
||||
self._log(f"[{self._frame_count:04d}] {label}")
|
||||
self._log(
|
||||
f" hex: {f.data[:64].hex()}"
|
||||
+ (" ..." if len(f.data) > 64 else "")
|
||||
)
|
||||
self._emit_json("s3", f)
|
||||
|
||||
def feed_server(self, data: bytes) -> None:
|
||||
"""Bytes FROM the primary upstream server (BW request frames)."""
|
||||
self._raw_server.write(data)
|
||||
self._raw_server.flush()
|
||||
self._byte_count_server += len(data)
|
||||
label = _label_bw_frame(data, prefix=" →BW[primary]")
|
||||
self._log(f" {label}")
|
||||
|
||||
def feed_mirror(self, data: bytes) -> None:
|
||||
"""Bytes FROM the mirror server (logged, not forwarded to device)."""
|
||||
if self._raw_mirror:
|
||||
self._raw_mirror.write(data)
|
||||
self._raw_mirror.flush()
|
||||
self._byte_count_mirror += len(data)
|
||||
label = _label_bw_frame(data, prefix=" →BW[mirror] ")
|
||||
self._log(f" {label} [MIRROR — not sent to device]")
|
||||
|
||||
def close(self, reason: str = "connection closed") -> None:
|
||||
self._log(f"# Session ended: {reason}")
|
||||
self._log(
|
||||
f"# Totals — client={self._byte_count_client}B "
|
||||
f"server={self._byte_count_server}B "
|
||||
f"mirror={self._byte_count_mirror}B "
|
||||
f"s3_frames={self._frame_count}"
|
||||
)
|
||||
handles = [self._raw_client, self._raw_server, self._log_fh, self._jsonl_fh]
|
||||
if self._raw_mirror:
|
||||
handles.append(self._raw_mirror)
|
||||
for fh in handles:
|
||||
try:
|
||||
fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
log.info(
|
||||
"Session closed (%s): %dB client, %dB server, %dB mirror, %d S3 frames → %s",
|
||||
reason,
|
||||
self._byte_count_client, self._byte_count_server,
|
||||
self._byte_count_mirror, self._frame_count,
|
||||
self.dir,
|
||||
)
|
||||
|
||||
# ── internals ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _log(self, msg: str) -> None:
|
||||
print(msg, file=self._log_fh, flush=True)
|
||||
print(msg)
|
||||
|
||||
def _emit_json(self, direction: str, frame: S3Frame) -> None:
|
||||
record = {
|
||||
"dir": direction,
|
||||
"sub": frame.sub,
|
||||
"page_key": frame.page_key,
|
||||
"data_len": len(frame.data),
|
||||
"data_hex": frame.data.hex(),
|
||||
"checksum_valid": frame.checksum_valid,
|
||||
}
|
||||
print(json.dumps(record), file=self._jsonl_fh, flush=True)
|
||||
|
||||
|
||||
# ── Bridge / splitter connection handler ──────────────────────────────────────
|
||||
|
||||
class BridgeHandler:
|
||||
"""
|
||||
Handles inbound device connections.
|
||||
|
||||
Modes (determined by which upstreams are configured):
|
||||
standalone — no upstream_host / no mirror_host
|
||||
bridge — upstream_host set, no mirror_host
|
||||
splitter — upstream_host AND mirror_host both set
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
capture_dir: Path,
|
||||
upstream_host: Optional[str],
|
||||
upstream_port: Optional[int],
|
||||
mirror_host: Optional[str] = None,
|
||||
mirror_port: Optional[int] = None,
|
||||
):
|
||||
self.capture_dir = capture_dir
|
||||
self.upstream_host = upstream_host
|
||||
self.upstream_port = upstream_port
|
||||
self.mirror_host = mirror_host
|
||||
self.mirror_port = mirror_port
|
||||
|
||||
async def handle(
|
||||
self,
|
||||
client_reader: asyncio.StreamReader,
|
||||
client_writer: asyncio.StreamWriter,
|
||||
) -> None:
|
||||
peer = client_writer.get_extra_info("peername", ("?", 0))
|
||||
peer_str = f"{peer[0]}:{peer[1]}"
|
||||
log.info("Inbound connection from %s", peer_str)
|
||||
|
||||
has_mirror = bool(self.mirror_host)
|
||||
session = CaptureSession(self.capture_dir, peer_str, has_mirror=has_mirror)
|
||||
|
||||
if not self.upstream_host:
|
||||
# ── Standalone mode ──────────────────────────────────────────────
|
||||
log.info("Standalone mode — recording inbound traffic only")
|
||||
try:
|
||||
while True:
|
||||
data = await client_reader.read(4096)
|
||||
if not data:
|
||||
break
|
||||
session.feed_client(data)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
log.warning("Standalone read error: %s", exc)
|
||||
finally:
|
||||
session.close("standalone capture ended")
|
||||
try:
|
||||
client_writer.close()
|
||||
await client_writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
# ── Bridge / splitter mode ───────────────────────────────────────────
|
||||
# Connect to primary upstream (required)
|
||||
try:
|
||||
up_reader, up_writer = await asyncio.open_connection(
|
||||
self.upstream_host, self.upstream_port
|
||||
)
|
||||
log.info("Connected to primary %s:%s", self.upstream_host, self.upstream_port)
|
||||
except Exception as exc:
|
||||
log.error("Failed to connect to primary upstream: %s", exc)
|
||||
session.close(f"primary connect failed: {exc}")
|
||||
client_writer.close()
|
||||
return
|
||||
|
||||
# Connect to mirror upstream (optional — failure is non-fatal)
|
||||
mir_reader: Optional[asyncio.StreamReader] = None
|
||||
mir_writer: Optional[asyncio.StreamWriter] = None
|
||||
if self.mirror_host:
|
||||
try:
|
||||
mir_reader, mir_writer = await asyncio.open_connection(
|
||||
self.mirror_host, self.mirror_port
|
||||
)
|
||||
log.info("Connected to mirror %s:%s", self.mirror_host, self.mirror_port)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Mirror connect failed — continuing without mirror: %s", exc
|
||||
)
|
||||
session._log(f"# WARNING: mirror connect failed: {exc}")
|
||||
|
||||
# Build relay tasks
|
||||
#
|
||||
# ┌──────────┐ device bytes ┌─────────────┐
|
||||
# │ Device │ ─────────────► │ PRIMARY │ responses ──► device
|
||||
# └──────────┘ └─────────────┘
|
||||
# │
|
||||
# │ device bytes (copy)
|
||||
# ▼
|
||||
# ┌─────────────┐
|
||||
# │ MIRROR │ responses discarded (logged only)
|
||||
# └─────────────┘
|
||||
#
|
||||
tasks = [
|
||||
asyncio.create_task(
|
||||
self._relay_device(client_reader, up_writer, mir_writer, session),
|
||||
name="device→upstreams",
|
||||
),
|
||||
asyncio.create_task(
|
||||
self._relay_simple(up_reader, client_writer, session, "server"),
|
||||
name="primary→device",
|
||||
),
|
||||
]
|
||||
if mir_reader is not None:
|
||||
tasks.append(asyncio.create_task(
|
||||
self._relay_drain(mir_reader, session),
|
||||
name="mirror→drain",
|
||||
))
|
||||
|
||||
try:
|
||||
# Wait for the device-to-upstreams relay to exit first (device
|
||||
# disconnected or primary dropped). Then cancel the rest.
|
||||
done, pending = await asyncio.wait(
|
||||
tasks,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for t in pending:
|
||||
t.cancel()
|
||||
try:
|
||||
await t
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
except Exception as exc:
|
||||
log.warning("Bridge relay error: %s", exc)
|
||||
finally:
|
||||
session.close("relay ended")
|
||||
for writer in filter(None, [client_writer, up_writer, mir_writer]):
|
||||
try:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ── Relay helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
async def _relay_device(
|
||||
self,
|
||||
reader: asyncio.StreamReader,
|
||||
primary_writer: asyncio.StreamWriter,
|
||||
mirror_writer: Optional[asyncio.StreamWriter],
|
||||
session: CaptureSession,
|
||||
) -> None:
|
||||
"""
|
||||
Read bytes from the device, write to the primary server, and also
|
||||
write a copy to the mirror server (if connected). Mirror write
|
||||
failures are non-fatal — we log and continue.
|
||||
"""
|
||||
try:
|
||||
while True:
|
||||
data = await reader.read(4096)
|
||||
if not data:
|
||||
break
|
||||
session.feed_client(data)
|
||||
|
||||
# Primary write — failure IS fatal (lose primary = lose prod)
|
||||
primary_writer.write(data)
|
||||
await primary_writer.drain()
|
||||
|
||||
# Mirror write — failure is non-fatal
|
||||
if mirror_writer is not None:
|
||||
try:
|
||||
mirror_writer.write(data)
|
||||
await mirror_writer.drain()
|
||||
except Exception as exc:
|
||||
log.warning("Mirror write failed (non-fatal): %s", exc)
|
||||
session._log(f"# WARNING: mirror write failed: {exc}")
|
||||
mirror_writer = None # stop trying
|
||||
|
||||
except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
|
||||
async def _relay_simple(
|
||||
self,
|
||||
reader: asyncio.StreamReader,
|
||||
writer: asyncio.StreamWriter,
|
||||
session: CaptureSession,
|
||||
direction: str,
|
||||
) -> None:
|
||||
"""Standard single-pipe relay (primary→device or vice-versa)."""
|
||||
try:
|
||||
while True:
|
||||
data = await reader.read(4096)
|
||||
if not data:
|
||||
break
|
||||
if direction == "server":
|
||||
session.feed_server(data)
|
||||
else:
|
||||
session.feed_client(data)
|
||||
writer.write(data)
|
||||
await writer.drain()
|
||||
except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
|
||||
async def _relay_drain(
|
||||
self,
|
||||
reader: asyncio.StreamReader,
|
||||
session: CaptureSession,
|
||||
) -> None:
|
||||
"""
|
||||
Read mirror server responses, log them to session, do NOT forward to
|
||||
device. The device only ever sees primary server responses.
|
||||
"""
|
||||
try:
|
||||
while True:
|
||||
data = await reader.read(4096)
|
||||
if not data:
|
||||
break
|
||||
session.feed_mirror(data)
|
||||
except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
|
||||
|
||||
# ── Main ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
async def main(args: argparse.Namespace) -> None:
|
||||
capture_dir = Path(__file__).parent / "captures"
|
||||
capture_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
upstream_host: Optional[str] = None
|
||||
upstream_port: Optional[int] = None
|
||||
mirror_host: Optional[str] = None
|
||||
mirror_port: Optional[int] = None
|
||||
|
||||
if not args.standalone:
|
||||
if not args.upstream:
|
||||
print("ERROR: --upstream HOST:PORT is required unless --standalone is set.")
|
||||
sys.exit(1)
|
||||
parts = args.upstream.rsplit(":", 1)
|
||||
if len(parts) != 2:
|
||||
print("ERROR: --upstream must be HOST:PORT (e.g. 203.0.113.5:12345)")
|
||||
sys.exit(1)
|
||||
upstream_host = parts[0]
|
||||
upstream_port = int(parts[1])
|
||||
|
||||
if args.mirror:
|
||||
parts = args.mirror.rsplit(":", 1)
|
||||
if len(parts) != 2:
|
||||
print("ERROR: --mirror must be HOST:PORT (e.g. 192.168.1.50:12345)")
|
||||
sys.exit(1)
|
||||
mirror_host = parts[0]
|
||||
mirror_port = int(parts[1])
|
||||
|
||||
handler = BridgeHandler(
|
||||
capture_dir,
|
||||
upstream_host, upstream_port,
|
||||
mirror_host, mirror_port,
|
||||
)
|
||||
|
||||
server = await asyncio.start_server(
|
||||
handler.handle,
|
||||
host="0.0.0.0",
|
||||
port=args.port,
|
||||
)
|
||||
|
||||
# ── Startup banner ────────────────────────────────────────────────────────
|
||||
if args.standalone:
|
||||
mode = "STANDALONE capture (no forwarding)"
|
||||
elif mirror_host:
|
||||
mode = f"SPLITTER primary={upstream_host}:{upstream_port} mirror={mirror_host}:{mirror_port}"
|
||||
else:
|
||||
mode = f"BRIDGE → {upstream_host}:{upstream_port}"
|
||||
|
||||
addrs = ", ".join(str(s.getsockname()) for s in server.sockets)
|
||||
print(f"\n{'='*70}")
|
||||
print(f" ACH bridge/splitter listening on {addrs}")
|
||||
print(f" Mode: {mode}")
|
||||
print(f" Captures: {capture_dir}/ach_<timestamp>/")
|
||||
print(f"{'='*70}")
|
||||
|
||||
if upstream_host and not mirror_host:
|
||||
print(f"\n DISCOVERY PHASE")
|
||||
print(f" Point your TEST unit's ACEmanager call-home destination to:")
|
||||
print(f" <this machine's LAN IP> : {args.port}")
|
||||
print(f" All traffic will be forwarded to {upstream_host}:{upstream_port}")
|
||||
elif mirror_host:
|
||||
print(f"\n SPLITTER MODE — PRODUCTION SAFE")
|
||||
print(f" Units connect as normal. Every byte is forwarded to:")
|
||||
print(f" PRIMARY (authoritative): {upstream_host}:{upstream_port}")
|
||||
print(f" MIRROR (your server): {mirror_host}:{mirror_port}")
|
||||
print(f" Only PRIMARY responses reach the device.")
|
||||
print(f" Mirror failures are logged and do not affect the device.")
|
||||
else:
|
||||
print(f"\n STANDALONE MODE — capture only, nothing forwarded")
|
||||
print(f" Point a unit at <this machine's LAN IP> : {args.port}")
|
||||
|
||||
print(f"\n Waiting for inbound connections... (Ctrl-C to stop)\n")
|
||||
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Transparent TCP bridge / splitter for Instantel MiniMate Plus "
|
||||
"call-home (ACH) traffic."
|
||||
),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
p.add_argument(
|
||||
"--upstream", "-u",
|
||||
metavar="HOST:PORT",
|
||||
help=(
|
||||
"Primary upstream ACH server to forward to "
|
||||
"(e.g. 203.0.113.5:12345). "
|
||||
"Omit with --standalone for capture-only mode."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--mirror", "-m",
|
||||
metavar="HOST:PORT",
|
||||
help=(
|
||||
"Mirror / secondary server to receive a copy of all device bytes "
|
||||
"(splitter mode). Mirror responses are logged but NOT forwarded "
|
||||
"to the device. Mirror failures are non-fatal."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--port", "-p",
|
||||
type=int,
|
||||
default=12345,
|
||||
help="Local port to listen on (default: 12345).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--standalone", "-s",
|
||||
action="store_true",
|
||||
help="Capture-only mode: accept connection, do not forward anywhere.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Enable debug logging.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
)
|
||||
try:
|
||||
asyncio.run(main(args))
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopped.")
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ach_mitm.py — TCP man-in-the-middle proxy for capturing Blastware ACH sessions.
|
||||
|
||||
The unit calls home to THIS proxy instead of directly to Blastware. The proxy
|
||||
forwards every byte in both directions to the real Blastware ACH server and saves
|
||||
the traffic to separate raw capture files that the Analyzer can load directly.
|
||||
|
||||
Setup
|
||||
-----
|
||||
1. Start Blastware's ACH server on the BW PC as normal (it listens on its port).
|
||||
2. Run this proxy on any machine the unit can reach:
|
||||
|
||||
python bridges/ach_mitm.py --bw-host 192.168.1.50 --bw-port 9999
|
||||
|
||||
3. Point the unit's ACEmanager call-home destination to THIS machine's IP and
|
||||
the --listen-port (default 9999).
|
||||
4. Trigger a call-home (or wait for the unit to call in).
|
||||
5. The proxy transparently forwards everything and saves two files per session:
|
||||
|
||||
ach_mitm_<ts>/raw_bw_<ts>.bin -- bytes Blastware sent to unit (BW TX)
|
||||
ach_mitm_<ts>/raw_s3_<ts>.bin -- bytes unit sent to Blastware (S3 TX)
|
||||
|
||||
Both files load directly in the Analyzer (File > Open Capture).
|
||||
|
||||
The proxy exits cleanly when either side drops the connection.
|
||||
|
||||
Use case: capturing Blastware operations we haven't reverse-engineered yet,
|
||||
e.g. event deletion, factory reset, firmware update.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import logging
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("ach_mitm")
|
||||
|
||||
|
||||
def _pipe(src: socket.socket, dst: socket.socket, label: str, outfile) -> None:
|
||||
"""Forward bytes from src to dst, writing everything to outfile."""
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(4096)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
outfile.write(data)
|
||||
outfile.flush()
|
||||
log.debug("%s %d bytes", label, len(data))
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
log.info("%s pipe closed", label)
|
||||
# Signal the other direction to stop by shutting down our end.
|
||||
try:
|
||||
dst.shutdown(socket.SHUT_WR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def handle(unit_sock: socket.socket, peer: str, bw_host: str, bw_port: int,
|
||||
output_dir: Path) -> None:
|
||||
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
session_dir = output_dir / f"ach_mitm_{ts}"
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log.info("Session %s unit=%s forwarding to %s:%d", ts, peer, bw_host, bw_port)
|
||||
|
||||
# Connect upstream to Blastware.
|
||||
bw_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
bw_sock.connect((bw_host, bw_port))
|
||||
except OSError as exc:
|
||||
log.error("Cannot reach Blastware at %s:%d: %s", bw_host, bw_port, exc)
|
||||
unit_sock.close()
|
||||
return
|
||||
|
||||
log.info("Connected to Blastware at %s:%d", bw_host, bw_port)
|
||||
|
||||
bw_path = session_dir / f"raw_bw_{ts}.bin" # Blastware → unit (BW TX)
|
||||
s3_path = session_dir / f"raw_s3_{ts}.bin" # unit → Blastware (S3 TX)
|
||||
|
||||
with open(bw_path, "wb") as bw_fh, open(s3_path, "wb") as s3_fh:
|
||||
# Two threads: one per direction.
|
||||
t_bw = threading.Thread(
|
||||
target=_pipe, args=(bw_sock, unit_sock, "BW->unit", bw_fh), daemon=True
|
||||
)
|
||||
t_s3 = threading.Thread(
|
||||
target=_pipe, args=(unit_sock, bw_sock, "unit->BW", s3_fh), daemon=True
|
||||
)
|
||||
t_bw.start()
|
||||
t_s3.start()
|
||||
t_bw.join()
|
||||
t_s3.join()
|
||||
|
||||
bw_bytes = bw_path.stat().st_size
|
||||
s3_bytes = s3_path.stat().st_size
|
||||
log.info(
|
||||
"Session %s done BW->unit: %d bytes unit->BW: %d bytes -> %s",
|
||||
ts, bw_bytes, s3_bytes, session_dir,
|
||||
)
|
||||
|
||||
unit_sock.close()
|
||||
bw_sock.close()
|
||||
|
||||
|
||||
def serve(args: argparse.Namespace) -> None:
|
||||
output_dir = Path(args.output)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(("0.0.0.0", args.listen_port))
|
||||
server.listen(5)
|
||||
server.settimeout(1.0)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" ACH MITM proxy")
|
||||
print(f" Listening on 0.0.0.0:{args.listen_port}")
|
||||
print(f" Forwarding to {args.bw_host}:{args.bw_port}")
|
||||
print(f" Captures in {output_dir.resolve()}/ach_mitm_<ts>/")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n Point the unit's ACEmanager call-home to this machine on port {args.listen_port}")
|
||||
print(f" Ctrl-C to stop\n")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
client_sock, addr = server.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
peer = f"{addr[0]}:{addr[1]}"
|
||||
log.info("Accepted connection from %s", peer)
|
||||
t = threading.Thread(
|
||||
target=handle,
|
||||
args=(client_sock, peer, args.bw_host, args.bw_port, output_dir),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopping.")
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--bw-host", required=True,
|
||||
help="IP or hostname of the Blastware ACH server")
|
||||
ap.add_argument("--bw-port", type=int, default=9999,
|
||||
help="Port Blastware is listening on (default: 9999)")
|
||||
ap.add_argument("--listen-port", type=int, default=9999,
|
||||
help="Port this proxy listens on (default: 9999)")
|
||||
ap.add_argument("--output", default="bridges/captures/mitm",
|
||||
help="Directory for capture files")
|
||||
ap.add_argument("--log-level", default="INFO",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR"])
|
||||
args = ap.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, args.log_level),
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
|
||||
serve(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,903 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ach_server.py — Minimal inbound ACH (Auto Call Home) server for MiniMate Plus.
|
||||
|
||||
This IS your test server. Run it on any machine on the same network, point a
|
||||
unit's ACEmanager call-home destination at it, and it will speak the full BW
|
||||
protocol to the device: handshake, pull device info, download all events, save
|
||||
everything as JSON.
|
||||
|
||||
The key thing this script tells you that no amount of packet sniffing can:
|
||||
- Does the device speak first (push) or wait for us to send POLL (pull)?
|
||||
|
||||
If startup() completes normally → it's pull protocol, same as Blastware.
|
||||
If startup() times out → the device sent something first; check raw_rx.bin.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python bridges/ach_server.py [--port 12345] [--output bridges/captures/]
|
||||
|
||||
Setup
|
||||
-----
|
||||
1. Run this script on a machine on your local network.
|
||||
2. In ACEmanager → Application → ALEOS Application Framework (or equivalent)
|
||||
find the Call Home / ACH settings. Set:
|
||||
Remote Host: <this machine's LAN IP>
|
||||
Remote Port: 12345
|
||||
3. Trigger the unit (wait for a vibration event, or use the manual call-home
|
||||
button if your firmware version has one).
|
||||
4. The unit connects. This script handshakes, downloads all events,
|
||||
and saves a timestamped session directory.
|
||||
|
||||
Output per session
|
||||
------------------
|
||||
bridges/captures/ach_inbound_<ts>/
|
||||
device_info.json — serial number, firmware version, calibration date, etc.
|
||||
events.json — all events: timestamp, PPV per channel, peaks, metadata
|
||||
raw_rx_<ts>.bin — raw bytes from the device (S3 side) for Analyzer
|
||||
raw_tx_<ts>.bin — raw bytes we sent to the device (BW side) for Analyzer
|
||||
session_<ts>.log — detailed protocol log
|
||||
|
||||
What to look for
|
||||
----------------
|
||||
Push vs pull: Check session_<ts>.log. If the first line after "Connected"
|
||||
shows bytes arriving BEFORE the POLL probe was sent, it's push. If POLL
|
||||
gets a clean response, it's pull.
|
||||
|
||||
Frequency: Look at raw_rx.bin in the Analyzer. SUB 5A (0xA5 responses) carry
|
||||
bulk waveform data — if frequency is sent pre-computed there will be float32
|
||||
values before the ADC sample blocks.
|
||||
|
||||
ACH-specific framing: Does the unit send anything extra before the DLE+STX
|
||||
framing starts? raw_rx.bin will show raw bytes including any preamble.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from minimateplus.transport import SocketTransport
|
||||
from minimateplus.client import MiniMateClient
|
||||
from minimateplus.models import DeviceInfo, Event, MonitorLogEntry
|
||||
from sfm.database import SeismoDb
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
log = logging.getLogger("ach_server")
|
||||
|
||||
# ── Per-unit state (downloaded events index) ──────────────────────────────────
|
||||
# Persisted as <output_dir>/ach_state.json
|
||||
# Format (current — v2):
|
||||
# {
|
||||
# "BE11529": {
|
||||
# "downloaded_events": { # key_hex → ISO timestamp string
|
||||
# "01110000": "2026-04-11T00:42:17",
|
||||
# "0111245a": "2026-04-11T01:04:30"
|
||||
# },
|
||||
# "max_downloaded_key": "0111245a",
|
||||
# "last_seen": "2026-04-11T01:04:36",
|
||||
# "serial": "BE11529",
|
||||
# "peer": "63.43.212.232:51920"
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# Why (key, timestamp) and not key alone:
|
||||
# The device's event-key counter resets to 0x01110000 after every memory
|
||||
# erase (internal or external). A bare-key dedup (the v1 format) cannot
|
||||
# distinguish a re-recorded event with the same key from one we already
|
||||
# downloaded. The 0C waveform record's timestamp IS unique per physical
|
||||
# event, so we pair (key, timestamp) and treat a key with a different
|
||||
# timestamp as a new event regardless of `max_downloaded_key`.
|
||||
#
|
||||
# Legacy v1 format (`downloaded_keys: list[str]` only) is auto-migrated on
|
||||
# read: the keys are kept under a sentinel of "" (empty string) timestamp so
|
||||
# the (key, timestamp) compare always sees a mismatch and forces a one-time
|
||||
# re-download. After that pass the state is rewritten in v2 form.
|
||||
|
||||
_state_lock = threading.Lock()
|
||||
|
||||
|
||||
def _load_state(state_path: Path) -> dict:
|
||||
"""
|
||||
Load ach_state.json, transparently migrating any legacy
|
||||
`downloaded_keys: list` entries into the v2 `downloaded_events: dict`
|
||||
schema. Returns the migrated state.
|
||||
"""
|
||||
if not state_path.exists():
|
||||
return {}
|
||||
try:
|
||||
with open(state_path) as f:
|
||||
state = json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
# Per-unit migration: legacy list → dict-with-empty-timestamps
|
||||
for unit_key, unit_state in list(state.items()):
|
||||
if not isinstance(unit_state, dict):
|
||||
continue
|
||||
if "downloaded_events" in unit_state:
|
||||
continue
|
||||
legacy_keys = unit_state.get("downloaded_keys")
|
||||
if isinstance(legacy_keys, list):
|
||||
unit_state["downloaded_events"] = {k: "" for k in legacy_keys}
|
||||
log.info(
|
||||
"ach_state: migrated %s from v1 (downloaded_keys list) → v2 "
|
||||
"(downloaded_events dict, %d keys with empty timestamps; "
|
||||
"they will re-validate on next session)",
|
||||
unit_key, len(legacy_keys),
|
||||
)
|
||||
else:
|
||||
unit_state["downloaded_events"] = {}
|
||||
# keep legacy field for one cycle; cleared on next save
|
||||
unit_state.pop("downloaded_keys", None)
|
||||
|
||||
return state
|
||||
|
||||
|
||||
def _save_state(state_path: Path, state: dict) -> None:
|
||||
with _state_lock:
|
||||
with open(state_path, "w") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
|
||||
# ── Per-session handler ────────────────────────────────────────────────────────
|
||||
|
||||
class AchSession:
|
||||
"""
|
||||
Handles one inbound unit connection in its own thread.
|
||||
Wraps the socket in a SocketTransport → MiniMateClient, then runs the
|
||||
standard connect → get_device_info → get_events sequence.
|
||||
|
||||
State tracking (ach_state.json in output_dir):
|
||||
On each successful download we record the SET of event keys downloaded.
|
||||
On the next call-home we compare: if all device keys are already in the
|
||||
set, there's nothing new. If any key is new (including after the device
|
||||
was wiped and re-recorded), we download and save only those events.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sock: socket.socket,
|
||||
peer: str,
|
||||
output_dir: Path,
|
||||
timeout: float,
|
||||
events_only: bool,
|
||||
max_events: Optional[int],
|
||||
state_path: Path,
|
||||
db: "SeismoDb",
|
||||
store: "WaveformStore",
|
||||
clear_after_download: bool = False,
|
||||
restart_monitoring: bool = False,
|
||||
force_redownload: bool = False,
|
||||
) -> None:
|
||||
self.sock = sock
|
||||
self.peer = peer
|
||||
self.output_dir = output_dir
|
||||
self.timeout = timeout
|
||||
self.events_only = events_only
|
||||
self.max_events = max_events
|
||||
self.state_path = state_path
|
||||
self.db = db
|
||||
self.store = store
|
||||
self.clear_after_download = clear_after_download
|
||||
self.restart_monitoring = restart_monitoring
|
||||
# `force_redownload` tells this session to ignore ach_state and
|
||||
# re-download every event currently on the device, regardless of any
|
||||
# (key, timestamp) match. Useful as a manual override when state has
|
||||
# become inconsistent with what's actually on disk / in the DB.
|
||||
self.force_redownload = force_redownload
|
||||
|
||||
def run(self) -> None:
|
||||
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
# Session dir and file handler are created lazily — only after startup
|
||||
# succeeds. This prevents internet scanners and dropped connections from
|
||||
# littering the output directory with empty session folders.
|
||||
try:
|
||||
self._run_inner(ts)
|
||||
except Exception as exc:
|
||||
log.error("Session failed (%s): %s", self.peer, exc, exc_info=True)
|
||||
finally:
|
||||
try:
|
||||
self.sock.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _run_inner(self, ts: str) -> None:
|
||||
transport = SocketTransport(self.sock, peer=self.peer)
|
||||
|
||||
# Collect raw bytes in memory until startup succeeds, then flush to disk.
|
||||
raw_rx_buf: list[bytes] = [] # device → us (S3 side)
|
||||
raw_tx_buf: list[bytes] = [] # us → device (BW side)
|
||||
_orig_read = transport.read
|
||||
_orig_write = transport.write
|
||||
|
||||
def tapped_read(n: int) -> bytes:
|
||||
data = _orig_read(n)
|
||||
if data:
|
||||
raw_rx_buf.append(data)
|
||||
return data
|
||||
|
||||
def tapped_write(data: bytes) -> None:
|
||||
_orig_write(data)
|
||||
if data:
|
||||
raw_tx_buf.append(data)
|
||||
|
||||
transport.read = tapped_read # type: ignore[method-assign]
|
||||
transport.write = tapped_write # type: ignore[method-assign]
|
||||
|
||||
serial: Optional[str] = None
|
||||
|
||||
# ── Step 1: startup handshake ─────────────────────────────────────────
|
||||
# Do this BEFORE creating the session directory so that scanner probes
|
||||
# and dropped connections leave no trace on disk.
|
||||
try:
|
||||
from minimateplus.protocol import MiniMateProtocol
|
||||
client = MiniMateClient(transport=transport, timeout=self.timeout)
|
||||
client.open()
|
||||
proto = MiniMateProtocol(transport, recv_timeout=self.timeout)
|
||||
proto.startup()
|
||||
except Exception as exc:
|
||||
log.warning("Startup failed from %s: %s -- ignoring", self.peer, exc)
|
||||
return # no session dir created
|
||||
|
||||
# Startup succeeded — this is a real unit. Create session dir now.
|
||||
session_dir = self.output_dir / f"ach_inbound_{ts}"
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = session_dir / f"session_{ts}.log"
|
||||
raw_rx_path = session_dir / f"raw_rx_{ts}.bin" # device → us (S3 side)
|
||||
raw_tx_path = session_dir / f"raw_tx_{ts}.bin" # us → device (BW side)
|
||||
|
||||
# Flush buffered bytes to files and switch to direct file writes.
|
||||
raw_rx_fh = open(raw_rx_path, "wb")
|
||||
raw_tx_fh = open(raw_tx_path, "wb")
|
||||
for chunk in raw_rx_buf:
|
||||
raw_rx_fh.write(chunk)
|
||||
for chunk in raw_tx_buf:
|
||||
raw_tx_fh.write(chunk)
|
||||
raw_rx_buf.clear()
|
||||
raw_tx_buf.clear()
|
||||
|
||||
def tapped_read_file(n: int) -> bytes:
|
||||
data = _orig_read(n)
|
||||
if data:
|
||||
raw_rx_fh.write(data)
|
||||
raw_rx_fh.flush()
|
||||
return data
|
||||
|
||||
def tapped_write_file(data: bytes) -> None:
|
||||
_orig_write(data)
|
||||
if data:
|
||||
raw_tx_fh.write(data)
|
||||
raw_tx_fh.flush()
|
||||
|
||||
transport.read = tapped_read_file # type: ignore[method-assign]
|
||||
transport.write = tapped_write_file # type: ignore[method-assign]
|
||||
|
||||
# Wire up file handler now that the session dir exists.
|
||||
fh = logging.FileHandler(log_path, encoding="utf-8")
|
||||
fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s"))
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.addHandler(fh)
|
||||
|
||||
try:
|
||||
# ── Step 2: device info ───────────────────────────────────────────
|
||||
device_info = None
|
||||
if not self.events_only:
|
||||
log.info("Step 2/3: reading device info")
|
||||
try:
|
||||
device_info = client.connect()
|
||||
serial = device_info.serial
|
||||
_save_json(session_dir / "device_info.json", _device_info_to_dict(device_info))
|
||||
log.info(
|
||||
" [OK] Device: serial=%s firmware=%s model=%s events=%d",
|
||||
serial,
|
||||
device_info.firmware_version,
|
||||
device_info.model,
|
||||
device_info.event_count or 0,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.error(" [FAIL] Device info failed: %s", exc)
|
||||
else:
|
||||
log.info("Step 2/3: skipping device info (--events-only)")
|
||||
|
||||
# ── Step 3: check for new events by comparing key sets ────────────
|
||||
log.info("Step 3/3: checking for new events")
|
||||
|
||||
state = _load_state(self.state_path)
|
||||
unit_key = serial or self.peer # fall back to IP if no serial
|
||||
unit_state = state.get(unit_key, {})
|
||||
|
||||
# downloaded_events is the v2 (key_hex → timestamp_iso) dict.
|
||||
# Empty-string timestamps are migrated v1 entries — they force a
|
||||
# one-time re-download because the (key, timestamp) compare always
|
||||
# mismatches against any non-empty timestamp from a fresh 0C read.
|
||||
seen_events: dict[str, str] = dict(unit_state.get("downloaded_events", {}))
|
||||
max_seen_key: str = unit_state.get("max_downloaded_key", "00000000")
|
||||
|
||||
if self.force_redownload:
|
||||
log.info(" --force-redownload-all set — ignoring %d cached "
|
||||
"(key, timestamp) entries for this session",
|
||||
len(seen_events))
|
||||
seen_events = {}
|
||||
|
||||
# Walk the event index (browse-mode, no 5A) to get the actual current
|
||||
# key list. The SUB 08 event_count field is a lifetime "total events
|
||||
# ever recorded" counter that does NOT decrement on erase — confirmed
|
||||
# 2026-04-13. list_event_keys() via the 1E/1F chain is the only
|
||||
# reliable way to know what is actually stored on the device right now.
|
||||
log.info(" Checking device key list (browse walk, no waveform download)...")
|
||||
try:
|
||||
device_keys = client.list_event_keys()
|
||||
except Exception as exc:
|
||||
log.warning(" list_event_keys failed: %s -- falling back to full download", exc)
|
||||
device_keys = None
|
||||
|
||||
current_count = len(device_keys) if device_keys is not None else 0
|
||||
|
||||
log.info(" Unit has %d stored event(s); %d (key, ts) entr(ies) previously downloaded",
|
||||
current_count, len(seen_events))
|
||||
|
||||
if device_keys is not None and current_count == 0:
|
||||
log.info(" [OK] No events on device -- nothing to download")
|
||||
log.info("Session complete (no events) -> %s", session_dir)
|
||||
return
|
||||
|
||||
if device_keys is not None:
|
||||
# ── Post-erase detection (best-effort, key-only signal) ───────
|
||||
# After erase the device's key counter resets to 01110000.
|
||||
# If the device's current max key is below our high-water mark
|
||||
# we know erase happened. This catches the cleanest case but
|
||||
# does NOT catch erase-then-record-many-events (where the new
|
||||
# max may climb past the old max). The (key, timestamp) check
|
||||
# in get_events() is what handles those.
|
||||
if device_keys and max_seen_key != "00000000":
|
||||
max_device_key = max(device_keys)
|
||||
if max_device_key < max_seen_key:
|
||||
log.info(
|
||||
" Post-erase reset detected: "
|
||||
"device max key %s < historical max %s "
|
||||
"-- discarding stale (key, ts) state for this session",
|
||||
max_device_key, max_seen_key,
|
||||
)
|
||||
seen_events = {}
|
||||
|
||||
# Note: no early-exit "all already downloaded" short-circuit
|
||||
# here. Without per-event timestamps we cannot tell whether
|
||||
# device_keys ⊆ seen_events.keys() actually means we have
|
||||
# those physical events. get_events() will read 0C on its
|
||||
# skip path and decide per event.
|
||||
|
||||
# Apply max_events cap
|
||||
# stop_idx: when we know the count from list_event_keys, use it as
|
||||
# an upper bound. When list_event_keys failed (device_keys is None),
|
||||
# pass None — get_events will run until the null sentinel naturally.
|
||||
stop_idx: Optional[int] = (current_count - 1) if device_keys is not None else None
|
||||
if self.max_events is not None:
|
||||
cap = self.max_events - 1
|
||||
stop_idx = cap if stop_idx is None else min(stop_idx, cap)
|
||||
if device_keys is not None and self.max_events < current_count:
|
||||
log.warning(
|
||||
" max_events=%d cap: will download events 0-%d only "
|
||||
"(unit has %d total)",
|
||||
self.max_events, stop_idx, current_count,
|
||||
)
|
||||
|
||||
try:
|
||||
# Pass `seen_events` (key → ISO timestamp) so the client can
|
||||
# read 0C on its skip path and only skip 5A when the per-event
|
||||
# timestamp matches what we already have on disk. When force_-
|
||||
# redownload is set, seen_events was already cleared above.
|
||||
#
|
||||
# Filter out empty-string timestamps (legacy v1 entries) — the
|
||||
# client's 0C-on-skip-path only trusts entries with a
|
||||
# populated timestamp; otherwise it falls through to a full
|
||||
# 5A download.
|
||||
skip_dict = {k: ts for k, ts in seen_events.items() if ts}
|
||||
|
||||
all_events = client.get_events(
|
||||
full_waveform=True,
|
||||
stop_after_index=stop_idx,
|
||||
skip_waveform_for_events=skip_dict if skip_dict else None,
|
||||
)
|
||||
|
||||
# New events are those that came back with _a5_frames populated
|
||||
# (= 5A actually ran on this session). Skipped events have
|
||||
# _a5_frames = None because the client matched (key, timestamp)
|
||||
# against skip_dict and bypassed 5A.
|
||||
new_events = [
|
||||
e for e in all_events
|
||||
if getattr(e, "_a5_frames", None)
|
||||
]
|
||||
skipped = len(all_events) - len(new_events)
|
||||
|
||||
log.info(" [OK] Walked %d event(s): %d downloaded, %d skipped (matched (key, ts) in state)",
|
||||
len(all_events), len(new_events), skipped)
|
||||
|
||||
# ── Persist event file + A5 sidecar to the waveform store ──
|
||||
# Saves ride alongside the existing JSON dump so the on-disk
|
||||
# event file and events.json reference the same set of events.
|
||||
waveform_records: dict[str, dict] = {}
|
||||
for ev in new_events:
|
||||
if not ev._a5_frames:
|
||||
continue
|
||||
try:
|
||||
rec = self.store.save(
|
||||
ev,
|
||||
serial=serial or "UNKNOWN",
|
||||
a5_frames=ev._a5_frames,
|
||||
)
|
||||
if ev._waveform_key is not None:
|
||||
waveform_records[ev._waveform_key.hex()] = rec
|
||||
log.info(
|
||||
" [WAVE] saved %s (%d bytes)",
|
||||
rec["filename"], rec["filesize"],
|
||||
)
|
||||
except Exception as exc:
|
||||
key_hex = ev._waveform_key.hex() if ev._waveform_key else "????????"
|
||||
log.warning(
|
||||
" [WARN] Waveform store save failed for %s: %s",
|
||||
key_hex, exc,
|
||||
)
|
||||
|
||||
if new_events:
|
||||
_save_json(
|
||||
session_dir / "events.json",
|
||||
[_event_to_dict(e, waveform_records) for e in new_events],
|
||||
)
|
||||
|
||||
for ev in new_events:
|
||||
pv = ev.peak_values
|
||||
pi = ev.project_info
|
||||
key_hex = ev._waveform_key.hex() if ev._waveform_key else "????????"
|
||||
log.info(
|
||||
" NEW [%s] %s Tran=%.4f Vert=%.4f Long=%.4f VS=%.4f project=%r",
|
||||
key_hex,
|
||||
str(ev.timestamp) if ev.timestamp else "?",
|
||||
pv.tran if pv else 0,
|
||||
pv.vert if pv else 0,
|
||||
pv.long if pv else 0,
|
||||
pv.peak_vector_sum if pv else 0,
|
||||
pi.project if pi else "",
|
||||
)
|
||||
else:
|
||||
log.info(" [OK] No new events since last call-home -- nothing to save")
|
||||
|
||||
# ── Monitor log entries (partial records / continuous monitoring) ──
|
||||
# Browse walk (0A + 1F only) to collect monitor log entries for
|
||||
# recording intervals where no threshold was crossed. This is a
|
||||
# second 1E-based pass over the device's record list, separate from
|
||||
# the get_events() download loop above.
|
||||
log.info(" Collecting monitor log entries (browse walk)...")
|
||||
new_monitor_entries: list[MonitorLogEntry] = []
|
||||
try:
|
||||
new_monitor_entries = client.get_monitor_log_entries(
|
||||
skip_keys=seen_keys if seen_keys else None,
|
||||
)
|
||||
if new_monitor_entries:
|
||||
_save_json(
|
||||
session_dir / "monitor_log.json",
|
||||
[_monitor_log_entry_to_dict(e) for e in new_monitor_entries],
|
||||
)
|
||||
log.info(
|
||||
" [OK] %d new monitor log entry(s) saved",
|
||||
len(new_monitor_entries),
|
||||
)
|
||||
for ml in new_monitor_entries:
|
||||
log.info(
|
||||
" MONLOG [%s] %s → %s (%s)",
|
||||
ml.key,
|
||||
ml.start_time.isoformat() if ml.start_time else "?",
|
||||
ml.stop_time.isoformat() if ml.stop_time else "?",
|
||||
f"{ml.duration_seconds:.0f}s" if ml.duration_seconds is not None else "?s",
|
||||
)
|
||||
else:
|
||||
log.info(" [OK] No new monitor log entries")
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
" [WARN] Monitor log collection failed: %s -- continuing",
|
||||
exc,
|
||||
)
|
||||
|
||||
# ── Persist to SQLite DB ─────────────────────────────────────
|
||||
_session_start = datetime.datetime.now()
|
||||
try:
|
||||
_ev_ins, _ev_skip = self.db.insert_events(
|
||||
new_events,
|
||||
serial=serial or self.peer,
|
||||
session_id=None,
|
||||
waveform_records=waveform_records,
|
||||
)
|
||||
_ml_ins, _ml_skip = self.db.insert_monitor_log(
|
||||
new_monitor_entries, session_id=None
|
||||
)
|
||||
_session_id = self.db.insert_ach_session(
|
||||
serial=serial or self.peer,
|
||||
peer=self.peer,
|
||||
events_downloaded=_ev_ins,
|
||||
monitor_entries=_ml_ins,
|
||||
duration_seconds=(datetime.datetime.now() - _session_start).total_seconds(),
|
||||
session_time=_session_start,
|
||||
)
|
||||
log.info(
|
||||
" [DB] session=%s events +%d (skip %d) monitor +%d (skip %d)",
|
||||
_session_id[:8], _ev_ins, _ev_skip, _ml_ins, _ml_skip,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(" [WARN] DB write failed: %s -- continuing", exc)
|
||||
|
||||
# ── Optional: erase device memory after successful download ────
|
||||
erased_successfully = False
|
||||
if self.clear_after_download and new_events:
|
||||
log.info(" Clearing device memory (--clear-after-download)...")
|
||||
try:
|
||||
client.delete_all_events()
|
||||
log.info(" [OK] Device memory cleared")
|
||||
erased_successfully = True
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
" [WARN] Event deletion failed: %s -- events NOT cleared",
|
||||
exc,
|
||||
)
|
||||
|
||||
# ── Update persistent state ───────────────────────────────────
|
||||
# Build a fresh (key → ISO timestamp) map from THIS session's
|
||||
# results. For each event currently on the device, prefer the
|
||||
# timestamp we just observed (from 0C); fall back to whatever
|
||||
# was already in seen_events for that key (so we don't lose an
|
||||
# entry just because get_events skipped it on the (key, ts)
|
||||
# match path).
|
||||
def _ts_iso(ev) -> str:
|
||||
ts = getattr(ev, "timestamp", None)
|
||||
if ts is None:
|
||||
return ""
|
||||
try:
|
||||
return datetime.datetime(
|
||||
ts.year, ts.month, ts.day,
|
||||
ts.hour or 0, ts.minute or 0, ts.second or 0,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
return str(ts)
|
||||
|
||||
current_events_map: dict[str, str] = {}
|
||||
for ev in all_events:
|
||||
if ev._waveform_key is None:
|
||||
continue
|
||||
key_hex = ev._waveform_key.hex()
|
||||
ts_iso = _ts_iso(ev) or seen_events.get(key_hex, "")
|
||||
current_events_map[key_hex] = ts_iso
|
||||
|
||||
# Monitor-log entries don't have a 0C-style timestamp, but
|
||||
# they DO have a start_time; use that so the monitor-log keys
|
||||
# are properly entered into the (key, ts) map.
|
||||
for ml in new_monitor_entries:
|
||||
key_hex = ml.key
|
||||
ts = ml.start_time
|
||||
ts_iso = ts.isoformat() if ts else seen_events.get(key_hex, "")
|
||||
# If a triggered event already populated this key, keep
|
||||
# whichever has a non-empty timestamp.
|
||||
if key_hex not in current_events_map or not current_events_map[key_hex]:
|
||||
current_events_map[key_hex] = ts_iso
|
||||
|
||||
if erased_successfully:
|
||||
updated_events: dict[str, str] = {}
|
||||
new_max_key = "00000000"
|
||||
log.info(
|
||||
" State reset after erase -- next session will download "
|
||||
"from key 0 (device counter resets after erase)"
|
||||
)
|
||||
else:
|
||||
# Merge: keep prior (key, ts) entries we still have evidence
|
||||
# of (for survivors of any partial failure), plus this
|
||||
# session's authoritative (key, ts) pairs.
|
||||
updated_events = dict(seen_events)
|
||||
updated_events.update(current_events_map)
|
||||
new_max_key = (
|
||||
max(updated_events.keys())
|
||||
if updated_events else max_seen_key
|
||||
)
|
||||
|
||||
state[unit_key] = {
|
||||
"downloaded_events": updated_events,
|
||||
"max_downloaded_key": new_max_key,
|
||||
"last_seen": datetime.datetime.now().isoformat(),
|
||||
"serial": serial,
|
||||
"peer": self.peer,
|
||||
}
|
||||
_save_state(self.state_path, state)
|
||||
|
||||
except Exception as exc:
|
||||
log.error(" [FAIL] Event download failed: %s", exc, exc_info=True)
|
||||
|
||||
# ── Optional: restart monitoring after successful download ─────────
|
||||
if self.restart_monitoring:
|
||||
log.info(" Restarting monitoring on device (--restart-monitoring)...")
|
||||
try:
|
||||
client.start_monitoring()
|
||||
log.info(" [OK] Monitoring restarted")
|
||||
except Exception as exc:
|
||||
log.warning(" [WARN] Failed to restart monitoring: %s", exc)
|
||||
|
||||
finally:
|
||||
raw_rx_fh.close()
|
||||
raw_tx_fh.close()
|
||||
client.close() # closes transport / socket cleanly
|
||||
root_logger.removeHandler(fh)
|
||||
fh.close()
|
||||
|
||||
log.info("Session complete -> %s", session_dir)
|
||||
log.info("="*60)
|
||||
|
||||
|
||||
# ── JSON helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _save_json(path: Path, obj: object) -> None:
|
||||
with open(path, "w") as f:
|
||||
json.dump(obj, f, indent=2, default=str)
|
||||
log.debug("Saved %s", path)
|
||||
|
||||
|
||||
def _device_info_to_dict(d: DeviceInfo) -> dict:
|
||||
cc = d.compliance_config
|
||||
return {
|
||||
"serial": d.serial,
|
||||
"firmware_version": d.firmware_version,
|
||||
"dsp_version": d.dsp_version,
|
||||
"model": d.model,
|
||||
"event_count": d.event_count,
|
||||
# compliance config fields (None if 1A read failed)
|
||||
"setup_name": cc.setup_name if cc else None,
|
||||
"sample_rate": cc.sample_rate if cc else None,
|
||||
"record_time": cc.record_time if cc else None,
|
||||
"trigger_level_geo": cc.trigger_level_geo if cc else None,
|
||||
"alarm_level_geo": cc.alarm_level_geo if cc else None,
|
||||
"geo_adc_scale": cc.geo_adc_scale if cc else None, # hw scale factor (in/s)/V
|
||||
"geo_range": cc.geo_range if cc else None, # 0x01=Normal 10in/s, 0x00=Sensitive 1.25in/s (unconfirmed)
|
||||
"project": cc.project if cc else None,
|
||||
"client": cc.client if cc else None,
|
||||
"operator": cc.operator if cc else None,
|
||||
"sensor_location": cc.sensor_location if cc else None,
|
||||
}
|
||||
|
||||
|
||||
def _event_to_dict(
|
||||
e: Event,
|
||||
waveform_records: Optional[dict[str, dict]] = None,
|
||||
) -> dict:
|
||||
pv = e.peak_values
|
||||
pi = e.project_info
|
||||
peaks = {}
|
||||
if pv:
|
||||
peaks = {
|
||||
"transverse": pv.tran,
|
||||
"vertical": pv.vert,
|
||||
"longitudinal": pv.long,
|
||||
"vector_sum": pv.peak_vector_sum,
|
||||
"mic": pv.micl,
|
||||
}
|
||||
samples = {}
|
||||
if e.raw_samples:
|
||||
samples = {
|
||||
ch: vals[:20] # first 20 sample-sets to keep the file sane
|
||||
for ch, vals in e.raw_samples.items()
|
||||
}
|
||||
samples["__note__"] = "first 20 sample-sets only; see raw_rx.bin for full waveform"
|
||||
|
||||
rec: dict = {}
|
||||
if waveform_records and e._waveform_key is not None:
|
||||
rec = waveform_records.get(e._waveform_key.hex(), {}) or {}
|
||||
|
||||
return {
|
||||
"timestamp": str(e.timestamp) if e.timestamp else None,
|
||||
"project": pi.project if pi else None,
|
||||
"client": pi.client if pi else None,
|
||||
"operator": pi.operator if pi else None,
|
||||
"sensor_location": pi.sensor_location if pi else None,
|
||||
"peaks": peaks,
|
||||
"raw_samples_preview": samples,
|
||||
"blastware_filename": rec.get("filename"),
|
||||
"blastware_filesize": rec.get("filesize"),
|
||||
"a5_pickle_filename": rec.get("a5_pickle_filename"),
|
||||
}
|
||||
|
||||
|
||||
def _monitor_log_entry_to_dict(e: MonitorLogEntry) -> dict:
|
||||
return {
|
||||
"key": e.key,
|
||||
"start_time": e.start_time.isoformat() if e.start_time else None,
|
||||
"stop_time": e.stop_time.isoformat() if e.stop_time else None,
|
||||
"duration_seconds": e.duration_seconds,
|
||||
"serial": e.serial,
|
||||
"geo_threshold_ips": e.geo_threshold_ips,
|
||||
}
|
||||
|
||||
|
||||
# ── Main server loop ───────────────────────────────────────────────────────────
|
||||
|
||||
def serve(args: argparse.Namespace) -> None:
|
||||
output_dir = Path(args.output)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
state_path = output_dir / "ach_state.json"
|
||||
db = SeismoDb(output_dir / "seismo_relay.db")
|
||||
store = WaveformStore(output_dir / "waveforms")
|
||||
|
||||
server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server_sock.bind(("0.0.0.0", args.port))
|
||||
server_sock.listen(5)
|
||||
# Wake up every second so Ctrl-C is handled promptly on Windows.
|
||||
# Without this, accept() blocks indefinitely and ignores KeyboardInterrupt.
|
||||
server_sock.settimeout(1.0)
|
||||
|
||||
max_ev = args.max_events
|
||||
print(f"\n{'='*60}")
|
||||
print(f" ACH inbound server listening on 0.0.0.0:{args.port}")
|
||||
print(f" Output: {output_dir.resolve()}/ach_inbound_<timestamp>/")
|
||||
print(f" State file: {state_path}")
|
||||
print(f" Max events per session: {max_ev if max_ev else 'unlimited'}")
|
||||
print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}")
|
||||
print(f" Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}")
|
||||
print(f" Force re-download all (ignore state): {'YES' if args.force_redownload_all else 'no'}")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n Point your test unit's ACEmanager call-home settings to:")
|
||||
print(f" Remote Host: <this machine's LAN IP>")
|
||||
print(f" Remote Port: {args.port}")
|
||||
print(f"\n Waiting for inbound connections... (Ctrl-C to stop)\n")
|
||||
|
||||
allow_ips = set(args.allow_ips)
|
||||
if allow_ips:
|
||||
print(f" Allowlist: {', '.join(sorted(allow_ips))}")
|
||||
else:
|
||||
print(" Allowlist: NONE -- accepting all IPs (add --allow-ip to restrict)")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
client_sock, addr = server_sock.accept()
|
||||
except socket.timeout:
|
||||
continue # no connection this second; loop back and check for Ctrl-C
|
||||
try:
|
||||
peer_ip = addr[0]
|
||||
peer = f"{addr[0]}:{addr[1]}"
|
||||
|
||||
if allow_ips and peer_ip not in allow_ips:
|
||||
log.info("Rejected connection from %s (not in allowlist)", peer)
|
||||
client_sock.close()
|
||||
continue
|
||||
|
||||
log.info("Accepted connection from %s", peer)
|
||||
session = AchSession(
|
||||
sock=client_sock,
|
||||
peer=peer,
|
||||
output_dir=output_dir,
|
||||
timeout=args.timeout,
|
||||
events_only=args.events_only,
|
||||
max_events=max_ev,
|
||||
state_path=state_path,
|
||||
db=db,
|
||||
store=store,
|
||||
clear_after_download=args.clear_after_download,
|
||||
restart_monitoring=args.restart_monitoring,
|
||||
force_redownload=args.force_redownload_all,
|
||||
)
|
||||
t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}")
|
||||
t.start()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.error("Accept error: %s", exc)
|
||||
finally:
|
||||
server_sock.close()
|
||||
print("\nServer stopped.")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Minimal inbound ACH server — speak BW protocol to calling MiniMate Plus units.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
p.add_argument(
|
||||
"--port", "-p",
|
||||
type=int,
|
||||
default=12345,
|
||||
help="Port to listen on (default: 12345).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output", "-o",
|
||||
default=str(Path(__file__).parent / "captures"),
|
||||
metavar="DIR",
|
||||
help="Directory to write session captures (default: bridges/captures/).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--timeout", "-t",
|
||||
type=float,
|
||||
default=30.0,
|
||||
help="Protocol receive timeout in seconds (default: 30.0).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--events-only",
|
||||
action="store_true",
|
||||
help="Skip the device-info step and go straight to event download.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-events",
|
||||
type=int,
|
||||
default=None,
|
||||
metavar="N",
|
||||
help=(
|
||||
"Safety cap: download at most N events per session (default: unlimited). "
|
||||
"Useful if a unit has many old events stored — prevents a very long first run."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--allow-ip",
|
||||
metavar="IP",
|
||||
action="append",
|
||||
dest="allow_ips",
|
||||
default=[],
|
||||
help=(
|
||||
"Only accept connections from this IP address (repeat for multiple). "
|
||||
"Example: --allow-ip 63.43.212.232 "
|
||||
"If not specified, all IPs are accepted (not recommended for public servers)."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--restart-monitoring",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"After downloading events, send SUB 0x96 (start monitoring) before "
|
||||
"disconnecting. Required for RV55 units whose firmware does not assert "
|
||||
"DCD on disconnect — without this the unit stays idle after a call-home."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--clear-after-download",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"After successfully downloading new events, erase all events from the "
|
||||
"device memory (SUB 0xA3 → 0x1C → 0x06 → 0xA2 sequence, confirmed from "
|
||||
"4-11-26 MITM capture). Only fires when at least one new event was saved. "
|
||||
"This mirrors the standard Blastware ACH workflow."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--force-redownload-all",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Manual override: ignore ach_state.json's downloaded_events map "
|
||||
"for this session and re-download every event currently on the "
|
||||
"device, regardless of (key, timestamp) match. Useful when state "
|
||||
"has become inconsistent with the on-disk waveform store / DB."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Enable debug logging.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
)
|
||||
try:
|
||||
serve(args)
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopped.")
|
||||
+34
-29
@@ -58,16 +58,24 @@ class BridgeGUI(tk.Tk):
|
||||
tk.Entry(self, textvariable=self.logdir_var, width=24).grid(row=1, column=3, sticky="we", **pad)
|
||||
tk.Button(self, text="Browse", command=self._choose_dir).grid(row=1, column=4, sticky="w", **pad)
|
||||
|
||||
# Row 2: Raw taps
|
||||
self.raw_bw_var = tk.StringVar(value="")
|
||||
self.raw_s3_var = tk.StringVar(value="")
|
||||
tk.Checkbutton(self, text="Save BW->S3 raw", command=self._toggle_raw_bw, onvalue="1", offvalue="").grid(row=2, column=0, sticky="w", **pad)
|
||||
tk.Entry(self, textvariable=self.raw_bw_var, width=28).grid(row=2, column=1, columnspan=3, sticky="we", **pad)
|
||||
tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_bw_var, "bw")).grid(row=2, column=4, **pad)
|
||||
# Row 2: Raw taps — ON by default; "auto" = timestamped name; blank checkbox = disabled
|
||||
self.raw_bw_enabled = tk.IntVar(value=1)
|
||||
self.raw_s3_enabled = tk.IntVar(value=1)
|
||||
# Path fields: empty means "auto" (bridge picks a timestamped name)
|
||||
self.raw_bw_path_var = tk.StringVar(value="")
|
||||
self.raw_s3_path_var = tk.StringVar(value="")
|
||||
|
||||
tk.Checkbutton(self, text="Save S3->BW raw", command=self._toggle_raw_s3, onvalue="1", offvalue="").grid(row=3, column=0, sticky="w", **pad)
|
||||
tk.Entry(self, textvariable=self.raw_s3_var, width=28).grid(row=3, column=1, columnspan=3, sticky="we", **pad)
|
||||
tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_s3_var, "s3")).grid(row=3, column=4, **pad)
|
||||
tk.Checkbutton(self, text="BW→S3 raw (auto)", variable=self.raw_bw_enabled,
|
||||
command=self._toggle_raw_bw).grid(row=2, column=0, sticky="w", **pad)
|
||||
tk.Entry(self, textvariable=self.raw_bw_path_var, width=28,
|
||||
fg="grey").grid(row=2, column=1, columnspan=3, sticky="we", **pad)
|
||||
tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_bw_path_var, "bw")).grid(row=2, column=4, **pad)
|
||||
|
||||
tk.Checkbutton(self, text="S3→BW raw (auto)", variable=self.raw_s3_enabled,
|
||||
command=self._toggle_raw_s3).grid(row=3, column=0, sticky="w", **pad)
|
||||
tk.Entry(self, textvariable=self.raw_s3_path_var, width=28,
|
||||
fg="grey").grid(row=3, column=1, columnspan=3, sticky="we", **pad)
|
||||
tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_s3_path_var, "s3")).grid(row=3, column=4, **pad)
|
||||
|
||||
# Row 4: Status + buttons
|
||||
self.status_var = tk.StringVar(value="Idle")
|
||||
@@ -102,13 +110,11 @@ class BridgeGUI(tk.Tk):
|
||||
var.set(filename)
|
||||
|
||||
def _toggle_raw_bw(self) -> None:
|
||||
if not self.raw_bw_var.get():
|
||||
# default name
|
||||
self.raw_bw_var.set(os.path.join(self.logdir_var.get(), "raw_bw.bin"))
|
||||
# Checkbox toggled — no path action needed; enabled state drives the flag.
|
||||
pass
|
||||
|
||||
def _toggle_raw_s3(self) -> None:
|
||||
if not self.raw_s3_var.get():
|
||||
self.raw_s3_var.set(os.path.join(self.logdir_var.get(), "raw_s3.bin"))
|
||||
pass
|
||||
|
||||
def start_bridge(self) -> None:
|
||||
if self.process and self.process.poll() is None:
|
||||
@@ -126,23 +132,22 @@ class BridgeGUI(tk.Tk):
|
||||
|
||||
args = [sys.executable, BRIDGE_PATH, "--bw", bw, "--s3", s3, "--baud", baud, "--logdir", logdir]
|
||||
|
||||
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
# Raw tap flags.
|
||||
# Checkbox on + empty path → pass "auto" (bridge generates timestamped name).
|
||||
# Checkbox on + explicit path → pass that path.
|
||||
# Checkbox off → pass "" to disable (overrides bridge's auto default).
|
||||
raw_bw_explicit = self.raw_bw_path_var.get().strip()
|
||||
raw_s3_explicit = self.raw_s3_path_var.get().strip()
|
||||
|
||||
raw_bw = self.raw_bw_var.get().strip()
|
||||
raw_s3 = self.raw_s3_var.get().strip()
|
||||
if self.raw_bw_enabled.get():
|
||||
args += ["--raw-bw", raw_bw_explicit if raw_bw_explicit else "auto"]
|
||||
else:
|
||||
args += ["--raw-bw", ""] # explicit disable
|
||||
|
||||
# If the user left the default generic name, replace with a timestamped one
|
||||
# so each session gets its own file.
|
||||
if raw_bw:
|
||||
if os.path.basename(raw_bw) in ("raw_bw.bin", "raw_bw"):
|
||||
raw_bw = os.path.join(os.path.dirname(raw_bw) or logdir, f"raw_bw_{ts}.bin")
|
||||
self.raw_bw_var.set(raw_bw)
|
||||
args += ["--raw-bw", raw_bw]
|
||||
if raw_s3:
|
||||
if os.path.basename(raw_s3) in ("raw_s3.bin", "raw_s3"):
|
||||
raw_s3 = os.path.join(os.path.dirname(raw_s3) or logdir, f"raw_s3_{ts}.bin")
|
||||
self.raw_s3_var.set(raw_s3)
|
||||
args += ["--raw-s3", raw_s3]
|
||||
if self.raw_s3_enabled.get():
|
||||
args += ["--raw-s3", raw_s3_explicit if raw_s3_explicit else "auto"]
|
||||
else:
|
||||
args += ["--raw-s3", ""] # explicit disable
|
||||
|
||||
try:
|
||||
self.process = subprocess.Popen(
|
||||
|
||||
@@ -93,8 +93,11 @@ class SessionLogger:
|
||||
self._bin_fh = open(bin_path, "ab", buffering=0)
|
||||
self._lock = threading.Lock()
|
||||
# Optional pure-byte taps (no headers). BW=Blastware tx, S3=device tx.
|
||||
# These can be opened/closed on demand via start_raw_capture/stop_raw_capture.
|
||||
self._raw_bw = open(raw_bw_path, "ab", buffering=0) if raw_bw_path else None
|
||||
self._raw_s3 = open(raw_s3_path, "ab", buffering=0) if raw_s3_path else None
|
||||
self._cap_bw_path: Optional[str] = raw_bw_path
|
||||
self._cap_s3_path: Optional[str] = raw_s3_path
|
||||
|
||||
def log_line(self, line: str) -> None:
|
||||
with self._lock:
|
||||
@@ -124,6 +127,43 @@ class SessionLogger:
|
||||
self.log_line(f"[{ts}] [INFO] {msg}")
|
||||
self.bin_write_record(REC_INFO, msg.encode("utf-8", errors="replace"))
|
||||
|
||||
def start_raw_capture(self, label: str, logdir: str) -> tuple:
|
||||
"""Open new raw tap files for a named capture. Returns (bw_path, s3_path)."""
|
||||
ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in label)[:40] if label else ""
|
||||
suffix = f"_{safe}" if safe else ""
|
||||
bw_path = os.path.join(logdir, f"raw_bw_{ts}{suffix}.bin")
|
||||
s3_path = os.path.join(logdir, f"raw_s3_{ts}{suffix}.bin")
|
||||
with self._lock:
|
||||
# Close any previously open taps first
|
||||
if self._raw_bw:
|
||||
self._raw_bw.close()
|
||||
if self._raw_s3:
|
||||
self._raw_s3.close()
|
||||
self._raw_bw = open(bw_path, "ab", buffering=0)
|
||||
self._raw_s3 = open(s3_path, "ab", buffering=0)
|
||||
self._cap_bw_path = bw_path
|
||||
self._cap_s3_path = s3_path
|
||||
self.log_info(f"raw capture started: label={label!r} bw={bw_path} s3={s3_path}")
|
||||
return bw_path, s3_path
|
||||
|
||||
def stop_raw_capture(self) -> tuple:
|
||||
"""Close raw tap files. Returns (bw_path, s3_path) for the capture just closed."""
|
||||
with self._lock:
|
||||
bw = self._cap_bw_path
|
||||
s3 = self._cap_s3_path
|
||||
if self._raw_bw:
|
||||
self._raw_bw.close()
|
||||
self._raw_bw = None
|
||||
if self._raw_s3:
|
||||
self._raw_s3.close()
|
||||
self._raw_s3 = None
|
||||
self._cap_bw_path = None
|
||||
self._cap_s3_path = None
|
||||
if bw:
|
||||
self.log_info(f"raw capture stopped: bw={bw} s3={s3}")
|
||||
return bw, s3
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
try:
|
||||
@@ -291,8 +331,18 @@ def forward_loop(
|
||||
time.sleep(0.002)
|
||||
|
||||
|
||||
def annotation_loop(logger: SessionLogger, stop: threading.Event) -> None:
|
||||
print("[MARK] Type 'm' + Enter to annotate the capture. Ctrl+C to stop.")
|
||||
def annotation_loop(logger: SessionLogger, logdir: str, stop: threading.Event) -> None:
|
||||
"""
|
||||
Reads stdin commands while the bridge runs.
|
||||
|
||||
Commands:
|
||||
m — prompt for a mark label (interactive)
|
||||
CAP_START:<label> — begin a raw tap capture with the given label
|
||||
CAP_STOP — stop the current raw tap capture
|
||||
Responses (printed to stdout, parsed by the GUI):
|
||||
[CAP_START] <bw_path>\\t<s3_path>
|
||||
[CAP_STOP] <bw_path>\\t<s3_path>
|
||||
"""
|
||||
while not stop.is_set():
|
||||
try:
|
||||
line = input()
|
||||
@@ -303,7 +353,21 @@ def annotation_loop(logger: SessionLogger, stop: threading.Event) -> None:
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if line.lower() == "m":
|
||||
if line.startswith("CAP_START:"):
|
||||
label = line[10:].strip()
|
||||
bw_path, s3_path = logger.start_raw_capture(label, logdir)
|
||||
print(f"[CAP_START] {bw_path}\t{s3_path}")
|
||||
sys.stdout.flush()
|
||||
|
||||
elif line == "CAP_STOP":
|
||||
bw_path, s3_path = logger.stop_raw_capture()
|
||||
if bw_path:
|
||||
print(f"[CAP_STOP] {bw_path}\t{s3_path}")
|
||||
else:
|
||||
print("[CAP_STOP] no active capture")
|
||||
sys.stdout.flush()
|
||||
|
||||
elif line.lower() == "m":
|
||||
try:
|
||||
sys.stdout.write(" Label: ")
|
||||
sys.stdout.flush()
|
||||
@@ -315,8 +379,9 @@ def annotation_loop(logger: SessionLogger, stop: threading.Event) -> None:
|
||||
print(f" [MARK written] {label}")
|
||||
else:
|
||||
print(" (empty label — mark cancelled)")
|
||||
|
||||
else:
|
||||
print(" (type 'm' + Enter to annotate)")
|
||||
print(f" (unknown command: {line!r})")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -325,8 +390,14 @@ def main() -> int:
|
||||
ap.add_argument("--s3", default="COM5", help="S3-side COM port (default: COM5)")
|
||||
ap.add_argument("--baud", type=int, default=38400, help="Baud rate (default: 38400)")
|
||||
ap.add_argument("--logdir", default=".", help="Directory to write session logs into (default: .)")
|
||||
ap.add_argument("--raw-bw", default=None, help="Optional file to append raw bytes sent from BW->S3 (no headers)")
|
||||
ap.add_argument("--raw-s3", default=None, help="Optional file to append raw bytes sent from S3->BW (no headers)")
|
||||
ap.add_argument("--raw-bw", default="auto",
|
||||
help="File to append raw bytes sent from BW->S3 (no headers). "
|
||||
"Default 'auto' generates a timestamped name in --logdir. "
|
||||
"Pass an empty string to disable.")
|
||||
ap.add_argument("--raw-s3", default="auto",
|
||||
help="File to append raw bytes sent from S3->BW (no headers). "
|
||||
"Default 'auto' generates a timestamped name in --logdir. "
|
||||
"Pass an empty string to disable.")
|
||||
ap.add_argument("--quiet", action="store_true", help="No console heartbeat output")
|
||||
ap.add_argument("--status-every", type=float, default=0.0, help="Seconds between console heartbeat lines (default: 0 = off)")
|
||||
args = ap.parse_args()
|
||||
@@ -349,12 +420,16 @@ def main() -> int:
|
||||
# If raw tap flags were passed without a path (bare --raw-bw / --raw-s3),
|
||||
# or if the sentinel value "auto" is used, generate a timestamped name.
|
||||
# If a specific path was provided, use it as-is (caller's responsibility).
|
||||
raw_bw_path = args.raw_bw
|
||||
raw_s3_path = args.raw_s3
|
||||
if raw_bw_path in (None, "", "auto"):
|
||||
raw_bw_path = os.path.join(args.logdir, f"raw_bw_{ts}.bin") if args.raw_bw is not None else None
|
||||
if raw_s3_path in (None, "", "auto"):
|
||||
raw_s3_path = os.path.join(args.logdir, f"raw_s3_{ts}.bin") if args.raw_s3 is not None else None
|
||||
# Resolve raw tap paths.
|
||||
# "auto" (default) → timestamped file in logdir (always captured).
|
||||
# Explicit path → use verbatim.
|
||||
# None or "" → disabled (pass --raw-bw "" to suppress capture).
|
||||
raw_bw_path: Optional[str] = args.raw_bw if args.raw_bw else None
|
||||
raw_s3_path: Optional[str] = args.raw_s3 if args.raw_s3 else None
|
||||
if raw_bw_path == "auto":
|
||||
raw_bw_path = os.path.join(args.logdir, f"raw_bw_{ts}.bin")
|
||||
if raw_s3_path == "auto":
|
||||
raw_s3_path = os.path.join(args.logdir, f"raw_s3_{ts}.bin")
|
||||
|
||||
logger = SessionLogger(log_path, bin_path, raw_bw_path=raw_bw_path, raw_s3_path=raw_s3_path)
|
||||
|
||||
@@ -391,7 +466,7 @@ def main() -> int:
|
||||
t_ann = threading.Thread(
|
||||
target=annotation_loop,
|
||||
name="Annotator",
|
||||
args=(logger, stop),
|
||||
args=(logger, args.logdir, stop),
|
||||
daemon=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
serial_watch.py — Instantel Series-3 serial monitor with S3 frame parsing.
|
||||
|
||||
Taps the RS-232 line between the MiniMate Plus and its modem (RV50/RV55).
|
||||
Saves raw binary captures compatible with the rest of the analysis toolchain,
|
||||
plus a human-readable frame log.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python bridges/serial_watch.py # interactive COM picker
|
||||
python bridges/serial_watch.py --port COM3 # specify port
|
||||
python bridges/serial_watch.py --port COM3 --ack-ok # reply OK to AT commands
|
||||
# (useful if modem is absent
|
||||
# and you want the device to
|
||||
# proceed past AT negotiation)
|
||||
python bridges/serial_watch.py --list # list available ports
|
||||
|
||||
Output
|
||||
------
|
||||
bridges/captures/serial_<ISO-timestamp>/
|
||||
raw_s3_<ts>.bin — raw bytes from device (feeds directly into S3FrameParser)
|
||||
session_<ts>.log — human-readable frame + control-line log
|
||||
session_<ts>.jsonl — JSON-lines frame log
|
||||
|
||||
The raw_s3_*.bin file is byte-for-byte compatible with the existing capture
|
||||
format used by bridges/parse_capture.py and all analysis scripts.
|
||||
|
||||
What to look for in a call-home capture
|
||||
----------------------------------------
|
||||
1. Does the device talk first after CONNECT, or does it wait?
|
||||
- If raw_s3_*.bin has bytes before any AT/POLL exchange → PUSH protocol
|
||||
- If it stays silent → PULL protocol (same as Blastware manual download)
|
||||
|
||||
2. Look for "Operating System" ASCII at the start — the device sends this 16-byte
|
||||
boot string on cold start before entering DLE-framed mode.
|
||||
|
||||
3. RING/CONNECT from the modem appear as ASCII before the DLE frames — the parser
|
||||
handles these automatically (scans forward to DLE+STX).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import serial
|
||||
from serial.tools import list_ports
|
||||
except ModuleNotFoundError:
|
||||
print(
|
||||
"pyserial not found. Install with:\n python -m pip install pyserial",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Add project root so we can import the frame parser
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from minimateplus.framing import S3FrameParser, S3Frame
|
||||
|
||||
import json
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _ts() -> str:
|
||||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
|
||||
|
||||
|
||||
def _hexdump(b: bytes) -> str:
|
||||
return " ".join(f"{x:02X}" for x in b)
|
||||
|
||||
|
||||
def _printable(b: bytes) -> str:
|
||||
return b.decode("latin1", errors="replace")
|
||||
|
||||
|
||||
_KNOWN_SUBS = {
|
||||
0xA4: "POLL_RSP", 0xA5: "BULK_WAVEFORM_RSP", 0xE0: "ADVANCE_EVENT_RSP",
|
||||
0xE1: "EVENT_IDX_FIRST_RSP", 0xE3: "MONITOR_STATUS_RSP", 0xEA: "SERIAL_NUM_RSP",
|
||||
0xF3: "WAVEFORM_RECORD_RSP", 0xF5: "WAVEFORM_HEADER_RSP", 0xF7: "EVENT_INDEX_RSP",
|
||||
0xF9: "UNK_06_RSP", 0xFE: "DEVICE_INFO_RSP",
|
||||
0x69: "START_MONITOR_ACK", 0x68: "STOP_MONITOR_ACK",
|
||||
0x97: "EVT_IDX_WRITE_ACK", 0x8C: "CONFIRM_B_ACK", 0x8E: "COMPLIANCE_WRITE_ACK",
|
||||
0x8D: "CONFIRM_A_ACK", 0x7D: "TRIGGER_WRITE_ACK", 0x7C: "TRIGGER_CONFIRM_ACK",
|
||||
0x96: "WAVEFORM_WRITE_ACK", 0x8B: "CONFIRM_C_ACK",
|
||||
}
|
||||
|
||||
|
||||
def _label_frame(frame: S3Frame) -> str:
|
||||
name = _KNOWN_SUBS.get(frame.sub, f"UNK_0x{frame.sub:02X}")
|
||||
chk = "✓" if frame.checksum_valid else "✗ BAD_CHK"
|
||||
peek = frame.data[:24].hex() + ("…" if len(frame.data) > 24 else "")
|
||||
return (
|
||||
f"S3 SUB=0x{frame.sub:02X} ({name:<22}) "
|
||||
f"page=0x{frame.page_key:04X} data={len(frame.data):4d}B {chk} {peek}"
|
||||
)
|
||||
|
||||
|
||||
# ── Logger ────────────────────────────────────────────────────────────────────
|
||||
|
||||
class Logger:
|
||||
def __init__(self, log_path: Path, jsonl_path: Path, raw_path: Path) -> None:
|
||||
self._log = log_path.open("a", encoding="utf-8", newline="")
|
||||
self._jl = jsonl_path.open("a", encoding="utf-8", newline="")
|
||||
self._raw = raw_path.open("ab")
|
||||
self._lock = threading.Lock()
|
||||
self._frame_count = 0
|
||||
|
||||
def info(self, msg: str) -> None:
|
||||
line = f"[{_ts()}] INFO | {msg}"
|
||||
with self._lock:
|
||||
print(line)
|
||||
print(line, file=self._log, flush=True)
|
||||
|
||||
def ctrl(self, msg: str) -> None:
|
||||
line = f"[{_ts()}] CTRL | {msg}"
|
||||
with self._lock:
|
||||
print(line)
|
||||
print(line, file=self._log, flush=True)
|
||||
|
||||
def data_hex(self, msg: str) -> None:
|
||||
line = f"[{_ts()}] HEX | {msg}"
|
||||
with self._lock:
|
||||
print(line)
|
||||
print(line, file=self._log, flush=True)
|
||||
|
||||
def data_ascii(self, msg: str) -> None:
|
||||
line = f"[{_ts()}] DATA | {msg}"
|
||||
with self._lock:
|
||||
print(line)
|
||||
print(line, file=self._log, flush=True)
|
||||
|
||||
def frame(self, f: S3Frame) -> None:
|
||||
with self._lock:
|
||||
self._frame_count += 1
|
||||
label = f"[{_ts()}] FRAME | #{self._frame_count:04d} {_label_frame(f)}"
|
||||
print(label)
|
||||
print(label, file=self._log, flush=True)
|
||||
record = {
|
||||
"frame": self._frame_count,
|
||||
"sub": f.sub,
|
||||
"page_key": f.page_key,
|
||||
"data_len": len(f.data),
|
||||
"data_hex": f.data.hex(),
|
||||
"checksum_valid": f.checksum_valid,
|
||||
}
|
||||
print(json.dumps(record), file=self._jl, flush=True)
|
||||
|
||||
def write_raw(self, data: bytes) -> None:
|
||||
with self._lock:
|
||||
self._raw.write(data)
|
||||
self._raw.flush()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
for fh in (self._log, self._jl, self._raw):
|
||||
try:
|
||||
fh.flush()
|
||||
fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Control-line monitor thread ───────────────────────────────────────────────
|
||||
|
||||
def _monitor_control_lines(
|
||||
ser: serial.Serial,
|
||||
logger: Logger,
|
||||
stop: threading.Event,
|
||||
interval: float,
|
||||
) -> None:
|
||||
prev = dict(CTS=None, DSR=None, DCD=None, RI=None)
|
||||
try:
|
||||
prev.update(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd)
|
||||
try:
|
||||
prev["RI"] = ser.ri
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.ctrl(f"Init error: {exc}")
|
||||
return
|
||||
|
||||
logger.ctrl(
|
||||
f"Initial: CTS={prev['CTS']} DSR={prev['DSR']} DCD={prev['DCD']} RI={prev['RI']}"
|
||||
)
|
||||
while not stop.is_set():
|
||||
try:
|
||||
cur = dict(CTS=ser.cts, DSR=ser.dsr, DCD=ser.cd, RI=None)
|
||||
try:
|
||||
cur["RI"] = ser.ri
|
||||
except Exception:
|
||||
pass
|
||||
for name, val in cur.items():
|
||||
if val != prev[name]:
|
||||
logger.ctrl(f"{name} → {val}")
|
||||
prev[name] = val
|
||||
except serial.SerialException as exc:
|
||||
logger.ctrl(f"Poll error: {exc}")
|
||||
break
|
||||
stop.wait(interval)
|
||||
|
||||
|
||||
# ── Serial open ───────────────────────────────────────────────────────────────
|
||||
|
||||
_PARITY = {
|
||||
"N": serial.PARITY_NONE, "E": serial.PARITY_EVEN, "O": serial.PARITY_ODD,
|
||||
"M": serial.PARITY_MARK, "S": serial.PARITY_SPACE,
|
||||
}
|
||||
_STOPBITS = {
|
||||
1: serial.STOPBITS_ONE, 1.5: serial.STOPBITS_ONE_POINT_FIVE, 2: serial.STOPBITS_TWO,
|
||||
}
|
||||
|
||||
|
||||
def _open_serial(args: argparse.Namespace, logger: Logger) -> serial.Serial | None:
|
||||
for attempt in range(1, args.open_retries + 2):
|
||||
logger.info(
|
||||
f"Opening {args.port} @ {args.baud},{args.bytesize}{args.parity}{args.stopbits} "
|
||||
f"rtscts={args.rtscts} xonxoff={args.xonxoff} dsrdtr={args.dsrdtr} "
|
||||
f"(attempt {attempt})"
|
||||
)
|
||||
try:
|
||||
ser = serial.Serial(
|
||||
port=args.port,
|
||||
baudrate=args.baud,
|
||||
bytesize=args.bytesize,
|
||||
parity=_PARITY[args.parity],
|
||||
stopbits=_STOPBITS[args.stopbits],
|
||||
timeout=args.timeout,
|
||||
xonxoff=args.xonxoff,
|
||||
rtscts=args.rtscts,
|
||||
dsrdtr=args.dsrdtr,
|
||||
write_timeout=0,
|
||||
)
|
||||
try:
|
||||
ser.setDTR(args.dtr == "on")
|
||||
ser.setRTS(args.rts == "on")
|
||||
logger.ctrl(f"Set DTR={args.dtr} RTS={args.rts}")
|
||||
except Exception as exc:
|
||||
logger.ctrl(f"DTR/RTS set failed: {exc}")
|
||||
|
||||
if args.send_break > 0:
|
||||
try:
|
||||
ser.break_condition = True
|
||||
time.sleep(args.send_break / 1000.0)
|
||||
ser.break_condition = False
|
||||
logger.ctrl(f"BREAK held {args.send_break} ms")
|
||||
except Exception as exc:
|
||||
logger.ctrl(f"BREAK failed: {exc}")
|
||||
|
||||
return ser
|
||||
|
||||
except serial.SerialException as exc:
|
||||
logger.info(f"Open failed: {exc}")
|
||||
if attempt <= args.open_retries:
|
||||
time.sleep(args.open_retry_delay)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ── Port picker ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _list_ports() -> list:
|
||||
ports = list(list_ports.comports())
|
||||
if not ports:
|
||||
print("No serial ports found.")
|
||||
return []
|
||||
print("Available serial ports:")
|
||||
for i, p in enumerate(ports, 1):
|
||||
print(f" {i:2d}) {p.device:<12} {p.description or ''}")
|
||||
return ports
|
||||
|
||||
|
||||
def _pick_port() -> str:
|
||||
ports = _list_ports()
|
||||
if not ports:
|
||||
sys.exit(1)
|
||||
if len(ports) == 1:
|
||||
print(f"Auto-selecting: {ports[0].device}")
|
||||
return ports[0].device
|
||||
while True:
|
||||
sel = input("Select port (number or name, e.g. COM3): ").strip()
|
||||
if sel.isdigit() and 1 <= int(sel) <= len(ports):
|
||||
return ports[int(sel) - 1].device
|
||||
for p in ports:
|
||||
if p.device.upper() == sel.upper():
|
||||
return p.device
|
||||
print("Not recognised. Enter list number or exact port name.")
|
||||
|
||||
|
||||
# ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Monitor Instantel Series-3 serial traffic with S3 frame parsing."
|
||||
)
|
||||
ap.add_argument("--port", "-p",
|
||||
help="COM port (e.g. COM3). Omit to be prompted.")
|
||||
ap.add_argument("--baud", "-b", type=int, default=38400)
|
||||
ap.add_argument("--bytesize", type=int, choices=[5, 6, 7, 8], default=8)
|
||||
ap.add_argument("--parity", choices=["N", "E", "O", "M", "S"], default="N")
|
||||
ap.add_argument("--stopbits", type=float, choices=[1, 1.5, 2], default=1)
|
||||
ap.add_argument("--rtscts", action="store_true")
|
||||
ap.add_argument("--xonxoff", action="store_true")
|
||||
ap.add_argument("--dsrdtr", action="store_true")
|
||||
ap.add_argument("--dtr", choices=["on", "off"], default="on")
|
||||
ap.add_argument("--rts", choices=["on", "off"], default="on")
|
||||
ap.add_argument("--send-break", type=int, default=0,
|
||||
help="Hold BREAK for N ms after open.")
|
||||
ap.add_argument("--show", choices=["ascii", "hex", "both", "frames"],
|
||||
default="frames",
|
||||
help="'frames' (default) shows only parsed S3 frames. "
|
||||
"'ascii'/'hex'/'both' also show raw bytes.")
|
||||
ap.add_argument("--encoding", default="latin1")
|
||||
ap.add_argument("--read-chunk", type=int, default=4096)
|
||||
ap.add_argument("--timeout", type=float, default=0.05)
|
||||
ap.add_argument("--poll-lines-interval", type=float, default=0.2)
|
||||
ap.add_argument("--open-retries", type=int, default=0)
|
||||
ap.add_argument("--open-retry-delay", type=float, default=0.8)
|
||||
ap.add_argument("--ack-ok", action="store_true",
|
||||
help="Auto-reply OK to AT* commands (except ATDT). "
|
||||
"Useful for testing without a real modem.")
|
||||
ap.add_argument("--list", action="store_true",
|
||||
help="List available serial ports and exit.")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.list:
|
||||
_list_ports()
|
||||
return
|
||||
|
||||
args.port = args.port or _pick_port()
|
||||
|
||||
# Build output paths
|
||||
ts_str = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
out_dir = Path(__file__).parent / "captures" / f"serial_{ts_str}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log_path = out_dir / f"session_{ts_str}.log"
|
||||
jsonl_path = out_dir / f"session_{ts_str}.jsonl"
|
||||
raw_path = out_dir / f"raw_s3_{ts_str}.bin"
|
||||
|
||||
logger = Logger(log_path, jsonl_path, raw_path)
|
||||
logger.info(f"Output directory: {out_dir}")
|
||||
logger.info(f"raw_s3 → {raw_path.name} (compatible with parse_capture.py)")
|
||||
|
||||
ser = _open_serial(args, logger)
|
||||
if ser is None:
|
||||
logger.info("Could not open serial port. Exiting.")
|
||||
logger.close()
|
||||
sys.exit(1)
|
||||
|
||||
s3_parser = S3FrameParser()
|
||||
rx_buf = bytearray()
|
||||
stop_evt = threading.Event()
|
||||
|
||||
ctrl_thread = threading.Thread(
|
||||
target=_monitor_control_lines,
|
||||
args=(ser, logger, stop_evt, args.poll_lines_interval),
|
||||
daemon=True,
|
||||
)
|
||||
ctrl_thread.start()
|
||||
logger.info("Monitoring started. Waiting for call-home. Press Ctrl+C to stop.")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
data = ser.read(args.read_chunk)
|
||||
except serial.SerialException as exc:
|
||||
logger.info(f"Read error: {exc}")
|
||||
break
|
||||
|
||||
if not data:
|
||||
continue
|
||||
|
||||
# 1. Save raw bytes
|
||||
logger.write_raw(data)
|
||||
|
||||
# 2. Optional raw display
|
||||
if args.show in ("ascii", "both"):
|
||||
txt = _printable(data)
|
||||
for line in txt.splitlines():
|
||||
logger.data_ascii(line)
|
||||
if args.show in ("hex", "both"):
|
||||
logger.data_hex(_hexdump(data))
|
||||
|
||||
# 3. Parse S3 frames
|
||||
for byte in data:
|
||||
result = s3_parser.feed(bytes([byte]))
|
||||
if result:
|
||||
frames = result if isinstance(result, list) else [result]
|
||||
for f in frames:
|
||||
logger.frame(f)
|
||||
|
||||
# 4. AT command handling for --ack-ok
|
||||
if args.ack_ok:
|
||||
rx_buf.extend(data)
|
||||
while b"\r" in rx_buf or b"\n" in rx_buf:
|
||||
for sep in (b"\r", b"\n"):
|
||||
idx = rx_buf.find(sep)
|
||||
if idx != -1:
|
||||
line_bytes = bytes(rx_buf[:idx])
|
||||
del rx_buf[:idx + 1]
|
||||
break
|
||||
else:
|
||||
break
|
||||
|
||||
line_str = line_bytes.decode("latin1", errors="ignore").strip().upper()
|
||||
if line_str.startswith("AT") and not line_str.startswith("ATDT"):
|
||||
try:
|
||||
ser.write(b"\r\nOK\r\n")
|
||||
ser.flush()
|
||||
logger.info(f"AT ack: {line_str!r} → OK")
|
||||
except Exception as exc:
|
||||
logger.info(f"AT ack write failed: {exc}")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Ctrl+C — stopping.")
|
||||
|
||||
finally:
|
||||
stop_evt.set()
|
||||
try:
|
||||
ser.close()
|
||||
except Exception:
|
||||
pass
|
||||
ctrl_thread.join(timeout=1.0)
|
||||
logger.info(f"Capture saved to: {out_dir}")
|
||||
logger.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://terra-mechanics.com/schemas/seismo-relay/device-config/v1",
|
||||
"title": "MiniMate Plus Device Config",
|
||||
"description": "Writable configuration fields for an Instantel MiniMate Plus seismograph, as exposed by the seismo-relay SFM API (POST /device/config). All fields are optional — only supplied fields are written; all others are round-tripped from the device.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
|
||||
"properties": {
|
||||
|
||||
"sample_rate": {
|
||||
"title": "Sample Rate",
|
||||
"description": "ADC sample rate in samples per second. Must be one of the three supported rates.",
|
||||
"type": "integer",
|
||||
"enum": [1024, 2048, 4096],
|
||||
"examples": [1024]
|
||||
},
|
||||
|
||||
"record_time": {
|
||||
"title": "Record Time",
|
||||
"description": "Waveform record duration in seconds. Typical values are 1.0–15.0 s. The device stores this as a 32-bit IEEE 754 float.",
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 60.0,
|
||||
"examples": [3.0]
|
||||
},
|
||||
|
||||
"trigger_level_geo": {
|
||||
"title": "Trigger Level (Geo)",
|
||||
"description": "Geophone trigger threshold in in/s. Event recording begins when any geo channel exceeds this level.",
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"examples": [0.5]
|
||||
},
|
||||
|
||||
"alarm_level_geo": {
|
||||
"title": "Alarm Level (Geo)",
|
||||
"description": "Geophone alarm threshold in in/s. An alarm is flagged when any geo channel exceeds this level.",
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"examples": [1.0]
|
||||
},
|
||||
|
||||
"max_range_geo": {
|
||||
"title": "Max Range (Geo)",
|
||||
"description": "Full-scale calibration constant for geo channels in in/s. This is a factory-calibrated value — only modify if you have a calibration certificate. Default for MiniMate Plus is approximately 6.206 in/s.",
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"examples": [6.206]
|
||||
},
|
||||
|
||||
"project": {
|
||||
"title": "Project",
|
||||
"description": "Project name or description. Stored in the compliance config block and echoed on event reports. Max 41 ASCII characters.",
|
||||
"type": "string",
|
||||
"maxLength": 41,
|
||||
"examples": ["Bridge Inspection 2026"]
|
||||
},
|
||||
|
||||
"client_name": {
|
||||
"title": "Client",
|
||||
"description": "Client or company name. Max 41 ASCII characters.",
|
||||
"type": "string",
|
||||
"maxLength": 41,
|
||||
"examples": ["City of Portland"]
|
||||
},
|
||||
|
||||
"operator": {
|
||||
"title": "Operator",
|
||||
"description": "Operator or technician name. Stored as 'User Name:' in the device. Max 41 ASCII characters.",
|
||||
"type": "string",
|
||||
"maxLength": 41,
|
||||
"examples": ["Brian Harrison"]
|
||||
},
|
||||
|
||||
"seis_loc": {
|
||||
"title": "Sensor Location",
|
||||
"description": "Sensor location description. Stored as 'Seis Loc:' in the device. Max 41 ASCII characters.",
|
||||
"type": "string",
|
||||
"maxLength": 41,
|
||||
"examples": ["South Abutment — 3 m from blast"]
|
||||
},
|
||||
|
||||
"notes": {
|
||||
"title": "Extended Notes",
|
||||
"description": "Free-form notes. Stored as 'Extended Notes' in the device. Max 41 ASCII characters.",
|
||||
"type": "string",
|
||||
"maxLength": 41,
|
||||
"examples": ["Pre-blast baseline, no charges"]
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
"examples": [
|
||||
{
|
||||
"project": "Bridge Inspection 2026",
|
||||
"client_name": "City of Portland",
|
||||
"operator": "Brian Harrison",
|
||||
"seis_loc": "South Abutment",
|
||||
"notes": "Pre-blast baseline"
|
||||
},
|
||||
{
|
||||
"sample_rate": 1024,
|
||||
"record_time": 3.0,
|
||||
"trigger_level_geo": 0.5,
|
||||
"alarm_level_geo": 1.0
|
||||
},
|
||||
{
|
||||
"sample_rate": 2048,
|
||||
"record_time": 5.0,
|
||||
"trigger_level_geo": 0.25,
|
||||
"alarm_level_geo": 0.75,
|
||||
"project": "Quarry Blast Monitoring",
|
||||
"client_name": "Acme Quarry LLC",
|
||||
"operator": "Brian Harrison",
|
||||
"seis_loc": "Nearest Structure — East Wall",
|
||||
"notes": "Production blast series B"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Parse SUB 0x1C (monitoring status) response frames.
|
||||
|
||||
SUB 0x1C returns device monitoring status with different payload sizes depending on state:
|
||||
- IDLE (not monitoring): 58 bytes with full details
|
||||
- MONITORING (actively streaming): 12 bytes condensed format
|
||||
"""
|
||||
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonitoringStatus:
|
||||
"""Parsed SUB 0x1C response fields."""
|
||||
|
||||
monitor_mode: int # 0x2c = OFF, 0x00 = ON
|
||||
day: int # 1–31
|
||||
hour: int # 0–23
|
||||
month: int # 1–12
|
||||
year: int # 2000–2100
|
||||
minute: int # 0–59 (uncertain encoding)
|
||||
second: int # 0–59 (uncertain encoding)
|
||||
battery_voltage_v: float # Volts (6–8V typical)
|
||||
memory_total_kb: float # Kilobytes
|
||||
memory_free_kb: float # Kilobytes
|
||||
raw_payload: bytes
|
||||
|
||||
def __str__(self) -> str:
|
||||
mode_str = "OFF" if self.monitor_mode == 0x2c else "ON"
|
||||
date_str = f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
|
||||
time_str = f"{self.hour:02d}:{self.minute:02d}:{self.second:02d}"
|
||||
return (
|
||||
f"MonitoringStatus(\n"
|
||||
f" mode={mode_str} (0x{self.monitor_mode:02x})\n"
|
||||
f" datetime={date_str} {time_str}\n"
|
||||
f" battery={self.battery_voltage_v:.2f}V\n"
|
||||
f" memory=total {self.memory_total_kb:.1f} KB, "
|
||||
f"free {self.memory_free_kb:.1f} KB\n"
|
||||
f")"
|
||||
)
|
||||
|
||||
|
||||
def parse_0x1c_response(data: bytes) -> Optional[MonitoringStatus]:
|
||||
"""
|
||||
Parse a SUB 0x1C response payload (after S3 header removed).
|
||||
|
||||
Args:
|
||||
data: Destuffed payload bytes (without the 5-byte S3 header)
|
||||
|
||||
Returns:
|
||||
MonitoringStatus object, or None if parse fails
|
||||
"""
|
||||
|
||||
if len(data) < 39:
|
||||
# Minimum size for idle response
|
||||
print(f"[!] Payload too short: {len(data)} bytes (need >=39)")
|
||||
return None
|
||||
|
||||
try:
|
||||
monitor_mode = data[0x00]
|
||||
|
||||
day = data[0x0d]
|
||||
hour = data[0x0e]
|
||||
month = data[0x0f]
|
||||
year = struct.unpack('>H', data[0x10:0x12])[0]
|
||||
minute = data[0x12]
|
||||
second = data[0x13]
|
||||
|
||||
# Battery voltage: uint16 BE, divide by 100
|
||||
# At offset [2f:31]
|
||||
voltage_raw = struct.unpack('>H', data[0x2f:0x31])[0]
|
||||
battery_voltage_v = voltage_raw / 100.0
|
||||
|
||||
# Memory total: uint32 BE, in bytes
|
||||
# At offset [31:35]
|
||||
memory_total_bytes = struct.unpack('>I', data[0x31:0x35])[0]
|
||||
memory_total_kb = memory_total_bytes / 1024.0
|
||||
|
||||
# Memory free: uint32 BE, in bytes
|
||||
# At offset [35:39]
|
||||
memory_free_bytes = struct.unpack('>I', data[0x35:0x39])[0]
|
||||
memory_free_kb = memory_free_bytes / 1024.0
|
||||
|
||||
return MonitoringStatus(
|
||||
monitor_mode=monitor_mode,
|
||||
day=day,
|
||||
hour=hour,
|
||||
month=month,
|
||||
year=year,
|
||||
minute=minute,
|
||||
second=second,
|
||||
battery_voltage_v=battery_voltage_v,
|
||||
memory_total_kb=memory_total_kb,
|
||||
memory_free_kb=memory_free_kb,
|
||||
raw_payload=data
|
||||
)
|
||||
|
||||
except (struct.error, IndexError) as e:
|
||||
print(f"[!] Parse error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def hex_dump(data: bytes, offset: int = 0) -> str:
|
||||
"""Pretty-print hex dump of binary data."""
|
||||
lines = []
|
||||
for i in range(0, len(data), 16):
|
||||
chunk = data[i:i+16]
|
||||
hex_str = ' '.join(f'{b:02x}' for b in chunk)
|
||||
ascii_str = ''.join(chr(b) if 32 <= b < 127 else '.' for b in chunk)
|
||||
lines.append(f" {offset+i:04x}: {hex_str:<48} {ascii_str}")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: parse_0x1c_response.py <hex_string_or_file>")
|
||||
print()
|
||||
print("Example (hex string):")
|
||||
print(" python3 parse_0x1c_response.py 2c00000000000000000000000008100407ea00013b2d...")
|
||||
print()
|
||||
print("Example (from capture file, idle frame):")
|
||||
print(" Idle response (58 bytes):")
|
||||
idle_hex = (
|
||||
"2c00000000000000000000000008100407ea00013b2d000000000000"
|
||||
"010107cb00060000010107cb0015000000001002a8000efff2000e9e52ef"
|
||||
)
|
||||
status = parse_0x1c_response(bytes.fromhex(idle_hex))
|
||||
print(hex_dump(bytes.fromhex(idle_hex)))
|
||||
print()
|
||||
if status:
|
||||
print(status)
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
# Parse input
|
||||
input_str = sys.argv[1]
|
||||
|
||||
try:
|
||||
payload = bytes.fromhex(input_str)
|
||||
except ValueError:
|
||||
print(f"[!] Invalid hex string: {input_str}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Parsing {len(payload)} bytes:")
|
||||
print(hex_dump(payload))
|
||||
print()
|
||||
|
||||
status = parse_0x1c_response(payload)
|
||||
if status:
|
||||
print(status)
|
||||
else:
|
||||
print("[!] Failed to parse")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,274 @@
|
||||
# SUB 0x1C — Monitoring Status Response Format
|
||||
|
||||
**Capture file:** `/sessions/intelligent-nice-wright/mnt/seismo-relay/bridges/captures/4-8-26/2ndtry/raw_s3_20260408_015927.bin`
|
||||
|
||||
**Analysis date:** 2026-04-08
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
SUB 0x1C is a monitoring status query that returns different sized responses depending on device state:
|
||||
|
||||
- **IDLE/OFF (unit not monitoring):** 58-byte response with detailed fields
|
||||
- **MONITORING/ON (unit actively monitoring):** 12-byte response with condensed format
|
||||
|
||||
The key fields CONFIRMED from wire capture analysis:
|
||||
|
||||
| Field | Offset | Format | Value (Idle) | Notes |
|
||||
|-------|--------|--------|-------------|-------|
|
||||
| **Monitor Mode** | [00] | uint8 | 0x2c (OFF) | 0x2c = Idle, 0x00 = Monitoring |
|
||||
| **Day** | [0d] | uint8 | 0x08 | 1–31 |
|
||||
| **Hour** | [0e] | uint8 | 0x10 | 0–23 (16 = 4 PM) |
|
||||
| **Month** | [0f] | uint8 | 0x04 | 1–12 (April) |
|
||||
| **Year** | [10:12] | uint16 BE | 0x07ea | 2026 |
|
||||
| **Minute** | [12] | uint8 | 0x00 | 0–59 |
|
||||
| **Second** | [13] | uint8 | 0x01 | 0–59 (but this seems off) |
|
||||
| **Battery Voltage** | [2f:31] | uint16 BE, ÷100 | 0x02a8 | 680 → 6.80V |
|
||||
| **Memory Total** | [31:35] | uint32 BE | 0x000efff2 | 983,026 bytes = 960.0 KB |
|
||||
| **Memory Free** | [35:39] | uint32 BE | 0x000e9e52 | 958,034 bytes = 935.6 KB |
|
||||
|
||||
---
|
||||
|
||||
## Idle Frame (58 bytes) — Full Hex Dump
|
||||
|
||||
```
|
||||
00: 2c 00 00 00 00 00 00 00 00 00 00 00 00 08 10 04 ,...............
|
||||
10: 07 ea 00 01 3b 2d 00 00 00 00 00 00 01 01 07 cb ....;-..........
|
||||
20: 00 06 00 00 01 01 07 cb 00 15 00 00 00 00 10 02 ................
|
||||
30: a8 00 0e ff f2 00 0e 9e 52 ef ........R.
|
||||
```
|
||||
|
||||
### Field Breakdown
|
||||
|
||||
**[00:01] = Monitor Mode**
|
||||
```
|
||||
Offset 00: 0x2c = 44 (decimal)
|
||||
Interpretation: Unit is NOT currently monitoring (idle/off state)
|
||||
Counter-example in monitoring frame: 0x00 (ON state)
|
||||
```
|
||||
|
||||
**[01:0d] = Padding/Reserved (12 bytes of zeros)**
|
||||
```
|
||||
Offsets 01-0c: all 0x00
|
||||
```
|
||||
|
||||
**[0d:12] = Timestamp (5 bytes)**
|
||||
```
|
||||
Offset 0d: 0x08 = 8 → DAY
|
||||
Offset 0e: 0x10 = 16 → HOUR (4 PM)
|
||||
Offset 0f: 0x04 = 4 → MONTH (April)
|
||||
Offset 10-11: 0x07ea → YEAR (big-endian: 2026)
|
||||
= 2026-04-08, 16:??:??
|
||||
```
|
||||
|
||||
**[12:14] = Time (minute/second, ambiguous)**
|
||||
```
|
||||
Offset 12: 0x00 = 0 → Likely MINUTE
|
||||
Offset 13: 0x01 = 1 → Likely SECOND
|
||||
But this seems too low; may be wrong interpretation
|
||||
```
|
||||
|
||||
**[14:16] = Unknown (2 bytes)**
|
||||
```
|
||||
Offset 14: 0x3b = 59 (decimal) - could be seconds?
|
||||
Offset 15: 0x2d = 45 (decimal)
|
||||
```
|
||||
|
||||
**[16:2f] = Unknown/Filler (25 bytes)**
|
||||
```
|
||||
Contains various device-specific configuration or state bytes.
|
||||
Some patterns suggest repeating data structures (e.g., 01 01 07 cb appears twice).
|
||||
```
|
||||
|
||||
**[2f:31] = Battery Voltage (2 bytes, uint16 BE, divide by 100)**
|
||||
```
|
||||
Offset 2f-30: 0x02a8
|
||||
= 680 (decimal)
|
||||
÷ 100 = 6.80 volts
|
||||
Expected: ~6.8V ✓ CONFIRMED
|
||||
```
|
||||
|
||||
**[31:35] = Memory Total (4 bytes, uint32 BE)**
|
||||
```
|
||||
Offset 31-34: 0x000efff2
|
||||
= 983,026 (decimal, bytes)
|
||||
÷ 1024 = 960.0 KB ✓ CONFIRMED
|
||||
(Device spec: ~960 KB)
|
||||
```
|
||||
|
||||
**[35:39] = Memory Free (4 bytes, uint32 BE)**
|
||||
```
|
||||
Offset 35-38: 0x000e9e52
|
||||
= 958,034 (decimal, bytes)
|
||||
÷ 1024 = 935.6 KB ✓ CONFIRMED
|
||||
(Expected: ~936 KB)
|
||||
```
|
||||
|
||||
**[39:3a] = Trailing byte**
|
||||
```
|
||||
Offset 39: 0xef = 239
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitoring Frame (12 bytes) — Condensed Response
|
||||
|
||||
When the unit is actively monitoring, the response shrinks to 12 bytes:
|
||||
|
||||
```
|
||||
00: 00 00 00 00 2c 00 00 00 00 00 00 1f ....,.......
|
||||
```
|
||||
|
||||
### Changes from Idle
|
||||
|
||||
| Field | Idle Frame | Monitoring Frame | Note |
|
||||
|-------|------------|------------------|------|
|
||||
| Monitor Mode | [00] = 0x2c | [04] = 0x2c → may shift or invert | Moved to offset [04]? |
|
||||
| Size | 58 bytes | 12 bytes | Truncated response; only status, no detail |
|
||||
| [0b] | varies | 0x1f | New/different byte at end |
|
||||
|
||||
**Interpretation:**
|
||||
- The response layout changes based on monitoring state
|
||||
- In monitoring mode, many detailed fields are suppressed
|
||||
- The monitor_mode indicator may move or encode differently
|
||||
|
||||
---
|
||||
|
||||
## Date/Time Interpretation
|
||||
|
||||
The timestamp at [0d:12] uses this layout (confirmed from capture):
|
||||
|
||||
```
|
||||
[0d] = DAY (1–31) = 0x08 = 8
|
||||
[0e] = HOUR (0–23) = 0x10 = 16 (4 PM)
|
||||
[0f] = MONTH (1–12) = 0x04 = 4 (April)
|
||||
[10:12] = YEAR (uint16 BE) = 0x07ea = 2026
|
||||
```
|
||||
|
||||
**Timestamp extracted:** 2026-04-08 16:??:??
|
||||
|
||||
Minutes and seconds are less clear:
|
||||
- [12] = 0x00 → possibly minute
|
||||
- [13] = 0x01 → possibly second (but unusually low)
|
||||
- [14] = 0x3b = 59 (redundant second marker?)
|
||||
|
||||
---
|
||||
|
||||
## Voltage Encoding
|
||||
|
||||
Battery voltage is stored as **uint16 big-endian, divide by 100:**
|
||||
|
||||
```
|
||||
[2f:31] = 0x02a8
|
||||
Raw value: 680
|
||||
Voltage: 680 / 100 = 6.80 V
|
||||
Expected: ~6.8V ✓
|
||||
```
|
||||
|
||||
Other attempted decodings (all ruled out):
|
||||
- `÷1000`: 0.680V (too low)
|
||||
- `÷10`: 68V (too high)
|
||||
- float32 BE/LE: no match in range 6–8V
|
||||
- Fixed-point: no other range matched
|
||||
|
||||
---
|
||||
|
||||
## Memory Encoding
|
||||
|
||||
Both fields use **uint32 big-endian, in bytes:**
|
||||
|
||||
```
|
||||
Memory Total:
|
||||
[31:35] = 0x000efff2 = 983,026 bytes = 960.0 KB
|
||||
|
||||
Memory Free:
|
||||
[35:39] = 0x000e9e52 = 958,034 bytes = 935.6 KB
|
||||
|
||||
Sanity check: free < total ✓
|
||||
Free percentage: 935.6 / 960.0 = 97.5% (plausible)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitor Mode Field Transitions
|
||||
|
||||
**Idle/OFF State:**
|
||||
```
|
||||
[00] = 0x2c (decimal 44)
|
||||
```
|
||||
|
||||
**Monitoring/ON State (response shrinks to 12 bytes):**
|
||||
```
|
||||
Byte layout shifts; [04] carries 0x2c or another value
|
||||
Possible interpretation: the byte moves, or encoding inverts
|
||||
```
|
||||
|
||||
**Confirmed behavior:**
|
||||
- When idle: byte [00] = 0x2c, response is 58 bytes
|
||||
- When monitoring: byte position shifts to [04], response is 12 bytes
|
||||
- Value 0x2c appears to mean "OFF" or "not actively streaming"
|
||||
- Value 0x00 appears to mean "ON" or "actively streaming"
|
||||
|
||||
---
|
||||
|
||||
## Unknown Fields (for future analysis)
|
||||
|
||||
The following regions have been observed but their purpose is unclear:
|
||||
|
||||
| Range | Hex (Idle) | Notes |
|
||||
|-------|----------|-------|
|
||||
| [01:0d] | all 0x00 | Padding or reserved? |
|
||||
| [14:16] | 3b 2d | 59, 45 — possibly countdown timers or other state |
|
||||
| [16:2f] | mixed | Appears to contain device configuration snapshots; pattern repeats suggest sub-structures (e.g., trigger levels, calibration dates) |
|
||||
|
||||
---
|
||||
|
||||
## Wire Frame Structure (S3 Format)
|
||||
|
||||
Raw S3 response for SUB 0x1C (response SUB = 0xE3):
|
||||
|
||||
```
|
||||
[DLE=0x10][STX=0x02][destuffed_payload+chk][bare ETX=0x03]
|
||||
|
||||
Destuffed payload:
|
||||
[0] CMD = 0x00
|
||||
[1] flags = 0x10
|
||||
[2] SUB = 0xE3 (response)
|
||||
[3] PAGE_HI = 0x00
|
||||
[4] PAGE_LO = 0x00
|
||||
[5+] data = 58 or 12 bytes (depending on mode)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary Table (Idle/OFF State)
|
||||
|
||||
| Field | Bytes | Value | Interpretation |
|
||||
|-------|-------|-------|------------------|
|
||||
| Monitor Mode | [00] | 0x2c | Device idle (not streaming) |
|
||||
| Reserved | [01:0d] | 0x00×12 | Padding |
|
||||
| **Date/Time** | — | — | — |
|
||||
| Day | [0d] | 0x08 | 8th |
|
||||
| Hour | [0e] | 0x10 | 16 (4 PM) |
|
||||
| Month | [0f] | 0x04 | April |
|
||||
| Year | [10:12] | 0x07ea | 2026 |
|
||||
| Minute | [12] | 0x00 | 00 (uncertain) |
|
||||
| Second | [13] | 0x01 | 01 (uncertain) |
|
||||
| Unknown | [14:2f] | — | 27 bytes of mixed data |
|
||||
| **Battery** | — | — | — |
|
||||
| Voltage | [2f:31] | 0x02a8 | 6.80 V (BE ÷100) |
|
||||
| **Memory** | — | — | — |
|
||||
| Total | [31:35] | 0x000efff2 | 960.0 KB (BE) |
|
||||
| Free | [35:39] | 0x000e9e52 | 935.6 KB (BE) |
|
||||
| Trailer | [39:3a] | 0xef | Unknown (1 byte) |
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Verify minute/second fields** — Compare against multiple captures to confirm [12:14] layout
|
||||
2. **Decode unknown region [16:2f]** — Likely contains trigger levels, calibration dates, alarm thresholds
|
||||
3. **Monitoring mode byte position** — Confirm whether it truly moves to [04] in the monitoring response or if response layout is completely different
|
||||
4. **Min/max voltage limits** — Check if voltage ever deviates from 6.8V to validate encoding
|
||||
5. **Memory dynamics** — Track total/free across sessions to understand flash layout
|
||||
@@ -0,0 +1,225 @@
|
||||
SUB 0x1C MONITORING STATUS RESPONSE — FINAL FIELD LOCATIONS
|
||||
============================================================
|
||||
|
||||
Source: raw_s3_20260408_015927.bin (2ndtry capture)
|
||||
Frames analyzed:
|
||||
- IDLE (OFF): Frame 90 at file offset 4115 (58-byte response)
|
||||
- MONITORING (ON): Frame 106 at file offset 4922 (12-byte response)
|
||||
|
||||
================================================================================
|
||||
IDLE/OFF RESPONSE (58 bytes) — COMPLETE FIELD MAP
|
||||
================================================================================
|
||||
|
||||
HEX DUMP:
|
||||
00: 2c 00 00 00 00 00 00 00 00 00 00 00 00 08 10 04
|
||||
10: 07 ea 00 01 3b 2d 00 00 00 00 00 00 01 01 07 cb
|
||||
20: 00 06 00 00 01 01 07 cb 00 15 00 00 00 00 10 02
|
||||
30: a8 00 0e ff f2 00 0e 9e 52 ef
|
||||
|
||||
CONFIRMED FIELDS:
|
||||
─────────────────────────────────────────────────────────────────
|
||||
|
||||
[00] MONITOR_MODE
|
||||
Value: 0x2c (44 decimal)
|
||||
Meaning: Device is IDLE (not monitoring)
|
||||
When ON: 0x00
|
||||
|
||||
[0d] DAY
|
||||
Value: 0x08 (8 decimal)
|
||||
Range: 1–31
|
||||
Date: 8th
|
||||
|
||||
[0e] HOUR
|
||||
Value: 0x10 (16 decimal)
|
||||
Range: 0–23
|
||||
Interpretation: 4:00 PM (16:00)
|
||||
|
||||
[0f] MONTH
|
||||
Value: 0x04 (4 decimal)
|
||||
Range: 1–12
|
||||
Meaning: April
|
||||
|
||||
[10:12] YEAR (uint16 BE)
|
||||
Value: 0x07ea
|
||||
Decimal: 2026
|
||||
Full date: 2026-04-08
|
||||
|
||||
[12] MINUTE
|
||||
Value: 0x00 (0 decimal)
|
||||
Range: 0–59
|
||||
Note: May have different encoding in other captures
|
||||
|
||||
[13] SECOND
|
||||
Value: 0x01 (1 decimal)
|
||||
Range: 0–59
|
||||
Note: Unusually low; likely indicates sampling at minute turn-over
|
||||
|
||||
[2f:31] BATTERY_VOLTAGE (uint16 BE, ÷100)
|
||||
Raw bytes: 0x02a8
|
||||
Raw decimal: 680
|
||||
Voltage: 680 ÷ 100 = 6.80 V
|
||||
✓ CONFIRMED: Expected ~6.8V
|
||||
Alternative encodings tested and ruled out:
|
||||
- BE/1000: 0.68V (too low)
|
||||
- BE/10: 68V (too high)
|
||||
- float32 BE/LE: no match
|
||||
- Fixed-point variations: no match
|
||||
|
||||
[31:35] MEMORY_TOTAL (uint32 BE, in bytes)
|
||||
Raw bytes: 0x000efff2
|
||||
Decimal: 983,026 bytes
|
||||
Kilobytes: 983,026 ÷ 1024 = 960.0 KB
|
||||
✓ CONFIRMED: Expected ~960 KB
|
||||
|
||||
[35:39] MEMORY_FREE (uint32 BE, in bytes)
|
||||
Raw bytes: 0x000e9e52
|
||||
Decimal: 958,034 bytes
|
||||
Kilobytes: 958,034 ÷ 1024 = 935.6 KB
|
||||
✓ CONFIRMED: Expected ~936 KB
|
||||
Sanity check: 935.6 / 960.0 = 97.5% (plausible)
|
||||
|
||||
UNIDENTIFIED REGIONS:
|
||||
─────────────────────────────────────────────────────────────────
|
||||
|
||||
[01:0d] PADDING/RESERVED (12 bytes)
|
||||
All zeros: 00 00 00 00 00 00 00 00 00 00 00 00
|
||||
|
||||
[14:16] UNKNOWN (2 bytes)
|
||||
Value: 0x3b2d (59, 45)
|
||||
Possibly event countdown or state field
|
||||
|
||||
[16:2f] CONFIGURATION SNAPSHOT (25 bytes)
|
||||
Contains repeating patterns suggesting sub-structures:
|
||||
- Possibly trigger levels
|
||||
- Possibly calibration data
|
||||
- Possibly alarm settings
|
||||
|
||||
[39] TRAILER (1 byte)
|
||||
Value: 0xef (239)
|
||||
Purpose unknown
|
||||
|
||||
================================================================================
|
||||
MONITORING/ON RESPONSE (12 bytes) — CONDENSED FORMAT
|
||||
================================================================================
|
||||
|
||||
HEX DUMP:
|
||||
00: 00 00 00 00 2c 00 00 00 00 00 00 1f
|
||||
|
||||
INTERPRETATION:
|
||||
─────────────────────────────────────────────────────────────────
|
||||
|
||||
When the unit is actively monitoring, the response shrinks to 12 bytes.
|
||||
Response layout appears different from idle format.
|
||||
|
||||
[04] POSSIBLE MONITOR_MODE (shifted position?)
|
||||
Value: 0x2c
|
||||
Note: In idle response this was at [00]
|
||||
|
||||
[0b] TRAILER (1 byte)
|
||||
Value: 0x1f (31 decimal)
|
||||
Different from idle trailer (0xef at [39])
|
||||
|
||||
All other bytes: 0x00 padding
|
||||
|
||||
HYPOTHESIS:
|
||||
When monitoring, the device suppresses detailed fields and returns only:
|
||||
- Monitor mode status (position may shift)
|
||||
- A condensed state indicator
|
||||
|
||||
================================================================================
|
||||
TIME FIELD SUMMARY (3 INTERPRETATIONS)
|
||||
================================================================================
|
||||
|
||||
OBSERVED BYTES:
|
||||
[0d] = 0x08 (day)
|
||||
[0e] = 0x10 (hour)
|
||||
[0f] = 0x04 (month)
|
||||
[10:12] = 0x07ea (year)
|
||||
[12] = 0x00 (minute)
|
||||
[13] = 0x01 (second)
|
||||
|
||||
INTERPRETATION #1 (MOST LIKELY):
|
||||
2026-04-08 16:00:01
|
||||
|
||||
INTERPRETATION #2 (IF BYTES ARE SWAPPED):
|
||||
Could be 2026-04-08 04:10:?? (but less likely)
|
||||
|
||||
INTERPRETATION #3 (IF TIME IS ELSEWHERE):
|
||||
Bytes at [14:16] = 0x3b2d could indicate 59 seconds, 45 ???
|
||||
But structure is unclear
|
||||
|
||||
CONFIDENCE: MEDIUM
|
||||
The date part (day/month/year) is confirmed at 2026-04-08.
|
||||
The hour=16 (4 PM) seems reasonable.
|
||||
Minute=00 and second=01 seem offset but may reflect the sample time.
|
||||
|
||||
================================================================================
|
||||
VOLTAGE ENCODING VERIFICATION
|
||||
================================================================================
|
||||
|
||||
Test: uint16 BE ÷ 100
|
||||
Raw bytes: 0x02a8
|
||||
As BE uint16: 680
|
||||
After ÷100: 6.80 V
|
||||
Expected: ~6.8V ✓ MATCH
|
||||
|
||||
Eliminated alternatives:
|
||||
÷1000: 0.68V ✗ (too low)
|
||||
÷10: 68V ✗ (too high)
|
||||
float32 BE: no 6.8V match ✗
|
||||
float32 LE: no 6.8V match ✗
|
||||
Fixed-point 8.8: no match ✗
|
||||
Fixed-point 16.0: no match ✗
|
||||
|
||||
CONCLUSION: uint16 BE ÷ 100 is correct encoding.
|
||||
|
||||
================================================================================
|
||||
MEMORY ENCODING VERIFICATION
|
||||
================================================================================
|
||||
|
||||
Test: uint32 BE (bytes), convert to KB
|
||||
|
||||
Memory Total:
|
||||
Raw bytes: 0x000efff2
|
||||
As BE uint32: 983,026
|
||||
In KB: 983,026 ÷ 1024 = 960.0 KB
|
||||
Spec: ~960 KB ✓ MATCH
|
||||
|
||||
Memory Free:
|
||||
Raw bytes: 0x000e9e52
|
||||
As BE uint32: 958,034
|
||||
In KB: 958,034 ÷ 1024 = 935.6 KB
|
||||
Spec: ~936 KB ✓ MATCH
|
||||
|
||||
Sanity check: free (935.6) < total (960.0) ✓
|
||||
Usage: (960.0 - 935.6) / 960.0 = 2.5% (plausible)
|
||||
|
||||
CONCLUSION: uint32 BE (in bytes), divide by 1024 for KB.
|
||||
|
||||
================================================================================
|
||||
PYTHON PARSING REFERENCE
|
||||
================================================================================
|
||||
|
||||
from struct import unpack
|
||||
|
||||
data = bytes.fromhex("2c00000000000000000000000008100407ea00013b2d...")
|
||||
|
||||
monitor_mode = data[0x00]
|
||||
day = data[0x0d]
|
||||
hour = data[0x0e]
|
||||
month = data[0x0f]
|
||||
year = unpack('>H', data[0x10:0x12])[0]
|
||||
minute = data[0x12]
|
||||
second = data[0x13]
|
||||
|
||||
voltage_v = unpack('>H', data[0x2f:0x31])[0] / 100.0
|
||||
memory_total_kb = unpack('>I', data[0x31:0x35])[0] / 1024.0
|
||||
memory_free_kb = unpack('>I', data[0x35:0x39])[0] / 1024.0
|
||||
|
||||
print(f"Monitor: {['ON', 'OFF'][monitor_mode == 0x2c]}")
|
||||
print(f"Date: {year:04d}-{month:02d}-{day:02d}")
|
||||
print(f"Time: {hour:02d}:{minute:02d}:{second:02d}")
|
||||
print(f"Battery: {voltage_v:.2f} V")
|
||||
print(f"Memory: {memory_total_kb:.1f} KB total, {memory_free_kb:.1f} KB free")
|
||||
|
||||
================================================================================
|
||||
+634
@@ -0,0 +1,634 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
experiments.py — Protocol minimization experiments for MiniMate Plus.
|
||||
|
||||
Goal: figure out which steps in Blastware's sequences are truly required vs.
|
||||
cargo-culted, so we can build a faster, smarter client.
|
||||
|
||||
Each experiment is self-contained (opens its own TCP connection) and reports
|
||||
PASS / FAIL / INCONCLUSIVE with timing and notes.
|
||||
|
||||
Usage:
|
||||
python experiments.py [--host IP] [--port PORT] [exp1 exp2 ...]
|
||||
|
||||
Run all: python experiments.py
|
||||
Run specific: python experiments.py cold_status fast_event_count no_5a
|
||||
|
||||
Available experiments
|
||||
---------------------
|
||||
cold_status EXP1 Monitor status (1C) with NO prior POLL
|
||||
fast_event_count EXP2 Event count via POLL+08 only — skip identity reads
|
||||
no_5a EXP3 Event record (0C) without bulk waveform stream (5A)
|
||||
skip_1e EXP4 0A/0C directly with cached key — skip initial 1E
|
||||
fewer_polls EXP5 Only 1 POLL before 5A instead of Blastware's 3
|
||||
compliance_only EXP6 Write compliance ONLY (71x3→72), skip event index+trigger+waveform
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING, # experiment output is via print(); set DEBUG for wire trace
|
||||
format="%(asctime)s %(levelname)-7s %(name)-20s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger("experiments")
|
||||
|
||||
# ── Imports ───────────────────────────────────────────────────────────────────
|
||||
|
||||
from minimateplus.transport import TcpTransport
|
||||
from minimateplus.protocol import (
|
||||
MiniMateProtocol,
|
||||
ProtocolError,
|
||||
TimeoutError as ProtoTimeout,
|
||||
SUB_MONITOR_STATUS,
|
||||
SUB_SERIAL_NUMBER,
|
||||
SUB_FULL_CONFIG,
|
||||
SUB_EVENT_INDEX,
|
||||
SUB_COMPLIANCE,
|
||||
SUB_WRITE_CONFIRM_A,
|
||||
SUB_WRITE_CONFIRM_B,
|
||||
)
|
||||
from minimateplus.framing import build_bw_frame, SESSION_RESET
|
||||
from minimateplus.client import (
|
||||
MiniMateClient,
|
||||
_decode_compliance_config_into,
|
||||
_encode_compliance_config,
|
||||
)
|
||||
from minimateplus.models import DeviceInfo
|
||||
|
||||
|
||||
DEFAULT_HOST = "63.43.212.232"
|
||||
DEFAULT_PORT = 9034
|
||||
|
||||
|
||||
# ── Result container ──────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
name: str
|
||||
outcome: str # "PASS" | "FAIL" | "INCONCLUSIVE"
|
||||
elapsed: float = 0.0
|
||||
notes: str = ""
|
||||
details: dict = field(default_factory=dict)
|
||||
|
||||
def __str__(self) -> str:
|
||||
sym = {"PASS": "✅", "FAIL": "❌", "INCONCLUSIVE": "⚠️ "}.get(self.outcome, "?")
|
||||
lines = [f" {sym} {self.outcome:13s} {self.name} ({self.elapsed:.1f}s)"]
|
||||
if self.notes:
|
||||
lines.append(f" {self.notes}")
|
||||
for k, v in self.details.items():
|
||||
lines.append(f" {k}: {v}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── Connection helpers ────────────────────────────────────────────────────────
|
||||
|
||||
def connect_proto(host: str, port: int, timeout: float = 15.0) -> tuple[TcpTransport, MiniMateProtocol]:
|
||||
"""Open a raw TCP connection and return (transport, proto) without any handshake."""
|
||||
t = TcpTransport(host, port)
|
||||
t.connect()
|
||||
proto = MiniMateProtocol(t, recv_timeout=timeout)
|
||||
return t, proto
|
||||
|
||||
|
||||
def connect_client(host: str, port: int, timeout: float = 30.0) -> tuple[MiniMateClient, DeviceInfo]:
|
||||
"""Open a MiniMateClient and run the full connect() handshake."""
|
||||
transport = TcpTransport(host, port)
|
||||
client = MiniMateClient(transport=transport, timeout=timeout)
|
||||
client.open()
|
||||
info = client.connect()
|
||||
return client, info
|
||||
|
||||
|
||||
# ── Experiment runner ─────────────────────────────────────────────────────────
|
||||
|
||||
def run(name: str, fn, *args, **kwargs) -> Result:
|
||||
print(f"\n{'─'*60}")
|
||||
print(f" Running: {name}")
|
||||
print(f"{'─'*60}")
|
||||
t0 = time.time()
|
||||
try:
|
||||
outcome, notes, details = fn(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
outcome = "FAIL"
|
||||
notes = f"Uncaught exception: {exc}"
|
||||
details = {}
|
||||
log.exception("Experiment %s raised:", name)
|
||||
elapsed = time.time() - t0
|
||||
r = Result(name=name, outcome=outcome, elapsed=elapsed, notes=notes, details=details)
|
||||
print(str(r))
|
||||
return r
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP1 — Monitor status (1C) with NO prior POLL
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Blastware always does a full POLL handshake before any other command.
|
||||
# We want to know: can we query SUB 1C (battery, memory, monitoring state)
|
||||
# cold, with only a SESSION_RESET signal and no POLL at all?
|
||||
#
|
||||
# If PASS: status checks become near-instant (no ~1s POLL round-trip).
|
||||
# If FAIL: we need POLL first, but maybe we can cache it.
|
||||
|
||||
def exp_cold_status(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""SUB 1C without any POLL — just SESSION_RESET + 1C probe + 1C data."""
|
||||
t, proto = connect_proto(host, port)
|
||||
try:
|
||||
print(" Sending SESSION_RESET only (no POLL)")
|
||||
t.write(SESSION_RESET)
|
||||
time.sleep(0.1)
|
||||
|
||||
print(" Sending SUB 1C probe (no POLL first)…")
|
||||
rsp_sub = (0xFF - SUB_MONITOR_STATUS) & 0xFF # 0xE3
|
||||
t.write(build_bw_frame(SUB_MONITOR_STATUS, 0x00))
|
||||
probe = proto._recv_one(expected_sub=rsp_sub, timeout=8.0)
|
||||
print(f" 1C probe OK page_key=0x{probe.page_key:04X} data={probe.data.hex()}")
|
||||
|
||||
t.write(build_bw_frame(SUB_MONITOR_STATUS, 0x2C))
|
||||
data_rsp = proto._recv_one(expected_sub=rsp_sub, timeout=8.0)
|
||||
|
||||
section = data_rsp.data
|
||||
print(f" 1C data OK {len(section)} bytes hex: {section.hex()}")
|
||||
|
||||
# Decode battery + memory from the end of the section
|
||||
details = {"raw_bytes": len(section)}
|
||||
if len(section) >= 10:
|
||||
batt_raw = struct.unpack_from(">H", section, len(section) - 10)[0]
|
||||
mem_total = struct.unpack_from(">I", section, len(section) - 8)[0]
|
||||
mem_free = struct.unpack_from(">I", section, len(section) - 4)[0]
|
||||
is_monitoring = (section[1] == 0x10)
|
||||
details["battery_v"] = f"{batt_raw / 100:.2f} V"
|
||||
details["memory_total"] = f"{mem_total:,} bytes"
|
||||
details["memory_free"] = f"{mem_free:,} bytes"
|
||||
details["monitoring"] = is_monitoring
|
||||
print(f" battery={batt_raw/100:.2f}V mem_free={mem_free:,} monitoring={is_monitoring}")
|
||||
|
||||
return "PASS", "SUB 1C responded without any POLL — cold status read works!", details
|
||||
|
||||
except ProtoTimeout:
|
||||
return "FAIL", "Device did not respond to 1C without POLL (timeout)", {}
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"Protocol error: {exc}", {}
|
||||
finally:
|
||||
t.disconnect()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP2 — Fast event count: POLL + SUB 08 only (skip identity reads)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Blastware's connect() does: POLL → 15 → 01 → 1A → 08
|
||||
# We want to know: can we skip 15/01/1A and go straight from POLL to 08?
|
||||
#
|
||||
# Reading identity (15, 01) and full compliance (1A, ~2126 bytes over TCP)
|
||||
# takes several seconds each connect. If we only need event count, skipping
|
||||
# them would be a huge win.
|
||||
#
|
||||
# If PASS: fast status poll = POLL + 08 only (~2 round trips vs ~8+).
|
||||
|
||||
def exp_fast_event_count(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""POLL startup → SUB 08 only, skip serial/config/compliance reads."""
|
||||
t, proto = connect_proto(host, port)
|
||||
try:
|
||||
print(" Running startup (POLL only)…")
|
||||
proto.startup()
|
||||
print(" POLL OK — now reading SUB 08 (event index) directly…")
|
||||
|
||||
idx_raw = proto.read_event_index()
|
||||
print(f" SUB 08 OK {len(idx_raw)} bytes")
|
||||
|
||||
# Try to decode event count from SUB 08 payload
|
||||
# The raw block is 88 bytes; bytes [3:7] may be a count (uint32 BE)
|
||||
details = {"idx_raw_len": len(idx_raw)}
|
||||
if len(idx_raw) >= 7:
|
||||
count_candidate = struct.unpack_from(">I", idx_raw, 3)[0]
|
||||
details["count_candidate"] = count_candidate
|
||||
print(f" idx[3:7] as uint32 BE = {count_candidate} (may or may not be event count)")
|
||||
|
||||
# Also verify we can read 1E without the identity reads having been done
|
||||
print(" Reading 1E (event header) to confirm event access works…")
|
||||
key4, data8 = proto.read_event_first()
|
||||
is_empty = data8[4:8] == b"\x00\x00\x00\x00"
|
||||
details["first_key"] = key4.hex()
|
||||
details["is_empty"] = is_empty
|
||||
print(f" 1E OK key={key4.hex()} empty={is_empty}")
|
||||
|
||||
return "PASS", "POLL+08+1E all work without identity reads (15/01/1A skipped)", details
|
||||
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"Protocol error: {exc}", {}
|
||||
finally:
|
||||
t.disconnect()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP3 — Get event record (0C) without bulk waveform stream (5A)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Blastware's event download = 1E → 0A → 1E-arm → 0C → 1F(dl) → POLL×3 → 5A → 1F(browse)
|
||||
#
|
||||
# The 5A bulk stream is the slow part (several large frames, ~1s+ per event).
|
||||
# We only need 5A for: client, operator, seis_loc, notes (not in 0C).
|
||||
# If you don't need those fields, can we do: 1E → 0A → 0C → 1F(browse) ?
|
||||
#
|
||||
# Two variants tested:
|
||||
# 3a: Skip 1E-arm AND 5A — just 0A → 0C → 1F(browse)
|
||||
# 3b: Include 1E-arm but skip 5A+POLL — 0A → 1E-arm → 0C → 1F(browse)
|
||||
#
|
||||
# If PASS: event peak values available without the slow bulk stream.
|
||||
# If FAIL on 3a but PASS on 3b: 1E-arm required even without 5A.
|
||||
|
||||
def exp_no_5a(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""Event record via 0A→0C without 5A or POLL×3. Tests both with and without 1E-arm."""
|
||||
t, proto = connect_proto(host, port)
|
||||
try:
|
||||
print(" Startup (POLL)…")
|
||||
proto.startup()
|
||||
|
||||
# Get the first event key via 1E
|
||||
key4, data8 = proto.read_event_first()
|
||||
if data8[4:8] == b"\x00\x00\x00\x00":
|
||||
return "INCONCLUSIVE", "Device has no stored events — cannot test", {}
|
||||
print(f" First event key: {key4.hex()}")
|
||||
|
||||
details: dict = {"key": key4.hex()}
|
||||
|
||||
# ── Variant 3a: 0A → 0C → 1F(browse), no 1E-arm ─────────────────────
|
||||
print("\n [3a] 0A → 0C → 1F(browse) (NO 1E-arm, NO 5A)")
|
||||
try:
|
||||
_hdr, rec_len = proto.read_waveform_header(key4)
|
||||
print(f" 0A OK rec_len=0x{rec_len:02X}")
|
||||
record_3a = proto.read_waveform_record(key4)
|
||||
print(f" 0C OK {len(record_3a)} bytes")
|
||||
# Check for recognizable content
|
||||
has_tran = b"Tran" in record_3a
|
||||
has_vert = b"Vert" in record_3a
|
||||
has_long = b"Long" in record_3a
|
||||
print(f" 0C content check: Tran={has_tran} Vert={has_vert} Long={has_long}")
|
||||
details["3a_0c_bytes"] = len(record_3a)
|
||||
details["3a_has_peaks"] = has_tran and has_vert and has_long
|
||||
|
||||
# Now try browse 1F without any 5A
|
||||
key4_next, data8_next = proto.advance_event(browse=True)
|
||||
null_sentinel = data8_next[4:8] == b"\x00\x00\x00\x00"
|
||||
print(f" 1F(browse) → key={key4_next.hex()} null={null_sentinel}")
|
||||
details["3a_1f_ok"] = True
|
||||
details["3a_outcome"] = "PASS"
|
||||
except ProtocolError as exc:
|
||||
print(f" 3a FAILED: {exc}")
|
||||
details["3a_outcome"] = f"FAIL: {exc}"
|
||||
# Try to recover by reconnecting for 3b
|
||||
t.disconnect()
|
||||
t2, proto2 = connect_proto(host, port)
|
||||
proto2.startup()
|
||||
key4, data8 = proto2.read_event_first()
|
||||
if data8[4:8] == b"\x00\x00\x00\x00":
|
||||
return "FAIL", f"3a failed and device empty on retry: {exc}", details
|
||||
t, proto = t2, proto2
|
||||
|
||||
# ── Variant 3b: 0A → 1E-arm → 0C → 1F(browse), no 5A ───────────────
|
||||
print("\n [3b] 0A → 1E-arm(0xFE) → 0C → 1F(browse) (NO POLL×3, NO 5A)")
|
||||
try:
|
||||
_hdr, rec_len = proto.read_waveform_header(key4)
|
||||
print(f" 0A OK rec_len=0x{rec_len:02X}")
|
||||
|
||||
# 1E download-arm (token=0xFE) between 0A and 0C
|
||||
proto.read_event_first(token=0xFE)
|
||||
print(" 1E-arm OK")
|
||||
|
||||
record_3b = proto.read_waveform_record(key4)
|
||||
print(f" 0C OK {len(record_3b)} bytes")
|
||||
has_tran = b"Tran" in record_3b
|
||||
print(f" 0C content check: Tran={has_tran} Vert={b'Vert' in record_3b}")
|
||||
details["3b_0c_bytes"] = len(record_3b)
|
||||
details["3b_has_peaks"] = has_tran
|
||||
|
||||
# Browse 1F without 5A / POLL×3
|
||||
key4_next2, data8_next2 = proto.advance_event(browse=True)
|
||||
null_sentinel2 = data8_next2[4:8] == b"\x00\x00\x00\x00"
|
||||
print(f" 1F(browse) → key={key4_next2.hex()} null={null_sentinel2}")
|
||||
details["3b_1f_ok"] = True
|
||||
details["3b_outcome"] = "PASS"
|
||||
except ProtocolError as exc:
|
||||
print(f" 3b FAILED: {exc}")
|
||||
details["3b_outcome"] = f"FAIL: {exc}"
|
||||
|
||||
# Summarize
|
||||
a_ok = details.get("3a_outcome") == "PASS"
|
||||
b_ok = details.get("3b_outcome") == "PASS"
|
||||
if a_ok:
|
||||
return "PASS", "3a: 0A→0C works with NO 1E-arm and NO 5A. Huge speedup possible!", details
|
||||
elif b_ok:
|
||||
return "PASS", "3b: 0A→1E-arm→0C works without 5A (1E-arm still needed before 0C)", details
|
||||
else:
|
||||
return "FAIL", "Both 3a and 3b failed — 5A may be required for device state", details
|
||||
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"Protocol error during setup: {exc}", {}
|
||||
finally:
|
||||
try:
|
||||
t.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP4 — Skip initial 1E if we already know the event key
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# In Blastware, every session starts with 1E to discover the first key.
|
||||
# But if we already fetched and cached the event keys from a previous session,
|
||||
# can we skip 1E entirely and go straight to 0A(cached_key)?
|
||||
#
|
||||
# Practical use case: we poll the device every N minutes. We already know
|
||||
# all the event keys from last time. On re-connect, can we go direct to 0A?
|
||||
#
|
||||
# If PASS: subsequent polls that don't add new events can skip 1E discovery.
|
||||
|
||||
def exp_skip_1e(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""Get the first event key, disconnect, reconnect, go straight to 0A (skip 1E)."""
|
||||
# Phase 1: get the key
|
||||
t, proto = connect_proto(host, port)
|
||||
try:
|
||||
proto.startup()
|
||||
key4, data8 = proto.read_event_first()
|
||||
if data8[4:8] == b"\x00\x00\x00\x00":
|
||||
return "INCONCLUSIVE", "No events stored — cannot test", {}
|
||||
print(f" Phase 1: got event key = {key4.hex()}")
|
||||
finally:
|
||||
t.disconnect()
|
||||
time.sleep(0.5)
|
||||
|
||||
# Phase 2: fresh connection, skip 1E, go straight to 0A with cached key
|
||||
t2, proto2 = connect_proto(host, port)
|
||||
try:
|
||||
print(" Phase 2: fresh connection — startup + 0A directly (no 1E)")
|
||||
proto2.startup()
|
||||
|
||||
_hdr, rec_len = proto2.read_waveform_header(key4)
|
||||
print(f" 0A OK rec_len=0x{rec_len:02X}")
|
||||
|
||||
record = proto2.read_waveform_record(key4)
|
||||
has_peaks = b"Tran" in record
|
||||
print(f" 0C OK {len(record)} bytes has_peaks={has_peaks}")
|
||||
|
||||
details = {
|
||||
"cached_key": key4.hex(),
|
||||
"0c_bytes": len(record),
|
||||
"has_peaks": has_peaks,
|
||||
}
|
||||
return "PASS", "0A works with cached key — 1E discovery can be skipped on known sessions", details
|
||||
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"0A failed with cached key (device needs 1E first?): {exc}", {"key": key4.hex()}
|
||||
finally:
|
||||
t2.disconnect()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP5 — Fewer POLLs before 5A (try POLL×1 instead of Blastware's POLL×3)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Blastware always sends 3 full POLL probe+data cycles between 1F and 5A.
|
||||
# Each POLL is a round trip. Can we get away with just 1?
|
||||
#
|
||||
# WARNING: If POLL×1 fails, the device may be in a bad state. We try to
|
||||
# recover with an extra POLL×2 and a fresh 5A attempt. Even on failure we
|
||||
# try to leave the device in a usable state.
|
||||
#
|
||||
# Strategy: run the full event sequence up to 1F(download), then try 5A
|
||||
# with only 1 POLL. If 5A responds → PASS. If timeout → try 2 more POLLs
|
||||
# and check if the device recovers.
|
||||
|
||||
def exp_fewer_polls(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""Full sequence to 1F, then only 1 POLL before 5A (Blastware does 3)."""
|
||||
t, proto = connect_proto(host, port)
|
||||
try:
|
||||
proto.startup()
|
||||
|
||||
key4, data8 = proto.read_event_first()
|
||||
if data8[4:8] == b"\x00\x00\x00\x00":
|
||||
return "INCONCLUSIVE", "No events stored — cannot test", {}
|
||||
print(f" Event key: {key4.hex()}")
|
||||
|
||||
# Full setup: 0A → 1E-arm → 0C → 1F(download)
|
||||
_hdr, rec_len = proto.read_waveform_header(key4)
|
||||
print(f" 0A OK rec_len=0x{rec_len:02X}")
|
||||
proto.read_event_first(token=0xFE) # 1E-arm
|
||||
print(" 1E-arm OK")
|
||||
proto.read_waveform_record(key4)
|
||||
print(" 0C OK")
|
||||
arm_key4, _ = proto.advance_event(browse=False) # 1F(download) — arms 5A
|
||||
print(f" 1F(download) OK arm_key={arm_key4.hex()}")
|
||||
|
||||
# Only 1 POLL (Blastware does 3)
|
||||
print(" Sending 1 POLL (instead of 3)…")
|
||||
proto.poll()
|
||||
print(" POLL ok — now probing 5A…")
|
||||
|
||||
try:
|
||||
frames = proto.read_bulk_waveform_stream(key4, stop_after_metadata=True, max_chunks=12)
|
||||
print(f" 5A OK after 1 POLL — {len(frames)} frames received")
|
||||
details = {"poll_count": 1, "frames": len(frames)}
|
||||
return "PASS", "5A works with only 1 POLL (saved 2 round-trips per event)!", details
|
||||
|
||||
except ProtoTimeout:
|
||||
print(" 5A timed out after 1 POLL — device needs more POLLs")
|
||||
# Attempt recovery: send 2 more POLLs and see if 5A then works
|
||||
print(" Attempting recovery: 2 more POLLs…")
|
||||
try:
|
||||
proto.poll()
|
||||
proto.poll()
|
||||
frames2 = proto.read_bulk_waveform_stream(key4, stop_after_metadata=True, max_chunks=12)
|
||||
print(f" 5A worked after total 3 POLLs ({len(frames2)} frames)")
|
||||
return "FAIL", "5A needs 3 POLLs — 1 is not enough (recovery confirmed 3 still works)", {
|
||||
"poll_count_tried": 1, "recovery_polls": 3, "recovery_frames": len(frames2)
|
||||
}
|
||||
except ProtocolError as exc2:
|
||||
return "FAIL", f"5A failed even after 3 total POLLs — device may need reconnect: {exc2}", {}
|
||||
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"Setup failed: {exc}", {}
|
||||
finally:
|
||||
t.disconnect()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# EXP6 — Compliance-only write (71×3→72), skip event index + trigger + waveform
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Blastware's full write sequence: 68→73 | 71×3→72 | 82→83 | 69→74→72
|
||||
# We want to know: can we write ONLY the compliance block (71×3→72)?
|
||||
#
|
||||
# Test procedure:
|
||||
# 1. Read current compliance config (SUB 1A)
|
||||
# 2. Patch the "notes" field to a test marker
|
||||
# 3. Write ONLY 71×3→72 (skip 68, 73, 82, 83, 69, 74, final 72)
|
||||
# 4. Read back (SUB 1A) and verify the change was written
|
||||
# 5. Restore original value
|
||||
#
|
||||
# If PASS: we can push individual config fields without touching event index,
|
||||
# trigger config, or waveform data — huge simplification.
|
||||
# If FAIL: the device needs the full write sequence (may reject partial write).
|
||||
#
|
||||
# SAFETY: We restore original data in a finally block. If the restore write
|
||||
# fails, the device will have the test marker in "notes" — harmless but visible.
|
||||
|
||||
_EXP6_MARKER = "[exp6-test]"
|
||||
|
||||
def exp_compliance_only(host: str, port: int) -> tuple[str, str, dict]:
|
||||
"""Write compliance block alone (71×3→72), verify, and restore."""
|
||||
client, info = connect_client(host, port)
|
||||
original_raw: Optional[bytes] = None
|
||||
try:
|
||||
proto = client._proto
|
||||
if proto is None:
|
||||
return "FAIL", "Could not get protocol handle from client", {}
|
||||
|
||||
# 1. Read current compliance
|
||||
print(" Reading current compliance config (SUB 1A)…")
|
||||
original_raw = proto.read_compliance_config()
|
||||
print(f" Got {len(original_raw)} bytes of compliance config")
|
||||
|
||||
# Find current notes value for display
|
||||
info_obj = DeviceInfo()
|
||||
_decode_compliance_config_into(original_raw, info_obj)
|
||||
cc = info_obj.compliance_config
|
||||
orig_notes = cc.notes if cc else "(unknown)"
|
||||
print(f" Current notes field: {orig_notes!r}")
|
||||
|
||||
# 2. Build modified payload with test marker in notes
|
||||
test_notes = _EXP6_MARKER
|
||||
modified_raw = _encode_compliance_config(
|
||||
original_raw,
|
||||
notes=test_notes,
|
||||
)
|
||||
print(f" Encoded modified compliance payload ({len(modified_raw)} bytes)")
|
||||
print(f" Patching notes: {orig_notes!r} → {test_notes!r}")
|
||||
|
||||
# 3. Write ONLY the compliance block: 71×3 → 72
|
||||
print(" Writing compliance ONLY (71×3→72) — skipping 68/73/82/83/69/74…")
|
||||
proto.write_compliance_config_raw(modified_raw)
|
||||
print(" Write complete — device acked 71×3→72")
|
||||
|
||||
# 4. Read back and verify
|
||||
print(" Reading back compliance config to verify…")
|
||||
readback_raw = proto.read_compliance_config()
|
||||
readback_info = DeviceInfo()
|
||||
_decode_compliance_config_into(readback_raw, readback_info)
|
||||
rb_cc = readback_info.compliance_config
|
||||
readback_notes = rb_cc.notes if rb_cc else "(decode failed)"
|
||||
print(f" Read-back notes: {readback_notes!r}")
|
||||
|
||||
write_worked = (readback_notes == test_notes)
|
||||
print(f" Write verified: {write_worked}")
|
||||
|
||||
details = {
|
||||
"original_notes": orig_notes,
|
||||
"written_notes": test_notes,
|
||||
"readback_notes": readback_notes,
|
||||
"write_verified": write_worked,
|
||||
}
|
||||
|
||||
if write_worked:
|
||||
return "PASS", "Compliance-only write works! No event index or trigger writes needed.", details
|
||||
else:
|
||||
return "FAIL", f"Write was not reflected in read-back (got {readback_notes!r})", details
|
||||
|
||||
except ProtocolError as exc:
|
||||
return "FAIL", f"Protocol error: {exc}", {}
|
||||
|
||||
finally:
|
||||
# Restore original compliance data regardless of outcome
|
||||
if original_raw is not None:
|
||||
print(" Restoring original compliance config…")
|
||||
try:
|
||||
proto2 = client._proto
|
||||
if proto2:
|
||||
proto2.write_compliance_config_raw(
|
||||
_encode_compliance_config(original_raw) # no-op patch = verbatim
|
||||
)
|
||||
print(" Restore complete")
|
||||
else:
|
||||
print(" WARNING: protocol handle gone — could not restore")
|
||||
except Exception as exc_r:
|
||||
print(f" WARNING: restore failed: {exc_r}")
|
||||
client.close()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Registry + main
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
EXPERIMENTS = {
|
||||
"cold_status": ("EXP1", exp_cold_status, "Monitor status (1C) with no POLL"),
|
||||
"fast_event_count": ("EXP2", exp_fast_event_count, "Event count via POLL+08, skip identity reads"),
|
||||
"no_5a": ("EXP3", exp_no_5a, "Event record (0C) without bulk waveform (5A)"),
|
||||
"skip_1e": ("EXP4", exp_skip_1e, "0A/0C with cached key — skip initial 1E"),
|
||||
"fewer_polls": ("EXP5", exp_fewer_polls, "1 POLL before 5A instead of Blastware's 3"),
|
||||
"compliance_only": ("EXP6", exp_compliance_only, "Compliance-only write (71×3→72), no other blocks"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="MiniMate Plus protocol minimization experiments")
|
||||
ap.add_argument("--host", default=DEFAULT_HOST)
|
||||
ap.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
ap.add_argument("--debug", action="store_true", help="Enable DEBUG wire logging")
|
||||
ap.add_argument("experiments", nargs="*",
|
||||
help=f"Which to run (default: all). Choices: {', '.join(EXPERIMENTS)}")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.debug:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
which = args.experiments or list(EXPERIMENTS.keys())
|
||||
unknown = [e for e in which if e not in EXPERIMENTS]
|
||||
if unknown:
|
||||
print(f"Unknown experiments: {unknown}")
|
||||
print(f"Available: {', '.join(EXPERIMENTS)}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n{'═'*60}")
|
||||
print(f" MiniMate Plus Protocol Minimization Experiments")
|
||||
print(f" Target: {args.host}:{args.port}")
|
||||
print(f" Running: {', '.join(which)}")
|
||||
print(f"{'═'*60}")
|
||||
|
||||
results: list[Result] = []
|
||||
for key in which:
|
||||
tag, fn, desc = EXPERIMENTS[key]
|
||||
label = f"{tag}: {desc}"
|
||||
r = run(label, fn, args.host, args.port)
|
||||
results.append(r)
|
||||
time.sleep(1.5) # brief pause between experiments — let device settle
|
||||
|
||||
print(f"\n\n{'═'*60}")
|
||||
print(" SUMMARY")
|
||||
print(f"{'═'*60}")
|
||||
for r in results:
|
||||
sym = {"PASS": "✅", "FAIL": "❌", "INCONCLUSIVE": "⚠️ "}.get(r.outcome, "?")
|
||||
print(f" {sym} {r.outcome:13s} {r.name}")
|
||||
print(f"{'═'*60}")
|
||||
|
||||
passed = sum(1 for r in results if r.outcome == "PASS")
|
||||
failed = sum(1 for r in results if r.outcome == "FAIL")
|
||||
skipped = sum(1 for r in results if r.outcome == "INCONCLUSIVE")
|
||||
print(f" {passed} passed {failed} failed {skipped} inconclusive")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.")
|
||||
sys.exit(0)
|
||||
@@ -20,8 +20,16 @@ Typical usage (TCP / modem):
|
||||
"""
|
||||
|
||||
from .client import MiniMateClient
|
||||
from .models import DeviceInfo, Event
|
||||
from .transport import SerialTransport, TcpTransport
|
||||
from .models import DeviceInfo, Event, MonitorLogEntry
|
||||
from .transport import CapturingTransport, SerialTransport, TcpTransport
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__all__ = ["MiniMateClient", "DeviceInfo", "Event", "SerialTransport", "TcpTransport"]
|
||||
__all__ = [
|
||||
"MiniMateClient",
|
||||
"DeviceInfo",
|
||||
"Event",
|
||||
"MonitorLogEntry",
|
||||
"SerialTransport",
|
||||
"TcpTransport",
|
||||
"CapturingTransport",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,974 @@
|
||||
"""
|
||||
blastware_file.py — Blastware binary file codec for bidirectional interoperability.
|
||||
|
||||
Reads and writes the proprietary Instantel/Blastware file formats:
|
||||
Waveform events (.CE0W, .VM0H, .440, .7M0, etc.) (extension encoding UNKNOWN — see below)
|
||||
.MLG — Monitor log (monitoring session history)
|
||||
|
||||
All waveform formats share a common 22-byte file header prefix and identical
|
||||
internal binary structure (same type tag 00 12 03 00, same STRT record layout).
|
||||
Blastware identifies the file type by extension, not by a magic marker.
|
||||
|
||||
EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22:
|
||||
|
||||
Direct / manual download: AB0 (3-char, no type character)
|
||||
Call-home (ACH) download: AB0W or AB0H (4-char, W=waveform H=histogram)
|
||||
|
||||
AB = 2-char base-36 of (total_seconds % 1296), where
|
||||
total_seconds = (event_local_time − 1985-01-01T00:00:00_local).
|
||||
0 = always literal digit zero.
|
||||
Verified against 3,248 call-home files from a 10-year production archive.
|
||||
|
||||
The 10-year archive contains only ACH files (all end in W or H).
|
||||
Manual Blastware downloads produce 3-char AB0 extensions — same encoding
|
||||
but without the trailing type character.
|
||||
|
||||
Old firmware (S338, 3-char extensions): encoding unknown / same as manual?
|
||||
Micromate Series 4 uses a different scheme (literal datetime in filename).
|
||||
|
||||
─── File structure overview ─────────────────────────────────────────────────────
|
||||
|
||||
Waveform file structure (confirmed from example-events/4-3-26-multi/M529LIY6 (example event)):
|
||||
|
||||
[22B header] [21B STRT record] [body bytes] [26B footer]
|
||||
|
||||
Header (22 bytes):
|
||||
10 00 01 80 00 00 — fixed prefix
|
||||
49 6e 73 74 61 6e 74 65 6c 00 — b'Instantel\x00'
|
||||
07 2c — fixed
|
||||
00 12 03 00 — waveform file type tag (shared by all waveform extensions)
|
||||
|
||||
STRT record (21 bytes, immediately follows header):
|
||||
53 54 52 54 — b'STRT'
|
||||
ff fe — fixed (2 bytes)
|
||||
[key4] — 4-byte waveform event key
|
||||
[key4] — 4-byte waveform event key (repeated)
|
||||
[zeros] — 7 bytes padding
|
||||
[rectime] — uint8 record time in seconds
|
||||
|
||||
Body (variable — reconstructed from A5 frame data):
|
||||
The body bytes are derived from the raw A5 frame wire content, specifically
|
||||
from the DLE-decoded representation of each frame's contribution. See the
|
||||
_frame_body_bytes() helper for the exact algorithm.
|
||||
|
||||
Footer (26 bytes):
|
||||
0e 08
|
||||
[ts1: 8B big-endian timestamp] — start timestamp
|
||||
[ts2: 8B big-endian timestamp] — stop timestamp
|
||||
00 01 00 02 00 00
|
||||
[crc: 2B] — CRC (algorithm unconfirmed; written as 0x00 0x00 placeholder)
|
||||
|
||||
Timestamp format (big-endian, 8 bytes):
|
||||
[day] [month] [year_HI] [year_LO] [0x00] [hour] [min] [sec]
|
||||
|
||||
MLG (monitor log, confirmed from example-events/4-3-26-multi/BE11529.MLG):
|
||||
|
||||
[308B header] [N × 292B records]
|
||||
|
||||
Header (308 bytes):
|
||||
Offset 0x00: 10 00 01 80 00 00 Instantel\x00 07 2c 22 01 0e a0 — fixed (16B)
|
||||
Offset 0x10: ... (unknown structure, written as zeros + serial)
|
||||
Offset 0x2A: serial number (8 bytes, null-padded ASCII, e.g. "BE11529")
|
||||
... zero-padded to 308 bytes total
|
||||
|
||||
Record (292 bytes each):
|
||||
[2B CRC] — unknown algorithm; written as 0x00 0x00
|
||||
22 01 0e 80 — record marker
|
||||
[ts1: 8B big-endian timestamp] — start time
|
||||
[ts2: 8B big-endian timestamp] — stop time (zeros if no stop)
|
||||
[4B flags] — see MLG_FLAGS_* constants below
|
||||
[10B serial] — null-padded serial number ASCII
|
||||
[text] — for trigger records: [0x08][8B ts1_copy] then ASCII "Geo: X.XXX in/s"
|
||||
for monitoring records: b'' (or minimal separator)
|
||||
[zero-padded to 292 bytes]
|
||||
|
||||
─── Critical implementation notes ──────────────────────────────────────────────
|
||||
|
||||
Waveform body reconstruction algorithm (confirmed 2026-04-21 from verification against
|
||||
M529LIY6 (example event) using raw_s3_20260403_153508.bin capture):
|
||||
|
||||
The waveform body bytes come from the A5 frame content, stripped of DLE-framing
|
||||
artifacts. Each A5 frame contributes a different slice of its data section,
|
||||
with DLE+{0x02,0x03,0x04} byte pairs stripped.
|
||||
|
||||
Skip amounts per frame index (offsets into frame.data):
|
||||
A5[0] (probe): data[strt_pos + 21 + 7] (skip header + STRT record)
|
||||
strt_pos found by searching frame.data[7:] for b'STRT';
|
||||
the contribution starts at strt_pos + 21 within data[7:]
|
||||
which equals strt_pos + 21 + 7 within frame.data.
|
||||
A5[1]: data[13] (skip 7-byte frame.data prefix + 6 header bytes)
|
||||
A5[2..N]: data[12] (skip 7-byte frame.data prefix + 5 header bytes)
|
||||
Terminator A5: data[11] (1 byte less than chunk frames; terminator inner header
|
||||
is 4 bytes instead of 5 — confirmed 2026-04-21)
|
||||
|
||||
DLE strip rule (applied AFTER slicing):
|
||||
Strip any 0x10 byte that is immediately followed by 0x02, 0x03, or 0x04.
|
||||
This undoes the DLE-escape that S3FrameParser preserves as literal pairs.
|
||||
Applied to frame.data[skip:] + bytes([frame.chk_byte]) together, then
|
||||
conditionally exclude the trailing chk_byte from the output.
|
||||
|
||||
chk_byte absorption:
|
||||
When frame.data[-1] == 0x10 AND frame.chk_byte ∈ {0x02, 0x03, 0x04},
|
||||
the last byte of frame.data is the DLE prefix of a split DLE+chk pair.
|
||||
Including chk_byte in the strip buffer allows the pair to be stripped as
|
||||
a unit. After stripping, the trailing chk_byte is ALWAYS removed — because
|
||||
_strip_inner_frame_dles keeps the byte after the DLE (the chk_byte value),
|
||||
and that value is the checksum, never payload. This applies to all three
|
||||
cases (chk ∈ {0x02, 0x03, 0x04}) identically.
|
||||
|
||||
MLG CRC:
|
||||
The algorithm that produces the 2-byte CRC at the start of each MLG record
|
||||
is unknown. All examined records use non-zero values that do not match
|
||||
CRC-16/CCITT, CRC-16/IBM, CRC-32 (truncated), word sums, XOR variants, or
|
||||
any of the 40+ polynomial/init combinations tested. The writer emits 0x0000.
|
||||
This produces files that Blastware may reject or display without the CRC check —
|
||||
the exact impact on BW import is unknown (TODO: test).
|
||||
|
||||
─── Public API ──────────────────────────────────────────────────────────────────
|
||||
|
||||
blastware_filename(event, serial)
|
||||
Return the correct Blastware filename for an event (e.g. "M529LIY6.CE0W").
|
||||
Full AB0T extension encoding confirmed 2026-04-22 against 3,248 archive files.
|
||||
Extension matches what Blastware itself would generate for the same event.
|
||||
|
||||
write_blastware_file(event, a5_frames, path)
|
||||
Create a Blastware waveform file from an Event and the full A5 frame list.
|
||||
All waveform extensions share the same binary format — the extension is set
|
||||
by blastware_filename() based on the event timestamp and type.
|
||||
|
||||
read_blastware_file(path) → Event
|
||||
Parse a Blastware waveform file into an Event object with waveform data populated.
|
||||
(Not yet implemented — placeholder raises NotImplementedError.)
|
||||
|
||||
write_mlg(entries, serial, path)
|
||||
Create a .MLG file from a list of MonitorLogEntry objects.
|
||||
|
||||
read_mlg(path) → list[MonitorLogEntry]
|
||||
Parse a .MLG file into MonitorLogEntry objects.
|
||||
(Not yet implemented — placeholder raises NotImplementedError.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from .framing import S3Frame
|
||||
from .models import Event, MonitorLogEntry, Timestamp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ── File header constants ─────────────────────────────────────────────────────
|
||||
|
||||
# Common 16-byte prefix shared by waveform files and MLG (confirmed from binary inspection).
|
||||
_FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c"
|
||||
# = 10 00 01 80 00 00 49 73 74 61 6e 74 65 6c 00 07 2c (17 bytes)
|
||||
# Confirmed breakdown: 10 00 01 80 00 00 = fixed; "Instantel\x00" = 10B; 07 2c = fixed
|
||||
|
||||
# Simpler construction:
|
||||
_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes
|
||||
|
||||
# Waveform file type tag (4 bytes after common prefix) — shared by ALL waveform extensions
|
||||
_WAVEFORM_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6 (example event) — same tag for .CE0W, .VM0H, etc.
|
||||
|
||||
# MLG type tag (4 bytes after common prefix)
|
||||
_MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14
|
||||
|
||||
# Total header sizes
|
||||
_WAVEFORM_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate.
|
||||
# From binary: first 22 bytes = header, then STRT at byte 22.
|
||||
# 17-byte common prefix + 4-byte type tag = 21 bytes. But observed header is 22B.
|
||||
# Checking: 6 fixed + 10 "Instantel\x00" + 2 "07 2c" = 18B prefix, then 4B type tag = 22B.
|
||||
# Re-count: b"\x10\x00\x01\x80\x00\x00" = 6B + b"Instantel\x00" = 10B + b"\x07\x2c" = 2B = 18B prefix.
|
||||
_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 18 bytes
|
||||
_WAVEFORM_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅
|
||||
_MLG_HEADER_SIZE = 308 # confirmed from BE11529.MLG
|
||||
|
||||
# MLG record marker (4 bytes after 2-byte CRC at start of each record)
|
||||
_MLG_RECORD_MARKER = b"\x22\x01\x0e\x80"
|
||||
_MLG_RECORD_SIZE = 292 # bytes per record (confirmed from BE11529.MLG)
|
||||
|
||||
# MLG record flags (4 bytes at record[22:26])
|
||||
# Confirmed from BE11529.MLG binary inspection:
|
||||
MLG_FLAGS_START_ONLY = b"\xff\xff\x00\x00" # monitoring start with no stop
|
||||
MLG_FLAGS_TRIGGER = b"\x01\x00\x02\x00" # triggered event (has ts1 + ts2)
|
||||
MLG_FLAGS_INTERVAL = b"\x02\x00\x00\x00" # monitoring interval (has ts1 + ts2)
|
||||
|
||||
|
||||
# ── Timestamp helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
def _encode_ts_be(ts: Optional[datetime.datetime]) -> bytes:
|
||||
"""
|
||||
Encode a datetime as an 8-byte big-endian Blastware timestamp.
|
||||
|
||||
Format (waveform file and MLG record timestamps):
|
||||
[day][month][year_HI][year_LO][0x00][hour][min][sec]
|
||||
|
||||
Big-endian year confirmed from M529LIY6 (example event) footer:
|
||||
footer bytes [2..9] = 01 04 07 ea 00 00 1c 08
|
||||
→ day=1 month=4 year=0x07ea=2026 hour=0 min=28 sec=8 ✅
|
||||
|
||||
Returns 8 zero bytes if ts is None.
|
||||
"""
|
||||
if ts is None:
|
||||
return bytes(8)
|
||||
return bytes([
|
||||
ts.day,
|
||||
ts.month,
|
||||
(ts.year >> 8) & 0xFF,
|
||||
ts.year & 0xFF,
|
||||
0x00,
|
||||
ts.hour,
|
||||
ts.minute,
|
||||
ts.second,
|
||||
])
|
||||
|
||||
|
||||
def _decode_ts_be(raw: bytes) -> Optional[datetime.datetime]:
|
||||
"""
|
||||
Decode an 8-byte big-endian Blastware timestamp.
|
||||
|
||||
Returns None if the bytes are all zero or structurally invalid.
|
||||
"""
|
||||
if len(raw) < 8 or raw == bytes(8):
|
||||
return None
|
||||
day = raw[0]
|
||||
month = raw[1]
|
||||
year = (raw[2] << 8) | raw[3]
|
||||
hour = raw[5]
|
||||
minute = raw[6]
|
||||
sec = raw[7]
|
||||
try:
|
||||
return datetime.datetime(year, month, day, hour, minute, sec)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _ts_from_model(ts: Optional[Timestamp]) -> Optional[datetime.datetime]:
|
||||
"""Convert a models.Timestamp to datetime.datetime, or None."""
|
||||
if ts is None:
|
||||
return None
|
||||
try:
|
||||
return datetime.datetime(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
# ── DLE strip helper ──────────────────────────────────────────────────────────
|
||||
|
||||
def _strip_inner_frame_dles(data: bytes) -> bytes:
|
||||
"""
|
||||
Strip DLE (0x10) framing markers from A5 inner-frame content.
|
||||
|
||||
The A5 (bulk waveform stream) response body contains DLE-encoded sub-frame
|
||||
structure. S3FrameParser preserves DLE+XX pairs as two literal bytes in
|
||||
frame.data. Only the DLE marker byte needs to be removed; the following
|
||||
byte is actual payload content.
|
||||
|
||||
Rule: when 0x10 is immediately followed by {0x02, 0x03, 0x04}, strip the
|
||||
0x10 (DLE marker) and keep the following byte as payload.
|
||||
|
||||
Lone 0x10 bytes not followed by {0x02, 0x03, 0x04} are kept as-is.
|
||||
|
||||
Confirmed correct by verifying reconstructed waveform body against M529LIY6 (example event):
|
||||
- 0x10 0x02 in terminator → 0x02 kept ✓
|
||||
- 0x10 0x04 in terminator (month byte) → 0x04 kept ✓
|
||||
"""
|
||||
out = bytearray()
|
||||
i = 0
|
||||
while i < len(data):
|
||||
b = data[i]
|
||||
if b == 0x10 and i + 1 < len(data) and data[i + 1] in {0x02, 0x03, 0x04}:
|
||||
# Strip the DLE marker; the next byte is payload and will be appended
|
||||
# in the next loop iteration.
|
||||
i += 1
|
||||
continue
|
||||
out.append(b)
|
||||
i += 1
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes:
|
||||
"""
|
||||
Extract the waveform body contribution from one A5 S3Frame.
|
||||
|
||||
The contribution is frame.data[skip:] with inner-frame DLE pairs stripped
|
||||
per _strip_inner_frame_dles(). The chk_byte is temporarily appended before
|
||||
stripping to handle the split-pair edge case where a DLE at the end of
|
||||
frame.data is paired with chk_byte.
|
||||
|
||||
Split-pair edge case (confirmed for A5[8] of M529LIY6 (example event), 2026-04-21):
|
||||
|
||||
S3FrameParser appends DLE+XX pairs as two literal bytes when XX ∉ {DLE, ETX}.
|
||||
When the LAST occurrence of such a pair straddles the payload/checksum boundary
|
||||
(i.e., DLE is the last byte of raw_payload and XX is the checksum), the parser
|
||||
splits them:
|
||||
- DLE ends up as the last byte of frame.data (frame.data[-1] == 0x10)
|
||||
- XX is stored as frame.chk_byte
|
||||
|
||||
To strip the pair correctly, we reunite the bytes before calling the strip
|
||||
function. Since chk_byte is the checksum (not payload data), it is excluded
|
||||
from the final output regardless of whether it was part of a pair.
|
||||
|
||||
Post-strip chk_byte removal (ALL cases):
|
||||
_strip_inner_frame_dles strips the 0x10 and KEEPS chk_byte in all cases.
|
||||
Chk_byte is always the checksum (not payload), so always strip it off.
|
||||
|
||||
Args:
|
||||
frame: S3Frame with frame.data and frame.chk_byte populated.
|
||||
skip: Number of leading bytes in frame.data to exclude (frame header).
|
||||
|
||||
Returns:
|
||||
bytes — the waveform body contribution for this frame.
|
||||
"""
|
||||
if skip >= len(frame.data):
|
||||
return b""
|
||||
|
||||
relevant = frame.data[skip:]
|
||||
|
||||
# Detect split DLE+chk pair at the frame boundary.
|
||||
has_split_pair = (
|
||||
len(relevant) > 0
|
||||
and relevant[-1] == 0x10
|
||||
and frame.chk_byte in {0x02, 0x03, 0x04}
|
||||
)
|
||||
|
||||
if has_split_pair:
|
||||
# Reunite the split pair so the strip function sees both bytes together.
|
||||
buf = relevant + bytes([frame.chk_byte])
|
||||
stripped = _strip_inner_frame_dles(buf)
|
||||
# _strip_inner_frame_dles strips the DLE (0x10) and KEEPS chk_byte.
|
||||
# chk_byte is the received checksum — never payload — so remove it.
|
||||
# This is correct for all values in {0x02, 0x03, 0x04}.
|
||||
if stripped:
|
||||
stripped = stripped[:-1]
|
||||
return stripped
|
||||
else:
|
||||
return _strip_inner_frame_dles(relevant)
|
||||
|
||||
|
||||
# ── Filename helper ───────────────────────────────────────────────────────────
|
||||
|
||||
_INSTANTEL_EPOCH = datetime.datetime(1985, 1, 1, 0, 0, 0)
|
||||
"""
|
||||
Instantel timestamp epoch — January 1, 1985, 00:00:00 local time.
|
||||
Confirmed 2026-04-21: stem values for 6 independent events (April 1–9, 2026)
|
||||
all converge to this epoch when decoded as floor(seconds_since_epoch / 1296).
|
||||
1985 is the year Instantel was founded.
|
||||
"""
|
||||
|
||||
_STEM_UNIT_SEC = 1296 # = 36^2 seconds ≈ 21.6 minutes per stem unit
|
||||
|
||||
_STEM_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
||||
# ── Waveform file extension encoding ─────────────────────────────────────────
|
||||
#
|
||||
# NEW FIRMWARE (V10.72+) — FULLY DECODED (confirmed 2026-04-21, 10-year archive):
|
||||
#
|
||||
# Extension format: AB0T (4 characters)
|
||||
# AB = 2-char base-36 encoding of (seconds_since_epoch % 1296)
|
||||
# i.e. the number of seconds into the current 21.6-minute stem window
|
||||
# Range: 0 ("00") to 1295 ("ZZ")
|
||||
# 0 = always literal '0'
|
||||
# T = event type: 'W' = Full Waveform, 'H' = Full Histogram
|
||||
#
|
||||
# Combined with the 4-char stem (which encodes seconds_since_epoch // 1296),
|
||||
# the FULL filename gives a second-resolution timestamp:
|
||||
# total_seconds = stem_val * 1296 + ab_val
|
||||
# timestamp = EPOCH + timedelta(seconds=total_seconds)
|
||||
#
|
||||
# Verified against three S353L4H0 events (all three match to the second):
|
||||
# S353L4H0.3M0W Full Waveform 2025-06-23 13:57:22 AB=3M=130 ✓
|
||||
# S353L4H0.8S0H Full Histogram 2025-06-23 14:00:28 AB=8S=316 ✓
|
||||
# S353L4H0.9X0W Full Waveform 2025-06-23 14:01:09 AB=9X=357 ✓
|
||||
#
|
||||
# OLD FIRMWARE (S338, 3-char extensions ending in '0') — UNKNOWN:
|
||||
# Observed (old firmware / manual downloads): .440, .470, .7M0, .9T0, .EI0, etc.
|
||||
# The V10.72 formula does NOT apply to these.
|
||||
# Extension is NOT recording mode (refuted 2026-04-21: continuous → .EI0, not .9T0).
|
||||
# blastware_filename() computes the correct AB0 extension for V10.72 firmware.
|
||||
#
|
||||
# WRONG earlier assumption (do not re-introduce):
|
||||
# Extension was believed to encode recording mode × sample rate.
|
||||
# Refuted by continuous-mode event producing .EI0 instead of .9T0.
|
||||
|
||||
|
||||
def _make_stem(ts_local: datetime.datetime) -> str:
|
||||
"""
|
||||
Encode a local timestamp as a 4-character uppercase base-36 stem.
|
||||
|
||||
Algorithm (confirmed 2026-04-21 from 6 known file/timestamp pairs):
|
||||
stem_int = floor((ts_local - Jan_1_1985_midnight_local) / 1296_seconds)
|
||||
stem = 4-char uppercase base-36 encoding of stem_int
|
||||
|
||||
Unit = 36² = 1296 seconds ≈ 21.6 minutes. Events within the same 1296-second
|
||||
window receive the same stem; their extension distinguishes them.
|
||||
"""
|
||||
delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds())
|
||||
n = delta_sec // _STEM_UNIT_SEC
|
||||
s = ""
|
||||
for _ in range(4):
|
||||
s = _STEM_CHARS[n % 36] + s
|
||||
n //= 36
|
||||
return s
|
||||
|
||||
|
||||
def blastware_filename(event: Event, serial: str, ach: bool = False) -> str:
|
||||
"""
|
||||
Return the correct Blastware filename for an event.
|
||||
|
||||
CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year archive.
|
||||
|
||||
Filename format: <prefix_letter><serial3><stem><AB>0[T]
|
||||
where:
|
||||
|
||||
prefix_letter = chr(ord('B') + floor(serial_numeric / 1000))
|
||||
— encodes the production generation (batch of 1000 units)
|
||||
— e.g. BE6907→H, BE11529→M, BE14036→P, BE18003→T
|
||||
|
||||
serial3 = f"{serial_numeric % 1000:03d}"
|
||||
— last 3 digits of numeric serial, zero-padded
|
||||
|
||||
stem = 4-char base-36 of floor(total_seconds / 1296)
|
||||
— encodes which 21.6-minute window the event fell in
|
||||
|
||||
AB = 2-char base-36 of (total_seconds % 1296)
|
||||
— encodes seconds within the window (0–1295)
|
||||
|
||||
0 = always literal digit zero
|
||||
|
||||
T = 'W' or 'H' — ONLY appended for call-home (ACH) downloads (ach=True).
|
||||
Manual / direct downloads produce a 3-char extension (AB0) with no type char.
|
||||
Call-home downloads produce a 4-char extension (AB0W or AB0H).
|
||||
|
||||
total_seconds = (event_local_time − 1985-01-01T00:00:00_local) in seconds
|
||||
|
||||
The 10-year production archive contains only call-home files (all end in W or H).
|
||||
Manual Blastware downloads produce 3-char extensions — the same AB0 prefix but
|
||||
without the trailing type character.
|
||||
|
||||
Micromate Series 4 uses a completely different naming scheme (literal datetime
|
||||
in filename); this function does not apply to Micromate units.
|
||||
|
||||
Args:
|
||||
event: Event object with timestamp set.
|
||||
serial: Device serial number string (e.g. "BE11529").
|
||||
ach: If True, append W/H type character (call-home style).
|
||||
If False (default), omit type character (direct download style).
|
||||
|
||||
Returns:
|
||||
Filename string, e.g. "M529LIY6.CE0" (direct) or "M529LIY6.CE0H" (ACH).
|
||||
"""
|
||||
# ── Serial prefix ──────────────────────────────────────────────────────────
|
||||
serial_digits = "".join(c for c in serial if c.isdigit())
|
||||
if len(serial_digits) >= 1:
|
||||
serial_numeric = int(serial_digits)
|
||||
generation = serial_numeric // 1000
|
||||
prefix_letter = chr(ord('B') + generation)
|
||||
serial3 = f"{serial_numeric % 1000:03d}"
|
||||
else:
|
||||
prefix_letter = "M" # fallback
|
||||
serial3 = "000"
|
||||
prefix = prefix_letter + serial3
|
||||
|
||||
# ── Stem + AB extension from timestamp ────────────────────────────────────
|
||||
if event.timestamp is not None:
|
||||
try:
|
||||
ts_local = datetime.datetime(
|
||||
event.timestamp.year, event.timestamp.month, event.timestamp.day,
|
||||
event.timestamp.hour, event.timestamp.minute, event.timestamp.second,
|
||||
)
|
||||
delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds())
|
||||
stem = _make_stem(ts_local)
|
||||
ab_val = delta_sec % _STEM_UNIT_SEC
|
||||
ab_str = _STEM_CHARS[ab_val // 36] + _STEM_CHARS[ab_val % 36]
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
stem = "0000"
|
||||
ab_str = "00"
|
||||
else:
|
||||
stem = "0000"
|
||||
ab_str = "00"
|
||||
|
||||
# ── Type character (ACH only) ─────────────────────────────────────────────
|
||||
if ach:
|
||||
if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont
|
||||
type_char = 'H'
|
||||
else:
|
||||
type_char = 'W'
|
||||
ext = f".{ab_str}0{type_char}"
|
||||
else:
|
||||
ext = f".{ab_str}0"
|
||||
|
||||
return prefix + stem + ext
|
||||
|
||||
|
||||
# ── A5 frame classifier ───────────────────────────────────────────────────────────
|
||||
|
||||
# ASCII markers that identify a compliance-config / metadata frame.
|
||||
# These strings appear in the A5 bulk stream as part of the device's
|
||||
# compliance setup payload. They should NEVER appear in raw ADC waveform
|
||||
# frames (which are binary-heavy, < 20 % printable ASCII).
|
||||
_METADATA_FRAME_MARKERS = (
|
||||
b"Project:",
|
||||
b"Client:",
|
||||
b"Standard Recording Setup",
|
||||
b"Extended Notes",
|
||||
b"User Name:",
|
||||
b"Seis Loc:",
|
||||
)
|
||||
|
||||
|
||||
def classify_frame(frame: S3Frame) -> str:
|
||||
"""
|
||||
Classify an A5 bulk waveform stream frame by its content.
|
||||
|
||||
Returns one of:
|
||||
"terminator" — page_key == 0x0000
|
||||
"probe_or_strt" — data contains b"STRT\xff\xfe" (the initial probe response)
|
||||
"metadata" — data contains ASCII compliance-config markers
|
||||
"waveform" — predominantly binary (< 20 % printable ASCII)
|
||||
"unknown" — none of the above criteria matched
|
||||
|
||||
Used by write_blastware_file() to filter non-waveform frames out of
|
||||
the reconstructed body so that metadata blocks (Project:, Client:, …)
|
||||
and spurious STRT records do not corrupt the output file.
|
||||
"""
|
||||
if frame.page_key == 0x0000:
|
||||
return "terminator"
|
||||
data = bytes(frame.data)
|
||||
if b"STRT\xff\xfe" in data:
|
||||
return "probe_or_strt"
|
||||
if any(m in data for m in _METADATA_FRAME_MARKERS):
|
||||
return "metadata"
|
||||
if len(data) > 0:
|
||||
printable = sum(1 for b in data if 32 <= b < 127)
|
||||
if printable / len(data) < 0.20:
|
||||
return "waveform"
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ── Waveform file writer ───────────────────────────────────────────────────────────
|
||||
|
||||
def write_blastware_file(
|
||||
event: Event,
|
||||
a5_frames: list[S3Frame],
|
||||
path: Union[str, Path],
|
||||
) -> None:
|
||||
"""
|
||||
Write a Blastware waveform file from a downloaded event.
|
||||
|
||||
Args:
|
||||
event: Event object (populated by get_events() or download_waveform()).
|
||||
Used for the STRT record (key, rectime) and footer timestamps.
|
||||
a5_frames: Complete A5 frame list INCLUDING the terminator frame
|
||||
(page_key=0x0000). Pass include_terminator=True to
|
||||
read_bulk_waveform_stream() when collecting frames.
|
||||
Must have at least 2 frames (probe + terminator).
|
||||
path: Destination file path. Parent directory must exist.
|
||||
Extension should be set via blastware_filename().
|
||||
|
||||
File layout:
|
||||
[22B header] [21B STRT] [body bytes] [26B footer]
|
||||
|
||||
Raises:
|
||||
ValueError: if a5_frames is empty or has no terminator (page_key=0).
|
||||
OSError: if the file cannot be written.
|
||||
|
||||
Confirmed correct waveform body reconstruction against M529LIY6 (example event) (2026-04-21).
|
||||
"""
|
||||
if not a5_frames:
|
||||
raise ValueError("a5_frames must not be empty")
|
||||
|
||||
path = Path(path)
|
||||
|
||||
# ── Extract STRT record from probe frame ────────────────────────────────
|
||||
# The STRT record (21 bytes) lives verbatim inside A5[0].data[7:].
|
||||
# It is stored as-is in the waveform file — do NOT reconstruct it from Event
|
||||
# fields, as bytes [10:14] and [14:20] contain device-specific values
|
||||
# (not simply key4 repeated or zero-padded). Confirmed 2026-04-21.
|
||||
#
|
||||
# STRT layout (21 bytes, observed in M529LIY6 files):
|
||||
# [0:4] b'STRT'
|
||||
# [4:6] 0xff 0xfe (fixed)
|
||||
# [6:10] key4 (event key)
|
||||
# [10:14] device-specific field (NOT a key4 repeat)
|
||||
# [14:20] device-specific fields (NOT zeros)
|
||||
# [20] rectime uint8 seconds
|
||||
# Extract STRT from the DLE-stripped probe frame.
|
||||
#
|
||||
# frame.data[7:] is the raw wire representation; it may contain DLE+{02,03,04}
|
||||
# inner-frame pairs that S3FrameParser preserves as two literal bytes. The
|
||||
# Blastware file stores the stripped form, so we must strip before extracting.
|
||||
#
|
||||
# Example (M529LK0Y, 2026-04-21): STRT contains value 0x02 encoded as [10 02]
|
||||
# on the wire. Without stripping, STRT is 22 raw bytes → write_blastware_file writes the
|
||||
# DLE prefix into the file AND begins the body 1 byte too early (probe_skip off
|
||||
# by 1). Stripping fixes both.
|
||||
#
|
||||
# probe_skip must be computed in the RAW frame.data domain (it is used as the
|
||||
# `skip` argument to _frame_body_bytes which operates on raw frame.data).
|
||||
# We walk the raw bytes counting stripped bytes until we have passed
|
||||
# strt_pos + 21 stripped bytes, giving the raw offset of the first body byte.
|
||||
w0_raw = bytes(a5_frames[0].data[7:])
|
||||
w0_stripped = _strip_inner_frame_dles(w0_raw)
|
||||
strt_pos_stripped = w0_stripped.find(b"STRT")
|
||||
|
||||
if strt_pos_stripped >= 0:
|
||||
strt = bytes(w0_stripped[strt_pos_stripped : strt_pos_stripped + 21])
|
||||
|
||||
# Walk raw bytes to find the raw-domain end of the STRT (= body start).
|
||||
target_stripped = strt_pos_stripped + 21
|
||||
stripped_so_far = 0
|
||||
raw_i = 0
|
||||
while stripped_so_far < target_stripped and raw_i < len(w0_raw):
|
||||
if (w0_raw[raw_i] == 0x10
|
||||
and raw_i + 1 < len(w0_raw)
|
||||
and w0_raw[raw_i + 1] in {0x02, 0x03, 0x04}):
|
||||
raw_i += 2 # DLE pair → 1 stripped byte, 2 raw bytes
|
||||
else:
|
||||
raw_i += 1 # normal byte → 1 stripped byte, 1 raw byte
|
||||
stripped_so_far += 1
|
||||
probe_skip = 7 + raw_i # raw bytes to skip: 7 header + raw STRT length
|
||||
else:
|
||||
# Fallback: construct a minimal STRT if probe frame lacks it
|
||||
key4 = event._waveform_key if hasattr(event, '_waveform_key') and event._waveform_key else bytes(4)
|
||||
rectime = event.rectime_seconds if event.rectime_seconds is not None else 0
|
||||
strt = b"STRT" + b"\xff\xfe" + key4 + bytes(14) + bytes([rectime & 0xFF])
|
||||
probe_skip = 7 + 21
|
||||
|
||||
log.debug(
|
||||
"write_blastware_file: strt_pos_stripped=%d probe_skip=%d "
|
||||
"probe_data_len=%d strt_hex=%s",
|
||||
strt_pos_stripped if strt_pos_stripped >= 0 else -1,
|
||||
probe_skip,
|
||||
len(a5_frames[0].data),
|
||||
strt.hex() if len(strt) >= 4 else "(short)",
|
||||
)
|
||||
|
||||
if len(strt) != 21:
|
||||
raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}")
|
||||
|
||||
# ── Build waveform file header ─────────────────────────────────────────────────────
|
||||
header = _FILE_HEADER_PREFIX + _WAVEFORM_TYPE_TAG
|
||||
assert len(header) == _WAVEFORM_HEADER_SIZE, f"Waveform header must be {_WAVEFORM_HEADER_SIZE} bytes"
|
||||
|
||||
# ── Build body from A5 frames ────────────────────────────────────────────
|
||||
# The waveform body is reconstructed from ALL A5 frames (data + terminator).
|
||||
# The terminator frame's contribution includes the 26-byte footer at its end.
|
||||
#
|
||||
# Reconstruction layout (confirmed from M529LIY6 captures, 2026-04-21):
|
||||
# all_bytes = contributions from A5[0..N] + terminator_contribution
|
||||
# body = all_bytes[:-26] (everything except the last 26 bytes)
|
||||
# footer = all_bytes[-26:] (last 26 bytes = the waveform file footer)
|
||||
#
|
||||
# The footer bytes come directly from the terminator frame's inner content —
|
||||
# using them verbatim ensures timestamps match the device's recorded values.
|
||||
|
||||
# Separate terminator from data frames.
|
||||
# Search from the FRONT for the first terminator (page_key == 0x0000).
|
||||
# Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a
|
||||
# subsequent event (a known get_events side-effect), the last frame will
|
||||
# not be the terminator and the footer will be mis-identified.
|
||||
# TERM detection (v0.14.0): last frame if page_key != 0x0010 (sample marker)
|
||||
term_idx: Optional[int] = None
|
||||
if a5_frames and a5_frames[-1].page_key != 0x0010:
|
||||
term_idx = len(a5_frames) - 1
|
||||
|
||||
if term_idx is not None:
|
||||
body_frames = a5_frames[:term_idx]
|
||||
term_frame = a5_frames[term_idx]
|
||||
else:
|
||||
body_frames = a5_frames
|
||||
term_frame = None
|
||||
|
||||
# Frame contribution loop (v0.14.0 BW-exact walk).
|
||||
# Skip values:
|
||||
# probe (fi=0): probe_skip
|
||||
# meta@0x1002 (fi=1): 13 (6-byte inner header)
|
||||
# meta@0x1004 (fi=2): 13 (6-byte inner header)
|
||||
# sample chunks (fi=3+): 12 (5-byte inner header)
|
||||
last_fi = len(body_frames) - 1
|
||||
|
||||
log.debug(
|
||||
"write_blastware_file: %d body_frames last_fi=%d",
|
||||
len(body_frames), last_fi,
|
||||
)
|
||||
|
||||
all_bytes = bytearray()
|
||||
|
||||
for fi, frame in enumerate(body_frames):
|
||||
if fi == 0:
|
||||
skip = probe_skip
|
||||
elif fi in (1, 2):
|
||||
skip = 13 # metadata pages
|
||||
else:
|
||||
skip = 12 # sample chunks
|
||||
|
||||
contribution = _frame_body_bytes(frame, skip)
|
||||
log.debug("write_blastware_file: fi=%d skip=%d raw_data=%d contribution=%d",
|
||||
fi, skip, len(frame.data), len(contribution))
|
||||
all_bytes.extend(contribution)
|
||||
|
||||
# Terminator contributes its content, which ends with the 26-byte footer.
|
||||
# skip=11 (not 12) because the terminator's inner frame header is 4 bytes,
|
||||
# one shorter than chunk frames' 5-byte inner header. Confirmed 2026-04-21.
|
||||
if term_frame is not None:
|
||||
term_contribution = _frame_body_bytes(term_frame, 11)
|
||||
log.debug(
|
||||
"write_blastware_file: term_frame data_len=%d skip=11 "
|
||||
"contribution_len=%d first8=%s",
|
||||
len(term_frame.data),
|
||||
len(term_contribution),
|
||||
term_contribution[:8].hex() if len(term_contribution) >= 8 else term_contribution.hex(),
|
||||
)
|
||||
all_bytes.extend(term_contribution)
|
||||
|
||||
log.debug(
|
||||
"write_blastware_file: all_bytes total=%d last28=%s",
|
||||
len(all_bytes),
|
||||
bytes(all_bytes[-28:]).hex() if len(all_bytes) >= 28 else bytes(all_bytes).hex(),
|
||||
)
|
||||
|
||||
# NOTE: The "duplicate header+STRT strip" logic from v0.13.x has been
|
||||
# REMOVED in v0.14.2. Under the v0.14.0 BW-exact 5A walk, body assembly
|
||||
# is just contiguous concatenation of frame contributions in stream order
|
||||
# (probe → meta@0x1002 → meta@0x1004 → samples → TERM), exactly as BW
|
||||
# writes its files. The previous strip was matching the `00 12 03 00 STRT`
|
||||
# byte sequence in legitimate waveform data — sample chunks at counter
|
||||
# 0x1000 and beyond often contain those bytes coincidentally — and
|
||||
# zeroing 25 bytes of valid samples per match. Compared to a known-good
|
||||
# BW reference for the same 3-sec event 0, the strip introduced 26 bytes
|
||||
# of zeros that BW did not have, then propagated alignment differences
|
||||
# through the rest of the body. See decode_test/5-1-26/bw vs SFM diff
|
||||
# at file[0x1012..0x102B] (2026-05-04 analysis).
|
||||
|
||||
# Find the first valid 0e 08 footer marker (v0.14.0).
|
||||
footer_pos = -1
|
||||
pos = 0
|
||||
while True:
|
||||
pos = bytes(all_bytes).find(b"\x0e\x08", pos)
|
||||
if pos < 0 or pos + 26 > len(all_bytes):
|
||||
break
|
||||
yr = (all_bytes[pos + 4] << 8) | all_bytes[pos + 5]
|
||||
if 2015 <= yr <= 2050:
|
||||
footer_pos = pos
|
||||
break
|
||||
pos += 1
|
||||
if footer_pos >= 0:
|
||||
body = bytes(all_bytes[:footer_pos])
|
||||
footer = bytes(all_bytes[footer_pos:footer_pos + 26])
|
||||
log.debug(
|
||||
"write_blastware_file: real 0e 08 footer at all_bytes[%d]; "
|
||||
"truncating %d post-footer bytes",
|
||||
footer_pos, len(all_bytes) - footer_pos - 26,
|
||||
)
|
||||
elif len(all_bytes) >= 26:
|
||||
body = bytes(all_bytes[:-26])
|
||||
footer = bytes(all_bytes[-26:])
|
||||
else:
|
||||
body = bytes(all_bytes)
|
||||
start_dt = _ts_from_model(event.timestamp)
|
||||
stop_dt: Optional[datetime.datetime] = None
|
||||
if start_dt is not None and event.rectime_seconds:
|
||||
stop_dt = start_dt + datetime.timedelta(seconds=event.rectime_seconds)
|
||||
footer = (
|
||||
b"\x0e\x08"
|
||||
+ _encode_ts_be(start_dt)
|
||||
+ _encode_ts_be(stop_dt)
|
||||
+ b"\x00\x01\x00\x02\x00\x00"
|
||||
+ b"\x00\x00"
|
||||
)
|
||||
|
||||
# ── Write file ───────────────────────────────────────────────────────────
|
||||
with open(path, "wb") as f:
|
||||
f.write(header)
|
||||
f.write(strt)
|
||||
f.write(body)
|
||||
f.write(footer)
|
||||
|
||||
|
||||
def read_blastware_file(path: Union[str, Path]) -> Event:
|
||||
"""
|
||||
Parse a Blastware waveform file into an Event object.
|
||||
|
||||
NOT YET IMPLEMENTED.
|
||||
|
||||
Args:
|
||||
path: Path to the waveform file.
|
||||
|
||||
Returns:
|
||||
Event object with waveform data populated.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: always (pending implementation).
|
||||
"""
|
||||
raise NotImplementedError("read_blastware_file() is not yet implemented")
|
||||
|
||||
|
||||
# ── MLG file writer ───────────────────────────────────────────────────────────
|
||||
|
||||
def _build_mlg_header(serial: str) -> bytes:
|
||||
"""
|
||||
Build the 308-byte MLG file header.
|
||||
|
||||
Header structure (confirmed from BE11529.MLG binary inspection):
|
||||
Offset 0x00: 10 00 01 80 00 00 Instantel\x00 07 2c 22 01 0e a0 (22B)
|
||||
Offset 0x16: ... (16B unknown — observed as zeros in BE11529.MLG)
|
||||
Offset 0x2A: serial number (8 bytes, null-padded ASCII)
|
||||
... rest zero-padded to 308 bytes
|
||||
|
||||
The serial string "BE11529" appears at offset 0x2A (42 decimal).
|
||||
"""
|
||||
buf = bytearray(_MLG_HEADER_SIZE)
|
||||
|
||||
# Common prefix + MLG type tag
|
||||
prefix = _FILE_HEADER_PREFIX + _MLG_TYPE_TAG # 22 bytes
|
||||
buf[0:len(prefix)] = prefix
|
||||
|
||||
# Serial number at offset 0x2A
|
||||
serial_bytes = serial.encode("ascii", errors="replace")[:8]
|
||||
serial_padded = serial_bytes.ljust(8, b"\x00")
|
||||
buf[0x2A : 0x2A + 8] = serial_padded
|
||||
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def _build_mlg_record(
|
||||
entry: MonitorLogEntry,
|
||||
serial: str,
|
||||
) -> bytes:
|
||||
"""
|
||||
Build one 292-byte MLG record from a MonitorLogEntry.
|
||||
|
||||
Record layout (confirmed from BE11529.MLG binary inspection):
|
||||
[0:2] CRC — 2-byte CRC (algorithm unknown; written as 0x0000)
|
||||
[2:6] marker — 22 01 0e 80
|
||||
[6:14] ts1 — 8B big-endian start timestamp
|
||||
[14:22] ts2 — 8B big-endian stop timestamp
|
||||
[22:26] flags — 4B record flags (see MLG_FLAGS_* constants)
|
||||
[26:36] serial — 10B null-padded serial number
|
||||
[36:] text — for triggered events: [0x08][8B ts1_copy]["Geo: X.XXX in/s"]
|
||||
for monitoring intervals: b"" or minimal separator
|
||||
[... zero-padded to 292 bytes]
|
||||
|
||||
Flags based on entry type:
|
||||
- MonitorLogEntry with start_time only (no stop_time): MLG_FLAGS_START_ONLY
|
||||
- MonitorLogEntry with both times and geo_threshold_ips set: MLG_FLAGS_TRIGGER
|
||||
- MonitorLogEntry with both times (monitoring interval): MLG_FLAGS_INTERVAL
|
||||
|
||||
The triggered-event text block (flags = MLG_FLAGS_TRIGGER):
|
||||
[0x08] [ts1: 8B] [ASCII "Geo: X.XXX in/s\x00"]
|
||||
Confirmed from BE11529.MLG records at offset 0x0134 and 0x0258.
|
||||
"""
|
||||
buf = bytearray(_MLG_RECORD_SIZE)
|
||||
|
||||
start_dt = (
|
||||
datetime.datetime(
|
||||
entry.start_time.year, entry.start_time.month, entry.start_time.day,
|
||||
entry.start_time.hour, entry.start_time.minute, entry.start_time.second,
|
||||
)
|
||||
if entry.start_time else None
|
||||
)
|
||||
stop_dt = (
|
||||
datetime.datetime(
|
||||
entry.stop_time.year, entry.stop_time.month, entry.stop_time.day,
|
||||
entry.stop_time.hour, entry.stop_time.minute, entry.stop_time.second,
|
||||
)
|
||||
if entry.stop_time else None
|
||||
)
|
||||
|
||||
# [0:2] CRC placeholder
|
||||
buf[0:2] = b"\x00\x00"
|
||||
|
||||
# [2:6] Record marker
|
||||
buf[2:6] = _MLG_RECORD_MARKER
|
||||
|
||||
# [6:14] ts1
|
||||
buf[6:14] = _encode_ts_be(start_dt)
|
||||
|
||||
# [14:22] ts2
|
||||
buf[14:22] = _encode_ts_be(stop_dt)
|
||||
|
||||
# [22:26] flags
|
||||
if stop_dt is None:
|
||||
flags = MLG_FLAGS_START_ONLY
|
||||
elif entry.geo_threshold_ips is not None:
|
||||
flags = MLG_FLAGS_TRIGGER
|
||||
else:
|
||||
flags = MLG_FLAGS_INTERVAL
|
||||
buf[22:26] = flags
|
||||
|
||||
# [26:36] serial (10B null-padded)
|
||||
serial_bytes = serial.encode("ascii", errors="replace")[:10]
|
||||
buf[26 : 26 + len(serial_bytes)] = serial_bytes
|
||||
|
||||
# [36:] text content
|
||||
pos = 36
|
||||
if flags == MLG_FLAGS_TRIGGER:
|
||||
# Extra ts1 copy: [0x08][ts1: 8B]
|
||||
buf[pos] = 0x08
|
||||
pos += 1
|
||||
buf[pos : pos + 8] = _encode_ts_be(start_dt)
|
||||
pos += 8
|
||||
|
||||
if entry.geo_threshold_ips is not None:
|
||||
geo_text = f"Geo: {entry.geo_threshold_ips:.3f} in/s\x00".encode("ascii")
|
||||
buf[pos : pos + len(geo_text)] = geo_text
|
||||
pos += len(geo_text)
|
||||
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def write_mlg(
|
||||
entries: list[MonitorLogEntry],
|
||||
serial: str,
|
||||
path: Union[str, Path],
|
||||
) -> None:
|
||||
"""
|
||||
Write a Blastware .MLG monitor log file.
|
||||
|
||||
Args:
|
||||
entries: List of MonitorLogEntry objects (from get_monitor_log_entries()).
|
||||
Each entry produces one 292-byte record in the file.
|
||||
serial: Device serial number string (e.g. "BE11529").
|
||||
Written to the file header and each record.
|
||||
path: Destination file path. Extension is not enforced — use ".MLG".
|
||||
|
||||
File layout:
|
||||
[308B header] [N × 292B records]
|
||||
|
||||
Note: The 2-byte CRC at the start of each record is written as 0x0000.
|
||||
The CRC algorithm is unknown (see module docstring).
|
||||
|
||||
Raises:
|
||||
OSError: if the file cannot be written.
|
||||
"""
|
||||
path = Path(path)
|
||||
header = _build_mlg_header(serial)
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(header)
|
||||
for entry in entries:
|
||||
record = _build_mlg_record(entry, serial)
|
||||
f.write(record)
|
||||
|
||||
|
||||
def read_mlg(path: Union[str, Path]) -> list[MonitorLogEntry]:
|
||||
"""
|
||||
Parse a Blastware .MLG file into a list of MonitorLogEntry objects.
|
||||
|
||||
NOT YET IMPLEMENTED.
|
||||
|
||||
Args:
|
||||
path: Path to the .MLG file.
|
||||
|
||||
Returns:
|
||||
List of MonitorLogEntry objects.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: always (pending implementation).
|
||||
"""
|
||||
raise NotImplementedError("read_mlg() is not yet implemented")
|
||||
+2169
-147
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,533 @@
|
||||
"""
|
||||
minimateplus/event_file_io.py — modern event-file (.sfm.json sidecar) IO.
|
||||
|
||||
This module is the single home for event-file conversion code that doesn't
|
||||
fit cleanly inside `blastware_file.py` (which is the BW binary codec):
|
||||
|
||||
- sidecar JSON read/write (the modern per-event metadata file)
|
||||
- read_blastware_file() — reverse of write_blastware_file, used by
|
||||
the BW-importer flow when SFM is ingesting files produced by
|
||||
Blastware's own ACH (where the source A5 frames aren't available).
|
||||
|
||||
Sidecar schema v1 layout — see docs in the project plan or the schema
|
||||
declared in `event_to_sidecar_dict()`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from .models import Event, PeakValues, ProjectInfo, Timestamp
|
||||
from . import blastware_file as _bw # avoid circular reference at module load
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Schema version for the sidecar JSON. Bump when fields change shape.
|
||||
# Older readers must reject anything > SCHEMA_VERSION; newer fields added
|
||||
# inside `extensions` are forward-compatible without a bump.
|
||||
SCHEMA_VERSION = 1
|
||||
SIDECAR_KIND = "sfm.event"
|
||||
|
||||
# Default tool_version stamp; callers can override. Hard-coded here
|
||||
# rather than read via importlib.metadata because the latter reflects the
|
||||
# *installed* dist-info, which doesn't update when pyproject.toml is
|
||||
# bumped without a `pip install` re-run — leading to confusing stale
|
||||
# version stamps in sidecars. Bump this constant and CHANGELOG.md
|
||||
# together at release time.
|
||||
TOOL_VERSION = "0.15.0"
|
||||
|
||||
try:
|
||||
# Best-effort: prefer the installed metadata when it's NEWER than the
|
||||
# baked-in constant (e.g. a downstream packager bumped the wheel
|
||||
# without editing this file). Otherwise fall back to TOOL_VERSION.
|
||||
from importlib.metadata import version as _pkg_version
|
||||
_meta_v = _pkg_version("seismo-relay")
|
||||
def _vtuple(s):
|
||||
try:
|
||||
return tuple(int(p) for p in s.split(".")[:3])
|
||||
except Exception:
|
||||
return (0, 0, 0)
|
||||
_TOOL_VERSION_DEFAULT = (
|
||||
_meta_v if _vtuple(_meta_v) > _vtuple(TOOL_VERSION) else TOOL_VERSION
|
||||
)
|
||||
except Exception:
|
||||
_TOOL_VERSION_DEFAULT = TOOL_VERSION
|
||||
|
||||
|
||||
# ── Sidecar dict construction ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _ts_iso(ts: Optional[Timestamp]) -> Optional[str]:
|
||||
if ts is None:
|
||||
return None
|
||||
try:
|
||||
return datetime.datetime(
|
||||
ts.year, ts.month, ts.day,
|
||||
ts.hour or 0, ts.minute or 0, ts.second or 0,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
return str(ts)
|
||||
|
||||
|
||||
def _peak_values_to_dict(pv: Optional[PeakValues]) -> dict:
|
||||
if pv is None:
|
||||
return {
|
||||
"transverse": None,
|
||||
"vertical": None,
|
||||
"longitudinal": None,
|
||||
"vector_sum": None,
|
||||
"mic_psi": None,
|
||||
}
|
||||
return {
|
||||
"transverse": pv.tran,
|
||||
"vertical": pv.vert,
|
||||
"longitudinal": pv.long,
|
||||
"vector_sum": pv.peak_vector_sum,
|
||||
"mic_psi": pv.micl,
|
||||
}
|
||||
|
||||
|
||||
def _project_info_to_dict(pi: Optional[ProjectInfo]) -> dict:
|
||||
if pi is None:
|
||||
return {
|
||||
"project": None,
|
||||
"client": None,
|
||||
"operator": None,
|
||||
"sensor_location": None,
|
||||
}
|
||||
return {
|
||||
"project": pi.project,
|
||||
"client": pi.client,
|
||||
"operator": pi.operator,
|
||||
"sensor_location": pi.sensor_location,
|
||||
}
|
||||
|
||||
|
||||
def event_to_sidecar_dict(
|
||||
event: Event,
|
||||
*,
|
||||
serial: str,
|
||||
blastware_filename: str,
|
||||
blastware_filesize: int,
|
||||
blastware_sha256: str,
|
||||
source_kind: str = "sfm-live",
|
||||
a5_pickle_filename: Optional[str] = None,
|
||||
tool_version: str = _TOOL_VERSION_DEFAULT,
|
||||
captured_at: Optional[datetime.datetime] = None,
|
||||
review: Optional[dict] = None,
|
||||
extensions: Optional[dict] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Build a v1 sidecar dict from an Event + the surrounding metadata.
|
||||
|
||||
Pure helper — no file I/O. Callers stitch the result into a sidecar
|
||||
via `write_sidecar()` (or POST it back via the PATCH endpoint).
|
||||
"""
|
||||
if source_kind not in {"sfm-live", "sfm-ach", "bw-import"}:
|
||||
raise ValueError(f"unknown source_kind: {source_kind!r}")
|
||||
|
||||
captured_at = captured_at or datetime.datetime.utcnow()
|
||||
|
||||
# Stash raw 0C record bytes in `extensions.raw_records` so future
|
||||
# field-decoding work (Peak Acceleration, ZC Freq, Time of Peak,
|
||||
# sensor self-check results, etc.) can run offline against committed
|
||||
# sidecars without a live device. Cheap (~280 bytes base64) and
|
||||
# forward-compatible (older readers ignore unknown extensions keys).
|
||||
ext_dict: dict = dict(extensions) if extensions else {}
|
||||
raw_0c = getattr(event, "_raw_record", None)
|
||||
if raw_0c:
|
||||
rr = ext_dict.setdefault("raw_records", {})
|
||||
# Don't clobber a raw_0c that callers explicitly passed in via
|
||||
# `extensions=...` (e.g. round-trip preservation in patch_sidecar).
|
||||
rr.setdefault("waveform_record_b64", base64.b64encode(raw_0c).decode("ascii"))
|
||||
rr.setdefault("waveform_record_len", len(raw_0c))
|
||||
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"kind": SIDECAR_KIND,
|
||||
|
||||
"event": {
|
||||
"serial": serial,
|
||||
"timestamp": _ts_iso(event.timestamp),
|
||||
"waveform_key": event._waveform_key.hex() if event._waveform_key else None,
|
||||
"record_type": event.record_type,
|
||||
"sample_rate": event.sample_rate,
|
||||
"rectime_seconds": event.rectime_seconds,
|
||||
"total_samples": event.total_samples,
|
||||
"pretrig_samples": event.pretrig_samples,
|
||||
},
|
||||
|
||||
"peak_values": _peak_values_to_dict(event.peak_values),
|
||||
"project_info": _project_info_to_dict(event.project_info),
|
||||
|
||||
"blastware": {
|
||||
"filename": blastware_filename,
|
||||
"filesize": blastware_filesize,
|
||||
"sha256": blastware_sha256,
|
||||
"available": True,
|
||||
},
|
||||
|
||||
"source": {
|
||||
"kind": source_kind,
|
||||
"captured_at": captured_at.isoformat() + "Z" if captured_at.tzinfo is None else captured_at.isoformat(),
|
||||
"tool_version": tool_version,
|
||||
"a5_pickle_filename": a5_pickle_filename,
|
||||
},
|
||||
|
||||
"review": review or {
|
||||
"false_trigger": False,
|
||||
"reviewer": None,
|
||||
"reviewed_at": None,
|
||||
"notes": "",
|
||||
},
|
||||
|
||||
"extensions": ext_dict,
|
||||
}
|
||||
|
||||
|
||||
# ── Sidecar IO ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def write_sidecar(path: Union[str, Path], data: dict) -> None:
|
||||
"""
|
||||
Atomic write of a sidecar dict to <path>.
|
||||
|
||||
Validates schema_version is supported before writing so we don't
|
||||
silently drop a future-format sidecar over the wire.
|
||||
"""
|
||||
path = Path(path)
|
||||
sv = data.get("schema_version")
|
||||
if not isinstance(sv, int) or sv < 1 or sv > SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
f"write_sidecar: unsupported schema_version={sv!r} "
|
||||
f"(this build supports 1..{SCHEMA_VERSION})"
|
||||
)
|
||||
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with tmp.open("w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, sort_keys=False, default=str)
|
||||
f.write("\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def read_sidecar(path: Union[str, Path]) -> dict:
|
||||
"""
|
||||
Load a sidecar JSON file.
|
||||
|
||||
Raises FileNotFoundError if missing, ValueError on bad shape /
|
||||
unsupported schema_version. Unknown keys at the top level are
|
||||
preserved in the returned dict (forward-compat).
|
||||
"""
|
||||
path = Path(path)
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"sidecar at {path}: top-level is not a JSON object")
|
||||
sv = data.get("schema_version")
|
||||
if not isinstance(sv, int) or sv < 1:
|
||||
raise ValueError(f"sidecar at {path}: missing or invalid schema_version")
|
||||
if sv > SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
f"sidecar at {path}: schema_version={sv} > supported {SCHEMA_VERSION}; "
|
||||
"upgrade seismo-relay to read this file"
|
||||
)
|
||||
if data.get("kind") != SIDECAR_KIND:
|
||||
raise ValueError(f"sidecar at {path}: unexpected kind={data.get('kind')!r}")
|
||||
return data
|
||||
|
||||
|
||||
def patch_sidecar(
|
||||
path: Union[str, Path],
|
||||
*,
|
||||
review: Optional[dict] = None,
|
||||
extensions: Optional[dict] = None,
|
||||
reviewer_now: bool = True,
|
||||
) -> dict:
|
||||
"""
|
||||
Atomically apply a JSON-merge-patch to a sidecar file's `review`
|
||||
and/or `extensions` blocks. Other top-level keys are untouched.
|
||||
|
||||
`review_now`: when True (default) and `review` is non-empty, stamps
|
||||
`review.reviewed_at` with the current UTC time so the review-time is
|
||||
auditable without the caller having to pass it.
|
||||
|
||||
Returns the new full sidecar dict.
|
||||
"""
|
||||
path = Path(path)
|
||||
data = read_sidecar(path)
|
||||
|
||||
if review:
|
||||
merged = dict(data.get("review") or {})
|
||||
merged.update({k: v for k, v in review.items() if v is not None or k in merged})
|
||||
if reviewer_now:
|
||||
merged["reviewed_at"] = datetime.datetime.utcnow().isoformat() + "Z"
|
||||
data["review"] = merged
|
||||
|
||||
if extensions:
|
||||
merged_ext = dict(data.get("extensions") or {})
|
||||
merged_ext.update(extensions)
|
||||
data["extensions"] = merged_ext
|
||||
|
||||
write_sidecar(path, data)
|
||||
return data
|
||||
|
||||
|
||||
def sidecar_path_for(blastware_path: Union[str, Path]) -> Path:
|
||||
"""Convention: <bw_path>.sfm.json sits next to the BW binary."""
|
||||
p = Path(blastware_path)
|
||||
return p.with_name(p.name + ".sfm.json")
|
||||
|
||||
|
||||
def file_sha256(path: Union[str, Path], chunk_size: int = 65536) -> str:
|
||||
"""Compute SHA-256 of a file as a hex string."""
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
# ── Blastware-file reader ─────────────────────────────────────────────────────
|
||||
#
|
||||
# Reverse of `blastware_file.write_blastware_file`. Used by the BW-import
|
||||
# flow to ingest files produced by Blastware's own ACH (where the source
|
||||
# A5 frames are not available).
|
||||
#
|
||||
# File structure (recap):
|
||||
# [22B header] [21B STRT record] [body bytes] [26B footer]
|
||||
#
|
||||
# The body holds:
|
||||
# - 6B preamble (00 00 ff ff ff ff) immediately after the STRT
|
||||
# - 4-channel interleaved int16 LE samples
|
||||
# - Embedded ASCII metadata strings (Project: / Client: / User Name: /
|
||||
# Seis Loc: / Extended Notes) from the device's session-start config
|
||||
#
|
||||
# The 0C waveform record (per-event peaks, project name) is NOT in the
|
||||
# BW file — those are computed by the device firmware and only carried
|
||||
# in the live SUB 0C response. read_blastware_file() therefore computes
|
||||
# peaks from the raw samples assuming Normal-range (10 in/s full-scale)
|
||||
# geophone sensitivity. Imported events surface that assumption via the
|
||||
# sidecar's `peak_values.computed_from_samples` flag.
|
||||
|
||||
|
||||
# Geophone scale factor, in/s per ADC unit, for Normal range (10 in/s FS).
|
||||
# Confirmed from CLAUDE.md (geo_hardware_constant = 6.206053 in/s per V,
|
||||
# ADC full-scale = 1.61133 V Normal range = 10.0 in/s peak; per-count
|
||||
# resolution ≈ 10.0 / 32768).
|
||||
_GEO_NORMAL_FS_INS = 10.0
|
||||
_GEO_SENSITIVE_FS_INS = 1.250
|
||||
_INT16_FS = 32768.0
|
||||
|
||||
# Microphone scale factor, psi per ADC count. Approximate — exact factor
|
||||
# depends on the geophone-vs-mic ADC scaling and the firmware reference.
|
||||
# We mark mic_psi as "computed approximate" in the sidecar.
|
||||
_MIC_FS_PSI = 0.0125 / _INT16_FS # ~0.5 psi full-scale assumption
|
||||
|
||||
|
||||
def _decode_strt(strt: bytes) -> dict:
|
||||
"""
|
||||
Decode the 21-byte STRT record from a BW file.
|
||||
|
||||
Returns dict with waveform_key (4B), total_samples, pretrig_samples,
|
||||
rectime_seconds. Falls back to None on truncated/missing fields.
|
||||
"""
|
||||
if len(strt) < 21 or strt[0:4] != b"STRT":
|
||||
return {}
|
||||
return {
|
||||
"waveform_key": strt[6:10].hex(),
|
||||
"total_samples": struct.unpack_from(">H", strt, 8)[0],
|
||||
"pretrig_samples": struct.unpack_from(">H", strt, 16)[0],
|
||||
"rectime_seconds": strt[18],
|
||||
}
|
||||
|
||||
|
||||
def _find_first_string(buf: bytes, label: bytes, max_len: int = 256) -> Optional[str]:
|
||||
"""
|
||||
Search `buf` for `label` (e.g. b"Project:") and return the
|
||||
null-terminated ASCII string that follows, stripped.
|
||||
"""
|
||||
pos = buf.find(label)
|
||||
if pos < 0:
|
||||
return None
|
||||
start = pos + len(label)
|
||||
end = buf.find(b"\x00", start, start + max_len)
|
||||
if end < 0:
|
||||
end = start + max_len
|
||||
text = buf[start:end].decode("ascii", errors="replace").strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _decode_samples_4ch_int16_le(stream: bytes) -> dict[str, list[int]]:
|
||||
"""
|
||||
Decode a 4-channel interleaved int16 LE byte stream into per-channel
|
||||
lists. Channels are [Tran, Vert, Long, Mic] = [ch0, ch1, ch2, ch3].
|
||||
Truncates to a multiple of 8 bytes (one full sample-set).
|
||||
"""
|
||||
n_complete = (len(stream) // 8) * 8
|
||||
if n_complete == 0:
|
||||
return {"Tran": [], "Vert": [], "Long": [], "MicL": []}
|
||||
fmt = "<" + "h" * (n_complete // 2)
|
||||
flat = list(struct.unpack(fmt, stream[:n_complete]))
|
||||
return {
|
||||
"Tran": flat[0::4],
|
||||
"Vert": flat[1::4],
|
||||
"Long": flat[2::4],
|
||||
"MicL": flat[3::4],
|
||||
}
|
||||
|
||||
|
||||
def _peaks_from_samples(samples: dict[str, list[int]]) -> PeakValues:
|
||||
"""
|
||||
Compute approximate peaks from raw int16 samples assuming Normal-range
|
||||
geophone sensitivity. Used by the BW-importer when the 0C waveform
|
||||
record (the device's authoritative peaks) is unavailable.
|
||||
"""
|
||||
def _peak_ins(ch: list[int]) -> float:
|
||||
if not ch:
|
||||
return 0.0
|
||||
m = max(abs(int(v)) for v in ch)
|
||||
return m / _INT16_FS * _GEO_NORMAL_FS_INS
|
||||
|
||||
tran = _peak_ins(samples.get("Tran", []))
|
||||
vert = _peak_ins(samples.get("Vert", []))
|
||||
long_ = _peak_ins(samples.get("Long", []))
|
||||
|
||||
# Mic in psi (approximate)
|
||||
mic_ch = samples.get("MicL", []) or []
|
||||
mic = max((abs(int(v)) for v in mic_ch), default=0) * _MIC_FS_PSI
|
||||
|
||||
# Peak vector sum: max over time of sqrt(T^2 + V^2 + L^2)
|
||||
pvs = 0.0
|
||||
n = min(len(samples.get("Tran", [])), len(samples.get("Vert", [])), len(samples.get("Long", [])))
|
||||
if n:
|
||||
scale = _GEO_NORMAL_FS_INS / _INT16_FS
|
||||
T = samples["Tran"]; V = samples["Vert"]; L = samples["Long"]
|
||||
for i in range(n):
|
||||
t = T[i] * scale
|
||||
v = V[i] * scale
|
||||
l = L[i] * scale
|
||||
mag = (t*t + v*v + l*l) ** 0.5
|
||||
if mag > pvs:
|
||||
pvs = mag
|
||||
|
||||
return PeakValues(
|
||||
tran=tran, vert=vert, long=long_,
|
||||
peak_vector_sum=pvs, micl=mic,
|
||||
)
|
||||
|
||||
|
||||
def read_blastware_file(path: Union[str, Path]) -> Event:
|
||||
"""
|
||||
Parse a Blastware waveform file into an Event.
|
||||
|
||||
Recovers:
|
||||
- waveform_key, rectime_seconds, total_samples, pretrig_samples
|
||||
(from the STRT record)
|
||||
- timestamp (from the footer's start-time field)
|
||||
- project_info (from ASCII labels embedded in the body)
|
||||
- raw_samples (Tran/Vert/Long/MicL int16 lists)
|
||||
- peak_values (computed from raw_samples; approximate — see notes
|
||||
on _peaks_from_samples about Normal-range assumption)
|
||||
|
||||
Does NOT recover the source A5 frames (they aren't in the BW file).
|
||||
The returned Event has `_a5_frames = None`, signalling that
|
||||
byte-for-byte regeneration of the BW file from this Event alone is
|
||||
not possible — the on-disk BW file IS the byte-for-byte source.
|
||||
"""
|
||||
path = Path(path)
|
||||
raw = path.read_bytes()
|
||||
if len(raw) < _bw._WAVEFORM_HEADER_SIZE + 21 + 26:
|
||||
raise ValueError(f"{path}: file too short ({len(raw)} bytes) to be a BW event")
|
||||
|
||||
# Header: validate magic prefix.
|
||||
header = raw[:_bw._WAVEFORM_HEADER_SIZE]
|
||||
if not header.startswith(_bw._FILE_HEADER_PREFIX):
|
||||
raise ValueError(f"{path}: not a Blastware file (bad header prefix)")
|
||||
|
||||
# STRT record: 21 bytes immediately after the header.
|
||||
strt_raw = raw[_bw._WAVEFORM_HEADER_SIZE : _bw._WAVEFORM_HEADER_SIZE + 21]
|
||||
strt_fields = _decode_strt(strt_raw)
|
||||
if not strt_fields:
|
||||
raise ValueError(f"{path}: STRT record missing or malformed")
|
||||
|
||||
# Footer: locate the 0e 08 marker, validating the year is in a sane range.
|
||||
body_start = _bw._WAVEFORM_HEADER_SIZE + 21
|
||||
footer_pos = -1
|
||||
pos = body_start
|
||||
while True:
|
||||
pos = raw.find(b"\x0e\x08", pos)
|
||||
if pos < 0 or pos + 26 > len(raw):
|
||||
break
|
||||
yr = (raw[pos + 4] << 8) | raw[pos + 5]
|
||||
if 2015 <= yr <= 2050:
|
||||
footer_pos = pos
|
||||
break
|
||||
pos += 1
|
||||
|
||||
if footer_pos < 0 and len(raw) >= 26:
|
||||
footer_pos = len(raw) - 26
|
||||
if footer_pos < body_start:
|
||||
raise ValueError(f"{path}: footer not found")
|
||||
|
||||
body = raw[body_start : footer_pos]
|
||||
footer = raw[footer_pos : footer_pos + 26]
|
||||
|
||||
# Footer layout:
|
||||
# [0:2] 0e 08 marker
|
||||
# [2:10] ts1 (start) BE 8B
|
||||
# [10:18] ts2 (stop) BE 8B
|
||||
# [18:24] 00 01 00 02 00 00
|
||||
# [24:26] crc
|
||||
ts1 = _bw._decode_ts_be(footer[2:10])
|
||||
ts2 = _bw._decode_ts_be(footer[10:18])
|
||||
|
||||
# Body: first 6 bytes are the preamble (00 00 ff ff ff ff). Strip
|
||||
# them before decoding samples. Any trailing tail past the last
|
||||
# full sample-set is silently truncated by _decode_samples_4ch.
|
||||
sample_bytes = body[6:] if body[:6].hex() in ("0000ffffffff", "0000FFFFFFFF") else body
|
||||
samples = _decode_samples_4ch_int16_le(sample_bytes)
|
||||
|
||||
# Metadata strings (label-anchored search across the body).
|
||||
project = _find_first_string(body, b"Project:")
|
||||
client = _find_first_string(body, b"Client:")
|
||||
user = _find_first_string(body, b"User Name:")
|
||||
seisloc = _find_first_string(body, b"Seis Loc:")
|
||||
|
||||
# Build the Event.
|
||||
ev = Event(index=-1)
|
||||
if strt_fields.get("waveform_key"):
|
||||
ev._waveform_key = bytes.fromhex(strt_fields["waveform_key"])
|
||||
ev.record_type = "Waveform"
|
||||
ev.rectime_seconds = strt_fields.get("rectime_seconds")
|
||||
ev.total_samples = strt_fields.get("total_samples")
|
||||
ev.pretrig_samples = strt_fields.get("pretrig_samples")
|
||||
|
||||
if ts1 is not None:
|
||||
ev.timestamp = Timestamp(
|
||||
raw=footer[2:10],
|
||||
flag=0x10,
|
||||
year=ts1.year, unknown_byte=0, month=ts1.month, day=ts1.day,
|
||||
hour=ts1.hour, minute=ts1.minute, second=ts1.second,
|
||||
)
|
||||
|
||||
ev.project_info = ProjectInfo(
|
||||
project=project, client=client, operator=user, sensor_location=seisloc,
|
||||
)
|
||||
ev.raw_samples = samples
|
||||
ev.peak_values = _peaks_from_samples(samples)
|
||||
ev._a5_frames = None # not recoverable from BW file
|
||||
|
||||
return ev
|
||||
+428
-13
@@ -90,6 +90,102 @@ def checksum(payload: bytes) -> int:
|
||||
|
||||
# ── BW→S3 frame builder ───────────────────────────────────────────────────────
|
||||
|
||||
# SUB byte for 5A — used by build_5a_frame below (protocol.py has the full
|
||||
# constant set; defined here to avoid a circular import).
|
||||
SUB_5A = 0x5A
|
||||
|
||||
|
||||
def build_5a_frame(offset_word: int, raw_params: bytes) -> bytes:
|
||||
"""
|
||||
Build a BW→S3 frame for SUB 5A (BULK_WAVEFORM_STREAM) that exactly
|
||||
matches Blastware's wire output.
|
||||
|
||||
SUB 5A uses a DIFFERENT frame format from all other read commands:
|
||||
1. The offset field (bytes [4:6]) is written RAW — the 0x10 in
|
||||
offset_hi=0x10 is NOT DLE-stuffed, unlike build_bw_frame().
|
||||
2. The checksum uses a DLE-aware sum: for each 0x10 XX pair in the
|
||||
stuffed section, only XX contributes; lone bytes contribute normally.
|
||||
This differs from the standard SUM8 checksum on the unstuffed payload.
|
||||
|
||||
Both differences are confirmed from the 1-2-26 BW TX capture (all 10 frames
|
||||
verified against this algorithm on 2026-04-02).
|
||||
|
||||
Args:
|
||||
offset_word: 16-bit offset. For probe/chunks/metadata pages this is
|
||||
`0x1002`. For the proper TERM frame this is computed by
|
||||
`bulk_waveform_term_v2()` from the STRT-derived
|
||||
`end_offset`.
|
||||
raw_params: 10, 11, or 12 params bytes (from `bulk_waveform_params`
|
||||
for probes/samples, `bulk_waveform_term_v2` for TERM, or
|
||||
a manually-built 12-byte block for the metadata pages
|
||||
0x1002 / 0x1004). See gotcha #3 below — params region
|
||||
uses partial DLE stuffing of 0x10 bytes.
|
||||
|
||||
Returns:
|
||||
Complete frame bytes: [ACK][STX][stuffed_section][chk][ETX]
|
||||
"""
|
||||
if len(raw_params) not in (10, 11, 12):
|
||||
# 10 = termination params; 11 = regular probe / chunk params;
|
||||
# 12 = metadata-page params (extra trailing 0x00 — BW byte-perfect quirk
|
||||
# for the two fixed metadata reads at counter=0x1002 and 0x1004).
|
||||
raise ValueError(f"raw_params must be 10/11/12 bytes, got {len(raw_params)}")
|
||||
|
||||
# Build stuffed section between STX and checksum
|
||||
s = bytearray()
|
||||
s += b"\x10\x10" # DLE-stuffed BW_CMD
|
||||
s += b"\x00" # flags
|
||||
s += bytes([SUB_5A]) # sub = 0x5A
|
||||
s += b"\x00" # field3
|
||||
s += bytes([(offset_word >> 8) & 0xFF, # offset_hi — raw, NOT stuffed
|
||||
offset_word & 0xFF]) # offset_lo
|
||||
# Params — partial DLE stuffing of 0x10 bytes (CONFIRMED 2026-05-05).
|
||||
#
|
||||
# The device's de-stuffing rule for params is:
|
||||
# • `10 10` → de-stuffs to `10`
|
||||
# • `10 02/03/04` → kept literal (these are inner-frame markers)
|
||||
# • `10 X` other → de-stuffs to just `X` (drops the 0x10)
|
||||
#
|
||||
# So for any 0x10 byte in the *logical* params that is followed by a
|
||||
# byte NOT in {0x02, 0x03, 0x04, 0x10}, we must double the 0x10 on the
|
||||
# wire (`10 X` → `10 10 X`) so the device's de-stuffer reproduces the
|
||||
# original `10 X` pair. Without this, counter values with `0x10` in
|
||||
# the high byte (e.g. counter=0x1000 has params bytes `10 00`) are
|
||||
# silently corrupted to `0x__00` on the device side, and the device
|
||||
# responds for the wrong address — for counter=0x1000 it returns the
|
||||
# probe response (counter=0x0000), which contains the file header +
|
||||
# STRT. That STRT block then lands in the assembled file body and
|
||||
# Blastware rejects the file as malformed.
|
||||
#
|
||||
# Confirmed against BW capture 5-1-26 / bwcap3sec frame 20: params
|
||||
# logical bytes `00 01 11 10 00 00 00 00 00 00 00` (counter=0x1000)
|
||||
# are encoded on the wire as `00 01 11 10 10 00 00 00 00 00 00 00`.
|
||||
# BW frames 13/14 (meta @ 0x1002 / 0x1004) leave `10 02` and `10 04`
|
||||
# raw — the device handles those literal pairs correctly.
|
||||
i = 0
|
||||
while i < len(raw_params):
|
||||
b = raw_params[i]
|
||||
s.append(b)
|
||||
if (
|
||||
b == 0x10
|
||||
and i + 1 < len(raw_params)
|
||||
and raw_params[i + 1] not in (0x02, 0x03, 0x04, 0x10)
|
||||
):
|
||||
s.append(0x10) # double the 0x10 so it survives device de-stuffing
|
||||
i += 1
|
||||
|
||||
# DLE-aware checksum: for 0x10 XX pairs count XX; for lone bytes count them
|
||||
chk, i = 0, 0
|
||||
while i < len(s):
|
||||
if s[i] == DLE and i + 1 < len(s):
|
||||
chk = (chk + s[i + 1]) & 0xFF
|
||||
i += 2
|
||||
else:
|
||||
chk = (chk + s[i]) & 0xFF
|
||||
i += 1
|
||||
|
||||
return bytes([ACK, STX]) + bytes(s) + bytes([chk, ETX])
|
||||
|
||||
|
||||
def build_bw_frame(sub: int, offset: int = 0, params: bytes = bytes(10)) -> bytes:
|
||||
"""
|
||||
Build a BW→S3 read-command frame.
|
||||
@@ -134,6 +230,109 @@ def build_bw_frame(sub: int, offset: int = 0, params: bytes = bytes(10)) -> byte
|
||||
return wire
|
||||
|
||||
|
||||
def build_bw_write_frame(
|
||||
sub: int,
|
||||
data: bytes,
|
||||
*,
|
||||
offset: int = 0,
|
||||
params: bytes = bytes(10),
|
||||
) -> bytes:
|
||||
"""
|
||||
Build a BW→S3 write-command frame.
|
||||
|
||||
Write frames extend the standard 16-byte read header with a variable-length
|
||||
data payload. They use a different checksum formula from read frames.
|
||||
|
||||
**CRITICAL: Write frames use minimal DLE stuffing.**
|
||||
|
||||
Unlike read frames (build_bw_frame), write frames do NOT apply full DLE
|
||||
stuffing to the payload. Only the BW_CMD byte (0x10) at position [0] is
|
||||
doubled to 0x10 0x10 on the wire. All other bytes — flags, sub, offset,
|
||||
params, data, and checksum — are written RAW with no stuffing, even if they
|
||||
contain 0x10 bytes (e.g. offset_hi=0x10 for compliance chunks, or 0x10
|
||||
bytes in the write data payload).
|
||||
|
||||
Confirmed from 3-11-26 BW TX capture (frames 102–112): all 11 write frames
|
||||
match the rule "double BW_CMD only; everything else raw." ✅ 2026-04-07.
|
||||
|
||||
Wire layout:
|
||||
[41] ACK
|
||||
[02] STX
|
||||
[10 10] BW_CMD doubled (the ONLY DLE stuffing applied)
|
||||
[00] flags
|
||||
[sub] write command byte (0x68–0x83)
|
||||
[00] always zero
|
||||
[hi][lo] offset as uint16 BE (raw; NOT stuffed even if hi=0x10)
|
||||
[params] 10 bytes (raw)
|
||||
[data] variable-length write payload (raw; NOT stuffed)
|
||||
[chk] checksum byte (raw; NOT stuffed even if 0x10)
|
||||
[03] ETX
|
||||
|
||||
De-stuffed payload (for checksum computation):
|
||||
[0] BW_CMD 0x10
|
||||
[1] flags 0x00
|
||||
[2] SUB write command byte
|
||||
[3] 0x00 always zero
|
||||
[4] offset_hi
|
||||
[5] offset_lo
|
||||
[6:16] params 10 bytes
|
||||
[16:] data write payload
|
||||
[-1] chk
|
||||
|
||||
**Checksum formula (confirmed 2026-03-12 from 3-11-26 BW TX capture):**
|
||||
chk = (sum(b for b in payload[2:] if b != 0x10) + 0x10) % 256
|
||||
where payload = destuffed content BEFORE appending chk.
|
||||
This skips all 0x10 bytes in payload[2:] (sub onwards), including any
|
||||
0x10 bytes in the offset, params, data, and the checksum byte itself.
|
||||
|
||||
The offset field [4:6] meaning per write SUB:
|
||||
- SUBs 68, 69, 82 (single-chunk writes): offset = data[1] + 2, where
|
||||
data[1] is an embedded length field in the write payload.
|
||||
Confirmed from capture: 68→0x5A (data[1]=0x58+2), 82→0x1C
|
||||
(data[1]=0x1A+2), 69→0xCA (data[1]=0xC8+2).
|
||||
- SUB 71 (multi-chunk compliance): 0x1004 for full chunks, 0x002C
|
||||
for the final partial chunk.
|
||||
- Confirm frames (72, 73, 74, 83): offset=0, no data.
|
||||
|
||||
Args:
|
||||
sub: Write command SUB byte.
|
||||
data: Write payload (variable length; empty for confirm frames).
|
||||
offset: 16-bit value placed at [4:6]. See per-SUB notes above.
|
||||
params: 10 bytes placed at [6:16]. All-zero for most writes; compliance
|
||||
chunk writes use chunk-specific values.
|
||||
|
||||
Returns:
|
||||
Complete frame bytes ready to write to the transport.
|
||||
"""
|
||||
if len(params) != 10:
|
||||
raise ValueError(f"params must be exactly 10 bytes, got {len(params)}")
|
||||
if offset > 0xFFFF:
|
||||
raise ValueError(f"offset must fit in uint16, got {offset:#06x}")
|
||||
|
||||
offset_hi = (offset >> 8) & 0xFF
|
||||
offset_lo = offset & 0xFF
|
||||
|
||||
# Destuffed payload (used only for checksum; not sent directly)
|
||||
payload_no_chk = bytes([BW_CMD, 0x00, sub, 0x00, offset_hi, offset_lo]) + params + data
|
||||
|
||||
# Large-frame checksum: sum payload[2:] skipping all 0x10 bytes, add 0x10.
|
||||
# Applied to the destuffed representation — confirms correctly against
|
||||
# all 11 write frames in the 3-11-26/170151 BW TX capture. ✅
|
||||
chk = (sum(b for b in payload_no_chk[2:] if b != 0x10) + 0x10) & 0xFF
|
||||
|
||||
# Wire construction: only BW_CMD is doubled; everything else is raw.
|
||||
# Do NOT use dle_stuff() here — that would incorrectly double 0x10 bytes
|
||||
# in the offset, params, and data sections.
|
||||
wire = (
|
||||
bytes([ACK, STX]) # Frame prefix (not part of payload)
|
||||
+ bytes([BW_CMD, BW_CMD]) # BW_CMD doubled (only DLE stuffing applied)
|
||||
+ payload_no_chk[1:] # flags, sub, offset, params, data — RAW
|
||||
+ bytes([chk]) # checksum — RAW
|
||||
+ bytes([ETX]) # Frame terminator
|
||||
)
|
||||
return wire
|
||||
|
||||
|
||||
def waveform_key_params(key4: bytes) -> bytes:
|
||||
"""
|
||||
Build the 10-byte params block that carries a 4-byte waveform key.
|
||||
@@ -164,24 +363,224 @@ def token_params(token: int = 0) -> bytes:
|
||||
Build the 10-byte params block that carries a single token byte.
|
||||
|
||||
Used for SUBs 1E (EVENT_HEADER) and 1F (EVENT_ADVANCE).
|
||||
The token goes at params[6], which maps to payload[12].
|
||||
The token goes at params[7], which maps to payload[13].
|
||||
|
||||
Confirmed from BOTH 3-31-26 and 4-3-26 BW TX captures:
|
||||
raw params bytes: 00 00 00 00 00 00 00 fe 00 00
|
||||
token is at index 7 (not 6 — that was wrong).
|
||||
|
||||
Confirmed from 3-31-26 capture:
|
||||
- token=0x00: first-event read / browse mode (no download marking)
|
||||
- token=0xfe: download mode (causes 1F to skip partial bins and
|
||||
advance to the next full record)
|
||||
|
||||
The device echoes the token at data[8] of the S3 response (payload[13]),
|
||||
distinct from the next-event key at data[11:15] (payload[16:20]).
|
||||
|
||||
Args:
|
||||
token: single byte to place at params[6] / payload[12].
|
||||
token: single byte to place at params[7] / payload[13].
|
||||
|
||||
Returns:
|
||||
10-byte params block with token at position [6].
|
||||
10-byte params block with token at position [7].
|
||||
"""
|
||||
p = bytearray(10)
|
||||
p[6] = token
|
||||
p[7] = token
|
||||
return bytes(p)
|
||||
|
||||
|
||||
def bulk_waveform_params(key4: bytes, counter: int, *, is_probe: bool = False) -> bytes:
|
||||
"""
|
||||
Build the 10-byte params block for SUB 5A (BULK_WAVEFORM_STREAM) requests.
|
||||
|
||||
Confirmed 2026-04-02 from 1-2-26 BW TX capture analysis:
|
||||
|
||||
Probe / first request (is_probe=True, counter=0):
|
||||
params[0] = 0x00
|
||||
params[1:5] = key4 (all 4 key bytes; counter overlaps key4[2:4] = 0x0000)
|
||||
params[5:] = zeros
|
||||
|
||||
Regular chunk requests (is_probe=False):
|
||||
params[0] = 0x00
|
||||
params[1:3] = key4[0:2] (first 2 key bytes as session handle)
|
||||
params[3:5] = counter (BE uint16) (chunk position, increments by 0x0400)
|
||||
params[5:] = zeros
|
||||
|
||||
Termination request: DO NOT use this helper — see bulk_waveform_term_params().
|
||||
|
||||
Args:
|
||||
key4: 4-byte waveform key from EVENT_HEADER (1E) response.
|
||||
counter: Chunk position counter (uint16 BE). Pass 0 for probe.
|
||||
is_probe: If True, embed full key4 (probe step only).
|
||||
|
||||
Returns:
|
||||
11-byte params block. (BW confirmed: chunk frames carry 11 params bytes,
|
||||
not 10; the extra trailing 0x00 was confirmed from 1-2-26 wire capture
|
||||
on 2026-04-02.)
|
||||
"""
|
||||
if len(key4) != 4:
|
||||
raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
|
||||
p = bytearray(11) # 11 bytes confirmed from BW wire capture
|
||||
p[0] = 0x00
|
||||
p[1] = key4[0]
|
||||
p[2] = key4[1]
|
||||
if is_probe:
|
||||
# Full key4; counter=0 is implied (overlaps with key4[2:4] which must be 0x0000)
|
||||
p[3] = key4[2]
|
||||
p[4] = key4[3]
|
||||
else:
|
||||
p[3] = (counter >> 8) & 0xFF
|
||||
p[4] = counter & 0xFF
|
||||
return bytes(p)
|
||||
|
||||
|
||||
def bulk_waveform_term_params(key4: bytes, counter: int) -> bytes:
|
||||
"""
|
||||
⛔ DEPRECATED — DO NOT USE IN NEW CODE.
|
||||
|
||||
This is the v1 termination params helper, paired with the broken
|
||||
`_BULK_TERM_OFFSET = 0x005A` magic offset_word. Together they produce a
|
||||
~100-byte device-side terminator response that does NOT contain the
|
||||
partial-last-chunk waveform tail or the 26-byte file footer. Files
|
||||
reconstructed using this terminator are missing their last ~512 bytes of
|
||||
waveform data and have a synthesized footer that disagrees with what BW
|
||||
would have written.
|
||||
|
||||
**For new code, use `bulk_waveform_term_v2(key4, end_offset, last_chunk_counter)`**
|
||||
which computes the correct offset_word + params from the STRT-derived
|
||||
`end_offset`. v2 produces wire bytes that match BW exactly across all
|
||||
tested events (4-27-26 / 5-1-26 / 5-4-26 captures).
|
||||
|
||||
This function is retained ONLY for the defensive fallback path in
|
||||
`read_bulk_waveform_stream()` that triggers when STRT parsing fails or no
|
||||
chunks are fetched (= a malformed event or an unexpected device state).
|
||||
The fallback already logs a WARNING when it activates; if you see that
|
||||
warning, the bug is upstream — STRT should have been parseable.
|
||||
"""
|
||||
if len(key4) != 4:
|
||||
raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
|
||||
p = bytearray(10)
|
||||
p[0] = key4[0]
|
||||
p[1] = key4[1]
|
||||
p[2] = (counter >> 8) & 0xFF
|
||||
return bytes(p)
|
||||
|
||||
|
||||
def bulk_waveform_term_v2(
|
||||
key4: bytes,
|
||||
end_offset: int,
|
||||
last_chunk_counter: int,
|
||||
) -> tuple[int, bytes]:
|
||||
"""
|
||||
Compute the SUB 5A TERM frame's offset_word and 10-byte params block.
|
||||
|
||||
Confirmed across 3 events (4-27-26 + 5-1-26 captures):
|
||||
|
||||
next_boundary = last_chunk_counter + 0x0200
|
||||
offset_word = end_offset - next_boundary (residual byte count)
|
||||
params[0] = key4[0] (= 0x01 on every observed device)
|
||||
params[1] = key4[1] (= 0x11)
|
||||
params[2] = (next_boundary >> 8) & 0xFF
|
||||
params[3] = next_boundary & 0xFF
|
||||
params[4:10] = zeros
|
||||
|
||||
Verification:
|
||||
| end_offset | last_chunk | next_boundary | offset_word | params[2:4] |
|
||||
| 0x1ABE | 0x1800 | 0x1A00 | 0x00BE | 1A 00 |
|
||||
| 0x21F2 | 0x1E00 | 0x2000 | 0x01F2 | 20 00 |
|
||||
| 0x417E | 0x3E38 | 0x4038 | 0x0146 | 40 38 |
|
||||
|
||||
The device receives `requested_address = (params[2] << 8) | offset_word`
|
||||
and replies with `(end_offset - next_boundary)` bytes of waveform tail
|
||||
starting at `next_boundary` — including the 26-byte file footer.
|
||||
|
||||
Args:
|
||||
key4: 4-byte waveform key for this event.
|
||||
end_offset: Event-end pointer (= `(end_key[2] << 8) | end_key[3]`
|
||||
from the STRT record at data[23:27] of A5[0]).
|
||||
last_chunk_counter: Counter of the last full 0x0200-byte chunk fetched
|
||||
(the chunk that covers [last_chunk_counter,
|
||||
last_chunk_counter + 0x0200)).
|
||||
|
||||
Returns:
|
||||
(offset_word, params10) tuple. Pass as
|
||||
`build_5a_frame(offset_word, params)`.
|
||||
|
||||
Raises:
|
||||
ValueError: on inconsistent inputs.
|
||||
"""
|
||||
if len(key4) != 4:
|
||||
raise ValueError(f"waveform key must be 4 bytes, got {len(key4)}")
|
||||
next_boundary = last_chunk_counter + 0x0200
|
||||
if next_boundary > 0xFFFF:
|
||||
raise ValueError(
|
||||
f"next_boundary 0x{next_boundary:04X} exceeds uint16; check inputs"
|
||||
)
|
||||
if end_offset <= last_chunk_counter:
|
||||
raise ValueError(
|
||||
f"end_offset 0x{end_offset:04X} must be > "
|
||||
f"last_chunk_counter 0x{last_chunk_counter:04X}"
|
||||
)
|
||||
offset_word = end_offset - next_boundary
|
||||
if offset_word < 0:
|
||||
# Last chunk overshot end_offset; caller should have stopped one chunk
|
||||
# earlier. Treat as zero residual.
|
||||
offset_word = 0
|
||||
if offset_word > 0xFFFF:
|
||||
raise ValueError(
|
||||
f"offset_word 0x{offset_word:04X} exceeds uint16"
|
||||
)
|
||||
p = bytearray(10)
|
||||
p[0] = key4[0]
|
||||
p[1] = key4[1]
|
||||
p[2] = (next_boundary >> 8) & 0xFF
|
||||
p[3] = next_boundary & 0xFF
|
||||
return offset_word, bytes(p)
|
||||
|
||||
|
||||
# ── End-offset extraction from STRT record ────────────────────────────────────
|
||||
|
||||
STRT_MARKER = b"STRT"
|
||||
|
||||
|
||||
def parse_strt_end_offset(a5_data: bytes) -> Optional[int]:
|
||||
"""
|
||||
Extract the event-end offset from the STRT record in an A5 response payload.
|
||||
|
||||
The first A5 response (the probe response, or the first chunk for events
|
||||
with non-zero start_key[2:4]) contains a STRT record at byte offset 17 of
|
||||
`data`. Layout:
|
||||
|
||||
data[17:21] "STRT"
|
||||
data[21:23] ff fe sentinel
|
||||
data[23:27] end_key ← 4-byte key of where this event ENDS
|
||||
data[27:31] start_key
|
||||
...
|
||||
|
||||
Returns `(end_key[2] << 8) | end_key[3]` — the absolute device-buffer
|
||||
address where the event ends. Use this to bound the chunk loop and to
|
||||
compute the TERM frame.
|
||||
|
||||
Verified end_offset values:
|
||||
| event start_key | end_key | end_offset |
|
||||
| 01110000 | 01111ABE | 0x1ABE |
|
||||
| 01110000 | 011121F2 | 0x21F2 |
|
||||
| 011121F2 | 0111417E | 0x417E |
|
||||
|
||||
Args:
|
||||
a5_data: The `data` field of an A5 response frame (frame.data).
|
||||
|
||||
Returns:
|
||||
The end_offset (uint16) if STRT is found, else None.
|
||||
"""
|
||||
pos = a5_data.find(STRT_MARKER)
|
||||
if pos < 0 or pos + 10 > len(a5_data):
|
||||
return None
|
||||
# data[pos+4:pos+6] is "ff fe"; data[pos+6:pos+10] is end_key.
|
||||
end_key = a5_data[pos + 6 : pos + 10]
|
||||
if len(end_key) < 4:
|
||||
return None
|
||||
return (end_key[2] << 8) | end_key[3]
|
||||
|
||||
|
||||
# ── Pre-built POLL frames ─────────────────────────────────────────────────────
|
||||
#
|
||||
# POLL (SUB 0x5B) uses the same two-step pattern as all other reads — the
|
||||
@@ -190,6 +589,14 @@ def token_params(token: int = 0) -> bytes:
|
||||
POLL_PROBE = build_bw_frame(0x5B, 0x00) # length-probe POLL (offset = 0)
|
||||
POLL_DATA = build_bw_frame(0x5B, 0x30) # data-request POLL (offset = 0x30)
|
||||
|
||||
# Session-reset signal (ACK + ETX, no STX/payload).
|
||||
# Confirmed from 4-8-26 BW TX captures: Blastware sends this 2-byte sequence
|
||||
# immediately before the first POLL probe, and again between the POLL probe
|
||||
# and the POLL data request. Required to wake a unit that is actively
|
||||
# monitoring — without it the unit does not respond to POLL over TCP.
|
||||
# Harmless for idle units (they respond to POLL regardless).
|
||||
SESSION_RESET = bytes([0x41, 0x03])
|
||||
|
||||
|
||||
# ── S3 response dataclass ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -201,6 +608,11 @@ class S3Frame:
|
||||
page_lo: int # PAGE_LO from header
|
||||
data: bytes # payload data section (payload[5:], checksum already stripped)
|
||||
checksum_valid: bool
|
||||
chk_byte: int = 0 # actual checksum byte received from wire (body[-1])
|
||||
# needed for waveform file reconstruction: when the last data byte
|
||||
# is 0x10 and chk_byte ∈ {0x02, 0x03, 0x04}, the DLE+chk pair
|
||||
# must be included in the DLE-strip operation to correctly
|
||||
# reconstruct the Blastware binary body.
|
||||
|
||||
@property
|
||||
def page_key(self) -> int:
|
||||
@@ -209,7 +621,6 @@ class S3Frame:
|
||||
|
||||
|
||||
# ── Streaming S3 frame parser ─────────────────────────────────────────────────
|
||||
|
||||
class S3FrameParser:
|
||||
"""
|
||||
Incremental byte-stream parser for S3→BW response frames.
|
||||
@@ -241,13 +652,15 @@ class S3FrameParser:
|
||||
_IN_FRAME_DLE = 3
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._state = self._IDLE
|
||||
self._body = bytearray() # accumulates de-stuffed frame bytes
|
||||
self._state = self._IDLE
|
||||
self._body = bytearray() # accumulates de-stuffed frame bytes
|
||||
self.frames: list[S3Frame] = []
|
||||
self.bytes_fed: int = 0 # cumulative raw bytes fed since last reset
|
||||
|
||||
def reset(self) -> None:
|
||||
self._state = self._IDLE
|
||||
self._state = self._IDLE
|
||||
self._body.clear()
|
||||
self.bytes_fed = 0
|
||||
|
||||
def feed(self, data: bytes) -> list[S3Frame]:
|
||||
"""
|
||||
@@ -256,6 +669,7 @@ class S3FrameParser:
|
||||
Returns a list of S3Frame objects completed during this call.
|
||||
All completed frames are also appended to self.frames.
|
||||
"""
|
||||
self.bytes_fed += len(data)
|
||||
completed: list[S3Frame] = []
|
||||
for b in data:
|
||||
frame = self._step(b)
|
||||
@@ -333,9 +747,10 @@ class S3FrameParser:
|
||||
return None
|
||||
|
||||
return S3Frame(
|
||||
sub = raw_payload[2],
|
||||
page_hi = raw_payload[3],
|
||||
page_lo = raw_payload[4],
|
||||
data = raw_payload[5:],
|
||||
sub = raw_payload[2],
|
||||
page_hi = raw_payload[3],
|
||||
page_lo = raw_payload[4],
|
||||
data = raw_payload[5:],
|
||||
checksum_valid = (chk_received == chk_computed),
|
||||
chk_byte = chk_received,
|
||||
)
|
||||
|
||||
+306
-9
@@ -14,6 +14,7 @@ Notes on certainty:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import struct
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
@@ -98,14 +99,17 @@ class Timestamp:
|
||||
|
||||
Wire layout (✅ CONFIRMED 2026-04-01 against Blastware event report):
|
||||
byte[0]: day (uint8)
|
||||
byte[1]: sub_code / mode flag (0x10 = Waveform mode) 🔶
|
||||
byte[1]: sub_code / mode flag (0x10 = Waveform single-shot)
|
||||
byte[2]: month (uint8)
|
||||
bytes[3–4]: year (big-endian uint16)
|
||||
byte[5]: unknown (0x00 in all observed samples ❓)
|
||||
byte[5]: unknown (0x00 in all observed samples)
|
||||
byte[6]: hour (uint8)
|
||||
byte[7]: minute (uint8)
|
||||
byte[8]: second (uint8)
|
||||
|
||||
Used for sub_code=0x10 records only. For sub_code=0x03 (continuous
|
||||
mode) use from_continuous_record() — the layout is shifted by 1 byte.
|
||||
|
||||
Args:
|
||||
data: at least 9 bytes; only the first 9 are consumed.
|
||||
|
||||
@@ -120,7 +124,7 @@ class Timestamp:
|
||||
f"Waveform record timestamp requires at least 9 bytes, got {len(data)}"
|
||||
)
|
||||
day = data[0]
|
||||
sub_code = data[1] # 0x10 = Waveform; histogram code not yet confirmed
|
||||
sub_code = data[1] # 0x10 = Waveform single-shot
|
||||
month = data[2]
|
||||
year = struct.unpack_from(">H", data, 3)[0]
|
||||
unknown_byte = data[5]
|
||||
@@ -139,6 +143,116 @@ class Timestamp:
|
||||
second=second,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_continuous_record(cls, data: bytes) -> "Timestamp":
|
||||
"""
|
||||
Decode a 10-byte timestamp from the first bytes of a sub_code=0x03
|
||||
(Waveform Continuous) 210-byte record.
|
||||
|
||||
Wire layout (✅ CONFIRMED 2026-04-03 against Blastware event report,
|
||||
event recorded at 15:20:17 April 3 2026, raw: 10 03 10 04 07 ea 00 0f 14 11):
|
||||
byte[0]: unknown_a (0x10 observed — meaning TBD)
|
||||
byte[1]: day (uint8)
|
||||
byte[2]: unknown_b (0x10 observed — meaning TBD)
|
||||
bytes[3]: month (uint8)
|
||||
bytes[4–5]: year (big-endian uint16)
|
||||
byte[6]: unknown (0x00 in all observed samples)
|
||||
byte[7]: hour (uint8)
|
||||
byte[8]: minute (uint8)
|
||||
byte[9]: second (uint8)
|
||||
|
||||
This is the sub_code=0x10 layout shifted forward by 1 byte, with two
|
||||
extra unknown bytes at [0] and [2]. The sub_code (0x03) itself is at
|
||||
byte[1] in the raw record, which also encodes the day — but the day
|
||||
value (3 = April 3rd) happens to differ from the sub_code (0x03) only
|
||||
in semantics; the byte is shared.
|
||||
|
||||
Args:
|
||||
data: at least 10 bytes; only the first 10 are consumed.
|
||||
|
||||
Returns:
|
||||
Decoded Timestamp with hour/minute/second populated.
|
||||
|
||||
Raises:
|
||||
ValueError: if data is fewer than 10 bytes.
|
||||
"""
|
||||
if len(data) < 10:
|
||||
raise ValueError(
|
||||
f"Continuous record timestamp requires at least 10 bytes, got {len(data)}"
|
||||
)
|
||||
unknown_a = data[0] # 0x10 observed; meaning unknown
|
||||
day = data[1] # doubles as the sub_code byte (0x03) — day=3 on Apr 3
|
||||
unknown_b = data[2] # 0x10 observed; meaning unknown
|
||||
month = data[3]
|
||||
year = struct.unpack_from(">H", data, 4)[0]
|
||||
unknown_byte = data[6]
|
||||
hour = data[7]
|
||||
minute = data[8]
|
||||
second = data[9]
|
||||
return cls(
|
||||
raw=bytes(data[:10]),
|
||||
flag=unknown_a,
|
||||
year=year,
|
||||
unknown_byte=unknown_byte,
|
||||
month=month,
|
||||
day=day,
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
second=second,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_short_record(cls, data: bytes) -> "Timestamp":
|
||||
"""
|
||||
Decode an 8-byte timestamp header from a 210-byte waveform record.
|
||||
|
||||
Wire layout (✅ CONFIRMED 2026-05-01 against live SFM run on BE11529 in
|
||||
Continuous mode, day-of-month = 1 May, raw: 01 05 07 ea 00 0d 15 25):
|
||||
byte[0]: day (uint8)
|
||||
byte[1]: month (uint8)
|
||||
bytes[2-3]: year (big-endian uint16)
|
||||
byte[4]: unknown (0x00 in observed sample)
|
||||
byte[5]: hour (uint8)
|
||||
byte[6]: minute (uint8)
|
||||
byte[7]: second (uint8)
|
||||
|
||||
This is a third format observed in the wild — distinct from the 9-byte
|
||||
(single-shot, sub_code=0x10 at [1]) and 10-byte (continuous, 0x10 at
|
||||
[0] AND [2]) layouts. No marker bytes; disambiguated by where the
|
||||
year lands when scanned at byte 2/3/4.
|
||||
|
||||
Args:
|
||||
data: at least 8 bytes; only the first 8 are consumed.
|
||||
|
||||
Returns:
|
||||
Decoded Timestamp.
|
||||
|
||||
Raises:
|
||||
ValueError: if data is fewer than 8 bytes.
|
||||
"""
|
||||
if len(data) < 8:
|
||||
raise ValueError(
|
||||
f"Short record timestamp requires at least 8 bytes, got {len(data)}"
|
||||
)
|
||||
day = data[0]
|
||||
month = data[1]
|
||||
year = struct.unpack_from(">H", data, 2)[0]
|
||||
unknown_byte = data[4]
|
||||
hour = data[5]
|
||||
minute = data[6]
|
||||
second = data[7]
|
||||
return cls(
|
||||
raw=bytes(data[:8]),
|
||||
flag=0,
|
||||
year=year,
|
||||
unknown_byte=unknown_byte,
|
||||
month=month,
|
||||
day=day,
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
second=second,
|
||||
)
|
||||
|
||||
@property
|
||||
def clock_set(self) -> bool:
|
||||
"""False when year == 1995 (factory default / battery-lost state)."""
|
||||
@@ -183,6 +297,9 @@ class DeviceInfo:
|
||||
# ── From SUB 1A (COMPLIANCE_CONFIG_RESPONSE) ──────────────────────────────
|
||||
compliance_config: Optional["ComplianceConfig"] = None # E5 response, read in connect()
|
||||
|
||||
# ── From SUB 08 (EVENT_INDEX_RESPONSE) ────────────────────────────────────
|
||||
event_count: Optional[int] = None # stored event count from F7 response 🔶
|
||||
|
||||
def __str__(self) -> str:
|
||||
fw = self.firmware_version or f"?.{self.firmware_minor}"
|
||||
mdl = self.model or "MiniMate Plus"
|
||||
@@ -204,7 +321,7 @@ class ChannelConfig:
|
||||
label: str # e.g. "Tran", "Vert", "Long", "MicL" ✅
|
||||
trigger_level: float # in/s (geo) or psi (MicL) ✅
|
||||
alarm_level: float # in/s (geo) or psi (MicL) ✅
|
||||
max_range: float # full-scale calibration constant (e.g. 6.206) 🔶
|
||||
max_range: float # hardware/firmware sensitivity constant (e.g. 6.206053) ✅ confirmed same on all units
|
||||
unit_label: str # e.g. "in./s" or "psi" ✅
|
||||
|
||||
|
||||
@@ -273,15 +390,34 @@ class ComplianceConfig:
|
||||
raw: Optional[bytes] = None # full 2090-byte payload (for debugging)
|
||||
|
||||
# Recording parameters (✅ CONFIRMED from §7.6)
|
||||
record_time: Optional[float] = None # seconds (7.0, 10.0, 13.0, etc.)
|
||||
sample_rate: Optional[int] = None # sps (1024, 2048, 4096, etc.) — NOT YET FOUND ❓
|
||||
recording_mode: Optional[int] = None # uint8: 0x00=Single Shot, 0x01=Continuous,
|
||||
# 0x03=Histogram, 0x04=Histogram+Continuous ✅ confirmed 2026-04-20
|
||||
# Read (E5): data[anchor_pos - 8] (6-byte anchor)
|
||||
# Write (SUB 71): data[anchor_pos - 7]
|
||||
sample_rate: Optional[int] = None # sps (1024, 2048, 4096)
|
||||
histogram_interval_sec: Optional[int] = None # uint16 BE, seconds ✅ confirmed 2026-04-20
|
||||
# anchor_pos - 4 (same offset in read & write)
|
||||
# Valid values: 2, 5, 15, 60, 300, 900
|
||||
# Mode-gated: only active in Histogram/Histogram+Continuous
|
||||
record_time: Optional[float] = None # seconds (e.g. 3.0, 5.0, 8.0, 10.0)
|
||||
|
||||
# Trigger/alarm levels (✅ CONFIRMED per-channel at §7.6)
|
||||
# For now we store the first geo channel (Transverse) as representatives;
|
||||
# full per-channel data would require structured Channel objects.
|
||||
trigger_level_geo: Optional[float] = None # in/s (first geo channel)
|
||||
alarm_level_geo: Optional[float] = None # in/s (first geo channel)
|
||||
max_range_geo: Optional[float] = None # in/s full-scale range
|
||||
trigger_level_geo: Optional[float] = None # in/s (first geo channel) ✅
|
||||
alarm_level_geo: Optional[float] = None # in/s (first geo channel) ✅
|
||||
geo_adc_scale: Optional[float] = None # ADC-to-velocity scale factor (float32 at Tran+28) ✅
|
||||
# = inverse sensitivity = 1/sensitivity (in/s per V)
|
||||
# Formula (Interface Handbook §4.5): Range = 1.61133 V × scale_factor
|
||||
# → 1.61133 × 6.206053 = 10.000 in/s (Normal range) ✅
|
||||
# Firmware uses: PPV (in/s) = ADC_voltage (V) × 6.206053
|
||||
# Identical on BE11529 and BE18189 — same Instantel geophone hardware.
|
||||
# NOT a user-configurable setting. Must NOT be written.
|
||||
geo_range: Optional[int] = None # range/sensitivity selector — CONFIRMED 2026-04-20
|
||||
# 0x00 = Normal 10.000 in/s (standard gain)
|
||||
# 0x01 = Sensitive 1.250 in/s (high gain)
|
||||
# Offset: Tran+33 in both E5 read and SUB 71 write payloads
|
||||
# (same 2126-byte buffer is round-tripped; applied to Tran/Vert/Long)
|
||||
|
||||
# Project/setup strings (sourced from E5 / SUB 71 write payload)
|
||||
# These are the FULL project metadata from compliance config,
|
||||
@@ -294,6 +430,78 @@ class ComplianceConfig:
|
||||
notes: Optional[str] = None # extended notes / additional info
|
||||
|
||||
|
||||
# ── Call Home Config ──────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class CallHomeConfig:
|
||||
"""
|
||||
Auto Call Home (ACH) configuration from SUB 0x2C (response 0xD3).
|
||||
|
||||
Read with a standard two-step protocol (probe offset=0x00, data offset=0x7C).
|
||||
Written via SUB 0x7E (write, 127-byte payload) + SUB 0x7F (confirm).
|
||||
|
||||
Confirmed from 4-20-26 call home settings captures (11 BW + S3 capture pairs).
|
||||
|
||||
Raw payload layout (data[11:] from S3 response, 125 bytes):
|
||||
[0] 0x00 header byte
|
||||
[1] 0x7C = 124 inner length (= offset for SUB 0x7E write - 2)
|
||||
[2] 0xDC constant
|
||||
[3:5] 0x00 0x00 padding
|
||||
[5] auto_call_home_enabled (0x00=off, 0x01=on) ✅
|
||||
[6:46] dial_string 40-byte null-padded ASCII ✅
|
||||
[46:87] auto_answer_raw AT command strings (not decoded) ✅ present
|
||||
[87] after_event_recorded (0x01=on, 0x00=off) ✅
|
||||
[91] at_specified_times (0x01=on, 0x00=off) ✅
|
||||
[93] time1_enabled (0x01=on, 0x00=off) ✅
|
||||
[95] time2_enabled (0x01=on, 0x00=off) ✅
|
||||
[101] time1_hour uint8 decimal 0-23 ✅
|
||||
[102] time1_min uint8 decimal 0-59 ✅
|
||||
[105] time2_hour uint8 decimal 0-23 ✅
|
||||
[106] time2_min uint8 decimal 0-59 ✅
|
||||
[117] DLE prefix (0x10) ┐ DLE-escaped num_retries=3 (0x03)
|
||||
[118] 0x03 ┘ device stores/returns 0x03 DLE-escaped ✅
|
||||
[120] time_between_retries_sec uint8 (= 0x0F = 15 s default) ✅
|
||||
[122] wait_for_connection_sec uint8 (= 0x3C = 60 s default) ✅
|
||||
[124] warm_up_time_sec uint8 (= 0x3C = 60 s default) ✅
|
||||
|
||||
Write payload = raw 125 bytes + b'\\x00\\x00' (2 trailing zeros) = 127 bytes.
|
||||
Offset for SUB 0x7E: data[1] + 2 = 0x7C + 2 = 0x7E = 126.
|
||||
|
||||
Note on DLE-escaped 0x03: The device's S3 response DLE-escapes ETX (0x03)
|
||||
bytes as \\x10\\x03. The S3FrameParser preserves both bytes in frame.data.
|
||||
Subsequent fields after offset 117 are therefore at raw_offset = logical+1.
|
||||
The raw payload must be round-tripped verbatim in write; do NOT reapply DLE
|
||||
destuffing or stripping.
|
||||
"""
|
||||
raw: Optional[bytes] = None # raw 125-byte read payload (for round-trip write)
|
||||
|
||||
# ── Main enable ──────────────────────────────────────────────────────────
|
||||
auto_call_home_enabled: Optional[bool] = None # raw[5] ✅
|
||||
|
||||
# ── Dial string ──────────────────────────────────────────────────────────
|
||||
dial_string: Optional[str] = None # raw[6:46] 40-byte null-padded ASCII ✅
|
||||
|
||||
# ── When to call ─────────────────────────────────────────────────────────
|
||||
after_event_recorded: Optional[bool] = None # raw[87] ✅
|
||||
at_specified_times: Optional[bool] = None # raw[91] ✅
|
||||
|
||||
# ── Time slot 1 ──────────────────────────────────────────────────────────
|
||||
time1_enabled: Optional[bool] = None # raw[93] ✅
|
||||
time1_hour: Optional[int] = None # raw[101] 0-23 ✅
|
||||
time1_min: Optional[int] = None # raw[102] 0-59 ✅
|
||||
|
||||
# ── Time slot 2 ──────────────────────────────────────────────────────────
|
||||
time2_enabled: Optional[bool] = None # raw[95] ✅
|
||||
time2_hour: Optional[int] = None # raw[105] 0-23 ✅
|
||||
time2_min: Optional[int] = None # raw[106] 0-59 ✅
|
||||
|
||||
# ── Retry / timeout settings (read-only; not writable via set_call_home_config) ──
|
||||
num_retries: Optional[int] = None # raw[117:119]=10 03 → value 3 ✅
|
||||
time_between_retries_sec: Optional[int] = None # raw[120] (shifted +1 by DLE) ✅
|
||||
wait_for_connection_sec: Optional[int] = None # raw[122] ✅
|
||||
warm_up_time_sec: Optional[int] = None # raw[124] ✅
|
||||
|
||||
|
||||
# ── Event ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
@@ -324,12 +532,23 @@ class Event:
|
||||
# Raw ADC samples keyed by channel label. Not fetched unless explicitly
|
||||
# requested (large data transfer — up to several MB per event).
|
||||
raw_samples: Optional[dict] = None # {"Tran": [...], "Vert": [...], ...}
|
||||
total_samples: Optional[int] = None # from STRT record: expected total sample-sets
|
||||
pretrig_samples: Optional[int] = None # from STRT record: pre-trigger sample count
|
||||
rectime_seconds: Optional[int] = None # from STRT record: record duration (seconds)
|
||||
|
||||
# ── Debug / introspection ─────────────────────────────────────────────────
|
||||
# Raw 210-byte waveform record bytes, set when debug mode is active.
|
||||
# Exposed by the SFM server via ?debug=true so field layouts can be verified.
|
||||
_raw_record: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
# 4-byte waveform key used to request this event via SUB 5A.
|
||||
# Set by get_events(); required by download_waveform().
|
||||
_waveform_key: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
# Raw A5 frames from the full bulk waveform download (full_waveform=True).
|
||||
# Populated by get_events() when full_waveform=True; used by write_blastware_file().
|
||||
_a5_frames: Optional[list] = field(default=None, repr=False)
|
||||
|
||||
def __str__(self) -> str:
|
||||
ts = str(self.timestamp) if self.timestamp else "no timestamp"
|
||||
ppv = ""
|
||||
@@ -346,3 +565,81 @@ class Event:
|
||||
parts.append(f"M={pv.micl:.6f}")
|
||||
ppv = " [" + ", ".join(parts) + " in/s]"
|
||||
return f"Event#{self.index} {ts}{ppv}"
|
||||
|
||||
|
||||
# ── MonitorLogEntry ───────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class MonitorLogEntry:
|
||||
"""
|
||||
A monitor log entry decoded from a SUB 0x0A (WAVEFORM_HEADER) response
|
||||
whose first byte is 0x2C (partial record, recording mode = continuous
|
||||
monitoring without a triggered event).
|
||||
|
||||
These are the "partial bins" that Blastware stores between triggered events.
|
||||
Each entry represents one monitoring interval — the span of time during
|
||||
which the unit was actively monitoring but no threshold crossing occurred.
|
||||
|
||||
Confirmed from 4-11-26 MITM capture analysis (2026-04-11):
|
||||
|
||||
Header layout (full response data[0:]):
|
||||
data[0] = 0x2C (partial record type / data length in probe response)
|
||||
data[1:5] = 0x00 × 4
|
||||
data[5:9] = event key (4 bytes, big-endian hex)
|
||||
data[9:11] = 0x00 × 2
|
||||
data[11:] = timestamp_start (9 or 10 bytes depending on recording mode)
|
||||
+ timestamp_stop (same format)
|
||||
+ separator (4–5 bytes, variable)
|
||||
+ serial null-terminated (e.g. "BE11529\\0")
|
||||
+ "Geo: X.XXX in/s\\0" (trigger threshold string)
|
||||
|
||||
Timestamp format detection:
|
||||
data[11] == 0x10 → 10-byte sub_code=0x03 (continuous) format
|
||||
data[12] == 0x10 → 9-byte sub_code=0x10 (single-shot) format
|
||||
|
||||
In contrast to Event (triggered records, type 0x46), MonitorLogEntry
|
||||
records do NOT have a waveform record (SUB 0x0C) or bulk waveform stream
|
||||
(SUB 5A). All available metadata is in the 0x0A header alone.
|
||||
"""
|
||||
index: int # 0-based position in device record list
|
||||
key: str # 8-hex event key (e.g. "01114290") ✅
|
||||
|
||||
start_time: Optional[datetime.datetime] = None # monitoring session start ✅
|
||||
stop_time: Optional[datetime.datetime] = None # monitoring session stop ✅
|
||||
serial: Optional[str] = None # device serial (e.g. "BE11529") ✅
|
||||
geo_threshold_ips: Optional[float] = None # trigger level from "Geo: X.XXX in/s" ✅
|
||||
|
||||
# Raw bytes for debugging / future decoding
|
||||
raw_header: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> Optional[float]:
|
||||
"""Duration of monitoring interval in seconds, or None if times unavailable."""
|
||||
if self.start_time and self.stop_time:
|
||||
return (self.stop_time - self.start_time).total_seconds()
|
||||
return None
|
||||
|
||||
def __str__(self) -> str:
|
||||
start = self.start_time.isoformat() if self.start_time else "?"
|
||||
stop = self.stop_time.isoformat() if self.stop_time else "?"
|
||||
dur = f" ({self.duration_seconds:.0f}s)" if self.duration_seconds is not None else ""
|
||||
return f"MonitorLog#{self.index} key={self.key} {start}→{stop}{dur}"
|
||||
|
||||
|
||||
# ── MonitorStatus ─────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class MonitorStatus:
|
||||
"""
|
||||
Current monitoring state decoded from SUB 0x1C response.
|
||||
|
||||
Confirmed field locations from 4-8-26/2ndtry BW capture:
|
||||
battery_v : data[11 + 0x2F : 11 + 0x31] uint16 BE ÷ 100 e.g. 680 → 6.80 V
|
||||
memory_total: data[11 + 0x31 : 11 + 0x35] uint32 BE bytes e.g. 983040 → 960 KB
|
||||
memory_free : data[11 + 0x35 : 11 + 0x39] uint32 BE bytes (subset of total)
|
||||
is_monitoring: inferred from payload length — idle = 44 bytes, monitoring = 12 bytes
|
||||
"""
|
||||
is_monitoring: bool # True if unit is actively recording ✅
|
||||
battery_v: Optional[float] = None # Battery voltage in volts ✅
|
||||
memory_total: Optional[int] = None # Total flash memory in bytes ✅
|
||||
memory_free: Optional[int] = None # Free flash memory in bytes ✅
|
||||
|
||||
+999
-54
File diff suppressed because it is too large
Load Diff
@@ -418,3 +418,138 @@ class TcpTransport(BaseTransport):
|
||||
def __repr__(self) -> str:
|
||||
state = "connected" if self.is_connected else "disconnected"
|
||||
return f"TcpTransport({self.host!r}, port={self.port}, {state})"
|
||||
|
||||
|
||||
# ── Inbound / accepted-socket transport ───────────────────────────────────────
|
||||
|
||||
class SocketTransport(TcpTransport):
|
||||
"""
|
||||
Like TcpTransport but wraps an already-accepted inbound socket.
|
||||
|
||||
Used by the ACH inbound server (bridges/ach_server.py) — the device dials
|
||||
IN to us, so by the time we create this transport the socket is already live.
|
||||
connect() is a no-op; everything else (read, write, read_until_idle, …) is
|
||||
inherited unchanged from TcpTransport.
|
||||
|
||||
Args:
|
||||
sock: An already-connected socket.socket returned by server_socket.accept().
|
||||
peer: Human-readable peer label for repr / logging (e.g. "203.0.113.5:54321").
|
||||
"""
|
||||
|
||||
def __init__(self, sock: socket.socket, peer: str = "inbound") -> None:
|
||||
# Bypass TcpTransport.__init__ — we already have a live socket.
|
||||
self.host = peer
|
||||
self.port = 0
|
||||
self.connect_timeout = 0.0
|
||||
self._sock = sock
|
||||
sock.settimeout(self._RECV_TIMEOUT)
|
||||
|
||||
def connect(self) -> None:
|
||||
"""No-op — socket was already accepted inbound."""
|
||||
pass # Already have a live socket; nothing to open.
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._sock is not None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SocketTransport(peer={self.host!r})"
|
||||
|
||||
|
||||
# ── Capturing transport (MITM-style raw byte mirror) ──────────────────────────
|
||||
|
||||
class CapturingTransport(BaseTransport):
|
||||
"""
|
||||
Wraps another BaseTransport and mirrors every byte to two raw capture files:
|
||||
|
||||
raw_bw_<...>.bin — bytes WE wrote to the device (BW-side TX)
|
||||
raw_s3_<...>.bin — bytes the device wrote back (S3-side TX)
|
||||
|
||||
The file naming and on-wire byte layout are identical to the captures
|
||||
produced by `bridges/ach_mitm.py`, so the resulting `.bin` files can be
|
||||
loaded directly by the Analyzer (File > Open Capture) and parsed by the
|
||||
same tooling used for genuine Blastware MITM captures.
|
||||
|
||||
All BaseTransport methods are forwarded to the inner transport; the only
|
||||
side-effect is that successful read/write byte streams are appended to the
|
||||
two open binary files.
|
||||
|
||||
Args:
|
||||
inner: An already-built BaseTransport (SerialTransport / TcpTransport).
|
||||
bw_path: File path for the "BW TX" stream (bytes we send). Opened "wb".
|
||||
s3_path: File path for the "S3 TX" stream (bytes the device sends).
|
||||
Opened "wb".
|
||||
|
||||
Example:
|
||||
with CapturingTransport(TcpTransport("1.2.3.4", 9034),
|
||||
"raw_bw.bin", "raw_s3.bin") as t:
|
||||
client = MiniMateClient(transport=t)
|
||||
client.connect()
|
||||
client.get_events()
|
||||
# both .bin files now hold the full bidirectional capture.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: BaseTransport, bw_path: str, s3_path: str) -> None:
|
||||
self._inner = inner
|
||||
self._bw_path = bw_path
|
||||
self._s3_path = s3_path
|
||||
self._bw_fh = None
|
||||
self._s3_fh = None
|
||||
# Forward inner attrs so callers can introspect (e.g. .host, .port).
|
||||
self.host = getattr(inner, "host", None)
|
||||
self.port = getattr(inner, "port", None)
|
||||
|
||||
# ── BaseTransport interface ───────────────────────────────────────────────
|
||||
|
||||
def connect(self) -> None:
|
||||
if self._bw_fh is None:
|
||||
self._bw_fh = open(self._bw_path, "wb", buffering=0)
|
||||
if self._s3_fh is None:
|
||||
self._s3_fh = open(self._s3_path, "wb", buffering=0)
|
||||
self._inner.connect()
|
||||
|
||||
def disconnect(self) -> None:
|
||||
try:
|
||||
self._inner.disconnect()
|
||||
finally:
|
||||
for fh_attr in ("_bw_fh", "_s3_fh"):
|
||||
fh = getattr(self, fh_attr)
|
||||
if fh is not None:
|
||||
try:
|
||||
fh.flush()
|
||||
fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
setattr(self, fh_attr, None)
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._inner.is_connected
|
||||
|
||||
def write(self, data: bytes) -> None:
|
||||
self._inner.write(data)
|
||||
if data and self._bw_fh is not None:
|
||||
try:
|
||||
self._bw_fh.write(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def read(self, n: int) -> bytes:
|
||||
got = self._inner.read(n)
|
||||
if got and self._s3_fh is not None:
|
||||
try:
|
||||
self._s3_fh.write(got)
|
||||
except Exception:
|
||||
pass
|
||||
return got
|
||||
|
||||
@property
|
||||
def bw_path(self) -> str:
|
||||
return self._bw_path
|
||||
|
||||
@property
|
||||
def s3_path(self) -> str:
|
||||
return self._s3_path
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"CapturingTransport({self._inner!r}, bw={self._bw_path!r}, s3={self._s3_path!r})"
|
||||
|
||||
@@ -53,7 +53,9 @@ SUB_TABLE: dict[int, tuple[str, str, str]] = {
|
||||
0x82: ("TRIGGER_CONFIG_WRITE", "BW→S3", "0x1C bytes; trigger config block; mirrors SUB 1C"),
|
||||
0x83: ("TRIGGER_WRITE_CONFIRM", "BW→S3", "Short frame; commit step after 0x82"),
|
||||
# S3→BW responses
|
||||
0x5A: ("BULK_WAVEFORM_STREAM", "BW→S3", "Bulk waveform chunk request; response is A5 stream"),
|
||||
0xA4: ("POLL_RESPONSE", "S3→BW", "Response to SUB 5B poll"),
|
||||
0xA5: ("BULK_WAVEFORM_RESPONSE", "S3→BW", "Response to SUB 5A; waveform chunks + metadata"),
|
||||
0xFE: ("FULL_CONFIG_RESPONSE", "S3→BW", "Response to SUB 01"),
|
||||
0xF9: ("CHANNEL_CONFIG_RESPONSE", "S3→BW", "Response to SUB 06"),
|
||||
0xF7: ("EVENT_INDEX_RESPONSE", "S3→BW", "Response to SUB 08; contains backlight/power-save"),
|
||||
|
||||
+36
-21
@@ -33,7 +33,7 @@ STX = 0x02
|
||||
ETX = 0x03
|
||||
ACK = 0x41
|
||||
|
||||
__version__ = "0.2.2"
|
||||
__version__ = "0.2.5"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -184,9 +184,9 @@ def validate_bw_body_auto(body: bytes) -> Optional[Tuple[bytes, bytes, str]]:
|
||||
def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]:
|
||||
frames: List[Frame] = []
|
||||
|
||||
IDLE = 0
|
||||
IN_FRAME = 1
|
||||
AFTER_DLE = 2
|
||||
IDLE = 0
|
||||
IN_FRAME = 1
|
||||
IN_FRAME_DLE = 2 # saw DLE inside frame — waiting for next byte
|
||||
|
||||
state = IDLE
|
||||
body = bytearray()
|
||||
@@ -206,28 +206,26 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]:
|
||||
state = IN_FRAME
|
||||
i += 2
|
||||
continue
|
||||
# ACK bytes, boot strings, garbage — silently ignored
|
||||
|
||||
elif state == IN_FRAME:
|
||||
if b == DLE:
|
||||
state = AFTER_DLE
|
||||
state = IN_FRAME_DLE
|
||||
i += 1
|
||||
continue
|
||||
body.append(b)
|
||||
|
||||
else: # AFTER_DLE
|
||||
if b == DLE:
|
||||
body.append(DLE)
|
||||
state = IN_FRAME
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if b == ETX:
|
||||
# Bare ETX = real S3 frame terminator (confirmed from S3FrameParser)
|
||||
end_offset = i + 1
|
||||
trailer_start = i + 1
|
||||
trailer_end = trailer_start + trailer_len
|
||||
trailer = blob[trailer_start:trailer_end]
|
||||
|
||||
# For S3 mode we don't assume checksum type here yet.
|
||||
# S3 checksums are deliberately not validated here.
|
||||
# Large S3 responses (A5 bulk waveform, E5 compliance) embed
|
||||
# inner DLE+ETX sub-frame terminators whose trailing 0x03 byte
|
||||
# lands where the parser would expect the SUM8 checksum, causing
|
||||
# false failures. The live protocol (protocol.py _validate_frame)
|
||||
# also skips S3 checksum enforcement for the same reason.
|
||||
frames.append(Frame(
|
||||
index=idx,
|
||||
start_offset=start_offset,
|
||||
@@ -244,13 +242,27 @@ def parse_s3(blob: bytes, trailer_len: int) -> List[Frame]:
|
||||
state = IDLE
|
||||
i = trailer_end
|
||||
continue
|
||||
body.append(b)
|
||||
|
||||
else: # IN_FRAME_DLE
|
||||
if b == DLE:
|
||||
# DLE DLE → literal 0x10 in payload
|
||||
body.append(DLE)
|
||||
state = IN_FRAME
|
||||
i += 1
|
||||
continue
|
||||
if b == ETX:
|
||||
# DLE+ETX inside a frame = inner-frame terminator (A4/E5 sub-frames).
|
||||
# Treat as literal data, NOT the outer frame end.
|
||||
body.append(DLE)
|
||||
body.append(ETX)
|
||||
state = IN_FRAME
|
||||
i += 1
|
||||
continue
|
||||
# Unexpected DLE + byte → treat as literal data
|
||||
body.append(DLE)
|
||||
body.append(b)
|
||||
state = IN_FRAME
|
||||
i += 1
|
||||
continue
|
||||
|
||||
i += 1
|
||||
|
||||
@@ -298,10 +310,13 @@ def parse_bw(blob: bytes, trailer_len: int, validate_checksum: bool) -> List[Fra
|
||||
|
||||
if b == ETX:
|
||||
# Candidate end-of-frame.
|
||||
# Accept ETX if the next bytes look like a real next-frame start (ACK+STX),
|
||||
# or we're at EOF. This prevents chopping on in-payload 0x03.
|
||||
next_is_start = (i + 2 < n and blob[i + 1] == ACK and blob[i + 2] == STX)
|
||||
at_eof = (i == n - 1)
|
||||
# Skip any SESSION_RESET (41 03) sequences — sent before POLL to wake
|
||||
# monitoring units — to find the real next frame start (ACK+STX).
|
||||
j = i + 1
|
||||
while j + 1 < n and blob[j] == ACK and blob[j + 1] == ETX:
|
||||
j += 2
|
||||
next_is_start = (j + 1 < n and blob[j] == ACK and blob[j + 1] == STX)
|
||||
at_eof = (i == n - 1) or (j >= n)
|
||||
|
||||
if not (next_is_start or at_eof):
|
||||
# Not a real boundary -> payload byte
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
poc_set_project.py — POC test for set_project_info() against a live MiniMate Plus.
|
||||
|
||||
Usage:
|
||||
python poc_set_project.py [--host IP] [--port PORT]
|
||||
|
||||
Default target: BE11529 at 63.43.212.232:9034
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger("poc_set_project")
|
||||
|
||||
from minimateplus import MiniMateClient
|
||||
from minimateplus.transport import TcpTransport
|
||||
|
||||
|
||||
DEFAULT_HOST = "63.43.212.232"
|
||||
DEFAULT_PORT = 9034
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="POC: write project info to MiniMate Plus")
|
||||
ap.add_argument("--host", default=DEFAULT_HOST, help="Modem IP address")
|
||||
ap.add_argument("--port", type=int, default=DEFAULT_PORT, help="TCP port")
|
||||
ap.add_argument("--project", default="POC Write Test")
|
||||
ap.add_argument("--client-name", default="Terra-Mechanics Inc.")
|
||||
ap.add_argument("--operator", default="B. Harrison")
|
||||
ap.add_argument("--seis-loc", default="Lab Bench - POC")
|
||||
ap.add_argument("--notes", default="set_project_info POC 2026-04-07")
|
||||
args = ap.parse_args()
|
||||
|
||||
log.info("Connecting to %s:%d", args.host, args.port)
|
||||
transport = TcpTransport(args.host, port=args.port)
|
||||
|
||||
with MiniMateClient(transport=transport, timeout=60.0) as client:
|
||||
log.info("Performing POLL handshake + identity read …")
|
||||
info = client.connect()
|
||||
log.info("Connected: serial=%s firmware=%s", info.serial, info.firmware_version)
|
||||
|
||||
log.info("Calling set_project_info() …")
|
||||
client.set_project_info(
|
||||
project=args.project,
|
||||
client_name=args.client_name,
|
||||
operator=args.operator,
|
||||
seis_loc=args.seis_loc,
|
||||
notes=args.notes,
|
||||
)
|
||||
log.info("set_project_info() returned — write sequence complete")
|
||||
|
||||
log.info("Done. Reconnect Blastware to verify the fields were written.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
except Exception as exc:
|
||||
log.exception("Fatal: %s", exc)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,23 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "seismo-relay"
|
||||
version = "0.15.0"
|
||||
description = "Python client and REST server for MiniMate Plus seismographs"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.104",
|
||||
"uvicorn[standard]>=0.24",
|
||||
"pyserial>=3.5",
|
||||
"sqlalchemy>=2.0",
|
||||
"python-multipart>=0.0.7",
|
||||
"h5py>=3.10",
|
||||
"numpy>=1.24",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
# Auto-discovers minimateplus/, sfm/, bridges/ as packages
|
||||
where = ["."]
|
||||
include = ["minimateplus*", "sfm*", "bridges*"]
|
||||
@@ -0,0 +1,7 @@
|
||||
fastapi
|
||||
uvicorn
|
||||
sqlalchemy
|
||||
pyserial
|
||||
python-multipart
|
||||
h5py
|
||||
numpy
|
||||
@@ -0,0 +1,346 @@
|
||||
"""
|
||||
scripts/backfill_sidecars.py — generate .sfm.json sidecars AND .h5
|
||||
clean-waveform files for existing events already in the waveform store
|
||||
that predate those features.
|
||||
|
||||
Walks `<store_root>/<serial>/<filename>` and for each BW event file:
|
||||
|
||||
Sidecar (.sfm.json):
|
||||
- Skip when an existing sidecar's blastware.sha256 matches the
|
||||
current BW file's sha256.
|
||||
- Else regenerate: prefer .a5.pkl (full fidelity); fall back to
|
||||
parsing the BW binary directly (peaks computed from samples).
|
||||
|
||||
Clean waveform (.h5):
|
||||
- Skip when <filename>.h5 already exists (idempotent).
|
||||
- Else write from .a5.pkl (preferred) or BW binary parse (fallback).
|
||||
|
||||
Usage:
|
||||
python scripts/backfill_sidecars.py [--store-root PATH]
|
||||
[--db-path PATH]
|
||||
[--dry-run]
|
||||
[--skip-hdf5]
|
||||
[-v]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Allow running from the repo root without installation.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from minimateplus import event_file_io
|
||||
from sfm import event_hdf5
|
||||
from sfm.waveform_store import WaveformStore, _frame_to_dict, _dict_to_frame # noqa: F401
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
log = logging.getLogger("backfill_sidecars")
|
||||
|
||||
|
||||
def _looks_like_event_file(path: Path) -> bool:
|
||||
"""Same heuristic as the importer CLI."""
|
||||
if not path.is_file():
|
||||
return False
|
||||
if path.name.endswith((".a5.pkl", ".sfm.json")):
|
||||
return False
|
||||
ext = path.suffix.lstrip(".")
|
||||
if not (3 <= len(ext) <= 4):
|
||||
return False
|
||||
if not (ext[-1].upper() in {"W", "H"} or ext.endswith("0")):
|
||||
return False
|
||||
try:
|
||||
return path.stat().st_size >= 70
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument(
|
||||
"--db-path",
|
||||
default=str(Path(__file__).resolve().parent.parent / "bridges" / "captures" / "seismo_relay.db"),
|
||||
)
|
||||
p.add_argument("--store-root", default=None)
|
||||
p.add_argument("--dry-run", action="store_true")
|
||||
p.add_argument(
|
||||
"--skip-hdf5", action="store_true",
|
||||
help="Don't generate .h5 clean-waveform files (only sidecars).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--force", action="store_true",
|
||||
help=(
|
||||
"Regenerate sidecars + .h5 even when an existing sidecar's "
|
||||
"blastware.sha256 matches the current BW file. Use this after "
|
||||
"upgrading seismo-relay to pull in decoder bug fixes (e.g. the "
|
||||
"STRT-rectime byte-offset fix in v0.15.x)."
|
||||
),
|
||||
)
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
db_path = Path(args.db_path).expanduser().resolve()
|
||||
store_root = (
|
||||
Path(args.store_root).expanduser().resolve()
|
||||
if args.store_root else db_path.parent / "waveforms"
|
||||
)
|
||||
if not store_root.exists():
|
||||
print(f"error: store root does not exist: {store_root}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
store = WaveformStore(store_root)
|
||||
db = SeismoDb(db_path)
|
||||
|
||||
written = skipped = errors = 0
|
||||
for serial_dir in sorted(p for p in store_root.iterdir() if p.is_dir()):
|
||||
serial = serial_dir.name
|
||||
for path in sorted(serial_dir.iterdir()):
|
||||
if not _looks_like_event_file(path):
|
||||
continue
|
||||
sidecar_path = store.sidecar_path_for(serial, path.name)
|
||||
try:
|
||||
bw_sha = event_file_io.file_sha256(path)
|
||||
except Exception as exc:
|
||||
log.error("sha256 failed for %s: %s", path, exc)
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
# Skip when an up-to-date sidecar already exists.
|
||||
#
|
||||
# Two-part freshness check:
|
||||
# 1. blastware.sha256 must match the current BW file (proves
|
||||
# the sidecar describes THIS file).
|
||||
# 2. source.tool_version must be ≥ current TOOL_VERSION (proves
|
||||
# the sidecar was written by a build that includes any
|
||||
# decoder fixes shipped since).
|
||||
# Either part failing → regenerate. --force bypasses both.
|
||||
if sidecar_path.exists() and not args.force:
|
||||
try:
|
||||
existing = event_file_io.read_sidecar(sidecar_path)
|
||||
sha_ok = existing.get("blastware", {}).get("sha256") == bw_sha
|
||||
src_ver = existing.get("source", {}).get("tool_version", "")
|
||||
def _vt(s):
|
||||
try:
|
||||
return tuple(int(p) for p in str(s).split(".")[:3])
|
||||
except Exception:
|
||||
return (0, 0, 0)
|
||||
ver_ok = _vt(src_ver) >= _vt(event_file_io.TOOL_VERSION)
|
||||
if sha_ok and ver_ok:
|
||||
skipped += 1
|
||||
continue
|
||||
if sha_ok and not ver_ok:
|
||||
log.info(
|
||||
"regenerating %s (sidecar tool_version=%s < current %s)",
|
||||
sidecar_path.name, src_ver or "(none)",
|
||||
event_file_io.TOOL_VERSION,
|
||||
)
|
||||
except Exception:
|
||||
pass # fall through to rewrite
|
||||
|
||||
# Decide path: A5-based (high-fidelity) or BW-only.
|
||||
a5_path = serial_dir / f"{path.name}.a5.pkl"
|
||||
try:
|
||||
if a5_path.exists():
|
||||
frames = store.load_a5(serial, path.name)
|
||||
if not frames:
|
||||
raise RuntimeError("a5_pickle present but unreadable")
|
||||
# Build an Event by replaying the A5 decoders. Note:
|
||||
# the .a5.pkl alone CANNOT recover timestamp /
|
||||
# record_type / waveform_key / per-channel peaks —
|
||||
# those live in the 0C record, which isn't saved
|
||||
# separately. We seed those from the DB row + the
|
||||
# existing sidecar below so a re-backfill doesn't
|
||||
# nuke fields the original save populated.
|
||||
from minimateplus.client import (
|
||||
_decode_a5_metadata_into,
|
||||
_decode_a5_waveform,
|
||||
)
|
||||
from minimateplus.models import Event, PeakValues, ProjectInfo, Timestamp
|
||||
ev = Event(index=-1)
|
||||
_decode_a5_metadata_into(frames, ev)
|
||||
_decode_a5_waveform(frames, ev)
|
||||
source_kind = "sfm-live"
|
||||
a5_filename = a5_path.name
|
||||
else:
|
||||
ev = event_file_io.read_blastware_file(path)
|
||||
source_kind = "bw-import"
|
||||
a5_filename = None
|
||||
from minimateplus.models import Event, PeakValues, ProjectInfo, Timestamp
|
||||
|
||||
# ── Seed missing fields from the SeismoDb events row ──
|
||||
# The DB row was populated at original save time with peaks,
|
||||
# project info, timestamp, record_type, sample_rate, etc.
|
||||
# All of those survive intact in SQLite; pull them onto the
|
||||
# rebuilt Event so the regenerated sidecar matches what was
|
||||
# there before the backfill ran.
|
||||
db_row = None
|
||||
try:
|
||||
import sqlite3 as _sql
|
||||
with _sql.connect(str(db.db_path)) as _conn:
|
||||
_conn.row_factory = _sql.Row
|
||||
db_row = _conn.execute(
|
||||
"SELECT * FROM events "
|
||||
"WHERE serial=? AND blastware_filename=? "
|
||||
"LIMIT 1",
|
||||
(serial, path.name),
|
||||
).fetchone()
|
||||
except Exception as exc:
|
||||
log.debug("DB lookup failed for %s: %s", path.name, exc)
|
||||
|
||||
if db_row is not None:
|
||||
if ev.sample_rate is None and db_row["sample_rate"]:
|
||||
ev.sample_rate = int(db_row["sample_rate"])
|
||||
if not ev.record_type and db_row["record_type"]:
|
||||
ev.record_type = db_row["record_type"]
|
||||
if ev._waveform_key is None and db_row["waveform_key"]:
|
||||
try:
|
||||
ev._waveform_key = bytes.fromhex(db_row["waveform_key"])
|
||||
except Exception:
|
||||
pass
|
||||
# Timestamp from the ISO-8601 string in the DB row.
|
||||
if ev.timestamp is None and db_row["timestamp"]:
|
||||
try:
|
||||
import datetime as _dt
|
||||
_t = _dt.datetime.fromisoformat(db_row["timestamp"])
|
||||
ev.timestamp = Timestamp(
|
||||
raw=b"", flag=0x10,
|
||||
year=_t.year, unknown_byte=0,
|
||||
month=_t.month, day=_t.day,
|
||||
hour=_t.hour, minute=_t.minute, second=_t.second,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# Peaks from the DB row when the A5 decode didn't supply them.
|
||||
if ev.peak_values is None:
|
||||
ev.peak_values = PeakValues(
|
||||
tran=db_row["tran_ppv"],
|
||||
vert=db_row["vert_ppv"],
|
||||
long=db_row["long_ppv"],
|
||||
peak_vector_sum=db_row["peak_vector_sum"],
|
||||
micl=db_row["mic_ppv"],
|
||||
)
|
||||
# Project info from the DB row when the A5 metadata-page
|
||||
# decode didn't pick it up.
|
||||
if ev.project_info is None or all(
|
||||
v in (None, "")
|
||||
for v in (
|
||||
(ev.project_info.project if ev.project_info else None),
|
||||
(ev.project_info.client if ev.project_info else None),
|
||||
(ev.project_info.operator if ev.project_info else None),
|
||||
(ev.project_info.sensor_location if ev.project_info else None),
|
||||
)
|
||||
):
|
||||
ev.project_info = ProjectInfo(
|
||||
project=db_row["project"],
|
||||
client=db_row["client"],
|
||||
operator=db_row["operator"],
|
||||
sensor_location=db_row["sensor_location"],
|
||||
)
|
||||
|
||||
# Derive total_samples when we have both rectime + sample_rate.
|
||||
# The decoder's STRT-derived value can be a buffer offset
|
||||
# rather than a sample count — drop it in that case.
|
||||
if ev.sample_rate and ev.rectime_seconds:
|
||||
derived = int(round(ev.sample_rate * ev.rectime_seconds))
|
||||
if (ev.total_samples is None
|
||||
or ev.total_samples > derived * 2
|
||||
or ev.total_samples < derived // 4):
|
||||
ev.total_samples = derived
|
||||
|
||||
# Preserve user-edited review state + extensions from the
|
||||
# existing sidecar (false_trigger flag, notes, etc.) so a
|
||||
# backfill never wipes them out.
|
||||
preserved_review = None
|
||||
preserved_ext = None
|
||||
if sidecar_path.exists():
|
||||
try:
|
||||
_existing = event_file_io.read_sidecar(sidecar_path)
|
||||
preserved_review = _existing.get("review")
|
||||
preserved_ext = _existing.get("extensions")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sidecar = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial=serial,
|
||||
blastware_filename=path.name,
|
||||
blastware_filesize=path.stat().st_size,
|
||||
blastware_sha256=bw_sha,
|
||||
source_kind=source_kind,
|
||||
a5_pickle_filename=a5_filename,
|
||||
review=preserved_review,
|
||||
extensions=preserved_ext,
|
||||
)
|
||||
|
||||
# Also emit the .h5 clean-waveform file when missing OR when
|
||||
# --force was passed (so a re-backfill picks up decoder fixes).
|
||||
hdf5_path = store.hdf5_path_for(serial, path.name)
|
||||
hdf5_filename = hdf5_path.name if hdf5_path.exists() else None
|
||||
hdf5_action = "kept"
|
||||
need_h5 = not args.skip_hdf5 and (args.force or not hdf5_path.exists())
|
||||
if need_h5:
|
||||
if args.dry_run:
|
||||
hdf5_action = "would (re)write"
|
||||
else:
|
||||
try:
|
||||
event_hdf5.write_event_hdf5(
|
||||
hdf5_path, ev,
|
||||
serial=serial,
|
||||
geo_range="normal",
|
||||
source_kind=source_kind,
|
||||
)
|
||||
hdf5_filename = hdf5_path.name
|
||||
hdf5_action = "rewrote" if hdf5_path.exists() else "wrote"
|
||||
except Exception as exc:
|
||||
log.warning("HDF5 write failed for %s: %s", path.name, exc)
|
||||
hdf5_action = "FAILED"
|
||||
|
||||
if args.dry_run:
|
||||
print(f" [DRY ] would write {sidecar_path.name} "
|
||||
f"+ .h5 ({hdf5_action}) source={source_kind}")
|
||||
written += 1
|
||||
continue
|
||||
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
# Best-effort: keep the SQL row's sidecar_filename in sync
|
||||
# by upserting via insert_events (it dedups on serial+ts).
|
||||
try:
|
||||
db.insert_events(
|
||||
[ev], serial=serial,
|
||||
waveform_records=(
|
||||
{ev._waveform_key.hex(): {
|
||||
"filename": path.name,
|
||||
"filesize": path.stat().st_size,
|
||||
"a5_pickle_filename": a5_filename,
|
||||
"sidecar_filename": sidecar_path.name,
|
||||
}}
|
||||
if ev._waveform_key else None
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning("DB upsert failed for %s: %s", path.name, exc)
|
||||
|
||||
print(f" [OK ] {path.name} → {sidecar_path.name} "
|
||||
f"+ h5 ({hdf5_action}) source={source_kind}")
|
||||
written += 1
|
||||
|
||||
except Exception as exc:
|
||||
log.error("backfill failed for %s: %s", path, exc, exc_info=args.verbose)
|
||||
errors += 1
|
||||
|
||||
print(f"\nDone. written={written} skipped(uptodate)={skipped} errors={errors}")
|
||||
return 0 if errors == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+1323
-72
File diff suppressed because it is too large
Load Diff
+505
@@ -0,0 +1,505 @@
|
||||
"""
|
||||
sfm/cache.py — Persistent SQLite cache for SFM device data.
|
||||
|
||||
Caching strategy
|
||||
----------------
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
| Data | Mutability | Invalidation |
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
| Device info | Effectively immutable (firmware, | Manual clear / force |
|
||||
| (serial, model, | serial never change) | refresh query param |
|
||||
| compliance cfg) | | |
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
| Event headers | Append-only (new events added, | Fetch new ones when |
|
||||
| (peaks, ts, | old never modified) | device event count > |
|
||||
| project info) | | cached count |
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
| Full waveforms | Immutable once recorded | Never (permanent cache) |
|
||||
| (raw ADC samples)| | |
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
| Monitor status | Frequently changing | TTL = 30 seconds |
|
||||
| (battery, memory)| | |
|
||||
+------------------+----------------------------------+-------------------------+
|
||||
|
||||
Keys
|
||||
----
|
||||
All cached rows are keyed by (host, tcp_port) for TCP connections, or (port, baud)
|
||||
for serial connections. Within a device, events are keyed by index (0-based).
|
||||
|
||||
The device serial number is stored once we learn it, and used for display / debugging
|
||||
only — the network address is the primary routing key (same as how the rest of the SFM
|
||||
code operates).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import orm
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"sqlalchemy is required for the SFM cache.\n"
|
||||
"Install it with: pip install sqlalchemy"
|
||||
)
|
||||
|
||||
log = logging.getLogger("sfm.cache")
|
||||
|
||||
# ── Schema ────────────────────────────────────────────────────────────────────
|
||||
|
||||
Base = orm.declarative_base()
|
||||
|
||||
_MONITOR_STATUS_TTL = 30 # seconds
|
||||
|
||||
|
||||
class CachedDevice(Base):
|
||||
"""
|
||||
Device identity + compliance config, keyed by connection address.
|
||||
|
||||
Stores the full serialised JSON blob returned by /device/info so the
|
||||
endpoint can return it verbatim on a cache hit without re-connecting.
|
||||
"""
|
||||
__tablename__ = "cached_devices"
|
||||
|
||||
# Connection key — either TCP (host+port) or serial (port+baud)
|
||||
conn_key = sa.Column(sa.String, primary_key=True) # e.g. "tcp:1.2.3.4:12345"
|
||||
serial = sa.Column(sa.String, nullable=True) # e.g. "BE11529"
|
||||
info_json = sa.Column(sa.Text, nullable=False) # full /device/info response JSON
|
||||
updated_at = sa.Column(sa.Float, nullable=False) # Unix timestamp of last write
|
||||
|
||||
# When a config write happens we set this flag so the next /device/info call
|
||||
# fetches fresh data instead of serving stale compliance config.
|
||||
config_dirty = sa.Column(sa.Boolean, default=False, nullable=False)
|
||||
|
||||
|
||||
class CachedEvent(Base):
|
||||
"""
|
||||
Per-event header + peak values + project info, keyed by (conn_key, index).
|
||||
|
||||
Events are immutable once recorded on the device; once we have an event in
|
||||
the cache it never needs to be re-downloaded unless explicitly requested.
|
||||
|
||||
The two extra columns `waveform_key` and `event_timestamp` are an
|
||||
integrity stamp: when set_event() / set_waveform() are called with a
|
||||
different (waveform_key, event_timestamp) for the same (conn_key, index),
|
||||
we know the device was erased and re-recorded — the cached row no longer
|
||||
refers to the same physical event and the entire device's cache is
|
||||
flushed before the new entry is written. This catches the post-erase
|
||||
key-reuse bug where the device's first new event (key 01110000) collides
|
||||
with the first event we previously downloaded.
|
||||
"""
|
||||
__tablename__ = "cached_events"
|
||||
|
||||
conn_key = sa.Column(sa.String, primary_key=True)
|
||||
index = sa.Column(sa.Integer, primary_key=True)
|
||||
event_json = sa.Column(sa.Text, nullable=False) # serialised Event dict
|
||||
cached_at = sa.Column(sa.Float, nullable=False) # Unix timestamp
|
||||
waveform_key = sa.Column(sa.String, nullable=True) # 8-hex device key
|
||||
event_timestamp = sa.Column(sa.String, nullable=True) # ISO-8601 from 0C
|
||||
|
||||
|
||||
class CachedWaveform(Base):
|
||||
"""
|
||||
Full raw ADC waveform for a single event (SUB 5A full download).
|
||||
|
||||
These are large (up to several MB) and expensive to fetch over cellular.
|
||||
Once downloaded they are immutable and cached permanently — but the
|
||||
cache row is invalidated when the device is erased and a new event lands
|
||||
at the same index (see CachedEvent docstring).
|
||||
"""
|
||||
__tablename__ = "cached_waveforms"
|
||||
|
||||
conn_key = sa.Column(sa.String, primary_key=True)
|
||||
index = sa.Column(sa.Integer, primary_key=True)
|
||||
waveform_json = sa.Column(sa.Text, nullable=False) # full /device/event/{idx}/waveform response JSON
|
||||
cached_at = sa.Column(sa.Float, nullable=False)
|
||||
waveform_key = sa.Column(sa.String, nullable=True) # 8-hex device key
|
||||
event_timestamp = sa.Column(sa.String, nullable=True) # ISO-8601 from 0C
|
||||
|
||||
|
||||
class CachedMonitorStatus(Base):
|
||||
"""
|
||||
Monitor status (battery, memory, is_monitoring) with a short TTL.
|
||||
|
||||
These change frequently during field operations so we keep them only for
|
||||
MONITOR_STATUS_TTL seconds before re-fetching from the device.
|
||||
"""
|
||||
__tablename__ = "cached_monitor_status"
|
||||
|
||||
conn_key = sa.Column(sa.String, primary_key=True)
|
||||
status_json = sa.Column(sa.Text, nullable=False)
|
||||
cached_at = sa.Column(sa.Float, nullable=False)
|
||||
|
||||
|
||||
# ── Cache store ───────────────────────────────────────────────────────────────
|
||||
|
||||
class SFMCache:
|
||||
"""
|
||||
SQLite-backed cache for SFM device data.
|
||||
|
||||
Usage
|
||||
-----
|
||||
cache = SFMCache() # stores in sfm/data/sfm_cache.db by default
|
||||
cache = SFMCache(":memory:") # in-memory (tests / ephemeral mode)
|
||||
|
||||
All public methods accept a *conn_key* string — use make_conn_key() to
|
||||
build a consistent key from the transport parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | Path | None = None) -> None:
|
||||
in_memory = (db_path == ":memory:")
|
||||
if db_path is None:
|
||||
# Default: alongside this file in sfm/data/
|
||||
db_path = Path(__file__).parent / "data" / "sfm_cache.db"
|
||||
if not in_memory:
|
||||
db_path = Path(db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
url = "sqlite:///:memory:" if in_memory else f"sqlite:///{db_path}"
|
||||
engine = sa.create_engine(url, connect_args={"check_same_thread": False})
|
||||
Base.metadata.create_all(engine)
|
||||
self._Session = orm.sessionmaker(bind=engine)
|
||||
# In-place schema migration: add the (waveform_key, event_timestamp)
|
||||
# integrity-stamp columns to legacy cache DBs that predate the
|
||||
# post-erase eviction logic. ALTER TABLE ADD COLUMN is idempotent
|
||||
# via the column-presence check below.
|
||||
with engine.begin() as conn:
|
||||
for table in ("cached_events", "cached_waveforms"):
|
||||
cols = {
|
||||
r[1]
|
||||
for r in conn.exec_driver_sql(f"PRAGMA table_info({table})").fetchall()
|
||||
}
|
||||
for new_col, ddl in (
|
||||
("waveform_key", "TEXT"),
|
||||
("event_timestamp", "TEXT"),
|
||||
):
|
||||
if new_col not in cols:
|
||||
log.info("cache schema: %s ADD COLUMN %s %s", table, new_col, ddl)
|
||||
conn.exec_driver_sql(f"ALTER TABLE {table} ADD COLUMN {new_col} {ddl}")
|
||||
log.info("SFM cache opened: %s", db_path)
|
||||
|
||||
# ── Connection key ────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def make_conn_key(
|
||||
host: Optional[str],
|
||||
tcp_port: int,
|
||||
port: Optional[str],
|
||||
baud: int,
|
||||
) -> str:
|
||||
"""Return a stable string key for this transport configuration."""
|
||||
if host:
|
||||
return f"tcp:{host}:{tcp_port}"
|
||||
return f"serial:{port}:{baud}"
|
||||
|
||||
# ── Device info ───────────────────────────────────────────────────────────
|
||||
|
||||
def get_device_info(self, conn_key: str) -> Optional[dict]:
|
||||
"""
|
||||
Return cached device info dict, or None if not cached / config_dirty.
|
||||
"""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedDevice, conn_key)
|
||||
if row is None or row.config_dirty:
|
||||
return None
|
||||
return json.loads(row.info_json)
|
||||
|
||||
def set_device_info(self, conn_key: str, info: dict) -> None:
|
||||
"""Store device info and clear any dirty flag."""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedDevice, conn_key)
|
||||
serial = info.get("serial")
|
||||
if row is None:
|
||||
row = CachedDevice(
|
||||
conn_key=conn_key,
|
||||
serial=serial,
|
||||
info_json=json.dumps(info),
|
||||
updated_at=time.time(),
|
||||
config_dirty=False,
|
||||
)
|
||||
s.add(row)
|
||||
else:
|
||||
row.serial = serial
|
||||
row.info_json = json.dumps(info)
|
||||
row.updated_at = time.time()
|
||||
row.config_dirty = False
|
||||
s.commit()
|
||||
log.debug("cached device info for %s (serial=%s)", conn_key, serial)
|
||||
|
||||
def mark_config_dirty(self, conn_key: str) -> None:
|
||||
"""
|
||||
Called after a successful POST /device/config write.
|
||||
|
||||
Forces the next /device/info call to re-read compliance config from the
|
||||
device instead of serving the now-stale cached version.
|
||||
"""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedDevice, conn_key)
|
||||
if row:
|
||||
row.config_dirty = True
|
||||
s.commit()
|
||||
log.debug("marked config dirty for %s", conn_key)
|
||||
|
||||
# ── Events ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_cached_event_count(self, conn_key: str) -> int:
|
||||
"""Return the number of events we have cached for this device."""
|
||||
with self._Session() as s:
|
||||
return s.query(CachedEvent).filter_by(conn_key=conn_key).count()
|
||||
|
||||
def get_all_events(self, conn_key: str) -> Optional[list[dict]]:
|
||||
"""
|
||||
Return all cached events as a list of dicts, sorted by index.
|
||||
Returns None if nothing is cached yet.
|
||||
"""
|
||||
with self._Session() as s:
|
||||
rows = (
|
||||
s.query(CachedEvent)
|
||||
.filter_by(conn_key=conn_key)
|
||||
.order_by(CachedEvent.index)
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
return None
|
||||
return [json.loads(r.event_json) for r in rows]
|
||||
|
||||
def get_event(self, conn_key: str, index: int) -> Optional[dict]:
|
||||
"""Return a single cached event by index, or None if not cached."""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedEvent, (conn_key, index))
|
||||
return json.loads(row.event_json) if row else None
|
||||
|
||||
@staticmethod
|
||||
def _event_signature(ev: dict) -> tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Extract the (waveform_key_hex, timestamp_iso) integrity stamp from
|
||||
a serialised event dict. Either field may be None if the source
|
||||
Event was missing it; the comparison logic in set_events/set_waveform
|
||||
treats "both sides have a value AND they differ" as the only
|
||||
eviction trigger, so partial data never spuriously flushes cache.
|
||||
"""
|
||||
key = ev.get("waveform_key") or ev.get("_waveform_key")
|
||||
if isinstance(key, (bytes, bytearray)):
|
||||
key = bytes(key).hex()
|
||||
ts = ev.get("timestamp")
|
||||
if isinstance(ts, dict):
|
||||
# _serialise_timestamp returns a dict like {"iso": "...", ...}
|
||||
ts = ts.get("iso") or ts.get("string") or None
|
||||
return (key if isinstance(key, str) else None,
|
||||
ts if isinstance(ts, str) else None)
|
||||
|
||||
def _maybe_flush_on_mismatch(
|
||||
self,
|
||||
s,
|
||||
conn_key: str,
|
||||
index: int,
|
||||
new_key: Optional[str],
|
||||
new_ts: Optional[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Check whether the cached entry at (conn_key, index) has a different
|
||||
(waveform_key, timestamp) than the incoming one. If so, treat it as
|
||||
a post-erase key-reuse signal and flush ALL cached events/waveforms
|
||||
for this device, then return True.
|
||||
Returns False when no flush was needed.
|
||||
"""
|
||||
if not new_key and not new_ts:
|
||||
return False # nothing to compare against
|
||||
existing = s.get(CachedEvent, (conn_key, index))
|
||||
if existing is None:
|
||||
existing = s.get(CachedWaveform, (conn_key, index))
|
||||
if existing is None:
|
||||
return False
|
||||
old_key = existing.waveform_key
|
||||
old_ts = existing.event_timestamp
|
||||
# Only flush when both sides have populated values and they differ.
|
||||
differs = (
|
||||
(new_key and old_key and new_key != old_key)
|
||||
or (new_ts and old_ts and new_ts != old_ts)
|
||||
)
|
||||
if not differs:
|
||||
return False
|
||||
log.warning(
|
||||
"cache: device %s — index %d (key=%s, ts=%s) replaces (key=%s, ts=%s); "
|
||||
"flushing all cached events/waveforms for this device "
|
||||
"(post-erase key reuse detected)",
|
||||
conn_key, index, new_key, new_ts, old_key, old_ts,
|
||||
)
|
||||
s.query(CachedEvent).filter_by(conn_key=conn_key).delete()
|
||||
s.query(CachedWaveform).filter_by(conn_key=conn_key).delete()
|
||||
return True
|
||||
|
||||
def set_events(self, conn_key: str, events: list[dict]) -> None:
|
||||
"""
|
||||
Upsert a list of event dicts. Existing rows are updated; new rows are
|
||||
inserted. This is used to add newly-discovered events to the cache.
|
||||
|
||||
Eviction: if any incoming event has a different (waveform_key,
|
||||
timestamp) than the row currently cached at the same index, we flush
|
||||
the entire device's cache before inserting the new entries. Catches
|
||||
post-erase key reuse where index 0 silently switches identity.
|
||||
"""
|
||||
now = time.time()
|
||||
with self._Session() as s:
|
||||
# Eviction check: scan incoming events for any (index, key, ts)
|
||||
# that conflicts with a cached row. A single conflict triggers
|
||||
# a full device-wide flush so we don't end up with a mixed-era
|
||||
# cache.
|
||||
for ev in events:
|
||||
key, ts = self._event_signature(ev)
|
||||
if self._maybe_flush_on_mismatch(s, conn_key, ev["index"], key, ts):
|
||||
s.commit()
|
||||
break # cache is now empty for this device; carry on
|
||||
|
||||
for ev in events:
|
||||
idx = ev["index"]
|
||||
key, ts = self._event_signature(ev)
|
||||
row = s.get(CachedEvent, (conn_key, idx))
|
||||
if row is None:
|
||||
row = CachedEvent(
|
||||
conn_key=conn_key,
|
||||
index=idx,
|
||||
event_json=json.dumps(ev),
|
||||
cached_at=now,
|
||||
waveform_key=key,
|
||||
event_timestamp=ts,
|
||||
)
|
||||
s.add(row)
|
||||
log.debug("cached new event %d for %s", idx, conn_key)
|
||||
else:
|
||||
# Refresh in case project_info was backfilled after initial store
|
||||
row.event_json = json.dumps(ev)
|
||||
if key:
|
||||
row.waveform_key = key
|
||||
if ts:
|
||||
row.event_timestamp = ts
|
||||
s.commit()
|
||||
|
||||
# ── Waveforms ─────────────────────────────────────────────────────────────
|
||||
|
||||
def get_waveform(self, conn_key: str, index: int) -> Optional[dict]:
|
||||
"""Return a cached full waveform response dict, or None if not cached."""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedWaveform, (conn_key, index))
|
||||
if row is None:
|
||||
return None
|
||||
log.debug("waveform cache hit: %s event %d", conn_key, index)
|
||||
return json.loads(row.waveform_json)
|
||||
|
||||
def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None:
|
||||
"""
|
||||
Store a full waveform response dict permanently.
|
||||
|
||||
Like set_events, this checks the (waveform_key, timestamp) signature
|
||||
of the incoming entry against what's currently cached at the same
|
||||
index. A mismatch flushes the entire device's cache before insert.
|
||||
"""
|
||||
key, ts = self._event_signature(waveform)
|
||||
with self._Session() as s:
|
||||
self._maybe_flush_on_mismatch(s, conn_key, index, key, ts)
|
||||
row = s.get(CachedWaveform, (conn_key, index))
|
||||
if row is None:
|
||||
row = CachedWaveform(
|
||||
conn_key=conn_key,
|
||||
index=index,
|
||||
waveform_json=json.dumps(waveform),
|
||||
cached_at=time.time(),
|
||||
waveform_key=key,
|
||||
event_timestamp=ts,
|
||||
)
|
||||
s.add(row)
|
||||
else:
|
||||
row.waveform_json = json.dumps(waveform)
|
||||
row.cached_at = time.time()
|
||||
if key:
|
||||
row.waveform_key = key
|
||||
if ts:
|
||||
row.event_timestamp = ts
|
||||
s.commit()
|
||||
log.debug("cached waveform for %s event %d (key=%s, ts=%s)",
|
||||
conn_key, index, key, ts)
|
||||
|
||||
# ── Monitor status ────────────────────────────────────────────────────────
|
||||
|
||||
def get_monitor_status(self, conn_key: str) -> Optional[dict]:
|
||||
"""Return cached monitor status if it's within TTL, else None."""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedMonitorStatus, conn_key)
|
||||
if row is None:
|
||||
return None
|
||||
age = time.time() - row.cached_at
|
||||
if age > _MONITOR_STATUS_TTL:
|
||||
log.debug("monitor status expired (age=%.1fs) for %s", age, conn_key)
|
||||
return None
|
||||
return json.loads(row.status_json)
|
||||
|
||||
def set_monitor_status(self, conn_key: str, status: dict) -> None:
|
||||
"""Store monitor status."""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedMonitorStatus, conn_key)
|
||||
if row is None:
|
||||
row = CachedMonitorStatus(
|
||||
conn_key=conn_key,
|
||||
status_json=json.dumps(status),
|
||||
cached_at=time.time(),
|
||||
)
|
||||
s.add(row)
|
||||
else:
|
||||
row.status_json = json.dumps(status)
|
||||
row.cached_at = time.time()
|
||||
s.commit()
|
||||
|
||||
def invalidate_monitor_status(self, conn_key: str) -> None:
|
||||
"""
|
||||
Called after start/stop monitoring so the next status poll re-reads from device.
|
||||
"""
|
||||
with self._Session() as s:
|
||||
row = s.get(CachedMonitorStatus, conn_key)
|
||||
if row:
|
||||
s.delete(row)
|
||||
s.commit()
|
||||
|
||||
# ── Cache management ──────────────────────────────────────────────────────
|
||||
|
||||
def clear_device(self, conn_key: str) -> dict:
|
||||
"""
|
||||
Remove all cached data for a device. Returns counts of deleted rows.
|
||||
"""
|
||||
counts = {}
|
||||
with self._Session() as s:
|
||||
counts["device_info"] = s.query(CachedDevice).filter_by(conn_key=conn_key).delete()
|
||||
counts["events"] = s.query(CachedEvent).filter_by(conn_key=conn_key).delete()
|
||||
counts["waveforms"] = s.query(CachedWaveform).filter_by(conn_key=conn_key).delete()
|
||||
counts["monitor_status"] = s.query(CachedMonitorStatus).filter_by(conn_key=conn_key).delete()
|
||||
s.commit()
|
||||
log.info("cleared cache for %s: %s", conn_key, counts)
|
||||
return counts
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""Return row counts for all cache tables (for /cache/stats endpoint)."""
|
||||
with self._Session() as s:
|
||||
return {
|
||||
"devices": s.query(CachedDevice).count(),
|
||||
"events": s.query(CachedEvent).count(),
|
||||
"waveforms": s.query(CachedWaveform).count(),
|
||||
"monitor_status": s.query(CachedMonitorStatus).count(),
|
||||
}
|
||||
|
||||
|
||||
# ── Module-level singleton ────────────────────────────────────────────────────
|
||||
# Instantiated once when the module is imported; shared across all requests.
|
||||
|
||||
_cache: Optional[SFMCache] = None
|
||||
|
||||
|
||||
def get_cache() -> SFMCache:
|
||||
"""Return the module-level cache singleton, initialising it on first call."""
|
||||
global _cache
|
||||
if _cache is None:
|
||||
_cache = SFMCache()
|
||||
return _cache
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
"""
|
||||
sfm/database.py — SQLite persistence layer for seismo-relay.
|
||||
|
||||
Three tables, all keyed by unit serial number:
|
||||
|
||||
ach_sessions — one row per inbound ACH call-home
|
||||
events — one row per triggered waveform event (deduped by serial+timestamp)
|
||||
monitor_log — one row per monitoring interval (deduped by serial+start_time)
|
||||
|
||||
The DB file lives at:
|
||||
<output_dir>/seismo_relay.db (default: bridges/captures/seismo_relay.db)
|
||||
|
||||
Usage
|
||||
-----
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
db = SeismoDb("bridges/captures/seismo_relay.db")
|
||||
|
||||
# Write a call-home session
|
||||
session_id = db.insert_ach_session(serial="BE11529", peer="1.2.3.4:51920",
|
||||
events_downloaded=3, monitor_entries=2,
|
||||
duration_seconds=47.3)
|
||||
|
||||
# Write events (silently skips duplicates)
|
||||
db.insert_events(events, serial="BE11529", session_id=session_id)
|
||||
|
||||
# Write monitor log entries
|
||||
db.insert_monitor_log(entries, session_id=session_id)
|
||||
|
||||
# Query
|
||||
rows = db.query_events(serial="BE11529", from_dt=datetime(...), to_dt=datetime(...))
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import sqlite3
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from minimateplus.models import Event, MonitorLogEntry
|
||||
|
||||
log = logging.getLogger("sfm.database")
|
||||
|
||||
# ── Schema ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
_SCHEMA = """
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ach_sessions (
|
||||
id TEXT PRIMARY KEY, -- UUID
|
||||
serial TEXT NOT NULL,
|
||||
session_time TEXT NOT NULL, -- ISO-8601 UTC
|
||||
peer TEXT, -- "ip:port"
|
||||
events_downloaded INTEGER NOT NULL DEFAULT 0,
|
||||
monitor_entries INTEGER NOT NULL DEFAULT 0,
|
||||
duration_seconds REAL,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ach_sessions_serial ON ach_sessions(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_ach_sessions_time ON ach_sessions(session_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id TEXT PRIMARY KEY, -- UUID
|
||||
serial TEXT NOT NULL,
|
||||
waveform_key TEXT NOT NULL, -- 8-hex device key (dedup field)
|
||||
session_id TEXT, -- FK → ach_sessions.id
|
||||
timestamp TEXT, -- ISO-8601 local time from device
|
||||
tran_ppv REAL, -- in/s
|
||||
vert_ppv REAL, -- in/s
|
||||
long_ppv REAL, -- in/s
|
||||
peak_vector_sum REAL, -- in/s
|
||||
mic_ppv REAL, -- psi or dB depending on setup
|
||||
project TEXT,
|
||||
client TEXT,
|
||||
operator TEXT,
|
||||
sensor_location TEXT,
|
||||
sample_rate INTEGER,
|
||||
record_type TEXT, -- "single_shot" | "continuous"
|
||||
false_trigger INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=yes (manual flag)
|
||||
blastware_filename TEXT, -- event file within waveform store; extension is per-event (AB0T encodes timestamp)
|
||||
blastware_filesize INTEGER, -- bytes; NULL if no event file saved
|
||||
a5_pickle_filename TEXT, -- "<filename>.a5.pkl" sidecar
|
||||
sidecar_filename TEXT, -- "<filename>.sfm.json" review/metadata sidecar
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, timestamp)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_events_serial ON events(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS monitor_log (
|
||||
id TEXT PRIMARY KEY, -- UUID
|
||||
serial TEXT NOT NULL,
|
||||
waveform_key TEXT NOT NULL, -- 8-hex device key (dedup field)
|
||||
session_id TEXT, -- FK → ach_sessions.id
|
||||
start_time TEXT, -- ISO-8601
|
||||
stop_time TEXT, -- ISO-8601
|
||||
duration_seconds REAL,
|
||||
geo_threshold_ips REAL, -- in/s
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, start_time)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_serial ON monitor_log(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_start ON monitor_log(start_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_session ON monitor_log(session_id);
|
||||
"""
|
||||
|
||||
|
||||
# ── SeismoDb class ─────────────────────────────────────────────────────────────
|
||||
|
||||
class SeismoDb:
|
||||
"""
|
||||
Thin SQLite wrapper for seismo-relay persistence.
|
||||
|
||||
Thread-safe: each call opens, uses, and closes a connection with
|
||||
check_same_thread=False and WAL mode enabled. For the ACH server's
|
||||
single-writer / occasional-reader pattern this is more than sufficient.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | Path) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_schema()
|
||||
log.info("SeismoDb initialised at %s", self.db_path)
|
||||
|
||||
# ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
return conn
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(_SCHEMA)
|
||||
self._migrate(conn)
|
||||
|
||||
def _migrate(self, conn: sqlite3.Connection) -> None:
|
||||
"""Apply in-place schema migrations for existing databases."""
|
||||
|
||||
# Migration 1: change events UNIQUE from (serial, waveform_key) [or any
|
||||
# waveform_key-based variant] to (serial, timestamp).
|
||||
# Rationale: device key counter resets to 01110000 after every erase, so
|
||||
# waveform_key is not a stable dedup field across erase cycles. The event
|
||||
# timestamp (from the device clock) is the correct natural key.
|
||||
row = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='events'"
|
||||
).fetchone()
|
||||
if row and "UNIQUE(serial, timestamp)" not in row[0]:
|
||||
log.info("_migrate: rebuilding events table — UNIQUE(serial, timestamp)")
|
||||
conn.executescript("""
|
||||
ALTER TABLE events RENAME TO events_old;
|
||||
|
||||
CREATE TABLE events (
|
||||
id TEXT PRIMARY KEY,
|
||||
serial TEXT NOT NULL,
|
||||
waveform_key TEXT NOT NULL,
|
||||
session_id TEXT,
|
||||
timestamp TEXT,
|
||||
tran_ppv REAL,
|
||||
vert_ppv REAL,
|
||||
long_ppv REAL,
|
||||
peak_vector_sum REAL,
|
||||
mic_ppv REAL,
|
||||
project TEXT,
|
||||
client TEXT,
|
||||
operator TEXT,
|
||||
sensor_location TEXT,
|
||||
sample_rate INTEGER,
|
||||
record_type TEXT,
|
||||
false_trigger INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, timestamp)
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO events SELECT * FROM events_old;
|
||||
DROP TABLE events_old;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_events_serial ON events(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
|
||||
""")
|
||||
log.info("_migrate: events table rebuilt OK")
|
||||
|
||||
# Migration 1b: add Blastware-file columns to existing events tables.
|
||||
# New columns are NULLable so old rows just read NULL.
|
||||
existing_cols = {
|
||||
r[1] for r in conn.execute("PRAGMA table_info(events)").fetchall()
|
||||
}
|
||||
for col, ddl in (
|
||||
("blastware_filename", "TEXT"),
|
||||
("blastware_filesize", "INTEGER"),
|
||||
("a5_pickle_filename", "TEXT"),
|
||||
("sidecar_filename", "TEXT"),
|
||||
):
|
||||
if col not in existing_cols:
|
||||
log.info("_migrate: events ADD COLUMN %s %s", col, ddl)
|
||||
conn.execute(f"ALTER TABLE events ADD COLUMN {col} {ddl}")
|
||||
|
||||
# Migration 2: change monitor_log UNIQUE from (serial, waveform_key) to
|
||||
# (serial, start_time) — same reasoning as events.
|
||||
row = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='monitor_log'"
|
||||
).fetchone()
|
||||
if row and "UNIQUE(serial, start_time)" not in row[0]:
|
||||
log.info("_migrate: rebuilding monitor_log table — UNIQUE(serial, start_time)")
|
||||
conn.executescript("""
|
||||
ALTER TABLE monitor_log RENAME TO monitor_log_old;
|
||||
|
||||
CREATE TABLE monitor_log (
|
||||
id TEXT PRIMARY KEY,
|
||||
serial TEXT NOT NULL,
|
||||
waveform_key TEXT NOT NULL,
|
||||
session_id TEXT,
|
||||
start_time TEXT,
|
||||
stop_time TEXT,
|
||||
duration_seconds REAL,
|
||||
geo_threshold_ips REAL,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, start_time)
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO monitor_log SELECT * FROM monitor_log_old;
|
||||
DROP TABLE monitor_log_old;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_serial ON monitor_log(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_start ON monitor_log(start_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_monitor_log_session ON monitor_log(session_id);
|
||||
""")
|
||||
log.info("_migrate: monitor_log table rebuilt OK")
|
||||
|
||||
@staticmethod
|
||||
def _iso(dt: Optional[datetime.datetime]) -> Optional[str]:
|
||||
return dt.isoformat() if dt is not None else None
|
||||
|
||||
@staticmethod
|
||||
def _new_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
# ── ACH sessions ──────────────────────────────────────────────────────────
|
||||
|
||||
def insert_ach_session(
|
||||
self,
|
||||
*,
|
||||
serial: str,
|
||||
peer: Optional[str] = None,
|
||||
events_downloaded: int = 0,
|
||||
monitor_entries: int = 0,
|
||||
duration_seconds: Optional[float] = None,
|
||||
session_time: Optional[datetime.datetime] = None,
|
||||
) -> str:
|
||||
"""Insert a new ACH session row. Returns the new session UUID."""
|
||||
sid = self._new_id()
|
||||
ts = self._iso(session_time or datetime.datetime.utcnow())
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO ach_sessions
|
||||
(id, serial, session_time, peer,
|
||||
events_downloaded, monitor_entries, duration_seconds)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(sid, serial, ts, peer,
|
||||
events_downloaded, monitor_entries, duration_seconds),
|
||||
)
|
||||
log.debug("ach_session inserted: %s serial=%s events=%d monitor=%d",
|
||||
sid, serial, events_downloaded, monitor_entries)
|
||||
return sid
|
||||
|
||||
def get_sessions(
|
||||
self,
|
||||
serial: Optional[str] = None,
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
"""Return recent ACH sessions, newest first."""
|
||||
with self._connect() as conn:
|
||||
if serial:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM ach_sessions WHERE serial=? "
|
||||
"ORDER BY session_time DESC LIMIT ?",
|
||||
(serial, limit),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM ach_sessions ORDER BY session_time DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
# ── Events ────────────────────────────────────────────────────────────────
|
||||
|
||||
def insert_events(
|
||||
self,
|
||||
events: list[Event],
|
||||
*,
|
||||
serial: str,
|
||||
session_id: Optional[str] = None,
|
||||
waveform_records: Optional[dict[str, dict]] = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Insert triggered events. Silently skips duplicates (serial+timestamp).
|
||||
Returns (inserted, skipped).
|
||||
|
||||
``waveform_records`` (optional): dict keyed by event waveform_key (hex)
|
||||
whose value is a record from ``WaveformStore.save()``:
|
||||
{"filename": str, "filesize": int, "a5_pickle_filename": str}
|
||||
|
||||
For events whose key is in this dict, the matching columns are
|
||||
populated. If a row with the same (serial, timestamp) already exists
|
||||
(dedup hit), the matching waveform record is upserted onto the
|
||||
existing row so a re-download via the live endpoint refreshes the
|
||||
file metadata.
|
||||
"""
|
||||
inserted = skipped = 0
|
||||
wave_recs = waveform_records or {}
|
||||
with self._connect() as conn:
|
||||
for ev in events:
|
||||
key = ev._waveform_key.hex() if ev._waveform_key else None
|
||||
if key is None:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
ts = None
|
||||
if ev.timestamp:
|
||||
try:
|
||||
ts = datetime.datetime(
|
||||
ev.timestamp.year, ev.timestamp.month, ev.timestamp.day,
|
||||
ev.timestamp.hour, ev.timestamp.minute, ev.timestamp.second,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
ts = str(ev.timestamp)
|
||||
|
||||
pv = ev.peak_values
|
||||
pi = ev.project_info
|
||||
rec = wave_recs.get(key) or {}
|
||||
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO events
|
||||
(id, serial, waveform_key, session_id, timestamp,
|
||||
tran_ppv, vert_ppv, long_ppv, peak_vector_sum, mic_ppv,
|
||||
project, client, operator, sensor_location,
|
||||
sample_rate, record_type,
|
||||
blastware_filename, blastware_filesize,
|
||||
a5_pickle_filename, sidecar_filename)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
self._new_id(), serial, key, session_id, ts,
|
||||
pv.tran if pv else None,
|
||||
pv.vert if pv else None,
|
||||
pv.long if pv else None,
|
||||
pv.peak_vector_sum if pv else None,
|
||||
pv.micl if pv else None,
|
||||
pi.project if pi else None,
|
||||
pi.client if pi else None,
|
||||
pi.operator if pi else None,
|
||||
pi.sensor_location if pi else None,
|
||||
ev.sample_rate,
|
||||
ev.record_type,
|
||||
rec.get("filename"),
|
||||
rec.get("filesize"),
|
||||
rec.get("a5_pickle_filename"),
|
||||
rec.get("sidecar_filename"),
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
except sqlite3.IntegrityError:
|
||||
skipped += 1
|
||||
# Upsert waveform fields onto the existing dedup row so a
|
||||
# re-download via the live endpoint refreshes filename /
|
||||
# size / sidecar without churning the rest of the row.
|
||||
if rec and ts:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE events
|
||||
SET blastware_filename = ?,
|
||||
blastware_filesize = ?,
|
||||
a5_pickle_filename = ?,
|
||||
sidecar_filename = ?
|
||||
WHERE serial = ? AND timestamp = ?
|
||||
""",
|
||||
(
|
||||
rec.get("filename"),
|
||||
rec.get("filesize"),
|
||||
rec.get("a5_pickle_filename"),
|
||||
rec.get("sidecar_filename"),
|
||||
serial,
|
||||
ts,
|
||||
),
|
||||
)
|
||||
|
||||
log.debug("insert_events serial=%s inserted=%d skipped=%d",
|
||||
serial, inserted, skipped)
|
||||
return inserted, skipped
|
||||
|
||||
def get_event(self, event_id: str) -> Optional[dict]:
|
||||
"""Return one event row by id, or None."""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM events WHERE id = ?", (event_id,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def query_events(
|
||||
self,
|
||||
serial: Optional[str] = None,
|
||||
from_dt: Optional[datetime.datetime] = None,
|
||||
to_dt: Optional[datetime.datetime] = None,
|
||||
false_trigger: Optional[bool] = None,
|
||||
limit: int = 500,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Query events with optional filters. Returns newest first."""
|
||||
clauses: list[str] = []
|
||||
params: list = []
|
||||
|
||||
if serial:
|
||||
clauses.append("serial = ?")
|
||||
params.append(serial)
|
||||
if from_dt:
|
||||
clauses.append("timestamp >= ?")
|
||||
params.append(from_dt.isoformat())
|
||||
if to_dt:
|
||||
clauses.append("timestamp <= ?")
|
||||
params.append(to_dt.isoformat())
|
||||
if false_trigger is not None:
|
||||
clauses.append("false_trigger = ?")
|
||||
params.append(1 if false_trigger else 0)
|
||||
|
||||
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
params += [limit, offset]
|
||||
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM events {where} "
|
||||
f"ORDER BY timestamp DESC LIMIT ? OFFSET ?",
|
||||
params,
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def set_false_trigger(self, event_id: str, value: bool) -> bool:
|
||||
"""Set or clear the false_trigger flag on an event. Returns True if found."""
|
||||
with self._connect() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE events SET false_trigger=? WHERE id=?",
|
||||
(1 if value else 0, event_id),
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
def update_event_review(self, event_id: str, review: dict) -> bool:
|
||||
"""
|
||||
Sync derived index columns from a sidecar's `review` block.
|
||||
|
||||
Currently the only derived index is `events.false_trigger` — kept
|
||||
in sync so `/db/events?false_trigger=true` queries don't have to
|
||||
scan every sidecar JSON on disk. The sidecar JSON itself remains
|
||||
the source of truth for the full review state.
|
||||
|
||||
Returns True when the row exists, False otherwise. No-op fields
|
||||
(review without `false_trigger`) leave the column untouched.
|
||||
"""
|
||||
if not isinstance(review, dict):
|
||||
return False
|
||||
if "false_trigger" not in review:
|
||||
# Nothing derived to update; just confirm the row exists.
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM events WHERE id=?", (event_id,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
flag = 1 if review.get("false_trigger") else 0
|
||||
with self._connect() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE events SET false_trigger=? WHERE id=?",
|
||||
(flag, event_id),
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
# ── Monitor log ───────────────────────────────────────────────────────────
|
||||
|
||||
def insert_monitor_log(
|
||||
self,
|
||||
entries: list[MonitorLogEntry],
|
||||
*,
|
||||
session_id: Optional[str] = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Insert monitor log entries. Silently skips duplicates (serial+start_time).
|
||||
Returns (inserted, skipped).
|
||||
"""
|
||||
inserted = skipped = 0
|
||||
with self._connect() as conn:
|
||||
for e in entries:
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO monitor_log
|
||||
(id, serial, waveform_key, session_id,
|
||||
start_time, stop_time, duration_seconds,
|
||||
geo_threshold_ips)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
self._new_id(),
|
||||
e.serial or "",
|
||||
e.key,
|
||||
session_id,
|
||||
self._iso(e.start_time),
|
||||
self._iso(e.stop_time),
|
||||
e.duration_seconds,
|
||||
e.geo_threshold_ips,
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
except sqlite3.IntegrityError:
|
||||
skipped += 1
|
||||
|
||||
log.debug("insert_monitor_log inserted=%d skipped=%d", inserted, skipped)
|
||||
return inserted, skipped
|
||||
|
||||
def query_monitor_log(
|
||||
self,
|
||||
serial: Optional[str] = None,
|
||||
from_dt: Optional[datetime.datetime] = None,
|
||||
to_dt: Optional[datetime.datetime] = None,
|
||||
limit: int = 500,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Query monitor log entries with optional filters. Returns newest first."""
|
||||
clauses: list[str] = []
|
||||
params: list = []
|
||||
|
||||
if serial:
|
||||
clauses.append("serial = ?")
|
||||
params.append(serial)
|
||||
if from_dt:
|
||||
clauses.append("start_time >= ?")
|
||||
params.append(from_dt.isoformat())
|
||||
if to_dt:
|
||||
clauses.append("start_time <= ?")
|
||||
params.append(to_dt.isoformat())
|
||||
|
||||
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
params += [limit, offset]
|
||||
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM monitor_log {where} "
|
||||
f"ORDER BY start_time DESC LIMIT ? OFFSET ?",
|
||||
params,
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
# ── Fleet overview ────────────────────────────────────────────────────────
|
||||
|
||||
def query_units(self) -> list[dict]:
|
||||
"""
|
||||
Return one row per known serial with summary stats:
|
||||
last_seen, total_events, total_monitor_entries.
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
s.serial,
|
||||
MAX(s.session_time) AS last_seen,
|
||||
SUM(s.events_downloaded) AS total_events,
|
||||
SUM(s.monitor_entries) AS total_monitor_entries,
|
||||
COUNT(*) AS total_sessions
|
||||
FROM ach_sessions s
|
||||
GROUP BY s.serial
|
||||
ORDER BY last_seen DESC
|
||||
"""
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
sfm.dump_0c — inspect the raw 210-byte SUB 0C waveform record stored in a
|
||||
sidecar JSON's `extensions.raw_records.waveform_record_b64`.
|
||||
|
||||
Usage:
|
||||
|
||||
python -m sfm.dump_0c <sidecar.sfm.json> [<sidecar.sfm.json> ...]
|
||||
|
||||
Prints, for each input:
|
||||
- A header summarising the sidecar's metadata-block claims (peaks,
|
||||
project, timestamp) — the "what BW says this event measured" view.
|
||||
- A 16-byte-wide hex dump of the raw 0C record, annotated with known
|
||||
field anchors (STRT, channel labels, project strings).
|
||||
- A "candidate float regions" scan that brute-forces every byte
|
||||
position as a float32 BE and prints any that yield a value in a
|
||||
plausible range (1e-7 to 1e3) — useful for hunting where Peak
|
||||
Acceleration / Peak Displacement / ZC Freq / Time of Peak live.
|
||||
|
||||
Pairing the printed candidates with the BW Event Report values lets
|
||||
us nail down byte offsets for the missing fields without a live
|
||||
device.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ── Annotations for known anchors in a 210-byte 0C record ──────────────────
|
||||
|
||||
# Anchors we look for and label inline in the hex dump. Each is a needle
|
||||
# (bytes to find) and a short label. Found via .find() — the first
|
||||
# occurrence wins.
|
||||
_ANCHORS = [
|
||||
(b"Tran", "Tran label (PPV @ +6, PVS @ -12)"),
|
||||
(b"Vert", "Vert label (PPV @ +6)"),
|
||||
(b"Long", "Long label (PPV @ +6)"),
|
||||
(b"MicL", "MicL label (peak psi @ +6)"),
|
||||
(b"Project:", "Project: label"),
|
||||
(b"Client:", "Client: label"),
|
||||
(b"User Name:", "User Name: label"),
|
||||
(b"Seis Loc:", "Seis Loc: label"),
|
||||
(b"Extended Notes", "Extended Notes label"),
|
||||
]
|
||||
|
||||
|
||||
def _hex_dump(data: bytes, anchors: dict[int, str]) -> str:
|
||||
"""Return a 16-byte-wide hex+ASCII dump, with anchor labels printed
|
||||
on the line that contains the anchor's start byte."""
|
||||
lines = []
|
||||
for off in range(0, len(data), 16):
|
||||
chunk = data[off : off + 16]
|
||||
hex_part = " ".join(f"{b:02x}" for b in chunk)
|
||||
ascii_part = "".join(chr(b) if 32 <= b < 127 else "." for b in chunk)
|
||||
line = f" {off:04x} {hex_part:<47} |{ascii_part}|"
|
||||
|
||||
# If any anchor lands on a byte in this row, append a tag
|
||||
tags = [
|
||||
f"[{a:#04x}: {label}]"
|
||||
for a, label in anchors.items()
|
||||
if off <= a < off + 16
|
||||
]
|
||||
if tags:
|
||||
line += " " + " ".join(tags)
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _scan_float32_be(data: bytes, lo: float, hi: float) -> list[tuple[int, float]]:
|
||||
"""Brute-force every offset where data[off:off+4] is a float32 BE in
|
||||
(lo, hi). Includes negatives in the symmetric range."""
|
||||
hits = []
|
||||
for i in range(len(data) - 3):
|
||||
try:
|
||||
v = struct.unpack_from(">f", data, i)[0]
|
||||
except struct.error:
|
||||
continue
|
||||
if v != v: # NaN
|
||||
continue
|
||||
if abs(v) < 1e-30 or abs(v) > 1e10: # crap range
|
||||
continue
|
||||
a = abs(v)
|
||||
if lo <= a <= hi:
|
||||
hits.append((i, v))
|
||||
return hits
|
||||
|
||||
|
||||
def _scan_uint16_be(data: bytes, lo: int, hi: int) -> list[tuple[int, int]]:
|
||||
"""Find every offset where uint16 BE is in [lo, hi]."""
|
||||
hits = []
|
||||
for i in range(len(data) - 1):
|
||||
v = (data[i] << 8) | data[i + 1]
|
||||
if lo <= v <= hi:
|
||||
hits.append((i, v))
|
||||
return hits
|
||||
|
||||
|
||||
def _summarize_sidecar(side: dict) -> str:
|
||||
ev = side.get("event", {})
|
||||
pv = side.get("peak_values", {})
|
||||
pi = side.get("project_info", {})
|
||||
bw = side.get("blastware", {})
|
||||
return (
|
||||
f" serial: {ev.get('serial')}\n"
|
||||
f" timestamp: {ev.get('timestamp')}\n"
|
||||
f" waveform: {ev.get('waveform_key')} ({ev.get('record_type')})\n"
|
||||
f" sample_rate:{ev.get('sample_rate')} sps rectime:{ev.get('rectime_seconds')}s\n"
|
||||
f" bw file: {bw.get('filename')} ({bw.get('filesize')} B)\n"
|
||||
f" peaks: "
|
||||
f"Tran={pv.get('transverse'):.5f} "
|
||||
f"Vert={pv.get('vertical'):.5f} "
|
||||
f"Long={pv.get('longitudinal'):.5f} "
|
||||
f"PVS={pv.get('vector_sum'):.5f} in/s "
|
||||
f"Mic={pv.get('mic_psi'):.6e} psi"
|
||||
if all(pv.get(k) is not None for k in
|
||||
("transverse", "vertical", "longitudinal", "vector_sum", "mic_psi"))
|
||||
else f" peaks: {pv}\n project: {pi}"
|
||||
) + (
|
||||
f"\n project: {pi.get('project')!r} / {pi.get('client')!r} / "
|
||||
f"operator={pi.get('operator')!r} loc={pi.get('sensor_location')!r}"
|
||||
)
|
||||
|
||||
|
||||
def dump_one(path: Path) -> int:
|
||||
side = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
raw_b64 = (
|
||||
side.get("extensions", {})
|
||||
.get("raw_records", {})
|
||||
.get("waveform_record_b64")
|
||||
)
|
||||
if not raw_b64:
|
||||
print(f"\n=== {path} ===")
|
||||
print(" ! no extensions.raw_records.waveform_record_b64 — sidecar")
|
||||
print(" pre-dates raw-0C persistence (added in v0.15.x). Re-save")
|
||||
print(" the event from the device to capture the bytes.")
|
||||
return 1
|
||||
|
||||
raw = base64.b64decode(raw_b64)
|
||||
|
||||
# Build anchor map
|
||||
anchors: dict[int, str] = {}
|
||||
for needle, label in _ANCHORS:
|
||||
i = raw.find(needle)
|
||||
if i >= 0:
|
||||
anchors[i] = label
|
||||
|
||||
print(f"\n=== {path} ===")
|
||||
print("metadata claimed by sidecar:")
|
||||
print(_summarize_sidecar(side))
|
||||
|
||||
print(f"\nraw 0C record ({len(raw)} bytes):")
|
||||
print(_hex_dump(raw, anchors))
|
||||
|
||||
# Float32 BE candidates in geo-relevant ranges
|
||||
geo_hits = _scan_float32_be(raw, 1e-5, 50.0)
|
||||
# Filter: only show hits that are NOT trivially the per-channel labels'
|
||||
# +6 PPV floats already documented (those will land in any sweep too).
|
||||
print("\nfloat32 BE candidates (1e-5 .. 50.0):")
|
||||
for off, v in geo_hits:
|
||||
annotation = ""
|
||||
for needle, _ in _ANCHORS[:4]: # geo + mic labels
|
||||
i = raw.find(needle)
|
||||
if i >= 0 and off == i + 6:
|
||||
annotation = f" ← {needle.decode()} PPV (label+6)"
|
||||
break
|
||||
print(f" {off:#04x} ({off:3d}) {v:>+15.6f}{annotation}")
|
||||
|
||||
print("\nuint16 BE candidates ZC-Freq-ish (1..200):")
|
||||
for off, v in _scan_uint16_be(raw, 1, 200):
|
||||
if v < 5: # too noisy at very low end
|
||||
continue
|
||||
print(f" {off:#04x} ({off:3d}) = {v}")
|
||||
|
||||
print("\nuint16 BE candidates Time-of-Peak-ish if stored as ms (1..30000):")
|
||||
for off, v in _scan_uint16_be(raw, 1, 30000):
|
||||
if v < 100: # noise filter
|
||||
continue
|
||||
# Only the first ~80 are worth showing — too many hits otherwise
|
||||
if off > 80:
|
||||
break
|
||||
print(f" {off:#04x} ({off:3d}) = {v} ms ?")
|
||||
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Inspect a saved 0C waveform record from a sidecar JSON.",
|
||||
)
|
||||
p.add_argument(
|
||||
"sidecars",
|
||||
nargs="+",
|
||||
type=Path,
|
||||
help="Path(s) to <event>.sfm.json sidecar file(s).",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
rc = 0
|
||||
for path in args.sidecars:
|
||||
try:
|
||||
rc |= dump_one(path)
|
||||
except Exception as exc:
|
||||
print(f"\n=== {path} ===\n ERROR: {exc}", file=sys.stderr)
|
||||
rc |= 2
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,530 @@
|
||||
"""
|
||||
sfm/event_hdf5.py — HDF5 codec for the canonical "clean waveform" file.
|
||||
|
||||
Layout written to `<filename>.h5`:
|
||||
|
||||
/
|
||||
├─ samples/
|
||||
│ ├─ Tran (float32, in/s) shape: (N,)
|
||||
│ ├─ Vert (float32, in/s) shape: (N,)
|
||||
│ ├─ Long (float32, in/s) shape: (N,)
|
||||
│ └─ MicL (float32, psi) shape: (N,)
|
||||
├─ samples_int16/ (optional)
|
||||
│ ├─ Tran (int16, raw ADC counts) shape: (N,)
|
||||
│ └─ ... per channel (only when present in the source)
|
||||
└─ root attrs (event metadata):
|
||||
schema_version int = 1
|
||||
kind str = "sfm.event.hdf5"
|
||||
serial str
|
||||
waveform_key str (8-hex)
|
||||
timestamp str (ISO-8601)
|
||||
record_type str
|
||||
sample_rate int (sps)
|
||||
pretrig_samples int
|
||||
total_samples int
|
||||
rectime_seconds float
|
||||
geo_range str "normal" | "sensitive"
|
||||
geo_full_scale_ips float (10.0 or 1.250)
|
||||
project str
|
||||
client str
|
||||
operator str
|
||||
sensor_location str
|
||||
peak_tran_ips float (from 0C; authoritative)
|
||||
peak_vert_ips float
|
||||
peak_long_ips float
|
||||
peak_pvs_ips float
|
||||
peak_mic_psi float
|
||||
tool_version str
|
||||
captured_at str (ISO-8601 UTC)
|
||||
source_kind str "sfm-live" | "sfm-ach" | "bw-import"
|
||||
|
||||
Why HDF5 and not just JSON for the canonical clean format:
|
||||
- Native float32 arrays (no base64 dance, no per-value JSON parsing).
|
||||
- Per-dataset gzip compression — sample arrays compress 3-5×.
|
||||
- Cross-language: h5py (Python), HDF5.jl (Julia), io.netcdf (R), etc.
|
||||
Analysis pipelines don't have to know anything about Blastware.
|
||||
- Self-describing via attributes; future fields don't break readers.
|
||||
|
||||
The plot-ready `sfm.plot.v1` JSON returned by the REST endpoints is
|
||||
derived from this HDF5 (or computed on-the-fly when no .h5 exists yet).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
from minimateplus.event_file_io import TOOL_VERSION as _DEFAULT_TOOL_VERSION
|
||||
from minimateplus.models import Event
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
HDF5_KIND = "sfm.event.hdf5"
|
||||
|
||||
# Geophone full-scale velocity per range (in/s). Confirmed in CLAUDE.md
|
||||
# from 4-20-26 captures: Normal=0x00 → 10 in/s, Sensitive=0x01 → 1.25 in/s.
|
||||
_GEO_FS_BY_RANGE = {
|
||||
"normal": 10.000,
|
||||
"sensitive": 1.2500,
|
||||
0: 10.000,
|
||||
1: 1.2500,
|
||||
}
|
||||
_INT16_FS = 32768.0
|
||||
|
||||
# Default mic conversion: ADC count → psi. Approximate; exact factor
|
||||
# depends on firmware reference voltage and mic sensitivity, neither of
|
||||
# which is independently confirmed. We try to refine it from the device-
|
||||
# reported peak when available (peak_mic_psi / max_abs_int16).
|
||||
_MIC_DEFAULT_FS_PSI = 0.0125 # ≈ 0.5 psi at full scale (rough)
|
||||
|
||||
|
||||
def _resolve_geo_full_scale(geo_range) -> float:
|
||||
"""Map a geo_range value (string or int from compliance config) to the
|
||||
full-scale velocity in in/s. Defaults to Normal range (10.0) when the
|
||||
value is unknown — same default as Blastware itself."""
|
||||
if geo_range is None:
|
||||
return _GEO_FS_BY_RANGE["normal"]
|
||||
if isinstance(geo_range, str):
|
||||
return _GEO_FS_BY_RANGE.get(geo_range.lower(), _GEO_FS_BY_RANGE["normal"])
|
||||
return _GEO_FS_BY_RANGE.get(int(geo_range), _GEO_FS_BY_RANGE["normal"])
|
||||
|
||||
|
||||
def _normalise_range(geo_range) -> str:
|
||||
"""Return 'normal' or 'sensitive' (string) regardless of input form."""
|
||||
if isinstance(geo_range, str):
|
||||
v = geo_range.lower()
|
||||
if v in ("normal", "sensitive"):
|
||||
return v
|
||||
return "normal"
|
||||
if geo_range == 1:
|
||||
return "sensitive"
|
||||
return "normal"
|
||||
|
||||
|
||||
def _ts_iso(ts) -> str:
|
||||
if ts is None:
|
||||
return ""
|
||||
try:
|
||||
return datetime.datetime(
|
||||
ts.year, ts.month, ts.day,
|
||||
ts.hour or 0, ts.minute or 0, ts.second or 0,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
return str(ts)
|
||||
|
||||
|
||||
def _samples_to_float(
|
||||
samples_int16: list[int],
|
||||
full_scale: float,
|
||||
) -> np.ndarray:
|
||||
"""Convert int16 ADC counts → float32 physical units.
|
||||
|
||||
Uses _INT16_FS=32768 (not 32767) so that a count of -32768 maps to
|
||||
exactly -full_scale and +32767 maps to ~+full_scale * 32767/32768.
|
||||
Matches the device firmware's documented mapping (see CLAUDE.md
|
||||
geo_hardware_constant rationale).
|
||||
"""
|
||||
if not samples_int16:
|
||||
return np.array([], dtype=np.float32)
|
||||
arr = np.asarray(samples_int16, dtype=np.int32) # int32 to avoid overflow during scale
|
||||
return (arr.astype(np.float32) * (full_scale / _INT16_FS)).astype(np.float32)
|
||||
|
||||
|
||||
def _mic_scale_factor(
|
||||
samples_int16: list[int],
|
||||
peak_mic_psi: Optional[float],
|
||||
) -> float:
|
||||
"""Resolve the per-count psi factor for the microphone channel.
|
||||
|
||||
When the device reports a peak mic value via the 0C record, we
|
||||
back-solve the per-count factor from `peak_psi / max(|samples|)` so
|
||||
the plotted waveform peaks land exactly at the device-reported value.
|
||||
Otherwise fall back to the rough _MIC_DEFAULT_FS_PSI estimate.
|
||||
"""
|
||||
if peak_mic_psi is not None and peak_mic_psi > 0 and samples_int16:
|
||||
max_count = max(abs(int(v)) for v in samples_int16) or 1
|
||||
return float(peak_mic_psi) / float(max_count)
|
||||
return _MIC_DEFAULT_FS_PSI / _INT16_FS
|
||||
|
||||
|
||||
def write_event_hdf5(
|
||||
path: Union[str, Path],
|
||||
event: Event,
|
||||
*,
|
||||
serial: str,
|
||||
geo_range = "normal",
|
||||
source_kind: str = "sfm-live",
|
||||
tool_version: Optional[str] = None,
|
||||
captured_at: Optional[datetime.datetime] = None,
|
||||
include_int16: bool = True,
|
||||
) -> dict:
|
||||
"""
|
||||
Persist a decoded Event as an HDF5 file with samples in physical units.
|
||||
|
||||
Returns a small summary dict suitable for logging:
|
||||
{"path": Path, "n_samples": int, "geo_full_scale_ips": float}
|
||||
"""
|
||||
path = Path(path)
|
||||
raw = event.raw_samples or {}
|
||||
pv = event.peak_values
|
||||
pi = event.project_info
|
||||
|
||||
geo_fs = _resolve_geo_full_scale(geo_range)
|
||||
geo_range_str = _normalise_range(geo_range)
|
||||
captured_at = captured_at or datetime.datetime.utcnow()
|
||||
tool_version = tool_version or _DEFAULT_TOOL_VERSION
|
||||
|
||||
# Per-channel float32 arrays in physical units.
|
||||
geo_arrays = {}
|
||||
for ch in ("Tran", "Vert", "Long"):
|
||||
geo_arrays[ch] = _samples_to_float(raw.get(ch, []), geo_fs)
|
||||
|
||||
# Mic channel — the per-count factor is resolved from the device-reported
|
||||
# peak when available so the plot peaks the BW value exactly.
|
||||
mic_int16 = raw.get("MicL", [])
|
||||
mic_factor = _mic_scale_factor(
|
||||
mic_int16,
|
||||
getattr(pv, "micl", None) if pv else None,
|
||||
)
|
||||
if mic_int16:
|
||||
mic_arr = (np.asarray(mic_int16, dtype=np.int32).astype(np.float32) * mic_factor).astype(np.float32)
|
||||
else:
|
||||
mic_arr = np.array([], dtype=np.float32)
|
||||
|
||||
n_samples = max(
|
||||
(len(geo_arrays[ch]) for ch in geo_arrays),
|
||||
default=0,
|
||||
)
|
||||
|
||||
# Atomic write: temp file + os.replace.
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with h5py.File(tmp, "w") as f:
|
||||
# Root attrs — event-level metadata.
|
||||
attrs = f.attrs
|
||||
attrs["schema_version"] = SCHEMA_VERSION
|
||||
attrs["kind"] = HDF5_KIND
|
||||
attrs["serial"] = serial or ""
|
||||
attrs["waveform_key"] = event._waveform_key.hex() if event._waveform_key else ""
|
||||
attrs["timestamp"] = _ts_iso(event.timestamp)
|
||||
attrs["record_type"] = event.record_type or ""
|
||||
attrs["sample_rate"] = int(event.sample_rate or 0)
|
||||
attrs["pretrig_samples"] = int(event.pretrig_samples or 0)
|
||||
attrs["total_samples"] = int(event.total_samples or n_samples)
|
||||
attrs["rectime_seconds"] = float(event.rectime_seconds or 0.0)
|
||||
attrs["geo_range"] = geo_range_str
|
||||
attrs["geo_full_scale_ips"] = float(geo_fs)
|
||||
attrs["project"] = (pi.project if pi else "") or ""
|
||||
attrs["client"] = (pi.client if pi else "") or ""
|
||||
attrs["operator"] = (pi.operator if pi else "") or ""
|
||||
attrs["sensor_location"] = (pi.sensor_location if pi else "") or ""
|
||||
attrs["peak_tran_ips"] = float(pv.tran if pv and pv.tran is not None else 0.0)
|
||||
attrs["peak_vert_ips"] = float(pv.vert if pv and pv.vert is not None else 0.0)
|
||||
attrs["peak_long_ips"] = float(pv.long if pv and pv.long is not None else 0.0)
|
||||
attrs["peak_pvs_ips"] = float(pv.peak_vector_sum if pv and pv.peak_vector_sum is not None else 0.0)
|
||||
attrs["peak_mic_psi"] = float(pv.micl if pv and pv.micl is not None else 0.0)
|
||||
attrs["tool_version"] = tool_version or ""
|
||||
attrs["captured_at"] = captured_at.isoformat() + "Z" if captured_at.tzinfo is None else captured_at.isoformat()
|
||||
attrs["source_kind"] = source_kind
|
||||
|
||||
# /samples — physical-units float32 (the primary data).
|
||||
sgrp = f.create_group("samples")
|
||||
for ch, arr in geo_arrays.items():
|
||||
sgrp.create_dataset(
|
||||
ch, data=arr, dtype="float32",
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
sgrp.create_dataset(
|
||||
"MicL", data=mic_arr, dtype="float32",
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
|
||||
# /samples_int16 — optional raw ADC counts (preserved for analysis
|
||||
# tools that want pre-conversion data). Cheap to include.
|
||||
if include_int16:
|
||||
igrp = f.create_group("samples_int16")
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
vals = raw.get(ch, [])
|
||||
if vals:
|
||||
igrp.create_dataset(
|
||||
ch, data=np.asarray(vals, dtype=np.int16),
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
igrp.attrs["mic_psi_per_count"] = float(mic_factor)
|
||||
|
||||
import os
|
||||
os.replace(tmp, path)
|
||||
|
||||
log.info(
|
||||
"write_event_hdf5: %s n_samples=%d geo_fs=%.3f filesize=%d",
|
||||
path, n_samples, geo_fs, path.stat().st_size,
|
||||
)
|
||||
return {
|
||||
"path": path,
|
||||
"n_samples": n_samples,
|
||||
"geo_full_scale_ips": geo_fs,
|
||||
}
|
||||
|
||||
|
||||
def read_event_hdf5(path: Union[str, Path]) -> dict:
|
||||
"""
|
||||
Load an event HDF5 into a plain dict (no Event reconstruction —
|
||||
callers that want an Event can use the data directly).
|
||||
|
||||
Returns:
|
||||
{
|
||||
"schema_version": int,
|
||||
"kind": str,
|
||||
"attrs": dict[str, …], # all root attributes
|
||||
"samples": { # float32 lists in physical units
|
||||
"Tran": ndarray, "Vert": ndarray, "Long": ndarray, "MicL": ndarray,
|
||||
},
|
||||
"samples_int16": {…} or None,
|
||||
"mic_psi_per_count": float | None,
|
||||
}
|
||||
|
||||
Raises FileNotFoundError if missing, ValueError on bad shape /
|
||||
unsupported schema_version.
|
||||
"""
|
||||
path = Path(path)
|
||||
with h5py.File(path, "r") as f:
|
||||
attrs = {k: _h5_attr_value(v) for k, v in f.attrs.items()}
|
||||
sv = attrs.get("schema_version", 0)
|
||||
if not isinstance(sv, int) or sv < 1 or sv > SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
f"{path}: unsupported HDF5 schema_version={sv} "
|
||||
f"(this build supports 1..{SCHEMA_VERSION})"
|
||||
)
|
||||
if attrs.get("kind") != HDF5_KIND:
|
||||
raise ValueError(f"{path}: kind != {HDF5_KIND!r} (got {attrs.get('kind')!r})")
|
||||
|
||||
samples = {}
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
ds = f.get(f"samples/{ch}")
|
||||
samples[ch] = np.asarray(ds[()]) if ds is not None else np.array([], dtype=np.float32)
|
||||
|
||||
samples_int16 = None
|
||||
mic_psi = None
|
||||
igrp = f.get("samples_int16")
|
||||
if igrp is not None:
|
||||
samples_int16 = {}
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
ds = igrp.get(ch)
|
||||
if ds is not None:
|
||||
samples_int16[ch] = np.asarray(ds[()])
|
||||
mic_attr = igrp.attrs.get("mic_psi_per_count")
|
||||
if mic_attr is not None:
|
||||
mic_psi = float(mic_attr)
|
||||
|
||||
return {
|
||||
"schema_version": sv,
|
||||
"kind": attrs.get("kind"),
|
||||
"attrs": attrs,
|
||||
"samples": samples,
|
||||
"samples_int16": samples_int16,
|
||||
"mic_psi_per_count": mic_psi,
|
||||
}
|
||||
|
||||
|
||||
def _h5_attr_value(v):
|
||||
"""Convert an h5py attribute value to a plain Python type."""
|
||||
if isinstance(v, bytes):
|
||||
return v.decode("utf-8", errors="replace")
|
||||
if isinstance(v, np.generic):
|
||||
return v.item()
|
||||
return v
|
||||
|
||||
|
||||
# ── Plot-ready JSON ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def event_to_plot_json(
|
||||
event: Event,
|
||||
*,
|
||||
serial: str,
|
||||
geo_range = "normal",
|
||||
event_id: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Build a `sfm.plot.v1` JSON dict directly from an Event (skipping HDF5).
|
||||
|
||||
Used by:
|
||||
- `/device/event/{idx}/waveform` (live device path)
|
||||
- The CLI / tests for in-memory conversion sanity-checks.
|
||||
|
||||
Stored events go through `plot_json_from_hdf5()` so the wire format
|
||||
is identical regardless of whether the data came from the live device
|
||||
or the on-disk HDF5.
|
||||
"""
|
||||
raw = event.raw_samples or {}
|
||||
pv = event.peak_values
|
||||
geo_fs = _resolve_geo_full_scale(geo_range)
|
||||
geo_range_str = _normalise_range(geo_range)
|
||||
sr = int(event.sample_rate or 0) or 1024
|
||||
pretrig = int(event.pretrig_samples or 0)
|
||||
|
||||
geo_arrays = {ch: _samples_to_float(raw.get(ch, []), geo_fs).tolist()
|
||||
for ch in ("Tran", "Vert", "Long")}
|
||||
mic_int16 = raw.get("MicL", [])
|
||||
mic_factor = _mic_scale_factor(
|
||||
mic_int16,
|
||||
getattr(pv, "micl", None) if pv else None,
|
||||
)
|
||||
mic_arr = [float(v) * mic_factor for v in mic_int16] if mic_int16 else []
|
||||
|
||||
n = max(
|
||||
(len(geo_arrays[ch]) for ch in geo_arrays),
|
||||
default=len(mic_arr),
|
||||
)
|
||||
return _build_plot_dict(
|
||||
n_samples=n,
|
||||
sample_rate=sr,
|
||||
pretrig_samples=pretrig,
|
||||
total_samples=int(event.total_samples or n),
|
||||
rectime_seconds=float(event.rectime_seconds or 0.0),
|
||||
timestamp_iso=_ts_iso(event.timestamp),
|
||||
serial=serial,
|
||||
record_type=event.record_type,
|
||||
waveform_key=event._waveform_key.hex() if event._waveform_key else None,
|
||||
geo_range=geo_range_str,
|
||||
geo_fs=geo_fs,
|
||||
channels_floats={
|
||||
"Tran": geo_arrays["Tran"],
|
||||
"Vert": geo_arrays["Vert"],
|
||||
"Long": geo_arrays["Long"],
|
||||
"MicL": mic_arr,
|
||||
},
|
||||
peaks_dict={
|
||||
"tran": getattr(pv, "tran", None) if pv else None,
|
||||
"vert": getattr(pv, "vert", None) if pv else None,
|
||||
"long": getattr(pv, "long", None) if pv else None,
|
||||
"pvs": getattr(pv, "peak_vector_sum", None) if pv else None,
|
||||
"mic": getattr(pv, "micl", None) if pv else None,
|
||||
},
|
||||
event_id=event_id,
|
||||
index=index if index is not None else event.index,
|
||||
)
|
||||
|
||||
|
||||
def plot_json_from_hdf5(
|
||||
path: Union[str, Path],
|
||||
*,
|
||||
event_id: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""Build a `sfm.plot.v1` JSON dict from a stored .h5 file."""
|
||||
data = read_event_hdf5(path)
|
||||
a = data["attrs"]
|
||||
s = data["samples"]
|
||||
return _build_plot_dict(
|
||||
n_samples=len(s["Tran"]) if "Tran" in s else 0,
|
||||
sample_rate=int(a.get("sample_rate", 1024) or 1024),
|
||||
pretrig_samples=int(a.get("pretrig_samples", 0) or 0),
|
||||
total_samples=int(a.get("total_samples", 0) or 0),
|
||||
rectime_seconds=float(a.get("rectime_seconds", 0.0) or 0.0),
|
||||
timestamp_iso=a.get("timestamp", ""),
|
||||
serial=a.get("serial", ""),
|
||||
record_type=a.get("record_type", ""),
|
||||
waveform_key=a.get("waveform_key", "") or None,
|
||||
geo_range=a.get("geo_range", "normal"),
|
||||
geo_fs=float(a.get("geo_full_scale_ips", 10.0) or 10.0),
|
||||
channels_floats={
|
||||
"Tran": s.get("Tran", np.array([])).tolist(),
|
||||
"Vert": s.get("Vert", np.array([])).tolist(),
|
||||
"Long": s.get("Long", np.array([])).tolist(),
|
||||
"MicL": s.get("MicL", np.array([])).tolist(),
|
||||
},
|
||||
peaks_dict={
|
||||
"tran": float(a.get("peak_tran_ips", 0.0) or 0.0) or None,
|
||||
"vert": float(a.get("peak_vert_ips", 0.0) or 0.0) or None,
|
||||
"long": float(a.get("peak_long_ips", 0.0) or 0.0) or None,
|
||||
"pvs": float(a.get("peak_pvs_ips", 0.0) or 0.0) or None,
|
||||
"mic": float(a.get("peak_mic_psi", 0.0) or 0.0) or None,
|
||||
},
|
||||
event_id=event_id,
|
||||
index=index,
|
||||
)
|
||||
|
||||
|
||||
def _build_plot_dict(
|
||||
*,
|
||||
n_samples: int,
|
||||
sample_rate: int,
|
||||
pretrig_samples: int,
|
||||
total_samples: int,
|
||||
rectime_seconds: float,
|
||||
timestamp_iso: str,
|
||||
serial: str,
|
||||
record_type: Optional[str],
|
||||
waveform_key: Optional[str],
|
||||
geo_range: str,
|
||||
geo_fs: float,
|
||||
channels_floats: dict[str, list[float]],
|
||||
peaks_dict: dict[str, Optional[float]],
|
||||
event_id: Optional[str],
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
dt_ms = (1000.0 / sample_rate) if sample_rate > 0 else 0.0
|
||||
t0_ms = -pretrig_samples * dt_ms
|
||||
|
||||
def _ch(unit: str, values: list[float], peak: Optional[float]) -> dict:
|
||||
# Locate the peak's time within the values array (max abs).
|
||||
if values:
|
||||
mags = [abs(v) for v in values]
|
||||
i = mags.index(max(mags))
|
||||
peak_t_ms = round(t0_ms + i * dt_ms, 4)
|
||||
peak_value = peak if peak is not None else values[i]
|
||||
else:
|
||||
peak_t_ms = None
|
||||
peak_value = peak
|
||||
return {
|
||||
"unit": unit,
|
||||
"values": values,
|
||||
"peak": peak_value,
|
||||
"peak_t_ms": peak_t_ms,
|
||||
}
|
||||
|
||||
return {
|
||||
"schema": "sfm.plot.v1",
|
||||
"event_id": event_id,
|
||||
"index": index,
|
||||
"serial": serial,
|
||||
"timestamp": timestamp_iso,
|
||||
"record_type": record_type,
|
||||
"waveform_key": waveform_key,
|
||||
|
||||
"time_axis": {
|
||||
"sample_rate": sample_rate,
|
||||
"pretrig_samples": pretrig_samples,
|
||||
"total_samples": total_samples or n_samples,
|
||||
"n_samples": n_samples,
|
||||
"t0_ms": round(t0_ms, 4),
|
||||
"dt_ms": round(dt_ms, 6),
|
||||
"rectime_seconds": rectime_seconds,
|
||||
},
|
||||
|
||||
"geo_range": geo_range,
|
||||
"geo_full_scale_ips": geo_fs,
|
||||
"trigger_ms": 0.0,
|
||||
|
||||
"channels": {
|
||||
"Tran": _ch("in/s", channels_floats.get("Tran", []), peaks_dict.get("tran")),
|
||||
"Vert": _ch("in/s", channels_floats.get("Vert", []), peaks_dict.get("vert")),
|
||||
"Long": _ch("in/s", channels_floats.get("Long", []), peaks_dict.get("long")),
|
||||
"MicL": _ch("psi", channels_floats.get("MicL", []), peaks_dict.get("mic")),
|
||||
},
|
||||
|
||||
"peak_values": {
|
||||
"transverse": peaks_dict.get("tran"),
|
||||
"vertical": peaks_dict.get("vert"),
|
||||
"longitudinal": peaks_dict.get("long"),
|
||||
"vector_sum": peaks_dict.get("pvs"),
|
||||
"mic_psi": peaks_dict.get("mic"),
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
sfm/import_bw.py — CLI for ingesting Blastware-format event files.
|
||||
|
||||
Walks a path (file or directory), parses each recognised event-file
|
||||
binary, copies it into the canonical waveform store, writes the
|
||||
.sfm.json sidecar, and upserts a row in seismo_relay.db.
|
||||
|
||||
Use cases:
|
||||
- Migrating a Blastware ACH inbox into SFM
|
||||
- One-off imports of files emailed in by field crews
|
||||
- Bulk-loading historical archives
|
||||
|
||||
Usage:
|
||||
python -m sfm.import_bw <path-or-dir> [--serial BE11529]
|
||||
[--db-path bridges/captures/seismo_relay.db]
|
||||
[--store-root bridges/captures/waveforms]
|
||||
[--dry-run]
|
||||
[-v]
|
||||
|
||||
Examples:
|
||||
python -m sfm.import_bw ~/Downloads/M529LKIQ.7M0W
|
||||
python -m sfm.import_bw /path/to/blastware_archive --serial BE11529
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
# Allow running from the repo root without installation.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from sfm.database import SeismoDb
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
log = logging.getLogger("sfm.import_bw")
|
||||
|
||||
|
||||
# Blastware event-file extensions: 4-char `AB0T` (T = W or H) for ACH
|
||||
# downloads, 3-char `AB0` for direct downloads. We discover candidates
|
||||
# by length + last-char rather than enumerating every (A, B) pair.
|
||||
def _looks_like_bw_event(path: Path) -> bool:
|
||||
"""Heuristic: 3-char or 4-char extension, ends with W/H/0, and the
|
||||
file is at least 70 bytes (header + STRT + footer minimum)."""
|
||||
if not path.is_file():
|
||||
return False
|
||||
ext = path.suffix.lstrip(".")
|
||||
if not (3 <= len(ext) <= 4):
|
||||
return False
|
||||
if not (ext[-1].upper() in {"W", "H"} or ext.endswith("0")):
|
||||
return False
|
||||
try:
|
||||
return path.stat().st_size >= 70
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _walk(path: Path) -> Iterator[Path]:
|
||||
"""Yield candidate BW event-file paths under `path` (file or dir)."""
|
||||
if path.is_file():
|
||||
if _looks_like_bw_event(path):
|
||||
yield path
|
||||
return
|
||||
if path.is_dir():
|
||||
for p in sorted(path.rglob("*")):
|
||||
if _looks_like_bw_event(p):
|
||||
yield p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Import Blastware-format event files into the SFM store + DB.",
|
||||
)
|
||||
p.add_argument("path", help="File or directory to import.")
|
||||
p.add_argument(
|
||||
"--serial", default=None, metavar="SERIAL",
|
||||
help="Override the serial-number hint (e.g. BE11529). Defaults to "
|
||||
"the value decoded from each BW filename's prefix.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--db-path",
|
||||
default=str(Path(__file__).resolve().parent.parent / "bridges" / "captures" / "seismo_relay.db"),
|
||||
help="Path to seismo_relay.db (default: bridges/captures/seismo_relay.db).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--store-root",
|
||||
default=None,
|
||||
help="Root of the waveform store (default: <db_dir>/waveforms).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="Parse and report per-file outcomes; don't write anything.",
|
||||
)
|
||||
p.add_argument("-v", "--verbose", action="store_true", help="Debug logging.")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
src = Path(args.path).expanduser().resolve()
|
||||
if not src.exists():
|
||||
print(f"error: {src} does not exist", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
db_path = Path(args.db_path).expanduser().resolve()
|
||||
store_root = (
|
||||
Path(args.store_root).expanduser().resolve()
|
||||
if args.store_root else db_path.parent / "waveforms"
|
||||
)
|
||||
|
||||
db = None if args.dry_run else SeismoDb(db_path)
|
||||
store = None if args.dry_run else WaveformStore(store_root)
|
||||
|
||||
candidates = list(_walk(src))
|
||||
if not candidates:
|
||||
print(f"No BW event-file candidates found under {src}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"Importing {len(candidates)} file(s) from {src}...")
|
||||
if args.dry_run:
|
||||
print("(dry-run — no writes will occur)")
|
||||
|
||||
ok = err = skipped = 0
|
||||
for path in candidates:
|
||||
try:
|
||||
bw_bytes = path.read_bytes()
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: read failed: {exc}")
|
||||
err += 1
|
||||
continue
|
||||
|
||||
if args.dry_run:
|
||||
# Just parse to verify integrity; don't touch DB or store.
|
||||
from minimateplus import event_file_io
|
||||
try:
|
||||
ev = event_file_io.read_blastware_file(path)
|
||||
ts = ev.timestamp and (
|
||||
f"{ev.timestamp.year}-{ev.timestamp.month:02d}-{ev.timestamp.day:02d} "
|
||||
f"{ev.timestamp.hour:02d}:{ev.timestamp.minute:02d}:{ev.timestamp.second:02d}"
|
||||
) or "?"
|
||||
pv = ev.peak_values
|
||||
pvs = pv.peak_vector_sum if pv and pv.peak_vector_sum is not None else 0.0
|
||||
print(f" [OK ] {path.name} ts={ts} PVS={pvs:.4f}")
|
||||
ok += 1
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: parse failed: {exc}")
|
||||
err += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
ev, rec = store.save_imported_bw(
|
||||
bw_bytes, source_path=path, serial_hint=args.serial,
|
||||
)
|
||||
# Resolve serial for the DB row. Prefer the hint, then the
|
||||
# one decoded from the filename (already done by the store).
|
||||
serial_used = args.serial or _infer_serial(path.name) or "UNKNOWN"
|
||||
ins, sk = db.insert_events(
|
||||
[ev], serial=serial_used,
|
||||
waveform_records=(
|
||||
{ev._waveform_key.hex(): rec}
|
||||
if ev._waveform_key else None
|
||||
),
|
||||
)
|
||||
tag = "OK " if ins else ("SKIP" if sk else "OK ")
|
||||
print(f" [{tag}] {path.name} → {rec['filename']} "
|
||||
f"({rec['filesize']} B, sha256={rec['sha256'][:12]}…) "
|
||||
f"serial={serial_used} ins={ins} skip={sk}")
|
||||
if ins:
|
||||
ok += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: import failed: {exc}")
|
||||
log.debug("traceback", exc_info=True)
|
||||
err += 1
|
||||
|
||||
print(f"\nDone. ok={ok} skipped={skipped} errors={err}")
|
||||
return 0 if err == 0 else 1
|
||||
|
||||
|
||||
def _infer_serial(filename: str):
|
||||
"""Reuse WaveformStore's filename → serial decoder for log output."""
|
||||
from sfm.waveform_store import _serial_from_bw_filename
|
||||
return _serial_from_bw_filename(filename)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
sfm/live_cache.py — Thread-safe in-memory cache for live SFM device data.
|
||||
|
||||
Extracted from sfm/server.py so the cache logic is importable and testable
|
||||
without pulling in fastapi/uvicorn.
|
||||
|
||||
Caching strategy
|
||||
----------------
|
||||
Keyed by `conn_key` ("tcp:host:port" or "serial:port:baud"). Does NOT
|
||||
persist across server restarts.
|
||||
|
||||
device_info cached until POST /device/config marks it dirty
|
||||
events cached by (conn_key, device_event_count); re-fetched when
|
||||
a quick count_events() probe shows new events on the device
|
||||
monitor_status 30-second TTL (changes frequently during monitoring)
|
||||
waveforms permanent within a process — but auto-evicted at the device
|
||||
level when a (waveform_key, timestamp) mismatch is detected
|
||||
at the same index (post-erase key reuse — the device's
|
||||
event-key counter resets to 0x01110000 after every erase,
|
||||
so the same `(conn_key, index)` slot can refer to a
|
||||
brand-new physical event).
|
||||
|
||||
All endpoints accept ?force=true to bypass the cache and re-read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
_MONITOR_STATUS_TTL = 30.0 # seconds
|
||||
|
||||
|
||||
class LiveCache:
|
||||
"""
|
||||
Thread-safe in-memory cache for live SFM device data.
|
||||
One singleton per server process.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._device_info: dict[str, dict] = {}
|
||||
self._events: dict[str, tuple[int, list]] = {}
|
||||
self._monitor_status: dict[str, tuple[float, dict]] = {}
|
||||
self._config_dirty: dict[str, bool] = {}
|
||||
self._waveforms: dict[tuple, dict] = {}
|
||||
|
||||
# ── Connection key ────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def make_conn_key(
|
||||
host: Optional[str],
|
||||
tcp_port: int,
|
||||
port: Optional[str],
|
||||
baud: int,
|
||||
) -> str:
|
||||
if host:
|
||||
return f"tcp:{host}:{tcp_port}"
|
||||
return f"serial:{port}:{baud}"
|
||||
|
||||
# ── Eviction signature ────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _event_signature(ev: dict) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Return (waveform_key_hex, timestamp_iso) from a serialised event."""
|
||||
key = ev.get("waveform_key") or ev.get("_waveform_key")
|
||||
if isinstance(key, (bytes, bytearray)):
|
||||
key = bytes(key).hex()
|
||||
ts = ev.get("timestamp")
|
||||
if isinstance(ts, dict):
|
||||
ts = ts.get("iso") or ts.get("string") or None
|
||||
return (key if isinstance(key, str) else None,
|
||||
ts if isinstance(ts, str) else None)
|
||||
|
||||
def _flush_device(self, conn_key: str) -> None:
|
||||
"""Drop all cached events + waveforms for one device. Caller holds lock."""
|
||||
self._events.pop(conn_key, None)
|
||||
stale_wf_keys = [k for k in self._waveforms if k[0] == conn_key]
|
||||
for k in stale_wf_keys:
|
||||
self._waveforms.pop(k, None)
|
||||
|
||||
# ── Device info ───────────────────────────────────────────────────────────
|
||||
|
||||
def get_device_info(self, conn_key: str) -> Optional[dict]:
|
||||
with self._lock:
|
||||
if self._config_dirty.get(conn_key):
|
||||
return None
|
||||
return self._device_info.get(conn_key)
|
||||
|
||||
def set_device_info(self, conn_key: str, info: dict) -> None:
|
||||
with self._lock:
|
||||
self._device_info[conn_key] = info
|
||||
self._config_dirty[conn_key] = False
|
||||
|
||||
# ── Events ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_events(self, conn_key: str, device_count: int) -> Optional[list]:
|
||||
with self._lock:
|
||||
if self._config_dirty.get(conn_key):
|
||||
return None
|
||||
entry = self._events.get(conn_key)
|
||||
if entry is None:
|
||||
return None
|
||||
cached_count, events = entry
|
||||
return events if cached_count == device_count else None
|
||||
|
||||
def set_events(self, conn_key: str, device_count: int, events: list) -> None:
|
||||
"""
|
||||
Replace the cached events list for `conn_key`. If any incoming event
|
||||
has a different (waveform_key, timestamp) than the cached entry at
|
||||
the same index, flush the entire conn_key's event + waveform cache
|
||||
first. Catches post-erase key reuse.
|
||||
"""
|
||||
with self._lock:
|
||||
cached_entry = self._events.get(conn_key)
|
||||
cached_events = cached_entry[1] if cached_entry else []
|
||||
cached_by_index = {e.get("index"): e for e in cached_events}
|
||||
|
||||
evict = False
|
||||
for ev in events:
|
||||
idx = ev.get("index")
|
||||
if idx is None:
|
||||
continue
|
||||
cached = cached_by_index.get(idx)
|
||||
if cached is None:
|
||||
continue
|
||||
new_key, new_ts = self._event_signature(ev)
|
||||
old_key, old_ts = self._event_signature(cached)
|
||||
if (new_key and old_key and new_key != old_key) or \
|
||||
(new_ts and old_ts and new_ts != old_ts):
|
||||
evict = True
|
||||
break
|
||||
|
||||
if evict:
|
||||
self._flush_device(conn_key)
|
||||
|
||||
self._events[conn_key] = (device_count, events)
|
||||
|
||||
# ── Monitor status ────────────────────────────────────────────────────────
|
||||
|
||||
def get_monitor_status(self, conn_key: str) -> Optional[dict]:
|
||||
with self._lock:
|
||||
entry = self._monitor_status.get(conn_key)
|
||||
if entry is None:
|
||||
return None
|
||||
fetched_at, status = entry
|
||||
if time.time() - fetched_at > _MONITOR_STATUS_TTL:
|
||||
return None
|
||||
return status
|
||||
|
||||
def set_monitor_status(self, conn_key: str, status: dict) -> None:
|
||||
with self._lock:
|
||||
self._monitor_status[conn_key] = (time.time(), status)
|
||||
|
||||
def invalidate_monitor_status(self, conn_key: str) -> None:
|
||||
with self._lock:
|
||||
self._monitor_status.pop(conn_key, None)
|
||||
|
||||
# ── Config dirty flag ─────────────────────────────────────────────────────
|
||||
|
||||
def mark_config_dirty(self, conn_key: str) -> None:
|
||||
with self._lock:
|
||||
self._config_dirty[conn_key] = True
|
||||
self._events.pop(conn_key, None)
|
||||
|
||||
# ── Waveforms (permanent cache, evicted on (key,ts) mismatch) ─────────────
|
||||
|
||||
def get_waveform(self, conn_key: str, index: int) -> Optional[dict]:
|
||||
with self._lock:
|
||||
return self._waveforms.get((conn_key, index))
|
||||
|
||||
def set_waveform(self, conn_key: str, index: int, waveform: dict) -> None:
|
||||
"""
|
||||
Cache a waveform. Evicts the device's whole cache when the existing
|
||||
entry at the same index has a different (waveform_key, timestamp).
|
||||
"""
|
||||
with self._lock:
|
||||
existing = self._waveforms.get((conn_key, index))
|
||||
if existing is not None:
|
||||
new_key, new_ts = self._event_signature(waveform)
|
||||
old_key, old_ts = self._event_signature(existing)
|
||||
differs = (
|
||||
(new_key and old_key and new_key != old_key)
|
||||
or (new_ts and old_ts and new_ts != old_ts)
|
||||
)
|
||||
if differs:
|
||||
self._flush_device(conn_key)
|
||||
self._waveforms[(conn_key, index)] = waveform
|
||||
+1436
-30
File diff suppressed because it is too large
Load Diff
+2783
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,446 @@
|
||||
"""
|
||||
sfm/waveform_store.py — On-disk store for Blastware-format event files.
|
||||
|
||||
Layout (flat per-serial, four files per event):
|
||||
|
||||
<root>/<serial>/<filename> ← event file (BW-readable binary)
|
||||
<root>/<serial>/<filename>.a5.pkl ← pickled list of A5 S3Frame dicts
|
||||
<root>/<serial>/<filename>.h5 ← clean waveform arrays (HDF5)
|
||||
<root>/<serial>/<filename>.sfm.json ← modern sidecar (peaks, project,
|
||||
review state, extensions)
|
||||
|
||||
`<filename>` is whatever `minimateplus.blastware_file.blastware_filename`
|
||||
produces for the event. The extension is NOT a fixed type tag — it
|
||||
encodes the event timestamp (`AB0T` format).
|
||||
|
||||
Roles:
|
||||
- BW binary: what Blastware reads. Untouched. The user-facing review
|
||||
waveform viewer.
|
||||
- .a5.pkl: regenerative source. Lets the BW binary be rebuilt
|
||||
byte-for-byte if the encoder changes. Never delete.
|
||||
- .h5: clean per-channel waveform arrays in physical units (in/s for
|
||||
geo, psi for mic) plus event metadata. Canonical format for
|
||||
downstream analysis tools and the `/device/event/{idx}/waveform`
|
||||
endpoint's plot-JSON output.
|
||||
- .sfm.json: small, queryable metadata + review state. SQL
|
||||
`events.false_trigger` is a derived index kept in sync via
|
||||
`patch_sidecar()`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import pickle
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from minimateplus import event_file_io
|
||||
from minimateplus.blastware_file import blastware_filename, write_blastware_file
|
||||
from minimateplus.framing import S3Frame
|
||||
from minimateplus.models import Event
|
||||
from sfm import event_hdf5
|
||||
|
||||
log = logging.getLogger("sfm.waveform_store")
|
||||
|
||||
A5_PICKLE_VERSION = 1
|
||||
|
||||
|
||||
def _frame_to_dict(f: S3Frame) -> dict:
|
||||
return {
|
||||
"sub": f.sub,
|
||||
"page_hi": f.page_hi,
|
||||
"page_lo": f.page_lo,
|
||||
"data": bytes(f.data),
|
||||
"chk_byte": f.chk_byte,
|
||||
"checksum_valid": f.checksum_valid,
|
||||
}
|
||||
|
||||
|
||||
def _dict_to_frame(d: dict) -> S3Frame:
|
||||
return S3Frame(
|
||||
sub=d["sub"],
|
||||
page_hi=d["page_hi"],
|
||||
page_lo=d["page_lo"],
|
||||
data=bytes(d["data"]),
|
||||
checksum_valid=d.get("checksum_valid", True),
|
||||
chk_byte=d.get("chk_byte", 0),
|
||||
)
|
||||
|
||||
|
||||
class WaveformStore:
|
||||
"""
|
||||
Persistent store for Blastware-format waveform files + their A5 source frames.
|
||||
|
||||
Thread safety: write_blastware_file is single-shot; concurrent saves of the
|
||||
*same* filename would race, but the filename encodes second-resolution
|
||||
timestamps + serial, so collisions across threads/processes are vanishingly
|
||||
unlikely in practice.
|
||||
"""
|
||||
|
||||
def __init__(self, root: str | Path) -> None:
|
||||
self.root = Path(root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
log.info("WaveformStore root=%s", self.root)
|
||||
|
||||
# ── path helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
def _serial_dir(self, serial: str) -> Path:
|
||||
d = self.root / serial
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
def paths_for(self, serial: str, filename: str) -> tuple[Path, Path]:
|
||||
"""Return (blastware_path, a5_pickle_path) for a given serial+filename.
|
||||
|
||||
For the sidecar path use `sidecar_path_for()` — kept separate so
|
||||
existing callers don't need to unpack a 3-tuple.
|
||||
"""
|
||||
d = self._serial_dir(serial)
|
||||
return d / filename, d / f"{filename}.a5.pkl"
|
||||
|
||||
def sidecar_path_for(self, serial: str, filename: str) -> Path:
|
||||
"""Return absolute path to the .sfm.json sidecar for a given event."""
|
||||
return self._serial_dir(serial) / f"{filename}.sfm.json"
|
||||
|
||||
def hdf5_path_for(self, serial: str, filename: str) -> Path:
|
||||
"""Return absolute path to the .h5 clean-waveform file for a given event."""
|
||||
return self._serial_dir(serial) / f"{filename}.h5"
|
||||
|
||||
def open_blastware(self, serial: str, filename: str) -> Optional[Path]:
|
||||
"""Return absolute path to an existing event file or None."""
|
||||
bw_path, _ = self.paths_for(serial, filename)
|
||||
return bw_path if bw_path.exists() else None
|
||||
|
||||
# ── save / load ─────────────────────────────────────────────────────────────
|
||||
|
||||
def save(
|
||||
self,
|
||||
ev: Event,
|
||||
serial: str,
|
||||
a5_frames: list[S3Frame],
|
||||
*,
|
||||
source_kind: str = "sfm-live",
|
||||
geo_range = "normal",
|
||||
) -> dict:
|
||||
"""
|
||||
Write all four event-file artifacts for one event:
|
||||
- <filename> BW binary
|
||||
- <filename>.a5.pkl raw A5 frame pickle
|
||||
- <filename>.h5 clean waveform (HDF5)
|
||||
- <filename>.sfm.json modern sidecar (metadata + review)
|
||||
|
||||
Returns a record dict suitable for persisting alongside the DB row:
|
||||
|
||||
{
|
||||
"filename": "M529LKIQ.7M0W",
|
||||
"filesize": 8708,
|
||||
"sha256": "a1b2c3...",
|
||||
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
||||
"hdf5_filename": "M529LKIQ.7M0W.h5",
|
||||
"sidecar_filename": "M529LKIQ.7M0W.sfm.json",
|
||||
}
|
||||
|
||||
`source_kind` flows into `sidecar.source.kind` — callers should
|
||||
pass "sfm-live" (default) for the live endpoint and "sfm-ach" for
|
||||
the ACH ingestion path. BW-imported events use save_imported_bw()
|
||||
instead.
|
||||
|
||||
`geo_range` controls the ADC-counts → in/s scaling in the HDF5
|
||||
file ("normal" = 10 in/s FS, "sensitive" = 1.25 in/s FS).
|
||||
Defaults to "normal" — callers with compliance-config access
|
||||
should pass the actual unit setting so the saved samples are in
|
||||
the right units.
|
||||
|
||||
Idempotent: if the event file already exists, it is overwritten
|
||||
with the freshly-encoded version (same bytes for the same
|
||||
a5_frames) and the sidecar's review block is preserved across
|
||||
re-saves.
|
||||
"""
|
||||
if not a5_frames:
|
||||
raise ValueError("WaveformStore.save: a5_frames is empty")
|
||||
if not serial:
|
||||
raise ValueError("WaveformStore.save: serial is required")
|
||||
|
||||
filename = blastware_filename(ev, serial)
|
||||
bw_path, a5_path = self.paths_for(serial, filename)
|
||||
sidecar_path = self.sidecar_path_for(serial, filename)
|
||||
hdf5_path = self.hdf5_path_for(serial, filename)
|
||||
|
||||
# 1. encode the event file (defensive unlink prevents trailing-byte
|
||||
# leaks from a previous larger file on synced/odd filesystems).
|
||||
try:
|
||||
bw_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
write_blastware_file(ev, a5_frames, bw_path)
|
||||
filesize = bw_path.stat().st_size
|
||||
sha256 = event_file_io.file_sha256(bw_path)
|
||||
|
||||
# 2. write the .a5.pkl sidecar
|
||||
try:
|
||||
a5_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
payload = {
|
||||
"version": A5_PICKLE_VERSION,
|
||||
"frames": [_frame_to_dict(f) for f in a5_frames],
|
||||
}
|
||||
with a5_path.open("wb") as fp:
|
||||
pickle.dump(payload, fp, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
# 3. write the .h5 clean-waveform file (samples in physical units).
|
||||
# Best-effort: a write failure shouldn't sink the rest of the save
|
||||
# (the HDF5 can be regenerated later from the .a5.pkl).
|
||||
hdf5_filename: Optional[str] = None
|
||||
try:
|
||||
event_hdf5.write_event_hdf5(
|
||||
hdf5_path, ev,
|
||||
serial=serial,
|
||||
geo_range=geo_range,
|
||||
source_kind=source_kind,
|
||||
)
|
||||
hdf5_filename = hdf5_path.name
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save: HDF5 write failed for %s: %s — continuing without .h5",
|
||||
hdf5_path, exc,
|
||||
)
|
||||
|
||||
# 4. write the .sfm.json sidecar. Preserve any existing review
|
||||
# block + extensions across re-saves so user edits aren't lost
|
||||
# when the same event is re-downloaded (e.g. via Force refresh).
|
||||
existing_review = None
|
||||
existing_extensions = None
|
||||
if sidecar_path.exists():
|
||||
try:
|
||||
old = event_file_io.read_sidecar(sidecar_path)
|
||||
existing_review = old.get("review")
|
||||
existing_extensions = old.get("extensions")
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save: existing sidecar at %s unreadable (%s); overwriting",
|
||||
sidecar_path, exc,
|
||||
)
|
||||
|
||||
sidecar = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial=serial,
|
||||
blastware_filename=filename,
|
||||
blastware_filesize=filesize,
|
||||
blastware_sha256=sha256,
|
||||
source_kind=source_kind,
|
||||
a5_pickle_filename=a5_path.name,
|
||||
review=existing_review,
|
||||
extensions=existing_extensions,
|
||||
)
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
log.info(
|
||||
"WaveformStore.save serial=%s filename=%s filesize=%d frames=%d "
|
||||
"h5=%s sidecar=%s",
|
||||
serial, filename, filesize, len(a5_frames),
|
||||
hdf5_filename or "(skipped)", sidecar_path.name,
|
||||
)
|
||||
return {
|
||||
"filename": filename,
|
||||
"filesize": filesize,
|
||||
"sha256": sha256,
|
||||
"a5_pickle_filename": a5_path.name,
|
||||
"hdf5_filename": hdf5_filename,
|
||||
"sidecar_filename": sidecar_path.name,
|
||||
}
|
||||
|
||||
def save_imported_bw(
|
||||
self,
|
||||
bw_bytes: bytes,
|
||||
source_path: Path,
|
||||
*,
|
||||
serial_hint: Optional[str] = None,
|
||||
) -> tuple[Event, dict]:
|
||||
"""
|
||||
Ingest a Blastware event file produced by an external tool
|
||||
(Blastware's own ACH, manual download, etc.) where the source A5
|
||||
frames aren't available.
|
||||
|
||||
Workflow:
|
||||
1. Parse the bytes via event_file_io.read_blastware_file (writes
|
||||
a temp file to do that, since the parser takes a path).
|
||||
2. Resolve serial from BW filename (`<P><serial3>...`) or use
|
||||
serial_hint. Falls back to "UNKNOWN".
|
||||
3. Copy the BW bytes verbatim into <root>/<serial>/<filename>.
|
||||
4. Write the .sfm.json sidecar with source.kind = "bw-import"
|
||||
and a5_pickle_filename = None. Does NOT write a .a5.pkl
|
||||
(no A5 source available; byte-for-byte regeneration not
|
||||
possible — the on-disk BW file IS the byte-for-byte source).
|
||||
|
||||
Returns (event, record_dict) so callers can both insert into
|
||||
SeismoDb and surface the parsed Event.
|
||||
"""
|
||||
# Stash the bytes to a temp path so read_blastware_file (path-based)
|
||||
# can parse without us duplicating its logic.
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(suffix=".bw", delete=False) as tmp:
|
||||
tmp.write(bw_bytes)
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
ev = event_file_io.read_blastware_file(tmp_path)
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# Resolve serial. blastware_filename derives a 4-char prefix from
|
||||
# the numeric serial (e.g. BE11529 → M529); we go the other way
|
||||
# via the source filename if a hint wasn't given.
|
||||
serial = serial_hint or _serial_from_bw_filename(source_path.name) or "UNKNOWN"
|
||||
|
||||
# Use the source filename verbatim — it already encodes timestamp
|
||||
# + record type per BW's AB0T scheme, and we want to preserve it
|
||||
# so the file BW knows about can be opened back in BW.
|
||||
filename = source_path.name
|
||||
bw_path = self._serial_dir(serial) / filename
|
||||
|
||||
# 1. copy bytes
|
||||
bw_path.write_bytes(bw_bytes)
|
||||
filesize = bw_path.stat().st_size
|
||||
sha256 = event_file_io.file_sha256(bw_path)
|
||||
|
||||
# 2. write the .h5 clean-waveform file from the parsed Event.
|
||||
# Note: peaks here are computed from raw samples (the BW file
|
||||
# doesn't carry the device-authoritative 0C peaks). Best-effort.
|
||||
hdf5_path = self.hdf5_path_for(serial, filename)
|
||||
hdf5_filename: Optional[str] = None
|
||||
try:
|
||||
event_hdf5.write_event_hdf5(
|
||||
hdf5_path, ev,
|
||||
serial=serial,
|
||||
geo_range="normal", # BW file doesn't carry the range; assume Normal
|
||||
source_kind="bw-import",
|
||||
)
|
||||
hdf5_filename = hdf5_path.name
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save_imported_bw: HDF5 write failed for %s: %s — continuing",
|
||||
hdf5_path, exc,
|
||||
)
|
||||
|
||||
# 3. write sidecar with source.kind = bw-import
|
||||
sidecar_path = self.sidecar_path_for(serial, filename)
|
||||
existing_review = None
|
||||
if sidecar_path.exists():
|
||||
try:
|
||||
existing_review = event_file_io.read_sidecar(sidecar_path).get("review")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sidecar = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial=serial,
|
||||
blastware_filename=filename,
|
||||
blastware_filesize=filesize,
|
||||
blastware_sha256=sha256,
|
||||
source_kind="bw-import",
|
||||
a5_pickle_filename=None,
|
||||
review=existing_review,
|
||||
)
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
log.info(
|
||||
"WaveformStore.save_imported_bw serial=%s filename=%s filesize=%d "
|
||||
"h5=%s (no .a5.pkl — A5 source unavailable for BW-imported files)",
|
||||
serial, filename, filesize, hdf5_filename or "(skipped)",
|
||||
)
|
||||
return ev, {
|
||||
"filename": filename,
|
||||
"filesize": filesize,
|
||||
"sha256": sha256,
|
||||
"a5_pickle_filename": None,
|
||||
"hdf5_filename": hdf5_filename,
|
||||
"sidecar_filename": sidecar_path.name,
|
||||
}
|
||||
|
||||
def load_a5(self, serial: str, filename: str) -> Optional[list[S3Frame]]:
|
||||
"""
|
||||
Re-hydrate the pickled A5 frame stream for a stored event.
|
||||
Returns None if the sidecar is missing.
|
||||
"""
|
||||
_, a5_path = self.paths_for(serial, filename)
|
||||
if not a5_path.exists():
|
||||
return None
|
||||
with a5_path.open("rb") as fp:
|
||||
payload = pickle.load(fp)
|
||||
if not isinstance(payload, dict) or "frames" not in payload:
|
||||
log.warning("WaveformStore.load_a5: malformed sidecar at %s", a5_path)
|
||||
return None
|
||||
return [_dict_to_frame(d) for d in payload["frames"]]
|
||||
|
||||
# ── modern .sfm.json sidecar accessors ──────────────────────────────────────
|
||||
|
||||
def load_sidecar(self, serial: str, filename: str) -> Optional[dict]:
|
||||
"""Return the parsed .sfm.json sidecar dict, or None if missing."""
|
||||
path = self.sidecar_path_for(serial, filename)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return event_file_io.read_sidecar(path)
|
||||
except Exception as exc:
|
||||
log.warning("load_sidecar: failed to read %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
def patch_sidecar(
|
||||
self,
|
||||
serial: str,
|
||||
filename: str,
|
||||
*,
|
||||
review: Optional[dict] = None,
|
||||
extensions: Optional[dict] = None,
|
||||
reviewer_now: bool = True,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
JSON-merge-patch the .sfm.json sidecar's review/extensions blocks.
|
||||
Returns the new full dict, or None if the sidecar doesn't exist.
|
||||
"""
|
||||
path = self.sidecar_path_for(serial, filename)
|
||||
if not path.exists():
|
||||
return None
|
||||
return event_file_io.patch_sidecar(
|
||||
path,
|
||||
review=review,
|
||||
extensions=extensions,
|
||||
reviewer_now=reviewer_now,
|
||||
)
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _serial_from_bw_filename(name: str) -> Optional[str]:
|
||||
"""
|
||||
Reverse of `blastware_filename`'s serial-prefix encoding.
|
||||
|
||||
BW filename format (V10.72): `<P><serial3><stem4>.<ext>`
|
||||
where P = chr(ord('B') + floor(serial // 1000))
|
||||
and serial3 = f"{serial % 1000:03d}".
|
||||
|
||||
Examples (from CLAUDE.md verification archive):
|
||||
P036... → BE14036 H907... → BE6907
|
||||
M529... → BE11529 T003... → BE18003
|
||||
|
||||
Returns the inferred BE-prefix serial (e.g. "BE11529") or None when
|
||||
the filename doesn't match the expected pattern.
|
||||
"""
|
||||
if not name:
|
||||
return None
|
||||
# First letter encodes the thousands group; next 3 chars encode the
|
||||
# last 3 digits of the serial.
|
||||
base = name.split(".", 1)[0]
|
||||
if len(base) < 4 or not base[0].isalpha() or not base[1:4].isdigit():
|
||||
return None
|
||||
prefix_letter = base[0].upper()
|
||||
if prefix_letter < "B":
|
||||
return None
|
||||
thousands = ord(prefix_letter) - ord("B")
|
||||
serial_num = thousands * 1000 + int(base[1:4])
|
||||
return f"BE{serial_num}"
|
||||
@@ -0,0 +1,602 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>SFM Waveform Viewer</title>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.min.js"></script>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
body {
|
||||
background: #0d1117;
|
||||
color: #c9d1d9;
|
||||
font-family: 'Segoe UI', system-ui, sans-serif;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
header {
|
||||
background: #161b22;
|
||||
border-bottom: 1px solid #30363d;
|
||||
padding: 12px 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: #f0f6fc;
|
||||
white-space: nowrap;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
.conn-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
label { color: #8b949e; font-size: 12px; }
|
||||
|
||||
input[type="text"], input[type="number"] {
|
||||
background: #0d1117;
|
||||
border: 1px solid #30363d;
|
||||
border-radius: 6px;
|
||||
color: #c9d1d9;
|
||||
padding: 5px 8px;
|
||||
font-size: 13px;
|
||||
width: 100px;
|
||||
}
|
||||
input[type="number"] { width: 70px; }
|
||||
input:focus { outline: none; border-color: #388bfd; }
|
||||
|
||||
button {
|
||||
background: #1f6feb;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
padding: 5px 14px;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
button:hover { background: #388bfd; }
|
||||
button:active { background: #1158c7; }
|
||||
button:disabled { background: #21262d; color: #484f58; cursor: not-allowed; }
|
||||
|
||||
#status-bar {
|
||||
background: #161b22;
|
||||
border-bottom: 1px solid #21262d;
|
||||
padding: 5px 20px;
|
||||
font-size: 12px;
|
||||
color: #8b949e;
|
||||
min-height: 26px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 20px;
|
||||
}
|
||||
#status-bar.error { color: #f85149; }
|
||||
#status-bar.ok { color: #3fb950; }
|
||||
#status-bar.loading { color: #d29922; }
|
||||
|
||||
.meta-pill {
|
||||
background: #21262d;
|
||||
border-radius: 4px;
|
||||
padding: 2px 8px;
|
||||
color: #c9d1d9;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
#charts {
|
||||
padding: 12px 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.chart-wrap {
|
||||
background: #161b22;
|
||||
border: 1px solid #21262d;
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px 8px;
|
||||
}
|
||||
|
||||
.chart-label {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.chart-canvas-wrap { position: relative; height: 130px; }
|
||||
|
||||
#empty-state {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 60vh;
|
||||
color: #484f58;
|
||||
gap: 8px;
|
||||
}
|
||||
#empty-state svg { opacity: 0.3; }
|
||||
#empty-state p { font-size: 14px; }
|
||||
|
||||
.ch-tran { color: #58a6ff; }
|
||||
.ch-vert { color: #3fb950; }
|
||||
.ch-long { color: #d29922; }
|
||||
.ch-mic { color: #bc8cff; }
|
||||
|
||||
#unit-bar {
|
||||
background: #0d1117;
|
||||
border-bottom: 1px solid #21262d;
|
||||
padding: 8px 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
flex-wrap: wrap;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.unit-field { display: flex; flex-direction: column; gap: 1px; }
|
||||
.unit-field .uf-label { color: #484f58; font-size: 10px; text-transform: uppercase; letter-spacing: 0.05em; }
|
||||
.unit-field .uf-value { color: #c9d1d9; font-family: monospace; font-size: 13px; }
|
||||
.unit-field .uf-value.highlight { color: #58a6ff; font-weight: 600; }
|
||||
|
||||
.event-chips {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
flex-wrap: wrap;
|
||||
margin-left: 8px;
|
||||
}
|
||||
|
||||
.event-chip {
|
||||
background: #21262d;
|
||||
border: 1px solid #30363d;
|
||||
border-radius: 5px;
|
||||
color: #8b949e;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
padding: 3px 10px;
|
||||
transition: all 0.12s;
|
||||
}
|
||||
.event-chip:hover { background: #1f6feb; border-color: #1f6feb; color: #fff; }
|
||||
.event-chip.active { background: #1f6feb; border-color: #388bfd; color: #fff; font-weight: 600; }
|
||||
|
||||
#connect-btn {
|
||||
background: #238636;
|
||||
margin-left: auto;
|
||||
}
|
||||
#connect-btn:hover { background: #2ea043; }
|
||||
#connect-btn:disabled { background: #21262d; color: #484f58; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>SFM Waveform Viewer</h1>
|
||||
<div class="conn-group">
|
||||
<label>API</label>
|
||||
<input type="text" id="api-base" style="width:180px" />
|
||||
</div>
|
||||
<div class="conn-group">
|
||||
<label>Device host</label>
|
||||
<input type="text" id="dev-host" value="" placeholder="e.g. 10.0.0.5" />
|
||||
<label>TCP port</label>
|
||||
<input type="number" id="dev-tcp-port" value="9034" />
|
||||
</div>
|
||||
<button id="connect-btn" onclick="connectUnit()">Connect</button>
|
||||
<button id="load-btn" onclick="loadWaveform()" disabled>Load Waveform</button>
|
||||
<button id="prev-btn" onclick="stepEvent(-1)" disabled>◀ Prev</button>
|
||||
<button id="next-btn" onclick="stepEvent(+1)" disabled>Next ▶</button>
|
||||
</header>
|
||||
|
||||
<!-- Unit info bar — hidden until connected -->
|
||||
<div id="unit-bar" style="display:none">
|
||||
<div class="unit-field">
|
||||
<span class="uf-label">Serial</span>
|
||||
<span class="uf-value" id="u-serial">—</span>
|
||||
</div>
|
||||
<div class="unit-field">
|
||||
<span class="uf-label">Firmware</span>
|
||||
<span class="uf-value" id="u-fw">—</span>
|
||||
</div>
|
||||
<div class="unit-field">
|
||||
<span class="uf-label">Sample rate</span>
|
||||
<span class="uf-value" id="u-sr">—</span>
|
||||
</div>
|
||||
<div class="unit-field">
|
||||
<span class="uf-label">Events</span>
|
||||
<span class="uf-value highlight" id="u-count">—</span>
|
||||
</div>
|
||||
<div class="event-chips" id="event-chips"></div>
|
||||
</div>
|
||||
|
||||
<div id="status-bar">Ready — enter device host and click Connect.</div>
|
||||
|
||||
<div id="empty-state">
|
||||
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5">
|
||||
<polyline points="22 12 18 12 15 21 9 3 6 12 2 12"/>
|
||||
</svg>
|
||||
<p>No waveform loaded</p>
|
||||
</div>
|
||||
|
||||
<div id="charts" style="display:none"></div>
|
||||
|
||||
<script>
|
||||
const CHANNEL_COLORS = {
|
||||
Tran: '#58a6ff',
|
||||
Vert: '#3fb950',
|
||||
Long: '#d29922',
|
||||
Mic: '#bc8cff',
|
||||
};
|
||||
|
||||
let charts = {};
|
||||
let lastData = null;
|
||||
let unitInfo = null;
|
||||
let geoAdcScale = 10.0; // in/s full-scale for geo channels; updated on connect
|
||||
let eventList = []; // populated from /device/events after connect
|
||||
let currentEventIndex = 0;
|
||||
|
||||
function setStatus(msg, cls = '') {
|
||||
const bar = document.getElementById('status-bar');
|
||||
bar.textContent = msg;
|
||||
bar.className = cls;
|
||||
}
|
||||
|
||||
function appendMeta(label, value) {
|
||||
const bar = document.getElementById('status-bar');
|
||||
const pill = document.createElement('span');
|
||||
pill.className = 'meta-pill';
|
||||
pill.textContent = `${label}: ${value}`;
|
||||
bar.appendChild(pill);
|
||||
}
|
||||
|
||||
async function connectUnit() {
|
||||
const apiBase = document.getElementById('api-base').value.replace(/\/$/, '');
|
||||
const devHost = document.getElementById('dev-host').value.trim();
|
||||
const tcpPort = document.getElementById('dev-tcp-port').value;
|
||||
|
||||
if (!devHost) { setStatus('Enter a device host first.', 'error'); return; }
|
||||
|
||||
const btn = document.getElementById('connect-btn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Connecting…';
|
||||
setStatus('Connecting to unit…', 'loading');
|
||||
|
||||
const url = `${apiBase}/device/info?host=${encodeURIComponent(devHost)}&tcp_port=${tcpPort}`;
|
||||
try {
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json().catch(() => ({ detail: resp.statusText }));
|
||||
throw new Error(err.detail || resp.statusText);
|
||||
}
|
||||
unitInfo = await resp.json();
|
||||
geoAdcScale = unitInfo.compliance_config?.geo_adc_scale ?? 10.0;
|
||||
} catch (e) {
|
||||
setStatus(`Error: ${e.message}`, 'error');
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Connect';
|
||||
return;
|
||||
}
|
||||
|
||||
// Populate unit bar from /device/info
|
||||
document.getElementById('u-serial').textContent = unitInfo.serial || '—';
|
||||
document.getElementById('u-fw').textContent = unitInfo.firmware_version || '—';
|
||||
const sr = unitInfo.compliance_config?.sample_rate;
|
||||
document.getElementById('u-sr').textContent = sr ? `${sr} sps` : '—';
|
||||
|
||||
// Fetch real event list from /device/events — SUB 08 count is unreliable
|
||||
setStatus('Fetching event list…', 'loading');
|
||||
const eventsUrl = `${apiBase}/device/events?host=${encodeURIComponent(devHost)}&tcp_port=${tcpPort}`;
|
||||
try {
|
||||
const evResp = await fetch(eventsUrl);
|
||||
if (!evResp.ok) {
|
||||
const err = await evResp.json().catch(() => ({ detail: evResp.statusText }));
|
||||
throw new Error(err.detail || evResp.statusText);
|
||||
}
|
||||
const evData = await evResp.json();
|
||||
eventList = evData.events || [];
|
||||
} catch (e) {
|
||||
setStatus(`Error fetching events: ${e.message}`, 'error');
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Reconnect';
|
||||
return;
|
||||
}
|
||||
|
||||
const count = eventList.length;
|
||||
document.getElementById('u-count').textContent = count;
|
||||
|
||||
// Build event chips with timestamps
|
||||
const chipsEl = document.getElementById('event-chips');
|
||||
chipsEl.innerHTML = '';
|
||||
eventList.forEach((ev, i) => {
|
||||
const chip = document.createElement('button');
|
||||
chip.className = 'event-chip' + (i === 0 ? ' active' : '');
|
||||
const label = ev.timestamp?.display ?? `Event ${ev.index}`;
|
||||
chip.textContent = label;
|
||||
chip.title = ev.record_type || '';
|
||||
chip.onclick = () => selectEvent(i);
|
||||
chipsEl.appendChild(chip);
|
||||
});
|
||||
|
||||
document.getElementById('unit-bar').style.display = 'flex';
|
||||
document.getElementById('load-btn').disabled = count === 0;
|
||||
document.getElementById('prev-btn').disabled = true;
|
||||
document.getElementById('next-btn').disabled = count <= 1;
|
||||
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Reconnect';
|
||||
|
||||
if (count === 0) {
|
||||
setStatus('Connected — no events stored on device.', 'ok');
|
||||
} else {
|
||||
setStatus(`Connected — ${count} event${count !== 1 ? 's' : ''} stored. Select an event or click Load Waveform.`, 'ok');
|
||||
}
|
||||
}
|
||||
|
||||
function selectEvent(idx) {
|
||||
currentEventIndex = idx;
|
||||
// Update chip highlight
|
||||
document.querySelectorAll('.event-chip').forEach((c, i) => {
|
||||
c.classList.toggle('active', i === idx);
|
||||
});
|
||||
document.getElementById('prev-btn').disabled = idx <= 0;
|
||||
document.getElementById('next-btn').disabled = idx >= eventList.length - 1;
|
||||
loadWaveform();
|
||||
}
|
||||
|
||||
async function loadWaveform() {
|
||||
const apiBase = document.getElementById('api-base').value.replace(/\/$/, '');
|
||||
const devHost = document.getElementById('dev-host').value.trim();
|
||||
const tcpPort = document.getElementById('dev-tcp-port').value;
|
||||
const evIndex = currentEventIndex;
|
||||
|
||||
if (!devHost) { setStatus('Enter a device host first.', 'error'); return; }
|
||||
|
||||
const btn = document.getElementById('load-btn');
|
||||
btn.disabled = true;
|
||||
setStatus('Fetching waveform…', 'loading');
|
||||
|
||||
const url = `${apiBase}/device/event/${evIndex}/waveform?host=${encodeURIComponent(devHost)}&tcp_port=${tcpPort}`;
|
||||
|
||||
let data;
|
||||
try {
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json().catch(() => ({ detail: resp.statusText }));
|
||||
throw new Error(err.detail || resp.statusText);
|
||||
}
|
||||
data = await resp.json();
|
||||
} catch (e) {
|
||||
setStatus(`Error: ${e.message}`, 'error');
|
||||
btn.disabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
lastData = data;
|
||||
renderWaveform(data);
|
||||
btn.disabled = false;
|
||||
}
|
||||
|
||||
function stepEvent(delta) {
|
||||
const count = eventList.length;
|
||||
const next = Math.max(0, Math.min(count - 1, currentEventIndex + delta));
|
||||
selectEvent(next);
|
||||
}
|
||||
|
||||
function renderWaveform(data) {
|
||||
const sr = data.sample_rate || 1024;
|
||||
const pretrig = data.pretrig_samples || 0;
|
||||
const decoded = data.samples_decoded || 0;
|
||||
const total = data.total_samples || decoded;
|
||||
const channels = data.channels || {};
|
||||
const recType = data.record_type || 'Unknown';
|
||||
|
||||
// Status bar
|
||||
const bar = document.getElementById('status-bar');
|
||||
bar.innerHTML = '';
|
||||
bar.className = 'ok';
|
||||
const ts = data.timestamp;
|
||||
if (ts) {
|
||||
bar.textContent = `Event #${data.index} — ${ts.display} `;
|
||||
} else {
|
||||
bar.textContent = `Event #${data.index} `;
|
||||
}
|
||||
appendMeta('type', recType);
|
||||
appendMeta('sr', `${sr} sps`);
|
||||
appendMeta('samples', `${decoded.toLocaleString()} / ${total.toLocaleString()}`);
|
||||
appendMeta('pretrig', pretrig);
|
||||
appendMeta('rectime', `${data.rectime_seconds ?? '?'}s`);
|
||||
|
||||
// No waveform data — show a clear reason instead of empty charts
|
||||
if (decoded === 0) {
|
||||
document.getElementById('empty-state').style.display = 'flex';
|
||||
document.getElementById('empty-state').querySelector('p').textContent =
|
||||
recType === 'Waveform'
|
||||
? 'Waveform decode returned no samples — check server logs'
|
||||
: `Record type "${recType}" — waveform decode not yet supported for this mode`;
|
||||
document.getElementById('charts').style.display = 'none';
|
||||
Object.values(charts).forEach(c => c.destroy());
|
||||
charts = {};
|
||||
return;
|
||||
}
|
||||
|
||||
// Build time axis (ms)
|
||||
const times = Array.from({ length: decoded }, (_, i) =>
|
||||
((i - pretrig) / sr * 1000).toFixed(2)
|
||||
);
|
||||
|
||||
// Show charts area
|
||||
document.getElementById('empty-state').style.display = 'none';
|
||||
const chartsDiv = document.getElementById('charts');
|
||||
chartsDiv.style.display = 'flex';
|
||||
chartsDiv.innerHTML = '';
|
||||
|
||||
// Destroy old Chart instances
|
||||
Object.values(charts).forEach(c => c.destroy());
|
||||
charts = {};
|
||||
|
||||
// Mic peak PSI from 0C waveform record — used to scale raw mic counts
|
||||
const micPeakPsi = data.peak_values?.micl_psi ?? null;
|
||||
const DBL_REF_PSI = 2.9e-9; // 20 µPa in psi
|
||||
|
||||
for (const [ch, color] of Object.entries(CHANNEL_COLORS)) {
|
||||
const samples = channels[ch];
|
||||
if (!samples || samples.length === 0) continue;
|
||||
|
||||
// Convert raw ADC counts to physical units
|
||||
const isGeo = ch !== 'Mic';
|
||||
let plotSamples, peakLabel, yUnit, tooltipFmt, tickFmt;
|
||||
|
||||
if (isGeo) {
|
||||
// Geo channels: counts × (range / 32767) → in/s
|
||||
const scale = geoAdcScale / 32767;
|
||||
plotSamples = samples.map(c => c * scale);
|
||||
const peakIns = Math.max(...plotSamples.map(Math.abs));
|
||||
peakLabel = `${peakIns.toFixed(5)} in/s`;
|
||||
yUnit = 'in/s';
|
||||
tooltipFmt = v => `${ch}: ${v.toFixed(5)} in/s`;
|
||||
tickFmt = v => v.toFixed(4);
|
||||
} else {
|
||||
// Mic: derive psi/count scale from the 0C peak value, display as psi; show dBL in header
|
||||
const peakCounts = Math.max(...samples.map(Math.abs));
|
||||
const micScale = (micPeakPsi !== null && peakCounts > 0)
|
||||
? Math.abs(micPeakPsi) / peakCounts
|
||||
: 1.0;
|
||||
plotSamples = samples.map(c => c * micScale);
|
||||
const peakPsi = Math.max(...plotSamples.map(Math.abs));
|
||||
const peakDbl = peakPsi > 0 ? 20 * Math.log10(peakPsi / DBL_REF_PSI) : -Infinity;
|
||||
peakLabel = `${peakDbl.toFixed(1)} dBL (${peakPsi.toExponential(3)} psi)`;
|
||||
yUnit = 'psi';
|
||||
tooltipFmt = v => `${ch}: ${v.toExponential(3)} psi`;
|
||||
tickFmt = v => v.toExponential(1);
|
||||
}
|
||||
|
||||
const wrap = document.createElement('div');
|
||||
wrap.className = 'chart-wrap';
|
||||
|
||||
const lbl = document.createElement('div');
|
||||
lbl.className = `chart-label ch-${ch.toLowerCase()}`;
|
||||
lbl.textContent = `${ch} — peak ${peakLabel}`;
|
||||
wrap.appendChild(lbl);
|
||||
|
||||
const canvasWrap = document.createElement('div');
|
||||
canvasWrap.className = 'chart-canvas-wrap';
|
||||
const canvas = document.createElement('canvas');
|
||||
canvasWrap.appendChild(canvas);
|
||||
wrap.appendChild(canvasWrap);
|
||||
chartsDiv.appendChild(wrap);
|
||||
|
||||
// Downsample for rendering if very long (keep chart responsive)
|
||||
const MAX_POINTS = 4000;
|
||||
let renderTimes = times;
|
||||
let renderData = plotSamples;
|
||||
if (plotSamples.length > MAX_POINTS) {
|
||||
const step = Math.ceil(plotSamples.length / MAX_POINTS);
|
||||
renderTimes = times.filter((_, i) => i % step === 0);
|
||||
renderData = plotSamples.filter((_, i) => i % step === 0);
|
||||
}
|
||||
|
||||
const chart = new Chart(canvas, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: renderTimes,
|
||||
datasets: [{
|
||||
data: renderData,
|
||||
borderColor: color,
|
||||
borderWidth: 1,
|
||||
pointRadius: 0,
|
||||
tension: 0,
|
||||
}],
|
||||
},
|
||||
options: {
|
||||
animation: false,
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: { display: false },
|
||||
tooltip: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
callbacks: {
|
||||
title: items => `t = ${items[0].label} ms`,
|
||||
label: item => tooltipFmt(item.raw),
|
||||
},
|
||||
},
|
||||
},
|
||||
scales: {
|
||||
x: {
|
||||
type: 'category',
|
||||
ticks: {
|
||||
color: '#484f58',
|
||||
maxTicksLimit: 10,
|
||||
maxRotation: 0,
|
||||
callback: (val, i) => renderTimes[i] + ' ms',
|
||||
},
|
||||
grid: { color: '#21262d' },
|
||||
},
|
||||
y: {
|
||||
ticks: {
|
||||
color: '#484f58',
|
||||
maxTicksLimit: 5,
|
||||
callback: v => tickFmt(v),
|
||||
},
|
||||
grid: { color: '#21262d' },
|
||||
title: {
|
||||
display: true,
|
||||
text: yUnit,
|
||||
color: '#484f58',
|
||||
font: { size: 10 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [{
|
||||
// Draw trigger line at t=0
|
||||
id: 'triggerLine',
|
||||
afterDraw(chart) {
|
||||
const ctx = chart.ctx;
|
||||
const xAxis = chart.scales.x;
|
||||
const yAxis = chart.scales.y;
|
||||
|
||||
// Find index of t=0
|
||||
const zeroIdx = renderTimes.findIndex(t => parseFloat(t) >= 0);
|
||||
if (zeroIdx < 0) return;
|
||||
|
||||
const x = xAxis.getPixelForValue(zeroIdx);
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x, yAxis.top);
|
||||
ctx.lineTo(x, yAxis.bottom);
|
||||
ctx.strokeStyle = 'rgba(248, 81, 73, 0.7)';
|
||||
ctx.lineWidth = 1.5;
|
||||
ctx.setLineDash([4, 3]);
|
||||
ctx.stroke();
|
||||
ctx.restore();
|
||||
},
|
||||
}],
|
||||
});
|
||||
|
||||
charts[ch] = chart;
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-detect API base from wherever this page was served from
|
||||
document.getElementById('api-base').value = window.location.origin;
|
||||
|
||||
// Allow Enter key on connection inputs to trigger connect
|
||||
['api-base', 'dev-host', 'dev-tcp-port'].forEach(id => {
|
||||
document.getElementById(id).addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter') connectUnit();
|
||||
});
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
test_5a_protocol.py — Regression test for the v0.14.x SUB 5A protocol fixes.
|
||||
|
||||
Verifies that SFM's framing helpers reproduce Blastware's exact wire bytes
|
||||
for every 5A request frame in the 5-1-26 "bwcap3sec" capture, AND that the
|
||||
file builder produces a byte-identical file when fed the BW capture's A5
|
||||
responses.
|
||||
|
||||
Together these two tests protect all four v0.14.x fixes:
|
||||
|
||||
v0.14.0 — STRT-bounded chunk walk (probe @ 0, metadata pages @ 0x1002 +
|
||||
0x1004, samples @ 0x0600..0x1E00 step 0x0200, TERM at residual)
|
||||
v0.14.1 — event-N probe counter is `start_offset`, not `start_offset+0x46`
|
||||
(covered by the multi-event captures, not this 3-sec event-1
|
||||
capture — but the helpers are the same code path)
|
||||
v0.14.2 — file body assembly is contiguous concatenation, no de-duplication
|
||||
v0.14.3 — partial DLE stuffing of `0x10` bytes in 5A params (counter=0x1000
|
||||
wire bytes are `10 10 00`, not `10 00`)
|
||||
|
||||
If any of these fixes regresses, this test fails immediately with a clear
|
||||
byte-level diff.
|
||||
|
||||
Run:
|
||||
python -m pytest tests/test_5a_protocol.py -v
|
||||
or:
|
||||
python tests/test_5a_protocol.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
# Allow running from the project root without installation
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from minimateplus.framing import (
|
||||
S3FrameParser,
|
||||
build_5a_frame,
|
||||
bulk_waveform_params,
|
||||
bulk_waveform_term_v2,
|
||||
)
|
||||
|
||||
|
||||
# ── Capture loading ────────────────────────────────────────────────────────────
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# Reference BW MITM capture: BW saving a 3-sec event 0 (start_key=01110000,
|
||||
# end_offset=0x21F2). 17 5A frames: probe + 2 metadata pages + 13 samples + TERM.
|
||||
BW_TX_PATH = os.path.join(
|
||||
ROOT,
|
||||
"bridges/captures/5-1-26/comcheck/bwcap3sec/"
|
||||
"raw_bw_20260501_165723_copy_3sec_waveform_to_disk.bin",
|
||||
)
|
||||
BW_S3_PATH = os.path.join(
|
||||
ROOT,
|
||||
"bridges/captures/5-1-26/comcheck/bwcap3sec/"
|
||||
"raw_s3_20260501_165723_copy_3sec_waveform_to_disk.bin",
|
||||
)
|
||||
|
||||
# BW's saved Blastware file for the same event (used for file-builder verification).
|
||||
BW_SAVED_FILE = os.path.join(
|
||||
ROOT, "example-events/decode_test/5-1-26/bw/M529LKIQ.G10",
|
||||
)
|
||||
|
||||
|
||||
def _split_bw_frames(data: bytes) -> list[bytes]:
|
||||
"""Split BW TX bytes into individual frames (ACK STX … bare ETX)."""
|
||||
frames: list[bytes] = []
|
||||
i = 0
|
||||
while i < len(data):
|
||||
if data[i] != 0x41 or i + 1 >= len(data) or data[i + 1] != 0x02:
|
||||
i += 1
|
||||
continue
|
||||
j = i + 2
|
||||
while j < len(data):
|
||||
if data[j] == 0x03:
|
||||
break
|
||||
if data[j] == 0x10 and j + 1 < len(data):
|
||||
j += 2
|
||||
continue
|
||||
j += 1
|
||||
if j >= len(data):
|
||||
break
|
||||
frames.append(data[i : j + 1])
|
||||
i = j + 1
|
||||
return frames
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bw_5a_frames() -> list[bytes]:
|
||||
"""All 5A frames from the BW TX capture, in wire order."""
|
||||
if not os.path.exists(BW_TX_PATH):
|
||||
pytest.skip(f"BW capture not found: {BW_TX_PATH}")
|
||||
raw = open(BW_TX_PATH, "rb").read()
|
||||
frames = [
|
||||
f for f in _split_bw_frames(raw)
|
||||
if len(f) >= 6 and f[5] == 0x5A # body[3] == 0x5A (SUB)
|
||||
]
|
||||
assert len(frames) == 17, f"expected 17 5A frames in capture, got {len(frames)}"
|
||||
return frames
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bw_a5_frames():
|
||||
"""All A5 (response) frames from the matching S3 capture."""
|
||||
if not os.path.exists(BW_S3_PATH):
|
||||
pytest.skip(f"BW S3 capture not found: {BW_S3_PATH}")
|
||||
raw = open(BW_S3_PATH, "rb").read()
|
||||
p = S3FrameParser()
|
||||
p.feed(raw)
|
||||
a5 = [f for f in p.frames if f.sub == 0xA5]
|
||||
assert len(a5) == 17, f"expected 17 A5 frames in capture, got {len(a5)}"
|
||||
return a5
|
||||
|
||||
|
||||
# ── 5A request frame byte-perfect verification ────────────────────────────────
|
||||
|
||||
KEY4 = bytes.fromhex("01110000") # start_key for the 3-sec event 0
|
||||
END_OFFSET = 0x21F2 # parsed from STRT in the BW capture
|
||||
LAST_CHUNK_COUNTER = 0x1E00 # last full 0x0200-byte chunk before TERM
|
||||
|
||||
SAMPLE_COUNTERS = (
|
||||
0x0600, 0x0800, 0x0A00, 0x0C00, 0x0E00,
|
||||
0x1000, 0x1200, 0x1400, 0x1600, 0x1800,
|
||||
0x1A00, 0x1C00, 0x1E00,
|
||||
)
|
||||
|
||||
|
||||
def _meta_params(key: bytes, counter: int) -> bytes:
|
||||
"""Build the 12-byte metadata-page params block (matches BW for 0x1002 / 0x1004)."""
|
||||
return bytes(
|
||||
[
|
||||
0x00, key[0], key[1],
|
||||
(counter >> 8) & 0xFF, counter & 0xFF,
|
||||
0, 0, 0, 0, 0, 0, 0,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_probe_frame_byte_perfect(bw_5a_frames):
|
||||
"""Probe @ counter=0x0000 (frame 0)."""
|
||||
sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, 0, is_probe=True))
|
||||
assert sfm == bw_5a_frames[0], (
|
||||
f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[0].hex()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("idx,counter", [(1, 0x1002), (2, 0x1004)])
|
||||
def test_metadata_page_frames_byte_perfect(bw_5a_frames, idx, counter):
|
||||
"""Metadata pages @ counter=0x1002 and 0x1004 (frames 1 and 2)."""
|
||||
sfm = build_5a_frame(0x1002, _meta_params(KEY4, counter))
|
||||
assert sfm == bw_5a_frames[idx], (
|
||||
f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[idx].hex()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("i,counter", list(enumerate(SAMPLE_COUNTERS)))
|
||||
def test_sample_chunk_frames_byte_perfect(bw_5a_frames, i, counter):
|
||||
"""
|
||||
Sample chunks @ counter=0x0600..0x1E00, step 0x0200 (frames 3..15).
|
||||
|
||||
Critically, frame 8 (counter=0x1000) requires the v0.14.3 partial DLE
|
||||
stuffing fix — wire params include `10 10 00` for the counter, not `10 00`.
|
||||
"""
|
||||
sfm = build_5a_frame(0x1002, bulk_waveform_params(KEY4, counter))
|
||||
bw_idx = 3 + i
|
||||
assert sfm == bw_5a_frames[bw_idx], (
|
||||
f"\ncounter=0x{counter:04X}"
|
||||
f"\nSFM: {sfm.hex()}"
|
||||
f"\nBW: {bw_5a_frames[bw_idx].hex()}"
|
||||
)
|
||||
|
||||
|
||||
def test_term_frame_byte_perfect(bw_5a_frames):
|
||||
"""TERM frame at residual (frame 16)."""
|
||||
offset_word, params = bulk_waveform_term_v2(KEY4, END_OFFSET, LAST_CHUNK_COUNTER)
|
||||
sfm = build_5a_frame(offset_word, params)
|
||||
assert sfm == bw_5a_frames[16], (
|
||||
f"\nSFM: {sfm.hex()}\nBW: {bw_5a_frames[16].hex()}"
|
||||
)
|
||||
|
||||
|
||||
def test_strt_end_offset_parsing(bw_a5_frames):
|
||||
"""The probe response (A5[0]) carries STRT at byte 17 with end_offset=0x21F2."""
|
||||
from minimateplus.framing import parse_strt_end_offset
|
||||
|
||||
end_offset = parse_strt_end_offset(bw_a5_frames[0].data)
|
||||
assert end_offset == END_OFFSET, (
|
||||
f"expected end_offset=0x{END_OFFSET:04X}, got "
|
||||
f"{f'0x{end_offset:04X}' if end_offset is not None else 'None'}"
|
||||
)
|
||||
|
||||
|
||||
# ── File builder byte-perfect verification ────────────────────────────────────
|
||||
|
||||
def test_blastware_file_builder_byte_perfect(bw_a5_frames):
|
||||
"""
|
||||
Feed the BW capture's A5 frames into write_blastware_file() and verify the
|
||||
output is byte-identical to BW's saved M529LKIQ.G10 reference file.
|
||||
|
||||
This protects the v0.14.2 strip-removal fix and the file-builder skip
|
||||
values (probe=38, meta=13, samples=12, TERM=11).
|
||||
"""
|
||||
if not os.path.exists(BW_SAVED_FILE):
|
||||
pytest.skip(f"BW saved file not found: {BW_SAVED_FILE}")
|
||||
|
||||
import tempfile
|
||||
|
||||
from minimateplus.blastware_file import write_blastware_file
|
||||
from minimateplus.models import Event
|
||||
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = KEY4
|
||||
ev.rectime_seconds = 3
|
||||
ev.timestamp = None # let the builder pull the footer from the TERM frame
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".G10", delete=False) as tf:
|
||||
tmp_path = tf.name
|
||||
try:
|
||||
write_blastware_file(ev, bw_a5_frames, tmp_path)
|
||||
sfm_bytes = open(tmp_path, "rb").read()
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
bw_bytes = open(BW_SAVED_FILE, "rb").read()
|
||||
|
||||
assert len(sfm_bytes) == len(bw_bytes), (
|
||||
f"file size mismatch: SFM={len(sfm_bytes)} BW={len(bw_bytes)}"
|
||||
)
|
||||
|
||||
if sfm_bytes != bw_bytes:
|
||||
# Find first diff for actionable error message
|
||||
for i in range(len(bw_bytes)):
|
||||
if bw_bytes[i] != sfm_bytes[i]:
|
||||
ctx_start = max(0, i - 8)
|
||||
ctx_end = min(len(bw_bytes), i + 16)
|
||||
pytest.fail(
|
||||
f"file diverges at byte 0x{i:04X}\n"
|
||||
f" BW : {bw_bytes[ctx_start:ctx_end].hex()}\n"
|
||||
f" SFM: {sfm_bytes[ctx_start:ctx_end].hex()}\n"
|
||||
f" {' ' * (i - ctx_start)}^^"
|
||||
)
|
||||
|
||||
|
||||
# ── Standalone runner ─────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
test_cache_invalidation.py — verify post-erase key-reuse correctness.
|
||||
|
||||
The device's event-key counter resets to 0x01110000 after every memory erase,
|
||||
so a bare-key dedup (the old behaviour) silently treats a freshly-recorded
|
||||
event 0 as if it were the previously-downloaded one. These tests exercise
|
||||
the (key, timestamp)-based eviction logic in:
|
||||
|
||||
- bridges/ach_server.py (state-file migration + force flag)
|
||||
- sfm/server.py (_LiveCache.set_events / set_waveform)
|
||||
|
||||
Run:
|
||||
python tests/test_cache_invalidation.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import pytest
|
||||
except ImportError:
|
||||
pytest = None # type: ignore
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
# ── ACH state migration ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_ach_state_legacy_migration(tmp_path: Path):
|
||||
"""
|
||||
Legacy v1 state with a `downloaded_keys` list is migrated on _load_state
|
||||
to the v2 `downloaded_events` dict. All legacy keys come back with empty
|
||||
timestamps so the (key, ts) compare in get_events() always falls through
|
||||
to a fresh download.
|
||||
"""
|
||||
from bridges.ach_server import _load_state
|
||||
|
||||
state_path = tmp_path / "ach_state.json"
|
||||
legacy = {
|
||||
"BE11529": {
|
||||
"downloaded_keys": ["01110000", "0111245a"],
|
||||
"max_downloaded_key": "0111245a",
|
||||
"last_seen": "2026-04-11T01:04:36",
|
||||
"serial": "BE11529",
|
||||
"peer": "63.43.212.232:51920",
|
||||
},
|
||||
}
|
||||
state_path.write_text(json.dumps(legacy))
|
||||
|
||||
migrated = _load_state(state_path)
|
||||
|
||||
unit = migrated["BE11529"]
|
||||
assert "downloaded_keys" not in unit
|
||||
assert unit["downloaded_events"] == {
|
||||
"01110000": "",
|
||||
"0111245a": "",
|
||||
}
|
||||
# max_downloaded_key is preserved verbatim
|
||||
assert unit["max_downloaded_key"] == "0111245a"
|
||||
|
||||
|
||||
def test_ach_state_v2_passes_through(tmp_path: Path):
|
||||
"""A v2 state file is returned verbatim — no migration touches it."""
|
||||
from bridges.ach_server import _load_state
|
||||
|
||||
state_path = tmp_path / "ach_state.json"
|
||||
v2 = {
|
||||
"BE11529": {
|
||||
"downloaded_events": {
|
||||
"01110000": "2026-04-15T14:23:45",
|
||||
"0111245a": "2026-04-16T09:01:12",
|
||||
},
|
||||
"max_downloaded_key": "0111245a",
|
||||
"serial": "BE11529",
|
||||
},
|
||||
}
|
||||
state_path.write_text(json.dumps(v2))
|
||||
|
||||
loaded = _load_state(state_path)
|
||||
assert loaded["BE11529"]["downloaded_events"] == v2["BE11529"]["downloaded_events"]
|
||||
|
||||
|
||||
def test_ach_state_missing_returns_empty(tmp_path: Path):
|
||||
"""Nonexistent state path → empty dict (not an error)."""
|
||||
from bridges.ach_server import _load_state
|
||||
assert _load_state(tmp_path / "absent.json") == {}
|
||||
|
||||
|
||||
# ── _LiveCache eviction ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _ev(index: int, key: str, ts: str) -> dict:
|
||||
return {"index": index, "waveform_key": key, "timestamp": ts}
|
||||
|
||||
|
||||
def test_live_cache_set_events_no_eviction_when_keys_match():
|
||||
"""No flush when incoming events match the cached (key, ts) at each index."""
|
||||
from sfm.live_cache import LiveCache as _LiveCache
|
||||
|
||||
c = _LiveCache()
|
||||
conn = "tcp:1.2.3.4:12345"
|
||||
c.set_events(conn, 2, [_ev(0, "01110000", "2026-04-15T14:23:45"),
|
||||
_ev(1, "0111245a", "2026-04-16T09:01:12")])
|
||||
c.set_waveform(conn, 0, _ev(0, "01110000", "2026-04-15T14:23:45"))
|
||||
|
||||
# Same events again — must not flush.
|
||||
c.set_events(conn, 2, [_ev(0, "01110000", "2026-04-15T14:23:45"),
|
||||
_ev(1, "0111245a", "2026-04-16T09:01:12")])
|
||||
|
||||
assert c._events[conn][0] == 2
|
||||
assert (conn, 0) in c._waveforms
|
||||
|
||||
|
||||
def test_live_cache_set_events_flushes_on_post_erase_collision():
|
||||
"""
|
||||
Index 0 keeps the same key (01110000 reuses) but the timestamp differs
|
||||
→ device was erased + re-recorded → flush all events + waveforms for the
|
||||
device.
|
||||
"""
|
||||
from sfm.live_cache import LiveCache as _LiveCache
|
||||
|
||||
c = _LiveCache()
|
||||
conn = "tcp:1.2.3.4:12345"
|
||||
# First "session": index 0 key=01110000 ts=2026-04-15.
|
||||
c.set_events(conn, 1, [_ev(0, "01110000", "2026-04-15T14:23:45")])
|
||||
c.set_waveform(conn, 0, _ev(0, "01110000", "2026-04-15T14:23:45"))
|
||||
assert (conn, 0) in c._waveforms
|
||||
|
||||
# Second "session" after erase: index 0 still key=01110000 but new ts.
|
||||
c.set_events(conn, 1, [_ev(0, "01110000", "2026-05-06T12:34:56")])
|
||||
|
||||
# Stale waveform for index 0 must have been flushed by the eviction path
|
||||
# before the new event was inserted. The new events list IS in cache but
|
||||
# the cached waveform from the prior session is gone.
|
||||
assert (conn, 0) not in c._waveforms
|
||||
assert c._events[conn][1][0]["timestamp"] == "2026-05-06T12:34:56"
|
||||
|
||||
|
||||
def test_live_cache_set_waveform_flushes_on_mismatch():
|
||||
"""set_waveform alone should also evict when (key, ts) differs."""
|
||||
from sfm.live_cache import LiveCache as _LiveCache
|
||||
|
||||
c = _LiveCache()
|
||||
conn = "tcp:1.2.3.4:12345"
|
||||
c.set_waveform(conn, 0, _ev(0, "01110000", "2026-04-15T14:23:45"))
|
||||
c.set_waveform(conn, 1, _ev(1, "0111245a", "2026-04-16T09:01:12"))
|
||||
|
||||
# Index 0 swap: same key, new timestamp.
|
||||
c.set_waveform(conn, 0, _ev(0, "01110000", "2026-05-06T12:34:56"))
|
||||
|
||||
# Index 1's stale waveform must be flushed — keeping it would mix eras.
|
||||
assert (conn, 1) not in c._waveforms
|
||||
# The newly-inserted index 0 entry is what's there.
|
||||
assert c._waveforms[(conn, 0)]["timestamp"] == "2026-05-06T12:34:56"
|
||||
|
||||
|
||||
def test_live_cache_partial_signature_does_not_flush():
|
||||
"""
|
||||
If incoming event lacks waveform_key OR timestamp, we cannot prove a
|
||||
mismatch — eviction must NOT trigger. Avoids spurious flushes from
|
||||
legacy / partial event shapes.
|
||||
"""
|
||||
from sfm.live_cache import LiveCache as _LiveCache
|
||||
|
||||
c = _LiveCache()
|
||||
conn = "tcp:1.2.3.4:12345"
|
||||
c.set_waveform(conn, 0, _ev(0, "01110000", "2026-04-15T14:23:45"))
|
||||
|
||||
# Incoming entry missing the timestamp — cannot prove a mismatch.
|
||||
c.set_waveform(conn, 0, {"index": 0, "waveform_key": "01110000"})
|
||||
|
||||
# Cache should contain the new entry; the implementation overwrites
|
||||
# the index-0 row but does NOT flush other indices. Since there are no
|
||||
# other indices in this test, just check the entry exists.
|
||||
assert (conn, 0) in c._waveforms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if pytest is not None:
|
||||
pytest.main([__file__, "-v"])
|
||||
else:
|
||||
import inspect
|
||||
import traceback as _tb
|
||||
|
||||
passed = failed = 0
|
||||
for _name, _fn in sorted(globals().items()):
|
||||
if not _name.startswith("test_") or not callable(_fn):
|
||||
continue
|
||||
try:
|
||||
_sig = inspect.signature(_fn)
|
||||
if "tmp_path" in _sig.parameters:
|
||||
with tempfile.TemporaryDirectory() as _td:
|
||||
_fn(Path(_td))
|
||||
else:
|
||||
_fn()
|
||||
print(f"PASS {_name}")
|
||||
passed += 1
|
||||
except Exception:
|
||||
print(f"FAIL {_name}")
|
||||
_tb.print_exc()
|
||||
failed += 1
|
||||
print(f"\n{passed} passed, {failed} failed")
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
@@ -0,0 +1,401 @@
|
||||
"""
|
||||
test_event_file_io.py — sidecar write/read/patch round-trips,
|
||||
WaveformStore sidecar integration, and the BW-import path.
|
||||
|
||||
Run:
|
||||
python tests/test_event_file_io.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import pytest
|
||||
except ImportError:
|
||||
pytest = None # type: ignore
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from minimateplus import event_file_io
|
||||
from minimateplus.framing import S3Frame
|
||||
from minimateplus.models import Event, Timestamp
|
||||
|
||||
|
||||
# ── Fixtures shared with test_waveform_store.py ───────────────────────────────
|
||||
|
||||
|
||||
def _make_synthetic_event() -> tuple[Event, list[S3Frame]]:
|
||||
"""Same shape as tests/test_waveform_store.py — minimum viable Event +
|
||||
A5 stream that makes write_blastware_file emit a parseable file.
|
||||
|
||||
STRT is exactly 21 bytes; rectime_seconds lands at byte 18 to match
|
||||
`_decode_a5_waveform`'s expected layout (which is also what
|
||||
`read_blastware_file()` reads back)."""
|
||||
key4 = bytes.fromhex("01110000")
|
||||
rectime = 3
|
||||
strt = bytearray(21)
|
||||
strt[0:4] = b"STRT"
|
||||
strt[4:6] = b"\xff\xfe"
|
||||
strt[6:10] = key4 # end_key (per data[23:27] in CLAUDE.md)
|
||||
strt[10:14] = key4 # start_key (per data[27:31])
|
||||
strt[18] = rectime
|
||||
strt = bytes(strt)
|
||||
|
||||
probe_data = bytes(7) + strt + bytes(32)
|
||||
probe = S3Frame(sub=0xA5, page_hi=0x10, page_lo=0x00, data=probe_data,
|
||||
checksum_valid=True, chk_byte=0x00)
|
||||
|
||||
sample = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x10,
|
||||
data=bytes(7) + bytes(0x0200), checksum_valid=True,
|
||||
chk_byte=0x00)
|
||||
|
||||
# Build a valid 26-byte footer (0e 08 + ts1 + ts2 + 6 const + 2 crc)
|
||||
# and embed it at the END of the terminator's contribution so
|
||||
# write_blastware_file finds the real `0e 08` marker rather than
|
||||
# falling back to slicing the last 26 bytes of zero garbage.
|
||||
# ts byte order: [day][month][year_HI][year_LO][0x00][hour][min][sec]
|
||||
footer = (
|
||||
b"\x0e\x08"
|
||||
+ bytes([6, 5, 0x07, 0xea, 0, 12, 34, 56]) # ts1 = 2026-05-06 12:34:56
|
||||
+ bytes([6, 5, 0x07, 0xea, 0, 12, 35, 6]) # ts2 = ts1 + ~10s
|
||||
+ b"\x00\x01\x00\x02\x00\x00"
|
||||
+ b"\x00\x00"
|
||||
)
|
||||
assert len(footer) == 26
|
||||
term_data = bytes(11) + bytes(38) + footer # 11 prefix + 38 pad + 26 footer = 75
|
||||
term = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x00,
|
||||
data=term_data, checksum_valid=True, chk_byte=0x00)
|
||||
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = key4
|
||||
ev.timestamp = Timestamp(
|
||||
raw=b"", flag=0x10, year=2026, unknown_byte=0,
|
||||
month=5, day=6, hour=12, minute=34, second=56,
|
||||
)
|
||||
ev.rectime_seconds = rectime
|
||||
ev.record_type = "Waveform"
|
||||
ev._a5_frames = [probe, sample, term]
|
||||
return ev, [probe, sample, term]
|
||||
|
||||
|
||||
# ── Sidecar write/read round-trip ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_event_to_sidecar_dict_shape():
|
||||
ev, _ = _make_synthetic_event()
|
||||
d = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial="BE11529",
|
||||
blastware_filename="M529LKIQ.7M0W",
|
||||
blastware_filesize=1024,
|
||||
blastware_sha256="abcd" * 16,
|
||||
source_kind="sfm-live",
|
||||
a5_pickle_filename="M529LKIQ.7M0W.a5.pkl",
|
||||
)
|
||||
|
||||
assert d["schema_version"] == event_file_io.SCHEMA_VERSION
|
||||
assert d["kind"] == event_file_io.SIDECAR_KIND
|
||||
assert d["event"]["serial"] == "BE11529"
|
||||
assert d["event"]["timestamp"] == "2026-05-06T12:34:56"
|
||||
assert d["event"]["waveform_key"] == "01110000"
|
||||
assert d["blastware"]["sha256"] == "abcd" * 16
|
||||
assert d["source"]["kind"] == "sfm-live"
|
||||
assert d["review"] == {
|
||||
"false_trigger": False, "reviewer": None,
|
||||
"reviewed_at": None, "notes": "",
|
||||
}
|
||||
assert d["extensions"] == {}
|
||||
|
||||
|
||||
def test_sidecar_write_and_read_round_trip(tmp_path: Path):
|
||||
ev, _ = _make_synthetic_event()
|
||||
path = tmp_path / "M529LKIQ.7M0W.sfm.json"
|
||||
src = event_file_io.event_to_sidecar_dict(
|
||||
ev, serial="BE11529",
|
||||
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
||||
blastware_sha256="x" * 64, source_kind="sfm-ach",
|
||||
)
|
||||
event_file_io.write_sidecar(path, src)
|
||||
loaded = event_file_io.read_sidecar(path)
|
||||
assert loaded["event"] == src["event"]
|
||||
assert loaded["blastware"] == src["blastware"]
|
||||
assert loaded["source"]["kind"] == "sfm-ach"
|
||||
|
||||
|
||||
def test_sidecar_persists_raw_0c_record_in_extensions(tmp_path: Path):
|
||||
"""An Event with _raw_record populated should land its 210 bytes
|
||||
base64-encoded in extensions.raw_records.waveform_record_b64, so
|
||||
later analysis (e.g. mapping Peak Acceleration / Time of Peak / ZC
|
||||
Freq byte offsets) can run offline against the saved sidecar."""
|
||||
import base64
|
||||
|
||||
ev, _ = _make_synthetic_event()
|
||||
# Synthesize a 210-byte 0C record with embedded label needles so
|
||||
# the dump tool's anchor scan has something to find.
|
||||
raw = bytearray(210)
|
||||
raw[10:14] = b"Tran"
|
||||
raw[60:64] = b"Vert"
|
||||
raw[110:114] = b"Long"
|
||||
raw[160:164] = b"MicL"
|
||||
ev._raw_record = bytes(raw)
|
||||
|
||||
d = event_file_io.event_to_sidecar_dict(
|
||||
ev, serial="BE11529",
|
||||
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
||||
blastware_sha256="x" * 64, source_kind="sfm-live",
|
||||
)
|
||||
|
||||
rr = d["extensions"]["raw_records"]
|
||||
assert rr["waveform_record_len"] == 210
|
||||
decoded = base64.b64decode(rr["waveform_record_b64"])
|
||||
assert decoded == ev._raw_record
|
||||
|
||||
# Round-trip through write/read
|
||||
path = tmp_path / "raw0c.sfm.json"
|
||||
event_file_io.write_sidecar(path, d)
|
||||
loaded = event_file_io.read_sidecar(path)
|
||||
assert (
|
||||
base64.b64decode(loaded["extensions"]["raw_records"]["waveform_record_b64"])
|
||||
== ev._raw_record
|
||||
)
|
||||
|
||||
|
||||
def test_sidecar_omits_raw_records_when_event_has_no_0c(tmp_path: Path):
|
||||
"""Events without a _raw_record (e.g. constructed by importers that
|
||||
never see 0C) should NOT add an empty raw_records block — keep the
|
||||
sidecar clean for those flows."""
|
||||
ev, _ = _make_synthetic_event()
|
||||
assert ev._raw_record is None
|
||||
|
||||
d = event_file_io.event_to_sidecar_dict(
|
||||
ev, serial="BE11529",
|
||||
blastware_filename="M529LKIQ.7M0W", blastware_filesize=1024,
|
||||
blastware_sha256="x" * 64, source_kind="bw-import",
|
||||
)
|
||||
assert d["extensions"] == {}
|
||||
|
||||
|
||||
def test_sidecar_rejects_unsupported_schema_version(tmp_path: Path):
|
||||
path = tmp_path / "future.sfm.json"
|
||||
path.write_text(json.dumps({
|
||||
"schema_version": event_file_io.SCHEMA_VERSION + 1,
|
||||
"kind": event_file_io.SIDECAR_KIND,
|
||||
}))
|
||||
try:
|
||||
event_file_io.read_sidecar(path)
|
||||
except ValueError as exc:
|
||||
assert "schema_version" in str(exc)
|
||||
return
|
||||
raise AssertionError("read_sidecar should have rejected unsupported version")
|
||||
|
||||
|
||||
def test_sidecar_extensions_survive_round_trip(tmp_path: Path):
|
||||
"""Forward-compat: unknown keys inside `extensions` survive a r/w cycle."""
|
||||
ev, _ = _make_synthetic_event()
|
||||
path = tmp_path / "x.sfm.json"
|
||||
d = event_file_io.event_to_sidecar_dict(
|
||||
ev, serial="BE11529",
|
||||
blastware_filename="X", blastware_filesize=0, blastware_sha256="",
|
||||
source_kind="sfm-live",
|
||||
extensions={"vendor.acme.gps": {"lat": 40.7, "lon": -74.0}},
|
||||
)
|
||||
event_file_io.write_sidecar(path, d)
|
||||
back = event_file_io.read_sidecar(path)
|
||||
assert back["extensions"]["vendor.acme.gps"]["lat"] == 40.7
|
||||
|
||||
|
||||
def test_sidecar_patch_review_stamps_reviewed_at(tmp_path: Path):
|
||||
ev, _ = _make_synthetic_event()
|
||||
path = tmp_path / "patch.sfm.json"
|
||||
event_file_io.write_sidecar(
|
||||
path,
|
||||
event_file_io.event_to_sidecar_dict(
|
||||
ev, serial="BE11529",
|
||||
blastware_filename="X", blastware_filesize=0, blastware_sha256="",
|
||||
source_kind="sfm-live",
|
||||
),
|
||||
)
|
||||
new = event_file_io.patch_sidecar(
|
||||
path,
|
||||
review={"false_trigger": True, "notes": "truck thump", "reviewer": "brian"},
|
||||
)
|
||||
assert new["review"]["false_trigger"] is True
|
||||
assert new["review"]["notes"] == "truck thump"
|
||||
assert new["review"]["reviewer"] == "brian"
|
||||
assert new["review"]["reviewed_at"], "reviewed_at must be auto-stamped"
|
||||
|
||||
on_disk = event_file_io.read_sidecar(path)
|
||||
assert on_disk["review"]["false_trigger"] is True
|
||||
|
||||
|
||||
# ── WaveformStore integration ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_waveform_store_save_writes_sidecar(tmp_path: Path):
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
ev, frames = _make_synthetic_event()
|
||||
rec = store.save(ev, serial="BE11529", a5_frames=frames, source_kind="sfm-live")
|
||||
|
||||
assert rec["sidecar_filename"].endswith(".sfm.json")
|
||||
assert rec["sha256"] and len(rec["sha256"]) == 64
|
||||
|
||||
sc = store.load_sidecar("BE11529", rec["filename"])
|
||||
assert sc is not None
|
||||
assert sc["blastware"]["filename"] == rec["filename"]
|
||||
assert sc["blastware"]["sha256"] == rec["sha256"]
|
||||
assert sc["source"]["kind"] == "sfm-live"
|
||||
# The .a5.pkl reference should match the actual filename on disk.
|
||||
assert sc["source"]["a5_pickle_filename"] == rec["a5_pickle_filename"]
|
||||
|
||||
|
||||
def test_waveform_store_save_preserves_review_across_resave(tmp_path: Path):
|
||||
"""Re-saving the same event must preserve a user's prior review edits."""
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
ev, frames = _make_synthetic_event()
|
||||
rec = store.save(ev, serial="BE11529", a5_frames=frames)
|
||||
|
||||
# User flips false_trigger and adds a note.
|
||||
store.patch_sidecar(
|
||||
"BE11529", rec["filename"],
|
||||
review={"false_trigger": True, "notes": "hello"},
|
||||
)
|
||||
|
||||
# A second save (e.g. Force refresh re-download) must keep those edits.
|
||||
store.save(ev, serial="BE11529", a5_frames=frames)
|
||||
sc = store.load_sidecar("BE11529", rec["filename"])
|
||||
assert sc["review"]["false_trigger"] is True
|
||||
assert sc["review"]["notes"] == "hello"
|
||||
|
||||
|
||||
def test_waveform_store_patch_sidecar_returns_none_when_missing(tmp_path: Path):
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
out = store.patch_sidecar("BE99999", "no.such.W", review={"notes": "x"})
|
||||
assert out is None
|
||||
|
||||
|
||||
# ── DB integration: sidecar_filename column + update_event_review ─────────────
|
||||
|
||||
|
||||
def test_seismodb_persists_sidecar_filename_and_review_sync(tmp_path: Path):
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
db = SeismoDb(tmp_path / "seismo_relay.db")
|
||||
ev, _ = _make_synthetic_event()
|
||||
|
||||
rec = {
|
||||
"filename": "M529LKIQ.7M0W",
|
||||
"filesize": 8708,
|
||||
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
||||
"sidecar_filename": "M529LKIQ.7M0W.sfm.json",
|
||||
}
|
||||
inserted, _ = db.insert_events(
|
||||
[ev], serial="BE11529",
|
||||
waveform_records={ev._waveform_key.hex(): rec},
|
||||
)
|
||||
assert inserted == 1
|
||||
|
||||
rows = db.query_events(serial="BE11529")
|
||||
row = rows[0]
|
||||
assert row["sidecar_filename"] == rec["sidecar_filename"]
|
||||
|
||||
# update_event_review keeps false_trigger column in sync with sidecar.
|
||||
assert db.update_event_review(row["id"], {"false_trigger": True}) is True
|
||||
again = db.get_event(row["id"])
|
||||
assert again["false_trigger"] == 1
|
||||
|
||||
# Empty review block (no false_trigger key) → no-op but row exists.
|
||||
assert db.update_event_review(row["id"], {"notes": "x"}) is True
|
||||
|
||||
|
||||
# ── BW-file reader (read_blastware_file) ─────────────────────────────────────
|
||||
|
||||
|
||||
def test_read_blastware_file_round_trip(tmp_path: Path):
|
||||
"""write → read → key/timestamp/rectime survive."""
|
||||
from minimateplus.blastware_file import write_blastware_file, blastware_filename
|
||||
|
||||
ev, frames = _make_synthetic_event()
|
||||
bw_path = tmp_path / blastware_filename(ev, "BE11529")
|
||||
write_blastware_file(ev, frames, bw_path)
|
||||
|
||||
parsed = event_file_io.read_blastware_file(bw_path)
|
||||
assert parsed._waveform_key == ev._waveform_key
|
||||
assert parsed.rectime_seconds == ev.rectime_seconds
|
||||
# Timestamp lands via the footer; year/month/day/hour/min/sec all survive.
|
||||
assert parsed.timestamp is not None
|
||||
assert parsed.timestamp.year == ev.timestamp.year
|
||||
assert parsed.timestamp.month == ev.timestamp.month
|
||||
assert parsed.timestamp.day == ev.timestamp.day
|
||||
assert parsed.timestamp.hour == ev.timestamp.hour
|
||||
assert parsed.timestamp.minute == ev.timestamp.minute
|
||||
assert parsed.timestamp.second == ev.timestamp.second
|
||||
# No A5 source recoverable.
|
||||
assert parsed._a5_frames is None
|
||||
# Peaks computed from samples (synthetic = zero samples → zero peaks).
|
||||
assert parsed.peak_values is not None
|
||||
assert parsed.peak_values.peak_vector_sum == 0.0
|
||||
|
||||
|
||||
def test_save_imported_bw_round_trip(tmp_path: Path):
|
||||
"""save_imported_bw stores a copy + sidecar with source.kind = bw-import."""
|
||||
from minimateplus.blastware_file import write_blastware_file, blastware_filename
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
# Produce a BW file outside the store.
|
||||
ev, frames = _make_synthetic_event()
|
||||
fname = blastware_filename(ev, "BE11529")
|
||||
src = tmp_path / fname
|
||||
write_blastware_file(ev, frames, src)
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
parsed_ev, rec = store.save_imported_bw(src.read_bytes(), source_path=src)
|
||||
|
||||
assert rec["filename"] == fname
|
||||
assert rec["a5_pickle_filename"] is None # no A5 source for BW imports
|
||||
sc = store.load_sidecar("BE11529", fname)
|
||||
assert sc is not None
|
||||
assert sc["source"]["kind"] == "bw-import"
|
||||
assert sc["source"]["a5_pickle_filename"] is None
|
||||
# The stored binary should match the source byte-for-byte (we just copied).
|
||||
stored_path = store.open_blastware("BE11529", fname)
|
||||
assert stored_path is not None
|
||||
assert stored_path.read_bytes() == src.read_bytes()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if pytest is not None:
|
||||
pytest.main([__file__, "-v"])
|
||||
else:
|
||||
import inspect
|
||||
import traceback as _tb
|
||||
|
||||
passed = failed = 0
|
||||
for _name, _fn in sorted(globals().items()):
|
||||
if not _name.startswith("test_") or not callable(_fn):
|
||||
continue
|
||||
try:
|
||||
_sig = inspect.signature(_fn)
|
||||
if "tmp_path" in _sig.parameters:
|
||||
with tempfile.TemporaryDirectory() as _td:
|
||||
_fn(Path(_td))
|
||||
else:
|
||||
_fn()
|
||||
print(f"PASS {_name}")
|
||||
passed += 1
|
||||
except Exception:
|
||||
print(f"FAIL {_name}")
|
||||
_tb.print_exc()
|
||||
failed += 1
|
||||
print(f"\n{passed} passed, {failed} failed")
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
@@ -0,0 +1,296 @@
|
||||
"""
|
||||
test_event_hdf5.py — HDF5 codec round-trip + plot.v1 JSON shape sanity.
|
||||
|
||||
Run:
|
||||
python tests/test_event_hdf5.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import pytest
|
||||
except ImportError:
|
||||
pytest = None # type: ignore
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from minimateplus.framing import S3Frame
|
||||
from minimateplus.models import Event, PeakValues, ProjectInfo, Timestamp
|
||||
from sfm import event_hdf5
|
||||
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_event_with_samples(n: int = 256) -> Event:
|
||||
"""An Event with synthetic int16 ADC samples on all four channels.
|
||||
|
||||
Channel content:
|
||||
- Tran: ramp from -16384 to +16383 (peak ≈ 5 in/s for Normal range)
|
||||
- Vert: full-scale dirac at index n//2 (peak = 10 in/s)
|
||||
- Long: zeros
|
||||
- MicL: small ramp
|
||||
Peak values are set on the event the way the device's 0C record
|
||||
would supply them — used by the HDF5 writer for the mic per-count
|
||||
factor.
|
||||
"""
|
||||
tran = [int((i / max(n - 1, 1)) * 32767 - 16384) for i in range(n)]
|
||||
vert = [0] * n
|
||||
if n:
|
||||
vert[n // 2] = 32767
|
||||
long_ = [0] * n
|
||||
mic = [int((i / max(n - 1, 1)) * 5000) for i in range(n)]
|
||||
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = bytes.fromhex("01110000")
|
||||
ev.timestamp = Timestamp(
|
||||
raw=b"", flag=0x10,
|
||||
year=2026, unknown_byte=0, month=5, day=7,
|
||||
hour=10, minute=0, second=0,
|
||||
)
|
||||
ev.record_type = "Waveform"
|
||||
ev.sample_rate = 1024
|
||||
ev.pretrig_samples = n // 4
|
||||
ev.total_samples = n
|
||||
ev.rectime_seconds = n / 1024.0
|
||||
ev.raw_samples = {"Tran": tran, "Vert": vert, "Long": long_, "MicL": mic}
|
||||
ev.peak_values = PeakValues(
|
||||
tran=5.0, vert=10.0, long=0.0,
|
||||
peak_vector_sum=10.0, micl=0.001,
|
||||
)
|
||||
ev.project_info = ProjectInfo(
|
||||
project="TestProj", client="TestClient",
|
||||
operator="brian", sensor_location="loc-A",
|
||||
)
|
||||
return ev
|
||||
|
||||
|
||||
# ── HDF5 round-trip ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_hdf5_round_trip_preserves_metadata(tmp_path: Path):
|
||||
ev = _make_event_with_samples()
|
||||
h5 = tmp_path / "test.h5"
|
||||
event_hdf5.write_event_hdf5(
|
||||
h5, ev, serial="BE11529", geo_range="normal",
|
||||
)
|
||||
|
||||
data = event_hdf5.read_event_hdf5(h5)
|
||||
a = data["attrs"]
|
||||
assert a["schema_version"] == event_hdf5.SCHEMA_VERSION
|
||||
assert a["kind"] == event_hdf5.HDF5_KIND
|
||||
assert a["serial"] == "BE11529"
|
||||
assert a["waveform_key"] == "01110000"
|
||||
assert a["sample_rate"] == 1024
|
||||
assert a["pretrig_samples"] == 64
|
||||
assert a["geo_range"] == "normal"
|
||||
assert a["geo_full_scale_ips"] == 10.0
|
||||
assert a["project"] == "TestProj"
|
||||
assert a["client"] == "TestClient"
|
||||
assert a["operator"] == "brian"
|
||||
# Float attrs may round-trip with tiny precision noise.
|
||||
assert abs(a["peak_tran_ips"] - 5.0) < 1e-6
|
||||
assert abs(a["peak_vert_ips"] - 10.0) < 1e-6
|
||||
|
||||
|
||||
def test_hdf5_samples_in_physical_units_normal_range(tmp_path: Path):
|
||||
"""Vert hits ADC full-scale (32767) → with Normal range FS=10 in/s,
|
||||
the HDF5 sample value should be ≈ 10 * 32767/32768 in/s."""
|
||||
ev = _make_event_with_samples()
|
||||
h5 = tmp_path / "n.h5"
|
||||
event_hdf5.write_event_hdf5(h5, ev, serial="BE11529", geo_range="normal")
|
||||
data = event_hdf5.read_event_hdf5(h5)
|
||||
|
||||
vert = data["samples"]["Vert"]
|
||||
assert vert.dtype.name == "float32"
|
||||
assert max(abs(v) for v in vert) > 9.99 # full-scale ≈ 10.0
|
||||
# The dirac was at n//2 → 32767 ADC counts.
|
||||
expected_peak = 10.0 * 32767 / 32768
|
||||
assert abs(max(vert) - expected_peak) < 1e-3
|
||||
|
||||
|
||||
def test_hdf5_samples_in_physical_units_sensitive_range(tmp_path: Path):
|
||||
"""Same fixture but Sensitive range → full-scale 1.250 in/s."""
|
||||
ev = _make_event_with_samples()
|
||||
h5 = tmp_path / "s.h5"
|
||||
event_hdf5.write_event_hdf5(h5, ev, serial="BE11529", geo_range="sensitive")
|
||||
data = event_hdf5.read_event_hdf5(h5)
|
||||
|
||||
vert = data["samples"]["Vert"]
|
||||
expected_peak = 1.250 * 32767 / 32768
|
||||
assert abs(max(vert) - expected_peak) < 1e-4
|
||||
|
||||
|
||||
def test_hdf5_includes_int16_samples(tmp_path: Path):
|
||||
ev = _make_event_with_samples()
|
||||
h5 = tmp_path / "i.h5"
|
||||
event_hdf5.write_event_hdf5(h5, ev, serial="BE11529")
|
||||
data = event_hdf5.read_event_hdf5(h5)
|
||||
assert data["samples_int16"] is not None
|
||||
assert "Tran" in data["samples_int16"]
|
||||
assert data["samples_int16"]["Vert"].dtype.name == "int16"
|
||||
|
||||
|
||||
def test_hdf5_rejects_unsupported_schema(tmp_path: Path):
|
||||
"""Round-tripping with a tampered schema_version raises ValueError."""
|
||||
import h5py
|
||||
h5 = tmp_path / "future.h5"
|
||||
with h5py.File(h5, "w") as f:
|
||||
f.attrs["schema_version"] = 99
|
||||
f.attrs["kind"] = event_hdf5.HDF5_KIND
|
||||
try:
|
||||
event_hdf5.read_event_hdf5(h5)
|
||||
except ValueError as exc:
|
||||
assert "schema_version" in str(exc)
|
||||
return
|
||||
raise AssertionError("read_event_hdf5 should reject unsupported schema_version")
|
||||
|
||||
|
||||
# ── plot.v1 JSON shape ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_event_to_plot_json_shape():
|
||||
ev = _make_event_with_samples()
|
||||
j = event_hdf5.event_to_plot_json(ev, serial="BE11529", geo_range="normal")
|
||||
assert j["schema"] == "sfm.plot.v1"
|
||||
assert j["serial"] == "BE11529"
|
||||
assert j["geo_range"] == "normal"
|
||||
assert j["geo_full_scale_ips"] == 10.0
|
||||
assert j["trigger_ms"] == 0.0
|
||||
|
||||
t = j["time_axis"]
|
||||
assert t["sample_rate"] == 1024
|
||||
assert t["pretrig_samples"] == 64
|
||||
assert t["n_samples"] == 256
|
||||
# t0_ms = -pretrig * dt_ms = -64 * (1000/1024) ≈ -62.5
|
||||
assert abs(t["t0_ms"] - (-64 * 1000 / 1024)) < 1e-3
|
||||
assert abs(t["dt_ms"] - (1000 / 1024)) < 1e-6
|
||||
|
||||
chans = j["channels"]
|
||||
for name in ("Tran", "Vert", "Long", "MicL"):
|
||||
assert name in chans, f"missing channel: {name}"
|
||||
assert chans[name]["unit"] in ("in/s", "psi")
|
||||
assert "values" in chans[name]
|
||||
assert "peak" in chans[name]
|
||||
assert "peak_t_ms" in chans[name]
|
||||
|
||||
# Values are in physical units: Vert peak ≈ 10 in/s.
|
||||
assert max(chans["Vert"]["values"]) > 9.99
|
||||
|
||||
|
||||
def test_event_to_plot_json_peak_t_ms_locates_dirac():
|
||||
"""The Vert channel's full-scale dirac at sample n//2 should produce
|
||||
peak_t_ms = (n//2 - pretrig) * dt_ms."""
|
||||
ev = _make_event_with_samples(n=256)
|
||||
j = event_hdf5.event_to_plot_json(ev, serial="BE11529")
|
||||
expected = (128 - 64) * (1000 / 1024) # = 62.5 ms
|
||||
assert abs(j["channels"]["Vert"]["peak_t_ms"] - expected) < 1e-2
|
||||
|
||||
|
||||
def test_plot_json_from_hdf5_round_trip(tmp_path: Path):
|
||||
"""plot_json_from_hdf5 produces the same shape as event_to_plot_json."""
|
||||
ev = _make_event_with_samples()
|
||||
h5 = tmp_path / "rt.h5"
|
||||
event_hdf5.write_event_hdf5(h5, ev, serial="BE11529", geo_range="normal")
|
||||
|
||||
j_disk = event_hdf5.plot_json_from_hdf5(h5, event_id="abc-123")
|
||||
j_mem = event_hdf5.event_to_plot_json(ev, serial="BE11529", geo_range="normal", event_id="abc-123")
|
||||
|
||||
# Top-level shape parity
|
||||
for k in ("schema", "serial", "geo_range", "geo_full_scale_ips",
|
||||
"trigger_ms", "record_type", "waveform_key", "event_id"):
|
||||
assert j_disk.get(k) == j_mem.get(k), f"mismatch on {k}"
|
||||
assert j_disk["time_axis"]["sample_rate"] == j_mem["time_axis"]["sample_rate"]
|
||||
assert j_disk["time_axis"]["n_samples"] == j_mem["time_axis"]["n_samples"]
|
||||
|
||||
# Sample values must match within float32 precision.
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
a = j_disk["channels"][ch]["values"]
|
||||
b = j_mem["channels"][ch]["values"]
|
||||
assert len(a) == len(b)
|
||||
if a:
|
||||
mx = max(abs(x - y) for x, y in zip(a, b))
|
||||
assert mx < 1e-3, f"{ch}: max diff {mx}"
|
||||
|
||||
|
||||
# ── WaveformStore integration with HDF5 ───────────────────────────────────────
|
||||
|
||||
|
||||
def _make_synthetic_event_for_save() -> tuple[Event, list[S3Frame]]:
|
||||
"""Same flavour as test_event_file_io.py but ensures _make_event_with_samples
|
||||
is also wired into the BW write path so we can exercise WaveformStore.save."""
|
||||
ev = _make_event_with_samples(n=128)
|
||||
# Build a minimum 3-frame A5 stream (probe + sample + term) — same
|
||||
# shape used in the other test files. The encoder only really needs
|
||||
# the STRT in the probe + a non-zero body and a footer in the term.
|
||||
key4 = ev._waveform_key
|
||||
rectime = int(ev.rectime_seconds or 0) or 1
|
||||
strt = bytearray(21)
|
||||
strt[0:4] = b"STRT"
|
||||
strt[4:6] = b"\xff\xfe"
|
||||
strt[6:10] = key4
|
||||
strt[10:14] = key4
|
||||
strt[18] = rectime
|
||||
probe = S3Frame(sub=0xA5, page_hi=0x10, page_lo=0x00,
|
||||
data=bytes(7) + bytes(strt) + bytes(32),
|
||||
checksum_valid=True, chk_byte=0x00)
|
||||
sample = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x10,
|
||||
data=bytes(7) + bytes(0x0200), checksum_valid=True, chk_byte=0x00)
|
||||
footer = (
|
||||
b"\x0e\x08"
|
||||
+ bytes([7, 5, 0x07, 0xea, 0, 10, 0, 0])
|
||||
+ bytes([7, 5, 0x07, 0xea, 0, 10, 0, 1])
|
||||
+ b"\x00\x01\x00\x02\x00\x00\x00\x00"
|
||||
)
|
||||
term = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x00,
|
||||
data=bytes(11) + bytes(38) + footer, checksum_valid=True, chk_byte=0x00)
|
||||
ev._a5_frames = [probe, sample, term]
|
||||
return ev, [probe, sample, term]
|
||||
|
||||
|
||||
def test_waveform_store_save_emits_hdf5(tmp_path: Path):
|
||||
from sfm.waveform_store import WaveformStore
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
ev, frames = _make_synthetic_event_for_save()
|
||||
rec = store.save(ev, serial="BE11529", a5_frames=frames, geo_range="normal")
|
||||
|
||||
assert rec["hdf5_filename"], "hdf5_filename should be present in save() record"
|
||||
h5 = store.hdf5_path_for("BE11529", rec["filename"])
|
||||
assert h5.exists(), "WaveformStore.save should produce a .h5 file"
|
||||
# The HDF5 round-trip should match the event's metadata.
|
||||
data = event_hdf5.read_event_hdf5(h5)
|
||||
assert data["attrs"]["serial"] == "BE11529"
|
||||
assert data["attrs"]["geo_range"] == "normal"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if pytest is not None:
|
||||
pytest.main([__file__, "-v"])
|
||||
else:
|
||||
import inspect
|
||||
import traceback as _tb
|
||||
|
||||
passed = failed = 0
|
||||
for _name, _fn in sorted(globals().items()):
|
||||
if not _name.startswith("test_") or not callable(_fn):
|
||||
continue
|
||||
try:
|
||||
_sig = inspect.signature(_fn)
|
||||
if "tmp_path" in _sig.parameters:
|
||||
with tempfile.TemporaryDirectory() as _td:
|
||||
_fn(Path(_td))
|
||||
else:
|
||||
_fn()
|
||||
print(f"PASS {_name}")
|
||||
passed += 1
|
||||
except Exception:
|
||||
print(f"FAIL {_name}")
|
||||
_tb.print_exc()
|
||||
failed += 1
|
||||
print(f"\n{passed} passed, {failed} failed")
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
@@ -0,0 +1,302 @@
|
||||
"""
|
||||
test_waveform_store.py — unit tests for sfm/waveform_store.py and the
|
||||
SeismoDb columns + insert_events upsert path that the store depends on.
|
||||
|
||||
These tests exercise the *store + DB plumbing* in isolation — they do not
|
||||
re-test write_blastware_file (covered separately) and do not require a live
|
||||
device or a wire capture.
|
||||
|
||||
Run:
|
||||
python -m pytest tests/test_waveform_store.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import pytest
|
||||
except ImportError: # allow running standalone without pytest installed
|
||||
pytest = None # type: ignore
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from minimateplus.framing import S3Frame
|
||||
from minimateplus.models import Event, Timestamp
|
||||
|
||||
|
||||
# ── Test fixtures ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_synthetic_event() -> tuple[Event, list[S3Frame]]:
|
||||
"""
|
||||
Build a minimal Event + a 3-frame A5 stream that satisfies
|
||||
write_blastware_file's STRT-extraction path.
|
||||
|
||||
Frame 0 (probe): contains a STRT record at the canonical position so
|
||||
write_blastware_file finds it without falling back.
|
||||
Frame 1 (sample): 0x0200 bytes of zeros at page_key=0x0010 (sample marker).
|
||||
Frame 2 (TERM): page_key=0x0000 marks the terminator.
|
||||
"""
|
||||
key4 = bytes.fromhex("01110000")
|
||||
rectime = 3
|
||||
strt = b"STRT" + b"\xff\xfe" + key4 + key4 + bytes(7) + bytes([rectime])
|
||||
|
||||
# Probe payload prefix: 7 zero bytes then STRT (matches blastware_file._strip
|
||||
# logic which looks for STRT in data[7:]). Tail with 32 zero bytes of fake
|
||||
# body so reconstruction has something to slice.
|
||||
probe_data = bytes(7) + strt + bytes(32)
|
||||
probe = S3Frame(sub=0xA5, page_hi=0x10, page_lo=0x00, data=probe_data,
|
||||
checksum_valid=True, chk_byte=0x00)
|
||||
|
||||
sample = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x10,
|
||||
data=bytes(7) + bytes(0x0200), checksum_valid=True,
|
||||
chk_byte=0x00)
|
||||
|
||||
term = S3Frame(sub=0xA5, page_hi=0x00, page_lo=0x00,
|
||||
data=bytes(7) + bytes(64), checksum_valid=True,
|
||||
chk_byte=0x00)
|
||||
|
||||
ev = Event(index=0)
|
||||
ev._waveform_key = key4
|
||||
ev.timestamp = Timestamp(
|
||||
raw=b"",
|
||||
flag=0x10,
|
||||
year=2026,
|
||||
unknown_byte=0,
|
||||
month=5,
|
||||
day=6,
|
||||
hour=12,
|
||||
minute=34,
|
||||
second=56,
|
||||
)
|
||||
ev.rectime_seconds = rectime
|
||||
ev.record_type = "Waveform"
|
||||
ev._a5_frames = [probe, sample, term]
|
||||
return ev, [probe, sample, term]
|
||||
|
||||
|
||||
# ── Frame round-trip ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_frame_dict_round_trip():
|
||||
"""_frame_to_dict and _dict_to_frame must round-trip every field."""
|
||||
from sfm.waveform_store import _dict_to_frame, _frame_to_dict
|
||||
|
||||
f = S3Frame(
|
||||
sub=0xA5, page_hi=0x12, page_lo=0x34,
|
||||
data=b"\x10\x02\x00\xab\xcd",
|
||||
checksum_valid=False,
|
||||
chk_byte=0x42,
|
||||
)
|
||||
d = _frame_to_dict(f)
|
||||
g = _dict_to_frame(d)
|
||||
assert g.sub == f.sub
|
||||
assert g.page_hi == f.page_hi
|
||||
assert g.page_lo == f.page_lo
|
||||
assert g.data == f.data
|
||||
assert g.checksum_valid == f.checksum_valid
|
||||
assert g.chk_byte == f.chk_byte
|
||||
|
||||
|
||||
# ── Store save/load round-trip ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_waveform_store_save_load_round_trip(tmp_path: Path):
|
||||
"""save() writes both files; load_a5() returns equivalent frames."""
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
ev, frames = _make_synthetic_event()
|
||||
|
||||
rec = store.save(ev, serial="BE11529", a5_frames=frames)
|
||||
|
||||
assert rec["filename"].startswith("M529")
|
||||
assert rec["filesize"] > 0
|
||||
assert rec["a5_pickle_filename"] == rec["filename"] + ".a5.pkl"
|
||||
|
||||
bw_path = store.open_blastware("BE11529", rec["filename"])
|
||||
assert bw_path is not None
|
||||
assert bw_path.exists()
|
||||
assert bw_path.stat().st_size == rec["filesize"]
|
||||
|
||||
# Sidecar exists and round-trips
|
||||
loaded = store.load_a5("BE11529", rec["filename"])
|
||||
assert loaded is not None
|
||||
assert len(loaded) == len(frames)
|
||||
for orig, got in zip(frames, loaded):
|
||||
assert got.sub == orig.sub
|
||||
assert got.page_hi == orig.page_hi
|
||||
assert got.page_lo == orig.page_lo
|
||||
assert got.data == orig.data
|
||||
|
||||
|
||||
def test_waveform_store_missing_returns_none(tmp_path: Path):
|
||||
"""open_blastware / load_a5 return None for nonexistent entries."""
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
assert store.open_blastware("BE99999", "no_such.7M0W") is None
|
||||
assert store.load_a5("BE99999", "no_such.7M0W") is None
|
||||
|
||||
|
||||
def test_waveform_store_idempotent_save(tmp_path: Path):
|
||||
"""Saving the same event twice produces the same event-file bytes."""
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
store = WaveformStore(tmp_path / "waveforms")
|
||||
ev, frames = _make_synthetic_event()
|
||||
|
||||
rec1 = store.save(ev, serial="BE11529", a5_frames=frames)
|
||||
bw_path = store.open_blastware("BE11529", rec1["filename"])
|
||||
bytes1 = bw_path.read_bytes()
|
||||
|
||||
rec2 = store.save(ev, serial="BE11529", a5_frames=frames)
|
||||
bytes2 = bw_path.read_bytes()
|
||||
|
||||
assert rec1["filename"] == rec2["filename"]
|
||||
assert bytes1 == bytes2
|
||||
|
||||
|
||||
# ── DB integration ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_seismodb_persists_waveform_columns(tmp_path: Path):
|
||||
"""insert_events writes the new columns when waveform_records is supplied."""
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
db = SeismoDb(tmp_path / "seismo_relay.db")
|
||||
ev, _ = _make_synthetic_event()
|
||||
|
||||
rec = {
|
||||
"filename": "M529LKIQ.7M0W",
|
||||
"filesize": 8708,
|
||||
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
||||
}
|
||||
inserted, skipped = db.insert_events(
|
||||
[ev],
|
||||
serial="BE11529",
|
||||
waveform_records={ev._waveform_key.hex(): rec},
|
||||
)
|
||||
assert inserted == 1
|
||||
assert skipped == 0
|
||||
|
||||
rows = db.query_events(serial="BE11529")
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["blastware_filename"] == rec["filename"]
|
||||
assert row["blastware_filesize"] == rec["filesize"]
|
||||
assert row["a5_pickle_filename"] == rec["a5_pickle_filename"]
|
||||
|
||||
# get_event by id returns the same fields
|
||||
row2 = db.get_event(row["id"])
|
||||
assert row2 is not None
|
||||
assert row2["blastware_filename"] == rec["filename"]
|
||||
|
||||
|
||||
def test_seismodb_dedup_upserts_waveform_fields(tmp_path: Path):
|
||||
"""Re-inserting the same (serial, timestamp) refreshes waveform fields."""
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
db = SeismoDb(tmp_path / "seismo_relay.db")
|
||||
ev, _ = _make_synthetic_event()
|
||||
|
||||
db.insert_events([ev], serial="BE11529") # no waveform record yet
|
||||
rows = db.query_events(serial="BE11529")
|
||||
assert rows[0]["blastware_filename"] is None
|
||||
|
||||
rec = {
|
||||
"filename": "M529LKIQ.7M0W",
|
||||
"filesize": 4242,
|
||||
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
||||
}
|
||||
inserted, skipped = db.insert_events(
|
||||
[ev],
|
||||
serial="BE11529",
|
||||
waveform_records={ev._waveform_key.hex(): rec},
|
||||
)
|
||||
assert inserted == 0 # dedup'd
|
||||
assert skipped == 1
|
||||
rows = db.query_events(serial="BE11529")
|
||||
assert rows[0]["blastware_filename"] == rec["filename"]
|
||||
assert rows[0]["blastware_filesize"] == 4242
|
||||
|
||||
|
||||
def test_seismodb_migration_adds_columns(tmp_path: Path):
|
||||
"""An existing DB without the new columns gets them added on init."""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "old.db"
|
||||
# Build a "v0" events table without the new columns.
|
||||
with sqlite3.connect(str(db_path)) as conn:
|
||||
conn.executescript("""
|
||||
CREATE TABLE events (
|
||||
id TEXT PRIMARY KEY,
|
||||
serial TEXT NOT NULL,
|
||||
waveform_key TEXT NOT NULL,
|
||||
session_id TEXT,
|
||||
timestamp TEXT,
|
||||
tran_ppv REAL,
|
||||
vert_ppv REAL,
|
||||
long_ppv REAL,
|
||||
peak_vector_sum REAL,
|
||||
mic_ppv REAL,
|
||||
project TEXT,
|
||||
client TEXT,
|
||||
operator TEXT,
|
||||
sensor_location TEXT,
|
||||
sample_rate INTEGER,
|
||||
record_type TEXT,
|
||||
false_trigger INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, timestamp)
|
||||
);
|
||||
INSERT INTO events
|
||||
(id, serial, waveform_key, timestamp)
|
||||
VALUES
|
||||
('legacy-id', 'BE11529', '01110000',
|
||||
'2026-04-01T12:00:00');
|
||||
""")
|
||||
|
||||
# Initialise SeismoDb against the old DB — migration should run.
|
||||
from sfm.database import SeismoDb
|
||||
|
||||
db = SeismoDb(db_path)
|
||||
rows = db.query_events(serial="BE11529")
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["blastware_filename"] is None
|
||||
assert "blastware_filesize" in rows[0]
|
||||
assert "a5_pickle_filename" in rows[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if pytest is not None:
|
||||
pytest.main([__file__, "-v"])
|
||||
else:
|
||||
# Standalone runner — does not require pytest.
|
||||
import inspect
|
||||
import tempfile
|
||||
import traceback as _tb
|
||||
|
||||
passed = failed = 0
|
||||
for _name, _fn in sorted(globals().items()):
|
||||
if not _name.startswith("test_") or not callable(_fn):
|
||||
continue
|
||||
try:
|
||||
_sig = inspect.signature(_fn)
|
||||
if "tmp_path" in _sig.parameters:
|
||||
with tempfile.TemporaryDirectory() as _td:
|
||||
_fn(Path(_td))
|
||||
else:
|
||||
_fn()
|
||||
print(f"PASS {_name}")
|
||||
passed += 1
|
||||
except Exception:
|
||||
print(f"FAIL {_name}")
|
||||
_tb.print_exc()
|
||||
failed += 1
|
||||
print(f"\n{passed} passed, {failed} failed")
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
@@ -0,0 +1,436 @@
|
||||
"""
|
||||
test_write_frames.py — Verify write frame construction against BW capture.
|
||||
|
||||
Validates that build_bw_write_frame() reproduces the exact wire bytes that
|
||||
Blastware sent during the 3-11-26/170151 compliance-config write session.
|
||||
|
||||
Frames tested (BW TX frame indices 102–112):
|
||||
102 — SUB 0x68 event index write
|
||||
103 — SUB 0x73 confirm B
|
||||
104 — SUB 0x71 compliance write chunk 1
|
||||
105 — SUB 0x71 compliance write chunk 2
|
||||
106 — SUB 0x71 compliance write chunk 3
|
||||
107 — SUB 0x72 confirm A
|
||||
108 — SUB 0x82 trigger config write
|
||||
109 — SUB 0x83 trigger confirm
|
||||
110 — SUB 0x69 waveform data write
|
||||
111 — SUB 0x74 confirm C
|
||||
112 — SUB 0x72 confirm A (end of sequence)
|
||||
|
||||
Run:
|
||||
python -m pytest tests/test_write_frames.py -v
|
||||
or:
|
||||
python tests/test_write_frames.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
# Allow running from the project root without installation
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from minimateplus.framing import build_bw_write_frame
|
||||
|
||||
|
||||
# ── Capture loading ────────────────────────────────────────────────────────────
|
||||
|
||||
CAPTURE_PATH = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"bridges",
|
||||
"captures",
|
||||
"3-11-26",
|
||||
"raw_bw_20260311_170151.bin",
|
||||
)
|
||||
|
||||
|
||||
def _load_bw_frames(path: str) -> list[bytes]:
|
||||
"""
|
||||
Parse a raw BW capture file into a list of BW frames.
|
||||
|
||||
BW frames start with ACK=0x41 followed by STX=0x02. The frame boundary is
|
||||
the position of the NEXT 0x41 0x02 sequence (the ETX=0x03 terminator is the
|
||||
last byte before the next frame start).
|
||||
|
||||
NOTE: A naive scan for ETX=0x03 fails because 0x03 can appear inside the
|
||||
DLE-stuffed payload. This parser uses consecutive 0x41 0x02 starts as
|
||||
boundaries, which is safe because the ACK byte (0x41) is never DLE-stuffed.
|
||||
"""
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
|
||||
boundaries: list[int] = []
|
||||
i = 0
|
||||
while i < len(raw) - 1:
|
||||
if raw[i] == 0x41 and raw[i + 1] == 0x02:
|
||||
boundaries.append(i)
|
||||
i += 1
|
||||
boundaries.append(len(raw))
|
||||
|
||||
frames = []
|
||||
for k in range(len(boundaries) - 1):
|
||||
frames.append(raw[boundaries[k] : boundaries[k + 1]])
|
||||
return frames
|
||||
|
||||
|
||||
def _destuff(data: bytes) -> bytes:
|
||||
"""Undo DLE stuffing: replace every 0x10 0x10 pair with a single 0x10."""
|
||||
result = bytearray()
|
||||
k = 0
|
||||
while k < len(data):
|
||||
if data[k] == 0x10 and k + 1 < len(data) and data[k + 1] == 0x10:
|
||||
result.append(0x10)
|
||||
k += 2
|
||||
else:
|
||||
result.append(data[k])
|
||||
k += 1
|
||||
return bytes(result)
|
||||
|
||||
|
||||
def _decode_bw_frame(wire: bytes) -> tuple[int, int, bytes, bytes, int]:
|
||||
"""
|
||||
Decode a BW wire frame into its components.
|
||||
|
||||
Returns:
|
||||
(sub, offset, params, data, chk)
|
||||
sub — SUB byte (payload[2])
|
||||
offset — uint16 from payload[4:6]
|
||||
params — 10-byte params field (payload[6:16])
|
||||
data — write payload bytes (payload[16:-1])
|
||||
chk — checksum byte (payload[-1])
|
||||
"""
|
||||
inner = wire[2:-1] # strip ACK+STX and trailing ETX
|
||||
payload = _destuff(inner)
|
||||
sub = payload[2]
|
||||
offset = (payload[4] << 8) | payload[5]
|
||||
params = payload[6:16]
|
||||
data = payload[16:-1]
|
||||
chk = payload[-1]
|
||||
return sub, offset, params, data, chk
|
||||
|
||||
|
||||
# ── Test fixtures ──────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bw_frames() -> list[bytes]:
|
||||
if not os.path.exists(CAPTURE_PATH):
|
||||
pytest.skip(f"Capture file not found: {CAPTURE_PATH}")
|
||||
return _load_bw_frames(CAPTURE_PATH)
|
||||
|
||||
|
||||
# ── Individual frame tests ─────────────────────────────────────────────────────
|
||||
|
||||
class TestWriteFrameReconstruction:
|
||||
"""Verify build_bw_write_frame() reproduces the exact wire bytes from the capture."""
|
||||
|
||||
def test_frame_102_event_index_write_sub68(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x68 — event index write (frame 102)."""
|
||||
cap_wire = bw_frames[102]
|
||||
sub_cap, offset_cap, params_cap, data_cap, chk_cap = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x68
|
||||
assert params_cap == bytes(10)
|
||||
|
||||
# Reconstruct using build_bw_write_frame with the same data and offset
|
||||
built = build_bw_write_frame(0x68, data_cap, offset=offset_cap, params=params_cap)
|
||||
assert built == cap_wire, (
|
||||
f"SUB 0x68 wire mismatch\n"
|
||||
f" built: {built.hex()}\n"
|
||||
f" capt: {cap_wire.hex()}"
|
||||
)
|
||||
|
||||
def test_frame_103_confirm_b_sub73(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x73 — confirm B (zero-data confirm frame 103)."""
|
||||
cap_wire = bw_frames[103]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x73
|
||||
assert data_cap == b""
|
||||
assert offset_cap == 0x0000
|
||||
|
||||
built = build_bw_write_frame(0x73, b"")
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_104_compliance_chunk1_sub71(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x71 chunk 1 — 1027-byte compliance write (frame 104)."""
|
||||
cap_wire = bw_frames[104]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x71
|
||||
assert offset_cap == 0x1004
|
||||
assert params_cap == bytes(10)
|
||||
assert len(data_cap) == 1027
|
||||
|
||||
built = build_bw_write_frame(
|
||||
0x71, data_cap,
|
||||
offset=0x1004,
|
||||
params=bytes(10),
|
||||
)
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_105_compliance_chunk2_sub71(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x71 chunk 2 — 1055-byte compliance write (frame 105)."""
|
||||
cap_wire = bw_frames[105]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
_CHUNK2_PARAMS = bytes([0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00])
|
||||
|
||||
assert sub_cap == 0x71
|
||||
assert offset_cap == 0x1004
|
||||
assert params_cap == _CHUNK2_PARAMS
|
||||
assert len(data_cap) == 1055
|
||||
|
||||
built = build_bw_write_frame(
|
||||
0x71, data_cap,
|
||||
offset=0x1004,
|
||||
params=_CHUNK2_PARAMS,
|
||||
)
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_106_compliance_chunk3_sub71(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x71 chunk 3 — 46-byte compliance write (frame 106)."""
|
||||
cap_wire = bw_frames[106]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
_CHUNK3_PARAMS = bytes([0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
|
||||
|
||||
assert sub_cap == 0x71
|
||||
assert offset_cap == 0x002C
|
||||
assert params_cap == _CHUNK3_PARAMS
|
||||
assert len(data_cap) == 46
|
||||
|
||||
built = build_bw_write_frame(
|
||||
0x71, data_cap,
|
||||
offset=0x002C,
|
||||
params=_CHUNK3_PARAMS,
|
||||
)
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_107_confirm_a_sub72(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x72 — confirm A after compliance write (frame 107)."""
|
||||
cap_wire = bw_frames[107]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x72
|
||||
assert data_cap == b""
|
||||
assert offset_cap == 0x0000
|
||||
|
||||
built = build_bw_write_frame(0x72, b"")
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_108_trigger_config_write_sub82(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x82 — trigger config write (frame 108)."""
|
||||
cap_wire = bw_frames[108]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x82
|
||||
assert params_cap == bytes(10)
|
||||
assert len(data_cap) == 29
|
||||
|
||||
# Verify offset formula: data[1] + 2
|
||||
assert offset_cap == data_cap[1] + 2, (
|
||||
f"Trigger write offset formula mismatch: "
|
||||
f"data[1]={data_cap[1]} → expected {data_cap[1]+2}, got {offset_cap}"
|
||||
)
|
||||
|
||||
built = build_bw_write_frame(
|
||||
0x82, data_cap,
|
||||
offset=offset_cap,
|
||||
params=params_cap,
|
||||
)
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_109_trigger_confirm_sub83(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x83 — trigger confirm (frame 109)."""
|
||||
cap_wire = bw_frames[109]
|
||||
sub_cap, _, _, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x83
|
||||
assert data_cap == b""
|
||||
|
||||
built = build_bw_write_frame(0x83, b"")
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_110_waveform_data_write_sub69(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x69 — waveform data write (frame 110)."""
|
||||
cap_wire = bw_frames[110]
|
||||
sub_cap, offset_cap, params_cap, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x69
|
||||
assert params_cap == bytes(10)
|
||||
assert len(data_cap) == 204
|
||||
|
||||
# Verify offset formula: data[1] + 2
|
||||
assert offset_cap == data_cap[1] + 2, (
|
||||
f"Waveform write offset formula mismatch: "
|
||||
f"data[1]={data_cap[1]} → expected {data_cap[1]+2}, got {offset_cap}"
|
||||
)
|
||||
|
||||
built = build_bw_write_frame(
|
||||
0x69, data_cap,
|
||||
offset=offset_cap,
|
||||
params=params_cap,
|
||||
)
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_111_confirm_c_sub74(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x74 — confirm C after waveform data write (frame 111)."""
|
||||
cap_wire = bw_frames[111]
|
||||
sub_cap, _, _, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x74
|
||||
assert data_cap == b""
|
||||
|
||||
built = build_bw_write_frame(0x74, b"")
|
||||
assert built == cap_wire
|
||||
|
||||
def test_frame_112_confirm_a_sub72_end(self, bw_frames: list[bytes]) -> None:
|
||||
"""SUB 0x72 — final confirm A at end of write sequence (frame 112)."""
|
||||
cap_wire = bw_frames[112]
|
||||
sub_cap, _, _, data_cap, _ = _decode_bw_frame(cap_wire)
|
||||
|
||||
assert sub_cap == 0x72
|
||||
assert data_cap == b""
|
||||
|
||||
built = build_bw_write_frame(0x72, b"")
|
||||
assert built == cap_wire
|
||||
|
||||
|
||||
class TestOffsetFormula:
|
||||
"""Verify the offset = data[1] + 2 formula for single-chunk write commands."""
|
||||
|
||||
def test_event_index_offset_formula(self, bw_frames: list[bytes]) -> None:
|
||||
"""Frame 102 (SUB 0x68): offset = data[1] + 2."""
|
||||
_, offset_cap, _, data_cap, _ = _decode_bw_frame(bw_frames[102])
|
||||
assert offset_cap == data_cap[1] + 2
|
||||
|
||||
def test_trigger_config_offset_formula(self, bw_frames: list[bytes]) -> None:
|
||||
"""Frame 108 (SUB 0x82): offset = data[1] + 2."""
|
||||
_, offset_cap, _, data_cap, _ = _decode_bw_frame(bw_frames[108])
|
||||
assert offset_cap == data_cap[1] + 2
|
||||
|
||||
def test_waveform_data_offset_formula(self, bw_frames: list[bytes]) -> None:
|
||||
"""Frame 110 (SUB 0x69): offset = data[1] + 2."""
|
||||
_, offset_cap, _, data_cap, _ = _decode_bw_frame(bw_frames[110])
|
||||
assert offset_cap == data_cap[1] + 2
|
||||
|
||||
|
||||
class TestChecksumVerification:
|
||||
"""Verify large-frame DLE-aware checksum for all write frames."""
|
||||
|
||||
def _verify_checksum(self, wire: bytes, label: str) -> None:
|
||||
inner = wire[2:-1]
|
||||
payload = _destuff(inner)
|
||||
chk = payload[-1]
|
||||
computed = (sum(b for b in payload[2:-1] if b != 0x10) + 0x10) & 0xFF
|
||||
assert computed == chk, (
|
||||
f"{label}: checksum mismatch — computed=0x{computed:02X}, got=0x{chk:02X}"
|
||||
)
|
||||
|
||||
def test_all_write_frame_checksums(self, bw_frames: list[bytes]) -> None:
|
||||
write_frames = {
|
||||
102: "SUB 0x68 event index write",
|
||||
103: "SUB 0x73 confirm B",
|
||||
104: "SUB 0x71 compliance chunk 1",
|
||||
105: "SUB 0x71 compliance chunk 2",
|
||||
106: "SUB 0x71 compliance chunk 3",
|
||||
107: "SUB 0x72 confirm A",
|
||||
108: "SUB 0x82 trigger config write",
|
||||
109: "SUB 0x83 trigger confirm",
|
||||
110: "SUB 0x69 waveform data write",
|
||||
111: "SUB 0x74 confirm C",
|
||||
112: "SUB 0x72 confirm A (end)",
|
||||
}
|
||||
for idx, label in write_frames.items():
|
||||
self._verify_checksum(bw_frames[idx], f"Frame {idx} ({label})")
|
||||
|
||||
|
||||
class TestComplianceChunkSizes:
|
||||
"""Verify compliance write chunk sizes and sequence."""
|
||||
|
||||
def test_chunk1_size(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, _, data, _ = _decode_bw_frame(bw_frames[104])
|
||||
assert len(data) == 1027, f"Chunk 1 should be 1027 bytes, got {len(data)}"
|
||||
|
||||
def test_chunk2_size(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, _, data, _ = _decode_bw_frame(bw_frames[105])
|
||||
assert len(data) == 1055, f"Chunk 2 should be 1055 bytes, got {len(data)}"
|
||||
|
||||
def test_chunk3_size(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, _, data, _ = _decode_bw_frame(bw_frames[106])
|
||||
assert len(data) == 46, f"Chunk 3 should be 46 bytes, got {len(data)}"
|
||||
|
||||
def test_total_compliance_data(self, bw_frames: list[bytes]) -> None:
|
||||
total = sum(
|
||||
len(_decode_bw_frame(bw_frames[i])[3]) for i in [104, 105, 106]
|
||||
)
|
||||
assert total == 2128, f"Total compliance write data should be 2128 bytes, got {total}"
|
||||
|
||||
def test_chunk1_params(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, params, _, _ = _decode_bw_frame(bw_frames[104])
|
||||
assert params == bytes(10)
|
||||
|
||||
def test_chunk2_params(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, params, _, _ = _decode_bw_frame(bw_frames[105])
|
||||
assert params == bytes([0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00])
|
||||
|
||||
def test_chunk3_params(self, bw_frames: list[bytes]) -> None:
|
||||
_, _, params, _, _ = _decode_bw_frame(bw_frames[106])
|
||||
assert params == bytes([0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
|
||||
|
||||
def test_chunk1_offset(self, bw_frames: list[bytes]) -> None:
|
||||
_, offset, _, _, _ = _decode_bw_frame(bw_frames[104])
|
||||
assert offset == 0x1004
|
||||
|
||||
def test_chunk2_offset(self, bw_frames: list[bytes]) -> None:
|
||||
_, offset, _, _, _ = _decode_bw_frame(bw_frames[105])
|
||||
assert offset == 0x1004
|
||||
|
||||
def test_chunk3_offset(self, bw_frames: list[bytes]) -> None:
|
||||
_, offset, _, _, _ = _decode_bw_frame(bw_frames[106])
|
||||
assert offset == 0x002C
|
||||
|
||||
|
||||
# ── Standalone runner ──────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not os.path.exists(CAPTURE_PATH):
|
||||
print(f"ERROR: Capture file not found: {CAPTURE_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
frames = _load_bw_frames(CAPTURE_PATH)
|
||||
print(f"Loaded {len(frames)} BW frames from capture")
|
||||
|
||||
write_frame_indices = list(range(102, 113))
|
||||
all_pass = True
|
||||
print()
|
||||
print(f"{'Frame':>6} {'SUB':>5} {'Offset':>8} {'DataLen':>8} {'Chk OK':>7} {'Rebuilt':>8}")
|
||||
print("-" * 60)
|
||||
for idx in write_frame_indices:
|
||||
wire = frames[idx]
|
||||
sub, offset, params, data, chk = _decode_bw_frame(wire)
|
||||
payload = _destuff(wire[2:-1])
|
||||
computed = (sum(b for b in payload[2:-1] if b != 0x10) + 0x10) & 0xFF
|
||||
chk_ok = computed == chk
|
||||
|
||||
built = build_bw_write_frame(sub, data, offset=offset, params=params)
|
||||
rebuilt_ok = built == wire
|
||||
|
||||
status = "✅" if (chk_ok and rebuilt_ok) else "❌"
|
||||
print(
|
||||
f" {idx:4d} 0x{sub:02X} 0x{offset:04X} {len(data):8d} "
|
||||
f"{'✅' if chk_ok else '❌':>7} {'✅' if rebuilt_ok else '❌':>8} {status}"
|
||||
)
|
||||
if not (chk_ok and rebuilt_ok):
|
||||
all_pass = False
|
||||
|
||||
print()
|
||||
if all_pass:
|
||||
print("All 11 write frames verified ✅")
|
||||
else:
|
||||
print("FAILURES DETECTED ❌")
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user