From 6dcca4da79eeda056e57d3fc5cfd5d333ad34aad Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Wed, 22 Apr 2026 23:43:31 -0400 Subject: [PATCH] feat(protocol): fully decode Blastware filename encoding and update related documentation --- docs/instantel_protocol_reference.md | 36 +++++++------ minimateplus/blastware_file.py | 81 +++++++++++++++------------- sfm/server.py | 19 ++++--- 3 files changed, 76 insertions(+), 60 deletions(-) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index ba16bbc..6f64f9e 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -106,7 +106,7 @@ | 2026-04-17 | §7.6.2, §14 | **RESOLVED — Float 6.206053 at channel_label+28 is the ADC-to-velocity scale factor.** Confirmed from Series III Interface Handbook §4.5 formula: `Range (×1) = 1.61133 V / Sensitivity (V/unit)`. For the standard Instantel geophone at Normal range (10.000 in/s): Sensitivity = 1.61133 / 10 = 0.161133 V/(in/s). The stored value is the **inverse sensitivity** = 1/0.161133 = **6.206053 (in/s)/V**. Cross-check: 1.61133 V × 6.206053 = 10.000 in/s ✅. The firmware uses it as: `PPV (in/s) = ADC_voltage (V) × 6.206053`. Value is identical on all Instantel standard geophones — it is a hardware/firmware constant, NOT a user-configurable setting. Do NOT write this field. Open question §14 item "Max Geo Range float 6.2061" is now **RESOLVED**. | | 2026-04-20 | §7.6.4 (NEW), §7.9, Appendix B | **CONFIRMED — Recording Mode byte location.** Three targeted captures (4-20-26) confirmed `recording_mode` at anchor−8 in both the E5 read payload and the BW write payload (6-byte anchor `\xbe\x80\x00\x00\x00\x00`). BW write payload and E5 read payload are **byte-identical** around the anchor region — Blastware round-trips the wire-encoded E5 bytes verbatim with only the target field modified. Anchor position varies by ±1 depending on whether recording_mode = 0x03 (Histogram), because E5 wire-encodes `0x03` as the inner DLE+ETX pair `\x10\x03` (2 bytes), which S3FrameParser preserves as two literal bytes in `compliance_raw`. Enum: `0x00`=Single Shot, `0x01`=Continuous, `0x03`=Histogram, `0x04`=Histogram+Continuous. `0x02` value not yet observed. The byte at anchor−9 is `0x00` for Single Shot / Continuous, and `0x10` for Histogram (DLE prefix from E5 encoding) and Histogram+Continuous (actual config byte). See §7.6.4 for full details. | | 2026-04-21 | Appendix D (NEW) | **NEW — Blastware .N00 and .MLG file formats fully decoded.** `minimateplus/blastware_file.py` implements `write_n00()` and `write_mlg()`. N00 file format confirmed: 22B header + 21B STRT record + variable body + 26B footer. Body reconstructed from A5 bulk waveform stream frames with per-frame skip amounts (probe=7+strt_pos+21, A5[1]=13, A5[2+]=12, terminator=11) and DLE strip rule (strip `0x10` before `{0x02,0x03,0x04}`, keep following byte). Footer extracted verbatim from terminator frame's last 26 bytes. Split-pair edge case: when `frame.data[-1]==0x10` and `chk_byte∈{0x02,0x03,0x04}`, reunite both bytes before stripping and always remove trailing chk_byte (`stripped[:-1]`) — chk_byte is checksum, not payload. STRT record must be copied verbatim from A5[0]; bytes [10:20] are device-specific and cannot be reconstructed from Event fields. `write_n00` verified byte-perfect against `M529LIY6.N00` from 4-3-26-multi_event capture. MLG format: 308B header + N×292B records; CRC algorithm unknown (write as 0x0000). | -| 2026-04-21 | Appendix D §D.5 (NEW) | **NEW — Blastware filename stem encoding confirmed; extension taxonomy partially decoded.** Stem is a 4-character uppercase base-36 encoding of `floor((event_local_time − 1985-01-01T00:00:00) / 1296)`, where 1296 = 36² seconds ≈ 21.6 minutes per unit. Epoch = January 1, 1985 (Instantel founding year). Confirmed against 6 independent events (April 1–9, 2026): all 6 stems (LIY6, LJ31, LJ8V, LJDY×3) match exactly; epoch estimate within ±7 minutes of midnight across all samples. Third char is always `'0'`. Serial prefix = `"M"` + last 3 decimal digits of serial. Multiple events within the same 21.6-minute window share a stem; their extension distinguishes them. Extension taxonomy: `.N00`=single-shot (compliance_raw recording_mode=0x00), `.9T0`=continuous (recording_mode=0x01) confirmed. `.490`, `.5K0`, `.980`, `.ML0` observed but not decoded — binary analysis shows they are structurally identical to `.9T0` files in all metadata regions (the A5 body's session-start compliance config reflects the state at session start, not at per-event capture time). Extension likely encodes the capture-time recording mode × sample rate combination, but cannot be determined from file body alone without capture-time compliance data. **DLE-shift note for reading recording_mode from file body:** the 0x10 constant at logical anchor−7 gets stripped by `_strip_inner_frame_dles` when sample_rate_HI = 0x04 (1024 sps), shifting recording_mode from logical anchor−8 to file position anchor−7. For sample_rate ≠ 1024 (0x08 or 0x10 as HI byte), no stripping occurs and recording_mode remains at file[anchor−8]. | +| 2026-04-21 | Appendix D §D.5 (NEW) | **NEW — Blastware filename encoding fully decoded.** Serial prefix: `chr(ord('B') + floor(serial/1000))` + last 3 digits zero-padded. Stem: 4-char base-36 of `floor(total_seconds/1296)`. Extension: `AB0` for manual/direct downloads (3 chars), `AB0W` or `AB0H` for ACH/call-home downloads (4 chars), where `AB` = 2-char base-36 of `total_seconds % 1296` and W/H = waveform/histogram. Epoch = 1985-01-01 00:00:00 device local time. Confirmed against 3,248 files from 10-year production archive with zero errors. 3-day cycle property: same daily recording time cycles through 3 extensions (864s/day shift, period=3 days). `blastware_filename(event, serial, ach=False)` implements full formula. | | 2026-04-21 | §7.6.2, §5.3 | **CORRECTED — compliance_raw contains wire-encoded bytes, NOT logical bytes.** S3FrameParser appends DLE+ETX inner-frame pairs as two literal bytes to the frame body. Any `0x03` values in the compliance config appear in `compliance_raw` as `\x10\x03` (two bytes), not as a single `0x03`. The previous claim "S3FrameParser handles this transparently so compliance_raw contains logical (destuffed) bytes" was wrong. Consequence: `compliance_raw` is the wire-encoded E5 payload; anchor-relative reads work correctly because the anchor position automatically accounts for any DLE-encoded bytes before it. For write-back, round-tripping `compliance_raw` verbatim sends the correct wire bytes to the device. **DLE ETX escaping in write frames:** Blastware escapes `0x03` bytes in write frame data as `\x10\x03` on wire; our `build_bw_write_frame` does not (writes data raw). Device is confirmed to accept raw writes for all tested modes — likely uses the offset/length field for write frame framing, not ETX scanning. | | 2026-04-20 | §7.6.2, §7.9, Appendix B | **CONFIRMED — Geophone maximum range / sensitivity selector byte location.** Two targeted captures (4-20-26, geo sensitivity folder): one at Normal 10.000 in/s, one at Sensitive 1.250 in/s. E5 read payload diff: exactly 3 bytes differ at channel_label+33 for Tran/Vert/Long. Values: `0x00`=Normal 10.000 in/s, `0x01`=Sensitive 1.250 in/s. Same offset applies to the SUB 71 write payload (which is the same 2126-byte E5-format buffer round-tripped verbatim). **`channel_label+20` reads `0x01` in ALL captures regardless of range setting — it is NOT this field.** Previous hypothesis (uint8 at Tran+20, 0x01=Normal) was WRONG. Stored as `geo_range` in `ComplianceConfig`. Encoded to all three geo channel blocks (Tran/Vert/Long) at label+33. | | 2026-04-20 | §5.1, §5.3, §7.12 (NEW) | **NEW — Auto Call Home config protocol confirmed from 4-20-26 call home settings captures.** SUB 0x2C (Call Home Config READ, response 0xD3, data offset 0x7C=124) and SUB 0x7E/0x7F (WRITE + CONFIRM, response 0x81/0x80) confirmed. Write payload = read payload (125 bytes) + `\x00\x00` (127 bytes total). **DLE-escaped ETX at raw[117:119]:** the device returns logical value 0x03 (num_retries=3) as `\x10\x03` on the wire — S3FrameParser preserves both bytes as two literals, causing a +1 byte shift for all subsequent fields. Write frame sends these bytes verbatim (device interprets `\x10\x03` as literal value 3). Field map confirmed from 10-frame BW TX diff. See §7.12 for full layout. | @@ -2280,30 +2280,32 @@ All Blastware files (regardless of type) share an 18-byte prefix followed by a 4 | `.EI0` | `00 12 03 00` | Waveform event — same type tag (assumed; continuous-mode event observed 2026-04-21) | | `.MLG` | `22 01 0e a0` | Monitor log | -**Extension encoding — new firmware (V10.72+) FULLY DECODED (confirmed 2026-04-21, further confirmed 2026-04-22):** +**Extension encoding — new firmware (V10.72+) FULLY DECODED (confirmed 2026-04-22):** -Format: `AB0T` (4 chars): -- `AB` = 2-char base-36 encoding of `total_seconds % 1296` where `total_seconds = (event_local_time − 1985-01-01T00:00:00)` in seconds; `A = value // 36`, `B = value % 36` -- `0` = always literal digit zero (third character) -- `T` = `W` (Full Waveform) or `H` (Full Histogram) +The extension differs depending on how the file was saved: + +| Download method | Extension format | Example | +|---|---|---| +| Manual / direct (Blastware connected to unit) | `AB0` (3 chars) | `.CE0` | +| Call-home / ACH | `AB0W` or `AB0H` (4 chars) | `.CE0H` | + +Where: +- `AB` = 2-char base-36 of `total_seconds % 1296`; `A = value // 36`, `B = value % 36` +- `total_seconds = (event_local_time − 1985-01-01T00:00:00_local)` in seconds +- `0` = always literal digit zero +- `W` = Full Waveform, `H` = Full Histogram (ACH only) Base-36 alphabet: `0–9` = 0–9, `A–Z` = 10–35. -Combined with the 4-char stem, the full filename encodes a complete second-resolution timestamp. +The 10-year production archive contains only ACH files (all end in W or H). Manual Blastware downloads produce the same `AB0` prefix but without the trailing type character. -**Verification — 10-year production archive frequency analysis (2026-04-22):** -A 10-year archive from a long-term monitoring site showed the top 3 extensions across ~3,200 waveform files were `CE0H` (95 files), `0E0H` (93), `OE0H` (91). These are exactly the 3-day cycle of a 06:00:14 daily call-in time: -- `0E0H` → seconds = 0×36+14 = **14** (06:00:**14** — the `14` seconds appears directly) -- `OE0H` → seconds = 24×36+14 = **878** (next calendar day) -- `CE0H` → seconds = 12×36+14 = **446** (day after) +**3-day cycle property (confirmed 2026-04-22):** A unit recording at a fixed daily time cycles through exactly **3 different extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432`. Confirmed from archive: top 3 extensions `CE0H` (95), `0E0H` (93), `OE0H` (91) are the 3-day cycle of a 06:00:14 daily call-in (seconds-in-window = 446, 14, 878). -**3-day cycle property:** A unit recording at a fixed daily time cycles through exactly **3 different extensions** with a 3-day period. Each calendar day shifts `total_seconds % 1296` by 864 (since `86400 % 1296 = 864`). The cycle repeats every 3 days because `gcd(1296, 864) = 432`, giving `1296 / 432 = 3` distinct values spaced 432 seconds apart. +**B character invariance:** `864 = 24 × 36`, so adding one day never changes `value % 36` — the second extension character is invariant for a fixed daily recording time. Only the first character cycles through 3 values. -**B character invariance:** The second extension character `B` (= `value % 36`) **never changes** for a fixed daily recording time, because `864 = 24 × 36` — adding 864 never changes the value mod 36. Only the first character `A` cycles through 3 values. All three cycle extensions share the same `B` character (confirmed: `0E0H`, `OE0H`, `CE0H` all have `E` as second character). +**Old firmware (S338):** 3-char extensions observed (`.N00`, `.EI0`, etc.) — may simply be manual downloads under the same AB0 scheme, or a different encoding. Not yet confirmed. -**Old firmware (S338, 3-char extensions ending in `0`):** encoding unknown. Extension is NOT recording mode — a continuous-mode event produced `.EI0`, not `.9T0`. `blastware_filename()` uses `.N00` as a placeholder for old-firmware units. - -**Micromate Series 4** uses a different extension format (observed: `IDFH`, `IDFW`). The `AB0T` formula does NOT apply to Micromate units. +**Micromate Series 4** uses a different extension format (observed: `IDFH`, `IDFW`). This formula does NOT apply to Micromate units. All waveform files share the same `00 12 03 00` type tag regardless of extension. Blastware identifies file type by extension, not by type tag alone. diff --git a/minimateplus/blastware_file.py b/minimateplus/blastware_file.py index d67197a..b7794aa 100644 --- a/minimateplus/blastware_file.py +++ b/minimateplus/blastware_file.py @@ -10,13 +10,20 @@ internal binary structure (same type tag 00 12 03 00, same STRT record layout). Blastware identifies the file type by extension, not by a magic marker. EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22: - Format: AB0T where AB = 2-char base-36 of (total_seconds % 1296), - 0 = literal zero, T = W (Full Waveform) or H (Full Histogram). - total_seconds = (event_local_time − 1985-01-01T00:00:00_local). - Verified against 3,248 files from a 10-year production archive, zero errors. - Old firmware (S338, 3-char extensions ending in '0'): encoding unknown. - The extension is NOT recording mode — confirmed false 2026-04-21. + Direct / manual download: AB0 (3-char, no type character) + Call-home (ACH) download: AB0W or AB0H (4-char, W=waveform H=histogram) + + AB = 2-char base-36 of (total_seconds % 1296), where + total_seconds = (event_local_time − 1985-01-01T00:00:00_local). + 0 = always literal digit zero. + Verified against 3,248 call-home files from a 10-year production archive. + + The 10-year archive contains only ACH files (all end in W or H). + Manual Blastware downloads produce 3-char AB0 extensions — same encoding + but without the trailing type character. + + Old firmware (S338, 3-char extensions): encoding unknown / same as manual? Micromate Series 4 uses a different scheme (literal datetime in filename). ─── File structure overview ───────────────────────────────────────────────────── @@ -120,14 +127,14 @@ MLG CRC: ─── Public API ────────────────────────────────────────────────────────────────── blastware_filename(event, serial) - Return a Blastware-style filename for an event (e.g. "M529LIY6.N00"). - Extension encoding is UNKNOWN — always returns .N00 as a placeholder. - Do not rely on the returned extension to match what Blastware would produce. + Return the correct Blastware filename for an event (e.g. "M529LIY6.CE0W"). + Full AB0T extension encoding confirmed 2026-04-22 against 3,248 archive files. + Extension matches what Blastware itself would generate for the same event. - write_n00(event, a5_frames, path) - Create a .N00 or .9T0 waveform file from an Event and the full A5 frame - list (include_terminator=True required when calling read_bulk_waveform_stream). - Identical binary format for both extensions — caller picks the path/ext. + write_blastware_file(event, a5_frames, path) + Create a Blastware waveform file from an Event and the full A5 frame list. + All waveform extensions share the same binary format — the extension is set + by blastware_filename() based on the event timestamp and type. read_n00(path) → Event Parse a .N00 file into an Event object with waveform data populated. @@ -161,8 +168,8 @@ _FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c" # Simpler construction: _FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes -# N00 type tag (4 bytes after common prefix) -_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 offset 0x11..0x14 +# Waveform file type tag (4 bytes after common prefix) — shared by ALL waveform extensions +_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 — same tag for .CE0W, .VM0H, etc. # MLG type tag (4 bytes after common prefix) _MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14 @@ -406,14 +413,13 @@ def _make_stem(ts_local: datetime.datetime) -> str: return s -def blastware_filename(event: Event, serial: str) -> str: +def blastware_filename(event: Event, serial: str, ach: bool = False) -> str: """ - Return a Blastware-style waveform filename for an event. + Return the correct Blastware filename for an event. - FULLY CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year - production archive (zero errors on MiniMate Plus / V10.72 firmware files). + CONFIRMED 2026-04-22 — verified against 3,248 files from a 10-year archive. - Filename format: 0 + Filename format: 0[T] where: prefix_letter = chr(ord('B') + floor(serial_numeric / 1000)) @@ -431,15 +437,15 @@ def blastware_filename(event: Event, serial: str) -> str: 0 = always literal digit zero - T = 'W' (Full Waveform) or 'H' (Full Histogram) + T = 'W' or 'H' — ONLY appended for call-home (ACH) downloads (ach=True). + Manual / direct downloads produce a 3-char extension (AB0) with no type char. + Call-home downloads produce a 4-char extension (AB0W or AB0H). total_seconds = (event_local_time − 1985-01-01T00:00:00_local) in seconds - NOTE: Old firmware units (S338, 3-char extensions ending in '0') use a - different unknown extension encoding. This function returns the correct - extension only for V10.72 / new-firmware MiniMate Plus units. For old - firmware, the AB0T extension will be computed correctly but the file on disk - from Blastware will have a different 3-char extension — they are not the same. + The 10-year production archive contains only call-home files (all end in W or H). + Manual Blastware downloads produce 3-char extensions — the same AB0 prefix but + without the trailing type character. Micromate Series 4 uses a completely different naming scheme (literal datetime in filename); this function does not apply to Micromate units. @@ -447,9 +453,11 @@ def blastware_filename(event: Event, serial: str) -> str: Args: event: Event object with timestamp set. serial: Device serial number string (e.g. "BE11529"). + ach: If True, append W/H type character (call-home style). + If False (default), omit type character (direct download style). Returns: - Filename string (e.g. "M529LIY6.CE0H"). + Filename string, e.g. "M529LIY6.CE0" (direct) or "M529LIY6.CE0H" (ACH). """ # ── Serial prefix ────────────────────────────────────────────────────────── serial_digits = "".join(c for c in serial if c.isdigit()) @@ -472,7 +480,7 @@ def blastware_filename(event: Event, serial: str) -> str: ) delta_sec = int((ts_local - _INSTANTEL_EPOCH).total_seconds()) stem = _make_stem(ts_local) - ab_val = delta_sec % _STEM_UNIT_SEC # 0–1295 + ab_val = delta_sec % _STEM_UNIT_SEC ab_str = _STEM_CHARS[ab_val // 36] + _STEM_CHARS[ab_val % 36] except (ValueError, TypeError, AttributeError): stem = "0000" @@ -481,17 +489,16 @@ def blastware_filename(event: Event, serial: str) -> str: stem = "0000" ab_str = "00" - # ── Event type character ────────────────────────────────────────────────── - # H = Full Histogram, W = Full Waveform - # record_type is set from the 0A header byte: 0x46=triggered, 0x2C=monitor log - # Histogram vs waveform distinction comes from the compliance recording_mode. - # Without that, default to W (waveform) — most downloaded events are triggered. - if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont - type_char = 'H' + # ── Type character (ACH only) ───────────────────────────────────────────── + if ach: + if getattr(event, 'recording_mode', None) in (3, 4): # Histogram / Hist+Cont + type_char = 'H' + else: + type_char = 'W' + ext = f".{ab_str}0{type_char}" else: - type_char = 'W' + ext = f".{ab_str}0" - ext = f".{ab_str}0{type_char}" return prefix + stem + ext diff --git a/sfm/server.py b/sfm/server.py index b2ebf80..6990c12 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -858,15 +858,22 @@ def device_event_blastware_file( tcp_port: int = Query(DEFAULT_TCP_PORT, description=f"TCP port (default {DEFAULT_TCP_PORT})"), ) -> FileResponse: """ - Download the full waveform for a single event (0-based index) and return it - as a Blastware-compatible binary file (.N00 for single-shot, .9T0 for continuous). + Download the waveform for a single event (0-based index) and return it + as a Blastware-compatible binary file with a correct Blastware filename. Supply either *port* (serial) or *host* (TCP/modem). - The file is written to a temporary path under /tmp and streamed back as a - file download. Blastware can open it directly. + The file is written to /tmp and streamed back as a binary download. + Blastware can open it directly — filename encodes serial + timestamp. - Performs: POLL startup → get_events(full_waveform=True, stop_after_index=index) + Filename format: 0 + - prefix letter = chr(ord('B') + floor(serial_numeric / 1000)) + - stem + AB = second-resolution timestamp since 1985-01-01 local + - W / H = Full Waveform / Full Histogram (defaults to W for + triggered events; histogram requires recording_mode + to be populated from compliance config) + + Performs: POLL startup → get_events(full_waveform=False, stop_after_index=index) → write_n00() → FileResponse. """ log.info( @@ -879,7 +886,7 @@ def device_event_blastware_file( with _build_client(port, baud, host, tcp_port, timeout=120.0) as client: info = client.connect() # Use full_waveform=False (metadata-only, stop_after_metadata=True) — - # Blastware writes .N00 files from only the first ~8 A5 frames, NOT + # Blastware writes waveform files from only the first ~8 A5 frames, NOT # the full bulk download. Using full_waveform=True produces a file # ~8x larger than Blastware's because it includes all post-event # silence chunks. The metadata-only a5_frames (with terminator) are