fix(protocol): improve terminator frame detection in write_blastware_file.

fix: rename .n00 to just blastware file (.n00 was false positive)
This commit is contained in:
2026-04-23 01:33:44 -04:00
parent 8cb8b86192
commit 3eeafd24aa
6 changed files with 70 additions and 59 deletions
+55 -44
View File
@@ -2,7 +2,7 @@
blastware_file.py — Blastware binary file codec for bidirectional interoperability.
Reads and writes the proprietary Instantel/Blastware file formats:
.N00 / .9T0 / .EI0 / etc. — Waveform event (extension encoding UNKNOWN — see below)
Waveform events (.CE0W, .VM0H, .440, .7M0, etc.) (extension encoding UNKNOWN — see below)
.MLG — Monitor log (monitoring session history)
All waveform formats share a common 22-byte file header prefix and identical
@@ -28,7 +28,7 @@ EXTENSION ENCODING — V10.72 firmware FULLY CONFIRMED 2026-04-22:
─── File structure overview ─────────────────────────────────────────────────────
N00 (single-shot waveform, confirmed from example-events/4-3-26-multi/M529LIY6.N00):
Waveform file structure (confirmed from example-events/4-3-26-multi/M529LIY6 (example event)):
[22B header] [21B STRT record] [body bytes] [26B footer]
@@ -36,7 +36,7 @@ N00 (single-shot waveform, confirmed from example-events/4-3-26-multi/M529LIY6.N
10 00 01 80 00 00 — fixed prefix
49 6e 73 74 61 6e 74 65 6c 00 — b'Instantel\x00'
07 2c — fixed
00 12 03 00 — N00 type marker
00 12 03 00 — waveform file type tag (shared by all waveform extensions)
STRT record (21 bytes, immediately follows header):
53 54 52 54 — b'STRT'
@@ -84,10 +84,10 @@ MLG (monitor log, confirmed from example-events/4-3-26-multi/BE11529.MLG):
─── Critical implementation notes ──────────────────────────────────────────────
N00 body reconstruction algorithm (confirmed 2026-04-21 from verification against
M529LIY6.N00 using raw_s3_20260403_153508.bin capture):
Waveform body reconstruction algorithm (confirmed 2026-04-21 from verification against
M529LIY6 (example event) using raw_s3_20260403_153508.bin capture):
The N00 body bytes come from the A5 frame content, stripped of DLE-framing
The waveform body bytes come from the A5 frame content, stripped of DLE-framing
artifacts. Each A5 frame contributes a different slice of its data section,
with DLE+{0x02,0x03,0x04} byte pairs stripped.
@@ -136,8 +136,8 @@ MLG CRC:
All waveform extensions share the same binary format — the extension is set
by blastware_filename() based on the event timestamp and type.
read_n00(path) → Event
Parse a .N00 file into an Event object with waveform data populated.
read_blastware_file(path) → Event
Parse a Blastware waveform file into an Event object with waveform data populated.
(Not yet implemented — placeholder raises NotImplementedError.)
write_mlg(entries, serial, path)
@@ -160,7 +160,7 @@ from .models import Event, MonitorLogEntry, Timestamp
# ── File header constants ─────────────────────────────────────────────────────
# Common 16-byte prefix shared by N00 and MLG (confirmed from binary inspection).
# Common 16-byte prefix shared by waveform files and MLG (confirmed from binary inspection).
_FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c"
# = 10 00 01 80 00 00 49 73 74 61 6e 74 65 6c 00 07 2c (17 bytes)
# Confirmed breakdown: 10 00 01 80 00 00 = fixed; "Instantel\x00" = 10B; 07 2c = fixed
@@ -169,19 +169,19 @@ _FILE_HEADER_PREFIX = bytes.fromhex("1000018000004973") + b"tantel\x00\x07\x2c"
_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 17 bytes
# Waveform file type tag (4 bytes after common prefix) — shared by ALL waveform extensions
_N00_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6.N00 — same tag for .CE0W, .VM0H, etc.
_WAVEFORM_TYPE_TAG = b"\x00\x12\x03\x00" # confirmed from M529LIY6 (example event) — same tag for .CE0W, .VM0H, etc.
# MLG type tag (4 bytes after common prefix)
_MLG_TYPE_TAG = b"\x22\x01\x0e\xa0" # confirmed from BE11529.MLG offset 0x11..0x14
# Total header sizes
_N00_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate.
_WAVEFORM_HEADER_SIZE = 22 # 17 + 4 = 21... wait. Let me recalculate.
# From binary: first 22 bytes = header, then STRT at byte 22.
# 17-byte common prefix + 4-byte type tag = 21 bytes. But observed header is 22B.
# Checking: 6 fixed + 10 "Instantel\x00" + 2 "07 2c" = 18B prefix, then 4B type tag = 22B.
# Re-count: b"\x10\x00\x01\x80\x00\x00" = 6B + b"Instantel\x00" = 10B + b"\x07\x2c" = 2B = 18B prefix.
_FILE_HEADER_PREFIX = b"\x10\x00\x01\x80\x00\x00Instantel\x00\x07\x2c" # 18 bytes
_N00_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅
_WAVEFORM_HEADER_SIZE = 22 # 18 + 4 = 22 bytes ✅
_MLG_HEADER_SIZE = 308 # confirmed from BE11529.MLG
# MLG record marker (4 bytes after 2-byte CRC at start of each record)
@@ -201,10 +201,10 @@ def _encode_ts_be(ts: Optional[datetime.datetime]) -> bytes:
"""
Encode a datetime as an 8-byte big-endian Blastware timestamp.
Format (N00 and MLG record timestamps):
Format (waveform file and MLG record timestamps):
[day][month][year_HI][year_LO][0x00][hour][min][sec]
Big-endian year confirmed from M529LIY6.N00 footer:
Big-endian year confirmed from M529LIY6 (example event) footer:
footer bytes [2..9] = 01 04 07 ea 00 00 1c 08
→ day=1 month=4 year=0x07ea=2026 hour=0 min=28 sec=8 ✅
@@ -270,7 +270,7 @@ def _strip_inner_frame_dles(data: bytes) -> bytes:
Lone 0x10 bytes not followed by {0x02, 0x03, 0x04} are kept as-is.
Confirmed correct by verifying reconstructed N00 body against M529LIY6.N00:
Confirmed correct by verifying reconstructed waveform body against M529LIY6 (example event):
- 0x10 0x02 in terminator → 0x02 kept ✓
- 0x10 0x04 in terminator (month byte) → 0x04 kept ✓
"""
@@ -290,14 +290,14 @@ def _strip_inner_frame_dles(data: bytes) -> bytes:
def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes:
"""
Extract the N00 body contribution from one A5 S3Frame.
Extract the waveform body contribution from one A5 S3Frame.
The contribution is frame.data[skip:] with inner-frame DLE pairs stripped
per _strip_inner_frame_dles(). The chk_byte is temporarily appended before
stripping to handle the split-pair edge case where a DLE at the end of
frame.data is paired with chk_byte.
Split-pair edge case (confirmed for A5[8] of M529LIY6.N00, 2026-04-21):
Split-pair edge case (confirmed for A5[8] of M529LIY6 (example event), 2026-04-21):
S3FrameParser appends DLE+XX pairs as two literal bytes when XX ∉ {DLE, ETX}.
When the LAST occurrence of such a pair straddles the payload/checksum boundary
@@ -319,7 +319,7 @@ def _frame_body_bytes(frame: S3Frame, skip: int) -> bytes:
skip: Number of leading bytes in frame.data to exclude (frame header).
Returns:
bytes — the N00 body contribution for this frame.
bytes — the waveform body contribution for this frame.
"""
if skip >= len(frame.data):
return b""
@@ -383,10 +383,10 @@ _STEM_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
# S353L4H0.9X0W Full Waveform 2025-06-23 14:01:09 AB=9X=357 ✓
#
# OLD FIRMWARE (S338, 3-char extensions ending in '0') — UNKNOWN:
# Observed: .N00, .9T0, .EI0, .490, .5K0, .980, .ML0
# Observed (old firmware / manual downloads): .440, .470, .7M0, .9T0, .EI0, etc.
# The V10.72 formula does NOT apply to these.
# Extension is NOT recording mode (refuted 2026-04-21: continuous → .EI0, not .9T0).
# blastware_filename() returns .N00 as a placeholder for old-firmware units.
# blastware_filename() computes the correct AB0 extension for V10.72 firmware.
#
# WRONG earlier assumption (do not re-introduce):
# Extension was believed to encode recording mode × sample rate.
@@ -502,15 +502,15 @@ def blastware_filename(event: Event, serial: str, ach: bool = False) -> str:
return prefix + stem + ext
# ── N00 file writer ───────────────────────────────────────────────────────────
# ── Waveform file writer ───────────────────────────────────────────────────────────
def write_n00(
def write_blastware_file(
event: Event,
a5_frames: list[S3Frame],
path: Union[str, Path],
) -> None:
"""
Write a Blastware .N00 waveform file from a downloaded event.
Write a Blastware waveform file from a downloaded event.
Args:
event: Event object (populated by get_events() or download_waveform()).
@@ -520,7 +520,7 @@ def write_n00(
read_bulk_waveform_stream() when collecting frames.
Must have at least 2 frames (probe + terminator).
path: Destination file path. Parent directory must exist.
Extension is not enforced — caller should use ".N00".
Extension should be set via blastware_filename().
File layout:
[22B header] [21B STRT] [body bytes] [26B footer]
@@ -529,7 +529,7 @@ def write_n00(
ValueError: if a5_frames is empty or has no terminator (page_key=0).
OSError: if the file cannot be written.
Confirmed correct N00 body reconstruction against M529LIY6.N00 (2026-04-21).
Confirmed correct waveform body reconstruction against M529LIY6 (example event) (2026-04-21).
"""
if not a5_frames:
raise ValueError("a5_frames must not be empty")
@@ -538,11 +538,11 @@ def write_n00(
# ── Extract STRT record from probe frame ────────────────────────────────
# The STRT record (21 bytes) lives verbatim inside A5[0].data[7:].
# It is stored as-is in the N00 file — do NOT reconstruct it from Event
# It is stored as-is in the waveform file — do NOT reconstruct it from Event
# fields, as bytes [10:14] and [14:20] contain device-specific values
# (not simply key4 repeated or zero-padded). Confirmed 2026-04-21.
#
# STRT layout (21 bytes, observed in M529LIY6.N00):
# STRT layout (21 bytes, observed in M529LIY6 files):
# [0:4] b'STRT'
# [4:6] 0xff 0xfe (fixed)
# [6:10] key4 (event key)
@@ -556,7 +556,7 @@ def write_n00(
# Blastware file stores the stripped form, so we must strip before extracting.
#
# Example (M529LK0Y, 2026-04-21): STRT contains value 0x02 encoded as [10 02]
# on the wire. Without stripping, STRT is 22 raw bytes → write_n00 writes the
# on the wire. Without stripping, STRT is 22 raw bytes → write_blastware_file writes the
# DLE prefix into the file AND begins the body 1 byte too early (probe_skip off
# by 1). Stripping fixes both.
#
@@ -594,28 +594,39 @@ def write_n00(
if len(strt) != 21:
raise ValueError(f"STRT record must be 21 bytes, got {len(strt)}")
# ── Build N00 header ─────────────────────────────────────────────────────
header = _FILE_HEADER_PREFIX + _N00_TYPE_TAG
assert len(header) == _N00_HEADER_SIZE, f"N00 header must be {_N00_HEADER_SIZE} bytes"
# ── Build waveform file header ─────────────────────────────────────────────────────
header = _FILE_HEADER_PREFIX + _WAVEFORM_TYPE_TAG
assert len(header) == _WAVEFORM_HEADER_SIZE, f"Waveform header must be {_WAVEFORM_HEADER_SIZE} bytes"
# ── Build body from A5 frames ────────────────────────────────────────────
# The N00 body is reconstructed from ALL A5 frames (data + terminator).
# The waveform body is reconstructed from ALL A5 frames (data + terminator).
# The terminator frame's contribution includes the 26-byte footer at its end.
#
# Reconstruction layout (confirmed from M529LIY6.N00, 2026-04-21):
# Reconstruction layout (confirmed from M529LIY6 captures, 2026-04-21):
# all_bytes = contributions from A5[0..N] + terminator_contribution
# body = all_bytes[:-26] (everything except the last 26 bytes)
# footer = all_bytes[-26:] (last 26 bytes = the N00 footer)
# footer = all_bytes[-26:] (last 26 bytes = the waveform file footer)
#
# The footer bytes come directly from the terminator frame's inner content —
# using them verbatim ensures timestamps match the device's recorded values.
# Separate terminator from data frames
body_frames = a5_frames
term_frame: Optional[S3Frame] = None
if a5_frames and a5_frames[-1].page_key == 0x0000:
body_frames = a5_frames[:-1]
term_frame = a5_frames[-1]
# Separate terminator from data frames.
# Search from the FRONT for the first terminator (page_key == 0x0000).
# Do NOT use a5_frames[-1] — if _a5_frames contains stray frames from a
# subsequent event (a known get_events side-effect), the last frame will
# not be the terminator and the footer will be mis-identified.
term_idx: Optional[int] = None
for _i, _f in enumerate(a5_frames):
if _f.page_key == 0x0000:
term_idx = _i
break
if term_idx is not None:
body_frames = a5_frames[:term_idx]
term_frame = a5_frames[term_idx]
else:
body_frames = a5_frames
term_frame = None
all_bytes = bytearray()
@@ -660,14 +671,14 @@ def write_n00(
f.write(footer)
def read_n00(path: Union[str, Path]) -> Event:
def read_blastware_file(path: Union[str, Path]) -> Event:
"""
Parse a Blastware .N00 file into an Event object.
Parse a Blastware waveform file into an Event object.
NOT YET IMPLEMENTED.
Args:
path: Path to the .N00 file.
path: Path to the waveform file.
Returns:
Event object with waveform data populated.
@@ -675,7 +686,7 @@ def read_n00(path: Union[str, Path]) -> Event:
Raises:
NotImplementedError: always (pending implementation).
"""
raise NotImplementedError("read_n00() is not yet implemented")
raise NotImplementedError("read_blastware_file() is not yet implemented")
# ── MLG file writer ───────────────────────────────────────────────────────────
+7 -7
View File
@@ -608,7 +608,7 @@ class MiniMateClient:
)
if a5_frames:
a5_ok = True
ev._a5_frames = a5_frames # store for write_n00
ev._a5_frames = a5_frames # store for write_blastware_file
_decode_a5_metadata_into(a5_frames, ev)
_decode_a5_waveform(a5_frames, ev)
log.info(
@@ -624,7 +624,7 @@ class MiniMateClient:
)
if a5_frames:
a5_ok = True
ev._a5_frames = a5_frames # store for write_n00
ev._a5_frames = a5_frames # store for write_blastware_file
_decode_a5_metadata_into(a5_frames, ev)
log.debug(
"get_events: 5A metadata client=%r operator=%r",
@@ -783,29 +783,29 @@ class MiniMateClient:
def save_blastware_file(self, event: "Event", path: "Union[str, Path]", serial: str) -> None:
"""
Download the full waveform for *event* and save it as a Blastware-
compatible .N00 / .9T0 file at *path*.
compatible Blastware waveform file at *path*.
This is a convenience wrapper that calls download_waveform() (which
performs the complete SUB 5A BULK_WAVEFORM_STREAM download) and then
calls write_n00() from blastware_file.py to encode the result.
calls write_blastware_file() from blastware_file.py to encode the result.
Args:
event: Event object with waveform key populated (from get_events()).
path: Destination file path. Caller should use blastware_filename()
to pick the correct .N00 / .9T0 extension.
to pick the correct extension via blastware_filename().
serial: Device serial number (e.g. "BE11529") — passed to
blastware_filename() for reference, but the caller supplies
the final path.
"""
from pathlib import Path as _Path
from .blastware_file import write_n00 as _write_n00
from .blastware_file import write_blastware_file as _write_blastware_file
a5_frames = self.download_waveform(event)
if not a5_frames:
raise RuntimeError(
f"save_blastware_file: no A5 frames received for event#{event.index}"
)
_write_n00(event, a5_frames, path)
_write_blastware_file(event, a5_frames, path)
log.info(
"save_blastware_file: wrote %s (%d A5 frames)",
path, len(a5_frames),
+1 -1
View File
@@ -458,7 +458,7 @@ class S3Frame:
data: bytes # payload data section (payload[5:], checksum already stripped)
checksum_valid: bool
chk_byte: int = 0 # actual checksum byte received from wire (body[-1])
# needed for N00 file reconstruction: when the last data byte
# needed for waveform file reconstruction: when the last data byte
# is 0x10 and chk_byte ∈ {0x02, 0x03, 0x04}, the DLE+chk pair
# must be included in the DLE-strip operation to correctly
# reconstruct the Blastware binary body.
+1 -1
View File
@@ -494,7 +494,7 @@ class Event:
_waveform_key: Optional[bytes] = field(default=None, repr=False)
# Raw A5 frames from the full bulk waveform download (full_waveform=True).
# Populated by get_events() when full_waveform=True; used by write_n00().
# Populated by get_events() when full_waveform=True; used by write_blastware_file().
_a5_frames: Optional[list] = field(default=None, repr=False)
def __str__(self) -> str:
+2 -2
View File
@@ -545,7 +545,7 @@ class MiniMateProtocol:
By default the termination frame (page_key=0x0000) is NOT included in the
returned list. Pass include_terminator=True to append it; the blastware_file
writer needs the terminator frame's body to reconstruct the N00 footer.
writer needs the terminator frame's body to reconstruct the waveform file footer.
Args:
key4: 4-byte waveform key from EVENT_HEADER (1E).
@@ -557,7 +557,7 @@ class MiniMateProtocol:
(default 32; a typical event uses 9 large frames).
include_terminator: If True, append the terminator A5 frame
(page_key=0x0000) to the returned list. The
terminator carries the N00 footer bytes.
terminator carries the waveform file footer bytes.
Default False preserves existing caller behaviour.
Returns:
+4 -4
View File
@@ -61,7 +61,7 @@ from minimateplus import MiniMateClient
from minimateplus.protocol import ProtocolError
from minimateplus.models import CallHomeConfig, ComplianceConfig, DeviceInfo, Event, PeakValues, ProjectInfo, Timestamp
from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT
from minimateplus.blastware_file import write_n00, blastware_filename
from minimateplus.blastware_file import write_blastware_file, blastware_filename
from sfm.cache import SFMCache, get_cache
from sfm.database import SeismoDb
@@ -874,7 +874,7 @@ def device_event_blastware_file(
to be populated from compliance config)
Performs: POLL startup get_events(full_waveform=False, stop_after_index=index)
write_n00() FileResponse.
write_blastware_file() FileResponse.
"""
log.info(
"GET /device/event/%d/blastware_file port=%s host=%s",
@@ -890,7 +890,7 @@ def device_event_blastware_file(
# the full bulk download. Using full_waveform=True produces a file
# ~8x larger than Blastware's because it includes all post-event
# silence chunks. The metadata-only a5_frames (with terminator) are
# sufficient for byte-perfect write_n00 output.
# sufficient for byte-perfect write_blastware_file output.
events = client.get_events(full_waveform=False, stop_after_index=index)
matching = [ev for ev in events if ev.index == index]
return matching[0] if matching else None, info
@@ -928,7 +928,7 @@ def device_event_blastware_file(
# Write to /tmp so FastAPI can stream it back
out_path = Path("/tmp") / filename
write_n00(ev, a5_frames, out_path)
write_blastware_file(ev, a5_frames, out_path)
log.info(
"blastware_file: wrote %s (%d A5 frames, serial=%s)",
out_path, len(a5_frames), serial,