feat: update s3_bridge to v0.4.0 with annotation markers and dual log output

bridges/s3-bridge/s3_bridge.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """
 s3_bridge.py — S3 <-> Blastware serial bridge with frame-aware session logging
-Version: v0.3.0
+Version: v0.4.0
 
 Key features:
 - Low CPU: avoids per-byte console printing
@@ -9,12 +9,21 @@ Key features:
 - Frame-aware logging: buffers per direction until ETX (0x03), then logs full frame on one line
 - Also logs plain ASCII bursts (e.g., "Operating System") cleanly
 - Dual log output: hex text log (.log) AND raw binary log (.bin) written simultaneously
-- Session log files created on start, closed on Ctrl+C
+- Interactive annotation: type 'm' + Enter to stamp a [MARK] into both logs mid-capture
+- Binary sentinel markers: out-of-band FF FF FF FF <len> <label> in .bin for programmatic correlation
+- Auto-marks on session start and end
 
 Usage examples:
   python s3_bridge.py
   python s3_bridge.py --bw COM5 --s3 COM4 --baud 38400
   python s3_bridge.py --quiet
+
+Annotation:
+  While running, type 'm' and press Enter. You will be prompted for a label.
+  The mark is written to the .log as:
+      [HH:MM:SS.mmm] >>> MARK: your label here
+  And to the .bin as an out-of-band sentinel (never valid frame data):
+      FF FF FF FF <1-byte length> <label bytes>
 """
 
 from __future__ import annotations
@@ -31,7 +40,12 @@ from typing import Optional
 import serial
 
 
-VERSION = "v0.3.0"
+VERSION = "v0.4.0"
+
+# Sentinel prefix for binary markers. Four 0xFF bytes can never appear in
+# valid Instantel DLE-framed data (0xFF is not a legal protocol byte in any
+# framing position), so this sequence is unambiguously out-of-band.
+BIN_MARK_SENTINEL = b"\xFF\xFF\xFF\xFF"
 
 
 def now_ts() -> str:
@@ -74,6 +88,20 @@ class SessionLogger:
         with self._lock:
             self._bin_fh.write(data)
 
+    def log_mark(self, label: str) -> None:
+        """
+        Write an annotation mark to both logs simultaneously.
+
+        .log  — visually distinct line: [TS] >>> MARK: label
+        .bin  — out-of-band sentinel: FF FF FF FF <len> <label utf-8, max 255 bytes>
+        """
+        ts = now_ts()
+        label_bytes = label.encode("utf-8", errors="replace")[:255]
+        sentinel = BIN_MARK_SENTINEL + bytes([len(label_bytes)]) + label_bytes
+        with self._lock:
+            self._fh.write(f"[{ts}] >>> MARK: {label}\n")
+            self._bin_fh.write(sentinel)
+
     def close(self) -> None:
         with self._lock:
             try:
@@ -202,6 +230,43 @@ def forward_loop(
             time.sleep(0.002)
 
 
+def annotation_loop(logger: SessionLogger, stop: threading.Event) -> None:
+    """
+    Runs on the main thread (or a dedicated thread) reading stdin.
+    Type 'm' + Enter to trigger an annotation prompt.
+    Any other non-empty input is ignored with a hint.
+    Bare Enter (empty line) is silently ignored to prevent accidental marks.
+    """
+    print("[MARK] Type 'm' + Enter to annotate the capture. Ctrl+C to stop.")
+    while not stop.is_set():
+        try:
+            line = input()
+        except EOFError:
+            # stdin closed (e.g. piped input exhausted)
+            break
+        except KeyboardInterrupt:
+            break
+
+        line = line.strip()
+        if not line:
+            continue  # bare Enter — ignore silently
+
+        if line.lower() == "m":
+            try:
+                sys.stdout.write("  Label: ")
+                sys.stdout.flush()
+                label = input().strip()
+            except (EOFError, KeyboardInterrupt):
+                break
+            if label:
+                logger.log_mark(label)
+                print(f"  [MARK written] {label}")
+            else:
+                print("  (empty label — mark cancelled)")
+        else:
+            print("  (type 'm' + Enter to annotate)")
+
+
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--bw", default="COM5", help="Blastware-side COM port (default: COM5)")
@@ -229,8 +294,10 @@ def main() -> int:
     logger = SessionLogger(log_path, bin_path)
     print(f"[LOG] Writing hex log to  {log_path}")
     print(f"[LOG] Writing binary log to {bin_path}")
+
     logger.log_line(f"[{now_ts()}] [INFO] s3_bridge {VERSION} start")
     logger.log_line(f"[{now_ts()}] [INFO] BW={args.bw} S3={args.s3} baud={args.baud}")
+    logger.log_mark(f"SESSION START — BW={args.bw} S3={args.s3} baud={args.baud}")
 
     stop = threading.Event()
 
@@ -251,12 +318,19 @@ def main() -> int:
         args=("S3->BW", s3, bw, logger, stop, args.quiet, args.status_every),
         daemon=True,
     )
+    # Annotation loop runs in its own daemon thread so it doesn't block shutdown
+    t_ann = threading.Thread(
+        target=annotation_loop,
+        name="Annotator",
+        args=(logger, stop),
+        daemon=True,
+    )
 
     t1.start()
     t2.start()
+    t_ann.start()
 
     try:
-        # Wait until Ctrl+C
         while not stop.is_set():
             time.sleep(0.05)
     finally:
@@ -266,6 +340,7 @@ def main() -> int:
         stop.set()
         t1.join(timeout=1.0)
         t2.join(timeout=1.0)
+        # t_ann is daemon — don't join, it may be blocked on input()
 
         try:
             bw.close()
@@ -276,6 +351,7 @@ def main() -> int:
         except Exception:
             pass
 
+        logger.log_mark("SESSION END")
         logger.log_line(f"[{now_ts()}] [INFO] ports closed, session end")
         print("[LOG] Closing session log")
         logger.close()
@@ -284,4 +360,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

docs/instantel_protocol_reference.md

@@ -11,10 +11,10 @@
 
 | Date | Section | Change |
 |---|---|---|
-| 2026-02-26 | Initial | Document created from first hex dump analysis |
+| 2026-02-25 | Initial | Document created from first hex dump analysis |
-| 2026-02-26 | §2 Frame Structure | **CORRECTED:** Frame uses DLE-STX (`0x10 0x02`) and DLE-ETX (`0x10 0x03`), not bare `0x02`/`0x03`. `0x41` confirmed as ACK not STX. DLE stuffing rule added. |
+| 2026-02-25 | §2 Frame Structure | **CORRECTED:** Frame uses DLE-STX (`0x10 0x02`) and DLE-ETX (`0x10 0x03`), not bare `0x02`/`0x03`. `0x41` confirmed as ACK not STX. DLE stuffing rule added. |
-| 2026-02-26 | §8 Timestamp | **UPDATED:** Year `0x07CB = 1995` confirmed as MiniMate hardware default date when RTC battery is disconnected. Not an encoding error. Confidence upgraded from ❓ to 🔶. |
+| 2026-02-25 | §8 Timestamp | **UPDATED:** Year `0x07CB = 1995` confirmed as MiniMate hardware default date when RTC battery is disconnected. Not an encoding error. Confidence upgraded from ❓ to 🔶. |
-| 2026-02-26 | §10 DLE Stuffing | **UPGRADED:** Section upgraded from ❓ SPECULATIVE to ✅ CONFIRMED. Full stuffing rules and parser state machine documented. |
+| 2026-02-25 | §10 DLE Stuffing | **UPGRADED:** Section upgraded from ❓ SPECULATIVE to ✅ CONFIRMED. Full stuffing rules and parser state machine documented. |
 | 2026-02-26 | §11 Checksum | **UPDATED:** Frame builder and parser rewritten to handle DLE framing and byte stuffing correctly. |
 | 2026-02-26 | §14 Open Questions | DLE question removed (resolved). Timestamp year question removed (resolved). |
 | 2026-02-26 | §7.2 Serial Number Response | **CORRECTED:** Trailing bytes are `0x79 0x11` only (2 bytes, not 3). `0x20` was misidentified as a trailing byte — it is the frame checksum. |
@@ -29,6 +29,8 @@
 | 2026-02-26 | §5.2 Response SUBs | **STRENGTHENED:** `0xFF - SUB` rule wording clarified — high confidence, no counterexample, not yet formally proven. |
 | 2026-02-26 | §15 → Appendix A | **RENAMED:** Binary log format section moved to Appendix A with explicit note that it describes tooling behavior, not protocol. |
 | 2026-02-26 | Header | **ADDED:** Certainty legend clarification — ratings apply to protocol semantics only, not tooling behavior. |
+| 2026-02-26 | §7.6 Channel Config Float Layout | **NEW SECTION:** Trigger level confirmed as IEEE 754 BE float in in/s. Alarm level identified as adjacent float = 1.0 in/s. Unit string `"in./s"` embedded inline. `0x082A` removed as trigger level candidate. |
+| 2026-02-25 | Appendix A | **UPDATED:** v0.4.0 — annotation markers added. `.bin` sentinel format documented. Parser caveat added for SUB `5A` raw ADC payloads. |
 
 ---
 
@@ -183,7 +185,7 @@ Step 4 — Device sends actual data payload:
 | `25` | **WAVEFORM PAGE B?** | Paged waveform read, possibly channel group B. | 🔶 INFERRED |
 | `1F` | **EVENT ADVANCE / CLOSE** | Sent after waveform download completes. Likely advances internal record pointer. | 🔶 INFERRED |
 | `09` | **UNKNOWN READ A** | Read command, response (`F6`) returns 0xCA (202) bytes. Purpose unknown. | 🔶 INFERRED |
-| `1A` | **CHANNEL SCALING / COMPLIANCE CONFIG READ** | Read command, response (`E5`) returns large block containing IEEE 754 floats and `0x082A` (≈ 0.209 in/s trigger threshold candidate). Bidirectional during compliance setup. | 🔶 INFERRED |
+| `1A` | **CHANNEL SCALING / COMPLIANCE CONFIG READ** | Read command, response (`E5`) returns large block containing IEEE 754 floats including trigger level, alarm level, max range, and unit strings. Contains `0x082A` — purpose unknown, possibly alarm threshold or record config. | 🔶 INFERRED |
 | `2E` | **UNKNOWN READ B** | Read command, response (`D1`) returns 0x1A (26) bytes. Purpose unknown. | 🔶 INFERRED |
 
 All requests use CMD byte `0x02`. All responses use CMD byte `0x10 0x02` (which, after de-stuffing, is just the DLE+CMD combination — see §3).
@@ -379,7 +381,29 @@ Confirmed ASCII strings extracted from payload:
 "MicL"                                          ← Microphone / air overpressure
 ```
 
-Peak values as IEEE 754 big-endian floats — event 1:
+### 7.6 Channel Config Float Layout (SUB E5 / SUB 71)
+
+> ✅ **CONFIRMED — 2026-02-26** from trigger level change capture (session `193237`). Trigger changed `0.500 → 0.200 in/s`, alarm level independently read as `1.0 in/s`.
+
+The SUB `1A` read response (`E5`) and SUB `71` write block contain per-channel threshold and scaling values packed as **IEEE 754 big-endian floats**, with an inline unit string:
+
+```
+[max_range float]  [trigger float]  ["in.\0"]  [alarm float]  ["/s\0\0"]
+ 40 C6 97 FD        3E 4C CC CD      69 6E 2E    3F 80 00 00    2F 73 00 00
+ = 6.206            = 0.200 in/s     "in."       = 1.000 in/s   "/s"
+```
+
+| Float | Value observed | Meaning | Certainty |
+|---|---|---|---|
+| `40 C6 97 FD` | 6.206 | Maximum range (likely full-scale ADC range in in/s) | 🔶 INFERRED |
+| `3E 4C CC CD` | 0.200 | **Geophone trigger level** — changed `0.500 → 0.200` in capture | ✅ CONFIRMED |
+| `3F 80 00 00` | 1.000 | **Geophone alarm level** — matched UI value of 1.0 in/s | ✅ CONFIRMED |
+
+Unit strings `"in.\0"` and `"/s\0\0"` are embedded inline between the floats, confirming values are stored natively in **imperial units (in/s)** regardless of display locale.
+
+> ❓ **`0x082A` (= 2090)** — appears in the same block but did not change when trigger or alarm level was adjusted. Previous hypothesis that it was the trigger level is incorrect. Possibly record time, sample count, or a different threshold. Needs a targeted capture changing a known integer field to identify.
+
+> 🔶 **Pending confirmation:** Alarm level identification is based on value match (`3F 80 00 00` = 1.0 = UI value). A capture changing the alarm level will confirm the exact byte offset.
 ```
 Tran:  3D BB 45 7A  =  0.0916  (in/s — unit config dependent)
 Vert:  3D B9 56 E1  =  0.0907
@@ -608,7 +632,7 @@ Build in this order — each step is independently testable:
 | Channels | Tran, Vert, Long, MicL (4 channels) |
 | Sample Rate | ~1024 sps (🔶 INFERRED) |
 | Bridge Config | COM5 (Blastware) ↔ COM4 (Device), 38400 baud |
-| Capture Tool | s3_bridge v0.4.0 |
+| Capture Tool | s3_bridge v0.4.0 (annotation markers, dual .log/.bin output) |
 
 ---
 
@@ -616,10 +640,13 @@ Build in this order — each step is independently testable:
 
 ## Appendix A — s3_bridge Capture Format
 > ✅ **CONFIRMED — 2026-02-26**
+> ⚠️ **Updated for v0.4.0 — annotation markers added.**
 
 > ⚠️ **This behavior is not part of the Instantel protocol. It is an artifact of the bridge logger implementation.**
 
-The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger makes one modification:
+### A.1 Binary modifications
+
+The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger makes one modification to frame data:
 
 | Wire sequence | In .bin file | Notes |
 |---|---|---|
@@ -633,6 +660,33 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger
 
 > ⚠️ This means checksums cannot be verified on frames where the stuffed payload ends in `0x10` — that trailing `0x10` would normally be the DLE prefix of ETX, but the logger strips it, making the frame boundary ambiguous in that edge case. In practice this has not been observed in captured data.
 
+### A.2 Annotation markers (v0.4.0+)
+
+When the operator types `m` + Enter during a capture, both files receive a marker at that timestamp.
+
+**`.log` format:**
+```
+[HH:MM:SS.mmm] >>> MARK: label text here
+```
+The `>>>` prefix never appears in frame log lines (which use `[direction]`) and is trivially skippable by a parser.
+
+**`.bin` format — out-of-band sentinel:**
+```
+FF FF FF FF <len: 1 byte> <label: len bytes, UTF-8>
+```
+
+The four `0xFF` sentinel bytes are chosen because `0xFF` is not a valid byte in any Instantel framing position:
+- Not a valid ACK (`0x41`), DLE (`0x10`), STX (`0x02`), or ETX (`0x03`)
+- The `0xFF - SUB` response pattern produces values like `0xA4`, `0xEA`, `0xFE` — never a bare `0xFF` in the framing layer
+
+**⚠️ Parser caveat — SUB `5A` raw ADC payloads:**
+The sentinel assumption is robust for the framing layer, but the raw ADC sample data in SUB `5A` bulk waveform streams is less constrained. High-amplitude samples could theoretically produce `FF FF FF FF` within the data portion of a frame. **Do not scan the entire `.bin` file as a flat byte stream for sentinels.** Instead:
+1. Parse frame boundaries first (walk `0x41` ACK → `0x10 0x02` STX → ... → bare `0x03` ETX)
+2. Only scan for `FF FF FF FF` in the **gaps between frames** — sentinels are always written between complete frames, never mid-frame
+3. Any `FF FF FF FF` appearing inside a frame boundary is ADC data, not a marker
+
+Session start and end are automatically marked in both files.
+
 ---
 
 ## 14. Open Questions / Still Needs Cracking
@@ -646,6 +700,9 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger
 | Purpose of SUB `09` / response `F6` — 202-byte read block | MEDIUM | 2026-02-26 |
 | Purpose of SUB `2E` / response `D1` — 26-byte read block | MEDIUM | 2026-02-26 |
 | Full field mapping of SUB `1A` / response `E5` — channel scaling / compliance config block | MEDIUM | 2026-02-26 |
+| `0x082A` in channel config block — not trigger or alarm level. Possibly record time, sample count, or secondary threshold. Needs targeted capture. | MEDIUM | 2026-02-26 |
+| Geophone alarm level float offset confirmation — value match suggests `3F 80 00 00` at known position, needs change capture to confirm. | LOW | 2026-02-26 |
+| Max range float `40 C6 97 FD` = 6.206 — meaning unclear. Screenshot shows "Normal 10.000 in/s" range setting. | LOW | 2026-02-26 |
 | Full trigger configuration field mapping (SUB `1C` / write `82`) | LOW | 2026-02-26 |
 | Whether SUB `24`/`25` are distinct from SUB `5A` or redundant | LOW | 2026-02-26 |
 | Meaning of `0x07 E7` field in config block | LOW | 2026-02-26 |