feat: added raw capture pipeline. added simple windows gui.

bridges/gui_bridge.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""
+gui_bridge.py — simple Tk GUI wrapper for s3_bridge.py (Windows-friendly).
+
+Features:
+  - Select BW and S3 COM ports, baud, log directory.
+  - Optional raw taps (BW->S3, S3->BW).
+  - Start/Stop buttons spawn/terminate s3_bridge as a subprocess.
+  - Live stdout view from the bridge process.
+
+Requires only the stdlib (Tkinter is bundled on Windows/Python).
+"""
+
+from __future__ import annotations
+
+import os
+import queue
+import subprocess
+import sys
+import threading
+import tkinter as tk
+from tkinter import filedialog, messagebox, scrolledtext
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BRIDGE_PATH = os.path.join(SCRIPT_DIR, "s3_bridge.py")
+
+
+class BridgeGUI(tk.Tk):
+    def __init__(self) -> None:
+        super().__init__()
+        self.title("S3 Bridge GUI")
+        self.process: subprocess.Popen | None = None
+        self.stdout_q: queue.Queue[str] = queue.Queue()
+        self._build_widgets()
+        self._poll_stdout()
+
+    def _build_widgets(self) -> None:
+        pad = {"padx": 6, "pady": 4}
+
+        # Row 0: Ports
+        tk.Label(self, text="BW COM:").grid(row=0, column=0, sticky="e", **pad)
+        self.bw_var = tk.StringVar(value="COM4")
+        tk.Entry(self, textvariable=self.bw_var, width=10).grid(row=0, column=1, sticky="w", **pad)
+
+        tk.Label(self, text="S3 COM:").grid(row=0, column=2, sticky="e", **pad)
+        self.s3_var = tk.StringVar(value="COM5")
+        tk.Entry(self, textvariable=self.s3_var, width=10).grid(row=0, column=3, sticky="w", **pad)
+
+        # Row 1: Baud
+        tk.Label(self, text="Baud:").grid(row=1, column=0, sticky="e", **pad)
+        self.baud_var = tk.StringVar(value="38400")
+        tk.Entry(self, textvariable=self.baud_var, width=10).grid(row=1, column=1, sticky="w", **pad)
+
+        # Row 1: Logdir chooser
+        tk.Label(self, text="Log dir:").grid(row=1, column=2, sticky="e", **pad)
+        self.logdir_var = tk.StringVar(value=".")
+        tk.Entry(self, textvariable=self.logdir_var, width=24).grid(row=1, column=3, sticky="we", **pad)
+        tk.Button(self, text="Browse", command=self._choose_dir).grid(row=1, column=4, sticky="w", **pad)
+
+        # Row 2: Raw taps
+        self.raw_bw_var = tk.StringVar(value="")
+        self.raw_s3_var = tk.StringVar(value="")
+        tk.Checkbutton(self, text="Save BW->S3 raw", command=self._toggle_raw_bw, onvalue="1", offvalue="").grid(row=2, column=0, sticky="w", **pad)
+        tk.Entry(self, textvariable=self.raw_bw_var, width=28).grid(row=2, column=1, columnspan=3, sticky="we", **pad)
+        tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_bw_var, "bw")).grid(row=2, column=4, **pad)
+
+        tk.Checkbutton(self, text="Save S3->BW raw", command=self._toggle_raw_s3, onvalue="1", offvalue="").grid(row=3, column=0, sticky="w", **pad)
+        tk.Entry(self, textvariable=self.raw_s3_var, width=28).grid(row=3, column=1, columnspan=3, sticky="we", **pad)
+        tk.Button(self, text="...", command=lambda: self._choose_file(self.raw_s3_var, "s3")).grid(row=3, column=4, **pad)
+
+        # Row 4: Status + buttons
+        self.status_var = tk.StringVar(value="Idle")
+        tk.Label(self, textvariable=self.status_var, anchor="w").grid(row=4, column=0, columnspan=5, sticky="we", **pad)
+
+        tk.Button(self, text="Start", command=self.start_bridge, width=12).grid(row=5, column=0, columnspan=2, **pad)
+        tk.Button(self, text="Stop", command=self.stop_bridge, width=12).grid(row=5, column=2, columnspan=2, **pad)
+
+        # Row 6: Log view
+        self.log_view = scrolledtext.ScrolledText(self, height=20, width=90, state="disabled")
+        self.log_view.grid(row=6, column=0, columnspan=5, sticky="nsew", **pad)
+
+        # Grid weights
+        for c in range(5):
+            self.grid_columnconfigure(c, weight=1)
+        self.grid_rowconfigure(6, weight=1)
+
+    def _choose_dir(self) -> None:
+        path = filedialog.askdirectory()
+        if path:
+            self.logdir_var.set(path)
+
+    def _choose_file(self, var: tk.StringVar, direction: str) -> None:
+        filename = filedialog.asksaveasfilename(
+            title=f"Raw tap file for {direction}",
+            defaultextension=".bin",
+            filetypes=[("Binary", "*.bin"), ("All files", "*.*")]
+        )
+        if filename:
+            var.set(filename)
+
+    def _toggle_raw_bw(self) -> None:
+        if not self.raw_bw_var.get():
+            # default name
+            self.raw_bw_var.set(os.path.join(self.logdir_var.get(), "raw_bw.bin"))
+
+    def _toggle_raw_s3(self) -> None:
+        if not self.raw_s3_var.get():
+            self.raw_s3_var.set(os.path.join(self.logdir_var.get(), "raw_s3.bin"))
+
+    def start_bridge(self) -> None:
+        if self.process and self.process.poll() is None:
+            messagebox.showinfo("Bridge", "Bridge is already running.")
+            return
+
+        bw = self.bw_var.get().strip()
+        s3 = self.s3_var.get().strip()
+        baud = self.baud_var.get().strip()
+        logdir = self.logdir_var.get().strip() or "."
+
+        if not bw or not s3:
+            messagebox.showerror("Error", "Please enter both BW and S3 COM ports.")
+            return
+
+        args = [sys.executable, BRIDGE_PATH, "--bw", bw, "--s3", s3, "--baud", baud, "--logdir", logdir]
+
+        raw_bw = self.raw_bw_var.get().strip()
+        raw_s3 = self.raw_s3_var.get().strip()
+        if raw_bw:
+            args += ["--raw-bw", raw_bw]
+        if raw_s3:
+            args += ["--raw-s3", raw_s3]
+
+        try:
+            self.process = subprocess.Popen(
+                args,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                bufsize=1,
+            )
+        except Exception as e:
+            messagebox.showerror("Error", f"Failed to start bridge: {e}")
+            return
+
+        threading.Thread(target=self._reader_thread, daemon=True).start()
+        self.status_var.set("Running...")
+        self._append_log("== Bridge started ==\n")
+
+    def stop_bridge(self) -> None:
+        if self.process and self.process.poll() is None:
+            self.process.terminate()
+            try:
+                self.process.wait(timeout=3)
+            except subprocess.TimeoutExpired:
+                self.process.kill()
+        self.status_var.set("Stopped")
+        self._append_log("== Bridge stopped ==\n")
+
+    def _reader_thread(self) -> None:
+        if not self.process or not self.process.stdout:
+            return
+        for line in self.process.stdout:
+            self.stdout_q.put(line)
+        self.stdout_q.put("<<process-exit>>")
+
+    def _poll_stdout(self) -> None:
+        try:
+            while True:
+                line = self.stdout_q.get_nowait()
+                if line == "<<process-exit>>":
+                    self.status_var.set("Stopped")
+                    break
+                self._append_log(line)
+        except queue.Empty:
+            pass
+        finally:
+            self.after(100, self._poll_stdout)
+
+    def _append_log(self, text: str) -> None:
+        self.log_view.configure(state="normal")
+        self.log_view.insert(tk.END, text)
+        self.log_view.see(tk.END)
+        self.log_view.configure(state="disabled")
+
+
+def main() -> int:
+    app = BridgeGUI()
+    app.mainloop()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bridges/raw_capture.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+raw_capture.py — minimal serial logger for raw byte collection.
+
+Opens a single COM port, streams all bytes to a timestamped binary file,
+and does no parsing or forwarding. Useful when you just need the raw
+wire data without DLE framing or Blastware bridging.
+
+Record format (little-endian):
+  [ts_us:8][len:4][payload:len]
+Exactly one record type is used, so there is no type byte.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import os
+import signal
+import sys
+import time
+from typing import Optional
+
+import serial
+
+
+def now_ts() -> str:
+    t = _dt.datetime.now()
+    return t.strftime("%H:%M:%S.") + f"{int(t.microsecond/1000):03d}"
+
+
+def pack_u32_le(n: int) -> bytes:
+    return bytes((n & 0xFF, (n >> 8) & 0xFF, (n >> 16) & 0xFF, (n >> 24) & 0xFF))
+
+
+def pack_u64_le(n: int) -> bytes:
+    out = []
+    for i in range(8):
+        out.append((n >> (8 * i)) & 0xFF)
+    return bytes(out)
+
+
+def open_serial(port: str, baud: int, timeout: float) -> serial.Serial:
+    return serial.Serial(
+        port=port,
+        baudrate=baud,
+        bytesize=serial.EIGHTBITS,
+        parity=serial.PARITY_NONE,
+        stopbits=serial.STOPBITS_ONE,
+        timeout=timeout,
+        write_timeout=timeout,
+    )
+
+
+class RawWriter:
+    def __init__(self, path: str):
+        self.path = path
+        self._fh = open(path, "ab", buffering=0)
+
+    def write(self, payload: bytes, ts_us: Optional[int] = None) -> None:
+        if ts_us is None:
+            ts_us = int(time.time() * 1_000_000)
+        header = pack_u64_le(ts_us) + pack_u32_le(len(payload))
+        self._fh.write(header)
+        if payload:
+            self._fh.write(payload)
+
+    def close(self) -> None:
+        try:
+            self._fh.flush()
+        finally:
+            self._fh.close()
+
+
+def capture_loop(port: serial.Serial, writer: RawWriter, stop_flag: "StopFlag", status_every_s: float) -> None:
+    last_status = time.monotonic()
+    bytes_written = 0
+
+    while not stop_flag.is_set():
+        try:
+            n = port.in_waiting
+            chunk = port.read(n if n and n < 4096 else (4096 if n else 1))
+        except serial.SerialException as e:
+            print(f"[{now_ts()}] [ERROR] serial exception: {e!r}", file=sys.stderr)
+            break
+
+        if chunk:
+            writer.write(chunk)
+            bytes_written += len(chunk)
+
+        if status_every_s > 0:
+            now = time.monotonic()
+            if now - last_status >= status_every_s:
+                print(f"[{now_ts()}] captured {bytes_written} bytes", flush=True)
+                last_status = now
+
+        if not chunk:
+            time.sleep(0.002)
+
+
+class StopFlag:
+    def __init__(self):
+        self._set = False
+
+    def set(self):
+        self._set = True
+
+    def is_set(self) -> bool:
+        return self._set
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Raw serial capture to timestamped binary file (no forwarding).")
+    ap.add_argument("--port", default="COM5", help="Serial port to capture (default: COM5)")
+    ap.add_argument("--baud", type=int, default=38400, help="Baud rate (default: 38400)")
+    ap.add_argument("--timeout", type=float, default=0.05, help="Serial read timeout in seconds (default: 0.05)")
+    ap.add_argument("--logdir", default=".", help="Directory to write captures (default: .)")
+    ap.add_argument("--status-every", type=float, default=5.0, help="Seconds between progress lines (0 disables)")
+    args = ap.parse_args()
+
+    os.makedirs(args.logdir, exist_ok=True)
+    ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S")
+    bin_path = os.path.join(args.logdir, f"raw_capture_{ts}.bin")
+
+    print(f"[INFO] Opening {args.port} @ {args.baud}...")
+    try:
+        ser = open_serial(args.port, args.baud, args.timeout)
+    except Exception as e:
+        print(f"[ERROR] failed to open port: {e!r}", file=sys.stderr)
+        return 2
+
+    writer = RawWriter(bin_path)
+    print(f"[INFO] Writing raw bytes to {bin_path}")
+    print("[INFO] Press Ctrl+C to stop.")
+
+    stop = StopFlag()
+
+    def handle_sigint(sig, frame):
+        stop.set()
+
+    signal.signal(signal.SIGINT, handle_sigint)
+
+    try:
+        capture_loop(ser, writer, stop, args.status_every)
+    finally:
+        writer.close()
+        try:
+            ser.close()
+        except Exception:
+            pass
+        print(f"[INFO] Capture stopped. Total bytes written: {os.path.getsize(bin_path)}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bridges/s3-bridge/s3_bridge.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """
 s3_bridge.py — S3 <-> Blastware serial bridge with raw binary capture + DLE-aware text framing
-Version: v0.5.0
+Version: v0.5.1
 
 What’s new vs v0.4.0:
 - .bin is now a TRUE raw capture stream with direction + timestamps (record container format).
@@ -10,6 +10,8 @@ What’s new vs v0.4.0:
     - frame end   = 0x10 0x03 (DLE ETX)
   (No longer splits on bare 0x03.)
 - Marks/Info are stored as proper record types in .bin (no unsafe sentinel bytes).
+- Optional raw taps: use --raw-bw / --raw-s3 to also dump byte-for-byte traffic per direction
+  with no headers (for tools that just need a flat stream).
 
 BIN record format (little-endian):
   [type:1][ts_us:8][len:4][payload:len]
@@ -84,12 +86,15 @@ def pack_u64_le(n: int) -> bytes:
 
 
 class SessionLogger:
-    def __init__(self, path: str, bin_path: str):
+    def __init__(self, path: str, bin_path: str, raw_bw_path: Optional[str] = None, raw_s3_path: Optional[str] = None):
         self.path = path
         self.bin_path = bin_path
         self._fh = open(path, "a", buffering=1, encoding="utf-8", errors="replace")
         self._bin_fh = open(bin_path, "ab", buffering=0)
         self._lock = threading.Lock()
+        # Optional pure-byte taps (no headers). BW=Blastware tx, S3=device tx.
+        self._raw_bw = open(raw_bw_path, "ab", buffering=0) if raw_bw_path else None
+        self._raw_s3 = open(raw_s3_path, "ab", buffering=0) if raw_s3_path else None
 
     def log_line(self, line: str) -> None:
         with self._lock:
@@ -103,6 +108,11 @@ class SessionLogger:
             self._bin_fh.write(header)
             if payload:
                 self._bin_fh.write(payload)
+            # Raw taps: write only the payload bytes (no headers)
+            if rec_type == REC_BW and self._raw_bw:
+                self._raw_bw.write(payload)
+            if rec_type == REC_S3 and self._raw_s3:
+                self._raw_s3.write(payload)
 
     def log_mark(self, label: str) -> None:
         ts = now_ts()
@@ -122,6 +132,10 @@ class SessionLogger:
             finally:
                 self._fh.close()
                 self._bin_fh.close()
+                if self._raw_bw:
+                    self._raw_bw.close()
+                if self._raw_s3:
+                    self._raw_s3.close()
 
 
 class DLEFrameSniffer:
@@ -311,6 +325,8 @@ def main() -> int:
     ap.add_argument("--s3", default="COM5", help="S3-side COM port (default: COM5)")
     ap.add_argument("--baud", type=int, default=38400, help="Baud rate (default: 38400)")
     ap.add_argument("--logdir", default=".", help="Directory to write session logs into (default: .)")
+    ap.add_argument("--raw-bw", default=None, help="Optional file to append raw bytes sent from BW->S3 (no headers)")
+    ap.add_argument("--raw-s3", default=None, help="Optional file to append raw bytes sent from S3->BW (no headers)")
     ap.add_argument("--quiet", action="store_true", help="No console heartbeat output")
     ap.add_argument("--status-every", type=float, default=0.0, help="Seconds between console heartbeat lines (default: 0 = off)")
     args = ap.parse_args()
@@ -329,10 +345,14 @@ def main() -> int:
     ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S")
     log_path = os.path.join(args.logdir, f"s3_session_{ts}.log")
     bin_path = os.path.join(args.logdir, f"s3_session_{ts}.bin")
-    logger = SessionLogger(log_path, bin_path)
+    logger = SessionLogger(log_path, bin_path, raw_bw_path=args.raw_bw, raw_s3_path=args.raw_s3)
 
     print(f"[LOG] Writing hex log to  {log_path}")
     print(f"[LOG] Writing binary log to {bin_path}")
+    if args.raw_bw:
+        print(f"[LOG] Raw tap BW->S3  -> {args.raw_bw}")
+    if args.raw_s3:
+        print(f"[LOG] Raw tap S3->BW -> {args.raw_s3}")
 
     logger.log_info(f"s3_bridge {VERSION} start")
     logger.log_info(f"BW={args.bw} S3={args.s3} baud={args.baud}")
@@ -396,4 +416,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

parsers/s3_parser.py

@@ -20,6 +20,14 @@ STX = 0x02
 ETX = 0x03
 EOT = 0x04
 
+# How the capture was produced:
+# - Raw serial captures include DLE+ETX (`0x10 0x03`).
+# - The s3_bridge `.bin` logger strips the DLE byte from ETX, so frames end with a
+#   bare `0x03`. See docs/instantel_protocol_reference.md §Appendix A.
+ETX_MODE_AUTO = "auto"
+ETX_MODE_RAW = "raw"          # expect DLE+ETX
+ETX_MODE_STRIPPED = "stripped"  # expect bare ETX
+
 @dataclass
 class Frame:
     index: int
@@ -92,7 +100,7 @@ CRC_FUNCS = {
     "CRC-16/X-25": crc16_x25,
 }
 
-def parse_frames(blob: bytes, trailer_len: int) -> List[Frame]:
+def parse_frames(blob: bytes, trailer_len: int, etx_mode: str = ETX_MODE_AUTO) -> List[Frame]:
     frames: List[Frame] = []
     i = 0
     idx = 0
@@ -101,21 +109,40 @@ def parse_frames(blob: bytes, trailer_len: int) -> List[Frame]:
     def is_dle_seq(pos: int, second: int) -> bool:
         return pos + 1 < n and blob[pos] == DLE and blob[pos + 1] == second
 
+    # Auto-detect whether ETX is bare (logger-stripped) or DLE+ETX (raw wire).
+    if etx_mode == ETX_MODE_AUTO:
+        raw_etx = sum(1 for p in range(n - 1) if is_dle_seq(p, ETX))
+        stx_count = sum(1 for p in range(n - 1) if is_dle_seq(p, STX))
+        # Heuristic: the logger-stripped .bin files have plenty of STX but
+        # almost no DLE+ETX. If ETX count is far below STX count, assume stripped.
+        if raw_etx and raw_etx >= max(1, int(0.8 * stx_count)):
+            etx_mode = ETX_MODE_RAW
+        else:
+            etx_mode = ETX_MODE_STRIPPED
+
     while i < n - 1:
         if is_dle_seq(i, STX):
             start = i
             i += 2  # move past DLE STX
             payload_start = i
 
-            # find DLE ETX
+            # find end-of-frame marker
-            while i < n - 1 and not is_dle_seq(i, ETX):
+            while i < n:
+                if etx_mode == ETX_MODE_RAW and is_dle_seq(i, ETX):
+                    payload_end = i  # up to (but not including) DLE ETX
+                    i += 2  # skip DLE ETX
+                    end = i
+                    break
+                if etx_mode == ETX_MODE_STRIPPED and blob[i] == ETX:
+                    payload_end = i  # up to (but not including) bare ETX
+                    i += 1  # skip ETX
+                    end = i
+                    break
                 i += 1
 
-            if i >= n - 1:
+            else:
-                break  # truncated
+                # Ran off the end without finding ETX
-            payload_end = i  # bytes up to (but not including) DLE ETX
+                break
-            i += 2  # skip DLE ETX
-            end = i
 
             payload_raw = blob[payload_start:payload_end]
             payload = unescape_dle(payload_raw)
@@ -162,11 +189,18 @@ def main() -> None:
     ap.add_argument("--trailer-len", type=int, default=2, help="Bytes to capture after DLE ETX (default: 2)")
     ap.add_argument("--crc", action="store_true", help="Attempt CRC match using first 2 trailer bytes")
     ap.add_argument("--crc-endian", choices=["little", "big"], default="little", help="CRC endian when reading trailer")
+    ap.add_argument(
+        "--etx-mode",
+        choices=[ETX_MODE_AUTO, ETX_MODE_RAW, ETX_MODE_STRIPPED],
+        default=ETX_MODE_AUTO,
+        help="How to detect end-of-frame: 'raw' expects DLE+ETX, "
+             "'stripped' expects bare ETX (s3_bridge .bin), 'auto' picks based on presence of DLE+ETX."
+    )
     ap.add_argument("--out", type=Path, default=None, help="Write JSONL output to this file")
     args = ap.parse_args()
 
     blob = args.binfile.read_bytes()
-    frames = parse_frames(blob, trailer_len=args.trailer_len)
+    frames = parse_frames(blob, trailer_len=args.trailer_len, etx_mode=args.etx_mode)
 
     little = (args.crc_endian == "little")
     if args.crc:
@@ -208,4 +242,4 @@ def main() -> None:
             print(f"... ({len(lines) - 10} more)")
 
 if __name__ == "__main__":
-    main()
+    main()