From cc1388df655f49ced9f496ebf7aba9e0ed4692f5 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 25 Sep 2026 00:37:10 -0400 Subject: [PATCH] feat(scratch): recover captures from the socat relay log -- validated byte-exact The bench relay runs socat with -x, which hex-dumps every forwarded byte in both directions. That makes its log a complete second copy of every capture taken through it, independent of whether seismo_lab was recording. On 2026-09-25 a capture's .bin files never left the Windows machine and the session was rebuilt from the relay log instead. When the real bins turned up afterwards, the reconstruction was byte-for-byte IDENTICAL in both directions (3,595 and 4,004 bytes) -- verified again through the committed script, not just the ad-hoc version used at the time. So this is a validated fallback, not a lossy approximation. Adds scratch/socat_log_split.py, with --from-line/--to-line for picking one session out of a log that spans several (split on the "accepting connection" markers, or the frame walk runs sessions together). Also documents the -x flag and the fallback in the session-provenance section, so the next person runs the relay in a way that keeps the safety net. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ru8Lg9HkkYvX9VWWo65SmL --- docs/micromate_protocol_reference.md | 19 ++++- scratch/socat_log_split.py | 119 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 scratch/socat_log_split.py diff --git a/docs/micromate_protocol_reference.md b/docs/micromate_protocol_reference.md index fb280af..1111523 100644 --- a/docs/micromate_protocol_reference.md +++ b/docs/micromate_protocol_reference.md @@ -1757,12 +1757,29 @@ Three sittings, all on the same unit: | 2026-09-23 | 5 events (4 waveform, 1 histogram) | the event chain + `0x5A` | | 2026-09-24 | 5 events | **Thor pushing a setup**, via a recording relay | -The 2026-09-24 sitting used a different topology worth noting, because it is +The 2026-09-24/25 sittings used a topology worth keeping, because it is reusable: `socat` on mint-mac shares `/dev/ttyACM0` on TCP, seismo_lab's TCP bridge relays Thor to it and records both directions. Thor is configured with the unit at `127.0.0.1:` exactly as if it were a field modem. No modem, no SIM, and **nothing on the production Thor box is touched.** +**Run the relay with `-x` and keep its log.** It hex-dumps every forwarded byte +in both directions, which makes it a complete second copy of every capture, +independent of whether seismo_lab was recording: + +``` +socat -d -d -x TCP-LISTEN:12345,reuseaddr,fork /dev/ttyACM0,raw,echo=0,b115200 \ + > ~/mm-captures/socat_.log 2>&1 +``` + +That paid off on 2026-09-25, when a capture's `.bin` files never left the Windows +machine — the session was rebuilt from the relay log with +`scratch/socat_log_split.py`. When the real bins arrived later the +reconstruction was **byte-for-byte identical in both directions** (3,595 and +4,004 bytes), so the log is a validated fallback rather than an approximation. +Sessions concatenate in one log; split them on the `accepting connection` +markers. + ⚠ Firmware is the single biggest caveat on this document. Every finding here is from one unit on the Blastware build. A `11.0BD` unit has not been touched. diff --git a/scratch/socat_log_split.py b/scratch/socat_log_split.py new file mode 100644 index 0000000..9dd3427 --- /dev/null +++ b/scratch/socat_log_split.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +""" +socat_log_split.py — recover a capture pair from a `socat -x` relay log. + +Why this exists +--------------- +The bench relay that puts Thor in front of a USB-attached Micromate is: + + socat -d -d -x TCP-LISTEN:12345,reuseaddr,fork /dev/ttyACM0,raw,echo=0,b115200 \ + > ~/mm-captures/socat_.log 2>&1 + +`-x` makes socat hex-dump every byte it forwards, in both directions, with +timestamps. That log is therefore a **complete second copy of every capture** +taken through the relay — independent of whether seismo_lab was recording. + +On 2026-09-25 that mattered: a capture's `.bin` files never made it off the +Windows machine, and the session was rebuilt from this log instead. When the +real bins turned up later, the reconstruction was **byte-for-byte identical in +both directions** (3,595 and 4,004 bytes). So this is a validated fallback, not +a lossy approximation. + +Log format +---------- +``` +> 2026/09/25 00:30:35.000276659 length=21 from=0 to=20 + 41 02 10 10 00 5b 00 00 30 00 ... +2026/09/25 00:30:35 socat[32190] N write(5, 0x..., 21) completed +< 2026/09/25 00:30:35.000384100 length=64 from=0 to=63 + 02 00 c5 a4 00 00 30 00 ... +``` + +`>` is data heading toward the serial device (Thor → unit). `<` is data coming +back (unit → Thor). Hex lines are space-separated and indented; socat's own +status lines start with a date and carry no payload. + +Usage +----- + # whole log + python scratch/socat_log_split.py socat_20260924_181248.log --out-dir ./recovered + + # one session — line numbers from the "accepting connection" markers + grep -n "accepting connection" socat_*.log + python scratch/socat_log_split.py socat_*.log --from-line 919 --out-dir ./recovered + +Then parse the result as usual: + + python scratch/mm_frame_parse.py recovered/raw_bw.bin recovered/raw_s3.bin + +⚠ A log spanning several sessions concatenates them. Split by line number using +the `accepting connection` markers, or the frame walk will run sessions together. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +_HEX = re.compile(r"\A[0-9a-f]{2}\Z") +_SOCAT_STATUS = re.compile(r"\A\d{4}/\d{2}/\d{2}") + + +def split(lines) -> tuple[bytes, bytes]: + """Return (to_device, from_device) byte streams.""" + to_dev, from_dev = bytearray(), bytearray() + cur = None + for line in lines: + if line.startswith(">"): + cur = to_dev + continue + if line.startswith("<"): + cur = from_dev + continue + if _SOCAT_STATUS.match(line): + # socat's own status line ends the current dump block. + cur = None + continue + if cur is None or not line.startswith(" "): + continue + toks = line.split() + if toks and all(_HEX.match(t) for t in toks): + cur.extend(int(t, 16) for t in toks) + return bytes(to_dev), bytes(from_dev) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("log", help="a socat -x log file") + ap.add_argument("--out-dir", default=".", help="where to write the .bin pair") + ap.add_argument("--from-line", type=int, default=1, + help="first log line to read (1-based) — use the " + "'accepting connection' marker of the session you want") + ap.add_argument("--to-line", type=int, default=None, + help="last log line to read (1-based, inclusive)") + ap.add_argument("--prefix", default="raw", help="output basename prefix") + args = ap.parse_args() + + lines = Path(args.log).read_text(errors="replace").splitlines() + lo = max(args.from_line - 1, 0) + hi = args.to_line if args.to_line is not None else len(lines) + to_dev, from_dev = split(lines[lo:hi]) + + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + bw = out / f"{args.prefix}_bw.bin" + s3 = out / f"{args.prefix}_s3.bin" + bw.write_bytes(to_dev) + s3.write_bytes(from_dev) + print(f"Thor -> unit {len(to_dev):>7} bytes {bw}") + print(f"unit -> Thor {len(from_dev):>7} bytes {s3}") + if not to_dev or not from_dev: + print("⚠ one direction is empty — check --from-line / --to-line") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())