From 9f1050b5e74c7b6b56145b13cae42ac854cf946a Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 16 Sep 2026 20:54:46 +0000 Subject: [PATCH] =?UTF-8?q?feat(ach):=20rescue-on-connect=20=E2=80=94=20st?= =?UTF-8?q?op=20monitoring=20/=20disable=20ACH=20from=20the=20server=20sid?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A unit whose geophone offset has grown past its trigger level records back-to-back and, with ACH set to "after event recorded", re-dials every time. The wedged_unit_recovery runbook handles that by reaching the unit inbound and clearing the modem's Destination Address so it stops dialing. That fails when the device is wedged mid-modem-init. BE12599 (2026-09-16) sat repeating a 29-byte AT setup string — ATQ1/ATE0/ATS0=2, no ATD — every 75 s. The modem is in TCP data mode, never interprets it, never answers OK, so the device never progresses into S3 mode and ignores every frame we send. Worse, each attempt makes ALEOS log "tcpmode trying to send to invalid socket" and re-run "Initialize Auto answer on port 9034", which orphans any held inbound session — slow_drip reports a clean 120 s hold with bytes_received=0 because the modem stopped bridging after the first re-init. Inbound cannot win that race. But the modem auto-dials its Destination whenever serial data arrives while closed, so pointing Destination at an ach_server turns those 75 s attempts into a device-initiated session that the modem bridges correctly. Adds --stop-monitoring, --disable-ach and --rescue. They run as step 1.5, after the handshake and before the event walk, each independently guarded so a failure does not abort the download. Outcome is written to rescue.json. Startup banner reports both, and warns when --restart-monitoring would undo --stop-monitoring. Prefer --stop-monitoring alone on first contact: --disable-ach stops the unit calling, which is the only channel to a unit in this state, and halting the recording ends the loop on its own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Qcu9ByJfuKBQxmrWb8rSrN --- CHANGELOG.md | 21 ++++++++++++ bridges/ach_server.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2650bdd..8dbbfd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,27 @@ backfill**: a report regenerated for an existing event simply gains the new panels. ### Added +- **Rescue-on-connect for `bridges/ach_server.py`** — `--stop-monitoring` + (SUB 0x97), `--disable-ach` (SUB 0x2C read → 0x7E write → 0x7F confirm) and + `--rescue` (both). They fire immediately after the startup handshake and + **before** the event walk, so a unit that is recording back-to-back on a + stuck-triggered geophone is quieted as early in the session as possible. + Each action is independently guarded — a failure does not abort the download + — and the outcome is written to `rescue.json` in the session directory. + + This inverts the `docs/runbooks/wedged_unit_recovery.md` approach. That + runbook reaches the unit *inbound* and clears the modem's Destination Address + to stop it dialing. When the device is instead wedged mid-modem-init — ALEOS + logs `tcpmode trying to send to invalid socket` and re-runs `Initialize Auto + answer` every ~75 s, orphaning any held inbound session — inbound cannot win. + Pointing the modem's Destination at an `ach_server` and letting the unit call + *us* gives a device-initiated session the modem bridges properly. + + ⚠ Prefer `--stop-monitoring` alone on first contact. `--disable-ach` stops + the unit calling, which is the only channel to a unit in this state; stopping + the recording ends the call-home loop on its own when ACH is + "after event recorded". + - **Blastware-compatible channel FFT (`waveform_fft`).** Reproduces Blastware's FFT Report: DC-removed, no window, zero-padded to 4096 (0.25 Hz bins at 1024 sps), single-sided `2/N` amplitude. Matches Blastware's dominant diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 0bfd8af..bbc72b8 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -177,6 +177,8 @@ class AchSession: store: "WaveformStore", clear_after_download: bool = False, restart_monitoring: bool = False, + rescue_stop_monitoring: bool = False, + rescue_disable_ach: bool = False, force_redownload: bool = False, ) -> None: self.sock = sock @@ -190,6 +192,9 @@ class AchSession: self.store = store self.clear_after_download = clear_after_download self.restart_monitoring = restart_monitoring + # Rescue actions for a runaway unit — fired before the event walk. + self.rescue_stop_monitoring = rescue_stop_monitoring + self.rescue_disable_ach = rescue_disable_ach # `force_redownload` tells this session to ignore ach_state and # re-download every event currently on the device, regardless of any # (key, timestamp) match. Useful as a manual override when state has @@ -290,6 +295,41 @@ class AchSession: root_logger.addHandler(fh) try: + # ── Step 1.5: rescue actions ────────────────────────────────────── + # Fired BEFORE the event walk so a runaway unit is quieted as early + # in the session as possible. A unit whose geophone sits above the + # trigger threshold records back-to-back and, with ACH set to "after + # event recorded", re-dials every time — saturating its own firmware + # so it never services inbound requests. See + # docs/runbooks/wedged_unit_recovery.md. + # + # Each action is independently guarded: a failure here must not + # abort the download that follows. + if self.rescue_stop_monitoring or self.rescue_disable_ach: + rescue: dict = {"peer": self.peer, "ts": ts} + + if self.rescue_stop_monitoring: + log.info("Step 1.5: RESCUE — stop monitoring (SUB 0x97)") + try: + client.stop_monitoring() + rescue["stop_monitoring"] = "ok" + log.info(" stop monitoring OK — device should stop recording") + except Exception as exc: + rescue["stop_monitoring"] = f"failed: {exc}" + log.error(" stop monitoring FAILED: %s", exc) + + if self.rescue_disable_ach: + log.info("Step 1.5: RESCUE — disable auto call home (SUB 0x2C/0x7E/0x7F)") + try: + client.set_call_home_config(auto_call_home_enabled=False) + rescue["disable_ach"] = "ok" + log.info(" disable ACH OK — unit should stop calling home") + except Exception as exc: + rescue["disable_ach"] = f"failed: {exc}" + log.error(" disable ACH FAILED: %s", exc) + + _save_json(session_dir / "rescue.json", rescue) + # ── Step 2: device info ─────────────────────────────────────────── device_info = None if not self.events_only: @@ -747,6 +787,13 @@ def serve(args: argparse.Namespace) -> None: print(f" Max events per session: {max_ev if max_ev else 'unlimited'}") print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}") print(f" Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}") + _stop_mon = args.stop_monitoring or args.rescue + _dis_ach = args.disable_ach or args.rescue + print(f" RESCUE stop monitoring on connect: {'YES' if _stop_mon else 'no'}") + print(f" RESCUE disable auto call home: {'YES' if _dis_ach else 'no'}") + if _stop_mon and args.restart_monitoring: + print(" !! --restart-monitoring will re-start the unit after download,") + print(" undoing --stop-monitoring. Drop one of them.") print(f" Force re-download all (ignore state): {'YES' if args.force_redownload_all else 'no'}") print(f"{'='*60}") print(f"\n Point your test unit's ACEmanager call-home settings to:") @@ -788,6 +835,8 @@ def serve(args: argparse.Namespace) -> None: store=store, clear_after_download=args.clear_after_download, restart_monitoring=args.restart_monitoring, + rescue_stop_monitoring=args.stop_monitoring or args.rescue, + rescue_disable_ach=args.disable_ach or args.rescue, force_redownload=args.force_redownload_all, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") @@ -862,6 +911,32 @@ def parse_args() -> argparse.Namespace: "DCD on disconnect — without this the unit stays idle after a call-home." ), ) + p.add_argument( + "--stop-monitoring", + action="store_true", + default=False, + help=( + "RESCUE: send SUB 0x97 (stop monitoring) immediately after the " + "handshake, before any event download. Use on a unit that is " + "recording back-to-back because of a stuck-triggered geophone." + ), + ) + p.add_argument( + "--disable-ach", + action="store_true", + default=False, + help=( + "RESCUE: disable Auto Call Home on the device (SUB 0x2C read → " + "0x7E write → 0x7F confirm) immediately after the handshake. The " + "unit stops dialing out until ACH is explicitly re-enabled." + ), + ) + p.add_argument( + "--rescue", + action="store_true", + default=False, + help="Shorthand for --stop-monitoring --disable-ach.", + ) p.add_argument( "--clear-after-download", action="store_true",