diff --git a/CHANGELOG.md b/CHANGELOG.md index 2650bdd..8dbbfd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,27 @@ backfill**: a report regenerated for an existing event simply gains the new panels. ### Added +- **Rescue-on-connect for `bridges/ach_server.py`** — `--stop-monitoring` + (SUB 0x97), `--disable-ach` (SUB 0x2C read → 0x7E write → 0x7F confirm) and + `--rescue` (both). They fire immediately after the startup handshake and + **before** the event walk, so a unit that is recording back-to-back on a + stuck-triggered geophone is quieted as early in the session as possible. + Each action is independently guarded — a failure does not abort the download + — and the outcome is written to `rescue.json` in the session directory. + + This inverts the `docs/runbooks/wedged_unit_recovery.md` approach. That + runbook reaches the unit *inbound* and clears the modem's Destination Address + to stop it dialing. When the device is instead wedged mid-modem-init — ALEOS + logs `tcpmode trying to send to invalid socket` and re-runs `Initialize Auto + answer` every ~75 s, orphaning any held inbound session — inbound cannot win. + Pointing the modem's Destination at an `ach_server` and letting the unit call + *us* gives a device-initiated session the modem bridges properly. + + ⚠ Prefer `--stop-monitoring` alone on first contact. `--disable-ach` stops + the unit calling, which is the only channel to a unit in this state; stopping + the recording ends the call-home loop on its own when ACH is + "after event recorded". + - **Blastware-compatible channel FFT (`waveform_fft`).** Reproduces Blastware's FFT Report: DC-removed, no window, zero-padded to 4096 (0.25 Hz bins at 1024 sps), single-sided `2/N` amplitude. Matches Blastware's dominant diff --git a/bridges/ach_server.py b/bridges/ach_server.py index 0bfd8af..bbc72b8 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -177,6 +177,8 @@ class AchSession: store: "WaveformStore", clear_after_download: bool = False, restart_monitoring: bool = False, + rescue_stop_monitoring: bool = False, + rescue_disable_ach: bool = False, force_redownload: bool = False, ) -> None: self.sock = sock @@ -190,6 +192,9 @@ class AchSession: self.store = store self.clear_after_download = clear_after_download self.restart_monitoring = restart_monitoring + # Rescue actions for a runaway unit — fired before the event walk. + self.rescue_stop_monitoring = rescue_stop_monitoring + self.rescue_disable_ach = rescue_disable_ach # `force_redownload` tells this session to ignore ach_state and # re-download every event currently on the device, regardless of any # (key, timestamp) match. Useful as a manual override when state has @@ -290,6 +295,41 @@ class AchSession: root_logger.addHandler(fh) try: + # ── Step 1.5: rescue actions ────────────────────────────────────── + # Fired BEFORE the event walk so a runaway unit is quieted as early + # in the session as possible. A unit whose geophone sits above the + # trigger threshold records back-to-back and, with ACH set to "after + # event recorded", re-dials every time — saturating its own firmware + # so it never services inbound requests. See + # docs/runbooks/wedged_unit_recovery.md. + # + # Each action is independently guarded: a failure here must not + # abort the download that follows. + if self.rescue_stop_monitoring or self.rescue_disable_ach: + rescue: dict = {"peer": self.peer, "ts": ts} + + if self.rescue_stop_monitoring: + log.info("Step 1.5: RESCUE — stop monitoring (SUB 0x97)") + try: + client.stop_monitoring() + rescue["stop_monitoring"] = "ok" + log.info(" stop monitoring OK — device should stop recording") + except Exception as exc: + rescue["stop_monitoring"] = f"failed: {exc}" + log.error(" stop monitoring FAILED: %s", exc) + + if self.rescue_disable_ach: + log.info("Step 1.5: RESCUE — disable auto call home (SUB 0x2C/0x7E/0x7F)") + try: + client.set_call_home_config(auto_call_home_enabled=False) + rescue["disable_ach"] = "ok" + log.info(" disable ACH OK — unit should stop calling home") + except Exception as exc: + rescue["disable_ach"] = f"failed: {exc}" + log.error(" disable ACH FAILED: %s", exc) + + _save_json(session_dir / "rescue.json", rescue) + # ── Step 2: device info ─────────────────────────────────────────── device_info = None if not self.events_only: @@ -747,6 +787,13 @@ def serve(args: argparse.Namespace) -> None: print(f" Max events per session: {max_ev if max_ev else 'unlimited'}") print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}") print(f" Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}") + _stop_mon = args.stop_monitoring or args.rescue + _dis_ach = args.disable_ach or args.rescue + print(f" RESCUE stop monitoring on connect: {'YES' if _stop_mon else 'no'}") + print(f" RESCUE disable auto call home: {'YES' if _dis_ach else 'no'}") + if _stop_mon and args.restart_monitoring: + print(" !! --restart-monitoring will re-start the unit after download,") + print(" undoing --stop-monitoring. Drop one of them.") print(f" Force re-download all (ignore state): {'YES' if args.force_redownload_all else 'no'}") print(f"{'='*60}") print(f"\n Point your test unit's ACEmanager call-home settings to:") @@ -788,6 +835,8 @@ def serve(args: argparse.Namespace) -> None: store=store, clear_after_download=args.clear_after_download, restart_monitoring=args.restart_monitoring, + rescue_stop_monitoring=args.stop_monitoring or args.rescue, + rescue_disable_ach=args.disable_ach or args.rescue, force_redownload=args.force_redownload_all, ) t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}") @@ -862,6 +911,32 @@ def parse_args() -> argparse.Namespace: "DCD on disconnect — without this the unit stays idle after a call-home." ), ) + p.add_argument( + "--stop-monitoring", + action="store_true", + default=False, + help=( + "RESCUE: send SUB 0x97 (stop monitoring) immediately after the " + "handshake, before any event download. Use on a unit that is " + "recording back-to-back because of a stuck-triggered geophone." + ), + ) + p.add_argument( + "--disable-ach", + action="store_true", + default=False, + help=( + "RESCUE: disable Auto Call Home on the device (SUB 0x2C read → " + "0x7E write → 0x7F confirm) immediately after the handshake. The " + "unit stops dialing out until ACH is explicitly re-enabled." + ), + ) + p.add_argument( + "--rescue", + action="store_true", + default=False, + help="Shorthand for --stop-monitoring --disable-ach.", + ) p.add_argument( "--clear-after-download", action="store_true",