Release v0.31.0 — report parity + the inverted rescue (0.29.0 → 0.31.0) #40

Merged
serversdown merged 31 commits from dev into main 2026-09-18 16:46:45 -04:00
2 changed files with 96 additions and 0 deletions
Showing only changes of commit 9f1050b5e7 - Show all commits
+21
View File
@@ -17,6 +17,27 @@ backfill**: a report regenerated for an existing event simply gains the new
panels.
### Added
- **Rescue-on-connect for `bridges/ach_server.py`** — `--stop-monitoring`
(SUB 0x97), `--disable-ach` (SUB 0x2C read → 0x7E write → 0x7F confirm) and
`--rescue` (both). They fire immediately after the startup handshake and
**before** the event walk, so a unit that is recording back-to-back on a
stuck-triggered geophone is quieted as early in the session as possible.
Each action is independently guarded — a failure does not abort the download
— and the outcome is written to `rescue.json` in the session directory.
This inverts the `docs/runbooks/wedged_unit_recovery.md` approach. That
runbook reaches the unit *inbound* and clears the modem's Destination Address
to stop it dialing. When the device is instead wedged mid-modem-init — ALEOS
logs `tcpmode trying to send to invalid socket` and re-runs `Initialize Auto
answer` every ~75 s, orphaning any held inbound session — inbound cannot win.
Pointing the modem's Destination at an `ach_server` and letting the unit call
*us* gives a device-initiated session the modem bridges properly.
⚠ Prefer `--stop-monitoring` alone on first contact. `--disable-ach` stops
the unit calling, which is the only channel to a unit in this state; stopping
the recording ends the call-home loop on its own when ACH is
"after event recorded".
- **Blastware-compatible channel FFT (`waveform_fft`).** Reproduces Blastware's
FFT Report: DC-removed, no window, zero-padded to 4096 (0.25 Hz bins at
1024 sps), single-sided `2/N` amplitude. Matches Blastware's dominant
+75
View File
@@ -177,6 +177,8 @@ class AchSession:
store: "WaveformStore",
clear_after_download: bool = False,
restart_monitoring: bool = False,
rescue_stop_monitoring: bool = False,
rescue_disable_ach: bool = False,
force_redownload: bool = False,
) -> None:
self.sock = sock
@@ -190,6 +192,9 @@ class AchSession:
self.store = store
self.clear_after_download = clear_after_download
self.restart_monitoring = restart_monitoring
# Rescue actions for a runaway unit — fired before the event walk.
self.rescue_stop_monitoring = rescue_stop_monitoring
self.rescue_disable_ach = rescue_disable_ach
# `force_redownload` tells this session to ignore ach_state and
# re-download every event currently on the device, regardless of any
# (key, timestamp) match. Useful as a manual override when state has
@@ -290,6 +295,41 @@ class AchSession:
root_logger.addHandler(fh)
try:
# ── Step 1.5: rescue actions ──────────────────────────────────────
# Fired BEFORE the event walk so a runaway unit is quieted as early
# in the session as possible. A unit whose geophone sits above the
# trigger threshold records back-to-back and, with ACH set to "after
# event recorded", re-dials every time — saturating its own firmware
# so it never services inbound requests. See
# docs/runbooks/wedged_unit_recovery.md.
#
# Each action is independently guarded: a failure here must not
# abort the download that follows.
if self.rescue_stop_monitoring or self.rescue_disable_ach:
rescue: dict = {"peer": self.peer, "ts": ts}
if self.rescue_stop_monitoring:
log.info("Step 1.5: RESCUE — stop monitoring (SUB 0x97)")
try:
client.stop_monitoring()
rescue["stop_monitoring"] = "ok"
log.info(" stop monitoring OK — device should stop recording")
except Exception as exc:
rescue["stop_monitoring"] = f"failed: {exc}"
log.error(" stop monitoring FAILED: %s", exc)
if self.rescue_disable_ach:
log.info("Step 1.5: RESCUE — disable auto call home (SUB 0x2C/0x7E/0x7F)")
try:
client.set_call_home_config(auto_call_home_enabled=False)
rescue["disable_ach"] = "ok"
log.info(" disable ACH OK — unit should stop calling home")
except Exception as exc:
rescue["disable_ach"] = f"failed: {exc}"
log.error(" disable ACH FAILED: %s", exc)
_save_json(session_dir / "rescue.json", rescue)
# ── Step 2: device info ───────────────────────────────────────────
device_info = None
if not self.events_only:
@@ -747,6 +787,13 @@ def serve(args: argparse.Namespace) -> None:
print(f" Max events per session: {max_ev if max_ev else 'unlimited'}")
print(f" Clear device after download: {'YES' if args.clear_after_download else 'no'}")
print(f" Restart monitoring after download: {'YES' if args.restart_monitoring else 'no'}")
_stop_mon = args.stop_monitoring or args.rescue
_dis_ach = args.disable_ach or args.rescue
print(f" RESCUE stop monitoring on connect: {'YES' if _stop_mon else 'no'}")
print(f" RESCUE disable auto call home: {'YES' if _dis_ach else 'no'}")
if _stop_mon and args.restart_monitoring:
print(" !! --restart-monitoring will re-start the unit after download,")
print(" undoing --stop-monitoring. Drop one of them.")
print(f" Force re-download all (ignore state): {'YES' if args.force_redownload_all else 'no'}")
print(f"{'='*60}")
print(f"\n Point your test unit's ACEmanager call-home settings to:")
@@ -788,6 +835,8 @@ def serve(args: argparse.Namespace) -> None:
store=store,
clear_after_download=args.clear_after_download,
restart_monitoring=args.restart_monitoring,
rescue_stop_monitoring=args.stop_monitoring or args.rescue,
rescue_disable_ach=args.disable_ach or args.rescue,
force_redownload=args.force_redownload_all,
)
t = threading.Thread(target=session.run, daemon=True, name=f"ach-{peer}")
@@ -862,6 +911,32 @@ def parse_args() -> argparse.Namespace:
"DCD on disconnect — without this the unit stays idle after a call-home."
),
)
p.add_argument(
"--stop-monitoring",
action="store_true",
default=False,
help=(
"RESCUE: send SUB 0x97 (stop monitoring) immediately after the "
"handshake, before any event download. Use on a unit that is "
"recording back-to-back because of a stuck-triggered geophone."
),
)
p.add_argument(
"--disable-ach",
action="store_true",
default=False,
help=(
"RESCUE: disable Auto Call Home on the device (SUB 0x2C read → "
"0x7E write → 0x7F confirm) immediately after the handshake. The "
"unit stops dialing out until ACH is explicitly re-enabled."
),
)
p.add_argument(
"--rescue",
action="store_true",
default=False,
help="Shorthand for --stop-monitoring --disable-ach.",
)
p.add_argument(
"--clear-after-download",
action="store_true",