Add runbook for recovering wedged units and new scripts for device management

- Created a comprehensive runbook (`wedged_unit_recovery.md`) detailing the recovery process for units stuck in a call-home loop, including symptoms, recovery steps, and explanations of the failure mode. - Added `blind_stop.sh` script to send stop-monitoring commands in a tight loop for unresponsive devices. - Introduced `rescue_device.sh` script to disable Auto Call Home and erase events from a busy device. - Implemented `slow_drip.sh` script to send stop-monitoring frames at a slow rate to prevent UART overrun. - Developed `spam_stop.sh` script to rapidly send stop-monitoring commands to a device. - Created `watch_unit.sh` script for passive monitoring of device reachability, logging results over time.
2026-05-17 07:58:13 +00:00
parent ae7edac83f
commit 1fff8179d6
8 changed files with 1401 additions and 10 deletions
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Rescue an uncooperative MiniMate that's busy with another ACH session.
+#
+# Hammers POST /device/rescue in a tight loop with a short timeout.  When the
+# device is in an ACH session our SYN either gets refused or silently dropped
+# (5s connect timeout inside the endpoint) and we retry immediately.  When the
+# device is between sessions, our TCP wins, the endpoint disables Auto Call
+# Home and erases events inside the same session, then returns success.
+#
+# Usage:
+#   ./rescue_device.sh <host> [tcp_port] [--no-erase] [--no-disable-ach]
+#
+# Examples:
+#   ./rescue_device.sh 166.246.130.1 9034
+#   ./rescue_device.sh 166.246.130.1 9034 --no-erase     # just silence it
+#
+# Environment:
+#   SFM_BASE_URL    Defaults to http://localhost:8200 (SFM direct).
+#                   Set to http://localhost:8001/api/sfm to route through
+#                   Terra-View's proxy.  Direct mode avoids the proxy's
+#                   60s timeout, which matters for long-running endpoints.
+#   MAX_ATTEMPTS    Cap on retries (default 600 ≈ 30+ min).
+#   SLEEP_S         Backoff between attempts (default 1).
+#   MAX_TIME_S      Per-request timeout (default 60).
+#   CONNECT_TIMEOUT TCP connect timeout (default 5).
+#   RECV_TIMEOUT    Per-frame S3 recv timeout (default 5).  If POLL or any
+#                   subsequent frame doesn't respond within this window, the
+#                   rescue endpoint bails and this script retries.
+
+set -u
+
+host="${1:-}"
+tcp_port="${2:-9034}"
+shift 2 2>/dev/null || shift $# 2>/dev/null
+
+if [[ -z "$host" ]]; then
+  echo "usage: $0 <host> [tcp_port] [--no-erase] [--no-disable-ach]" >&2
+  exit 2
+fi
+
+disable_ach="true"
+erase="true"
+for arg in "$@"; do
+  case "$arg" in
+    --no-erase)        erase="false" ;;
+    --no-disable-ach)  disable_ach="false" ;;
+    *) echo "unknown flag: $arg" >&2; exit 2 ;;
+  esac
+done
+
+base="${SFM_BASE_URL:-http://localhost:8200}"
+max_attempts="${MAX_ATTEMPTS:-600}"
+sleep_s="${SLEEP_S:-1}"
+max_time_s="${MAX_TIME_S:-60}"
+connect_timeout="${CONNECT_TIMEOUT:-5}"
+recv_timeout="${RECV_TIMEOUT:-5}"
+
+url="${base}/device/rescue?host=${host}&tcp_port=${tcp_port}&disable_ach=${disable_ach}&erase=${erase}&connect_timeout=${connect_timeout}&recv_timeout=${recv_timeout}"
+
+echo "rescue: target ${host}:${tcp_port}  disable_ach=${disable_ach}  erase=${erase}"
+echo "rescue: connect_timeout=${connect_timeout}s  recv_timeout=${recv_timeout}s"
+echo "rescue: POST ${url}"
+echo "rescue: up to ${max_attempts} attempts, ${sleep_s}s between, ${max_time_s}s per request"
+echo
+
+started=$(date +%s)
+for ((i=1; i<=max_attempts; i++)); do
+  printf "[%3d] %s  " "$i" "$(date +%H:%M:%S)"
+  http_code=$(curl -sS -o /tmp/rescue_resp.$$ -w "%{http_code}" \
+    --max-time "$max_time_s" \
+    -X POST "$url" || echo "000")
+  body=$(cat /tmp/rescue_resp.$$ 2>/dev/null || true)
+  rm -f /tmp/rescue_resp.$$
+
+  case "$http_code" in
+    200|201)
+      elapsed=$(( $(date +%s) - started ))
+      echo "OK  (${elapsed}s total)"
+      echo "$body"
+      exit 0
+      ;;
+    503)
+      # Connection refused / timeout — device busy in another session.  Retry fast.
+      echo "busy (503)"
+      ;;
+    000)
+      echo "curl error (network)"
+      ;;
+    *)
+      echo "HTTP $http_code"
+      echo "  $body" | head -c 400
+      echo
+      ;;
+  esac
+  sleep "$sleep_s"
+done
+
+echo "rescue: gave up after ${max_attempts} attempts" >&2
+exit 1