diff --git a/deploy/mi50-watchdog/README.md b/deploy/mi50-watchdog/README.md new file mode 100644 index 0000000..c961a9e --- /dev/null +++ b/deploy/mi50-watchdog/README.md @@ -0,0 +1,54 @@ +# MI50 runaway watchdog (fallback layer "A") + +Independent host-side backstop to Lyra's in-app dream-cycle budget (layer "C", +`lyra/dream.py`). Stops the llama.cpp backend if the MI50 is busy too long or too +hot, and pings Brian. See +`docs/superpowers/specs/2026-07-04-mi50-runaway-guards-design.md`. + +## What it does + +Runs on the **Proxmox host** (`10.0.0.4`) via a systemd timer, every ~2 min: + +- **Duration:** if the GPU is busy (`rocm-smi` use% > 0) for **3600s continuously**, + it stops the container. Any idle read resets the streak, so a legitimate ~40-min + manual workload never trips it. +- **Temperature:** if junction ≥ **97°C** for **3 consecutive checks (~6 min)**, it + stops the container — independent of duration. +- On either trip: `pct exec 202 -- docker stop lyra-brain`, clear state, `logger` a + line, and POST to your ntfy topic. + +All thresholds are `Environment=` overrides in the `.service`. + +## Install (on the Proxmox host, as root) + +```sh +# copy the three files up (from the repo, on lyra-cortex): +scp -i ~/.ssh/id_lyra_proxmox deploy/mi50-watchdog/mi50-watchdog.sh \ + root@10.0.0.4:/usr/local/sbin/mi50-watchdog.sh +scp -i ~/.ssh/id_lyra_proxmox deploy/mi50-watchdog/mi50-watchdog.{service,timer} \ + root@10.0.0.4:/etc/systemd/system/ + +# on the host: +chmod +x /usr/local/sbin/mi50-watchdog.sh +# set your ntfy topic (same one Lyra uses) in the service: +sed -i 's/CHANGE_ME/YOUR_NTFY_TOPIC/' /etc/systemd/system/mi50-watchdog.service +systemctl daemon-reload +systemctl enable --now mi50-watchdog.timer +``` + +## Verify (when the card is back and healthy) + +```sh +# dry run once, watch what it decides: +NTFY_URL= /usr/local/sbin/mi50-watchdog.sh; echo "exit $?" +journalctl -t mi50-watchdog -n 20 --no-pager + +# force a trip test with tiny thresholds (won't touch a healthy idle card unless busy): +MAX_BUSY_SEC=60 TEMP_KILL_C=40 TEMP_KILL_STREAK=1 /usr/local/sbin/mi50-watchdog.sh +# confirm it stopped lyra-brain + sent the ntfy, then restart the container. + +systemctl list-timers mi50-watchdog.timer # confirm it's scheduled +``` + +**Not yet installed / live-verified** — staged here on 2026-07-04 while the card is +off and Brian is away. Install + trip-test when the MI50 is back. diff --git a/deploy/mi50-watchdog/mi50-watchdog.service b/deploy/mi50-watchdog/mi50-watchdog.service new file mode 100644 index 0000000..8dbdeee --- /dev/null +++ b/deploy/mi50-watchdog/mi50-watchdog.service @@ -0,0 +1,16 @@ +[Unit] +Description=MI50 runaway watchdog (stop the llama.cpp backend if the GPU is busy too long or too hot) +After=network-online.target + +[Service] +Type=oneshot +# Fill in your ntfy topic so it can ping Brian when it trips (leave URL empty to log only). +Environment=NTFY_URL=https://ntfy.sh +Environment=NTFY_TOPIC=CHANGE_ME +# Optional overrides (defaults shown): +# Environment=MAX_BUSY_SEC=3600 +# Environment=TEMP_KILL_C=97 +# Environment=TEMP_KILL_STREAK=3 +# Environment=CTID=202 +# Environment=CONTAINER=lyra-brain +ExecStart=/usr/local/sbin/mi50-watchdog.sh diff --git a/deploy/mi50-watchdog/mi50-watchdog.sh b/deploy/mi50-watchdog/mi50-watchdog.sh new file mode 100755 index 0000000..cf5b64c --- /dev/null +++ b/deploy/mi50-watchdog/mi50-watchdog.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# MI50 runaway watchdog — fallback layer "A". +# +# Runs on the Proxmox HOST (10.0.0.4) via a systemd timer (every ~2 min). It is the +# independent backstop to Lyra's own in-app dream-cycle budget ("C", in lyra/dream.py): +# if the MI50 is busy too LONG or runs too HOT, it stops the llama.cpp backend and +# pings Brian — regardless of what caused it. Trips on duration only after a full hour +# of *continuous* busy, so a legitimate ~40-min manual workload runs untouched. +# +# The GPU lives on the host; the llama.cpp container ("lyra-brain") runs inside LXC +# CT202. So temp/use come from host rocm-smi, and the stop goes via `pct exec`. +# +# See docs/superpowers/specs/2026-07-04-mi50-runaway-guards-design.md +set -uo pipefail + +# --- tunables (override in the .service via Environment=) --- +CTID="${CTID:-202}" # LXC holding the docker container +CONTAINER="${CONTAINER:-lyra-brain}" +MAX_BUSY_SEC="${MAX_BUSY_SEC:-3600}" # 1 hr continuous busy -> stop +TEMP_KILL_C="${TEMP_KILL_C:-97}" # junction >= this ... +TEMP_KILL_STREAK="${TEMP_KILL_STREAK:-3}" # ... for this many consecutive checks (~6 min) +NTFY_URL="${NTFY_URL:-}" # e.g. https://ntfy.sh (empty => log only) +NTFY_TOPIC="${NTFY_TOPIC:-}" +BUSY_STATE="${BUSY_STATE:-/run/mi50-watchdog.busy_since}" +HOT_STATE="${HOT_STATE:-/run/mi50-watchdog.hot_streak}" + +now="$(date +%s)" + +alert() { # $1 title, $2 message + logger -t mi50-watchdog "$2" + if [[ -n "$NTFY_URL" && -n "$NTFY_TOPIC" ]]; then + curl -s -m 8 -H "Title: $1" -H "Priority: urgent" -H "Tags: warning" \ + -d "$2" "$NTFY_URL/$NTFY_TOPIC" >/dev/null 2>&1 || true + fi +} + +stop_backend() { # $1 reason + pct exec "$CTID" -- docker stop "$CONTAINER" >/dev/null 2>&1 || true + rm -f "$BUSY_STATE" "$HOT_STATE" + alert "MI50 watchdog stopped the card" "$1" +} + +# Nothing to guard if the backend isn't even running. +running="$(pct exec "$CTID" -- docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo false)" +if [[ "$running" != "true" ]]; then + rm -f "$BUSY_STATE" "$HOT_STATE" + exit 0 +fi + +use="$(rocm-smi --showuse 2>/dev/null | awk -F: '/GPU use \(%\)/ {gsub(/[^0-9]/, "", $NF); print $NF; exit}')" +junction="$(rocm-smi --showtemp 2>/dev/null | awk -F: '/junction/ {gsub(/[^0-9.]/, "", $NF); print $NF; exit}')" + +# --- duration rule: accumulate continuous busy time in a state file --- +busy=0 +[[ "${use:-}" =~ ^[0-9]+$ ]] && (( use > 0 )) && busy=1 +if (( busy )); then + [[ -f "$BUSY_STATE" ]] || echo "$now" > "$BUSY_STATE" + since="$(cat "$BUSY_STATE" 2>/dev/null || echo "$now")" + elapsed=$(( now - since )) + if (( elapsed >= MAX_BUSY_SEC )); then + stop_backend "MI50 busy ${elapsed}s continuously (>= ${MAX_BUSY_SEC}s) — stopped ${CONTAINER}." + exit 0 + fi +else + rm -f "$BUSY_STATE" # idle breaks the streak +fi + +# --- temperature rule: independent of duration --- +if [[ "${junction:-}" =~ ^[0-9.]+$ ]]; then + jint="${junction%.*}" + if (( jint >= TEMP_KILL_C )); then + streak=$(( $(cat "$HOT_STATE" 2>/dev/null || echo 0) + 1 )) + echo "$streak" > "$HOT_STATE" + if (( streak >= TEMP_KILL_STREAK )); then + stop_backend "MI50 junction ${jint}C >= ${TEMP_KILL_C}C for ${streak} checks — stopped ${CONTAINER}." + exit 0 + fi + else + rm -f "$HOT_STATE" # cooled off, reset the streak + fi +fi +exit 0 diff --git a/deploy/mi50-watchdog/mi50-watchdog.timer b/deploy/mi50-watchdog/mi50-watchdog.timer new file mode 100644 index 0000000..2b91115 --- /dev/null +++ b/deploy/mi50-watchdog/mi50-watchdog.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run the MI50 runaway watchdog every 2 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=2min +AccuracySec=15s + +[Install] +WantedBy=timers.target