feat: wedge detection and automatic recovery orchestration

Adds the recovery layer for the NL-43 TCP wedge failure mode (control listener on 2255 dies while device/FTP stay alive — see SLM-stress-test/NL43_RX55_TCP_Wedge_Investigation_2026-02-18.md). - Failure classification: poll failures probed raw and classified as wedged (refused = stack alive, listener gone) vs offline (timeout). FTP-port probe upgrades wedge confidence to definitive. - RecoveryManager state machine: confirm wedge -> trigger reset -> wait boot -> reconnect -> resume measurement via start_cycle (keeps overwrite protection) -> full incident trail in device logs. - Pluggable reset backends: manual (notify, default) and webhook (HTTP to relay controller: Pi GPIO, Shelly, Tasmota, etc.). - Safety: auto_recovery_enabled off by default, 2-failure confirmation threshold, windowed attempt limit (2 per 6h), one recovery per device. - Endpoints: GET recovery/status, POST recovery/probe (diagnostic), POST recovery/trigger (manual, bypasses threshold). - Migration: migrate_add_recovery_fields.py - Tests: test_wedge_recovery.py — fake wedge-able NL-43 + fake relay webhook; 23 assertions covering classification, end-to-end recovery with resume, and gating. All passing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-05 05:03:25 +00:00
parent 450509d210
commit 3c22f1d70c
7 changed files with 1205 additions and 0 deletions
@@ -13,6 +13,7 @@ import asyncio
 from app.database import get_db
 from app.models import NL43Config, NL43Status
 from app.services import NL43Client, persist_snapshot
+from app.recovery import classify_failure, recovery_manager

 logger = logging.getLogger(__name__)

@@ -53,6 +54,38 @@ class ConfigPayload(BaseModel):
    poll_enabled: bool | None = None
    poll_interval_seconds: int | None = None

+    # Wedge recovery configuration
+    auto_recovery_enabled: bool | None = None
+    reset_backend: str | None = None
+    reset_webhook_url: str | None = None
+    reset_webhook_method: str | None = None
+    recovery_boot_wait_seconds: int | None = Field(None, ge=10, le=600)
+    recovery_reconnect_timeout: int | None = Field(None, ge=30, le=3600)
+    recovery_auto_resume: bool | None = None
+    recovery_max_attempts: int | None = Field(None, ge=1, le=10)
+    recovery_window_minutes: int | None = Field(None, ge=30, le=1440)
+
+    @field_validator("reset_backend")
+    @classmethod
+    def validate_reset_backend(cls, v):
+        if v is not None and v.lower() not in ("manual", "webhook"):
+            raise ValueError("reset_backend must be 'manual' or 'webhook'")
+        return v.lower() if v else v
+
+    @field_validator("reset_webhook_method")
+    @classmethod
+    def validate_reset_webhook_method(cls, v):
+        if v is not None and v.upper() not in ("GET", "POST"):
+            raise ValueError("reset_webhook_method must be 'GET' or 'POST'")
+        return v.upper() if v else v
+
+    @field_validator("reset_webhook_url")
+    @classmethod
+    def validate_reset_webhook_url(cls, v):
+        if v is not None and v != "" and not v.lower().startswith(("http://", "https://")):
+            raise ValueError("reset_webhook_url must start with http:// or https://")
+        return v
+
    @field_validator("host")
    @classmethod
    def validate_host(cls, v):
@@ -348,6 +381,15 @@ def get_config(unit_id: str, db: Session = Depends(get_db)):
            "ftp_username": cfg.ftp_username,
            "ftp_password": cfg.ftp_password,
            "web_enabled": cfg.web_enabled,
+            "auto_recovery_enabled": cfg.auto_recovery_enabled,
+            "reset_backend": cfg.reset_backend,
+            "reset_webhook_url": cfg.reset_webhook_url,
+            "reset_webhook_method": cfg.reset_webhook_method,
+            "recovery_boot_wait_seconds": cfg.recovery_boot_wait_seconds,
+            "recovery_reconnect_timeout": cfg.recovery_reconnect_timeout,
+            "recovery_auto_resume": cfg.recovery_auto_resume,
+            "recovery_max_attempts": cfg.recovery_max_attempts,
+            "recovery_window_minutes": cfg.recovery_window_minutes,
        },
    }

@@ -407,6 +449,24 @@ async def upsert_config(unit_id: str, payload: ConfigPayload, db: Session = Depe
        cfg.poll_enabled = payload.poll_enabled
    if payload.poll_interval_seconds is not None:
        cfg.poll_interval_seconds = payload.poll_interval_seconds
+    if payload.auto_recovery_enabled is not None:
+        cfg.auto_recovery_enabled = payload.auto_recovery_enabled
+    if payload.reset_backend is not None:
+        cfg.reset_backend = payload.reset_backend
+    if payload.reset_webhook_url is not None:
+        cfg.reset_webhook_url = payload.reset_webhook_url or None
+    if payload.reset_webhook_method is not None:
+        cfg.reset_webhook_method = payload.reset_webhook_method
+    if payload.recovery_boot_wait_seconds is not None:
+        cfg.recovery_boot_wait_seconds = payload.recovery_boot_wait_seconds
+    if payload.recovery_reconnect_timeout is not None:
+        cfg.recovery_reconnect_timeout = payload.recovery_reconnect_timeout
+    if payload.recovery_auto_resume is not None:
+        cfg.recovery_auto_resume = payload.recovery_auto_resume
+    if payload.recovery_max_attempts is not None:
+        cfg.recovery_max_attempts = payload.recovery_max_attempts
+    if payload.recovery_window_minutes is not None:
+        cfg.recovery_window_minutes = payload.recovery_window_minutes

    db.commit()
    db.refresh(cfg)
@@ -803,6 +863,89 @@ async def get_measurement_state(unit_id: str, db: Session = Depends(get_db)):
        raise HTTPException(status_code=502, detail=str(e))


+# ============================================================================
+# WEDGE RECOVERY ENDPOINTS
+# ============================================================================
+
+@router.get("/{unit_id}/recovery/status")
+def get_recovery_status(unit_id: str, db: Session = Depends(get_db)):
+    """Get wedge detection and recovery status for a device."""
+    cfg = db.query(NL43Config).filter_by(unit_id=unit_id).first()
+    if not cfg:
+        raise HTTPException(status_code=404, detail="NL43 config not found")
+
+    status = db.query(NL43Status).filter_by(unit_id=unit_id).first()
+    manager_status = recovery_manager.get_status(unit_id)
+
+    return {
+        "status": "ok",
+        "data": {
+            "unit_id": unit_id,
+            "connection_state": status.connection_state if status else "unknown",
+            "recovery_state": status.recovery_state if status else "idle",
+            "last_wedge_at": status.last_wedge_at.isoformat() + "Z" if status and status.last_wedge_at else None,
+            "wedge_count": status.wedge_count if status else 0,
+            "last_recovery_at": status.last_recovery_at.isoformat() + "Z" if status and status.last_recovery_at else None,
+            "last_recovery_result": status.last_recovery_result if status else None,
+            "auto_recovery_enabled": cfg.auto_recovery_enabled,
+            "reset_backend": cfg.reset_backend,
+            **manager_status,
+        },
+    }
+
+
+@router.post("/{unit_id}/recovery/probe")
+async def probe_device(unit_id: str, db: Session = Depends(get_db)):
+    """Probe the device's control and FTP ports and classify its state.
+
+    Does not trigger recovery — diagnostic only. Useful for confirming a
+    wedge before manually triggering a reset.
+    """
+    cfg = db.query(NL43Config).filter_by(unit_id=unit_id).first()
+    if not cfg:
+        raise HTTPException(status_code=404, detail="NL43 config not found")
+
+    classification = await classify_failure(
+        cfg.host,
+        cfg.tcp_port,
+        ftp_port=(cfg.ftp_port or 21) if cfg.ftp_enabled else None,
+    )
+    return {"status": "ok", "unit_id": unit_id, "data": classification}
+
+
+@router.post("/{unit_id}/recovery/trigger")
+async def trigger_recovery(unit_id: str, db: Session = Depends(get_db)):
+    """Manually trigger the recovery flow for a device.
+
+    Bypasses the consecutive-failure confirmation threshold but still
+    respects the one-recovery-at-a-time guard. Use after confirming a wedge
+    via /recovery/probe, or to test the reset hardware end-to-end.
+    """
+    cfg = db.query(NL43Config).filter_by(unit_id=unit_id).first()
+    if not cfg:
+        raise HTTPException(status_code=404, detail="NL43 config not found")
+
+    if recovery_manager.is_active(unit_id):
+        raise HTTPException(status_code=409, detail="Recovery already in progress for this device")
+
+    classification = await classify_failure(
+        cfg.host,
+        cfg.tcp_port,
+        ftp_port=(cfg.ftp_port or 21) if cfg.ftp_enabled else None,
+    )
+
+    started = recovery_manager.start_recovery(cfg, classification)
+    if not started:
+        raise HTTPException(status_code=409, detail="Could not start recovery (already in progress)")
+
+    return {
+        "status": "ok",
+        "unit_id": unit_id,
+        "message": f"Recovery started (backend={cfg.reset_backend or 'manual'})",
+        "classification": classification,
+    }
+
+
@router.post("/{unit_id}/sleep")
 async def sleep_device(unit_id: str, db: Session = Depends(get_db)):
    """Put the device into sleep mode for battery conservation."""