3 Commits

Author SHA1 Message Date
serversdown ad6071b790 fix(alerts): reset rule state + close open event on rule edit/delete
invalidate() only dropped the rule cache, not the per-(unit,rule) state machine —
so editing a rule's metric/threshold left a stale 'active' phase that mis-evaluated
against the new config (spurious clear, or suppressed onset), and deleting an
in-alarm rule left an open AlertEvent that kept the client portal stuck "in alarm"
forever. update/delete now call _reset_rule_runtime: forget_rule() drops the state
machine and any open event for that rule is closed.

Verified: existing evaluator tests + cooldown scenario still pass; compiles.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 23:40:52 +00:00
serversdown cfdeada9d6 fix(alerts): enforce cooldown_s between onsets
cooldown_s was stored + shown in the UI but never read, so a repeatedly-breaching
signal (e.g. intermittent traffic noise) would flood the alert history with an
event per spike. The evaluator now suppresses a new onset within cooldown_s of the
last, holding the edge so it fires the moment the window lapses if still breaching.
Hysteresis still gates clears. getattr-guarded so partial rule fixtures don't crash.

Verified: existing 4 evaluator tests pass; cooldown scenario (onset → clear →
suppressed re-breach → onset after window) passes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 22:47:39 +00:00
serversdown b51fefca2b feat(alerts): enabled rules pin the monitor on (24/7 evaluation)
The evaluator only runs inside the monitor loop, so a rule on an idle device
never fired. Now creating/updating/deleting an alert rule calls
_sync_keepalive_to_rules: if the unit has any enabled rule, persist
NL43Config.monitor_enabled=True (so the boot auto-start re-enables it after a
restart) and turn on runtime keepalive. Never auto-OFF — a device may be kept
alive for other reasons; operators control that on /admin/slmm. Alert CRUD
endpoints are now async to await the monitor manager.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 19:36:16 +00:00
2 changed files with 55 additions and 3 deletions
+15
View File
@@ -40,6 +40,7 @@ class RuleState:
edge_since: Optional[float] = None # when the current edge condition began (clock time) edge_since: Optional[float] = None # when the current edge condition began (clock time)
peak: float = 0.0 peak: float = 0.0
event_id: Optional[int] = None # the open AlertEvent row (for the clear update) event_id: Optional[int] = None # the open AlertEvent row (for the clear update)
last_onset: Optional[float] = None # time of the last onset (for cooldown)
def _exceeds(value: float, rule) -> bool: def _exceeds(value: float, rule) -> bool:
@@ -68,9 +69,17 @@ def _evaluate_step(state: RuleState, value: float, now: float, rule) -> Optional
if state.edge_since is None: if state.edge_since is None:
state.edge_since = now state.edge_since = now
if now - state.edge_since >= duration: if now - state.edge_since >= duration:
# Cooldown: suppress a new onset within cooldown_s of the last one
# (stops a repeatedly-breaching signal from flooding the history).
# Hold edge_since so it fires the moment cooldown lapses if still
# breaching — don't reset it here.
cooldown = getattr(rule, "cooldown_s", 0) or 0
if state.last_onset is not None and (now - state.last_onset) < cooldown:
return None
state.phase = "active" state.phase = "active"
state.edge_since = None state.edge_since = None
state.peak = value state.peak = value
state.last_onset = now
return "onset" return "onset"
else: else:
state.edge_since = None state.edge_since = None
@@ -166,6 +175,12 @@ class AlertEvaluator:
else: else:
self._rule_cache.pop(unit_id, None) self._rule_cache.pop(unit_id, None)
def forget_rule(self, unit_id: str, rule_id: int) -> None:
"""Drop a rule's per-(unit, rule) state machine after the rule is edited or
deleted, so a stale 'active' phase / open event_id from the old config
doesn't bleed into the new one (mis-firing a clear or suppressing an onset)."""
self._states.pop((unit_id, rule_id), None)
# -- scheduling ---------------------------------------------------------- # -- scheduling ----------------------------------------------------------
def _in_schedule(self, rule) -> bool: def _in_schedule(self, rule) -> bool:
+40 -3
View File
@@ -409,14 +409,47 @@ def _event_dict(e: AlertEvent) -> dict:
} }
async def _sync_keepalive_to_rules(unit_id: str, db: Session):
"""Keep a unit's monitor running while it has enabled alert rules, so the
evaluator runs 24/7 even with no browser watching. Turns keepalive ON (and
persists monitor_enabled so it survives a restart via the boot auto-start)
when enabled rules exist; never turns it OFF — a device may be kept alive for
other reasons, so operators control that on /admin/slmm."""
has_enabled = (db.query(AlertRule)
.filter_by(unit_id=unit_id, enabled=True).first() is not None)
if not has_enabled:
return
cfg = db.query(NL43Config).filter_by(unit_id=unit_id).first()
if cfg and not cfg.monitor_enabled:
cfg.monitor_enabled = True
db.commit()
from app.monitor import monitor_manager
m = await monitor_manager.get(unit_id)
await m.set_keepalive(True)
def _reset_rule_runtime(unit_id: str, rule_id: int, db: Session):
"""After a rule edit/delete: drop its evaluator state machine and close any open
event, so a stale 'active' phase doesn't mis-evaluate against the new config and
the client portal doesn't stay 'in alarm' on a rule that changed or is gone."""
from app.alerts import alert_evaluator
alert_evaluator.forget_rule(unit_id, rule_id)
now = datetime.utcnow()
for evt in db.query(AlertEvent).filter_by(unit_id=unit_id, rule_id=rule_id, status="active").all():
evt.clear_at = now
evt.status = "cleared"
db.commit()
@router.post("/{unit_id}/alerts/rules") @router.post("/{unit_id}/alerts/rules")
def create_alert_rule(unit_id: str, payload: AlertRulePayload, db: Session = Depends(get_db)): async def create_alert_rule(unit_id: str, payload: AlertRulePayload, db: Session = Depends(get_db)):
rule = AlertRule(unit_id=unit_id, **payload.model_dump()) rule = AlertRule(unit_id=unit_id, **payload.model_dump())
db.add(rule) db.add(rule)
db.commit() db.commit()
db.refresh(rule) db.refresh(rule)
from app.alerts import alert_evaluator from app.alerts import alert_evaluator
alert_evaluator.invalidate(unit_id) alert_evaluator.invalidate(unit_id)
await _sync_keepalive_to_rules(unit_id, db)
return {"status": "ok", "rule": _rule_dict(rule)} return {"status": "ok", "rule": _rule_dict(rule)}
@@ -427,7 +460,7 @@ def list_alert_rules(unit_id: str, db: Session = Depends(get_db)):
@router.put("/{unit_id}/alerts/rules/{rule_id}") @router.put("/{unit_id}/alerts/rules/{rule_id}")
def update_alert_rule(unit_id: str, rule_id: int, payload: AlertRulePayload, db: Session = Depends(get_db)): async def update_alert_rule(unit_id: str, rule_id: int, payload: AlertRulePayload, db: Session = Depends(get_db)):
rule = db.query(AlertRule).filter_by(id=rule_id, unit_id=unit_id).first() rule = db.query(AlertRule).filter_by(id=rule_id, unit_id=unit_id).first()
if not rule: if not rule:
raise HTTPException(status_code=404, detail="Alert rule not found") raise HTTPException(status_code=404, detail="Alert rule not found")
@@ -437,11 +470,13 @@ def update_alert_rule(unit_id: str, rule_id: int, payload: AlertRulePayload, db:
db.refresh(rule) db.refresh(rule)
from app.alerts import alert_evaluator from app.alerts import alert_evaluator
alert_evaluator.invalidate(unit_id) alert_evaluator.invalidate(unit_id)
_reset_rule_runtime(unit_id, rule_id, db)
await _sync_keepalive_to_rules(unit_id, db)
return {"status": "ok", "rule": _rule_dict(rule)} return {"status": "ok", "rule": _rule_dict(rule)}
@router.delete("/{unit_id}/alerts/rules/{rule_id}") @router.delete("/{unit_id}/alerts/rules/{rule_id}")
def delete_alert_rule(unit_id: str, rule_id: int, db: Session = Depends(get_db)): async def delete_alert_rule(unit_id: str, rule_id: int, db: Session = Depends(get_db)):
rule = db.query(AlertRule).filter_by(id=rule_id, unit_id=unit_id).first() rule = db.query(AlertRule).filter_by(id=rule_id, unit_id=unit_id).first()
if not rule: if not rule:
raise HTTPException(status_code=404, detail="Alert rule not found") raise HTTPException(status_code=404, detail="Alert rule not found")
@@ -449,6 +484,8 @@ def delete_alert_rule(unit_id: str, rule_id: int, db: Session = Depends(get_db))
db.commit() db.commit()
from app.alerts import alert_evaluator from app.alerts import alert_evaluator
alert_evaluator.invalidate(unit_id) alert_evaluator.invalidate(unit_id)
_reset_rule_runtime(unit_id, rule_id, db) # close its open event so the portal doesn't stay red
await _sync_keepalive_to_rules(unit_id, db) # no-op if no enabled rules remain
return {"status": "ok", "deleted": rule_id} return {"status": "ok", "deleted": rule_id}