feat: best-effort mirror (dual-send) of heartbeats + events

Optional second destination so each heartbeat and event is posted to a
mirror server (the office NAS) alongside the primary — dual-write to
de-risk the migration. Default off (blank mirror URLs); existing installs
unchanged.

- event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass()
  (reliable event mirror with its OWN state file, total exception isolation,
  never raises into the primary path).
- series3_watcher: MIRROR_API_URL/MIRROR_SFM_URL/MIRROR_SFM_STATE_FILE config;
  best-effort heartbeat mirror; isolated event-mirror pass after the primary.
- settings dialog + config-template: new 'Mirror' tab / keys.
- 8 new tests incl. the isolation invariant. 44 passing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-24 23:16:47 +00:00
parent 730f215e23
commit 4adaa09d75
5 changed files with 337 additions and 0 deletions
+60
View File
@@ -110,6 +110,12 @@ def load_config(path: str) -> Dict[str, Any]:
# first deploy in a folder that's been accumulating for years.
# See README "First-time deployment" section.
"SFM_MAX_FORWARDS_PER_PASS": get_int("SFM_MAX_FORWARDS_PER_PASS", 500),
# Mirror (dual-send) — best-effort second destination, default OFF.
# See docs/mirror-dual-send-design.md.
"MIRROR_API_URL": get_str("MIRROR_API_URL", ""),
"MIRROR_SFM_URL": get_str("MIRROR_SFM_URL", ""),
"MIRROR_SFM_STATE_FILE": get_str("MIRROR_SFM_STATE_FILE", ""),
}
@@ -427,6 +433,33 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
else:
print("[CFG] SFM_FORWARD_ENABLED=false (event forwarding disabled)")
# ---- Mirror (dual-send) setup ----
# Best-effort second destination. The event mirror gets its OWN state
# file so its idempotency is independent of the primary's.
MIRROR_API_URL = cfg.get("MIRROR_API_URL", "")
MIRROR_SFM_URL = cfg.get("MIRROR_SFM_URL", "")
mirror_state = None
if sfm_state is not None and MIRROR_SFM_URL:
try:
from event_forwarder import ForwardState
mirror_state_file = cfg.get("MIRROR_SFM_STATE_FILE") or os.path.join(
os.path.dirname(LOG_FILE) or here, "sfm_forwarded_mirror.json"
)
mirror_state = ForwardState(mirror_state_file)
print("[CFG] mirror event forwarding → {} state={} ({} known)".format(
MIRROR_SFM_URL, mirror_state_file, mirror_state.count()))
log_message(LOG_FILE, ENABLE_LOGGING,
"[mirror] event mirror enabled url={} state={} known={}".format(
MIRROR_SFM_URL, mirror_state_file, mirror_state.count()))
except Exception as e:
print("[WARN] mirror forwarder init failed: {}".format(e))
log_message(LOG_FILE, ENABLE_LOGGING,
"[warn] mirror forwarder init failed: {}".format(e))
mirror_state = None
if MIRROR_API_URL:
log_message(LOG_FILE, ENABLE_LOGGING,
"[mirror] heartbeat mirror enabled url={}".format(MIRROR_API_URL))
while not stop_event.is_set():
try:
now_local = datetime.now().isoformat()
@@ -497,6 +530,13 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
hb_payload["log_tail"] = _read_log_tail(cfg.get("LOG_FILE", ""), 25)
response = send_api_payload(hb_payload, cfg.get("API_URL", ""))
last_api_ts = now_ts
# Best-effort heartbeat mirror — never affects primary.
if MIRROR_API_URL:
try:
send_api_payload(hb_payload, MIRROR_API_URL)
except Exception as e:
log_message(LOG_FILE, ENABLE_LOGGING,
"[mirror] heartbeat post failed: {}".format(e))
if response is not None:
state["api_status"] = "ok"
state["last_api"] = datetime.now()
@@ -550,6 +590,26 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
print(err)
log_message(LOG_FILE, ENABLE_LOGGING, err)
state["sfm_status"] = "fail"
# Best-effort event mirror — own state, fast-fail guard,
# fully isolated. Runs after the primary each tick.
if mirror_state is not None:
from event_forwarder import mirror_forward_pass
m_counts = mirror_forward_pass(
WATCH_PATH, MIRROR_SFM_URL, mirror_state,
max_age_days=MAX_EVENT_AGE_DAYS,
quiescence_seconds=int(cfg.get("SFM_QUIESCENCE_SECONDS", 5)),
missing_report_grace_seconds=int(cfg.get("SFM_MISSING_REPORT_GRACE_SECONDS", 60)),
timeout=int(cfg.get("SFM_HTTP_TIMEOUT", 60)),
max_per_pass=int(cfg.get("SFM_MAX_FORWARDS_PER_PASS", 500)),
logger=lambda m: log_message(LOG_FILE, ENABLE_LOGGING, "[mirror] " + m),
)
if m_counts is None:
log_message(LOG_FILE, ENABLE_LOGGING,
"[mirror] forward skipped (unreachable/error): {}".format(MIRROR_SFM_URL))
else:
log_message(LOG_FILE, ENABLE_LOGGING,
"[mirror] pass forwarded={forwarded} errors={errors}".format(**m_counts))
else:
state["sfm_status"] = "disabled"