feat: best-effort mirror (dual-send) of heartbeats + events
Optional second destination so each heartbeat and event is posted to a mirror server (the office NAS) alongside the primary — dual-write to de-risk the migration. Default off (blank mirror URLs); existing installs unchanged. - event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass() (reliable event mirror with its OWN state file, total exception isolation, never raises into the primary path). - series3_watcher: MIRROR_API_URL/MIRROR_SFM_URL/MIRROR_SFM_STATE_FILE config; best-effort heartbeat mirror; isolated event-mirror pass after the primary. - settings dialog + config-template: new 'Mirror' tab / keys. - 8 new tests incl. the isolation invariant. 44 passing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -110,6 +110,12 @@ def load_config(path: str) -> Dict[str, Any]:
|
||||
# first deploy in a folder that's been accumulating for years.
|
||||
# See README "First-time deployment" section.
|
||||
"SFM_MAX_FORWARDS_PER_PASS": get_int("SFM_MAX_FORWARDS_PER_PASS", 500),
|
||||
|
||||
# Mirror (dual-send) — best-effort second destination, default OFF.
|
||||
# See docs/mirror-dual-send-design.md.
|
||||
"MIRROR_API_URL": get_str("MIRROR_API_URL", ""),
|
||||
"MIRROR_SFM_URL": get_str("MIRROR_SFM_URL", ""),
|
||||
"MIRROR_SFM_STATE_FILE": get_str("MIRROR_SFM_STATE_FILE", ""),
|
||||
}
|
||||
|
||||
|
||||
@@ -427,6 +433,33 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
else:
|
||||
print("[CFG] SFM_FORWARD_ENABLED=false (event forwarding disabled)")
|
||||
|
||||
# ---- Mirror (dual-send) setup ----
|
||||
# Best-effort second destination. The event mirror gets its OWN state
|
||||
# file so its idempotency is independent of the primary's.
|
||||
MIRROR_API_URL = cfg.get("MIRROR_API_URL", "")
|
||||
MIRROR_SFM_URL = cfg.get("MIRROR_SFM_URL", "")
|
||||
mirror_state = None
|
||||
if sfm_state is not None and MIRROR_SFM_URL:
|
||||
try:
|
||||
from event_forwarder import ForwardState
|
||||
mirror_state_file = cfg.get("MIRROR_SFM_STATE_FILE") or os.path.join(
|
||||
os.path.dirname(LOG_FILE) or here, "sfm_forwarded_mirror.json"
|
||||
)
|
||||
mirror_state = ForwardState(mirror_state_file)
|
||||
print("[CFG] mirror event forwarding → {} state={} ({} known)".format(
|
||||
MIRROR_SFM_URL, mirror_state_file, mirror_state.count()))
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[mirror] event mirror enabled url={} state={} known={}".format(
|
||||
MIRROR_SFM_URL, mirror_state_file, mirror_state.count()))
|
||||
except Exception as e:
|
||||
print("[WARN] mirror forwarder init failed: {}".format(e))
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[warn] mirror forwarder init failed: {}".format(e))
|
||||
mirror_state = None
|
||||
if MIRROR_API_URL:
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[mirror] heartbeat mirror enabled url={}".format(MIRROR_API_URL))
|
||||
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
now_local = datetime.now().isoformat()
|
||||
@@ -497,6 +530,13 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
hb_payload["log_tail"] = _read_log_tail(cfg.get("LOG_FILE", ""), 25)
|
||||
response = send_api_payload(hb_payload, cfg.get("API_URL", ""))
|
||||
last_api_ts = now_ts
|
||||
# Best-effort heartbeat mirror — never affects primary.
|
||||
if MIRROR_API_URL:
|
||||
try:
|
||||
send_api_payload(hb_payload, MIRROR_API_URL)
|
||||
except Exception as e:
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[mirror] heartbeat post failed: {}".format(e))
|
||||
if response is not None:
|
||||
state["api_status"] = "ok"
|
||||
state["last_api"] = datetime.now()
|
||||
@@ -550,6 +590,26 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
print(err)
|
||||
log_message(LOG_FILE, ENABLE_LOGGING, err)
|
||||
state["sfm_status"] = "fail"
|
||||
|
||||
# Best-effort event mirror — own state, fast-fail guard,
|
||||
# fully isolated. Runs after the primary each tick.
|
||||
if mirror_state is not None:
|
||||
from event_forwarder import mirror_forward_pass
|
||||
m_counts = mirror_forward_pass(
|
||||
WATCH_PATH, MIRROR_SFM_URL, mirror_state,
|
||||
max_age_days=MAX_EVENT_AGE_DAYS,
|
||||
quiescence_seconds=int(cfg.get("SFM_QUIESCENCE_SECONDS", 5)),
|
||||
missing_report_grace_seconds=int(cfg.get("SFM_MISSING_REPORT_GRACE_SECONDS", 60)),
|
||||
timeout=int(cfg.get("SFM_HTTP_TIMEOUT", 60)),
|
||||
max_per_pass=int(cfg.get("SFM_MAX_FORWARDS_PER_PASS", 500)),
|
||||
logger=lambda m: log_message(LOG_FILE, ENABLE_LOGGING, "[mirror] " + m),
|
||||
)
|
||||
if m_counts is None:
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[mirror] forward skipped (unreachable/error): {}".format(MIRROR_SFM_URL))
|
||||
else:
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[mirror] pass forwarded={forwarded} errors={errors}".format(**m_counts))
|
||||
else:
|
||||
state["sfm_status"] = "disabled"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user