feat: best-effort mirror (dual-send) of heartbeats + events

Optional second destination so each heartbeat and event is posted to a
mirror server (the office NAS) alongside the primary — dual-write to
de-risk the migration. Default off (blank mirror URLs); existing installs
unchanged.

- event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass()
  (reliable event mirror with its OWN state file, total exception isolation,
  never raises into the primary path).
- series4_ingest: mirror_api_url/mirror_sfm_url/mirror_sfm_state_file config;
  best-effort heartbeat mirror; isolated event-mirror pass after the primary.
- settings dialog: new 'Mirror' tab.
- 9 new tests incl. the isolation invariant (down mirror = fast no-op that
  never touches primary state). 42 passing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-24 23:10:50 +00:00
parent dae67afeb9
commit 7d22442bad
5 changed files with 328 additions and 1 deletions
+59
View File
@@ -69,6 +69,15 @@ def load_config(config_path: str) -> Dict[str, Any]:
"sfm_state_file": "", # blank → <log_dir>/thor_forwarded.json
"sfm_max_forwards_per_pass": 500,
"sfm_max_event_age_days": 365,
# Mirror (dual-send) — best-effort second destination, default OFF.
# See docs/mirror-dual-send-design.md. Heartbeat-mirror fires iff
# api_url AND mirror_api_url are set; event-mirror runs iff SFM
# forwarding is on AND mirror_sfm_url is set. Own state file →
# idempotency independent of the primary.
"mirror_api_url": "",
"mirror_sfm_url": "",
"mirror_sfm_state_file": "", # blank → <log_dir>/thor_forwarded_mirror.json
}
with open(config_path, "r", encoding="utf-8") as f:
@@ -329,6 +338,12 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
sfm_state_path = str(cfg.get("sfm_state_file", "")).strip() or \
os.path.join(state["log_dir"], "thor_forwarded.json")
# Mirror (dual-send) config
MIRROR_API_URL = str(cfg.get("mirror_api_url", "")).strip()
MIRROR_SFM_URL = str(cfg.get("mirror_sfm_url", "")).strip()
mirror_state_path = str(cfg.get("mirror_sfm_state_file", "")).strip() or \
os.path.join(state["log_dir"], "thor_forwarded_mirror.json")
log_message(log_file, ENABLE_LOGGING,
"[cfg] THORDATA_PATH={} SCAN_INTERVAL={}s API_INTERVAL={}s API={} SFM={}".format(
THORDATA_PATH, SCAN_INTERVAL, API_INTERVAL, bool(API_URL),
@@ -362,6 +377,24 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
else:
state["sfm_status"] = "disabled"
# Mirror event-forward state — independent of the primary's. Created
# only when the primary forwarder is on AND a mirror URL is set.
mirror_state_obj: Optional[event_forwarder.ForwardState] = None
if sfm_state_obj is not None and MIRROR_SFM_URL:
try:
mirror_state_obj = event_forwarder.ForwardState(mirror_state_path)
log_message(log_file, ENABLE_LOGGING,
"[mirror] event mirror ready url={} state={} known={}".format(
MIRROR_SFM_URL, mirror_state_path, mirror_state_obj.count()))
print("[MIRROR] event mirror ready url={}".format(MIRROR_SFM_URL))
except Exception as exc:
log_message(log_file, ENABLE_LOGGING,
"[mirror] event mirror init failed: {}".format(exc))
mirror_state_obj = None
if MIRROR_API_URL:
log_message(log_file, ENABLE_LOGGING,
"[mirror] heartbeat mirror enabled url={}".format(MIRROR_API_URL))
state["last_forward"] = None
state["last_forward_counts"] = None
@@ -419,6 +452,13 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
payload["log_tail"] = _read_log_tail(log_file, 25)
response = send_api_payload(payload, API_URL, API_TIMEOUT)
last_api_ts = now_ts
# Best-effort heartbeat mirror — never affects primary.
if MIRROR_API_URL:
try:
send_api_payload(payload, MIRROR_API_URL, API_TIMEOUT)
except Exception as exc:
log_message(log_file, ENABLE_LOGGING,
"[mirror] heartbeat post failed: {}".format(exc))
if response is not None:
state["api_status"] = "ok"
if response.get("update_available"):
@@ -459,6 +499,25 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
print(msg)
log_message(log_file, ENABLE_LOGGING, msg)
# Best-effort event mirror — own state, fast-fail guard,
# fully isolated. Runs after the primary each tick.
if mirror_state_obj is not None:
m_counts = event_forwarder.mirror_forward_pass(
THORDATA_PATH, MIRROR_SFM_URL, mirror_state_obj,
max_age_days=SFM_MAX_AGE_DAYS,
quiescence_seconds=SFM_QUIESCENCE,
missing_report_grace_seconds=SFM_GRACE,
timeout=SFM_HTTP_TIMEOUT,
max_per_pass=SFM_MAX_PER_PASS,
logger=lambda m: log_message(log_file, ENABLE_LOGGING, "[mirror] " + m),
)
if m_counts is None:
log_message(log_file, ENABLE_LOGGING,
"[mirror] forward skipped (unreachable/error): {}".format(MIRROR_SFM_URL))
else:
log_message(log_file, ENABLE_LOGGING,
"[mirror] pass forwarded={forwarded} errors={errors}".format(**m_counts))
except Exception as e:
err = "[loop-error] {}".format(e)
print(err)