feat: best-effort mirror (dual-send) of heartbeats + events
Optional second destination so each heartbeat and event is posted to a mirror server (the office NAS) alongside the primary — dual-write to de-risk the migration. Default off (blank mirror URLs); existing installs unchanged. - event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass() (reliable event mirror with its OWN state file, total exception isolation, never raises into the primary path). - series4_ingest: mirror_api_url/mirror_sfm_url/mirror_sfm_state_file config; best-effort heartbeat mirror; isolated event-mirror pass after the primary. - settings dialog: new 'Mirror' tab. - 9 new tests incl. the isolation invariant (down mirror = fast no-op that never touches primary state). 42 passing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -69,6 +69,15 @@ def load_config(config_path: str) -> Dict[str, Any]:
|
||||
"sfm_state_file": "", # blank → <log_dir>/thor_forwarded.json
|
||||
"sfm_max_forwards_per_pass": 500,
|
||||
"sfm_max_event_age_days": 365,
|
||||
|
||||
# Mirror (dual-send) — best-effort second destination, default OFF.
|
||||
# See docs/mirror-dual-send-design.md. Heartbeat-mirror fires iff
|
||||
# api_url AND mirror_api_url are set; event-mirror runs iff SFM
|
||||
# forwarding is on AND mirror_sfm_url is set. Own state file →
|
||||
# idempotency independent of the primary.
|
||||
"mirror_api_url": "",
|
||||
"mirror_sfm_url": "",
|
||||
"mirror_sfm_state_file": "", # blank → <log_dir>/thor_forwarded_mirror.json
|
||||
}
|
||||
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
@@ -329,6 +338,12 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
sfm_state_path = str(cfg.get("sfm_state_file", "")).strip() or \
|
||||
os.path.join(state["log_dir"], "thor_forwarded.json")
|
||||
|
||||
# Mirror (dual-send) config
|
||||
MIRROR_API_URL = str(cfg.get("mirror_api_url", "")).strip()
|
||||
MIRROR_SFM_URL = str(cfg.get("mirror_sfm_url", "")).strip()
|
||||
mirror_state_path = str(cfg.get("mirror_sfm_state_file", "")).strip() or \
|
||||
os.path.join(state["log_dir"], "thor_forwarded_mirror.json")
|
||||
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[cfg] THORDATA_PATH={} SCAN_INTERVAL={}s API_INTERVAL={}s API={} SFM={}".format(
|
||||
THORDATA_PATH, SCAN_INTERVAL, API_INTERVAL, bool(API_URL),
|
||||
@@ -362,6 +377,24 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
else:
|
||||
state["sfm_status"] = "disabled"
|
||||
|
||||
# Mirror event-forward state — independent of the primary's. Created
|
||||
# only when the primary forwarder is on AND a mirror URL is set.
|
||||
mirror_state_obj: Optional[event_forwarder.ForwardState] = None
|
||||
if sfm_state_obj is not None and MIRROR_SFM_URL:
|
||||
try:
|
||||
mirror_state_obj = event_forwarder.ForwardState(mirror_state_path)
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] event mirror ready url={} state={} known={}".format(
|
||||
MIRROR_SFM_URL, mirror_state_path, mirror_state_obj.count()))
|
||||
print("[MIRROR] event mirror ready url={}".format(MIRROR_SFM_URL))
|
||||
except Exception as exc:
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] event mirror init failed: {}".format(exc))
|
||||
mirror_state_obj = None
|
||||
if MIRROR_API_URL:
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] heartbeat mirror enabled url={}".format(MIRROR_API_URL))
|
||||
|
||||
state["last_forward"] = None
|
||||
state["last_forward_counts"] = None
|
||||
|
||||
@@ -419,6 +452,13 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
payload["log_tail"] = _read_log_tail(log_file, 25)
|
||||
response = send_api_payload(payload, API_URL, API_TIMEOUT)
|
||||
last_api_ts = now_ts
|
||||
# Best-effort heartbeat mirror — never affects primary.
|
||||
if MIRROR_API_URL:
|
||||
try:
|
||||
send_api_payload(payload, MIRROR_API_URL, API_TIMEOUT)
|
||||
except Exception as exc:
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] heartbeat post failed: {}".format(exc))
|
||||
if response is not None:
|
||||
state["api_status"] = "ok"
|
||||
if response.get("update_available"):
|
||||
@@ -459,6 +499,25 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
print(msg)
|
||||
log_message(log_file, ENABLE_LOGGING, msg)
|
||||
|
||||
# Best-effort event mirror — own state, fast-fail guard,
|
||||
# fully isolated. Runs after the primary each tick.
|
||||
if mirror_state_obj is not None:
|
||||
m_counts = event_forwarder.mirror_forward_pass(
|
||||
THORDATA_PATH, MIRROR_SFM_URL, mirror_state_obj,
|
||||
max_age_days=SFM_MAX_AGE_DAYS,
|
||||
quiescence_seconds=SFM_QUIESCENCE,
|
||||
missing_report_grace_seconds=SFM_GRACE,
|
||||
timeout=SFM_HTTP_TIMEOUT,
|
||||
max_per_pass=SFM_MAX_PER_PASS,
|
||||
logger=lambda m: log_message(log_file, ENABLE_LOGGING, "[mirror] " + m),
|
||||
)
|
||||
if m_counts is None:
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] forward skipped (unreachable/error): {}".format(MIRROR_SFM_URL))
|
||||
else:
|
||||
log_message(log_file, ENABLE_LOGGING,
|
||||
"[mirror] pass forwarded={forwarded} errors={errors}".format(**m_counts))
|
||||
|
||||
except Exception as e:
|
||||
err = "[loop-error] {}".format(e)
|
||||
print(err)
|
||||
|
||||
Reference in New Issue
Block a user