feat: best-effort mirror (dual-send) of heartbeats + events

Optional second destination so each heartbeat and event is posted to a
mirror server (the office NAS) alongside the primary — dual-write to
de-risk the migration. Default off (blank mirror URLs); existing installs
unchanged.

- event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass()
  (reliable event mirror with its OWN state file, total exception isolation,
  never raises into the primary path).
- series4_ingest: mirror_api_url/mirror_sfm_url/mirror_sfm_state_file config;
  best-effort heartbeat mirror; isolated event-mirror pass after the primary.
- settings dialog: new 'Mirror' tab.
- 9 new tests incl. the isolation invariant (down mirror = fast no-op that
  never touches primary state). 42 passing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-24 23:10:50 +00:00
parent dae67afeb9
commit 7d22442bad
5 changed files with 328 additions and 1 deletions
+129
View File
@@ -712,5 +712,134 @@ class TestForwardPending(unittest.TestCase):
self.assertEqual(len(_FakeImportHandler.received), 2)
# ── Mirror (dual-send) ───────────────────────────────────────────────────────
class TestMirrorReachable(unittest.TestCase):
def test_empty_url_is_unreachable(self):
self.assertFalse(ef.mirror_reachable("", timeout=1.0))
def test_dead_port_is_unreachable_and_fast(self):
# Nothing listens on 127.0.0.1:1 → connection refused → False,
# and it must NOT hang (this is the whole point of the guard).
t0 = time.time()
self.assertFalse(ef.mirror_reachable("http://127.0.0.1:1", timeout=2.0))
self.assertLess(time.time() - t0, 2.5)
def test_any_http_response_counts_as_reachable(self):
# The fake server 501s on GET /health, but it's UP — so reachable.
server, base = _start_fake_server()
try:
self.assertTrue(ef.mirror_reachable(base, timeout=2.0))
finally:
server.shutdown()
server.server_close()
class TestMirrorForwardPass(unittest.TestCase):
"""The dual-send entry point: best-effort, isolated, own state."""
def setUp(self):
_FakeImportHandler.received = []
self.server, self.base_url = _start_fake_server()
def tearDown(self):
self.server.shutdown()
self.server.server_close()
def _one_event(self, root: Path) -> None:
unit = _make_thordata(root, "Project A", "UM11719")
_make_event(unit, "UM11719_20231219163444.IDFW", age_seconds=200, content=b"binary")
_make_txt(unit, "UM11719_20231219163444.IDFW", age_seconds=100, content=b"report")
def test_empty_mirror_url_is_noop(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._one_event(root)
mstate = ef.ForwardState(str(root / "mirror.json"))
self.assertIsNone(ef.mirror_forward_pass(str(root), "", mstate, max_age_days=30))
self.assertEqual(len(_FakeImportHandler.received), 0)
def test_skips_when_unreachable_without_posting(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._one_event(root)
mstate = ef.ForwardState(str(root / "mirror.json"))
result = ef.mirror_forward_pass(
str(root), self.base_url, mstate,
reachable_fn=lambda url, timeout=3.0: False, # force "down"
max_age_days=30, quiescence_seconds=5,
missing_report_grace_seconds=60, timeout=5.0,
)
self.assertIsNone(result)
self.assertEqual(mstate.count(), 0) # nothing forwarded
self.assertEqual(len(_FakeImportHandler.received), 0) # never POSTed
def test_forwards_when_reachable_using_its_own_state(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._one_event(root)
mstate = ef.ForwardState(str(root / "mirror.json"))
counts = ef.mirror_forward_pass(
str(root), self.base_url, mstate,
reachable_fn=lambda url, timeout=3.0: True,
max_age_days=30, quiescence_seconds=5,
missing_report_grace_seconds=60, timeout=5.0,
)
self.assertIsNotNone(counts)
self.assertEqual(counts["forwarded"], 1)
self.assertEqual(mstate.count(), 1)
self.assertEqual(len(_FakeImportHandler.received), 1)
def test_never_raises_when_forward_blows_up(self):
# Even if forward_pending raises, the mirror swallows it → None.
def _boom(*a, **k):
raise RuntimeError("boom")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._one_event(root)
mstate = ef.ForwardState(str(root / "mirror.json"))
orig = ef.forward_pending
ef.forward_pending = _boom
try:
result = ef.mirror_forward_pass(
str(root), self.base_url, mstate,
reachable_fn=lambda url, timeout=3.0: True,
max_age_days=30,
)
finally:
ef.forward_pending = orig
self.assertIsNone(result)
def test_down_mirror_leaves_primary_state_untouched(self):
# The invariant: a real down mirror (via the real reachability
# probe) is a fast no-op and the primary forward is unaffected.
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._one_event(root)
primary_state = ef.ForwardState(str(root / "primary.json"))
pcounts = ef.forward_pending(
str(root), self.base_url, primary_state,
max_age_days=30, quiescence_seconds=5,
missing_report_grace_seconds=60, timeout=5.0,
)
self.assertEqual(pcounts["forwarded"], 1)
primary_snapshot = primary_state.count()
mstate = ef.ForwardState(str(root / "mirror.json"))
t0 = time.time()
result = ef.mirror_forward_pass(
str(root), "http://127.0.0.1:1", mstate, # nothing listens
max_age_days=30, quiescence_seconds=5,
missing_report_grace_seconds=60, timeout=5.0,
)
self.assertIsNone(result)
self.assertLess(time.time() - t0, 3.5) # fast-failed, didn't hang
self.assertEqual(mstate.count(), 0)
self.assertEqual(primary_state.count(), primary_snapshot) # untouched
if __name__ == "__main__":
unittest.main()