feat: best-effort mirror (dual-send) of heartbeats + events
Optional second destination so each heartbeat and event is posted to a mirror server (the office NAS) alongside the primary — dual-write to de-risk the migration. Default off (blank mirror URLs); existing installs unchanged. - event_forwarder: mirror_reachable() fast-fail probe + mirror_forward_pass() (reliable event mirror with its OWN state file, total exception isolation, never raises into the primary path). - series4_ingest: mirror_api_url/mirror_sfm_url/mirror_sfm_state_file config; best-effort heartbeat mirror; isolated event-mirror pass after the primary. - settings dialog: new 'Mirror' tab. - 9 new tests incl. the isolation invariant (down mirror = fast no-op that never touches primary state). 42 passing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -712,5 +712,134 @@ class TestForwardPending(unittest.TestCase):
|
||||
self.assertEqual(len(_FakeImportHandler.received), 2)
|
||||
|
||||
|
||||
# ── Mirror (dual-send) ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMirrorReachable(unittest.TestCase):
|
||||
|
||||
def test_empty_url_is_unreachable(self):
|
||||
self.assertFalse(ef.mirror_reachable("", timeout=1.0))
|
||||
|
||||
def test_dead_port_is_unreachable_and_fast(self):
|
||||
# Nothing listens on 127.0.0.1:1 → connection refused → False,
|
||||
# and it must NOT hang (this is the whole point of the guard).
|
||||
t0 = time.time()
|
||||
self.assertFalse(ef.mirror_reachable("http://127.0.0.1:1", timeout=2.0))
|
||||
self.assertLess(time.time() - t0, 2.5)
|
||||
|
||||
def test_any_http_response_counts_as_reachable(self):
|
||||
# The fake server 501s on GET /health, but it's UP — so reachable.
|
||||
server, base = _start_fake_server()
|
||||
try:
|
||||
self.assertTrue(ef.mirror_reachable(base, timeout=2.0))
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
class TestMirrorForwardPass(unittest.TestCase):
|
||||
"""The dual-send entry point: best-effort, isolated, own state."""
|
||||
|
||||
def setUp(self):
|
||||
_FakeImportHandler.received = []
|
||||
self.server, self.base_url = _start_fake_server()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
|
||||
def _one_event(self, root: Path) -> None:
|
||||
unit = _make_thordata(root, "Project A", "UM11719")
|
||||
_make_event(unit, "UM11719_20231219163444.IDFW", age_seconds=200, content=b"binary")
|
||||
_make_txt(unit, "UM11719_20231219163444.IDFW", age_seconds=100, content=b"report")
|
||||
|
||||
def test_empty_mirror_url_is_noop(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._one_event(root)
|
||||
mstate = ef.ForwardState(str(root / "mirror.json"))
|
||||
self.assertIsNone(ef.mirror_forward_pass(str(root), "", mstate, max_age_days=30))
|
||||
self.assertEqual(len(_FakeImportHandler.received), 0)
|
||||
|
||||
def test_skips_when_unreachable_without_posting(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._one_event(root)
|
||||
mstate = ef.ForwardState(str(root / "mirror.json"))
|
||||
result = ef.mirror_forward_pass(
|
||||
str(root), self.base_url, mstate,
|
||||
reachable_fn=lambda url, timeout=3.0: False, # force "down"
|
||||
max_age_days=30, quiescence_seconds=5,
|
||||
missing_report_grace_seconds=60, timeout=5.0,
|
||||
)
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(mstate.count(), 0) # nothing forwarded
|
||||
self.assertEqual(len(_FakeImportHandler.received), 0) # never POSTed
|
||||
|
||||
def test_forwards_when_reachable_using_its_own_state(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._one_event(root)
|
||||
mstate = ef.ForwardState(str(root / "mirror.json"))
|
||||
counts = ef.mirror_forward_pass(
|
||||
str(root), self.base_url, mstate,
|
||||
reachable_fn=lambda url, timeout=3.0: True,
|
||||
max_age_days=30, quiescence_seconds=5,
|
||||
missing_report_grace_seconds=60, timeout=5.0,
|
||||
)
|
||||
self.assertIsNotNone(counts)
|
||||
self.assertEqual(counts["forwarded"], 1)
|
||||
self.assertEqual(mstate.count(), 1)
|
||||
self.assertEqual(len(_FakeImportHandler.received), 1)
|
||||
|
||||
def test_never_raises_when_forward_blows_up(self):
|
||||
# Even if forward_pending raises, the mirror swallows it → None.
|
||||
def _boom(*a, **k):
|
||||
raise RuntimeError("boom")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._one_event(root)
|
||||
mstate = ef.ForwardState(str(root / "mirror.json"))
|
||||
orig = ef.forward_pending
|
||||
ef.forward_pending = _boom
|
||||
try:
|
||||
result = ef.mirror_forward_pass(
|
||||
str(root), self.base_url, mstate,
|
||||
reachable_fn=lambda url, timeout=3.0: True,
|
||||
max_age_days=30,
|
||||
)
|
||||
finally:
|
||||
ef.forward_pending = orig
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_down_mirror_leaves_primary_state_untouched(self):
|
||||
# The invariant: a real down mirror (via the real reachability
|
||||
# probe) is a fast no-op and the primary forward is unaffected.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._one_event(root)
|
||||
|
||||
primary_state = ef.ForwardState(str(root / "primary.json"))
|
||||
pcounts = ef.forward_pending(
|
||||
str(root), self.base_url, primary_state,
|
||||
max_age_days=30, quiescence_seconds=5,
|
||||
missing_report_grace_seconds=60, timeout=5.0,
|
||||
)
|
||||
self.assertEqual(pcounts["forwarded"], 1)
|
||||
primary_snapshot = primary_state.count()
|
||||
|
||||
mstate = ef.ForwardState(str(root / "mirror.json"))
|
||||
t0 = time.time()
|
||||
result = ef.mirror_forward_pass(
|
||||
str(root), "http://127.0.0.1:1", mstate, # nothing listens
|
||||
max_age_days=30, quiescence_seconds=5,
|
||||
missing_report_grace_seconds=60, timeout=5.0,
|
||||
)
|
||||
self.assertIsNone(result)
|
||||
self.assertLess(time.time() - t0, 3.5) # fast-failed, didn't hang
|
||||
self.assertEqual(mstate.count(), 0)
|
||||
self.assertEqual(primary_state.count(), primary_snapshot) # untouched
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user