serversdown/seismo-relay
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

perf(micromate): 16 KB per request, not 1024 -- 14x fewer round trips

Browse Source 
The chunk-size ceiling, measured on UM20147 with every size checked
byte-for-byte against a known-good download:

  1024 / 2048 / 4096 / 8192 / 16384  ->  served in full
  32768 / 65535                      ->  SILENTLY CLAMPED to 16384

The clamp is the important part: a 32,768 B request returns 16,384 B of perfectly
good data and no error.  Nothing in the response says it was truncated; the
length is the only signal.

THAT DICTATES HOW THE LOOP MUST BE WRITTEN.  read_event_file() now tracks its
offset by BYTES RECEIVED rather than striding by chunk index.  A fixed stride
would either fail on a clamp or skip the bytes it never collected; tracking what
actually arrived makes a clamp cost one extra request, and makes the loop
self-correcting against any short response -- precisely the failure mode that has
bitten the Series III side repeatedly.

Consequence of that change, recorded because it reverses an earlier decision: a
short chunk is NO LONGER AN ERROR.  It used to raise, on the principle that a
silently short event is this codebase's recurring bug.  But the device returns
short legitimately, and the real protection is the offset arithmetic plus the
final total-length check -- which still raises on a genuinely truncated event.
The test was rewritten rather than deleted, and says why.

CHUNK_SIZE = 16384.  For UM20147's events: 4,796 B goes 5 requests -> 1,
30,230 B goes 30 -> 2, and 72,560 B goes 71 -> 5.  Over cellular at ~0.65 s per
round trip that is ~46 s -> ~3.2 s on the large one.

Measured on one unit over USB, so THOR_CHUNK_SIZE = 1024 stays available and the
replay tests pin it -- reproducing THOR's exact traffic is one argument away, and
client.download_event()/get_event() take chunk_size for a link where large
responses are not surviving.

Full suite: 472 passed, 16 pre-existing failures unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ru8Lg9HkkYvX9VWWo65SmL
This commit is contained in:
serversdown
2026-10-02 14:14:02 -04:00
co-authored by Claude Opus 5
parent 248ddc9cda
commit 423608ccd1
5 changed files with 194 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_micromate_events.py
+5 -3
View File
@@ -293,7 +293,9 @@ def test_replaying_thors_session_reproduces_every_byte_and_decodes_every_event()
decoded, checked = 0, 0
for ref in mm.iter_events():
result = mm.get_event(ref) # verify=True by default
# Pin THOR's chunk size: this test asserts byte-identity with
# THOR's frames, and our default is 16x larger.
result = mm.get_event(ref, chunk_size=1024)
decoded += 1
assert ref.serial == "UM12947"
assert ref.filename and ref.filename.endswith(ref.suffix)
@@ -334,10 +336,10 @@ def test_a_corrupted_stored_peak_is_caught_by_the_self_check():
for ref in mm.iter_events():
if ref.is_histogram:
mm.download_event(ref) # keep the replay in step
mm.download_event(ref, chunk_size=1024) # keep the replay in step
continue
ref.peak_vector_sum_ips *= 1.5 # as a bad decode would look
with pytest.raises(DecodeMismatch, match="decode is suspect"):
mm.get_event(ref)
mm.get_event(ref, chunk_size=1024)
return
pytest.fail("no waveform event found in the capture")
tests/test_micromate_protocol.py
+87 -14
View File
@@ -174,12 +174,13 @@ def test_download_reproduces_thors_chunk_sequence_byte_for_byte():
payload = payload[:SIZE_4A81]
responses = []
for i in range(4):
want = min(P.CHUNK_SIZE, SIZE_4A81 - i * P.CHUNK_SIZE)
body = payload[i * P.CHUNK_SIZE: i * P.CHUNK_SIZE + want]
want = min(P.THOR_CHUNK_SIZE, SIZE_4A81 - i * P.THOR_CHUNK_SIZE)
body = payload[i * P.THOR_CHUNK_SIZE: i * P.THOR_CHUNK_SIZE + want]
responses.append(frame(0xA5, bytes(11) + body, page=want // 256))
p, t = proto(responses)
got = p.read_event_file(bytes.fromhex("055d4a81"), SIZE_4A81)
got = p.read_event_file(bytes.fromhex("055d4a81"), SIZE_4A81,
chunk_size=P.THOR_CHUNK_SIZE)
assert t.written == REQ_CHUNKS_4A81
assert got == payload
@@ -198,11 +199,12 @@ def test_chunk_count_and_final_offset(size, n_chunks, last_offset):
"""The first six rows are the six bench events, with THOR's real offsets."""
responses = []
for i in range(n_chunks):
want = min(P.CHUNK_SIZE, size - i * P.CHUNK_SIZE)
want = min(P.THOR_CHUNK_SIZE, size - i * P.THOR_CHUNK_SIZE)
responses.append(frame(0xA5, bytes(11) + bytes(want)))
p, t = proto(responses)
p.read_event_file(bytes.fromhex("055d4a81"), size)
p.read_event_file(bytes.fromhex("055d4a81"), size,
chunk_size=P.THOR_CHUNK_SIZE)
assert len(t.written) == n_chunks
# offset is payload[4:5] of the request; recover it from the built frame
@@ -214,17 +216,29 @@ def test_chunk_count_and_final_offset(size, n_chunks, last_offset):
)
def test_a_short_chunk_raises_rather_than_truncating():
"""A silently short event is the failure mode this codebase keeps hitting."""
p, _ = proto([frame(0xA5, bytes(11) + bytes(900))]) # asked for 1024
with pytest.raises(ShortRead, match="asked for 1024 B, got 900"):
p.read_event_file(bytes.fromhex("055d4a81"), 1024)
def test_a_short_chunk_is_absorbed_and_the_remainder_refetched():
"""⚠ Changed 2026-10-02: a short chunk is no longer an error.
It used to raise, on the principle that a silently short event is the failure
mode this codebase keeps hitting. But the device *legitimately* returns
short — it clamps an over-large request to 16,384 B without saying so — and
the real protection is tracking the offset by bytes received, which makes a
short response cost one extra request instead of corrupting the file. The
total length is still checked, so a genuinely truncated event still raises.
"""
p, t = proto([frame(0xA5, bytes(11) + b"\xaa" * 900),
frame(0xA5, bytes(11) + b"\xbb" * 124)])
got = p.read_event_file(bytes.fromhex("055d4a81"), 1024,
chunk_size=P.THOR_CHUNK_SIZE)
assert got == b"\xaa" * 900 + b"\xbb" * 124
assert len(t.written) == 2, "the 124 B remainder was refetched"
def test_a_chunk_too_short_to_hold_its_header_raises():
p, _ = proto([frame(0xA5, bytes(4))])
with pytest.raises(ShortRead, match="too short to hold a chunk header"):
p.read_event_file(bytes.fromhex("055d4a81"), 1024)
p.read_event_file(bytes.fromhex("055d4a81"), 1024,
chunk_size=P.THOR_CHUNK_SIZE)
def test_download_rejects_a_nonsense_size():
@@ -391,7 +405,8 @@ def test_every_captured_download_frame_is_one_we_would_have_sent():
for e in events:
p, t = proto([bytes([STX]) + stuff(r + bytes([checksum(r)])) + bytes([ETX])
for r in e["rsps"]])
got = p.read_event_file(e["key"], e["size"])
got = p.read_event_file(e["key"], e["size"],
chunk_size=P.THOR_CHUNK_SIZE)
assert t.written == e["reqs"], (
f"event {e['key'].hex()}: our {len(t.written)} frames differ from "
f"THOR's {len(e['reqs'])}"
@@ -452,14 +467,72 @@ def test_a_72kb_event_downloads_in_71_chunks():
n = 71
responses = []
for i in range(n):
want = min(P.CHUNK_SIZE, size - i * P.CHUNK_SIZE)
want = min(P.THOR_CHUNK_SIZE, size - i * P.THOR_CHUNK_SIZE)
responses.append(frame(0xA5, bytes(11) + bytes(want)))
p, t = proto(responses)
got = p.read_event_file(bytes.fromhex("055d4a83"), size)
got = p.read_event_file(bytes.fromhex("055d4a83"), size,
chunk_size=P.THOR_CHUNK_SIZE)
assert len(got) == size
assert len(t.written) == n
# Chunk 64 is the first past the old cap; its params must carry the 0x01.
wire = t.written[64]
assert bytes.fromhex("000100") in wire, "the carry into params[1] is on the wire"
# ── The 16 KB ceiling, and the silent clamp ───────────────────────────────────
def test_default_chunk_size_is_the_measured_ceiling_not_thors():
assert P.CHUNK_SIZE == 16384
assert P.THOR_CHUNK_SIZE == 1024
def test_a_silently_clamped_response_is_absorbed_not_failed():
"""⚠ Ask for more than the device serves and it CLAMPS — silently.
Measured on UM20147: a 32,768 B request returns exactly 16,384 B of correct
data, no error. A loop striding by a fixed chunk size would either fail on
that or skip the bytes it never collected. Driving by bytes received makes
it a non-event: one more request.
Here a 20,000 B event is fetched with chunk_size=16384 against a device
pretending to clamp at 8192, so the walk must take 3 requests at
offsets 0 / 8192 / 16384.
"""
size, clamp = 20000, 8192
payload = bytes(range(256)) * 100
payload = payload[:size]
served, responses = 0, []
while served < size:
n = min(clamp, size - served)
responses.append(frame(0xA5, bytes(11) + payload[served:served + n]))
served += n
p, t = proto(responses)
got = p.read_event_file(bytes.fromhex("055d4a83"), size, chunk_size=16384)
assert got == payload
assert len(t.written) == 3, "two clamped requests plus the remainder"
def test_a_chunk_that_returns_nothing_raises_rather_than_spinning():
"""A byte-driven loop must not loop forever on zero progress."""
p, _ = proto([frame(0xA5, bytes(11))] * 4)
with pytest.raises(ShortRead, match="got none"):
p.read_event_file(bytes.fromhex("055d4a83"), 5000)
def test_the_72kb_event_now_takes_5_requests_not_71():
size = 72560
served, responses = 0, []
while served < size:
n = min(P.CHUNK_SIZE, size - served)
responses.append(frame(0xA5, bytes(11) + bytes(n)))
served += n
p, t = proto(responses)
got = p.read_event_file(bytes.fromhex("055d4a83"), size)
assert len(got) == size
assert len(t.written) == 5, "14x fewer round trips than THOR's 71"