feat(forward): SFM event forwarder (v1.5.0)

When SFM_FORWARD_ENABLED=true and SFM_URL is set, every Blastware
event binary in the ACH watch folder is forwarded to an SFM server's
/db/import/blastware_file endpoint as a multipart POST.  The paired
<binary>.TXT ASCII report (which Blastware's ACH writes alongside
each event) is shipped in the same request, letting the SFM server
index the full per-channel stats — PPV, ZC Freq, Time of Peak, Peak
Acceleration / Displacement, Peak Vector Sum + time, sensor
self-check Pass/Fail per channel, and monitor-log timestamps —
without depending on the still-undecoded BW waveform body codec.

New module event_forwarder.py:
  - is_event_binary() filename matcher (BW's <P><serial3><stem>.<ext>
    scheme; rejects .MLG, .TXT, .log, .ini, .h5, etc.)
  - ForwardState (.json file keyed by sha256 — idempotent across
    restarts and auto-updates)
  - find_pending_events() with quiescence + grace-period guards
  - Hand-rolled multipart encoder (stdlib-only)
  - forward_event_pair() / forward_pending() — POST loop with
    structured per-event outcomes

Wired into series3_watcher.run_watcher() on its own cadence
(SFM_FORWARD_INTERVAL_SECONDS, default 60s) so it doesn't slow the
existing 5-min heartbeat scan.

Default-off: existing 1.4.x deployments keep their old behaviour
after auto-updating until an operator sets SFM_URL +
SFM_FORWARD_ENABLED=true and restarts.

17 unit tests in test_event_forwarder.py cover filename matching,
state idempotency, scan logic (quiescence, grace, max age,
already-forwarded, .TXT pairing), multipart byte shape, and an
end-to-end POST against a tiny stdlib http.server fake.

Bumps version 1.4.4 → 1.5.0 (minor — additive feature, no API break).
Requires SFM server v0.16+ for the paired-.TXT import endpoint.
This commit is contained in:
2026-05-09 00:03:31 +00:00
parent 010016d515
commit f4ec6ef945
8 changed files with 1052 additions and 5 deletions
+100 -1
View File
@@ -80,6 +80,30 @@ def load_config(path: str) -> Dict[str, Any]:
# Auto-updater source
"UPDATE_SOURCE": get_str("UPDATE_SOURCE", "gitea"),
"UPDATE_URL": get_str("UPDATE_URL", ""),
# SFM event forwarder — when enabled, forwards each Blastware
# event binary (+ paired .TXT report when present) to an SFM
# server's /db/import/blastware_file endpoint. Default-off so
# existing 1.4.x deployments don't change behaviour on
# auto-update; operators flip it on by setting SFM_URL +
# SFM_FORWARD_ENABLED=true in config.ini.
"SFM_FORWARD_ENABLED": get_bool("SFM_FORWARD_ENABLED", False),
"SFM_URL": get_str("SFM_URL", ""),
"SFM_FORWARD_INTERVAL_SECONDS": get_int("SFM_FORWARD_INTERVAL_SECONDS", 60),
# Files modified within the last N seconds are skipped (BW may
# still be writing them).
"SFM_QUIESCENCE_SECONDS": get_int("SFM_QUIESCENCE_SECONDS", 5),
# If a binary's .TXT report hasn't appeared after this many
# seconds, forward the binary alone rather than blocking
# forever.
"SFM_MISSING_REPORT_GRACE_SECONDS": get_int(
"SFM_MISSING_REPORT_GRACE_SECONDS", 60
),
# Per-request HTTP timeout (seconds).
"SFM_HTTP_TIMEOUT": get_int("SFM_HTTP_TIMEOUT", 60),
# State file for forwarded-sha256 idempotency tracking.
# Defaults next to the log file for easy operator access.
"SFM_STATE_FILE": get_str("SFM_STATE_FILE", ""),
}
@@ -217,7 +241,7 @@ def scan_latest(
# --- API heartbeat / SFM telemetry helpers ---
VERSION = "1.4.4"
VERSION = "1.5.0"
def _read_log_tail(log_file: str, n: int = 25) -> Optional[list]:
@@ -366,6 +390,36 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
sniff_cache: Dict[str, Tuple[float, str]] = {}
last_api_ts: float = 0.0
last_forward_ts: float = 0.0
# ---- SFM event-forwarder setup ----
# Default-off; only initialised when both flag and URL are set.
sfm_state = None
if cfg.get("SFM_FORWARD_ENABLED") and cfg.get("SFM_URL"):
try:
from event_forwarder import ForwardState
state_file = cfg.get("SFM_STATE_FILE") or os.path.join(
os.path.dirname(LOG_FILE) or here, "sfm_forwarded.json"
)
sfm_state = ForwardState(state_file)
print(
"[CFG] SFM_FORWARD_ENABLED=true SFM_URL={} state={} ({} already-forwarded)".format(
cfg.get("SFM_URL"), state_file, sfm_state.count(),
)
)
log_message(
LOG_FILE, ENABLE_LOGGING,
"[cfg] sfm forwarder enabled url={} state={} already_forwarded={}".format(
cfg.get("SFM_URL"), state_file, sfm_state.count(),
),
)
except Exception as e:
print("[WARN] SFM forwarder init failed: {}".format(e))
log_message(LOG_FILE, ENABLE_LOGGING,
"[warn] sfm forwarder init failed: {}".format(e))
sfm_state = None
else:
print("[CFG] SFM_FORWARD_ENABLED=false (event forwarding disabled)")
while not stop_event.is_set():
try:
@@ -447,6 +501,51 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
else:
state["api_status"] = "disabled"
# ---- SFM event forwarder ----
# Same scan loop as the heartbeat, but on its own cadence
# (SFM_FORWARD_INTERVAL_SECONDS). Default-off — sfm_state
# is None unless config explicitly enabled it AND supplied
# an SFM_URL.
if sfm_state is not None:
now_ts = time.time()
fwd_interval = int(cfg.get("SFM_FORWARD_INTERVAL_SECONDS", 60))
if now_ts - last_forward_ts >= fwd_interval:
try:
from event_forwarder import forward_pending
counts = forward_pending(
WATCH_PATH,
cfg.get("SFM_URL", ""),
sfm_state,
max_age_days=MAX_EVENT_AGE_DAYS,
quiescence_seconds=int(cfg.get("SFM_QUIESCENCE_SECONDS", 5)),
missing_report_grace_seconds=int(
cfg.get("SFM_MISSING_REPORT_GRACE_SECONDS", 60)
),
timeout=int(cfg.get("SFM_HTTP_TIMEOUT", 60)),
logger=lambda m: log_message(LOG_FILE, ENABLE_LOGGING, m),
)
last_forward_ts = now_ts
if counts["scanned"] > 0:
summary = (
"[forward] scanned={} forwarded={} "
"with_report={} errors={}".format(
counts["scanned"], counts["forwarded"],
counts["with_report"], counts["errors"],
)
)
print(summary)
log_message(LOG_FILE, ENABLE_LOGGING, summary)
state["sfm_status"] = "ok" if counts["errors"] == 0 else "errors"
state["last_forward"] = datetime.now()
state["last_forward_counts"] = counts
except Exception as e:
err = "[forward-error] {}".format(e)
print(err)
log_message(LOG_FILE, ENABLE_LOGGING, err)
state["sfm_status"] = "fail"
else:
state["sfm_status"] = "disabled"
except Exception as e:
err = "[loop-error] {}".format(e)
print(err)