feat(forward): SFM event forwarder (v1.5.0)
When SFM_FORWARD_ENABLED=true and SFM_URL is set, every Blastware
event binary in the ACH watch folder is forwarded to an SFM server's
/db/import/blastware_file endpoint as a multipart POST. The paired
<binary>.TXT ASCII report (which Blastware's ACH writes alongside
each event) is shipped in the same request, letting the SFM server
index the full per-channel stats — PPV, ZC Freq, Time of Peak, Peak
Acceleration / Displacement, Peak Vector Sum + time, sensor
self-check Pass/Fail per channel, and monitor-log timestamps —
without depending on the still-undecoded BW waveform body codec.
New module event_forwarder.py:
- is_event_binary() filename matcher (BW's <P><serial3><stem>.<ext>
scheme; rejects .MLG, .TXT, .log, .ini, .h5, etc.)
- ForwardState (.json file keyed by sha256 — idempotent across
restarts and auto-updates)
- find_pending_events() with quiescence + grace-period guards
- Hand-rolled multipart encoder (stdlib-only)
- forward_event_pair() / forward_pending() — POST loop with
structured per-event outcomes
Wired into series3_watcher.run_watcher() on its own cadence
(SFM_FORWARD_INTERVAL_SECONDS, default 60s) so it doesn't slow the
existing 5-min heartbeat scan.
Default-off: existing 1.4.x deployments keep their old behaviour
after auto-updating until an operator sets SFM_URL +
SFM_FORWARD_ENABLED=true and restarts.
17 unit tests in test_event_forwarder.py cover filename matching,
state idempotency, scan logic (quiescence, grace, max age,
already-forwarded, .TXT pairing), multipart byte shape, and an
end-to-end POST against a tiny stdlib http.server fake.
Bumps version 1.4.4 → 1.5.0 (minor — additive feature, no API break).
Requires SFM server v0.16+ for the paired-.TXT import endpoint.
This commit is contained in:
+100
-1
@@ -80,6 +80,30 @@ def load_config(path: str) -> Dict[str, Any]:
|
||||
# Auto-updater source
|
||||
"UPDATE_SOURCE": get_str("UPDATE_SOURCE", "gitea"),
|
||||
"UPDATE_URL": get_str("UPDATE_URL", ""),
|
||||
|
||||
# SFM event forwarder — when enabled, forwards each Blastware
|
||||
# event binary (+ paired .TXT report when present) to an SFM
|
||||
# server's /db/import/blastware_file endpoint. Default-off so
|
||||
# existing 1.4.x deployments don't change behaviour on
|
||||
# auto-update; operators flip it on by setting SFM_URL +
|
||||
# SFM_FORWARD_ENABLED=true in config.ini.
|
||||
"SFM_FORWARD_ENABLED": get_bool("SFM_FORWARD_ENABLED", False),
|
||||
"SFM_URL": get_str("SFM_URL", ""),
|
||||
"SFM_FORWARD_INTERVAL_SECONDS": get_int("SFM_FORWARD_INTERVAL_SECONDS", 60),
|
||||
# Files modified within the last N seconds are skipped (BW may
|
||||
# still be writing them).
|
||||
"SFM_QUIESCENCE_SECONDS": get_int("SFM_QUIESCENCE_SECONDS", 5),
|
||||
# If a binary's .TXT report hasn't appeared after this many
|
||||
# seconds, forward the binary alone rather than blocking
|
||||
# forever.
|
||||
"SFM_MISSING_REPORT_GRACE_SECONDS": get_int(
|
||||
"SFM_MISSING_REPORT_GRACE_SECONDS", 60
|
||||
),
|
||||
# Per-request HTTP timeout (seconds).
|
||||
"SFM_HTTP_TIMEOUT": get_int("SFM_HTTP_TIMEOUT", 60),
|
||||
# State file for forwarded-sha256 idempotency tracking.
|
||||
# Defaults next to the log file for easy operator access.
|
||||
"SFM_STATE_FILE": get_str("SFM_STATE_FILE", ""),
|
||||
}
|
||||
|
||||
|
||||
@@ -217,7 +241,7 @@ def scan_latest(
|
||||
|
||||
|
||||
# --- API heartbeat / SFM telemetry helpers ---
|
||||
VERSION = "1.4.4"
|
||||
VERSION = "1.5.0"
|
||||
|
||||
|
||||
def _read_log_tail(log_file: str, n: int = 25) -> Optional[list]:
|
||||
@@ -366,6 +390,36 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
|
||||
sniff_cache: Dict[str, Tuple[float, str]] = {}
|
||||
last_api_ts: float = 0.0
|
||||
last_forward_ts: float = 0.0
|
||||
|
||||
# ---- SFM event-forwarder setup ----
|
||||
# Default-off; only initialised when both flag and URL are set.
|
||||
sfm_state = None
|
||||
if cfg.get("SFM_FORWARD_ENABLED") and cfg.get("SFM_URL"):
|
||||
try:
|
||||
from event_forwarder import ForwardState
|
||||
state_file = cfg.get("SFM_STATE_FILE") or os.path.join(
|
||||
os.path.dirname(LOG_FILE) or here, "sfm_forwarded.json"
|
||||
)
|
||||
sfm_state = ForwardState(state_file)
|
||||
print(
|
||||
"[CFG] SFM_FORWARD_ENABLED=true SFM_URL={} state={} ({} already-forwarded)".format(
|
||||
cfg.get("SFM_URL"), state_file, sfm_state.count(),
|
||||
)
|
||||
)
|
||||
log_message(
|
||||
LOG_FILE, ENABLE_LOGGING,
|
||||
"[cfg] sfm forwarder enabled url={} state={} already_forwarded={}".format(
|
||||
cfg.get("SFM_URL"), state_file, sfm_state.count(),
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
print("[WARN] SFM forwarder init failed: {}".format(e))
|
||||
log_message(LOG_FILE, ENABLE_LOGGING,
|
||||
"[warn] sfm forwarder init failed: {}".format(e))
|
||||
sfm_state = None
|
||||
else:
|
||||
print("[CFG] SFM_FORWARD_ENABLED=false (event forwarding disabled)")
|
||||
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
@@ -447,6 +501,51 @@ def run_watcher(state: Dict[str, Any], stop_event: threading.Event) -> None:
|
||||
else:
|
||||
state["api_status"] = "disabled"
|
||||
|
||||
# ---- SFM event forwarder ----
|
||||
# Same scan loop as the heartbeat, but on its own cadence
|
||||
# (SFM_FORWARD_INTERVAL_SECONDS). Default-off — sfm_state
|
||||
# is None unless config explicitly enabled it AND supplied
|
||||
# an SFM_URL.
|
||||
if sfm_state is not None:
|
||||
now_ts = time.time()
|
||||
fwd_interval = int(cfg.get("SFM_FORWARD_INTERVAL_SECONDS", 60))
|
||||
if now_ts - last_forward_ts >= fwd_interval:
|
||||
try:
|
||||
from event_forwarder import forward_pending
|
||||
counts = forward_pending(
|
||||
WATCH_PATH,
|
||||
cfg.get("SFM_URL", ""),
|
||||
sfm_state,
|
||||
max_age_days=MAX_EVENT_AGE_DAYS,
|
||||
quiescence_seconds=int(cfg.get("SFM_QUIESCENCE_SECONDS", 5)),
|
||||
missing_report_grace_seconds=int(
|
||||
cfg.get("SFM_MISSING_REPORT_GRACE_SECONDS", 60)
|
||||
),
|
||||
timeout=int(cfg.get("SFM_HTTP_TIMEOUT", 60)),
|
||||
logger=lambda m: log_message(LOG_FILE, ENABLE_LOGGING, m),
|
||||
)
|
||||
last_forward_ts = now_ts
|
||||
if counts["scanned"] > 0:
|
||||
summary = (
|
||||
"[forward] scanned={} forwarded={} "
|
||||
"with_report={} errors={}".format(
|
||||
counts["scanned"], counts["forwarded"],
|
||||
counts["with_report"], counts["errors"],
|
||||
)
|
||||
)
|
||||
print(summary)
|
||||
log_message(LOG_FILE, ENABLE_LOGGING, summary)
|
||||
state["sfm_status"] = "ok" if counts["errors"] == 0 else "errors"
|
||||
state["last_forward"] = datetime.now()
|
||||
state["last_forward_counts"] = counts
|
||||
except Exception as e:
|
||||
err = "[forward-error] {}".format(e)
|
||||
print(err)
|
||||
log_message(LOG_FILE, ENABLE_LOGGING, err)
|
||||
state["sfm_status"] = "fail"
|
||||
else:
|
||||
state["sfm_status"] = "disabled"
|
||||
|
||||
except Exception as e:
|
||||
err = "[loop-error] {}".format(e)
|
||||
print(err)
|
||||
|
||||
Reference in New Issue
Block a user