From 2ba20c7809420928201369b33f7cebd097f5a94c Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Mon, 13 Apr 2026 17:14:36 -0400 Subject: [PATCH 01/26] feat(sfm): add SFM proxy router and event data page - backend/routers/sfm.py: HTTP proxy to SFM backend (localhost:8200), mirrors the SLMM proxy pattern. SFM_BASE_URL env var for docker-compose. Catch-all /{path} forwards to SFM root (no /api/ prefix). 60s timeout. - templates/sfm.html: full SFM dashboard with 5 tabs: Events (DB listing, filters by serial/date/false-trigger, flag/unflag FT), Units (known serials + stats, filter events by unit), Monitor Log (continuous monitoring intervals), ACH Sessions (call-home history), Live Device (TCP connect, device info cards, start/stop monitoring, push project config, download events from device, operation log). - backend/main.py: import sfm router, include router, add GET /sfm route - templates/base.html: add SFM Live Data nav link under Seismographs Co-Authored-By: Claude Sonnet 4.6 --- backend/main.py | 9 +- backend/routers/sfm.py | 130 +++++ templates/base.html | 6 + templates/sfm.html | 1117 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1261 insertions(+), 1 deletion(-) create mode 100644 backend/routers/sfm.py create mode 100644 templates/sfm.html diff --git a/backend/main.py b/backend/main.py index 89cca81..81dfac9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -18,7 +18,7 @@ logging.basicConfig( logger = logging.getLogger(__name__) from backend.database import engine, Base, get_db -from backend.routers import roster, units, photos, roster_edit, roster_rename, dashboard, dashboard_tabs, activity, slmm, slm_ui, slm_dashboard, seismo_dashboard, projects, project_locations, scheduler, modem_dashboard +from backend.routers import roster, units, photos, roster_edit, roster_rename, dashboard, dashboard_tabs, activity, slmm, slm_ui, slm_dashboard, seismo_dashboard, sfm, projects, project_locations, scheduler, modem_dashboard from backend.services.snapshot import emit_status_snapshot from backend.models import IgnoredUnit from backend.utils.timezone import get_user_timezone @@ -97,6 +97,7 @@ app.include_router(slmm.router) app.include_router(slm_ui.router) app.include_router(slm_dashboard.router) app.include_router(seismo_dashboard.router) +app.include_router(sfm.router) app.include_router(modem_dashboard.router) from backend.routers import settings @@ -233,6 +234,12 @@ async def seismographs_page(request: Request): return templates.TemplateResponse("seismographs.html", {"request": request}) +@app.get("/sfm", response_class=HTMLResponse) +async def sfm_page(request: Request): + """SFM live event data and device control dashboard""" + return templates.TemplateResponse("sfm.html", {"request": request}) + + @app.get("/modems", response_class=HTMLResponse) async def modems_page(request: Request): """Field modems management dashboard""" diff --git a/backend/routers/sfm.py b/backend/routers/sfm.py new file mode 100644 index 0000000..5126284 --- /dev/null +++ b/backend/routers/sfm.py @@ -0,0 +1,130 @@ +""" +SFM (Seismograph Field Module) Proxy Router + +Proxies requests from terra-view to the standalone SFM backend service. +SFM runs on port 8200 and handles MiniMate Plus seismograph communication +and event database queries. + +SFM endpoints are at root level (e.g. /db/units, /device/info) — no /api/ prefix. +""" + +from fastapi import APIRouter, HTTPException, Request, Response +import httpx +import logging +import os + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/sfm", tags=["sfm"]) + +# SFM backend URL - configurable via environment variable +SFM_BASE_URL = os.getenv("SFM_BASE_URL", "http://localhost:8200") + + +@router.get("/health") +async def check_sfm_health(): + """ + Check if the SFM backend service is reachable and healthy. + """ + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.get(f"{SFM_BASE_URL}/health") + + if response.status_code == 200: + data = response.json() + return { + "status": "ok", + "sfm_status": "connected", + "sfm_url": SFM_BASE_URL, + "sfm_response": data + } + else: + return { + "status": "degraded", + "sfm_status": "error", + "sfm_url": SFM_BASE_URL, + "detail": f"SFM returned status {response.status_code}" + } + + except httpx.ConnectError: + return { + "status": "error", + "sfm_status": "unreachable", + "sfm_url": SFM_BASE_URL, + "detail": "Cannot connect to SFM backend. Is it running?" + } + except Exception as e: + return { + "status": "error", + "sfm_status": "error", + "sfm_url": SFM_BASE_URL, + "detail": str(e) + } + + +# HTTP catch-all — proxies everything else to SFM backend +@router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) +async def proxy_to_sfm(path: str, request: Request): + """ + Proxy all requests to the SFM backend service. + + SFM endpoints have no /api/ prefix — target URL is {SFM_BASE_URL}/{path}. + Timeout is 60s to allow for live device round-trips (event downloads can + take 30-45s for a full event list). + """ + # Build target URL — SFM endpoints live at root, not /api/ + target_url = f"{SFM_BASE_URL}/{path}" + + # Forward query params + query_params = dict(request.query_params) + + # Read body for mutation requests + body = None + if request.method in ["POST", "PUT", "PATCH"]: + try: + body = await request.body() + except Exception as e: + logger.error(f"Failed to read request body: {e}") + body = None + + # Strip hop-by-hop headers + headers = dict(request.headers) + headers_to_exclude = ["host", "content-length", "transfer-encoding", "connection"] + proxy_headers = {k: v for k, v in headers.items() if k.lower() not in headers_to_exclude} + + logger.info(f"Proxying {request.method} {path} → SFM: {target_url}") + + try: + async with httpx.AsyncClient(timeout=60.0) as client: + response = await client.request( + method=request.method, + url=target_url, + params=query_params, + headers=proxy_headers, + content=body + ) + return Response( + content=response.content, + status_code=response.status_code, + headers=dict(response.headers), + media_type=response.headers.get("content-type") + ) + + except httpx.ConnectError: + logger.error(f"Failed to connect to SFM backend at {SFM_BASE_URL}") + raise HTTPException( + status_code=503, + detail=f"SFM backend service unavailable. Is SFM running on {SFM_BASE_URL}?" + ) + except httpx.TimeoutException: + logger.error(f"Timeout connecting to SFM backend at {SFM_BASE_URL}") + raise HTTPException( + status_code=504, + detail="SFM backend timeout" + ) + except Exception as e: + logger.error(f"Error proxying to SFM: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to proxy request to SFM: {str(e)}" + ) diff --git a/templates/base.html b/templates/base.html index 36bd6c0..5cf5e81 100644 --- a/templates/base.html +++ b/templates/base.html @@ -122,6 +122,12 @@ Seismographs + + + + + SFM Live Data + diff --git a/templates/sfm.html b/templates/sfm.html new file mode 100644 index 0000000..e55e0c3 --- /dev/null +++ b/templates/sfm.html @@ -0,0 +1,1117 @@ +{% extends "base.html" %} + +{% block title %}SFM Event Data - Seismo Fleet Manager{% endblock %} + +{% block content %} +
+
+

SFM Event Data

+

Live device control and ACH event database

+
+
+ + Checking SFM… + + +
+
+ + +
+
+ Known Units + — +
+
+ Total Events + — +
+
+ Monitor Intervals + — +
+
+ ACH Sessions + — +
+
+ + +
+
+ +
+ + +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + +
+ + +
+
+
+ Loading events… +
+
+
+ + + + + + + + + + + + + +
+ + + + + + + +{% endblock %} -- 2.52.0 From ec661ee079e78d23a93bc43182ba5210a0e32b17 Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 11 May 2026 19:36:38 +0000 Subject: [PATCH 02/26] refactor(sfm): drop ACH/monitor/live-device UI; scope SFM tab to watcher-forwarded events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /sfm page was originally designed around a Python ACH-server replacement that would land call-home sessions, monitor-log intervals, and live-device control alongside triggered events. That work is paused — deployment uses Blastware's official ACH server and series3- watcher forwards events to SFM's /db/import/blastware_file. The sessions/monitor-log/live-device surfaces have no path to populate under this architecture and were rendering 0/0 everywhere. Removed (UI only — SFM backend untouched): - KPI tiles "Monitor Intervals" + "ACH Sessions" (always 0 under watcher-forward pipeline) - Tabs Monitor Log / ACH Sessions / Live Device + their loaders - Units card columns total_monitor_entries + total_sessions - Orphaned helpers fmtDuration / fmtBytes - Live-device state vars + status poll timer - Subtitle and empty-state copy updated to match reality - Sidebar: "SFM Live Data" -> "SFM Events" SFM-side code (ach_sessions/monitor_log tables, /db/sessions, /db/monitor_log, /device/* endpoints, protocol RE library) is preserved intact — re-surfacing the tabs later is a UI-only revert. backend/routers/sfm.py catch-all proxy unchanged. Co-Authored-By: Claude Opus 4.7 --- templates/base.html | 2 +- templates/sfm.html | 657 +------------------------------------------- 2 files changed, 8 insertions(+), 651 deletions(-) diff --git a/templates/base.html b/templates/base.html index 5cf5e81..4385b0e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -126,7 +126,7 @@ - SFM Live Data + SFM Events diff --git a/templates/sfm.html b/templates/sfm.html index e55e0c3..770eacb 100644 --- a/templates/sfm.html +++ b/templates/sfm.html @@ -6,7 +6,7 @@

SFM Event Data

-

Live device control and ACH event database

+

Blastware ACH events forwarded by series3-watcher

@@ -19,7 +19,7 @@
-
+
Known Units — @@ -28,14 +28,6 @@ Total Events —
-
- Monitor Intervals - — -
-
- ACH Sessions - — -
@@ -50,18 +42,6 @@ class="sfm-tab shrink-0 px-6 py-4 text-sm font-medium border-b-2 transition-colors"> Units - - -
@@ -133,259 +113,6 @@
- - - - - - - - - @@ -439,10 +166,6 @@ +#} + diff --git a/templates/sfm.html b/templates/sfm.html index 770eacb..68d4558 100644 --- a/templates/sfm.html +++ b/templates/sfm.html @@ -115,21 +115,9 @@ - - +{# Shared event-detail modal — rendered by /static/event-modal.js #} +{% include 'partials/event_detail_modal.html' %} + @@ -654,7 +675,8 @@ function toggleCard(cardName) { // Restore card states from localStorage on page load function restoreCardStates() { const cardStates = JSON.parse(localStorage.getItem('dashboardCardStates') || '{}'); - const cardNames = ['fleet-summary', 'recent-alerts', 'recent-callins', 'todays-actions', 'fleet-map', 'fleet-status']; + // Note: todays-actions has its own collapse handling (see toggleTodaysSchedule / onTodaysActionsSwap) + const cardNames = ['fleet-summary', 'recent-alerts', 'recent-callins', 'fleet-map', 'fleet-status']; cardNames.forEach(cardName => { const content = document.getElementById(`${cardName}-content`); @@ -839,89 +861,139 @@ async function loadRecentPhotos() { loadRecentPhotos(); setInterval(loadRecentPhotos, 30000); -// Load and display recent call-ins -let showingAllCallins = false; -const DEFAULT_CALLINS_DISPLAY = 5; - +// Load and display recent call-ins. +// Source: SFM events (forwarded by series3-watcher from Blastware ACH). +// Each event = one call-home. Heartbeat-derived endpoint /api/recent-callins +// is being phased out but kept as a backup. async function loadRecentCallins() { + const callinsList = document.getElementById('recent-callins-list'); try { - const response = await fetch('/api/recent-callins?hours=6'); + const response = await fetch('/api/recent-event-callins?limit=10'); if (!response.ok) { throw new Error('Failed to load recent call-ins'); } const data = await response.json(); - const callinsList = document.getElementById('recent-callins-list'); - const showAllButton = document.getElementById('show-all-callins'); - if (data.call_ins && data.call_ins.length > 0) { - // Determine how many to show - const displayCount = showingAllCallins ? data.call_ins.length : Math.min(DEFAULT_CALLINS_DISPLAY, data.call_ins.length); - const callinsToDisplay = data.call_ins.slice(0, displayCount); - - // Build HTML for call-ins list - let html = ''; - callinsToDisplay.forEach(callin => { - // Status color - const statusColor = callin.status === 'OK' ? 'green' : callin.status === 'Pending' ? 'yellow' : 'red'; - const statusClass = callin.status === 'OK' ? 'bg-green-500' : callin.status === 'Pending' ? 'bg-yellow-500' : 'bg-red-500'; - - // Build location/note line - let subtitle = ''; - if (callin.location) { - subtitle = callin.location; - } else if (callin.note) { - subtitle = callin.note; - } - - html += ` -
-
- -
- - ${callin.unit_id} - - ${subtitle ? `

${subtitle}

` : ''} -
-
- ${callin.time_ago} -
`; - }); - - callinsList.innerHTML = html; - - // Show/hide the "Show all" button - if (data.call_ins.length > DEFAULT_CALLINS_DISPLAY) { - showAllButton.classList.remove('hidden'); - showAllButton.textContent = showingAllCallins - ? `Show fewer (${DEFAULT_CALLINS_DISPLAY})` - : `Show all (${data.call_ins.length})`; - } else { - showAllButton.classList.add('hidden'); - } - } else { - callinsList.innerHTML = '

No units have called in within the past 6 hours

'; - showAllButton.classList.add('hidden'); + if (!data.call_ins || data.call_ins.length === 0) { + callinsList.innerHTML = '

No recent event call-ins from SFM

'; + return; } + + // Two-column dense grid on lg+, single column below. + let html = '
'; + data.call_ins.forEach(c => { + const isFalse = c.false_trigger; + const pvs = c.peak_vector_sum; + const pvsStr = (pvs !== null && pvs !== undefined) + ? Number(pvs).toFixed(3) + ' in/s' + : '—'; + + // Subtitle: prefer sensor_location, fallback to project. + const subtitle = c.sensor_location || c.project || ''; + + // Status dot: amber for false trigger, green for real event, + // gray if unit not in roster. + const dotClass = !c.in_roster + ? 'bg-gray-400' + : (isFalse ? 'bg-amber-400' : 'bg-green-500'); + + // Format event timestamp short (e.g. "05-13 05:00"). + let tsShort = ''; + if (c.event_timestamp) { + const ts = c.event_timestamp.replace('T', ' '); + // "2026-05-13 05:00:13" → "05-13 05:00" + tsShort = ts.length >= 16 ? ts.slice(5, 16) : ts; + } + + const unitLink = c.in_roster + ? `${c.unit_id}` + : `${c.unit_id}`; + + html += ` +
+
+ +
+
+ ${unitLink} + ${isFalse ? 'false' : ''} + ${pvsStr} +
+ ${subtitle ? `

${subtitle}

` : ''} +
+
+
+ ${c.time_ago} + ${tsShort ? `${tsShort}` : ''} +
+
`; + }); + html += '
'; + callinsList.innerHTML = html; } catch (error) { console.error('Error loading recent call-ins:', error); - document.getElementById('recent-callins-list').innerHTML = '

Failed to load recent call-ins

'; + callinsList.innerHTML = '

Failed to load recent call-ins

'; } } -// Toggle show all/show fewer -document.addEventListener('DOMContentLoaded', function() { - const showAllButton = document.getElementById('show-all-callins'); - showAllButton.addEventListener('click', function() { - showingAllCallins = !showingAllCallins; - loadRecentCallins(); - }); -}); - -// Load recent call-ins on page load and refresh every 30 seconds +// Load recent call-ins on page load and refresh every 30 seconds. loadRecentCallins(); setInterval(loadRecentCallins, 30000); + +// ===== Today's Schedule horizontal card ===== +function toggleTodaysSchedule() { + const content = document.getElementById('todays-actions-content'); + const chevron = document.getElementById('todays-actions-chevron'); + if (!content || !chevron) return; + const isCollapsed = content.classList.toggle('collapsed'); + chevron.classList.toggle('collapsed', isCollapsed); + // Remember the user's explicit choice so we don't fight them on the next + // 30s htmx refresh. + localStorage.setItem('todaysScheduleUserToggled', '1'); + localStorage.setItem('todaysScheduleCollapsed', isCollapsed ? '1' : '0'); +} + +function onTodaysActionsSwap(el) { + // Read pending/total counts from the rendered partial to drive + // auto-expand + the header badge. + const badge = document.getElementById('todays-actions-badge'); + const content = document.getElementById('todays-actions-content'); + const chevron = document.getElementById('todays-actions-chevron'); + if (!content || !chevron) return; + + // Count yellow status indicators in the rendered partial as a proxy for + // "pending action present today". + const pendingDots = el.querySelectorAll('.bg-yellow-400').length; + const pendingTimes = el.querySelectorAll('.text-yellow-600').length; + const hasPending = pendingDots > 0 || pendingTimes > 0; + + if (badge) { + if (hasPending) { + const n = Math.max(pendingDots, pendingTimes); + badge.textContent = `${n} pending today`; + badge.classList.remove('hidden'); + } else { + badge.classList.add('hidden'); + } + } + + // Auto-expand only if the user hasn't manually toggled this session AND + // there's something pending. Once the user collapses/expands manually, + // their preference sticks. + const userToggled = localStorage.getItem('todaysScheduleUserToggled') === '1'; + if (!userToggled && hasPending) { + content.classList.remove('collapsed'); + chevron.classList.remove('collapsed'); + } else if (!userToggled && !hasPending) { + content.classList.add('collapsed'); + chevron.classList.add('collapsed'); + } else if (userToggled) { + const stored = localStorage.getItem('todaysScheduleCollapsed') === '1'; + content.classList.toggle('collapsed', stored); + chevron.classList.toggle('collapsed', stored); + } +} {% endblock %} -- 2.52.0 From 449e031589ec92415e7956df27c0584aaaa89d88 Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 13 May 2026 22:58:34 +0000 Subject: [PATCH 21/26] feat(status): use SFM event forwards as primary seismograph last-seen, heartbeat as backup emit_status_snapshot() now consults SFM /db/units (cached 15s) before falling back to Emitter.last_seen for each seismograph. The fresher of the two wins and the choice is recorded in a new per-unit last_seen_source field ("sfm" | "heartbeat" | "none"). sfm_reachable is exposed alongside so the UI can show degraded state. Fallback is transparent: if SFM is unreachable or has no record for a serial, the watcher heartbeat path takes over and the unit just shows the HB badge instead of SFM. No schema changes; SLMs are untouched (they don't go through SFM); modems inherit source from their pair. active_table.html grows a small "SFM" / "HB" badge next to the age column so operators can see at a glance which path is currently driving each unit's status. Co-Authored-By: Claude Opus 4.7 --- backend/services/snapshot.py | 126 +++++++++++++++++++++++++-- templates/partials/active_table.html | 9 +- 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/backend/services/snapshot.py b/backend/services/snapshot.py index da54f65..f01cef6 100644 --- a/backend/services/snapshot.py +++ b/backend/services/snapshot.py @@ -1,9 +1,77 @@ from datetime import datetime, timezone +import logging +import os +import threading +import time +from typing import Optional + +import httpx from sqlalchemy.orm import Session from backend.database import get_db_session from backend.models import Emitter, RosterUnit, IgnoredUnit +log = logging.getLogger(__name__) + +SFM_BASE_URL = os.getenv("SFM_BASE_URL", "http://localhost:8200") + +# Tiny module-level cache: /api/status-snapshot is polled every 10s by the +# dashboard, and we don't want to hammer SFM with one /db/units roundtrip per +# call. 15s TTL keeps the cache mostly hot, with occasional refreshes. +_SFM_CACHE_TTL_SECONDS = 15.0 +_sfm_cache_lock = threading.Lock() +_sfm_cache: dict = {"fetched_at": 0.0, "data": None, "reachable": False} + + +def _parse_sfm_timestamp(ts_str: Optional[str]) -> Optional[datetime]: + """SFM /db/units returns naive ISO timestamps (no tz suffix). Treat them + as UTC, mirroring how the watcher heartbeat stores Emitter.last_seen.""" + if not ts_str: + return None + try: + ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + except ValueError: + return None + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +def fetch_sfm_unit_last_seen() -> tuple[dict[str, datetime], bool]: + """Return ({serial: last_seen_utc}, sfm_reachable). + + Cached for _SFM_CACHE_TTL_SECONDS. On any HTTP error returns ({}, False) + so callers transparently fall back to the watcher-heartbeat path. + """ + now = time.monotonic() + with _sfm_cache_lock: + if _sfm_cache["data"] is not None and (now - _sfm_cache["fetched_at"]) < _SFM_CACHE_TTL_SECONDS: + return _sfm_cache["data"], _sfm_cache["reachable"] + + data: dict[str, datetime] = {} + reachable = False + try: + with httpx.Client(timeout=4.0) as client: + resp = client.get(f"{SFM_BASE_URL}/db/units") + resp.raise_for_status() + payload = resp.json() or [] + for row in payload: + serial = row.get("serial") + ts = _parse_sfm_timestamp(row.get("last_seen")) + if serial and ts is not None: + data[serial] = ts + reachable = True + except httpx.HTTPError as e: + log.warning("SFM /db/units unreachable for status snapshot: %s", e) + except Exception as e: # noqa: BLE001 — defensive against malformed payload + log.warning("SFM /db/units parse error: %s", e) + + with _sfm_cache_lock: + _sfm_cache["fetched_at"] = now + _sfm_cache["data"] = data + _sfm_cache["reachable"] = reachable + return data, reachable + def ensure_utc(dt): if dt is None: @@ -69,6 +137,11 @@ def emit_status_snapshot(): emitters = {e.id: e for e in db.query(Emitter).all()} ignored = {i.id for i in db.query(IgnoredUnit).all()} + # SFM event-forwards are now the primary "last seen" signal for + # seismographs. Watcher heartbeats stay as a backup — if SFM is down + # or hasn't seen a serial, we fall back to Emitter.last_seen. + sfm_last_seen_map, sfm_reachable = fetch_sfm_unit_last_seen() + units = {} # --- Merge roster entries first --- @@ -93,24 +166,49 @@ def emit_status_snapshot(): last_seen = None fname = "" else: - if e: - last_seen = ensure_utc(e.last_seen) - # RECALCULATE status based on current time, not stored value + device_type = r.device_type or "seismograph" + emitter_last_seen = ensure_utc(e.last_seen) if e else None + fname = e.last_file if e else "" + + # SFM-primary, heartbeat-backup logic — only for seismographs. + # (SLMs / modems aren't forwarded into SFM's events store.) + sfm_last_seen = sfm_last_seen_map.get(unit_id) if device_type == "seismograph" else None + + if sfm_last_seen and emitter_last_seen: + # Both sources reported — use whichever is more recent. + if sfm_last_seen >= emitter_last_seen: + last_seen = sfm_last_seen + last_seen_source = "sfm" + else: + last_seen = emitter_last_seen + last_seen_source = "heartbeat" + elif sfm_last_seen: + last_seen = sfm_last_seen + last_seen_source = "sfm" + elif emitter_last_seen: + last_seen = emitter_last_seen + # If SFM was reachable but doesn't have this serial, it + # means the unit is calling home to the watcher but not + # being forwarded — still a working state for now. + last_seen_source = "heartbeat" + else: + last_seen = None + last_seen_source = "none" + + if last_seen is not None: status = calculate_status(last_seen, status_ok_threshold, status_pending_threshold) age = format_age(last_seen) - fname = e.last_file else: - # Rostered but no emitter data status = "Missing" - last_seen = None age = "N/A" - fname = "" units[unit_id] = { "id": unit_id, "status": status, "age": age, "last": last_seen.isoformat() if last_seen else None, + "last_seen_source": last_seen_source, + "sfm_reachable": sfm_reachable, "fname": fname, "deployed": r.deployed, "note": r.note or "", @@ -136,14 +234,23 @@ def emit_status_snapshot(): # --- Add unexpected emitter-only units --- for unit_id, e in emitters.items(): if unit_id not in roster: - last_seen = ensure_utc(e.last_seen) + emitter_last_seen = ensure_utc(e.last_seen) + sfm_last_seen = sfm_last_seen_map.get(unit_id) + if sfm_last_seen and (not emitter_last_seen or sfm_last_seen >= emitter_last_seen): + last_seen = sfm_last_seen + last_seen_source = "sfm" + else: + last_seen = emitter_last_seen + last_seen_source = "heartbeat" # RECALCULATE status for unknown units too status = calculate_status(last_seen, status_ok_threshold, status_pending_threshold) units[unit_id] = { "id": unit_id, "status": status, "age": format_age(last_seen), - "last": last_seen.isoformat(), + "last": last_seen.isoformat() if last_seen else None, + "last_seen_source": last_seen_source, + "sfm_reachable": sfm_reachable, "fname": e.last_file, "deployed": False, # default "note": "", @@ -192,6 +299,7 @@ def emit_status_snapshot(): unit_data["status"] = paired_unit.get("status", "Missing") unit_data["age"] = paired_unit.get("age", "N/A") unit_data["last"] = paired_unit.get("last") + unit_data["last_seen_source"] = paired_unit.get("last_seen_source", "none") unit_data["derived_from"] = paired_unit_id # Separate buckets for UI diff --git a/templates/partials/active_table.html b/templates/partials/active_table.html index e72a4d5..085766c 100644 --- a/templates/partials/active_table.html +++ b/templates/partials/active_table.html @@ -36,7 +36,14 @@ -
+
+ {% if unit.last_seen_source == 'sfm' %} + SFM + {% elif unit.last_seen_source == 'heartbeat' %} + HB + {% endif %} {{ unit.age }} -- 2.52.0 From 583af1948e5d59a6e38b3b02fd44454b00b34ba9 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 01:19:33 +0000 Subject: [PATCH 22/26] doc: add server migration plan docs --- docs/SYNOLOGY_DEPLOYMENT.md | 436 ++++++++++++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 docs/SYNOLOGY_DEPLOYMENT.md diff --git a/docs/SYNOLOGY_DEPLOYMENT.md b/docs/SYNOLOGY_DEPLOYMENT.md new file mode 100644 index 0000000..0434647 --- /dev/null +++ b/docs/SYNOLOGY_DEPLOYMENT.md @@ -0,0 +1,436 @@ +# Synology NAS Deployment Guide + +This guide covers migrating the terra-view stack from a generic Linux host +(currently the home server at `10.0.0.44`) to an always-on Synology NAS in +the office, including data migration and the minimal external-access +networking layer. + +## Table of Contents + +1. [Architecture overview](#architecture-overview) +2. [Pre-requisites](#pre-requisites) +3. [Phase 1 — Pre-stage on the NAS (no downtime)](#phase-1--pre-stage-on-the-nas-no-downtime) +4. [Phase 2 — Data migration (~10 min window)](#phase-2--data-migration-10-min-window) +5. [Phase 3 — Repoint the watcher (download2-PC)](#phase-3--repoint-the-watcher-download2-pc) +6. [Phase 4 — External access for remote operators](#phase-4--external-access-for-remote-operators) +7. [Phase 5 — Decommission home server](#phase-5--decommission-home-server) +8. [Verification checklist](#verification-checklist) +9. [Rollback plan](#rollback-plan) +10. [Gotchas](#gotchas) + +--- + +## Architecture overview + +The terra-view stack is three containers: + +| Service | Port | What writes to it | Where it lives | +|---------------|-------|-----------------------------|----------------| +| terra-view | 8001 | Operators (UI), watchers (heartbeat) | Synology NAS | +| SFM | 8200 | Watchers (Blastware ACH forwards) | Synology NAS | +| SLMM | 8100 | terra-view (proxied), SLMs on LAN | Synology NAS | + +Everything that **writes** to the stack lives inside the office LAN: + +- **download2-PC** is the series3-watcher host. It has a static office IP and + POSTs to terra-view's heartbeat endpoint plus SFM's Blastware import + endpoint. Both flows are LAN-internal. +- **Sound level meters (NL-43)** sit on the office LAN; SLMM reaches them + via `network_mode: host`. + +The **only** thing that needs to cross the office firewall is operator UI +access from outside the office (laptops, phones, working from home). That +makes the external networking layer trivial — see Phase 4. + +--- + +## Pre-requisites + +On the Synology side: + +- **DSM 7.2+** with **Container Manager** installed (Package Center). + Older "Docker" package works too — same engine, different menu names. +- **x86_64 model** (Plus / Value / XS series). ARM j-series will build but + expect a slower first build. +- **Static LAN IP** reserved for the NAS in the office router's DHCP table. + Devices on the LAN must have a stable target. +- **SSH enabled** — Control Panel → Terminal & SNMP → Enable SSH service. +- **Shared folder** for the stack — e.g. `/volume1/docker/`. + +On the home server side: + +- Working terra-view / SFM / SLMM stack you want to migrate. +- `rsync` available (it almost certainly is). + +You will also need: + +- An admin account on the Synology with sudo privileges. +- Network access between the home server and the NAS during the migration + window (or USB-drive shuttle if not). + +--- + +## Phase 1 — Pre-stage on the NAS (no downtime) + +Goal: get the NAS booting an empty stack so you can validate the build and +networking *before* touching any production data. + +### 1.1 Clone the repos + +SSH to the NAS as admin: + +```bash +sudo mkdir -p /volume1/docker +cd /volume1/docker +sudo git clone terra-view +sudo git clone slmm +sudo git clone seismo-relay +cd terra-view +sudo git checkout main # or whichever branch you ship from +``` + +### 1.2 Build images + +```bash +cd /volume1/docker/terra-view +sudo docker compose build +``` + +First build takes 5–15 min depending on model. + +### 1.3 Boot the empty stack + +```bash +sudo docker compose up -d +``` + +Hit `http://:1001` (dev profile) or `:8001` (prod profile) from +another office machine. You should see an empty fleet roster. If that +works, the NAS can run the stack — proven before any production data is +at risk. + +### 1.4 Stop the NAS stack again + +```bash +sudo docker compose stop +``` + +We're ready for the data migration. + +--- + +## Phase 2 — Data migration (~10 min window) + +The terra-view stack is stateful in three places. All three must be moved +together for consistency. + +| Service | Data location (home server) | +|------------|----------------------------------------------| +| terra-view | `/home/serversdown/terra-view/data/` | +| SLMM | `/home/serversdown/slmm/data/` | +| SFM | `/home/serversdown/seismo-relay/data/` | + +### 2.1 Stop writes on both sides + +On the NAS: + +```bash +cd /volume1/docker/terra-view +sudo docker compose stop +``` + +On the home server: + +```bash +cd /home/serversdown/terra-view +docker compose stop terra-view slmm sfm +``` + +### 2.2 rsync the data dirs + +From the home server (or anywhere with SSH access to both): + +```bash +rsync -avh /home/serversdown/terra-view/data/ admin@:/volume1/docker/terra-view/data/ +rsync -avh /home/serversdown/slmm/data/ admin@:/volume1/docker/slmm/data/ +rsync -avh /home/serversdown/seismo-relay/data/ admin@:/volume1/docker/seismo-relay/data/ +``` + +### 2.3 Fix ownership on the NAS + +Synology admin is usually UID `1026`, GID `100`. Inside containers running +as root, this doesn't matter — but if you've configured `user:` in any +compose file it will. Safe default: + +```bash +ssh admin@ "sudo chown -R 1026:100 \ + /volume1/docker/terra-view/data \ + /volume1/docker/slmm/data \ + /volume1/docker/seismo-relay/data" +``` + +### 2.4 Run any pending migrations + +Some earlier feature work added migration scripts that need to run once +per database. After the rsync, before starting the stack, check what's +pending: + +```bash +ssh admin@ +cd /volume1/docker/terra-view +ls backend/migrate_*.py +``` + +Run each one inside the container (after starting it temporarily) or apply +them on the host with the same Python environment. Idempotent migrations +re-run safely. + +### 2.5 Start the NAS stack + +```bash +ssh admin@ \ + "cd /volume1/docker/terra-view && sudo docker compose up -d" +``` + +### 2.6 Spot-check + +- Dashboard loads with real units +- `/sfm` page lists historical events +- A photo loads on a unit detail page +- SFM/HB badge mix on the active table matches what you saw on the home + server + +If anything's off, see [Rollback plan](#rollback-plan). + +--- + +## Phase 3 — Repoint the watcher (download2-PC) + +The download2-PC is the one client we have to reconfigure. It currently +POSTs to the home server. Two endpoints to change: + +1. **terra-view heartbeat URL** — + `http://:8001/api/series3/heartbeat` + → `http://:8001/api/series3/heartbeat` + +2. **SFM Blastware import URL** — + `http://:8200/db/import/blastware_file` + → `http://:8200/db/import/blastware_file` + + Or, if you want to keep SFM container-internal and not publish 8200 on + the LAN at all, point it through terra-view's existing SFM proxy: + → `http://:8001/api/sfm/db/import/blastware_file` + +Update the config, restart the watcher service, and confirm the next +heartbeat lands in the NAS DB (check the Recent Call-Ins card on the +dashboard). + +> **Tip:** keep the home server running in parallel for 1–2 days. If you +> forget to repoint something, it'll still flow into the old DB and you +> can resync. + +--- + +## Phase 4 — External access for remote operators + +Only the terra-view UI needs to be reachable from outside the office. Two +clean options — pick one. + +### Option A — Tailscale (recommended for small teams) + +Zero port forwards, zero certs, zero public DNS, zero reverse proxy. + +1. Install Tailscale from Synology Package Center, sign in. +2. Install Tailscale on each operator's laptop/phone, sign in to the same + tailnet. +3. Operators access `http://:8001` from anywhere. + +That's the whole setup. The office network has no external exposure at +all. + +### Option B — Reverse proxy with Let's Encrypt + +If you want a `https://terraview.yourdomain.com` URL that any browser can +reach: + +#### B.1 Port forward on the office router + +``` +WAN 443 → :443 +WAN 80 → :80 (only needed for Let's Encrypt HTTP-01; + skip if you use DNS-01 challenge) +``` + +Do **not** forward 1001, 8001, 8100, or 8200. + +#### B.2 Public DNS + +- Free: Synology DDNS (Control Panel → External Access → DDNS) — gives + you `something.synology.me`. +- Better: your own domain with an A record → office WAN IP, or a CNAME → + Synology DDNS hostname (handles dynamic IPs automatically). + +#### B.3 Let's Encrypt certificate + +Control Panel → Security → Certificate → Add → "Get a certificate from +Let's Encrypt." DSM handles renewal. + +#### B.4 Synology reverse proxy + +Control Panel → Login Portal → Advanced → Reverse Proxy → Create: + +``` +Source: Hostname terraview.yourdomain.com + Protocol HTTPS + Port 443 +Destination: Hostname localhost + Protocol HTTP + Port 8001 +``` + +Under "Custom Header", add: + +| Header | Value | +|---------------------|------------------------------------| +| `X-Forwarded-For` | `$proxy_add_x_forwarded_for` | +| `X-Forwarded-Proto` | `$scheme` | +| `Host` | `$host` | + +Tick the WebSocket support checkbox. + +#### B.5 DSM firewall + +Control Panel → Security → Firewall → enable: + +- 443/TCP from `Anywhere` — allow +- 80/TCP from `Anywhere` — allow (cert renewal only) +- Everything else from WAN — deny +- All from LAN — allow + +Optional: geo-block to your country if your operators are domestic only. +Big reduction in scanning noise. + +--- + +## Phase 5 — Decommission home server + +After 1–2 weeks of stable NAS operation: + +1. Take a final `docker compose down` on the home server. +2. Archive `/home/serversdown/{terra-view,slmm,seismo-relay}/data/` to a + backup volume. +3. Free the home server hardware. + +--- + +## Verification checklist + +After Phase 2 (data migration): + +- [ ] `http://:8001/` loads dashboard with real units +- [ ] Recent Alerts, Call-Ins (2 cols), Fleet Summary across the top +- [ ] SFM/HB badge mix on the active table looks sane +- [ ] `/sfm` page lists historical events (the same count as before) +- [ ] A unit detail page loads with photos rendering +- [ ] `/api/recent-event-callins` returns 200 with real data +- [ ] `/api/status-snapshot` returns 200, `sfm_reachable: true` + +After Phase 3 (watcher cutover): + +- [ ] Next heartbeat from download2-PC lands in NAS DB +- [ ] A new event arrives in `/sfm` page on the NAS within the next + Blastware ACH cycle +- [ ] No errors in `docker logs terra-view-terra-view-1` + +After Phase 4 (external access): + +- [ ] (Option A) Operator laptop on tailnet can reach + `http://:8001` +- [ ] (Option B) `https://terraview.yourdomain.com` resolves, cert is + valid, dashboard loads +- [ ] (Option B) Office DSM admin (5001) is **not** reachable from outside + +--- + +## Rollback plan + +The home server stays alive in parallel through Phases 2–3 as a safety +net. If anything goes wrong on the NAS: + +1. On the home server: + ```bash + cd /home/serversdown/terra-view + docker compose up -d + ``` +2. Point download2-PC back at the home server IP. +3. NAS data isn't lost — it's just sitting idle. Investigate, fix, retry. + +The "irreversible" point is when you decommission the home server in +Phase 5. Until then, you can always fall back. + +--- + +## Gotchas + +1. **Synology UID/GID quirks.** Synology admin is usually `1026:100`. + Containers running as root inside don't care, but if your compose + files set `user:`, mismatched UIDs cause SQLite "readonly database" + errors. Easiest fix: omit `user:` and let containers run as root. + +2. **`network_mode: host` for SLMM.** Required for LAN-direct comms with + sound level meters. On Synology this binds to the NAS's interface — + confirm nothing else on the NAS uses ports 8100 or 21 (FTP). + +3. **Auto-start on boot.** Container Manager → Project → Settings → + enable "Auto-restart". Otherwise a DSM update or NAS reboot drops the + stack. + +4. **`restart: unless-stopped` in compose.** Verify every service has it. + DSM occasionally restarts Docker during DSM updates — this flag + ensures everything comes back. + +5. **Hyper Backup.** Schedule a daily snapshot of + `/volume1/docker/terra-view/data/` to a USB drive or off-site. SQLite + + small photo dir = trivially small backups. The DB-Management UI's + built-in snapshots are an additional layer but not a replacement. + +6. **NAT loopback (Option B only).** If your office router doesn't + support hairpinning, machines INSIDE the office can't reach the NAS + by its public hostname — they have to use the LAN IP. Most modern + routers handle this; some ISP-provided ones don't. Test from a laptop + on the office Wi-Fi. + +7. **Let's Encrypt rate limits (Option B only).** 5 issuances per domain + per week. Don't fat-finger DNS or you'll be locked out. Test with the + staging endpoint first if unsure. + +8. **`host.docker.internal` resolution.** terra-view's + `SFM_BASE_URL=http://host.docker.internal:8200` relies on Docker's + internal DNS. Works on DSM 7.2+ in bridge mode. If you see "name not + resolved" errors, fall back to explicit container names with a custom + network in compose. + +9. **SFM stale rows.** The SFM SQLite has a few rows in `monitor_log` + and `ach_sessions` from earlier Python-ACH experiments. Harmless + to bring over — invisible to terra-view's UI under the + watcher-forward pipeline. + +--- + +## Suggested timeline + +For a low-risk migration: + +- **Week 1**: Phase 1. Get the NAS booting an empty stack. No production + touch. +- **Week 2, day 1**: Phase 2. Migrate data. 10-min window. Keep home + server alive in parallel. +- **Week 2, day 1**: Phase 3. Repoint download2-PC. Watch heartbeats + land on the NAS for the rest of the day. +- **Week 3**: Phase 4. Add Tailscale or reverse-proxy access for remote + operators. +- **Week 4–5**: Monitor. Confirm everything's stable. Then Phase 5 + (decommission home server). + +Splitting "make it work on LAN" from "expose it remotely" means you +debug one thing at a time. -- 2.52.0 From 155f0b007a92e0a56d2841d1f3a45ea20ed26caf Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 17:53:28 +0000 Subject: [PATCH 23/26] feat(events): event modal + sortable tables polish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Event modal (event-modal.js): - Record Type now derived from Blastware filename's last-char code (H=Histogram, W=Waveform, M=Manual, E=Event, C=Combo). Falls back to whatever SFM reported if the code isn't recognized. Client-side workaround — SFM still hardcodes "Waveform" server-side and needs a proper fix in its sidecar parser. - PSI mic tile dropped; mic section now renders 3 tiles (dB(L), ZC Frequency, Time of Peak) instead of 4. - New "View JSON" toggle exposes a prettified inline JSON viewer with a Copy-to-clipboard button alongside the existing "Download sidecar JSON" link. - "Project Info" section header renamed to "User Notes" to reflect that these are operator-typed fields, not the terra-view project assignment. Sortable tables (sfm.html + unit_detail.html): - Both Events tables now have clickable column headers with ↕/↓/↑ indicators. Default sort is Timestamp DESC. Clicking the same column toggles direction; clicking a different column switches and resets to DESC. Sort is purely client-side over the cached rowset, so no extra fetches. Co-Authored-By: Claude Opus 4.7 --- backend/static/event-modal.js | 84 +++++++++++++++++++++++++++++++---- templates/sfm.html | 70 ++++++++++++++++++++++++----- templates/unit_detail.html | 77 ++++++++++++++++++++++++++++---- 3 files changed, 203 insertions(+), 28 deletions(-) diff --git a/backend/static/event-modal.js b/backend/static/event-modal.js index 28a97d8..cf32f7f 100644 --- a/backend/static/event-modal.js +++ b/backend/static/event-modal.js @@ -62,6 +62,27 @@
`; } + function _deriveRecordType(filename, fallback) { + // SFM currently hardcodes record_type="Waveform" for every event. + // The actual type is encoded in the LAST character of the Blastware + // filename's extension (e.g. "O121LL5E.IS0H" → "H" → Histogram). + // We derive it client-side until SFM is fixed; if the suffix isn't + // a known code we fall back to whatever SFM reported. + if (!filename) return fallback || '—'; + const dotIdx = filename.lastIndexOf('.'); + if (dotIdx < 0 || dotIdx === filename.length - 1) return fallback || '—'; + const ext = filename.slice(dotIdx + 1); + const lastChar = ext.slice(-1).toUpperCase(); + const typeMap = { + 'H': 'Histogram', + 'W': 'Waveform', + 'M': 'Manual', + 'E': 'Event', + 'C': 'Combo', + }; + return typeMap[lastChar] || (fallback || '—'); + } + function _sectionHeader(title, sub) { return `

${_esc(title)}${sub ? ` ${_esc(sub)}` : ''} @@ -72,21 +93,23 @@ function _renderEventHeader(s) { const ev = s.event || {}; + const bw = s.blastware || {}; const ts = ev.timestamp ? ev.timestamp.replace('T', ' ').slice(0, 19) : '—'; + const recType = _deriveRecordType(bw.filename || ev.blastware_filename, ev.record_type); return `
Serial ${_esc(ev.serial)}
Timestamp ${ts}
-
Record Type ${_esc(ev.record_type || '—')}
+
Record Type ${_esc(recType)}
Sample Rate ${ev.sample_rate ?? '—'} sps
Rec Time ${ev.rectime_seconds != null ? ev.rectime_seconds + ' s' : '—'}
Waveform Key ${_esc(ev.waveform_key || '—')}
`; } - function _renderProjectInfo(s) { + function _renderUserNotes(s) { // The "user notes" metadata the operator typed into the BW device. // These are the strings the future metadata-driven parser will use. - const p = s.project_info || {}; + const p = s.user_notes || {}; return `
Project ${_esc(p.project || '—')}
Client ${_esc(p.client || '—')}
@@ -120,20 +143,21 @@ } function _renderMic(s) { + // Operators only care about dB(L); PSI tile was dropped 2026-05. + // We still render the row if any mic data is present so ZC freq / + // time-of-peak stay visible even when bw_report.mic is missing. const mic = (s.bw_report && s.bw_report.mic) || null; const pv = s.peak_values || {}; if (!mic && pv.mic_psi == null) return ''; const dbl = mic?.pspl_dbl; - const psi = pv.mic_psi; const zcHz = mic?.zc_freq_hz; const tPk = mic?.time_of_peak_s; const wt = mic?.weighting; - return `
+ return `
${_kvCard('Peak Mic dB(L)', _fmt(dbl, 1), { sub: wt || '' })} - ${_kvCard('Peak Mic psi', _fmt(psi, 4))} ${_kvCard('ZC Frequency', _fmt(zcHz, 1, 'Hz'))} ${_kvCard('Time of Peak', tPk != null ? _fmt(tPk, 2, 's') : '—')}
`; @@ -223,6 +247,14 @@ Blastware file unavailable `} + @@ -232,6 +264,16 @@ Download sidecar JSON
+ `; return `${downloadButtons} @@ -294,8 +336,8 @@ ${_sectionHeader('Event')} ${_renderEventHeader(s)} - ${_sectionHeader('Project Info', '(operator-typed at session start)')} - ${_renderProjectInfo(s)} + ${_sectionHeader('User Notes')} + ${_renderUserNotes(s)} ${_sectionHeader('Peak Particle Velocity')} ${_renderPeakValues(s)} @@ -323,6 +365,32 @@ if (modal) modal.classList.add('hidden'); }; + window.toggleEventJsonViewer = function () { + const viewer = document.getElementById('event-json-viewer'); + const label = document.getElementById('event-json-toggle-label'); + if (!viewer) return; + const isHidden = viewer.classList.toggle('hidden'); + if (label) label.textContent = isHidden ? 'View JSON' : 'Hide JSON'; + }; + + window.copyEventJson = function () { + const pre = document.getElementById('event-json-pre'); + const label = document.getElementById('event-json-copy-label'); + if (!pre) return; + navigator.clipboard.writeText(pre.textContent).then(() => { + if (label) { + label.textContent = 'Copied!'; + setTimeout(() => { label.textContent = 'Copy'; }, 1500); + } + }).catch(err => { + console.error('clipboard write failed', err); + if (label) { + label.textContent = 'Failed'; + setTimeout(() => { label.textContent = 'Copy'; }, 1500); + } + }); + }; + // Close on Escape. document.addEventListener('keydown', function (e) { if (e.key === 'Escape') window.closeEventDetailModal(); diff --git a/templates/sfm.html b/templates/sfm.html index a9123ee..bbb94e2 100644 --- a/templates/sfm.html +++ b/templates/sfm.html @@ -220,6 +220,12 @@ async function loadStats() { } // ── Events tab ─────────────────────────────────────────────────────────────── +// Module-level cache so sort can re-render without re-fetching. +let _eventsCache = []; +let _eventsTotal = 0; +let _eventsSortKey = 'timestamp'; +let _eventsSortDir = 'desc'; // 'asc' | 'desc' + async function loadEvents() { const container = document.getElementById('events-container'); container.innerHTML = '
Loading events…
'; @@ -241,19 +247,61 @@ async function loadEvents() { const r = await fetch('/api/sfm/db/events?' + params.toString()); if (!r.ok) { throw new Error('HTTP ' + r.status); } const d = await r.json(); - renderEventsTable(d.events, d.count, container); + _eventsCache = d.events || []; + _eventsTotal = d.count || 0; + renderEventsTable(_eventsCache, _eventsTotal, container); } catch (e) { container.innerHTML = `
Failed to load events: ${e.message}
`; } } +function sortEvents(key) { + // Toggle direction if same column clicked; otherwise default to desc. + if (_eventsSortKey === key) { + _eventsSortDir = _eventsSortDir === 'desc' ? 'asc' : 'desc'; + } else { + _eventsSortKey = key; + _eventsSortDir = 'desc'; + } + renderEventsTable(_eventsCache, _eventsTotal, document.getElementById('events-container')); +} + +function _applySort(events) { + const key = _eventsSortKey; + const dir = _eventsSortDir === 'asc' ? 1 : -1; + return [...events].sort((a, b) => { + let av = a[key], bv = b[key]; + // Nulls always sort last regardless of dir. + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + if (typeof av === 'number' && typeof bv === 'number') return (av - bv) * dir; + return String(av).localeCompare(String(bv)) * dir; + }); +} + +function _sortIndicator(key) { + if (_eventsSortKey !== key) return '↕'; + return _eventsSortDir === 'desc' + ? '↓' + : '↑'; +} + +function _sortableTh(label, key) { + return ` + ${label}${_sortIndicator(key)} + `; +} + function renderEventsTable(events, total, container) { if (!events || events.length === 0) { container.innerHTML = '

No events found matching the current filters.

'; return; } - const rows = events.map(ev => { + const sorted = _applySort(events); + const rows = sorted.map(ev => { const ts = ev.timestamp ? ev.timestamp.replace('T', ' ').slice(0, 19) : '—'; const tran = fmtPPV(ev.tran_ppv); const vert = fmtPPV(ev.vert_ppv); @@ -288,15 +336,15 @@ function renderEventsTable(events, total, container) { - - - - - - - - - + ${_sortableTh('Timestamp', 'timestamp')} + ${_sortableTh('Serial', 'serial')} + ${_sortableTh('Project', 'project')} + ${_sortableTh('Tran', 'tran_ppv')} + ${_sortableTh('Vert', 'vert_ppv')} + ${_sortableTh('Long', 'long_ppv')} + ${_sortableTh('PVS', 'peak_vector_sum')} + ${_sortableTh('Mic', 'mic_ppv')} + ${_sortableTh('Flags', 'false_trigger')} ${rows} diff --git a/templates/unit_detail.html b/templates/unit_detail.html index aa39df4..8821781 100644 --- a/templates/unit_detail.html +++ b/templates/unit_detail.html @@ -2142,6 +2142,15 @@ function clearUnitEventFilters() { loadUnitEvents(); } +// Module-level state for the unit-events table sort. Cache lets us re-sort +// without a refetch when the user clicks a column header. +let _ueEventsCache = []; +let _ueEventsTotal = 0; +let _ueEventsBucket = 'all'; +let _ueAssignmentsTotal = 0; +let _ueSortKey = 'timestamp'; +let _ueSortDir = 'desc'; + async function loadUnitEvents() { if (!currentUnit || currentUnit.device_type !== 'seismograph') return; const container = document.getElementById('ue-events-container'); @@ -2166,13 +2175,62 @@ async function loadUnitEvents() { throw new Error(err.detail || 'HTTP ' + r.status); } const d = await r.json(); + _ueEventsCache = d.events || []; + _ueEventsTotal = d.count || 0; + _ueEventsBucket = bucket; + _ueAssignmentsTotal = d.assignments_total || 0; renderUnitEventStats(d.stats); - renderUnitEventTable(d.events, d.count, container, bucket, d.assignments_total); + renderUnitEventTable(_ueEventsCache, _ueEventsTotal, container, bucket, _ueAssignmentsTotal); } catch (e) { container.innerHTML = `
Failed to load events: ${e.message}
`; } } +function sortUnitEvents(key) { + if (_ueSortKey === key) { + _ueSortDir = _ueSortDir === 'desc' ? 'asc' : 'desc'; + } else { + _ueSortKey = key; + _ueSortDir = 'desc'; + } + renderUnitEventTable(_ueEventsCache, _ueEventsTotal, + document.getElementById('ue-events-container'), _ueEventsBucket, _ueAssignmentsTotal); +} + +function _ueApplySort(events) { + const key = _ueSortKey; + const dir = _ueSortDir === 'asc' ? 1 : -1; + return [...events].sort((a, b) => { + let av, bv; + if (key === 'attribution') { + // Sort by location name so attributed rows group together. + av = a.attribution ? (a.attribution.location_name || '') : ''; + bv = b.attribution ? (b.attribution.location_name || '') : ''; + } else { + av = a[key]; bv = b[key]; + } + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + if (typeof av === 'number' && typeof bv === 'number') return (av - bv) * dir; + return String(av).localeCompare(String(bv)) * dir; + }); +} + +function _ueSortIndicator(key) { + if (_ueSortKey !== key) return '↕'; + return _ueSortDir === 'desc' + ? '↓' + : '↑'; +} + +function _ueSortableTh(label, key) { + return ``; +} + function renderUnitEventStats(stats) { const s = stats || {}; document.getElementById('ue-stat-total').textContent = (s.event_count ?? 0).toLocaleString(); @@ -2269,7 +2327,8 @@ function renderUnitEventTable(events, total, container, bucket, assignmentsTotal return; } - const rows = events.map(ev => { + const sorted = _ueApplySort(events); + const rows = sorted.map(ev => { const ts = ev.timestamp ? ev.timestamp.replace('T', ' ').slice(0, 19) : '—'; const tran = _ueFmtPPV(ev.tran_ppv); const vert = _ueFmtPPV(ev.vert_ppv); @@ -2295,13 +2354,13 @@ function renderUnitEventTable(events, total, container, bucket, assignmentsTotal
TimestampSerialProjectTranVertLongPVSMicFlags
+ ${label}${_ueSortIndicator(key)} +
- - - - - - - + ${_ueSortableTh('Timestamp', 'timestamp')} + ${_ueSortableTh('Tran', 'tran_ppv')} + ${_ueSortableTh('Vert', 'vert_ppv')} + ${_ueSortableTh('Long', 'long_ppv')} + ${_ueSortableTh('PVS', 'peak_vector_sum')} + ${_ueSortableTh('Flags', 'false_trigger')} + ${_ueSortableTh('Attribution', 'attribution')} ${rows} -- 2.52.0 From 904ff044408b25baf1bebfe3af8014bc51accd72 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 17:53:43 +0000 Subject: [PATCH 24/26] feat(admin): SFM + SLMM diagnostic pages under Developer settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New /admin/sfm page (linked from Settings → Developer): - Health banner — green/red with version + last-checked timestamp - Connection panel — shows SFM_BASE_URL terra-view is configured with - 4 KPI tiles — known units, total events, stale monitor_log rows, stale ach_sessions rows (the deprecated tables from the paused Python-ACH experiment, useful for confirming nothing's growing them) - Per-unit roll-up table — serial, last_seen, event count, stale per-unit counts, sourced from SFM's /db/units - Recent events with forwarding latency — color-coded gap between the event's recorded timestamp and SFM ingest time, so operators can spot watchers that are forwarding stale files (e.g. after a jobsite outage) - Raw API tester — text input + GET button against any /api/sfm/* path, response rendered as prettified JSON New /admin/slmm page — same layout, stripped down to health + connection + raw API tester. For per-device SLM control the existing /sound-level-meters dashboard remains the right entry point. Backend (backend/routers/admin_modules.py): - GET /admin/sfm, GET /admin/slmm — HTML pages - GET /api/admin/sfm/overview — single aggregated probe that returns health, units, last 25 events with computed latency, stale-table counts, cache stats. Tolerant of partial failures: any sub-fetch error is captured into errors{} so a flaky SFM endpoint doesn't break the whole page - GET /api/admin/slmm/overview — health + connection info only for now Co-Authored-By: Claude Opus 4.7 --- backend/main.py | 3 + backend/routers/admin_modules.py | 209 ++++++++++++++++++++++++ templates/admin_sfm.html | 264 +++++++++++++++++++++++++++++++ templates/admin_slmm.html | 138 ++++++++++++++++ templates/settings.html | 28 +++- 5 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 backend/routers/admin_modules.py create mode 100644 templates/admin_sfm.html create mode 100644 templates/admin_slmm.html diff --git a/backend/main.py b/backend/main.py index 3009107..8029f19 100644 --- a/backend/main.py +++ b/backend/main.py @@ -106,6 +106,9 @@ app.include_router(settings.router) from backend.routers import watcher_manager app.include_router(watcher_manager.router) +from backend.routers import admin_modules +app.include_router(admin_modules.router) + # Projects system routers app.include_router(projects.router) app.include_router(project_locations.router) diff --git a/backend/routers/admin_modules.py b/backend/routers/admin_modules.py new file mode 100644 index 0000000..8f9dc6c --- /dev/null +++ b/backend/routers/admin_modules.py @@ -0,0 +1,209 @@ +""" +Admin / diagnostic pages for the device modules (SFM, SLMM). + +These pages live under /admin/{module} and exist purely so an operator can +peek under the hood and confirm the module is reachable, what data it's +holding, and whether the proxy from terra-view is healthy. + +Routes: + GET /admin/sfm — SFM diagnostic page + GET /admin/slmm — SLMM diagnostic page + +API helpers (called by the HTML pages via fetch): + GET /api/admin/sfm/overview — aggregated SFM health + db stats in one call + GET /api/admin/slmm/overview — aggregated SLMM health + device count + +The pages are intentionally read-only. Any actual administration of SFM +or SLMM happens in those modules directly. +""" + +import logging +import os +from datetime import datetime, timezone +from typing import Any, Dict + +import httpx +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse, JSONResponse +from sqlalchemy.orm import Session + +from backend.database import get_db +from backend.templates_config import templates + +log = logging.getLogger(__name__) + +router = APIRouter() + +SFM_BASE_URL = os.getenv("SFM_BASE_URL", "http://localhost:8200") +SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") + + +# ── SFM ─────────────────────────────────────────────────────────────────────── + + +@router.get("/admin/sfm", response_class=HTMLResponse) +def admin_sfm_page(request: Request): + return templates.TemplateResponse("admin_sfm.html", { + "request": request, + "sfm_base_url": SFM_BASE_URL, + }) + + +@router.get("/api/admin/sfm/overview") +async def admin_sfm_overview() -> JSONResponse: + """Aggregated SFM diagnostic snapshot. + + Returns health, db stats, stale-table counts, per-unit summary, and + recent events with forwarding latency. Tolerant of partial failures: + any individual sub-fetch error is captured into its section, so a flaky + sub-endpoint doesn't break the whole page. + """ + overview: Dict[str, Any] = { + "sfm_base_url": SFM_BASE_URL, + "checked_at": datetime.now(timezone.utc).isoformat(), + "health": None, + "reachable": False, + "units": [], + "events": [], + "stale": { + "monitor_log": None, + "sessions": None, + }, + "cache_stats": None, + "errors": {}, + } + + async with httpx.AsyncClient(timeout=5.0) as client: + # Health + try: + r = await client.get(f"{SFM_BASE_URL}/health") + r.raise_for_status() + overview["health"] = r.json() + overview["reachable"] = overview["health"].get("status") == "ok" + except Exception as e: # noqa: BLE001 + overview["errors"]["health"] = str(e) + overview["reachable"] = False + + # If SFM is down, no point hitting the rest. + if not overview["reachable"]: + return JSONResponse(overview) + + # Units + try: + r = await client.get(f"{SFM_BASE_URL}/db/units") + r.raise_for_status() + overview["units"] = r.json() or [] + except Exception as e: # noqa: BLE001 + overview["errors"]["units"] = str(e) + + # Recent events (newest 25 — bigger sample of the call-home stream) + try: + r = await client.get(f"{SFM_BASE_URL}/db/events", params={"limit": 25}) + r.raise_for_status() + payload = r.json() or {} + events = payload.get("events", []) or [] + # Compute forwarding latency: created_at (SFM ingest) − timestamp (event). + now = datetime.now(timezone.utc) + for ev in events: + ev.pop("waveform_blob", None) + ev.pop("a5_pickle_filename", None) + ts_str = ev.get("timestamp") + ca_str = ev.get("created_at") + latency_seconds = None + try: + if ts_str and ca_str: + ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + ca = datetime.fromisoformat(ca_str.replace("Z", "+00:00")) + if ts.tzinfo is None: ts = ts.replace(tzinfo=timezone.utc) + if ca.tzinfo is None: ca = ca.replace(tzinfo=timezone.utc) + latency_seconds = (ca - ts).total_seconds() + except ValueError: + pass + ev["forwarding_latency_seconds"] = latency_seconds + overview["events"] = events + except Exception as e: # noqa: BLE001 + overview["errors"]["events"] = str(e) + + # Stale tables (deprecated by the watcher-forward pipeline but still + # present in SFM's SQLite). Surface as counts only. + for key, path in (("monitor_log", "/db/monitor_log"), + ("sessions", "/db/sessions")): + try: + r = await client.get(f"{SFM_BASE_URL}{path}", params={"limit": 1}) + r.raise_for_status() + payload = r.json() or {} + # SFM returns count = total when limit covers all rows; we + # query with limit=1 just to be polite, then ask again with + # a high limit if we need the real total. + first_count = payload.get("count") + if first_count is None: + overview["stale"][key] = None + continue + # Re-query with high limit to get the true total. + r2 = await client.get(f"{SFM_BASE_URL}{path}", params={"limit": 100000}) + r2.raise_for_status() + overview["stale"][key] = (r2.json() or {}).get("count") + except Exception as e: # noqa: BLE001 + overview["errors"][f"stale_{key}"] = str(e) + + # Cache stats (in-memory device cache on SFM) + try: + r = await client.get(f"{SFM_BASE_URL}/cache/stats") + r.raise_for_status() + overview["cache_stats"] = r.json() + except Exception as e: # noqa: BLE001 + overview["errors"]["cache_stats"] = str(e) + + # Aggregate counts the UI can render without re-walking arrays + overview["totals"] = { + "units": len(overview["units"]), + "events_total": sum(u.get("total_events", 0) for u in overview["units"]), + "stale_monitor_log": overview["stale"]["monitor_log"], + "stale_sessions": overview["stale"]["sessions"], + } + + return JSONResponse(overview) + + +# ── SLMM ────────────────────────────────────────────────────────────────────── + + +@router.get("/admin/slmm", response_class=HTMLResponse) +def admin_slmm_page(request: Request): + return templates.TemplateResponse("admin_slmm.html", { + "request": request, + "slmm_base_url": SLMM_BASE_URL, + }) + + +@router.get("/api/admin/slmm/overview") +async def admin_slmm_overview() -> JSONResponse: + """Aggregated SLMM diagnostic snapshot.""" + overview: Dict[str, Any] = { + "slmm_base_url": SLMM_BASE_URL, + "checked_at": datetime.now(timezone.utc).isoformat(), + "health": None, + "reachable": False, + "devices": [], + "errors": {}, + } + + async with httpx.AsyncClient(timeout=5.0) as client: + try: + r = await client.get(f"{SLMM_BASE_URL}/health") + r.raise_for_status() + overview["health"] = r.json() + overview["reachable"] = True + except Exception as e: # noqa: BLE001 + overview["errors"]["health"] = str(e) + return JSONResponse(overview) + + # Pull a roster of configured devices (SLMM exposes per-unit + # config + status under /api/nl43/*). This is a best-effort probe + # — SLMM doesn't expose a "list all devices" endpoint, so we ask + # terra-view's RosterUnit table what serials it knows about for + # SLMs and just check each one. For now, just surface the health + # payload and let the operator click through to /sound-level-meters + # for the per-device details. + + return JSONResponse(overview) diff --git a/templates/admin_sfm.html b/templates/admin_sfm.html new file mode 100644 index 0000000..e73ea09 --- /dev/null +++ b/templates/admin_sfm.html @@ -0,0 +1,264 @@ +{% extends "base.html" %} + +{% block title %}SFM Admin - Seismo Fleet Manager{% endblock %} + +{% block content %} +
+
+ ← Back to Developer Tools +

SFM Admin

+

Diagnostics for the Seismograph Field Module (SFM) backend.

+
+ +
+ + +
+

Loading SFM status…

+
+ + +
+

Connection

+
+
+ terra-view → SFM URL +
{{ sfm_base_url }}
+
+
+ Last checked +
—
+
+
+ Version +
—
+
+
+
+ + +
+
+
Known Units
+
—
+
+
+
Total Events
+
—
+
+
+
Stale: monitor_log
+
—
+
+
+
Stale: ach_sessions
+
—
+
+
+ + +
+

Per-Unit Roll-up

+

All seismograph serials SFM has ever seen, with their last-event timestamp and total event count. Sourced from GET /db/units.

+
+
TimestampTranVertLongPVSFlagsAttribution
+ + + + + + + + + + + + +
SerialLast SeenEventsMonitor (stale)Sessions (stale)
Loading…
+
+

+ + +
+

Recent Events — Forwarding Latency

+

The last 25 events SFM ingested, with the gap between the event's recorded timestamp and when SFM received the forward. Large latencies indicate the watcher is forwarding stale files (e.g. after a network outage).

+
+ + + + + + + + + + + + + +
RecordedSerialForwardedLatencyFile
Loading…
+
+
+ + +
+

Raw API Tester

+

Send a GET request to any SFM endpoint via the terra-view /api/sfm/* proxy. Path is relative to SFM root (no leading slash).

+
+ /api/sfm/ + + +
+ +
+ + +{% endblock %} diff --git a/templates/admin_slmm.html b/templates/admin_slmm.html new file mode 100644 index 0000000..c9056b1 --- /dev/null +++ b/templates/admin_slmm.html @@ -0,0 +1,138 @@ +{% extends "base.html" %} + +{% block title %}SLMM Admin - Seismo Fleet Manager{% endblock %} + +{% block content %} +
+
+ ← Back to Developer Tools +

SLMM Admin

+

Diagnostics for the Sound Level Meter Manager (SLMM) backend.

+
+ +
+ + +
+

Loading SLMM status…

+
+ + +
+

Connection

+
+
+ terra-view → SLMM URL +
{{ slmm_base_url }}
+
+
+ Last checked +
—
+
+
+ Version +
—
+
+
+
+ For per-device SLM control, see the Sound Level Meters dashboard. +
+
+ + +
+

Raw API Tester

+

Send a GET request to any SLMM endpoint via the terra-view /api/slmm/* proxy.

+
+ /api/slmm/ + + +
+ +
+ + +{% endblock %} diff --git a/templates/settings.html b/templates/settings.html index 9e2bc6e..cb586a7 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -561,7 +561,33 @@ - {# SFM Admin moved back to main nav as "Events" — see sidebar. #} + +
+
+
SFM Admin
+
+ Diagnose the SFM backend — health, per-unit event counts, forwarding latency, stale tables, raw API probe. +
+
+ + Open + +
+ + +
+
+
SLMM Admin
+
+ Diagnose the SLMM backend — health check + raw API probe. For per-device control use the SLM dashboard. +
+
+ + Open + +
{# Metadata Backfill + Project Tidy moved to Tools (they're operator workflows, not admin/dev surfaces). Find them -- 2.52.0 From 9775dca1142698a997b9dcc0eba2574fb2f3a1a7 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 18:00:38 +0000 Subject: [PATCH 25/26] fix(event-modal): update user notes rendering to align with SFM API naming --- backend/static/event-modal.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/static/event-modal.js b/backend/static/event-modal.js index cf32f7f..b41d961 100644 --- a/backend/static/event-modal.js +++ b/backend/static/event-modal.js @@ -109,7 +109,10 @@ function _renderUserNotes(s) { // The "user notes" metadata the operator typed into the BW device. // These are the strings the future metadata-driven parser will use. - const p = s.user_notes || {}; + // NOTE: SFM's sidecar JSON still names this block `project_info` — + // we render it as "User Notes" (the actual BW term) but read the + // field by its SFM-API name. Rename in SFM is a future cleanup. + const p = s.project_info || {}; return `
Project ${_esc(p.project || '—')}
Client ${_esc(p.client || '—')}
-- 2.52.0 From 4378290c9c528f7993e7f242186ed53b98fe2686 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 20:53:07 +0000 Subject: [PATCH 26/26] chore(release): bump to v0.10.0 Version bumped in backend/main.py, README header/highlights/history, and CHANGELOG.md with a comprehensive 0.10.0 entry covering the SFM integration work shipped on this branch. Highlights: - SFM event store is now the authoritative vibration data source - SFM-primary seismograph status with heartbeat fallback - Dashboard rework (top-row reorder, schedule moved, SFM-sourced Recent Call-Ins) - Event detail modal + sortable events tables across all event surfaces - Events attribution engine + metadata-backfill tool - /admin/sfm + /admin/slmm diagnostic pages under Developer settings - Tools workflow hub at /tools (Pair Devices, Project Tidy, Backfill) - Sidebar reorganization (single Devices entry w/ tabs, new Events + Tools entries) Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 31 +++++++++++++++++++++++++++--- backend/main.py | 2 +- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3973d36..a71d3cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,56 @@ All notable changes to Terra-View will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.0] - 2026-05-14 + +This release brings terra-view onto the SFM (Seismograph Field Module) event pipeline. Triggered events forwarded by series3-watcher now land in SFM, and terra-view reads from that store as the authoritative source for vibration data. The watcher heartbeat is preserved as a transparent fallback signal. + +### Added +- **SFM Integration**: New fleet-wide events page at `/sfm` listing every event ingested by SFM, with filters for serial, date range, false-trigger flag, and limit. Unit detail pages and project-location pages show their own attributed subsets of the same event stream. +- **Event Detail Modal**: Shared across `/sfm`, unit detail, and project-location pages — clicking any event opens a rich modal showing peaks per channel (PVS color-coded by magnitude), microphone dB(L) + ZC frequency + time of peak, sensor self-check table with pass/fail per channel, device/recording metadata (firmware, battery, calibration date, geo range), and download buttons for the original Blastware binary and the sidecar JSON. Includes an inline pretty-printed JSON viewer with copy-to-clipboard. +- **Events Attribution Engine** (`backend/services/sfm_events.py`): Per-event attribution against `UnitAssignment` time windows. Events outside any assignment window surface in an "Unattributed" bucket with the nearest-assignment diagnostic (which location, signed delta in days). +- **Metadata Backfill Tool** (`/tools` → Backfill from event metadata): Scans operator-typed `project` and `sensor_location` strings in event sidecars, fuzzy-clusters them via `rapidfuzz.WRatio`, and proposes retroactive `UnitAssignment` records to attribute orphan events. Tracks operator decisions per cluster across re-scans. +- **Project Tidy Tool** (`/tools` → Project Tidy): Fuzzy-detect duplicate projects and bulk-merge them with a single click. Source projects soft-deleted with full audit trail. +- **Vibration Summary on Project Pages**: New roll-up card on vibration project detail pages showing per-location event counts, the project's "Overall Peak" PVS (false triggers excluded), last event timestamp, and a Top Locations by Activity list. +- **SFM-Primary Seismograph Status**: `emit_status_snapshot()` now consults SFM's `/db/units` (cached 15s) before falling back to `Emitter.last_seen` for each seismograph. The fresher signal wins; the choice is recorded in a new per-unit `last_seen_source` field. A small `SFM` (orange) or `HB` (gray) badge on each unit's active-table row shows which path is currently driving the status. +- **Dashboard Rework**: Top row reordered to Recent Alerts → Recent Call-Ins (double-wide) → Fleet Summary. Today's Schedule moved to a horizontal collapsible card below the Fleet Map, auto-expanding only when pending actions exist. Recent Call-Ins now sources from a new `/api/recent-event-callins` endpoint backed by SFM event forwards instead of the watcher-heartbeat endpoint. +- **Sortable Events Tables**: `/sfm` and unit-detail SFM Events tables now have clickable column headers with ↕/↓/↑ indicators. Default sort is Timestamp DESC. Click same column to toggle direction; click different column to switch and reset to DESC. Pure client-side over cached rows — no re-fetches. +- **Developer → SFM Admin** (`/admin/sfm`): Health banner with reachability indicator, terra-view↔SFM connection panel, 4 KPI tiles (known units, total events, stale `monitor_log` rows, stale `ach_sessions` rows), per-unit roll-up table, recent-events table with color-coded forwarding latency (so stale watcher forwards stand out), and a raw API tester for any `/api/sfm/*` path. +- **Developer → SLMM Admin** (`/admin/slmm`): Stripped-down companion page — health, connection info, raw API tester. +- **Tools Workflow Hub** (`/tools`): New top-level sidebar entry consolidating Pair Devices, Project Tidy, Metadata Backfill, Reports (info card), and Swap Detection (placeholder). +- **Sidebar Reorganization**: Devices → Projects → Events → Tools → Job Planner → Settings. Devices is now a single entry with internal tabs (All Devices / Seismographs / Sound Level Meters / Modems / Pair Devices) replacing five separate sidebar items. +- **Synology Deployment Doc** (`docs/SYNOLOGY_DEPLOYMENT.md`): End-to-end playbook for migrating the stack to an always-on office NAS — phased rollout (pre-stage, data rsync, watcher repoint, external access, decommission), Tailscale vs reverse-proxy options, rollback plan, and gotchas. + +### Changed +- **Overall Peak excludes false triggers**: The project-level "Overall Peak" KPI tile (and the underlying `_compute_stats()` function in `sfm_events.py`) now skip events flagged as false triggers when computing the highest PVS, so operators see the highest real event rather than the biggest sensor glitch. `false_trigger_count` still includes flagged events so operators can see how many were filtered out. +- **`RosterUnit.note` Editing**: Inline edit on seismograph cards is more forgiving and now auto-saves on blur. +- **Sidebar Nav Renamed**: Old "Fleet" sidebar entry → "Devices" (renamed because it always meant the device list, not the broader fleet view). + +### Fixed +- **Status drift between watcher heartbeat and actual event arrivals**: Seismographs are now reported with whichever signal is more recent — eliminates the case where a unit had recent SFM events but a stale heartbeat (or vice-versa) showed the wrong status. +- **Event modal: Record Type always showed "Waveform"**: Workaround client-side — Record Type now derived from the Blastware filename's last-char code (`H`=Histogram, `W`=Waveform, `M`=Manual, `E`=Event, `C`=Combo). The proper fix lives in SFM's sidecar parser; tracked separately. +- **Event modal: Mic PSI tile removed**: Operators only care about dB(L); the redundant PSI tile was dropped. + +### Migration Notes +Run on each database before deploying. Every migration is idempotent. + +```bash +# Cleanest: re-run all migrations in chronological order. +# Already-applied migrations no-op safely. +for f in backend/migrate_*.py; do + docker exec terra-view-terra-view-1 python3 "/app/backend/$(basename $f)" +done +``` + +Migrations new in this release: +- `migrate_add_metadata_backfill.py` — adds `unit_assignments.source` column and `metadata_backfill_decisions` table for the Metadata Backfill tool + +### Deployment Notes +- **`SFM_BASE_URL`**: Confirm prod's `docker-compose.yml` sets this for the terra-view service (typically `http://sfm:8200` for the in-stack SFM container, or an external URL if SFM lives elsewhere). +- **Watcher repoint**: series3-watcher's `sfm_forward_url` should point at `https:///api/sfm` (proxy-based — no second port forward needed). Watcher composes the full path `/db/import/blastware_file` itself. + +--- + ## [0.9.4] - 2026-04-06 ### Added diff --git a/README.md b/README.md index 011f764..9ab2bc2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Terra-View v0.9.4 +# Terra-View v0.10.0 Backend API and HTMX-powered web interface for managing a mixed fleet of seismographs and field modems. Track deployments, monitor health in real time, merge roster intent with incoming telemetry, and control your fleet through a unified database and dashboard. ## Features @@ -496,6 +496,19 @@ docker compose down -v ## Release Highlights +### v0.10.0 — 2026-05-14 +- **SFM Integration**: terra-view now consumes events from the SFM (Seismograph Field Module) backend in real time, with a fleet-wide events page at `/sfm`, per-unit attribution against project assignment windows, and a project-level vibration roll-up that uses SFM data as the single source of truth. +- **SFM-Primary Seismograph Status**: Deployed seismograph status (OK/Pending/Missing) now flows from SFM event forwards first; the watcher heartbeat stays as a transparent backup. Each unit's active table row shows a small `SFM` or `HB` badge so operators can see at a glance which signal is currently driving the status. +- **Dashboard Rework**: Top row reordered to Recent Alerts → Recent Call-Ins (double-wide) → Fleet Summary. Today's Schedule moves to a horizontal collapsible card below the Fleet Map, auto-expanding only when there's a pending action. Recent Call-Ins now sources from SFM event forwards instead of the legacy watcher-heartbeat endpoint. +- **Event Detail Modal**: Click any event anywhere in the app to open a rich detail modal showing peak particle velocity per channel, microphone dB(L), sensor self-check results, device/recording metadata, and download buttons for the original Blastware binary and sidecar JSON. Includes an inline JSON viewer with one-click copy. +- **Sortable Events Tables**: Every events table (project events, unit-detail events, fleet-wide /sfm) now supports clickable column-header sorting with directional indicators. Defaults to newest-first. +- **Events Attribution & Backfill**: Each SFM event is automatically attributed to a project/location based on `UnitAssignment` time windows. Unattributed events get a diagnostic showing the nearest assignment and a delta-days gap. The metadata-backfill tool in `/tools` scans operator-typed project/sensor-location strings in event sidecars and clusters them via fuzzy matching to propose new assignment retroactives. +- **Projects Tools**: New `/tools` workflow hub consolidates Pair Devices, Project Tidy (fuzzy-detect + merge duplicate projects), Metadata Backfill, Reports, and Swap Detection (placeholder). +- **Sidebar Reorganization**: Devices → Projects → Events → Tools → Job Planner → Settings. Devices is now a single entry with internal tabs (All Devices / Seismographs / Sound Level Meters / Modems / Pair Devices). +- **Developer → SFM Admin**: New `/admin/sfm` page surfacing SFM health, per-unit roll-up from `/db/units`, recent-events table with forwarding latency (so operators can spot stale watcher forwards), stale-table counts, and a raw API tester. Companion `/admin/slmm` page covers SLMM health + raw API. +- **"Overall Peak" excludes False Triggers**: The project-level Overall Peak KPI tile now excludes events flagged as false triggers — operators see the highest real event, not the biggest sensor glitch. +- **Synology Deployment Doc**: New `docs/SYNOLOGY_DEPLOYMENT.md` covers migrating the stack to an always-on office NAS, including phased rollout, data rsync, watcher repoint, external-access (Tailscale or reverse-proxy), and rollback plan. + ### v0.8.0 — 2026-03-18 - **Watcher Manager**: Admin page for monitoring field watcher agents with live status cards, log tails, and one-click update triggering - **Watcher Status Fix**: Agent status now reflects heartbeat connectivity (missing if not heard from in >60 min) rather than unit-level data staleness @@ -599,9 +612,21 @@ MIT ## Version -**Current: 0.8.0** — Watcher Manager admin page, live agent status refresh, watcher connectivity-based status (2026-03-18) +**Current: 0.10.0** — SFM integration, SFM-primary seismograph status, dashboard rework, sortable events tables, event detail modal, /admin/sfm + /admin/slmm diagnostic pages, Tools workflow hub (2026-05-14) -Previous: 0.7.1 — Out-for-calibration status, reservation modal, migration fixes (2026-03-12) +Previous: 0.9.4 — Modular project types, deleted project management, swap modal search, roster auto-refresh fix (2026-04-06) + +0.9.3 — Monitoring session detail page, configurable period windows, vibration project redesign, modem assignment on locations (2026-03-28) + +0.9.2 — Deployment records, allocated status, quick-info unit modal, inline seismograph editing (2026-03-27) + +0.9.1 — Fix location slots not persisting on save/reload (2026-03-20) + +0.9.0 — Job Planner redesign, monitoring locations, estimated units, smart color picker, calendar bar tooltips, toast notifications (2026-03-19) + +0.8.0 — Watcher Manager admin page, live agent status refresh, watcher connectivity-based status (2026-03-18) + +0.7.1 — Out-for-calibration status, reservation modal, migration fixes (2026-03-12) 0.7.0 — Project status management, manual SD card upload, combined report wizard, NL32 support, MonitoringSession rename (2026-03-07) diff --git a/backend/main.py b/backend/main.py index 8029f19..c7d39d7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -30,7 +30,7 @@ Base.metadata.create_all(bind=engine) ENVIRONMENT = os.getenv("ENVIRONMENT", "production") # Initialize FastAPI app -VERSION = "0.9.4" +VERSION = "0.10.0" if ENVIRONMENT == "development": _build = os.getenv("BUILD_NUMBER", "0") if _build and _build != "0": -- 2.52.0