From d5a0163852d23315d6973e58e74be8f00bd615ce Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 22:22:40 +0000 Subject: [PATCH 01/13] feat(locations): soft-remove monitoring locations without destroying history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a client drops a location from scope mid-project (e.g. the office half of a museum+office monitoring job), operators couldn't previously mark it as no-longer-active without either deleting it (which would orphan historical events) or leaving it in the active list looking deployable. Now there's a proper middle ground. Data model - MonitoringLocation gets two new nullable columns: - removed_at — NULL means active; set means soft-removed - removal_reason — optional operator note Migration: backend/migrate_add_location_removed.py (idempotent) Endpoints - POST /api/projects/{p}/locations/{l}/remove Body: { effective_date?: ISO-datetime, reason?: str } Side effects (cascade): 1. Closes active UnitAssignment rows at this location (assigned_until = effective_date, status = "completed") 2. Cancels pending ScheduledActions at this location 3. Marks location.removed_at = effective_date Returns counts of assignments closed + actions cancelled. - POST /api/projects/{p}/locations/{l}/restore Clears removed_at + removal_reason. Does NOT auto-reopen assignments — operator creates new ones if resuming monitoring. Active-surface filters - locations-json defaults to active-only; pass include_removed=true for historical / reporting views. Schedule modal dropdowns now exclude removed locations automatically. - Metadata-backfill fuzzy matcher excludes removed locations from proposed targets (don't want backfill creating new assignments at decommissioned locations). - Vibration-summary per_location rollup includes removed locations (so historical event totals stay accurate) but tags each with removed_at so the UI can show a badge. UI - Project detail page's Monitoring Locations section now splits into: Active locations (full card with Assign / Edit / Remove / Delete) Removed locations (collapsed
, greyed cards, Restore button, shows removal date + reason) - New per-card "Remove" button → opens confirmation modal explaining the cascade, with optional effective-date (defaults to now, backdateable) and reason fields. - Unit detail's SFM Events attribution cell shows a small "removed" badge next to historical attributions whose location is no longer active. Same pattern in vibration_summary's top-locations list. - Soft-removal indicator surfaced through the events_for_unit attribution payload as location_removed_at. Co-Authored-By: Claude Opus 4.7 --- backend/migrate_add_location_removed.py | 63 ++++++ backend/models.py | 8 + backend/routers/metadata_backfill.py | 3 + backend/routers/project_locations.py | 214 ++++++++++++++++-- backend/services/sfm_events.py | 23 +- .../partials/projects/location_list.html | 104 +++++++-- .../partials/projects/vibration_summary.html | 4 + templates/projects/detail.html | 143 ++++++++++++ templates/unit_detail.html | 7 +- 9 files changed, 531 insertions(+), 38 deletions(-) create mode 100644 backend/migrate_add_location_removed.py diff --git a/backend/migrate_add_location_removed.py b/backend/migrate_add_location_removed.py new file mode 100644 index 0000000..7c7fee7 --- /dev/null +++ b/backend/migrate_add_location_removed.py @@ -0,0 +1,63 @@ +""" +Migration: add `removed_at` + `removal_reason` columns to `monitoring_locations`. + +Lets operators mark a location as no longer actively monitored without +deleting it (so historical events stay attributed correctly). Mirrors +the timestamp-based "closed state" pattern already used by +`unit_assignments.assigned_until`. + +Behavior: + - `removed_at IS NULL` → location is active (default for all existing + rows after this migration) + - `removed_at` set → location is removed; historical events still + attribute to it but it's hidden from active + surfaces (assign dropdowns, calendar, etc.) + - `removal_reason` → optional operator note (e.g. "client dropped + from scope") + +Idempotent — safe to re-run. Non-destructive — adds only. + +Run with: + docker exec terra-view-terra-view-1 python3 /app/backend/migrate_add_location_removed.py +""" + +import os +import sqlite3 + +DB_PATH = "./data/seismo_fleet.db" + + +def _has_column(cur: sqlite3.Cursor, table: str, column: str) -> bool: + cur.execute(f"PRAGMA table_info({table})") + return any(row[1] == column for row in cur.fetchall()) + + +def migrate_database() -> None: + if not os.path.exists(DB_PATH): + print(f"Database not found at {DB_PATH}") + return + + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + added = [] + if not _has_column(cur, "monitoring_locations", "removed_at"): + cur.execute("ALTER TABLE monitoring_locations ADD COLUMN removed_at DATETIME") + added.append("removed_at") + if not _has_column(cur, "monitoring_locations", "removal_reason"): + cur.execute("ALTER TABLE monitoring_locations ADD COLUMN removal_reason TEXT") + added.append("removal_reason") + + conn.commit() + conn.close() + + if added: + print(f" Added columns to monitoring_locations: {', '.join(added)}") + else: + print(" monitoring_locations already has removed_at + removal_reason — nothing to do.") + + +if __name__ == "__main__": + print("Running migration: add removed_at + removal_reason to monitoring_locations") + migrate_database() + print("Done.") diff --git a/backend/models.py b/backend/models.py index 5ab7761..8e17197 100644 --- a/backend/models.py +++ b/backend/models.py @@ -235,6 +235,14 @@ class MonitoringLocation(Base): # For vibration: {"ground_type": "bedrock", "depth": "10m"} location_metadata = Column(Text, nullable=True) + # Soft-removal: NULL means active. When set, the location is hidden from + # active surfaces (assign dropdowns, calendar, scheduler, dashboard + # vibration summary) but historical events generated before this time + # still attribute to it. Mirrors the closed-state pattern used by + # UnitAssignment.assigned_until. + removed_at = Column(DateTime, nullable=True) + removal_reason = Column(Text, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) diff --git a/backend/routers/metadata_backfill.py b/backend/routers/metadata_backfill.py index 3e002a1..f086020 100644 --- a/backend/routers/metadata_backfill.py +++ b/backend/routers/metadata_backfill.py @@ -361,6 +361,9 @@ def locations_search( db.query(MonitoringLocation) .filter(MonitoringLocation.project_id == project_id) .filter(MonitoringLocation.location_type == "vibration") + # Don't propose creating assignments at removed locations — they + # were intentionally decommissioned and shouldn't be backfill targets. + .filter(MonitoringLocation.removed_at == None) # noqa: E711 .all() ) diff --git a/backend/routers/project_locations.py b/backend/routers/project_locations.py index 733fd38..7fda2eb 100644 --- a/backend/routers/project_locations.py +++ b/backend/routers/project_locations.py @@ -31,6 +31,7 @@ from backend.models import ( MonitoringSession, DataFile, UnitHistory, + ScheduledAction, ) from backend.templates_config import templates from backend.utils.timezone import local_to_utc @@ -138,7 +139,7 @@ async def get_project_locations( ): """ Get all monitoring locations for a project. - Returns HTML partial with location list. + Returns HTML partial with location list, split into active + removed. """ project = db.query(Project).filter_by(id=project_id).first() if not project: @@ -152,10 +153,14 @@ async def get_project_locations( locations = query.order_by(MonitoringLocation.name).all() - # Enrich with assignment info - locations_data = [] + # Enrich with assignment info, splitting active vs removed. + active_data: list = [] + removed_data: list = [] for location in locations: - # Get active assignment (active = assigned_until IS NULL) + # Get active assignment (active = assigned_until IS NULL). For + # removed locations this will normally be None because the + # /remove cascade closes them, but check anyway for resilience + # against legacy data. assignment = db.query(UnitAssignment).filter( and_( UnitAssignment.location_id == location.id, @@ -172,17 +177,23 @@ async def get_project_locations( location_id=location.id ).count() - locations_data.append({ - "location": location, - "assignment": assignment, + item = { + "location": location, + "assignment": assignment, "assigned_unit": assigned_unit, "session_count": session_count, - }) + } + if location.removed_at is None: + active_data.append(item) + else: + removed_data.append(item) return templates.TemplateResponse("partials/projects/location_list.html", { - "request": request, - "project": project, - "locations": locations_data, + "request": request, + "project": project, + "locations": active_data, # back-compat alias + "active_locations": active_data, + "removed_locations": removed_data, }) @@ -191,10 +202,15 @@ async def get_project_locations_json( project_id: str, db: Session = Depends(get_db), location_type: Optional[str] = Query(None), + include_removed: bool = Query(False), ): """ Get all monitoring locations for a project as JSON. Used by the schedule modal to populate location dropdown. + + Removed locations are filtered out by default (you can't schedule + a new action at a removed location). Pass `include_removed=true` + to get them too — useful for historical / reporting views. """ project = db.query(Project).filter_by(id=project_id).first() if not project: @@ -205,16 +221,21 @@ async def get_project_locations_json( if location_type: query = query.filter_by(location_type=location_type) + if not include_removed: + query = query.filter(MonitoringLocation.removed_at == None) # noqa: E711 + locations = query.order_by(MonitoringLocation.name).all() return [ { - "id": loc.id, - "name": loc.name, - "location_type": loc.location_type, - "description": loc.description, - "address": loc.address, - "coordinates": loc.coordinates, + "id": loc.id, + "name": loc.name, + "location_type": loc.location_type, + "description": loc.description, + "address": loc.address, + "coordinates": loc.coordinates, + "removed_at": loc.removed_at.isoformat() if loc.removed_at else None, + "removal_reason": loc.removal_reason, } for loc in locations ] @@ -335,6 +356,165 @@ async def delete_location( return {"success": True, "message": "Location deleted successfully"} +@router.post("/locations/{location_id}/remove") +async def remove_location( + project_id: str, + location_id: str, + request: Request, + db: Session = Depends(get_db), +): + """ + Soft-remove a monitoring location — mark it as no longer actively + monitored without destroying it. + + Use case: a client drops a location from scope mid-project, but the + historical events recorded there should remain attributed. Deleting + would orphan those events; this preserves them. + + Cascading side-effects: + 1. All active UnitAssignment rows at this location are closed + (assigned_until = effective_date, status = "completed"). + Units become available for other deployments. + 2. All pending ScheduledAction rows at this location are cancelled + (execution_status = "cancelled"). + 3. Historical events stay attributed (attribution is window-based; + events with timestamp < effective_date still match the + now-closed assignment windows). + + Accepts JSON body: + - effective_date: ISO datetime (optional, defaults to now) + - reason: operator note (optional) + """ + location = db.query(MonitoringLocation).filter_by( + id=location_id, + project_id=project_id, + ).first() + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if location.removed_at is not None: + raise HTTPException( + status_code=400, + detail=f"Location is already removed (as of {location.removed_at.isoformat()}).", + ) + + # Body is optional — POST with no body is fine and means "remove now, + # no reason given." + try: + payload = await request.json() + except Exception: + payload = {} + + # Effective date: accept "YYYY-MM-DDTHH:MM" from datetime-local inputs or + # full ISO. Defaults to now if absent/empty. + raw_eff = payload.get("effective_date") + if raw_eff: + try: + effective_date = datetime.fromisoformat(raw_eff) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail=f"Invalid effective_date: {raw_eff!r}", + ) + else: + effective_date = datetime.utcnow() + + reason = (payload.get("reason") or "").strip() or None + + # 1. Close active assignments at this location. + active_assignments = db.query(UnitAssignment).filter( + and_( + UnitAssignment.location_id == location_id, + UnitAssignment.assigned_until == None, # noqa: E711 — SQL NULL + ) + ).all() + + for a in active_assignments: + a.status = "completed" + a.assigned_until = effective_date + _record_assignment_history( + db, + unit_id=a.unit_id, + change_type="assignment_ended", + old_value=location.name, + new_value="location removed", + notes=f"Location '{location.name}' marked as removed" + + (f" — {reason}" if reason else ""), + ) + + # 2. Cancel pending scheduled actions at this location. + pending_actions = db.query(ScheduledAction).filter( + and_( + ScheduledAction.location_id == location_id, + ScheduledAction.execution_status == "pending", + ScheduledAction.scheduled_time >= effective_date, + ) + ).all() + + for sa in pending_actions: + sa.execution_status = "cancelled" + sa.error_message = ( + f"Cancelled: location '{location.name}' marked as removed" + + (f" — {reason}" if reason else "") + ) + + # 3. Mark the location itself as removed. + location.removed_at = effective_date + location.removal_reason = reason + location.updated_at = datetime.utcnow() + + db.commit() + + return { + "success": True, + "message": f"Location '{location.name}' marked as removed", + "effective_date": effective_date.isoformat(), + "assignments_closed": len(active_assignments), + "actions_cancelled": len(pending_actions), + } + + +@router.post("/locations/{location_id}/restore") +async def restore_location( + project_id: str, + location_id: str, + db: Session = Depends(get_db), +): + """ + Restore a previously-removed monitoring location to active. + + Clears `removed_at` and `removal_reason`. Does NOT automatically + re-open the assignments or scheduled actions that were closed when + the location was removed — those stay closed and the operator can + create new ones if they want to resume monitoring. + """ + location = db.query(MonitoringLocation).filter_by( + id=location_id, + project_id=project_id, + ).first() + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if location.removed_at is None: + raise HTTPException( + status_code=400, + detail="Location is already active.", + ) + + location.removed_at = None + location.removal_reason = None + location.updated_at = datetime.utcnow() + + db.commit() + + return { + "success": True, + "message": f"Location '{location.name}' restored to active", + } + + # ============================================================================ # Unit Assignments # ============================================================================ diff --git a/backend/services/sfm_events.py b/backend/services/sfm_events.py index b866818..5fd46dd 100644 --- a/backend/services/sfm_events.py +++ b/backend/services/sfm_events.py @@ -318,13 +318,18 @@ async def events_for_unit( loc = loc_map.get(a.location_id) proj = proj_map.get(a.project_id) return { - "assignment_id": a.id, - "location_id": a.location_id, - "location_name": loc.name if loc else None, - "project_id": a.project_id, - "project_name": proj.name if proj else None, - "assigned_at": _iso_utc(a.assigned_at), - "assigned_until": _iso_utc(a.assigned_until), + "assignment_id": a.id, + "location_id": a.location_id, + "location_name": loc.name if loc else None, + # Soft-removal indicator so the UI can render a "(removed)" + # badge next to historical attributions whose location is no + # longer actively monitored. + "location_removed_at": (loc.removed_at.isoformat() + if loc and loc.removed_at else None), + "project_id": a.project_id, + "project_name": proj.name if proj else None, + "assigned_at": _iso_utc(a.assigned_at), + "assigned_until": _iso_utc(a.assigned_until), } # 2. Fetch all events for this serial in one shot. @@ -515,6 +520,10 @@ async def vibration_summary_for_project( "event_count": ec, "peak_pvs": ev_peak, "last_event": ev_last, + # Soft-removal state — UI can show a "(removed)" badge in the + # per-location list so operators see at a glance that a row's + # numbers are historical-only. + "removed_at": loc.removed_at.isoformat() if loc.removed_at else None, }) per_location.sort(key=lambda r: r["event_count"], reverse=True) diff --git a/templates/partials/projects/location_list.html b/templates/partials/projects/location_list.html index 9c8219a..f274373 100644 --- a/templates/partials/projects/location_list.html +++ b/templates/partials/projects/location_list.html @@ -1,7 +1,21 @@ - -{% if locations %} + + +{% if not active_locations and not removed_locations %} +
+ + + +

No locations added yet

+
+{% else %} + +{# ─── Active locations ─── #} +{% if active_locations %}
- {% for item in locations %} + {% for item in active_locations %}
@@ -24,11 +38,13 @@
{% if item.assignment %} - {% else %} - {% endif %} @@ -37,7 +53,14 @@ class="text-xs px-3 py-1 rounded-full bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300"> Edit - +
@@ -54,11 +77,66 @@
{% endfor %}
-{% else %} -
- - - -

No locations added yet

-
+{% endif %} + +{# ─── Removed locations (collapsed by default) ─── #} +{% if removed_locations %} +
+ + + + + + Removed locations + {{ removed_locations | length }} + +

Historical only — events stay attributed, but no new assignments or schedules can be created here.

+ + +
+ {% for item in removed_locations %} +
+
+
+
+ + {{ item.location.name }} + + + Removed + + + {{ item.location.removed_at.strftime('%Y-%m-%d') if item.location.removed_at else '—' }} + +
+ {% if item.location.removal_reason %} +

"{{ item.location.removal_reason }}"

+ {% endif %} + {% if item.location.description %} +

{{ item.location.description }}

+ {% endif %} + {% if item.location.address %} +

{{ item.location.address }}

+ {% endif %} +
+ +
+ +
+
+ +
+ Historical sessions: {{ item.session_count }} +
+
+ {% endfor %} +
+
+{% endif %} + {% endif %} diff --git a/templates/partials/projects/vibration_summary.html b/templates/partials/projects/vibration_summary.html index 4197af9..ce32f8a 100644 --- a/templates/partials/projects/vibration_summary.html +++ b/templates/partials/projects/vibration_summary.html @@ -60,6 +60,10 @@ class="flex items-center justify-between py-1.5 px-3 rounded hover:bg-gray-50 dark:hover:bg-slate-700/50 transition-colors"> 📍 {{ loc.location_name }} + {% if loc.removed_at %} + removed + {% endif %} {{ "{:,}".format(loc.event_count) }} event{{ '' if loc.event_count == 1 else 's' }} diff --git a/templates/projects/detail.html b/templates/projects/detail.html index 70c3dc5..4f75550 100644 --- a/templates/projects/detail.html +++ b/templates/projects/detail.html @@ -778,6 +778,61 @@
+ + + `; @@ -608,6 +616,33 @@ function openAssignmentEditModal(encodedJson) { document.getElementById('assignment-edit-modal').classList.remove('hidden'); } +async function deleteAssignment(assignmentId, unitId, windowLabel) { + // For mis-clicks / accidental duplicate assignments. Backend refuses + // if there's a real recording session inside the window — those should + // go through Edit or Unassign instead. + const msg = `Delete this assignment?\n\n` + + `Unit: ${unitId}\n` + + `Window: ${windowLabel}\n\n` + + `This is for assignments created in error. Events that fell ` + + `in this window will become unattributed. The unit's deployment ` + + `history will log the deletion for audit.\n\n` + + `If the unit actually was deployed here, use Edit or Unassign instead.`; + if (!confirm(msg)) return; + + try { + const r = await fetch(`/api/projects/${projectId}/assignments/${assignmentId}`, { + method: 'DELETE', + }); + if (!r.ok) { + const err = await r.json().catch(() => ({detail: 'HTTP ' + r.status})); + throw new Error(err.detail || 'HTTP ' + r.status); + } + await loadLocationEvents(); // Refresh stats + table without this assignment. + } catch (err) { + alert(err.message || 'Failed to delete assignment.'); + } +} + function closeAssignmentEditModal() { document.getElementById('assignment-edit-modal').classList.add('hidden'); } From ef0008822e5ea69c4ef214526c8fff4f29288f36 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 14 May 2026 23:29:51 +0000 Subject: [PATCH 04/13] feat(timeline): merge consecutive same-location assignments + per-unit Gantt chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a unit had its assignment closed-then-reopened (e.g. via the recent location remove/restore flow) or had metadata-backfill auto- create a retroactive window adjacent to a manual one, the deployment timeline showed N stacked rows that represented one continuous deployment. Visual noise that didn't match reality. Merge feature - New endpoint POST /api/projects/{p}/assignments/merge - Body: { assignment_ids: [uuid, ...] } - Keeps earliest record, extends its window to span all inputs, deletes the others, logs `assignment_merged` to UnitHistory - Validates: all assignments share same unit + location, all belong to the same project - deployment_timeline_for_unit() now auto-detects mergeable groups (consecutive same-location assignments within 7-day gap tolerance) and returns them in `merge_groups` as a list of id-lists - Unit detail page shows a blue banner above the timeline list when groups exist, with one "Merge into one" button per group. Each mergeable row gets a small "mergeable" badge to make the relationship obvious. Per-unit Gantt chart (Phase 1 of the deployment-history calendar) - Plain-SVG horizontal timeline rendered above the existing Deployment Timeline list, ~140px tall - One colored bar per assignment, color-keyed by location (auto- assigned palette + legend) - Reduced opacity for closed bars; small white dot at the right edge of active bars; today marker as a dashed orange vertical line - Month gridlines (or every-3-month gridlines when domain > 24 months) - Metadata-backfilled assignments get a blue outline so you spot which were auto-attributed - Mergeable groups get a dashed blue underline tying their bars together visually - Click any bar → smooth-scrolls the matching list row into view and flashes a ring around it - Hover any bar → tooltip with location + window + event count - Auto-hides on units with no deployment history Co-Authored-By: Claude Opus 4.7 --- backend/routers/project_locations.py | 136 ++++++++++ backend/services/deployment_timeline.py | 44 +++- templates/unit_detail.html | 323 +++++++++++++++++++++++- 3 files changed, 492 insertions(+), 11 deletions(-) diff --git a/backend/routers/project_locations.py b/backend/routers/project_locations.py index d62d6ff..02003d6 100644 --- a/backend/routers/project_locations.py +++ b/backend/routers/project_locations.py @@ -916,6 +916,142 @@ async def delete_assignment( } +@router.post("/assignments/merge") +async def merge_assignments( + project_id: str, + request: Request, + db: Session = Depends(get_db), +): + """ + Merge multiple consecutive UnitAssignment rows for the same (unit, location) + into a single record spanning their combined window. + + Use case: a unit's deployment timeline shows 3 stacked rows for the + same location because the assignment was closed-and-reopened (e.g. via + location remove + restore) or because metadata-backfill auto-created + a retroactive window adjacent to a manual one. Operator sees three + rows but they represent one continuous deployment. + + Body JSON: + { "assignment_ids": ["", "", ...] } + + Validation: + - All assignments must belong to project_id + - All must share the same unit_id AND location_id + - At least 2 ids must be provided + + Merge rules: + - Keeps the EARLIEST-starting assignment as the surviving row + - assigned_at = min(assigned_at across all) + - assigned_until = max(assigned_until), or NULL if any input was active + - status = "active" if any input was active, else "completed" + - source = source of the earliest record (preserves original ingest provenance) + - notes = earliest's notes + "Merged N records ()" + - Other records are DELETED + - One UnitHistory `assignment_merged` row is written for audit + + No tolerance check — the operator is asking for the merge, so we trust + the intent. The UI can pre-filter to only offer "consecutive" merges + if it wants. + """ + try: + payload = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + ids = payload.get("assignment_ids") or [] + if not isinstance(ids, list) or len(ids) < 2: + raise HTTPException( + status_code=400, + detail="Need at least 2 assignment_ids to merge.", + ) + + assignments = ( + db.query(UnitAssignment) + .filter(UnitAssignment.project_id == project_id) + .filter(UnitAssignment.id.in_(ids)) + .all() + ) + + if len(assignments) != len(set(ids)): + raise HTTPException( + status_code=404, + detail=f"Some assignments not found (got {len(assignments)} of {len(set(ids))}).", + ) + + unit_ids = {a.unit_id for a in assignments} + loc_ids = {a.location_id for a in assignments} + if len(unit_ids) > 1 or len(loc_ids) > 1: + raise HTTPException( + status_code=400, + detail="Can only merge assignments that share the same unit and location.", + ) + + # Order chronologically. + assignments.sort(key=lambda a: a.assigned_at) + earliest = assignments[0] + others = assignments[1:] + + # Compute merged window. + any_active = any(a.assigned_until is None for a in assignments) + if any_active: + merged_until = None + else: + merged_until = max(a.assigned_until for a in assignments) + + # Build a brief audit-style note describing what got merged. + bits = [] + for a in assignments: + win = ( + f"{a.assigned_at:%Y-%m-%d}" + f"→{(a.assigned_until and a.assigned_until.strftime('%Y-%m-%d')) or 'active'}" + ) + bits.append(f"{win} [{a.source}]") + merge_note_suffix = f"Merged {len(assignments)} records: " + " + ".join(bits) + new_notes = (earliest.notes + " • " + merge_note_suffix) if earliest.notes else merge_note_suffix + + # Resolve names for the audit log before mutating. + location = db.query(MonitoringLocation).filter_by(id=earliest.location_id).first() + location_label = location.name if location else earliest.location_id + + # Mutate the survivor. + earliest.assigned_at = min(a.assigned_at for a in assignments) + earliest.assigned_until = merged_until + earliest.status = "active" if any_active else "completed" + earliest.notes = new_notes + + # Delete the rest. + deleted_ids = [a.id for a in others] + for a in others: + db.delete(a) + + _record_assignment_history( + db, + unit_id=earliest.unit_id, + change_type="assignment_merged", + old_value=f"{len(assignments)} rows at {location_label}", + new_value=( + f"1 row {earliest.assigned_at:%Y-%m-%d}" + f"→{(merged_until and merged_until.strftime('%Y-%m-%d')) or 'active'}" + ), + notes=merge_note_suffix, + ) + + db.commit() + db.refresh(earliest) + + return { + "success": True, + "message": f"Merged {len(assignments)} assignments into one.", + "kept_id": earliest.id, + "deleted_ids": deleted_ids, + "merged_window": { + "assigned_at": earliest.assigned_at.isoformat(), + "assigned_until": earliest.assigned_until.isoformat() if earliest.assigned_until else None, + }, + } + + @router.post("/locations/{location_id}/swap") async def swap_unit_on_location( project_id: str, diff --git a/backend/services/deployment_timeline.py b/backend/services/deployment_timeline.py index 21fa8af..6690b52 100644 --- a/backend/services/deployment_timeline.py +++ b/backend/services/deployment_timeline.py @@ -46,6 +46,13 @@ log = logging.getLogger("backend.services.deployment_timeline") # clutter from a sub-second handoff during a swap workflow. _MIN_GAP_SECONDS = 24 * 3600 # 1 day +# When detecting "mergeable" groups of consecutive same-location assignments, +# treat assignments separated by no more than this many seconds as adjacent. +# Generous enough to catch overnight handoffs and weekend gaps where the +# operator forgot to log, but tight enough that genuinely separate +# deployments months apart don't get suggested for merging. +_MERGE_GAP_TOLERANCE_SECONDS = 7 * 24 * 3600 # 7 days + # Per-call timeout when querying SFM for the event overlay. _SFM_TIMEOUT = 10.0 _SFM_FETCH_CEILING = 5000 @@ -245,12 +252,41 @@ async def deployment_timeline_for_unit( "history_notes": h.notes, }) - # 6. Sort newest first. Active assignments (no end) sort by start time, + # 6. Detect mergeable groups — runs of consecutive assignments to the + # same location with small gaps between them. Each group becomes a + # list of assignment_ids; the UI offers a "Merge into one" action + # on any group >= 2. + merge_groups: list[list[str]] = [] + if len(assignments) >= 2: + # Sort ascending for the linear scan. + sorted_assignments = sorted(assignments, key=lambda a: a.assigned_at) + cur_group: list[UnitAssignment] = [sorted_assignments[0]] + for a in sorted_assignments[1:]: + prev = cur_group[-1] + same_location = a.location_id == prev.location_id + prev_end = prev.assigned_until or now + gap_seconds = (a.assigned_at - prev_end).total_seconds() if a.assigned_at else 0 + # Within tolerance and same location → extend the current group. + # Negative gaps (overlap) also count as adjacent. + if same_location and gap_seconds <= _MERGE_GAP_TOLERANCE_SECONDS: + cur_group.append(a) + else: + if len(cur_group) >= 2: + merge_groups.append([x.id for x in cur_group]) + cur_group = [a] + if len(cur_group) >= 2: + merge_groups.append([x.id for x in cur_group]) + + # 7. Sort newest first. Active assignments (no end) sort by start time, # same as everything else. entries.sort(key=lambda e: e.get("starts_at") or "", reverse=True) return { - "unit_id": unit.id, - "device_type": unit.device_type, - "entries": entries, + "unit_id": unit.id, + "device_type": unit.device_type, + "entries": entries, + # List of assignment_id lists; each inner list is a mergeable group. + # Empty if nothing is mergeable. UI shows a "Merge" button on any + # row whose assignment_id appears in a group. + "merge_groups": merge_groups, } diff --git a/templates/unit_detail.html b/templates/unit_detail.html index ff2fcc6..beec92b 100644 --- a/templates/unit_detail.html +++ b/templates/unit_detail.html @@ -287,6 +287,16 @@ ↻ Refresh + + + +

Loading timeline…

@@ -1986,6 +1996,10 @@ loadUnitData().then(() => { // Replaces the legacy loadDeploymentHistory() + loadUnitHistory() pair. // Derives entries from unit_assignments + unit_history + SFM event overlay. +// Cache the most recent timeline payload so the merge action can look up +// which assignment_ids belong together in a mergeable group. +let _dtCurrentTimeline = { entries: [], merge_groups: [] }; + async function loadDeploymentTimeline() { const container = document.getElementById('deploymentTimeline'); container.innerHTML = '

Loading timeline…

'; @@ -1994,12 +2008,58 @@ async function loadDeploymentTimeline() { const r = await fetch(`/api/units/${currentUnit.id}/deployment_timeline`); if (!r.ok) throw new Error('HTTP ' + r.status); const d = await r.json(); - renderDeploymentTimeline(d.entries || [], container); + _dtCurrentTimeline = { + entries: d.entries || [], + merge_groups: d.merge_groups || [], + }; + renderDeploymentTimeline(_dtCurrentTimeline.entries, container, _dtCurrentTimeline.merge_groups); } catch (e) { container.innerHTML = `

Failed to load timeline: ${e.message}

`; } } +// Returns the merge_group (list of assignment_ids) that this assignment is +// part of, or null if it isn't in any mergeable group. +function _dtFindMergeGroup(assignmentId) { + for (const group of _dtCurrentTimeline.merge_groups || []) { + if (group.includes(assignmentId)) return group; + } + return null; +} + +async function mergeAssignmentGroup(assignmentIds) { + if (!Array.isArray(assignmentIds) || assignmentIds.length < 2) return; + const msg = `Merge ${assignmentIds.length} consecutive assignment records into one?\n\n` + + `The earliest record is kept and its window extended to span all ` + + `of them. The other ${assignmentIds.length - 1} record(s) are deleted.\n\n` + + `Original metadata (notes + ingest source) is preserved. This is ` + + `logged to the unit's history as "assignment_merged".`; + if (!confirm(msg)) return; + + try { + // All assignments share the same project_id (validated server-side). + // Pick the first entry's project_id from the cache. + const first = (_dtCurrentTimeline.entries || []).find(e => + e.kind === 'assignment' && assignmentIds.includes(e.assignment_id) + ); + const projectId = first ? first.project_id : null; + if (!projectId) throw new Error('Could not resolve project id for this group'); + + const r = await fetch(`/api/projects/${projectId}/assignments/merge`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ assignment_ids: assignmentIds }), + }); + if (!r.ok) { + const err = await r.json().catch(() => ({detail: 'HTTP ' + r.status})); + throw new Error(err.detail || 'HTTP ' + r.status); + } + await loadDeploymentTimeline(); + } catch (e) { + alert(e.message || 'Failed to merge assignments.'); + } +} + function _dtFmtDate(iso) { if (!iso) return '—'; return iso.slice(0, 10); @@ -2044,6 +2104,17 @@ function _dtRenderAssignment(e) { ? 'active' : ''; + // If this assignment belongs to a mergeable group, show a small + // indicator badge — the group-level "Merge" action lives in the + // banner at the top of the section to avoid N redundant buttons. + const mergeGroup = _dtFindMergeGroup(e.assignment_id); + const mergeableBadge = mergeGroup + ? ` + mergeable + ` + : ''; + const overlay = evCount > 0 ? `
${evCount.toLocaleString()} event${evCount === 1 ? '' : 's'} @@ -2056,7 +2127,7 @@ function _dtRenderAssignment(e) { ? `
${_dtEsc(e.notes)}
` : ''; - return `
+ return `
@@ -2065,7 +2136,10 @@ function _dtRenderAssignment(e) {
${start} → ${end}${dur}
- ${activeBadge} +
+ ${mergeableBadge} + ${activeBadge} +
${locLink}
${projLine} @@ -2118,18 +2192,253 @@ function _dtRenderStateChange(e) {
`; } -function renderDeploymentTimeline(entries, container) { - if (!entries.length) { - container.innerHTML = '

No deployment history yet. Assign this unit to a project location to start a deployment record.

'; +// ── Gantt chart ───────────────────────────────────────────────────────────── +// Renders all assignment windows as colored horizontal bars on an SVG +// timeline. Click a bar to scroll its detail row into view in the list +// below. Color per location, opacity reduced for closed assignments. +// "Mergeable" groups get a unifying outline overlay so they're visible at +// a glance as one logical deployment. +const _ganttColorPalette = [ + '#f48b1c', '#142a66', '#7d234d', '#0e7490', '#15803d', '#a16207', + '#9333ea', '#dc2626', '#0d9488', '#1d4ed8', '#be185d', '#65a30d', +]; +function _ganttColorFor(locId, locColorMap) { + if (locColorMap[locId]) return locColorMap[locId]; + const idx = Object.keys(locColorMap).length % _ganttColorPalette.length; + locColorMap[locId] = _ganttColorPalette[idx]; + return locColorMap[locId]; +} + +function _ganttParseDate(iso) { + if (!iso) return null; + const d = new Date(iso.replace(' ', 'T')); + return isNaN(d.getTime()) ? null : d; +} + +function _ganttFmtMonth(d) { + return d.toLocaleDateString('en-US', { month: 'short', year: '2-digit' }); +} + +function renderDeploymentGantt(entries, mergeGroups) { + const wrapper = document.getElementById('deploymentGantt'); + const svg = document.getElementById('deploymentGanttSvg'); + const legend = document.getElementById('deploymentGanttLegend'); + if (!wrapper || !svg) return; + + const assignments = (entries || []).filter(e => e.kind === 'assignment' && e.starts_at); + if (assignments.length === 0) { + wrapper.classList.add('hidden'); return; } + wrapper.classList.remove('hidden'); + + // Compute time domain. Pad the end by a few days when an active + // assignment is present so the "active" bar doesn't reach the very + // edge of the chart. + const now = new Date(); + let minDate = null, maxDate = null; + for (const a of assignments) { + const start = _ganttParseDate(a.starts_at); + const end = a.is_active ? now : (_ganttParseDate(a.ends_at) || now); + if (start && (!minDate || start < minDate)) minDate = start; + if (end && (!maxDate || end > maxDate)) maxDate = end; + } + if (!minDate || !maxDate) { wrapper.classList.add('hidden'); return; } + // Tiny padding at both ends (3% of total span). + const span = maxDate - minDate; + const pad = Math.max(span * 0.03, 24 * 3600 * 1000); // at least 1 day + minDate = new Date(minDate.getTime() - pad); + maxDate = new Date(maxDate.getTime() + pad); + + // Build a quick "which mergeGroup is this id in?" map. + const idToGroup = {}; + (mergeGroups || []).forEach((g, idx) => g.forEach(id => { idToGroup[id] = idx; })); + + // Compute SVG geometry. + const width = Math.max(svg.clientWidth || svg.parentElement.clientWidth || 800, 400); + const height = 140; + const padLeft = 8; + const padRight = 8; + const padTop = 32; // room for month labels above the bars + const padBottom = 18; // room for assignment-count axis below + const usableW = width - padLeft - padRight; + const usableH = height - padTop - padBottom; + const totalRange = maxDate - minDate; + const xFor = (d) => padLeft + (d - minDate) / totalRange * usableW; + + // Choose one-row-per-bar OR stack overlapping bars. Since same-unit + // assignments rarely overlap (only via the brief unassign/reassign + // race), a single row is usually fine. But just in case, stack with + // simple top-down packing. + const lanes = []; // each lane = [{x1, x2, ...}, ...] + function placeInLane(start, end) { + for (let i = 0; i < lanes.length; i++) { + const last = lanes[i][lanes[i].length - 1]; + if (last.x2 + 2 < start) { + lanes[i].push({ x1: start, x2: end }); + return i; + } + } + lanes.push([{ x1: start, x2: end }]); + return lanes.length - 1; + } + const placed = assignments.map(a => { + const start = _ganttParseDate(a.starts_at); + const end = a.is_active ? now : (_ganttParseDate(a.ends_at) || now); + const x1 = xFor(start); + const x2 = xFor(end); + const lane = placeInLane(x1, x2); + return { a, x1, x2, lane }; + }); + const laneCount = Math.max(lanes.length, 1); + const barH = Math.max(14, Math.min(28, Math.floor(usableH / laneCount) - 4)); + const laneSpacing = barH + 4; + + // Month gridlines + labels. Tick on the 1st of each month inside the + // domain. If span > 24mo, tick every 3 months instead. + const months = []; + let monthCursor = new Date(minDate.getFullYear(), minDate.getMonth(), 1); + const tickEveryMonths = (totalRange > 24 * 30 * 86400 * 1000) ? 3 : 1; + while (monthCursor <= maxDate) { + if (monthCursor >= minDate) months.push(new Date(monthCursor)); + monthCursor.setMonth(monthCursor.getMonth() + tickEveryMonths); + } + + // Build the SVG string. + const isDark = document.documentElement.classList.contains('dark'); + const gridColor = isDark ? '#374151' : '#e5e7eb'; + const labelColor = isDark ? '#9ca3af' : '#6b7280'; + const todayColor = '#f48b1c'; + + const locColorMap = {}; + const usedLocs = {}; + + let parts = []; + // Month gridlines. + months.forEach(m => { + const x = xFor(m); + parts.push(``); + parts.push(`${_ganttFmtMonth(m)}`); + }); + // Today marker. + if (now >= minDate && now <= maxDate) { + const x = xFor(now); + parts.push(``); + parts.push(`today`); + } + + // Bars. + placed.forEach(p => { + const a = p.a; + const color = _ganttColorFor(a.location_id || '_', locColorMap); + usedLocs[a.location_name || '(no location)'] = color; + const y = padTop + p.lane * laneSpacing; + const opacity = a.is_active ? 1.0 : 0.85; + const stroke = (a.source === 'metadata_backfill') ? '#3b82f6' : 'none'; + const strokeWidth = (a.source === 'metadata_backfill') ? 2 : 0; + + const barW = Math.max(p.x2 - p.x1, 3); + const tipDates = `${(a.starts_at || '').slice(0,10)} → ${a.is_active ? 'active' : (a.ends_at || '').slice(0,10)}`; + const tip = `${(a.location_name || '?').replace(/"/g, '"')} (${tipDates})${a.event_overlay && a.event_overlay.event_count ? ' • ' + a.event_overlay.event_count + ' events' : ''}`; + + parts.push(` + ${tip} + + ${a.is_active ? `` : ''} + `); + + // Mergeable highlight — thin dashed underline below the bar. + if (idToGroup[a.assignment_id] !== undefined) { + const uy = y + barH + 1; + parts.push(``); + } + }); + + svg.innerHTML = parts.join(''); + svg.setAttribute('viewBox', `0 0 ${width} ${height}`); + + // Build legend (one swatch per distinct location). + const legendItems = Object.entries(usedLocs).map(([name, color]) => + `${_dtEsc(name)}` + ); + if (mergeGroups && mergeGroups.length > 0) { + legendItems.push(`mergeable group`); + } + if (placed.some(p => p.a.source === 'metadata_backfill')) { + legendItems.push(`auto-backfilled`); + } + legend.innerHTML = legendItems.join(''); +} + +// Click-on-bar handler. Just scroll the matching list row into view + +// briefly flash it so the eye finds it. +function _ganttScrollTo(assignmentId) { + const target = document.querySelector(`[data-assignment-row="${assignmentId}"]`); + if (!target) return; + target.scrollIntoView({ behavior: 'smooth', block: 'center' }); + target.classList.add('ring-2', 'ring-seismo-orange'); + setTimeout(() => target.classList.remove('ring-2', 'ring-seismo-orange'), 1500); +} + +function renderDeploymentTimeline(entries, container, mergeGroups) { + if (!entries.length) { + container.innerHTML = '

No deployment history yet. Assign this unit to a project location to start a deployment record.

'; + // Hide the Gantt block too. + const g = document.getElementById('deploymentGantt'); + if (g) g.classList.add('hidden'); + return; + } + + // Render the Gantt chart first (above the list). + renderDeploymentGantt(entries, mergeGroups); + + // Build the mergeable-groups banner. Each group offers one "Merge into + // one" button. Skipped when no groups exist. + let bannerHtml = ''; + if (mergeGroups && mergeGroups.length > 0) { + const rows = mergeGroups.map(group => { + // Look up the entries to describe what we're merging. + const groupEntries = (entries || []).filter(e => + e.kind === 'assignment' && group.includes(e.assignment_id) + ); + if (groupEntries.length === 0) return ''; + const locName = groupEntries[0].location_name || 'unnamed location'; + const earliest = groupEntries.map(e => e.starts_at).filter(Boolean).sort()[0] || ''; + const latest = groupEntries.map(e => e.ends_at).filter(Boolean).sort().reverse()[0] || 'present'; + const idsJson = JSON.stringify(group).replace(/"/g, '"'); + return `
+
+ ${group.length} consecutive records at ${_dtEsc(locName)} + ${_dtFmtDate(earliest)} → ${_dtFmtDate(latest)} +
+ +
`; + }).join(''); + bannerHtml = `
+
+ + + +
+ Consecutive deployments at the same location detected. Combine them into a single record to clean up the view (notes + ingest sources are preserved). +
+
+ ${rows} +
`; + } + const html = entries.map(e => { if (e.kind === 'assignment') return _dtRenderAssignment(e); if (e.kind === 'gap') return _dtRenderGap(e); if (e.kind === 'state_change') return _dtRenderStateChange(e); return ''; }).join(''); - container.innerHTML = html; + + container.innerHTML = bannerHtml + '
' + html + '
'; } // ── SFM Events section ────────────────────────────────────────────────────── From c48c6e5bcac4f725a6dffbb14c6513d648214033 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 02:28:52 +0000 Subject: [PATCH 05/13] fix(assignments): delete_assignment used wrong column name on MonitoringSession The safety check that refuses to delete assignments with real recording history referenced MonitoringSession.start_time, but the actual column is MonitoringSession.started_at. Every DELETE call to /assignments/{id} crashed with AttributeError before doing anything. Now uses started_at correctly. Verified end-to-end on dev. Co-Authored-By: Claude Opus 4.7 --- backend/routers/project_locations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/routers/project_locations.py b/backend/routers/project_locations.py index 02003d6..7059192 100644 --- a/backend/routers/project_locations.py +++ b/backend/routers/project_locations.py @@ -874,8 +874,8 @@ async def delete_assignment( and_( MonitoringSession.location_id == assignment.location_id, MonitoringSession.unit_id == assignment.unit_id, - MonitoringSession.start_time >= window_start, - MonitoringSession.start_time <= window_end, + MonitoringSession.started_at >= window_start, + MonitoringSession.started_at <= window_end, ) ).count() From ba1f28ee53cfd7531a5f4b22ef064921510c90d6 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 03:59:38 +0000 Subject: [PATCH 06/13] fix(backfill): typeahead picks broken by JSON.stringify quote collision in onclick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inline onclick on each typeahead dropdown item was: onclick="onTypeaheadPick(event, 'cid', 'location', 'loc-id', ${JSON.stringify(m.name)})" For any name with spaces/punctuation (i.e. every real location name like "Area 1 - Loc 1 - 87 Jenks"), JSON.stringify emits double quotes around the value, which collide with the onclick attribute's own double quotes and terminate the attribute early. The dropdown rendered fine via .innerHTML, but the browser's HTML parser saw a broken attribute and never bound the click handler — clicks on dropdown items silently did nothing. Same pattern that broke the location Remove button yesterday. Same fix: move args into data-* attributes and dispatch through a tiny trampoline that reads from this.dataset. Robust against any character in project/location names. Co-Authored-By: Claude Opus 4.7 --- templates/admin/metadata_backfill.html | 31 ++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/templates/admin/metadata_backfill.html b/templates/admin/metadata_backfill.html index 84459cd..41bd18e 100644 --- a/templates/admin/metadata_backfill.html +++ b/templates/admin/metadata_backfill.html @@ -275,7 +275,13 @@ async function _fetchTypeahead(input, fieldKind) { return; } + // Args go in data-* attributes (not inline onclick) to avoid the quote + // collision when location names contain characters JSON.stringify quotes + // (e.g. anything with spaces/punctuation — basically every real name). + // _esc() escapes for HTML attribute context (entities for <>&"), then + // the browser decodes them when reading the dataset value. dropdown.innerHTML = items.map((it, idx) => { + const cid = _esc(input.dataset.clusterId); if (it.kind === 'match') { const m = it.payload; const scoreBadge = m.score >= 0.99 @@ -291,16 +297,24 @@ async function _fetchTypeahead(input, fieldKind) { } const metaLine = meta.length ? `
${meta.join(' · ')}
` : ''; return ``; } return ``; @@ -308,6 +322,19 @@ async function _fetchTypeahead(input, fieldKind) { dropdown.classList.remove('hidden'); } +// Trampoline — reads the click target's data-* attributes and forwards +// to onTypeaheadPick. Keeps the inline onclick attribute free of any +// string interpolation that could collide with HTML quoting. +function _typeaheadPickFromButton(btn) { + onTypeaheadPick( + null, + btn.dataset.cid, + btn.dataset.fieldKind, + btn.dataset.entityId || '', + btn.dataset.entityName || '' + ); +} + function onTypeaheadPick(event, clusterId, fieldKind, entityId, name) { // entityId is empty string for "create new", or a UUID for matched existing. const inputs = document.querySelectorAll(`input[data-cluster-id="${clusterId}"]`); From ad55d4ca09bbd1d4026aabf213b725c364dc80bc Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 04:10:48 +0000 Subject: [PATCH 07/13] fix(backfill): location matching over-confident on boilerplate-shared names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rapidfuzz.fuzz.WRatio inflates scores when two strings share substring tokens, even when the shared tokens are common boilerplate. For project names this is desirable (catches typos like '1-80' vs 'I-80') but for location names it produces obvious false positives: 'Area 2 - Brookville Dam - Loc 2 East' vs 'Area 1 - Loc 1 - 87 Jenks' → WRatio 85.5 (above 0.80 fuzzy threshold) These share only 'area' + 'loc' + a digit but score 85%+ because WRatio weights partial-substring overlap heavily. Operator reported the backfill tool suggesting completely unrelated locations as 86% matches. Fix: introduce `location_similarity()` — token_set_ratio + multi-digit mismatch penalty. Used for location matching everywhere; WRatio stays as the scorer for project names where its leniency is correct. The multi-digit penalty (-0.30) triggers when both strings contain 2+- digit numbers and none overlap. Catches the harder "same project, different address identifier" case: 'Area 1 - Loc 2 - 68 Jenks' vs 'Area 1 - Loc 1 - 87 Jenks' token_set_ratio = 0.91 (would still match without penalty) multi-digit tokens {68} and {87} disjoint → -0.30 → 0.61 (rejected) Single-digit tokens ('Loc 1', 'Area 2') are excluded from the penalty because they're often coincidentally shared. Updated: - backend/services/metadata_backfill.py: new location_similarity() function; _find_best_match() gains a `kind` parameter that selects scorer; cluster-match call site passes kind='location' - backend/routers/metadata_backfill.py: locations_search endpoint (the typeahead dropdown's data source) uses location_similarity instead of similarity for the same reason Verified all six test cases land correctly: - user-reported false positive: 0.85 → 0.59 (rejected) - '87 Jenks' vs '68 Jenks': 0.90 → 0.61 (rejected) - NRL-01 vs NRL-02: 0.83 → 0.53 (rejected) - 'Loc 2 - 735 Bunola' vs 'Loc 2 735 Bunola Rd': 1.00 (still matches) - punctuation-only difference: 1.00 (still matches) Co-Authored-By: Claude Opus 4.7 --- backend/routers/metadata_backfill.py | 6 ++- backend/services/metadata_backfill.py | 62 ++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/backend/routers/metadata_backfill.py b/backend/routers/metadata_backfill.py index f086020..34e0d2d 100644 --- a/backend/routers/metadata_backfill.py +++ b/backend/routers/metadata_backfill.py @@ -376,7 +376,11 @@ def locations_search( if q_norm in l_norm: scored.append((l, 1.0)) continue - score = svc.similarity(q_norm, l_norm) + # Use the location-specific scorer (token_set_ratio + multi-digit + # penalty) instead of WRatio — same reason as the cluster-match + # path: location names share too much boilerplate vocabulary for + # WRatio to discriminate reliably. + score = svc.location_similarity(q_norm, l_norm) if score >= 0.50: scored.append((l, score)) diff --git a/backend/services/metadata_backfill.py b/backend/services/metadata_backfill.py index 303328a..30e234c 100644 --- a/backend/services/metadata_backfill.py +++ b/backend/services/metadata_backfill.py @@ -162,6 +162,11 @@ def similarity(a: str, b: str) -> float: too short to fuzzy-match safely (see _MIN_FUZZY_LEN comment) AND the strings don't exact-match. This guardrails the 'one common word inside a longer phrase' false positive. + + USE FOR: project names (where typos like '1-80' vs 'I-80' should + still match). For location names use `location_similarity()` — + WRatio is too lenient on the shared boilerplate vocabulary in + location strings ('Area', 'Loc', 'Bridge', 'Dam', etc.). """ if not a or not b: return 0.0 @@ -172,6 +177,50 @@ def similarity(a: str, b: str) -> float: return rapidfuzz.fuzz.WRatio(a, b) / 100.0 +# Multi-digit penalty applied when two location names have completely +# disjoint multi-digit numeric tokens (e.g. "87 Jenks" vs "68 Jenks"). +# Single-digit numbers ("Loc 1", "Area 2") are often shared coincidentally, +# but address-style multi-digit numbers are strong identifiers — if they +# differ, the locations are usually different physical places. +_LOCATION_DIGIT_MISMATCH_PENALTY = 0.30 + + +def location_similarity(a: str, b: str) -> float: + """Stricter similarity score for location-name matching. + + Location names share so much boilerplate vocabulary ('Area', 'Loc', + 'Bridge', 'Dam') that rapidfuzz.WRatio inflates obvious mismatches. + Example: 'Area 2 - Brookville Dam - Loc 2 East' vs 'Area 1 - Loc 1 - + 87 Jenks' scores 85.5 via WRatio despite being unrelated locations. + + This scorer uses `token_set_ratio` as the base (sensitive to actual + word overlap, not just substring containment). It then applies a + multi-digit penalty: if both strings contain 2+-digit numbers and + none overlap, subtract 0.30. Catches the "same project, different + address-style identifier" case ('87 Jenks' vs '68 Jenks') that pure + token-set scoring still rates above 0.90. + + Single-digit numbers ('Loc 1', 'Area 2') are excluded from the + penalty because they're often shared boilerplate ("Loc 1" in every + project) rather than discriminating identifiers. + """ + if not a or not b: + return 0.0 + if a == b: + return 1.0 + if min(len(a), len(b)) < _MIN_FUZZY_LEN: + return 0.0 + + base = rapidfuzz.fuzz.token_set_ratio(a, b) / 100.0 + + multidigits_a = set(re.findall(r"\d{2,}", a)) + multidigits_b = set(re.findall(r"\d{2,}", b)) + if multidigits_a and multidigits_b and not (multidigits_a & multidigits_b): + base = max(0.0, base - _LOCATION_DIGIT_MISMATCH_PENALTY) + + return base + + # ── Cluster + Suggestion dataclasses ─────────────────────────────────────────── @@ -572,15 +621,24 @@ async def _scan_clusters( def _find_best_match( candidate_norm: str, candidates: list[tuple[str, str]], # (id, normalised_name) + *, + kind: str = "project", # "project" | "location" ) -> tuple[Optional[str], Optional[float], str]: """Return (best_id, best_score, classification). classification ∈ {"exact", "fuzzy", "ambiguous", "no_match"} + + The `kind` parameter selects the scorer. Project matching uses + rapidfuzz.WRatio (lenient — catches typos like '1-80' vs 'I-80'). + Location matching uses `location_similarity` (stricter — catches + boilerplate-shared-but-actually-different strings like 'Loc 2 - 68 + Jenks' vs 'Loc 1 - 87 Jenks'). """ if not candidate_norm or not candidates: return None, None, "no_match" - scored = [(cid, similarity(candidate_norm, cnorm)) for cid, cnorm in candidates] + scorer = location_similarity if kind == "location" else similarity + scored = [(cid, scorer(candidate_norm, cnorm)) for cid, cnorm in candidates] scored.sort(key=lambda x: x[1], reverse=True) best_id, best_score = scored[0] @@ -725,7 +783,7 @@ def _build_suggestion(db: Session, cluster: Cluster) -> Suggestion: ) location_candidates = [(l.id, _normalise(l.name)) for l in location_candidates_objs] if cluster.location_norm: - loc_id, loc_score, loc_match = _find_best_match(cluster.location_norm, location_candidates) + loc_id, loc_score, loc_match = _find_best_match(cluster.location_norm, location_candidates, kind="location") else: loc_id, loc_score, loc_match = None, None, "create_new" else: From 295f9637b3d3979f6c2b01e6ec8f80c8684b6dd4 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 04:54:33 +0000 Subject: [PATCH 08/13] fix(merge-project): dropdown unclickable + modal too short to show it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in the project-merge modal: 1. Dropdown options had the same JSON.stringify quote-collision in their inline onclick that broke the location Remove button and the metadata-backfill typeahead earlier this week: onclick="onMergePickTarget('${id}', ${JSON.stringify(m.name)})" For 'I-80 Area 1' that renders as onclick="...(\"I-80 Area 1\")" — the inner double quotes terminate the onclick attribute early, and the browser never binds the click handler. Operator clicked items in the dropdown and nothing happened. Fixed via data-target-id / data-target-name attributes and a _mergePickFromButton(btn) trampoline. 2. Modal body had `flex-1 overflow-y-auto` with no min-height, so the container shrunk tight around the input. When the typeahead dropdown appeared below the input it got clipped by the body's overflow and the operator had to scroll inside the modal to see the options. Fixed by adding min-height: 480px to the modal container + min-h- [320px] on the body so there's always room for the dropdown + the preview pane that appears below after a target is picked. Co-Authored-By: Claude Opus 4.7 --- .../partials/projects/project_header.html | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/templates/partials/projects/project_header.html b/templates/partials/projects/project_header.html index 6aeff79..9e49749 100644 --- a/templates/partials/projects/project_header.html +++ b/templates/partials/projects/project_header.html @@ -87,9 +87,14 @@
- + -
-

Upcoming Actions

- {% if upcoming_actions %} -
- {% for action in upcoming_actions %} -
-

{{ action.action_type }}

-

{{ action.scheduled_time|local_datetime }} {{ timezone_abbr() }}

- {% if action.description %} -

{{ action.description }}

- {% endif %} -
- {% endfor %} -
- {% else %} -

No scheduled actions.

- {% endif %} + +
+
+

Location Map

+ {% if upcoming_actions %} + + {{ upcoming_actions | length }} upcoming action{{ '' if upcoming_actions | length == 1 else 's' }} → + + {% endif %} +
+
+ +
+ + From f063383e61db166297add494a1f98f7b57767944 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 06:22:08 +0000 Subject: [PATCH 12/13] fix(project-overview): Leaflet map z-index leak covered modals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The location map's tile-pane (z-index 200), marker-pane (600), and control-pane (800) outranked the page modals' z-50 because the map's container didn't establish its own stacking context. Modals opened over the page rendered BEHIND the map tiles (visible in the Edit Location, Assign, Remove, etc. modals — anywhere overlapping the right column). Fixed with `isolation: isolate` on the map container. That CSS property forces a new stacking context without needing to rewrite Leaflet's internal z-indexes, so all the map's panes stay contained inside the card and z-50 modals correctly render on top. Co-Authored-By: Claude Opus 4.7 --- templates/partials/projects/project_dashboard.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/partials/projects/project_dashboard.html b/templates/partials/projects/project_dashboard.html index 7d9387b..5b859e2 100644 --- a/templates/partials/projects/project_dashboard.html +++ b/templates/partials/projects/project_dashboard.html @@ -95,8 +95,12 @@ {% endif %} +
+ style="height: 320px; background: rgba(0,0,0,0.05); isolation: isolate;"> From ba9cdb43473a17845d1e4aae6d1e5915f72e1b93 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 15 May 2026 06:27:38 +0000 Subject: [PATCH 13/13] chore(release): bump to v0.11.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-facing polish release on top of v0.10.0's SFM integration: - Soft-remove monitoring locations (preserves history) - Per-unit deployment Gantt chart - Merge consecutive same-location assignments - Delete assignment for mis-clicks (with safety check) - Drag-to-reorder location cards (HTML5 native) - Three-dot kebab menu replaces inline pill buttons - Event count on vibration cards (instead of "Sessions: 0") - Project overview location map (replaces Upcoming Actions) - Stricter backfill location matcher (no false positives on boilerplate-shared names like "Area 1" vs "Area 2") - 3× JSON.stringify quote-collision bug fixes (Remove button, backfill typeahead, project-merge dropdown) - Merge-project modal min-height fix - Leaflet stacking-context fix (no more map-over-modal) - delete_assignment column name fix (start_time → started_at) Migrations added this release: - migrate_add_location_removed.py - migrate_add_location_sort_order.py Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 21 +++++++++++++++++--- backend/main.py | 2 +- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71d3cf..6c9aa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,57 @@ All notable changes to Terra-View will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.11.0] - 2026-05-15 + +Operator-facing polish release. All work builds on the v0.10.0 SFM integration foundation — this release is about making the day-to-day workflows (managing locations, cleaning up bad attributions, browsing deployments) faster and less error-prone. + +### Added +- **Soft-remove monitoring locations** (`POST /api/projects/{p}/locations/{l}/remove` + `/restore`): mark a location as no longer actively monitored without destroying historical events. Cascade-closes active unit assignments and cancels pending scheduled actions at the location. Restored locations rejoin the active list (assignments are NOT auto-reopened — operator creates new ones if resuming). Project page splits locations into Active and Removed sections; removed cards are greyed out, badged with the removal date + reason, and offer a Restore button. +- **Per-unit deployment Gantt chart** above the existing Deployment Timeline list on every seismograph unit detail page. Plain-SVG rendering, color per location, today marker (orange dashed line), reduced-opacity bars for closed assignments, blue outlines on metadata-backfilled assignments, dashed blue underlines marking mergeable groups. Click a bar to scroll the matching list row into view with a flash highlight. +- **Merge consecutive same-location assignments** (`POST /api/projects/{p}/assignments/merge`): operators often end up with several rows representing one continuous deployment (after remove/restore, or metadata-backfill adjacent to a manual record). Now auto-detected and surfaceable in the timeline header — one click combines them into a single record. Preserves the earliest record's notes + ingest source, writes an `assignment_merged` audit entry, deletes the others. +- **Delete assignment for mis-clicks** (`DELETE /api/projects/{p}/assignments/{a}`): hard-deletes a bogus assignment row that was never a real deployment. Trash icon in each row of the location's Deployment History panel. Refuses the delete if any `MonitoringSession` exists in the assignment's window — those should go through Unassign instead, which preserves audit history. Writes an `assignment_deleted` UnitHistory row. +- **Drag-to-reorder location cards**: each active card has a six-dot drag handle on the left. Drag/drop reorders the DOM and persists via `POST /api/projects/{p}/locations/reorder`. Implementation uses native HTML5 drag-and-drop (no library). New locations land at the end (`sort_order = max + 1`); removed locations stay sorted by removal date. +- **Three-dot kebab menu on location cards**: replaces the four inline pill buttons (Unassign / Edit / Remove / Delete) with a single ⋮ menu. Click ⋮ to open; click outside or Escape to close; only one menu open at a time. +- **Event count on vibration location cards**: vibration cards now show "{N} events" sourced from SFM via concurrent fan-out, instead of "Sessions: 0" (sessions don't exist under the watcher-forward pipeline). Sound locations still show session counts. +- **Project overview location map**: right column of every project's overview replaces the lightly-used Upcoming Actions panel with a Leaflet map. One pin per active monitoring location (parsed from the `coordinates` field). Click pin → scrolls + flashes the matching card. Tooltip on hover. Locations without coordinates surface as an inline hint below the map. If the project has pending scheduled actions, a small "{N} upcoming actions →" link appears in the card header that switches to the Schedules tab. + +### Changed +- **Backfill location fuzzy matcher is now stricter**: `rapidfuzz.WRatio` was over-confident on location names because their shared boilerplate vocabulary ("Area", "Loc", numbers) inflated scores. Example false positive that prompted the change: `"Area 2 - Brookville Dam - Loc 2 East"` vs `"Area 1 - Loc 1 - 87 Jenks"` scored 86% via WRatio. Now uses `token_set_ratio` as the base scorer plus a 0.30 penalty when the two strings have disjoint multi-digit numeric tokens. Catches the "same project, different address number" case (`"68 Jenks"` vs `"87 Jenks"`) that pure token-set scoring still rated above 0.90. Project matching keeps WRatio (where its leniency is desirable for typos like `1-80` vs `I-80`). + +### Fixed +- **Three separate JSON.stringify quote-collision bugs**: any inline `onclick="...({...} | tojson)"` or `onclick="...${JSON.stringify(x)}..."` where `x` contained any character that JSON quotes (essentially every real-world string) broke the HTML attribute and silently un-bound the click handler. Surfaced in three places this release; all fixed by switching to `data-*` attributes plus a trampoline function reading from `this.dataset`: + - **Location Remove button** on the project page + - **Metadata-backfill typeahead dropdown** (existing project + location pickers) + - **Project-merge typeahead dropdown** (in the per-project header) +- **Project-merge modal too short to show typeahead options without scrolling**: modal body's `flex-1 overflow-y-auto` collapsed tight; added `min-height: 480px` to the modal container + `min-h-[320px]` to the body so the dropdown always has room. +- **Project location map covered modals**: Leaflet's internal panes carry z-indexes 200–800 by default and the map container didn't establish a stacking context, so those z-indexes leaked into the root and outranked modals' `z-50`. Fixed by adding `isolation: isolate` to the map container. +- **`delete_assignment` crashed with `AttributeError`**: the safety check queried `MonitoringSession.start_time` but the actual column is `started_at`. Every DELETE call to `/assignments/{id}` failed with 500 before doing anything. + +### Migration Notes +Run on each database before deploying. Both migrations are idempotent and non-destructive. + +```bash +docker exec terra-view-terra-view-1 python3 /app/backend/migrate_add_location_removed.py +docker exec terra-view-terra-view-1 python3 /app/backend/migrate_add_location_sort_order.py +``` + +Or sweep all migrations at once (safe — already-applied ones no-op): + +```bash +for f in backend/migrate_*.py; do + docker exec terra-view-terra-view-1 python3 "/app/backend/$(basename $f)" +done +``` + +New columns added this release: +- `monitoring_locations.removed_at` (DATETIME, nullable) — NULL means active +- `monitoring_locations.removal_reason` (TEXT, nullable) +- `monitoring_locations.sort_order` (INTEGER, default 0) — seeded to alphabetical-index per project on first migration + +**Deploy order matters**: migrations must run BEFORE the new code is up, otherwise the running app will throw 500s on the unrecognized columns. Idempotent migrations make this recoverable but it's better avoided — the v0.11.0 deploy on prod hit this exact window after the v0.10.0 release. + +--- + ## [0.10.0] - 2026-05-14 This release brings terra-view onto the SFM (Seismograph Field Module) event pipeline. Triggered events forwarded by series3-watcher now land in SFM, and terra-view reads from that store as the authoritative source for vibration data. The watcher heartbeat is preserved as a transparent fallback signal. diff --git a/README.md b/README.md index 9ab2bc2..4a74e14 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Terra-View v0.10.0 +# Terra-View v0.11.0 Backend API and HTMX-powered web interface for managing a mixed fleet of seismographs and field modems. Track deployments, monitor health in real time, merge roster intent with incoming telemetry, and control your fleet through a unified database and dashboard. ## Features @@ -496,6 +496,19 @@ docker compose down -v ## Release Highlights +### v0.11.0 — 2026-05-15 +- **Soft-Remove Monitoring Locations**: Mark a location as no longer actively monitored without destroying history. Closes active unit assignments and cancels pending scheduled actions; historical events stay attributed. Restore brings it back. Surfaces as a Removed Locations collapsed section on the project page. +- **Per-Unit Deployment Gantt**: Visual timeline above the deployment history list on each unit detail page. Color-coded bars per location, today marker, mergeable-group dashed underlines, click a bar to scroll its detail row into view. +- **Merge Consecutive Deployments**: Auto-detects runs of same-location assignments within a 7-day gap and offers a one-click "Merge into one" button. Preserves notes, ingest source, and writes an `assignment_merged` audit entry. +- **Delete Assignment for Mis-Clicks**: Trash icon on each row of the location's Deployment History panel. Hard-deletes the assignment with a safety check that refuses if real MonitoringSessions sit inside the window (those should go through Unassign instead). +- **Drag-to-Reorder Location Cards**: Six-dot drag handle on each card; drop order persists via a new `/locations/reorder` endpoint. Removed locations stay sorted by removal date (their order is historical). +- **Three-Dot Kebab Menu**: Replaces the inline Unassign / Edit / Remove / Delete pill row with a single ⋮ menu. Much cleaner card layout, especially for projects with many locations. +- **Event Count on Vibration Cards**: Vibration locations now show "{N} events" instead of "Sessions: 0" (sessions don't exist under the watcher-forward pipeline). Sound locations are unchanged. +- **Project Location Map**: Right column of the project overview is now a Leaflet map with a pin per location. Click pin → scrolls + flashes the matching card. Replaces the lightly-used Upcoming Actions panel (still discoverable via a link to the Schedules tab when actions exist). +- **Stricter Location Fuzzy Matching**: Metadata-backfill no longer suggests obviously-wrong matches. WRatio was over-confident on location names ("Area 2 - Brookville Dam - Loc 2" vs "Area 1 - Loc 1 - 87 Jenks" used to score 86%); now uses `token_set_ratio` + a multi-digit penalty so disjoint address numbers correctly demote the score. +- **Fixed: Multiple typeahead dropdowns weren't clickable**: Same JSON.stringify quote-collision bug surfaced in three places (location Remove button, backfill typeahead, project-merge dropdown). All three fixed by switching to `data-*` attributes + trampoline functions. +- **Fixed: Merge-project modal had to be scrolled to see options**: Modal body's `flex-1 overflow-y-auto` collapsed too tight; added `min-height` so the dropdown has room to render below the input. + ### v0.10.0 — 2026-05-14 - **SFM Integration**: terra-view now consumes events from the SFM (Seismograph Field Module) backend in real time, with a fleet-wide events page at `/sfm`, per-unit attribution against project assignment windows, and a project-level vibration roll-up that uses SFM data as the single source of truth. - **SFM-Primary Seismograph Status**: Deployed seismograph status (OK/Pending/Missing) now flows from SFM event forwards first; the watcher heartbeat stays as a transparent backup. Each unit's active table row shows a small `SFM` or `HB` badge so operators can see at a glance which signal is currently driving the status. @@ -612,9 +625,11 @@ MIT ## Version -**Current: 0.10.0** — SFM integration, SFM-primary seismograph status, dashboard rework, sortable events tables, event detail modal, /admin/sfm + /admin/slmm diagnostic pages, Tools workflow hub (2026-05-14) +**Current: 0.11.0** — Soft-remove locations, per-unit Gantt, merge/delete assignments, drag-to-reorder, three-dot kebab menu, event count on vibration cards, project location map, stricter backfill fuzzy match, modal/typeahead bug fixes (2026-05-15) -Previous: 0.9.4 — Modular project types, deleted project management, swap modal search, roster auto-refresh fix (2026-04-06) +Previous: 0.10.0 — SFM integration, SFM-primary seismograph status, dashboard rework, sortable events tables, event detail modal, /admin/sfm + /admin/slmm diagnostic pages, Tools workflow hub (2026-05-14) + +0.9.4 — Modular project types, deleted project management, swap modal search, roster auto-refresh fix (2026-04-06) 0.9.3 — Monitoring session detail page, configurable period windows, vibration project redesign, modem assignment on locations (2026-03-28) diff --git a/backend/main.py b/backend/main.py index c7d39d7..f88e18c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -30,7 +30,7 @@ Base.metadata.create_all(bind=engine) ENVIRONMENT = os.getenv("ENVIRONMENT", "production") # Initialize FastAPI app -VERSION = "0.10.0" +VERSION = "0.11.0" if ENVIRONMENT == "development": _build = os.getenv("BUILD_NUMBER", "0") if _build and _build != "0":