From abcfba179f09e1796a467b726e869098d38a8780 Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:12:15 +0000 Subject: [PATCH 1/8] refactor(reports): funnel scheduler stop/download/cycle through one ingest _execute_stop and _execute_download no longer hand-roll ZIP extraction; all three actions now call a shared _ingest_and_link helper (ingest via ingest_nrl_zip, link the unit, drop the empty placeholder session). Every capture path produces the same clean, .rnh-parsed, percentile-aware, deduped, Leq-only session. _execute_download previously created no session at all (TODO); it now ingests like the others. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/services/scheduler.py | 215 ++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 99 deletions(-) diff --git a/backend/services/scheduler.py b/backend/services/scheduler.py index e1ce32e..fc16ee9 100644 --- a/backend/services/scheduler.py +++ b/backend/services/scheduler.py @@ -309,18 +309,11 @@ class SchedulerService: 2. Enable FTP 3. Download measurement folder to SLMM local storage - After stop_cycle, if download succeeded, this method fetches the ZIP - from SLMM and extracts it into Terra-View's project directory, creating - DataFile records for each file. + After stop_cycle, if download succeeded, this method ingests the folder + into Terra-View through the shared NRL ingest (same path as cycle and the + manual SD-card upload) so the resulting session is Leq-only, has its + `.rnh` parsed (percentile slot map + weightings), and is deduped. """ - import hashlib - import io - import os - import zipfile - import httpx - from pathlib import Path - from backend.models import DataFile - # Parse notes for download preference include_download = True try: @@ -365,79 +358,39 @@ class SchedulerService: db.commit() - # If SLMM downloaded the folder successfully, fetch the ZIP from SLMM - # and extract it into Terra-View's project directory, creating DataFile records - files_created = 0 - if include_download and cycle_response.get("download_success") and active_session: + # If SLMM downloaded the folder successfully, ingest it into Terra-View + # through the shared NRL ingest (the same path cycle and the manual SD + # upload use): keeps only the .rnh + Leq .rnd, parses the header + # (percentile slot map + weightings), dedups, and links the unit. The + # transient "recording" marker session is dropped in favour of the clean + # ingested row. (Replaces the old inline unzip that stored every file — + # incl. the 1-second _Lp_ data — without parsing the .rnh.) + ingest_result = None + ingested_session_id = None + if (include_download and cycle_response.get("download_success") + and active_session and action.device_type == "slm"): folder_name = cycle_response.get("downloaded_folder") # e.g. "Auto_0058" - remote_path = f"/NL-43/{folder_name}" - - try: - SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") - async with httpx.AsyncClient(timeout=600.0) as client: - zip_response = await client.post( - f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download-folder", - json={"remote_path": remote_path} + if folder_name: + try: + ingest_result = await self._ingest_and_link( + db, + location_id=action.location_id, + unit_id=unit_id, + folder_name=folder_name, + placeholder_session=active_session, ) - - if zip_response.is_success and len(zip_response.content) > 22: - base_dir = Path(f"data/Projects/{action.project_id}/{active_session.id}/{folder_name}") - base_dir.mkdir(parents=True, exist_ok=True) - - file_type_map = { - '.wav': 'audio', '.mp3': 'audio', - '.csv': 'data', '.txt': 'data', '.json': 'data', '.dat': 'data', - '.rnd': 'data', '.rnh': 'data', - '.log': 'log', - '.zip': 'archive', - '.jpg': 'image', '.jpeg': 'image', '.png': 'image', - '.pdf': 'document', - } - - with zipfile.ZipFile(io.BytesIO(zip_response.content)) as zf: - for zip_info in zf.filelist: - if zip_info.is_dir(): - continue - file_data = zf.read(zip_info.filename) - file_path = base_dir / zip_info.filename - file_path.parent.mkdir(parents=True, exist_ok=True) - with open(file_path, 'wb') as f: - f.write(file_data) - checksum = hashlib.sha256(file_data).hexdigest() - ext = os.path.splitext(zip_info.filename)[1].lower() - data_file = DataFile( - id=str(uuid.uuid4()), - session_id=active_session.id, - file_path=str(file_path.relative_to("data")), - file_type=file_type_map.get(ext, 'data'), - file_size_bytes=len(file_data), - downloaded_at=datetime.utcnow(), - checksum=checksum, - file_metadata=json.dumps({ - "source": "stop_cycle", - "remote_path": remote_path, - "unit_id": unit_id, - "folder_name": folder_name, - "relative_path": zip_info.filename, - }), - ) - db.add(data_file) - files_created += 1 - - db.commit() - logger.info(f"Created {files_created} DataFile records for session {active_session.id} from {folder_name}") - else: - logger.warning(f"ZIP from SLMM for {folder_name} was empty or failed, skipping DataFile creation") - - except Exception as e: - logger.error(f"Failed to extract ZIP and create DataFile records for {folder_name}: {e}") - # Don't fail the stop action — the device was stopped successfully + ingested_session_id = ingest_result.get("session_id") + logger.info(f"[STOP] Ingested {folder_name}: {ingest_result}") + except Exception as e: + logger.error(f"Failed to ingest {folder_name} on stop: {e}", exc_info=True) + # Don't fail the stop action — the device was stopped successfully + ingest_result = {"success": False, "error": str(e)} return { "status": "stopped", - "session_id": active_session.id if active_session else None, + "session_id": ingested_session_id or (active_session.id if active_session else None), "cycle_response": cycle_response, - "files_created": files_created, + "ingest": ingest_result, } async def _execute_download( @@ -490,12 +443,38 @@ class SchedulerService: files=None, # Download all files in current measurement folder ) - # TODO: Create DataFile records for downloaded files + # Ingest the downloaded folder into Terra-View via the shared NRL ingest + # (same path as stop/cycle): clean Leq-only session, .rnh parsed + # (percentiles + weightings), deduped, unit linked. No placeholder + # session here — a standalone download isn't tied to a "recording" marker. + ingest_result = None + if action.device_type == "slm": + folder_name = (response or {}).get("folder_name") + if not folder_name: + try: + folder_name = f"Auto_{int((response or {}).get('index_number')):04d}" + except (ValueError, TypeError): + folder_name = None + if not folder_name: + ingest_result = {"success": False, "error": "no folder_name/index_number from download"} + else: + try: + ingest_result = await self._ingest_and_link( + db, + location_id=action.location_id, + unit_id=unit_id, + folder_name=folder_name, + ) + logger.info(f"[DOWNLOAD] Ingested {folder_name}: {ingest_result}") + except Exception as e: + logger.error(f"Failed to ingest {folder_name} on download: {e}", exc_info=True) + ingest_result = {"success": False, "error": str(e)} return { "status": "downloaded", "destination_path": destination_path, "device_response": response, + "ingest": ingest_result, } async def _execute_cycle( @@ -650,27 +629,17 @@ class SchedulerService: else: folder_name = f"Auto_{idx:04d}" try: - ing = await self._ingest_cycle_folder(db, action.location_id, unit_id, folder_name) + ing = await self._ingest_and_link( + db, + location_id=action.location_id, + unit_id=unit_id, + folder_name=folder_name, + placeholder_session=active_session, + ) result["steps"]["ingest"] = ing - db.commit() - if ing.get("success"): - from backend.models import DataFile - sid = ing.get("session_id") - # ingest_nrl_zip leaves unit_id None — tie the data session to the - # unit that recorded it so it stays linked after we drop the placeholder. - if sid: - s = db.query(MonitoringSession).filter_by(id=sid).first() - if s and not s.unit_id: - s.unit_id = unit_id - db.commit() - # The just-closed "recording" session was only a marker; its data now - # lives in the ingested (unit-linked) session. Drop the empty placeholder - # and repoint old_session_id at the real row. - if active_session and db.query(DataFile).filter_by(session_id=active_session.id).count() == 0: - if sid: - result["old_session_id"] = sid - db.delete(active_session) - db.commit() + # The marker session was dropped; repoint old_session_id at the real row. + if ing.get("placeholder_dropped") and ing.get("session_id"): + result["old_session_id"] = ing["session_id"] logger.info(f"[CYCLE] Ingested {folder_name}: {ing}") except Exception as e: logger.error(f"[CYCLE] Ingest failed for {folder_name}: {e}", exc_info=True) @@ -790,6 +759,54 @@ class SchedulerService: except IngestError as e: return {"success": False, "error": str(e)} + async def _ingest_and_link( + self, + db, + *, + location_id: str, + unit_id: str, + folder_name: str, + placeholder_session=None, + ) -> dict: + """Ingest a just-finished Auto_#### folder and tie it to the unit. + + This is the ONE ingest path that stop / cycle / download all funnel + through, so every route produces the same clean session: Leq-only, + `.rnh` parsed (percentile slot map + weightings captured), deduped. + + Steps: + 1. Fetch + ingest the folder via the shared NRL ingest + (`_ingest_cycle_folder` → `ingest_nrl_zip`). + 2. `ingest_nrl_zip` leaves `unit_id` None — link it to the unit that + recorded the data so the session stays attributed. + 3. If a `placeholder_session` (the transient "recording" marker) was + passed and it never accumulated DataFiles of its own, drop it — its + data now lives in the ingested, unit-linked session. + + Returns the ingest result dict (with `success`); adds `placeholder_dropped` + when step 3 removed the marker. On ingest failure the placeholder is + left untouched. + """ + ing = await self._ingest_cycle_folder(db, location_id, unit_id, folder_name) + if not ing.get("success"): + return ing + + sid = ing.get("session_id") + if sid: + s = db.query(MonitoringSession).filter_by(id=sid).first() + if s and not s.unit_id: + s.unit_id = unit_id + db.commit() + + if placeholder_session is not None and sid: + from backend.models import DataFile + if db.query(DataFile).filter_by(session_id=placeholder_session.id).count() == 0: + db.delete(placeholder_session) + db.commit() + ing["placeholder_dropped"] = True + + return ing + # ======================================================================== # Recurring Schedule Generation # ======================================================================== -- 2.52.0 From 2ecf1f54d5c3f55a5554b0b47ee71e283dd7cb90 Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:12:15 +0000 Subject: [PATCH 2/8] fix(reports): derive session recording window from the Leq rows The NL-43 .rnh carries no measurement timestamps, so _ingest_file_entries was stamping every session with utcnow() and no duration. Derive started_at/stopped_at/duration from the Leq .rnd 'Start Time' column when the header lacks them (interval from the .rnh, else inferred from row spacing). Adds an optional unit_id so callers that know the recording unit attribute the session at creation, and returns duration_seconds. Side effect: NL-43 dedupe now works (it keyed on a previously-empty start_time_str). Affects all ingest paths: manual upload, FTP cycle, stop, download, and manual FTP download. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/routers/project_locations.py | 85 ++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/backend/routers/project_locations.py b/backend/routers/project_locations.py index b1c38f8..1074a5a 100644 --- a/backend/routers/project_locations.py +++ b/backend/routers/project_locations.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Request, Depends, HTTPException, Query from fastapi.responses import HTMLResponse, JSONResponse from sqlalchemy.orm import Session from sqlalchemy import and_, or_ -from datetime import datetime +from datetime import datetime, timedelta from zoneinfo import ZoneInfo from typing import Optional import uuid @@ -1753,6 +1753,48 @@ def _classify_file(filename: str) -> str: return "data" +def _rnd_interval_seconds(s: Optional[str]) -> Optional[int]: + """Parse an NL-43 interval string ('15m' / '1s' / '1h') into seconds.""" + import re + m = re.match(r"\s*(\d+)\s*([smh])", (s or "").strip().lower()) + if not m: + return None + return int(m.group(1)) * {"s": 1, "m": 60, "h": 3600}[m.group(2)] + + +def _leq_window_local(leq_bytes: bytes): + """Recording window from a Leq .rnd's 'Start Time' column (meter-local time). + + Returns (first_start, last_start, row_count, inferred_interval_seconds). + This is the source of truth for the recording window on NL-43 units, whose + .rnh carries no measurement timestamps. Reuses the report's AU2 normaliser + so NL-43 and AU2 files parse identically. + """ + import csv as _csv + from backend.routers.projects import _normalize_rnd_rows # lazy: avoid import cycle + try: + text = leq_bytes.decode("utf-8", errors="replace") + rows = list(_csv.DictReader(io.StringIO(text))) + except Exception: + return None, None, 0, None + try: + rows, _ = _normalize_rnd_rows(rows) + except Exception: + pass + times = [] + for r in rows: + v = (r.get("Start Time") or "").strip() + try: + times.append(datetime.strptime(v, "%Y/%m/%d %H:%M:%S")) + except (ValueError, TypeError): + continue + if not times: + return None, None, 0, None + times.sort() + inferred = int((times[1] - times[0]).total_seconds()) if len(times) >= 2 else None + return times[0], times[-1], len(times), inferred + + def _is_wanted_nrl_file(fname: str) -> bool: """Keep only the files an NRL ingest cares about: .rnh metadata + the averaged Leq .rnd. Drops the 1-second _Lp_ files and everything else. @@ -1832,6 +1874,7 @@ def _ingest_file_entries( *, source: str = "manual_upload", dedupe: bool = False, + unit_id: Optional[str] = None, ) -> dict: """Core NRL ingest, shared by the HTTP upload and the programmatic FTP pull. @@ -1840,6 +1883,10 @@ def _ingest_file_entries( location's project. Metric-agnostic: the full Leq file is written to disk and every column preserved; metric selection happens in the report layer. + `unit_id` attributes the session to the recording unit when the caller knows + it (manual FTP download / SD upload from a known unit). Left None for paths + that link the unit afterwards (the scheduler's `_ingest_and_link`). + Raises IngestError if no usable files are present. """ # --- Filter to the files we keep (.rnh + Leq .rnd) --- @@ -1872,6 +1919,32 @@ def _ingest_file_entries( index_number = rnh_meta.get("index_number", "") start_time_str = rnh_meta.get("start_time_str", "") + # The NL-43 .rnh has NO measurement timestamps — the real recording window + # lives in the Leq .rnd's "Start Time" column. Whenever the header didn't + # give us a start (and/or stop), derive it from the Leq rows so the session + # gets the true window + duration (and a stable start_time_str for dedupe). + if not start_time_str or stopped_at_local is None: + leq_entry = next( + ((f, b) for f, b in file_entries + if f.lower().endswith(".rnd") and ("_leq_" in f.lower() or f.lower().startswith("au2_"))), + None, + ) + if leq_entry is not None: + first_dt, last_dt, _n, inferred = _leq_window_local(leq_entry[1]) + interval_s = _rnd_interval_seconds(rnh_meta.get("leq_interval")) or inferred or 0 + if first_dt and not start_time_str: + started_at_local = first_dt + start_time_str = first_dt.strftime("%Y/%m/%d %H:%M:%S") + if last_dt and stopped_at_local is None: + stopped_at_local = last_dt + timedelta(seconds=interval_s) + # Recompute UTC + duration from the resolved window. + started_at = local_to_utc(started_at_local) + stopped_at = local_to_utc(stopped_at_local) if stopped_at_local else None + duration_seconds = ( + int((stopped_at - started_at).total_seconds()) + if (started_at and stopped_at) else duration_seconds + ) + # --- Dedupe: skip if this exact measurement is already ingested --- if dedupe: existing = _find_existing_session(db, location.id, store_name, started_at, start_time_str) @@ -1887,6 +1960,7 @@ def _ingest_file_entries( "store_name": store_name, "started_at": started_at.isoformat() if started_at else None, "stopped_at": stopped_at.isoformat() if stopped_at else None, + "duration_seconds": duration_seconds, } # --- Create MonitoringSession (local times drive period/label) --- @@ -1901,7 +1975,7 @@ def _ingest_file_entries( id=session_id, project_id=location.project_id, location_id=location.id, - unit_id=None, + unit_id=unit_id, session_type="sound", started_at=started_at, stopped_at=stopped_at, @@ -1976,6 +2050,7 @@ def _ingest_file_entries( "store_name": store_name, "started_at": started_at.isoformat() if started_at else None, "stopped_at": stopped_at.isoformat() if stopped_at else None, + "duration_seconds": duration_seconds, } @@ -1986,13 +2061,15 @@ def ingest_nrl_zip( *, source: str = "ftp_pull", dedupe: bool = True, + unit_id: Optional[str] = None, ) -> dict: """Programmatically ingest an Auto_#### ZIP (e.g. a scheduled FTP pull). Extracts the ZIP (flattening any nested Auto_Leq/Auto_Lp_ folders), keeps the .rnh + Leq .rnd, parses the header, and creates a MonitoringSession + DataFile rows for `location_id`. Defaults to dedupe=True so repeated daily - pulls of the same closed folder don't create duplicate sessions. + pulls of the same closed folder don't create duplicate sessions. Pass + `unit_id` to attribute the session to the recording unit at creation. Returns the same dict shape as the HTTP upload, plus a `deduped` flag. Raises IngestError on a bad ZIP, no usable files, or unknown location. @@ -2014,7 +2091,7 @@ def ingest_nrl_zip( except zipfile.BadZipFile: raise IngestError("Downloaded data is not a valid ZIP archive.") - return _ingest_file_entries(location, file_entries, db, source=source, dedupe=dedupe) + return _ingest_file_entries(location, file_entries, db, source=source, dedupe=dedupe, unit_id=unit_id) @router.post("/nrl/{location_id}/upload-data") -- 2.52.0 From 7716a4b51d9018921d4974b30c4c972231a25d2f Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:12:15 +0000 Subject: [PATCH 3/8] feat(reports): manual FTP "Download & Save" saves a parsed session ftp-download-folder-to-server and ftp-download-to-server now route NRL data through the shared ingest (ingest_nrl_zip / _ingest_file_entries) instead of hand-rolling DataFile rows on a now/zero-duration session. Folder save requires the unit be assigned to a location; non-NRL single files keep the generic save path. The FTP browser popup now reports how long the measurement ran. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/routers/projects.py | 408 +++++++------------ templates/partials/projects/ftp_browser.html | 24 +- 2 files changed, 171 insertions(+), 261 deletions(-) diff --git a/backend/routers/projects.py b/backend/routers/projects.py index 3c61236..ea73d52 100644 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -1638,9 +1638,13 @@ async def ftp_download_to_server( db: Session = Depends(get_db), ): """ - Download a file from an SLM to the server via FTP. - Creates a DataFile record and stores the file in data/Projects/{project_id}/ - Sound Monitoring projects only. + Download a single file from an SLM to the server via FTP. + + NRL measurement files (.rnh / _Leq_ .rnd) are routed through the shared NRL + ingest so the session is parsed and attributed to the unit (a lone .rnh still + yields the real recording window + duration). Any other file type — or a + unit with no location — falls back to a generic stored DataFile, preserving + the original behaviour. Sound Monitoring projects only. """ import httpx import os @@ -1658,7 +1662,55 @@ async def ftp_download_to_server( if not unit_id or not remote_path: raise HTTPException(status_code=400, detail="Missing unit_id or remote_path") - # Get or create active session for this location/unit + filename = os.path.basename(remote_path) + + # Download the file from SLMM + SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") + try: + async with httpx.AsyncClient(timeout=300.0) as client: + response = await client.post( + f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download", + json={"remote_path": remote_path} + ) + except httpx.TimeoutException: + raise HTTPException(status_code=504, detail="Timeout downloading file from SLM") + except Exception as e: + logger.error(f"Error reaching SLMM for file download: {e}") + raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}") + + if not response.is_success: + raise HTTPException( + status_code=response.status_code, + detail=f"Failed to download from SLMM: {response.text}", + ) + file_content = response.content + + # NRL measurement file + known location → shared ingest (parsed + attributed). + from backend.routers.project_locations import ( + _ingest_file_entries, IngestError, _is_wanted_nrl_file, + ) + if location_id and _is_wanted_nrl_file(filename): + location = db.query(MonitoringLocation).filter_by(id=location_id).first() + if location: + try: + result = _ingest_file_entries( + location, [(filename, file_content)], db, + source="ftp_manual", dedupe=False, unit_id=unit_id, + ) + except IngestError as e: + raise HTTPException(status_code=400, detail=str(e)) + return { + "success": True, + "message": f"Imported {filename} as NRL measurement data", + "ingested": True, + "session_id": result["session_id"], + "file_size": len(file_content), + "started_at": result["started_at"], + "stopped_at": result["stopped_at"], + "duration_seconds": result["duration_seconds"], + } + + # --- Generic path: any other file type (or no location) — store as-is --- session = db.query(MonitoringSession).filter( and_( MonitoringSession.project_id == project_id, @@ -1668,7 +1720,6 @@ async def ftp_download_to_server( ) ).first() - # If no active session, create one if not session: _ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first() session = MonitoringSession( @@ -1687,115 +1738,50 @@ async def ftp_download_to_server( db.commit() db.refresh(session) - # Download file from SLMM - SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") + ext = os.path.splitext(filename)[1].lower() + file_type_map = { + '.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio', + '.rnd': 'measurement', + '.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data', + '.log': 'log', + '.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive', + '.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image', + '.pdf': 'document', '.doc': 'document', '.docx': 'document', + } + file_type = file_type_map.get(ext, 'data') - try: - async with httpx.AsyncClient(timeout=300.0) as client: - response = await client.post( - f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download", - json={"remote_path": remote_path} - ) + project_dir = Path(f"data/Projects/{project_id}/{session.id}") + project_dir.mkdir(parents=True, exist_ok=True) + file_path = project_dir / filename + with open(file_path, 'wb') as f: + f.write(file_content) + checksum = hashlib.sha256(file_content).hexdigest() - if not response.is_success: - raise HTTPException( - status_code=response.status_code, - detail=f"Failed to download from SLMM: {response.text}" - ) + data_file = DataFile( + id=str(uuid.uuid4()), + session_id=session.id, + file_path=str(file_path.relative_to("data")), # Store relative to data/ + file_type=file_type, + file_size_bytes=len(file_content), + downloaded_at=datetime.utcnow(), + checksum=checksum, + file_metadata=json.dumps({ + "source": "ftp", + "remote_path": remote_path, + "unit_id": unit_id, + "location_id": location_id, + }) + ) + db.add(data_file) + db.commit() - # Extract filename from remote_path - filename = os.path.basename(remote_path) - - # Determine file type from extension - ext = os.path.splitext(filename)[1].lower() - file_type_map = { - # Audio files - '.wav': 'audio', - '.mp3': 'audio', - '.flac': 'audio', - '.m4a': 'audio', - '.aac': 'audio', - # Sound level meter measurement files - '.rnd': 'measurement', - # Data files - '.csv': 'data', - '.txt': 'data', - '.json': 'data', - '.xml': 'data', - '.dat': 'data', - # Log files - '.log': 'log', - # Archives - '.zip': 'archive', - '.tar': 'archive', - '.gz': 'archive', - '.7z': 'archive', - '.rar': 'archive', - # Images - '.jpg': 'image', - '.jpeg': 'image', - '.png': 'image', - '.gif': 'image', - # Documents - '.pdf': 'document', - '.doc': 'document', - '.docx': 'document', - } - file_type = file_type_map.get(ext, 'data') - - # Create directory structure: data/Projects/{project_id}/{session_id}/ - project_dir = Path(f"data/Projects/{project_id}/{session.id}") - project_dir.mkdir(parents=True, exist_ok=True) - - # Save file to disk - file_path = project_dir / filename - file_content = response.content - - with open(file_path, 'wb') as f: - f.write(file_content) - - # Calculate checksum - checksum = hashlib.sha256(file_content).hexdigest() - - # Create DataFile record - data_file = DataFile( - id=str(uuid.uuid4()), - session_id=session.id, - file_path=str(file_path.relative_to("data")), # Store relative to data/ - file_type=file_type, - file_size_bytes=len(file_content), - downloaded_at=datetime.utcnow(), - checksum=checksum, - file_metadata=json.dumps({ - "source": "ftp", - "remote_path": remote_path, - "unit_id": unit_id, - "location_id": location_id, - }) - ) - - db.add(data_file) - db.commit() - - return { - "success": True, - "message": f"Downloaded {filename} to server", - "file_id": data_file.id, - "file_path": str(file_path), - "file_size": len(file_content), - } - - except httpx.TimeoutException: - raise HTTPException( - status_code=504, - detail="Timeout downloading file from SLM" - ) - except Exception as e: - logger.error(f"Error downloading file to server: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to download file to server: {str(e)}" - ) + return { + "success": True, + "message": f"Downloaded {filename} to server", + "file_id": data_file.id, + "file_path": str(file_path), + "file_size": len(file_content), + } @router.post("/{project_id}/ftp-download-folder-to-server") @@ -1805,20 +1791,20 @@ async def ftp_download_folder_to_server( db: Session = Depends(get_db), ): """ - Download an entire folder from an SLM to the server via FTP. - Extracts all files from the ZIP and preserves folder structure. - Creates individual DataFile records for each file. - Sound Monitoring projects only. + Download an entire Auto_#### measurement folder from an SLM to the server. + + Routes the downloaded ZIP through the shared NRL ingest — the same path the + scheduled FTP pull, the daily cycle, and the manual SD-card upload use. That + means: keep the .rnh + Leq .rnd, parse the header (real recording start/stop + + duration, percentile slot map, weightings), drop the 1-second _Lp_ files, + and create one clean MonitoringSession attributed to the unit. Sound + Monitoring projects only. """ import httpx import os - import hashlib - import zipfile - import io _require_module(db.query(Project).filter_by(id=project_id).first(), "sound_monitoring", db) - from pathlib import Path - from backend.models import DataFile + from backend.routers.project_locations import ingest_nrl_zip, IngestError data = await request.json() unit_id = data.get("unit_id") @@ -1827,160 +1813,66 @@ async def ftp_download_folder_to_server( if not unit_id or not remote_path: raise HTTPException(status_code=400, detail="Missing unit_id or remote_path") - - # Get or create active session for this location/unit - session = db.query(MonitoringSession).filter( - and_( - MonitoringSession.project_id == project_id, - MonitoringSession.location_id == location_id, - MonitoringSession.unit_id == unit_id, - MonitoringSession.status.in_(["recording", "paused"]) + if not location_id: + raise HTTPException( + status_code=400, + detail=("This unit isn't assigned to a monitoring location. Assign it to an " + "NRL first so the downloaded measurement attaches to the right location."), ) - ).first() - # If no active session, create one - if not session: - _ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first() - session = MonitoringSession( - id=str(uuid.uuid4()), - project_id=project_id, - location_id=location_id, - unit_id=unit_id, - session_type="sound", # SLMs are sound monitoring devices - status="completed", - started_at=datetime.utcnow(), - stopped_at=datetime.utcnow(), - device_model=_ftp_unit.slm_model if _ftp_unit else None, - session_metadata='{"source": "ftp_folder_download", "note": "Auto-created for FTP folder download"}' - ) - db.add(session) - db.commit() - db.refresh(session) - - # Download folder from SLMM (returns ZIP) + # Download the folder from SLMM (returns a ZIP of the Auto_#### folder) SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") - try: - async with httpx.AsyncClient(timeout=600.0) as client: # Longer timeout for folders + async with httpx.AsyncClient(timeout=600.0) as client: # longer timeout for folders response = await client.post( f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download-folder", json={"remote_path": remote_path} ) - - if not response.is_success: - raise HTTPException( - status_code=response.status_code, - detail=f"Failed to download folder from SLMM: {response.text}" - ) - - # Extract folder name from remote_path - folder_name = os.path.basename(remote_path.rstrip('/')) - - # Create base directory: data/Projects/{project_id}/{session_id}/{folder_name}/ - base_dir = Path(f"data/Projects/{project_id}/{session.id}/{folder_name}") - base_dir.mkdir(parents=True, exist_ok=True) - - # Extract ZIP and save individual files - zip_content = response.content - created_files = [] - total_size = 0 - - # File type mapping for classification - file_type_map = { - # Audio files - '.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio', - # Data files - '.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data', - # Log files - '.log': 'log', - # Archives - '.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive', - # Images - '.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image', - # Documents - '.pdf': 'document', '.doc': 'document', '.docx': 'document', - } - - with zipfile.ZipFile(io.BytesIO(zip_content)) as zf: - for zip_info in zf.filelist: - # Skip directories - if zip_info.is_dir(): - continue - - # Read file from ZIP - file_data = zf.read(zip_info.filename) - - # Determine file path (preserve structure within folder) - # zip_info.filename might be like "Auto_0001/measurement.wav" - file_path = base_dir / zip_info.filename - file_path.parent.mkdir(parents=True, exist_ok=True) - - # Write file to disk - with open(file_path, 'wb') as f: - f.write(file_data) - - # Calculate checksum - checksum = hashlib.sha256(file_data).hexdigest() - - # Determine file type - ext = os.path.splitext(zip_info.filename)[1].lower() - file_type = file_type_map.get(ext, 'data') - - # Create DataFile record - data_file = DataFile( - id=str(uuid.uuid4()), - session_id=session.id, - file_path=str(file_path.relative_to("data")), - file_type=file_type, - file_size_bytes=len(file_data), - downloaded_at=datetime.utcnow(), - checksum=checksum, - file_metadata=json.dumps({ - "source": "ftp_folder", - "remote_path": remote_path, - "unit_id": unit_id, - "location_id": location_id, - "folder_name": folder_name, - "relative_path": zip_info.filename, - }) - ) - - db.add(data_file) - created_files.append({ - "filename": zip_info.filename, - "size": len(file_data), - "type": file_type - }) - total_size += len(file_data) - - db.commit() - - return { - "success": True, - "message": f"Downloaded folder {folder_name} with {len(created_files)} files", - "folder_name": folder_name, - "file_count": len(created_files), - "total_size": total_size, - "files": created_files, - } - except httpx.TimeoutException: raise HTTPException( status_code=504, - detail="Timeout downloading folder from SLM (large folders may take a while)" - ) - except zipfile.BadZipFile: - raise HTTPException( - status_code=500, - detail="Downloaded file is not a valid ZIP archive" + detail="Timeout downloading folder from SLM (large folders may take a while)", ) except Exception as e: - logger.error(f"Error downloading folder to server: {e}") + logger.error(f"Error reaching SLMM for folder download: {e}") + raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}") + + if not response.is_success: raise HTTPException( - status_code=500, - detail=f"Failed to download folder to server: {str(e)}" + status_code=response.status_code, + detail=f"Failed to download folder from SLMM: {response.text}", ) + # Ingest through the shared NRL core. dedupe=False so a re-download of a + # still-growing folder captures the latest intervals (matches manual upload). + try: + result = ingest_nrl_zip( + location_id, response.content, db, + source="ftp_manual", dedupe=False, unit_id=unit_id, + ) + except IngestError as e: + # No usable .rnd/.rnh in the folder, or unknown location. + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error ingesting downloaded folder: {e}") + raise HTTPException(status_code=500, detail=f"Failed to ingest downloaded folder: {str(e)}") + + folder_name = os.path.basename(remote_path.rstrip('/')) + return { + "success": True, + "message": ( + f"Imported {result['leq_files']} Leq file(s) from {folder_name} " + f"({result['files_imported']} stored; 1-second _Lp_ data skipped)" + ), + "folder_name": folder_name, + "session_id": result["session_id"], + "file_count": result["files_imported"], + "leq_files": result["leq_files"], + "started_at": result["started_at"], + "stopped_at": result["stopped_at"], + "duration_seconds": result["duration_seconds"], + } + # ============================================================================ # Project Types diff --git a/templates/partials/projects/ftp_browser.html b/templates/partials/projects/ftp_browser.html index 331acf2..2c8f7a8 100644 --- a/templates/partials/projects/ftp_browser.html +++ b/templates/partials/projects/ftp_browser.html @@ -542,8 +542,11 @@ async function downloadFolderToServer(unitId, remotePath, folderName) { const data = await response.json(); if (response.ok) { - // Show success message - alert(`✓ Folder "${folderName}" downloaded successfully!\n\n${data.file_count} files extracted\nTotal size: ${formatFileSize(data.total_size)}\n\nFiles are now available in the Project Files section below.`); + // Show success message — surface how long the measurement ran + alert(`✓ Folder "${folderName}" saved!\n\n` + + (data.message || `${data.file_count} file(s) imported`) + + formatRunLength(data) + + `\n\nNow saved as a session in the Project Files section below.`); // Refresh the unified files list htmx.trigger('#unified-files', 'refresh'); @@ -585,7 +588,11 @@ async function downloadToServer(unitId, remotePath, fileName) { if (response.ok) { // Show success message - alert(`✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}\nSize: ${formatFileSize(data.file_size)}`); + const sizeLine = `\nSize: ${formatFileSize(data.file_size)}`; + const msg = data.ingested + ? `✓ ${fileName} imported as measurement data!` + formatRunLength(data) + sizeLine + : `✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}` + sizeLine; + alert(msg); // Refresh the unified files list htmx.trigger('#unified-files', 'refresh'); @@ -607,6 +614,17 @@ function formatFileSize(bytes) { return (bytes / 1073741824).toFixed(2) + ' GB'; } +// Build a "how long did it run" line from an ingest response. Duration is +// timezone-independent (stop − start), so it's the reliable number to show. +function formatRunLength(data) { + if (data.duration_seconds == null) return ''; + const s = data.duration_seconds; + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + let txt = h > 0 ? `${h}h ${m}m` : `${m}m`; + return `\n\nRecorded for: ${txt}`; +} + // Check FTP status for all units on load // Use setTimeout to ensure DOM elements exist when HTMX loads this partial setTimeout(function() { -- 2.52.0 From aa21c81c2eafd780627be5677a77a0955243e2fd Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:46:59 +0000 Subject: [PATCH 4/8] feat(reports): per-NRL Data Files tab reaches parity with the project-wide tab The per-NRL Data Files tab now reuses the same FTP browser + unified-files partials as the project-wide tab, scoped to the one NRL: ftp-browser and files-unified take an optional location_id. nrl_detail.html drops the flat file_list view for 'Download Files from SLMs' (Browse Files -> Download & Save) plus the grouped 'Project Files' view (edit times / download-all / delete), keeping the NRL upload and adding a refresh button. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/routers/projects.py | 27 ++++++++++++++++++++------- templates/nrl_detail.html | 27 ++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/backend/routers/projects.py b/backend/routers/projects.py index ea73d52..b1407de 100644 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -1591,24 +1591,32 @@ async def get_sessions_calendar( async def get_ftp_browser( project_id: str, request: Request, + location_id: Optional[str] = None, db: Session = Depends(get_db), ): """ Get FTP browser interface for downloading files from assigned SLMs. Returns HTML partial with FTP browser. Sound Monitoring projects only. + + When `location_id` is given, scope to just the unit(s) assigned to that NRL + (used by the per-NRL Data Files tab, which mirrors the project-wide tab). """ from backend.models import DataFile project = db.query(Project).filter_by(id=project_id).first() _require_module(project, "sound_monitoring", db) - # Get all assignments for this project (active = assigned_until IS NULL) - assignments = db.query(UnitAssignment).filter( + # Active assignments for this project (active = assigned_until IS NULL), + # optionally scoped to a single NRL/location. + q = db.query(UnitAssignment).filter( and_( UnitAssignment.project_id == project_id, UnitAssignment.assigned_until == None, ) - ).all() + ) + if location_id: + q = q.filter(UnitAssignment.location_id == location_id) + assignments = q.all() # Enrich with unit and location details units_data = [] @@ -1882,21 +1890,26 @@ async def ftp_download_folder_to_server( async def get_unified_files( project_id: str, request: Request, + location_id: Optional[str] = None, db: Session = Depends(get_db), ): """ Get unified view of all files in this project. Groups files by recording session with full metadata. Returns HTML partial with hierarchical file listing. + + When `location_id` is given, scope to a single NRL/location (used by the + per-NRL Data Files tab so it mirrors the project-wide tab). """ from backend.models import DataFile from pathlib import Path import json - # Get all sessions for this project - sessions = db.query(MonitoringSession).filter_by( - project_id=project_id - ).order_by(MonitoringSession.started_at.desc()).all() + # Sessions for this project (optionally scoped to one NRL/location) + q = db.query(MonitoringSession).filter_by(project_id=project_id) + if location_id: + q = q.filter(MonitoringSession.location_id == location_id) + sessions = q.order_by(MonitoringSession.started_at.desc()).all() sessions_data = [] for session in sessions: diff --git a/templates/nrl_detail.html b/templates/nrl_detail.html index cafc12a..656435d 100644 --- a/templates/nrl_detail.html +++ b/templates/nrl_detail.html @@ -357,6 +357,16 @@ -
-
Loading data files...
+
Loading files...
@@ -715,7 +732,7 @@ function submitUpload() { status.textContent = parts.join(' '); status.className = 'text-sm text-green-600 dark:text-green-400'; input.value = ''; - htmx.trigger(document.getElementById('data-files-list'), 'load'); + htmx.trigger(document.getElementById('unified-files'), 'refresh'); } else { status.textContent = `Error: ${data.detail || 'Upload failed'}`; status.className = 'text-sm text-red-600 dark:text-red-400'; -- 2.52.0 From 1801d4eb74e60ce4bcacb5dca5a4a7bd3b7d19df Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:58:34 +0000 Subject: [PATCH 5/8] fix(reports): resolve loadFTPFiles collision breaking Browse Files on the NRL tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ftp_browser.html and slm_live_view.html are both loaded on the per-NRL detail page (Data Files + Command Center tabs) and each defined loadFTPFiles / downloadToServer / downloadFTPFile / enableFTP / formatFileSize as globals — last to load won. 'Browse Files' then called slm_live_view's loadFTPFiles, which renders into the hidden Command Center's #ftp-files-list, so the FTP request fired but nothing appeared. Prefix ftp_browser's five colliding functions with fb* so each partial keeps its own. (Element IDs don't collide: per-unit vs fixed.) Co-Authored-By: Claude Opus 4.8 (1M context) --- templates/partials/projects/ftp_browser.html | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/templates/partials/projects/ftp_browser.html b/templates/partials/projects/ftp_browser.html index 2c8f7a8..5c7373b 100644 --- a/templates/partials/projects/ftp_browser.html +++ b/templates/partials/projects/ftp_browser.html @@ -32,7 +32,7 @@ Settings - - - - -