From 7716a4b51d9018921d4974b30c4c972231a25d2f Mon Sep 17 00:00:00 2001 From: serversdown Date: Mon, 15 Jun 2026 18:12:15 +0000 Subject: [PATCH] feat(reports): manual FTP "Download & Save" saves a parsed session ftp-download-folder-to-server and ftp-download-to-server now route NRL data through the shared ingest (ingest_nrl_zip / _ingest_file_entries) instead of hand-rolling DataFile rows on a now/zero-duration session. Folder save requires the unit be assigned to a location; non-NRL single files keep the generic save path. The FTP browser popup now reports how long the measurement ran. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/routers/projects.py | 408 +++++++------------ templates/partials/projects/ftp_browser.html | 24 +- 2 files changed, 171 insertions(+), 261 deletions(-) diff --git a/backend/routers/projects.py b/backend/routers/projects.py index 3c61236..ea73d52 100644 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -1638,9 +1638,13 @@ async def ftp_download_to_server( db: Session = Depends(get_db), ): """ - Download a file from an SLM to the server via FTP. - Creates a DataFile record and stores the file in data/Projects/{project_id}/ - Sound Monitoring projects only. + Download a single file from an SLM to the server via FTP. + + NRL measurement files (.rnh / _Leq_ .rnd) are routed through the shared NRL + ingest so the session is parsed and attributed to the unit (a lone .rnh still + yields the real recording window + duration). Any other file type — or a + unit with no location — falls back to a generic stored DataFile, preserving + the original behaviour. Sound Monitoring projects only. """ import httpx import os @@ -1658,7 +1662,55 @@ async def ftp_download_to_server( if not unit_id or not remote_path: raise HTTPException(status_code=400, detail="Missing unit_id or remote_path") - # Get or create active session for this location/unit + filename = os.path.basename(remote_path) + + # Download the file from SLMM + SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") + try: + async with httpx.AsyncClient(timeout=300.0) as client: + response = await client.post( + f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download", + json={"remote_path": remote_path} + ) + except httpx.TimeoutException: + raise HTTPException(status_code=504, detail="Timeout downloading file from SLM") + except Exception as e: + logger.error(f"Error reaching SLMM for file download: {e}") + raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}") + + if not response.is_success: + raise HTTPException( + status_code=response.status_code, + detail=f"Failed to download from SLMM: {response.text}", + ) + file_content = response.content + + # NRL measurement file + known location → shared ingest (parsed + attributed). + from backend.routers.project_locations import ( + _ingest_file_entries, IngestError, _is_wanted_nrl_file, + ) + if location_id and _is_wanted_nrl_file(filename): + location = db.query(MonitoringLocation).filter_by(id=location_id).first() + if location: + try: + result = _ingest_file_entries( + location, [(filename, file_content)], db, + source="ftp_manual", dedupe=False, unit_id=unit_id, + ) + except IngestError as e: + raise HTTPException(status_code=400, detail=str(e)) + return { + "success": True, + "message": f"Imported {filename} as NRL measurement data", + "ingested": True, + "session_id": result["session_id"], + "file_size": len(file_content), + "started_at": result["started_at"], + "stopped_at": result["stopped_at"], + "duration_seconds": result["duration_seconds"], + } + + # --- Generic path: any other file type (or no location) — store as-is --- session = db.query(MonitoringSession).filter( and_( MonitoringSession.project_id == project_id, @@ -1668,7 +1720,6 @@ async def ftp_download_to_server( ) ).first() - # If no active session, create one if not session: _ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first() session = MonitoringSession( @@ -1687,115 +1738,50 @@ async def ftp_download_to_server( db.commit() db.refresh(session) - # Download file from SLMM - SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") + ext = os.path.splitext(filename)[1].lower() + file_type_map = { + '.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio', + '.rnd': 'measurement', + '.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data', + '.log': 'log', + '.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive', + '.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image', + '.pdf': 'document', '.doc': 'document', '.docx': 'document', + } + file_type = file_type_map.get(ext, 'data') - try: - async with httpx.AsyncClient(timeout=300.0) as client: - response = await client.post( - f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download", - json={"remote_path": remote_path} - ) + project_dir = Path(f"data/Projects/{project_id}/{session.id}") + project_dir.mkdir(parents=True, exist_ok=True) + file_path = project_dir / filename + with open(file_path, 'wb') as f: + f.write(file_content) + checksum = hashlib.sha256(file_content).hexdigest() - if not response.is_success: - raise HTTPException( - status_code=response.status_code, - detail=f"Failed to download from SLMM: {response.text}" - ) + data_file = DataFile( + id=str(uuid.uuid4()), + session_id=session.id, + file_path=str(file_path.relative_to("data")), # Store relative to data/ + file_type=file_type, + file_size_bytes=len(file_content), + downloaded_at=datetime.utcnow(), + checksum=checksum, + file_metadata=json.dumps({ + "source": "ftp", + "remote_path": remote_path, + "unit_id": unit_id, + "location_id": location_id, + }) + ) + db.add(data_file) + db.commit() - # Extract filename from remote_path - filename = os.path.basename(remote_path) - - # Determine file type from extension - ext = os.path.splitext(filename)[1].lower() - file_type_map = { - # Audio files - '.wav': 'audio', - '.mp3': 'audio', - '.flac': 'audio', - '.m4a': 'audio', - '.aac': 'audio', - # Sound level meter measurement files - '.rnd': 'measurement', - # Data files - '.csv': 'data', - '.txt': 'data', - '.json': 'data', - '.xml': 'data', - '.dat': 'data', - # Log files - '.log': 'log', - # Archives - '.zip': 'archive', - '.tar': 'archive', - '.gz': 'archive', - '.7z': 'archive', - '.rar': 'archive', - # Images - '.jpg': 'image', - '.jpeg': 'image', - '.png': 'image', - '.gif': 'image', - # Documents - '.pdf': 'document', - '.doc': 'document', - '.docx': 'document', - } - file_type = file_type_map.get(ext, 'data') - - # Create directory structure: data/Projects/{project_id}/{session_id}/ - project_dir = Path(f"data/Projects/{project_id}/{session.id}") - project_dir.mkdir(parents=True, exist_ok=True) - - # Save file to disk - file_path = project_dir / filename - file_content = response.content - - with open(file_path, 'wb') as f: - f.write(file_content) - - # Calculate checksum - checksum = hashlib.sha256(file_content).hexdigest() - - # Create DataFile record - data_file = DataFile( - id=str(uuid.uuid4()), - session_id=session.id, - file_path=str(file_path.relative_to("data")), # Store relative to data/ - file_type=file_type, - file_size_bytes=len(file_content), - downloaded_at=datetime.utcnow(), - checksum=checksum, - file_metadata=json.dumps({ - "source": "ftp", - "remote_path": remote_path, - "unit_id": unit_id, - "location_id": location_id, - }) - ) - - db.add(data_file) - db.commit() - - return { - "success": True, - "message": f"Downloaded {filename} to server", - "file_id": data_file.id, - "file_path": str(file_path), - "file_size": len(file_content), - } - - except httpx.TimeoutException: - raise HTTPException( - status_code=504, - detail="Timeout downloading file from SLM" - ) - except Exception as e: - logger.error(f"Error downloading file to server: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to download file to server: {str(e)}" - ) + return { + "success": True, + "message": f"Downloaded {filename} to server", + "file_id": data_file.id, + "file_path": str(file_path), + "file_size": len(file_content), + } @router.post("/{project_id}/ftp-download-folder-to-server") @@ -1805,20 +1791,20 @@ async def ftp_download_folder_to_server( db: Session = Depends(get_db), ): """ - Download an entire folder from an SLM to the server via FTP. - Extracts all files from the ZIP and preserves folder structure. - Creates individual DataFile records for each file. - Sound Monitoring projects only. + Download an entire Auto_#### measurement folder from an SLM to the server. + + Routes the downloaded ZIP through the shared NRL ingest — the same path the + scheduled FTP pull, the daily cycle, and the manual SD-card upload use. That + means: keep the .rnh + Leq .rnd, parse the header (real recording start/stop + + duration, percentile slot map, weightings), drop the 1-second _Lp_ files, + and create one clean MonitoringSession attributed to the unit. Sound + Monitoring projects only. """ import httpx import os - import hashlib - import zipfile - import io _require_module(db.query(Project).filter_by(id=project_id).first(), "sound_monitoring", db) - from pathlib import Path - from backend.models import DataFile + from backend.routers.project_locations import ingest_nrl_zip, IngestError data = await request.json() unit_id = data.get("unit_id") @@ -1827,160 +1813,66 @@ async def ftp_download_folder_to_server( if not unit_id or not remote_path: raise HTTPException(status_code=400, detail="Missing unit_id or remote_path") - - # Get or create active session for this location/unit - session = db.query(MonitoringSession).filter( - and_( - MonitoringSession.project_id == project_id, - MonitoringSession.location_id == location_id, - MonitoringSession.unit_id == unit_id, - MonitoringSession.status.in_(["recording", "paused"]) + if not location_id: + raise HTTPException( + status_code=400, + detail=("This unit isn't assigned to a monitoring location. Assign it to an " + "NRL first so the downloaded measurement attaches to the right location."), ) - ).first() - # If no active session, create one - if not session: - _ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first() - session = MonitoringSession( - id=str(uuid.uuid4()), - project_id=project_id, - location_id=location_id, - unit_id=unit_id, - session_type="sound", # SLMs are sound monitoring devices - status="completed", - started_at=datetime.utcnow(), - stopped_at=datetime.utcnow(), - device_model=_ftp_unit.slm_model if _ftp_unit else None, - session_metadata='{"source": "ftp_folder_download", "note": "Auto-created for FTP folder download"}' - ) - db.add(session) - db.commit() - db.refresh(session) - - # Download folder from SLMM (returns ZIP) + # Download the folder from SLMM (returns a ZIP of the Auto_#### folder) SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100") - try: - async with httpx.AsyncClient(timeout=600.0) as client: # Longer timeout for folders + async with httpx.AsyncClient(timeout=600.0) as client: # longer timeout for folders response = await client.post( f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download-folder", json={"remote_path": remote_path} ) - - if not response.is_success: - raise HTTPException( - status_code=response.status_code, - detail=f"Failed to download folder from SLMM: {response.text}" - ) - - # Extract folder name from remote_path - folder_name = os.path.basename(remote_path.rstrip('/')) - - # Create base directory: data/Projects/{project_id}/{session_id}/{folder_name}/ - base_dir = Path(f"data/Projects/{project_id}/{session.id}/{folder_name}") - base_dir.mkdir(parents=True, exist_ok=True) - - # Extract ZIP and save individual files - zip_content = response.content - created_files = [] - total_size = 0 - - # File type mapping for classification - file_type_map = { - # Audio files - '.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio', - # Data files - '.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data', - # Log files - '.log': 'log', - # Archives - '.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive', - # Images - '.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image', - # Documents - '.pdf': 'document', '.doc': 'document', '.docx': 'document', - } - - with zipfile.ZipFile(io.BytesIO(zip_content)) as zf: - for zip_info in zf.filelist: - # Skip directories - if zip_info.is_dir(): - continue - - # Read file from ZIP - file_data = zf.read(zip_info.filename) - - # Determine file path (preserve structure within folder) - # zip_info.filename might be like "Auto_0001/measurement.wav" - file_path = base_dir / zip_info.filename - file_path.parent.mkdir(parents=True, exist_ok=True) - - # Write file to disk - with open(file_path, 'wb') as f: - f.write(file_data) - - # Calculate checksum - checksum = hashlib.sha256(file_data).hexdigest() - - # Determine file type - ext = os.path.splitext(zip_info.filename)[1].lower() - file_type = file_type_map.get(ext, 'data') - - # Create DataFile record - data_file = DataFile( - id=str(uuid.uuid4()), - session_id=session.id, - file_path=str(file_path.relative_to("data")), - file_type=file_type, - file_size_bytes=len(file_data), - downloaded_at=datetime.utcnow(), - checksum=checksum, - file_metadata=json.dumps({ - "source": "ftp_folder", - "remote_path": remote_path, - "unit_id": unit_id, - "location_id": location_id, - "folder_name": folder_name, - "relative_path": zip_info.filename, - }) - ) - - db.add(data_file) - created_files.append({ - "filename": zip_info.filename, - "size": len(file_data), - "type": file_type - }) - total_size += len(file_data) - - db.commit() - - return { - "success": True, - "message": f"Downloaded folder {folder_name} with {len(created_files)} files", - "folder_name": folder_name, - "file_count": len(created_files), - "total_size": total_size, - "files": created_files, - } - except httpx.TimeoutException: raise HTTPException( status_code=504, - detail="Timeout downloading folder from SLM (large folders may take a while)" - ) - except zipfile.BadZipFile: - raise HTTPException( - status_code=500, - detail="Downloaded file is not a valid ZIP archive" + detail="Timeout downloading folder from SLM (large folders may take a while)", ) except Exception as e: - logger.error(f"Error downloading folder to server: {e}") + logger.error(f"Error reaching SLMM for folder download: {e}") + raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}") + + if not response.is_success: raise HTTPException( - status_code=500, - detail=f"Failed to download folder to server: {str(e)}" + status_code=response.status_code, + detail=f"Failed to download folder from SLMM: {response.text}", ) + # Ingest through the shared NRL core. dedupe=False so a re-download of a + # still-growing folder captures the latest intervals (matches manual upload). + try: + result = ingest_nrl_zip( + location_id, response.content, db, + source="ftp_manual", dedupe=False, unit_id=unit_id, + ) + except IngestError as e: + # No usable .rnd/.rnh in the folder, or unknown location. + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error ingesting downloaded folder: {e}") + raise HTTPException(status_code=500, detail=f"Failed to ingest downloaded folder: {str(e)}") + + folder_name = os.path.basename(remote_path.rstrip('/')) + return { + "success": True, + "message": ( + f"Imported {result['leq_files']} Leq file(s) from {folder_name} " + f"({result['files_imported']} stored; 1-second _Lp_ data skipped)" + ), + "folder_name": folder_name, + "session_id": result["session_id"], + "file_count": result["files_imported"], + "leq_files": result["leq_files"], + "started_at": result["started_at"], + "stopped_at": result["stopped_at"], + "duration_seconds": result["duration_seconds"], + } + # ============================================================================ # Project Types diff --git a/templates/partials/projects/ftp_browser.html b/templates/partials/projects/ftp_browser.html index 331acf2..2c8f7a8 100644 --- a/templates/partials/projects/ftp_browser.html +++ b/templates/partials/projects/ftp_browser.html @@ -542,8 +542,11 @@ async function downloadFolderToServer(unitId, remotePath, folderName) { const data = await response.json(); if (response.ok) { - // Show success message - alert(`✓ Folder "${folderName}" downloaded successfully!\n\n${data.file_count} files extracted\nTotal size: ${formatFileSize(data.total_size)}\n\nFiles are now available in the Project Files section below.`); + // Show success message — surface how long the measurement ran + alert(`✓ Folder "${folderName}" saved!\n\n` + + (data.message || `${data.file_count} file(s) imported`) + + formatRunLength(data) + + `\n\nNow saved as a session in the Project Files section below.`); // Refresh the unified files list htmx.trigger('#unified-files', 'refresh'); @@ -585,7 +588,11 @@ async function downloadToServer(unitId, remotePath, fileName) { if (response.ok) { // Show success message - alert(`✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}\nSize: ${formatFileSize(data.file_size)}`); + const sizeLine = `\nSize: ${formatFileSize(data.file_size)}`; + const msg = data.ingested + ? `✓ ${fileName} imported as measurement data!` + formatRunLength(data) + sizeLine + : `✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}` + sizeLine; + alert(msg); // Refresh the unified files list htmx.trigger('#unified-files', 'refresh'); @@ -607,6 +614,17 @@ function formatFileSize(bytes) { return (bytes / 1073741824).toFixed(2) + ' GB'; } +// Build a "how long did it run" line from an ingest response. Duration is +// timezone-independent (stop − start), so it's the reliable number to show. +function formatRunLength(data) { + if (data.duration_seconds == null) return ''; + const s = data.duration_seconds; + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + let txt = h > 0 ? `${h}h ${m}m` : `${m}m`; + return `\n\nRecorded for: ${txt}`; +} + // Check FTP status for all units on load // Use setTimeout to ensure DOM elements exist when HTMX loads this partial setTimeout(function() {