serversdown/terra-view
1
0
Fork 0
Code Issues 17 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

feat(reports): manual FTP "Download & Save" saves a parsed session

Browse Source 
ftp-download-folder-to-server and ftp-download-to-server now route NRL data through the shared ingest (ingest_nrl_zip / _ingest_file_entries) instead of hand-rolling DataFile rows on a now/zero-duration session. Folder save requires the unit be assigned to a location; non-NRL single files keep the generic save path. The FTP browser popup now reports how long the measurement ran.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
serversdown
2026-06-15 18:12:15 +00:00
parent 2ecf1f54d5
commit 7716a4b51d
2 changed files with 171 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/routers/projects.py
+110 -218
View File
@@ -1638,9 +1638,13 @@ async def ftp_download_to_server(
db: Session = Depends(get_db),
):
"""
Download a file from an SLM to the server via FTP.
Creates a DataFile record and stores the file in data/Projects/{project_id}/
Sound Monitoring projects only.
Download a single file from an SLM to the server via FTP.
NRL measurement files (.rnh / _Leq_ .rnd) are routed through the shared NRL
ingest so the session is parsed and attributed to the unit (a lone .rnh still
yields the real recording window + duration). Any other file type — or a
unit with no location — falls back to a generic stored DataFile, preserving
the original behaviour. Sound Monitoring projects only.
"""
import httpx
import os
@@ -1658,7 +1662,55 @@ async def ftp_download_to_server(
if not unit_id or not remote_path:
raise HTTPException(status_code=400, detail="Missing unit_id or remote_path")
# Get or create active session for this location/unit
filename = os.path.basename(remote_path)
# Download the file from SLMM
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
try:
async with httpx.AsyncClient(timeout=300.0) as client:
response = await client.post(
f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download",
json={"remote_path": remote_path}
)
except httpx.TimeoutException:
raise HTTPException(status_code=504, detail="Timeout downloading file from SLM")
except Exception as e:
logger.error(f"Error reaching SLMM for file download: {e}")
raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}")
if not response.is_success:
raise HTTPException(
status_code=response.status_code,
detail=f"Failed to download from SLMM: {response.text}",
)
file_content = response.content
# NRL measurement file + known location → shared ingest (parsed + attributed).
from backend.routers.project_locations import (
_ingest_file_entries, IngestError, _is_wanted_nrl_file,
)
if location_id and _is_wanted_nrl_file(filename):
location = db.query(MonitoringLocation).filter_by(id=location_id).first()
if location:
try:
result = _ingest_file_entries(
location, [(filename, file_content)], db,
source="ftp_manual", dedupe=False, unit_id=unit_id,
)
except IngestError as e:
raise HTTPException(status_code=400, detail=str(e))
return {
"success": True,
"message": f"Imported {filename} as NRL measurement data",
"ingested": True,
"session_id": result["session_id"],
"file_size": len(file_content),
"started_at": result["started_at"],
"stopped_at": result["stopped_at"],
"duration_seconds": result["duration_seconds"],
}
# --- Generic path: any other file type (or no location) — store as-is ---
session = db.query(MonitoringSession).filter(
and_(
MonitoringSession.project_id == project_id,
@@ -1668,7 +1720,6 @@ async def ftp_download_to_server(
)
).first()
# If no active session, create one
if not session:
_ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first()
session = MonitoringSession(
@@ -1687,77 +1738,25 @@ async def ftp_download_to_server(
db.commit()
db.refresh(session)
# Download file from SLMM
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
try:
async with httpx.AsyncClient(timeout=300.0) as client:
response = await client.post(
f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download",
json={"remote_path": remote_path}
)
if not response.is_success:
raise HTTPException(
status_code=response.status_code,
detail=f"Failed to download from SLMM: {response.text}"
)
# Extract filename from remote_path
filename = os.path.basename(remote_path)
# Determine file type from extension
ext = os.path.splitext(filename)[1].lower()
file_type_map = {
# Audio files
'.wav': 'audio',
'.mp3': 'audio',
'.flac': 'audio',
'.m4a': 'audio',
'.aac': 'audio',
# Sound level meter measurement files
'.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio',
'.rnd': 'measurement',
# Data files
'.csv': 'data',
'.txt': 'data',
'.json': 'data',
'.xml': 'data',
'.dat': 'data',
# Log files
'.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data',
'.log': 'log',
# Archives
'.zip': 'archive',
'.tar': 'archive',
'.gz': 'archive',
'.7z': 'archive',
'.rar': 'archive',
# Images
'.jpg': 'image',
'.jpeg': 'image',
'.png': 'image',
'.gif': 'image',
# Documents
'.pdf': 'document',
'.doc': 'document',
'.docx': 'document',
'.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive',
'.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image',
'.pdf': 'document', '.doc': 'document', '.docx': 'document',
}
file_type = file_type_map.get(ext, 'data')
# Create directory structure: data/Projects/{project_id}/{session_id}/
project_dir = Path(f"data/Projects/{project_id}/{session.id}")
project_dir.mkdir(parents=True, exist_ok=True)
# Save file to disk
file_path = project_dir / filename
file_content = response.content
with open(file_path, 'wb') as f:
f.write(file_content)
# Calculate checksum
checksum = hashlib.sha256(file_content).hexdigest()
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
@@ -1773,7 +1772,6 @@ async def ftp_download_to_server(
"location_id": location_id,
})
)
db.add(data_file)
db.commit()
@@ -1785,18 +1783,6 @@ async def ftp_download_to_server(
"file_size": len(file_content),
}
except httpx.TimeoutException:
raise HTTPException(
status_code=504,
detail="Timeout downloading file from SLM"
)
except Exception as e:
logger.error(f"Error downloading file to server: {e}")
raise HTTPException(
status_code=500,
detail=f"Failed to download file to server: {str(e)}"
)
@router.post("/{project_id}/ftp-download-folder-to-server")
async def ftp_download_folder_to_server(
@@ -1805,20 +1791,20 @@ async def ftp_download_folder_to_server(
db: Session = Depends(get_db),
):
"""
Download an entire folder from an SLM to the server via FTP.
Extracts all files from the ZIP and preserves folder structure.
Creates individual DataFile records for each file.
Sound Monitoring projects only.
Download an entire Auto_#### measurement folder from an SLM to the server.
Routes the downloaded ZIP through the shared NRL ingest — the same path the
scheduled FTP pull, the daily cycle, and the manual SD-card upload use. That
means: keep the .rnh + Leq .rnd, parse the header (real recording start/stop
+ duration, percentile slot map, weightings), drop the 1-second _Lp_ files,
and create one clean MonitoringSession attributed to the unit. Sound
Monitoring projects only.
"""
import httpx
import os
import hashlib
import zipfile
import io
_require_module(db.query(Project).filter_by(id=project_id).first(), "sound_monitoring", db)
from pathlib import Path
from backend.models import DataFile
from backend.routers.project_locations import ingest_nrl_zip, IngestError
data = await request.json()
unit_id = data.get("unit_id")
@@ -1827,160 +1813,66 @@ async def ftp_download_folder_to_server(
if not unit_id or not remote_path:
raise HTTPException(status_code=400, detail="Missing unit_id or remote_path")
# Get or create active session for this location/unit
session = db.query(MonitoringSession).filter(
and_(
MonitoringSession.project_id == project_id,
MonitoringSession.location_id == location_id,
MonitoringSession.unit_id == unit_id,
MonitoringSession.status.in_(["recording", "paused"])
if not location_id:
raise HTTPException(
status_code=400,
detail=("This unit isn't assigned to a monitoring location. Assign it to an "
"NRL first so the downloaded measurement attaches to the right location."),
)
).first()
# If no active session, create one
if not session:
_ftp_unit = db.query(RosterUnit).filter_by(id=unit_id).first()
session = MonitoringSession(
id=str(uuid.uuid4()),
project_id=project_id,
location_id=location_id,
unit_id=unit_id,
session_type="sound", # SLMs are sound monitoring devices
status="completed",
started_at=datetime.utcnow(),
stopped_at=datetime.utcnow(),
device_model=_ftp_unit.slm_model if _ftp_unit else None,
session_metadata='{"source": "ftp_folder_download", "note": "Auto-created for FTP folder download"}'
)
db.add(session)
db.commit()
db.refresh(session)
# Download folder from SLMM (returns ZIP)
# Download the folder from SLMM (returns a ZIP of the Auto_#### folder)
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
try:
async with httpx.AsyncClient(timeout=600.0) as client: # Longer timeout for folders
async with httpx.AsyncClient(timeout=600.0) as client: # longer timeout for folders
response = await client.post(
f"{SLMM_BASE_URL}/api/nl43/{unit_id}/ftp/download-folder",
json={"remote_path": remote_path}
)
except httpx.TimeoutException:
raise HTTPException(
status_code=504,
detail="Timeout downloading folder from SLM (large folders may take a while)",
)
except Exception as e:
logger.error(f"Error reaching SLMM for folder download: {e}")
raise HTTPException(status_code=502, detail=f"Failed to reach SLMM: {str(e)}")
if not response.is_success:
raise HTTPException(
status_code=response.status_code,
detail=f"Failed to download folder from SLMM: {response.text}"
detail=f"Failed to download folder from SLMM: {response.text}",
)
# Extract folder name from remote_path
# Ingest through the shared NRL core. dedupe=False so a re-download of a
# still-growing folder captures the latest intervals (matches manual upload).
try:
result = ingest_nrl_zip(
location_id, response.content, db,
source="ftp_manual", dedupe=False, unit_id=unit_id,
)
except IngestError as e:
# No usable .rnd/.rnh in the folder, or unknown location.
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Error ingesting downloaded folder: {e}")
raise HTTPException(status_code=500, detail=f"Failed to ingest downloaded folder: {str(e)}")
folder_name = os.path.basename(remote_path.rstrip('/'))
# Create base directory: data/Projects/{project_id}/{session_id}/{folder_name}/
base_dir = Path(f"data/Projects/{project_id}/{session.id}/{folder_name}")
base_dir.mkdir(parents=True, exist_ok=True)
# Extract ZIP and save individual files
zip_content = response.content
created_files = []
total_size = 0
# File type mapping for classification
file_type_map = {
# Audio files
'.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio',
# Data files
'.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data',
# Log files
'.log': 'log',
# Archives
'.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive',
# Images
'.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image',
# Documents
'.pdf': 'document', '.doc': 'document', '.docx': 'document',
}
with zipfile.ZipFile(io.BytesIO(zip_content)) as zf:
for zip_info in zf.filelist:
# Skip directories
if zip_info.is_dir():
continue
# Read file from ZIP
file_data = zf.read(zip_info.filename)
# Determine file path (preserve structure within folder)
# zip_info.filename might be like "Auto_0001/measurement.wav"
file_path = base_dir / zip_info.filename
file_path.parent.mkdir(parents=True, exist_ok=True)
# Write file to disk
with open(file_path, 'wb') as f:
f.write(file_data)
# Calculate checksum
checksum = hashlib.sha256(file_data).hexdigest()
# Determine file type
ext = os.path.splitext(zip_info.filename)[1].lower()
file_type = file_type_map.get(ext, 'data')
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
file_path=str(file_path.relative_to("data")),
file_type=file_type,
file_size_bytes=len(file_data),
downloaded_at=datetime.utcnow(),
checksum=checksum,
file_metadata=json.dumps({
"source": "ftp_folder",
"remote_path": remote_path,
"unit_id": unit_id,
"location_id": location_id,
"folder_name": folder_name,
"relative_path": zip_info.filename,
})
)
db.add(data_file)
created_files.append({
"filename": zip_info.filename,
"size": len(file_data),
"type": file_type
})
total_size += len(file_data)
db.commit()
return {
"success": True,
"message": f"Downloaded folder {folder_name} with {len(created_files)} files",
"message": (
f"Imported {result['leq_files']} Leq file(s) from {folder_name} "
f"({result['files_imported']} stored; 1-second _Lp_ data skipped)"
),
"folder_name": folder_name,
"file_count": len(created_files),
"total_size": total_size,
"files": created_files,
"session_id": result["session_id"],
"file_count": result["files_imported"],
"leq_files": result["leq_files"],
"started_at": result["started_at"],
"stopped_at": result["stopped_at"],
"duration_seconds": result["duration_seconds"],
}
except httpx.TimeoutException:
raise HTTPException(
status_code=504,
detail="Timeout downloading folder from SLM (large folders may take a while)"
)
except zipfile.BadZipFile:
raise HTTPException(
status_code=500,
detail="Downloaded file is not a valid ZIP archive"
)
except Exception as e:
logger.error(f"Error downloading folder to server: {e}")
raise HTTPException(
status_code=500,
detail=f"Failed to download folder to server: {str(e)}"
)
# ============================================================================
# Project Types
templates/partials/projects/ftp_browser.html
+21 -3
View File
@@ -542,8 +542,11 @@ async function downloadFolderToServer(unitId, remotePath, folderName) {
const data = await response.json();
if (response.ok) {
// Show success message
alert(`✓ Folder "${folderName}" downloaded successfully!\n\n${data.file_count} files extracted\nTotal size: ${formatFileSize(data.total_size)}\n\nFiles are now available in the Project Files section below.`);
// Show success message — surface how long the measurement ran
alert(`✓ Folder "${folderName}" saved!\n\n` +
(data.message || `${data.file_count} file(s) imported`) +
formatRunLength(data) +
`\n\nNow saved as a session in the Project Files section below.`);
// Refresh the unified files list
htmx.trigger('#unified-files', 'refresh');
@@ -585,7 +588,11 @@ async function downloadToServer(unitId, remotePath, fileName) {
if (response.ok) {
// Show success message
alert(`✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}\nSize: ${formatFileSize(data.file_size)}`);
const sizeLine = `\nSize: ${formatFileSize(data.file_size)}`;
const msg = data.ingested
? `✓ ${fileName} imported as measurement data!` + formatRunLength(data) + sizeLine
: `✓ ${fileName} downloaded to server successfully!\n\nFile ID: ${data.file_id}` + sizeLine;
alert(msg);
// Refresh the unified files list
htmx.trigger('#unified-files', 'refresh');
@@ -607,6 +614,17 @@ function formatFileSize(bytes) {
return (bytes / 1073741824).toFixed(2) + ' GB';
}
// Build a "how long did it run" line from an ingest response. Duration is
// timezone-independent (stop − start), so it's the reliable number to show.
function formatRunLength(data) {
if (data.duration_seconds == null) return '';
const s = data.duration_seconds;
const h = Math.floor(s / 3600);
const m = Math.floor((s % 3600) / 60);
let txt = h > 0 ? `${h}h ${m}m` : `${m}m`;
return `\n\nRecorded for: ${txt}`;
}
// Check FTP status for all units on load
// Use setTimeout to ensure DOM elements exist when HTMX loads this partial
setTimeout(function() {