serversdown/terra-view
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Project data management phase 1. Files can be downloaded to server and downloaded locally.

Browse Source
This commit is contained in:
serversdwn
2026-01-16 07:39:22 +00:00
parent 54754e2279
commit 6c7ce5aad0
9 changed files with 783 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
backend/routers/projects.py
View File

@@ -522,51 +522,6 @@ async def get_project_sessions(
})
@router.get("/{project_id}/files", response_class=HTMLResponse)
async def get_project_files(
project_id: str,
request: Request,
db: Session = Depends(get_db),
file_type: Optional[str] = Query(None),
):
"""
Get all data files from all sessions in this project.
Returns HTML partial with file list.
Optional file_type filter: audio, data, log, etc.
"""
from backend.models import DataFile
# Join through RecordingSession to get project files
query = db.query(DataFile).join(
RecordingSession,
DataFile.session_id == RecordingSession.id
).filter(RecordingSession.project_id == project_id)
# Filter by file type if provided
if file_type:
query = query.filter(DataFile.file_type == file_type)
files = query.order_by(DataFile.created_at.desc()).all()
# Enrich with session details
files_data = []
for file in files:
session = None
if file.session_id:
session = db.query(RecordingSession).filter_by(id=file.session_id).first()
files_data.append({
"file": file,
"session": session,
})
return templates.TemplateResponse("partials/projects/file_list.html", {
"request": request,
"project_id": project_id,
"files": files_data,
})
@router.get("/{project_id}/ftp-browser", response_class=HTMLResponse)
async def get_ftp_browser(
project_id: str,
@@ -649,10 +604,11 @@ async def ftp_download_to_server(
project_id=project_id,
location_id=location_id,
unit_id=unit_id,
session_type="sound", # SLMs are sound monitoring devices
status="completed",
started_at=datetime.utcnow(),
stopped_at=datetime.utcnow(),
notes="Auto-created for FTP download"
session_metadata='{"source": "ftp_download", "note": "Auto-created for FTP download"}'
)
db.add(session)
db.commit()
@@ -680,12 +636,35 @@ async def ftp_download_to_server(
# Determine file type from extension
ext = os.path.splitext(filename)[1].lower()
file_type_map = {
# Audio files
'.wav': 'audio',
'.mp3': 'audio',
'.flac': 'audio',
'.m4a': 'audio',
'.aac': 'audio',
# Data files
'.csv': 'data',
'.txt': 'data',
'.log': 'log',
'.json': 'data',
'.xml': 'data',
'.dat': 'data',
# Log files
'.log': 'log',
# Archives
'.zip': 'archive',
'.tar': 'archive',
'.gz': 'archive',
'.7z': 'archive',
'.rar': 'archive',
# Images
'.jpg': 'image',
'.jpeg': 'image',
'.png': 'image',
'.gif': 'image',
# Documents
'.pdf': 'document',
'.doc': 'document',
'.docx': 'document',
}
file_type = file_type_map.get(ext, 'data')
@@ -751,12 +730,15 @@ async def ftp_download_folder_to_server(
db: Session = Depends(get_db),
):
"""
Download an entire folder from an SLM to the server via FTP as a ZIP file.
Creates a DataFile record and stores the ZIP in data/Projects/{project_id}/
Download an entire folder from an SLM to the server via FTP.
Extracts all files from the ZIP and preserves folder structure.
Creates individual DataFile records for each file.
"""
import httpx
import os
import hashlib
import zipfile
import io
from pathlib import Path
from backend.models import DataFile
@@ -785,16 +767,17 @@ async def ftp_download_folder_to_server(
project_id=project_id,
location_id=location_id,
unit_id=unit_id,
session_type="sound", # SLMs are sound monitoring devices
status="completed",
started_at=datetime.utcnow(),
stopped_at=datetime.utcnow(),
notes="Auto-created for FTP folder download"
session_metadata='{"source": "ftp_folder_download", "note": "Auto-created for FTP folder download"}'
)
db.add(session)
db.commit()
db.refresh(session)
# Download folder from SLMM
# Download folder from SLMM (returns ZIP)
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
try:
@@ -812,49 +795,93 @@ async def ftp_download_folder_to_server(
# Extract folder name from remote_path
folder_name = os.path.basename(remote_path.rstrip('/'))
filename = f"{folder_name}.zip"
# Create directory structure: data/Projects/{project_id}/{session_id}/
project_dir = Path(f"data/Projects/{project_id}/{session.id}")
project_dir.mkdir(parents=True, exist_ok=True)
# Create base directory: data/Projects/{project_id}/{session_id}/{folder_name}/
base_dir = Path(f"data/Projects/{project_id}/{session.id}/{folder_name}")
base_dir.mkdir(parents=True, exist_ok=True)
# Save ZIP file to disk
file_path = project_dir / filename
file_content = response.content
# Extract ZIP and save individual files
zip_content = response.content
created_files = []
total_size = 0
with open(file_path, 'wb') as f:
f.write(file_content)
# File type mapping for classification
file_type_map = {
# Audio files
'.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio',
# Data files
'.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data',
# Log files
'.log': 'log',
# Archives
'.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive',
# Images
'.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image',
# Documents
'.pdf': 'document', '.doc': 'document', '.docx': 'document',
}
# Calculate checksum
checksum = hashlib.sha256(file_content).hexdigest()
with zipfile.ZipFile(io.BytesIO(zip_content)) as zf:
for zip_info in zf.filelist:
# Skip directories
if zip_info.is_dir():
continue
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
file_path=str(file_path.relative_to("data")), # Store relative to data/
file_type='archive', # ZIP archives
file_size_bytes=len(file_content),
downloaded_at=datetime.utcnow(),
checksum=checksum,
file_metadata=json.dumps({
"source": "ftp_folder",
"remote_path": remote_path,
"unit_id": unit_id,
"location_id": location_id,
"folder_name": folder_name,
})
)
# Read file from ZIP
file_data = zf.read(zip_info.filename)
# Determine file path (preserve structure within folder)
# zip_info.filename might be like "Auto_0001/measurement.wav"
file_path = base_dir / zip_info.filename
file_path.parent.mkdir(parents=True, exist_ok=True)
# Write file to disk
with open(file_path, 'wb') as f:
f.write(file_data)
# Calculate checksum
checksum = hashlib.sha256(file_data).hexdigest()
# Determine file type
ext = os.path.splitext(zip_info.filename)[1].lower()
file_type = file_type_map.get(ext, 'data')
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
file_path=str(file_path.relative_to("data")),
file_type=file_type,
file_size_bytes=len(file_data),
downloaded_at=datetime.utcnow(),
checksum=checksum,
file_metadata=json.dumps({
"source": "ftp_folder",
"remote_path": remote_path,
"unit_id": unit_id,
"location_id": location_id,
"folder_name": folder_name,
"relative_path": zip_info.filename,
})
)
db.add(data_file)
created_files.append({
"filename": zip_info.filename,
"size": len(file_data),
"type": file_type
})
total_size += len(file_data)
db.add(data_file)
db.commit()
return {
"success": True,
"message": f"Downloaded folder {folder_name} to server as ZIP",
"file_id": data_file.id,
"file_path": str(file_path),
"file_size": len(file_content),
"message": f"Downloaded folder {folder_name} with {len(created_files)} files",
"folder_name": folder_name,
"file_count": len(created_files),
"total_size": total_size,
"files": created_files,
}
except httpx.TimeoutException:
@@ -862,6 +889,11 @@ async def ftp_download_folder_to_server(
status_code=504,
detail="Timeout downloading folder from SLM (large folders may take a while)"
)
except zipfile.BadZipFile:
raise HTTPException(
status_code=500,
detail="Downloaded file is not a valid ZIP archive"
)
except Exception as e:
logger.error(f"Error downloading folder to server: {e}")
raise HTTPException(
@@ -874,6 +906,121 @@ async def ftp_download_folder_to_server(
# Project Types
# ============================================================================
@router.get("/{project_id}/files-unified", response_class=HTMLResponse)
async def get_unified_files(
project_id: str,
request: Request,
db: Session = Depends(get_db),
):
"""
Get unified view of all files in this project.
Groups files by recording session with full metadata.
Returns HTML partial with hierarchical file listing.
"""
from backend.models import DataFile
from pathlib import Path
import json
# Get all sessions for this project
sessions = db.query(RecordingSession).filter_by(
project_id=project_id
).order_by(RecordingSession.started_at.desc()).all()
sessions_data = []
for session in sessions:
# Get files for this session
files = db.query(DataFile).filter_by(session_id=session.id).all()
# Skip sessions with no files
if not files:
continue
# Get session context
unit = None
location = None
if session.unit_id:
unit = db.query(RosterUnit).filter_by(id=session.unit_id).first()
if session.location_id:
location = db.query(MonitoringLocation).filter_by(id=session.location_id).first()
files_data = []
for file in files:
# Check if file exists on disk
file_path = Path("data") / file.file_path
exists_on_disk = file_path.exists()
# Get actual file size if exists
actual_size = file_path.stat().st_size if exists_on_disk else None
# Parse metadata JSON
metadata = {}
try:
if file.file_metadata:
metadata = json.loads(file.file_metadata)
except Exception as e:
logger.warning(f"Failed to parse metadata for file {file.id}: {e}")
files_data.append({
"file": file,
"exists_on_disk": exists_on_disk,
"actual_size": actual_size,
"metadata": metadata,
})
sessions_data.append({
"session": session,
"unit": unit,
"location": location,
"files": files_data,
})
return templates.TemplateResponse("partials/projects/unified_files.html", {
"request": request,
"project_id": project_id,
"sessions": sessions_data,
})
@router.get("/{project_id}/files/{file_id}/download")
async def download_project_file(
project_id: str,
file_id: str,
db: Session = Depends(get_db),
):
"""
Download a data file from a project.
Returns the file for download.
"""
from backend.models import DataFile
from fastapi.responses import FileResponse
from pathlib import Path
# Get the file record
file_record = db.query(DataFile).filter_by(id=file_id).first()
if not file_record:
raise HTTPException(status_code=404, detail="File not found")
# Verify file belongs to this project
session = db.query(RecordingSession).filter_by(id=file_record.session_id).first()
if not session or session.project_id != project_id:
raise HTTPException(status_code=403, detail="File does not belong to this project")
# Build full file path
file_path = Path("data") / file_record.file_path
if not file_path.exists():
raise HTTPException(status_code=404, detail="File not found on disk")
# Extract filename for download
filename = file_path.name
return FileResponse(
path=str(file_path),
filename=filename,
media_type="application/octet-stream"
)
@router.get("/types/list", response_class=HTMLResponse)
async def get_project_types(request: Request, db: Session = Depends(get_db)):
"""

19
backend/routers/roster_edit.py
View File

@@ -458,16 +458,20 @@ def set_retired(unit_id: str, retired: bool = Form(...), db: Session = Depends(g
@router.delete("/{unit_id}")
def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
async def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
"""
Permanently delete a unit from the database.
Checks roster, emitters, and ignored_units tables and deletes from any table where the unit exists.
For SLM devices, also removes from SLMM to stop background polling.
"""
deleted = False
was_slm = False
# Try to delete from roster table
roster_unit = db.query(RosterUnit).filter(RosterUnit.id == unit_id).first()
if roster_unit:
was_slm = roster_unit.device_type == "slm"
db.delete(roster_unit)
deleted = True
@@ -488,6 +492,19 @@ def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="Unit not found")
db.commit()
# If it was an SLM, also delete from SLMM
if was_slm:
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.delete(f"{SLMM_BASE_URL}/api/nl43/{unit_id}/config")
if response.status_code in [200, 404]:
logger.info(f"Deleted SLM {unit_id} from SLMM")
else:
logger.warning(f"Failed to delete SLM {unit_id} from SLMM: {response.status_code}")
except Exception as e:
logger.error(f"Error deleting SLM {unit_id} from SLMM: {e}")
return {"message": "Unit deleted", "id": unit_id}

72
backend/routers/settings.py
View File

@@ -477,3 +477,75 @@ async def upload_snapshot(file: UploadFile = File(...)):
except Exception as e:
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
# ============================================================================
# SLMM SYNC ENDPOINTS
# ============================================================================
@router.post("/slmm/sync-all")
async def sync_all_slms(db: Session = Depends(get_db)):
"""
Manually trigger full sync of all SLM devices from Terra-View roster to SLMM.
This ensures SLMM database matches Terra-View roster (source of truth).
Also cleans up orphaned devices in SLMM that are not in Terra-View.
"""
from backend.services.slmm_sync import sync_all_slms_to_slmm, cleanup_orphaned_slmm_devices
try:
# Sync all SLMs
sync_results = await sync_all_slms_to_slmm(db)
# Clean up orphaned devices
cleanup_results = await cleanup_orphaned_slmm_devices(db)
return {
"status": "ok",
"sync": sync_results,
"cleanup": cleanup_results
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Sync failed: {str(e)}")
@router.get("/slmm/status")
async def get_slmm_sync_status(db: Session = Depends(get_db)):
"""
Get status of SLMM synchronization.
Shows which devices are in Terra-View roster vs SLMM database.
"""
from backend.services.slmm_sync import get_slmm_devices
try:
# Get devices from both systems
roster_slms = db.query(RosterUnit).filter_by(device_type="slm").all()
slmm_devices = await get_slmm_devices()
if slmm_devices is None:
raise HTTPException(status_code=503, detail="SLMM service unavailable")
roster_unit_ids = {unit.unit_type for unit in roster_slms}
slmm_unit_ids = set(slmm_devices)
# Find differences
in_roster_only = roster_unit_ids - slmm_unit_ids
in_slmm_only = slmm_unit_ids - roster_unit_ids
in_both = roster_unit_ids & slmm_unit_ids
return {
"status": "ok",
"terra_view_total": len(roster_unit_ids),
"slmm_total": len(slmm_unit_ids),
"synced": len(in_both),
"missing_from_slmm": list(in_roster_only),
"orphaned_in_slmm": list(in_slmm_only),
"in_sync": len(in_roster_only) == 0 and len(in_slmm_only) == 0
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"Status check failed: {str(e)}")