serversdown/terra-view
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Project data management phase 1. Files can be downloaded to server and downloaded locally.

Browse Source
This commit is contained in:
serversdwn
2026-01-16 07:39:22 +00:00
parent 54754e2279
commit 6c7ce5aad0
9 changed files with 783 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/main.py
View File

@@ -1,6 +1,6 @@
import os
import logging
from fastapi import FastAPI, Request, Depends
from fastapi import FastAPI, Request, Depends, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
@@ -111,6 +111,26 @@ async def startup_event():
await start_scheduler()
logger.info("Scheduler service started")
# Sync all SLMs to SLMM on startup
logger.info("Syncing SLM devices to SLMM...")
try:
from backend.services.slmm_sync import sync_all_slms_to_slmm, cleanup_orphaned_slmm_devices
from backend.database import SessionLocal
db = SessionLocal()
try:
# Sync all SLMs from roster to SLMM
sync_results = await sync_all_slms_to_slmm(db)
logger.info(f"SLM sync complete: {sync_results}")
# Clean up orphaned devices in SLMM
cleanup_results = await cleanup_orphaned_slmm_devices(db)
logger.info(f"SLMM cleanup complete: {cleanup_results}")
finally:
db.close()
except Exception as e:
logger.error(f"Error syncing SLMs to SLMM on startup: {e}")
@app.on_event("shutdown")
def shutdown_event():
"""Clean up services on app shutdown"""

315
backend/routers/projects.py
View File

@@ -522,51 +522,6 @@ async def get_project_sessions(
})
@router.get("/{project_id}/files", response_class=HTMLResponse)
async def get_project_files(
project_id: str,
request: Request,
db: Session = Depends(get_db),
file_type: Optional[str] = Query(None),
):
"""
Get all data files from all sessions in this project.
Returns HTML partial with file list.
Optional file_type filter: audio, data, log, etc.
"""
from backend.models import DataFile
# Join through RecordingSession to get project files
query = db.query(DataFile).join(
RecordingSession,
DataFile.session_id == RecordingSession.id
).filter(RecordingSession.project_id == project_id)
# Filter by file type if provided
if file_type:
query = query.filter(DataFile.file_type == file_type)
files = query.order_by(DataFile.created_at.desc()).all()
# Enrich with session details
files_data = []
for file in files:
session = None
if file.session_id:
session = db.query(RecordingSession).filter_by(id=file.session_id).first()
files_data.append({
"file": file,
"session": session,
})
return templates.TemplateResponse("partials/projects/file_list.html", {
"request": request,
"project_id": project_id,
"files": files_data,
})
@router.get("/{project_id}/ftp-browser", response_class=HTMLResponse)
async def get_ftp_browser(
project_id: str,
@@ -649,10 +604,11 @@ async def ftp_download_to_server(
project_id=project_id,
location_id=location_id,
unit_id=unit_id,
session_type="sound", # SLMs are sound monitoring devices
status="completed",
started_at=datetime.utcnow(),
stopped_at=datetime.utcnow(),
notes="Auto-created for FTP download"
session_metadata='{"source": "ftp_download", "note": "Auto-created for FTP download"}'
)
db.add(session)
db.commit()
@@ -680,12 +636,35 @@ async def ftp_download_to_server(
# Determine file type from extension
ext = os.path.splitext(filename)[1].lower()
file_type_map = {
# Audio files
'.wav': 'audio',
'.mp3': 'audio',
'.flac': 'audio',
'.m4a': 'audio',
'.aac': 'audio',
# Data files
'.csv': 'data',
'.txt': 'data',
'.log': 'log',
'.json': 'data',
'.xml': 'data',
'.dat': 'data',
# Log files
'.log': 'log',
# Archives
'.zip': 'archive',
'.tar': 'archive',
'.gz': 'archive',
'.7z': 'archive',
'.rar': 'archive',
# Images
'.jpg': 'image',
'.jpeg': 'image',
'.png': 'image',
'.gif': 'image',
# Documents
'.pdf': 'document',
'.doc': 'document',
'.docx': 'document',
}
file_type = file_type_map.get(ext, 'data')
@@ -751,12 +730,15 @@ async def ftp_download_folder_to_server(
db: Session = Depends(get_db),
):
"""
Download an entire folder from an SLM to the server via FTP as a ZIP file.
Creates a DataFile record and stores the ZIP in data/Projects/{project_id}/
Download an entire folder from an SLM to the server via FTP.
Extracts all files from the ZIP and preserves folder structure.
Creates individual DataFile records for each file.
"""
import httpx
import os
import hashlib
import zipfile
import io
from pathlib import Path
from backend.models import DataFile
@@ -785,16 +767,17 @@ async def ftp_download_folder_to_server(
project_id=project_id,
location_id=location_id,
unit_id=unit_id,
session_type="sound", # SLMs are sound monitoring devices
status="completed",
started_at=datetime.utcnow(),
stopped_at=datetime.utcnow(),
notes="Auto-created for FTP folder download"
session_metadata='{"source": "ftp_folder_download", "note": "Auto-created for FTP folder download"}'
)
db.add(session)
db.commit()
db.refresh(session)
# Download folder from SLMM
# Download folder from SLMM (returns ZIP)
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
try:
@@ -812,49 +795,93 @@ async def ftp_download_folder_to_server(
# Extract folder name from remote_path
folder_name = os.path.basename(remote_path.rstrip('/'))
filename = f"{folder_name}.zip"
# Create directory structure: data/Projects/{project_id}/{session_id}/
project_dir = Path(f"data/Projects/{project_id}/{session.id}")
project_dir.mkdir(parents=True, exist_ok=True)
# Create base directory: data/Projects/{project_id}/{session_id}/{folder_name}/
base_dir = Path(f"data/Projects/{project_id}/{session.id}/{folder_name}")
base_dir.mkdir(parents=True, exist_ok=True)
# Save ZIP file to disk
file_path = project_dir / filename
file_content = response.content
# Extract ZIP and save individual files
zip_content = response.content
created_files = []
total_size = 0
with open(file_path, 'wb') as f:
f.write(file_content)
# File type mapping for classification
file_type_map = {
# Audio files
'.wav': 'audio', '.mp3': 'audio', '.flac': 'audio', '.m4a': 'audio', '.aac': 'audio',
# Data files
'.csv': 'data', '.txt': 'data', '.json': 'data', '.xml': 'data', '.dat': 'data',
# Log files
'.log': 'log',
# Archives
'.zip': 'archive', '.tar': 'archive', '.gz': 'archive', '.7z': 'archive', '.rar': 'archive',
# Images
'.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.gif': 'image',
# Documents
'.pdf': 'document', '.doc': 'document', '.docx': 'document',
}
# Calculate checksum
checksum = hashlib.sha256(file_content).hexdigest()
with zipfile.ZipFile(io.BytesIO(zip_content)) as zf:
for zip_info in zf.filelist:
# Skip directories
if zip_info.is_dir():
continue
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
file_path=str(file_path.relative_to("data")), # Store relative to data/
file_type='archive', # ZIP archives
file_size_bytes=len(file_content),
downloaded_at=datetime.utcnow(),
checksum=checksum,
file_metadata=json.dumps({
"source": "ftp_folder",
"remote_path": remote_path,
"unit_id": unit_id,
"location_id": location_id,
"folder_name": folder_name,
})
)
# Read file from ZIP
file_data = zf.read(zip_info.filename)
# Determine file path (preserve structure within folder)
# zip_info.filename might be like "Auto_0001/measurement.wav"
file_path = base_dir / zip_info.filename
file_path.parent.mkdir(parents=True, exist_ok=True)
# Write file to disk
with open(file_path, 'wb') as f:
f.write(file_data)
# Calculate checksum
checksum = hashlib.sha256(file_data).hexdigest()
# Determine file type
ext = os.path.splitext(zip_info.filename)[1].lower()
file_type = file_type_map.get(ext, 'data')
# Create DataFile record
data_file = DataFile(
id=str(uuid.uuid4()),
session_id=session.id,
file_path=str(file_path.relative_to("data")),
file_type=file_type,
file_size_bytes=len(file_data),
downloaded_at=datetime.utcnow(),
checksum=checksum,
file_metadata=json.dumps({
"source": "ftp_folder",
"remote_path": remote_path,
"unit_id": unit_id,
"location_id": location_id,
"folder_name": folder_name,
"relative_path": zip_info.filename,
})
)
db.add(data_file)
created_files.append({
"filename": zip_info.filename,
"size": len(file_data),
"type": file_type
})
total_size += len(file_data)
db.add(data_file)
db.commit()
return {
"success": True,
"message": f"Downloaded folder {folder_name} to server as ZIP",
"file_id": data_file.id,
"file_path": str(file_path),
"file_size": len(file_content),
"message": f"Downloaded folder {folder_name} with {len(created_files)} files",
"folder_name": folder_name,
"file_count": len(created_files),
"total_size": total_size,
"files": created_files,
}
except httpx.TimeoutException:
@@ -862,6 +889,11 @@ async def ftp_download_folder_to_server(
status_code=504,
detail="Timeout downloading folder from SLM (large folders may take a while)"
)
except zipfile.BadZipFile:
raise HTTPException(
status_code=500,
detail="Downloaded file is not a valid ZIP archive"
)
except Exception as e:
logger.error(f"Error downloading folder to server: {e}")
raise HTTPException(
@@ -874,6 +906,121 @@ async def ftp_download_folder_to_server(
# Project Types
# ============================================================================
@router.get("/{project_id}/files-unified", response_class=HTMLResponse)
async def get_unified_files(
project_id: str,
request: Request,
db: Session = Depends(get_db),
):
"""
Get unified view of all files in this project.
Groups files by recording session with full metadata.
Returns HTML partial with hierarchical file listing.
"""
from backend.models import DataFile
from pathlib import Path
import json
# Get all sessions for this project
sessions = db.query(RecordingSession).filter_by(
project_id=project_id
).order_by(RecordingSession.started_at.desc()).all()
sessions_data = []
for session in sessions:
# Get files for this session
files = db.query(DataFile).filter_by(session_id=session.id).all()
# Skip sessions with no files
if not files:
continue
# Get session context
unit = None
location = None
if session.unit_id:
unit = db.query(RosterUnit).filter_by(id=session.unit_id).first()
if session.location_id:
location = db.query(MonitoringLocation).filter_by(id=session.location_id).first()
files_data = []
for file in files:
# Check if file exists on disk
file_path = Path("data") / file.file_path
exists_on_disk = file_path.exists()
# Get actual file size if exists
actual_size = file_path.stat().st_size if exists_on_disk else None
# Parse metadata JSON
metadata = {}
try:
if file.file_metadata:
metadata = json.loads(file.file_metadata)
except Exception as e:
logger.warning(f"Failed to parse metadata for file {file.id}: {e}")
files_data.append({
"file": file,
"exists_on_disk": exists_on_disk,
"actual_size": actual_size,
"metadata": metadata,
})
sessions_data.append({
"session": session,
"unit": unit,
"location": location,
"files": files_data,
})
return templates.TemplateResponse("partials/projects/unified_files.html", {
"request": request,
"project_id": project_id,
"sessions": sessions_data,
})
@router.get("/{project_id}/files/{file_id}/download")
async def download_project_file(
project_id: str,
file_id: str,
db: Session = Depends(get_db),
):
"""
Download a data file from a project.
Returns the file for download.
"""
from backend.models import DataFile
from fastapi.responses import FileResponse
from pathlib import Path
# Get the file record
file_record = db.query(DataFile).filter_by(id=file_id).first()
if not file_record:
raise HTTPException(status_code=404, detail="File not found")
# Verify file belongs to this project
session = db.query(RecordingSession).filter_by(id=file_record.session_id).first()
if not session or session.project_id != project_id:
raise HTTPException(status_code=403, detail="File does not belong to this project")
# Build full file path
file_path = Path("data") / file_record.file_path
if not file_path.exists():
raise HTTPException(status_code=404, detail="File not found on disk")
# Extract filename for download
filename = file_path.name
return FileResponse(
path=str(file_path),
filename=filename,
media_type="application/octet-stream"
)
@router.get("/types/list", response_class=HTMLResponse)
async def get_project_types(request: Request, db: Session = Depends(get_db)):
"""

19
backend/routers/roster_edit.py
View File

@@ -458,16 +458,20 @@ def set_retired(unit_id: str, retired: bool = Form(...), db: Session = Depends(g
@router.delete("/{unit_id}")
def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
async def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
"""
Permanently delete a unit from the database.
Checks roster, emitters, and ignored_units tables and deletes from any table where the unit exists.
For SLM devices, also removes from SLMM to stop background polling.
"""
deleted = False
was_slm = False
# Try to delete from roster table
roster_unit = db.query(RosterUnit).filter(RosterUnit.id == unit_id).first()
if roster_unit:
was_slm = roster_unit.device_type == "slm"
db.delete(roster_unit)
deleted = True
@@ -488,6 +492,19 @@ def delete_roster_unit(unit_id: str, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="Unit not found")
db.commit()
# If it was an SLM, also delete from SLMM
if was_slm:
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.delete(f"{SLMM_BASE_URL}/api/nl43/{unit_id}/config")
if response.status_code in [200, 404]:
logger.info(f"Deleted SLM {unit_id} from SLMM")
else:
logger.warning(f"Failed to delete SLM {unit_id} from SLMM: {response.status_code}")
except Exception as e:
logger.error(f"Error deleting SLM {unit_id} from SLMM: {e}")
return {"message": "Unit deleted", "id": unit_id}

72
backend/routers/settings.py
View File

@@ -477,3 +477,75 @@ async def upload_snapshot(file: UploadFile = File(...)):
except Exception as e:
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
# ============================================================================
# SLMM SYNC ENDPOINTS
# ============================================================================
@router.post("/slmm/sync-all")
async def sync_all_slms(db: Session = Depends(get_db)):
"""
Manually trigger full sync of all SLM devices from Terra-View roster to SLMM.
This ensures SLMM database matches Terra-View roster (source of truth).
Also cleans up orphaned devices in SLMM that are not in Terra-View.
"""
from backend.services.slmm_sync import sync_all_slms_to_slmm, cleanup_orphaned_slmm_devices
try:
# Sync all SLMs
sync_results = await sync_all_slms_to_slmm(db)
# Clean up orphaned devices
cleanup_results = await cleanup_orphaned_slmm_devices(db)
return {
"status": "ok",
"sync": sync_results,
"cleanup": cleanup_results
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Sync failed: {str(e)}")
@router.get("/slmm/status")
async def get_slmm_sync_status(db: Session = Depends(get_db)):
"""
Get status of SLMM synchronization.
Shows which devices are in Terra-View roster vs SLMM database.
"""
from backend.services.slmm_sync import get_slmm_devices
try:
# Get devices from both systems
roster_slms = db.query(RosterUnit).filter_by(device_type="slm").all()
slmm_devices = await get_slmm_devices()
if slmm_devices is None:
raise HTTPException(status_code=503, detail="SLMM service unavailable")
roster_unit_ids = {unit.unit_type for unit in roster_slms}
slmm_unit_ids = set(slmm_devices)
# Find differences
in_roster_only = roster_unit_ids - slmm_unit_ids
in_slmm_only = slmm_unit_ids - roster_unit_ids
in_both = roster_unit_ids & slmm_unit_ids
return {
"status": "ok",
"terra_view_total": len(roster_unit_ids),
"slmm_total": len(slmm_unit_ids),
"synced": len(in_both),
"missing_from_slmm": list(in_roster_only),
"orphaned_in_slmm": list(in_slmm_only),
"in_sync": len(in_roster_only) == 0 and len(in_slmm_only) == 0
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"Status check failed: {str(e)}")

227
backend/services/slmm_sync.py Normal file
View File

@@ -0,0 +1,227 @@
"""
SLMM Synchronization Service
This service ensures Terra-View roster is the single source of truth for SLM device configuration.
When SLM devices are added, edited, or deleted in Terra-View, changes are automatically synced to SLMM.
"""
import logging
import httpx
import os
from typing import Optional
from sqlalchemy.orm import Session
from backend.models import RosterUnit
logger = logging.getLogger(__name__)
SLMM_BASE_URL = os.getenv("SLMM_BASE_URL", "http://localhost:8100")
async def sync_slm_to_slmm(unit: RosterUnit) -> bool:
"""
Sync a single SLM device from Terra-View roster to SLMM.
Args:
unit: RosterUnit with device_type="slm"
Returns:
True if sync successful, False otherwise
"""
if unit.device_type != "slm":
logger.warning(f"Attempted to sync non-SLM unit {unit.id} to SLMM")
return False
if not unit.slm_host:
logger.warning(f"SLM {unit.id} has no host configured, skipping SLMM sync")
return False
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.put(
f"{SLMM_BASE_URL}/api/nl43/{unit.id}/config",
json={
"host": unit.slm_host,
"tcp_port": unit.slm_tcp_port or 2255,
"tcp_enabled": True,
"ftp_enabled": True,
"ftp_username": "USER", # Default NL43 credentials
"ftp_password": "0000",
"poll_enabled": not unit.retired, # Disable polling for retired units
"poll_interval_seconds": 60, # Default interval
}
)
if response.status_code in [200, 201]:
logger.info(f"✓ Synced SLM {unit.id} to SLMM at {unit.slm_host}:{unit.slm_tcp_port or 2255}")
return True
else:
logger.error(f"Failed to sync SLM {unit.id} to SLMM: {response.status_code} {response.text}")
return False
except httpx.TimeoutException:
logger.error(f"Timeout syncing SLM {unit.id} to SLMM")
return False
except Exception as e:
logger.error(f"Error syncing SLM {unit.id} to SLMM: {e}")
return False
async def delete_slm_from_slmm(unit_id: str) -> bool:
"""
Delete a device from SLMM database.
Args:
unit_id: The unit ID to delete
Returns:
True if deletion successful or device doesn't exist, False on error
"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.delete(
f"{SLMM_BASE_URL}/api/nl43/{unit_id}/config"
)
if response.status_code == 200:
logger.info(f"✓ Deleted SLM {unit_id} from SLMM")
return True
elif response.status_code == 404:
logger.info(f"SLM {unit_id} not found in SLMM (already deleted)")
return True
else:
logger.error(f"Failed to delete SLM {unit_id} from SLMM: {response.status_code} {response.text}")
return False
except httpx.TimeoutException:
logger.error(f"Timeout deleting SLM {unit_id} from SLMM")
return False
except Exception as e:
logger.error(f"Error deleting SLM {unit_id} from SLMM: {e}")
return False
async def sync_all_slms_to_slmm(db: Session) -> dict:
"""
Sync all SLM devices from Terra-View roster to SLMM.
This ensures SLMM database matches Terra-View roster as the source of truth.
Should be called on Terra-View startup and optionally via admin endpoint.
Args:
db: Database session
Returns:
Dictionary with sync results
"""
logger.info("Starting full SLM sync to SLMM...")
# Get all SLM units from roster
slm_units = db.query(RosterUnit).filter_by(device_type="slm").all()
results = {
"total": len(slm_units),
"synced": 0,
"skipped": 0,
"failed": 0
}
for unit in slm_units:
# Skip units without host configured
if not unit.slm_host:
results["skipped"] += 1
logger.debug(f"Skipped {unit.unit_type} - no host configured")
continue
# Sync to SLMM
success = await sync_slm_to_slmm(unit)
if success:
results["synced"] += 1
else:
results["failed"] += 1
logger.info(
f"SLM sync complete: {results['synced']} synced, "
f"{results['skipped']} skipped, {results['failed']} failed"
)
return results
async def get_slmm_devices() -> Optional[list]:
"""
Get list of all devices currently in SLMM database.
Returns:
List of device unit_ids, or None on error
"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.get(f"{SLMM_BASE_URL}/api/nl43/_polling/status")
if response.status_code == 200:
data = response.json()
return [device["unit_id"] for device in data["data"]["devices"]]
else:
logger.error(f"Failed to get SLMM devices: {response.status_code}")
return None
except Exception as e:
logger.error(f"Error getting SLMM devices: {e}")
return None
async def cleanup_orphaned_slmm_devices(db: Session) -> dict:
"""
Remove devices from SLMM that are not in Terra-View roster.
This cleans up orphaned test devices or devices that were manually added to SLMM.
Args:
db: Database session
Returns:
Dictionary with cleanup results
"""
logger.info("Checking for orphaned devices in SLMM...")
# Get all device IDs from SLMM
slmm_devices = await get_slmm_devices()
if slmm_devices is None:
return {"error": "Failed to get SLMM device list"}
# Get all SLM unit IDs from Terra-View roster
roster_units = db.query(RosterUnit.id).filter_by(device_type="slm").all()
roster_unit_ids = {unit.id for unit in roster_units}
# Find orphaned devices (in SLMM but not in roster)
orphaned = [uid for uid in slmm_devices if uid not in roster_unit_ids]
results = {
"total_in_slmm": len(slmm_devices),
"total_in_roster": len(roster_unit_ids),
"orphaned": len(orphaned),
"deleted": 0,
"failed": 0,
"orphaned_devices": orphaned
}
if not orphaned:
logger.info("No orphaned devices found in SLMM")
return results
logger.info(f"Found {len(orphaned)} orphaned devices in SLMM: {orphaned}")
# Delete orphaned devices
for unit_id in orphaned:
success = await delete_slm_from_slmm(unit_id)
if success:
results["deleted"] += 1
else:
results["failed"] += 1
logger.info(
f"Cleanup complete: {results['deleted']} deleted, {results['failed']} failed"
)
return results