serversdown
fdd0426884
- #1 XSS: escape user-controlled values (location name, baseline values, recent- report fields, SMTP status message) in the modals via the existing _mergeEsc helper — they were concatenated raw into innerHTML (stored XSS via location name). - #2 SMTP: an unrecognized REPORT_SMTP_SECURITY no longer silently downgrades to a plaintext connection while still calling login() — it falls back to starttls and warns; warn on intentional security=none + auth. - #3 scheduler: run the (blocking smtplib + Excel) nightly report in a worker thread (asyncio.to_thread + its own DB session) so it can't stall the loop that drives time-sensitive device cycles. New _run_one_report helper. - #4 cycle ingest: set unit_id on the ingested data session (ingest_nrl_zip leaves it None) before dropping the empty placeholder, preserving the unit<->session link; repoint old_session_id at the real row. - #7 robustness: wrap send_report_email in the orchestrator and run_nightly_report in /view + /run so a render/SMTP error returns a clean error instead of a raw 500 after artifacts are written. Verified: SMTP paths (typo->starttls, none, starttls, ssl), off-thread tick stamps last_run_date + writes the file, /view 200, escaping wired, app imports. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
435 lines
19 KiB
Python
435 lines
19 KiB
Python
"""
|
|
Nightly Report Router.
|
|
|
|
Manual triggers for the night-vs-baseline sound report — the same entry point
|
|
the scheduled morning tick will reuse. Two endpoints:
|
|
|
|
GET …/reports/nightly/view → render and return the HTML inline (preview).
|
|
No write, no email. Browser-friendly.
|
|
POST …/reports/nightly/run → full run: build → write report.html/json to
|
|
disk → (dry-run) email. Returns JSON result.
|
|
|
|
Dates are the *evening* date of the night being reported (the 7/7 in "night of
|
|
7/7 → morning 7/8"). Defaults to last night. Baseline is optional; pass the
|
|
baseline-week range to populate the comparison.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import uuid
|
|
from datetime import datetime, timedelta, date
|
|
from html import escape
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
from fastapi.responses import HTMLResponse
|
|
from sqlalchemy.orm import Session
|
|
|
|
from backend.database import get_db
|
|
from backend.models import Project, SoundReportConfig, MonitoringLocation
|
|
from backend.services.report_pipeline import (
|
|
METRIC_REGISTRY, DEFAULT_METRICS, DEFAULT_WINDOWS, _location_reference_baseline,
|
|
)
|
|
from backend.services.report_orchestrator import run_nightly_report
|
|
from backend.utils.timezone import utc_to_local
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(prefix="/api/projects/{project_id}/reports", tags=["reports"])
|
|
|
|
|
|
def _default_night_date() -> date:
|
|
"""Last night = yesterday in the user's local timezone."""
|
|
return (utc_to_local(datetime.utcnow()) - timedelta(days=1)).date()
|
|
|
|
|
|
def _parse_date(s: Optional[str], field: str) -> Optional[date]:
|
|
if not s:
|
|
return None
|
|
try:
|
|
return datetime.strptime(s, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise HTTPException(status_code=400, detail=f"{field} must be YYYY-MM-DD (got {s!r})")
|
|
|
|
|
|
def _parse_metrics(s: Optional[str]) -> list[str]:
|
|
if not s:
|
|
return list(DEFAULT_METRICS)
|
|
keys = [k.strip().lower() for k in s.split(",") if k.strip()]
|
|
unknown = [k for k in keys if k not in METRIC_REGISTRY]
|
|
if unknown:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Unknown metric(s): {unknown}. Known: {sorted(METRIC_REGISTRY)}",
|
|
)
|
|
return keys or list(DEFAULT_METRICS)
|
|
|
|
|
|
def _validate_hhmm(s) -> str:
|
|
"""Validate a local HH:MM (24h) time string."""
|
|
try:
|
|
hh, mm = str(s).split(":")
|
|
h, m = int(hh), int(mm)
|
|
if 0 <= h < 24 and 0 <= m < 60:
|
|
return f"{h:02d}:{m:02d}"
|
|
except (ValueError, AttributeError):
|
|
pass
|
|
raise HTTPException(status_code=400, detail=f"report_time must be HH:MM 24-hour (got {s!r})")
|
|
|
|
|
|
def _config_dict(cfg: Optional[SoundReportConfig], project_id: str) -> dict:
|
|
"""Serialise a config row (or defaults if none yet) to JSON."""
|
|
return {
|
|
"project_id": project_id,
|
|
"exists": cfg is not None,
|
|
"enabled": cfg.enabled if cfg else False,
|
|
"report_time": cfg.report_time if cfg else "08:00",
|
|
"metric_keys": cfg.metric_keys if cfg else ",".join(DEFAULT_METRICS),
|
|
"baseline_mode": cfg.baseline_mode if cfg else "captured",
|
|
"baseline_start": cfg.baseline_start.isoformat() if cfg and cfg.baseline_start else None,
|
|
"baseline_end": cfg.baseline_end.isoformat() if cfg and cfg.baseline_end else None,
|
|
"recipients": (cfg.recipients if cfg and cfg.recipients else ""),
|
|
"last_run_date": cfg.last_run_date.isoformat() if cfg and cfg.last_run_date else None,
|
|
}
|
|
|
|
|
|
@router.get("/config")
|
|
async def get_report_config(project_id: str, db: Session = Depends(get_db)):
|
|
"""Return the project's nightly-report config (or defaults if not set yet)."""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
return _config_dict(cfg, project_id)
|
|
|
|
|
|
@router.put("/config")
|
|
async def put_report_config(project_id: str, request: Request, db: Session = Depends(get_db)):
|
|
"""Create or update the project's nightly-report config (JSON body)."""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
data = await request.json()
|
|
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
created = cfg is None
|
|
if cfg is None:
|
|
cfg = SoundReportConfig(id=str(uuid.uuid4()), project_id=project_id)
|
|
db.add(cfg)
|
|
|
|
if "enabled" in data:
|
|
cfg.enabled = bool(data["enabled"])
|
|
if "report_time" in data:
|
|
cfg.report_time = _validate_hhmm(data["report_time"])
|
|
if "metric_keys" in data:
|
|
mk = data["metric_keys"]
|
|
mk = mk if isinstance(mk, str) else ",".join(mk or [])
|
|
cfg.metric_keys = ",".join(_parse_metrics(mk))
|
|
if "baseline_mode" in data:
|
|
bm = str(data["baseline_mode"]).lower()
|
|
if bm not in ("captured", "reference"):
|
|
raise HTTPException(status_code=400, detail="baseline_mode must be 'captured' or 'reference'")
|
|
cfg.baseline_mode = bm
|
|
if "baseline_start" in data or "baseline_end" in data:
|
|
bs = _parse_date(data.get("baseline_start") or None, "baseline_start")
|
|
be = _parse_date(data.get("baseline_end") or None, "baseline_end")
|
|
if (bs and not be) or (be and not bs):
|
|
raise HTTPException(status_code=400, detail="Provide both baseline dates, or neither.")
|
|
if bs and be and bs > be:
|
|
raise HTTPException(status_code=400, detail="baseline_start must be on or before baseline_end.")
|
|
cfg.baseline_start, cfg.baseline_end = bs, be
|
|
if "recipients" in data:
|
|
recips = data["recipients"]
|
|
if isinstance(recips, list):
|
|
recips = ",".join(recips)
|
|
cfg.recipients = (recips or "").strip() or None
|
|
|
|
db.commit()
|
|
db.refresh(cfg)
|
|
return {**_config_dict(cfg, project_id), "created": created}
|
|
|
|
|
|
def _resolve_params(project_id, db, night_date, baseline_start, baseline_end, metrics):
|
|
"""Validate inputs and resolve the baseline source.
|
|
|
|
Explicit baseline dates in the query override (captured mode with those
|
|
dates). Otherwise the project's saved config supplies the baseline (its
|
|
mode + dates) and the default metric set — so the manual view/run match
|
|
what the scheduled report does.
|
|
Returns (night_date, baseline_mode, baseline_start, baseline_end, metric_keys).
|
|
"""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
nd = _parse_date(night_date, "night_date") or _default_night_date()
|
|
bs = _parse_date(baseline_start, "baseline_start")
|
|
be = _parse_date(baseline_end, "baseline_end")
|
|
if (bs and not be) or (be and not bs):
|
|
raise HTTPException(status_code=400, detail="Provide both baseline_start and baseline_end, or neither.")
|
|
if bs and be and bs > be:
|
|
raise HTTPException(status_code=400, detail="baseline_start must be on or before baseline_end.")
|
|
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
if bs and be:
|
|
baseline_mode = "captured" # explicit dates win
|
|
elif cfg:
|
|
baseline_mode = cfg.baseline_mode # fall back to saved config
|
|
bs, be = cfg.baseline_start, cfg.baseline_end
|
|
else:
|
|
baseline_mode = "captured"
|
|
|
|
if metrics:
|
|
metric_keys = _parse_metrics(metrics)
|
|
elif cfg and cfg.metric_keys:
|
|
metric_keys = _parse_metrics(cfg.metric_keys)
|
|
else:
|
|
metric_keys = list(DEFAULT_METRICS)
|
|
|
|
return nd, baseline_mode, bs, be, metric_keys
|
|
|
|
|
|
@router.get("/nightly/view", response_class=HTMLResponse)
|
|
async def view_nightly_report(
|
|
project_id: str,
|
|
night_date: Optional[str] = Query(None, description="Evening date of the night (YYYY-MM-DD). Default: last night."),
|
|
baseline_start: Optional[str] = Query(None, description="Baseline range start (YYYY-MM-DD)."),
|
|
baseline_end: Optional[str] = Query(None, description="Baseline range end (YYYY-MM-DD)."),
|
|
metrics: Optional[str] = Query(None, description="Comma list, e.g. lmax,l01,l10,l90. Default: house set."),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Render the night report and return the HTML inline (preview — no write, no email)."""
|
|
nd, bmode, bs, be, metric_keys = _resolve_params(project_id, db, night_date, baseline_start, baseline_end, metrics)
|
|
try:
|
|
result = run_nightly_report(
|
|
db, project_id, nd,
|
|
metric_keys=metric_keys, baseline_mode=bmode, baseline_start=bs, baseline_end=be,
|
|
send=False, # preview: no email
|
|
)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e: # noqa: BLE001
|
|
logger.error("nightly/view failed for %s (%s): %s", project_id, nd, e, exc_info=True)
|
|
raise HTTPException(status_code=500, detail=f"Report generation failed: {e}")
|
|
return HTMLResponse(result["html"])
|
|
|
|
|
|
@router.post("/nightly/run")
|
|
async def run_nightly_report_endpoint(
|
|
project_id: str,
|
|
night_date: Optional[str] = Query(None, description="Evening date of the night (YYYY-MM-DD). Default: last night."),
|
|
baseline_start: Optional[str] = Query(None, description="Baseline range start (YYYY-MM-DD)."),
|
|
baseline_end: Optional[str] = Query(None, description="Baseline range end (YYYY-MM-DD)."),
|
|
metrics: Optional[str] = Query(None, description="Comma list, e.g. lmax,l01,l10,l90. Default: house set."),
|
|
send: bool = Query(True, description="Attempt email (dry-run until SMTP is configured)."),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Run the night report: build → write report.html/report.json to disk → email (best-effort).
|
|
|
|
This is the same path the scheduled morning tick will call. The `html` field
|
|
is omitted from the JSON response (it's large and on disk); use /view to see it.
|
|
"""
|
|
nd, bmode, bs, be, metric_keys = _resolve_params(project_id, db, night_date, baseline_start, baseline_end, metrics)
|
|
try:
|
|
result = run_nightly_report(
|
|
db, project_id, nd,
|
|
metric_keys=metric_keys, baseline_mode=bmode, baseline_start=bs, baseline_end=be,
|
|
send=send,
|
|
)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e: # noqa: BLE001
|
|
logger.error("nightly/run failed for %s (%s): %s", project_id, nd, e, exc_info=True)
|
|
raise HTTPException(status_code=500, detail=f"Report generation failed: {e}")
|
|
result.pop("html", None) # keep the JSON response lean — view it via /view or the file
|
|
result["view_url"] = (
|
|
f"/api/projects/{project_id}/reports/nightly/view"
|
|
f"?night_date={nd:%Y-%m-%d}"
|
|
+ (f"&baseline_start={bs:%Y-%m-%d}&baseline_end={be:%Y-%m-%d}" if bs and be else "")
|
|
+ (f"&metrics={','.join(metric_keys)}")
|
|
)
|
|
return result
|
|
|
|
|
|
# ============================================================================
|
|
# Test email + generated-report archive
|
|
# ============================================================================
|
|
|
|
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
|
|
|
|
@router.post("/test-email")
|
|
async def send_test_email(project_id: str, request: Request, db: Session = Depends(get_db)):
|
|
"""Send a small test email to verify the SMTP relay (dry-run if unconfigured).
|
|
|
|
Recipients: JSON body {"recipients": "..."} overrides; else the project's
|
|
configured recipients; else the REPORT_SMTP_RECIPIENTS env default.
|
|
"""
|
|
project = db.query(Project).filter_by(id=project_id).first()
|
|
if not project:
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
try:
|
|
data = await request.json()
|
|
except Exception:
|
|
data = {}
|
|
|
|
raw = (data or {}).get("recipients")
|
|
if not raw:
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
raw = cfg.recipients if cfg else None
|
|
recipients = None
|
|
if raw:
|
|
if isinstance(raw, list):
|
|
raw = ",".join(raw)
|
|
recipients = [r.strip() for r in raw.split(",") if r.strip()]
|
|
|
|
from backend.services.report_email import send_report_email
|
|
body = (
|
|
"<div style=\"font:14px Arial,sans-serif\">"
|
|
f"Terra-View test email for <b>{escape(project.name)}</b>.<br>"
|
|
"If you got this, the nightly sound-report email path is working.</div>"
|
|
)
|
|
return send_report_email("Terra-View — nightly report test email", body, recipients=recipients)
|
|
|
|
|
|
@router.get("/list")
|
|
async def list_reports(project_id: str, db: Session = Depends(get_db)):
|
|
"""List the generated report artifacts on disk for this project (newest first)."""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
base = Path("data/reports") / project_id
|
|
out = []
|
|
if base.exists():
|
|
for d in sorted((p for p in base.iterdir() if p.is_dir()), key=lambda p: p.name, reverse=True):
|
|
html_file = d / "report.html"
|
|
if html_file.exists():
|
|
st = html_file.stat()
|
|
out.append({
|
|
"night_date": d.name,
|
|
"view_url": f"/api/projects/{project_id}/reports/archive/{d.name}",
|
|
"xlsx_url": (f"/api/projects/{project_id}/reports/archive/{d.name}/xlsx"
|
|
if (d / "report.xlsx").exists() else None),
|
|
"size_bytes": st.st_size,
|
|
"generated_at": datetime.utcfromtimestamp(st.st_mtime).isoformat(),
|
|
})
|
|
return {"reports": out, "count": len(out)}
|
|
|
|
|
|
@router.get("/archive/{night_date}", response_class=HTMLResponse)
|
|
async def view_archived_report(project_id: str, night_date: str, db: Session = Depends(get_db)):
|
|
"""Serve a previously generated report.html from disk (the actual artifact)."""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
if not _DATE_RE.match(night_date):
|
|
raise HTTPException(status_code=400, detail="Invalid date (YYYY-MM-DD)")
|
|
safe = _parse_date(night_date, "night_date") # also guards path traversal
|
|
path = Path("data/reports") / project_id / f"{safe:%Y-%m-%d}" / "report.html"
|
|
if not path.exists():
|
|
raise HTTPException(status_code=404, detail="No saved report for that date")
|
|
return HTMLResponse(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
@router.get("/archive/{night_date}/xlsx")
|
|
async def download_archived_xlsx(project_id: str, night_date: str, db: Session = Depends(get_db)):
|
|
"""Download a previously generated report.xlsx from disk."""
|
|
from fastapi.responses import Response
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
if not _DATE_RE.match(night_date):
|
|
raise HTTPException(status_code=400, detail="Invalid date (YYYY-MM-DD)")
|
|
safe = _parse_date(night_date, "night_date")
|
|
path = Path("data/reports") / project_id / f"{safe:%Y-%m-%d}" / "report.xlsx"
|
|
if not path.exists():
|
|
raise HTTPException(status_code=404, detail="No saved spreadsheet for that date")
|
|
return Response(
|
|
content=path.read_bytes(),
|
|
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
headers={"Content-Disposition": f'attachment; filename="night_report_{safe:%Y-%m-%d}.xlsx"'},
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# Reference baseline (fixed values typed per location — limits / prior averages)
|
|
# ============================================================================
|
|
|
|
@router.get("/baseline")
|
|
async def get_baseline(project_id: str, db: Session = Depends(get_db)):
|
|
"""Return the baseline mode + per-location reference values + the metric/window
|
|
grid to render the editor."""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
mode = cfg.baseline_mode if cfg else "captured"
|
|
metric_keys = _parse_metrics(cfg.metric_keys) if cfg and cfg.metric_keys else list(DEFAULT_METRICS)
|
|
|
|
locations = db.query(MonitoringLocation).filter_by(
|
|
project_id=project_id, location_type="sound",
|
|
).order_by(MonitoringLocation.sort_order, MonitoringLocation.name).all()
|
|
locations = [l for l in locations if getattr(l, "removed_at", None) is None]
|
|
|
|
return {
|
|
"mode": mode,
|
|
"windows": [{"key": w.key, "label": w.label} for w in DEFAULT_WINDOWS],
|
|
"metrics": [{"key": k, "label": METRIC_REGISTRY[k].label} for k in metric_keys],
|
|
"locations": [
|
|
{"id": loc.id, "name": loc.name, "values": _location_reference_baseline(loc)}
|
|
for loc in locations
|
|
],
|
|
}
|
|
|
|
|
|
@router.put("/baseline")
|
|
async def put_baseline(project_id: str, request: Request, db: Session = Depends(get_db)):
|
|
"""Save the baseline mode (on config) and per-location reference values
|
|
(on each location's metadata). Body:
|
|
{"mode": "reference",
|
|
"locations": {"<loc_id>": {"nighttime": {"l10": 85}, "evening": {...}}}}
|
|
"""
|
|
if not db.query(Project).filter_by(id=project_id).first():
|
|
raise HTTPException(status_code=404, detail="Project not found")
|
|
data = await request.json()
|
|
|
|
if "mode" in data:
|
|
bm = str(data["mode"]).lower()
|
|
if bm not in ("captured", "reference"):
|
|
raise HTTPException(status_code=400, detail="mode must be 'captured' or 'reference'")
|
|
cfg = db.query(SoundReportConfig).filter_by(project_id=project_id).first()
|
|
if cfg is None:
|
|
cfg = SoundReportConfig(id=str(uuid.uuid4()), project_id=project_id)
|
|
db.add(cfg)
|
|
cfg.baseline_mode = bm
|
|
|
|
loc_values = data.get("locations") or {}
|
|
updated = 0
|
|
for loc_id, windows in loc_values.items():
|
|
loc = db.query(MonitoringLocation).filter_by(id=loc_id, project_id=project_id).first()
|
|
if not loc or not isinstance(windows, dict):
|
|
continue
|
|
try:
|
|
meta = json.loads(loc.location_metadata or "{}")
|
|
except (json.JSONDecodeError, TypeError):
|
|
meta = {}
|
|
clean: dict = {}
|
|
for wkey, mvals in windows.items():
|
|
if not isinstance(mvals, dict):
|
|
continue
|
|
cm = {}
|
|
for mkey, val in mvals.items():
|
|
if val in (None, ""):
|
|
continue
|
|
try:
|
|
cm[mkey] = round(float(val), 1)
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if cm:
|
|
clean[wkey] = cm
|
|
if clean:
|
|
meta["report_baseline"] = clean
|
|
else:
|
|
meta.pop("report_baseline", None)
|
|
loc.location_metadata = json.dumps(meta)
|
|
updated += 1
|
|
|
|
db.commit()
|
|
return {"ok": True, "locations_updated": updated}
|