feat(reports): FTP night-report pipeline foundation
Terra-View side of the daily night-vs-baseline sound report for the John Myler 24/7 job. Engine is built and verified end-to-end against real meter data; SMTP send + scheduler/capture wiring still pending. - ingest: refactor upload_nrl_data into a callable ingest_nrl_zip(location_id, zip_bytes, db) sharing one core with the HTTP endpoint. Capture the .rnh percentile map + weightings into session metadata; dedup on store-name + start time. Ingest stays metric-agnostic (every Leq column preserved). - report_pipeline.py: metric registry, Evening/Nighttime windows, correct aggregation (Lmax=max, Ln=arithmetic, Leq=logarithmic), baseline = typical night, per-location + per-project builders. - report_renderers.py: HTML email-body renderer (Last/Base/delta layout). - report_email.py: config-driven SMTP via stdlib (env vars) with a dry-run fallback so the pipeline runs without credentials. - report_orchestrator.py: compute -> render -> always write report.html + report.json to disk -> best-effort email. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Nightly Report Orchestrator.
|
||||
|
||||
Ties the pieces together: compute → render → write-to-disk → email.
|
||||
|
||||
This is what the daily cycle (or a manual trigger) calls. It ALWAYS writes the
|
||||
rendered report to disk — `data/reports/{project_id}/{night_date}/report.html`
|
||||
(+ `report.json` with the raw numbers) — so there's a viewable artifact even
|
||||
when email is in dry-run (SMTP not configured yet). The email step is
|
||||
best-effort and never aborts the run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from backend.services.report_pipeline import (
|
||||
ProjectNightReport, build_project_night_report, Window,
|
||||
)
|
||||
from backend.services.report_renderers import render_html_summary
|
||||
from backend.services.report_email import send_report_email, Attachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_OUTPUT_ROOT = "data/reports"
|
||||
|
||||
|
||||
def _report_to_dict(report: ProjectNightReport) -> dict:
|
||||
"""Serialise the report data model to plain JSON (for the on-disk record)."""
|
||||
return {
|
||||
"project_id": report.project_id,
|
||||
"project_name": report.project_name,
|
||||
"night_date": report.night_date.isoformat(),
|
||||
"metrics": [m.key for m in report.metrics],
|
||||
"locations": [
|
||||
{
|
||||
"name": loc.location_name,
|
||||
"night_interval_count": loc.night_interval_count,
|
||||
"baseline_nights_used": loc.baseline_nights_used,
|
||||
"notes": loc.notes,
|
||||
"windows": {
|
||||
w.key: {
|
||||
"label": w.label,
|
||||
"metrics": {
|
||||
m.key: {
|
||||
"label": m.label,
|
||||
"last_night": loc.table[w.key][m.key].last_night,
|
||||
"baseline": loc.table[w.key][m.key].baseline,
|
||||
"delta": loc.table[w.key][m.key].delta,
|
||||
}
|
||||
for m in loc.metrics
|
||||
},
|
||||
}
|
||||
for w in loc.windows
|
||||
},
|
||||
}
|
||||
for loc in report.locations
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_nightly_report(
|
||||
db: Session,
|
||||
project_id: str,
|
||||
night_date: date,
|
||||
*,
|
||||
metric_keys: Optional[list[str]] = None,
|
||||
windows: Optional[list[Window]] = None,
|
||||
baseline_start: Optional[date] = None,
|
||||
baseline_end: Optional[date] = None,
|
||||
recipients: Optional[list[str]] = None,
|
||||
output_root: str = DEFAULT_OUTPUT_ROOT,
|
||||
send: bool = True,
|
||||
) -> dict:
|
||||
"""Build, persist, and (dry-run) email the night report for a project.
|
||||
|
||||
Returns a result dict with the on-disk artifact paths and the email result.
|
||||
Designed to be called from the daily cycle or a manual trigger.
|
||||
"""
|
||||
report = build_project_night_report(
|
||||
db, project_id, night_date,
|
||||
metric_keys=metric_keys, windows=windows,
|
||||
baseline_start=baseline_start, baseline_end=baseline_end,
|
||||
)
|
||||
|
||||
html = render_html_summary(report)
|
||||
subject = f"{report.project_name} — night report {night_date:%m/%d/%y}"
|
||||
|
||||
# --- Always persist a viewable copy ---
|
||||
out_dir = Path(output_root) / project_id / f"{night_date:%Y-%m-%d}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
html_path = out_dir / "report.html"
|
||||
html_path.write_text(html, encoding="utf-8")
|
||||
json_path = out_dir / "report.json"
|
||||
json_path.write_text(json.dumps(_report_to_dict(report), indent=2), encoding="utf-8")
|
||||
|
||||
# --- Excel attachment (renderer not built yet — held until metric set is final) ---
|
||||
attachments: list[Attachment] = []
|
||||
|
||||
# --- Email (best-effort; dry-run until SMTP is configured) ---
|
||||
email_result = {"sent": False, "dry_run": False, "skipped": True, "error": None}
|
||||
if send:
|
||||
email_result = send_report_email(
|
||||
subject, html, attachments=attachments, recipients=recipients,
|
||||
)
|
||||
|
||||
result = {
|
||||
"project_id": project_id,
|
||||
"project_name": report.project_name,
|
||||
"night_date": night_date.isoformat(),
|
||||
"subject": subject,
|
||||
"location_count": len(report.locations),
|
||||
"html_path": str(html_path),
|
||||
"json_path": str(json_path),
|
||||
"email": email_result,
|
||||
}
|
||||
logger.info(
|
||||
"Nightly report for %s (%s): %d location(s) → %s; email=%s",
|
||||
report.project_name, night_date, len(report.locations), html_path,
|
||||
"sent" if email_result.get("sent") else
|
||||
("dry-run" if email_result.get("dry_run") else
|
||||
("skipped" if email_result.get("skipped") else f"error: {email_result.get('error')}")),
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user