Merge pull request 'Update to v0.22.0' (#30) from dev into main

Reviewed-on: #30

CHANGELOG.md

@@ -8,6 +8,40 @@ All notable changes to seismo-relay are documented here.
 
 ---
 
+## v0.22.0 — 2026-07-03
+
+Full-snapshot bundle support (SFM side).  Adds the DB-snapshot, recent-waveform,
+and gated-restore endpoints Terra-View drives to pull a complete prod→dev refresh
+— the `seismo_relay.db` plus the waveform files behind recent events — in one
+pass.  Pairs with **Terra-View v0.17.0**'s "Create full snapshot (incl. SFM)"
+control on Settings → Database.  New logic lives in `sfm/db_snapshot.py`, kept
+out of `server.py` so it's unit-testable without HTTP.
+
+### Added
+
+- **`GET /db/snapshot`** — streams a WAL-safe point-in-time copy of
+  `seismo_relay.db`, taken through the SQLite online backup API (not a raw
+  file grab), so the download stays consistent even while the ACH server is
+  writing under WAL.
+- **`GET /db/waveforms/recent.zip?n=…`** — zips the on-disk files (event file
+  + its `.h5` / sidecar siblings) for the *n* most-recent events, arcname
+  `<serial>/<filename>`, so a dev box can restore chart-openable events without
+  hauling the entire waveform store.  Events with no resolvable files on disk
+  are skipped.
+- **`POST /db/restore`** — gated by `SFM_DB_RESTORE_ENABLED` (dormant → 404
+  unless explicitly enabled; dev-only).  Validates the uploaded DB first, takes
+  a WAL-safe `.pre-restore-<ts>` safety backup of the current DB, then swaps in
+  the restored DB + waveforms.  Zip entries are path-traversal-guarded before
+  extraction.
+
+### Fixed
+
+- **Restore no longer blocks the event loop.**  `/db/restore` runs its
+  synchronous DB/file work in a threadpool instead of `async def`, so a large
+  restore can't stall inbound ACH connections or other API requests.
+
+---
+
 ## v0.21.1 — 2026-06-01
 
 Bug fixes against v0.21.0 surfaced after the first prod redeploy.  Three

README.md

@@ -1,4 +1,4 @@
-# seismo-relay  `v0.21.0`
+# seismo-relay  `v0.22.0`
 
 A ground-up replacement for **Blastware** — Instantel's aging Windows-only
 software for managing seismographs.  Supports both the **MiniMate Plus
@@ -54,6 +54,11 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55).
 > flow through the existing Event Report PDF pipeline without a
 > separate renderer.  Terra-View v0.13.0 ships in parallel and
 > closes Phase 1 of the SFM integration — see its CHANGELOG.
+> **v0.22.0 (2026-07-03)** adds the SFM side of the full-snapshot
+> bundle: `GET /db/snapshot` (WAL-safe DB copy), `GET /db/waveforms/
+> recent.zip` (recent events' waveform files), and a gated
+> `POST /db/restore` (`SFM_DB_RESTORE_ENABLED`, dev-only).  Terra-View
+> v0.17.0 drives them to pull a one-pass prod→dev refresh.
 > See [CHANGELOG.md](CHANGELOG.md) for full version history.
 
 ---
@@ -210,6 +215,18 @@ optionally paired with their ASCII sidecar reports; both dedup by
 | `POST` | `/db/import/blastware_file` | Series III: `.AB0*` / `.N00` binaries + paired `_ASCII.TXT`.  Source: `series3-watcher`. |
 | `POST` | `/db/import/idf_file` | Series IV: `.IDFH` / `.IDFW` binaries + paired `.IDFW.txt` / `.IDFH.txt`.  Source: `thor-watcher`. |
 
+### DB snapshot / restore endpoints
+
+Back the "full snapshot bundle" prod→dev refresh Terra-View orchestrates
+(Settings → Database).  Snapshot/zip are read-only; restore is gated and
+dev-only.
+
+| Method | URL | Description |
+|--------|-----|-------------|
+| `GET` | `/db/snapshot` | WAL-safe point-in-time copy of `seismo_relay.db` via the SQLite online backup API. |
+| `GET` | `/db/waveforms/recent.zip?n=…` | Zips the on-disk files (event file + `.h5` / sidecars) for the *n* most-recent events. |
+| `POST` | `/db/restore` | Validate-first restore of an uploaded DB + waveforms; takes a WAL-safe pre-restore safety backup, path-traversal-guards the zip.  Gated by `SFM_DB_RESTORE_ENABLED` (dormant → 404). |
+
 ---
 
 ## minimateplus library

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "seismo-relay"
-version = "0.21.1"
+version = "0.22.0"
 description = "Python client and REST server for MiniMate Plus seismographs"
 requires-python = ">=3.10"
 dependencies = [

sfm/db_snapshot.py

@@ -0,0 +1,106 @@
+"""
+db_snapshot.py — WAL-safe snapshot / restore helpers for seismo_relay.db and
+the waveform store, backing the /db/snapshot, /db/waveforms/recent.zip and
+/db/restore endpoints in server.py. Kept out of server.py so the logic is
+unit-testable without HTTP.
+"""
+from __future__ import annotations
+import os
+import shutil
+import sqlite3
+import zipfile
+import datetime
+from pathlib import Path
+
+
+def snapshot_db(src_path: Path, dest_path: Path) -> Path:
+    """WAL-safe copy of a live SQLite DB via the SQLite online backup API.
+    Consistent even while the source is being written under WAL, and never
+    blocks the source's writers."""
+    src = sqlite3.connect(str(src_path))
+    dest = sqlite3.connect(str(dest_path))
+    try:
+        with dest:
+            src.backup(dest)
+    finally:
+        src.close()
+        dest.close()
+    return Path(dest_path)
+
+
+def build_recent_waveforms_zip(db, store, n: int, dest_path: Path) -> dict:
+    """Zip every on-disk file (event file + .h5/.a5.pkl/.sfm.json siblings) for
+    the n most-recent events, arcname '<serial>/<filename>'. Events with no
+    resolvable files on disk are skipped. Returns {'events': int, 'files': int}."""
+    rows = db.query_events(limit=n, offset=0)
+    events = 0
+    files = 0
+    with zipfile.ZipFile(dest_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for row in rows:
+            serial = row.get("serial")
+            fn = row.get("blastware_filename")
+            if not serial or not fn:
+                continue
+            serial_dir = store.root / serial
+            matched = sorted(serial_dir.glob(f"{fn}*")) if serial_dir.is_dir() else []
+            matched = [p for p in matched if p.is_file()]
+            if not matched:
+                continue
+            events += 1
+            for p in matched:
+                zf.write(p, arcname=f"{serial}/{p.name}")
+                files += 1
+    return {"events": events, "files": files}
+
+
+def sfm_db_restore_enabled() -> bool:
+    return os.getenv("SFM_DB_RESTORE_ENABLED", "").strip().lower() in ("1", "true", "yes", "on")
+
+
+def _assert_sqlite(path: Path) -> None:
+    with open(path, "rb") as f:
+        head = f.read(16)
+    if head != b"SQLite format 3\x00":
+        raise ValueError(f"{path} is not a SQLite database")
+
+
+def restore_db_and_waveforms(db_path: Path, waveform_root: Path,
+                             new_db_path: Path, waveforms_zip_path) -> dict:
+    """Safety-backup the current seismo_relay.db, atomically swap in
+    new_db_path, clear stale -wal/-shm sidecars, then unpack the waveforms zip
+    under waveform_root. SFM opens a fresh connection per call, so an in-place
+    atomic replace is safe."""
+    db_path = Path(db_path)
+    new_db_path = Path(new_db_path)
+    _assert_sqlite(new_db_path)
+
+    safety_name = None
+    if db_path.exists():
+        ts = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+        safety = db_path.with_name(f"{db_path.name}.pre-restore-{ts}")
+        snapshot_db(db_path, safety)
+        safety_name = safety.name
+
+    # Clear the OUTGOING db's WAL sidecars so they cannot attach to the incoming file.
+    for sfx in ("-wal", "-shm"):
+        side = Path(str(db_path) + sfx)
+        if side.exists():
+            side.unlink()
+
+    incoming = db_path.with_name(db_path.name + ".incoming")
+    shutil.copy2(new_db_path, incoming)
+    os.replace(incoming, db_path)   # atomic within the same directory
+
+    unpacked = 0
+    if waveforms_zip_path and Path(waveforms_zip_path).exists():
+        waveform_root = Path(waveform_root)
+        waveform_root.mkdir(parents=True, exist_ok=True)
+        with zipfile.ZipFile(waveforms_zip_path) as zf:
+            for m in zf.namelist():
+                if m.endswith("/") or m.startswith("/") or ".." in Path(m).parts:
+                    continue
+                zf.extract(m, waveform_root)
+                unpacked += 1
+
+    return {"ok": True, "db_restored": True,
+            "waveforms_unpacked": unpacked, "safety_backup": safety_name}

sfm/server.py

@@ -72,6 +72,7 @@ from sfm.cache import SFMCache, get_cache
 from sfm.database import SeismoDb
 from sfm.live_cache import LiveCache as _LiveCache
 from sfm.waveform_store import WaveformStore
+from sfm import db_snapshot
 
 logging.basicConfig(
     level=logging.INFO,
@@ -2808,6 +2809,70 @@ def db_unit_waveforms_zip(
     return StreamingResponse(buf, media_type="application/zip", headers=headers)
 
 
+# ── Full-snapshot bundle endpoints (see sfm/db_snapshot.py) ───────────────────
+# SFM_DB_RESTORE_ENABLED gates /db/restore (dormant → 404). Set true on dev only.
+@app.get("/db/snapshot")
+def db_snapshot_download() -> FileResponse:
+    """WAL-safe copy of seismo_relay.db, for the terra-view full-snapshot bundle."""
+    import os
+    import tempfile
+    from starlette.background import BackgroundTask
+    fd, tmp = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    db_snapshot.snapshot_db(_get_db().db_path, Path(tmp))
+    return FileResponse(
+        tmp, media_type="application/x-sqlite3", filename="seismo_relay.db",
+        background=BackgroundTask(os.unlink, tmp),
+    )
+
+
+@app.get("/db/waveforms/recent.zip")
+def db_waveforms_recent_zip(n: int = Query(200, ge=1, le=2000)) -> FileResponse:
+    """Zip of the n most-recent events' waveform + sidecar files."""
+    import os
+    import tempfile
+    from starlette.background import BackgroundTask
+    fd, tmp = tempfile.mkstemp(suffix=".zip")
+    os.close(fd)
+    stats = db_snapshot.build_recent_waveforms_zip(_get_db(), _get_store(), n, Path(tmp))
+    return FileResponse(
+        tmp, media_type="application/zip", filename="waveforms.zip",
+        headers={"X-Event-Count": str(stats["events"]), "X-File-Count": str(stats["files"])},
+        background=BackgroundTask(os.unlink, tmp),
+    )
+
+
+@app.post("/db/restore")
+def db_restore(
+    db: UploadFile = File(...),
+    waveforms: Optional[UploadFile] = File(None),
+) -> dict:
+    """Destructive: swap seismo_relay.db + unpack waveforms. Dormant (404) unless
+    SFM_DB_RESTORE_ENABLED is set."""
+    import os
+    import shutil as _shutil
+    import tempfile
+    if not db_snapshot.sfm_db_restore_enabled():
+        raise HTTPException(status_code=404, detail="Not found")
+
+    def _save(upload) -> Path:
+        fd, tmp = tempfile.mkstemp()
+        with os.fdopen(fd, "wb") as out:
+            _shutil.copyfileobj(upload.file, out)
+        return Path(tmp)
+
+    db_tmp = _save(db)
+    wf_tmp = _save(waveforms) if waveforms is not None else None
+    try:
+        return db_snapshot.restore_db_and_waveforms(
+            _get_db().db_path, _get_store().root, db_tmp, wf_tmp,
+        )
+    finally:
+        db_tmp.unlink(missing_ok=True)
+        if wf_tmp is not None:
+            wf_tmp.unlink(missing_ok=True)
+
+
 @app.get("/db/monitor_log")
 def db_monitor_log(
     serial:  Optional[str] = Query(None, description="Filter by unit serial"),

tests/test_db_snapshot.py

@@ -0,0 +1,163 @@
+"""
+test_db_snapshot.py — unit tests for sfm/db_snapshot.py (snapshot/restore
+helpers behind the /db/snapshot, /db/waveforms/recent.zip, /db/restore routes).
+
+Run:
+    python -m pytest tests/test_db_snapshot.py -v
+"""
+from __future__ import annotations
+import os
+import sys
+import sqlite3
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def _make_db(path: Path, rows: int = 3) -> None:
+    conn = sqlite3.connect(str(path))
+    conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
+    conn.executemany("INSERT INTO t (v) VALUES (?)", [(f"row{i}",) for i in range(rows)])
+    conn.commit()
+    conn.close()
+
+
+def _seed_event(db_path: Path, serial: str, fn: str, ts: str) -> None:
+    conn = sqlite3.connect(str(db_path))
+    conn.execute(
+        "INSERT INTO events (id, serial, waveform_key, timestamp, "
+        "blastware_filename, a5_pickle_filename, sidecar_filename, false_trigger) "
+        "VALUES (?, ?, ?, ?, ?, ?, ?, 0)",
+        (f"id-{fn}", serial, fn[:8], ts, fn, f"{fn}.a5.pkl", f"{fn}.sfm.json"),
+    )
+    conn.commit()
+    conn.close()
+
+
+def test_snapshot_db_produces_queryable_copy(tmp_path):
+    from sfm.db_snapshot import snapshot_db
+    src = tmp_path / "src.db"
+    _make_db(src, rows=5)
+    dest = tmp_path / "snap.db"
+    out = snapshot_db(src, dest)
+    assert out == dest and dest.exists()
+    conn = sqlite3.connect(str(dest))
+    n = conn.execute("SELECT COUNT(*) FROM t").fetchone()[0]
+    conn.close()
+    assert n == 5
+
+
+def test_recent_waveforms_zip_bundles_all_per_event_files(tmp_path):
+    import zipfile
+    from sfm.database import SeismoDb
+    from sfm.waveform_store import WaveformStore
+    from sfm.db_snapshot import build_recent_waveforms_zip
+
+    db = SeismoDb(tmp_path / "seismo_relay.db")   # creates schema on init
+    store = WaveformStore(tmp_path / "waveforms")
+
+    def add(serial, fn, ts):
+        d = store.root / serial
+        d.mkdir(parents=True, exist_ok=True)
+        for suffix in ("", ".h5", ".a5.pkl", ".sfm.json"):
+            (d / f"{fn}{suffix}").write_bytes(b"data")
+        _seed_event(db.db_path, serial, fn, ts)
+
+    add("BE111", "AAAA1111.OE0H", "2026-06-01T00:00:00")
+    add("BE111", "BBBB2222.OE0H", "2026-06-02T00:00:00")   # newest
+
+    dest = tmp_path / "wf.zip"
+    stats = build_recent_waveforms_zip(db, store, n=1, dest_path=dest)
+
+    assert stats["events"] == 1
+    assert stats["files"] == 4
+    with zipfile.ZipFile(dest) as zf:
+        names = set(zf.namelist())
+    assert names == {
+        "BE111/BBBB2222.OE0H",
+        "BE111/BBBB2222.OE0H.h5",
+        "BE111/BBBB2222.OE0H.a5.pkl",
+        "BE111/BBBB2222.OE0H.sfm.json",
+    }
+
+
+def test_sfm_db_restore_enabled_reads_env(monkeypatch):
+    from sfm.db_snapshot import sfm_db_restore_enabled
+    monkeypatch.delenv("SFM_DB_RESTORE_ENABLED", raising=False)
+    assert sfm_db_restore_enabled() is False
+    monkeypatch.setenv("SFM_DB_RESTORE_ENABLED", "true")
+    assert sfm_db_restore_enabled() is True
+    monkeypatch.setenv("SFM_DB_RESTORE_ENABLED", "0")
+    assert sfm_db_restore_enabled() is False
+
+
+def test_restore_swaps_db_and_unpacks_waveforms(tmp_path):
+    import zipfile
+    from sfm.db_snapshot import restore_db_and_waveforms
+
+    db_path = tmp_path / "seismo_relay.db"
+    _make_db(db_path, rows=1)
+    (tmp_path / "seismo_relay.db-wal").write_bytes(b"stale")   # must be cleared
+
+    new_db = tmp_path / "incoming.db"
+    _make_db(new_db, rows=9)
+
+    wf_root = tmp_path / "waveforms"
+    wf_root.mkdir()
+    wf_zip = tmp_path / "wf.zip"
+    with zipfile.ZipFile(wf_zip, "w") as zf:
+        zf.writestr("BE111/AAAA.OE0H", b"x")
+        zf.writestr("BE111/AAAA.OE0H.h5", b"x")
+
+    result = restore_db_and_waveforms(db_path, wf_root, new_db, wf_zip)
+
+    conn = sqlite3.connect(str(db_path))
+    assert conn.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 9
+    conn.close()
+    assert not (tmp_path / "seismo_relay.db-wal").exists()
+    assert (wf_root / "BE111" / "AAAA.OE0H.h5").exists()
+    assert result["ok"] and result["waveforms_unpacked"] == 2
+    assert result["safety_backup"] is not None
+
+
+def test_restore_rejects_non_sqlite(tmp_path):
+    import pytest
+    from sfm.db_snapshot import restore_db_and_waveforms
+    db_path = tmp_path / "seismo_relay.db"
+    _make_db(db_path, rows=1)
+    bogus = tmp_path / "bogus.db"
+    bogus.write_bytes(b"not a sqlite file at all")
+    wf_root = tmp_path / "waveforms"
+    wf_root.mkdir()
+    original_bytes = db_path.read_bytes()
+    with pytest.raises(ValueError):
+        restore_db_and_waveforms(db_path, wf_root, bogus, None)
+    assert db_path.read_bytes() == original_bytes
+    assert list(tmp_path.glob("*.pre-restore-*")) == []
+
+
+def test_restore_skips_unsafe_zip_members(tmp_path):
+    import zipfile
+    from sfm.db_snapshot import restore_db_and_waveforms
+
+    db_path = tmp_path / "seismo_relay.db"
+    _make_db(db_path, rows=1)
+
+    new_db = tmp_path / "incoming.db"
+    _make_db(new_db, rows=2)
+
+    wf_root = tmp_path / "waveforms"
+    wf_root.mkdir()
+    wf_zip = tmp_path / "wf.zip"
+    with zipfile.ZipFile(wf_zip, "w") as zf:
+        zf.writestr("BE111/ok.h5", b"good")
+        zf.writestr("../evil.txt", b"escape via parent dir")
+        zf.writestr("/tmp/abs_evil.txt", b"escape via absolute path")
+        zf.writestr("baddir/", b"")
+
+    result = restore_db_and_waveforms(db_path, wf_root, new_db, wf_zip)
+
+    assert (wf_root / "BE111" / "ok.h5").exists()
+    assert result["waveforms_unpacked"] == 1
+    assert not (wf_root.parent / "evil.txt").exists()
+    assert not Path("/tmp/abs_evil.txt").exists()

tests/test_db_snapshot_endpoints.py

@@ -0,0 +1,53 @@
+"""
+test_db_snapshot_endpoints.py — HTTP smoke tests for the /db/snapshot,
+/db/waveforms/recent.zip and /db/restore routes. Skipped if the FastAPI
+TestClient stack (httpx) is not installed.
+
+Run:
+    python -m pytest tests/test_db_snapshot_endpoints.py -v
+"""
+from __future__ import annotations
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import pytest
+
+pytest.importorskip("httpx")   # starlette TestClient needs httpx
+
+
+def _client(tmp_path, monkeypatch):
+    import sfm.server as srv
+    from sfm.database import SeismoDb
+    from sfm.waveform_store import WaveformStore
+    from starlette.testclient import TestClient
+    db = SeismoDb(tmp_path / "seismo_relay.db")
+    store = WaveformStore(tmp_path / "waveforms")
+    monkeypatch.setattr(srv, "_db", db, raising=False)
+    monkeypatch.setattr(srv, "_store", store, raising=False)
+    return TestClient(srv.app)
+
+
+def test_snapshot_endpoint_returns_sqlite(tmp_path, monkeypatch):
+    client = _client(tmp_path, monkeypatch)
+    r = client.get("/db/snapshot")
+    assert r.status_code == 200
+    assert r.content[:16] == b"SQLite format 3\x00"
+
+
+def test_recent_waveforms_endpoint_ok(tmp_path, monkeypatch):
+    client = _client(tmp_path, monkeypatch)
+    r = client.get("/db/waveforms/recent.zip", params={"n": 5})
+    assert r.status_code == 200
+    assert r.headers["content-type"] == "application/zip"
+
+
+def test_restore_dormant_without_env(tmp_path, monkeypatch):
+    monkeypatch.delenv("SFM_DB_RESTORE_ENABLED", raising=False)
+    client = _client(tmp_path, monkeypatch)
+    r = client.post(
+        "/db/restore",
+        files={"db": ("x.db", b"SQLite format 3\x00", "application/octet-stream")},
+    )
+    assert r.status_code == 404