diff --git a/CHANGELOG.md b/CHANGELOG.md index 77478d7..4983325 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,40 @@ All notable changes to seismo-relay are documented here. --- +## v0.22.0 — 2026-07-03 + +Full-snapshot bundle support (SFM side). Adds the DB-snapshot, recent-waveform, +and gated-restore endpoints Terra-View drives to pull a complete prod→dev refresh +— the `seismo_relay.db` plus the waveform files behind recent events — in one +pass. Pairs with **Terra-View v0.17.0**'s "Create full snapshot (incl. SFM)" +control on Settings → Database. New logic lives in `sfm/db_snapshot.py`, kept +out of `server.py` so it's unit-testable without HTTP. + +### Added + +- **`GET /db/snapshot`** — streams a WAL-safe point-in-time copy of + `seismo_relay.db`, taken through the SQLite online backup API (not a raw + file grab), so the download stays consistent even while the ACH server is + writing under WAL. +- **`GET /db/waveforms/recent.zip?n=…`** — zips the on-disk files (event file + + its `.h5` / sidecar siblings) for the *n* most-recent events, arcname + `/`, so a dev box can restore chart-openable events without + hauling the entire waveform store. Events with no resolvable files on disk + are skipped. +- **`POST /db/restore`** — gated by `SFM_DB_RESTORE_ENABLED` (dormant → 404 + unless explicitly enabled; dev-only). Validates the uploaded DB first, takes + a WAL-safe `.pre-restore-` safety backup of the current DB, then swaps in + the restored DB + waveforms. Zip entries are path-traversal-guarded before + extraction. + +### Fixed + +- **Restore no longer blocks the event loop.** `/db/restore` runs its + synchronous DB/file work in a threadpool instead of `async def`, so a large + restore can't stall inbound ACH connections or other API requests. + +--- + ## v0.21.1 — 2026-06-01 Bug fixes against v0.21.0 surfaced after the first prod redeploy. Three diff --git a/README.md b/README.md index 8d41f7e..791f0b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# seismo-relay `v0.21.0` +# seismo-relay `v0.22.0` A ground-up replacement for **Blastware** — Instantel's aging Windows-only software for managing seismographs. Supports both the **MiniMate Plus @@ -54,6 +54,11 @@ over direct RS-232 or cellular modem (Sierra Wireless RV50 / RV55). > flow through the existing Event Report PDF pipeline without a > separate renderer. Terra-View v0.13.0 ships in parallel and > closes Phase 1 of the SFM integration — see its CHANGELOG. +> **v0.22.0 (2026-07-03)** adds the SFM side of the full-snapshot +> bundle: `GET /db/snapshot` (WAL-safe DB copy), `GET /db/waveforms/ +> recent.zip` (recent events' waveform files), and a gated +> `POST /db/restore` (`SFM_DB_RESTORE_ENABLED`, dev-only). Terra-View +> v0.17.0 drives them to pull a one-pass prod→dev refresh. > See [CHANGELOG.md](CHANGELOG.md) for full version history. --- @@ -210,6 +215,18 @@ optionally paired with their ASCII sidecar reports; both dedup by | `POST` | `/db/import/blastware_file` | Series III: `.AB0*` / `.N00` binaries + paired `_ASCII.TXT`. Source: `series3-watcher`. | | `POST` | `/db/import/idf_file` | Series IV: `.IDFH` / `.IDFW` binaries + paired `.IDFW.txt` / `.IDFH.txt`. Source: `thor-watcher`. | +### DB snapshot / restore endpoints + +Back the "full snapshot bundle" prod→dev refresh Terra-View orchestrates +(Settings → Database). Snapshot/zip are read-only; restore is gated and +dev-only. + +| Method | URL | Description | +|--------|-----|-------------| +| `GET` | `/db/snapshot` | WAL-safe point-in-time copy of `seismo_relay.db` via the SQLite online backup API. | +| `GET` | `/db/waveforms/recent.zip?n=…` | Zips the on-disk files (event file + `.h5` / sidecars) for the *n* most-recent events. | +| `POST` | `/db/restore` | Validate-first restore of an uploaded DB + waveforms; takes a WAL-safe pre-restore safety backup, path-traversal-guards the zip. Gated by `SFM_DB_RESTORE_ENABLED` (dormant → 404). | + --- ## minimateplus library diff --git a/pyproject.toml b/pyproject.toml index dafdc73..313e73d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "seismo-relay" -version = "0.21.1" +version = "0.22.0" description = "Python client and REST server for MiniMate Plus seismographs" requires-python = ">=3.10" dependencies = [ diff --git a/sfm/db_snapshot.py b/sfm/db_snapshot.py new file mode 100644 index 0000000..84531ab --- /dev/null +++ b/sfm/db_snapshot.py @@ -0,0 +1,106 @@ +""" +db_snapshot.py — WAL-safe snapshot / restore helpers for seismo_relay.db and +the waveform store, backing the /db/snapshot, /db/waveforms/recent.zip and +/db/restore endpoints in server.py. Kept out of server.py so the logic is +unit-testable without HTTP. +""" +from __future__ import annotations +import os +import shutil +import sqlite3 +import zipfile +import datetime +from pathlib import Path + + +def snapshot_db(src_path: Path, dest_path: Path) -> Path: + """WAL-safe copy of a live SQLite DB via the SQLite online backup API. + Consistent even while the source is being written under WAL, and never + blocks the source's writers.""" + src = sqlite3.connect(str(src_path)) + dest = sqlite3.connect(str(dest_path)) + try: + with dest: + src.backup(dest) + finally: + src.close() + dest.close() + return Path(dest_path) + + +def build_recent_waveforms_zip(db, store, n: int, dest_path: Path) -> dict: + """Zip every on-disk file (event file + .h5/.a5.pkl/.sfm.json siblings) for + the n most-recent events, arcname '/'. Events with no + resolvable files on disk are skipped. Returns {'events': int, 'files': int}.""" + rows = db.query_events(limit=n, offset=0) + events = 0 + files = 0 + with zipfile.ZipFile(dest_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for row in rows: + serial = row.get("serial") + fn = row.get("blastware_filename") + if not serial or not fn: + continue + serial_dir = store.root / serial + matched = sorted(serial_dir.glob(f"{fn}*")) if serial_dir.is_dir() else [] + matched = [p for p in matched if p.is_file()] + if not matched: + continue + events += 1 + for p in matched: + zf.write(p, arcname=f"{serial}/{p.name}") + files += 1 + return {"events": events, "files": files} + + +def sfm_db_restore_enabled() -> bool: + return os.getenv("SFM_DB_RESTORE_ENABLED", "").strip().lower() in ("1", "true", "yes", "on") + + +def _assert_sqlite(path: Path) -> None: + with open(path, "rb") as f: + head = f.read(16) + if head != b"SQLite format 3\x00": + raise ValueError(f"{path} is not a SQLite database") + + +def restore_db_and_waveforms(db_path: Path, waveform_root: Path, + new_db_path: Path, waveforms_zip_path) -> dict: + """Safety-backup the current seismo_relay.db, atomically swap in + new_db_path, clear stale -wal/-shm sidecars, then unpack the waveforms zip + under waveform_root. SFM opens a fresh connection per call, so an in-place + atomic replace is safe.""" + db_path = Path(db_path) + new_db_path = Path(new_db_path) + _assert_sqlite(new_db_path) + + safety_name = None + if db_path.exists(): + ts = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") + safety = db_path.with_name(f"{db_path.name}.pre-restore-{ts}") + snapshot_db(db_path, safety) + safety_name = safety.name + + # Clear the OUTGOING db's WAL sidecars so they cannot attach to the incoming file. + for sfx in ("-wal", "-shm"): + side = Path(str(db_path) + sfx) + if side.exists(): + side.unlink() + + incoming = db_path.with_name(db_path.name + ".incoming") + shutil.copy2(new_db_path, incoming) + os.replace(incoming, db_path) # atomic within the same directory + + unpacked = 0 + if waveforms_zip_path and Path(waveforms_zip_path).exists(): + waveform_root = Path(waveform_root) + waveform_root.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(waveforms_zip_path) as zf: + for m in zf.namelist(): + if m.endswith("/") or m.startswith("/") or ".." in Path(m).parts: + continue + zf.extract(m, waveform_root) + unpacked += 1 + + return {"ok": True, "db_restored": True, + "waveforms_unpacked": unpacked, "safety_backup": safety_name} diff --git a/sfm/server.py b/sfm/server.py index ed42775..f41098f 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -72,6 +72,7 @@ from sfm.cache import SFMCache, get_cache from sfm.database import SeismoDb from sfm.live_cache import LiveCache as _LiveCache from sfm.waveform_store import WaveformStore +from sfm import db_snapshot logging.basicConfig( level=logging.INFO, @@ -2808,6 +2809,70 @@ def db_unit_waveforms_zip( return StreamingResponse(buf, media_type="application/zip", headers=headers) +# ── Full-snapshot bundle endpoints (see sfm/db_snapshot.py) ─────────────────── +# SFM_DB_RESTORE_ENABLED gates /db/restore (dormant → 404). Set true on dev only. +@app.get("/db/snapshot") +def db_snapshot_download() -> FileResponse: + """WAL-safe copy of seismo_relay.db, for the terra-view full-snapshot bundle.""" + import os + import tempfile + from starlette.background import BackgroundTask + fd, tmp = tempfile.mkstemp(suffix=".db") + os.close(fd) + db_snapshot.snapshot_db(_get_db().db_path, Path(tmp)) + return FileResponse( + tmp, media_type="application/x-sqlite3", filename="seismo_relay.db", + background=BackgroundTask(os.unlink, tmp), + ) + + +@app.get("/db/waveforms/recent.zip") +def db_waveforms_recent_zip(n: int = Query(200, ge=1, le=2000)) -> FileResponse: + """Zip of the n most-recent events' waveform + sidecar files.""" + import os + import tempfile + from starlette.background import BackgroundTask + fd, tmp = tempfile.mkstemp(suffix=".zip") + os.close(fd) + stats = db_snapshot.build_recent_waveforms_zip(_get_db(), _get_store(), n, Path(tmp)) + return FileResponse( + tmp, media_type="application/zip", filename="waveforms.zip", + headers={"X-Event-Count": str(stats["events"]), "X-File-Count": str(stats["files"])}, + background=BackgroundTask(os.unlink, tmp), + ) + + +@app.post("/db/restore") +def db_restore( + db: UploadFile = File(...), + waveforms: Optional[UploadFile] = File(None), +) -> dict: + """Destructive: swap seismo_relay.db + unpack waveforms. Dormant (404) unless + SFM_DB_RESTORE_ENABLED is set.""" + import os + import shutil as _shutil + import tempfile + if not db_snapshot.sfm_db_restore_enabled(): + raise HTTPException(status_code=404, detail="Not found") + + def _save(upload) -> Path: + fd, tmp = tempfile.mkstemp() + with os.fdopen(fd, "wb") as out: + _shutil.copyfileobj(upload.file, out) + return Path(tmp) + + db_tmp = _save(db) + wf_tmp = _save(waveforms) if waveforms is not None else None + try: + return db_snapshot.restore_db_and_waveforms( + _get_db().db_path, _get_store().root, db_tmp, wf_tmp, + ) + finally: + db_tmp.unlink(missing_ok=True) + if wf_tmp is not None: + wf_tmp.unlink(missing_ok=True) + + @app.get("/db/monitor_log") def db_monitor_log( serial: Optional[str] = Query(None, description="Filter by unit serial"), diff --git a/tests/test_db_snapshot.py b/tests/test_db_snapshot.py new file mode 100644 index 0000000..9101353 --- /dev/null +++ b/tests/test_db_snapshot.py @@ -0,0 +1,163 @@ +""" +test_db_snapshot.py — unit tests for sfm/db_snapshot.py (snapshot/restore +helpers behind the /db/snapshot, /db/waveforms/recent.zip, /db/restore routes). + +Run: + python -m pytest tests/test_db_snapshot.py -v +""" +from __future__ import annotations +import os +import sys +import sqlite3 +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def _make_db(path: Path, rows: int = 3) -> None: + conn = sqlite3.connect(str(path)) + conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + conn.executemany("INSERT INTO t (v) VALUES (?)", [(f"row{i}",) for i in range(rows)]) + conn.commit() + conn.close() + + +def _seed_event(db_path: Path, serial: str, fn: str, ts: str) -> None: + conn = sqlite3.connect(str(db_path)) + conn.execute( + "INSERT INTO events (id, serial, waveform_key, timestamp, " + "blastware_filename, a5_pickle_filename, sidecar_filename, false_trigger) " + "VALUES (?, ?, ?, ?, ?, ?, ?, 0)", + (f"id-{fn}", serial, fn[:8], ts, fn, f"{fn}.a5.pkl", f"{fn}.sfm.json"), + ) + conn.commit() + conn.close() + + +def test_snapshot_db_produces_queryable_copy(tmp_path): + from sfm.db_snapshot import snapshot_db + src = tmp_path / "src.db" + _make_db(src, rows=5) + dest = tmp_path / "snap.db" + out = snapshot_db(src, dest) + assert out == dest and dest.exists() + conn = sqlite3.connect(str(dest)) + n = conn.execute("SELECT COUNT(*) FROM t").fetchone()[0] + conn.close() + assert n == 5 + + +def test_recent_waveforms_zip_bundles_all_per_event_files(tmp_path): + import zipfile + from sfm.database import SeismoDb + from sfm.waveform_store import WaveformStore + from sfm.db_snapshot import build_recent_waveforms_zip + + db = SeismoDb(tmp_path / "seismo_relay.db") # creates schema on init + store = WaveformStore(tmp_path / "waveforms") + + def add(serial, fn, ts): + d = store.root / serial + d.mkdir(parents=True, exist_ok=True) + for suffix in ("", ".h5", ".a5.pkl", ".sfm.json"): + (d / f"{fn}{suffix}").write_bytes(b"data") + _seed_event(db.db_path, serial, fn, ts) + + add("BE111", "AAAA1111.OE0H", "2026-06-01T00:00:00") + add("BE111", "BBBB2222.OE0H", "2026-06-02T00:00:00") # newest + + dest = tmp_path / "wf.zip" + stats = build_recent_waveforms_zip(db, store, n=1, dest_path=dest) + + assert stats["events"] == 1 + assert stats["files"] == 4 + with zipfile.ZipFile(dest) as zf: + names = set(zf.namelist()) + assert names == { + "BE111/BBBB2222.OE0H", + "BE111/BBBB2222.OE0H.h5", + "BE111/BBBB2222.OE0H.a5.pkl", + "BE111/BBBB2222.OE0H.sfm.json", + } + + +def test_sfm_db_restore_enabled_reads_env(monkeypatch): + from sfm.db_snapshot import sfm_db_restore_enabled + monkeypatch.delenv("SFM_DB_RESTORE_ENABLED", raising=False) + assert sfm_db_restore_enabled() is False + monkeypatch.setenv("SFM_DB_RESTORE_ENABLED", "true") + assert sfm_db_restore_enabled() is True + monkeypatch.setenv("SFM_DB_RESTORE_ENABLED", "0") + assert sfm_db_restore_enabled() is False + + +def test_restore_swaps_db_and_unpacks_waveforms(tmp_path): + import zipfile + from sfm.db_snapshot import restore_db_and_waveforms + + db_path = tmp_path / "seismo_relay.db" + _make_db(db_path, rows=1) + (tmp_path / "seismo_relay.db-wal").write_bytes(b"stale") # must be cleared + + new_db = tmp_path / "incoming.db" + _make_db(new_db, rows=9) + + wf_root = tmp_path / "waveforms" + wf_root.mkdir() + wf_zip = tmp_path / "wf.zip" + with zipfile.ZipFile(wf_zip, "w") as zf: + zf.writestr("BE111/AAAA.OE0H", b"x") + zf.writestr("BE111/AAAA.OE0H.h5", b"x") + + result = restore_db_and_waveforms(db_path, wf_root, new_db, wf_zip) + + conn = sqlite3.connect(str(db_path)) + assert conn.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 9 + conn.close() + assert not (tmp_path / "seismo_relay.db-wal").exists() + assert (wf_root / "BE111" / "AAAA.OE0H.h5").exists() + assert result["ok"] and result["waveforms_unpacked"] == 2 + assert result["safety_backup"] is not None + + +def test_restore_rejects_non_sqlite(tmp_path): + import pytest + from sfm.db_snapshot import restore_db_and_waveforms + db_path = tmp_path / "seismo_relay.db" + _make_db(db_path, rows=1) + bogus = tmp_path / "bogus.db" + bogus.write_bytes(b"not a sqlite file at all") + wf_root = tmp_path / "waveforms" + wf_root.mkdir() + original_bytes = db_path.read_bytes() + with pytest.raises(ValueError): + restore_db_and_waveforms(db_path, wf_root, bogus, None) + assert db_path.read_bytes() == original_bytes + assert list(tmp_path.glob("*.pre-restore-*")) == [] + + +def test_restore_skips_unsafe_zip_members(tmp_path): + import zipfile + from sfm.db_snapshot import restore_db_and_waveforms + + db_path = tmp_path / "seismo_relay.db" + _make_db(db_path, rows=1) + + new_db = tmp_path / "incoming.db" + _make_db(new_db, rows=2) + + wf_root = tmp_path / "waveforms" + wf_root.mkdir() + wf_zip = tmp_path / "wf.zip" + with zipfile.ZipFile(wf_zip, "w") as zf: + zf.writestr("BE111/ok.h5", b"good") + zf.writestr("../evil.txt", b"escape via parent dir") + zf.writestr("/tmp/abs_evil.txt", b"escape via absolute path") + zf.writestr("baddir/", b"") + + result = restore_db_and_waveforms(db_path, wf_root, new_db, wf_zip) + + assert (wf_root / "BE111" / "ok.h5").exists() + assert result["waveforms_unpacked"] == 1 + assert not (wf_root.parent / "evil.txt").exists() + assert not Path("/tmp/abs_evil.txt").exists() diff --git a/tests/test_db_snapshot_endpoints.py b/tests/test_db_snapshot_endpoints.py new file mode 100644 index 0000000..e5dd44b --- /dev/null +++ b/tests/test_db_snapshot_endpoints.py @@ -0,0 +1,53 @@ +""" +test_db_snapshot_endpoints.py — HTTP smoke tests for the /db/snapshot, +/db/waveforms/recent.zip and /db/restore routes. Skipped if the FastAPI +TestClient stack (httpx) is not installed. + +Run: + python -m pytest tests/test_db_snapshot_endpoints.py -v +""" +from __future__ import annotations +import os +import sys +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import pytest + +pytest.importorskip("httpx") # starlette TestClient needs httpx + + +def _client(tmp_path, monkeypatch): + import sfm.server as srv + from sfm.database import SeismoDb + from sfm.waveform_store import WaveformStore + from starlette.testclient import TestClient + db = SeismoDb(tmp_path / "seismo_relay.db") + store = WaveformStore(tmp_path / "waveforms") + monkeypatch.setattr(srv, "_db", db, raising=False) + monkeypatch.setattr(srv, "_store", store, raising=False) + return TestClient(srv.app) + + +def test_snapshot_endpoint_returns_sqlite(tmp_path, monkeypatch): + client = _client(tmp_path, monkeypatch) + r = client.get("/db/snapshot") + assert r.status_code == 200 + assert r.content[:16] == b"SQLite format 3\x00" + + +def test_recent_waveforms_endpoint_ok(tmp_path, monkeypatch): + client = _client(tmp_path, monkeypatch) + r = client.get("/db/waveforms/recent.zip", params={"n": 5}) + assert r.status_code == 200 + assert r.headers["content-type"] == "application/zip" + + +def test_restore_dormant_without_env(tmp_path, monkeypatch): + monkeypatch.delenv("SFM_DB_RESTORE_ENABLED", raising=False) + client = _client(tmp_path, monkeypatch) + r = client.post( + "/db/restore", + files={"db": ("x.db", b"SQLite format 3\x00", "application/octet-stream")}, + ) + assert r.status_code == 404