chore: docs and scripts organized. clutter cleared.

archive/README.md

@@ -0,0 +1,67 @@
+# SLMM Archive
+
+This directory contains legacy scripts that are no longer needed for normal operation but are preserved for reference.
+
+## Legacy Migrations (`legacy_migrations/`)
+
+These migration scripts were used during SLMM development (v0.1.x) to incrementally add database fields. They are **no longer needed** because:
+
+1. **Fresh databases** get the complete schema automatically from `app/models.py`
+2. **Existing databases** should already have these fields from previous runs
+3. **Current migration** is `migrate_add_polling_fields.py` (v0.2.0) in the parent directory
+
+### Archived Migration Files
+
+- `migrate_add_counter.py` - Added `counter` field to NL43Status
+- `migrate_add_measurement_start_time.py` - Added `measurement_start_time` field
+- `migrate_add_ftp_port.py` - Added `ftp_port` field to NL43Config
+- `migrate_field_names.py` - Renamed fields for consistency (one-time fix)
+- `migrate_revert_field_names.py` - Rollback for the rename migration
+
+**Do not delete** - These provide historical context for database schema evolution.
+
+---
+
+## Legacy Tools
+
+### `nl43_dod_poll.py`
+
+Manual polling script that queries a single NL-43 device for DOD (Device On-Demand) data.
+
+**Status**: Replaced by background polling system in v0.2.0
+
+**Why archived**:
+- Background poller (`app/background_poller.py`) now handles continuous polling automatically
+- No need for manual polling scripts
+- Kept for reference in case manual querying is needed for debugging
+
+**How to use** (if needed):
+```bash
+cd /home/serversdown/tmi/slmm/archive
+python3 nl43_dod_poll.py <host> <port> <unit_id>
+```
+
+---
+
+## Active Scripts (Still in Parent Directory)
+
+These scripts are **actively used** and documented in the main README:
+
+### Migrations
+- `migrate_add_polling_fields.py` - **v0.2.0 migration** - Adds background polling fields
+- `migrate_add_ftp_credentials.py` - **Legacy FTP migration** - Adds FTP auth fields
+
+### Testing
+- `test_polling.sh` - Comprehensive test suite for background polling features
+- `test_settings_endpoint.py` - Tests device settings API
+- `test_sleep_mode_auto_disable.py` - Tests automatic sleep mode handling
+
+### Utilities
+- `set_ftp_credentials.py` - Command-line tool to set FTP credentials for a device
+
+---
+
+## Version History
+
+- **v0.2.0** (2026-01-15) - Background polling system added, manual polling scripts archived
+- **v0.1.0** (2025-12-XX) - Initial release with incremental migrations

archive/legacy_migrations/migrate_add_counter.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Database migration: Add counter field to nl43_status table
+
+This adds the d0 (measurement interval counter) field to track the device's
+actual measurement progress for accurate timer synchronization.
+"""
+
+import sqlite3
+import sys
+
+DB_PATH = "data/slmm.db"
+
+def migrate():
+    print(f"Adding counter field to: {DB_PATH}")
+
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+
+        # Check if counter column already exists
+        cursor.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cursor.fetchall()]
+
+        if 'counter' in columns:
+            print("✓ Counter column already exists, no migration needed")
+            conn.close()
+            return
+
+        print("Starting migration...")
+
+        # Add counter column
+        cursor.execute("""
+            ALTER TABLE nl43_status
+            ADD COLUMN counter TEXT
+        """)
+
+        conn.commit()
+        print("✓ Added counter column")
+
+        # Verify
+        cursor.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cursor.fetchall()]
+
+        if 'counter' not in columns:
+            raise Exception("Counter column was not added successfully")
+
+        print("✓ Migration completed successfully")
+
+        conn.close()
+
+    except Exception as e:
+        print(f"✗ Migration failed: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    migrate()

archive/legacy_migrations/migrate_add_ftp_port.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""
+Migration script to add ftp_port column to nl43_config table.
+
+Usage:
+    python migrate_add_ftp_port.py
+"""
+
+import sqlite3
+import sys
+from pathlib import Path
+
+def migrate():
+    db_path = Path("data/slmm.db")
+
+    if not db_path.exists():
+        print(f"❌ Database not found at {db_path}")
+        print("   Run this script from the slmm directory")
+        return False
+
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        # Check if column already exists
+        cursor.execute("PRAGMA table_info(nl43_config)")
+        columns = [row[1] for row in cursor.fetchall()]
+
+        if "ftp_port" in columns:
+            print("✓ ftp_port column already exists")
+            conn.close()
+            return True
+
+        print("Adding ftp_port column to nl43_config table...")
+
+        # Add the ftp_port column with default value of 21
+        cursor.execute("""
+            ALTER TABLE nl43_config
+            ADD COLUMN ftp_port INTEGER DEFAULT 21
+        """)
+
+        conn.commit()
+        print("✓ Migration completed successfully")
+        print("  Added ftp_port column (default: 21)")
+
+        conn.close()
+        return True
+
+    except Exception as e:
+        print(f"❌ Migration failed: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = migrate()
+    sys.exit(0 if success else 1)

archive/legacy_migrations/migrate_add_measurement_start_time.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Database migration: Add measurement_start_time field to nl43_status table
+
+This tracks when a measurement session started by detecting the state transition
+from "Stop" to "Measure", enabling accurate elapsed time display even for
+manually-started measurements.
+"""
+
+import sqlite3
+import sys
+
+DB_PATH = "data/slmm.db"
+
+def migrate():
+    print(f"Adding measurement_start_time field to: {DB_PATH}")
+
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+
+        # Check if measurement_start_time column already exists
+        cursor.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cursor.fetchall()]
+
+        if 'measurement_start_time' in columns:
+            print("✓ measurement_start_time column already exists, no migration needed")
+            conn.close()
+            return
+
+        print("Starting migration...")
+
+        # Add measurement_start_time column
+        cursor.execute("""
+            ALTER TABLE nl43_status
+            ADD COLUMN measurement_start_time TEXT
+        """)
+
+        conn.commit()
+        print("✓ Added measurement_start_time column")
+
+        # Verify
+        cursor.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cursor.fetchall()]
+
+        if 'measurement_start_time' not in columns:
+            raise Exception("measurement_start_time column was not added successfully")
+
+        print("✓ Migration completed successfully")
+
+        conn.close()
+
+    except Exception as e:
+        print(f"✗ Migration failed: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    migrate()

archive/legacy_migrations/migrate_field_names.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+Migration script to rename NL43 measurement field names to match actual device output.
+
+Changes:
+- lp -> laeq (A-weighted equivalent continuous sound level)
+- leq -> lae (A-weighted sound exposure level)
+- lmax -> lasmax (A-weighted slow maximum)
+- lmin -> lasmin (A-weighted slow minimum)
+- lpeak -> lapeak (A-weighted peak)
+"""
+
+import sqlite3
+import sys
+from pathlib import Path
+
+def migrate_database(db_path: str):
+    """Migrate the database schema to use correct field names."""
+
+    print(f"Migrating database: {db_path}")
+
+    # Connect to database
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+
+    try:
+        # Check if migration is needed
+        cur.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cur.fetchall()]
+
+        if 'laeq' in columns:
+            print("✓ Database already migrated")
+            return
+
+        if 'lp' not in columns:
+            print("✗ Database schema does not match expected format")
+            sys.exit(1)
+
+        print("Starting migration...")
+
+        # SQLite doesn't support column renaming directly, so we need to:
+        # 1. Create new table with correct column names
+        # 2. Copy data from old table
+        # 3. Drop old table
+        # 4. Rename new table
+
+        # Create new table with correct column names
+        cur.execute("""
+            CREATE TABLE nl43_status_new (
+                unit_id VARCHAR PRIMARY KEY,
+                last_seen DATETIME,
+                measurement_state VARCHAR,
+                laeq VARCHAR,
+                lae VARCHAR,
+                lasmax VARCHAR,
+                lasmin VARCHAR,
+                lapeak VARCHAR,
+                battery_level VARCHAR,
+                power_source VARCHAR,
+                sd_remaining_mb VARCHAR,
+                sd_free_ratio VARCHAR,
+                raw_payload TEXT
+            )
+        """)
+        print("✓ Created new table with correct column names")
+
+        # Copy data from old table to new table
+        cur.execute("""
+            INSERT INTO nl43_status_new
+            (unit_id, last_seen, measurement_state, laeq, lae, lasmax, lasmin, lapeak,
+             battery_level, power_source, sd_remaining_mb, sd_free_ratio, raw_payload)
+            SELECT
+                unit_id, last_seen, measurement_state, lp, leq, lmax, lmin, lpeak,
+                battery_level, power_source, sd_remaining_mb, sd_free_ratio, raw_payload
+            FROM nl43_status
+        """)
+        rows_copied = cur.rowcount
+        print(f"✓ Copied {rows_copied} rows from old table")
+
+        # Drop old table
+        cur.execute("DROP TABLE nl43_status")
+        print("✓ Dropped old table")
+
+        # Rename new table
+        cur.execute("ALTER TABLE nl43_status_new RENAME TO nl43_status")
+        print("✓ Renamed new table to nl43_status")
+
+        # Commit changes
+        conn.commit()
+        print("✓ Migration completed successfully")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"✗ Migration failed: {e}")
+        sys.exit(1)
+    finally:
+        conn.close()
+
+if __name__ == "__main__":
+    # Default database path
+    db_path = Path(__file__).parent / "data" / "slmm.db"
+
+    # Allow custom path as command line argument
+    if len(sys.argv) > 1:
+        db_path = Path(sys.argv[1])
+
+    if not db_path.exists():
+        print(f"✗ Database not found: {db_path}")
+        sys.exit(1)
+
+    migrate_database(str(db_path))

archive/legacy_migrations/migrate_revert_field_names.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Migration script to revert NL43 measurement field names back to correct DRD format.
+
+The previous migration was incorrect. According to NL43 DRD documentation:
+- d0 = counter (1-600) - NOT a measurement!
+- d1 = Lp (instantaneous sound pressure level)
+- d2 = Leq (equivalent continuous sound level)
+- d3 = Lmax (maximum level)
+- d4 = Lmin (minimum level)
+- d5 = Lpeak (peak level)
+
+Changes:
+- laeq -> lp (was incorrectly mapped to counter field!)
+- lae -> leq
+- lasmax -> lmax
+- lasmin -> lmin
+- lapeak -> lpeak
+"""
+
+import sqlite3
+import sys
+from pathlib import Path
+
+def migrate_database(db_path: str):
+    """Revert database schema to correct DRD field names."""
+
+    print(f"Reverting database migration: {db_path}")
+
+    # Connect to database
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+
+    try:
+        # Check if migration is needed
+        cur.execute("PRAGMA table_info(nl43_status)")
+        columns = [row[1] for row in cur.fetchall()]
+
+        if 'lp' in columns:
+            print("✓ Database already has correct field names")
+            return
+
+        if 'laeq' not in columns:
+            print("✗ Database schema does not match expected format")
+            sys.exit(1)
+
+        print("Starting revert migration...")
+
+        # Create new table with correct column names
+        cur.execute("""
+            CREATE TABLE nl43_status_new (
+                unit_id VARCHAR PRIMARY KEY,
+                last_seen DATETIME,
+                measurement_state VARCHAR,
+                lp VARCHAR,
+                leq VARCHAR,
+                lmax VARCHAR,
+                lmin VARCHAR,
+                lpeak VARCHAR,
+                battery_level VARCHAR,
+                power_source VARCHAR,
+                sd_remaining_mb VARCHAR,
+                sd_free_ratio VARCHAR,
+                raw_payload TEXT
+            )
+        """)
+        print("✓ Created new table with correct DRD field names")
+
+        # Copy data from old table to new table
+        # Note: laeq was incorrectly mapped to d0 (counter), so we discard it
+        # The actual measurements start from d1
+        cur.execute("""
+            INSERT INTO nl43_status_new
+            (unit_id, last_seen, measurement_state, lp, leq, lmax, lmin, lpeak,
+             battery_level, power_source, sd_remaining_mb, sd_free_ratio, raw_payload)
+            SELECT
+                unit_id, last_seen, measurement_state, lae, lasmax, lasmin, lapeak, NULL,
+                battery_level, power_source, sd_remaining_mb, sd_free_ratio, raw_payload
+            FROM nl43_status
+        """)
+        rows_copied = cur.rowcount
+        print(f"✓ Copied {rows_copied} rows (note: discarded incorrect 'laeq' counter field)")
+
+        # Drop old table
+        cur.execute("DROP TABLE nl43_status")
+        print("✓ Dropped old table")
+
+        # Rename new table
+        cur.execute("ALTER TABLE nl43_status_new RENAME TO nl43_status")
+        print("✓ Renamed new table to nl43_status")
+
+        # Commit changes
+        conn.commit()
+        print("✓ Revert migration completed successfully")
+        print("\nNote: The 'lp' field will be populated correctly on next device measurement")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"✗ Migration failed: {e}")
+        sys.exit(1)
+    finally:
+        conn.close()
+
+if __name__ == "__main__":
+    # Default database path
+    db_path = Path(__file__).parent / "data" / "slmm.db"
+
+    # Allow custom path as command line argument
+    if len(sys.argv) > 1:
+        db_path = Path(sys.argv[1])
+
+    if not db_path.exists():
+        print(f"✗ Database not found: {db_path}")
+        sys.exit(1)
+
+    migrate_database(str(db_path))

archive/nl43_dod_poll.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""
+Diagnostic poller for NL-43 TCP connectivity.
+
+Every interval, open a TCP connection, send DOD?, read response, and log results.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import socket
+import time
+from pathlib import Path
+
+# ---- Configuration (edit as needed) ----
+HOST = "192.168.0.10"
+PORT = 2255
+INTERVAL_SECONDS = 5 * 60
+CONNECT_TIMEOUT_SECONDS = 5.0
+READ_TIMEOUT_SECONDS = 5.0
+LOG_PATH = Path("nl43_dod_poll.log")
+# ---------------------------------------
+
+
+def _timestamp() -> str:
+    return dt.datetime.utcnow().isoformat(timespec="seconds") + "Z"
+
+
+def _read_line(sock_file) -> str:
+    line = sock_file.readline()
+    if not line:
+        raise ConnectionError("Socket closed before full response")
+    return line.decode("ascii", errors="ignore").strip()
+
+
+def _poll_once() -> tuple[bool, str, str, str, str]:
+    sock = None
+    result_code = ""
+    data_line = ""
+    try:
+        sock = socket.create_connection((HOST, PORT), timeout=CONNECT_TIMEOUT_SECONDS)
+        sock.settimeout(READ_TIMEOUT_SECONDS)
+
+        sock.sendall(b"DOD?\r\n")
+
+        with sock.makefile("rb") as sock_file:
+            result_code = _read_line(sock_file)
+            if result_code.startswith("$"):
+                result_code = result_code[1:].strip()
+
+            if result_code != "R+0000":
+                return False, "other", f"device_result={result_code}", result_code, data_line
+
+            data_line = _read_line(sock_file)
+            if data_line.startswith("$"):
+                data_line = data_line[1:].strip()
+
+        return True, "none", "ok", result_code, data_line
+    except socket.timeout:
+        return False, "timeout", "socket_timeout", result_code, data_line
+    except ConnectionRefusedError:
+        return False, "refused", "connection_refused", result_code, data_line
+    except OSError as exc:
+        return False, "other", f"os_error={exc.__class__.__name__}", result_code, data_line
+    except Exception as exc:
+        return False, "other", f"error={exc.__class__.__name__}", result_code, data_line
+    finally:
+        if sock is not None:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            sock.close()
+
+
+def _log_line(text: str) -> None:
+    print(text, flush=True)
+    with LOG_PATH.open("a", encoding="ascii") as handle:
+        handle.write(text + "\n")
+
+
+def main() -> None:
+    while True:
+        start = time.monotonic()
+        ok, error_type, detail, result_code, data_line = _poll_once()
+
+        status = "success" if ok else "failure"
+        msg = (
+            f"ts={_timestamp()} status={status} error_type={error_type} "
+            f"detail={detail} result_code={result_code} data={data_line}"
+        )
+        _log_line(msg)
+
+        elapsed = time.monotonic() - start
+        sleep_for = max(0.0, INTERVAL_SECONDS - elapsed)
+        time.sleep(sleep_for)
+
+
+if __name__ == "__main__":
+    main()