From 69e856db6a11ff2cd87c2b79418c186a59a29fe1 Mon Sep 17 00:00:00 2001 From: serversdwn Date: Tue, 10 Feb 2026 07:08:24 +0000 Subject: [PATCH] feat: update to version 0.1.3 with new CLI flags, logging enhancements, and Windows MSI installer --- .gitignore | Bin 52 -> 392 bytes CHANGELOG.md | 13 +++ README.md | 87 +++++++++++++++--- RELEASE.md | 39 ++++++++ installer/Product.wxs | 114 ++++++++++++++++++++++++ installer/README.md | 21 +++++ installer/build.ps1 | 30 +++++++ installer/tools/README.txt | 1 + series4_ingest.py | 176 +++++++++++++++++++++++++++---------- 9 files changed, 424 insertions(+), 57 deletions(-) create mode 100644 RELEASE.md create mode 100644 installer/Product.wxs create mode 100644 installer/README.md create mode 100644 installer/build.ps1 create mode 100644 installer/tools/README.txt diff --git a/.gitignore b/.gitignore index 4a4c7556cc056f4a56e3fb87447fd623e38ec21b..e2fc18f33eed2707d1e288e976680f912a0edd43 100644 GIT binary patch literal 392 zcmZXQ%L;-}5QhJC&^rk2TIh2GEm{P%3va1mCYn?)UwvonV##5|nfd3N%i}(ir`hou-NFe~@=`UHjnt{YPt?~Zcuw`DweV8< z40#(=+fTm>-j(Sh9sXi9+~^ZKLN0zdcf&6#|{mIdc(g32u2xtHR diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c0d36d..e71e62a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.3] - 2026-02-09 + +### Added +- CLI flags: `--config`, `--log-dir`, `--once`, `--version` +- File logging with rotation (default in `C:\ProgramData\ThorIngest\logs`) + +### Changed +- Default config location to `C:\ProgramData\ThorIngest\config.json` with fallback to local `config.json` +- Updated console output to use structured logging + +### Added +- Windows MSI installer scaffolding under `installer/` (WiX + NSSM setup) + ## [0.1.1] - 2025-12-08 ### Changed diff --git a/README.md b/README.md index a2e1514..accfd20 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Series 4 Ingest Agent -**Version:** 0.1.2 +**Version:** 0.1.3 Micromate (Series 4) ingest agent for Seismo Fleet Manager (SFM). @@ -27,10 +27,16 @@ Series 4 Ingest Agent is a Python-based monitoring tool that scans for Micromate ## Installation -1. Clone or copy this repository to your THOR PC/VM -2. Ensure Python 3.6+ is installed -3. Configure `config.json` (see Configuration section) -4. Run the agent: +1. Download and run the MSI installer (Windows, admin required) +2. Edit config at `C:\ProgramData\ThorIngest\config.json` +3. The service starts automatically and runs on boot + +No command line is required for normal use. The installer sets up the Windows Service. + +For development or manual runs: +1. Ensure Python 3.6+ is installed +2. Configure `config.json` (see Configuration section) +3. Run the agent: ```bash python series4_ingest.py @@ -38,7 +44,11 @@ python series4_ingest.py ## Configuration -All configuration is managed through `config.json` in the same directory as the script. +All configuration is managed through `config.json` in: + +``` +C:\ProgramData\ThorIngest\config.json +``` ### config.json Structure @@ -75,16 +85,35 @@ If `config.json` is missing or malformed, the agent will: ## Usage -### Start the Agent +### Start the Agent (manual) ```bash python series4_ingest.py ``` +### Common Flags + +- `--config `: Override config file path +- `--log-dir `: Override log directory +- `--once`: Run one scan and exit +- `--version`: Print version and exit + +### Run Once (manual) + +```bash +python series4_ingest.py --once +``` + +### Show Version + +```bash +python series4_ingest.py --version +``` + ### Console Output Example ``` -Series 4 Ingest Agent — Micromate Heartbeat (v0.1.2) +Series 4 Ingest Agent — Micromate Heartbeat (v0.1.3) THORDATA root: C:\THORDATA Now: 2025-12-08 14:30:00 -------------------------------------------------------------------------------- @@ -96,10 +125,48 @@ Total units: 3 Next scan in 60 seconds... ``` -### Stop the Agent +### Stop the Agent (manual) Press `Ctrl+C` to gracefully stop the agent. +## Windows Service Installation + +The MSI installs a Windows Service named `ThorIngest` using NSSM. It runs at startup. + +**Check status** + +``` +sc query ThorIngest +``` + +**Start / stop** + +``` +sc start ThorIngest +sc stop ThorIngest +``` + +**Service Logs** + +Logs are written to: +``` +C:\ProgramData\ThorIngest\logs\thor_ingest.log +``` + +## Updating + +1. Download the newer MSI from the internal file share/URL. +2. Run the MSI to upgrade in place. +3. Your existing config and logs are preserved. + +## Building the EXE (PyInstaller) + +```bash +pyinstaller --onefile --name thor-ingest-agent series4_ingest.py +``` + +The output EXE will be in `dist/`. + ## Status Classification Units are classified based on the age of their last MLG file: @@ -178,7 +245,7 @@ C:\THORDATA\ ## Troubleshooting ### Config file not found -If you see `[WARN] Config file not found`, create `config.json` in the same directory as `series4_ingest.py`. +If you see `[WARN] Config file not found`, create `config.json` in `C:\ProgramData\ThorIngest\config.json` (or pass `--config`). ### THORDATA path doesn't exist Verify the `thordata_path` in `config.json` points to the correct directory. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..2fda9c8 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,39 @@ +# Release Packaging (Windows) + +Goal: Users download a release artifact and install without using the command line. + +## Deliverables + +- `ThorIngest.msi` (primary installer) +- Optional: `thor-ingest-agent.exe` (for internal diagnostics) + +## Build Steps (internal) + +1. Build EXE: + +```bash +pyinstaller --onefile --name thor-ingest-agent series4_ingest.py +``` + +2. Place NSSM: + +``` +thor-ingest-agent/installer/tools/nssm.exe +``` + +3. Build MSI: + +```powershell +./installer/build.ps1 +``` + +4. Publish: + +- Upload `installer/ThorIngest.msi` to the internal release URL. +- Provide a short “Download and run the installer” instruction to users. + +## Install UX + +- Double‑click the MSI +- Minimal UI (Next/Install/Finish) +- Service installed and started automatically diff --git a/installer/Product.wxs b/installer/Product.wxs new file mode 100644 index 0000000..8d26523 --- /dev/null +++ b/installer/Product.wxs @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NOT Installed + NOT Installed + NOT Installed + NOT Installed + NOT Installed + NOT Installed + REMOVE="ALL" + + + + + + + diff --git a/installer/README.md b/installer/README.md new file mode 100644 index 0000000..0213959 --- /dev/null +++ b/installer/README.md @@ -0,0 +1,21 @@ +# Thor Ingest MSI Installer + +This folder contains the WiX MSI definition and build script. + +## Prerequisites + +- WiX Toolset v3.11+ (set `WIX` env var to the WiX `bin` path) +- `nssm.exe` placed at `installer/tools/nssm.exe` +- `thor-ingest-agent.exe` built into `thor-ingest-agent/dist/` + +## Build + +```powershell +./build.ps1 +``` + +The MSI output will be: + +``` +installer/ThorIngest.msi +``` diff --git a/installer/build.ps1 b/installer/build.ps1 new file mode 100644 index 0000000..c850fd5 --- /dev/null +++ b/installer/build.ps1 @@ -0,0 +1,30 @@ +$ErrorActionPreference = "Stop" + +$WixBin = $env:WIX +if (-not $WixBin) { + Write-Error "WIX environment variable not set. Set WIX to your WiX Toolset bin path (e.g., C:\Program Files (x86)\WiX Toolset v3.11\bin)." +} + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$InstallerDir = $ScriptDir +$RootDir = Resolve-Path (Join-Path $InstallerDir "..") + +$ProductWxs = Join-Path $InstallerDir "Product.wxs" +$DistDir = Join-Path $RootDir "dist" +$NssmPath = Join-Path $InstallerDir "tools\nssm.exe" + +if (-not (Test-Path $DistDir)) { + Write-Error "dist folder not found. Build the EXE with PyInstaller first." +} + +if (-not (Test-Path $NssmPath)) { + Write-Error "Missing nssm.exe at $NssmPath. Place the NSSM binary there before building." +} + +$candle = Join-Path $WixBin "candle.exe" +$light = Join-Path $WixBin "light.exe" + +& $candle -nologo $ProductWxs -out (Join-Path $InstallerDir "Product.wixobj") +& $light -nologo -ext WixUIExtension -ext WixUtilExtension -out (Join-Path $InstallerDir "ThorIngest.msi") (Join-Path $InstallerDir "Product.wixobj") + +Write-Host "Built MSI: $InstallerDir\ThorIngest.msi" diff --git a/installer/tools/README.txt b/installer/tools/README.txt new file mode 100644 index 0000000..ee3ed47 --- /dev/null +++ b/installer/tools/README.txt @@ -0,0 +1 @@ +Place nssm.exe in this folder before building the MSI. diff --git a/series4_ingest.py b/series4_ingest.py index 8da07e4..61b984a 100644 --- a/series4_ingest.py +++ b/series4_ingest.py @@ -1,5 +1,5 @@ """ -Series 4 Ingest Agent — v0.1.2 +Series 4 Ingest Agent — v0.1.3 Micromate (Series 4) ingest agent for Seismo Fleet Manager (SFM). @@ -14,6 +14,9 @@ Behavior: No roster. SFM backend decides what to do with each unit. """ +import argparse +import logging +from logging.handlers import RotatingFileHandler import os import re import time @@ -30,9 +33,14 @@ try: except ImportError: urllib = None # type: ignore +__version__ = "0.1.3" + # ---------------- Config ---------------- -def load_config(config_path: str = "config.json") -> Dict[str, Any]: +DEFAULT_CONFIG_PATH = r"C:\ProgramData\ThorIngest\config.json" +DEFAULT_LOG_DIR = r"C:\ProgramData\ThorIngest\logs" + +def load_config(config_path: Optional[str] = None) -> Dict[str, Any]: """ Load configuration from a JSON file. @@ -48,12 +56,28 @@ def load_config(config_path: str = "config.json") -> Dict[str, Any]: "debug": True } - # Try to find config file relative to script location + # Resolve config path order: + # 1) explicit config_path (if provided) + # 2) ProgramData default + # 3) local directory config.json (script dir) script_dir = os.path.dirname(os.path.abspath(__file__)) - full_config_path = os.path.join(script_dir, config_path) + candidates = [] + if config_path: + candidates.append(config_path) + candidates.append(DEFAULT_CONFIG_PATH) + candidates.append(os.path.join(script_dir, "config.json")) - if not os.path.exists(full_config_path): - print(f"[WARN] Config file not found at {full_config_path}, using defaults", file=sys.stderr) + full_config_path = None + for path in candidates: + if os.path.exists(path): + full_config_path = path + break + + if full_config_path is None: + print( + f"[WARN] Config file not found in: {', '.join(candidates)}, using defaults", + file=sys.stderr + ) return defaults try: @@ -68,16 +92,15 @@ def load_config(config_path: str = "config.json") -> Dict[str, Any]: print(f"[WARN] Error loading config file: {e}, using defaults", file=sys.stderr) return defaults -# Load configuration -config = load_config() +THORDATA_PATH = r"C:\THORDATA" +SCAN_INTERVAL = 60 +LATE_DAYS = 2 +STALE_DAYS = 60 +SFM_ENDPOINT = "" +SFM_TIMEOUT = 5 +DEBUG = True -THORDATA_PATH = config["thordata_path"] -SCAN_INTERVAL = config["scan_interval"] -LATE_DAYS = config["late_days"] -STALE_DAYS = config["stale_days"] -SFM_ENDPOINT = config["sfm_endpoint"] -SFM_TIMEOUT = config["sfm_timeout"] -DEBUG = config["debug"] +logger = logging.getLogger("thor_ingest") # Regex: UM12345_YYYYMMDDHHMMSS.MLG MLG_PATTERN = re.compile(r"^(UM\d+)_([0-9]{14})\.MLG$", re.IGNORECASE) @@ -88,7 +111,7 @@ MLG_PATTERN = re.compile(r"^(UM\d+)_([0-9]{14})\.MLG$", re.IGNORECASE) def debug(msg: str) -> None: if DEBUG: - print(f"[DEBUG] {msg}", file=sys.stderr, flush=True) + logger.debug(msg) def parse_mlg_filename(name: str) -> Optional[Tuple[str, datetime]]: @@ -151,8 +174,8 @@ def scan_thordata(root: str) -> Dict[str, Dict[str, Any]]: try: unit_dirs = os.listdir(project_path) except OSError as e: - debug(f"Failed to list project '{project_path}': {e}") - continue + debug(f"Failed to list project '{project_path}': {e}") + continue for unit_name in unit_dirs: unit_path = os.path.join(project_path, unit_name) @@ -249,6 +272,8 @@ def format_age(td: timedelta) -> str: def clear_console() -> None: """Clear the console screen (Windows / *nix).""" + if not sys.stdout.isatty(): + return if os.name == "nt": os.system("cls") else: @@ -265,13 +290,13 @@ def print_heartbeat(unit_map: Dict[str, Dict[str, Any]]) -> None: now = datetime.now() clear_console() - print("Series 4 Ingest Agent — Micromate Heartbeat (v0.1.2)") - print(f"THORDATA root: {THORDATA_PATH}") - print(f"Now: {now.strftime('%Y-%m-%d %H:%M:%S')}") - print("-" * 80) + logger.info("Series 4 Ingest Agent — Micromate Heartbeat (v%s)", __version__) + logger.info("THORDATA root: %s", THORDATA_PATH) + logger.info("Now: %s", now.strftime('%Y-%m-%d %H:%M:%S')) + logger.info("-" * 80) if not unit_map: - print("No units found (no .MLG files detected).") + logger.info("No units found (no .MLG files detected).") return # Sort by unit_id for stable output @@ -285,15 +310,18 @@ def print_heartbeat(unit_map: Dict[str, Dict[str, Any]]) -> None: age_str = format_age(age_td) last_str = last_call.strftime("%Y-%m-%d %H:%M:%S") - print( - f"{unit_id:<8} {status:<6} Age: {age_str:<8} " - f"Last: {last_str} Project: {project}" + logger.info( + "%-8s %-6s Age: %-8s Last: %s Project: %s", + unit_id, + status, + age_str, + last_str, + project, ) - print("-" * 80) - print(f"Total units: {len(unit_map)}") - print(f"Next scan in {SCAN_INTERVAL} seconds...") - sys.stdout.flush() + logger.info("-" * 80) + logger.info("Total units: %s", len(unit_map)) + logger.info("Next scan in %s seconds...", SCAN_INTERVAL) def build_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: @@ -372,10 +400,9 @@ def emit_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> None: return if urllib is None: - print( - "[WARN] urllib not available; cannot POST to SFM. " - "Install standard Python or disable SFM_ENDPOINT.", - file=sys.stderr, + logger.warning( + "urllib not available; cannot POST to SFM. " + "Install standard Python or disable SFM_ENDPOINT." ) return @@ -394,44 +421,99 @@ def emit_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> None: _ = resp.read() # we don't care about the body for now debug(f"SFM POST OK: HTTP {resp.status}") except urllib.error.URLError as e: - print(f"[WARN] Failed to POST to SFM: {e}", file=sys.stderr) + logger.warning("Failed to POST to SFM: %s", e) except Exception as e: - print(f"[WARN] Unexpected error during SFM POST: {e}", file=sys.stderr) + logger.warning("Unexpected error during SFM POST: %s", e) + + +def setup_logging(log_dir: str) -> None: + os.makedirs(log_dir, exist_ok=True) + + logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) + + # File handler with rotation + file_handler = RotatingFileHandler( + os.path.join(log_dir, "thor_ingest.log"), + maxBytes=5 * 1024 * 1024, + backupCount=5, + encoding="utf-8", + ) + file_handler.setLevel(logging.DEBUG if DEBUG else logging.INFO) + file_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + file_handler.setFormatter(file_format) + + # Console handler (clean output) + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.DEBUG if DEBUG else logging.INFO) + console_format = logging.Formatter("%(message)s") + console_handler.setFormatter(console_format) + + # Avoid duplicate handlers if main is re-entered + if not logger.handlers: + logger.addHandler(file_handler) + logger.addHandler(console_handler) def main() -> None: - print("Starting Series 4 Ingest Agent (Micromate) v0.1.2") - print(f"THORDATA_PATH = {THORDATA_PATH}") - print(f"SCAN_INTERVAL = {SCAN_INTERVAL} seconds") - print(f"LATE_DAYS = {LATE_DAYS}, STALE_DAYS = {STALE_DAYS}") + parser = argparse.ArgumentParser(description="Series 4 Ingest Agent (Micromate)") + parser.add_argument("--config", help="Path to config.json") + parser.add_argument("--log-dir", help="Directory for log output") + parser.add_argument("--once", action="store_true", help="Run one scan and exit") + parser.add_argument("--version", action="store_true", help="Print version and exit") + args = parser.parse_args() + + if args.version: + print(__version__) + return + + config = load_config(args.config) + + global THORDATA_PATH, SCAN_INTERVAL, LATE_DAYS, STALE_DAYS, SFM_ENDPOINT, SFM_TIMEOUT, DEBUG + THORDATA_PATH = config["thordata_path"] + SCAN_INTERVAL = config["scan_interval"] + LATE_DAYS = config["late_days"] + STALE_DAYS = config["stale_days"] + SFM_ENDPOINT = config["sfm_endpoint"] + SFM_TIMEOUT = config["sfm_timeout"] + DEBUG = config["debug"] + + log_dir = args.log_dir or DEFAULT_LOG_DIR + setup_logging(log_dir) + + logger.info("Starting Series 4 Ingest Agent (Micromate) v%s", __version__) + logger.info("THORDATA_PATH = %s", THORDATA_PATH) + logger.info("SCAN_INTERVAL = %s seconds", SCAN_INTERVAL) + logger.info("LATE_DAYS = %s, STALE_DAYS = %s", LATE_DAYS, STALE_DAYS) if not os.path.isdir(THORDATA_PATH): - print(f"[WARN] THORDATA_PATH does not exist: {THORDATA_PATH}", file=sys.stderr) + logger.warning("THORDATA_PATH does not exist: %s", THORDATA_PATH) loop_counter = 0 try: while True: loop_counter += 1 - print(f"\n[LOOP] Iteration {loop_counter} starting...", flush=True) + logger.info("\n[LOOP] Iteration %s starting...", loop_counter) try: unit_map = scan_thordata(THORDATA_PATH) debug(f"scan_thordata found {len(unit_map)} units") print_heartbeat(unit_map) emit_sfm_payload(unit_map) - print("[LOOP] Iteration complete, entering sleep...", flush=True) + logger.info("[LOOP] Iteration complete, entering sleep...") except Exception as e: # Catch-all so a single error doesn't kill the loop - print(f"[ERROR] Exception in main loop: {e}", file=sys.stderr) - sys.stderr.flush() + logger.error("Exception in main loop: %s", e) + + if args.once: + break # Sleep in 1-second chunks to avoid VM time drift weirdness for i in range(SCAN_INTERVAL): time.sleep(1) - print("[LOOP] Woke up for next scan", flush=True) + logger.info("[LOOP] Woke up for next scan") except KeyboardInterrupt: - print("\nSeries 4 Ingest Agent stopped by user.") + logger.info("\nSeries 4 Ingest Agent stopped by user.")