serversdown/thor-watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: update to version 0.1.3 with new CLI flags, logging enhancements, and Windows MSI installer

Browse Source
This commit is contained in:
serversdwn
2026-02-10 07:08:24 +00:00
parent edeb6a6eba
commit 69e856db6a
9 changed files with 424 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
series4_ingest.py
View File

@@ -1,5 +1,5 @@
"""
Series 4 Ingest Agent — v0.1.2
Series 4 Ingest Agent — v0.1.3
Micromate (Series 4) ingest agent for Seismo Fleet Manager (SFM).
@@ -14,6 +14,9 @@ Behavior:
No roster. SFM backend decides what to do with each unit.
"""
import argparse
import logging
from logging.handlers import RotatingFileHandler
import os
import re
import time
@@ -30,9 +33,14 @@ try:
except ImportError:
urllib = None # type: ignore
__version__ = "0.1.3"
# ---------------- Config ----------------
def load_config(config_path: str = "config.json") -> Dict[str, Any]:
DEFAULT_CONFIG_PATH = r"C:\ProgramData\ThorIngest\config.json"
DEFAULT_LOG_DIR = r"C:\ProgramData\ThorIngest\logs"
def load_config(config_path: Optional[str] = None) -> Dict[str, Any]:
"""
Load configuration from a JSON file.
@@ -48,12 +56,28 @@ def load_config(config_path: str = "config.json") -> Dict[str, Any]:
"debug": True
}
# Try to find config file relative to script location
# Resolve config path order:
# 1) explicit config_path (if provided)
# 2) ProgramData default
# 3) local directory config.json (script dir)
script_dir = os.path.dirname(os.path.abspath(__file__))
full_config_path = os.path.join(script_dir, config_path)
candidates = []
if config_path:
candidates.append(config_path)
candidates.append(DEFAULT_CONFIG_PATH)
candidates.append(os.path.join(script_dir, "config.json"))
if not os.path.exists(full_config_path):
print(f"[WARN] Config file not found at {full_config_path}, using defaults", file=sys.stderr)
full_config_path = None
for path in candidates:
if os.path.exists(path):
full_config_path = path
break
if full_config_path is None:
print(
f"[WARN] Config file not found in: {', '.join(candidates)}, using defaults",
file=sys.stderr
)
return defaults
try:
@@ -68,16 +92,15 @@ def load_config(config_path: str = "config.json") -> Dict[str, Any]:
print(f"[WARN] Error loading config file: {e}, using defaults", file=sys.stderr)
return defaults
# Load configuration
config = load_config()
THORDATA_PATH = r"C:\THORDATA"
SCAN_INTERVAL = 60
LATE_DAYS = 2
STALE_DAYS = 60
SFM_ENDPOINT = ""
SFM_TIMEOUT = 5
DEBUG = True
THORDATA_PATH = config["thordata_path"]
SCAN_INTERVAL = config["scan_interval"]
LATE_DAYS = config["late_days"]
STALE_DAYS = config["stale_days"]
SFM_ENDPOINT = config["sfm_endpoint"]
SFM_TIMEOUT = config["sfm_timeout"]
DEBUG = config["debug"]
logger = logging.getLogger("thor_ingest")
# Regex: UM12345_YYYYMMDDHHMMSS.MLG
MLG_PATTERN = re.compile(r"^(UM\d+)_([0-9]{14})\.MLG$", re.IGNORECASE)
@@ -88,7 +111,7 @@ MLG_PATTERN = re.compile(r"^(UM\d+)_([0-9]{14})\.MLG$", re.IGNORECASE)
def debug(msg: str) -> None:
if DEBUG:
print(f"[DEBUG] {msg}", file=sys.stderr, flush=True)
logger.debug(msg)
def parse_mlg_filename(name: str) -> Optional[Tuple[str, datetime]]:
@@ -151,8 +174,8 @@ def scan_thordata(root: str) -> Dict[str, Dict[str, Any]]:
try:
unit_dirs = os.listdir(project_path)
except OSError as e:
debug(f"Failed to list project '{project_path}': {e}")
continue
debug(f"Failed to list project '{project_path}': {e}")
continue
for unit_name in unit_dirs:
unit_path = os.path.join(project_path, unit_name)
@@ -249,6 +272,8 @@ def format_age(td: timedelta) -> str:
def clear_console() -> None:
"""Clear the console screen (Windows / *nix)."""
if not sys.stdout.isatty():
return
if os.name == "nt":
os.system("cls")
else:
@@ -265,13 +290,13 @@ def print_heartbeat(unit_map: Dict[str, Dict[str, Any]]) -> None:
now = datetime.now()
clear_console()
print("Series 4 Ingest Agent — Micromate Heartbeat (v0.1.2)")
print(f"THORDATA root: {THORDATA_PATH}")
print(f"Now: {now.strftime('%Y-%m-%d %H:%M:%S')}")
print("-" * 80)
logger.info("Series 4 Ingest Agent — Micromate Heartbeat (v%s)", __version__)
logger.info("THORDATA root: %s", THORDATA_PATH)
logger.info("Now: %s", now.strftime('%Y-%m-%d %H:%M:%S'))
logger.info("-" * 80)
if not unit_map:
print("No units found (no .MLG files detected).")
logger.info("No units found (no .MLG files detected).")
return
# Sort by unit_id for stable output
@@ -285,15 +310,18 @@ def print_heartbeat(unit_map: Dict[str, Dict[str, Any]]) -> None:
age_str = format_age(age_td)
last_str = last_call.strftime("%Y-%m-%d %H:%M:%S")
print(
f"{unit_id:<8} {status:<6} Age: {age_str:<8} "
f"Last: {last_str} Project: {project}"
logger.info(
"%-8s %-6s Age: %-8s Last: %s Project: %s",
unit_id,
status,
age_str,
last_str,
project,
)
print("-" * 80)
print(f"Total units: {len(unit_map)}")
print(f"Next scan in {SCAN_INTERVAL} seconds...")
sys.stdout.flush()
logger.info("-" * 80)
logger.info("Total units: %s", len(unit_map))
logger.info("Next scan in %s seconds...", SCAN_INTERVAL)
def build_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
@@ -372,10 +400,9 @@ def emit_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> None:
return
if urllib is None:
print(
"[WARN] urllib not available; cannot POST to SFM. "
"Install standard Python or disable SFM_ENDPOINT.",
file=sys.stderr,
logger.warning(
"urllib not available; cannot POST to SFM. "
"Install standard Python or disable SFM_ENDPOINT."
)
return
@@ -394,44 +421,99 @@ def emit_sfm_payload(unit_map: Dict[str, Dict[str, Any]]) -> None:
_ = resp.read() # we don't care about the body for now
debug(f"SFM POST OK: HTTP {resp.status}")
except urllib.error.URLError as e:
print(f"[WARN] Failed to POST to SFM: {e}", file=sys.stderr)
logger.warning("Failed to POST to SFM: %s", e)
except Exception as e:
print(f"[WARN] Unexpected error during SFM POST: {e}", file=sys.stderr)
logger.warning("Unexpected error during SFM POST: %s", e)
def setup_logging(log_dir: str) -> None:
os.makedirs(log_dir, exist_ok=True)
logger.setLevel(logging.DEBUG if DEBUG else logging.INFO)
# File handler with rotation
file_handler = RotatingFileHandler(
os.path.join(log_dir, "thor_ingest.log"),
maxBytes=5 * 1024 * 1024,
backupCount=5,
encoding="utf-8",
)
file_handler.setLevel(logging.DEBUG if DEBUG else logging.INFO)
file_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
file_handler.setFormatter(file_format)
# Console handler (clean output)
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.DEBUG if DEBUG else logging.INFO)
console_format = logging.Formatter("%(message)s")
console_handler.setFormatter(console_format)
# Avoid duplicate handlers if main is re-entered
if not logger.handlers:
logger.addHandler(file_handler)
logger.addHandler(console_handler)
def main() -> None:
print("Starting Series 4 Ingest Agent (Micromate) v0.1.2")
print(f"THORDATA_PATH = {THORDATA_PATH}")
print(f"SCAN_INTERVAL = {SCAN_INTERVAL} seconds")
print(f"LATE_DAYS = {LATE_DAYS}, STALE_DAYS = {STALE_DAYS}")
parser = argparse.ArgumentParser(description="Series 4 Ingest Agent (Micromate)")
parser.add_argument("--config", help="Path to config.json")
parser.add_argument("--log-dir", help="Directory for log output")
parser.add_argument("--once", action="store_true", help="Run one scan and exit")
parser.add_argument("--version", action="store_true", help="Print version and exit")
args = parser.parse_args()
if args.version:
print(__version__)
return
config = load_config(args.config)
global THORDATA_PATH, SCAN_INTERVAL, LATE_DAYS, STALE_DAYS, SFM_ENDPOINT, SFM_TIMEOUT, DEBUG
THORDATA_PATH = config["thordata_path"]
SCAN_INTERVAL = config["scan_interval"]
LATE_DAYS = config["late_days"]
STALE_DAYS = config["stale_days"]
SFM_ENDPOINT = config["sfm_endpoint"]
SFM_TIMEOUT = config["sfm_timeout"]
DEBUG = config["debug"]
log_dir = args.log_dir or DEFAULT_LOG_DIR
setup_logging(log_dir)
logger.info("Starting Series 4 Ingest Agent (Micromate) v%s", __version__)
logger.info("THORDATA_PATH = %s", THORDATA_PATH)
logger.info("SCAN_INTERVAL = %s seconds", SCAN_INTERVAL)
logger.info("LATE_DAYS = %s, STALE_DAYS = %s", LATE_DAYS, STALE_DAYS)
if not os.path.isdir(THORDATA_PATH):
print(f"[WARN] THORDATA_PATH does not exist: {THORDATA_PATH}", file=sys.stderr)
logger.warning("THORDATA_PATH does not exist: %s", THORDATA_PATH)
loop_counter = 0
try:
while True:
loop_counter += 1
print(f"\n[LOOP] Iteration {loop_counter} starting...", flush=True)
logger.info("\n[LOOP] Iteration %s starting...", loop_counter)
try:
unit_map = scan_thordata(THORDATA_PATH)
debug(f"scan_thordata found {len(unit_map)} units")
print_heartbeat(unit_map)
emit_sfm_payload(unit_map)
print("[LOOP] Iteration complete, entering sleep...", flush=True)
logger.info("[LOOP] Iteration complete, entering sleep...")
except Exception as e:
# Catch-all so a single error doesn't kill the loop
print(f"[ERROR] Exception in main loop: {e}", file=sys.stderr)
sys.stderr.flush()
logger.error("Exception in main loop: %s", e)
if args.once:
break
# Sleep in 1-second chunks to avoid VM time drift weirdness
for i in range(SCAN_INTERVAL):
time.sleep(1)
print("[LOOP] Woke up for next scan", flush=True)
logger.info("[LOOP] Woke up for next scan")
except KeyboardInterrupt:
print("\nSeries 4 Ingest Agent stopped by user.")
logger.info("\nSeries 4 Ingest Agent stopped by user.")