From 1fdc6656750ba5909288d3bd99126edbde9e53c1 Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 28 Aug 2026 20:41:20 +0000 Subject: [PATCH] =?UTF-8?q?fix(offset):=20retract=20the=20v1=20detector=20?= =?UTF-8?q?=E2=80=94=20per-channel=20median,=20not=20dominant-axis=20mean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brian challenged the v1 finding that offsets "come and go", against field experience that a unit which develops one stays broken until the geophone is replaced. He was right; v1 had two flaws, both of which manufactured false recoveries: 1. It scored only the axis with the largest peak, so a real event on one axis hid a persistent pedestal on another. BE12599 on 2026-08-21 read "clean" because Long had a 1.065 in/s event, while Tran sat at +0.4732 in/s and was never examined. 2. It used the mean, which a real transient perturbs. The median is the resting baseline and a blast does not move it. Same event, Long channel: mean +0.0783 vs median -0.0050. offset_scan2.py flags a CHANNEL when |median| >= 0.025 in/s (5 A/D counts, Instantel's own criterion) and treats >=3 consecutive flagged events as the real signal. No m/p ratio guard is needed — that existed only to compensate for the mean. Corrected results: units with any flagged event 6 -> 19 of 45 units with a sustained pedestal 8 of 45 (18%) runs >=3 consecutive 29; 1-2 event runs (noise) 69 Also corrected: the affected channel is most often Vert, not Tran (v1 named whichever axis had the largest peak, so it was frequently wrong). BE10895 and BE18003 were invisible to v1. BE12599's fault began 2026-08-14, not 08-17. The decode itself was never in question and is confirmed against Blastware's own ASCII export: on K558LJN3.BK0W, BW shows Tran parked at +0.265..+0.375 in/s for the entire record while Vert and Long sit at ~0.005 — Instantel's "parallel lines above or below the zero line". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- docs/offset_investigation.md | 86 ++++++++++++++++++++++++++++ scratch/offset_scan2.py | 108 +++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 scratch/offset_scan2.py diff --git a/docs/offset_investigation.md b/docs/offset_investigation.md index daccf3c..f87fc68 100644 --- a/docs/offset_investigation.md +++ b/docs/offset_investigation.md @@ -1,5 +1,26 @@ # The "offset" fault — investigation journal +> ## ⚠ CORRECTED 2026-08-28 (same day) — the v1 detector was wrong +> +> Brian pushed back on the finding that offsets "come and go": in the field, +> once a unit develops one it stays broken until the geophone is replaced. +> He was right, and the challenge exposed **two real flaws** in the v1 detector: +> +> 1. **It scored only the axis with the largest peak.** A real event on one axis +> hid a persistent pedestal on another. BE12599 on 2026-08-21 read "clean" +> solely because Long had a 1.065 in/s event — Tran was sitting at +> **+0.4732 in/s** at that moment and was never examined. +> 2. **It used the MEAN**, which a real transient perturbs. The **median** is the +> resting baseline — most samples sit at it, so a blast does not move it. +> Same event, Long channel: mean **+0.0783** vs median **-0.0050**. +> +> Both flaws manufactured false recoveries. The corrected detector +> (`scratch/offset_scan2.py`, per-channel median) shows the pedestal is +> **persistent**, exactly as the field experience says. See §2b and §3b. +> +> Sections below that were written against v1 are marked; v1 numbers are kept +> for the reasoning trail, not as current fact. + A running record of the **offset** hardware fault on Instantel Series III seismographs: a geophone channel whose trace sits displaced from zero rather than centred on it. @@ -85,6 +106,60 @@ see offsets large enough to *dominate* the trace. A mild offset on a real blast is invisible. Instantel's A/D-mode check (§5) is the only thing that sees the mild end. Our base rate is therefore a **gross-offset** rate. +### 2b. Detector v2 — per-channel median (CURRENT) + +`scratch/offset_scan2.py`. Supersedes the above. + +``` +for each series-3 waveform binary: + for each geo channel independently: + pedestal = median(samples) # resting baseline, robust to blasts + flag the CHANNEL when |pedestal| >= 0.025 in/s (5 A/D counts) +a unit has a real fault when a channel is flagged on >=3 CONSECUTIVE events +``` + +Why median: a DC pedestal shifts every sample, so it moves the median. A real +event moves only a minority of samples, so it does not. This removes the need +for the `m/p` ratio guard entirely — that guard existed only to compensate for +using the mean. + +Why per-channel: the fault is on one geophone axis. Scoring only the dominant +axis means any event with motion elsewhere hides it. + +Why "3 consecutive": the 0.025 in/s floor is only ~2x a healthy channel's +resting median (observed 0.010-0.015), so isolated flags are noise. Persistence +is the discriminator — and it is what the field experience predicts. + +--- + +## 3b. Archive results, corrected (v2) + +| | v1 (dominant axis, mean) | **v2 (per-channel median)** | +|---|---|---| +| units with any flagged event | 6 of 45 | 19 of 45 | +| **units with a sustained pedestal (>=3 consecutive)** | — | **8 of 45 (18%)** | +| runs of >=3 consecutive | — | 29 | +| runs of 1-2 events (noise) | — | 69 | + +Units with a sustained pedestal: **BE9558, BE10895, BE11007, BE11529, BE12599, +BE13117, BE18003, BE18438**. BE10895 and BE18003 were invisible to v1. + +**The affected channel is most often Vert**, which v1 got wrong — it named +whichever axis had the largest peak. BE13117 and BE18438 are both Vert faults. + +Longest / clearest runs: + +| unit | ch | span | events | median in/s | +|---|---|---|---|---| +| BE13117 | Vert | 2023-05-03 → 05-04 | 194 | 0.035 → **1.915** | +| BE18438 | Vert | 2026-02-25 → 02-26 | 75 | 0.180 → 0.370 | +| BE9558 | Vert | 2020-02-11 (6 h) | 33 | 0.065 → 0.090 | +| BE12599 | Tran | 2026-08-14 → 08-23 | 8 | 0.030 → **0.565** | +| BE18003 | Vert | 2021-03-17 → 06-11 | 3 | 0.040 → 0.060 | + +BE12599 began **2026-08-14**, not 08-17 as v1 reported, and was still faulting +at the last event in the archive. + --- ## 3. Archive results (2026-08-28) @@ -253,6 +328,16 @@ swing test measures geophone frequency response and damping — it never examine DC zero. **A grossly offset unit passes its own self-check.** This is why the fault goes unnoticed until somebody looks at waveforms. +### "Offsets are transient / come and go on their own" — RETRACTED 2026-08-28 +v1 reported episodes lasting hours that ended spontaneously. **This was an +artifact of the v1 detector** (see the banner at the top). With the per-channel +median, the pedestal persists. Every clear case reads clean again only after a +multi-day-to-multi-month gap consistent with service: BE13117 6 days, BE18438 +24 days, BE9558 63 days **with a confirmed Instantel calibration inside the +gap**. BE12599 never reads clean — it is still faulting at the end of the +archive. This matches the operational experience: once a unit develops an +offset it stays broken until the geophone is replaced. + ### "Offsets develop N months after calibration" — CONFOUNDED, NOT A FINDING Tempting, and it looked strong: @@ -361,3 +446,4 @@ doubled two reported figures before it was caught. | 2026-08-28 | Calibration-timing correlation attempted and **rejected as confounded**. | | 2026-08-28 | Instantel FAQs supplied: autozero procedure, the **2027–2069** window, the **>5 counts** threshold. Explains the ~10% re-zero success rate. | | 2026-08-28 | Bimodality established; sensor check proven **blind** to offsets; `SUB 0x0E` identified as the best open lead. | +| 2026-08-28 | **v1 detector retracted.** Brian challenged the "come and go" finding against field experience. Two flaws found: dominant-axis-only scoring and mean-instead-of-median. Corrected detector shows persistent pedestals on **8 of 45 units**, and the gaps are service windows. | diff --git a/scratch/offset_scan2.py b/scratch/offset_scan2.py new file mode 100644 index 0000000..5fe2e62 --- /dev/null +++ b/scratch/offset_scan2.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Offset detector v2 — per-channel MEDIAN pedestal. + +Supersedes the dominant-axis / mean detector in offset_scan.py, which had two +flaws that manufactured false "recoveries": + + 1. It scored only the axis with the largest peak, so a real event on one axis + hid a persistent pedestal on another. BE12599 2026-08-21 read "clean" + because Long had a 1.065 in/s event, while Tran sat at +0.47 in/s. + 2. It used the MEAN, which a real transient perturbs. The median is the + resting baseline: most samples sit at it, so a blast does not move it. + Same event, Long: mean +0.0783 vs median -0.0050. + +Flags a CHANNEL when |median| >= --floor in/s (default 0.025 = 5 A/D counts, +Instantel's own criterion; 1 A/D count = 0.005 in/s). + +Emits one row per (event, channel) so persistence can be tracked per channel. +""" +from __future__ import annotations +import argparse, csv, re, statistics, sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from minimateplus.event_file_io import read_blastware_file + +GEO = ("Tran", "Vert", "Long") +K = 10.0 / 32000.0 # ADC counts -> in/s at the 10 in/s range +_WAVE_RE = re.compile(r"\.[A-Za-z0-9]{2}0[Ww]$") +_STEM_RE = re.compile(r"^([B-Z])(\d{3})") + + +def serial_from_name(n): + m = _STEM_RE.match(n) + return f"BE{(ord(m.group(1))-ord('B'))*1000+int(m.group(2))}" if m else "?" + + +def scan_one(ps): + p = Path(ps) + try: + ev = read_blastware_file(p) + s = ev.raw_samples or {} + if not all(s.get(c) for c in GEO): + return None + ts = ev.timestamp + stamp = (f"{ts.year:04d}-{ts.month:02d}-{ts.day:02d}T" + f"{ts.hour:02d}:{ts.minute:02d}:{ts.second:02d}") if ts else "" + out = [] + for ch in GEO: + a = s[ch] + out.append({ + "serial": serial_from_name(p.name), "timestamp": stamp, + "filename": p.name, "channel": ch, + "median_ips": round(statistics.median(a) * K, 4), + "mean_ips": round(statistics.fmean(a) * K, 4), + "peak_ips": round(max(abs(v) for v in a) * K, 4), + }) + return out + except Exception: + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dir", required=True) + ap.add_argument("--jobs", type=int, default=4) + ap.add_argument("--floor", type=float, default=0.025) + ap.add_argument("--out", required=True) + a = ap.parse_args() + + seen, files = set(), [] + for q in sorted(Path(a.dir).rglob("*")): + if q.is_file() and _WAVE_RE.search(q.name) and q.name not in seen: + seen.add(q.name); files.append(str(q)) + print(f"unique waveform binaries: {len(files)}", flush=True) + + rows = [] + with ProcessPoolExecutor(max_workers=a.jobs) as ex: + for n, f in enumerate(as_completed([ex.submit(scan_one, p) for p in files]), 1): + r = f.result() + if r: rows.extend(r) + if n % 2000 == 0: print(f" {n}/{len(files)}", flush=True) + + for r in rows: + r["offset"] = int(abs(r["median_ips"]) >= a.floor) + + cols = ["serial","timestamp","filename","channel","median_ips","mean_ips","peak_ips","offset"] + with open(a.out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=cols); w.writeheader(); w.writerows(rows) + + from collections import defaultdict + ev_flagged = {(r["serial"], r["filename"]) for r in rows if r["offset"]} + ev_all = {(r["serial"], r["filename"]) for r in rows} + per = defaultdict(set) + for r in rows: + if r["offset"]: per[r["serial"]].add(r["filename"]) + tot = defaultdict(set) + for r in rows: tot[r["serial"]].add(r["filename"]) + print(f"\nfloor = {a.floor} in/s ({a.floor/0.005:.0f} A/D counts)") + print(f"events with >=1 offset channel: {len(ev_flagged)} of {len(ev_all)}") + print(f"units affected: {len(per)} of {len(tot)}") + for s in sorted(per, key=lambda s: -len(per[s])): + print(f" {s:9} {len(per[s]):4} / {len(tot[s]):4} events") + print(f"\nwrote {a.out}") + + +if __name__ == "__main__": + main()