From 43ee6194120da629f0b34e7d3e86ff2b20ef55b5 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sat, 8 Aug 2026 23:16:03 +0200 Subject: [PATCH] fix(research): require the v2 reproduction, and correct the P1-cap denominator Two review findings, plus a lost-edit repair. v2_reconstruction is now a required variant. It carries every published figure the reproduction rests on (avg, p80, max, P3-pegged, W1-live), so a run without it could emit a confident, non-provisional recommendation having checked nothing against v2 at all -- while the methodology doc claims v2 and v3 are reproduced first. The default invocation is now derived from REQUIRED_VARIANTS so the two cannot drift, and a test asserts the default satisfies its own requirement. The doc and the P1_TREND_BREAK_ANCHORS comment still justified skipping the P1_SCORE_CAP with 17/408 = 4.2%, which is the all-session share and does not evaluate the rule. The rule names sessions with State >= 40: 47 of them, P1 sole argmax on 17 = 36.2%, against P2's 16 and P3's 14. Conclusion unchanged -- well under the 80% trigger -- but the published rationale now states the metric that actually decided it. Root cause of that survival: the earlier correction WAS made, but in a script that applied several substitutions and wrote the file once at the end. A later substitution raised, so the successful edits were discarded with it. The "Unlike P3 and V1 ... P3's do not" fix was lost the same way and is restored. Also adds tests for the refusal paths themselves -- missing required variant, unknown variant, custom window with no calendar anchor. They were verified by hand last round but left unpinned, which is the same shape of problem as the optional gates they exist to enforce. All return before any network call. Deliberately not done, as not load-bearing: recording the oas400 variant's missing-credit session count (the truncation conclusion rests on the distribution mismatch, which is already recorded), and generalising _pipeline_gates for arbitrary --end/--sessions windows. Co-Authored-By: Claude Opus 5 --- app/services/regime_monitor_service.py | 6 ++- docs/research/regime-monitor-v4.md | 10 +++-- scripts/run_regime_monitor_calibration.py | 11 +++-- tests/unit/test_regime_calibration_script.py | 42 ++++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/app/services/regime_monitor_service.py b/app/services/regime_monitor_service.py index dd2632f..8dae0f1 100644 --- a/app/services/regime_monitor_service.py +++ b/app/services/regime_monitor_service.py @@ -139,8 +139,10 @@ P3_DRAWDOWN_ANCHORS = ( # why the crossing gets a floor of 20 rather than starting at 0. Calibrated to # sit alongside P3 rather than swamp it -- the 200-DMA lags, so a 20% drawdown # typically coincides with ~10% below the average, where this reads ~61 against -# P3's ~59. Over the 408 sessions to 2026-07-24 P1 is the sole price argmax on -# 17 of them (4.2%), so it informs the pillar without owning it. +# P3's ~59. On the population the P1_SCORE_CAP rule actually names -- sessions +# with State >= 40 -- P1 is the sole price argmax on 17 of 47 (36.2%), against +# P2's 16 and P3's 14, so it informs the pillar without owning it and no cap +# was needed. P1_TREND_BREAK_ANCHORS = ( (0.0, 20.0), (3.0, 35.0), (8.0, 55.0), (15.0, 75.0), (25.0, 100.0), ) diff --git a/docs/research/regime-monitor-v4.md b/docs/research/regime-monitor-v4.md index 4649cf6..494bbfd 100644 --- a/docs/research/regime-monitor-v4.md +++ b/docs/research/regime-monitor-v4.md @@ -57,9 +57,11 @@ used to read 100. `max()` was **kept**. The defect was the step function feeding it, not the vote itself, and v3's "one capped vote for correlated reads" rationale still holds. -Over the window P1 is the sole price argmax on 17 of 408 sessions (4.2%), so it -informs the pillar without owning it — the `P1_SCORE_CAP` fallback considered -during design was measured as unnecessary and not shipped. +The `P1_SCORE_CAP` fallback drafted during design was to fire if P1 became the +sole price argmax on **more than 80% of sessions with State ≥ 40** — i.e. if it +had quietly become a second drawdown sensor. Measured on that population: 47 +qualifying sessions, P1 sole argmax on **17 of them (36.2%)**, against P2's 16 +and P3's 14. Well under the threshold, so the cap is not shipped. **The top State band moved 80 → 65.** See Calibration; this is the one change that is about the band rather than a sensor. @@ -148,7 +150,7 @@ against P3's ~59. V1 reaches full scale at 55 rather than at 2020's ~82: anchoring the top at a once-in-a-generation print would make VIX 50 — a genuine crisis — read only ~70. The anchors encode the long-run distribution as constants, the same argument the -credit level uses. Unlike P3 and V1, whose slopes ease off monotonically, P3's do +credit level uses. Unlike P1 and V1, whose slopes ease off monotonically, P3's do not (2.5, 3.75, 3.125, 2.33, 1.83) — its gentle onset is intentional and the monotone-slope test excludes it. diff --git a/scripts/run_regime_monitor_calibration.py b/scripts/run_regime_monitor_calibration.py index d6a2499..fb1ff8c 100644 --- a/scripts/run_regime_monitor_calibration.py +++ b/scripts/run_regime_monitor_calibration.py @@ -267,7 +267,11 @@ OAS_SOURCE_TRUNCATION = {"v2_reconstruction_oas400": 400} # A v4 recommendation is meaningless without both of these: the row-wise # state_v4 <= state_v3 invariant needs them, and it is a hard gate. -REQUIRED_VARIANTS = ("v3", "v4") +# v2_reconstruction is required too: it carries every published figure the +# reproduction rests on (avg/p80/max/P3-pegged/W1-live). Without it a run could +# emit a confident recommendation having checked nothing against v2 at all, +# while the methodology doc claims v2 and v3 are reproduced first. +REQUIRED_VARIANTS = ("v2_reconstruction", "v3", "v4") @contextlib.contextmanager @@ -681,7 +685,7 @@ def _parse_args() -> argparse.Namespace: help="per-session slice for the canonical run") p.add_argument("--oas-fetch-days", type=int, default=int(365.25 * 13), help="fetch range; matches v2's request (ICE truncates to ~3y)") - p.add_argument("--methodology", default="v3,v4") + p.add_argument("--methodology", default=",".join(REQUIRED_VARIANTS)) p.add_argument("--expected-first-session", default=None, help="assert the first replayed date. Defaults to the published " "window's start; REQUIRED when --end/--sessions are overridden, " @@ -748,7 +752,8 @@ async def _main() -> int: missing = [v for v in REQUIRED_VARIANTS if v not in variants] if missing: print(f"--methodology must include {list(REQUIRED_VARIANTS)}; missing {missing}. " - "The state_v4 <= state_v3 invariant is a hard gate and needs both.", + "v2_reconstruction carries the published reproduction figures, and " + "v3+v4 are needed for the state_v4 <= state_v3 invariant gate.", file=sys.stderr) return 2 diff --git a/tests/unit/test_regime_calibration_script.py b/tests/unit/test_regime_calibration_script.py index fdb80f1..0f9984f 100644 --- a/tests/unit/test_regime_calibration_script.py +++ b/tests/unit/test_regime_calibration_script.py @@ -139,3 +139,45 @@ class TestScenarios: # ...and it must clear the chosen threshold under either P2 assumption. assert min(rows["S3a"]["state"], rows["S3b"]["state"]) > 65.0 assert rows["S1"]["state"] < 20.0 + + +class TestRefusals: + """The harness's safety contract: it must decline rather than under-report. + + Both paths return before any network call, so these are fast and offline. + """ + + def _run(self, argv, monkeypatch): + import asyncio + import sys + + monkeypatch.setattr(sys, "argv", ["run_regime_monitor_calibration.py", *argv]) + return asyncio.run(calib._main()) + + def test_refuses_without_every_required_variant(self, monkeypatch): + assert self._run(["--methodology", "v3,v4"], monkeypatch) == 2 + assert self._run(["--methodology", "v2_reconstruction,v3"], monkeypatch) == 2 + + def test_refuses_an_unknown_variant(self, monkeypatch): + assert self._run(["--methodology", "v3,v4,nonsense"], monkeypatch) == 2 + + def test_refuses_a_custom_window_without_a_calendar_anchor(self, monkeypatch): + """--sessions alone can only ever be tautological, so the start date must + be supplied explicitly once the published window is left behind.""" + assert self._run( + ["--methodology", ",".join(calib.REQUIRED_VARIANTS), "--sessions", "100"], + monkeypatch, + ) == 2 + assert self._run( + ["--methodology", ",".join(calib.REQUIRED_VARIANTS), "--end", "2026-01-05"], + monkeypatch, + ) == 2 + + def test_the_default_invocation_satisfies_its_own_requirement(self, monkeypatch): + """A default that the requirement rejects would make every bare run fail.""" + import sys + + monkeypatch.setattr(sys, "argv", ["run_regime_monitor_calibration.py"]) + default = calib._parse_args().methodology.split(",") + assert set(calib.REQUIRED_VARIANTS) <= set(default) + assert set(calib.REQUIRED_VARIANTS) <= set(calib.VARIANTS)