diff --git a/README.md b/README.md index 5994b03..3879117 100644 --- a/README.md +++ b/README.md @@ -500,6 +500,19 @@ It first verifies exact control parity with the completed tuning matrix, then checkpoints consolidated JSON and Markdown reports under `reports/backtest-YYYYMMDD-gtl-confirmation-matrix.*`. +The confirmation matrix's only near-hit was strength-1000 intersection: it +improved full/train/post-2024 Sharpe but missed the unchanged drawdown guardrail +by 0.3 percentage points. The final narrow sensitivity check is: + +```bash +.venv/bin/python scripts/run_gtl_strength_sensitivity.py \ + backtest_snapshots/prod.sqlite --workers 12 +``` + +It checks eight coarse scales around 1000, verifies exact control and +strength-1000 replication, and requires two adjacent scales to pass every +original guardrail before calling the result stable. + ### Reading a local backtest report The deployed **Signals → Track Record** page is deliberately trimmed to validation diff --git a/docs/research/sr-levels-and-exits.md b/docs/research/sr-levels-and-exits.md index ef6ab17..9e49906 100644 --- a/docs/research/sr-levels-and-exits.md +++ b/docs/research/sr-levels-and-exits.md @@ -879,6 +879,33 @@ matrix before any research arm is accepted. backtest_snapshots/prod.sqlite --workers 12 ``` +Result: **13/13 arms completed with exact control parity; no arm passed all six +guardrails.** The decomposition does identify one near-hit: + +| Arm | Full Sharpe | Train | Post-2024 | CAGR | Max DD | Trades | +|---|---:|---:|---:|---:|---:|---:| +| Control | 2.03 | 1.28 | 2.78 | 50.0% | 21.4% | 321 | +| Strength-1000 intersection | **2.06** | **1.30** | **2.82** | **50.7%** | 21.7% | 316 | + +Strength confirmation removes only 49 of 1,086 qualified control setups. Those +removed setups average +0.142R, but turn negative after removing their largest +5% of outcomes (-0.044R); the retained 1,037 average +0.212R and +0.054R +ex-top-5%. This is consistent with a weak tail-dependence filter. It is not yet +a winner: the original drawdown guardrail remains fixed, and 21.7% is worse +than 21.4% even though the difference is small. + +The final parameter test is therefore deliberately one-dimensional. It sweeps +coarse strength scales around 1000 (625, 750, 875, 1000, 1125, 1250, 1500, +2000), using intersection only. It must reproduce both the frozen control and +the completed strength-1000 result exactly. Promotion requires at least two +adjacent non-control scales to pass all six original checks; an isolated winner +is rejected as sensitivity. + +```bash +.venv/bin/python scripts/run_gtl_strength_sensitivity.py \ + backtest_snapshots/prod.sqlite --workers 12 +``` + The post-2024 window has been opened and is now analysis data, not a valid final promotion holdout. These arms can isolate mechanism, but neither may ship without new future data or a separately pre-registered walk-forward protocol. diff --git a/reports/README.md b/reports/README.md index f06b495..b7fd4eb 100644 --- a/reports/README.md +++ b/reports/README.md @@ -79,3 +79,14 @@ point. The evidence-selected retained-versus-added decomposition writes: Those become the next decision point; detailed per-arm reports remain temporary unless explicitly retained. + +The completed confirmation matrix found one near-hit but no formal winner: +strength-1000 intersection improved full/train/post-2024 Sharpe and CAGR, while +max drawdown worsened from 21.4% to 21.7%. Retain that matrix as the cohort- +decomposition decision point. The final stability check writes: + +- `backtest-YYYYMMDD-gtl-strength-sensitivity.json` +- `backtest-YYYYMMDD-gtl-strength-sensitivity.md` + +Its pre-registered decision requires at least two adjacent scales to pass all +six unchanged checks. diff --git a/scripts/run_gtl_strength_sensitivity.py b/scripts/run_gtl_strength_sensitivity.py new file mode 100644 index 0000000..ea6e378 --- /dev/null +++ b/scripts/run_gtl_strength_sensitivity.py @@ -0,0 +1,287 @@ +"""Run the pre-registered GTL strength-confirmation sensitivity band. + +The strength-1000 intersection was the only composition arm to improve +full/train/test Sharpe, but it missed the unchanged drawdown guardrail. This +matrix checks whether that result is a stable one-dimensional plateau rather +than tuning the guardrail or launching another broad parameter search. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts import run_gtl_confirmation_matrix as composition # noqa: E402 +from scripts import run_gtl_tuning_matrix as common # noqa: E402 + +REFERENCE_TUNING = ROOT / "reports" / "backtest-20260713-gtl-tuning-matrix.json" +REFERENCE_COMPOSITION = ( + ROOT / "reports" / "backtest-20260713-gtl-confirmation-matrix.json" +) +STRENGTH_SCALES = (625.0, 750.0, 875.0, 1000.0, 1125.0, 1250.0, 1500.0, 2000.0) + +GTL_STRENGTH_ARMS: tuple[dict[str, Any], ...] = ( + composition._arm( + "control", + "Frozen GTL composition-path parity control.", + "intersection", + ), + *( + composition._arm( + f"strength_{int(scale)}_intersection", + f"Require control confirmation at traffic-strength scale {scale:g}.", + "intersection", + {"name": f"strength_{int(scale)}", "strength_scale": scale}, + ) + for scale in STRENGTH_SCALES + ), +) + + +def _args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "snapshot", + nargs="?", + default="backtest_snapshots/prod.sqlite", + help="Local SQLite snapshot path.", + ) + parser.add_argument("--workers", type=int, default=7) + parser.add_argument("--holdout-split", default="2024-07-01") + parser.add_argument( + "--out", + default=None, + help=( + "Consolidated JSON path. Defaults to " + "reports/backtest-YYYYMMDD-gtl-strength-sensitivity.json." + ), + ) + parser.add_argument("--keep-arm-reports", action="store_true") + return parser.parse_args() + + +def _reference_arm(path: Path, name: str) -> dict | None: + if not path.exists(): + return None + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + return next( + (arm for arm in payload.get("arms") or [] if arm.get("name") == name), + None, + ) + + +def _write_markdown(path: Path, payload: dict) -> None: + rows = [ + "# GTL strength-confirmation sensitivity", + "", + f"Status: **{payload['status']}** ", + f"Holdout split: `{payload['holdout_split']}` ", + f"Completed arms: {len(payload['arms'])}/{payload['arm_count']}", + "", + "| Arm | Qualified | Full Sharpe | CAGR | Max DD | Trades | Train Sharpe | Test Sharpe | Ex-top-5% R | Screen |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for arm in payload["arms"]: + full = arm.get("full_book") or {} + holdout = arm.get("holdout") or {} + screen = arm.get("screen") or {} + rows.append( + "| " + + " | ".join(( + arm["name"], + str(arm.get("qualified") or "-"), + common._fmt(full.get("sharpe")), + common._fmt(full.get("cagr_pct"), 1), + common._fmt(full.get("max_drawdown_pct"), 1), + str(full.get("trades") or "-"), + common._fmt((holdout.get("train") or {}).get("sharpe")), + common._fmt((holdout.get("test") or {}).get("sharpe")), + common._fmt(arm.get("qualified_net_avg_r_ex_top5"), 3), + f"{screen.get('passed', '-')}/{screen.get('total', '-')}", + )) + + " |" + ) + rows.extend(( + "", + "## Pre-registered interpretation", + "", + "The six original guardrails remain unchanged. A stable candidate requires at least two adjacent non-control scales to pass all six; an isolated passing scale is rejected as sensitivity, not promoted.", + "", + )) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(rows), encoding="utf-8") + + +def _stable_plateau_pairs(arms: list[dict]) -> list[list[str]]: + sensitivity = [arm for arm in arms if arm.get("name") != "control"] + return [ + [left["name"], right["name"]] + for left, right in zip(sensitivity, sensitivity[1:], strict=False) + if (left.get("screen") or {}).get("advances") + and (right.get("screen") or {}).get("advances") + ] + + +def main() -> None: + args = _args() + snapshot = Path(args.snapshot).resolve() + if not snapshot.exists(): + raise SystemExit(f"Snapshot not found: {snapshot}") + if args.workers < 1: + raise SystemExit("--workers must be at least 1") + try: + date.fromisoformat(args.holdout_split) + except ValueError as exc: + raise SystemExit("--holdout-split must use YYYY-MM-DD") from exc + + reference_control = _reference_arm(REFERENCE_TUNING, "control") + reference_1000 = _reference_arm( + REFERENCE_COMPOSITION, + "strength_intersection", + ) + if reference_control is None or reference_1000 is None: + raise SystemExit( + "The completed GTL tuning and confirmation matrices are required " + "for control/replication checks" + ) + + stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + default_out = ( + ROOT / "reports" / f"backtest-{stamp[:8]}-gtl-strength-sensitivity.json" + ) + out_path = Path(args.out) if args.out else default_out + if not out_path.is_absolute(): + out_path = ROOT / out_path + markdown_path = out_path.with_suffix(".md") + work_dir = ROOT / "reports" / f".gtl-strength-work-{stamp}" + work_dir.mkdir(parents=True, exist_ok=False) + + payload: dict[str, Any] = { + "status": "running", + "generated_at": datetime.now(timezone.utc).isoformat(), + "snapshot": str(snapshot), + "workers": args.workers, + "holdout_split": args.holdout_split, + "arm_count": len(GTL_STRENGTH_ARMS), + "strength_scales": list(STRENGTH_SCALES), + "arms": [], + "promotion_rule": ( + "At least two adjacent non-control scales must pass all six original " + "guardrails." + ), + } + common._write_json(out_path, payload) + _write_markdown(markdown_path, payload) + + control_report: dict | None = None + arm_outputs: list[Path] = [] + try: + for index, arm in enumerate(GTL_STRENGTH_ARMS, start=1): + name = arm["name"] + output = work_dir / f"{index:02d}-{name}.json" + arm_outputs.append(output) + print(f"\n[{index}/{len(GTL_STRENGTH_ARMS)}] {name}", flush=True) + print(f" {arm['description']}", flush=True) + composition._run_arm( + arm, + snapshot, + args.workers, + args.holdout_split, + output, + ) + report = common._load_report(output) + compact = common._compact_arm( + report, + composition._config(arm), + control_report, + ) + compact["description"] = arm["description"] + if control_report is None: + control_report = report + compact["screen"] = { + "checks": {}, "passed": 0, "total": 0, "advances": False, + } + if composition._signature(compact) != composition._signature( + reference_control + ): + raise RuntimeError( + "Strength sensitivity control does not reproduce frozen GTL control" + ) + payload["control_parity"] = "pass" + else: + compact["screen"] = common._screen_arm( + compact, + payload["arms"][0], + ) + if name == "strength_1000_intersection": + if composition._signature(compact) != composition._signature( + reference_1000 + ): + raise RuntimeError( + "Strength-1000 arm does not reproduce the completed composition result" + ) + payload["strength_1000_replication"] = "pass" + payload["arms"].append(compact) + payload["completed_at"] = datetime.now(timezone.utc).isoformat() + common._write_json(out_path, payload) + _write_markdown(markdown_path, payload) + except BaseException as exc: + payload["status"] = "failed" + payload["failed_at"] = datetime.now(timezone.utc).isoformat() + payload["error"] = f"{type(exc).__name__}: {exc}" + payload["work_dir"] = str(work_dir) + common._write_json(out_path, payload) + _write_markdown(markdown_path, payload) + raise + + payload["status"] = "complete" + payload["completed_at"] = datetime.now(timezone.utc).isoformat() + payload["advancing_arms"] = [ + arm["name"] + for arm in payload["arms"] + if (arm.get("screen") or {}).get("advances") + ] + payload["stable_plateau_pairs"] = _stable_plateau_pairs(payload["arms"]) + payload["stable_candidate"] = bool(payload["stable_plateau_pairs"]) + payload["ranking_by_full_sharpe"] = [ + arm["name"] + for arm in sorted( + payload["arms"], + key=lambda row: float( + (row.get("full_book") or {}).get("sharpe") or -math.inf + ), + reverse=True, + ) + ] + if args.keep_arm_reports: + payload["arm_report_directory"] = str(work_dir) + common._write_json(out_path, payload) + _write_markdown(markdown_path, payload) + + if not args.keep_arm_reports: + for path in arm_outputs: + path.unlink(missing_ok=True) + work_dir.rmdir() + + print("\nGTL strength sensitivity complete.") + print(f" JSON: {out_path}") + print(f" Markdown: {markdown_path}") + if payload["stable_candidate"]: + pairs = [" + ".join(pair) for pair in payload["stable_plateau_pairs"]] + print(" Stable passing plateau: " + "; ".join(pairs)) + else: + print(" No stable adjacent passing plateau.") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_gtl_strength_sensitivity.py b/tests/unit/test_gtl_strength_sensitivity.py new file mode 100644 index 0000000..9d3ce11 --- /dev/null +++ b/tests/unit/test_gtl_strength_sensitivity.py @@ -0,0 +1,31 @@ +"""Tests for the pre-registered strength-confirmation sensitivity band.""" + +from __future__ import annotations + +from scripts import run_gtl_strength_sensitivity as matrix + + +def test_strength_sensitivity_is_ordered_and_contains_replication_arm(): + assert len(matrix.GTL_STRENGTH_ARMS) == 9 + assert matrix.GTL_STRENGTH_ARMS[0]["name"] == "control" + assert list(matrix.STRENGTH_SCALES) == sorted(matrix.STRENGTH_SCALES) + assert "strength_1000_intersection" in { + arm["name"] for arm in matrix.GTL_STRENGTH_ARMS + } + for arm in matrix.GTL_STRENGTH_ARMS[1:]: + config = matrix.composition._config(arm) + assert config["mode"] == "intersection" + assert len(config["confirmations"]) == 1 + + +def test_stable_plateau_requires_adjacent_passing_scales(): + arms = [ + {"name": "control", "screen": {"advances": False}}, + {"name": "strength_625", "screen": {"advances": True}}, + {"name": "strength_750", "screen": {"advances": True}}, + {"name": "strength_875", "screen": {"advances": False}}, + {"name": "strength_1000", "screen": {"advances": True}}, + ] + assert matrix._stable_plateau_pairs(arms) == [ + ["strength_625", "strength_750"] + ]