Add GTL cohort composition backtest

scripts/run_backtest_snapshot.py

@@ -62,6 +62,7 @@ def _parse_args() -> argparse.Namespace:
             "production_structural_overlay",
             "explicit_target_ladder",
             "gtl_tuning",
+            "gtl_confirmation",
         ),
         default=None,
         help="Research-only S/R detector/gate arm.",
@@ -71,6 +72,14 @@ def _parse_args() -> argparse.Namespace:
         default=None,
         help="Research-only GTL configuration as a JSON object (requires --sr-variant gtl_tuning).",
     )
+    parser.add_argument(
+        "--gtl-confirm-config",
+        default=None,
+        help=(
+            "Research-only GTL intersection/union configuration as JSON "
+            "(requires --sr-variant gtl_confirmation)."
+        ),
+    )
     parser.add_argument(
         "--entry-start",
         default=None,
@@ -210,6 +219,12 @@ async def _main() -> None:
         if args.sr_variant != "gtl_tuning":
             raise SystemExit("--gtl-config requires --sr-variant gtl_tuning")
         os.environ["BACKTEST_GTL_CONFIG"] = args.gtl_config
+    if args.gtl_confirm_config:
+        if args.sr_variant != "gtl_confirmation":
+            raise SystemExit(
+                "--gtl-confirm-config requires --sr-variant gtl_confirmation"
+            )
+        os.environ["BACKTEST_GTL_CONFIRM_CONFIG"] = args.gtl_confirm_config
     if args.entry_start:
         os.environ["BACKTEST_ENTRY_START"] = args.entry_start
     if args.entry_end:

scripts/run_gtl_confirmation_matrix.py

@@ -0,0 +1,360 @@
+"""Run the evidence-selected GTL confirmation/union matrix with one command.
+
+The first GTL tuning matrix tested replacements. This follow-up decomposes the
+four informative variants into retained-only intersections and control-plus-
+addition unions while preserving frozen control geometry wherever possible.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+from datetime import date, datetime, timezone
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from scripts import run_gtl_tuning_matrix as common  # noqa: E402
+
+RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py"
+REFERENCE_MATRIX = ROOT / "reports" / "backtest-20260713-gtl-tuning-matrix.json"
+
+TOUCH = {"name": "touch_0_25pct", "touch_tolerance": 0.0025}
+STRENGTH = {"name": "strength_1000", "strength_scale": 1000.0}
+MERGE = {"name": "merge_0_25pct", "merge_tolerance": 0.0025}
+GRID_ONLY = {"name": "pivots_none", "include_pivots": False}
+
+
+def _arm(
+    name: str,
+    description: str,
+    mode: str,
+    *confirmations: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "name": name,
+        "description": description,
+        "mode": mode,
+        "confirmations": list(confirmations),
+    }
+
+
+# Pre-registered from the replacement matrix's paired cohorts. Replacement
+# arms that plainly removed strong control setups or added weak cohorts are not
+# repeated here.
+GTL_CONFIRMATION_ARMS: tuple[dict[str, Any], ...] = (
+    _arm("control", "Frozen explicit GTL; composition-path parity control.", "intersection"),
+    _arm(
+        "touch_intersection",
+        "Retain control setups also qualified with 0.25% touch padding.",
+        "intersection",
+        TOUCH,
+    ),
+    _arm(
+        "touch_union",
+        "Keep control and admit additions from 0.25% touch padding.",
+        "union",
+        TOUCH,
+    ),
+    _arm(
+        "strength_intersection",
+        "Retain control setups also qualified at strength scale 1000.",
+        "intersection",
+        STRENGTH,
+    ),
+    _arm(
+        "strength_union",
+        "Keep control and admit additions from strength scale 1000.",
+        "union",
+        STRENGTH,
+    ),
+    _arm(
+        "merge_intersection",
+        "Retain control setups also qualified with 0.25% proposal merging.",
+        "intersection",
+        MERGE,
+    ),
+    _arm(
+        "merge_union",
+        "Keep control and admit additions from 0.25% proposal merging.",
+        "union",
+        MERGE,
+    ),
+    _arm(
+        "grid_intersection",
+        "Retain control setups also qualified by the range grid without pivots.",
+        "intersection",
+        GRID_ONLY,
+    ),
+    _arm(
+        "grid_union",
+        "Keep control and admit additions from the range grid without pivots.",
+        "union",
+        GRID_ONLY,
+    ),
+    _arm(
+        "touch_strength_intersection",
+        "Require both tighter-touch and faster-strength confirmation.",
+        "intersection",
+        TOUCH,
+        STRENGTH,
+    ),
+    _arm(
+        "touch_merge_intersection",
+        "Require both tighter-touch and tighter-merge confirmation.",
+        "intersection",
+        TOUCH,
+        MERGE,
+    ),
+    _arm(
+        "strength_merge_intersection",
+        "Require both faster-strength and tighter-merge confirmation.",
+        "intersection",
+        STRENGTH,
+        MERGE,
+    ),
+    _arm(
+        "touch_strength_merge_intersection",
+        "Require all three high-breadth confirmation variants.",
+        "intersection",
+        TOUCH,
+        STRENGTH,
+        MERGE,
+    ),
+)
+
+
+def _args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "snapshot",
+        nargs="?",
+        default="backtest_snapshots/prod.sqlite",
+        help="Local SQLite snapshot path.",
+    )
+    parser.add_argument("--workers", type=int, default=7)
+    parser.add_argument("--holdout-split", default="2024-07-01")
+    parser.add_argument(
+        "--out",
+        default=None,
+        help=(
+            "Consolidated JSON path. Defaults to "
+            "reports/backtest-YYYYMMDD-gtl-confirmation-matrix.json."
+        ),
+    )
+    parser.add_argument("--keep-arm-reports", action="store_true")
+    return parser.parse_args()
+
+
+def _config(arm: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "name": arm["name"],
+        "mode": arm["mode"],
+        "confirmations": arm["confirmations"],
+    }
+
+
+def _signature(arm: dict) -> dict:
+    return {
+        "candidates": arm.get("candidates"),
+        "qualified": arm.get("qualified"),
+        "qualified_net_avg_r": arm.get("qualified_net_avg_r"),
+        "qualified_net_avg_r_ex_top5": arm.get("qualified_net_avg_r_ex_top5"),
+        "full_book": arm.get("full_book"),
+        "holdout": arm.get("holdout"),
+    }
+
+
+def _reference_control() -> dict | None:
+    if not REFERENCE_MATRIX.exists():
+        return None
+    with REFERENCE_MATRIX.open(encoding="utf-8") as handle:
+        payload = json.load(handle)
+    return next(
+        (arm for arm in payload.get("arms") or [] if arm.get("name") == "control"),
+        None,
+    )
+
+
+def _write_markdown(path: Path, payload: dict) -> None:
+    rows = [
+        "# GTL confirmation/union matrix",
+        "",
+        f"Status: **{payload['status']}**  ",
+        f"Holdout split: `{payload['holdout_split']}`  ",
+        f"Completed arms: {len(payload['arms'])}/{payload['arm_count']}",
+        "",
+        "| Arm | Mode | Qualified | Full Sharpe | CAGR | Max DD | Trades | Train Sharpe | Test Sharpe | Ex-top-5% R | Screen |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for arm in payload["arms"]:
+        full = arm.get("full_book") or {}
+        holdout = arm.get("holdout") or {}
+        screen = arm.get("screen") or {}
+        rows.append(
+            "| "
+            + " | ".join((
+                arm["name"],
+                arm["config"]["mode"],
+                str(arm.get("qualified") or "-"),
+                common._fmt(full.get("sharpe")),
+                common._fmt(full.get("cagr_pct"), 1),
+                common._fmt(full.get("max_drawdown_pct"), 1),
+                str(full.get("trades") or "-"),
+                common._fmt((holdout.get("train") or {}).get("sharpe")),
+                common._fmt((holdout.get("test") or {}).get("sharpe")),
+                common._fmt(arm.get("qualified_net_avg_r_ex_top5"), 3),
+                f"{screen.get('passed', '-')}/{screen.get('total', '-')}",
+            ))
+            + " |"
+        )
+    rows.extend((
+        "",
+        "## Interpretation guardrail",
+        "",
+        "Intersections test the retained control cohort; unions test control plus genuinely added setups. The post-2024 interval is a robustness check, not a pristine holdout. Passing does not authorize deployment.",
+        "",
+    ))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(rows), encoding="utf-8")
+
+
+def _run_arm(
+    arm: dict[str, Any],
+    snapshot: Path,
+    workers: int,
+    holdout_split: str,
+    output: Path,
+) -> None:
+    command = [
+        sys.executable,
+        str(RUNNER),
+        str(snapshot),
+        "--workers", str(workers),
+        "--allow-spawn",
+        "--sr-variant", "gtl_confirmation",
+        "--gtl-confirm-config", json.dumps(_config(arm), separators=(",", ":")),
+        "--holdout-split", holdout_split,
+        "--sr-audit",
+        "--out", str(output),
+    ]
+    subprocess.run(command, cwd=ROOT, check=True)
+
+
+def main() -> None:
+    args = _args()
+    snapshot = Path(args.snapshot).resolve()
+    if not snapshot.exists():
+        raise SystemExit(f"Snapshot not found: {snapshot}")
+    if args.workers < 1:
+        raise SystemExit("--workers must be at least 1")
+    try:
+        date.fromisoformat(args.holdout_split)
+    except ValueError as exc:
+        raise SystemExit("--holdout-split must use YYYY-MM-DD") from exc
+
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    default_out = ROOT / "reports" / f"backtest-{stamp[:8]}-gtl-confirmation-matrix.json"
+    out_path = Path(args.out) if args.out else default_out
+    if not out_path.is_absolute():
+        out_path = ROOT / out_path
+    markdown_path = out_path.with_suffix(".md")
+    work_dir = ROOT / "reports" / f".gtl-confirmation-work-{stamp}"
+    work_dir.mkdir(parents=True, exist_ok=False)
+
+    reference = _reference_control()
+    payload: dict[str, Any] = {
+        "status": "running",
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "snapshot": str(snapshot),
+        "workers": args.workers,
+        "holdout_split": args.holdout_split,
+        "arm_count": len(GTL_CONFIRMATION_ARMS),
+        "reference_matrix": str(REFERENCE_MATRIX) if reference else None,
+        "arms": [],
+    }
+    common._write_json(out_path, payload)
+    _write_markdown(markdown_path, payload)
+
+    control_report: dict | None = None
+    arm_outputs: list[Path] = []
+    try:
+        for index, arm in enumerate(GTL_CONFIRMATION_ARMS, start=1):
+            name = arm["name"]
+            output = work_dir / f"{index:02d}-{name}.json"
+            arm_outputs.append(output)
+            print(f"\n[{index}/{len(GTL_CONFIRMATION_ARMS)}] {name}", flush=True)
+            print(f"  {arm['description']}", flush=True)
+            _run_arm(arm, snapshot, args.workers, args.holdout_split, output)
+            report = common._load_report(output)
+            compact = common._compact_arm(report, _config(arm), control_report)
+            compact["description"] = arm["description"]
+            if control_report is None:
+                control_report = report
+                compact["screen"] = {
+                    "checks": {}, "passed": 0, "total": 0, "advances": False,
+                }
+                if reference is not None and _signature(compact) != _signature(reference):
+                    raise RuntimeError(
+                        "Confirmation-path control does not reproduce the frozen GTL matrix control"
+                    )
+                payload["control_parity"] = "pass" if reference is not None else "not_checked"
+            else:
+                compact["screen"] = common._screen_arm(compact, payload["arms"][0])
+            payload["arms"].append(compact)
+            payload["completed_at"] = datetime.now(timezone.utc).isoformat()
+            common._write_json(out_path, payload)
+            _write_markdown(markdown_path, payload)
+    except BaseException as exc:
+        payload["status"] = "failed"
+        payload["failed_at"] = datetime.now(timezone.utc).isoformat()
+        payload["error"] = f"{type(exc).__name__}: {exc}"
+        payload["work_dir"] = str(work_dir)
+        common._write_json(out_path, payload)
+        _write_markdown(markdown_path, payload)
+        raise
+
+    payload["status"] = "complete"
+    payload["completed_at"] = datetime.now(timezone.utc).isoformat()
+    payload["advancing_arms"] = [
+        arm["name"]
+        for arm in payload["arms"]
+        if (arm.get("screen") or {}).get("advances")
+    ]
+    payload["ranking_by_full_sharpe"] = [
+        arm["name"]
+        for arm in sorted(
+            payload["arms"],
+            key=lambda row: float(
+                (row.get("full_book") or {}).get("sharpe") or -math.inf
+            ),
+            reverse=True,
+        )
+    ]
+    if args.keep_arm_reports:
+        payload["arm_report_directory"] = str(work_dir)
+    common._write_json(out_path, payload)
+    _write_markdown(markdown_path, payload)
+
+    if not args.keep_arm_reports:
+        for path in arm_outputs:
+            path.unlink(missing_ok=True)
+        work_dir.rmdir()
+
+    print("\nGTL confirmation matrix complete.")
+    print(f"  JSON: {out_path}")
+    print(f"  Markdown: {markdown_path}")
+    if payload["advancing_arms"]:
+        print("  Passing arms: " + ", ".join(payload["advancing_arms"]))
+    else:
+        print("  No arm passed every pre-registered screen.")
+
+
+if __name__ == "__main__":
+    main()