signal-platform/scripts/run_gtl_confirmation_matrix.py

"""Run the evidence-selected GTL confirmation/union matrix with one command.

The first GTL tuning matrix tested replacements. This follow-up decomposes the
four informative variants into retained-only intersections and control-plus-
addition unions while preserving frozen control geometry wherever possible.
"""

from __future__ import annotations

import argparse
import json
import math
import subprocess
import sys
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

from scripts import run_gtl_tuning_matrix as common  # noqa: E402

RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py"
REFERENCE_MATRIX = ROOT / "reports" / "backtest-20260713-gtl-tuning-matrix.json"

TOUCH = {"name": "touch_0_25pct", "touch_tolerance": 0.0025}
STRENGTH = {"name": "strength_1000", "strength_scale": 1000.0}
MERGE = {"name": "merge_0_25pct", "merge_tolerance": 0.0025}
GRID_ONLY = {"name": "pivots_none", "include_pivots": False}


def _arm(
    name: str,
    description: str,
    mode: str,
    *confirmations: dict[str, Any],
) -> dict[str, Any]:
    return {
        "name": name,
        "description": description,
        "mode": mode,
        "confirmations": list(confirmations),
    }


# Pre-registered from the replacement matrix's paired cohorts. Replacement
# arms that plainly removed strong control setups or added weak cohorts are not
# repeated here.
GTL_CONFIRMATION_ARMS: tuple[dict[str, Any], ...] = (
    _arm("control", "Frozen explicit GTL; composition-path parity control.", "intersection"),
    _arm(
        "touch_intersection",
        "Retain control setups also qualified with 0.25% touch padding.",
        "intersection",
        TOUCH,
    ),
    _arm(
        "touch_union",
        "Keep control and admit additions from 0.25% touch padding.",
        "union",
        TOUCH,
    ),
    _arm(
        "strength_intersection",
        "Retain control setups also qualified at strength scale 1000.",
        "intersection",
        STRENGTH,
    ),
    _arm(
        "strength_union",
        "Keep control and admit additions from strength scale 1000.",
        "union",
        STRENGTH,
    ),
    _arm(
        "merge_intersection",
        "Retain control setups also qualified with 0.25% proposal merging.",
        "intersection",
        MERGE,
    ),
    _arm(
        "merge_union",
        "Keep control and admit additions from 0.25% proposal merging.",
        "union",
        MERGE,
    ),
    _arm(
        "grid_intersection",
        "Retain control setups also qualified by the range grid without pivots.",
        "intersection",
        GRID_ONLY,
    ),
    _arm(
        "grid_union",
        "Keep control and admit additions from the range grid without pivots.",
        "union",
        GRID_ONLY,
    ),
    _arm(
        "touch_strength_intersection",
        "Require both tighter-touch and faster-strength confirmation.",
        "intersection",
        TOUCH,
        STRENGTH,
    ),
    _arm(
        "touch_merge_intersection",
        "Require both tighter-touch and tighter-merge confirmation.",
        "intersection",
        TOUCH,
        MERGE,
    ),
    _arm(
        "strength_merge_intersection",
        "Require both faster-strength and tighter-merge confirmation.",
        "intersection",
        STRENGTH,
        MERGE,
    ),
    _arm(
        "touch_strength_merge_intersection",
        "Require all three high-breadth confirmation variants.",
        "intersection",
        TOUCH,
        STRENGTH,
        MERGE,
    ),
)


def _args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "snapshot",
        nargs="?",
        default="backtest_snapshots/prod.sqlite",
        help="Local SQLite snapshot path.",
    )
    parser.add_argument("--workers", type=int, default=7)
    parser.add_argument("--holdout-split", default="2024-07-01")
    parser.add_argument(
        "--out",
        default=None,
        help=(
            "Consolidated JSON path. Defaults to "
            "reports/backtest-YYYYMMDD-gtl-confirmation-matrix.json."
        ),
    )
    parser.add_argument("--keep-arm-reports", action="store_true")
    return parser.parse_args()


def _config(arm: dict[str, Any]) -> dict[str, Any]:
    return {
        "name": arm["name"],
        "mode": arm["mode"],
        "confirmations": arm["confirmations"],
    }


def _signature(arm: dict) -> dict:
    return {
        "candidates": arm.get("candidates"),
        "qualified": arm.get("qualified"),
        "qualified_net_avg_r": arm.get("qualified_net_avg_r"),
        "qualified_net_avg_r_ex_top5": arm.get("qualified_net_avg_r_ex_top5"),
        "full_book": arm.get("full_book"),
        "holdout": arm.get("holdout"),
    }


def _reference_control() -> dict | None:
    if not REFERENCE_MATRIX.exists():
        return None
    with REFERENCE_MATRIX.open(encoding="utf-8") as handle:
        payload = json.load(handle)
    return next(
        (arm for arm in payload.get("arms") or [] if arm.get("name") == "control"),
        None,
    )


def _write_markdown(path: Path, payload: dict) -> None:
    rows = [
        "# GTL confirmation/union matrix",
        "",
        f"Status: **{payload['status']}**  ",
        f"Holdout split: `{payload['holdout_split']}`  ",
        f"Completed arms: {len(payload['arms'])}/{payload['arm_count']}",
        "",
        "| Arm | Mode | Qualified | Full Sharpe | CAGR | Max DD | Trades | Train Sharpe | Test Sharpe | Ex-top-5% R | Screen |",
        "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
    ]
    for arm in payload["arms"]:
        full = arm.get("full_book") or {}
        holdout = arm.get("holdout") or {}
        screen = arm.get("screen") or {}
        rows.append(
            "| "
            + " | ".join((
                arm["name"],
                arm["config"]["mode"],
                str(arm.get("qualified") or "-"),
                common._fmt(full.get("sharpe")),
                common._fmt(full.get("cagr_pct"), 1),
                common._fmt(full.get("max_drawdown_pct"), 1),
                str(full.get("trades") or "-"),
                common._fmt((holdout.get("train") or {}).get("sharpe")),
                common._fmt((holdout.get("test") or {}).get("sharpe")),
                common._fmt(arm.get("qualified_net_avg_r_ex_top5"), 3),
                f"{screen.get('passed', '-')}/{screen.get('total', '-')}",
            ))
            + " |"
        )
    rows.extend((
        "",
        "## Interpretation guardrail",
        "",
        "Intersections test the retained control cohort; unions test control plus genuinely added setups. The post-2024 interval is a robustness check, not a pristine holdout. Passing does not authorize deployment.",
        "",
    ))
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("\n".join(rows), encoding="utf-8")


def _run_arm(
    arm: dict[str, Any],
    snapshot: Path,
    workers: int,
    holdout_split: str,
    output: Path,
) -> None:
    command = [
        sys.executable,
        str(RUNNER),
        str(snapshot),
        "--workers", str(workers),
        "--allow-spawn",
        "--sr-variant", "gtl_confirmation",
        "--gtl-confirm-config", json.dumps(_config(arm), separators=(",", ":")),
        "--holdout-split", holdout_split,
        "--sr-audit",
        "--out", str(output),
    ]
    subprocess.run(command, cwd=ROOT, check=True)


def main() -> None:
    args = _args()
    snapshot = Path(args.snapshot).resolve()
    if not snapshot.exists():
        raise SystemExit(f"Snapshot not found: {snapshot}")
    if args.workers < 1:
        raise SystemExit("--workers must be at least 1")
    try:
        date.fromisoformat(args.holdout_split)
    except ValueError as exc:
        raise SystemExit("--holdout-split must use YYYY-MM-DD") from exc

    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
    default_out = ROOT / "reports" / f"backtest-{stamp[:8]}-gtl-confirmation-matrix.json"
    out_path = Path(args.out) if args.out else default_out
    if not out_path.is_absolute():
        out_path = ROOT / out_path
    markdown_path = out_path.with_suffix(".md")
    work_dir = ROOT / "reports" / f".gtl-confirmation-work-{stamp}"
    work_dir.mkdir(parents=True, exist_ok=False)

    reference = _reference_control()
    payload: dict[str, Any] = {
        "status": "running",
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "snapshot": str(snapshot),
        "workers": args.workers,
        "holdout_split": args.holdout_split,
        "arm_count": len(GTL_CONFIRMATION_ARMS),
        "reference_matrix": str(REFERENCE_MATRIX) if reference else None,
        "arms": [],
    }
    common._write_json(out_path, payload)
    _write_markdown(markdown_path, payload)

    control_report: dict | None = None
    arm_outputs: list[Path] = []
    try:
        for index, arm in enumerate(GTL_CONFIRMATION_ARMS, start=1):
            name = arm["name"]
            output = work_dir / f"{index:02d}-{name}.json"
            arm_outputs.append(output)
            print(f"\n[{index}/{len(GTL_CONFIRMATION_ARMS)}] {name}", flush=True)
            print(f"  {arm['description']}", flush=True)
            _run_arm(arm, snapshot, args.workers, args.holdout_split, output)
            report = common._load_report(output)
            compact = common._compact_arm(report, _config(arm), control_report)
            compact["description"] = arm["description"]
            if control_report is None:
                control_report = report
                compact["screen"] = {
                    "checks": {}, "passed": 0, "total": 0, "advances": False,
                }
                if reference is not None and _signature(compact) != _signature(reference):
                    raise RuntimeError(
                        "Confirmation-path control does not reproduce the frozen GTL matrix control"
                    )
                payload["control_parity"] = "pass" if reference is not None else "not_checked"
            else:
                compact["screen"] = common._screen_arm(compact, payload["arms"][0])
            payload["arms"].append(compact)
            payload["completed_at"] = datetime.now(timezone.utc).isoformat()
            common._write_json(out_path, payload)
            _write_markdown(markdown_path, payload)
    except BaseException as exc:
        payload["status"] = "failed"
        payload["failed_at"] = datetime.now(timezone.utc).isoformat()
        payload["error"] = f"{type(exc).__name__}: {exc}"
        payload["work_dir"] = str(work_dir)
        common._write_json(out_path, payload)
        _write_markdown(markdown_path, payload)
        raise

    payload["status"] = "complete"
    payload["completed_at"] = datetime.now(timezone.utc).isoformat()
    payload["advancing_arms"] = [
        arm["name"]
        for arm in payload["arms"]
        if (arm.get("screen") or {}).get("advances")
    ]
    payload["ranking_by_full_sharpe"] = [
        arm["name"]
        for arm in sorted(
            payload["arms"],
            key=lambda row: float(
                (row.get("full_book") or {}).get("sharpe") or -math.inf
            ),
            reverse=True,
        )
    ]
    if args.keep_arm_reports:
        payload["arm_report_directory"] = str(work_dir)
    common._write_json(out_path, payload)
    _write_markdown(markdown_path, payload)

    if not args.keep_arm_reports:
        for path in arm_outputs:
            path.unlink(missing_ok=True)
        work_dir.rmdir()

    print("\nGTL confirmation matrix complete.")
    print(f"  JSON: {out_path}")
    print(f"  Markdown: {markdown_path}")
    if payload["advancing_arms"]:
        print("  Passing arms: " + ", ".join(payload["advancing_arms"]))
    else:
        print("  No arm passed every pre-registered screen.")


if __name__ == "__main__":
    main()