Add GTL cohort composition backtest

This commit is contained in:
2026-07-13 14:22:56 +02:00
parent 3f86aec0be
commit 623dc08875
8 changed files with 755 additions and 6 deletions
+14
View File
@@ -486,6 +486,20 @@ the run fails, they remain available for diagnosis; pass `--keep-arm-reports`
to retain them after success too. No arm changes live scanner defaults or
deploys anything.
The replacement matrix found no winning single constant. Its evidence-selected
follow-up keeps control geometry and isolates retained versus added cohorts in
one 13-arm confirmation/union run:
```bash
# macOS/Linux
.venv/bin/python scripts/run_gtl_confirmation_matrix.py \
backtest_snapshots/prod.sqlite --workers 12
```
It first verifies exact control parity with the completed tuning matrix, then
checkpoints consolidated JSON and Markdown reports under
`reports/backtest-YYYYMMDD-gtl-confirmation-matrix.*`.
### Reading a local backtest report
The deployed **Signals → Track Record** page is deliberately trimmed to validation
+187 -6
View File
@@ -111,6 +111,7 @@ STRUCTURAL_OVERLAY_WEIGHT = 0.05
STRUCTURAL_OVERLAY_SCORE_KEY = "structural_overlay_95_5_score"
EXPLICIT_TARGET_LADDER_VARIANT = "explicit_target_ladder"
GTL_TUNING_VARIANT = "gtl_tuning"
GTL_CONFIRMATION_VARIANT = "gtl_confirmation"
RANGE_RESIDUAL_VARIANTS = {
"rewrite_range504_structural_legacy_primary",
"rewrite_range504_structural_primary2",
@@ -167,6 +168,23 @@ class GTLResearchConfig:
strength_scale=self.strength_scale,
)
@dataclass(frozen=True)
class GTLConfirmationConfig:
"""Research-only composition of the frozen GTL and tuned variants."""
name: str = "control"
mode: str = "intersection"
confirmations: tuple[GTLResearchConfig, ...] = ()
def __post_init__(self) -> None:
if not self.name.strip():
raise ValueError("GTL confirmation config name must not be empty")
if self.mode not in {"intersection", "union"}:
raise ValueError("GTL confirmation mode must be intersection or union")
if self.mode == "union" and len(self.confirmations) != 1:
raise ValueError("GTL union mode requires exactly one tuned variant")
# Cross-sectional signal evaluation (factor IC). Each candidate signal is a
# point-in-time number computed from closes alone (sentiment/fundamentals have no
# history here), sampled one as-of per ISO week, and graded by how its rank
@@ -222,6 +240,7 @@ SR_RESEARCH_VARIANTS = {
"legacy_range_grid_neutral",
EXPLICIT_TARGET_LADDER_VARIANT,
GTL_TUNING_VARIANT,
GTL_CONFIRMATION_VARIANT,
STRUCTURAL_OVERLAY_VARIANT,
*RANGE_FACTOR_VARIANTS,
}
@@ -261,6 +280,47 @@ def _gtl_research_config() -> GTLResearchConfig:
return _parse_gtl_research_config(os.getenv("BACKTEST_GTL_CONFIG", ""))
@lru_cache(maxsize=32)
def _parse_gtl_confirmation_config(raw: str) -> GTLConfirmationConfig:
"""Parse one composition arm for the offline confirmation matrix."""
if not raw.strip():
return GTLConfirmationConfig()
try:
payload = json.loads(raw)
except json.JSONDecodeError as exc:
raise ValueError("BACKTEST_GTL_CONFIRM_CONFIG must be valid JSON") from exc
if not isinstance(payload, dict):
raise ValueError("BACKTEST_GTL_CONFIRM_CONFIG must be a JSON object")
allowed = {"name", "mode", "confirmations"}
unknown = sorted(set(payload) - allowed)
if unknown:
raise ValueError(
f"Unknown GTL confirmation config fields: {', '.join(unknown)}"
)
raw_confirmations = payload.get("confirmations", [])
if not isinstance(raw_confirmations, list):
raise ValueError("GTL confirmations must be a JSON array")
confirmations: list[GTLResearchConfig] = []
for item in raw_confirmations:
if not isinstance(item, dict):
raise ValueError("Each GTL confirmation must be a JSON object")
confirmations.append(_parse_gtl_research_config(json.dumps(item)))
try:
return GTLConfirmationConfig(
name=payload.get("name", "control"),
mode=payload.get("mode", "intersection"),
confirmations=tuple(confirmations),
)
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError(f"Invalid GTL confirmation config: {exc}") from exc
def _gtl_confirmation_config() -> GTLConfirmationConfig:
return _parse_gtl_confirmation_config(
os.getenv("BACKTEST_GTL_CONFIRM_CONFIG", "")
)
def _sr_detector_variant(sr_variant: str) -> str:
"""Map factor-gated research arms to the detector they hold fixed."""
if sr_variant in {"production_range504", STRUCTURAL_OVERLAY_VARIANT}:
@@ -302,6 +362,7 @@ def _primary_min_rr_for_variant(sr_variant: str, activation: dict) -> float:
"production_range504",
EXPLICIT_TARGET_LADDER_VARIANT,
GTL_TUNING_VARIANT,
GTL_CONFIRMATION_VARIANT,
STRUCTURAL_OVERLAY_VARIANT,
}
or sr_variant.endswith("_legacy_primary")
@@ -416,6 +477,7 @@ def _window_setups(
activation: dict,
*,
sr_variant: str | None = None,
gtl_research_config: GTLResearchConfig | None = None,
) -> list[dict]:
"""Rebuild the setup(s) at the last bar of ``window_records`` (the as-of date),
using only those bars. Returns one dict per tradeable direction."""
@@ -444,7 +506,11 @@ def _window_setups(
sr_variant = sr_variant or _sr_research_variant()
detector_variant = _sr_detector_variant(sr_variant)
gtl_config = _gtl_research_config() if sr_variant == GTL_TUNING_VARIANT else None
gtl_config = (
gtl_research_config or _gtl_research_config()
if sr_variant == GTL_TUNING_VARIANT
else None
)
range_504_log = _range_504_log(highs, lows)
if sr_variant == "legacy_geometry_neutral":
detected_levels = detect_sr_levels_legacy(
@@ -683,6 +749,87 @@ def _structural_overlay_window_setups(
return tagged
def _gtl_confirmation_window_setups(
window_records: list,
config: dict,
activation: dict,
*,
confirmation_config: GTLConfirmationConfig | None = None,
) -> list[dict]:
"""Compose tuned GTLs around the frozen ladder without changing it silently.
``intersection`` retains the frozen setup geometry and requires every tuned
variant to clear the core gate in the same direction. ``union`` preserves a
frozen setup whenever it already clears the core gate, and otherwise admits
the one tuned variant's setup. This makes retained, removed, and added
cohorts explicit instead of conflating them in a replacement arm.
"""
research = confirmation_config or _gtl_confirmation_config()
production = _window_setups(
window_records,
config,
activation,
sr_variant=EXPLICIT_TARGET_LADDER_VARIANT,
)
tuned_sets = [
_window_setups(
window_records,
config,
activation,
sr_variant=GTL_TUNING_VARIANT,
gtl_research_config=tuned_config,
)
for tuned_config in research.confirmations
]
tuned_by_direction = [
{row["direction"]: row for row in rows}
for rows in tuned_sets
]
def annotate(row: dict, passes: list[bool], source: str) -> dict:
tagged = dict(row)
tagged["sr_variant"] = GTL_CONFIRMATION_VARIANT
tagged["gtl_confirmation_name"] = research.name
tagged["gtl_confirmation_mode"] = research.mode
tagged["gtl_confirmation_source"] = source
tagged["gtl_confirmation_passes"] = passes
tagged["gtl_confirmation_all_pass"] = all(passes)
return tagged
if research.mode == "intersection":
tagged: list[dict] = []
for production_row in production:
direction = production_row["direction"]
passes = [
bool(rows.get(direction) and rows[direction].get("meets_core"))
for rows in tuned_by_direction
]
row = annotate(production_row, passes, "control")
row["meets_core"] = bool(production_row.get("meets_core")) and all(
passes
)
tagged.append(row)
return tagged
# Union mode is validated to contain exactly one tuned variant. Keep one
# setup per direction: frozen geometry wins whenever it already qualifies;
# tuned geometry is used only for a genuinely added core-qualified setup.
production_by_direction = {row["direction"]: row for row in production}
tuned_by_dir = tuned_by_direction[0]
tagged = []
for direction in sorted(set(production_by_direction) | set(tuned_by_dir)):
production_row = production_by_direction.get(direction)
tuned_row = tuned_by_dir.get(direction)
tuned_pass = bool(tuned_row and tuned_row.get("meets_core"))
if production_row is not None and production_row.get("meets_core"):
tagged.append(annotate(production_row, [tuned_pass], "control"))
elif tuned_pass and tuned_row is not None:
tagged.append(annotate(tuned_row, [True], "tuned_addition"))
elif production_row is not None:
tagged.append(annotate(production_row, [tuned_pass], "control"))
return tagged
def _stop_fill_r(direction: str, entry: float, stop: float, bar) -> float:
"""Realized R when the stop is hit on ``bar``: filled at the stop, or at the
bar's open when price gapped through it — so a gap can lose more than 1R,
@@ -787,16 +934,17 @@ def _replay_ticker(
)
vol_6m = _realized_vol_6m(closes, len(window) - 1)
setups = (
_structural_overlay_window_setups(window, config, activation)
if sr_variant == STRUCTURAL_OVERLAY_VARIANT
else _window_setups(
if sr_variant == STRUCTURAL_OVERLAY_VARIANT:
setups = _structural_overlay_window_setups(window, config, activation)
elif sr_variant == GTL_CONFIRMATION_VARIANT:
setups = _gtl_confirmation_window_setups(window, config, activation)
else:
setups = _window_setups(
window,
config,
activation,
sr_variant=sr_variant,
)
)
for s in setups:
outcome, outcome_date = evaluate_setup_against_bars(
s["direction"], s["stop"], s["target"], forward_bars, HORIZON
@@ -862,6 +1010,13 @@ def _replay_ticker(
"structural_overlay_gate_level_count": s.get(
"structural_overlay_gate_level_count"
),
"gtl_confirmation_name": s.get("gtl_confirmation_name"),
"gtl_confirmation_mode": s.get("gtl_confirmation_mode"),
"gtl_confirmation_source": s.get("gtl_confirmation_source"),
"gtl_confirmation_passes": s.get("gtl_confirmation_passes"),
"gtl_confirmation_all_pass": s.get(
"gtl_confirmation_all_pass"
),
"outcome": outcome,
"target_hit": target_hit,
"realized_r": realized_r,
@@ -934,6 +1089,9 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
range_logs: list[float] = []
overlay_rows = 0
overlay_pass = 0
confirmation_rows = 0
confirmation_pass = 0
confirmation_tuned_additions = 0
for cand in candidates:
sources = list(cand.get("primary_sources") or [])
for source in sources:
@@ -949,6 +1107,12 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
if cand.get("structural_overlay_pass") is not None:
overlay_rows += 1
overlay_pass += int(bool(cand["structural_overlay_pass"]))
if cand.get("gtl_confirmation_all_pass") is not None:
confirmation_rows += 1
confirmation_pass += int(bool(cand["gtl_confirmation_all_pass"]))
confirmation_tuned_additions += int(
cand.get("gtl_confirmation_source") == "tuned_addition"
)
def avg(values: list[float] | list[int]) -> float | None:
return round(sum(values) / len(values), 3) if values else None
@@ -973,6 +1137,9 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
"structural_overlay_weight": (
STRUCTURAL_OVERLAY_WEIGHT if overlay_rows else None
),
"gtl_confirmation_rows": confirmation_rows,
"gtl_confirmation_pass": confirmation_pass,
"gtl_confirmation_tuned_additions": confirmation_tuned_additions,
}
@@ -1019,6 +1186,15 @@ def _sr_candidate_audit(candidates: list[dict], min_percentile: float) -> list[d
"structural_overlay_gate_level_count": int(
cand.get("structural_overlay_gate_level_count", 0) or 0
),
"gtl_confirmation_name": cand.get("gtl_confirmation_name"),
"gtl_confirmation_mode": cand.get("gtl_confirmation_mode"),
"gtl_confirmation_source": cand.get("gtl_confirmation_source"),
"gtl_confirmation_passes": list(
cand.get("gtl_confirmation_passes") or []
),
"gtl_confirmation_all_pass": cand.get(
"gtl_confirmation_all_pass"
),
"rr": round(float(cand.get("rr", 0.0)), 6),
"primary_prob": round(float(cand.get("primary_prob", 0.0)), 6),
"primary_sources": list(cand.get("primary_sources") or []),
@@ -3430,6 +3606,11 @@ async def run_backtest(
if _sr_research_variant() == GTL_TUNING_VARIANT
else None
),
"gtl_confirmation_config": (
asdict(_gtl_confirmation_config())
if _sr_research_variant() == GTL_CONFIRMATION_VARIANT
else None
),
"range_factor_lookback": RANGE_FACTOR_LOOKBACK,
"range_factor_min_log": RANGE_FACTOR_MIN_LOG,
"range_factor_min_ratio": round(math.exp(RANGE_FACTOR_MIN_LOG), 4),
+34
View File
@@ -845,6 +845,40 @@ the top 5%. Passing identifies a candidate for forward paper validation, not an
automatic deployment. The post-2024 interval has already influenced this
research, so the split is a robustness check rather than a pristine holdout.
Result on the 2026-07-13 snapshot: **20/20 arms completed; no replacement arm
passed all six checks.** The frozen control remained best on full-period Sharpe
(2.03), CAGR (50.0%), and post-2024 Sharpe (2.78), with 321 production trades
and 21.4% drawdown. The closest replacement, 0.25% touch padding, still fell to
Sharpe 1.94 / CAGR 47.9%. This rejects direct constant replacement; the inherited
behavior is not explained by one obvious GTL knob.
The paired cohorts do expose a narrower mechanism worth testing:
| Variant | Retained control setups | Added by variant | Removed from control |
|---|---|---|---|
| 0.25% touch | 1,049 at +0.214R (+0.058 ex-top-5%) | 18 at -0.235R | 37 at -0.056R |
| Strength 1000 | 1,037 at +0.225R (+0.069) | 122 at +0.301R (+0.152) | 49 at +0.142R (-0.044) |
| 0.25% merge | 791 at +0.238R (+0.078) | 365 at +0.161R (+0.023) | 295 at +0.129R (-0.008) |
| Grid without pivots | 428 at +0.232R (+0.100) | 392 at +0.176R (+0.046) | 658 at +0.208R (+0.041) |
Replacement mixes the retained and added cohorts and also discards the removed
cohort, so its portfolio result cannot say which part helped. The follow-up
`gtl_confirmation` matrix therefore preserves frozen control geometry and
decomposes each selected variant into:
- **intersection** — only control setups also core-qualified by the variant;
- **union** — all core-qualified control setups plus genuinely added variant
setups, using tuned geometry only for those additions.
It also tests pre-registered intersections among the three high-breadth
confirmers. Its control path must exactly reproduce the completed tuning
matrix before any research arm is accepted.
```bash
.venv/bin/python scripts/run_gtl_confirmation_matrix.py \
backtest_snapshots/prod.sqlite --workers 12
```
The post-2024 window has been opened and is now analysis data, not a valid final
promotion holdout. These arms can isolate mechanism, but neither may ship without
new future data or a separately pre-registered walk-forward protocol.
+10
View File
@@ -69,3 +69,13 @@ The next decision point is generated by the one-command GTL parameter run:
These two consolidated files are the reports worth retaining. The runner uses
a hidden temporary directory for its 20 full per-arm reports and removes it
after successful consolidation unless `--keep-arm-reports` is supplied.
The completed 2026-07-13 matrix found no single-parameter replacement that
passed all robustness checks. Retain its JSON and Markdown as that decision
point. The evidence-selected retained-versus-added decomposition writes:
- `backtest-YYYYMMDD-gtl-confirmation-matrix.json`
- `backtest-YYYYMMDD-gtl-confirmation-matrix.md`
Those become the next decision point; detailed per-arm reports remain temporary
unless explicitly retained.
+15
View File
@@ -62,6 +62,7 @@ def _parse_args() -> argparse.Namespace:
"production_structural_overlay",
"explicit_target_ladder",
"gtl_tuning",
"gtl_confirmation",
),
default=None,
help="Research-only S/R detector/gate arm.",
@@ -71,6 +72,14 @@ def _parse_args() -> argparse.Namespace:
default=None,
help="Research-only GTL configuration as a JSON object (requires --sr-variant gtl_tuning).",
)
parser.add_argument(
"--gtl-confirm-config",
default=None,
help=(
"Research-only GTL intersection/union configuration as JSON "
"(requires --sr-variant gtl_confirmation)."
),
)
parser.add_argument(
"--entry-start",
default=None,
@@ -210,6 +219,12 @@ async def _main() -> None:
if args.sr_variant != "gtl_tuning":
raise SystemExit("--gtl-config requires --sr-variant gtl_tuning")
os.environ["BACKTEST_GTL_CONFIG"] = args.gtl_config
if args.gtl_confirm_config:
if args.sr_variant != "gtl_confirmation":
raise SystemExit(
"--gtl-confirm-config requires --sr-variant gtl_confirmation"
)
os.environ["BACKTEST_GTL_CONFIRM_CONFIG"] = args.gtl_confirm_config
if args.entry_start:
os.environ["BACKTEST_ENTRY_START"] = args.entry_start
if args.entry_end:
+360
View File
@@ -0,0 +1,360 @@
"""Run the evidence-selected GTL confirmation/union matrix with one command.
The first GTL tuning matrix tested replacements. This follow-up decomposes the
four informative variants into retained-only intersections and control-plus-
addition unions while preserving frozen control geometry wherever possible.
"""
from __future__ import annotations
import argparse
import json
import math
import subprocess
import sys
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts import run_gtl_tuning_matrix as common # noqa: E402
RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py"
REFERENCE_MATRIX = ROOT / "reports" / "backtest-20260713-gtl-tuning-matrix.json"
TOUCH = {"name": "touch_0_25pct", "touch_tolerance": 0.0025}
STRENGTH = {"name": "strength_1000", "strength_scale": 1000.0}
MERGE = {"name": "merge_0_25pct", "merge_tolerance": 0.0025}
GRID_ONLY = {"name": "pivots_none", "include_pivots": False}
def _arm(
name: str,
description: str,
mode: str,
*confirmations: dict[str, Any],
) -> dict[str, Any]:
return {
"name": name,
"description": description,
"mode": mode,
"confirmations": list(confirmations),
}
# Pre-registered from the replacement matrix's paired cohorts. Replacement
# arms that plainly removed strong control setups or added weak cohorts are not
# repeated here.
GTL_CONFIRMATION_ARMS: tuple[dict[str, Any], ...] = (
_arm("control", "Frozen explicit GTL; composition-path parity control.", "intersection"),
_arm(
"touch_intersection",
"Retain control setups also qualified with 0.25% touch padding.",
"intersection",
TOUCH,
),
_arm(
"touch_union",
"Keep control and admit additions from 0.25% touch padding.",
"union",
TOUCH,
),
_arm(
"strength_intersection",
"Retain control setups also qualified at strength scale 1000.",
"intersection",
STRENGTH,
),
_arm(
"strength_union",
"Keep control and admit additions from strength scale 1000.",
"union",
STRENGTH,
),
_arm(
"merge_intersection",
"Retain control setups also qualified with 0.25% proposal merging.",
"intersection",
MERGE,
),
_arm(
"merge_union",
"Keep control and admit additions from 0.25% proposal merging.",
"union",
MERGE,
),
_arm(
"grid_intersection",
"Retain control setups also qualified by the range grid without pivots.",
"intersection",
GRID_ONLY,
),
_arm(
"grid_union",
"Keep control and admit additions from the range grid without pivots.",
"union",
GRID_ONLY,
),
_arm(
"touch_strength_intersection",
"Require both tighter-touch and faster-strength confirmation.",
"intersection",
TOUCH,
STRENGTH,
),
_arm(
"touch_merge_intersection",
"Require both tighter-touch and tighter-merge confirmation.",
"intersection",
TOUCH,
MERGE,
),
_arm(
"strength_merge_intersection",
"Require both faster-strength and tighter-merge confirmation.",
"intersection",
STRENGTH,
MERGE,
),
_arm(
"touch_strength_merge_intersection",
"Require all three high-breadth confirmation variants.",
"intersection",
TOUCH,
STRENGTH,
MERGE,
),
)
def _args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"snapshot",
nargs="?",
default="backtest_snapshots/prod.sqlite",
help="Local SQLite snapshot path.",
)
parser.add_argument("--workers", type=int, default=7)
parser.add_argument("--holdout-split", default="2024-07-01")
parser.add_argument(
"--out",
default=None,
help=(
"Consolidated JSON path. Defaults to "
"reports/backtest-YYYYMMDD-gtl-confirmation-matrix.json."
),
)
parser.add_argument("--keep-arm-reports", action="store_true")
return parser.parse_args()
def _config(arm: dict[str, Any]) -> dict[str, Any]:
return {
"name": arm["name"],
"mode": arm["mode"],
"confirmations": arm["confirmations"],
}
def _signature(arm: dict) -> dict:
return {
"candidates": arm.get("candidates"),
"qualified": arm.get("qualified"),
"qualified_net_avg_r": arm.get("qualified_net_avg_r"),
"qualified_net_avg_r_ex_top5": arm.get("qualified_net_avg_r_ex_top5"),
"full_book": arm.get("full_book"),
"holdout": arm.get("holdout"),
}
def _reference_control() -> dict | None:
if not REFERENCE_MATRIX.exists():
return None
with REFERENCE_MATRIX.open(encoding="utf-8") as handle:
payload = json.load(handle)
return next(
(arm for arm in payload.get("arms") or [] if arm.get("name") == "control"),
None,
)
def _write_markdown(path: Path, payload: dict) -> None:
rows = [
"# GTL confirmation/union matrix",
"",
f"Status: **{payload['status']}** ",
f"Holdout split: `{payload['holdout_split']}` ",
f"Completed arms: {len(payload['arms'])}/{payload['arm_count']}",
"",
"| Arm | Mode | Qualified | Full Sharpe | CAGR | Max DD | Trades | Train Sharpe | Test Sharpe | Ex-top-5% R | Screen |",
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
]
for arm in payload["arms"]:
full = arm.get("full_book") or {}
holdout = arm.get("holdout") or {}
screen = arm.get("screen") or {}
rows.append(
"| "
+ " | ".join((
arm["name"],
arm["config"]["mode"],
str(arm.get("qualified") or "-"),
common._fmt(full.get("sharpe")),
common._fmt(full.get("cagr_pct"), 1),
common._fmt(full.get("max_drawdown_pct"), 1),
str(full.get("trades") or "-"),
common._fmt((holdout.get("train") or {}).get("sharpe")),
common._fmt((holdout.get("test") or {}).get("sharpe")),
common._fmt(arm.get("qualified_net_avg_r_ex_top5"), 3),
f"{screen.get('passed', '-')}/{screen.get('total', '-')}",
))
+ " |"
)
rows.extend((
"",
"## Interpretation guardrail",
"",
"Intersections test the retained control cohort; unions test control plus genuinely added setups. The post-2024 interval is a robustness check, not a pristine holdout. Passing does not authorize deployment.",
"",
))
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(rows), encoding="utf-8")
def _run_arm(
arm: dict[str, Any],
snapshot: Path,
workers: int,
holdout_split: str,
output: Path,
) -> None:
command = [
sys.executable,
str(RUNNER),
str(snapshot),
"--workers", str(workers),
"--allow-spawn",
"--sr-variant", "gtl_confirmation",
"--gtl-confirm-config", json.dumps(_config(arm), separators=(",", ":")),
"--holdout-split", holdout_split,
"--sr-audit",
"--out", str(output),
]
subprocess.run(command, cwd=ROOT, check=True)
def main() -> None:
args = _args()
snapshot = Path(args.snapshot).resolve()
if not snapshot.exists():
raise SystemExit(f"Snapshot not found: {snapshot}")
if args.workers < 1:
raise SystemExit("--workers must be at least 1")
try:
date.fromisoformat(args.holdout_split)
except ValueError as exc:
raise SystemExit("--holdout-split must use YYYY-MM-DD") from exc
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
default_out = ROOT / "reports" / f"backtest-{stamp[:8]}-gtl-confirmation-matrix.json"
out_path = Path(args.out) if args.out else default_out
if not out_path.is_absolute():
out_path = ROOT / out_path
markdown_path = out_path.with_suffix(".md")
work_dir = ROOT / "reports" / f".gtl-confirmation-work-{stamp}"
work_dir.mkdir(parents=True, exist_ok=False)
reference = _reference_control()
payload: dict[str, Any] = {
"status": "running",
"generated_at": datetime.now(timezone.utc).isoformat(),
"snapshot": str(snapshot),
"workers": args.workers,
"holdout_split": args.holdout_split,
"arm_count": len(GTL_CONFIRMATION_ARMS),
"reference_matrix": str(REFERENCE_MATRIX) if reference else None,
"arms": [],
}
common._write_json(out_path, payload)
_write_markdown(markdown_path, payload)
control_report: dict | None = None
arm_outputs: list[Path] = []
try:
for index, arm in enumerate(GTL_CONFIRMATION_ARMS, start=1):
name = arm["name"]
output = work_dir / f"{index:02d}-{name}.json"
arm_outputs.append(output)
print(f"\n[{index}/{len(GTL_CONFIRMATION_ARMS)}] {name}", flush=True)
print(f" {arm['description']}", flush=True)
_run_arm(arm, snapshot, args.workers, args.holdout_split, output)
report = common._load_report(output)
compact = common._compact_arm(report, _config(arm), control_report)
compact["description"] = arm["description"]
if control_report is None:
control_report = report
compact["screen"] = {
"checks": {}, "passed": 0, "total": 0, "advances": False,
}
if reference is not None and _signature(compact) != _signature(reference):
raise RuntimeError(
"Confirmation-path control does not reproduce the frozen GTL matrix control"
)
payload["control_parity"] = "pass" if reference is not None else "not_checked"
else:
compact["screen"] = common._screen_arm(compact, payload["arms"][0])
payload["arms"].append(compact)
payload["completed_at"] = datetime.now(timezone.utc).isoformat()
common._write_json(out_path, payload)
_write_markdown(markdown_path, payload)
except BaseException as exc:
payload["status"] = "failed"
payload["failed_at"] = datetime.now(timezone.utc).isoformat()
payload["error"] = f"{type(exc).__name__}: {exc}"
payload["work_dir"] = str(work_dir)
common._write_json(out_path, payload)
_write_markdown(markdown_path, payload)
raise
payload["status"] = "complete"
payload["completed_at"] = datetime.now(timezone.utc).isoformat()
payload["advancing_arms"] = [
arm["name"]
for arm in payload["arms"]
if (arm.get("screen") or {}).get("advances")
]
payload["ranking_by_full_sharpe"] = [
arm["name"]
for arm in sorted(
payload["arms"],
key=lambda row: float(
(row.get("full_book") or {}).get("sharpe") or -math.inf
),
reverse=True,
)
]
if args.keep_arm_reports:
payload["arm_report_directory"] = str(work_dir)
common._write_json(out_path, payload)
_write_markdown(markdown_path, payload)
if not args.keep_arm_reports:
for path in arm_outputs:
path.unlink(missing_ok=True)
work_dir.rmdir()
print("\nGTL confirmation matrix complete.")
print(f" JSON: {out_path}")
print(f" Markdown: {markdown_path}")
if payload["advancing_arms"]:
print(" Passing arms: " + ", ".join(payload["advancing_arms"]))
else:
print(" No arm passed every pre-registered screen.")
if __name__ == "__main__":
main()
+96
View File
@@ -805,6 +805,7 @@ def test_sr_research_variant_is_explicit_and_validated(monkeypatch):
"production_structural_overlay",
"explicit_target_ladder",
"gtl_tuning",
"gtl_confirmation",
):
monkeypatch.setenv("BACKTEST_SR_VARIANT", variant)
assert bt._sr_research_variant() == variant
@@ -864,6 +865,7 @@ def test_residual_arms_change_only_the_primary_rr_floor():
activation,
) == 1.5
assert bt._primary_min_rr_for_variant("gtl_tuning", activation) == 1.5
assert bt._primary_min_rr_for_variant("gtl_confirmation", activation) == 1.5
def test_gtl_research_config_parses_and_rejects_unknown_fields():
@@ -883,6 +885,29 @@ def test_gtl_research_config_parses_and_rejects_unknown_fields():
bt._parse_gtl_research_config("{")
def test_gtl_confirmation_config_parses_and_validates_composition():
config = bt._parse_gtl_confirmation_config(json.dumps({
"name": "touch_strength_intersection",
"mode": "intersection",
"confirmations": [
{"name": "touch", "touch_tolerance": 0.0025},
{"name": "strength", "strength_scale": 1000.0},
],
}))
assert config.name == "touch_strength_intersection"
assert config.mode == "intersection"
assert len(config.confirmations) == 2
assert config.confirmations[0].touch_tolerance == 0.0025
assert config.confirmations[1].strength_scale == 1000.0
with pytest.raises(ValueError, match="exactly one tuned variant"):
bt._parse_gtl_confirmation_config(json.dumps({
"name": "invalid_union",
"mode": "union",
"confirmations": [],
}))
def test_structural_overlay_tags_production_geometry_without_replacing_it(monkeypatch):
production = {
"direction": "long",
@@ -1031,6 +1056,77 @@ def test_window_setups_routes_gtl_tuning_config(monkeypatch):
assert captured["config"].grid_bins == 12
def test_gtl_confirmation_intersection_keeps_control_geometry(monkeypatch):
production = [{
"direction": "long",
"target": 111.0,
"rr": 2.2,
"meets_core": True,
"sr_variant": bt.EXPLICIT_TARGET_LADDER_VARIANT,
}]
def fake_window_setups(*args, sr_variant=None, gtl_research_config=None, **kwargs):
if sr_variant == bt.EXPLICIT_TARGET_LADDER_VARIANT:
return production
assert sr_variant == bt.GTL_TUNING_VARIANT
return [{
"direction": "long",
"target": 115.0,
"rr": 3.0,
"meets_core": gtl_research_config.name == "pass",
}]
monkeypatch.setattr(bt, "_window_setups", fake_window_setups)
research = bt.GTLConfirmationConfig(
name="two_filters",
mode="intersection",
confirmations=(
bt.GTLResearchConfig(name="pass"),
bt.GTLResearchConfig(name="fail"),
),
)
rows = bt._gtl_confirmation_window_setups(
[], {}, {}, confirmation_config=research
)
assert len(rows) == 1
assert rows[0]["target"] == 111.0
assert rows[0]["rr"] == 2.2
assert rows[0]["meets_core"] is False
assert rows[0]["gtl_confirmation_passes"] == [True, False]
assert production[0]["meets_core"] is True
def test_gtl_confirmation_union_uses_tuned_geometry_only_for_addition(monkeypatch):
production = [
{"direction": "long", "target": 111.0, "meets_core": False},
{"direction": "short", "target": 90.0, "meets_core": True},
]
tuned = [
{"direction": "long", "target": 115.0, "meets_core": True},
{"direction": "short", "target": 85.0, "meets_core": False},
]
def fake_window_setups(*args, sr_variant=None, **kwargs):
return production if sr_variant == bt.EXPLICIT_TARGET_LADDER_VARIANT else tuned
monkeypatch.setattr(bt, "_window_setups", fake_window_setups)
research = bt.GTLConfirmationConfig(
name="strength_union",
mode="union",
confirmations=(bt.GTLResearchConfig(name="strength"),),
)
rows = bt._gtl_confirmation_window_setups(
[], {}, {}, confirmation_config=research
)
by_direction = {row["direction"]: row for row in rows}
assert by_direction["long"]["target"] == 115.0
assert by_direction["long"]["gtl_confirmation_source"] == "tuned_addition"
assert by_direction["short"]["target"] == 90.0
assert by_direction["short"]["gtl_confirmation_source"] == "control"
@pytest.mark.parametrize(
"variant",
["legacy_geometry_neutral", "legacy_range_grid_neutral"],
@@ -0,0 +1,39 @@
"""Tests for the evidence-selected GTL composition matrix."""
from __future__ import annotations
from scripts import run_gtl_confirmation_matrix as matrix
def test_confirmation_matrix_is_pre_registered_and_well_formed():
assert len(matrix.GTL_CONFIRMATION_ARMS) == 13
assert matrix.GTL_CONFIRMATION_ARMS[0]["name"] == "control"
names = [arm["name"] for arm in matrix.GTL_CONFIRMATION_ARMS]
assert len(names) == len(set(names))
for arm in matrix.GTL_CONFIRMATION_ARMS:
config = matrix._config(arm)
assert config["mode"] in {"intersection", "union"}
if config["mode"] == "union":
assert len(config["confirmations"]) == 1
for confirmation in config["confirmations"]:
assert confirmation["name"] in {
"touch_0_25pct",
"strength_1000",
"merge_0_25pct",
"pivots_none",
}
def test_signature_uses_only_control_parity_fields():
arm = {
"candidates": 100,
"qualified": 10,
"qualified_net_avg_r": 0.2,
"qualified_net_avg_r_ex_top5": 0.05,
"full_book": {"sharpe": 2.0},
"holdout": {"train": {"sharpe": 1.0}},
"unrelated": "ignored",
}
assert "unrelated" not in matrix._signature(arm)
assert matrix._signature(arm)["full_book"] == {"sharpe": 2.0}