Make S/R matrix runner cross-platform

This commit is contained in:
2026-07-12 21:27:13 +02:00
parent 19b81c169d
commit 681f0f95da
4 changed files with 144 additions and 70 deletions
+16 -4
View File
@@ -412,10 +412,22 @@ Detector evidence (`sources`, rejection count, last rejection age) stays in the
pure backtest objects. It is deliberately not migrated into the production DB
schema until a variant passes validation.
Run the training matrix with `--entry-end 2024-06-30 --sr-audit`, choose one arm,
and record that lock before running exactly control and the locked arm with
`--entry-start 2024-07-01`. Use `scripts/compare_sr_variants.py` to produce the
paired cohort CSV and summary JSON.
The cross-platform matrix runner is only an orchestrator around the existing
`run_backtest_snapshot.py`; it contains no duplicate backtest logic. On macOS:
```bash
.venv/bin/python scripts/run_sr_v2_matrix.py train --workers 14
```
Choose one arm and record that lock before running exactly control and that arm:
```bash
.venv/bin/python scripts/run_sr_v2_matrix.py validate \
--locked-arm confirmed_rounds --workers 14
```
Replace `confirmed_rounds` with the recorded winner. The validation command also
calls `scripts/compare_sr_variants.py` to produce the paired cohort CSV and JSON.
The post-2024 interval has informed earlier research, so this is validation rather
than a pristine holdout; do not sweep variants on it. No deployment follows
+128
View File
@@ -0,0 +1,128 @@
"""Run the S/R v2 training matrix or one locked validation comparison.
This is a cross-platform orchestrator around ``run_backtest_snapshot.py``. It
contains no backtest logic; every arm still runs through the production-parity
Python harness.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py"
COMPARE = ROOT / "scripts" / "compare_sr_variants.py"
TRAINING_ARMS = (
"production_control",
"rr_aligned_control",
"rewrite",
"soft_zones",
"confirmed_rounds",
"gate_v2",
)
LOCKABLE_ARMS = TRAINING_ARMS[1:]
def _add_common(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--snapshot",
default="backtest_snapshots/prod.sqlite",
help="Local SQLite snapshot path.",
)
parser.add_argument("--workers", type=int, default=7)
def _args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
train = commands.add_parser("train", help="Run all arms before 2024-07-01.")
_add_common(train)
validate = commands.add_parser(
"validate",
help="Run production control and one locked arm from 2024-07-01.",
)
_add_common(validate)
validate.add_argument("--locked-arm", required=True, choices=LOCKABLE_ARMS)
return parser.parse_args()
def _run_arm(
arm: str,
snapshot: str,
workers: int,
*,
entry_flag: str,
entry_date: str,
output: Path,
) -> None:
print(f"Running S/R arm: {arm}", flush=True)
subprocess.run(
[
sys.executable,
str(RUNNER),
snapshot,
"--workers", str(workers),
"--allow-spawn",
"--sr-variant", arm,
entry_flag, entry_date,
"--sr-audit",
"--out", str(output),
],
cwd=ROOT,
check=True,
)
def _train(args: argparse.Namespace) -> None:
for arm in TRAINING_ARMS:
_run_arm(
arm,
args.snapshot,
args.workers,
entry_flag="--entry-end",
entry_date="2024-06-30",
output=ROOT / "reports" / f"backtest-sr-v2-train-{arm}.json",
)
print("Training matrix complete. Lock one arm before running validation.")
def _validate(args: argparse.Namespace) -> None:
reports: dict[str, Path] = {}
for arm in ("production_control", args.locked_arm):
output = ROOT / "reports" / f"backtest-sr-v2-validation-{arm}.json"
reports[arm] = output
_run_arm(
arm,
args.snapshot,
args.workers,
entry_flag="--entry-start",
entry_date="2024-07-01",
output=output,
)
subprocess.run(
[
sys.executable,
str(COMPARE),
str(reports["production_control"]),
str(reports[args.locked_arm]),
"--out-csv", str(ROOT / "reports" / "sr-v2-validation-cohorts.csv"),
"--out-json", str(ROOT / "reports" / "sr-v2-validation-comparison.json"),
],
cwd=ROOT,
check=True,
)
def main() -> None:
args = _args()
if args.command == "train":
_train(args)
else:
_validate(args)
if __name__ == "__main__":
main()
-32
View File
@@ -1,32 +0,0 @@
param(
[string]$Snapshot = "backtest_snapshots\prod.sqlite",
[string]$Python = ".venv\Scripts\python.exe",
[int]$Workers = 7
)
$ErrorActionPreference = "Stop"
$arms = @(
"production_control",
"rr_aligned_control",
"rewrite",
"soft_zones",
"confirmed_rounds",
"gate_v2"
)
foreach ($arm in $arms) {
$output = "reports\backtest-sr-v2-train-$arm.json"
Write-Host "Running S/R training arm: $arm"
& $Python scripts\run_backtest_snapshot.py $Snapshot `
--workers $Workers `
--allow-spawn `
--sr-variant $arm `
--entry-end 2024-06-30 `
--sr-audit `
--out $output
if ($LASTEXITCODE -ne 0) {
throw "S/R training arm failed: $arm"
}
}
Write-Host "Training matrix complete. Lock one arm before running validation."
-34
View File
@@ -1,34 +0,0 @@
param(
[Parameter(Mandatory = $true)]
[ValidateSet("rr_aligned_control", "rewrite", "soft_zones", "confirmed_rounds", "gate_v2")]
[string]$LockedArm,
[string]$Snapshot = "backtest_snapshots\prod.sqlite",
[string]$Python = ".venv\Scripts\python.exe",
[int]$Workers = 7
)
$ErrorActionPreference = "Stop"
$arms = @("production_control", $LockedArm)
foreach ($arm in $arms) {
$output = "reports\backtest-sr-v2-validation-$arm.json"
Write-Host "Running locked S/R validation arm: $arm"
& $Python scripts\run_backtest_snapshot.py $Snapshot `
--workers $Workers `
--allow-spawn `
--sr-variant $arm `
--entry-start 2024-07-01 `
--sr-audit `
--out $output
if ($LASTEXITCODE -ne 0) {
throw "S/R validation arm failed: $arm"
}
}
& $Python scripts\compare_sr_variants.py `
reports\backtest-sr-v2-validation-production_control.json `
"reports\backtest-sr-v2-validation-$LockedArm.json" `
--out-csv reports\sr-v2-validation-cohorts.csv `
--out-json reports\sr-v2-validation-comparison.json
if ($LASTEXITCODE -ne 0) {
throw "S/R validation comparison failed"
}