diff --git a/docs/research/sr-levels-and-exits.md b/docs/research/sr-levels-and-exits.md index 5dd3f01..3a7d48f 100644 --- a/docs/research/sr-levels-and-exits.md +++ b/docs/research/sr-levels-and-exits.md @@ -412,10 +412,22 @@ Detector evidence (`sources`, rejection count, last rejection age) stays in the pure backtest objects. It is deliberately not migrated into the production DB schema until a variant passes validation. -Run the training matrix with `--entry-end 2024-06-30 --sr-audit`, choose one arm, -and record that lock before running exactly control and the locked arm with -`--entry-start 2024-07-01`. Use `scripts/compare_sr_variants.py` to produce the -paired cohort CSV and summary JSON. +The cross-platform matrix runner is only an orchestrator around the existing +`run_backtest_snapshot.py`; it contains no duplicate backtest logic. On macOS: + +```bash +.venv/bin/python scripts/run_sr_v2_matrix.py train --workers 14 +``` + +Choose one arm and record that lock before running exactly control and that arm: + +```bash +.venv/bin/python scripts/run_sr_v2_matrix.py validate \ + --locked-arm confirmed_rounds --workers 14 +``` + +Replace `confirmed_rounds` with the recorded winner. The validation command also +calls `scripts/compare_sr_variants.py` to produce the paired cohort CSV and JSON. The post-2024 interval has informed earlier research, so this is validation rather than a pristine holdout; do not sweep variants on it. No deployment follows diff --git a/scripts/run_sr_v2_matrix.py b/scripts/run_sr_v2_matrix.py new file mode 100644 index 0000000..39a619e --- /dev/null +++ b/scripts/run_sr_v2_matrix.py @@ -0,0 +1,128 @@ +"""Run the S/R v2 training matrix or one locked validation comparison. + +This is a cross-platform orchestrator around ``run_backtest_snapshot.py``. It +contains no backtest logic; every arm still runs through the production-parity +Python harness. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py" +COMPARE = ROOT / "scripts" / "compare_sr_variants.py" +TRAINING_ARMS = ( + "production_control", + "rr_aligned_control", + "rewrite", + "soft_zones", + "confirmed_rounds", + "gate_v2", +) +LOCKABLE_ARMS = TRAINING_ARMS[1:] + + +def _add_common(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--snapshot", + default="backtest_snapshots/prod.sqlite", + help="Local SQLite snapshot path.", + ) + parser.add_argument("--workers", type=int, default=7) + + +def _args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + train = commands.add_parser("train", help="Run all arms before 2024-07-01.") + _add_common(train) + validate = commands.add_parser( + "validate", + help="Run production control and one locked arm from 2024-07-01.", + ) + _add_common(validate) + validate.add_argument("--locked-arm", required=True, choices=LOCKABLE_ARMS) + return parser.parse_args() + + +def _run_arm( + arm: str, + snapshot: str, + workers: int, + *, + entry_flag: str, + entry_date: str, + output: Path, +) -> None: + print(f"Running S/R arm: {arm}", flush=True) + subprocess.run( + [ + sys.executable, + str(RUNNER), + snapshot, + "--workers", str(workers), + "--allow-spawn", + "--sr-variant", arm, + entry_flag, entry_date, + "--sr-audit", + "--out", str(output), + ], + cwd=ROOT, + check=True, + ) + + +def _train(args: argparse.Namespace) -> None: + for arm in TRAINING_ARMS: + _run_arm( + arm, + args.snapshot, + args.workers, + entry_flag="--entry-end", + entry_date="2024-06-30", + output=ROOT / "reports" / f"backtest-sr-v2-train-{arm}.json", + ) + print("Training matrix complete. Lock one arm before running validation.") + + +def _validate(args: argparse.Namespace) -> None: + reports: dict[str, Path] = {} + for arm in ("production_control", args.locked_arm): + output = ROOT / "reports" / f"backtest-sr-v2-validation-{arm}.json" + reports[arm] = output + _run_arm( + arm, + args.snapshot, + args.workers, + entry_flag="--entry-start", + entry_date="2024-07-01", + output=output, + ) + subprocess.run( + [ + sys.executable, + str(COMPARE), + str(reports["production_control"]), + str(reports[args.locked_arm]), + "--out-csv", str(ROOT / "reports" / "sr-v2-validation-cohorts.csv"), + "--out-json", str(ROOT / "reports" / "sr-v2-validation-comparison.json"), + ], + cwd=ROOT, + check=True, + ) + + +def main() -> None: + args = _args() + if args.command == "train": + _train(args) + else: + _validate(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_sr_v2_training_matrix.ps1 b/scripts/run_sr_v2_training_matrix.ps1 deleted file mode 100644 index 066c169..0000000 --- a/scripts/run_sr_v2_training_matrix.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -param( - [string]$Snapshot = "backtest_snapshots\prod.sqlite", - [string]$Python = ".venv\Scripts\python.exe", - [int]$Workers = 7 -) - -$ErrorActionPreference = "Stop" -$arms = @( - "production_control", - "rr_aligned_control", - "rewrite", - "soft_zones", - "confirmed_rounds", - "gate_v2" -) - -foreach ($arm in $arms) { - $output = "reports\backtest-sr-v2-train-$arm.json" - Write-Host "Running S/R training arm: $arm" - & $Python scripts\run_backtest_snapshot.py $Snapshot ` - --workers $Workers ` - --allow-spawn ` - --sr-variant $arm ` - --entry-end 2024-06-30 ` - --sr-audit ` - --out $output - if ($LASTEXITCODE -ne 0) { - throw "S/R training arm failed: $arm" - } -} - -Write-Host "Training matrix complete. Lock one arm before running validation." diff --git a/scripts/run_sr_v2_validation.ps1 b/scripts/run_sr_v2_validation.ps1 deleted file mode 100644 index 0c49b74..0000000 --- a/scripts/run_sr_v2_validation.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -param( - [Parameter(Mandatory = $true)] - [ValidateSet("rr_aligned_control", "rewrite", "soft_zones", "confirmed_rounds", "gate_v2")] - [string]$LockedArm, - [string]$Snapshot = "backtest_snapshots\prod.sqlite", - [string]$Python = ".venv\Scripts\python.exe", - [int]$Workers = 7 -) - -$ErrorActionPreference = "Stop" -$arms = @("production_control", $LockedArm) -foreach ($arm in $arms) { - $output = "reports\backtest-sr-v2-validation-$arm.json" - Write-Host "Running locked S/R validation arm: $arm" - & $Python scripts\run_backtest_snapshot.py $Snapshot ` - --workers $Workers ` - --allow-spawn ` - --sr-variant $arm ` - --entry-start 2024-07-01 ` - --sr-audit ` - --out $output - if ($LASTEXITCODE -ne 0) { - throw "S/R validation arm failed: $arm" - } -} - -& $Python scripts\compare_sr_variants.py ` - reports\backtest-sr-v2-validation-production_control.json ` - "reports\backtest-sr-v2-validation-$LockedArm.json" ` - --out-csv reports\sr-v2-validation-cohorts.csv ` - --out-json reports\sr-v2-validation-comparison.json -if ($LASTEXITCODE -ne 0) { - throw "S/R validation comparison failed" -}