dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize GTL and retire S/R research harness

Browse Source
This commit is contained in:
dennisthiessen
2026-07-13 17:58:04 +02:00
parent 9d362bd568
commit bee5a5ce89
35 changed files with 374 additions and 5385779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/compare_sr_variants.py
-125
View File
@@ -1,125 +0,0 @@
"""Compare two audited local S/R backtest reports by setup identity.
Reports must be generated with ``--sr-audit``. The comparison is read-only
apart from its explicit CSV/JSON outputs under the caller-selected paths.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
from pathlib import Path
def _args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("control")
parser.add_argument("variant")
parser.add_argument("--out-csv", required=True)
parser.add_argument("--out-json", required=True)
return parser.parse_args()
def _load(path: str) -> dict:
with Path(path).open(encoding="utf-8") as handle:
report = json.load(handle)
if report.get("sr_candidate_audit") is None:
raise SystemExit(f"Report lacks sr_candidate_audit; rerun with --sr-audit: {path}")
return report
def _key(row: dict) -> tuple[str, str, str]:
return row["symbol"], row["date"], row["direction"]
def _cohort_stats(rows: list[dict]) -> dict:
net = [float(row.get("net_r", 0.0)) for row in rows]
hold = [float(row.get("hold30_r", 0.0)) for row in rows]
trimmed = sorted(net, reverse=True)[math.ceil(len(net) * 0.05):]
return {
"count": len(rows),
"net_avg_r": round(sum(net) / len(net), 4) if net else None,
"net_avg_r_ex_top5": round(sum(trimmed) / len(trimmed), 4) if trimmed else None,
"hold30_avg_r": round(sum(hold) / len(hold), 4) if hold else None,
}
def _production_book(report: dict) -> dict | None:
runs = ((report.get("portfolio_monitor") or {}).get("runs") or [])
row = next(
(
run for run in runs
if run.get("is_production") and run.get("lookback") == "all"
),
None,
)
if row is None:
return None
return {
key: row.get(key)
for key in ("sharpe", "cagr_pct", "max_drawdown_pct", "trades", "skipped_book_full")
}
def main() -> None:
args = _args()
control = _load(args.control)
variant = _load(args.variant)
control_rows = {_key(row): row for row in control["sr_candidate_audit"]}
variant_rows = {_key(row): row for row in variant["sr_candidate_audit"]}
control_q = {key for key, row in control_rows.items() if row.get("qualified")}
variant_q = {key for key, row in variant_rows.items() if row.get("qualified")}
retained = control_q & variant_q
added = variant_q - control_q
removed = control_q - variant_q
union = sorted(control_q | variant_q, key=lambda key: (key[1], key[0], key[2]))
csv_path = Path(args.out_csv)
csv_path.parent.mkdir(parents=True, exist_ok=True)
fields = [
"symbol", "date", "direction", "cohort",
"control_rr", "variant_rr", "control_prob", "variant_prob",
"control_sources", "variant_sources", "control_net_r", "variant_net_r",
"control_hold30_r", "variant_hold30_r",
]
with csv_path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
for key in union:
c = control_rows.get(key) or {}
v = variant_rows.get(key) or {}
cohort = "retained" if key in retained else "added" if key in added else "removed"
writer.writerow({
"symbol": key[0], "date": key[1], "direction": key[2], "cohort": cohort,
"control_rr": c.get("rr"), "variant_rr": v.get("rr"),
"control_prob": c.get("primary_prob"), "variant_prob": v.get("primary_prob"),
"control_sources": "+".join(c.get("primary_sources") or []),
"variant_sources": "+".join(v.get("primary_sources") or []),
"control_net_r": c.get("net_r"), "variant_net_r": v.get("net_r"),
"control_hold30_r": c.get("hold30_r"), "variant_hold30_r": v.get("hold30_r"),
})
summary = {
"control_report": str(Path(args.control)),
"variant_report": str(Path(args.variant)),
"control_variant": (control.get("params") or {}).get("sr_variant"),
"variant": (variant.get("params") or {}).get("sr_variant"),
"retained": _cohort_stats([variant_rows[key] for key in retained]),
"added": _cohort_stats([variant_rows[key] for key in added]),
"removed": _cohort_stats([control_rows[key] for key in removed]),
"control_book": _production_book(control),
"variant_book": _production_book(variant),
}
json_path = Path(args.out_json)
json_path.parent.mkdir(parents=True, exist_ok=True)
with json_path.open("w", encoding="utf-8") as handle:
json.dump(summary, handle, indent=2)
handle.write("\n")
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
main()
scripts/run_backtest_snapshot.py
+12 -44
View File
@@ -47,38 +47,13 @@ def _parse_args() -> argparse.Namespace:
)
parser.add_argument("--quiet", action="store_true", help="Hide progress output.")
parser.add_argument(
"--sr-variant",
choices=(
"production_control", "rr_aligned_control", "rewrite",
"soft_zones", "confirmed_rounds", "gate_v2",
"rewrite_legacy_primary", "soft_zones_legacy_primary",
"confirmed_rounds_legacy_primary", "gate_v2_legacy_primary",
"legacy_geometry_neutral", "legacy_pivots_only",
"legacy_traffic_grid_only", "legacy_range_grid_touch",
"legacy_range_grid_neutral",
"production_range504", "rewrite_range504_legacy_primary",
"rewrite_range504_structural_legacy_primary",
"rewrite_range504_structural_primary2",
"production_structural_overlay",
"explicit_target_ladder",
"--target-model",
choices=("production_gtl", "structural_sr"),
default="production_gtl",
help=(
"Target source: production_gtl matches the live scanner; "
"structural_sr is a comparison-only chart-S/R model."
),
default=None,
help="Research-only S/R detector/gate arm.",
)
parser.add_argument(
"--entry-start",
default=None,
help="Include entries on/after YYYY-MM-DD.",
)
parser.add_argument(
"--entry-end",
default=None,
help="Include entries on/before YYYY-MM-DD.",
)
parser.add_argument(
"--sr-audit",
action="store_true",
help="Include candidate-level S/R audit rows for paired comparison.",
)
parser.add_argument(
"--holdout-split",
@@ -171,10 +146,7 @@ def _print_summary(report: dict) -> None:
row
for row in ((report.get("portfolio_monitor") or {}).get("runs") or [])
if row.get("lookback") == "all"
and (
row.get("is_production")
or row.get("strategy") == "production_structural_overlay5_atr3"
)
and row.get("is_production")
]
if monitor_rows:
print(" live-path full-period comparison:")
@@ -198,14 +170,6 @@ async def _main() -> None:
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
if args.allow_spawn:
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
if args.sr_variant:
os.environ["BACKTEST_SR_VARIANT"] = args.sr_variant
if args.entry_start:
os.environ["BACKTEST_ENTRY_START"] = args.entry_start
if args.entry_end:
os.environ["BACKTEST_ENTRY_END"] = args.entry_end
if args.sr_audit:
os.environ["BACKTEST_SR_AUDIT"] = "1"
if args.holdout_split:
try:
date.fromisoformat(args.holdout_split)
@@ -240,7 +204,11 @@ async def _main() -> None:
try:
async with Session() as db:
report = await run_backtest(db, progress_cb=progress)
report = await run_backtest(
db,
progress_cb=progress,
target_model=args.target_model,
)
finally:
await engine.dispose()
scripts/run_sr_v2_matrix.py
-287
View File
@@ -1,287 +0,0 @@
"""Run S/R detector, hidden-feature, or locked validation comparisons.
This is a cross-platform orchestrator around ``run_backtest_snapshot.py``. It
contains no backtest logic; every arm still runs through the production-parity
Python harness.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RUNNER = ROOT / "scripts" / "run_backtest_snapshot.py"
COMPARE = ROOT / "scripts" / "compare_sr_variants.py"
TRAINING_ARMS = (
"production_control",
"rewrite_legacy_primary",
"soft_zones_legacy_primary",
"confirmed_rounds_legacy_primary",
"gate_v2_legacy_primary",
)
RANGE_GRID_ARMS = (
"legacy_traffic_grid_only",
"legacy_range_grid_touch",
"legacy_range_grid_neutral",
)
LOCKABLE_ARMS = TRAINING_ARMS[1:] + RANGE_GRID_ARMS
TRAFFIC_ARMS = (
"production_control",
"legacy_geometry_neutral",
"legacy_pivots_only",
*RANGE_GRID_ARMS,
)
RANGE_FACTOR_ARMS = (
"production_range504",
"rewrite_range504_legacy_primary",
)
RANGE_RESIDUAL_ARMS = (
"rewrite_range504_structural_legacy_primary",
"rewrite_range504_structural_primary2",
)
FULL_PERIOD_ARMS = (
"production_control",
"rewrite_range504_structural_legacy_primary",
)
def _add_common(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--snapshot",
default="backtest_snapshots/prod.sqlite",
help="Local SQLite snapshot path.",
)
parser.add_argument("--workers", type=int, default=7)
def _args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
train = commands.add_parser("train", help="Run all arms before 2024-07-01.")
_add_common(train)
traffic = commands.add_parser(
"traffic",
help="Isolate pivots, range-grid geometry, and touch strength on training data.",
)
_add_common(traffic)
traffic.add_argument(
"--only-arm",
choices=TRAFFIC_ARMS,
default=None,
help="Rerun one hidden-feature arm without repeating the full matrix.",
)
factor = commands.add_parser(
"factor",
help="Test the explicit 504-day range factor with old and clean detectors.",
)
_add_common(factor)
factor.add_argument(
"--only-arm",
choices=RANGE_FACTOR_ARMS,
default=None,
help="Run one range-factor arm without repeating the pair.",
)
residual = commands.add_parser(
"residual",
help="Isolate standalone rounds and the 1.5-2R primary-target veto.",
)
_add_common(residual)
residual.add_argument(
"--only-arm",
choices=RANGE_RESIDUAL_ARMS,
default=None,
help="Run one residual target-geometry arm without repeating the pair.",
)
full = commands.add_parser(
"full",
help="Run current production and the frozen candidate over the full snapshot.",
)
_add_common(full)
overlay = commands.add_parser(
"overlay",
help="Test the frozen 5% clean-structure rank overlay on production breadth.",
)
_add_common(overlay)
ladder = commands.add_parser(
"ladder",
help="Verify the explicit volume-free gate target ladder against production.",
)
_add_common(ladder)
validate = commands.add_parser(
"validate",
help="Run production control and one locked arm from 2024-07-01.",
)
_add_common(validate)
validate.add_argument("--locked-arm", required=True, choices=LOCKABLE_ARMS)
return parser.parse_args()
def _run_arm(
arm: str,
snapshot: str,
workers: int,
*,
entry_flag: str | None = None,
entry_date: str | None = None,
output: Path,
) -> None:
print(f"Running S/R arm: {arm}", flush=True)
command = [
sys.executable,
str(RUNNER),
snapshot,
"--workers", str(workers),
"--allow-spawn",
"--sr-variant", arm,
"--sr-audit",
"--out", str(output),
]
if entry_flag is not None and entry_date is not None:
command.extend((entry_flag, entry_date))
elif entry_flag is not None or entry_date is not None:
raise ValueError("entry_flag and entry_date must be provided together")
subprocess.run(
command,
cwd=ROOT,
check=True,
)
def _train(
args: argparse.Namespace,
arms: tuple[str, ...],
filename_prefix: str,
) -> None:
for arm in arms:
_run_arm(
arm,
args.snapshot,
args.workers,
entry_flag="--entry-end",
entry_date="2024-06-30",
output=ROOT / "reports" / f"{filename_prefix}-{arm}.json",
)
print("Training matrix complete. Review results before running validation.")
def _validate(args: argparse.Namespace) -> None:
reports: dict[str, Path] = {}
for arm in ("production_control", args.locked_arm):
output = ROOT / "reports" / f"backtest-sr-v2-validation-{arm}.json"
reports[arm] = output
_run_arm(
arm,
args.snapshot,
args.workers,
entry_flag="--entry-start",
entry_date="2024-07-01",
output=output,
)
subprocess.run(
[
sys.executable,
str(COMPARE),
str(reports["production_control"]),
str(reports[args.locked_arm]),
"--out-csv", str(ROOT / "reports" / "sr-v2-validation-cohorts.csv"),
"--out-json", str(ROOT / "reports" / "sr-v2-validation-comparison.json"),
],
cwd=ROOT,
check=True,
)
def _full(args: argparse.Namespace) -> None:
reports: dict[str, Path] = {}
for arm in FULL_PERIOD_ARMS:
output = ROOT / "reports" / f"backtest-sr-full-{arm}.json"
reports[arm] = output
_run_arm(
arm,
args.snapshot,
args.workers,
output=output,
)
subprocess.run(
[
sys.executable,
str(COMPARE),
str(reports["production_control"]),
str(reports["rewrite_range504_structural_legacy_primary"]),
"--out-csv", str(ROOT / "reports" / "sr-full-production-vs-candidate-cohorts.csv"),
"--out-json", str(ROOT / "reports" / "sr-full-production-vs-candidate-comparison.json"),
],
cwd=ROOT,
check=True,
)
print("Full-period production comparison complete.")
def _overlay(args: argparse.Namespace) -> None:
_run_arm(
"production_structural_overlay",
args.snapshot,
args.workers,
output=ROOT / "reports" / "backtest-sr-overlay-full.json",
)
print("Full-period structural ranking overlay complete.")
def _ladder(args: argparse.Namespace) -> None:
control = ROOT / "reports" / "backtest-sr-full-production_control.json"
if not control.exists():
raise SystemExit(f"Full-period production control not found: {control}")
variant = ROOT / "reports" / "backtest-sr-full-explicit_target_ladder.json"
_run_arm(
"explicit_target_ladder",
args.snapshot,
args.workers,
output=variant,
)
subprocess.run(
[
sys.executable,
str(COMPARE),
str(control),
str(variant),
"--out-csv", str(
ROOT / "reports" / "sr-explicit-target-ladder-cohorts.csv"
),
"--out-json", str(
ROOT / "reports" / "sr-explicit-target-ladder-comparison.json"
),
],
cwd=ROOT,
check=True,
)
print("Explicit target-ladder production parity comparison complete.")
def main() -> None:
args = _args()
if args.command == "train":
_train(args, TRAINING_ARMS, "backtest-sr-v2-train")
elif args.command == "traffic":
arms = (args.only_arm,) if args.only_arm else TRAFFIC_ARMS
_train(args, arms, "backtest-sr-traffic-train")
elif args.command == "factor":
arms = (args.only_arm,) if args.only_arm else RANGE_FACTOR_ARMS
_train(args, arms, "backtest-sr-range-factor-train")
elif args.command == "residual":
arms = (args.only_arm,) if args.only_arm else RANGE_RESIDUAL_ARMS
_train(args, arms, "backtest-sr-range-residual-train")
elif args.command == "full":
_full(args)
elif args.command == "overlay":
_overlay(args)
elif args.command == "ladder":
_ladder(args)
else:
_validate(args)
if __name__ == "__main__":
main()