feat: add selectable daily backtest cadence
This commit is contained in:
@@ -387,6 +387,7 @@ async def trigger_job(
|
|||||||
db,
|
db,
|
||||||
job_name,
|
job_name,
|
||||||
target_model=body.target_model if body is not None else None,
|
target_model=body.target_model if body is not None else None,
|
||||||
|
cadence=body.cadence if body is not None else None,
|
||||||
)
|
)
|
||||||
return APIEnvelope(status="success", data=result)
|
return APIEnvelope(status="success", data=result)
|
||||||
|
|
||||||
|
|||||||
+37
-10
@@ -37,8 +37,10 @@ from app.services import fundamental_service, ingestion_service, sentiment_servi
|
|||||||
from app.services.alert_service import dispatch_alerts
|
from app.services.alert_service import dispatch_alerts
|
||||||
from app.services.backtest_service import (
|
from app.services.backtest_service import (
|
||||||
BACKTEST_TARGET_MODELS,
|
BACKTEST_TARGET_MODELS,
|
||||||
|
DEFAULT_BACKTEST_CADENCE,
|
||||||
PRODUCTION_GTL_TARGET_MODEL,
|
PRODUCTION_GTL_TARGET_MODEL,
|
||||||
run_and_store as run_backtest_and_store,
|
run_and_store as run_backtest_and_store,
|
||||||
|
validate_backtest_cadence,
|
||||||
validate_backtest_target_model,
|
validate_backtest_target_model,
|
||||||
)
|
)
|
||||||
from app.services.benchmark_service import refresh_benchmark_prices
|
from app.services.benchmark_service import refresh_benchmark_prices
|
||||||
@@ -112,6 +114,7 @@ def _idle_runtime() -> dict[str, object]:
|
|||||||
|
|
||||||
_job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES}
|
_job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES}
|
||||||
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
||||||
|
_next_backtest_cadence = DEFAULT_BACKTEST_CADENCE
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -119,23 +122,44 @@ _next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def queue_backtest_target_model(target_model: str | None) -> str:
|
def queue_backtest_options(
|
||||||
"""Select the model for the next manual backtest run only.
|
target_model: str | None,
|
||||||
|
cadence: str | None,
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""Select model and cadence for the next manual backtest run only.
|
||||||
|
|
||||||
Scheduled runs and subsequent manual runs return to the production GTL.
|
Scheduled and subsequent manual runs return to production GTL at the
|
||||||
|
resource-safe weekly cadence.
|
||||||
"""
|
"""
|
||||||
global _next_backtest_target_model
|
global _next_backtest_target_model, _next_backtest_cadence
|
||||||
selected = validate_backtest_target_model(
|
selected_model = validate_backtest_target_model(
|
||||||
target_model or PRODUCTION_GTL_TARGET_MODEL
|
target_model or PRODUCTION_GTL_TARGET_MODEL
|
||||||
)
|
)
|
||||||
_next_backtest_target_model = selected
|
selected_cadence = validate_backtest_cadence(
|
||||||
|
cadence or DEFAULT_BACKTEST_CADENCE
|
||||||
|
)
|
||||||
|
_next_backtest_target_model = selected_model
|
||||||
|
_next_backtest_cadence = selected_cadence
|
||||||
|
return selected_model, selected_cadence
|
||||||
|
|
||||||
|
|
||||||
|
def queue_backtest_target_model(target_model: str | None) -> str:
|
||||||
|
"""Compatibility wrapper for callers selecting only the target model."""
|
||||||
|
selected, _ = queue_backtest_options(target_model, DEFAULT_BACKTEST_CADENCE)
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def _consume_backtest_options() -> tuple[str, str]:
|
||||||
|
global _next_backtest_target_model, _next_backtest_cadence
|
||||||
|
selected = (_next_backtest_target_model, _next_backtest_cadence)
|
||||||
|
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
||||||
|
_next_backtest_cadence = DEFAULT_BACKTEST_CADENCE
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
|
|
||||||
def _consume_backtest_target_model() -> str:
|
def _consume_backtest_target_model() -> str:
|
||||||
global _next_backtest_target_model
|
"""Compatibility wrapper consuming all queued one-run options."""
|
||||||
selected = _next_backtest_target_model
|
selected, _ = _consume_backtest_options()
|
||||||
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
|
|
||||||
@@ -1028,12 +1052,13 @@ async def compute_regime_monitor() -> None:
|
|||||||
async def run_backtest_job() -> None:
|
async def run_backtest_job() -> None:
|
||||||
"""Replay the price-derived engine over history and cache the report."""
|
"""Replay the price-derived engine over history and cache the report."""
|
||||||
job_name = "backtest"
|
job_name = "backtest"
|
||||||
target_model = _consume_backtest_target_model()
|
target_model, cadence = _consume_backtest_options()
|
||||||
_log_event(
|
_log_event(
|
||||||
logging.INFO,
|
logging.INFO,
|
||||||
"job_start",
|
"job_start",
|
||||||
job=job_name,
|
job=job_name,
|
||||||
target_model=target_model,
|
target_model=target_model,
|
||||||
|
cadence=cadence,
|
||||||
)
|
)
|
||||||
_runtime_start(job_name)
|
_runtime_start(job_name)
|
||||||
|
|
||||||
@@ -1051,6 +1076,7 @@ async def run_backtest_job() -> None:
|
|||||||
db,
|
db,
|
||||||
_on_progress,
|
_on_progress,
|
||||||
target_model=target_model,
|
target_model=target_model,
|
||||||
|
cadence=cadence,
|
||||||
)
|
)
|
||||||
|
|
||||||
_runtime_finish(
|
_runtime_finish(
|
||||||
@@ -1058,6 +1084,7 @@ async def run_backtest_job() -> None:
|
|||||||
processed=report.get("tickers", 0), total=report.get("tickers", 0),
|
processed=report.get("tickers", 0), total=report.get("tickers", 0),
|
||||||
message=(
|
message=(
|
||||||
f"{BACKTEST_TARGET_MODELS[target_model]}: "
|
f"{BACKTEST_TARGET_MODELS[target_model]}: "
|
||||||
|
f"{cadence} cadence, "
|
||||||
f"{report.get('candidates', 0)} setups, "
|
f"{report.get('candidates', 0)} setups, "
|
||||||
f"{report.get('qualified', 0)} qualified"
|
f"{report.get('qualified', 0)} qualified"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ class JobToggle(BaseModel):
|
|||||||
class JobTriggerRequest(BaseModel):
|
class JobTriggerRequest(BaseModel):
|
||||||
"""Optional parameters for a one-time manual job run."""
|
"""Optional parameters for a one-time manual job run."""
|
||||||
target_model: Literal["production_gtl", "structural_sr"] | None = None
|
target_model: Literal["production_gtl", "structural_sr"] | None = None
|
||||||
|
cadence: Literal["weekly", "daily"] | None = None
|
||||||
|
|
||||||
|
|
||||||
class RecommendationConfigUpdate(BaseModel):
|
class RecommendationConfigUpdate(BaseModel):
|
||||||
|
|||||||
@@ -607,6 +607,7 @@ async def trigger_job(
|
|||||||
job_name: str,
|
job_name: str,
|
||||||
*,
|
*,
|
||||||
target_model: str | None = None,
|
target_model: str | None = None,
|
||||||
|
cadence: str | None = None,
|
||||||
) -> dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Trigger a manual job run via the scheduler.
|
"""Trigger a manual job run via the scheduler.
|
||||||
|
|
||||||
@@ -616,6 +617,8 @@ async def trigger_job(
|
|||||||
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
||||||
if target_model is not None and job_name != "backtest":
|
if target_model is not None and job_name != "backtest":
|
||||||
raise ValidationError("target_model is supported only for the backtest job")
|
raise ValidationError("target_model is supported only for the backtest job")
|
||||||
|
if cadence is not None and job_name != "backtest":
|
||||||
|
raise ValidationError("cadence is supported only for the backtest job")
|
||||||
|
|
||||||
from app.scheduler import get_job_runtime_snapshot, scheduler
|
from app.scheduler import get_job_runtime_snapshot, scheduler
|
||||||
|
|
||||||
@@ -643,9 +646,9 @@ async def trigger_job(
|
|||||||
return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"}
|
return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"}
|
||||||
|
|
||||||
if job_name == "backtest":
|
if job_name == "backtest":
|
||||||
from app.scheduler import queue_backtest_target_model
|
from app.scheduler import queue_backtest_options
|
||||||
|
|
||||||
target_model = queue_backtest_target_model(target_model)
|
target_model, cadence = queue_backtest_options(target_model, cadence)
|
||||||
|
|
||||||
job.modify(next_run_time=None) # Reset, then trigger immediately
|
job.modify(next_run_time=None) # Reset, then trigger immediately
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
@@ -654,6 +657,8 @@ async def trigger_job(
|
|||||||
result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
|
result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
|
||||||
if target_model is not None:
|
if target_model is not None:
|
||||||
result["target_model"] = target_model
|
result["target_model"] = target_model
|
||||||
|
if cadence is not None:
|
||||||
|
result["cadence"] = cadence
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Historical backtest (Phase 1): replay the price-derived engine over stored
|
"""Historical backtest (Phase 1): replay the price-derived engine over stored
|
||||||
OHLCV and measure how the CURRENT config would have performed.
|
OHLCV and measure how the CURRENT config would have performed.
|
||||||
|
|
||||||
For each ticker we step through history (weekly), and at each as-of date D we
|
For each ticker we step through history at the selected entry cadence, and at each as-of date D we
|
||||||
rebuild the setup using only bars ≤ D (no lookahead), then walk the actual bars
|
rebuild the setup using only bars ≤ D (no lookahead), then walk the actual bars
|
||||||
after D to record the realized outcome. The report contains:
|
after D to record the realized outcome. The report contains:
|
||||||
|
|
||||||
@@ -100,7 +100,16 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
KEY_REPORT = "backtest_report"
|
KEY_REPORT = "backtest_report"
|
||||||
|
|
||||||
STEP_DAYS = 5 # weekly cadence (≈ 5 trading days)
|
WEEKLY_BACKTEST_CADENCE = "weekly"
|
||||||
|
DAILY_BACKTEST_CADENCE = "daily"
|
||||||
|
DEFAULT_BACKTEST_CADENCE = WEEKLY_BACKTEST_CADENCE
|
||||||
|
BACKTEST_CADENCE_SESSIONS = {
|
||||||
|
WEEKLY_BACKTEST_CADENCE: 5,
|
||||||
|
DAILY_BACKTEST_CADENCE: 1,
|
||||||
|
}
|
||||||
|
# Compatibility alias for research scripts built around the original weekly
|
||||||
|
# replay. New code should select a cadence and call ``backtest_step_sessions``.
|
||||||
|
STEP_DAYS = BACKTEST_CADENCE_SESSIONS[WEEKLY_BACKTEST_CADENCE]
|
||||||
MIN_LOOKBACK = 60 # bars needed before D for indicators (EMA cross needs 51)
|
MIN_LOOKBACK = 60 # bars needed before D for indicators (EMA cross needs 51)
|
||||||
HORIZON = 30 # trading days to resolve an outcome (matches the evaluator)
|
HORIZON = 30 # trading days to resolve an outcome (matches the evaluator)
|
||||||
ATR_MULTIPLIER = 1.5
|
ATR_MULTIPLIER = 1.5
|
||||||
@@ -157,6 +166,30 @@ def validate_backtest_target_model(value: str) -> str:
|
|||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def validate_backtest_cadence(value: str) -> str:
|
||||||
|
"""Validate the supported entry-replay cadences."""
|
||||||
|
normalized = value.strip().lower()
|
||||||
|
if normalized not in BACKTEST_CADENCE_SESSIONS:
|
||||||
|
allowed = ", ".join(BACKTEST_CADENCE_SESSIONS)
|
||||||
|
raise ValueError(
|
||||||
|
f"Unknown backtest cadence {value!r}; expected one of {allowed}"
|
||||||
|
)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def backtest_step_sessions(cadence: str) -> int:
|
||||||
|
return BACKTEST_CADENCE_SESSIONS[validate_backtest_cadence(cadence)]
|
||||||
|
|
||||||
|
|
||||||
|
def _ranking_period(as_of: date, cadence: str) -> tuple:
|
||||||
|
"""Cross-section key for activation ranks at the selected entry cadence."""
|
||||||
|
cadence = validate_backtest_cadence(cadence)
|
||||||
|
if cadence == DAILY_BACKTEST_CADENCE:
|
||||||
|
return ("date", as_of.toordinal())
|
||||||
|
iso = as_of.isocalendar()
|
||||||
|
return ("week", iso[0], iso[1])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# RESEARCH / DIAGNOSTIC FALLBACKS (retired experiments)
|
# RESEARCH / DIAGNOSTIC FALLBACKS (retired experiments)
|
||||||
#
|
#
|
||||||
@@ -456,14 +489,17 @@ def _replay_ticker(
|
|||||||
activation: dict,
|
activation: dict,
|
||||||
benchmark_closes: dict[date, float] | None = None,
|
benchmark_closes: dict[date, float] | None = None,
|
||||||
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
||||||
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Walk one ticker's history weekly, building setups and their realized outcomes."""
|
"""Walk one ticker at the selected cadence and resolve each setup outcome."""
|
||||||
|
cadence = validate_backtest_cadence(cadence)
|
||||||
|
step_sessions = backtest_step_sessions(cadence)
|
||||||
candidates: list[dict] = []
|
candidates: list[dict] = []
|
||||||
n = len(records)
|
n = len(records)
|
||||||
if n < MIN_LOOKBACK + HORIZON:
|
if n < MIN_LOOKBACK + HORIZON:
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
for i in range(MIN_LOOKBACK - 1, n - HORIZON, STEP_DAYS):
|
for i in range(MIN_LOOKBACK - 1, n - HORIZON, step_sessions):
|
||||||
window = records[: i + 1]
|
window = records[: i + 1]
|
||||||
forward = records[i + 1 :]
|
forward = records[i + 1 :]
|
||||||
forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
|
forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
|
||||||
@@ -512,6 +548,7 @@ def _replay_ticker(
|
|||||||
"symbol": symbol,
|
"symbol": symbol,
|
||||||
"date": records[i].date.isoformat(),
|
"date": records[i].date.isoformat(),
|
||||||
"iso_week": (iso[0], iso[1]),
|
"iso_week": (iso[0], iso[1]),
|
||||||
|
"ranking_period": _ranking_period(records[i].date, cadence),
|
||||||
"direction": s["direction"],
|
"direction": s["direction"],
|
||||||
"entry": s["entry"],
|
"entry": s["entry"],
|
||||||
"stop": s["stop"],
|
"stop": s["stop"],
|
||||||
@@ -968,6 +1005,7 @@ def _replay_and_signals(
|
|||||||
activation: dict,
|
activation: dict,
|
||||||
benchmark_closes: dict[date, float] | None = None,
|
benchmark_closes: dict[date, float] | None = None,
|
||||||
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
||||||
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
) -> tuple[list[dict], dict]:
|
) -> tuple[list[dict], dict]:
|
||||||
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can
|
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can
|
||||||
run in a worker process. Takes primitive column arrays (cheap to pickle),
|
run in a worker process. Takes primitive column arrays (cheap to pickle),
|
||||||
@@ -987,6 +1025,7 @@ def _replay_and_signals(
|
|||||||
activation,
|
activation,
|
||||||
benchmark_closes,
|
benchmark_closes,
|
||||||
target_model,
|
target_model,
|
||||||
|
cadence,
|
||||||
),
|
),
|
||||||
_signal_series(bars, benchmark_closes),
|
_signal_series(bars, benchmark_closes),
|
||||||
)
|
)
|
||||||
@@ -999,6 +1038,7 @@ def _replay_candidates_for_period(
|
|||||||
activation: dict,
|
activation: dict,
|
||||||
benchmark_closes: dict[date, float] | None,
|
benchmark_closes: dict[date, float] | None,
|
||||||
start_date: date,
|
start_date: date,
|
||||||
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Slim picklable replay used by local event studies.
|
"""Slim picklable replay used by local event studies.
|
||||||
|
|
||||||
@@ -1014,8 +1054,13 @@ def _replay_candidates_for_period(
|
|||||||
date_ords, opens, highs, lows, closes, volumes
|
date_ords, opens, highs, lows, closes, volumes
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
cadence = validate_backtest_cadence(cadence)
|
||||||
candidates: list[dict] = []
|
candidates: list[dict] = []
|
||||||
for i in range(MIN_LOOKBACK - 1, len(bars) - HORIZON, STEP_DAYS):
|
for i in range(
|
||||||
|
MIN_LOOKBACK - 1,
|
||||||
|
len(bars) - HORIZON,
|
||||||
|
backtest_step_sessions(cadence),
|
||||||
|
):
|
||||||
if bars[i].date < start_date:
|
if bars[i].date < start_date:
|
||||||
continue
|
continue
|
||||||
window = bars[: i + 1]
|
window = bars[: i + 1]
|
||||||
@@ -1036,6 +1081,7 @@ def _replay_candidates_for_period(
|
|||||||
"symbol": symbol,
|
"symbol": symbol,
|
||||||
"date": bars[i].date.isoformat(),
|
"date": bars[i].date.isoformat(),
|
||||||
"iso_week": (iso[0], iso[1]),
|
"iso_week": (iso[0], iso[1]),
|
||||||
|
"ranking_period": _ranking_period(bars[i].date, cadence),
|
||||||
"direction": "long",
|
"direction": "long",
|
||||||
"entry": setup["entry"],
|
"entry": setup["entry"],
|
||||||
"stop": setup["stop"],
|
"stop": setup["stop"],
|
||||||
@@ -1120,14 +1166,17 @@ def _assign_signal_percentiles(
|
|||||||
value_key: str,
|
value_key: str,
|
||||||
percentile_key: str,
|
percentile_key: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Per ISO week, rank candidates by ``value_key`` and attach a 0-100
|
"""Per replay period, rank candidates by ``value_key`` and attach a 0-100
|
||||||
percentile under ``percentile_key`` (100 = strongest). Missing values get
|
percentile under ``percentile_key`` (100 = strongest). Missing values get
|
||||||
None and therefore cannot clear a gate based on that signal."""
|
None and therefore cannot clear a gate based on that signal."""
|
||||||
by_week: dict = defaultdict(list)
|
by_period: dict = defaultdict(list)
|
||||||
for c in candidates:
|
for c in candidates:
|
||||||
if c.get(value_key) is not None:
|
if c.get(value_key) is not None:
|
||||||
by_week[c["iso_week"]].append(c)
|
# Hand-built/research candidates predating the cadence flag retain
|
||||||
for group in by_week.values():
|
# the weekly key as a compatibility fallback.
|
||||||
|
period = c.get("ranking_period") or c["iso_week"]
|
||||||
|
by_period[period].append(c)
|
||||||
|
for group in by_period.values():
|
||||||
ordered = sorted(group, key=lambda c: c[value_key])
|
ordered = sorted(group, key=lambda c: c[value_key])
|
||||||
n = len(ordered)
|
n = len(ordered)
|
||||||
for rank, c in enumerate(ordered):
|
for rank, c in enumerate(ordered):
|
||||||
@@ -1137,9 +1186,9 @@ def _assign_signal_percentiles(
|
|||||||
|
|
||||||
|
|
||||||
def _assign_momentum_percentiles(candidates: list[dict]) -> None:
|
def _assign_momentum_percentiles(candidates: list[dict]) -> None:
|
||||||
"""Per ISO week, rank candidates by their ticker's 12-1 momentum and attach a
|
"""Per replay period, rank candidates by 12-1 momentum and attach a
|
||||||
0-100 ``momentum_percentile`` (100 = highest momentum in the universe that
|
0-100 ``momentum_percentile`` (100 = highest momentum in the universe that
|
||||||
week). Candidates whose momentum is unknown (insufficient lookback) get None
|
period). Candidates whose momentum is unknown (insufficient lookback) get None
|
||||||
and therefore can't clear a momentum gate. Mutates ``candidates``."""
|
and therefore can't clear a momentum gate. Mutates ``candidates``."""
|
||||||
_assign_signal_percentiles(candidates, "momentum", "momentum_percentile")
|
_assign_signal_percentiles(candidates, "momentum", "momentum_percentile")
|
||||||
|
|
||||||
@@ -1152,7 +1201,7 @@ def _assign_residual_momentum_percentiles(candidates: list[dict]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _assign_low_volatility_percentiles(candidates: list[dict]) -> None:
|
def _assign_low_volatility_percentiles(candidates: list[dict]) -> None:
|
||||||
"""Per ISO week, attach volatility ranks where 100 = lowest 6-month vol."""
|
"""Per replay period, attach volatility ranks where 100 = lowest 6-month vol."""
|
||||||
_assign_signal_percentiles(candidates, "vol_6m", VOL_PERCENTILE_KEY)
|
_assign_signal_percentiles(candidates, "vol_6m", VOL_PERCENTILE_KEY)
|
||||||
for c in candidates:
|
for c in candidates:
|
||||||
raw = c.get(VOL_PERCENTILE_KEY)
|
raw = c.get(VOL_PERCENTILE_KEY)
|
||||||
@@ -2227,6 +2276,19 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
|
|||||||
"entry_variant": "residual80_highvol_blend80_20_fixed10",
|
"entry_variant": "residual80_highvol_blend80_20_fixed10",
|
||||||
"exit_policy": "hold",
|
"exit_policy": "hold",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"strategy": "production_live_no_lockdown",
|
||||||
|
"label": "Live setup + 3x ATR trail (no re-entry lockdown)",
|
||||||
|
"description": (
|
||||||
|
"Exact live activation, ordering, and Admin exit policy, with only "
|
||||||
|
"the post-stop re-entry lockdown disabled as the comparison baseline."
|
||||||
|
),
|
||||||
|
"entry_variant": "residual80_highvol_blend80_20_fixed10",
|
||||||
|
"exit_policy": "atr_trail3",
|
||||||
|
"reentry_lockdown_sessions": 0,
|
||||||
|
"use_live_config": True,
|
||||||
|
"comparison_arm": "live_no_lockdown",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"strategy": PRODUCTION_PORTFOLIO_STRATEGY,
|
"strategy": PRODUCTION_PORTFOLIO_STRATEGY,
|
||||||
"label": "Production: residual/high-vol 80/20 + 3x ATR trail + 5-session lockdown",
|
"label": "Production: residual/high-vol 80/20 + 3x ATR trail + 5-session lockdown",
|
||||||
@@ -2243,6 +2305,7 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
|
|||||||
# live Admin exit policy, instead of the frozen research-variant gate.
|
# live Admin exit policy, instead of the frozen research-variant gate.
|
||||||
"use_live_config": True,
|
"use_live_config": True,
|
||||||
"is_production": True,
|
"is_production": True,
|
||||||
|
"comparison_arm": "live_lockdown_5",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2603,6 +2666,7 @@ def _portfolio_monitor(
|
|||||||
"label": strategy["label"],
|
"label": strategy["label"],
|
||||||
"description": strategy["description"],
|
"description": strategy["description"],
|
||||||
"is_production": bool(strategy.get("is_production")),
|
"is_production": bool(strategy.get("is_production")),
|
||||||
|
"comparison_arm": strategy.get("comparison_arm"),
|
||||||
"entry_variant": strategy["entry_variant"],
|
"entry_variant": strategy["entry_variant"],
|
||||||
"ranking_key": ranking_key,
|
"ranking_key": ranking_key,
|
||||||
"exit_policy": exit_policy,
|
"exit_policy": exit_policy,
|
||||||
@@ -2620,6 +2684,7 @@ def _portfolio_monitor(
|
|||||||
"label": s["label"],
|
"label": s["label"],
|
||||||
"description": s["description"],
|
"description": s["description"],
|
||||||
"is_production": bool(s.get("is_production")),
|
"is_production": bool(s.get("is_production")),
|
||||||
|
"comparison_arm": s.get("comparison_arm"),
|
||||||
"reentry_lockdown_sessions": int(
|
"reentry_lockdown_sessions": int(
|
||||||
s.get("reentry_lockdown_sessions", 0)
|
s.get("reentry_lockdown_sessions", 0)
|
||||||
),
|
),
|
||||||
@@ -2641,6 +2706,46 @@ def _portfolio_monitor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _production_cadence_comparison(
|
||||||
|
monitor: dict | None,
|
||||||
|
cadence: str,
|
||||||
|
) -> dict | None:
|
||||||
|
"""Compact full-history live/no-lockdown vs live/5-session comparison."""
|
||||||
|
if not monitor:
|
||||||
|
return None
|
||||||
|
arms: list[dict] = []
|
||||||
|
for row in monitor.get("runs") or []:
|
||||||
|
comparison_arm = row.get("comparison_arm")
|
||||||
|
if not comparison_arm or row.get("lookback") != "all":
|
||||||
|
continue
|
||||||
|
compact = {
|
||||||
|
key: value
|
||||||
|
for key, value in row.items()
|
||||||
|
if key not in {"equity_curve", "benchmark_curve"}
|
||||||
|
}
|
||||||
|
arm_name = (
|
||||||
|
"prod_live_setup"
|
||||||
|
if comparison_arm == "live_no_lockdown"
|
||||||
|
else "cooldown_5"
|
||||||
|
)
|
||||||
|
compact["arm"] = f"{arm_name}_{cadence}"
|
||||||
|
compact["entry_cadence"] = cadence
|
||||||
|
arms.append(compact)
|
||||||
|
if not arms:
|
||||||
|
return None
|
||||||
|
arms.sort(key=lambda row: int(row.get("reentry_lockdown_sessions", 0)))
|
||||||
|
return {
|
||||||
|
"entry_cadence": cadence,
|
||||||
|
"lookback": "all",
|
||||||
|
"arms": arms,
|
||||||
|
"note": (
|
||||||
|
"Both arms use the exact same live gate, ordering, Admin exit policy, "
|
||||||
|
"fees, and candidate cadence. Only the five-session post-stop "
|
||||||
|
"re-entry lockdown changes."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _pct_loss(base: float | None, candidate: float | None) -> float | None:
|
def _pct_loss(base: float | None, candidate: float | None) -> float | None:
|
||||||
if base is None or candidate is None or base <= 0:
|
if base is None or candidate is None or base <= 0:
|
||||||
return None
|
return None
|
||||||
@@ -3019,9 +3124,11 @@ async def run_backtest(
|
|||||||
progress_cb: Callable[[int, int, str], None] | None = None,
|
progress_cb: Callable[[int, int, str], None] | None = None,
|
||||||
*,
|
*,
|
||||||
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
||||||
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Replay every ticker and aggregate the Phase-1 reports for the current config."""
|
"""Replay every ticker and aggregate the Phase-1 reports for the current config."""
|
||||||
target_model = validate_backtest_target_model(target_model)
|
target_model = validate_backtest_target_model(target_model)
|
||||||
|
cadence = validate_backtest_cadence(cadence)
|
||||||
config = await get_recommendation_config(db)
|
config = await get_recommendation_config(db)
|
||||||
activation = await get_activation_config(db)
|
activation = await get_activation_config(db)
|
||||||
|
|
||||||
@@ -3030,7 +3137,9 @@ async def run_backtest(
|
|||||||
total = len(tickers)
|
total = len(tickers)
|
||||||
|
|
||||||
candidates: list[dict] = []
|
candidates: list[dict] = []
|
||||||
# collected[signal_name][iso_week] -> list of (signal_value, forward_return)
|
# Signal IC remains a weekly, non-overlapping diagnostic regardless of the
|
||||||
|
# entry cadence. Production activation ranks are assigned from candidates
|
||||||
|
# at their own weekly or exact-date ``ranking_period`` below.
|
||||||
collected: dict = defaultdict(lambda: defaultdict(list))
|
collected: dict = defaultdict(lambda: defaultdict(list))
|
||||||
|
|
||||||
# Residual momentum needs a point-in-time benchmark return stream. Best-effort:
|
# Residual momentum needs a point-in-time benchmark return stream. Best-effort:
|
||||||
@@ -3087,6 +3196,7 @@ async def run_backtest(
|
|||||||
pool, _replay_and_signals, ticker.symbol, columns, config, activation,
|
pool, _replay_and_signals, ticker.symbol, columns, config, activation,
|
||||||
benchmark_closes,
|
benchmark_closes,
|
||||||
target_model,
|
target_model,
|
||||||
|
cadence,
|
||||||
))
|
))
|
||||||
for result in await asyncio.gather(*futures, return_exceptions=True):
|
for result in await asyncio.gather(*futures, return_exceptions=True):
|
||||||
if isinstance(result, Exception):
|
if isinstance(result, Exception):
|
||||||
@@ -3109,6 +3219,7 @@ async def run_backtest(
|
|||||||
_replay_and_signals, ticker.symbol, columns, config, activation,
|
_replay_and_signals, ticker.symbol, columns, config, activation,
|
||||||
benchmark_closes,
|
benchmark_closes,
|
||||||
target_model,
|
target_model,
|
||||||
|
cadence,
|
||||||
))
|
))
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Backtest replay failed for %s", ticker.symbol)
|
logger.exception("Backtest replay failed for %s", ticker.symbol)
|
||||||
@@ -3219,7 +3330,12 @@ async def run_backtest(
|
|||||||
"candidates": len(candidates),
|
"candidates": len(candidates),
|
||||||
"qualified": len(qualified),
|
"qualified": len(qualified),
|
||||||
"params": {
|
"params": {
|
||||||
"step_days": STEP_DAYS,
|
# Keep step_days for old report consumers; the value counts stored
|
||||||
|
# market sessions rather than calendar days.
|
||||||
|
"step_days": backtest_step_sessions(cadence),
|
||||||
|
"step_sessions": backtest_step_sessions(cadence),
|
||||||
|
"entry_cadence": cadence,
|
||||||
|
"signal_eval_cadence": WEEKLY_BACKTEST_CADENCE,
|
||||||
"horizon_days": HORIZON,
|
"horizon_days": HORIZON,
|
||||||
"min_lookback": MIN_LOOKBACK,
|
"min_lookback": MIN_LOOKBACK,
|
||||||
"cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
|
"cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
|
||||||
@@ -3288,6 +3404,11 @@ async def run_backtest(
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
"portfolio_monitor": portfolio_monitor_report,
|
"portfolio_monitor": portfolio_monitor_report,
|
||||||
|
"production_cadence_comparison": (
|
||||||
|
_production_cadence_comparison(portfolio_monitor_report, cadence)
|
||||||
|
if target_model == PRODUCTION_GTL_TARGET_MODEL
|
||||||
|
else None
|
||||||
|
),
|
||||||
"holdout": holdout_report,
|
"holdout": holdout_report,
|
||||||
"min_rr_sweep": min_rr_sweep_report,
|
"min_rr_sweep": min_rr_sweep_report,
|
||||||
"target_model_diagnostics": _target_model_diagnostics(
|
"target_model_diagnostics": _target_model_diagnostics(
|
||||||
@@ -3323,9 +3444,15 @@ async def run_and_store(
|
|||||||
progress_cb: Callable[[int, int, str], None] | None = None,
|
progress_cb: Callable[[int, int, str], None] | None = None,
|
||||||
*,
|
*,
|
||||||
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
||||||
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Run the backtest and cache the report in a SystemSetting. Job entrypoint."""
|
"""Run the backtest and cache the report in a SystemSetting. Job entrypoint."""
|
||||||
report = await run_backtest(db, progress_cb, target_model=target_model)
|
report = await run_backtest(
|
||||||
|
db,
|
||||||
|
progress_cb,
|
||||||
|
target_model=target_model,
|
||||||
|
cadence=cadence,
|
||||||
|
)
|
||||||
await update_setting(db, KEY_REPORT, json.dumps(report))
|
await update_setting(db, KEY_REPORT, json.dumps(report))
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|||||||
@@ -201,9 +201,11 @@ export interface TriggerJobResponse {
|
|||||||
status: 'triggered' | 'busy' | 'blocked' | 'not_found';
|
status: 'triggered' | 'busy' | 'blocked' | 'not_found';
|
||||||
message: string;
|
message: string;
|
||||||
target_model?: BacktestTargetModel;
|
target_model?: BacktestTargetModel;
|
||||||
|
cadence?: BacktestCadence;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type BacktestTargetModel = 'production_gtl' | 'structural_sr';
|
export type BacktestTargetModel = 'production_gtl' | 'structural_sr';
|
||||||
|
export type BacktestCadence = 'weekly' | 'daily';
|
||||||
|
|
||||||
export function listJobs() {
|
export function listJobs() {
|
||||||
return apiClient.get<JobStatus[]>('admin/jobs').then((r) => r.data);
|
return apiClient.get<JobStatus[]>('admin/jobs').then((r) => r.data);
|
||||||
@@ -219,7 +221,10 @@ export function toggleJob(jobName: string, enabled: boolean) {
|
|||||||
.then((r) => r.data);
|
.then((r) => r.data);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function triggerJob(jobName: string, options?: { target_model?: BacktestTargetModel }) {
|
export function triggerJob(
|
||||||
|
jobName: string,
|
||||||
|
options?: { target_model?: BacktestTargetModel; cadence?: BacktestCadence },
|
||||||
|
) {
|
||||||
return apiClient
|
return apiClient
|
||||||
.post<TriggerJobResponse>(`admin/jobs/${jobName}/trigger`, options)
|
.post<TriggerJobResponse>(`admin/jobs/${jobName}/trigger`, options)
|
||||||
.then((r) => r.data);
|
.then((r) => r.data);
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { useMemo, useState } from 'react';
|
|||||||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||||
import { useBacktestReport } from '../../hooks/useMarketRegime';
|
import { useBacktestReport } from '../../hooks/useMarketRegime';
|
||||||
import { triggerJob } from '../../api/admin';
|
import { triggerJob } from '../../api/admin';
|
||||||
import type { BacktestTargetModel } from '../../api/admin';
|
import type { BacktestCadence, BacktestTargetModel } from '../../api/admin';
|
||||||
import { Button } from '../ui/Button';
|
import { Button } from '../ui/Button';
|
||||||
import { Callout } from '../ui/Callout';
|
import { Callout } from '../ui/Callout';
|
||||||
import { Disclosure } from '../ui/Disclosure';
|
import { Disclosure } from '../ui/Disclosure';
|
||||||
@@ -145,6 +145,7 @@ export function BacktestPanel() {
|
|||||||
const [selectedStrategy, setSelectedStrategy] = useState('');
|
const [selectedStrategy, setSelectedStrategy] = useState('');
|
||||||
const [selectedLookback, setSelectedLookback] = useState('');
|
const [selectedLookback, setSelectedLookback] = useState('');
|
||||||
const [targetModel, setTargetModel] = useState<BacktestTargetModel>('production_gtl');
|
const [targetModel, setTargetModel] = useState<BacktestTargetModel>('production_gtl');
|
||||||
|
const [cadence, setCadence] = useState<BacktestCadence>('weekly');
|
||||||
|
|
||||||
const monitor = report?.portfolio_monitor ?? null;
|
const monitor = report?.portfolio_monitor ?? null;
|
||||||
const activeStrategy =
|
const activeStrategy =
|
||||||
@@ -161,11 +162,11 @@ export function BacktestPanel() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const run = useMutation({
|
const run = useMutation({
|
||||||
mutationFn: () => triggerJob('backtest', { target_model: targetModel }),
|
mutationFn: () => triggerJob('backtest', { target_model: targetModel, cadence }),
|
||||||
onSuccess: (res) => {
|
onSuccess: (res) => {
|
||||||
if (res.status === 'triggered') {
|
if (res.status === 'triggered') {
|
||||||
const label = targetModel === 'production_gtl' ? 'Live GTL' : 'Structural S/R comparison';
|
const label = targetModel === 'production_gtl' ? 'Live GTL' : 'Structural S/R comparison';
|
||||||
toast.addToast('success', `${label} backtest started — results appear when it finishes.`);
|
toast.addToast('success', `${label} ${cadence} backtest started — results appear when it finishes.`);
|
||||||
setTimeout(() => queryClient.invalidateQueries({ queryKey: ['backtest-report'] }), 8000);
|
setTimeout(() => queryClient.invalidateQueries({ queryKey: ['backtest-report'] }), 8000);
|
||||||
} else {
|
} else {
|
||||||
toast.addToast('info', res.message || 'Could not start backtest');
|
toast.addToast('info', res.message || 'Could not start backtest');
|
||||||
@@ -180,7 +181,7 @@ export function BacktestPanel() {
|
|||||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||||
<Disclosure summary="How this is measured">
|
<Disclosure summary="How this is measured">
|
||||||
<p className="max-w-2xl text-xs text-gray-400">
|
<p className="max-w-2xl text-xs text-gray-400">
|
||||||
The backtest replays the current config weekly through history — at each point the setup is
|
The backtest replays the current config at the selected cadence — at each point the setup is
|
||||||
rebuilt using only data up to that day (no lookahead) and the following ~30 trading days decide
|
rebuilt using only data up to that day (no lookahead) and the following ~30 trading days decide
|
||||||
its outcome — then simulates one capital-constrained book against the S&P 500. Sentiment and
|
its outcome — then simulates one capital-constrained book against the S&P 500. Sentiment and
|
||||||
fundamentals are held neutral (no point-in-time history). ~6 months is roughly one market regime,
|
fundamentals are held neutral (no point-in-time history). ~6 months is roughly one market regime,
|
||||||
@@ -238,6 +239,56 @@ export function BacktestPanel() {
|
|||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
<fieldset className="grid w-full grid-cols-2 gap-2 sm:w-[34rem]">
|
||||||
|
<legend className="mb-1 text-[11px] font-medium uppercase tracking-wider text-gray-500">
|
||||||
|
Entry cadence
|
||||||
|
</legend>
|
||||||
|
<label
|
||||||
|
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-blue-400/60 ${
|
||||||
|
cadence === 'weekly'
|
||||||
|
? 'border-blue-400/60 bg-blue-500/10'
|
||||||
|
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
className="sr-only"
|
||||||
|
type="radio"
|
||||||
|
name="backtest-cadence"
|
||||||
|
value="weekly"
|
||||||
|
checked={cadence === 'weekly'}
|
||||||
|
onChange={() => setCadence('weekly')}
|
||||||
|
/>
|
||||||
|
<span className="flex items-center justify-between gap-2 text-sm font-medium text-gray-100">
|
||||||
|
Weekly
|
||||||
|
<span className="rounded-full border border-blue-400/40 bg-blue-400/10 px-2 py-0.5 text-[9px] font-semibold uppercase tracking-widest text-blue-300">
|
||||||
|
Default
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
<span className="mt-1 block text-[11px] leading-4 text-gray-500">
|
||||||
|
Resource-safe server run at five-session intervals.
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label
|
||||||
|
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-amber-400/60 ${
|
||||||
|
cadence === 'daily'
|
||||||
|
? 'border-amber-400/50 bg-amber-500/10'
|
||||||
|
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
className="sr-only"
|
||||||
|
type="radio"
|
||||||
|
name="backtest-cadence"
|
||||||
|
value="daily"
|
||||||
|
checked={cadence === 'daily'}
|
||||||
|
onChange={() => setCadence('daily')}
|
||||||
|
/>
|
||||||
|
<span className="text-sm font-medium text-gray-200">Daily</span>
|
||||||
|
<span className="mt-1 block text-[11px] leading-4 text-amber-300/80">
|
||||||
|
Research run: roughly 5× the replay work; prefer the offline snapshot runner.
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
<Button onClick={() => run.mutate()} loading={run.isPending} className="shrink-0">
|
<Button onClick={() => run.mutate()} loading={run.isPending} className="shrink-0">
|
||||||
{run.isPending ? 'Starting…' : report ? 'Re-run backtest' : 'Run backtest'}
|
{run.isPending ? 'Starting…' : report ? 'Re-run backtest' : 'Run backtest'}
|
||||||
</Button>
|
</Button>
|
||||||
@@ -257,7 +308,8 @@ export function BacktestPanel() {
|
|||||||
<>
|
<>
|
||||||
<p className="text-[11px] text-gray-500">
|
<p className="text-[11px] text-gray-500">
|
||||||
Ran {timeAgo(report.generated_at)} · {report.tickers} tickers · {report.candidates} setups
|
Ran {timeAgo(report.generated_at)} · {report.tickers} tickers · {report.candidates} setups
|
||||||
({report.qualified} qualified) · weekly cadence, {report.params.horizon_days}-day horizon
|
({report.qualified} qualified) · {report.params.entry_cadence ?? 'weekly'} cadence,
|
||||||
|
{' '}{report.params.horizon_days}-day horizon
|
||||||
{report.params.cost_per_side_pct != null && (
|
{report.params.cost_per_side_pct != null && (
|
||||||
<> · net of {report.params.cost_per_side_pct}%/side costs</>
|
<> · net of {report.params.cost_per_side_pct}%/side costs</>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -356,6 +356,7 @@ export interface BacktestPortfolioMonitorRun extends BacktestPortfolioPolicy {
|
|||||||
label: string;
|
label: string;
|
||||||
description: string;
|
description: string;
|
||||||
is_production: boolean;
|
is_production: boolean;
|
||||||
|
comparison_arm?: 'live_no_lockdown' | 'live_lockdown_5' | null;
|
||||||
entry_variant: string;
|
entry_variant: string;
|
||||||
exit_policy: string;
|
exit_policy: string;
|
||||||
reentry_lockdown_sessions?: number;
|
reentry_lockdown_sessions?: number;
|
||||||
@@ -370,6 +371,7 @@ export interface BacktestPortfolioMonitor {
|
|||||||
label: string;
|
label: string;
|
||||||
description: string;
|
description: string;
|
||||||
is_production: boolean;
|
is_production: boolean;
|
||||||
|
comparison_arm?: 'live_no_lockdown' | 'live_lockdown_5' | null;
|
||||||
reentry_lockdown_sessions?: number;
|
reentry_lockdown_sessions?: number;
|
||||||
}[];
|
}[];
|
||||||
lookbacks: { lookback: string; label: string }[];
|
lookbacks: { lookback: string; label: string }[];
|
||||||
@@ -404,6 +406,9 @@ export interface BacktestReport {
|
|||||||
qualified: number;
|
qualified: number;
|
||||||
params: {
|
params: {
|
||||||
step_days: number;
|
step_days: number;
|
||||||
|
step_sessions?: number;
|
||||||
|
entry_cadence?: 'weekly' | 'daily';
|
||||||
|
signal_eval_cadence?: 'weekly';
|
||||||
horizon_days: number;
|
horizon_days: number;
|
||||||
min_lookback: number;
|
min_lookback: number;
|
||||||
cost_per_side_pct?: number;
|
cost_per_side_pct?: number;
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
"""Run the four production cadence/lockdown arms on one offline snapshot.
|
||||||
|
|
||||||
|
The command executes the complete backtest once weekly and once daily. Each
|
||||||
|
backtest contains two otherwise identical live-policy portfolio arms: no
|
||||||
|
post-stop lockdown and the production five-session lockdown. It writes both
|
||||||
|
full reports plus one compact four-arm comparison report.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"snapshot",
|
||||||
|
help="SQLite snapshot created by scripts/create_backtest_snapshot.py.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--out-dir",
|
||||||
|
default="reports",
|
||||||
|
help="Directory for the weekly, daily, and comparison JSON reports.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--prefix",
|
||||||
|
default=None,
|
||||||
|
help="Output prefix. Defaults to backtest-cadence-<timestamp>.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--workers",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Override worker count; on a powerful offline PC use CPU count minus one.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--allow-spawn",
|
||||||
|
action="store_true",
|
||||||
|
help="Enable multiprocessing spawn for the offline Windows run.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--quiet", action="store_true", help="Hide ticker progress.")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_url(path: Path) -> str:
|
||||||
|
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, payload: dict) -> None:
|
||||||
|
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _comparison_arms(report: dict) -> list[dict]:
|
||||||
|
comparison = report.get("production_cadence_comparison") or {}
|
||||||
|
arms = list(comparison.get("arms") or [])
|
||||||
|
if len(arms) != 2:
|
||||||
|
cadence = (report.get("params") or {}).get("entry_cadence", "unknown")
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Expected two live comparison arms for {cadence}; found {len(arms)}"
|
||||||
|
)
|
||||||
|
return arms
|
||||||
|
|
||||||
|
|
||||||
|
def _print_arm(row: dict) -> None:
|
||||||
|
print(
|
||||||
|
f" {row['arm']}: Sharpe {row.get('sharpe')}, "
|
||||||
|
f"CAGR {row.get('cagr_pct')}%, DD {row.get('max_drawdown_pct')}%, "
|
||||||
|
f"trades {row.get('trades')}, skipped cooldown {row.get('skipped_cooldown', 0)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _main() -> None:
|
||||||
|
args = _parse_args()
|
||||||
|
snapshot = Path(args.snapshot)
|
||||||
|
if not snapshot.exists():
|
||||||
|
raise SystemExit(f"Snapshot not found: {snapshot}")
|
||||||
|
|
||||||
|
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||||||
|
if args.allow_spawn:
|
||||||
|
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.services.backtest_service import run_backtest
|
||||||
|
|
||||||
|
if args.workers is not None:
|
||||||
|
settings.backtest_workers = args.workers
|
||||||
|
|
||||||
|
out_dir = Path(args.out_dir)
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
prefix = args.prefix or f"backtest-cadence-{datetime.now():%Y%m%d-%H%M%S}"
|
||||||
|
|
||||||
|
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||||
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||||
|
reports: dict[str, dict] = {}
|
||||||
|
try:
|
||||||
|
async with Session() as db:
|
||||||
|
for cadence in ("weekly", "daily"):
|
||||||
|
last_progress: tuple[int, int] | None = None
|
||||||
|
|
||||||
|
def progress(done: int, total: int, symbol: str) -> None:
|
||||||
|
nonlocal last_progress
|
||||||
|
if args.quiet or last_progress == (done, total):
|
||||||
|
return
|
||||||
|
last_progress = (done, total)
|
||||||
|
label = f" {symbol}" if symbol else ""
|
||||||
|
print(
|
||||||
|
f"{cadence} progress: {done}/{total}{label}",
|
||||||
|
end="\r",
|
||||||
|
)
|
||||||
|
|
||||||
|
reports[cadence] = await run_backtest(
|
||||||
|
db,
|
||||||
|
progress_cb=progress,
|
||||||
|
target_model="production_gtl",
|
||||||
|
cadence=cadence,
|
||||||
|
)
|
||||||
|
if not args.quiet:
|
||||||
|
print("")
|
||||||
|
_write_json(out_dir / f"{prefix}-{cadence}.json", reports[cadence])
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
arms = [
|
||||||
|
*_comparison_arms(reports["weekly"]),
|
||||||
|
*_comparison_arms(reports["daily"]),
|
||||||
|
]
|
||||||
|
expected = {
|
||||||
|
"prod_live_setup_weekly",
|
||||||
|
"prod_live_setup_daily",
|
||||||
|
"cooldown_5_weekly",
|
||||||
|
"cooldown_5_daily",
|
||||||
|
}
|
||||||
|
if {row.get("arm") for row in arms} != expected:
|
||||||
|
raise RuntimeError("The generated cadence report does not contain all four arms")
|
||||||
|
arm_order = {
|
||||||
|
"prod_live_setup_weekly": 0,
|
||||||
|
"prod_live_setup_daily": 1,
|
||||||
|
"cooldown_5_weekly": 2,
|
||||||
|
"cooldown_5_daily": 3,
|
||||||
|
}
|
||||||
|
arms.sort(key=lambda row: arm_order[str(row["arm"])])
|
||||||
|
|
||||||
|
comparison = {
|
||||||
|
"generated_at": datetime.now().astimezone().isoformat(),
|
||||||
|
"snapshot": str(snapshot.resolve()),
|
||||||
|
"target_model": "production_gtl",
|
||||||
|
"arms": arms,
|
||||||
|
"full_reports": {
|
||||||
|
cadence: str((out_dir / f"{prefix}-{cadence}.json").resolve())
|
||||||
|
for cadence in ("weekly", "daily")
|
||||||
|
},
|
||||||
|
"note": (
|
||||||
|
"All four arms use the same snapshot, activation settings, target model, "
|
||||||
|
"live Admin exit policy, fees, sizing, and portfolio constraints. Within "
|
||||||
|
"each cadence pair, only the five-session post-stop lockdown differs."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
comparison_path = out_dir / f"{prefix}-comparison.json"
|
||||||
|
_write_json(comparison_path, comparison)
|
||||||
|
|
||||||
|
print(f"Comparison written: {comparison_path}")
|
||||||
|
for row in arms:
|
||||||
|
_print_arm(row)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(_main())
|
||||||
@@ -32,7 +32,10 @@ def _parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--out",
|
"--out",
|
||||||
default=None,
|
default=None,
|
||||||
help="JSON report path. Defaults to reports/backtest-<timestamp>.json.",
|
help=(
|
||||||
|
"JSON report path. Defaults to "
|
||||||
|
"reports/backtest-<cadence>-<timestamp>.json."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--workers",
|
"--workers",
|
||||||
@@ -55,6 +58,15 @@ def _parse_args() -> argparse.Namespace:
|
|||||||
"structural_sr is a comparison-only chart-S/R model."
|
"structural_sr is a comparison-only chart-S/R model."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--cadence",
|
||||||
|
choices=("weekly", "daily"),
|
||||||
|
default="weekly",
|
||||||
|
help=(
|
||||||
|
"Entry replay cadence. Weekly is the resource-safe production default; "
|
||||||
|
"daily performs roughly five times as many setup evaluations."
|
||||||
|
),
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--holdout-split",
|
"--holdout-split",
|
||||||
default=None,
|
default=None,
|
||||||
@@ -63,9 +75,9 @@ def _parse_args() -> argparse.Namespace:
|
|||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def _default_output_path() -> Path:
|
def _default_output_path(cadence: str) -> Path:
|
||||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||||
return Path("reports") / f"backtest-{stamp}.json"
|
return Path("reports") / f"backtest-{cadence}-{stamp}.json"
|
||||||
|
|
||||||
|
|
||||||
def _pct(value: Any) -> str:
|
def _pct(value: Any) -> str:
|
||||||
@@ -93,6 +105,7 @@ def _print_summary(report: dict) -> None:
|
|||||||
|
|
||||||
print("")
|
print("")
|
||||||
print("Backtest summary")
|
print("Backtest summary")
|
||||||
|
print(f" entry cadence: {(report.get('params') or {}).get('entry_cadence', 'weekly')}")
|
||||||
print(f" candidates: {report.get('candidates')}")
|
print(f" candidates: {report.get('candidates')}")
|
||||||
print(f" qualified: {report.get('qualified')}")
|
print(f" qualified: {report.get('qualified')}")
|
||||||
print(f" all setups net avg R: {_r(all_setups.get('net_avg_r'))}")
|
print(f" all setups net avg R: {_r(all_setups.get('net_avg_r'))}")
|
||||||
@@ -183,7 +196,7 @@ async def _main() -> None:
|
|||||||
if args.workers is not None:
|
if args.workers is not None:
|
||||||
settings.backtest_workers = args.workers
|
settings.backtest_workers = args.workers
|
||||||
|
|
||||||
output = Path(args.out) if args.out else _default_output_path()
|
output = Path(args.out) if args.out else _default_output_path(args.cadence)
|
||||||
output.parent.mkdir(parents=True, exist_ok=True)
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||||
@@ -208,6 +221,7 @@ async def _main() -> None:
|
|||||||
db,
|
db,
|
||||||
progress_cb=progress,
|
progress_cb=progress,
|
||||||
target_model=args.target_model,
|
target_model=args.target_model,
|
||||||
|
cadence=args.cadence,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
await engine.dispose()
|
await engine.dispose()
|
||||||
|
|||||||
@@ -658,6 +658,36 @@ class TestSimulatePortfolio:
|
|||||||
for row in comparison_rows
|
for row in comparison_rows
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_production_cadence_comparison_names_exact_two_arms(self):
|
||||||
|
monitor = {
|
||||||
|
"runs": [
|
||||||
|
{
|
||||||
|
"comparison_arm": "live_no_lockdown",
|
||||||
|
"lookback": "all",
|
||||||
|
"reentry_lockdown_sessions": 0,
|
||||||
|
"trades": 10,
|
||||||
|
"equity_curve": [{"date": "2026-01-01", "value": 1.0}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comparison_arm": "live_lockdown_5",
|
||||||
|
"lookback": "all",
|
||||||
|
"reentry_lockdown_sessions": 5,
|
||||||
|
"trades": 8,
|
||||||
|
"benchmark_curve": [{"date": "2026-01-01", "value": 1.0}],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
comparison = bt._production_cadence_comparison(monitor, "daily")
|
||||||
|
|
||||||
|
assert comparison is not None
|
||||||
|
assert [row["arm"] for row in comparison["arms"]] == [
|
||||||
|
"prod_live_setup_daily",
|
||||||
|
"cooldown_5_daily",
|
||||||
|
]
|
||||||
|
assert all("equity_curve" not in row for row in comparison["arms"])
|
||||||
|
assert all("benchmark_curve" not in row for row in comparison["arms"])
|
||||||
|
|
||||||
def test_initial_stop_can_refresh_lower_and_survive_same_bar(self):
|
def test_initial_stop_can_refresh_lower_and_survive_same_bar(self):
|
||||||
closes = [100.0, 94.0, 96.0]
|
closes = [100.0, 94.0, 96.0]
|
||||||
prices = {"AAA": _sim_prices(self.ORD, closes)}
|
prices = {"AAA": _sim_prices(self.ORD, closes)}
|
||||||
@@ -929,6 +959,15 @@ def test_backtest_target_model_is_small_and_validated():
|
|||||||
bt.validate_backtest_target_model("legacy_range_grid_touch")
|
bt.validate_backtest_target_model("legacy_range_grid_touch")
|
||||||
|
|
||||||
|
|
||||||
|
def test_backtest_cadence_is_small_validated_and_session_based():
|
||||||
|
assert bt.validate_backtest_cadence(" WEEKLY ") == "weekly"
|
||||||
|
assert bt.validate_backtest_cadence("daily") == "daily"
|
||||||
|
assert bt.backtest_step_sessions("weekly") == 5
|
||||||
|
assert bt.backtest_step_sessions("daily") == 1
|
||||||
|
with pytest.raises(ValueError, match="Unknown backtest cadence"):
|
||||||
|
bt.validate_backtest_cadence("monthly")
|
||||||
|
|
||||||
|
|
||||||
def _flat_window_records():
|
def _flat_window_records():
|
||||||
return [
|
return [
|
||||||
SimpleNamespace(
|
SimpleNamespace(
|
||||||
@@ -1022,6 +1061,46 @@ def test_replay_ticker_candidates_carry_gate_fields():
|
|||||||
assert c.get("action") is not None
|
assert c.get("action") is not None
|
||||||
assert "risk_level" in c
|
assert "risk_level" in c
|
||||||
assert c["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL
|
assert c["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL
|
||||||
|
assert c["ranking_period"][0] == "week"
|
||||||
|
|
||||||
|
daily_cands = bt._replay_ticker(
|
||||||
|
"OSC",
|
||||||
|
bars,
|
||||||
|
dict(DEFAULT_RECOMMENDATION_CONFIG),
|
||||||
|
dict(ACTIVATION_DEFAULTS),
|
||||||
|
cadence="daily",
|
||||||
|
)
|
||||||
|
assert len(daily_cands) > len(cands)
|
||||||
|
assert all(c["ranking_period"][0] == "date" for c in daily_cands)
|
||||||
|
|
||||||
|
|
||||||
|
def test_daily_replay_uses_exact_date_ranking_periods():
|
||||||
|
candidates = [
|
||||||
|
{
|
||||||
|
"iso_week": (2026, 1),
|
||||||
|
"ranking_period": ("date", date(2026, 1, 5).toordinal()),
|
||||||
|
"momentum": 0.10,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"iso_week": (2026, 1),
|
||||||
|
"ranking_period": ("date", date(2026, 1, 5).toordinal()),
|
||||||
|
"momentum": 0.20,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"iso_week": (2026, 1),
|
||||||
|
"ranking_period": ("date", date(2026, 1, 6).toordinal()),
|
||||||
|
"momentum": 0.90,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"iso_week": (2026, 1),
|
||||||
|
"ranking_period": ("date", date(2026, 1, 6).toordinal()),
|
||||||
|
"momentum": 0.30,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
bt._assign_momentum_percentiles(candidates)
|
||||||
|
|
||||||
|
assert [row["momentum_percentile"] for row in candidates] == [0.0, 100.0, 100.0, 0.0]
|
||||||
|
|
||||||
|
|
||||||
async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None:
|
async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None:
|
||||||
@@ -1063,6 +1142,8 @@ async def test_run_backtest_smoke(session):
|
|||||||
assert report["params"]["cost_per_side_pct"] == pytest.approx(bt.COST_PER_SIDE * 100)
|
assert report["params"]["cost_per_side_pct"] == pytest.approx(bt.COST_PER_SIDE * 100)
|
||||||
assert report["params"]["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL
|
assert report["params"]["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL
|
||||||
assert report["params"]["is_production_target_model"] is True
|
assert report["params"]["is_production_target_model"] is True
|
||||||
|
assert report["params"]["entry_cadence"] == "weekly"
|
||||||
|
assert report["params"]["step_sessions"] == 5
|
||||||
assert (
|
assert (
|
||||||
report["params"]["production_reentry_lockdown_sessions"]
|
report["params"]["production_reentry_lockdown_sessions"]
|
||||||
== bt.REENTRY_LOCKDOWN_SESSIONS
|
== bt.REENTRY_LOCKDOWN_SESSIONS
|
||||||
@@ -1073,6 +1154,11 @@ async def test_run_backtest_smoke(session):
|
|||||||
# carries the hold-to-horizon grading alongside the target model
|
# carries the hold-to-horizon grading alongside the target model
|
||||||
ablation = {r["variant"]: r for r in report["gate_ablation"]}
|
ablation = {r["variant"]: r for r in report["gate_ablation"]}
|
||||||
assert ablation["all_floors"]["total"] == report["overall_qualified"]["total"]
|
assert ablation["all_floors"]["total"] == report["overall_qualified"]["total"]
|
||||||
|
|
||||||
|
daily_report = await bt.run_backtest(session, cadence="daily")
|
||||||
|
assert daily_report["params"]["entry_cadence"] == "daily"
|
||||||
|
assert daily_report["params"]["step_sessions"] == 1
|
||||||
|
assert daily_report["candidates"] > report["candidates"]
|
||||||
for row in report["gate_ablation"]:
|
for row in report["gate_ablation"]:
|
||||||
assert "hold_net_avg_r" in row
|
assert "hold_net_avg_r" in row
|
||||||
|
|
||||||
|
|||||||
@@ -3,11 +3,13 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.scheduler import (
|
from app.scheduler import (
|
||||||
|
_consume_backtest_options,
|
||||||
_consume_backtest_target_model,
|
_consume_backtest_target_model,
|
||||||
_parse_frequency,
|
_parse_frequency,
|
||||||
_resume_tickers,
|
_resume_tickers,
|
||||||
_last_successful,
|
_last_successful,
|
||||||
configure_scheduler,
|
configure_scheduler,
|
||||||
|
queue_backtest_options,
|
||||||
queue_backtest_target_model,
|
queue_backtest_target_model,
|
||||||
scheduler,
|
scheduler,
|
||||||
)
|
)
|
||||||
@@ -24,6 +26,15 @@ def test_manual_backtest_target_model_rejects_removed_research_arms():
|
|||||||
queue_backtest_target_model("production_control")
|
queue_backtest_target_model("production_control")
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_backtest_options_are_one_shot_and_default_back_to_weekly():
|
||||||
|
assert queue_backtest_options("structural_sr", "daily") == (
|
||||||
|
"structural_sr",
|
||||||
|
"daily",
|
||||||
|
)
|
||||||
|
assert _consume_backtest_options() == ("structural_sr", "daily")
|
||||||
|
assert _consume_backtest_options() == ("production_gtl", "weekly")
|
||||||
|
|
||||||
|
|
||||||
class TestParseFrequency:
|
class TestParseFrequency:
|
||||||
def test_hourly(self):
|
def test_hourly(self):
|
||||||
assert _parse_frequency("hourly") == {"hours": 1}
|
assert _parse_frequency("hourly") == {"hours": 1}
|
||||||
|
|||||||
Reference in New Issue
Block a user