dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add selectable daily backtest cadence

Browse Source
This commit is contained in:
dennisthiessen
2026-07-17 14:41:24 +02:00
parent bc50ba9136
commit 65a462271c
12 changed files with 550 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/backtest_service.py
+142 -15
View File
@@ -1,7 +1,7 @@
"""Historical backtest (Phase 1): replay the price-derived engine over stored
OHLCV and measure how the CURRENT config would have performed.
For each ticker we step through history (weekly), and at each as-of date D we
For each ticker we step through history at the selected entry cadence, and at each as-of date D we
rebuild the setup using only bars ≤ D (no lookahead), then walk the actual bars
after D to record the realized outcome. The report contains:
@@ -100,7 +100,16 @@ logger = logging.getLogger(__name__)
KEY_REPORT = "backtest_report"
STEP_DAYS = 5 # weekly cadence (≈ 5 trading days)
WEEKLY_BACKTEST_CADENCE = "weekly"
DAILY_BACKTEST_CADENCE = "daily"
DEFAULT_BACKTEST_CADENCE = WEEKLY_BACKTEST_CADENCE
BACKTEST_CADENCE_SESSIONS = {
WEEKLY_BACKTEST_CADENCE: 5,
DAILY_BACKTEST_CADENCE: 1,
}
# Compatibility alias for research scripts built around the original weekly
# replay. New code should select a cadence and call ``backtest_step_sessions``.
STEP_DAYS = BACKTEST_CADENCE_SESSIONS[WEEKLY_BACKTEST_CADENCE]
MIN_LOOKBACK = 60 # bars needed before D for indicators (EMA cross needs 51)
HORIZON = 30 # trading days to resolve an outcome (matches the evaluator)
ATR_MULTIPLIER = 1.5
@@ -157,6 +166,30 @@ def validate_backtest_target_model(value: str) -> str:
return normalized
def validate_backtest_cadence(value: str) -> str:
"""Validate the supported entry-replay cadences."""
normalized = value.strip().lower()
if normalized not in BACKTEST_CADENCE_SESSIONS:
allowed = ", ".join(BACKTEST_CADENCE_SESSIONS)
raise ValueError(
f"Unknown backtest cadence {value!r}; expected one of {allowed}"
)
return normalized
def backtest_step_sessions(cadence: str) -> int:
return BACKTEST_CADENCE_SESSIONS[validate_backtest_cadence(cadence)]
def _ranking_period(as_of: date, cadence: str) -> tuple:
"""Cross-section key for activation ranks at the selected entry cadence."""
cadence = validate_backtest_cadence(cadence)
if cadence == DAILY_BACKTEST_CADENCE:
return ("date", as_of.toordinal())
iso = as_of.isocalendar()
return ("week", iso[0], iso[1])
# ---------------------------------------------------------------------------
# RESEARCH / DIAGNOSTIC FALLBACKS (retired experiments)
#
@@ -456,14 +489,17 @@ def _replay_ticker(
activation: dict,
benchmark_closes: dict[date, float] | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> list[dict]:
"""Walk one ticker's history weekly, building setups and their realized outcomes."""
"""Walk one ticker at the selected cadence and resolve each setup outcome."""
cadence = validate_backtest_cadence(cadence)
step_sessions = backtest_step_sessions(cadence)
candidates: list[dict] = []
n = len(records)
if n < MIN_LOOKBACK + HORIZON:
return candidates
for i in range(MIN_LOOKBACK - 1, n - HORIZON, STEP_DAYS):
for i in range(MIN_LOOKBACK - 1, n - HORIZON, step_sessions):
window = records[: i + 1]
forward = records[i + 1 :]
forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
@@ -512,6 +548,7 @@ def _replay_ticker(
"symbol": symbol,
"date": records[i].date.isoformat(),
"iso_week": (iso[0], iso[1]),
"ranking_period": _ranking_period(records[i].date, cadence),
"direction": s["direction"],
"entry": s["entry"],
"stop": s["stop"],
@@ -968,6 +1005,7 @@ def _replay_and_signals(
activation: dict,
benchmark_closes: dict[date, float] | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> tuple[list[dict], dict]:
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can
run in a worker process. Takes primitive column arrays (cheap to pickle),
@@ -987,6 +1025,7 @@ def _replay_and_signals(
activation,
benchmark_closes,
target_model,
cadence,
),
_signal_series(bars, benchmark_closes),
)
@@ -999,6 +1038,7 @@ def _replay_candidates_for_period(
activation: dict,
benchmark_closes: dict[date, float] | None,
start_date: date,
cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> list[dict]:
"""Slim picklable replay used by local event studies.
@@ -1014,8 +1054,13 @@ def _replay_candidates_for_period(
date_ords, opens, highs, lows, closes, volumes
)
]
cadence = validate_backtest_cadence(cadence)
candidates: list[dict] = []
for i in range(MIN_LOOKBACK - 1, len(bars) - HORIZON, STEP_DAYS):
for i in range(
MIN_LOOKBACK - 1,
len(bars) - HORIZON,
backtest_step_sessions(cadence),
):
if bars[i].date < start_date:
continue
window = bars[: i + 1]
@@ -1036,6 +1081,7 @@ def _replay_candidates_for_period(
"symbol": symbol,
"date": bars[i].date.isoformat(),
"iso_week": (iso[0], iso[1]),
"ranking_period": _ranking_period(bars[i].date, cadence),
"direction": "long",
"entry": setup["entry"],
"stop": setup["stop"],
@@ -1120,14 +1166,17 @@ def _assign_signal_percentiles(
value_key: str,
percentile_key: str,
) -> None:
"""Per ISO week, rank candidates by ``value_key`` and attach a 0-100
"""Per replay period, rank candidates by ``value_key`` and attach a 0-100
percentile under ``percentile_key`` (100 = strongest). Missing values get
None and therefore cannot clear a gate based on that signal."""
by_week: dict = defaultdict(list)
by_period: dict = defaultdict(list)
for c in candidates:
if c.get(value_key) is not None:
by_week[c["iso_week"]].append(c)
for group in by_week.values():
# Hand-built/research candidates predating the cadence flag retain
# the weekly key as a compatibility fallback.
period = c.get("ranking_period") or c["iso_week"]
by_period[period].append(c)
for group in by_period.values():
ordered = sorted(group, key=lambda c: c[value_key])
n = len(ordered)
for rank, c in enumerate(ordered):
@@ -1137,9 +1186,9 @@ def _assign_signal_percentiles(
def _assign_momentum_percentiles(candidates: list[dict]) -> None:
"""Per ISO week, rank candidates by their ticker's 12-1 momentum and attach a
"""Per replay period, rank candidates by 12-1 momentum and attach a
0-100 ``momentum_percentile`` (100 = highest momentum in the universe that
week). Candidates whose momentum is unknown (insufficient lookback) get None
period). Candidates whose momentum is unknown (insufficient lookback) get None
and therefore can't clear a momentum gate. Mutates ``candidates``."""
_assign_signal_percentiles(candidates, "momentum", "momentum_percentile")
@@ -1152,7 +1201,7 @@ def _assign_residual_momentum_percentiles(candidates: list[dict]) -> None:
def _assign_low_volatility_percentiles(candidates: list[dict]) -> None:
"""Per ISO week, attach volatility ranks where 100 = lowest 6-month vol."""
"""Per replay period, attach volatility ranks where 100 = lowest 6-month vol."""
_assign_signal_percentiles(candidates, "vol_6m", VOL_PERCENTILE_KEY)
for c in candidates:
raw = c.get(VOL_PERCENTILE_KEY)
@@ -2227,6 +2276,19 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
"entry_variant": "residual80_highvol_blend80_20_fixed10",
"exit_policy": "hold",
},
{
"strategy": "production_live_no_lockdown",
"label": "Live setup + 3x ATR trail (no re-entry lockdown)",
"description": (
"Exact live activation, ordering, and Admin exit policy, with only "
"the post-stop re-entry lockdown disabled as the comparison baseline."
),
"entry_variant": "residual80_highvol_blend80_20_fixed10",
"exit_policy": "atr_trail3",
"reentry_lockdown_sessions": 0,
"use_live_config": True,
"comparison_arm": "live_no_lockdown",
},
{
"strategy": PRODUCTION_PORTFOLIO_STRATEGY,
"label": "Production: residual/high-vol 80/20 + 3x ATR trail + 5-session lockdown",
@@ -2243,6 +2305,7 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
# live Admin exit policy, instead of the frozen research-variant gate.
"use_live_config": True,
"is_production": True,
"comparison_arm": "live_lockdown_5",
},
)
@@ -2603,6 +2666,7 @@ def _portfolio_monitor(
"label": strategy["label"],
"description": strategy["description"],
"is_production": bool(strategy.get("is_production")),
"comparison_arm": strategy.get("comparison_arm"),
"entry_variant": strategy["entry_variant"],
"ranking_key": ranking_key,
"exit_policy": exit_policy,
@@ -2620,6 +2684,7 @@ def _portfolio_monitor(
"label": s["label"],
"description": s["description"],
"is_production": bool(s.get("is_production")),
"comparison_arm": s.get("comparison_arm"),
"reentry_lockdown_sessions": int(
s.get("reentry_lockdown_sessions", 0)
),
@@ -2641,6 +2706,46 @@ def _portfolio_monitor(
}
def _production_cadence_comparison(
monitor: dict | None,
cadence: str,
) -> dict | None:
"""Compact full-history live/no-lockdown vs live/5-session comparison."""
if not monitor:
return None
arms: list[dict] = []
for row in monitor.get("runs") or []:
comparison_arm = row.get("comparison_arm")
if not comparison_arm or row.get("lookback") != "all":
continue
compact = {
key: value
for key, value in row.items()
if key not in {"equity_curve", "benchmark_curve"}
}
arm_name = (
"prod_live_setup"
if comparison_arm == "live_no_lockdown"
else "cooldown_5"
)
compact["arm"] = f"{arm_name}_{cadence}"
compact["entry_cadence"] = cadence
arms.append(compact)
if not arms:
return None
arms.sort(key=lambda row: int(row.get("reentry_lockdown_sessions", 0)))
return {
"entry_cadence": cadence,
"lookback": "all",
"arms": arms,
"note": (
"Both arms use the exact same live gate, ordering, Admin exit policy, "
"fees, and candidate cadence. Only the five-session post-stop "
"re-entry lockdown changes."
),
}
def _pct_loss(base: float | None, candidate: float | None) -> float | None:
if base is None or candidate is None or base <= 0:
return None
@@ -3019,9 +3124,11 @@ async def run_backtest(
progress_cb: Callable[[int, int, str], None] | None = None,
*,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> dict:
"""Replay every ticker and aggregate the Phase-1 reports for the current config."""
target_model = validate_backtest_target_model(target_model)
cadence = validate_backtest_cadence(cadence)
config = await get_recommendation_config(db)
activation = await get_activation_config(db)
@@ -3030,7 +3137,9 @@ async def run_backtest(
total = len(tickers)
candidates: list[dict] = []
# collected[signal_name][iso_week] -> list of (signal_value, forward_return)
# Signal IC remains a weekly, non-overlapping diagnostic regardless of the
# entry cadence. Production activation ranks are assigned from candidates
# at their own weekly or exact-date ``ranking_period`` below.
collected: dict = defaultdict(lambda: defaultdict(list))
# Residual momentum needs a point-in-time benchmark return stream. Best-effort:
@@ -3087,6 +3196,7 @@ async def run_backtest(
pool, _replay_and_signals, ticker.symbol, columns, config, activation,
benchmark_closes,
target_model,
cadence,
))
for result in await asyncio.gather(*futures, return_exceptions=True):
if isinstance(result, Exception):
@@ -3109,6 +3219,7 @@ async def run_backtest(
_replay_and_signals, ticker.symbol, columns, config, activation,
benchmark_closes,
target_model,
cadence,
))
except Exception:
logger.exception("Backtest replay failed for %s", ticker.symbol)
@@ -3219,7 +3330,12 @@ async def run_backtest(
"candidates": len(candidates),
"qualified": len(qualified),
"params": {
"step_days": STEP_DAYS,
# Keep step_days for old report consumers; the value counts stored
# market sessions rather than calendar days.
"step_days": backtest_step_sessions(cadence),
"step_sessions": backtest_step_sessions(cadence),
"entry_cadence": cadence,
"signal_eval_cadence": WEEKLY_BACKTEST_CADENCE,
"horizon_days": HORIZON,
"min_lookback": MIN_LOOKBACK,
"cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
@@ -3288,6 +3404,11 @@ async def run_backtest(
),
},
"portfolio_monitor": portfolio_monitor_report,
"production_cadence_comparison": (
_production_cadence_comparison(portfolio_monitor_report, cadence)
if target_model == PRODUCTION_GTL_TARGET_MODEL
else None
),
"holdout": holdout_report,
"min_rr_sweep": min_rr_sweep_report,
"target_model_diagnostics": _target_model_diagnostics(
@@ -3323,9 +3444,15 @@ async def run_and_store(
progress_cb: Callable[[int, int, str], None] | None = None,
*,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> dict:
"""Run the backtest and cache the report in a SystemSetting. Job entrypoint."""
report = await run_backtest(db, progress_cb, target_model=target_model)
report = await run_backtest(
db,
progress_cb,
target_model=target_model,
cadence=cadence,
)
await update_setting(db, KEY_REPORT, json.dumps(report))
return report