From 65a462271cacf936b5d6ff56935fa8f3f5c87977 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Fri, 17 Jul 2026 14:41:24 +0200 Subject: [PATCH] feat: add selectable daily backtest cadence --- app/routers/admin.py | 1 + app/scheduler.py | 47 ++++- app/schemas/admin.py | 1 + app/services/admin_service.py | 9 +- app/services/backtest_service.py | 157 +++++++++++++-- frontend/src/api/admin.ts | 7 +- .../src/components/signals/BacktestPanel.tsx | 62 +++++- frontend/src/lib/types.ts | 5 + scripts/run_backtest_cadence_comparison.py | 179 ++++++++++++++++++ scripts/run_backtest_snapshot.py | 22 ++- tests/unit/test_backtest_service.py | 86 +++++++++ tests/unit/test_scheduler.py | 11 ++ 12 files changed, 550 insertions(+), 37 deletions(-) create mode 100644 scripts/run_backtest_cadence_comparison.py diff --git a/app/routers/admin.py b/app/routers/admin.py index 6323adc..1b7da35 100644 --- a/app/routers/admin.py +++ b/app/routers/admin.py @@ -387,6 +387,7 @@ async def trigger_job( db, job_name, target_model=body.target_model if body is not None else None, + cadence=body.cadence if body is not None else None, ) return APIEnvelope(status="success", data=result) diff --git a/app/scheduler.py b/app/scheduler.py index af61665..246634c 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -37,8 +37,10 @@ from app.services import fundamental_service, ingestion_service, sentiment_servi from app.services.alert_service import dispatch_alerts from app.services.backtest_service import ( BACKTEST_TARGET_MODELS, + DEFAULT_BACKTEST_CADENCE, PRODUCTION_GTL_TARGET_MODEL, run_and_store as run_backtest_and_store, + validate_backtest_cadence, validate_backtest_target_model, ) from app.services.benchmark_service import refresh_benchmark_prices @@ -112,6 +114,7 @@ def _idle_runtime() -> dict[str, object]: _job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES} _next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL +_next_backtest_cadence = DEFAULT_BACKTEST_CADENCE # --------------------------------------------------------------------------- @@ -119,23 +122,44 @@ _next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL # --------------------------------------------------------------------------- -def queue_backtest_target_model(target_model: str | None) -> str: - """Select the model for the next manual backtest run only. +def queue_backtest_options( + target_model: str | None, + cadence: str | None, +) -> tuple[str, str]: + """Select model and cadence for the next manual backtest run only. - Scheduled runs and subsequent manual runs return to the production GTL. + Scheduled and subsequent manual runs return to production GTL at the + resource-safe weekly cadence. """ - global _next_backtest_target_model - selected = validate_backtest_target_model( + global _next_backtest_target_model, _next_backtest_cadence + selected_model = validate_backtest_target_model( target_model or PRODUCTION_GTL_TARGET_MODEL ) - _next_backtest_target_model = selected + selected_cadence = validate_backtest_cadence( + cadence or DEFAULT_BACKTEST_CADENCE + ) + _next_backtest_target_model = selected_model + _next_backtest_cadence = selected_cadence + return selected_model, selected_cadence + + +def queue_backtest_target_model(target_model: str | None) -> str: + """Compatibility wrapper for callers selecting only the target model.""" + selected, _ = queue_backtest_options(target_model, DEFAULT_BACKTEST_CADENCE) + return selected + + +def _consume_backtest_options() -> tuple[str, str]: + global _next_backtest_target_model, _next_backtest_cadence + selected = (_next_backtest_target_model, _next_backtest_cadence) + _next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL + _next_backtest_cadence = DEFAULT_BACKTEST_CADENCE return selected def _consume_backtest_target_model() -> str: - global _next_backtest_target_model - selected = _next_backtest_target_model - _next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL + """Compatibility wrapper consuming all queued one-run options.""" + selected, _ = _consume_backtest_options() return selected @@ -1028,12 +1052,13 @@ async def compute_regime_monitor() -> None: async def run_backtest_job() -> None: """Replay the price-derived engine over history and cache the report.""" job_name = "backtest" - target_model = _consume_backtest_target_model() + target_model, cadence = _consume_backtest_options() _log_event( logging.INFO, "job_start", job=job_name, target_model=target_model, + cadence=cadence, ) _runtime_start(job_name) @@ -1051,6 +1076,7 @@ async def run_backtest_job() -> None: db, _on_progress, target_model=target_model, + cadence=cadence, ) _runtime_finish( @@ -1058,6 +1084,7 @@ async def run_backtest_job() -> None: processed=report.get("tickers", 0), total=report.get("tickers", 0), message=( f"{BACKTEST_TARGET_MODELS[target_model]}: " + f"{cadence} cadence, " f"{report.get('candidates', 0)} setups, " f"{report.get('qualified', 0)} qualified" ), diff --git a/app/schemas/admin.py b/app/schemas/admin.py index 60692e4..b5a0966 100644 --- a/app/schemas/admin.py +++ b/app/schemas/admin.py @@ -46,6 +46,7 @@ class JobToggle(BaseModel): class JobTriggerRequest(BaseModel): """Optional parameters for a one-time manual job run.""" target_model: Literal["production_gtl", "structural_sr"] | None = None + cadence: Literal["weekly", "daily"] | None = None class RecommendationConfigUpdate(BaseModel): diff --git a/app/services/admin_service.py b/app/services/admin_service.py index b1c07bf..fe0cc4e 100644 --- a/app/services/admin_service.py +++ b/app/services/admin_service.py @@ -607,6 +607,7 @@ async def trigger_job( job_name: str, *, target_model: str | None = None, + cadence: str | None = None, ) -> dict[str, str]: """Trigger a manual job run via the scheduler. @@ -616,6 +617,8 @@ async def trigger_job( raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}") if target_model is not None and job_name != "backtest": raise ValidationError("target_model is supported only for the backtest job") + if cadence is not None and job_name != "backtest": + raise ValidationError("cadence is supported only for the backtest job") from app.scheduler import get_job_runtime_snapshot, scheduler @@ -643,9 +646,9 @@ async def trigger_job( return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"} if job_name == "backtest": - from app.scheduler import queue_backtest_target_model + from app.scheduler import queue_backtest_options - target_model = queue_backtest_target_model(target_model) + target_model, cadence = queue_backtest_options(target_model, cadence) job.modify(next_run_time=None) # Reset, then trigger immediately from datetime import datetime, timezone @@ -654,6 +657,8 @@ async def trigger_job( result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"} if target_model is not None: result["target_model"] = target_model + if cadence is not None: + result["cadence"] = cadence return result diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index 071706d..b808172 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -1,7 +1,7 @@ """Historical backtest (Phase 1): replay the price-derived engine over stored OHLCV and measure how the CURRENT config would have performed. -For each ticker we step through history (weekly), and at each as-of date D we +For each ticker we step through history at the selected entry cadence, and at each as-of date D we rebuild the setup using only bars ≤ D (no lookahead), then walk the actual bars after D to record the realized outcome. The report contains: @@ -100,7 +100,16 @@ logger = logging.getLogger(__name__) KEY_REPORT = "backtest_report" -STEP_DAYS = 5 # weekly cadence (≈ 5 trading days) +WEEKLY_BACKTEST_CADENCE = "weekly" +DAILY_BACKTEST_CADENCE = "daily" +DEFAULT_BACKTEST_CADENCE = WEEKLY_BACKTEST_CADENCE +BACKTEST_CADENCE_SESSIONS = { + WEEKLY_BACKTEST_CADENCE: 5, + DAILY_BACKTEST_CADENCE: 1, +} +# Compatibility alias for research scripts built around the original weekly +# replay. New code should select a cadence and call ``backtest_step_sessions``. +STEP_DAYS = BACKTEST_CADENCE_SESSIONS[WEEKLY_BACKTEST_CADENCE] MIN_LOOKBACK = 60 # bars needed before D for indicators (EMA cross needs 51) HORIZON = 30 # trading days to resolve an outcome (matches the evaluator) ATR_MULTIPLIER = 1.5 @@ -157,6 +166,30 @@ def validate_backtest_target_model(value: str) -> str: return normalized +def validate_backtest_cadence(value: str) -> str: + """Validate the supported entry-replay cadences.""" + normalized = value.strip().lower() + if normalized not in BACKTEST_CADENCE_SESSIONS: + allowed = ", ".join(BACKTEST_CADENCE_SESSIONS) + raise ValueError( + f"Unknown backtest cadence {value!r}; expected one of {allowed}" + ) + return normalized + + +def backtest_step_sessions(cadence: str) -> int: + return BACKTEST_CADENCE_SESSIONS[validate_backtest_cadence(cadence)] + + +def _ranking_period(as_of: date, cadence: str) -> tuple: + """Cross-section key for activation ranks at the selected entry cadence.""" + cadence = validate_backtest_cadence(cadence) + if cadence == DAILY_BACKTEST_CADENCE: + return ("date", as_of.toordinal()) + iso = as_of.isocalendar() + return ("week", iso[0], iso[1]) + + # --------------------------------------------------------------------------- # RESEARCH / DIAGNOSTIC FALLBACKS (retired experiments) # @@ -456,14 +489,17 @@ def _replay_ticker( activation: dict, benchmark_closes: dict[date, float] | None = None, target_model: str = PRODUCTION_GTL_TARGET_MODEL, + cadence: str = DEFAULT_BACKTEST_CADENCE, ) -> list[dict]: - """Walk one ticker's history weekly, building setups and their realized outcomes.""" + """Walk one ticker at the selected cadence and resolve each setup outcome.""" + cadence = validate_backtest_cadence(cadence) + step_sessions = backtest_step_sessions(cadence) candidates: list[dict] = [] n = len(records) if n < MIN_LOOKBACK + HORIZON: return candidates - for i in range(MIN_LOOKBACK - 1, n - HORIZON, STEP_DAYS): + for i in range(MIN_LOOKBACK - 1, n - HORIZON, step_sessions): window = records[: i + 1] forward = records[i + 1 :] forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward] @@ -512,6 +548,7 @@ def _replay_ticker( "symbol": symbol, "date": records[i].date.isoformat(), "iso_week": (iso[0], iso[1]), + "ranking_period": _ranking_period(records[i].date, cadence), "direction": s["direction"], "entry": s["entry"], "stop": s["stop"], @@ -968,6 +1005,7 @@ def _replay_and_signals( activation: dict, benchmark_closes: dict[date, float] | None = None, target_model: str = PRODUCTION_GTL_TARGET_MODEL, + cadence: str = DEFAULT_BACKTEST_CADENCE, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), @@ -987,6 +1025,7 @@ def _replay_and_signals( activation, benchmark_closes, target_model, + cadence, ), _signal_series(bars, benchmark_closes), ) @@ -999,6 +1038,7 @@ def _replay_candidates_for_period( activation: dict, benchmark_closes: dict[date, float] | None, start_date: date, + cadence: str = DEFAULT_BACKTEST_CADENCE, ) -> list[dict]: """Slim picklable replay used by local event studies. @@ -1014,8 +1054,13 @@ def _replay_candidates_for_period( date_ords, opens, highs, lows, closes, volumes ) ] + cadence = validate_backtest_cadence(cadence) candidates: list[dict] = [] - for i in range(MIN_LOOKBACK - 1, len(bars) - HORIZON, STEP_DAYS): + for i in range( + MIN_LOOKBACK - 1, + len(bars) - HORIZON, + backtest_step_sessions(cadence), + ): if bars[i].date < start_date: continue window = bars[: i + 1] @@ -1036,6 +1081,7 @@ def _replay_candidates_for_period( "symbol": symbol, "date": bars[i].date.isoformat(), "iso_week": (iso[0], iso[1]), + "ranking_period": _ranking_period(bars[i].date, cadence), "direction": "long", "entry": setup["entry"], "stop": setup["stop"], @@ -1120,14 +1166,17 @@ def _assign_signal_percentiles( value_key: str, percentile_key: str, ) -> None: - """Per ISO week, rank candidates by ``value_key`` and attach a 0-100 + """Per replay period, rank candidates by ``value_key`` and attach a 0-100 percentile under ``percentile_key`` (100 = strongest). Missing values get None and therefore cannot clear a gate based on that signal.""" - by_week: dict = defaultdict(list) + by_period: dict = defaultdict(list) for c in candidates: if c.get(value_key) is not None: - by_week[c["iso_week"]].append(c) - for group in by_week.values(): + # Hand-built/research candidates predating the cadence flag retain + # the weekly key as a compatibility fallback. + period = c.get("ranking_period") or c["iso_week"] + by_period[period].append(c) + for group in by_period.values(): ordered = sorted(group, key=lambda c: c[value_key]) n = len(ordered) for rank, c in enumerate(ordered): @@ -1137,9 +1186,9 @@ def _assign_signal_percentiles( def _assign_momentum_percentiles(candidates: list[dict]) -> None: - """Per ISO week, rank candidates by their ticker's 12-1 momentum and attach a + """Per replay period, rank candidates by 12-1 momentum and attach a 0-100 ``momentum_percentile`` (100 = highest momentum in the universe that - week). Candidates whose momentum is unknown (insufficient lookback) get None + period). Candidates whose momentum is unknown (insufficient lookback) get None and therefore can't clear a momentum gate. Mutates ``candidates``.""" _assign_signal_percentiles(candidates, "momentum", "momentum_percentile") @@ -1152,7 +1201,7 @@ def _assign_residual_momentum_percentiles(candidates: list[dict]) -> None: def _assign_low_volatility_percentiles(candidates: list[dict]) -> None: - """Per ISO week, attach volatility ranks where 100 = lowest 6-month vol.""" + """Per replay period, attach volatility ranks where 100 = lowest 6-month vol.""" _assign_signal_percentiles(candidates, "vol_6m", VOL_PERCENTILE_KEY) for c in candidates: raw = c.get(VOL_PERCENTILE_KEY) @@ -2227,6 +2276,19 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = ( "entry_variant": "residual80_highvol_blend80_20_fixed10", "exit_policy": "hold", }, + { + "strategy": "production_live_no_lockdown", + "label": "Live setup + 3x ATR trail (no re-entry lockdown)", + "description": ( + "Exact live activation, ordering, and Admin exit policy, with only " + "the post-stop re-entry lockdown disabled as the comparison baseline." + ), + "entry_variant": "residual80_highvol_blend80_20_fixed10", + "exit_policy": "atr_trail3", + "reentry_lockdown_sessions": 0, + "use_live_config": True, + "comparison_arm": "live_no_lockdown", + }, { "strategy": PRODUCTION_PORTFOLIO_STRATEGY, "label": "Production: residual/high-vol 80/20 + 3x ATR trail + 5-session lockdown", @@ -2243,6 +2305,7 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = ( # live Admin exit policy, instead of the frozen research-variant gate. "use_live_config": True, "is_production": True, + "comparison_arm": "live_lockdown_5", }, ) @@ -2603,6 +2666,7 @@ def _portfolio_monitor( "label": strategy["label"], "description": strategy["description"], "is_production": bool(strategy.get("is_production")), + "comparison_arm": strategy.get("comparison_arm"), "entry_variant": strategy["entry_variant"], "ranking_key": ranking_key, "exit_policy": exit_policy, @@ -2620,6 +2684,7 @@ def _portfolio_monitor( "label": s["label"], "description": s["description"], "is_production": bool(s.get("is_production")), + "comparison_arm": s.get("comparison_arm"), "reentry_lockdown_sessions": int( s.get("reentry_lockdown_sessions", 0) ), @@ -2641,6 +2706,46 @@ def _portfolio_monitor( } +def _production_cadence_comparison( + monitor: dict | None, + cadence: str, +) -> dict | None: + """Compact full-history live/no-lockdown vs live/5-session comparison.""" + if not monitor: + return None + arms: list[dict] = [] + for row in monitor.get("runs") or []: + comparison_arm = row.get("comparison_arm") + if not comparison_arm or row.get("lookback") != "all": + continue + compact = { + key: value + for key, value in row.items() + if key not in {"equity_curve", "benchmark_curve"} + } + arm_name = ( + "prod_live_setup" + if comparison_arm == "live_no_lockdown" + else "cooldown_5" + ) + compact["arm"] = f"{arm_name}_{cadence}" + compact["entry_cadence"] = cadence + arms.append(compact) + if not arms: + return None + arms.sort(key=lambda row: int(row.get("reentry_lockdown_sessions", 0))) + return { + "entry_cadence": cadence, + "lookback": "all", + "arms": arms, + "note": ( + "Both arms use the exact same live gate, ordering, Admin exit policy, " + "fees, and candidate cadence. Only the five-session post-stop " + "re-entry lockdown changes." + ), + } + + def _pct_loss(base: float | None, candidate: float | None) -> float | None: if base is None or candidate is None or base <= 0: return None @@ -3019,9 +3124,11 @@ async def run_backtest( progress_cb: Callable[[int, int, str], None] | None = None, *, target_model: str = PRODUCTION_GTL_TARGET_MODEL, + cadence: str = DEFAULT_BACKTEST_CADENCE, ) -> dict: """Replay every ticker and aggregate the Phase-1 reports for the current config.""" target_model = validate_backtest_target_model(target_model) + cadence = validate_backtest_cadence(cadence) config = await get_recommendation_config(db) activation = await get_activation_config(db) @@ -3030,7 +3137,9 @@ async def run_backtest( total = len(tickers) candidates: list[dict] = [] - # collected[signal_name][iso_week] -> list of (signal_value, forward_return) + # Signal IC remains a weekly, non-overlapping diagnostic regardless of the + # entry cadence. Production activation ranks are assigned from candidates + # at their own weekly or exact-date ``ranking_period`` below. collected: dict = defaultdict(lambda: defaultdict(list)) # Residual momentum needs a point-in-time benchmark return stream. Best-effort: @@ -3087,6 +3196,7 @@ async def run_backtest( pool, _replay_and_signals, ticker.symbol, columns, config, activation, benchmark_closes, target_model, + cadence, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -3109,6 +3219,7 @@ async def run_backtest( _replay_and_signals, ticker.symbol, columns, config, activation, benchmark_closes, target_model, + cadence, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -3219,7 +3330,12 @@ async def run_backtest( "candidates": len(candidates), "qualified": len(qualified), "params": { - "step_days": STEP_DAYS, + # Keep step_days for old report consumers; the value counts stored + # market sessions rather than calendar days. + "step_days": backtest_step_sessions(cadence), + "step_sessions": backtest_step_sessions(cadence), + "entry_cadence": cadence, + "signal_eval_cadence": WEEKLY_BACKTEST_CADENCE, "horizon_days": HORIZON, "min_lookback": MIN_LOOKBACK, "cost_per_side_pct": round(COST_PER_SIDE * 100, 3), @@ -3288,6 +3404,11 @@ async def run_backtest( ), }, "portfolio_monitor": portfolio_monitor_report, + "production_cadence_comparison": ( + _production_cadence_comparison(portfolio_monitor_report, cadence) + if target_model == PRODUCTION_GTL_TARGET_MODEL + else None + ), "holdout": holdout_report, "min_rr_sweep": min_rr_sweep_report, "target_model_diagnostics": _target_model_diagnostics( @@ -3323,9 +3444,15 @@ async def run_and_store( progress_cb: Callable[[int, int, str], None] | None = None, *, target_model: str = PRODUCTION_GTL_TARGET_MODEL, + cadence: str = DEFAULT_BACKTEST_CADENCE, ) -> dict: """Run the backtest and cache the report in a SystemSetting. Job entrypoint.""" - report = await run_backtest(db, progress_cb, target_model=target_model) + report = await run_backtest( + db, + progress_cb, + target_model=target_model, + cadence=cadence, + ) await update_setting(db, KEY_REPORT, json.dumps(report)) return report diff --git a/frontend/src/api/admin.ts b/frontend/src/api/admin.ts index eb1ad72..4493aa7 100644 --- a/frontend/src/api/admin.ts +++ b/frontend/src/api/admin.ts @@ -201,9 +201,11 @@ export interface TriggerJobResponse { status: 'triggered' | 'busy' | 'blocked' | 'not_found'; message: string; target_model?: BacktestTargetModel; + cadence?: BacktestCadence; } export type BacktestTargetModel = 'production_gtl' | 'structural_sr'; +export type BacktestCadence = 'weekly' | 'daily'; export function listJobs() { return apiClient.get('admin/jobs').then((r) => r.data); @@ -219,7 +221,10 @@ export function toggleJob(jobName: string, enabled: boolean) { .then((r) => r.data); } -export function triggerJob(jobName: string, options?: { target_model?: BacktestTargetModel }) { +export function triggerJob( + jobName: string, + options?: { target_model?: BacktestTargetModel; cadence?: BacktestCadence }, +) { return apiClient .post(`admin/jobs/${jobName}/trigger`, options) .then((r) => r.data); diff --git a/frontend/src/components/signals/BacktestPanel.tsx b/frontend/src/components/signals/BacktestPanel.tsx index 0b87c84..c111299 100644 --- a/frontend/src/components/signals/BacktestPanel.tsx +++ b/frontend/src/components/signals/BacktestPanel.tsx @@ -2,7 +2,7 @@ import { useMemo, useState } from 'react'; import { useMutation, useQueryClient } from '@tanstack/react-query'; import { useBacktestReport } from '../../hooks/useMarketRegime'; import { triggerJob } from '../../api/admin'; -import type { BacktestTargetModel } from '../../api/admin'; +import type { BacktestCadence, BacktestTargetModel } from '../../api/admin'; import { Button } from '../ui/Button'; import { Callout } from '../ui/Callout'; import { Disclosure } from '../ui/Disclosure'; @@ -145,6 +145,7 @@ export function BacktestPanel() { const [selectedStrategy, setSelectedStrategy] = useState(''); const [selectedLookback, setSelectedLookback] = useState(''); const [targetModel, setTargetModel] = useState('production_gtl'); + const [cadence, setCadence] = useState('weekly'); const monitor = report?.portfolio_monitor ?? null; const activeStrategy = @@ -161,11 +162,11 @@ export function BacktestPanel() { ); const run = useMutation({ - mutationFn: () => triggerJob('backtest', { target_model: targetModel }), + mutationFn: () => triggerJob('backtest', { target_model: targetModel, cadence }), onSuccess: (res) => { if (res.status === 'triggered') { const label = targetModel === 'production_gtl' ? 'Live GTL' : 'Structural S/R comparison'; - toast.addToast('success', `${label} backtest started — results appear when it finishes.`); + toast.addToast('success', `${label} ${cadence} backtest started — results appear when it finishes.`); setTimeout(() => queryClient.invalidateQueries({ queryKey: ['backtest-report'] }), 8000); } else { toast.addToast('info', res.message || 'Could not start backtest'); @@ -180,7 +181,7 @@ export function BacktestPanel() {

- The backtest replays the current config weekly through history — at each point the setup is + The backtest replays the current config at the selected cadence — at each point the setup is rebuilt using only data up to that day (no lookahead) and the following ~30 trading days decide its outcome — then simulates one capital-constrained book against the S&P 500. Sentiment and fundamentals are held neutral (no point-in-time history). ~6 months is roughly one market regime, @@ -238,6 +239,56 @@ export function BacktestPanel() { +

+ + Entry cadence + + + +
@@ -257,7 +308,8 @@ export function BacktestPanel() { <>

Ran {timeAgo(report.generated_at)} · {report.tickers} tickers · {report.candidates} setups - ({report.qualified} qualified) · weekly cadence, {report.params.horizon_days}-day horizon + ({report.qualified} qualified) · {report.params.entry_cadence ?? 'weekly'} cadence, + {' '}{report.params.horizon_days}-day horizon {report.params.cost_per_side_pct != null && ( <> · net of {report.params.cost_per_side_pct}%/side costs )} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 7a771c6..beba854 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -356,6 +356,7 @@ export interface BacktestPortfolioMonitorRun extends BacktestPortfolioPolicy { label: string; description: string; is_production: boolean; + comparison_arm?: 'live_no_lockdown' | 'live_lockdown_5' | null; entry_variant: string; exit_policy: string; reentry_lockdown_sessions?: number; @@ -370,6 +371,7 @@ export interface BacktestPortfolioMonitor { label: string; description: string; is_production: boolean; + comparison_arm?: 'live_no_lockdown' | 'live_lockdown_5' | null; reentry_lockdown_sessions?: number; }[]; lookbacks: { lookback: string; label: string }[]; @@ -404,6 +406,9 @@ export interface BacktestReport { qualified: number; params: { step_days: number; + step_sessions?: number; + entry_cadence?: 'weekly' | 'daily'; + signal_eval_cadence?: 'weekly'; horizon_days: number; min_lookback: number; cost_per_side_pct?: number; diff --git a/scripts/run_backtest_cadence_comparison.py b/scripts/run_backtest_cadence_comparison.py new file mode 100644 index 0000000..4771cb7 --- /dev/null +++ b/scripts/run_backtest_cadence_comparison.py @@ -0,0 +1,179 @@ +"""Run the four production cadence/lockdown arms on one offline snapshot. + +The command executes the complete backtest once weekly and once daily. Each +backtest contains two otherwise identical live-policy portfolio arms: no +post-stop lockdown and the production five-session lockdown. It writes both +full reports plus one compact four-arm comparison report. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from datetime import datetime +from pathlib import Path + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "snapshot", + help="SQLite snapshot created by scripts/create_backtest_snapshot.py.", + ) + parser.add_argument( + "--out-dir", + default="reports", + help="Directory for the weekly, daily, and comparison JSON reports.", + ) + parser.add_argument( + "--prefix", + default=None, + help="Output prefix. Defaults to backtest-cadence-.", + ) + parser.add_argument( + "--workers", + type=int, + default=None, + help="Override worker count; on a powerful offline PC use CPU count minus one.", + ) + parser.add_argument( + "--allow-spawn", + action="store_true", + help="Enable multiprocessing spawn for the offline Windows run.", + ) + parser.add_argument("--quiet", action="store_true", help="Hide ticker progress.") + return parser.parse_args() + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _comparison_arms(report: dict) -> list[dict]: + comparison = report.get("production_cadence_comparison") or {} + arms = list(comparison.get("arms") or []) + if len(arms) != 2: + cadence = (report.get("params") or {}).get("entry_cadence", "unknown") + raise RuntimeError( + f"Expected two live comparison arms for {cadence}; found {len(arms)}" + ) + return arms + + +def _print_arm(row: dict) -> None: + print( + f" {row['arm']}: Sharpe {row.get('sharpe')}, " + f"CAGR {row.get('cagr_pct')}%, DD {row.get('max_drawdown_pct')}%, " + f"trades {row.get('trades')}, skipped cooldown {row.get('skipped_cooldown', 0)}" + ) + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Snapshot not found: {snapshot}") + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + from app.config import settings + from app.services.backtest_service import run_backtest + + if args.workers is not None: + settings.backtest_workers = args.workers + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + prefix = args.prefix or f"backtest-cadence-{datetime.now():%Y%m%d-%H%M%S}" + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + reports: dict[str, dict] = {} + try: + async with Session() as db: + for cadence in ("weekly", "daily"): + last_progress: tuple[int, int] | None = None + + def progress(done: int, total: int, symbol: str) -> None: + nonlocal last_progress + if args.quiet or last_progress == (done, total): + return + last_progress = (done, total) + label = f" {symbol}" if symbol else "" + print( + f"{cadence} progress: {done}/{total}{label}", + end="\r", + ) + + reports[cadence] = await run_backtest( + db, + progress_cb=progress, + target_model="production_gtl", + cadence=cadence, + ) + if not args.quiet: + print("") + _write_json(out_dir / f"{prefix}-{cadence}.json", reports[cadence]) + finally: + await engine.dispose() + + arms = [ + *_comparison_arms(reports["weekly"]), + *_comparison_arms(reports["daily"]), + ] + expected = { + "prod_live_setup_weekly", + "prod_live_setup_daily", + "cooldown_5_weekly", + "cooldown_5_daily", + } + if {row.get("arm") for row in arms} != expected: + raise RuntimeError("The generated cadence report does not contain all four arms") + arm_order = { + "prod_live_setup_weekly": 0, + "prod_live_setup_daily": 1, + "cooldown_5_weekly": 2, + "cooldown_5_daily": 3, + } + arms.sort(key=lambda row: arm_order[str(row["arm"])]) + + comparison = { + "generated_at": datetime.now().astimezone().isoformat(), + "snapshot": str(snapshot.resolve()), + "target_model": "production_gtl", + "arms": arms, + "full_reports": { + cadence: str((out_dir / f"{prefix}-{cadence}.json").resolve()) + for cadence in ("weekly", "daily") + }, + "note": ( + "All four arms use the same snapshot, activation settings, target model, " + "live Admin exit policy, fees, sizing, and portfolio constraints. Within " + "each cadence pair, only the five-session post-stop lockdown differs." + ), + } + comparison_path = out_dir / f"{prefix}-comparison.json" + _write_json(comparison_path, comparison) + + print(f"Comparison written: {comparison_path}") + for row in arms: + _print_arm(row) + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_backtest_snapshot.py b/scripts/run_backtest_snapshot.py index 458b5db..b78ecf8 100644 --- a/scripts/run_backtest_snapshot.py +++ b/scripts/run_backtest_snapshot.py @@ -32,7 +32,10 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--out", default=None, - help="JSON report path. Defaults to reports/backtest-.json.", + help=( + "JSON report path. Defaults to " + "reports/backtest--.json." + ), ) parser.add_argument( "--workers", @@ -55,6 +58,15 @@ def _parse_args() -> argparse.Namespace: "structural_sr is a comparison-only chart-S/R model." ), ) + parser.add_argument( + "--cadence", + choices=("weekly", "daily"), + default="weekly", + help=( + "Entry replay cadence. Weekly is the resource-safe production default; " + "daily performs roughly five times as many setup evaluations." + ), + ) parser.add_argument( "--holdout-split", default=None, @@ -63,9 +75,9 @@ def _parse_args() -> argparse.Namespace: return parser.parse_args() -def _default_output_path() -> Path: +def _default_output_path(cadence: str) -> Path: stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - return Path("reports") / f"backtest-{stamp}.json" + return Path("reports") / f"backtest-{cadence}-{stamp}.json" def _pct(value: Any) -> str: @@ -93,6 +105,7 @@ def _print_summary(report: dict) -> None: print("") print("Backtest summary") + print(f" entry cadence: {(report.get('params') or {}).get('entry_cadence', 'weekly')}") print(f" candidates: {report.get('candidates')}") print(f" qualified: {report.get('qualified')}") print(f" all setups net avg R: {_r(all_setups.get('net_avg_r'))}") @@ -183,7 +196,7 @@ async def _main() -> None: if args.workers is not None: settings.backtest_workers = args.workers - output = Path(args.out) if args.out else _default_output_path() + output = Path(args.out) if args.out else _default_output_path(args.cadence) output.parent.mkdir(parents=True, exist_ok=True) engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) @@ -208,6 +221,7 @@ async def _main() -> None: db, progress_cb=progress, target_model=args.target_model, + cadence=args.cadence, ) finally: await engine.dispose() diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index 1cf5337..4ff0784 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -658,6 +658,36 @@ class TestSimulatePortfolio: for row in comparison_rows ) + def test_production_cadence_comparison_names_exact_two_arms(self): + monitor = { + "runs": [ + { + "comparison_arm": "live_no_lockdown", + "lookback": "all", + "reentry_lockdown_sessions": 0, + "trades": 10, + "equity_curve": [{"date": "2026-01-01", "value": 1.0}], + }, + { + "comparison_arm": "live_lockdown_5", + "lookback": "all", + "reentry_lockdown_sessions": 5, + "trades": 8, + "benchmark_curve": [{"date": "2026-01-01", "value": 1.0}], + }, + ] + } + + comparison = bt._production_cadence_comparison(monitor, "daily") + + assert comparison is not None + assert [row["arm"] for row in comparison["arms"]] == [ + "prod_live_setup_daily", + "cooldown_5_daily", + ] + assert all("equity_curve" not in row for row in comparison["arms"]) + assert all("benchmark_curve" not in row for row in comparison["arms"]) + def test_initial_stop_can_refresh_lower_and_survive_same_bar(self): closes = [100.0, 94.0, 96.0] prices = {"AAA": _sim_prices(self.ORD, closes)} @@ -929,6 +959,15 @@ def test_backtest_target_model_is_small_and_validated(): bt.validate_backtest_target_model("legacy_range_grid_touch") +def test_backtest_cadence_is_small_validated_and_session_based(): + assert bt.validate_backtest_cadence(" WEEKLY ") == "weekly" + assert bt.validate_backtest_cadence("daily") == "daily" + assert bt.backtest_step_sessions("weekly") == 5 + assert bt.backtest_step_sessions("daily") == 1 + with pytest.raises(ValueError, match="Unknown backtest cadence"): + bt.validate_backtest_cadence("monthly") + + def _flat_window_records(): return [ SimpleNamespace( @@ -1022,6 +1061,46 @@ def test_replay_ticker_candidates_carry_gate_fields(): assert c.get("action") is not None assert "risk_level" in c assert c["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL + assert c["ranking_period"][0] == "week" + + daily_cands = bt._replay_ticker( + "OSC", + bars, + dict(DEFAULT_RECOMMENDATION_CONFIG), + dict(ACTIVATION_DEFAULTS), + cadence="daily", + ) + assert len(daily_cands) > len(cands) + assert all(c["ranking_period"][0] == "date" for c in daily_cands) + + +def test_daily_replay_uses_exact_date_ranking_periods(): + candidates = [ + { + "iso_week": (2026, 1), + "ranking_period": ("date", date(2026, 1, 5).toordinal()), + "momentum": 0.10, + }, + { + "iso_week": (2026, 1), + "ranking_period": ("date", date(2026, 1, 5).toordinal()), + "momentum": 0.20, + }, + { + "iso_week": (2026, 1), + "ranking_period": ("date", date(2026, 1, 6).toordinal()), + "momentum": 0.90, + }, + { + "iso_week": (2026, 1), + "ranking_period": ("date", date(2026, 1, 6).toordinal()), + "momentum": 0.30, + }, + ] + + bt._assign_momentum_percentiles(candidates) + + assert [row["momentum_percentile"] for row in candidates] == [0.0, 100.0, 100.0, 0.0] async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None: @@ -1063,6 +1142,8 @@ async def test_run_backtest_smoke(session): assert report["params"]["cost_per_side_pct"] == pytest.approx(bt.COST_PER_SIDE * 100) assert report["params"]["target_model"] == bt.PRODUCTION_GTL_TARGET_MODEL assert report["params"]["is_production_target_model"] is True + assert report["params"]["entry_cadence"] == "weekly" + assert report["params"]["step_sessions"] == 5 assert ( report["params"]["production_reentry_lockdown_sessions"] == bt.REENTRY_LOCKDOWN_SESSIONS @@ -1073,6 +1154,11 @@ async def test_run_backtest_smoke(session): # carries the hold-to-horizon grading alongside the target model ablation = {r["variant"]: r for r in report["gate_ablation"]} assert ablation["all_floors"]["total"] == report["overall_qualified"]["total"] + + daily_report = await bt.run_backtest(session, cadence="daily") + assert daily_report["params"]["entry_cadence"] == "daily" + assert daily_report["params"]["step_sessions"] == 1 + assert daily_report["candidates"] > report["candidates"] for row in report["gate_ablation"]: assert "hold_net_avg_r" in row diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 6f154de..494feb6 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -3,11 +3,13 @@ import pytest from app.scheduler import ( + _consume_backtest_options, _consume_backtest_target_model, _parse_frequency, _resume_tickers, _last_successful, configure_scheduler, + queue_backtest_options, queue_backtest_target_model, scheduler, ) @@ -24,6 +26,15 @@ def test_manual_backtest_target_model_rejects_removed_research_arms(): queue_backtest_target_model("production_control") +def test_manual_backtest_options_are_one_shot_and_default_back_to_weekly(): + assert queue_backtest_options("structural_sr", "daily") == ( + "structural_sr", + "daily", + ) + assert _consume_backtest_options() == ("structural_sr", "daily") + assert _consume_backtest_options() == ("production_gtl", "weekly") + + class TestParseFrequency: def test_hourly(self): assert _parse_frequency("hourly") == {"hours": 1}