dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: portfolio simulation + per-trade stats (gaps, hold time, best/worst)
Deploy / lint (push) Successful in 6s
Details
Deploy / test (push) Successful in 55s
Details
Deploy / deploy (push) Successful in 38s
Details

Browse Source 
Per-trade additions to the report:
- Gap-through-stop fills: stops now fill at the worse of the stop or the
  bar's open across every exit model (target, TP, trailing, time), so a
  loss can exceed -1R; targets never fill better than their level.
- best_r / worst_r, avg holding days, and net R per day of capital
  deployed on the summary buckets and the time-exit sweep.

Portfolio simulation (the stats a per-setup replay cannot give):
- One capital-constrained book over the qualified setups: 10k start, max
  10 concurrent positions (one per ticker, best momentum first), 1%
  fixed-fractional risk with a 20% no-leverage notional cap, entries at
  the detection close, 0.1%/side costs, daily mark-to-market.
- Two exit policies compared: S/R target race vs hold-to-horizon.
- Equity-curve stats: final equity, total return, CAGR, max drawdown,
  annualized daily Sharpe, win rate, avg P&L, best/worst trade, avg
  hold, entries skipped on a full book, and SPY price return over the
  same window (benchmark history refreshed to cover the replay span).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Dennis Thiessen
2026-07-02 11:56:29 +02:00
parent 942a22ce65
commit 0f43e755f4
4 changed files with 634 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/backtest_service.py
+356 -22
View File
@@ -216,12 +216,27 @@ def _window_setups(
return out
def _stop_fill_r(direction: str, entry: float, stop: float, bar) -> float:
"""Realized R when the stop is hit on ``bar``: filled at the stop, or at the
bar's open when price gapped through it — so a gap can lose more than −1R,
matching real fills. Targets are never filled better than their level, so
gap modeling only ever makes results more conservative."""
risk = abs(entry - stop)
if risk <= 0 or entry <= 0:
return -1.0
if direction == "long":
fill = min(stop, bar.open)
return (fill - entry) / risk
fill = max(stop, bar.open)
return (entry - fill) / risk
def _tp_primitives(
direction: str, entry: float, stop: float, forward: list, horizon: int
) -> tuple[float, bool, float, float]:
) -> tuple[float, bool, float, float, int | None, float]:
"""Primitives for the take-profit exit model, from the bars after detection.
Returns ``(risk_pct, stopped, mfe_pct, close_pct)``:
Returns ``(risk_pct, stopped, mfe_pct, close_pct, stop_day, stop_r)``:
- ``risk_pct`` fraction from entry to stop (the 1R distance)
- ``stopped`` whether the stop was hit within the horizon
- ``mfe_pct`` best favourable excursion (fraction) reachable *before* the
@@ -229,27 +244,34 @@ def _tp_primitives(
counts as a loss (matching the conservative target model);
over the whole horizon if the stop is never hit
- ``close_pct`` directional return at the horizon-end close (the timeout exit)
- ``stop_day`` 1-based trading day the stop was pierced, None if never
- ``stop_r`` realized R at the stop fill (≤ −1 when the bar gapped
through the stop — see _stop_fill_r); −1.0 when unused
From these any fixed take-profit level can be scored without re-walking bars:
tp reached before stop (``mfe_pct >= tp``) → +tp; else stop → −1R; else the
horizon-close move.
tp reached before stop (``mfe_pct >= tp``) → +tp; else stop → ``stop_r``;
else the horizon-close move.
"""
long = direction == "long"
risk_pct = abs(entry - stop) / entry if entry else 0.0
bars = forward[:horizon]
if not bars:
return risk_pct, False, 0.0, 0.0
return risk_pct, False, 0.0, 0.0, None, -1.0
mfe = 0.0
stopped = False
for r in bars:
stop_day: int | None = None
stop_r = -1.0
for i, r in enumerate(bars):
if (r.low <= stop) if long else (r.high >= stop):
stopped = True
stop_day = i + 1
stop_r = _stop_fill_r(direction, entry, stop, r)
break
fav = (r.high - entry) / entry if long else (entry - r.low) / entry
if fav > mfe:
mfe = fav
close_pct = ((bars[-1].close - entry) / entry) * (1.0 if long else -1.0)
return risk_pct, stopped, mfe, close_pct
return risk_pct, stopped, mfe, close_pct, stop_day, stop_r
def _trailing_exits(
@@ -281,12 +303,14 @@ def _trailing_exits(
if long:
stop_level = max(init_stop, peak * (1 - f))
if r.low <= stop_level:
result[round(f * 100)] = ((stop_level - entry) / entry) / risk
fill = min(stop_level, r.open) # gap through fills at the open
result[round(f * 100)] = ((fill - entry) / entry) / risk
continue
else:
stop_level = min(init_stop, peak * (1 + f))
if r.high >= stop_level:
result[round(f * 100)] = ((entry - stop_level) / entry) / risk
fill = max(stop_level, r.open)
result[round(f * 100)] = ((entry - fill) / entry) / risk
continue
remaining.append(f)
active = remaining
@@ -325,10 +349,12 @@ def _time_exits(
return {int(n): 0.0 for n in horizons}
stop_day: int | None = None # 1-based trading day the stop was pierced
stop_r = -1.0
closes: list[float] = []
for i, r in enumerate(bars):
if (r.low <= stop) if long else (r.high >= stop):
stop_day = i + 1
stop_r = _stop_fill_r(direction, entry, stop, r)
break
closes.append(r.close)
@@ -336,7 +362,7 @@ def _time_exits(
for h in horizons:
n = int(h)
if stop_day is not None and stop_day <= n:
result[n] = -1.0
result[n] = stop_r
else:
# closes can't be empty here: an empty closes means the stop hit on
# day 1, which the branch above catches for every n >= 1.
@@ -359,21 +385,29 @@ def _replay_ticker(symbol: str, records: list, config: dict, activation: dict) -
forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
for s in _window_setups(window, config, activation):
outcome, _ = evaluate_setup_against_bars(
outcome, outcome_date = evaluate_setup_against_bars(
s["direction"], s["stop"], s["target"], forward_bars, HORIZON
)
if outcome is None:
continue
# Trading days from detection to resolution (expired = full horizon).
hold_days = next(
(idx + 1 for idx, r in enumerate(forward[:HORIZON]) if r.date == outcome_date),
min(HORIZON, len(forward)),
)
target_hit = outcome == OUTCOME_TARGET_HIT
if outcome == OUTCOME_TARGET_HIT:
realized_r = s["rr"]
elif outcome in (OUTCOME_STOP_HIT, OUTCOME_AMBIGUOUS):
realized_r = -1.0
# Fill at the stop, or at the open when the bar gapped through it.
realized_r = _stop_fill_r(
s["direction"], s["entry"], s["stop"], forward[hold_days - 1]
)
else: # expired
realized_r = 0.0
# Take-profit exit primitives (parallel to the target-vs-stop outcome
# above; aggregated separately into the take-profit sweep).
risk_pct, tp_stopped, mfe_pct, tp_close_pct = _tp_primitives(
risk_pct, tp_stopped, mfe_pct, tp_close_pct, stop_day, tp_stop_r = _tp_primitives(
s["direction"], s["entry"], s["stop"], forward, HORIZON
)
trail_r = _trailing_exits(
@@ -388,6 +422,9 @@ def _replay_ticker(symbol: str, records: list, config: dict, activation: dict) -
"date": records[i].date.isoformat(),
"iso_week": (iso[0], iso[1]),
"direction": s["direction"],
"entry": s["entry"],
"stop": s["stop"],
"target": s["target"],
"rr": s["rr"],
"confidence": s["confidence"],
"primary_prob": s["primary_prob"],
@@ -401,8 +438,11 @@ def _replay_ticker(symbol: str, records: list, config: dict, activation: dict) -
"outcome": outcome,
"target_hit": target_hit,
"realized_r": realized_r,
"hold_days": hold_days,
"stop_day": stop_day,
"risk_pct": risk_pct,
"tp_stopped": tp_stopped,
"tp_stop_r": tp_stop_r,
"mfe_pct": mfe_pct,
"tp_close_pct": tp_close_pct,
"trail_r": trail_r,
@@ -418,6 +458,9 @@ def _bucket_stats(cands: list[dict]) -> dict:
decided = wins + losses
rs = [c["realized_r"] for c in cands]
net_rs = [c["realized_r"] - _cost_r(c) for c in cands]
holds = [c["hold_days"] for c in cands if c.get("hold_days")]
avg_hold = sum(holds) / len(holds) if holds else None
net_avg = sum(net_rs) / len(net_rs) if net_rs else None
return {
"total": len(cands),
"wins": wins,
@@ -426,8 +469,15 @@ def _bucket_stats(cands: list[dict]) -> dict:
"hit_rate": round(wins / decided * 100, 1) if decided else None,
"avg_r": round(sum(rs) / len(rs), 3) if rs else None,
"total_r": round(sum(rs), 2) if rs else None,
"net_avg_r": round(sum(net_rs) / len(net_rs), 3) if net_rs else None,
"net_avg_r": round(net_avg, 3) if net_avg is not None else None,
"net_total_r": round(sum(net_rs), 2) if net_rs else None,
"best_r": round(max(rs), 2) if rs else None,
"worst_r": round(min(rs), 2) if rs else None,
"avg_hold_days": round(avg_hold, 1) if avg_hold is not None else None,
# Capital efficiency: net expectancy per trading day the capital is tied up.
"net_r_per_day": (
round(net_avg / avg_hold, 4) if net_avg is not None and avg_hold else None
),
}
@@ -473,7 +523,7 @@ def _take_profit_bucket(cands: list[dict], tp: float) -> dict:
r = tp / risk
wins += 1
elif c.get("tp_stopped"):
r = -1.0
r = c.get("tp_stop_r", -1.0) # gap-aware stop fill, ≤ −1R
else:
r = (c.get("tp_close_pct", 0.0)) / risk
rs.append(r)
@@ -519,16 +569,24 @@ def _trailing_bucket(cands: list[dict], trail_pct: int) -> dict:
def _time_exit_bucket(cands: list[dict], hold_days: int) -> dict:
"""Stats for the hold-``hold_days`` exit: initial stop active, otherwise out
at the day-N close. Each candidate carries its realized R per hold length in
``time_r``; a "win" is an exit in profit (R > 0)."""
pairs = [
(c["time_r"][hold_days], _cost_r(c))
``time_r``; a "win" is an exit in profit (R > 0). The realized hold is the
full N days unless the stop cut it short (``stop_day``)."""
rows = [
(
c["time_r"][hold_days],
_cost_r(c),
min(hold_days, c.get("stop_day") or hold_days),
)
for c in cands
if c.get("time_r", {}).get(hold_days) is not None
]
total = len(pairs)
rs = [r for r, _ in pairs]
net_rs = [r - cost for r, cost in pairs]
total = len(rows)
rs = [r for r, _, _ in rows]
net_rs = [r - cost for r, cost, _ in rows]
holds = [h for _, _, h in rows]
wins = sum(1 for r in rs if r > 0)
avg_hold = sum(holds) / total if total else None
net_avg = sum(net_rs) / total if total else None
return {
"hold_days": hold_days,
"total": total,
@@ -536,8 +594,14 @@ def _time_exit_bucket(cands: list[dict], hold_days: int) -> dict:
"win_rate": round(wins / total * 100, 1) if total else None,
"avg_r": round(sum(rs) / total, 3) if total else None,
"total_r": round(sum(rs), 2) if total else None,
"net_avg_r": round(sum(net_rs) / total, 3) if total else None,
"net_avg_r": round(net_avg, 3) if net_avg is not None else None,
"net_total_r": round(sum(net_rs), 2) if total else None,
"best_r": round(max(rs), 2) if rs else None,
"worst_r": round(min(rs), 2) if rs else None,
"avg_hold_days": round(avg_hold, 1) if avg_hold is not None else None,
"net_r_per_day": (
round(net_avg / avg_hold, 4) if net_avg is not None and avg_hold else None
),
}
@@ -934,6 +998,214 @@ def _gate_ablation(candidates: list[dict], activation: dict, threshold: float) -
return rows
# ---------------------------------------------------------------------------
# Portfolio simulation
# ---------------------------------------------------------------------------
# Book parameters: fixed starting capital, a capped number of concurrent
# positions (one per ticker), fixed-fractional risk sizing with a no-leverage
# notional cap, and the same per-side cost as the per-trade tables. Entries are
# the QUALIFIED setups at their detection close, best momentum first while
# slots and cash allow.
SIM_STARTING_CAPITAL = 10_000.0
SIM_MAX_POSITIONS = 10
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
def _simulate_portfolio(
candidates: list[dict],
prices: dict[str, tuple],
spy_closes: dict | None,
exit_policy: str,
hold_days: int,
) -> dict | None:
"""Replay the qualified setups as ONE capital-constrained book and report
portfolio economics from the daily equity curve (return, CAGR, drawdown,
Sharpe) — the numbers the per-setup tables cannot give, because they grade
every setup as if capital were infinite.
``exit_policy``: "target" races the S/R target against the stop with a
timeout at ``hold_days``; "hold" keeps only the initial stop and exits at
the ``hold_days``-th close. Stops fill at the worse of stop or open (gaps
modeled); positions still open at the end are closed at their last mark.
Returns None when there is nothing to trade.
"""
entries_by_ord: dict[int, list[dict]] = defaultdict(list)
for c in candidates:
if not c.get("qualified") or c.get("direction") != "long":
continue
if not c.get("entry") or not c.get("stop"):
continue
entries_by_ord[date.fromisoformat(c["date"]).toordinal()].append(c)
if not entries_by_ord:
return None
# Per-symbol bar lookup: date ordinal -> index into the column arrays.
index_of: dict[str, dict[int, int]] = {
sym: {o: i for i, o in enumerate(cols[0])} for sym, cols in prices.items()
}
first_ord = min(entries_by_ord)
calendar = sorted({o for cols in prices.values() for o in cols[0] if o >= first_ord})
if not calendar:
return None
cash = SIM_STARTING_CAPITAL
positions: dict[str, dict] = {}
curve: list[tuple[int, float]] = []
trades: list[dict] = []
skipped_full = 0
def _bar(sym: str, o: int):
idx = index_of.get(sym, {}).get(o)
if idx is None:
return None
cols = prices[sym]
return SimpleNamespace(
open=cols[1][idx], high=cols[2][idx], low=cols[3][idx], close=cols[4][idx]
)
def _close_trade(sym: str, fill: float, reason: str) -> None:
nonlocal cash
pos = positions.pop(sym)
proceeds = pos["shares"] * fill
cost = proceeds * COST_PER_SIDE
cash += proceeds - cost
risk = pos["entry"] - pos["stop"]
trades.append({
"pnl": proceeds - pos["shares"] * pos["entry"] - cost - pos["entry_cost"],
"r": (fill - pos["entry"]) / risk if risk > 0 else 0.0,
"hold": pos["bars_held"],
"reason": reason,
})
def _marked_equity() -> float:
return cash + sum(p["shares"] * p["last_close"] for p in positions.values())
for o in calendar:
# 1) exits on today's bars (stop intraday, target intraday, time at close)
for sym in list(positions):
pos = positions[sym]
bar = _bar(sym, o)
if bar is None:
continue
pos["bars_held"] += 1
pos["last_close"] = bar.close
if bar.low <= pos["stop"]:
# Same-bar stop+target resolves as the loss (conservative, like
# the evaluator); gap through the stop fills at the open.
_close_trade(sym, min(pos["stop"], bar.open), "stop")
continue
if exit_policy == "target" and pos["target"] and bar.high >= pos["target"]:
_close_trade(sym, pos["target"], "target")
continue
if pos["bars_held"] >= hold_days:
_close_trade(sym, bar.close, "time")
# 2) entries at today's close, best momentum first
equity = _marked_equity()
todays = sorted(
entries_by_ord.get(o, ()),
key=lambda c: c.get("momentum_percentile") or 0.0,
reverse=True,
)
for c in todays:
sym = c["symbol"]
if sym in positions:
continue
if len(positions) >= SIM_MAX_POSITIONS:
skipped_full += 1
continue
entry, stop = float(c["entry"]), float(c["stop"])
risk_ps = entry - stop
if risk_ps <= 0 or entry <= 0:
continue
shares = min(
(equity * SIM_RISK_PER_TRADE) / risk_ps,
(equity * SIM_NOTIONAL_CAP) / entry,
max(cash, 0.0) / (entry * (1.0 + COST_PER_SIDE)),
)
if shares * entry < 1.0: # can't fund a meaningful position
continue
entry_cost = shares * entry * COST_PER_SIDE
cash -= shares * entry + entry_cost
positions[sym] = {
"shares": shares,
"entry": entry,
"stop": stop,
"target": float(c["target"]) if c.get("target") else None,
"entry_cost": entry_cost,
"bars_held": 0,
"last_close": entry,
}
equity = _marked_equity()
curve.append((o, _marked_equity()))
# Close whatever is still open at its last mark so final equity is realized.
for sym in list(positions):
_close_trade(sym, positions[sym]["last_close"], "open_at_end")
final_equity = cash
curve[-1] = (calendar[-1], final_equity)
total_return_pct = (final_equity / SIM_STARTING_CAPITAL - 1.0) * 100.0
years = (calendar[-1] - calendar[0]) / 365.25
cagr_pct = (
((final_equity / SIM_STARTING_CAPITAL) ** (1.0 / years) - 1.0) * 100.0
if years > 0.25 and final_equity > 0
else None
)
peak = float("-inf")
max_dd = 0.0
for _, eq in curve:
peak = max(peak, eq)
if peak > 0:
max_dd = max(max_dd, (peak - eq) / peak)
rets = [b / a - 1.0 for (_, a), (_, b) in zip(curve, curve[1:]) if a > 0]
sharpe = None
if len(rets) > 2:
mean = sum(rets) / len(rets)
var = sum((x - mean) ** 2 for x in rets) / (len(rets) - 1)
if var > 0:
sharpe = mean / math.sqrt(var) * math.sqrt(252)
pnls = [t["pnl"] for t in trades]
wins = sum(1 for p in pnls if p > 0)
spy_pct = None
if spy_closes:
from app.services.benchmark_service import benchmark_return_pct
spy_pct = benchmark_return_pct(
spy_closes, date.fromordinal(calendar[0]), date.fromordinal(calendar[-1])
)
return {
"starting_capital": SIM_STARTING_CAPITAL,
"final_equity": round(final_equity, 2),
"total_return_pct": round(total_return_pct, 1),
"cagr_pct": round(cagr_pct, 1) if cagr_pct is not None else None,
"max_drawdown_pct": round(max_dd * 100.0, 1),
"sharpe": round(sharpe, 2) if sharpe is not None else None,
"trades": len(trades),
"win_rate": round(wins / len(trades) * 100.0, 1) if trades else None,
"avg_trade_pnl": round(sum(pnls) / len(pnls), 2) if pnls else None,
"best_trade_r": round(max(t["r"] for t in trades), 2) if trades else None,
"worst_trade_r": round(min(t["r"] for t in trades), 2) if trades else None,
"best_trade_pnl": round(max(pnls), 2) if pnls else None,
"worst_trade_pnl": round(min(pnls), 2) if pnls else None,
"avg_hold_days": (
round(sum(t["hold"] for t in trades) / len(trades), 1) if trades else None
),
"skipped_book_full": skipped_full,
"spy_return_pct": round(spy_pct, 1) if spy_pct is not None else None,
"start_date": date.fromordinal(calendar[0]).isoformat(),
"end_date": date.fromordinal(calendar[-1]).isoformat(),
}
async def run_backtest(
db: AsyncSession,
progress_cb: Callable[[int, int, str], None] | None = None,
@@ -1037,6 +1309,43 @@ async def run_backtest(
cands = [c for c in candidates if _momentum_qualifies(c, threshold)]
sweep.append({"min_momentum_percentile": threshold, **_bucket_stats(cands)})
# Portfolio simulation: re-fetch bars for just the qualified symbols (memory-
# light vs retaining every ticker's columns through the replay) and replay
# the book once per exit policy. Best-effort — the report stands without it.
hold_horizon = max(TIME_EXIT_DAYS)
sim_policies: list[dict] = []
try:
qual_symbols = sorted({c["symbol"] for c in candidates if c.get("qualified")})
price_columns: dict[str, tuple] = {}
for sym in qual_symbols:
cols = await _fetch_columns(db, sym)
if cols is not None:
price_columns[sym] = cols
spy_closes: dict | None = None
try:
from app.services.benchmark_service import (
load_benchmark_closes,
refresh_benchmark_prices,
)
oldest = min((cols[0][0] for cols in price_columns.values()), default=None)
if oldest is not None:
days_needed = (date.today() - date.fromordinal(oldest)).days + 30
await refresh_benchmark_prices(db, days=days_needed)
spy_closes = await load_benchmark_closes(db)
except Exception:
logger.exception("Benchmark load for the portfolio sim failed")
for policy in ("target", "hold"):
sim = _simulate_portfolio(
candidates, price_columns, spy_closes, policy, hold_horizon
)
if sim is not None:
sim_policies.append({"policy": policy, **sim})
except Exception:
logger.exception("Portfolio simulation failed")
return {
"generated_at": datetime.now(timezone.utc).isoformat(),
"tickers": total,
@@ -1070,6 +1379,28 @@ async def run_backtest(
"take_profit_sweep": [_take_profit_bucket(qualified, tp) for tp in TP_LEVELS],
"trailing_sweep": [_trailing_bucket(qualified, round(f * 100)) for f in TRAIL_LEVELS],
"time_exit_sweep": [_time_exit_bucket(qualified, n) for n in TIME_EXIT_DAYS],
"portfolio_sim": {
"params": {
"starting_capital": SIM_STARTING_CAPITAL,
"max_positions": SIM_MAX_POSITIONS,
"risk_per_trade_pct": round(SIM_RISK_PER_TRADE * 100, 2),
"notional_cap_pct": round(SIM_NOTIONAL_CAP * 100, 1),
"cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
"hold_days": hold_horizon,
},
"policies": sim_policies,
"note": (
"One capital-constrained book over the same qualified setups the "
"tables above grade per-setup: at most "
f"{SIM_MAX_POSITIONS} concurrent positions (one per ticker), best "
"momentum first, fixed-fractional risk sizing with a no-leverage "
"cap, entries at the detection close, stops filled at the worse "
"of stop or open. 'target' races the S/R target against the stop "
"(timeout at the horizon); 'hold' keeps the initial stop and "
"exits at the horizon close. SPY return is price-only over the "
"same window. In-sample; no dividends."
),
},
"calibration": _calibration(candidates),
"signal_eval": _signal_evaluation(collected),
"signal_eval_note": (
@@ -1084,6 +1415,9 @@ async def run_backtest(
),
"note": (
"Sentiment & fundamentals held neutral (no point-in-time history). "
"Stops fill at the worse of the stop or the bar's open (gaps through "
"the stop are modeled, so a loss can exceed −1R); targets never fill "
"better than their level. "
"~6 months ≈ one market regime — treat as directional, not gospel."
),
}