From 442dc3f04b239eb413f12660c59af17cb571f693 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Tue, 11 Aug 2026 22:09:10 +0200 Subject: [PATCH] feat(backtest): add Sortino, Gain-to-Pain and dollar profit factor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three portfolio metrics computed where their inputs already live in _simulate_portfolio: Sortino off the existing daily return series, Gain-to-Pain off a monthly aggregation of the equity curve, profit factor off closed-trade dollar P&L. Gain-to-Pain follows Schwager — sum of ALL monthly returns over the absolute sum of the negative ones. The profit-factor-shaped variant, sum(positive)/|sum(negative)|, sits exactly 1.0 higher for every input since sum(all) = sum(pos) - |sum(neg)|; the test asserts against both so the wrong one cannot pass. Sortino divides by len(rets), the full-sample lower partial moment, not by the count of down days, which would shrink the denominator and inflate the ratio. No MAR field: calmar is already CAGR / max drawdown, the same number under the other name (docs/research/effective-risk-floor-ab.md). All three keys are emitted unconditionally even when None — the UI reads an absent key as "report predates these metrics", so presence is a contract. Co-Authored-By: Claude Opus 5 --- app/services/backtest_service.py | 51 ++++++++ ...ackRecordPanel.tsx => EvaluationPanel.tsx} | 0 tests/unit/test_backtest_service.py | 122 ++++++++++++++++++ 3 files changed, 173 insertions(+) rename frontend/src/components/signals/{TrackRecordPanel.tsx => EvaluationPanel.tsx} (100%) diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index beb978c..43c7818 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -2625,6 +2625,19 @@ def _simulate_portfolio( diag = sharpe_diagnostics(rets) sharpe = diag["sharpe"] + # Sortino: the same numerator as Sharpe over downside deviation about a zero + # target. The denominator divides by len(rets) — the full-sample lower partial + # moment — NOT by the count of down days, which would shrink the denominator + # and inflate the ratio. n >= 3 matches sharpe_diagnostics so the two appear + # together or not at all. No down days is +inf, reported as None. + sortino = None + downside = [r for r in rets if r < 0.0] + if len(rets) >= 3 and downside: + mean_ret = sum(rets) / len(rets) + dd = math.sqrt(sum(r * r for r in downside) / len(rets)) + if dd > 0: + sortino = round(mean_ret / dd * math.sqrt(252.0), 2) + # Per-calendar-year returns off the equity curve — shows whether every year # contributed or one exceptional stretch carried the result. yearly: list[dict] = [] @@ -2650,8 +2663,40 @@ def _simulate_portfolio( ), }) + # Gain-to-Pain off the same curve, on MONTHLY returns: Schwager's ratio is + # defined monthly and the daily variant is not comparable to published + # figures. Distinct loop variables from the yearly pass above — that one exits + # with last_eq at final equity, so reusing its names silently corrupts the + # first month. The monthly series itself is not emitted: 36-120 floats per + # strategy per lookback would bloat the single stored report blob. + monthly: list[float] = [] + month_start_eq = curve[0][1] + month_last_eq = curve[0][1] + cur_month = date.fromordinal(curve[0][0]).replace(day=1) + for o, eq in curve: + m = date.fromordinal(o).replace(day=1) + if m != cur_month: + if month_start_eq > 0: + monthly.append(month_last_eq / month_start_eq - 1.0) + cur_month = m + month_start_eq = month_last_eq + month_last_eq = eq + if month_start_eq > 0: + monthly.append(month_last_eq / month_start_eq - 1.0) + + # Schwager: SUM OF ALL monthly returns over the absolute sum of the negative + # ones. Not sum(positive)/|sum(negative)| — that is profit-factor-shaped and + # sits exactly 1.0 higher for every input, since sum(all) = sum(pos) - |sum(neg)|. + monthly_pain = -sum(r for r in monthly if r < 0.0) + gain_to_pain = round(sum(monthly) / monthly_pain, 2) if monthly_pain > 0 else None + pnls = [t["pnl"] for t in trades] wins = sum(1 for p in pnls if p > 0) + # Dollar-based, over closed-trade P&L. Distinct from the R-based profit_factor + # in _robustness_stats; the two never share an object. + gross_win = sum(p for p in pnls if p > 0) + gross_loss = -sum(p for p in pnls if p < 0) + profit_factor = round(gross_win / gross_loss, 2) if gross_loss > 0 else None reason_counts = { reason: sum(1 for t in trades if t["reason"] == reason) for reason in sorted({t["reason"] for t in trades}) @@ -2706,7 +2751,13 @@ def _simulate_portfolio( "total_return_pct": round(total_return_pct, 1), "cagr_pct": round(cagr_pct, 1) if cagr_pct is not None else None, "max_drawdown_pct": round(max_dd_pct, 1), + # calmar IS MAR here (CAGR / max drawdown) — one field, two names. "calmar": round(calmar, 2) if calmar is not None else None, + # Emitted unconditionally even when None: the UI treats an ABSENT key as + # "report predates these metrics", so presence is a contract. + "sortino": sortino, + "gain_to_pain": gain_to_pain, + "profit_factor": profit_factor, "sharpe": sharpe, "sharpe_se": diag["sharpe_se"], "psr": diag["psr"], diff --git a/frontend/src/components/signals/TrackRecordPanel.tsx b/frontend/src/components/signals/EvaluationPanel.tsx similarity index 100% rename from frontend/src/components/signals/TrackRecordPanel.tsx rename to frontend/src/components/signals/EvaluationPanel.tsx diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index c196a9c..43377a4 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -1688,3 +1688,125 @@ async def test_run_backtest_rolls_back_a_failed_portfolio_sim_load(session, monk assert called, "the portfolio-sim block never ran; test proves nothing" assert rolled_back, "a failed portfolio-sim load left the session un-rolled-back" assert report["tickers"] == 1 + + +class TestPortfolioQualityMetrics: + """Sortino / Gain-to-Pain / dollar profit factor. + + Each derives its expectation from the returned ``equity_curve`` rather than + hand-tracing position sizing, and each also asserts the *wrong* variant is + NOT what came back — the denominator and the numerator are exactly where + these ratios are usually got wrong. + """ + + ORD = date(2025, 1, 6).toordinal() + + @staticmethod + def _daily_returns(sim: dict) -> list[float]: + eq = [row["equity"] for row in sim["equity_curve"]] + return [b / a - 1.0 for a, b in zip(eq, eq[1:]) if a > 0] + + @staticmethod + def _monthly_returns(sim: dict) -> list[float]: + monthly: list[float] = [] + rows = sim["equity_curve"] + start = last = rows[0]["equity"] + cur = date.fromisoformat(rows[0]["date"]).replace(day=1) + for row in rows: + m = date.fromisoformat(row["date"]).replace(day=1) + if m != cur: + monthly.append(last / start - 1.0) + cur, start = m, last + last = row["equity"] + monthly.append(last / start - 1.0) + return monthly + + def _wobbly_sim(self) -> dict: + """~70 sessions crossing four month boundaries with a real mid drawdown, + so monthly returns include both signs (a short fixture yields one month + and zero pain, which reads as a broken formula).""" + closes = ( + [100.0 + i for i in range(20)] # climb + + [120.0 - 1.5 * i for i in range(20)] # drawdown + + [90.0 + 1.2 * i for i in range(30)] # recovery + ) + prices = {"AAA": _sim_prices(self.ORD, closes)} + cand = _sim_cand("AAA", self.ORD, entry=100.0, stop=80.0, target=400.0) + sim = bt._simulate_portfolio( + [cand], prices, None, "hold", 65, include_curve=True + ) + assert sim is not None + return sim + + def test_sortino_denominator_is_full_sample_not_downside_count(self): + sim = self._wobbly_sim() + rets = self._daily_returns(sim) + downside = [r for r in rets if r < 0.0] + assert downside, "fixture must produce down days or the test proves nothing" + + mean_ret = sum(rets) / len(rets) + correct = mean_ret / math.sqrt( + sum(r * r for r in downside) / len(rets) + ) * math.sqrt(252.0) + # The classic error: dividing by the count of down days shrinks the + # denominator and inflates the ratio. + inflated = mean_ret / math.sqrt( + sum(r * r for r in downside) / len(downside) + ) * math.sqrt(252.0) + + assert sim["sortino"] == pytest.approx(round(correct, 2), abs=0.01) + assert sim["sortino"] != pytest.approx(round(inflated, 2), abs=0.01) + + def test_gain_to_pain_is_schwager_on_monthly_returns(self): + sim = self._wobbly_sim() + monthly = self._monthly_returns(sim) + assert len(monthly) >= 3, "fixture must span several months" + pain = -sum(r for r in monthly if r < 0.0) + assert pain > 0, "fixture must have a losing month or pain is zero" + + schwager = sum(monthly) / pain + # sum(all) = sum(pos) - |sum(neg)|, so the profit-factor-shaped variant + # sits exactly 1.0 higher for every input. + profit_factor_shaped = sum(r for r in monthly if r > 0.0) / pain + assert profit_factor_shaped == pytest.approx(schwager + 1.0, abs=1e-9) + + assert sim["gain_to_pain"] == pytest.approx(round(schwager, 2), abs=0.01) + assert sim["gain_to_pain"] != pytest.approx( + round(profit_factor_shaped, 2), abs=0.01 + ) + + def test_profit_factor_is_dollar_based(self): + """One winner, one loser, on separate symbols so both fill.""" + up = [100.0 + 2.0 * i for i in range(8)] + down = [100.0 - 2.0 * i for i in range(8)] + prices = { + "WIN": _sim_prices(self.ORD, up), + "LOSE": _sim_prices(self.ORD, down), + } + cands = [ + _sim_cand("WIN", self.ORD, entry=100.0, stop=90.0, target=400.0, mp=95.0), + _sim_cand("LOSE", self.ORD, entry=100.0, stop=80.0, target=400.0, mp=94.0), + ] + sim = bt._simulate_portfolio([cands[0], cands[1]], prices, None, "hold", 5) + assert sim is not None + assert sim["trades"] == 2 + # With exactly two trades the reported best/worst ARE the win and the loss. + gross_win = sim["best_trade_pnl"] + gross_loss = -sim["worst_trade_pnl"] + assert gross_win > 0 and gross_loss > 0, "fixture must produce one of each" + assert sim["profit_factor"] == pytest.approx( + round(gross_win / gross_loss, 2), abs=0.01 + ) + + def test_keys_always_present_and_no_downside_is_none(self): + """Monotonic rise: no down days. Sortino must be None, never inf — and + all three keys must still be emitted, because the UI reads an ABSENT key + as 'report predates these metrics'.""" + closes = [100.0, 102.0, 104.0, 106.0, 108.0, 110.0] + prices = {"AAA": _sim_prices(self.ORD, closes)} + cand = _sim_cand("AAA", self.ORD, entry=100.0, stop=95.0, target=130.0) + sim = bt._simulate_portfolio([cand], prices, None, "hold", 3) + assert sim is not None + for key in ("sortino", "gain_to_pain", "profit_factor"): + assert key in sim + assert sim["sortino"] is None