promote residual momentum ranking

app/services/admin_service.py

@@ -39,8 +39,8 @@ SUPPORTED_TICKER_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
 # Track Record's qualified stats. The outcome evaluator deliberately ignores
 # these — every setup is evaluated so the gate itself can be validated.
 #
-# The core selection is cross-sectional 12-1 momentum (top percentile of the
-# universe, long-only). R:R and confidence are floors; high-conviction /
+# The core selection is residual cross-sectional 12-1 momentum (top percentile
+# of the universe, long-only). R:R and confidence are floors; high-conviction /
 # clean-read are optional tighteners (off by default).
 _ACTIVATION_FLOAT_KEYS: dict[str, str] = {
     "min_momentum_percentile": "activation_min_momentum_percentile",

app/services/backtest_service.py

@@ -93,6 +93,9 @@ ATR_MULTIPLIER = 1.5
 # signal, not the outcome of a target/stop structure built on top of one.
 MIN_CROSS_SECTION = 20    # min tickers present in a week to score that week
 MIN_RELIABLE_PERIODS = 12 # min non-overlapping windows before a signal's IC is trusted
+PRODUCTION_PERCENTILE_KEY = "activation_momentum_percentile"
+RAW_PERCENTILE_KEY = "momentum_percentile"
+RESIDUAL_PERCENTILE_KEY = "residual_momentum_percentile"
 
 
 def _wrap_levels(level_dicts: list[dict]) -> list[Any]:
@@ -845,12 +848,26 @@ def _assign_momentum_percentiles(candidates: list[dict]) -> None:
 
 
 def _assign_residual_momentum_percentiles(candidates: list[dict]) -> None:
-    """Research-only residual-momentum percentile used by strategy variants."""
+    """Residual-momentum percentile promoted to production activation ranking."""
     _assign_signal_percentiles(
-        candidates, "residual_momentum", "residual_momentum_percentile"
+        candidates, "residual_momentum", RESIDUAL_PERCENTILE_KEY
     )
 
 
+def _assign_activation_momentum_percentiles(candidates: list[dict]) -> None:
+    """Production activation rank: residual 12-1 when available, raw fallback.
+
+    The raw fallback mirrors the live scanner's behavior when benchmark history
+    is unavailable. In normal backtests, SPY is loaded and this is residual.
+    """
+    for c in candidates:
+        c[PRODUCTION_PERCENTILE_KEY] = (
+            c.get(RESIDUAL_PERCENTILE_KEY)
+            if c.get(RESIDUAL_PERCENTILE_KEY) is not None
+            else c.get(RAW_PERCENTILE_KEY)
+        )
+
+
 def _momentum_qualifies(cand: dict, threshold: float) -> bool:
     """Whether a candidate clears the floors (meets_core) and the momentum gate.
     Threshold 0 disables the momentum gate (floors only). The gate is long-only:
@@ -861,7 +878,7 @@ def _momentum_qualifies(cand: dict, threshold: float) -> bool:
         return True
     if cand["direction"] == "short":
         return False
-    mp = cand.get("momentum_percentile")
+    mp = cand.get(PRODUCTION_PERCENTILE_KEY)
     return mp is not None and mp >= threshold
 
 
@@ -890,7 +907,7 @@ def _gate_ablation(candidates: list[dict], activation: dict, threshold: float) -
             return True
         if c["direction"] == "short":
             return False
-        mp = c.get("momentum_percentile")
+        mp = c.get(PRODUCTION_PERCENTILE_KEY)
         return mp is not None and mp >= threshold
 
     def rr_ok(c: dict) -> bool:
@@ -965,7 +982,7 @@ def _simulate_portfolio(
     hold_days: int,
     *,
     qualified_fn: Callable[[dict], bool] | None = None,
-    ranking_key: str = "momentum_percentile",
+    ranking_key: str = PRODUCTION_PERCENTILE_KEY,
     max_positions: int = SIM_MAX_POSITIONS,
     risk_per_trade: float = SIM_RISK_PER_TRADE,
 ) -> dict | None:
@@ -1189,9 +1206,18 @@ def _simulate_portfolio(
 
 STRATEGY_VARIANTS: tuple[dict, ...] = (
     {
-        "variant": "production_raw_80_fixed10",
-        "label": "Production raw 80 / max 10",
-        "percentile_key": "momentum_percentile",
+        "variant": "production_residual_80_fixed10",
+        "label": "Production residual 80 / max 10",
+        "percentile_key": PRODUCTION_PERCENTILE_KEY,
+        "cutoff": 80.0,
+        "max_positions": 10,
+        "risk_per_trade": 0.01,
+        "risk_scale": None,
+    },
+    {
+        "variant": "legacy_raw_80_fixed10",
+        "label": "Legacy raw 80 / max 10",
+        "percentile_key": RAW_PERCENTILE_KEY,
         "cutoff": 80.0,
         "max_positions": 10,
         "risk_per_trade": 0.01,
@@ -1200,52 +1226,16 @@ STRATEGY_VARIANTS: tuple[dict, ...] = (
     {
         "variant": "raw_90_fixed10",
         "label": "Raw 90 / max 10",
-        "percentile_key": "momentum_percentile",
+        "percentile_key": RAW_PERCENTILE_KEY,
         "cutoff": 90.0,
         "max_positions": 10,
         "risk_per_trade": 0.01,
         "risk_scale": None,
     },
-    {
-        "variant": "raw_90_fixed15",
-        "label": "Raw 90 / max 15",
-        "percentile_key": "momentum_percentile",
-        "cutoff": 90.0,
-        "max_positions": 15,
-        "risk_per_trade": 0.01,
-        "risk_scale": None,
-    },
-    {
-        "variant": "residual_80_fixed10",
-        "label": "Residual 80 / max 10",
-        "percentile_key": "residual_momentum_percentile",
-        "cutoff": 80.0,
-        "max_positions": 10,
-        "risk_per_trade": 0.01,
-        "risk_scale": None,
-    },
     {
         "variant": "residual_80_fixed15",
-        "label": "Residual 80 / max 15",
-        "percentile_key": "residual_momentum_percentile",
-        "cutoff": 80.0,
-        "max_positions": 15,
-        "risk_per_trade": 0.01,
-        "risk_scale": None,
-    },
-    {
-        "variant": "residual_80_fixed20",
-        "label": "Residual 80 / max 20",
-        "percentile_key": "residual_momentum_percentile",
-        "cutoff": 80.0,
-        "max_positions": 20,
-        "risk_per_trade": 0.01,
-        "risk_scale": None,
-    },
-    {
-        "variant": "raw_80_fixed15",
-        "label": "Raw 80 / max 15",
-        "percentile_key": "momentum_percentile",
+        "label": "Residual 80 / max 15 capacity check",
+        "percentile_key": PRODUCTION_PERCENTILE_KEY,
         "cutoff": 80.0,
         "max_positions": 15,
         "risk_per_trade": 0.01,
@@ -1295,7 +1285,7 @@ def _strategy_variant_sims(
         rows.append({
             "variant": cfg["variant"],
             "label": cfg["label"],
-            "ranking": "residual" if "residual" in percentile_key else "raw",
+            "ranking": "raw" if percentile_key == RAW_PERCENTILE_KEY else "residual",
             "cutoff": cutoff,
             "max_positions": int(cfg["max_positions"]),
             "risk_per_trade_pct": round(float(cfg["risk_per_trade"]) * 100, 2),
@@ -1312,14 +1302,12 @@ def _pct_loss(base: float | None, candidate: float | None) -> float | None:
 
 
 def _build_research_recommendation(report: dict) -> dict:
-    """Advisory rules for research variants. These are deliberately conservative:
-    production only changes later if a portfolio variant beats the baseline under
-    transparent drawdown/Sharpe/CAGR constraints."""
+    """Advisory rules for the remaining research variants after residual promotion."""
     variants = {
         v.get("variant"): v
         for v in (report.get("strategy_variants") or {}).get("variants", [])
     }
-    base = variants.get("production_raw_80_fixed10")
+    base = variants.get("production_residual_80_fixed10")
     items: list[dict] = []
     if base is None:
         return {
@@ -1331,25 +1319,25 @@ def _build_research_recommendation(report: dict) -> dict:
     base_dd = base.get("max_drawdown_pct")
     base_cagr = base.get("cagr_pct")
 
-    residuals = [
-        v for key, v in variants.items()
-        if key.startswith("residual_80_") and v.get("risk_scale") is None
-    ]
-    residual = max(residuals, key=lambda v: v.get("sharpe") or -999, default=None)
+    capacity = variants.get("residual_80_fixed15")
     if (
-        residual and base_sharpe is not None and residual.get("sharpe") is not None
-        and base_dd is not None and residual.get("max_drawdown_pct") is not None
+        capacity and base_sharpe is not None and base_cagr is not None
+        and capacity.get("sharpe") is not None and capacity.get("cagr_pct") is not None
+        and capacity.get("max_drawdown_pct") is not None and base_dd is not None
     ):
-        sharpe_delta = residual["sharpe"] - base_sharpe
-        dd_delta = residual["max_drawdown_pct"] - base_dd
-        candidate = sharpe_delta >= 0.10 and dd_delta <= 2.0
+        candidate = (
+            capacity["sharpe"] > base_sharpe
+            and capacity["cagr_pct"] > base_cagr
+            and capacity["max_drawdown_pct"] <= base_dd + 1.0
+        )
         items.append({
-            "topic": "residual_momentum",
+            "topic": "capacity_15",
             "candidate": candidate,
             "text": (
-                f"Residual momentum {'is a promotion candidate' if candidate else 'stays research-only'}: "
-                f"{residual['label']} Sharpe {residual['sharpe']:.2f} vs {base_sharpe:.2f}, "
-                f"drawdown {residual['max_drawdown_pct']:.1f}% vs {base_dd:.1f}%."
+                f"Max-15 capacity {'is worth promoting' if candidate else 'is not needed yet'}: "
+                f"Sharpe {capacity['sharpe']:.2f} vs {base_sharpe:.2f}, "
+                f"CAGR {capacity['cagr_pct']:+.1f}% vs {base_cagr:+.1f}%, "
+                f"skipped {capacity.get('skipped_book_full', 0)} vs {base.get('skipped_book_full', 0)}."
             ),
         })
 
@@ -1385,8 +1373,8 @@ def _build_research_recommendation(report: dict) -> dict:
     return {
         "items": items,
         "note": (
-            "Advisory only. Production changes require a variant to pass the rule "
-            "and then be adopted explicitly in a later strategy-version change."
+            "Residual 12-1 momentum is now the production activation rank. "
+            "Remaining rows are research comparisons only."
         ),
     }
 
@@ -1473,7 +1461,8 @@ def _build_recommendation(report: dict) -> dict:
                     "text": f"Gate: keep the {label} (worth {delta:+.2f}R/trade under the hold exit).",
                 })
 
-    # Momentum cutoff: best per-trade net among the active-gate sweep rows.
+    # Activation cutoff: best per-trade net among the promoted residual-momentum
+    # sweep rows.
     sweep_rows = [
         r for r in report.get("sweep") or []
         if r.get("net_avg_r") is not None and (r.get("min_momentum_percentile") or 0) > 0
@@ -1483,7 +1472,7 @@ def _build_recommendation(report: dict) -> dict:
         items.append({
             "topic": "cutoff",
             "text": (
-                f"Momentum cutoff: {best_cut['min_momentum_percentile']:.0f} has the best "
+                f"Residual-momentum cutoff: {best_cut['min_momentum_percentile']:.0f} has the best "
                 f"per-trade net ({best_cut['net_avg_r']:+.2f}R over {best_cut['total']} setups)."
             ),
         })
@@ -1653,9 +1642,11 @@ async def run_backtest(
         progress_cb(total, total, "")
 
     # Cross-sectional momentum: rank every week's universe, then "qualified" means
-    # floors + top ``min_momentum_percentile`` by 12-1 momentum.
+    # floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum
+    # (raw 12-1 fallback only when benchmark data is unavailable).
     _assign_momentum_percentiles(candidates)
     _assign_residual_momentum_percentiles(candidates)
+    _assign_activation_momentum_percentiles(candidates)
     current_min_pct = float(activation.get("min_momentum_percentile", 80.0))
     for c in candidates:
         c["qualified"] = _momentum_qualifies(c, current_min_pct)
@@ -1779,10 +1770,9 @@ async def run_backtest(
         "strategy_variants": {
             "variants": strategy_variant_rows,
             "note": (
-                "Research-only hold-to-horizon portfolio variants. These compare "
-                "raw vs residual momentum ranking, cutoff 80 vs 90, and max 10/15/20 "
-                "position capacity. They do not change live "
-                "qualification or paper-trade behavior."
+                "Research-only hold-to-horizon portfolio variants. Production now "
+                "uses residual 12-1 momentum at cutoff 80; the remaining rows compare "
+                "the legacy raw rank, raw cutoff 90, and one max-15 capacity check."
             ),
         },
         "signal_eval": _signal_evaluation(collected),

app/services/benchmark_service.py

@@ -3,8 +3,8 @@
 Fetches the S&P 500 proxy (SPY) daily closes via Alpaca and persists them, so
 paper-trade alpha — a trade's return minus the benchmark's return over the same
 holding period — can be computed. The benchmark is a standalone series, NOT a
-tracked ``Ticker``, so it never contaminates the scanner, momentum-percentile
-ranking, or rankings.
+tracked ``Ticker``; its closes feed residual momentum and alpha, but it never
+becomes a trade candidate or rankings-table row.
 """
 
 from __future__ import annotations

app/services/momentum_service.py

@@ -1,17 +1,18 @@
-"""Cross-sectional 12-1 momentum ranking for the universe.
+"""Cross-sectional residual 12-1 momentum ranking for the universe.
 
 The activation gate selects the top ``min_momentum_percentile`` of the universe
-by 12-1 month momentum (return from ~12 months ago to ~1 month ago — the one
-price signal the backtest showed sorts forward returns). The daily scan ranks
-every ticker and stores each setup's percentile (see ``rr_scanner_service``), so
-the live list, the Track Record's qualified stats, and outcome evaluation all gate
-on the same value.
+by residual 12-1 month momentum: the stock's 12-1 return after subtracting its
+estimated benchmark beta contribution over the same formation window. The daily
+scan ranks every ticker and stores each setup's percentile (see
+``rr_scanner_service``), so the live list, the Track Record's qualified stats,
+and outcome evaluation all gate on the same value.
 """
 
 from __future__ import annotations
 
 import json
 import logging
+from datetime import date
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -35,29 +36,101 @@ def compute_12_1_momentum(closes: list[float]) -> float | None:
     return None
 
 
+def compute_residual_12_1_momentum(
+    dates: list[date],
+    closes: list[float],
+    benchmark_closes: dict[date, float],
+) -> float | None:
+    """12-1 momentum after removing linear benchmark exposure.
+
+    Estimate beta from daily stock/benchmark returns over the standard 12-1
+    formation window, then sum stock return minus beta * benchmark return. No
+    intercept is subtracted: fitting an intercept over the same window would make
+    residuals sum to roughly zero and destroy the ranking signal.
+    """
+    i = len(closes) - 1
+    if not benchmark_closes or len(dates) != len(closes) or i - _MOM_LOOKBACK < 0:
+        return None
+
+    stock_rets: list[float] = []
+    market_rets: list[float] = []
+    for k in range(i - _MOM_LOOKBACK + 1, i - _MOM_SKIP + 1):
+        prev_close = closes[k - 1]
+        bench_prev = benchmark_closes.get(dates[k - 1])
+        bench_cur = benchmark_closes.get(dates[k])
+        if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0:
+            continue
+        stock_rets.append(closes[k] / prev_close - 1.0)
+        market_rets.append(bench_cur / bench_prev - 1.0)
+
+    if len(stock_rets) < 100:
+        return None
+    mean_market = sum(market_rets) / len(market_rets)
+    mean_stock = sum(stock_rets) / len(stock_rets)
+    var_market = sum((x - mean_market) ** 2 for x in market_rets)
+    if var_market <= 0:
+        return None
+    cov = sum(
+        (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market)
+        for k in range(len(stock_rets))
+    )
+    beta = cov / var_market
+    return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets)))
+
+
+async def _load_activation_benchmark(db: AsyncSession) -> dict[date, float]:
+    """Load SPY closes for residual momentum; refresh once if the table is empty."""
+    try:
+        from app.services.benchmark_service import load_benchmark_closes, refresh_benchmark_prices
+
+        closes = await load_benchmark_closes(db)
+        if closes:
+            return closes
+        await refresh_benchmark_prices(db)
+        return await load_benchmark_closes(db)
+    except Exception:
+        logger.exception("Residual momentum benchmark load failed; falling back to raw momentum")
+        return {}
+
+
 async def compute_momentum_percentiles(db: AsyncSession) -> dict[str, float]:
-    """Compute each ticker's 12-1 momentum and rank the universe into a
-    ``{symbol: percentile}`` map (0–100, 100 = strongest momentum). Tickers
-    without a full year of history are absent (can't be ranked)."""
+    """Compute each ticker's activation momentum rank.
+
+    Production uses residual 12-1 momentum when benchmark data is available. If
+    SPY data is absent, fall back to raw 12-1 momentum rather than disabling the
+    scanner. Tickers without enough stock/benchmark history are absent.
+    """
     result = await db.execute(select(Ticker).order_by(Ticker.symbol))
     tickers = list(result.scalars().all())
 
-    momentum: dict[str, float] = {}
+    benchmark_closes = await _load_activation_benchmark(db)
+    using_residual = len(benchmark_closes) >= _MOM_LOOKBACK
+
+    values: dict[str, float] = {}
     for ticker in tickers:
         try:
             records = await query_ohlcv(db, ticker.symbol)
         except Exception:
             logger.exception("Momentum fetch failed for %s", ticker.symbol)
             continue
-        m = compute_12_1_momentum([float(r.close) for r in records])
-        if m is not None:
-            momentum[ticker.symbol] = m
+        closes = [float(r.close) for r in records]
+        value = (
+            compute_residual_12_1_momentum([r.date for r in records], closes, benchmark_closes)
+            if using_residual
+            else compute_12_1_momentum(closes)
+        )
+        if value is not None:
+            values[ticker.symbol] = value
 
-    ranked = sorted(momentum, key=lambda s: momentum[s])
+    ranked = sorted(values, key=lambda s: values[s])
     n = len(ranked)
     percentiles = {
         sym: round((rank / (n - 1) * 100.0) if n > 1 else 100.0, 2)
         for rank, sym in enumerate(ranked)
     }
-    logger.info(json.dumps({"event": "momentum_ranked", "tickers": n}))
+    logger.info(json.dumps({
+        "event": "momentum_ranked",
+        "signal": "residual_12_1" if using_residual else "raw_12_1_fallback",
+        "tickers": n,
+    }))
     return percentiles

app/services/qualification.py

@@ -2,12 +2,12 @@
 
 A single predicate, driven by the admin activation config, used by the
 performance stats (server) and mirrored on the frontend. The core selection is
-cross-sectional momentum: a setup's ticker must rank in the top
-``min_momentum_percentile`` of the universe by 12-1 month momentum — the one
-signal the backtest showed actually sorts forward returns. R:R and confidence
-remain as floors, and conviction/conflict survive as optional tighteners (off by
-default). The momentum percentile is computed across the universe and attached to
-each setup upstream; when it's absent the gate falls back to the floors.
+residual cross-sectional momentum: a setup's ticker must rank in the top
+``min_momentum_percentile`` of the universe by beta-adjusted 12-1 month momentum.
+R:R and confidence remain as floors, and conviction/conflict survive as optional
+tighteners (off by default). The activation percentile is computed across the
+universe and attached to each setup upstream; when it's absent the gate falls
+back to the floors.
 """
 
 from __future__ import annotations
@@ -65,12 +65,12 @@ def setup_qualifies(setup: Any, config: dict) -> bool:
             return False
     if (setup.confidence_score or 0.0) < config["min_confidence"]:
         return False
-    # Cross-sectional momentum: the core selection. A setup's ticker must rank in
-    # the top ``min_momentum_percentile`` of the universe by 12-1 momentum. The
-    # validated edge is long-only, so while the gate is active shorts (which fight
-    # the trend) never qualify. The percentile floor is only enforced when a
-    # percentile is attached (live setups / backtest); callers that don't attach
-    # it defer to the floors above.
+    # Residual cross-sectional momentum: the core selection. A setup's ticker
+    # must rank in the top ``min_momentum_percentile`` of the universe by
+    # beta-adjusted 12-1 momentum. The validated edge is long-only, so while the
+    # gate is active shorts (which fight the trend) never qualify. The percentile
+    # floor is only enforced when a percentile is attached (live setups /
+    # backtest); callers that don't attach it defer to the floors above.
     min_pct = float(config.get("min_momentum_percentile", 0.0))
     if min_pct > 0:
         if (getattr(setup, "direction", "long") or "long") == "short":
@@ -81,7 +81,7 @@ def setup_qualifies(setup: Any, config: dict) -> bool:
     # A NEUTRAL recommendation means the engine found no clear directional setup —
     # not an actionable signal, so by default it doesn't qualify (and can't be a
     # top pick). ``exclude_neutral`` defaults on; turn it off to also count
-    # no-clear-direction momentum leaders.
+    # no-clear-direction residual momentum leaders.
     if config.get("exclude_neutral"):
         if (setup.recommended_action or "NEUTRAL") == "NEUTRAL":
             return False

app/services/rr_scanner_service.py

@@ -31,7 +31,7 @@ from app.services.recommendation_service import enhance_trade_setup
 
 logger = logging.getLogger(__name__)
 
-STRATEGY_VERSION = "momentum_12_1_rr_time_v1"
+STRATEGY_VERSION = "residual_momentum_12_1_rr_time_v2"
 
 
 async def _get_ticker(db: AsyncSession, symbol: str) -> Ticker:
@@ -219,9 +219,9 @@ async def scan_ticker(
 ) -> list[TradeSetup]:
     """Scan a single ticker for trade setups meeting the R:R threshold.
 
-    ``momentum_percentile`` is the ticker's 12-1 momentum rank across the universe
-    (computed by the caller), stored on each setup so the activation gate can
-    select the top slice."""
+    ``momentum_percentile`` is the ticker's residual 12-1 momentum activation
+    rank across the universe (computed by the caller), stored on each setup so
+    the activation gate can select the top slice."""
     ticker = await _get_ticker(db, symbol)
 
     records = await query_ohlcv(db, symbol)
@@ -393,8 +393,9 @@ async def scan_all_tickers(
     tickers = list(result.scalars().all())
     total = len(tickers)
 
-    # Rank the universe by 12-1 momentum up front so each new setup carries its
-    # ticker's percentile (used by the activation gate). Best-effort.
+    # Rank the universe by residual 12-1 momentum up front so each new setup
+    # carries its activation percentile. Best-effort; the ranker falls back to
+    # raw 12-1 momentum only if benchmark data is unavailable.
     try:
         from app.services import momentum_service
 

app/services/watchlist_service.py

@@ -173,8 +173,8 @@ async def _enrich_entry(
         "dimensions": dims,
         "rr_ratio": setup.rr_ratio if setup else None,
         "rr_direction": setup.direction if setup else None,
-        # 12-1 cross-sectional momentum percentile (the top-pick selector); ticker-
-        # level, so any of the ticker's setups carries the same value.
+        # Residual 12-1 activation percentile (the top-pick selector); ticker-level,
+        # so any of the ticker's setups carries the same value.
         "momentum_percentile": setup.momentum_percentile if setup else None,
         "sr_levels": sr_levels,
         "last_close": last_close,