signal-platform/app/services/backtest_service.py

"""Historical backtest (Phase 1): replay the price-derived engine over stored
OHLCV and measure how the CURRENT config would have performed.

For each ticker we step through history at the selected entry cadence, and at each as-of date D we
rebuild the setup using only bars ≤ D (no lookahead), then walk the actual bars
after D to record the realized outcome. The report contains:

  - hit-rate / expectancy of qualified setups vs the all-setups control group,
    gross and net of costs, with robustness stats (median, profit factor,
    expectancy without the top winners)
  - the momentum-percentile sweep and the gate ablation (each floor removed in
    turn, graded under both the target and the hold-to-horizon exit)
  - the time-exit sweep (hold N days with the initial stop)
  - cross-sectional factor rank-IC ("signal edge")
  - a capital-constrained portfolio simulation (equity curve → CAGR, drawdown,
    Sharpe, SPY comparison)
  - a data-driven recommendation derived from this report's numbers

Limitation: sentiment and fundamentals have no point-in-time history, so they're
held neutral here — this calibrates the price/S-R machinery only.

Environment variables (see also run_backtest_snapshot.py):
  Production / general use:
    BACKTEST_HOLDOUT_SPLIT=YYYY-MM-DD   # disjoint train/test split
    BACKTEST_SNAPSHOT_OFFLINE=1
    BACKTEST_ALLOW_SPAWN=1              # for Windows multiprocessing

  Research / diagnostic only (retired experiments — do not use for live decisions):
    BACKTEST_ATR_TARGET_FALLBACK=3
    BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1
    BACKTEST_RESEARCH_EXITS=1
    BACKTEST_MIN_RR_SWEEP=1
"""

from __future__ import annotations

import asyncio
import bisect
import json
import logging
import math
import multiprocessing
import os
import statistics
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import ProcessPoolExecutor
from datetime import date, datetime, timezone
from types import SimpleNamespace
from typing import Any

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import settings
from app.models.ticker import Ticker
from app.services import settings_store
from app.services.admin_service import get_activation_config, update_setting
from app.services.indicator_service import _extract_ohlcv, compute_atr
from app.services.momentum_service import (
    STRATEGY_RANK_MOMENTUM_WEIGHT,
    compute_realized_vol_6m,
)
from app.services.outcome_service import (
    OUTCOME_AMBIGUOUS,
    OUTCOME_STOP_HIT,
    OUTCOME_TARGET_HIT,
    Bar,
    evaluate_setup_against_bars,
)
from app.services.price_service import query_ohlcv
from app.services.qualification import (
    HIGH_CONVICTION_ACTIONS,
    MIN_TARGET_PROBABILITY,
    _action_direction,
    best_target_probability,
    setup_qualifies,
)
from app.services.recommendation_service import (
    _choose_recommended_action,
    _classify_by_probability,
    _prune_floor_pinned_targets,
    _risk_level_from_conflicts,
    _select_primary_target,
    _zone_representative_levels,
    direction_analyzer,
    get_recommendation_config,
    probability_estimator,
    signal_conflict_detector,
    target_generator,
)
from app.services.scoring_service import (
    compute_momentum_from_closes,
    compute_technical_from_arrays,
)
from app.services.sr_service import detect_gate_target_ladder, detect_sr_levels
from app.services.trade_policy import REENTRY_LOCKDOWN_SESSIONS

logger = logging.getLogger(__name__)

KEY_REPORT = "backtest_report"

WEEKLY_BACKTEST_CADENCE = "weekly"
DAILY_BACKTEST_CADENCE = "daily"
DEFAULT_BACKTEST_CADENCE = WEEKLY_BACKTEST_CADENCE
BACKTEST_CADENCE_SESSIONS = {
    WEEKLY_BACKTEST_CADENCE: 5,
    DAILY_BACKTEST_CADENCE: 1,
}
# Compatibility alias for research scripts built around the original weekly
# replay. New code should select a cadence and call ``backtest_step_sessions``.
STEP_DAYS = BACKTEST_CADENCE_SESSIONS[WEEKLY_BACKTEST_CADENCE]
MIN_LOOKBACK = 60      # bars needed before D for indicators (EMA cross needs 51)
HORIZON = 30           # trading days to resolve an outcome (matches the evaluator)
ATR_MULTIPLIER = 1.5
PRODUCTION_GTL_TARGET_MODEL = "production_gtl"
STRUCTURAL_SR_TARGET_MODEL = "structural_sr"
BACKTEST_TARGET_MODELS = {
    PRODUCTION_GTL_TARGET_MODEL: "Live GTL (production)",
    STRUCTURAL_SR_TARGET_MODEL: "Structural S/R (comparison)",
}

# Cross-sectional signal evaluation (factor IC). Each candidate signal is a
# point-in-time number computed from closes alone (sentiment/fundamentals have no
# history here), sampled one as-of per ISO week, and graded by how its rank
# correlates with the forward HORIZON-day return ACROSS the universe — i.e. does
# ranking stocks by this signal sort tomorrow's winners from losers. This is the
# test the per-setup hit-rate report can't do: it measures predictive power of a
# signal, not the outcome of a target/stop structure built on top of one.
MIN_CROSS_SECTION = 20    # min tickers present in a week to score that week
MIN_RELIABLE_PERIODS = 12 # min non-overlapping windows before a signal's IC is trusted
PRODUCTION_PERCENTILE_KEY = "activation_momentum_percentile"
RAW_PERCENTILE_KEY = "momentum_percentile"
RESIDUAL_PERCENTILE_KEY = "residual_momentum_percentile"
VOL_PERCENTILE_KEY = "vol_6m_percentile"
LOW_VOL_PERCENTILE_KEY = "low_vol_6m_percentile"
RESIDUAL_LOW_VOL_BLEND_KEY = "residual_low_vol_blend_score"
RESIDUAL_HIGH_VOL_BLEND_90_10_KEY = "residual_high_vol_blend_90_10_score"
RESIDUAL_HIGH_VOL_BLEND_80_20_KEY = "residual_high_vol_blend_80_20_score"
RESIDUAL_HIGH_VOL_BLEND_KEY = "residual_high_vol_blend_score"
RESIDUAL_HIGH_VOL_BLEND_60_40_KEY = "residual_high_vol_blend_60_40_score"


def _wrap_levels(level_dicts: list[dict]) -> list[Any]:
    return [
        SimpleNamespace(
            id=i,
            price_level=float(d["price_level"]),
            type=d["type"],
            strength=int(d["strength"]),
            detection_method=d.get("detection_method", "unknown"),
            sources=list(d.get("sources") or [d.get("detection_method", "unknown")]),
            rejection_count=int(d.get("rejection_count", 0) or 0),
            last_rejection_age=d.get("last_rejection_age"),
        )
        for i, d in enumerate(level_dicts)
    ]


def validate_backtest_target_model(value: str) -> str:
    """Validate the small, user-facing set of supported backtest target models."""
    normalized = value.strip().lower()
    if normalized not in BACKTEST_TARGET_MODELS:
        allowed = ", ".join(BACKTEST_TARGET_MODELS)
        raise ValueError(f"Unknown backtest target model {value!r}; expected one of {allowed}")
    return normalized


def validate_backtest_cadence(value: str) -> str:
    """Validate the supported entry-replay cadences."""
    normalized = value.strip().lower()
    if normalized not in BACKTEST_CADENCE_SESSIONS:
        allowed = ", ".join(BACKTEST_CADENCE_SESSIONS)
        raise ValueError(
            f"Unknown backtest cadence {value!r}; expected one of {allowed}"
        )
    return normalized


def backtest_step_sessions(cadence: str) -> int:
    return BACKTEST_CADENCE_SESSIONS[validate_backtest_cadence(cadence)]


def _ranking_period(as_of: date, cadence: str) -> tuple:
    """Cross-section key for activation ranks at the selected entry cadence."""
    cadence = validate_backtest_cadence(cadence)
    if cadence == DAILY_BACKTEST_CADENCE:
        return ("date", as_of.toordinal())
    iso = as_of.isocalendar()
    return ("week", iso[0], iso[1])


# ---------------------------------------------------------------------------
# RESEARCH / DIAGNOSTIC FALLBACKS (retired experiments)
#
# These implement behavior from experiments that were rejected for production
# (clear-air synthetic targets, blanket ATR fallbacks). They are OFF by default
# and exist only to reproduce historical research results or run future ablations.
# See docs/research/sr-levels-and-exits.md.
# Do NOT enable for production decision making.
# ---------------------------------------------------------------------------

def _atr_target_fallback_k() -> float | None:
    """RESEARCH DIAGNOSTIC: k for a synthetic k*ATR target when no S/R level.
    Off (None) by default (production behavior). Set BACKTEST_ATR_TARGET_FALLBACK=3
    to enable. See docs/research/sr-levels-and-exits.md."""
    raw = os.getenv("BACKTEST_ATR_TARGET_FALLBACK", "").strip()
    if not raw:
        return None
    try:
        k = float(raw)
    except ValueError:
        return None
    return k if k > 0 else None


def _fallback_clear_air_only() -> bool:
    """RESEARCH DIAGNOSTIC: restrict fallback to genuine clear-air cases only.
    Set BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1."""
    return os.getenv("BACKTEST_FALLBACK_CLEAR_AIR_ONLY", "").strip().lower() in {
        "1", "true", "yes", "on",
    }


def _has_structure_ahead(direction: str, entry: float, sr_levels: list[Any]) -> bool:
    """Is there any S/R level in the direction of the trade? (Resistance above for
    a long, support below for a short.) False == clear air."""
    if direction == "long":
        return any(
            lv.type == "resistance" and float(lv.price_level) > entry for lv in sr_levels
        )
    return any(
        lv.type == "support" and float(lv.price_level) < entry for lv in sr_levels
    )


def _atr_fallback_target(
    direction: str, entry: float, stop: float, atr: float, k: float
) -> dict:
    """RESEARCH DIAGNOSTIC: synthetic target k*ATR (neutral strength)."""
    price = entry + k * atr if direction == "long" else entry - k * atr
    distance = abs(price - entry)
    risk = abs(entry - stop)
    return {
        "price": float(price),
        "distance_from_entry": float(distance),
        "distance_atr_multiple": float(k),
        "rr_ratio": float(distance / risk) if risk > 0 else 0.0,
        "classification": "Moderate",
        "sr_level_id": -1,  # synthetic: no S/R level behind it
        "sr_strength": 50.0,
    }


def _window_setups(
    window_records: list,
    config: dict,
    activation: dict,
    *,
    target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> list[dict]:
    """Rebuild the setup(s) at the last bar of ``window_records`` (the as-of date),
    using only those bars. Returns one dict per tradeable direction."""
    if len(window_records) < MIN_LOOKBACK:
        return []

    _, highs, lows, closes, volumes = _extract_ohlcv(window_records)
    entry = closes[-1]
    if entry <= 0:
        return []

    # 12-1 month momentum (skip the last month) — the universe ranks on this.
    # None until a year of history exists; such setups can't qualify on momentum.
    mom_12_1 = (
        closes[-22] / closes[-253] - 1.0
        if len(closes) >= 253 and closes[-253] > 0
        else None
    )

    try:
        atr = compute_atr(highs, lows, closes)["atr"]
    except Exception:
        return []
    if atr <= 0:
        return []

    target_model = validate_backtest_target_model(target_model)
    if target_model == PRODUCTION_GTL_TARGET_MODEL:
        detected_levels = detect_gate_target_ladder(
            highs,
            lows,
            closes,
        )
    else:
        detected_levels = detect_sr_levels(highs, lows, closes, volumes)
    sr_levels = _wrap_levels(detected_levels)
    if not sr_levels:
        return []

    gate_levels = list(sr_levels)

    technical = (compute_technical_from_arrays(highs, lows, closes, volumes)[0]) or 50.0
    momentum = (compute_momentum_from_closes(closes)[0]) or 50.0
    dim_scores = {"technical": technical, "momentum": momentum}

    conflicts = signal_conflict_detector.detect_conflicts(dim_scores, None, config)
    confidences = {
        "long": direction_analyzer.calculate_confidence("long", dim_scores, None, conflicts),
        "short": direction_analyzer.calculate_confidence("short", dim_scores, None, conflicts),
    }

    # First pass: build targets per direction
    per_dir: dict[str, dict] = {}
    for direction in ("long", "short"):
        stop = entry - atr * ATR_MULTIPLIER if direction == "long" else entry + atr * ATR_MULTIPLIER
        zone_levels = _zone_representative_levels(
            gate_levels,
            entry,
            strength_mode="sum",
        )
        targets = target_generator.generate_targets(direction, entry, stop, zone_levels, atr)
        if not targets:
            fallback_k = _atr_target_fallback_k()
            if fallback_k is None:
                continue
            # RESEARCH DIAGNOSTIC only (see _atr_target_fallback_k etc.)
            if _fallback_clear_air_only() and _has_structure_ahead(direction, entry, sr_levels):
                continue
            targets = [_atr_fallback_target(direction, entry, stop, atr, fallback_k)]
        for t in targets:
            t["probability"] = probability_estimator.estimate_probability(
                t, dim_scores, None, direction, config
            )
            t["classification"] = _classify_by_probability(t["probability"])
        # Collapse duplicate floor-pinned lottery targets (parity with
        # enhance_trade_setup).
        targets = _prune_floor_pinned_targets(targets)
        primary = _select_primary_target(
            targets,
            min_rr=1.5,
        )
        if primary is None:
            continue
        # Flag the primary so qualification's EV uses the primary target's
        # probability (matching production's enhance_trade_setup).
        for t in targets:
            t["is_primary"] = t is primary
        per_dir[direction] = {"stop": stop, "targets": targets, "primary": primary}

    available = set(per_dir.keys())
    if not available:
        return []

    action = _choose_recommended_action(confidences["long"], confidences["short"], config, available)

    out: list[dict] = []
    for direction, data in per_dir.items():
        targets, primary, stop = data["targets"], data["primary"], data["stop"]
        setup_conflicts = list(conflicts)
        if len(targets) < 3:
            setup_conflicts.append("target-availability: Fewer than 3 valid S/R targets available")
        risk_level = _risk_level_from_conflicts(setup_conflicts)
        rr = float(primary["rr_ratio"])
        target_price = float(primary["price"])

        setup_ns = SimpleNamespace(
            rr_ratio=rr,
            confidence_score=confidences[direction],
            recommended_action=action,
            risk_level=risk_level,
            targets=targets,
            direction=direction,
            target=target_price,
            stop_loss=stop,
            entry_price=entry,
        )
        # meets_core = clears every gate EXCEPT the cross-sectional momentum
        # percentile, which can only be assigned once all tickers' setups for a
        # week are known. run_backtest ranks momentum and finalizes `qualified`.
        core_config = {**activation, "min_momentum_percentile": 0.0}
        meets_core = setup_qualifies(setup_ns, core_config)
        best_prob = best_target_probability(setup_ns)
        out.append({
            "direction": direction,
            "entry": entry,
            "stop": stop,
            "target": target_price,
            "rr": rr,
            "confidence": confidences[direction],
            "primary_prob": float(primary["probability"]),
            "best_prob": best_prob,
            "momentum": mom_12_1,
            "meets_core": meets_core,
            "action": action,
            "risk_level": risk_level,
            "target_model": target_model,
            "primary_sources": list(primary.get("sr_sources") or []),
            "primary_strength": float(primary.get("sr_strength", 0.0)),
            "primary_rejection_count": int(
                primary.get("sr_rejection_count", 0) or 0
            ),
            "primary_last_rejection_age": primary.get("sr_last_rejection_age"),
            "primary_distance_atr": float(
                primary.get("distance_atr_multiple", 0.0)
            ),
            "raw_level_count": len(sr_levels),
            "gate_level_count": len(gate_levels),
        })
    return out


def _stop_fill_r(direction: str, entry: float, stop: float, bar) -> float:
    """Realized R when the stop is hit on ``bar``: filled at the stop, or at the
    bar's open when price gapped through it — so a gap can lose more than −1R,
    matching real fills. Targets are never filled better than their level, so
    gap modeling only ever makes results more conservative."""
    risk = abs(entry - stop)
    if risk <= 0 or entry <= 0:
        return -1.0
    if direction == "long":
        fill = min(stop, bar.open)
        return (fill - entry) / risk
    fill = max(stop, bar.open)
    return (entry - fill) / risk


def _risk_and_stop_day(
    direction: str, entry: float, stop: float, forward: list, horizon: int
) -> tuple[float, int | None]:
    """``(risk_pct, stop_day)`` from the bars after detection: the 1R stop
    distance as a fraction of entry, and the 1-based trading day the initial
    stop was first pierced within the horizon (None if never). Feeds the cost
    conversion and the time-exit hold accounting."""
    long = direction == "long"
    risk_pct = abs(entry - stop) / entry if entry else 0.0
    for i, r in enumerate(forward[:horizon]):
        if (r.low <= stop) if long else (r.high >= stop):
            return risk_pct, i + 1
    return risk_pct, None


def _time_exits(
    direction: str, entry: float, stop: float, forward: list, horizons
) -> dict[int, float]:
    """Realized R per hold-N-days exit, in one pass over the post-entry bars.

    The initial stop stays active (fill at the stop level → −1R); otherwise the
    trade exits at the day-N close (the last available close when history ends
    early). No target, no trailing — the classic momentum implementation: buy,
    hold ~N days, re-rank. Conservative bar logic: a bar that pierces the stop
    is a loss before that bar's close counts.
    """
    long = direction == "long"
    risk = abs(entry - stop) / entry if entry else 0.0
    if risk <= 0:
        return {int(n): 0.0 for n in horizons}
    bars = forward[: max(int(n) for n in horizons)]
    if not bars:
        return {int(n): 0.0 for n in horizons}

    stop_day: int | None = None  # 1-based trading day the stop was pierced
    stop_r = -1.0
    closes: list[float] = []
    for i, r in enumerate(bars):
        if (r.low <= stop) if long else (r.high >= stop):
            stop_day = i + 1
            stop_r = _stop_fill_r(direction, entry, stop, r)
            break
        closes.append(r.close)

    result: dict[int, float] = {}
    for h in horizons:
        n = int(h)
        if stop_day is not None and stop_day <= n:
            result[n] = stop_r
        else:
            # closes can't be empty here: an empty closes means the stop hit on
            # day 1, which the branch above catches for every n >= 1.
            c = closes[min(n, len(closes)) - 1]
            move = (c - entry) / entry if long else (entry - c) / entry
            result[n] = move / risk
    return result


def _replay_ticker(
    symbol: str,
    records: list,
    config: dict,
    activation: dict,
    benchmark_closes: dict[date, float] | None = None,
    target_model: str = PRODUCTION_GTL_TARGET_MODEL,
    cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> list[dict]:
    """Walk one ticker at the selected cadence and resolve each setup outcome."""
    cadence = validate_backtest_cadence(cadence)
    step_sessions = backtest_step_sessions(cadence)
    candidates: list[dict] = []
    n = len(records)
    if n < MIN_LOOKBACK + HORIZON:
        return candidates

    for i in range(MIN_LOOKBACK - 1, n - HORIZON, step_sessions):
        window = records[: i + 1]
        forward = records[i + 1 :]
        forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
        closes = [float(r.close) for r in window]
        dates = [r.date for r in window]
        residual_momentum = _residual_momentum_12_1(
            dates, closes, len(window) - 1, benchmark_closes
        )
        vol_6m = _realized_vol_6m(closes, len(window) - 1)

        setups = _window_setups(
            window,
            config,
            activation,
            target_model=target_model,
        )
        for s in setups:
            outcome, outcome_date = evaluate_setup_against_bars(
                s["direction"], s["stop"], s["target"], forward_bars, HORIZON
            )
            if outcome is None:
                continue
            # Trading days from detection to resolution (expired = full horizon).
            hold_days = next(
                (idx + 1 for idx, r in enumerate(forward[:HORIZON]) if r.date == outcome_date),
                min(HORIZON, len(forward)),
            )
            target_hit = outcome == OUTCOME_TARGET_HIT
            if outcome == OUTCOME_TARGET_HIT:
                realized_r = s["rr"]
            elif outcome in (OUTCOME_STOP_HIT, OUTCOME_AMBIGUOUS):
                # Fill at the stop, or at the open when the bar gapped through it.
                realized_r = _stop_fill_r(
                    s["direction"], s["entry"], s["stop"], forward[hold_days - 1]
                )
            else:  # expired
                realized_r = 0.0
            risk_pct, stop_day = _risk_and_stop_day(
                s["direction"], s["entry"], s["stop"], forward, HORIZON
            )
            time_r = _time_exits(
                s["direction"], s["entry"], s["stop"], forward, TIME_EXIT_DAYS
            )
            iso = records[i].date.isocalendar()
            candidates.append({
                "symbol": symbol,
                "date": records[i].date.isoformat(),
                "iso_week": (iso[0], iso[1]),
                "ranking_period": _ranking_period(records[i].date, cadence),
                "direction": s["direction"],
                "entry": s["entry"],
                "stop": s["stop"],
                "target": s["target"],
                "rr": s["rr"],
                "confidence": s["confidence"],
                "primary_prob": s["primary_prob"],
                "best_prob": s["best_prob"],
                "momentum": s["momentum"],
                "residual_momentum": residual_momentum,
                "vol_6m": vol_6m,
                "meets_core": s["meets_core"],
                # Gate fields the ablation recomputes floors from — without them
                # every candidate looks NEUTRAL and the ablation rows collapse.
                "action": s["action"],
                "risk_level": s["risk_level"],
                "target_model": s["target_model"],
                "primary_sources": s["primary_sources"],
                "primary_strength": s["primary_strength"],
                "primary_rejection_count": s["primary_rejection_count"],
                "primary_last_rejection_age": s["primary_last_rejection_age"],
                "primary_distance_atr": s["primary_distance_atr"],
                "raw_level_count": s["raw_level_count"],
                "gate_level_count": s["gate_level_count"],
                "outcome": outcome,
                "target_hit": target_hit,
                "realized_r": realized_r,
                "hold_days": hold_days,
                "stop_day": stop_day,
                "risk_pct": risk_pct,
                "time_r": time_r,
            })
    return candidates


def _bucket_stats(cands: list[dict]) -> dict:
    wins = sum(1 for c in cands if c["target_hit"])
    losses = sum(1 for c in cands if c["outcome"] in (OUTCOME_STOP_HIT, OUTCOME_AMBIGUOUS))
    expired = sum(1 for c in cands if c["outcome"] not in (OUTCOME_TARGET_HIT, OUTCOME_STOP_HIT, OUTCOME_AMBIGUOUS))
    decided = wins + losses
    rs = [c["realized_r"] for c in cands]
    net_rs = [c["realized_r"] - _cost_r(c) for c in cands]
    holds = [c["hold_days"] for c in cands if c.get("hold_days")]
    avg_hold = sum(holds) / len(holds) if holds else None
    net_avg = sum(net_rs) / len(net_rs) if net_rs else None
    return {
        "total": len(cands),
        "wins": wins,
        "losses": losses,
        "expired": expired,
        "hit_rate": round(wins / decided * 100, 1) if decided else None,
        "avg_r": round(sum(rs) / len(rs), 3) if rs else None,
        "total_r": round(sum(rs), 2) if rs else None,
        "net_avg_r": round(net_avg, 3) if net_avg is not None else None,
        "net_total_r": round(sum(net_rs), 2) if net_rs else None,
        "best_r": round(max(rs), 2) if rs else None,
        "worst_r": round(min(rs), 2) if rs else None,
        "avg_hold_days": round(avg_hold, 1) if avg_hold is not None else None,
        # Capital efficiency: net expectancy per trading day the capital is tied up.
        "net_r_per_day": (
            round(net_avg / avg_hold, 4) if net_avg is not None and avg_hold else None
        ),
        **_robustness_stats(net_rs),
    }


def _robustness_stats(net_rs: list[float]) -> dict:
    """Distribution-shape stats: the median (typical) trade, gross wins vs
    losses, and the expectancy with the top 5% of winners removed — the direct
    test of whether the edge depends on a handful of outliers."""
    if not net_rs:
        return {"median_net_r": None, "profit_factor": None, "net_avg_r_ex_top5": None}
    gains = sum(r for r in net_rs if r > 0)
    losses_abs = -sum(r for r in net_rs if r < 0)
    trimmed = sorted(net_rs, reverse=True)[math.ceil(len(net_rs) * 0.05):]
    return {
        "median_net_r": round(statistics.median(net_rs), 3),
        "profit_factor": round(gains / losses_abs, 2) if losses_abs > 0 else None,
        "net_avg_r_ex_top5": (
            round(sum(trimmed) / len(trimmed), 3) if trimmed else None
        ),
    }


def _target_model_diagnostics(candidates: list[dict], target_model: str) -> dict:
    """Compact target-source diagnostics for the selected supported model."""
    source_counts: dict[str, int] = defaultdict(int)
    round_only = 0
    strengths: list[float] = []
    distances: list[float] = []
    rejections: list[int] = []
    raw_counts: list[int] = []
    gate_counts: list[int] = []
    for cand in candidates:
        sources = list(cand.get("primary_sources") or [])
        for source in sources:
            source_counts[str(source)] += 1
        if set(sources) == {"round_number"}:
            round_only += 1
        strengths.append(float(cand.get("primary_strength", 0.0)))
        distances.append(float(cand.get("primary_distance_atr", 0.0)))
        rejections.append(int(cand.get("primary_rejection_count", 0) or 0))
        raw_counts.append(int(cand.get("raw_level_count", 0) or 0))
        gate_counts.append(int(cand.get("gate_level_count", 0) or 0))

    def avg(values: list[float] | list[int]) -> float | None:
        return round(sum(values) / len(values), 3) if values else None

    return {
        "target_model": target_model,
        "target_model_label": BACKTEST_TARGET_MODELS[target_model],
        "candidate_count": len(candidates),
        "primary_source_counts": dict(sorted(source_counts.items())),
        "primary_round_only": round_only,
        "primary_strength_100": sum(1 for value in strengths if value >= 100.0),
        "avg_primary_strength": avg(strengths),
        "avg_primary_distance_atr": avg(distances),
        "avg_primary_rejection_count": avg(rejections),
        "avg_raw_level_count": avg(raw_counts),
        "avg_gate_level_count": avg(gate_counts),
    }


# The fixed take-profit and trailing-stop sweeps were retired 2026-07: swept
# TPs never found an interior optimum (momentum's edge lives in the right tail)
# and wide trails converged to the hold-to-horizon exit, so the time-exit sweep
# is the exit-decision surface.

# Hold-N-days exits (initial stop stays active, exit at the day-N close) — the
# classic cross-sectional momentum implementation: buy, hold ~a month, re-rank.
TIME_EXIT_DAYS = (5, 10, 21, 30)

# Assumed transaction cost per side as a fraction of notional (commission +
# slippage). Aggregates report gross and net side by side; net subtracts a full
# round trip, converted into R via the setup's stop distance (the 1R unit).
COST_PER_SIDE = 0.001


def _cost_r(cand: dict) -> float:
    """Round-trip transaction cost in R units: two sides over the 1R stop
    distance. 0 when the candidate carries no usable risk_pct."""
    risk = cand.get("risk_pct") or 0.0
    return (2.0 * COST_PER_SIDE) / risk if risk > 0 else 0.0


def _time_exit_bucket(cands: list[dict], hold_days: int) -> dict:
    """Stats for the hold-``hold_days`` exit: initial stop active, otherwise out
    at the day-N close. Each candidate carries its realized R per hold length in
    ``time_r``; a "win" is an exit in profit (R > 0). The realized hold is the
    full N days unless the stop cut it short (``stop_day``)."""
    rows = [
        (
            c["time_r"][hold_days],
            _cost_r(c),
            min(hold_days, c.get("stop_day") or hold_days),
        )
        for c in cands
        if c.get("time_r", {}).get(hold_days) is not None
    ]
    total = len(rows)
    rs = [r for r, _, _ in rows]
    net_rs = [r - cost for r, cost, _ in rows]
    holds = [h for _, _, h in rows]
    wins = sum(1 for r in rs if r > 0)
    avg_hold = sum(holds) / total if total else None
    net_avg = sum(net_rs) / total if total else None
    return {
        "hold_days": hold_days,
        "total": total,
        "wins": wins,
        "win_rate": round(wins / total * 100, 1) if total else None,
        "avg_r": round(sum(rs) / total, 3) if total else None,
        "total_r": round(sum(rs), 2) if total else None,
        "net_avg_r": round(net_avg, 3) if net_avg is not None else None,
        "net_total_r": round(sum(net_rs), 2) if total else None,
        "best_r": round(max(rs), 2) if rs else None,
        "worst_r": round(min(rs), 2) if rs else None,
        "avg_hold_days": round(avg_hold, 1) if avg_hold is not None else None,
        "net_r_per_day": (
            round(net_avg / avg_hold, 4) if net_avg is not None and avg_hold else None
        ),
        **_robustness_stats(net_rs),
    }


# ---------------------------------------------------------------------------
# Cross-sectional signal evaluation (factor information-coefficient)
# ---------------------------------------------------------------------------

def _weekly_asof_indices(records: list) -> list[int]:
    """Index of the last bar in each ISO week — the weekly rebalance as-of bars.

    Keying on the calendar week (not the raw bar index) makes every ticker's
    as-of dates line up, so the cross-section on a given week is comparable.
    """
    last_by_week: dict[tuple[int, int], int] = {}
    for idx, r in enumerate(records):
        iso = r.date.isocalendar()
        last_by_week[(iso[0], iso[1])] = idx
    return sorted(last_by_week.values())


def _residual_momentum_12_1(
    dates: list[date],
    closes: list[float],
    i: int,
    benchmark_closes: dict[date, float] | None,
) -> float | None:
    """12-1 momentum after removing the stock's linear benchmark exposure.

    This is a practical beta-adjusted residual momentum approximation: estimate
    beta from daily stock/benchmark returns over the same formation window
    (12 months ending 1 month ago), then rank on cumulative stock return minus
    beta * benchmark return. We deliberately do not subtract a fitted intercept:
    with an intercept estimated over the same window, the arithmetic residuals
    sum to ~zero by construction, which would destroy the signal.
    """
    if not benchmark_closes or i - 252 < 0:
        return None

    stock_rets: list[float] = []
    market_rets: list[float] = []
    # Same daily intervals as mom_12_1: close[i-252] -> close[i-21].
    for k in range(i - 251, i - 20):
        prev_close = closes[k - 1]
        bench_prev = benchmark_closes.get(dates[k - 1])
        bench_cur = benchmark_closes.get(dates[k])
        if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0:
            continue
        stock_rets.append(closes[k] / prev_close - 1.0)
        market_rets.append(bench_cur / bench_prev - 1.0)

    if len(stock_rets) < 100:
        return None
    mean_market = sum(market_rets) / len(market_rets)
    mean_stock = sum(stock_rets) / len(stock_rets)
    var_market = sum((x - mean_market) ** 2 for x in market_rets)
    if var_market <= 0:
        return None
    cov = sum((stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) for k in range(len(stock_rets)))
    beta = cov / var_market
    return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets)))


def _realized_vol_6m(closes: list[float], i: int) -> float | None:
    """126-trading-day realized daily volatility at point-in-time index ``i``."""
    return compute_realized_vol_6m(closes[: i + 1])


def _signal_values(
    dates: list[date],
    closes: list[float],
    highs: list[float],
    i: int,
    benchmark_closes: dict[date, float] | None = None,
) -> dict[str, float]:
    """Point-in-time candidate signals at as-of index ``i`` (price-only).

    Momentum factors follow the standard "skip the last month" convention
    (return up to ~1 month ago) to avoid the short-term reversal effect, which
    ``reversal_1m`` isolates on purpose — we expect its IC to be negative if the
    universe mean-reverts. ``trend_200`` is price vs its 200-bar SMA. ``high_52w``
    is closeness to the trailing 52-week high (George/Hwang anchoring effect:
    higher = nearer the high, expect positive IC). ``vol_6m`` is 126-day realized
    volatility (expect negative IC if the low-volatility anomaly holds).
    """
    out: dict[str, float] = {}
    if i - 252 >= 0 and closes[i - 252] > 0:
        out["mom_12_1"] = closes[i - 21] / closes[i - 252] - 1.0
        residual = _residual_momentum_12_1(dates, closes, i, benchmark_closes)
        if residual is not None:
            out["mom_12_1_resid"] = residual
    if i - 126 >= 0 and closes[i - 126] > 0:
        out["mom_6_1"] = closes[i - 21] / closes[i - 126] - 1.0
    if i - 63 >= 0 and closes[i - 63] > 0:
        out["mom_3_1"] = closes[i - 21] / closes[i - 63] - 1.0
    if i - 21 >= 0 and closes[i - 21] > 0:
        out["reversal_1m"] = closes[i] / closes[i - 21] - 1.0
    if i - 199 >= 0:
        sma = sum(closes[i - 199 : i + 1]) / 200.0
        if sma > 0:
            out["trend_200"] = closes[i] / sma - 1.0
    if i - 251 >= 0:
        high_52w = max(highs[i - 251 : i + 1])
        if high_52w > 0:
            out["high_52w"] = closes[i] / high_52w
    vol_6m = _realized_vol_6m(closes, i)
    if vol_6m is not None:
        out["vol_6m"] = vol_6m
    return out


def _accumulate_signal_series(
    records: list,
    collected: dict,
    benchmark_closes: dict[date, float] | None = None,
) -> None:
    """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO
    week into ``collected[name][week_key]``. Forward return is close-to-close over
    HORIZON trading days. Mutates ``collected`` (a dict of dict of list)."""
    n = len(records)
    if n < HORIZON + 21:
        return
    closes = [float(r.close) for r in records]
    highs = [float(r.high) for r in records]
    dates = [r.date for r in records]
    for i in _weekly_asof_indices(records):
        j = i + HORIZON
        if j >= n or closes[i] <= 0:
            continue
        fwd = closes[j] / closes[i] - 1.0
        iso = records[i].date.isocalendar()
        week_key = (iso[0], iso[1])
        for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items():
            collected[name][week_key].append((val, fwd))


def _rank(xs: list[float]) -> list[float]:
    """Average (tie-corrected) ranks, 1-based."""
    order = sorted(range(len(xs)), key=lambda k: xs[k])
    ranks = [0.0] * len(xs)
    i = 0
    while i < len(xs):
        j = i
        while j + 1 < len(xs) and xs[order[j + 1]] == xs[order[i]]:
            j += 1
        avg_rank = (i + j) / 2.0 + 1.0
        for k in range(i, j + 1):
            ranks[order[k]] = avg_rank
        i = j + 1
    return ranks


def _pearson(a: list[float], b: list[float]) -> float | None:
    n = len(a)
    if n < 3:
        return None
    ma, mb = sum(a) / n, sum(b) / n
    va = sum((x - ma) ** 2 for x in a)
    vb = sum((y - mb) ** 2 for y in b)
    if va <= 0 or vb <= 0:
        return None
    cov = sum((a[k] - ma) * (b[k] - mb) for k in range(n))
    return cov / math.sqrt(va * vb)


def _spearman(xs: list[float], ys: list[float]) -> float | None:
    """Rank correlation = Pearson on the ranks. None if too few/degenerate."""
    if len(xs) < 3:
        return None
    return _pearson(_rank(xs), _rank(ys))


def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None:
    """Mean forward return of the top signal-quintile minus the bottom quintile."""
    n = len(pairs)
    if n < 10:
        return None
    ordered = sorted(pairs, key=lambda p: p[0])
    k = n // 5
    top = ordered[-k:]
    bottom = ordered[:k]
    return sum(p[1] for p in top) / k - sum(p[1] for p in bottom) / k


def _week_ordinal(week_key: tuple[int, int]) -> int:
    """Monotonic absolute week number from an (ISO year, ISO week) key."""
    year, week = week_key
    return year * 53 + week


def _nonoverlapping_weeks(
    week_keys: list[tuple[int, int]], stride: int
) -> list[tuple[int, int]]:
    """Thin to weeks at least ``stride`` apart so their forward windows don't
    overlap — greedy earliest-first. Removes the autocorrelation that would
    otherwise inflate the IC t-stat across adjacent weekly rebalances."""
    kept: list[tuple[int, int]] = []
    last: int | None = None
    for wk in sorted(week_keys, key=_week_ordinal):
        o = _week_ordinal(wk)
        if last is None or o - last >= stride:
            kept.append(wk)
            last = o
    return kept


def _signal_evaluation(collected: dict) -> list[dict]:
    """Per-signal factor diagnostics, one row per candidate signal:

      mean_ic               average rank-IC (Spearman of signal vs fwd ret)
      ic_t_stat             mean_ic / stderr — is the IC reliably non-zero?
      ic_positive_pct       share of windows the IC is positive (consistency)
      mean_quintile_spread  avg top-minus-bottom-quintile forward return
      reliable              True once there are >= MIN_RELIABLE_PERIODS windows

    IC is measured on NON-OVERLAPPING forward windows (weeks thinned to ~HORIZON
    apart) so the t-stat isn't inflated by autocorrelation. A signal with no edge
    lands near IC 0 / spread 0; one with too few independent windows is flagged
    unreliable rather than trusted on a lucky handful.
    """
    stride = max(1, round(HORIZON / 5))  # ISO weeks spanned by the forward window
    rows: list[dict] = []
    for name in sorted(collected):
        weeks_map = collected[name]
        usable = [wk for wk, recs in weeks_map.items() if len(recs) >= MIN_CROSS_SECTION]
        kept = _nonoverlapping_weeks(usable, stride)
        ics: list[float] = []
        spreads: list[float] = []
        sizes: list[int] = []
        for wk in kept:
            recs = weeks_map[wk]
            ic = _spearman([r[0] for r in recs], [r[1] for r in recs])
            if ic is not None:
                ics.append(ic)
            spread = _quintile_spread(recs)
            if spread is not None:
                spreads.append(spread)
            sizes.append(len(recs))
        if not ics:
            continue
        mean_ic = sum(ics) / len(ics)
        if len(ics) > 1:
            std = math.sqrt(sum((x - mean_ic) ** 2 for x in ics) / (len(ics) - 1))
        else:
            std = 0.0
        t_stat = mean_ic / std * math.sqrt(len(ics)) if std > 0 else None
        rows.append({
            "signal": name,
            "weeks": len(ics),
            "avg_cross_section": round(sum(sizes) / len(sizes), 1) if sizes else None,
            "mean_ic": round(mean_ic, 4),
            "ic_t_stat": round(t_stat, 2) if t_stat is not None else None,
            "ic_positive_pct": round(sum(1 for x in ics if x > 0) / len(ics) * 100, 1),
            "mean_quintile_spread": round(sum(spreads) / len(spreads), 4) if spreads else None,
            "reliable": len(ics) >= MIN_RELIABLE_PERIODS,
        })
    rows.sort(key=lambda r: r["mean_ic"], reverse=True)
    return rows


def _signal_series(records: list, benchmark_closes: dict[date, float] | None = None) -> dict:
    """Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict
    — no defaultdict/lambda — so it can cross a process boundary."""
    tmp: dict = defaultdict(lambda: defaultdict(list))
    _accumulate_signal_series(records, tmp, benchmark_closes)
    return {name: dict(weeks) for name, weeks in tmp.items()}


def _replay_and_signals(
    symbol: str,
    columns: tuple,
    config: dict,
    activation: dict,
    benchmark_closes: dict[date, float] | None = None,
    target_model: str = PRODUCTION_GTL_TARGET_MODEL,
    cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> tuple[list[dict], dict]:
    """The CPU-bound per-ticker work, as a top-level (picklable) function so it can
    run in a worker process. Takes primitive column arrays (cheap to pickle),
    rebuilds bar objects, and returns (candidates, signal_series)."""
    date_ords, opens, highs, lows, closes, volumes = columns
    bars = [
        SimpleNamespace(
            date=date.fromordinal(o), open=op, high=hi, low=lo, close=cl, volume=vo
        )
        for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes)
    ]
    return (
        _replay_ticker(
            symbol,
            bars,
            config,
            activation,
            benchmark_closes,
            target_model,
            cadence,
        ),
        _signal_series(bars, benchmark_closes),
    )


def _replay_candidates_for_period(
    symbol: str,
    columns: tuple,
    config: dict,
    activation: dict,
    benchmark_closes: dict[date, float] | None,
    start_date: date,
    cadence: str = DEFAULT_BACKTEST_CADENCE,
    include_short_candidates: bool = False,
) -> list[dict]:
    """Slim picklable replay used by local event studies.

    Unlike the full report worker it skips factor-series construction and only
    evaluates setup dates on or after ``start_date``. Long-only remains the
    compatibility default. Set ``include_short_candidates`` when the caller
    needs the production-faithful cross-sectional ranking universe; shorts can
    then contribute to percentiles while the portfolio simulator still trades
    only qualified longs.
    """
    date_ords, opens, highs, lows, closes, volumes = columns
    bars = [
        SimpleNamespace(
            date=date.fromordinal(o), open=op, high=hi, low=lo, close=cl, volume=vo
        )
        for o, op, hi, lo, cl, vo in zip(
            date_ords, opens, highs, lows, closes, volumes
        )
    ]
    cadence = validate_backtest_cadence(cadence)
    candidates: list[dict] = []
    for i in range(
        MIN_LOOKBACK - 1,
        len(bars) - HORIZON,
        backtest_step_sessions(cadence),
    ):
        if bars[i].date < start_date:
            continue
        window = bars[: i + 1]
        window_closes = [float(r.close) for r in window]
        window_dates = [r.date for r in window]
        residual_momentum = _residual_momentum_12_1(
            window_dates,
            window_closes,
            len(window) - 1,
            benchmark_closes,
        )
        vol_6m = _realized_vol_6m(window_closes, len(window) - 1)
        iso = bars[i].date.isocalendar()
        for setup in _window_setups(window, config, activation):
            if not include_short_candidates and setup["direction"] != "long":
                continue
            candidates.append({
                "symbol": symbol,
                "date": bars[i].date.isoformat(),
                "iso_week": (iso[0], iso[1]),
                "ranking_period": _ranking_period(bars[i].date, cadence),
                "direction": setup["direction"],
                "entry": setup["entry"],
                "stop": setup["stop"],
                "target": setup["target"],
                "rr": setup["rr"],
                "confidence": setup["confidence"],
                "primary_prob": setup["primary_prob"],
                "best_prob": setup["best_prob"],
                "momentum": setup["momentum"],
                "residual_momentum": residual_momentum,
                "vol_6m": vol_6m,
                "meets_core": setup["meets_core"],
                "action": setup["action"],
                "risk_level": setup["risk_level"],
            })
    return candidates


def _backtest_worker_count() -> int:
    """How many worker processes to replay tickers across. Capped to cpu_count-1
    so a core stays free for the web server; 1 means sequential."""
    configured = int(getattr(settings, "backtest_workers", 4))
    if configured <= 1:
        return 1
    cpu = os.cpu_count() or 1
    return max(1, min(configured, cpu - 1))


def _offline_snapshot_mode() -> bool:
    return os.getenv("BACKTEST_SNAPSHOT_OFFLINE", "").strip().lower() in {"1", "true", "yes", "on"}


async def _load_benchmark_closes_for_backtest(
    db: AsyncSession, *, days: int | None = None, refresh: bool = True
) -> dict[date, float]:
    from app.services.benchmark_service import load_benchmark_closes, refresh_benchmark_prices

    if refresh and not _offline_snapshot_mode():
        if days is None:
            await refresh_benchmark_prices(db)
        else:
            await refresh_benchmark_prices(db, days=days)
    return await load_benchmark_closes(db)


def _mp_context():
    """A start method safe to use from the threaded asyncio server: ``forkserver``
    (workers forked from a clean, single-threaded server — avoids the
    fork-with-threads deadlock) when available, else ``fork``. Returns None on
    spawn-only platforms (Windows), where the caller falls back to a thread."""
    methods = multiprocessing.get_all_start_methods()
    for method in ("forkserver", "fork"):
        if method in methods:
            return multiprocessing.get_context(method)
    if (
        _offline_snapshot_mode()
        and os.getenv("BACKTEST_ALLOW_SPAWN", "").strip().lower() in {"1", "true", "yes", "on"}
        and "spawn" in methods
    ):
        return multiprocessing.get_context("spawn")
    return None


async def _fetch_columns(db: AsyncSession, symbol: str) -> tuple | None:
    """Read one ticker's OHLCV and detach it to primitive column arrays in the
    event loop (safe ORM access), ready to hand to a worker. None if no data."""
    records = await query_ohlcv(db, symbol)
    if not records:
        return None
    return (
        [r.date.toordinal() for r in records],
        [float(r.open) for r in records],
        [float(r.high) for r in records],
        [float(r.low) for r in records],
        [float(r.close) for r in records],
        [int(r.volume) for r in records],
    )


def _assign_signal_percentiles(
    candidates: list[dict],
    value_key: str,
    percentile_key: str,
) -> None:
    """Per replay period, rank candidates by ``value_key`` and attach a 0-100
    percentile under ``percentile_key`` (100 = strongest). Missing values get
    None and therefore cannot clear a gate based on that signal."""
    by_period: dict = defaultdict(list)
    for c in candidates:
        if c.get(value_key) is not None:
            # Hand-built/research candidates predating the cadence flag retain
            # the weekly key as a compatibility fallback.
            period = c.get("ranking_period") or c["iso_week"]
            by_period[period].append(c)
    for group in by_period.values():
        ordered = sorted(group, key=lambda c: c[value_key])
        n = len(ordered)
        for rank, c in enumerate(ordered):
            c[percentile_key] = (rank / (n - 1) * 100.0) if n > 1 else 100.0
    for c in candidates:
        c.setdefault(percentile_key, None)


def _assign_momentum_percentiles(candidates: list[dict]) -> None:
    """Per replay period, rank candidates by 12-1 momentum and attach a
    0-100 ``momentum_percentile`` (100 = highest momentum in the universe that
    period). Candidates whose momentum is unknown (insufficient lookback) get None
    and therefore can't clear a momentum gate. Mutates ``candidates``."""
    _assign_signal_percentiles(candidates, "momentum", "momentum_percentile")


def _assign_residual_momentum_percentiles(candidates: list[dict]) -> None:
    """Residual-momentum percentile promoted to production activation ranking."""
    _assign_signal_percentiles(
        candidates, "residual_momentum", RESIDUAL_PERCENTILE_KEY
    )


def _assign_low_volatility_percentiles(candidates: list[dict]) -> None:
    """Per replay period, attach volatility ranks where 100 = lowest 6-month vol."""
    _assign_signal_percentiles(candidates, "vol_6m", VOL_PERCENTILE_KEY)
    for c in candidates:
        raw = c.get(VOL_PERCENTILE_KEY)
        c[LOW_VOL_PERCENTILE_KEY] = (100.0 - raw) if raw is not None else None


def _assign_activation_momentum_percentiles(candidates: list[dict]) -> None:
    """Production activation rank: residual 12-1 when available, raw fallback.

    The raw fallback mirrors the live scanner's behavior when benchmark history
    is unavailable. In normal backtests, SPY is loaded and this is residual.
    """
    for c in candidates:
        c[PRODUCTION_PERCENTILE_KEY] = (
            c.get(RESIDUAL_PERCENTILE_KEY)
            if c.get(RESIDUAL_PERCENTILE_KEY) is not None
            else c.get(RAW_PERCENTILE_KEY)
        )


def _assign_weighted_blend(
    candidates: list[dict],
    output_key: str,
    primary_key: str,
    primary_weight: float,
    secondary_key: str,
) -> None:
    secondary_weight = 1.0 - primary_weight
    for c in candidates:
        primary = c.get(primary_key)
        secondary = c.get(secondary_key)
        c[output_key] = (
            primary * primary_weight + secondary * secondary_weight
            if primary is not None and secondary is not None
            else None
        )


def _assign_residual_low_vol_blend(candidates: list[dict]) -> None:
    """Research rank: mostly residual momentum, with lower-vol names preferred."""
    _assign_weighted_blend(
        candidates,
        RESIDUAL_LOW_VOL_BLEND_KEY,
        PRODUCTION_PERCENTILE_KEY,
        0.7,
        LOW_VOL_PERCENTILE_KEY,
    )


def _assign_residual_high_vol_blend(candidates: list[dict]) -> None:
    """Research ranks: residual momentum blended with higher-vol preference."""
    for output_key, residual_weight in (
        (RESIDUAL_HIGH_VOL_BLEND_90_10_KEY, 0.9),
        # The production ordering weight comes from momentum_service so the
        # simulated production rank cannot drift from the live strategy_rank.
        (RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, STRATEGY_RANK_MOMENTUM_WEIGHT),
        (RESIDUAL_HIGH_VOL_BLEND_KEY, 0.7),
        (RESIDUAL_HIGH_VOL_BLEND_60_40_KEY, 0.6),
    ):
        _assign_weighted_blend(
            candidates,
            output_key,
            PRODUCTION_PERCENTILE_KEY,
            residual_weight,
            VOL_PERCENTILE_KEY,
        )


def _momentum_qualifies(cand: dict, threshold: float) -> bool:
    """Whether a candidate clears the floors (meets_core) and the momentum gate.
    Threshold 0 disables the momentum gate (floors only). The gate is long-only:
    while it's active, shorts (fighting the trend) never qualify."""
    if not cand["meets_core"]:
        return False
    if threshold <= 0:
        return True
    if cand["direction"] == "short":
        return False
    mp = cand.get(PRODUCTION_PERCENTILE_KEY)
    return mp is not None and mp >= threshold


def _gate_ablation(candidates: list[dict], activation: dict, threshold: float) -> list[dict]:
    """Which floors earn their keep: re-qualify the same candidates at the
    current momentum cutoff with one floor removed per row (long-only
    throughout, matching the live gate).

    ``all_floors`` uses the stored ``meets_core`` so it reproduces the qualified
    set exactly; the ablation rows recompute the remaining floors from stored
    candidate fields with the same comparisons as
    ``qualification.setup_qualifies``. Optional tighteners (high-conviction /
    conflict exclusion), when enabled, stay applied in every ablation row so
    only the named floor varies.
    """
    min_rr = float(activation.get("min_rr", 0.0))
    min_conf = float(activation.get("min_confidence", 0.0))
    exclude_neutral = bool(activation.get("exclude_neutral", False))
    require_high = bool(activation.get("require_high_conviction", False))
    exclude_conflicts = bool(activation.get("exclude_conflicts", False))

    def momentum_ok(c: dict) -> bool:
        # Mirrors the momentum part of _momentum_qualifies: long-only while the
        # gate is active; threshold 0 disables it (shorts pass too).
        if threshold <= 0:
            return True
        if c["direction"] == "short":
            return False
        mp = c.get(PRODUCTION_PERCENTILE_KEY)
        return mp is not None and mp >= threshold

    def rr_ok(c: dict) -> bool:
        return c["rr"] >= min_rr

    def conf_ok(c: dict) -> bool:
        return (c["confidence"] or 0.0) >= min_conf

    def neutral_ok(c: dict) -> bool:
        if not exclude_neutral:
            return True
        action_direction = _action_direction(c.get("action"))
        return action_direction != "neutral" and action_direction == c["direction"]

    def tighteners_ok(c: dict) -> bool:
        if require_high and (c.get("action") or "") not in HIGH_CONVICTION_ACTIONS:
            return False
        if exclude_conflicts and (c.get("risk_level") or "") != "Low":
            return False
        return True

    def core_ok(c: dict) -> bool:
        return bool(c["meets_core"])

    variants: list[tuple[str, list]] = [
        ("all_floors", [core_ok]),
        ("no_confidence_floor", [rr_ok, neutral_ok, tighteners_ok]),
        ("no_rr_floor", [conf_ok, neutral_ok, tighteners_ok]),
        ("no_neutral_exclusion", [rr_ok, conf_ok, tighteners_ok]),
        ("momentum_only", []),
    ]
    # Grade each variant under BOTH exit models: the target/stop outcome
    # (_bucket_stats) and the hold-to-horizon time exit. A floor that pays under
    # the target model may be meaningless once the exit is a fixed hold — the
    # hold_* columns are what a time-exit gate decision should read.
    hold_days = max(TIME_EXIT_DAYS)
    rows: list[dict] = []
    for name, checks in variants:
        matching = [
            c for c in candidates
            if momentum_ok(c) and all(check(c) for check in checks)
        ]
        hold = _time_exit_bucket(matching, hold_days)
        rows.append({
            "variant": name,
            **_bucket_stats(matching),
            "hold_days": hold_days,
            "hold_avg_r": hold["avg_r"],
            "hold_net_avg_r": hold["net_avg_r"],
            "hold_total_r": hold["total_r"],
        })
    return rows


# ---------------------------------------------------------------------------
# Portfolio simulation
# ---------------------------------------------------------------------------

# Book parameters: fixed starting capital, a capped number of concurrent
# positions (one per ticker), fixed-fractional risk sizing with a no-leverage
# notional cap, and the same per-side cost as the per-trade tables. Entries are
# the QUALIFIED setups at their detection close, best momentum first while
# slots and cash allow.
SIM_STARTING_CAPITAL = 10_000.0
SIM_MAX_POSITIONS = 10
SIM_RISK_PER_TRADE = 0.01  # fraction of equity risked per position (entry→stop)
SIM_NOTIONAL_CAP = 0.20    # max fraction of equity per position (no margin)
# The "atr_trail3" research policy's trail width. Must equal the live default
# (paper_trade_service.DEFAULT_ATR_MULTIPLIER) — enforced by the parity test.
# The production portfolio-monitor row additionally follows the *runtime* Admin
# exit policy, so tuning it live is reflected in the next backtest run.
ATR_TRAIL_MULTIPLIER = 3.0
# How live Admin exit modes map onto simulator exit policies. "trailing"
# (percent trail) has no simulator counterpart and falls back to the plain
# hold-to-horizon book; the row's live_exit_mode field keeps that visible.
LIVE_EXIT_MODE_TO_SIM = {
    "atr_trailing": "atr_trail3",
    "time": "hold",
    "target": "target",
    "trailing": "hold",
}


def _simulate_portfolio(
    candidates: list[dict],
    prices: dict[str, tuple],
    spy_closes: dict | None,
    exit_policy: str,
    hold_days: int,
    *,
    qualified_fn: Callable[[dict], bool] | None = None,
    ranking_key: str = PRODUCTION_PERCENTILE_KEY,
    max_positions: int = SIM_MAX_POSITIONS,
    risk_per_trade: float = SIM_RISK_PER_TRADE,
    atr_trail_multiplier: float = ATR_TRAIL_MULTIPLIER,
    cost_per_side: float = COST_PER_SIDE,
    reentry_cooldown_sessions: int = 0,
    initial_stop_refresh_fn: (
        Callable[[str, int, float, dict, Any], float | None] | None
    ) = None,
    post_stop_reentry_fn: (
        Callable[[str, int, dict, Any], dict | None] | None
    ) = None,
    start_date: date | None = None,
    end_date: date | None = None,
    include_curve: bool = False,
    include_trades: bool = False,
) -> dict | None:
    """Replay the qualified setups as ONE capital-constrained book and report
    portfolio economics from the daily equity curve (return, CAGR, drawdown,
    Sharpe) — the numbers the per-setup tables cannot give, because they grade
    every setup as if capital were infinite.

    ``exit_policy``: "target" races the S/R target against the stop with a
    timeout at ``hold_days``; "hold" keeps only the initial stop and exits at
    the ``hold_days``-th close. Research exits add price-derived early exits:
    "sma50", "low20", "technical40", and "atr_trail3". "atr_trail3_target"
    runs the ATR trail *and* the S/R take-profit together — the trade ends at
    whichever comes first. Stops fill at the worse of stop or open (gaps
    modeled); positions still open at the end are closed at their last mark.
    ``reentry_cooldown_sessions`` blocks a ticker for that many market sessions
    after an initial-stop loss. Profitable trailing-stop exits do not trigger
    it. ``initial_stop_refresh_fn`` may supply a lower, point-in-time valid long
    stop when the active initial stop is touched; the replacement is still
    checked against the same bar. ``post_stop_reentry_fn`` turns an initial
    stop-out into a stateful episode and is the only path by which that ticker
    can re-enter until the callback emits a new candidate. Returns None when
    there is nothing to trade. ``cost_per_side`` is charged on entry and exit
    and therefore changes both cash availability and subsequent position sizing.
    """
    cost_rate = float(cost_per_side)
    if not 0.0 <= cost_rate < 1.0:
        raise ValueError("cost_per_side must be between 0 (inclusive) and 1")
    if qualified_fn is None:
        def _default_qualified(c: dict) -> bool:
            return bool(c.get("qualified"))

        qualified_fn = _default_qualified

    entries_by_ord: dict[int, list[dict]] = defaultdict(list)
    start_ord = start_date.toordinal() if start_date is not None else None
    # Explicit simulator/holdout end dates are exclusive split boundaries.
    end_ord = end_date.toordinal() if end_date is not None else None
    for c in candidates:
        if not qualified_fn(c) or c.get("direction") != "long":
            continue
        entry_ord = date.fromisoformat(c["date"]).toordinal()
        if start_ord is not None and entry_ord < start_ord:
            continue
        if end_ord is not None and entry_ord >= end_ord:
            continue  # holdout: entries strictly before the split
        if not c.get("entry") or not c.get("stop"):
            continue
        entries_by_ord[entry_ord].append(c)
    if not entries_by_ord:
        return None

    # Per-symbol bar lookup: date ordinal -> index into the column arrays.
    index_of: dict[str, dict[int, int]] = {
        sym: {o: i for i, o in enumerate(cols[0])} for sym, cols in prices.items()
    }

    first_ord = start_ord if start_ord is not None else min(entries_by_ord)
    calendar = sorted({o for cols in prices.values() for o in cols[0] if o >= first_ord})
    if not calendar:
        return None

    if end_ord is not None:
        # Holdout train book: entries stop at the split, but the calendar would
        # otherwise still run to the last bar in the data — leaving the book in
        # flat cash for the whole test period and deflating CAGR/Sharpe into
        # something that looks like a result and isn't. Every open position
        # resolves within `hold_days` bars of the last entry, so cut there.
        last_entry_ord = max(entries_by_ord)
        cut = bisect.bisect_left(calendar, last_entry_ord) + hold_days + 1
        calendar = calendar[:cut]
        if not calendar:
            return None

    cash = SIM_STARTING_CAPITAL
    positions: dict[str, dict] = {}
    curve: list[tuple[int, float]] = []
    trades: list[dict] = []
    skipped_full = 0
    skipped_cooldown = 0
    cooldown_until_index: dict[str, int] = {}
    stop_refresh_attempts = 0
    stop_refreshes = 0
    stop_refresh_same_bar_hits = 0
    post_stop_states: dict[str, dict] = {}
    post_stop_events = 0
    reentry_events: list[dict] = []
    technical_cache: dict[tuple[str, int], float | None] = {}
    atr_cache: dict[tuple[str, int], float | None] = {}

    def _bar(sym: str, o: int):
        idx = index_of.get(sym, {}).get(o)
        if idx is None:
            return None
        cols = prices[sym]
        return SimpleNamespace(
            idx=idx,
            open=cols[1][idx],
            high=cols[2][idx],
            low=cols[3][idx],
            close=cols[4][idx],
        )

    def _sma(sym: str, idx: int, lookback: int) -> float | None:
        if idx + 1 < lookback:
            return None
        closes = prices[sym][4]
        return sum(closes[idx - lookback + 1 : idx + 1]) / lookback

    def _prior_low(sym: str, idx: int, lookback: int) -> float | None:
        if idx < lookback:
            return None
        lows = prices[sym][3]
        return min(lows[idx - lookback : idx])

    def _technical_score(sym: str, idx: int) -> float | None:
        key = (sym, idx)
        if key in technical_cache:
            return technical_cache[key]
        if idx + 1 < MIN_LOOKBACK:
            technical_cache[key] = None
            return None
        cols = prices[sym]
        try:
            score = compute_technical_from_arrays(
                cols[2][: idx + 1],
                cols[3][: idx + 1],
                cols[4][: idx + 1],
                cols[5][: idx + 1],
            )[0]
            technical_cache[key] = float(score) if score is not None else None
        except Exception:
            technical_cache[key] = None
        return technical_cache[key]

    def _atr(sym: str, idx: int) -> float | None:
        key = (sym, idx)
        if key in atr_cache:
            return atr_cache[key]
        cols = prices[sym]
        try:
            value = compute_atr(
                cols[2][: idx + 1],
                cols[3][: idx + 1],
                cols[4][: idx + 1],
            )["atr"]
            atr_cache[key] = float(value) if value and value > 0 else None
        except Exception:
            atr_cache[key] = None
        return atr_cache[key]

    def _close_trade(sym: str, fill: float, reason: str) -> dict:
        nonlocal cash
        pos = positions.pop(sym)
        proceeds = pos["shares"] * fill
        cost = proceeds * cost_rate
        cash += proceeds - cost
        risk = pos["entry"] - pos["initial_stop"]
        trades.append({
            "symbol": sym,
            "entry_ord": pos["entry_ord"],
            "exit_ord": o,
            "entry": pos["entry"],
            "initial_stop": pos["initial_stop"],
            "active_stop": pos["stop"],
            "fill": fill,
            "pnl": proceeds - pos["shares"] * pos["entry"] - cost - pos["entry_cost"],
            "r": (fill - pos["entry"]) / risk if risk > 0 else 0.0,
            "hold": pos["bars_held"],
            "reason": reason,
            "stop_refreshes": pos["stop_refreshes"],
            "is_reentry": pos["is_reentry"],
            "reentry_wait_sessions": pos["reentry_wait_sessions"],
            "transaction_cost": pos["entry_cost"] + cost,
        })
        return pos

    def _marked_equity() -> float:
        return cash + sum(p["shares"] * p["last_close"] for p in positions.values())

    cooldown_sessions = max(0, int(reentry_cooldown_sessions))
    for calendar_index, o in enumerate(calendar):
        # 1) exits on today's bars (stop intraday, target intraday, time at close)
        for sym in list(positions):
            pos = positions[sym]
            bar = _bar(sym, o)
            if bar is None:
                continue
            pos["bars_held"] += 1
            pos["last_close"] = bar.close
            if bar.low <= pos["stop"]:
                # Same-bar stop+target resolves as the loss (conservative, like
                # the evaluator); gap through the stop fills at the open.
                reason = (
                    "trailing_stop"
                    if pos["stop"] > pos["initial_stop"] + 1e-9
                    else "stop"
                )
                survived_refresh = False
                if reason == "stop" and initial_stop_refresh_fn is not None:
                    stop_refresh_attempts += 1
                    refreshed_stop = initial_stop_refresh_fn(
                        sym, o, float(pos["stop"]), pos, bar
                    )
                    if (
                        refreshed_stop is not None
                        and 0 < float(refreshed_stop) < pos["stop"] - 1e-9
                    ):
                        pos["stop"] = float(refreshed_stop)
                        pos["stop_refreshes"] += 1
                        stop_refreshes += 1
                        if bar.low > pos["stop"]:
                            survived_refresh = True
                        else:
                            stop_refresh_same_bar_hits += 1
                if not survived_refresh:
                    fill = min(pos["stop"], bar.open)
                    closed_pos = _close_trade(sym, fill, reason)
                    if reason == "stop" and cooldown_sessions:
                        cooldown_until_index[sym] = calendar_index + cooldown_sessions
                    if reason == "stop" and post_stop_reentry_fn is not None:
                        post_stop_events += 1
                        post_stop_states[sym] = {
                            "stop_ord": o,
                            "stop_calendar_index": calendar_index,
                            "stop_day_high": float(bar.high),
                            "stop_day_low": float(bar.low),
                            "stop_day_close": float(bar.close),
                            "exit_fill": float(fill),
                            "previous_entry": float(closed_pos["entry"]),
                            "previous_stop": float(closed_pos["initial_stop"]),
                            "previous_rank": closed_pos["entry_rank"],
                            "gate_went_unqualified": False,
                        }
                    continue
            if exit_policy in ("target", "atr_trail3_target") and pos["target"] and bar.high >= pos["target"]:
                _close_trade(sym, pos["target"], "target")
                continue
            if exit_policy == "sma50":
                sma = _sma(sym, bar.idx, 50)
                if sma is not None and bar.close < sma:
                    _close_trade(sym, bar.close, "sma50")
                    continue
            elif exit_policy == "low20":
                prior_low = _prior_low(sym, bar.idx, 20)
                if prior_low is not None and bar.close < prior_low:
                    _close_trade(sym, bar.close, "low20")
                    continue
            elif exit_policy == "technical40":
                technical = _technical_score(sym, bar.idx)
                if technical is not None and technical < 40.0:
                    _close_trade(sym, bar.close, "technical40")
                    continue
            if pos["bars_held"] >= hold_days:
                _close_trade(sym, bar.close, "time")
                continue
            if exit_policy in ("atr_trail3", "atr_trail3_target"):
                pos["highest_close"] = max(pos["highest_close"], bar.close)
                atr = _atr(sym, bar.idx)
                if atr is not None:
                    next_stop = pos["highest_close"] - atr_trail_multiplier * atr
                    if next_stop < bar.close:
                        pos["stop"] = max(pos["stop"], next_stop)

        # 2) entries at today's close, best momentum first
        equity = _marked_equity()
        fixed_todays = list(entries_by_ord.get(o, ()))
        reentry_todays: list[dict] = []
        if post_stop_reentry_fn is not None and (
            end_ord is None or o < end_ord
        ):
            fixed_todays = [
                candidate
                for candidate in fixed_todays
                if candidate["symbol"] not in post_stop_states
            ]
            for sym, state in list(post_stop_states.items()):
                bar = _bar(sym, o)
                if bar is None:
                    continue
                state["sessions_since_stop"] = (
                    calendar_index - state["stop_calendar_index"]
                )
                candidate = post_stop_reentry_fn(sym, o, state, bar)
                if candidate is None:
                    continue
                tagged = dict(candidate)
                tagged["_post_stop_reentry"] = True
                reentry_todays.append(tagged)
        todays = sorted(
            fixed_todays + reentry_todays,
            key=lambda c: c.get(ranking_key) or 0.0,
            reverse=True,
        )
        for c in todays:
            sym = c["symbol"]
            if sym in positions:
                continue
            if calendar_index < cooldown_until_index.get(sym, -1):
                skipped_cooldown += 1
                continue
            if len(positions) >= max_positions:
                skipped_full += 1
                continue
            entry, stop = float(c["entry"]), float(c["stop"])
            risk_ps = entry - stop
            if risk_ps <= 0 or entry <= 0:
                continue
            shares = min(
                (equity * risk_per_trade) / risk_ps,
                (equity * SIM_NOTIONAL_CAP) / entry,
                max(cash, 0.0) / (entry * (1.0 + cost_rate)),
            )
            if shares * entry < 1.0:  # can't fund a meaningful position
                continue
            entry_cost = shares * entry * cost_rate
            cash -= shares * entry + entry_cost
            is_reentry = bool(c.get("_post_stop_reentry"))
            reentry_wait_sessions: int | None = None
            if is_reentry:
                state = post_stop_states.pop(sym, None)
                if state is not None:
                    reentry_wait_sessions = int(state["sessions_since_stop"])
                    reentry_events.append({
                        "symbol": sym,
                        "stop_ord": state["stop_ord"],
                        "reentry_ord": o,
                        "wait_sessions": reentry_wait_sessions,
                        "reason": c.get("_reentry_reason"),
                    })
            positions[sym] = {
                "shares": shares,
                "entry": entry,
                "entry_ord": o,
                "initial_stop": stop,
                "stop": stop,
                "target": float(c["target"]) if c.get("target") else None,
                "entry_cost": entry_cost,
                "bars_held": 0,
                "last_close": entry,
                "highest_close": entry,
                "entry_rank": (
                    float(c[ranking_key]) if c.get(ranking_key) is not None else None
                ),
                "stop_refreshes": 0,
                "is_reentry": is_reentry,
                "reentry_wait_sessions": reentry_wait_sessions,
            }
            equity = _marked_equity()

        curve.append((o, _marked_equity()))

    # Close whatever is still open at its last mark so final equity is realized.
    for sym in list(positions):
        _close_trade(sym, positions[sym]["last_close"], "open_at_end")
    final_equity = cash
    curve[-1] = (calendar[-1], final_equity)

    total_return_pct = (final_equity / SIM_STARTING_CAPITAL - 1.0) * 100.0
    years = (calendar[-1] - calendar[0]) / 365.25
    cagr_pct = (
        ((final_equity / SIM_STARTING_CAPITAL) ** (1.0 / years) - 1.0) * 100.0
        if years > 0.25 and final_equity > 0
        else None
    )

    peak = float("-inf")
    max_dd = 0.0
    for _, eq in curve:
        peak = max(peak, eq)
        if peak > 0:
            max_dd = max(max_dd, (peak - eq) / peak)

    rets = [b / a - 1.0 for (_, a), (_, b) in zip(curve, curve[1:]) if a > 0]
    sharpe = None
    if len(rets) > 2:
        mean = sum(rets) / len(rets)
        var = sum((x - mean) ** 2 for x in rets) / (len(rets) - 1)
        if var > 0:
            sharpe = mean / math.sqrt(var) * math.sqrt(252)

    # Per-calendar-year returns off the equity curve — shows whether every year
    # contributed or one exceptional stretch carried the result.
    yearly: list[dict] = []
    year_start_eq = curve[0][1]
    cur_year = date.fromordinal(curve[0][0]).year
    last_eq = curve[0][1]
    for o, eq in curve:
        y = date.fromordinal(o).year
        if y != cur_year:
            yearly.append({
                "year": cur_year,
                "return_pct": (
                    round((last_eq / year_start_eq - 1) * 100, 1) if year_start_eq > 0 else None
                ),
            })
            cur_year = y
            year_start_eq = last_eq
        last_eq = eq
    yearly.append({
        "year": cur_year,
        "return_pct": (
            round((last_eq / year_start_eq - 1) * 100, 1) if year_start_eq > 0 else None
        ),
    })

    pnls = [t["pnl"] for t in trades]
    wins = sum(1 for p in pnls if p > 0)
    reason_counts = {
        reason: sum(1 for t in trades if t["reason"] == reason)
        for reason in sorted({t["reason"] for t in trades})
    }
    spy_pct = None
    if spy_closes:
        from app.services.benchmark_service import benchmark_return_pct

        spy_pct = benchmark_return_pct(
            spy_closes, date.fromordinal(calendar[0]), date.fromordinal(calendar[-1])
        )

    curve_payload: list[dict] | None = None
    benchmark_payload: list[dict] | None = None
    if include_curve:
        curve_base = curve[0][1] if curve else SIM_STARTING_CAPITAL
        curve_payload = [
            {
                "date": date.fromordinal(o).isoformat(),
                "equity": round(eq, 2),
                "return_pct": round((eq / curve_base - 1.0) * 100.0, 2)
                if curve_base > 0
                else None,
            }
            for o, eq in curve
        ]
        if spy_closes:
            benchmark_payload = []
            base_spy = None
            for o, _ in curve:
                d = date.fromordinal(o)
                close = spy_closes.get(d)
                if close is None or close <= 0:
                    continue
                if base_spy is None:
                    base_spy = close
                benchmark_payload.append({
                    "date": d.isoformat(),
                    "equity": round(SIM_STARTING_CAPITAL * close / base_spy, 2),
                    "return_pct": round((close / base_spy - 1.0) * 100.0, 2),
                })

    result = {
        "starting_capital": SIM_STARTING_CAPITAL,
        "cost_per_side_pct": round(cost_rate * 100.0, 3),
        "final_equity": round(final_equity, 2),
        "total_return_pct": round(total_return_pct, 1),
        "cagr_pct": round(cagr_pct, 1) if cagr_pct is not None else None,
        "max_drawdown_pct": round(max_dd * 100.0, 1),
        "sharpe": round(sharpe, 2) if sharpe is not None else None,
        "trades": len(trades),
        "win_rate": round(wins / len(trades) * 100.0, 1) if trades else None,
        "avg_trade_pnl": round(sum(pnls) / len(pnls), 2) if pnls else None,
        "best_trade_r": round(max(t["r"] for t in trades), 2) if trades else None,
        "worst_trade_r": round(min(t["r"] for t in trades), 2) if trades else None,
        "best_trade_pnl": round(max(pnls), 2) if pnls else None,
        "worst_trade_pnl": round(min(pnls), 2) if pnls else None,
        "avg_hold_days": (
            round(sum(t["hold"] for t in trades) / len(trades), 1) if trades else None
        ),
        "exit_reasons": reason_counts,
        "skipped_book_full": skipped_full,
        "spy_return_pct": round(spy_pct, 1) if spy_pct is not None else None,
        "yearly_returns": yearly,
        "start_date": date.fromordinal(calendar[0]).isoformat(),
        "end_date": date.fromordinal(calendar[-1]).isoformat(),
    }
    if curve_payload is not None:
        result["equity_curve"] = curve_payload
    if benchmark_payload is not None:
        result["benchmark_curve"] = benchmark_payload
    if cooldown_sessions:
        result["reentry_cooldown_sessions"] = cooldown_sessions
        result["skipped_cooldown"] = skipped_cooldown
    if initial_stop_refresh_fn is not None:
        result["stop_refresh_attempts"] = stop_refresh_attempts
        result["stop_refreshes"] = stop_refreshes
        result["stop_refresh_same_bar_hits"] = stop_refresh_same_bar_hits
    if post_stop_reentry_fn is not None:
        result["post_stop_events"] = post_stop_events
        result["post_stop_reentries"] = len(reentry_events)
        result["post_stop_states_open_at_end"] = len(post_stop_states)
        result["reentry_events"] = [
            {
                **{
                    key: value
                    for key, value in event.items()
                    if key not in {"stop_ord", "reentry_ord"}
                },
                "stop_date": date.fromordinal(event["stop_ord"]).isoformat(),
                "reentry_date": date.fromordinal(event["reentry_ord"]).isoformat(),
            }
            for event in reentry_events
        ]
    if include_trades:
        result["trade_details"] = [
            {
                **{
                    key: value
                    for key, value in trade.items()
                    if key not in {"entry_ord", "exit_ord"}
                },
                "entry_date": date.fromordinal(trade["entry_ord"]).isoformat(),
                "exit_date": date.fromordinal(trade["exit_ord"]).isoformat(),
            }
            for trade in trades
        ]
    return result


STRATEGY_VARIANTS: tuple[dict, ...] = (
    {
        "variant": "production_residual_80_fixed10",
        "label": "Production residual 80 / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "legacy_raw_80_fixed10",
        "label": "Legacy raw 80 / max 10",
        "percentile_key": RAW_PERCENTILE_KEY,
        "cutoff": 80.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "raw_90_fixed10",
        "label": "Raw 90 / max 10",
        "percentile_key": RAW_PERCENTILE_KEY,
        "cutoff": 90.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual_80_fixed15",
        "label": "Residual 80 / max 15 capacity check",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "max_positions": 15,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_lowvol50_fixed10",
        "label": "Residual 80 + low-vol 50 / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": (
            (PRODUCTION_PERCENTILE_KEY, 80.0),
            (LOW_VOL_PERCENTILE_KEY, 50.0),
        ),
        "ranking_key": PRODUCTION_PERCENTILE_KEY,
        "ranking": "residual+low_vol_filter",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_lowvol70_fixed10",
        "label": "Residual 80 + low-vol 70 / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": (
            (PRODUCTION_PERCENTILE_KEY, 80.0),
            (LOW_VOL_PERCENTILE_KEY, 70.0),
        ),
        "ranking_key": PRODUCTION_PERCENTILE_KEY,
        "ranking": "residual+low_vol_filter",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_lowvol_blend_fixed10",
        "label": "Residual 80 + low-vol blend / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": ((PRODUCTION_PERCENTILE_KEY, 80.0),),
        "ranking_key": RESIDUAL_LOW_VOL_BLEND_KEY,
        "ranking": "residual_low_vol_blend",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol50_fixed10",
        "label": "Residual 80 + high-vol 50 / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": (
            (PRODUCTION_PERCENTILE_KEY, 80.0),
            (VOL_PERCENTILE_KEY, 50.0),
        ),
        "ranking_key": PRODUCTION_PERCENTILE_KEY,
        "ranking": "residual+high_vol_filter",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol70_fixed10",
        "label": "Residual 80 + high-vol 70 / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": (
            (PRODUCTION_PERCENTILE_KEY, 80.0),
            (VOL_PERCENTILE_KEY, 70.0),
        ),
        "ranking_key": PRODUCTION_PERCENTILE_KEY,
        "ranking": "residual+high_vol_filter",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol_blend90_10_fixed10",
        "label": "Residual 80 + high-vol 90/10 blend / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": ((PRODUCTION_PERCENTILE_KEY, 80.0),),
        "ranking_key": RESIDUAL_HIGH_VOL_BLEND_90_10_KEY,
        "ranking": "residual_high_vol_blend_90_10",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol_blend80_20_fixed10",
        "label": "Residual 80 + high-vol 80/20 blend / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": ((PRODUCTION_PERCENTILE_KEY, 80.0),),
        "ranking_key": RESIDUAL_HIGH_VOL_BLEND_80_20_KEY,
        "ranking": "residual_high_vol_blend_80_20",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol_blend_fixed10",
        "label": "Residual 80 + high-vol 70/30 blend / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": ((PRODUCTION_PERCENTILE_KEY, 80.0),),
        "ranking_key": RESIDUAL_HIGH_VOL_BLEND_KEY,
        "ranking": "residual_high_vol_blend_70_30",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "residual80_highvol_blend60_40_fixed10",
        "label": "Residual 80 + high-vol 60/40 blend / max 10",
        "percentile_key": PRODUCTION_PERCENTILE_KEY,
        "cutoff": 80.0,
        "filters": ((PRODUCTION_PERCENTILE_KEY, 80.0),),
        "ranking_key": RESIDUAL_HIGH_VOL_BLEND_60_40_KEY,
        "ranking": "residual_high_vol_blend_60_40",
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "highvol70_fixed10",
        "label": "High-vol 70 / max 10",
        "percentile_key": VOL_PERCENTILE_KEY,
        "cutoff": 70.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "highvol80_fixed10",
        "label": "High-vol 80 / max 10",
        "percentile_key": VOL_PERCENTILE_KEY,
        "cutoff": 80.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "highvol90_fixed10",
        "label": "High-vol 90 / max 10",
        "percentile_key": VOL_PERCENTILE_KEY,
        "cutoff": 90.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
    {
        "variant": "lowvol80_fixed10",
        "label": "Low-vol 80 / max 10",
        "percentile_key": LOW_VOL_PERCENTILE_KEY,
        "cutoff": 80.0,
        "max_positions": 10,
        "risk_per_trade": 0.01,
        "risk_scale": None,
    },
)


def _qualifies_by_percentile(cand: dict, percentile_key: str, threshold: float) -> bool:
    """Variant qualification: production floors + long-only signal percentile.
    This does not mutate or replace the production ``qualified`` field."""
    if not cand.get("meets_core"):
        return False
    if threshold <= 0:
        return True
    if cand.get("direction") == "short":
        return False
    pct = cand.get(percentile_key)
    return pct is not None and pct >= threshold


def _variant_filters(cfg: dict) -> tuple[tuple[str, float], ...]:
    filters = cfg.get("filters")
    if filters is None:
        return ((str(cfg["percentile_key"]), float(cfg["cutoff"])),)
    return tuple((str(key), float(cutoff)) for key, cutoff in filters)


def _qualifies_strategy_variant(cand: dict, cfg: dict) -> bool:
    """Research-only variant qualification with optional secondary gates."""
    return all(
        _qualifies_by_percentile(cand, key, cutoff)
        for key, cutoff in _variant_filters(cfg)
    )


def _strategy_variant_sims(
    candidates: list[dict],
    prices: dict[str, tuple],
    _spy_closes: dict[date, float] | None,
    hold_days: int,
) -> list[dict]:
    """Research-only portfolio variants for comparing rank signals, cutoff and
    book capacity. Live qualification is untouched."""
    rows: list[dict] = []
    for cfg in STRATEGY_VARIANTS:
        percentile_key = str(cfg["percentile_key"])
        cutoff = float(cfg["cutoff"])
        ranking_key = str(cfg.get("ranking_key") or percentile_key)
        filters = [
            {"percentile_key": key, "cutoff": threshold}
            for key, threshold in _variant_filters(cfg)
        ]
        sim = _simulate_portfolio(
            candidates,
            prices,
            _spy_closes,
            "hold",
            hold_days,
            qualified_fn=lambda c, config=cfg: _qualifies_strategy_variant(c, config),
            ranking_key=ranking_key,
            max_positions=int(cfg["max_positions"]),
            risk_per_trade=float(cfg["risk_per_trade"]),
        )
        if sim is None:
            continue
        default_ranking = "residual"
        if percentile_key == RAW_PERCENTILE_KEY:
            default_ranking = "raw"
        elif percentile_key == VOL_PERCENTILE_KEY:
            default_ranking = "high_vol"
        elif percentile_key == LOW_VOL_PERCENTILE_KEY:
            default_ranking = "low_vol"
        rows.append({
            "variant": cfg["variant"],
            "label": cfg["label"],
            "ranking": cfg.get("ranking", default_ranking),
            "cutoff": cutoff,
            "filters": filters,
            "max_positions": int(cfg["max_positions"]),
            "risk_per_trade_pct": round(float(cfg["risk_per_trade"]) * 100, 2),
            "risk_scale": cfg["risk_scale"],
            **sim,
        })
    return rows


EXIT_ENTRY_VARIANT = "residual80_highvol_blend80_20_fixed10"
EXIT_POLICY_VARIANTS: tuple[dict, ...] = (
    {
        "exit_policy": "hold",
        "label": "80/20 entry + 30d hold / initial stop",
        "description": "Baseline: keep the initial ATR stop and exit at 30 trading days.",
    },
    {
        "exit_policy": "sma50",
        "label": "80/20 entry + SMA50 break",
        "description": "Exit at the close when price closes below its 50-day SMA.",
    },
    {
        "exit_policy": "low20",
        "label": "80/20 entry + 20-day low break",
        "description": "Exit at the close when price closes below the prior 20-day low.",
    },
    {
        "exit_policy": "technical40",
        "label": "80/20 entry + technical score < 40",
        "description": "Exit at the close when the price-derived technical score deteriorates below 40.",
    },
    {
        "exit_policy": "atr_trail3",
        "label": "80/20 entry + 3x ATR trailing stop",
        "description": "Trail the stop upward to highest close minus 3 ATR, evaluated from prior data.",
    },
)

# Take-profit exits, tested 2026-07-12 and REJECTED — see
# docs/research/sr-levels-and-exits.md. Honoring the S/R target is the worst
# exit of the seven: it lifts the win rate but truncates the right tail where
# momentum's edge lives. Kept so the result stays reproducible, off by default.
# Set BACKTEST_RESEARCH_EXITS=1 to include them in the exit comparison.
RESEARCH_EXIT_POLICY_VARIANTS: tuple[dict, ...] = (
    {
        "exit_policy": "target",
        "label": "80/20 entry + S/R target take-profit (no trail)",
        "description": "Take profit at the S/R target; initial stop only, no trailing.",
    },
    {
        "exit_policy": "atr_trail3_target",
        "label": "80/20 entry + 3x ATR trail AND S/R target take-profit",
        "description": "Production trail plus a take-profit at the S/R target, first one wins.",
    },
)


def _exit_policy_variants() -> tuple[dict, ...]:
    """The exit book. Research take-profit rows only when explicitly opted in, so
    the default report stays identical to the shipped baseline."""
    if os.getenv("BACKTEST_RESEARCH_EXITS", "").strip().lower() in {"1", "true", "yes", "on"}:
        return EXIT_POLICY_VARIANTS + RESEARCH_EXIT_POLICY_VARIANTS
    return EXIT_POLICY_VARIANTS


PORTFOLIO_MONITOR_LOOKBACKS: tuple[dict, ...] = (
    {"lookback": "6m", "label": "6 months", "days": 183},
    {"lookback": "1y", "label": "1 year", "days": 365},
    {"lookback": "3y", "label": "3 years", "days": 365 * 3},
    {"lookback": "5y", "label": "5 years", "days": 365 * 5},
    {"lookback": "all", "label": "All history", "days": None},
)

PRODUCTION_PORTFOLIO_STRATEGY = "residual80_highvol80_20_atr3"
PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
    {
        "strategy": "legacy_residual80_hold",
        "label": "Legacy residual 80 + 30d hold",
        "description": "Previous production baseline: residual momentum gate/rank with 30-trading-day hold.",
        "entry_variant": "production_residual_80_fixed10",
        "exit_policy": "hold",
    },
    {
        "strategy": "residual80_highvol80_20_hold",
        "label": "Residual/high-vol 80/20 + 30d hold",
        "description": "Promoted entry candidate before adding the ATR trailing exit.",
        "entry_variant": "residual80_highvol_blend80_20_fixed10",
        "exit_policy": "hold",
    },
    {
        "strategy": "production_live_no_lockdown",
        "label": "Live setup + 3x ATR trail (no re-entry lockdown)",
        "description": (
            "Exact live activation, ordering, and Admin exit policy, with only "
            "the post-stop re-entry lockdown disabled as the comparison baseline."
        ),
        "entry_variant": "residual80_highvol_blend80_20_fixed10",
        "exit_policy": "atr_trail3",
        "reentry_lockdown_sessions": 0,
        "use_live_config": True,
        "comparison_arm": "live_no_lockdown",
    },
    {
        "strategy": PRODUCTION_PORTFOLIO_STRATEGY,
        "label": "Production: residual/high-vol 80/20 + 3x ATR trail + 5-session lockdown",
        "description": (
            "The live strategy: production activation gate and Admin exit policy "
            "as currently configured, 80/20 residual/high-vol rank, and a "
            "five-session re-entry lockdown after an initial-stop exit."
        ),
        "entry_variant": "residual80_highvol_blend80_20_fixed10",
        "exit_policy": "atr_trail3",
        "reentry_lockdown_sessions": REENTRY_LOCKDOWN_SESSIONS,
        # The production row replays what the platform actually does right now:
        # the live qualification flag (runtime Admin activation settings) and the
        # live Admin exit policy, instead of the frozen research-variant gate.
        "use_live_config": True,
        "is_production": True,
        "comparison_arm": "live_lockdown_5",
    },
)


def _portfolio_monitor_strategies() -> tuple[dict, ...]:
    return PORTFOLIO_MONITOR_STRATEGIES


def _entry_variant_config(variant: str) -> dict | None:
    return next((cfg for cfg in STRATEGY_VARIANTS if cfg["variant"] == variant), None)


def _exit_policy_sims(
    candidates: list[dict],
    prices: dict[str, tuple],
    _spy_closes: dict[date, float] | None,
    hold_days: int,
) -> list[dict]:
    """Research-only exit variants over the promoted entry challenger."""
    entry_cfg = _entry_variant_config(EXIT_ENTRY_VARIANT)
    if entry_cfg is None:
        return []

    rows: list[dict] = []
    ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"])
    for cfg in _exit_policy_variants():
        sim = _simulate_portfolio(
            candidates,
            prices,
            _spy_closes,
            str(cfg["exit_policy"]),
            hold_days,
            qualified_fn=lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config),
            ranking_key=ranking_key,
            max_positions=int(entry_cfg["max_positions"]),
            risk_per_trade=float(entry_cfg["risk_per_trade"]),
        )
        if sim is None:
            continue
        rows.append({
            "entry_variant": EXIT_ENTRY_VARIANT,
            "exit_policy": cfg["exit_policy"],
            "label": cfg["label"],
            "description": cfg["description"],
            "hold_days": hold_days,
            **sim,
        })
    return rows


def _lookback_start(max_ord: int | None, days: int | None) -> date | None:
    if max_ord is None or days is None:
        return None
    return date.fromordinal(max_ord - days)


# The R:R floor is the last un-swept knob in the live gate: `gate_ablation` shows
# removing it halves expectancy, but the *level* (prod: 2.0) was hand-set in Admin
# and never tuned. Sweep it against portfolio Sharpe, not per-setup expectancy.
MIN_RR_SWEEP_VALUES: tuple[float, ...] = (0.0, 1.2, 1.5, 1.75, 2.0, 2.25, 2.5, 3.0, 4.0)


def _min_rr_sweep_enabled() -> bool:
    return os.getenv("BACKTEST_MIN_RR_SWEEP", "").strip().lower() in {"1", "true", "yes", "on"}


def _meets_core_at(cand: dict, activation: dict, min_rr: float) -> bool:
    """Recompute the live gate's ``meets_core`` for a candidate at a different
    R:R floor, from stored fields, mirroring ``qualification.setup_qualifies``.

    The live-R:R freshness check is skipped (it needs a current price, which
    historical candidates don't carry) and the momentum percentile is applied
    separately — exactly as ``core_config`` does when meets_core is first built.
    """
    if cand["rr"] < min_rr:
        return False
    primary_prob = cand.get("primary_prob")
    if primary_prob is None or float(primary_prob) < MIN_TARGET_PROBABILITY:
        return False
    if (cand["confidence"] or 0.0) < float(activation.get("min_confidence", 0.0)):
        return False
    if activation.get("exclude_neutral"):
        action_direction = _action_direction(cand.get("action"))
        if action_direction == "neutral" or action_direction != cand["direction"]:
            return False
    if activation.get("require_high_conviction") and (
        (cand.get("action") or "") not in HIGH_CONVICTION_ACTIONS
    ):
        return False
    if activation.get("exclude_conflicts") and (cand.get("risk_level") or "") != "Low":
        return False
    return True


def _min_rr_sweep(
    candidates: list[dict],
    prices: dict[str, tuple],
    _spy_closes: dict[date, float] | None,
    activation: dict,
    threshold: float,
    hold_days: int,
    live_exit_policy: dict | None = None,
) -> dict:
    """Portfolio economics of the production book at each R:R floor.

    Graded on Sharpe/CAGR/DD under the real exit — not per-setup expectancy —
    because that is the metric every other promotion decision used.
    """
    strategy = next((s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")), None)
    if strategy is None:
        return {}
    entry_cfg = _entry_variant_config(str(strategy["entry_variant"]))
    if entry_cfg is None:
        return {}

    exit_policy = str(strategy["exit_policy"])
    row_hold_days = hold_days
    trail_multiplier = ATR_TRAIL_MULTIPLIER
    reentry_lockdown_sessions = int(
        strategy.get("reentry_lockdown_sessions", 0)
    )
    if strategy.get("use_live_config") and live_exit_policy is not None:
        exit_policy = LIVE_EXIT_MODE_TO_SIM.get(
            str(live_exit_policy.get("mode", "atr_trailing")), "atr_trail3"
        )
        row_hold_days = int(live_exit_policy.get("hold_days", hold_days))
        trail_multiplier = float(live_exit_policy.get("atr_multiplier", ATR_TRAIL_MULTIPLIER))

    live_min_rr = float(activation.get("min_rr", 0.0))
    live_qualified = sum(1 for c in candidates if c.get("qualified"))

    # When a holdout split is set, sweep on the TEST window only. A threshold
    # picked off a full-history curve is fitted to that curve; the only way to
    # know whether the peak is real is to look for it in data the choice never saw.
    sweep_start = _holdout_split()

    rows: list[dict] = []
    for min_rr in MIN_RR_SWEEP_VALUES:
        def qualified_fn(c: dict, min_rr: float = min_rr) -> bool:
            if not _meets_core_at(c, activation, min_rr):
                return False
            if threshold <= 0:
                return True
            if c["direction"] == "short":
                return False
            mp = c.get(PRODUCTION_PERCENTILE_KEY)
            return mp is not None and mp >= threshold

        n_qualified = sum(1 for c in candidates if qualified_fn(c))
        sim = _simulate_portfolio(
            candidates,
            prices,
            _spy_closes,
            exit_policy,
            row_hold_days,
            qualified_fn=qualified_fn,
            ranking_key=str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"]),
            max_positions=int(entry_cfg["max_positions"]),
            risk_per_trade=float(entry_cfg["risk_per_trade"]),
            atr_trail_multiplier=trail_multiplier,
            reentry_cooldown_sessions=reentry_lockdown_sessions,
            start_date=sweep_start,
        )
        if sim is None:
            continue
        sim.pop("equity_curve", None)
        sim.pop("benchmark_curve", None)
        rows.append({
            "min_rr": min_rr,
            "is_live": abs(min_rr - live_min_rr) < 1e-9,
            "qualified_setups": n_qualified,
            **sim,
        })

    # Parity self-check: at the live floor the reconstructed gate must reproduce
    # the production qualified set exactly. If it doesn't, the sweep is measuring
    # some other gate and every row below is worthless.
    live_row = next((r for r in rows if r["is_live"]), None)
    reproduces = live_row is not None and live_row["qualified_setups"] == live_qualified

    return {
        "live_min_rr": live_min_rr,
        "live_qualified_setups": live_qualified,
        "reproduces_production_gate": reproduces,
        "exit_policy": exit_policy,
        "reentry_lockdown_sessions": reentry_lockdown_sessions,
        "entries_from": sweep_start.isoformat() if sweep_start else None,
        "window": "out-of-sample (test)" if sweep_start else "full history (in-sample)",
        "rows": rows,
        "note": (
            "Portfolio economics of the production book at each activation R:R floor "
            "(all other gate floors held at their live values). `reproduces_production_gate` "
            "must be true: the row at the live floor has to rebuild the exact qualified set, "
            "otherwise the sweep is grading a gate we don't run. Set BACKTEST_HOLDOUT_SPLIT "
            "to sweep on the held-out window instead — a threshold read off the full-history "
            "curve is fitted to it."
        ),
    }


def _holdout_split() -> date | None:
    """Train/test split date for out-of-sample validation, e.g.
    BACKTEST_HOLDOUT_SPLIT=2024-07-01. Off by default."""
    raw = os.getenv("BACKTEST_HOLDOUT_SPLIT", "").strip()
    if not raw:
        return None
    try:
        return date.fromisoformat(raw)
    except ValueError:
        return None


def _holdout_evaluation(
    candidates: list[dict],
    prices: dict[str, tuple],
    _spy_closes: dict[date, float] | None,
    hold_days: int,
    split: date,
    live_exit_policy: dict | None = None,
) -> dict:
    """The production strategy simulated on entries BEFORE the split (train) and
    on entries ON/AFTER it (test), as separate books.

    The lookback rows in ``portfolio_monitor`` are NOT a holdout — they are nested
    windows that all end today, so every one of them overlaps the data a rule was
    chosen on. This does the real thing: the test window is disjoint from the
    train window, so a rule settled on train has never seen it.
    """
    strategy = next(
        (s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")),
        None,
    )
    if strategy is None:
        return {}
    entry_cfg = _entry_variant_config(str(strategy["entry_variant"]))
    if entry_cfg is None:
        return {}

    exit_policy = str(strategy["exit_policy"])
    row_hold_days = hold_days
    trail_multiplier = ATR_TRAIL_MULTIPLIER
    reentry_lockdown_sessions = int(
        strategy.get("reentry_lockdown_sessions", 0)
    )
    if strategy.get("use_live_config") and live_exit_policy is not None:
        exit_policy = LIVE_EXIT_MODE_TO_SIM.get(
            str(live_exit_policy.get("mode", "atr_trailing")), "atr_trail3"
        )
        row_hold_days = int(live_exit_policy.get("hold_days", hold_days))
        trail_multiplier = float(
            live_exit_policy.get("atr_multiplier", ATR_TRAIL_MULTIPLIER)
        )
    qualified_fn = (
        None if strategy.get("use_live_config")
        else lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config)
    )

    rows: list[dict] = []
    for window, start, end in (
        ("train", None, split),
        ("test", split, None),
    ):
        sim = _simulate_portfolio(
            candidates,
            prices,
            _spy_closes,
            exit_policy,
            row_hold_days,
            qualified_fn=qualified_fn,
            ranking_key=str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"]),
            max_positions=int(entry_cfg["max_positions"]),
            risk_per_trade=float(entry_cfg["risk_per_trade"]),
            atr_trail_multiplier=trail_multiplier,
            reentry_cooldown_sessions=reentry_lockdown_sessions,
            start_date=start,
            end_date=end,
            include_curve=True,
        )
        if sim is None:
            continue
        rows.append({"window": window, "exit_policy": exit_policy, **sim})

    return {
        "split_date": split.isoformat(),
        "strategy": strategy["strategy"],
        "reentry_lockdown_sessions": reentry_lockdown_sessions,
        "rows": rows,
        "note": (
            "Train = entries before the split; test = entries on/after it. The two "
            "books are disjoint in entry date. Compare the TEST row across arms: a "
            "rule chosen by looking at full history has already seen train."
        ),
    }


def _portfolio_monitor(
    candidates: list[dict],
    prices: dict[str, tuple],
    _spy_closes: dict[date, float] | None,
    hold_days: int,
    live_exit_policy: dict | None = None,
) -> dict:
    latest_ord = max((max(cols[0]) for cols in prices.values() if cols[0]), default=None)
    rows: list[dict] = []
    strategies = _portfolio_monitor_strategies()
    for strategy in strategies:
        entry_cfg = _entry_variant_config(str(strategy["entry_variant"]))
        if entry_cfg is None:
            continue
        ranking_key = str(
            strategy.get("ranking_key")
            or entry_cfg.get("ranking_key")
            or entry_cfg["percentile_key"]
        )
        # Live-config rows replay the runtime qualification flag and Admin exit
        # policy. The overlay opts into this deliberately so only ordering
        # changes relative to the production row.
        use_live = bool(strategy.get("use_live_config"))
        reentry_lockdown_sessions = int(
            strategy.get("reentry_lockdown_sessions", 0)
        )
        exit_policy = str(strategy["exit_policy"])
        row_hold_days = hold_days
        trail_multiplier = ATR_TRAIL_MULTIPLIER
        live_exit_mode: str | None = None
        if use_live and live_exit_policy is not None:
            live_exit_mode = str(live_exit_policy.get("mode", "atr_trailing"))
            exit_policy = LIVE_EXIT_MODE_TO_SIM.get(live_exit_mode, "atr_trail3")
            row_hold_days = int(live_exit_policy.get("hold_days", hold_days))
            trail_multiplier = float(
                live_exit_policy.get("atr_multiplier", ATR_TRAIL_MULTIPLIER)
            )
        qualified_fn = (
            None if use_live
            else lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config)
        )
        for lookback in PORTFOLIO_MONITOR_LOOKBACKS:
            start = _lookback_start(latest_ord, lookback["days"])
            sim = _simulate_portfolio(
                candidates,
                prices,
                _spy_closes,
                exit_policy,
                row_hold_days,
                qualified_fn=qualified_fn,
                ranking_key=ranking_key,
                max_positions=int(entry_cfg["max_positions"]),
                risk_per_trade=float(entry_cfg["risk_per_trade"]),
                atr_trail_multiplier=trail_multiplier,
                reentry_cooldown_sessions=reentry_lockdown_sessions,
                start_date=start,
                include_curve=True,
            )
            if sim is None:
                continue
            rows.append({
                "strategy": strategy["strategy"],
                "label": strategy["label"],
                "description": strategy["description"],
                "is_production": bool(strategy.get("is_production")),
                "comparison_arm": strategy.get("comparison_arm"),
                "entry_variant": strategy["entry_variant"],
                "ranking_key": ranking_key,
                "exit_policy": exit_policy,
                "live_exit_mode": live_exit_mode,
                "reentry_lockdown_sessions": reentry_lockdown_sessions,
                "lookback": lookback["lookback"],
                "lookback_label": lookback["label"],
                **sim,
            })
    return {
        "production_strategy": PRODUCTION_PORTFOLIO_STRATEGY,
        "strategies": [
            {
                "strategy": s["strategy"],
                "label": s["label"],
                "description": s["description"],
                "is_production": bool(s.get("is_production")),
                "comparison_arm": s.get("comparison_arm"),
                "reentry_lockdown_sessions": int(
                    s.get("reentry_lockdown_sessions", 0)
                ),
            }
            for s in strategies
        ],
        "lookbacks": [
            {"lookback": lb["lookback"], "label": lb["label"]}
            for lb in PORTFOLIO_MONITOR_LOOKBACKS
        ],
        "runs": rows,
        "note": (
            "Portfolio monitor runs supported named strategies across cached lookbacks. "
            "The structural overlay appears only in its explicit research arm and changes "
            "ordering, not production qualification. Local snapshot backtests remain the "
            "research surface for broad variant sweeps. The production row applies the "
            "same five-session post-initial-stop re-entry lockdown as the live setup list."
        ),
    }


def _production_cadence_comparison(
    monitor: dict | None,
    cadence: str,
) -> dict | None:
    """Compact full-history live/no-lockdown vs live/5-session comparison."""
    if not monitor:
        return None
    arms: list[dict] = []
    for row in monitor.get("runs") or []:
        comparison_arm = row.get("comparison_arm")
        if not comparison_arm or row.get("lookback") != "all":
            continue
        compact = {
            key: value
            for key, value in row.items()
            if key not in {"equity_curve", "benchmark_curve"}
        }
        arm_name = (
            "prod_live_setup"
            if comparison_arm == "live_no_lockdown"
            else "cooldown_5"
        )
        compact["arm"] = f"{arm_name}_{cadence}"
        compact["entry_cadence"] = cadence
        arms.append(compact)
    if not arms:
        return None
    arms.sort(key=lambda row: int(row.get("reentry_lockdown_sessions", 0)))
    return {
        "entry_cadence": cadence,
        "lookback": "all",
        "arms": arms,
        "note": (
            "Both arms use the exact same live gate, ordering, Admin exit policy, "
            "fees, and candidate cadence. Only the five-session post-stop "
            "re-entry lockdown changes."
        ),
    }


def _pct_loss(base: float | None, candidate: float | None) -> float | None:
    if base is None or candidate is None or base <= 0:
        return None
    return (base - candidate) / base


def _is_promotion_candidate(
    row: dict,
    *,
    base_sharpe: float | None,
    base_dd: float | None,
    base_cagr: float | None,
) -> bool:
    if (
        base_sharpe is None or base_dd is None or base_cagr is None
        or row.get("sharpe") is None
        or row.get("cagr_pct") is None
        or row.get("max_drawdown_pct") is None
    ):
        return False
    cagr_loss = _pct_loss(base_cagr, row.get("cagr_pct"))
    return (
        row["sharpe"] > base_sharpe
        and row["max_drawdown_pct"] <= base_dd
        and cagr_loss is not None and cagr_loss < 0.10
    )


def _best_research_row(
    rows: list[dict],
    *,
    base_sharpe: float | None,
    base_dd: float | None,
    base_cagr: float | None,
) -> tuple[dict | None, bool]:
    candidates = [
        row for row in rows
        if _is_promotion_candidate(
            row, base_sharpe=base_sharpe, base_dd=base_dd, base_cagr=base_cagr
        )
    ]
    if candidates:
        return max(candidates, key=_sharpe_key), True
    return max(rows, key=_sharpe_key, default=None), False


def _sharpe_key(row: dict) -> float:
    sharpe = row.get("sharpe")
    return float(sharpe) if sharpe is not None else -999.0


def _build_research_recommendation(report: dict) -> dict:
    """Build advisory notes from any strategy variants present in the report."""
    variants = {
        v.get("variant"): v
        for v in (report.get("strategy_variants") or {}).get("variants", [])
    }
    base = variants.get("production_residual_80_fixed10")
    items: list[dict] = []
    if base is None:
        return {
            "items": [],
            "note": "Strategy variants unavailable; re-run the backtest after benchmark data is present.",
        }

    base_sharpe = base.get("sharpe")
    base_dd = base.get("max_drawdown_pct")
    base_cagr = base.get("cagr_pct")

    capacity = variants.get("residual_80_fixed15")
    if (
        capacity and base_sharpe is not None and base_cagr is not None
        and capacity.get("sharpe") is not None and capacity.get("cagr_pct") is not None
        and capacity.get("max_drawdown_pct") is not None and base_dd is not None
    ):
        candidate = (
            capacity["sharpe"] > base_sharpe
            and capacity["cagr_pct"] > base_cagr
            and capacity["max_drawdown_pct"] <= base_dd + 1.0
        )
        items.append({
            "topic": "capacity_15",
            "candidate": candidate,
            "text": (
                f"Max-15 capacity {'is worth promoting' if candidate else 'is not needed yet'}: "
                f"Sharpe {capacity['sharpe']:.2f} vs {base_sharpe:.2f}, "
                f"CAGR {capacity['cagr_pct']:+.1f}% vs {base_cagr:+.1f}%, "
                f"skipped {capacity.get('skipped_book_full', 0)} vs {base.get('skipped_book_full', 0)}."
            ),
        })

    raw_90s = [
        v for key, v in variants.items()
        if key.startswith("raw_90_") and v.get("risk_scale") is None
    ]
    raw_90 = max(raw_90s, key=_sharpe_key, default=None)
    if (
        raw_90 and base_sharpe is not None and base_dd is not None and base_cagr is not None
        and raw_90.get("sharpe") is not None and raw_90.get("cagr_pct") is not None
        and raw_90.get("max_drawdown_pct") is not None
    ):
        cagr_loss = _pct_loss(base_cagr, raw_90.get("cagr_pct"))
        raw_90_sharpe = raw_90.get("sharpe")
        candidate = (
            raw_90_sharpe is not None
            and raw_90_sharpe > base_sharpe
            and raw_90["max_drawdown_pct"] < base_dd
            and cagr_loss is not None and cagr_loss < 0.10
        )
        items.append({
            "topic": "cutoff_90",
            "candidate": candidate,
            "text": (
                f"Cutoff 90 {'is a promotion candidate' if candidate else 'stays research-only'}: "
                f"{raw_90['label']} Sharpe {raw_90_sharpe:.2f} vs {base_sharpe:.2f}, "
                f"drawdown {raw_90['max_drawdown_pct']:.1f}% vs {base_dd:.1f}%, "
                f"CAGR {raw_90.get('cagr_pct'):+.1f}% vs {base_cagr:+.1f}%."
            ),
        })

    low_vol_rows = [
        v for key, v in variants.items()
        if (key.startswith("residual80_lowvol") or key.startswith("lowvol"))
        and v.get("risk_scale") is None
    ]
    low_vol, candidate = _best_research_row(
        low_vol_rows, base_sharpe=base_sharpe, base_dd=base_dd, base_cagr=base_cagr
    )
    if (
        low_vol and base_sharpe is not None and base_dd is not None and base_cagr is not None
        and low_vol.get("sharpe") is not None and low_vol.get("cagr_pct") is not None
        and low_vol.get("max_drawdown_pct") is not None
    ):
        items.append({
            "topic": "low_vol_overlay",
            "candidate": candidate,
            "text": (
                f"Low-vol overlay {'is a promotion candidate' if candidate else 'stays research-only'}: "
                f"{low_vol['label']} Sharpe {low_vol['sharpe']:.2f} vs {base_sharpe:.2f}, "
                f"drawdown {low_vol['max_drawdown_pct']:.1f}% vs {base_dd:.1f}%, "
                f"CAGR {low_vol.get('cagr_pct'):+.1f}% vs {base_cagr:+.1f}%."
            ),
        })

    high_vol_rows = [
        v for key, v in variants.items()
        if (key.startswith("residual80_highvol") or key.startswith("highvol"))
        and v.get("risk_scale") is None
    ]
    high_vol, candidate = _best_research_row(
        high_vol_rows, base_sharpe=base_sharpe, base_dd=base_dd, base_cagr=base_cagr
    )
    if (
        high_vol and base_sharpe is not None and base_dd is not None and base_cagr is not None
        and high_vol.get("sharpe") is not None and high_vol.get("cagr_pct") is not None
        and high_vol.get("max_drawdown_pct") is not None
    ):
        items.append({
            "topic": "high_vol_overlay",
            "candidate": candidate,
            "text": (
                f"High-vol overlay {'is a promotion candidate' if candidate else 'stays research-only'}: "
                f"{high_vol['label']} Sharpe {high_vol['sharpe']:.2f} vs {base_sharpe:.2f}, "
                f"drawdown {high_vol['max_drawdown_pct']:.1f}% vs {base_dd:.1f}%, "
                f"CAGR {high_vol.get('cagr_pct'):+.1f}% vs {base_cagr:+.1f}%."
            ),
        })

    return {
        "items": items,
        "note": (
            "Residual 12-1 momentum is now the production activation rank. "
            "Remaining rows are research comparisons only."
        ),
    }


# ---------------------------------------------------------------------------
# Data-driven recommendation
# ---------------------------------------------------------------------------

# A floor whose removal costs less than this (R net per trade, under the hold
# exit) is judged not to be pulling its weight.
_FLOOR_KEEP_THRESHOLD = 0.02
# The hold exit must beat the target exit by at least this much to be advised.
_EXIT_SWITCH_THRESHOLD = 0.05


def _build_recommendation(report: dict) -> dict:
    """Strategy advice derived from THIS report's numbers — recomputed every
    run, so if the data flips, the advice flips. Rules are deliberately simple
    and transparent; thresholds are module constants above."""
    items: list[dict] = []
    headline = None
    monitor = report.get("portfolio_monitor") or {}
    production_strategy = monitor.get("production_strategy")
    production_rows = [
        row for row in monitor.get("runs", [])
        if row.get("strategy") == production_strategy
    ]
    production_row = (
        next((row for row in production_rows if row.get("lookback") == "all"), None)
        or next((row for row in production_rows if row.get("lookback") == "3y"), None)
        or (production_rows[0] if production_rows else None)
    )
    if production_row is not None:
        headline = (
            "Production baseline: residual/high-vol 80/20 entry rank with a "
            "3x ATR trailing exit, 30-trading-day max hold, and 5-session "
            "re-entry lockdown after an initial stop."
        )
        if (
            production_row.get("cagr_pct") is not None
            and production_row.get("sharpe") is not None
            and production_row.get("max_drawdown_pct") is not None
        ):
            items.append({
                "topic": "production",
                "text": (
                    f"Production monitor ({production_row.get('lookback_label', 'selected window')}): "
                    f"{production_row.get('cagr_pct'):+.1f}% CAGR, "
                    f"Sharpe {production_row.get('sharpe'):.2f}, "
                    f"max drawdown -{production_row.get('max_drawdown_pct', 0):.1f}%."
                ),
            })

    q = report.get("overall_qualified") or {}
    target_net = q.get("net_avg_r")

    # Legacy diagnostic: target/stop race vs the best fixed hold.
    time_rows = [r for r in report.get("time_exit_sweep") or [] if r.get("net_avg_r") is not None]
    best_hold = max(time_rows, key=lambda r: r["net_avg_r"], default=None)
    sim_rows = {
        p.get("policy"): p
        for p in (report.get("portfolio_sim") or {}).get("policies", [])
    }
    hold_sim = sim_rows.get("hold")
    if best_hold is not None and target_net is not None:
        if best_hold["net_avg_r"] > target_net + _EXIT_SWITCH_THRESHOLD:
            text = (
                f"Legacy exit diagnostic: hold {best_hold['hold_days']} trading days with the initial stop "
                f"({best_hold['net_avg_r']:+.2f}R net/trade vs {target_net:+.2f}R for the S/R target exit)."
            )
            target_sim = sim_rows.get("target")
            if (
                hold_sim is not None and target_sim is not None
                and hold_sim.get("cagr_pct") is not None and target_sim.get("cagr_pct") is not None
            ):
                text += (
                    f" The simulated book agrees: {hold_sim['cagr_pct']:+.1f}% vs "
                    f"{target_sim['cagr_pct']:+.1f}% CAGR at similar drawdown."
                )
            items.append({"topic": "exit", "text": text})
        else:
            items.append({
                "topic": "exit",
                "text": (
                    f"Legacy exit diagnostic: keep the S/R target exit ({target_net:+.2f}R net/trade) — "
                    "no fixed hold beats it by a meaningful margin."
                ),
            })

    # Gate floors, judged under the hold exit (the ablation's Hold column).
    ablation = {r["variant"]: r for r in report.get("gate_ablation") or []}
    base_row = ablation.get("all_floors")
    base_hold = (base_row or {}).get("hold_net_avg_r")
    floor_labels = {
        "no_confidence_floor": "confidence floor",
        "no_rr_floor": "R:R floor",
        "no_neutral_exclusion": "NEUTRAL exclusion",
    }
    if base_hold is not None:
        for variant, label in floor_labels.items():
            row = ablation.get(variant)
            if row is None or row.get("hold_net_avg_r") is None:
                continue
            delta = base_hold - row["hold_net_avg_r"]
            extra = row["total"] - base_row["total"]
            if delta <= _FLOOR_KEEP_THRESHOLD:
                items.append({
                    "topic": "gate",
                    "text": (
                        f"Gate: the {label} adds nothing — dropping it costs {delta:+.2f}R/trade "
                        f"and adds {extra} trades."
                    ),
                })
            else:
                items.append({
                    "topic": "gate",
                    "text": f"Gate: keep the {label} (worth {delta:+.2f}R/trade under the hold exit).",
                })

    # Activation cutoff: best per-trade net among the promoted residual-momentum
    # sweep rows.
    sweep_rows = [
        r for r in report.get("sweep") or []
        if r.get("net_avg_r") is not None and (r.get("min_momentum_percentile") or 0) > 0
    ]
    if sweep_rows:
        best_cut = max(sweep_rows, key=lambda r: r["net_avg_r"])
        items.append({
            "topic": "cutoff",
            "text": (
                f"Residual-momentum cutoff: {best_cut['min_momentum_percentile']:.0f} has the best "
                f"per-trade net ({best_cut['net_avg_r']:+.2f}R over {best_cut['total']} setups)."
            ),
        })

    # Book vs benchmark.
    book = hold_sim or sim_rows.get("target")
    if book is not None and book.get("spy_return_pct") is not None:
        edge = book["total_return_pct"] - book["spy_return_pct"]
        verdict = "beats" if edge > 0 else "LAGS"
        items.append({
            "topic": "benchmark",
            "text": (
                f"Book vs SPY: {verdict} buy-and-hold by {edge:+.1f} points "
                f"({book['total_return_pct']:+.1f}% vs {book['spy_return_pct']:+.1f}%), "
                f"max drawdown −{book['max_drawdown_pct']:.1f}%."
            ),
        })

    # Robustness: does the edge survive without the biggest winners? Judged on
    # the RECOMMENDED exit — outlier dependence under an exit we'd abandon
    # would be the wrong warning.
    hold_recommended = (
        best_hold is not None and target_net is not None
        and best_hold["net_avg_r"] > target_net + _EXIT_SWITCH_THRESHOLD
    )
    if hold_recommended and best_hold.get("net_avg_r_ex_top5") is not None:
        trimmed = best_hold["net_avg_r_ex_top5"]
        basis = f"under the recommended {best_hold['hold_days']}d hold"
    else:
        trimmed = q.get("net_avg_r_ex_top5")
        basis = "under the S/R target exit"
    if trimmed is not None:
        if trimmed > 0:
            items.append({
                "topic": "robustness",
                "text": (
                    f"Robustness: expectancy survives removing the top 5% of winners "
                    f"({trimmed:+.2f}R net/trade {basis}) — the edge is not a handful "
                    "of outliers."
                ),
            })
        else:
            items.append({
                "topic": "robustness",
                "text": (
                    f"Robustness WARNING: without the top 5% of winners the edge disappears "
                    f"({trimmed:+.2f}R net/trade {basis}) — outlier-dependent, treat the "
                    "headline expectancy with caution."
                ),
            })

    if headline is None and hold_recommended:
        cagr_note = (
            f" (~{hold_sim['cagr_pct']:.0f}% CAGR simulated)"
            if hold_sim is not None and hold_sim.get("cagr_pct") is not None
            else ""
        )
        headline = (
            f"Trade the qualified list long-only; hold {best_hold['hold_days']} trading days "
            f"with the initial ATR stop{cagr_note}."
        )

    return {
        "headline": headline,
        "items": items,
        "note": "Derived from this report's numbers on every run — the advice flips if the data does.",
    }


async def run_backtest(
    db: AsyncSession,
    progress_cb: Callable[[int, int, str], None] | None = None,
    *,
    target_model: str = PRODUCTION_GTL_TARGET_MODEL,
    cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> dict:
    """Replay every ticker and aggregate the Phase-1 reports for the current config."""
    target_model = validate_backtest_target_model(target_model)
    cadence = validate_backtest_cadence(cadence)
    config = await get_recommendation_config(db)
    activation = await get_activation_config(db)

    result = await db.execute(select(Ticker).order_by(Ticker.symbol))
    tickers = list(result.scalars().all())
    total = len(tickers)

    candidates: list[dict] = []
    # Signal IC remains a weekly, non-overlapping diagnostic regardless of the
    # entry cadence. Production activation ranks are assigned from candidates
    # at their own weekly or exact-date ``ranking_period`` below.
    collected: dict = defaultdict(lambda: defaultdict(list))

    # Residual momentum needs a point-in-time benchmark return stream. Best-effort:
    # if SPY benchmark data is unavailable, the residual signal simply won't be
    # emitted and the rest of the report remains valid.
    benchmark_closes: dict[date, float] = {}
    try:
        benchmark_closes = await _load_benchmark_closes_for_backtest(
            db, days=settings.ohlcv_history_days + 365
        )
    except Exception:
        logger.exception("Benchmark load for residual momentum failed")

    def _merge(result: tuple[list[dict], dict]) -> None:
        cands, series = result
        candidates.extend(cands)
        for name, weeks in series.items():
            for wk, pairs in weeks.items():
                collected[name][wk].extend(pairs)

    workers = _backtest_worker_count()
    ctx = _mp_context() if (workers > 1 and total > 1) else None
    loop = asyncio.get_running_loop()

    pool = None
    if ctx is not None:
        try:
            pool = ProcessPoolExecutor(max_workers=workers, mp_context=ctx)
        except Exception:
            logger.exception("Backtest process pool unavailable; falling back to sequential")

    if pool is not None:
        # Parallel: replay tickers across worker processes — true multi-core, since
        # the GIL only serializes work within a single process. Bars are fetched in
        # the event loop (ORM-safe) and a bounded batch is fanned out to the pool.
        logger.info(json.dumps({
            "event": "backtest_parallel", "workers": workers,
            "start_method": ctx.get_start_method(),
        }))
        chunk = workers * 2
        done = 0
        with pool:
            for start in range(0, total, chunk):
                batch = tickers[start : start + chunk]
                futures = []
                for ticker in batch:
                    try:
                        columns = await _fetch_columns(db, ticker.symbol)
                    except Exception:
                        logger.exception("Backtest fetch failed for %s", ticker.symbol)
                        continue
                    if columns is not None:
                        futures.append(loop.run_in_executor(
                            pool, _replay_and_signals, ticker.symbol, columns, config, activation,
                            benchmark_closes,
                            target_model,
                            cadence,
                        ))
                for result in await asyncio.gather(*futures, return_exceptions=True):
                    if isinstance(result, Exception):
                        logger.error(json.dumps({"event": "backtest_worker_error", "message": str(result)}))
                    else:
                        _merge(result)
                done += len(batch)
                if progress_cb is not None:
                    progress_cb(min(done, total), total, "")
    else:
        # Sequential fallback (Windows / 1 worker): run each replay in a worker
        # thread so the event loop — and the API server — stays responsive.
        for index, ticker in enumerate(tickers):
            if progress_cb is not None:
                progress_cb(index, total, ticker.symbol)
            try:
                columns = await _fetch_columns(db, ticker.symbol)
                if columns is not None:
                    _merge(await asyncio.to_thread(
                        _replay_and_signals, ticker.symbol, columns, config, activation,
                        benchmark_closes,
                        target_model,
                        cadence,
                    ))
            except Exception:
                logger.exception("Backtest replay failed for %s", ticker.symbol)

    if progress_cb is not None and total:
        progress_cb(total, total, "")

    # Cross-sectional momentum: rank every week's universe, then "qualified" means
    # floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum
    # (raw 12-1 fallback only when benchmark data is unavailable).
    _assign_momentum_percentiles(candidates)
    _assign_residual_momentum_percentiles(candidates)
    _assign_low_volatility_percentiles(candidates)
    _assign_activation_momentum_percentiles(candidates)
    _assign_residual_low_vol_blend(candidates)
    _assign_residual_high_vol_blend(candidates)
    current_min_pct = float(activation.get("min_momentum_percentile", 80.0))
    for c in candidates:
        c["qualified"] = _momentum_qualifies(c, current_min_pct)

    qualified = [c for c in candidates if c["qualified"]]
    longs = [c for c in qualified if c["direction"] == "long"]
    shorts = [c for c in qualified if c["direction"] == "short"]

    # Threshold sweep: re-apply the momentum gate at several percentile cutoffs
    # (floors held fixed) so the trade-off between how many setups qualify and
    # their expectancy is visible without re-replaying. 0 = floors only.
    sweep = []
    for threshold in (90.0, 80.0, 70.0, 60.0, 50.0, 0.0):
        cands = [c for c in candidates if _momentum_qualifies(c, threshold)]
        sweep.append({"min_momentum_percentile": threshold, **_bucket_stats(cands)})

    # Portfolio simulation: re-fetch bars for just the qualified symbols (memory-
    # light vs retaining every ticker's columns through the replay) and replay
    # the book once per exit policy. Best-effort — the report stands without it.
    hold_horizon = max(TIME_EXIT_DAYS)
    sim_policies: list[dict] = []
    strategy_variant_rows: list[dict] = []
    exit_policy_rows: list[dict] = []
    portfolio_monitor_report: dict | None = None
    holdout_report: dict | None = None
    min_rr_sweep_report: dict | None = None
    try:
        qual_symbols = sorted({
            c["symbol"]
            for c in candidates
            if c.get("qualified")
            or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS)
        })
        price_columns: dict[str, tuple] = {}
        for sym in qual_symbols:
            cols = await _fetch_columns(db, sym)
            if cols is not None:
                price_columns[sym] = cols

        spy_closes: dict | None = None
        try:
            oldest = min((cols[0][0] for cols in price_columns.values()), default=None)
            days_needed = None
            if oldest is not None and not _offline_snapshot_mode():
                days_needed = (date.today() - date.fromordinal(oldest)).days + 30
            spy_closes = await _load_benchmark_closes_for_backtest(
                db, days=days_needed, refresh=oldest is not None
            )
        except Exception:
            logger.exception("Benchmark load for the portfolio sim failed")

        for policy in ("target", "hold"):
            sim = _simulate_portfolio(
                candidates, price_columns, spy_closes, policy, hold_horizon
            )
            if sim is not None:
                sim_policies.append({"policy": policy, **sim})
        strategy_variant_rows = _strategy_variant_sims(
            candidates, price_columns, spy_closes, hold_horizon
        )
        exit_policy_rows = _exit_policy_sims(
            candidates, price_columns, spy_closes, hold_horizon
        )
        live_exit_policy: dict | None = None
        try:
            from app.services.paper_trade_service import get_exit_policy

            live_exit_policy = await get_exit_policy(db)
        except Exception:
            logger.exception("Live exit policy load failed; monitor uses defaults")
        portfolio_monitor_report = _portfolio_monitor(
            candidates, price_columns, spy_closes, hold_horizon,
            live_exit_policy=live_exit_policy,
        )
        split = _holdout_split()
        if split is not None:
            holdout_report = _holdout_evaluation(
                candidates, price_columns, spy_closes, hold_horizon, split,
                live_exit_policy=live_exit_policy,
            )
        if _min_rr_sweep_enabled():
            min_rr_sweep_report = _min_rr_sweep(
                candidates, price_columns, spy_closes, activation, current_min_pct,
                hold_horizon, live_exit_policy=live_exit_policy,
            )
    except Exception:
        logger.exception("Portfolio simulation failed")

    report = {
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "tickers": total,
        "candidates": len(candidates),
        "qualified": len(qualified),
        "params": {
            # Keep step_days for old report consumers; the value counts stored
            # market sessions rather than calendar days.
            "step_days": backtest_step_sessions(cadence),
            "step_sessions": backtest_step_sessions(cadence),
            "entry_cadence": cadence,
            "signal_eval_cadence": WEEKLY_BACKTEST_CADENCE,
            "horizon_days": HORIZON,
            "min_lookback": MIN_LOOKBACK,
            "cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
            "target_model": target_model,
            "target_model_label": BACKTEST_TARGET_MODELS[target_model],
            "is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL,
            "production_reentry_lockdown_sessions": REENTRY_LOCKDOWN_SESSIONS,
        },
        "activation": activation,
        "overall_qualified": _bucket_stats(qualified),
        "overall_all": _bucket_stats(candidates),
        "by_direction": {
            "long": _bucket_stats(longs),
            "short": _bucket_stats(shorts),
        },
        "min_momentum_percentile": current_min_pct,
        "sweep": sweep,
        "gate_ablation": _gate_ablation(candidates, activation, current_min_pct),
        "gate_ablation_note": (
            "Each row re-qualifies the same candidates at the current momentum "
            f"cutoff ({current_min_pct:.0f}) with one floor removed (long-only "
            "while the momentum gate is active). If dropping a floor doesn't "
            "hurt net expectancy, that floor isn't pulling its weight. The Hold "
            "columns grade the same variants under the hold-to-horizon time exit "
            "instead of the S/R target — the view that matters if the exit "
            "policy moves to a fixed hold."
        ),
        "time_exit_sweep": [_time_exit_bucket(qualified, n) for n in TIME_EXIT_DAYS],
        "portfolio_sim": {
            "params": {
                "starting_capital": SIM_STARTING_CAPITAL,
                "max_positions": SIM_MAX_POSITIONS,
                "risk_per_trade_pct": round(SIM_RISK_PER_TRADE * 100, 2),
                "notional_cap_pct": round(SIM_NOTIONAL_CAP * 100, 1),
                "cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
                "hold_days": hold_horizon,
            },
            "policies": sim_policies,
            "note": (
                "One capital-constrained book over the same qualified setups the "
                "tables above grade per-setup: at most "
                f"{SIM_MAX_POSITIONS} concurrent positions (one per ticker), best "
                "momentum first, fixed-fractional risk sizing with a no-leverage "
                "cap, entries at the detection close, stops filled at the worse "
                "of stop or open. 'target' races the S/R target against the stop "
                "(timeout at the horizon); 'hold' keeps the initial stop and "
                "exits at the horizon close. SPY return is price-only over the "
                "same window. In-sample; no dividends."
            ),
        },
        "strategy_variants": {
            "variants": strategy_variant_rows,
            "note": (
                "Research-only hold-to-horizon portfolio variants. Production now "
                "uses residual 12-1 momentum at cutoff 80; the remaining rows compare "
                "the legacy raw rank, raw cutoff 90, one max-15 capacity check, and "
                "volatility overlays."
            ),
        },
        "exit_policy_variants": {
            "variants": exit_policy_rows,
            "note": (
                "Research-only exit policies over the residual/high-vol 80/20 entry "
                "candidate. Every row uses the same entry qualification/ranking and "
                "changes only the exit discipline."
            ),
        },
        "portfolio_monitor": portfolio_monitor_report,
        "production_cadence_comparison": (
            _production_cadence_comparison(portfolio_monitor_report, cadence)
            if target_model == PRODUCTION_GTL_TARGET_MODEL
            else None
        ),
        "holdout": holdout_report,
        "min_rr_sweep": min_rr_sweep_report,
        "target_model_diagnostics": _target_model_diagnostics(
            candidates,
            target_model,
        ),
        "signal_eval": _signal_evaluation(collected),
        "signal_eval_note": (
            "Cross-sectional rank-IC of price-only signals vs the forward "
            f"{HORIZON}-day return (min {MIN_CROSS_SECTION} names/window). |IC| ≳ "
            "0.03 with a consistent sign is a real (if small) edge; near 0 means "
            "ranking on it sorts nothing. Momentum factors and high_52w are expected "
            "positive; reversal_1m and vol_6m expected negative (mean-reversion / "
            "low-vol anomaly). IC is measured on non-overlapping windows; signals "
            f"with fewer than {MIN_RELIABLE_PERIODS} independent windows are flagged "
            "unreliable (too few regimes — deepen history with the Data Backfill job)."
        ),
        "note": (
            "Sentiment & fundamentals held neutral (no point-in-time history). "
            "Stops fill at the worse of the stop or the bar's open (gaps through "
            "the stop are modeled, so a loss can exceed −1R); targets never fill "
            "better than their level. "
            "~6 months ≈ one market regime — treat as directional, not gospel."
        ),
    }
    report["recommendation"] = _build_recommendation(report)
    report["research_recommendation"] = _build_research_recommendation(report)
    return report


async def run_and_store(
    db: AsyncSession,
    progress_cb: Callable[[int, int, str], None] | None = None,
    *,
    target_model: str = PRODUCTION_GTL_TARGET_MODEL,
    cadence: str = DEFAULT_BACKTEST_CADENCE,
) -> dict:
    """Run the backtest and cache the report in a SystemSetting. Job entrypoint."""
    report = await run_backtest(
        db,
        progress_cb,
        target_model=target_model,
        cadence=cadence,
    )
    await update_setting(db, KEY_REPORT, json.dumps(report))
    return report


async def get_backtest_report(db: AsyncSession) -> dict | None:
    """Return the last cached backtest report, or None if never run."""
    setting = await settings_store.get_setting(db, KEY_REPORT)
    if setting is None:
        return None
    try:
        return json.loads(setting.value)
    except (TypeError, ValueError):
        return None