"""Point-in-time fundamentals rank-overlay research.

The production qualification gate is unchanged. The runner first measures
30-session factor IC, then reorders already-qualified candidates with quality,
growth, or balanced fundamental ranks. It supports the original registered
matrix and a split-safe follow-up sensitivity. It writes JSON, Markdown, CSV,
and a portable ZIP bundle.
"""

from __future__ import annotations

import argparse
import asyncio
import csv
import hashlib
import json
import multiprocessing
import os
import pickle
import subprocess
import sys
import zipfile
from collections import defaultdict
from concurrent.futures import ProcessPoolExecutor, as_completed
from datetime import date, datetime, time, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from zoneinfo import ZoneInfo

from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine

ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

CACHE_VERSION = "fundamentals-overlay-v1"
NY = ZoneInfo("America/New_York")
COMPOSITES = ("quality", "growth", "balanced")
ORIGINAL_PROTOCOL = "original"
SPLIT_SAFE_PROTOCOL = "split-safe"
WEIGHTS = (0.10, 0.20, 0.30, 0.40)
SPLIT_SAFE_WEIGHTS = (0.05, 0.10, 0.15)


def _arm_matrix(weights: tuple[float, ...]) -> tuple[dict[str, Any], ...]:
    """Build the control plus three bounded overlay families."""
    return (
        {
            "id": "control_w00",
            "label": "Production 80/20 momentum-volatility rank",
            "composite": None,
            "weight": 0.0,
        },
        *tuple(
            {
                "id": f"{composite}_w{round(weight * 100):02d}",
                "label": f"{composite.title()} overlay {round(weight * 100)}%",
                "composite": composite,
                "weight": weight,
            }
            for composite in COMPOSITES
            for weight in weights
        ),
    )


ARMS = _arm_matrix(WEIGHTS)
N_TRIALS = len(ARMS)
SPLIT_SAFE_ARMS = _arm_matrix(SPLIT_SAFE_WEIGHTS)
SPLIT_SAFE_N_TRIALS = len(SPLIT_SAFE_ARMS)


def _parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("snapshot")
    parser.add_argument(
        "--workers", type=int, default=max(1, multiprocessing.cpu_count() - 1)
    )
    parser.add_argument("--out", default=None)
    parser.add_argument(
        "--candidate-cache", default="reports/.cache/fundamentals-candidates.pkl"
    )
    parser.add_argument(
        "--fundamentals-cache", default="reports/.cache/fundamentals-scores.pkl"
    )
    parser.add_argument("--train-end", default="2024-01-01")
    parser.add_argument("--test-start", default="2025-01-01")
    parser.add_argument(
        "--protocol",
        choices=(ORIGINAL_PROTOCOL, SPLIT_SAFE_PROTOCOL),
        default=ORIGINAL_PROTOCOL,
    )
    parser.add_argument("--allow-spawn", action="store_true")
    parser.add_argument("--quiet", action="store_true")
    return parser.parse_args()


def _sqlite_url(path: Path) -> str:
    return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"


def _default_out(protocol: str = ORIGINAL_PROTOCOL) -> Path:
    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
    label = (
        "fundamentals-splitsafe"
        if protocol == SPLIT_SAFE_PROTOCOL
        else "fundamentals-overlay"
    )
    return Path("reports") / f"{label}-{stamp}.json"


def _snapshot_hash(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _cache_key(snapshot: Path, suffix: dict[str, Any]) -> dict[str, Any]:
    stat = snapshot.stat()
    return {
        "version": CACHE_VERSION,
        "snapshot": str(snapshot.resolve()),
        "size": stat.st_size,
        "mtime_ns": stat.st_mtime_ns,
        **suffix,
    }


def _load_cache(path: Path, key: dict[str, Any]) -> Any | None:
    if not path.exists():
        return None
    with path.open("rb") as handle:
        payload = pickle.load(handle)  # noqa: S301 - trusted local cache
    return payload.get("value") if payload.get("key") == key else None


def _save_cache(path: Path, key: dict[str, Any], value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("wb") as handle:
        pickle.dump(
            {"key": key, "value": value}, handle, protocol=pickle.HIGHEST_PROTOCOL
        )


def _git_commit() -> str | None:
    try:
        return subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=ROOT,
            capture_output=True,
            check=True,
            text=True,
        ).stdout.strip()
    except (OSError, subprocess.CalledProcessError):
        return None


async def _load_snapshot(snapshot: Path, quiet: bool) -> dict[str, Any]:
    from app.models.earnings_event import EarningsEvent
    from app.models.fundamental_snapshot import FundamentalSnapshot
    from app.models.ohlcv import OHLCVRecord
    from app.models.ticker import Ticker
    from app.services import backtest_service as bt
    from app.services.admin_service import get_activation_config
    from app.services.paper_trade_service import get_exit_policy
    from app.services.recommendation_service import get_recommendation_config

    engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
    session_factory = async_sessionmaker(
        engine, class_=AsyncSession, expire_on_commit=False
    )
    try:
        async with session_factory() as db:
            config = await get_recommendation_config(db)
            activation = await get_activation_config(db)
            exit_config = await get_exit_policy(db)
            benchmark = await bt._load_benchmark_closes_for_backtest(
                db, days=None, refresh=False
            )
            tickers = list(
                (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
            )
            snapshots = list(
                (
                    await db.execute(
                        select(FundamentalSnapshot).order_by(
                            FundamentalSnapshot.cik,
                            FundamentalSnapshot.accepted_at,
                        )
                    )
                ).scalars()
            )
            earnings_count = int(
                (
                    await db.execute(select(func.count()).select_from(EarningsEvent))
                ).scalar_one()
            )
            price_bounds = (
                await db.execute(
                    select(
                        func.min(OHLCVRecord.date),
                        func.max(OHLCVRecord.date),
                        func.count(),
                    )
                )
            ).one()
            prices: dict[str, tuple] = {}
            for index, ticker in enumerate(tickers, 1):
                columns = await bt._fetch_columns(db, ticker.symbol)
                if columns is not None:
                    prices[ticker.symbol] = columns
                if not quiet and index % 50 == 0:
                    print(f"loaded prices {index}/{len(tickers)}", flush=True)
    finally:
        await engine.dispose()

    by_cik: dict[str, list[Any]] = defaultdict(list)
    for row in snapshots:
        by_cik[str(row.cik)].append(row)
    ticker_rows = [
        {
            "symbol": ticker.symbol,
            "cik": str(ticker.cik) if ticker.cik else None,
            "sic": ticker.sic,
        }
        for ticker in tickers
    ]
    audit = {
        "tickers": len(tickers),
        "tickers_with_prices": len(prices),
        "tickers_with_cik": sum(row["cik"] is not None for row in ticker_rows),
        "unique_ciks": len({row["cik"] for row in ticker_rows if row["cik"]}),
        "fundamental_rows": len(snapshots),
        "fundamental_ciks": len(by_cik),
        "accepted_at_min": min((row.accepted_at for row in snapshots), default=None),
        "accepted_at_max": max((row.accepted_at for row in snapshots), default=None),
        "earnings_rows": earnings_count,
        "price_date_min": price_bounds[0],
        "price_date_max": price_bounds[1],
        "price_rows": int(price_bounds[2] or 0),
    }
    return {
        "config": config,
        "activation": activation,
        "exit_config": exit_config,
        "benchmark": benchmark,
        "ticker_rows": ticker_rows,
        "prices": prices,
        "snapshots_by_cik": dict(by_cik),
        "audit": audit,
    }


def _representatives(
    ticker_rows: list[dict], prices: dict[str, tuple]
) -> dict[str, str]:
    reps: dict[str, str] = {}
    for row in ticker_rows:
        cik = row.get("cik")
        symbol = str(row["symbol"])
        if not cik or symbol not in prices:
            continue
        if cik not in reps or symbol < reps[cik]:
            reps[cik] = symbol
    return reps


def _build_candidates(
    snapshot: Path,
    data: dict[str, Any],
    args: argparse.Namespace,
) -> tuple[list[dict], int]:
    from app.services import backtest_service as bt
    from scripts import run_research_matrix as shared

    cache_path = Path(args.candidate_cache)
    key = _cache_key(snapshot, {"kind": "daily-production-candidates"})
    cached = _load_cache(cache_path, key)
    if cached is not None:
        if not args.quiet:
            print(f"loaded candidate cache {cache_path}", flush=True)
        return list(cached["qualified"]), int(cached["entry_candidate_count"])

    prices = data["prices"]
    workers = max(1, min(args.workers, max(1, multiprocessing.cpu_count() - 1)))
    replay_rows: list[dict] = []
    replay_start = date(1900, 1, 1)

    def replay_one(symbol: str, columns: tuple) -> list[dict]:
        return bt._replay_candidates_for_period(
            symbol,
            columns,
            data["config"],
            data["activation"],
            data["benchmark"],
            replay_start,
            "daily",
            True,
            True,
        )

    if workers == 1:
        for index, (symbol, columns) in enumerate(prices.items(), 1):
            replay_rows.extend(replay_one(symbol, columns))
            if not args.quiet and index % 25 == 0:
                print(f"replay {index}/{len(prices)}", flush=True)
    else:
        context = bt._mp_context() or multiprocessing.get_context("spawn")
        with ProcessPoolExecutor(max_workers=workers, mp_context=context) as pool:
            futures = {
                pool.submit(
                    bt._replay_candidates_for_period,
                    symbol,
                    columns,
                    data["config"],
                    data["activation"],
                    data["benchmark"],
                    replay_start,
                    "daily",
                    True,
                    True,
                ): symbol
                for symbol, columns in prices.items()
            }
            for index, future in enumerate(as_completed(futures), 1):
                replay_rows.extend(future.result())
                if not args.quiet and index % 25 == 0:
                    print(f"replay {index}/{len(futures)}", flush=True)

    setups = [row for row in replay_rows if not row.get("_rank_only")]
    observations = [row for row in replay_rows if row.get("_universe_rank_observation")]
    ranks = shared._live_universe_rank_map(
        observations,
        data["benchmark"],
        bt.STRATEGY_RANK_MOMENTUM_WEIGHT,
    )
    cutoff = float(data["activation"].get("min_momentum_percentile", 80.0))
    qualified: list[dict] = []
    for setup in setups:
        if setup.get("direction") != "long":
            continue
        identity = (str(setup["symbol"]), str(setup["date"]))
        rank = ranks.get(identity)
        if rank is None:
            continue
        candidate = {
            key: value
            for key, value in setup.items()
            if not key.startswith("_universe_")
        }
        candidate[bt.PRODUCTION_PERCENTILE_KEY] = rank["momentum_percentile"]
        candidate[bt.VOL_PERCENTILE_KEY] = rank["volatility_percentile"]
        candidate[bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY] = rank["strategy_rank"]
        candidate["qualified"] = bt._momentum_qualifies(candidate, cutoff)
        if candidate["qualified"]:
            qualified.append(candidate)

    if not qualified:
        raise RuntimeError("no qualified long candidates after replay")
    value = {"qualified": qualified, "entry_candidate_count": len(setups)}
    _save_cache(cache_path, key, value)
    if not args.quiet:
        print(f"wrote candidate cache {cache_path}", flush=True)
    return qualified, len(setups)


def _weekly_factor_dates(
    representatives: dict[str, str], prices: dict[str, tuple]
) -> set[date]:
    from app.services import backtest_service as bt

    dates: set[date] = set()
    for symbol in representatives.values():
        columns = prices[symbol]
        records = [
            SimpleNamespace(date=date.fromordinal(int(value))) for value in columns[0]
        ]
        for index in bt._weekly_asof_indices(records):
            if index >= bt.MIN_LOOKBACK - 1 and index + bt.HORIZON < len(records):
                dates.add(records[index].date)
    return dates


def _utc(value: datetime) -> datetime:
    if value.tzinfo is None:
        return value.replace(tzinfo=timezone.utc)
    return value.astimezone(timezone.utc)


def _coverage_summary(rows: list[dict[str, int]]) -> dict[str, Any]:
    if not rows:
        return {}
    keys = sorted(rows[0])
    result: dict[str, Any] = {"dates": len(rows)}
    for key in keys:
        values = sorted(row[key] for row in rows)
        middle = len(values) // 2
        median = (
            values[middle]
            if len(values) % 2
            else (values[middle - 1] + values[middle]) / 2
        )
        result[key] = {
            "min": values[0],
            "median": median,
            "max": values[-1],
        }
    return result


def _build_scores(
    snapshot: Path,
    dates: set[date],
    representatives: dict[str, str],
    snapshots_by_cik: dict[str, list[Any]],
    split_safe: bool,
    args: argparse.Namespace,
) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]:
    from app.services import fundamentals_derivation as derivation
    from app.services import fundamentals_research as research

    factor_polarity = (
        research.SPLIT_SAFE_FACTOR_POLARITY if split_safe else research.FACTOR_POLARITY
    )
    ordered_dates = sorted(dates)
    date_fingerprint = hashlib.sha256(
        "|".join(value.isoformat() for value in ordered_dates).encode()
    ).hexdigest()
    cache_path = Path(args.fundamentals_cache)
    key = _cache_key(
        snapshot,
        {
            "kind": "point-in-time-scores",
            "score_profile": SPLIT_SAFE_PROTOCOL if split_safe else ORIGINAL_PROTOCOL,
            "date_fingerprint": date_fingerprint,
            "availability": "accepted before signal-date midnight America/New_York",
        },
    )
    cached = _load_cache(cache_path, key)
    if cached is not None:
        if not args.quiet:
            print(f"loaded fundamentals cache {cache_path}", flush=True)
        return cached["scores"], cached["coverage"]

    eligible_ciks = sorted(set(representatives) & set(snapshots_by_cik))
    rows_by_cik = {
        cik: sorted(snapshots_by_cik[cik], key=lambda row: _utc(row.accepted_at))
        for cik in eligible_ciks
    }
    positions = {cik: 0 for cik in eligible_ciks}
    visible = {cik: [] for cik in eligible_ciks}
    current_features: dict[str, dict[str, float | None]] = {}
    scores_by_date: dict[str, dict[str, dict[str, float | None]]] = {}
    coverage_rows: list[dict[str, int]] = []

    for date_index, signal_date in enumerate(ordered_dates, 1):
        cutoff = datetime.combine(signal_date, time.min, tzinfo=NY).astimezone(
            timezone.utc
        )
        for cik in eligible_ciks:
            rows = rows_by_cik[cik]
            position = positions[cik]
            changed = False
            while position < len(rows) and _utc(rows[position].accepted_at) <= cutoff:
                visible[cik].append(rows[position])
                position += 1
                changed = True
            positions[cik] = position
            if changed:
                current_features[cik] = research.raw_features(
                    derivation.derive(visible[cik])
                )
        scores = research.cross_section_scores(
            current_features,
            split_safe=split_safe,
        )
        scores_by_date[signal_date.isoformat()] = scores
        coverage_rows.append(
            {
                key: sum(row.get(key) is not None for row in scores.values())
                for key in (*factor_polarity, *research.COMPOSITE_KEYS)
            }
        )
        if not args.quiet and date_index % 100 == 0:
            print(f"fundamentals dates {date_index}/{len(ordered_dates)}", flush=True)

    coverage = _coverage_summary(coverage_rows)
    value = {"scores": scores_by_date, "coverage": coverage}
    _save_cache(cache_path, key, value)
    if not args.quiet:
        print(f"wrote fundamentals cache {cache_path}", flush=True)
    return scores_by_date, coverage


def _factor_diagnostics(
    representatives: dict[str, str],
    prices: dict[str, tuple],
    scores_by_date: dict[str, dict[str, dict[str, float | None]]],
    factor_keys: tuple[str, ...],
    train_end: date,
    test_start: date,
) -> dict[str, list[dict]]:
    from app.services import backtest_service as bt

    signal_keys = (*factor_keys, *COMPOSITES)
    observations: list[dict[str, Any]] = []
    for cik, symbol in representatives.items():
        columns = prices[symbol]
        ordinals, _opens, _highs, _lows, closes, _volumes = columns
        records = [
            SimpleNamespace(date=date.fromordinal(int(value))) for value in ordinals
        ]
        for index in bt._weekly_asof_indices(records):
            forward_index = index + bt.HORIZON
            if index < bt.MIN_LOOKBACK - 1 or forward_index >= len(records):
                continue
            if closes[index] <= 0:
                continue
            signal_date = records[index].date
            score = scores_by_date.get(signal_date.isoformat(), {}).get(cik, {})
            forward = closes[forward_index] / closes[index] - 1.0
            iso = signal_date.isocalendar()
            for key in signal_keys:
                value = score.get(key)
                if value is not None:
                    observations.append(
                        {
                            "signal": key,
                            "date": signal_date,
                            "week": (iso.year, iso.week),
                            "value": float(value),
                            "forward": float(forward),
                            "symbol": symbol,
                        }
                    )

    windows = {
        "train": lambda value: value < train_end,
        "validation": lambda value: train_end <= value < test_start,
        "test": lambda value: value >= test_start,
        "full": lambda _value: True,
    }
    result: dict[str, list[dict]] = {}
    for window, predicate in windows.items():
        collected: dict = defaultdict(lambda: defaultdict(list))
        for row in observations:
            if predicate(row["date"]):
                collected[row["signal"]][row["week"]].append(
                    {
                        "val": row["value"],
                        "fwd": row["forward"],
                        "symbol": row["symbol"],
                    }
                )
        result[window] = bt._signal_evaluation(collected)
    return result


def _attach_overlay_ranks(
    candidates: list[dict],
    ticker_rows: list[dict],
    scores_by_date: dict[str, dict[str, dict[str, float | None]]],
    arms: tuple[dict[str, Any], ...],
) -> dict[str, Any]:
    from app.services import backtest_service as bt
    from app.services import fundamentals_research as research

    symbol_to_cik = {
        str(row["symbol"]): row.get("cik") for row in ticker_rows if row.get("cik")
    }
    covered = {composite: 0 for composite in COMPOSITES}
    for candidate in candidates:
        cik = symbol_to_cik.get(str(candidate["symbol"]))
        score = scores_by_date.get(str(candidate["date"]), {}).get(cik, {})
        for composite in COMPOSITES:
            value = score.get(composite)
            candidate[f"fund_{composite}"] = value
            if value is not None:
                covered[composite] += 1
        base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
        for arm in arms:
            if arm["composite"] is None:
                continue
            candidate[_ranking_key(arm)] = research.overlay_rank(
                base,
                score.get(str(arm["composite"])),
                float(arm["weight"]),
            )
    total = len(candidates)
    return {
        composite: {
            "candidates": count,
            "pct": round(count / total * 100.0, 2) if total else 0.0,
        }
        for composite, count in covered.items()
    }


def _ranking_key(arm: dict[str, Any]) -> str:
    from app.services import backtest_service as bt

    if arm["composite"] is None:
        return bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY
    return "fund_overlay_{}_{:02d}".format(
        arm["composite"], round(float(arm["weight"]) * 100)
    )


def _window(arm: dict[str, Any], name: str) -> dict[str, Any] | None:
    return next(
        (row for row in arm.get("windows", []) if row.get("window") == name),
        None,
    )


def _trade_overlap(subject: set[str], control: set[str]) -> dict[str, Any]:
    union = subject | control
    return {
        "overlap_pct": round(len(subject & control) / len(union) * 100.0, 2)
        if union
        else 100.0,
        "added": len(subject - control),
        "removed": len(control - subject),
    }


def _winner_concentration(details: list[dict[str, Any]]) -> dict[str, Any]:
    pnls = sorted(
        (float(row["pnl"]) for row in details if row.get("pnl") is not None),
        reverse=True,
    )
    rs = sorted(
        (float(row["r"]) for row in details if row.get("r") is not None),
        reverse=True,
    )
    top_pnl = sum(pnls[:5])
    total_pnl = sum(pnls)
    remaining_rs = rs[5:]
    return {
        "top5_pnl": round(top_pnl, 2) if pnls else None,
        "net_pnl_ex_top5": round(total_pnl - top_pnl, 2) if pnls else None,
        "top5_share_of_positive_net_pct": (
            round(top_pnl / total_pnl * 100.0, 2) if total_pnl > 0 else None
        ),
        "avg_r_ex_top5": (
            round(sum(remaining_rs) / len(remaining_rs), 4) if remaining_rs else None
        ),
    }


def _run_arm(
    arm: dict[str, Any],
    candidates: list[dict],
    data: dict[str, Any],
    train_end: date,
    test_start: date,
    n_trials: int,
) -> tuple[dict[str, Any], dict[str, set[str]]]:
    from app.services import backtest_service as bt
    from scripts import run_research_matrix as shared

    strategy = next(
        row for row in bt.PORTFOLIO_MONITOR_STRATEGIES if row.get("is_production")
    )
    entry = bt._entry_variant_config(str(strategy["entry_variant"]))
    if entry is None:
        raise RuntimeError("production entry configuration missing")
    exit_config = data["exit_config"]
    exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get(
        str(exit_config.get("mode", "atr_trailing")), "atr_trail3"
    )
    hold_days = int(exit_config.get("hold_days", 30))
    trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER))
    ranking_key = _ranking_key(arm)
    reentry = bt._make_gate_reset_reentry_fn(
        candidates,
        data["prices"],
        cadence="daily",
        ranking_key=ranking_key,
    )
    windows: list[dict[str, Any]] = []
    trades_by_window: dict[str, set[str]] = {}
    full_trade_details: list[dict[str, Any]] = []
    for name, start, end in (
        ("train", None, train_end),
        ("validation", train_end, test_start),
        ("test", test_start, None),
        ("full", None, None),
    ):
        sim = bt._simulate_portfolio(
            candidates,
            data["prices"],
            data["benchmark"],
            exit_policy,
            hold_days,
            ranking_key=ranking_key,
            max_positions=int(entry["max_positions"]),
            risk_per_trade=float(entry["risk_per_trade"]),
            atr_trail_multiplier=trail,
            post_stop_reentry_fn=reentry,
            start_date=start,
            end_date=end,
            fill_mode=bt.FILL_MODE_CLOSE,
            include_trades=True,
        )
        if sim is None:
            windows.append({"window": name, "error": "no trades"})
            trades_by_window[name] = set()
            continue
        shared._assert_calendar_truncation(sim, hold_days, bt.FILL_MODE_CLOSE)
        details = sim.pop("trade_details", [])
        sim["winner_concentration"] = _winner_concentration(details)
        if name == "full":
            full_trade_details = details
        trades_by_window[name] = {
            "{}:{}".format(row.get("symbol"), row.get("entry_date")) for row in details
        }
        for heavy in ("equity_curve", "benchmark_curve", "reentry_events"):
            sim.pop(heavy, None)
        dsr = bt.deflated_sharpe_ratio(
            sim.get("sharpe"),
            sim.get("sharpe_se"),
            n_trials,
            n_returns=sim.get("n_returns"),
            return_skew=sim.get("return_skew"),
            return_kurtosis=sim.get("return_kurtosis"),
        )
        windows.append({"window": name, "dsr": dsr, **sim})
    return (
        {
            "id": arm["id"],
            "label": arm["label"],
            "composite": arm["composite"],
            "weight": arm["weight"],
            "ranking_key": ranking_key,
            "windows": windows,
            "full_trade_details": full_trade_details,
        },
        trades_by_window,
    )


def _development_grade(control: dict, arm: dict) -> dict[str, Any]:
    control_train = _window(control, "train") or {}
    control_validation = _window(control, "validation") or {}
    arm_train = _window(arm, "train") or {}
    arm_validation = _window(arm, "validation") or {}
    required = (
        control_train.get("sharpe"),
        control_validation.get("sharpe"),
        control_validation.get("max_drawdown_pct"),
        arm_train.get("sharpe"),
        arm_validation.get("sharpe"),
        arm_validation.get("max_drawdown_pct"),
    )
    if any(value is None for value in required):
        return {"pass": False, "reason": "missing train or validation statistic"}
    checks = {
        "train_sharpe_not_worse": arm_train["sharpe"] >= control_train["sharpe"],
        "validation_sharpe_not_worse": (
            arm_validation["sharpe"] >= control_validation["sharpe"]
        ),
        "validation_drawdown_within_2pp": (
            arm_validation["max_drawdown_pct"]
            <= control_validation["max_drawdown_pct"] + 2.0
        ),
    }
    return {
        "pass": all(checks.values()),
        "checks": checks,
        "train_sharpe_delta": round(arm_train["sharpe"] - control_train["sharpe"], 4),
        "validation_sharpe_delta": round(
            arm_validation["sharpe"] - control_validation["sharpe"], 4
        ),
    }


def _final_check(control: dict, selected: dict | None) -> dict[str, Any] | None:
    if selected is None:
        return None
    control_test = _window(control, "test") or {}
    selected_test = _window(selected, "test") or {}
    values = (
        control_test.get("sharpe"),
        control_test.get("max_drawdown_pct"),
        selected_test.get("sharpe"),
        selected_test.get("max_drawdown_pct"),
    )
    if any(value is None for value in values):
        return {"pass": False, "reason": "missing test statistic"}
    checks = {
        "test_sharpe_not_worse": selected_test["sharpe"] >= control_test["sharpe"],
        "test_drawdown_within_2pp": (
            selected_test["max_drawdown_pct"] <= control_test["max_drawdown_pct"] + 2.0
        ),
    }
    return {
        "arm_id": selected["id"],
        "pass": all(checks.values()),
        "checks": checks,
        "test_sharpe_delta": round(selected_test["sharpe"] - control_test["sharpe"], 4),
        "note": "Research evidence only; passing does not change production.",
    }


def _fmt(value: Any) -> str:
    if value is None:
        return "—"
    return f"{value:.4g}" if isinstance(value, float) else str(value)


def _markdown(report: dict[str, Any]) -> str:
    protocol_id = report.get("research_protocol", ORIGINAL_PROTOCOL)
    title = (
        "Split-safe fundamentals overlay sensitivity"
        if protocol_id == SPLIT_SAFE_PROTOCOL
        else "Point-in-time fundamentals overlay research"
    )
    lines = [
        f"# {title}",
        "",
        "Generated: {}".format(report.get("generated_at")),
        "",
        "## Protocol",
        "",
        "- Research protocol: **{}**.".format(protocol_id),
        "- Train ends before **{}**.".format(report["splits"]["train_end"]),
        "- Validation runs until **{}**.".format(report["splits"]["test_start"]),
        "- Test starts at that date and is not used to select the arm.",
        "- Qualification is unchanged; fundamentals only reorder qualified longs.",
        "- SEC filings become visible at midnight New York time after acceptance.",
        "- Pre-registered portfolio trials for DSR: **{}**.".format(report["n_trials"]),
        "",
        "## Data warnings",
        "",
    ]
    lines.extend(f"- {warning}" for warning in report.get("warnings", []))
    lines.extend(
        [
            "",
            "## Factor IC",
            "",
            "| window | signal | IC | t | positive | quintile spread | weeks | N |",
            "|---|---|---:|---:|---:|---:|---:|---:|",
        ]
    )
    for window, rows in report.get("factor_ic", {}).items():
        for row in rows:
            lines.append(
                "| {} | {} | {} | {} | {} | {} | {} | {} |".format(
                    window,
                    row.get("signal"),
                    _fmt(row.get("mean_ic")),
                    _fmt(row.get("ic_t_stat")),
                    _fmt(row.get("ic_positive_pct")),
                    _fmt(row.get("mean_quintile_spread")),
                    row.get("weeks"),
                    _fmt(row.get("avg_cross_section")),
                )
            )
    lines.extend(
        [
            "",
            "## Portfolio arms",
            "",
            "| arm | window | Sharpe | SE | DSR | CAGR | MaxDD | Calmar | trades | overlap |",
            "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|",
        ]
    )
    for arm in report.get("arms", []):
        for row in arm.get("windows", []):
            overlap = row.get("selection_vs_control", {}).get("overlap_pct")
            lines.append(
                "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |".format(
                    arm["id"],
                    row.get("window"),
                    _fmt(row.get("sharpe")),
                    _fmt(row.get("sharpe_se")),
                    _fmt(row.get("dsr")),
                    _fmt(row.get("cagr_pct")),
                    _fmt(row.get("max_drawdown_pct")),
                    _fmt(row.get("calmar")),
                    _fmt(row.get("trades")),
                    _fmt(overlap),
                )
            )
    selection = report.get("development_selection")
    lines.extend(["", "## Mechanical selection", ""])
    if selection:
        lines.append(
            "- Development-selected arm: **{}**. ".format(selection["arm_id"])
            + "The test result is reported only as a final check."
        )
        lines.append("- Final check: `{}`".format(report.get("final_check")))
    else:
        lines.append("- No overlay passed the train + validation requirements.")
    lines.extend(["", "Production remains unchanged pending human review.", ""])
    return "\n".join(lines)


def _write_outputs(report: dict[str, Any], out: Path, *, bundle: bool) -> None:
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8")
    markdown_path = out.with_suffix(".md")
    markdown_path.write_text(_markdown(report), encoding="utf-8")

    arms_csv = out.with_name(f"{out.stem}-arms.csv")
    with arms_csv.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(
            [
                "arm",
                "composite",
                "weight",
                "window",
                "sharpe",
                "sharpe_se",
                "dsr",
                "cagr_pct",
                "max_drawdown_pct",
                "calmar",
                "trades",
                "overlap_pct",
                "top5_pnl_share_pct",
                "avg_r_ex_top5",
            ]
        )
        for arm in report.get("arms", []):
            for row in arm.get("windows", []):
                writer.writerow(
                    [
                        arm["id"],
                        arm["composite"],
                        arm["weight"],
                        row.get("window"),
                        row.get("sharpe"),
                        row.get("sharpe_se"),
                        row.get("dsr"),
                        row.get("cagr_pct"),
                        row.get("max_drawdown_pct"),
                        row.get("calmar"),
                        row.get("trades"),
                        row.get("selection_vs_control", {}).get("overlap_pct"),
                        row.get("winner_concentration", {}).get(
                            "top5_share_of_positive_net_pct"
                        ),
                        row.get("winner_concentration", {}).get("avg_r_ex_top5"),
                    ]
                )

    factor_csv = out.with_name(f"{out.stem}-factor-ic.csv")
    with factor_csv.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(
            [
                "window",
                "signal",
                "mean_ic",
                "ic_t_stat",
                "ic_positive_pct",
                "mean_quintile_spread",
                "weeks",
                "avg_cross_section",
                "reliable",
            ]
        )
        for window, rows in report.get("factor_ic", {}).items():
            for row in rows:
                writer.writerow(
                    [
                        window,
                        row.get("signal"),
                        row.get("mean_ic"),
                        row.get("ic_t_stat"),
                        row.get("ic_positive_pct"),
                        row.get("mean_quintile_spread"),
                        row.get("weeks"),
                        row.get("avg_cross_section"),
                        row.get("reliable"),
                    ]
                )

    trades_csv = out.with_name(f"{out.stem}-trades.csv")
    trade_columns = [
        "arm",
        "symbol",
        "entry_date",
        "exit_date",
        "r",
        "pnl",
        "reason",
        "hold",
        "entry",
        "exit",
        "risk_dollars",
    ]
    with trades_csv.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=trade_columns, extrasaction="ignore")
        writer.writeheader()
        for arm in report.get("arms", []):
            for trade in arm.get("full_trade_details", []):
                writer.writerow({"arm": arm["id"], **trade})

    if bundle:
        bundle_path = out.with_suffix(".zip")
        with zipfile.ZipFile(bundle_path, "w", zipfile.ZIP_DEFLATED) as archive:
            for path in (out, markdown_path, arms_csv, factor_csv, trades_csv):
                archive.write(path, arcname=path.name)
            protocol = ROOT / "docs" / "research" / "fundamentals-weight-backtest.md"
            if protocol.exists():
                archive.write(protocol, arcname=protocol.name)


async def _main() -> None:
    from app.services import fundamentals_research as research

    args = _parse_args()
    split_safe = args.protocol == SPLIT_SAFE_PROTOCOL
    arms = SPLIT_SAFE_ARMS if split_safe else ARMS
    n_trials = len(arms)
    factor_keys = tuple(
        (
            research.SPLIT_SAFE_FACTOR_POLARITY
            if split_safe
            else research.FACTOR_POLARITY
        ).keys()
    )
    snapshot = Path(args.snapshot)
    out = Path(args.out) if args.out else _default_out(args.protocol)
    if not snapshot.exists():
        raise SystemExit(f"snapshot not found: {snapshot}")
    if args.workers < 1:
        raise SystemExit("--workers must be positive")
    train_end = date.fromisoformat(args.train_end)
    test_start = date.fromisoformat(args.test_start)
    if train_end >= test_start:
        raise SystemExit("--train-end must be earlier than --test-start")

    os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
    if args.allow_spawn:
        os.environ["BACKTEST_ALLOW_SPAWN"] = "1"

    data = await _load_snapshot(snapshot, args.quiet)
    audit = data["audit"]
    if audit["fundamental_rows"] == 0 or audit["fundamental_ciks"] < 5:
        raise SystemExit(
            "snapshot lacks usable fundamental_snapshots; create a fresh export "
            "with scripts/create_backtest_snapshot.py"
        )
    representatives = _representatives(data["ticker_rows"], data["prices"])
    candidates, entry_candidate_count = _build_candidates(snapshot, data, args)
    factor_dates = _weekly_factor_dates(representatives, data["prices"])
    all_dates = factor_dates | {
        date.fromisoformat(str(candidate["date"])) for candidate in candidates
    }
    scores_by_date, score_coverage = _build_scores(
        snapshot,
        all_dates,
        representatives,
        data["snapshots_by_cik"],
        split_safe,
        args,
    )
    candidate_coverage = _attach_overlay_ranks(
        candidates, data["ticker_rows"], scores_by_date, arms
    )
    factor_ic = _factor_diagnostics(
        representatives,
        data["prices"],
        scores_by_date,
        factor_keys,
        train_end,
        test_start,
    )

    warnings = [
        "Current tracked universe only: historical constituent membership and "
        "delisted names are unavailable, so absolute results have survivorship bias.",
        "Earnings surprise is excluded because the completed SUE study already "
        "failed its promotion bar for this strategy.",
        "The test window has already been observed; this follow-up is sensitivity "
        "evidence and live paper performance remains the final out-of-sample check.",
    ]
    if split_safe:
        warnings.insert(
            1,
            "Diluted-EPS growth and share-count change are excluded because filing-time "
            "values are not split-comparable without point-in-time split factors.",
        )
    else:
        warnings.insert(
            1,
            "Historical valuation is excluded because split-adjusted bars cannot be "
            "safely combined with filing-time EPS and shares without split factors.",
        )

    report: dict[str, Any] = {
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "research_protocol": args.protocol,
        "git_commit": _git_commit(),
        "snapshot": str(snapshot.resolve()),
        "snapshot_sha256": _snapshot_hash(snapshot),
        "splits": {
            "train_end": train_end.isoformat(),
            "test_start": test_start.isoformat(),
            "windows": {
                "train": f"entry date < {train_end.isoformat()}",
                "validation": (
                    f"{train_end.isoformat()} <= entry date < {test_start.isoformat()}"
                ),
                "test": f"entry date >= {test_start.isoformat()}",
            },
        },
        "n_trials": n_trials,
        "pre_registered_arms": list(arms),
        "protocol": {
            "id": args.protocol,
            "qualification": "unchanged production gate; rank overlay only",
            "cadence": "daily",
            "fill_mode": "close; production near-close proxy",
            "horizon_sessions": 30,
            "filing_availability": (
                "accepted_at before signal-date midnight America/New_York; "
                "conservative match for the daily pre-market SEC import"
            ),
            "missing_fundamental_score": 50.0,
            "factor_keys": list(factor_keys),
            "split_sensitive_metrics_excluded": (
                ["eps_growth_yoy", "share_count_change_yoy"] if split_safe else []
            ),
            "selection": (
                "highest validation Sharpe among arms with train and validation "
                "Sharpe not below control and validation drawdown within 2pp"
            ),
            "production_mutation": False,
        },
        "warnings": warnings,
        "data_audit": audit,
        "strategy_config": {
            "recommendation": data["config"],
            "activation": data["activation"],
            "exit": data["exit_config"],
        },
        "score_cross_section_coverage": score_coverage,
        "qualified_candidate_coverage": candidate_coverage,
        "entry_candidate_count": entry_candidate_count,
        "qualified_candidates": len(candidates),
        "factor_ic": factor_ic,
        "arms": [],
        "development_grades": {},
        "development_selection": None,
        "final_check": None,
    }
    _write_outputs(report, out, bundle=False)

    trade_sets: dict[str, dict[str, set[str]]] = {}
    control: dict[str, Any] | None = None
    for arm in arms:
        if not args.quiet:
            print("running {}".format(arm["id"]), flush=True)
        result, arm_trades = _run_arm(
            arm, candidates, data, train_end, test_start, n_trials
        )
        trade_sets[str(arm["id"])] = arm_trades
        if control is None:
            control = result
        else:
            for row in result["windows"]:
                name = str(row["window"])
                row["selection_vs_control"] = _trade_overlap(
                    arm_trades.get(name, set()),
                    trade_sets["control_w00"].get(name, set()),
                )
            report["development_grades"][str(arm["id"])] = _development_grade(
                control, result
            )
        report["arms"].append(result)
        _write_outputs(report, out, bundle=False)

    if control is None:
        raise RuntimeError("control arm did not run")
    eligible = [
        arm
        for arm in report["arms"][1:]
        if report["development_grades"].get(arm["id"], {}).get("pass")
    ]
    selected = max(
        eligible,
        key=lambda arm: (
            float((_window(arm, "validation") or {}).get("sharpe") or -999.0),
            -float(arm["weight"]),
        ),
        default=None,
    )
    if selected is not None:
        report["development_selection"] = {
            "arm_id": selected["id"],
            "chosen_without_test": True,
            "validation_sharpe": (_window(selected, "validation") or {}).get("sharpe"),
        }
    report["final_check"] = _final_check(control, selected)
    report["completed_at"] = datetime.now(timezone.utc).isoformat()
    _write_outputs(report, out, bundle=True)
    print(f"wrote {out}", flush=True)
    bundle_path = out.with_suffix(".zip")
    print("wrote {}".format(bundle_path), flush=True)


if __name__ == "__main__":
    asyncio.run(_main())