feat(backtest): make an incomplete run visibly incomplete

A snapshot whose schema predated a migration made every _fetch_columns call
raise, both dispatch paths caught the per-ticker exception and continued, and
the run wrote a report with zero candidates that was indistinguishable from an
honest run that found no setups. Nothing said the run had lost its universe.

run_backtest now builds a completeness ledger and emits it as
report["data_quality"]: expected / loaded / evaluated counts, per-symbol
failures with stage and reason, bar-less symbols named, and benchmark
availability. Every universe symbol lands in exactly one terminal bucket and
the arithmetic is checked, so a counter that stops being maintained reports
itself as a failure rather than under-counting quietly.

Benchmark availability is a separate axis: without SPY, residual 12-1 momentum
silently degrades to raw 12-1, and the report now says so instead of leaving it
in a log line.

Three statuses, because severity differs. "failed" (nothing evaluated, empty
universe, unbalanced ledger, missing benchmark) replaces the performance tiles
with the callout rather than warning above them — a banner over ordinary-looking
numbers is exactly how the zero-candidate run would still have read as a quiet
week. "degraded" keeps the tiles with a note. A report with no data_quality
block reads as unknown, not clean; defaulting old reports to complete would
reproduce the bug being fixed.

The same gate applies to the two other consumers of backtest numbers:
productionBaseRates returns null for a failed run, and EvaluationPanel will not
report drift against a broken baseline.

"complete" means the processing and coverage checks passed. It does not mean
the prices are correct — a complete run over silently wrong bars is still
complete, and the card says so.

Also threads an explicit terminal-events option through the replay, validated
before dispatch so a stale events file aborts the run instead of quietly
degrading it. Verified against a healthy snapshot (degraded, 505/506) and one
with tickers.cik dropped (failed, 0/506, 506 fetch failures).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ

tests/unit/test_backtest_data_quality.py

@@ -0,0 +1,141 @@
+"""Guards for the backtest completeness ledger.
+
+The failure this exists to prevent is concrete: a snapshot whose schema predates
+a migration makes every ``_fetch_columns`` call raise, both dispatch paths catch
+per-ticker exceptions and continue, and the run writes a report with zero
+candidates that is indistinguishable from an honest run that found no setups.
+The ledger has to turn that into a report that says so.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+
+from app.services.backtest_service import (
+    DATA_QUALITY_FAILURE_SAMPLE,
+    HORIZON,
+    MIN_LOOKBACK,
+    _DataQuality,
+    _has_short_history,
+)
+
+
+def _healthy(expected: int = 3, evaluated: int = 3) -> _DataQuality:
+    q = _DataQuality(expected=expected, evaluated=evaluated)
+    q.record_benchmark({date(2024, 1, 2): 470.0, date(2024, 1, 3): 471.0}, None)
+    return q
+
+
+def test_a_complete_run_is_ok():
+    report = _healthy().as_report()
+    assert report["status"] == "ok"
+    assert report["complete"] is True
+    assert report["reasons"] == []
+    assert report["loaded"] == report["evaluated"] == 3
+    assert report["benchmark"]["momentum_basis"] == "residual"
+    assert report["benchmark"]["first"] == "2024-01-02"
+    assert report["benchmark"]["last"] == "2024-01-03"
+
+
+def test_every_symbol_failing_to_load_reads_as_failed_not_as_no_setups():
+    """The unsynced-snapshot case: 3 expected, 3 fetch errors, 0 evaluated."""
+    q = _DataQuality(expected=3)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    for symbol in ("AAPL", "MSFT", "NVDA"):
+        q.record_failure(symbol, "fetch", RuntimeError("no such column: tickers.cik"))
+
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert report["complete"] is False
+    assert report["evaluated"] == 0
+    assert report["loaded"] == 0
+    assert report["failed"] == 3
+    assert report["failures_by_stage"] == {"fetch": 3}
+    assert "no symbol was evaluated successfully" in report["reasons"]
+    assert report["failures"][0]["symbol"] == "AAPL"
+    assert "tickers.cik" in report["failures"][0]["error"]
+
+
+def test_a_few_failures_degrade_but_do_not_fail_the_run():
+    q = _DataQuality(expected=10, evaluated=8, no_data_symbols=["RHM"])
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    q.record_failure("HBI", "replay", ValueError("bad bar"))
+
+    report = q.as_report()
+    assert report["status"] == "degraded"
+    assert report["complete"] is False
+    assert report["loaded"] == 9  # no_data is not a load failure
+    assert report["reasons"] == [
+        "1 symbol failed during replay",
+        "1 symbol had no stored bars (RHM)",
+    ]
+
+
+def test_bar_less_symbols_are_named_so_the_reason_is_actionable():
+    """A universe row with no bars recurs on every run until someone clears it.
+    An unnamed count would just nag; the symbol makes it a one-line fix."""
+    q = _DataQuality(expected=9, evaluated=2, no_data_symbols=list("ABCDEFG"))
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    q.evaluated = 2
+
+    report = q.as_report()
+    assert report["no_data"] == 7
+    assert report["no_data_symbols"] == ["A", "B", "C", "D", "E"]
+    assert report["reasons"] == ["7 symbols had no stored bars (A, B, C, D, E, …)"]
+
+
+def test_a_missing_benchmark_fails_the_run_because_the_signal_changed():
+    """Without SPY the gate ranks on raw 12-1 momentum, so the headline no
+    longer describes the production strategy even though every ticker loaded."""
+    q = _DataQuality(expected=3, evaluated=3)
+    q.record_benchmark({}, ConnectionError("alpaca unreachable"))
+
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert report["benchmark"]["momentum_basis"] == "raw_fallback"
+    assert report["benchmark"]["sessions"] == 0
+    assert "alpaca unreachable" in report["benchmark"]["error"]
+    assert any("fell back to raw momentum" in r for r in report["reasons"])
+
+
+def test_an_unbalanced_ledger_is_itself_a_failure():
+    """If a counter stops being maintained the block must not quietly report a
+    clean run — the arithmetic is the self-check."""
+    q = _healthy(expected=10, evaluated=3)  # 7 symbols unaccounted for
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert any("ledger does not balance" in r for r in report["reasons"])
+
+
+def test_an_empty_universe_is_a_failure():
+    q = _DataQuality(expected=0)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    assert q.as_report()["status"] == "failed"
+
+
+def test_the_failure_list_is_capped_but_the_count_is_not():
+    q = _DataQuality(expected=100)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    for i in range(60):
+        q.record_failure(f"SYM{i}", "fetch", RuntimeError("boom"))
+    q.evaluated = 40
+
+    report = q.as_report()
+    assert report["failed"] == 60
+    assert len(report["failures"]) == DATA_QUALITY_FAILURE_SAMPLE
+    assert report["failures_truncated"] is True
+
+
+def test_short_history_counts_a_recent_listing_without_failing_the_run():
+    q = _healthy()
+    q.short_history = 1
+    report = q.as_report()
+    assert report["short_history"] == 1
+    assert report["status"] == "ok", "a recent listing is not a data-quality failure"
+
+
+def test_has_short_history_uses_the_replay_minimum():
+    enough = ([0] * (MIN_LOOKBACK + HORIZON),) + ((),) * 5
+    one_short = ([0] * (MIN_LOOKBACK + HORIZON - 1),) + ((),) * 5
+    assert _has_short_history(enough) == 0
+    assert _has_short_history(one_short) == 1

tests/unit/test_backtest_service.py

@@ -1689,6 +1689,74 @@ async def test_run_backtest_rolls_back_a_failed_ticker_fetch(session, monkeypatc
     assert report["tickers"] == 2
     assert report["candidates"] >= 1
 
+    # ...and the report says one ticker was lost rather than presenting the
+    # remaining one as the whole universe.
+    quality = report["data_quality"]
+    assert quality["expected"] == 2
+    assert quality["evaluated"] == 1
+    assert quality["failed"] == 1
+    assert quality["failures"][0]["symbol"] == "AAA"
+    assert quality["failures"][0]["stage"] == "fetch"
+    assert "simulated OHLCV read failure" in quality["failures"][0]["error"]
+    assert quality["complete"] is False
+    assert quality["status"] == "degraded"
+
+
+async def test_run_backtest_reports_its_own_completeness(session):
+    """The healthy case: every ticker accounted for, so the report says so."""
+    await _seed_oscillating_ticker(session, "OSC")
+    report = await bt.run_backtest(session)
+
+    quality = report["data_quality"]
+    assert quality["expected"] == report["tickers"] == 1
+    assert quality["evaluated"] == 1
+    assert quality["failed"] == 0
+    assert quality["no_data"] == 0
+    # Ledger identity — the three terminal buckets must cover the universe.
+    assert quality["no_data"] + quality["failed"] + quality["evaluated"] == quality["expected"]
+
+
+async def test_run_backtest_marks_a_run_failed_when_every_ticker_fails(session, monkeypatch):
+    """The demonstrated failure: an unsynced snapshot raises on every read, the
+    replay loop swallows each one, and the run writes a report with no
+    candidates. It must come back "failed", not look like a quiet run.
+    """
+    await _seed_oscillating_ticker(session, "AAA")
+    await _seed_oscillating_ticker(session, "OSC")
+
+    async def always_failing_fetch(db, symbol):
+        raise RuntimeError("no such column: tickers.cik")
+
+    monkeypatch.setattr(bt, "_fetch_columns", always_failing_fetch)
+
+    report = await bt.run_backtest(session)
+
+    assert report["candidates"] == 0
+    quality = report["data_quality"]
+    assert quality["status"] == "failed"
+    assert quality["evaluated"] == 0
+    assert quality["loaded"] == 0
+    assert quality["failed"] == 2
+    assert "no symbol was evaluated successfully" in quality["reasons"]
+
+
+async def test_run_backtest_flags_a_missing_benchmark(session, monkeypatch):
+    """Residual momentum degrades to raw without SPY. The numbers then describe
+    a different gate than the one the report claims, so the run is not clean."""
+    await _seed_oscillating_ticker(session, "OSC")
+
+    async def no_benchmark(db, **kwargs):
+        return {}
+
+    monkeypatch.setattr(bt, "_load_benchmark_closes_for_backtest", no_benchmark)
+
+    report = await bt.run_backtest(session)
+    benchmark = report["data_quality"]["benchmark"]
+
+    assert benchmark["sessions"] == 0
+    assert benchmark["momentum_basis"] == "raw_fallback"
+    assert report["data_quality"]["status"] == "failed"
+
 
 async def test_run_backtest_rolls_back_a_failed_portfolio_sim_load(session, monkeypatch):
     """The portfolio-sim block loads the benchmark and the live exit policy from

tests/unit/test_backtest_terminal_events.py

@@ -0,0 +1,167 @@
+"""Replay bounds for a symbol carrying a research terminal mark.
+
+``_replay_ticker`` normally stops ``HORIZON`` bars before the end so every setup
+has a full resolution window. For a delisted name that is exactly wrong: the
+collapse happens in those final bars, so the signals that would have been caught
+by it are the ones the bound hides. ``terminal_date`` opts a single symbol into
+walking through its last real bar.
+
+The fixtures build a controlled setup rather than replaying a real ticker, so
+the tests do not depend on whether SIVB happened to qualify on a given day.
+"""
+
+from __future__ import annotations
+
+from datetime import date, timedelta
+from types import SimpleNamespace
+
+import pytest
+
+from app.services import backtest_service as bt
+
+
+def _bar(day: date, close: float, *, high: float | None = None, low: float | None = None):
+    return SimpleNamespace(
+        date=day,
+        open=close,
+        high=close * 1.01 if high is None else high,
+        low=close * 0.99 if low is None else low,
+        close=close,
+        volume=1_000_000,
+    )
+
+
+def _series(n: int, *, start: date = date(2021, 1, 4)) -> list:
+    """A steadily rising series long enough to clear MIN_LOOKBACK."""
+    bars = []
+    day = start
+    for i in range(n):
+        while day.weekday() >= 5:
+            day += timedelta(days=1)
+        bars.append(_bar(day, 100.0 + i * 0.25))
+        day += timedelta(days=1)
+    return bars
+
+
+def _terminal_bar(previous: date) -> SimpleNamespace:
+    day = previous + timedelta(days=1)
+    while day.weekday() >= 5:
+        day += timedelta(days=1)
+    return SimpleNamespace(
+        date=day, open=0.01, high=0.01, low=0.01, close=0.01, volume=0
+    )
+
+
+@pytest.fixture()
+def replay(monkeypatch):
+    """Force one long setup on every evaluated bar so signal dates are countable."""
+
+    def _fake_setups(window, config, activation, *, target_model):
+        close = float(window[-1].close)
+        return [{
+            "direction": "long",
+            "entry": close,
+            "stop": close * 0.95,
+            # Reachable within HORIZON on the rising fixture, but not within the
+            # four bars left before a terminal event.
+            "target": close * 1.02,
+            "rr": 2.0,
+            "confidence": 80,
+            "momentum": 0.1,
+            "best_prob": 0.5,
+            "primary_prob": 0.5,
+            "meets_core": True,
+            "action": "BUY",
+            "risk_level": "medium",
+            "target_model": target_model,
+            "primary_sources": [],
+            "primary_strength": 1.0,
+            "primary_rejection_count": 0,
+            "primary_last_rejection_age": None,
+            "primary_distance_atr": 1.0,
+            "raw_level_count": 1,
+            "gate_level_count": 1,
+        }]
+
+    monkeypatch.setattr(bt, "_window_setups", _fake_setups)
+
+    def _run(records, terminal_date=None):
+        return bt._replay_ticker(
+            "TEST", records, {}, {}, None,
+            bt.PRODUCTION_GTL_TARGET_MODEL, "daily", terminal_date,
+        )
+
+    return _run
+
+
+def test_setup_near_a_terminal_event_is_emitted_and_resolves_at_the_mark(replay):
+    """A setup five sessions before the halt is replayed, and if it is still open
+    at the terminal bar the near-zero price resolves it as a loss."""
+    real = _series(bt.MIN_LOOKBACK + 40)
+    records = real + [_terminal_bar(real[-1].date)]
+
+    without = replay(records)
+    with_mark = replay(records, terminal_date=records[-1].date)
+
+    last_real = real[-1].date
+    fifth_last = real[-5].date
+
+    without_dates = {c["date"] for c in without}
+    with_dates = {c["date"] for c in with_mark}
+
+    assert fifth_last.isoformat() not in without_dates, (
+        "baseline should hide signals inside the final HORIZON bars"
+    )
+    assert fifth_last.isoformat() in with_dates
+    assert last_real.isoformat() in with_dates
+
+    late = next(c for c in with_mark if c["date"] == fifth_last.isoformat())
+    assert late["outcome"] == bt.OUTCOME_STOP_HIT
+    assert late["realized_r"] < -1.0, "a gap to 0.01 must fill below the stop, not at it"
+
+
+def test_no_setup_is_generated_on_the_terminal_bar(replay):
+    real = _series(bt.MIN_LOOKBACK + 40)
+    terminal = _terminal_bar(real[-1].date)
+    records = real + [terminal]
+
+    candidates = replay(records, terminal_date=terminal.date)
+
+    assert terminal.date.isoformat() not in {c["date"] for c in candidates}
+    assert max(c["date"] for c in candidates) == real[-1].date.isoformat()
+
+
+def test_a_position_that_exits_before_the_event_keeps_its_own_exit(replay):
+    """An early setup resolves on its own bars; the terminal mark must not
+    retroactively turn it into a total loss."""
+    real = _series(bt.MIN_LOOKBACK + 40)
+    records = real + [_terminal_bar(real[-1].date)]
+
+    with_mark = replay(records, terminal_date=records[-1].date)
+    early_date = real[bt.MIN_LOOKBACK - 1].date.isoformat()
+
+    early = next(c for c in with_mark if c["date"] == early_date)
+    assert early["outcome"] == bt.OUTCOME_TARGET_HIT
+    assert early["realized_r"] == pytest.approx(2.0)
+
+
+def test_unmapped_symbols_and_runs_without_the_option_are_unchanged(replay):
+    """The ordinary replay bounds are untouched when no terminal date is given."""
+    records = _series(bt.MIN_LOOKBACK + 40)
+
+    baseline = replay(records)
+    again = replay(records, terminal_date=None)
+
+    assert [c["date"] for c in baseline] == [c["date"] for c in again]
+    assert max(c["date"] for c in baseline) < records[-1].date.isoformat(), (
+        "without a terminal mark the walk must still stop HORIZON bars early"
+    )
+
+
+def test_a_mismatched_terminal_date_is_rejected(replay):
+    """The date is validated against the final bar rather than trusted, so a
+    stale events file fails loudly instead of shifting the replay window."""
+    records = _series(bt.MIN_LOOKBACK + 40)
+
+    with pytest.raises(ValueError, match="does not match the final bar"):
+        replay(records, terminal_date=date(1999, 1, 1))

tests/unit/test_run_backtest_snapshot_terminal_events.py

@@ -0,0 +1,123 @@
+"""Runner-level guard: a stale terminal-event map must abort the whole run.
+
+``_replay_ticker`` rejects a mismatched date, but both dispatch paths in
+``run_backtest`` swallow per-ticker exceptions so one unreadable symbol cannot
+kill a long replay. That safety net would turn a wrong events file into a
+quietly incomplete report, so the map is validated before any replay starts —
+and the runner must therefore write no report at all.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.database import Base
+from app.models.ohlcv import OHLCVRecord
+from app.models.ticker import Ticker
+from app.services.backtest_service import TerminalEventError, run_backtest
+
+
+async def _make_snapshot(path: Path, *, symbol: str = "DEAD", bars: int = 40) -> date:
+    """A minimal snapshot with one ticker. Returns its final bar date."""
+    engine = create_async_engine(f"sqlite+aiosqlite:///{path.as_posix()}")
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+
+    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    day = date(2022, 1, 3)
+    last = day
+    async with Session() as db:
+        ticker = Ticker(symbol=symbol, name=symbol, created_at=datetime.utcnow())
+        db.add(ticker)
+        await db.flush()
+        for i in range(bars):
+            while day.weekday() >= 5:
+                day += timedelta(days=1)
+            price = 100.0 + i
+            db.add(OHLCVRecord(
+                ticker_id=ticker.id, date=day, open=price, high=price * 1.01,
+                low=price * 0.99, close=price, volume=1_000_000,
+                created_at=datetime.utcnow(),
+            ))
+            last = day
+            day += timedelta(days=1)
+        await db.commit()
+    await engine.dispose()
+    return last
+
+
+@pytest.mark.asyncio
+async def test_a_mismatched_terminal_date_aborts_the_run(tmp_path):
+    snapshot = tmp_path / "snap.sqlite"
+    final_bar = await _make_snapshot(snapshot)
+
+    engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
+    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    try:
+        async with Session() as db:
+            with pytest.raises(TerminalEventError, match="final bar"):
+                await run_backtest(db, terminal_events={"DEAD": final_bar + timedelta(days=5)})
+    finally:
+        await engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_an_unknown_symbol_aborts_the_run(tmp_path):
+    snapshot = tmp_path / "snap.sqlite"
+    await _make_snapshot(snapshot)
+
+    engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
+    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    try:
+        async with Session() as db:
+            with pytest.raises(TerminalEventError, match="not in this snapshot"):
+                await run_backtest(db, terminal_events={"NOPE": date(2022, 2, 1)})
+    finally:
+        await engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_the_runner_writes_no_report_when_validation_fails(tmp_path, monkeypatch):
+    """The report is written only after run_backtest returns, so an abort must
+    leave the output path untouched rather than a partial or empty report."""
+    snapshot = tmp_path / "snap.sqlite"
+    final_bar = await _make_snapshot(snapshot)
+
+    events = tmp_path / "events.json"
+    events.write_text(json.dumps({"DEAD": (final_bar + timedelta(days=5)).isoformat()}))
+    out = tmp_path / "report.json"
+
+    import scripts.run_backtest_snapshot as runner
+
+    monkeypatch.setattr(
+        runner.sys, "argv",
+        ["run_backtest_snapshot.py", str(snapshot), "--out", str(out),
+         "--terminal-events", str(events), "--quiet", "--workers", "1"],
+    )
+
+    with pytest.raises(TerminalEventError):
+        await runner._main()
+
+    assert not out.exists(), "a failed run must not leave a report behind"
+
+
+@pytest.mark.asyncio
+async def test_a_matching_map_is_accepted(tmp_path):
+    """The guard must not reject a correct map — the run proceeds normally."""
+    snapshot = tmp_path / "snap.sqlite"
+    final_bar = await _make_snapshot(snapshot)
+
+    engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
+    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    try:
+        async with Session() as db:
+            report = await run_backtest(db, terminal_events={"DEAD": final_bar})
+    finally:
+        await engine.dispose()
+
+    assert isinstance(report, dict)