feat(backtest): make an incomplete run visibly incomplete

A snapshot whose schema predated a migration made every _fetch_columns call
raise, both dispatch paths caught the per-ticker exception and continued, and
the run wrote a report with zero candidates that was indistinguishable from an
honest run that found no setups. Nothing said the run had lost its universe.

run_backtest now builds a completeness ledger and emits it as
report["data_quality"]: expected / loaded / evaluated counts, per-symbol
failures with stage and reason, bar-less symbols named, and benchmark
availability. Every universe symbol lands in exactly one terminal bucket and
the arithmetic is checked, so a counter that stops being maintained reports
itself as a failure rather than under-counting quietly.

Benchmark availability is a separate axis: without SPY, residual 12-1 momentum
silently degrades to raw 12-1, and the report now says so instead of leaving it
in a log line.

Three statuses, because severity differs. "failed" (nothing evaluated, empty
universe, unbalanced ledger, missing benchmark) replaces the performance tiles
with the callout rather than warning above them — a banner over ordinary-looking
numbers is exactly how the zero-candidate run would still have read as a quiet
week. "degraded" keeps the tiles with a note. A report with no data_quality
block reads as unknown, not clean; defaulting old reports to complete would
reproduce the bug being fixed.

The same gate applies to the two other consumers of backtest numbers:
productionBaseRates returns null for a failed run, and EvaluationPanel will not
report drift against a broken baseline.

"complete" means the processing and coverage checks passed. It does not mean
the prices are correct — a complete run over silently wrong bars is still
complete, and the card says so.

Also threads an explicit terminal-events option through the replay, validated
before dispatch so a stale events file aborts the run instead of quietly
degrading it. Verified against a healthy snapshot (degraded, 505/506) and one
with tickers.cik dropped (failed, 0/506, 506 fetch failures).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ

tests/unit/test_backtest_data_quality.py

@@ -0,0 +1,141 @@
+"""Guards for the backtest completeness ledger.
+
+The failure this exists to prevent is concrete: a snapshot whose schema predates
+a migration makes every ``_fetch_columns`` call raise, both dispatch paths catch
+per-ticker exceptions and continue, and the run writes a report with zero
+candidates that is indistinguishable from an honest run that found no setups.
+The ledger has to turn that into a report that says so.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+
+from app.services.backtest_service import (
+    DATA_QUALITY_FAILURE_SAMPLE,
+    HORIZON,
+    MIN_LOOKBACK,
+    _DataQuality,
+    _has_short_history,
+)
+
+
+def _healthy(expected: int = 3, evaluated: int = 3) -> _DataQuality:
+    q = _DataQuality(expected=expected, evaluated=evaluated)
+    q.record_benchmark({date(2024, 1, 2): 470.0, date(2024, 1, 3): 471.0}, None)
+    return q
+
+
+def test_a_complete_run_is_ok():
+    report = _healthy().as_report()
+    assert report["status"] == "ok"
+    assert report["complete"] is True
+    assert report["reasons"] == []
+    assert report["loaded"] == report["evaluated"] == 3
+    assert report["benchmark"]["momentum_basis"] == "residual"
+    assert report["benchmark"]["first"] == "2024-01-02"
+    assert report["benchmark"]["last"] == "2024-01-03"
+
+
+def test_every_symbol_failing_to_load_reads_as_failed_not_as_no_setups():
+    """The unsynced-snapshot case: 3 expected, 3 fetch errors, 0 evaluated."""
+    q = _DataQuality(expected=3)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    for symbol in ("AAPL", "MSFT", "NVDA"):
+        q.record_failure(symbol, "fetch", RuntimeError("no such column: tickers.cik"))
+
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert report["complete"] is False
+    assert report["evaluated"] == 0
+    assert report["loaded"] == 0
+    assert report["failed"] == 3
+    assert report["failures_by_stage"] == {"fetch": 3}
+    assert "no symbol was evaluated successfully" in report["reasons"]
+    assert report["failures"][0]["symbol"] == "AAPL"
+    assert "tickers.cik" in report["failures"][0]["error"]
+
+
+def test_a_few_failures_degrade_but_do_not_fail_the_run():
+    q = _DataQuality(expected=10, evaluated=8, no_data_symbols=["RHM"])
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    q.record_failure("HBI", "replay", ValueError("bad bar"))
+
+    report = q.as_report()
+    assert report["status"] == "degraded"
+    assert report["complete"] is False
+    assert report["loaded"] == 9  # no_data is not a load failure
+    assert report["reasons"] == [
+        "1 symbol failed during replay",
+        "1 symbol had no stored bars (RHM)",
+    ]
+
+
+def test_bar_less_symbols_are_named_so_the_reason_is_actionable():
+    """A universe row with no bars recurs on every run until someone clears it.
+    An unnamed count would just nag; the symbol makes it a one-line fix."""
+    q = _DataQuality(expected=9, evaluated=2, no_data_symbols=list("ABCDEFG"))
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    q.evaluated = 2
+
+    report = q.as_report()
+    assert report["no_data"] == 7
+    assert report["no_data_symbols"] == ["A", "B", "C", "D", "E"]
+    assert report["reasons"] == ["7 symbols had no stored bars (A, B, C, D, E, …)"]
+
+
+def test_a_missing_benchmark_fails_the_run_because_the_signal_changed():
+    """Without SPY the gate ranks on raw 12-1 momentum, so the headline no
+    longer describes the production strategy even though every ticker loaded."""
+    q = _DataQuality(expected=3, evaluated=3)
+    q.record_benchmark({}, ConnectionError("alpaca unreachable"))
+
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert report["benchmark"]["momentum_basis"] == "raw_fallback"
+    assert report["benchmark"]["sessions"] == 0
+    assert "alpaca unreachable" in report["benchmark"]["error"]
+    assert any("fell back to raw momentum" in r for r in report["reasons"])
+
+
+def test_an_unbalanced_ledger_is_itself_a_failure():
+    """If a counter stops being maintained the block must not quietly report a
+    clean run — the arithmetic is the self-check."""
+    q = _healthy(expected=10, evaluated=3)  # 7 symbols unaccounted for
+    report = q.as_report()
+    assert report["status"] == "failed"
+    assert any("ledger does not balance" in r for r in report["reasons"])
+
+
+def test_an_empty_universe_is_a_failure():
+    q = _DataQuality(expected=0)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    assert q.as_report()["status"] == "failed"
+
+
+def test_the_failure_list_is_capped_but_the_count_is_not():
+    q = _DataQuality(expected=100)
+    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
+    for i in range(60):
+        q.record_failure(f"SYM{i}", "fetch", RuntimeError("boom"))
+    q.evaluated = 40
+
+    report = q.as_report()
+    assert report["failed"] == 60
+    assert len(report["failures"]) == DATA_QUALITY_FAILURE_SAMPLE
+    assert report["failures_truncated"] is True
+
+
+def test_short_history_counts_a_recent_listing_without_failing_the_run():
+    q = _healthy()
+    q.short_history = 1
+    report = q.as_report()
+    assert report["short_history"] == 1
+    assert report["status"] == "ok", "a recent listing is not a data-quality failure"
+
+
+def test_has_short_history_uses_the_replay_minimum():
+    enough = ([0] * (MIN_LOOKBACK + HORIZON),) + ((),) * 5
+    one_short = ([0] * (MIN_LOOKBACK + HORIZON - 1),) + ((),) * 5
+    assert _has_short_history(enough) == 0
+    assert _has_short_history(one_short) == 1