signal-platform/tests/unit/test_backtest_data_quality.py

"""Guards for the backtest completeness ledger.

The failure this exists to prevent is concrete: a snapshot whose schema predates
a migration makes every ``_fetch_columns`` call raise, both dispatch paths catch
per-ticker exceptions and continue, and the run writes a report with zero
candidates that is indistinguishable from an honest run that found no setups.
The ledger has to turn that into a report that says so.
"""

from __future__ import annotations

from datetime import date

from app.services.backtest_service import (
    DATA_QUALITY_FAILURE_SAMPLE,
    HORIZON,
    MIN_LOOKBACK,
    _DataQuality,
    _has_short_history,
)


def _healthy(expected: int = 3, evaluated: int = 3) -> _DataQuality:
    q = _DataQuality(expected=expected, evaluated=evaluated)
    q.record_benchmark({date(2024, 1, 2): 470.0, date(2024, 1, 3): 471.0}, None)
    return q


def test_a_complete_run_is_ok():
    report = _healthy().as_report()
    assert report["status"] == "ok"
    assert report["complete"] is True
    assert report["reasons"] == []
    assert report["loaded"] == report["evaluated"] == 3
    assert report["benchmark"]["momentum_basis"] == "residual"
    assert report["benchmark"]["first"] == "2024-01-02"
    assert report["benchmark"]["last"] == "2024-01-03"


def test_every_symbol_failing_to_load_reads_as_failed_not_as_no_setups():
    """The unsynced-snapshot case: 3 expected, 3 fetch errors, 0 evaluated."""
    q = _DataQuality(expected=3)
    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
    for symbol in ("AAPL", "MSFT", "NVDA"):
        q.record_failure(symbol, "fetch", RuntimeError("no such column: tickers.cik"))

    report = q.as_report()
    assert report["status"] == "failed"
    assert report["complete"] is False
    assert report["evaluated"] == 0
    assert report["loaded"] == 0
    assert report["failed"] == 3
    assert report["failures_by_stage"] == {"fetch": 3}
    assert "no symbol was evaluated successfully" in report["reasons"]
    assert report["failures"][0]["symbol"] == "AAPL"
    assert "tickers.cik" in report["failures"][0]["error"]


def test_a_few_failures_degrade_but_do_not_fail_the_run():
    q = _DataQuality(expected=10, evaluated=8, no_data_symbols=["RHM"])
    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
    q.record_failure("HBI", "replay", ValueError("bad bar"))

    report = q.as_report()
    assert report["status"] == "degraded"
    assert report["complete"] is False
    assert report["loaded"] == 9  # no_data is not a load failure
    assert report["reasons"] == [
        "1 symbol failed during replay",
        "1 symbol had no stored bars (RHM)",
    ]


def test_bar_less_symbols_are_named_so_the_reason_is_actionable():
    """A universe row with no bars recurs on every run until someone clears it.
    An unnamed count would just nag; the symbol makes it a one-line fix."""
    q = _DataQuality(expected=9, evaluated=2, no_data_symbols=list("ABCDEFG"))
    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
    q.evaluated = 2

    report = q.as_report()
    assert report["no_data"] == 7
    assert report["no_data_symbols"] == ["A", "B", "C", "D", "E"]
    assert report["reasons"] == ["7 symbols had no stored bars (A, B, C, D, E, …)"]


def test_a_missing_benchmark_fails_the_run_because_the_signal_changed():
    """Without SPY the gate ranks on raw 12-1 momentum, so the headline no
    longer describes the production strategy even though every ticker loaded."""
    q = _DataQuality(expected=3, evaluated=3)
    q.record_benchmark({}, ConnectionError("alpaca unreachable"))

    report = q.as_report()
    assert report["status"] == "failed"
    assert report["benchmark"]["momentum_basis"] == "raw_fallback"
    assert report["benchmark"]["sessions"] == 0
    assert "alpaca unreachable" in report["benchmark"]["error"]
    assert any("fell back to raw momentum" in r for r in report["reasons"])


def test_an_unbalanced_ledger_is_itself_a_failure():
    """If a counter stops being maintained the block must not quietly report a
    clean run — the arithmetic is the self-check."""
    q = _healthy(expected=10, evaluated=3)  # 7 symbols unaccounted for
    report = q.as_report()
    assert report["status"] == "failed"
    assert any("ledger does not balance" in r for r in report["reasons"])


def test_an_empty_universe_is_a_failure():
    q = _DataQuality(expected=0)
    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
    assert q.as_report()["status"] == "failed"


def test_the_failure_list_is_capped_but_the_count_is_not():
    q = _DataQuality(expected=100)
    q.record_benchmark({date(2024, 1, 2): 470.0}, None)
    for i in range(60):
        q.record_failure(f"SYM{i}", "fetch", RuntimeError("boom"))
    q.evaluated = 40

    report = q.as_report()
    assert report["failed"] == 60
    assert len(report["failures"]) == DATA_QUALITY_FAILURE_SAMPLE
    assert report["failures_truncated"] is True


def test_short_history_counts_a_recent_listing_without_failing_the_run():
    q = _healthy()
    q.short_history = 1
    report = q.as_report()
    assert report["short_history"] == 1
    assert report["status"] == "ok", "a recent listing is not a data-quality failure"


def test_has_short_history_uses_the_replay_minimum():
    enough = ([0] * (MIN_LOOKBACK + HORIZON),) + ((),) * 5
    one_short = ([0] * (MIN_LOOKBACK + HORIZON - 1),) + ((),) * 5
    assert _has_short_history(enough) == 0
    assert _has_short_history(one_short) == 1