"""Guards for the backtest completeness ledger. The failure this exists to prevent is concrete: a snapshot whose schema predates a migration makes every ``_fetch_columns`` call raise, both dispatch paths catch per-ticker exceptions and continue, and the run writes a report with zero candidates that is indistinguishable from an honest run that found no setups. The ledger has to turn that into a report that says so. """ from __future__ import annotations from datetime import date from app.services.backtest_service import ( DATA_QUALITY_FAILURE_SAMPLE, HORIZON, MIN_LOOKBACK, _DataQuality, _has_short_history, ) def _healthy(expected: int = 3, evaluated: int = 3) -> _DataQuality: q = _DataQuality(expected=expected, evaluated=evaluated) q.record_benchmark({date(2024, 1, 2): 470.0, date(2024, 1, 3): 471.0}, None) return q def test_a_complete_run_is_ok(): report = _healthy().as_report() assert report["status"] == "ok" assert report["complete"] is True assert report["reasons"] == [] assert report["loaded"] == report["evaluated"] == 3 assert report["benchmark"]["momentum_basis"] == "residual" assert report["benchmark"]["first"] == "2024-01-02" assert report["benchmark"]["last"] == "2024-01-03" def test_every_symbol_failing_to_load_reads_as_failed_not_as_no_setups(): """The unsynced-snapshot case: 3 expected, 3 fetch errors, 0 evaluated.""" q = _DataQuality(expected=3) q.record_benchmark({date(2024, 1, 2): 470.0}, None) for symbol in ("AAPL", "MSFT", "NVDA"): q.record_failure(symbol, "fetch", RuntimeError("no such column: tickers.cik")) report = q.as_report() assert report["status"] == "failed" assert report["complete"] is False assert report["evaluated"] == 0 assert report["loaded"] == 0 assert report["failed"] == 3 assert report["failures_by_stage"] == {"fetch": 3} assert "no symbol was evaluated successfully" in report["reasons"] assert report["failures"][0]["symbol"] == "AAPL" assert "tickers.cik" in report["failures"][0]["error"] def test_a_few_failures_degrade_but_do_not_fail_the_run(): q = _DataQuality(expected=10, evaluated=8, no_data_symbols=["RHM"]) q.record_benchmark({date(2024, 1, 2): 470.0}, None) q.record_failure("HBI", "replay", ValueError("bad bar")) report = q.as_report() assert report["status"] == "degraded" assert report["complete"] is False assert report["loaded"] == 9 # no_data is not a load failure assert report["reasons"] == [ "1 symbol failed during replay", "1 symbol had no stored bars (RHM)", ] def test_bar_less_symbols_are_named_so_the_reason_is_actionable(): """A universe row with no bars recurs on every run until someone clears it. An unnamed count would just nag; the symbol makes it a one-line fix.""" q = _DataQuality(expected=9, evaluated=2, no_data_symbols=list("ABCDEFG")) q.record_benchmark({date(2024, 1, 2): 470.0}, None) q.evaluated = 2 report = q.as_report() assert report["no_data"] == 7 assert report["no_data_symbols"] == ["A", "B", "C", "D", "E"] assert report["reasons"] == ["7 symbols had no stored bars (A, B, C, D, E, …)"] def test_a_missing_benchmark_fails_the_run_because_the_signal_changed(): """Without SPY the gate ranks on raw 12-1 momentum, so the headline no longer describes the production strategy even though every ticker loaded.""" q = _DataQuality(expected=3, evaluated=3) q.record_benchmark({}, ConnectionError("alpaca unreachable")) report = q.as_report() assert report["status"] == "failed" assert report["benchmark"]["momentum_basis"] == "raw_fallback" assert report["benchmark"]["sessions"] == 0 assert "alpaca unreachable" in report["benchmark"]["error"] assert any("fell back to raw momentum" in r for r in report["reasons"]) def test_an_unbalanced_ledger_is_itself_a_failure(): """If a counter stops being maintained the block must not quietly report a clean run — the arithmetic is the self-check.""" q = _healthy(expected=10, evaluated=3) # 7 symbols unaccounted for report = q.as_report() assert report["status"] == "failed" assert any("ledger does not balance" in r for r in report["reasons"]) def test_an_empty_universe_is_a_failure(): q = _DataQuality(expected=0) q.record_benchmark({date(2024, 1, 2): 470.0}, None) assert q.as_report()["status"] == "failed" def test_the_failure_list_is_capped_but_the_count_is_not(): q = _DataQuality(expected=100) q.record_benchmark({date(2024, 1, 2): 470.0}, None) for i in range(60): q.record_failure(f"SYM{i}", "fetch", RuntimeError("boom")) q.evaluated = 40 report = q.as_report() assert report["failed"] == 60 assert len(report["failures"]) == DATA_QUALITY_FAILURE_SAMPLE assert report["failures_truncated"] is True def test_short_history_counts_a_recent_listing_without_failing_the_run(): q = _healthy() q.short_history = 1 report = q.as_report() assert report["short_history"] == 1 assert report["status"] == "ok", "a recent listing is not a data-quality failure" def test_has_short_history_uses_the_replay_minimum(): enough = ([0] * (MIN_LOOKBACK + HORIZON),) + ((),) * 5 one_short = ([0] * (MIN_LOOKBACK + HORIZON - 1),) + ((),) * 5 assert _has_short_history(enough) == 0 assert _has_short_history(one_short) == 1