From 25e821435d4bd08e12c8f426f2d84ce33934dc66 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Mon, 7 Sep 2026 19:46:49 +0200 Subject: [PATCH] fix(test): pin the benchmark in data-quality status assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed on test_run_backtest_rolls_back_a_failed_ticker_fetch: it asserted "degraded" and got "failed". The code was right and the test was wrong. run_backtest calls refresh_benchmark_prices unless it is in offline snapshot mode, so whether SPY closes exist depends on network access and credentials. That decides the status, because a missing benchmark means residual 12-1 momentum silently ran as raw 12-1 — a run that changed its own signal definition, which the ledger reports as failed. A dev box reaches Alpaca and sees "degraded"; CI does not and correctly sees "failed". Any test asserting on status now pins the benchmark instead of inheriting whatever the machine can reach. The every-ticker-fails test is pinned too, so its "failed" verdict is attributable to the fetch failures rather than passing for the wrong reason on a benchmark-less box, and the healthy-completeness test can now assert "ok" outright. Reproduced with BACKTEST_SNAPSHOT_OFFLINE=1, which skips the refresh and leaves the benchmark empty exactly as CI has it: the old test fails there, the new one passes. Full suite verified green both with and without a reachable benchmark. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ --- tests/unit/test_backtest_service.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index 1eec5b5..391d84e 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -1580,6 +1580,26 @@ def test_daily_replay_uses_exact_date_ranking_periods(): assert [row["momentum_percentile"] for row in candidates] == [0.0, 100.0, 100.0, 0.0] +def _pin_benchmark(monkeypatch, closes: dict | None = None) -> None: + """Fix the benchmark axis so a data-quality status is deterministic. + + ``run_backtest`` calls ``refresh_benchmark_prices`` unless it is in offline + snapshot mode, so whether SPY closes exist depends on network access and + credentials. That decides ``degraded`` vs ``failed`` — a missing benchmark + means residual momentum silently ran as raw momentum — so a test that + asserts on status must pin it rather than inherit whatever the machine can + reach. (This is why CI and a dev box disagreed: CI has no benchmark.) + """ + from datetime import date as _date + + pinned = {_date(2025, 1, 2): 470.0, _date(2025, 1, 3): 471.0} if closes is None else closes + + async def _stub(db, **kwargs): + return pinned + + monkeypatch.setattr(bt, "_load_benchmark_closes_for_backtest", _stub) + + async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None: t = Ticker(symbol=symbol) session.add(t) @@ -1681,6 +1701,7 @@ async def test_run_backtest_rolls_back_a_failed_ticker_fetch(session, monkeypatc monkeypatch.setattr(bt, "_fetch_columns", failing_fetch) monkeypatch.setattr(session, "rollback", tracking_rollback) + _pin_benchmark(monkeypatch) report = await bt.run_backtest(session) @@ -1702,12 +1723,15 @@ async def test_run_backtest_rolls_back_a_failed_ticker_fetch(session, monkeypatc assert quality["status"] == "degraded" -async def test_run_backtest_reports_its_own_completeness(session): +async def test_run_backtest_reports_its_own_completeness(session, monkeypatch): """The healthy case: every ticker accounted for, so the report says so.""" await _seed_oscillating_ticker(session, "OSC") + _pin_benchmark(monkeypatch) report = await bt.run_backtest(session) quality = report["data_quality"] + assert quality["status"] == "ok" + assert quality["complete"] is True assert quality["expected"] == report["tickers"] == 1 assert quality["evaluated"] == 1 assert quality["failed"] == 0 @@ -1728,6 +1752,9 @@ async def test_run_backtest_marks_a_run_failed_when_every_ticker_fails(session, raise RuntimeError("no such column: tickers.cik") monkeypatch.setattr(bt, "_fetch_columns", always_failing_fetch) + # Pinned so the "failed" verdict is attributable to the fetch failures, not + # to a benchmark that happens to be missing on this machine. + _pin_benchmark(monkeypatch) report = await bt.run_backtest(session)