fix(test): pin the benchmark in data-quality status assertions
Deploy / lint (push) Successful in 12s
Deploy / test (push) Successful in 2m14s
Deploy / deploy (push) Failing after 50s

CI failed on test_run_backtest_rolls_back_a_failed_ticker_fetch: it asserted
"degraded" and got "failed". The code was right and the test was wrong.

run_backtest calls refresh_benchmark_prices unless it is in offline snapshot
mode, so whether SPY closes exist depends on network access and credentials.
That decides the status, because a missing benchmark means residual 12-1
momentum silently ran as raw 12-1 — a run that changed its own signal
definition, which the ledger reports as failed. A dev box reaches Alpaca and
sees "degraded"; CI does not and correctly sees "failed".

Any test asserting on status now pins the benchmark instead of inheriting
whatever the machine can reach. The every-ticker-fails test is pinned too, so
its "failed" verdict is attributable to the fetch failures rather than passing
for the wrong reason on a benchmark-less box, and the healthy-completeness test
can now assert "ok" outright.

Reproduced with BACKTEST_SNAPSHOT_OFFLINE=1, which skips the refresh and leaves
the benchmark empty exactly as CI has it: the old test fails there, the new one
passes. Full suite verified green both with and without a reachable benchmark.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ
This commit is contained in:
dennisthiessenandClaude Opus 5 committed 2026-09-07 19:46:49 +02:00
1 parent ad3e47dc50
commit 25e821435d
1 file changed
+28 -1
+28 -1
View File
@@ -1580,6 +1580,26 @@ def test_daily_replay_uses_exact_date_ranking_periods():
assert [row["momentum_percentile"] for row in candidates] == [0.0, 100.0, 100.0, 0.0] assert [row["momentum_percentile"] for row in candidates] == [0.0, 100.0, 100.0, 0.0]
def _pin_benchmark(monkeypatch, closes: dict | None = None) -> None:
"""Fix the benchmark axis so a data-quality status is deterministic.
``run_backtest`` calls ``refresh_benchmark_prices`` unless it is in offline
snapshot mode, so whether SPY closes exist depends on network access and
credentials. That decides ``degraded`` vs ``failed`` — a missing benchmark
means residual momentum silently ran as raw momentum — so a test that
asserts on status must pin it rather than inherit whatever the machine can
reach. (This is why CI and a dev box disagreed: CI has no benchmark.)
"""
from datetime import date as _date
pinned = {_date(2025, 1, 2): 470.0, _date(2025, 1, 3): 471.0} if closes is None else closes
async def _stub(db, **kwargs):
return pinned
monkeypatch.setattr(bt, "_load_benchmark_closes_for_backtest", _stub)
async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None: async def _seed_oscillating_ticker(session, symbol: str, n: int = 160) -> None:
t = Ticker(symbol=symbol) t = Ticker(symbol=symbol)
session.add(t) session.add(t)
@@ -1681,6 +1701,7 @@ async def test_run_backtest_rolls_back_a_failed_ticker_fetch(session, monkeypatc
monkeypatch.setattr(bt, "_fetch_columns", failing_fetch) monkeypatch.setattr(bt, "_fetch_columns", failing_fetch)
monkeypatch.setattr(session, "rollback", tracking_rollback) monkeypatch.setattr(session, "rollback", tracking_rollback)
_pin_benchmark(monkeypatch)
report = await bt.run_backtest(session) report = await bt.run_backtest(session)
@@ -1702,12 +1723,15 @@ async def test_run_backtest_rolls_back_a_failed_ticker_fetch(session, monkeypatc
assert quality["status"] == "degraded" assert quality["status"] == "degraded"
async def test_run_backtest_reports_its_own_completeness(session): async def test_run_backtest_reports_its_own_completeness(session, monkeypatch):
"""The healthy case: every ticker accounted for, so the report says so.""" """The healthy case: every ticker accounted for, so the report says so."""
await _seed_oscillating_ticker(session, "OSC") await _seed_oscillating_ticker(session, "OSC")
_pin_benchmark(monkeypatch)
report = await bt.run_backtest(session) report = await bt.run_backtest(session)
quality = report["data_quality"] quality = report["data_quality"]
assert quality["status"] == "ok"
assert quality["complete"] is True
assert quality["expected"] == report["tickers"] == 1 assert quality["expected"] == report["tickers"] == 1
assert quality["evaluated"] == 1 assert quality["evaluated"] == 1
assert quality["failed"] == 0 assert quality["failed"] == 0
@@ -1728,6 +1752,9 @@ async def test_run_backtest_marks_a_run_failed_when_every_ticker_fails(session,
raise RuntimeError("no such column: tickers.cik") raise RuntimeError("no such column: tickers.cik")
monkeypatch.setattr(bt, "_fetch_columns", always_failing_fetch) monkeypatch.setattr(bt, "_fetch_columns", always_failing_fetch)
# Pinned so the "failed" verdict is attributable to the fetch failures, not
# to a benchmark that happens to be missing on this machine.
_pin_benchmark(monkeypatch)
report = await bt.run_backtest(session) report = await bt.run_backtest(session)