dennisthiessenandClaude Opus 5
f2ef8e5438
A snapshot whose schema predated a migration made every _fetch_columns call raise, both dispatch paths caught the per-ticker exception and continued, and the run wrote a report with zero candidates that was indistinguishable from an honest run that found no setups. Nothing said the run had lost its universe. run_backtest now builds a completeness ledger and emits it as report["data_quality"]: expected / loaded / evaluated counts, per-symbol failures with stage and reason, bar-less symbols named, and benchmark availability. Every universe symbol lands in exactly one terminal bucket and the arithmetic is checked, so a counter that stops being maintained reports itself as a failure rather than under-counting quietly. Benchmark availability is a separate axis: without SPY, residual 12-1 momentum silently degrades to raw 12-1, and the report now says so instead of leaving it in a log line. Three statuses, because severity differs. "failed" (nothing evaluated, empty universe, unbalanced ledger, missing benchmark) replaces the performance tiles with the callout rather than warning above them — a banner over ordinary-looking numbers is exactly how the zero-candidate run would still have read as a quiet week. "degraded" keeps the tiles with a note. A report with no data_quality block reads as unknown, not clean; defaulting old reports to complete would reproduce the bug being fixed. The same gate applies to the two other consumers of backtest numbers: productionBaseRates returns null for a failed run, and EvaluationPanel will not report drift against a broken baseline. "complete" means the processing and coverage checks passed. It does not mean the prices are correct — a complete run over silently wrong bars is still complete, and the card says so. Also threads an explicit terminal-events option through the replay, validated before dispatch so a stale events file aborts the run instead of quietly degrading it. Verified against a healthy snapshot (degraded, 505/506) and one with tickers.cik dropped (failed, 0/506, 506 fetch failures). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ
124 lines
4.6 KiB
Python
124 lines
4.6 KiB
Python
"""Runner-level guard: a stale terminal-event map must abort the whole run.
|
|
|
|
``_replay_ticker`` rejects a mismatched date, but both dispatch paths in
|
|
``run_backtest`` swallow per-ticker exceptions so one unreadable symbol cannot
|
|
kill a long replay. That safety net would turn a wrong events file into a
|
|
quietly incomplete report, so the map is validated before any replay starts —
|
|
and the runner must therefore write no report at all.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import date, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
from app.models.ohlcv import OHLCVRecord
|
|
from app.models.ticker import Ticker
|
|
from app.services.backtest_service import TerminalEventError, run_backtest
|
|
|
|
|
|
async def _make_snapshot(path: Path, *, symbol: str = "DEAD", bars: int = 40) -> date:
|
|
"""A minimal snapshot with one ticker. Returns its final bar date."""
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{path.as_posix()}")
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
day = date(2022, 1, 3)
|
|
last = day
|
|
async with Session() as db:
|
|
ticker = Ticker(symbol=symbol, name=symbol, created_at=datetime.utcnow())
|
|
db.add(ticker)
|
|
await db.flush()
|
|
for i in range(bars):
|
|
while day.weekday() >= 5:
|
|
day += timedelta(days=1)
|
|
price = 100.0 + i
|
|
db.add(OHLCVRecord(
|
|
ticker_id=ticker.id, date=day, open=price, high=price * 1.01,
|
|
low=price * 0.99, close=price, volume=1_000_000,
|
|
created_at=datetime.utcnow(),
|
|
))
|
|
last = day
|
|
day += timedelta(days=1)
|
|
await db.commit()
|
|
await engine.dispose()
|
|
return last
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_mismatched_terminal_date_aborts_the_run(tmp_path):
|
|
snapshot = tmp_path / "snap.sqlite"
|
|
final_bar = await _make_snapshot(snapshot)
|
|
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
try:
|
|
async with Session() as db:
|
|
with pytest.raises(TerminalEventError, match="final bar"):
|
|
await run_backtest(db, terminal_events={"DEAD": final_bar + timedelta(days=5)})
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_an_unknown_symbol_aborts_the_run(tmp_path):
|
|
snapshot = tmp_path / "snap.sqlite"
|
|
await _make_snapshot(snapshot)
|
|
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
try:
|
|
async with Session() as db:
|
|
with pytest.raises(TerminalEventError, match="not in this snapshot"):
|
|
await run_backtest(db, terminal_events={"NOPE": date(2022, 2, 1)})
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_the_runner_writes_no_report_when_validation_fails(tmp_path, monkeypatch):
|
|
"""The report is written only after run_backtest returns, so an abort must
|
|
leave the output path untouched rather than a partial or empty report."""
|
|
snapshot = tmp_path / "snap.sqlite"
|
|
final_bar = await _make_snapshot(snapshot)
|
|
|
|
events = tmp_path / "events.json"
|
|
events.write_text(json.dumps({"DEAD": (final_bar + timedelta(days=5)).isoformat()}))
|
|
out = tmp_path / "report.json"
|
|
|
|
import scripts.run_backtest_snapshot as runner
|
|
|
|
monkeypatch.setattr(
|
|
runner.sys, "argv",
|
|
["run_backtest_snapshot.py", str(snapshot), "--out", str(out),
|
|
"--terminal-events", str(events), "--quiet", "--workers", "1"],
|
|
)
|
|
|
|
with pytest.raises(TerminalEventError):
|
|
await runner._main()
|
|
|
|
assert not out.exists(), "a failed run must not leave a report behind"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_matching_map_is_accepted(tmp_path):
|
|
"""The guard must not reject a correct map — the run proceeds normally."""
|
|
snapshot = tmp_path / "snap.sqlite"
|
|
final_bar = await _make_snapshot(snapshot)
|
|
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{snapshot.as_posix()}")
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
try:
|
|
async with Session() as db:
|
|
report = await run_backtest(db, terminal_events={"DEAD": final_bar})
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
assert isinstance(report, dict)
|