From 477aa4b2dac582e1616feac25b87a731796dcf91 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Wed, 5 Aug 2026 10:40:45 +0200 Subject: [PATCH] fix: support legacy research snapshots on macOS --- docs/research/portfolio-capacity-bracket.md | 23 +++++- scripts/run_portfolio_construction_matrix.py | 49 +++++++++-- .../unit/test_portfolio_capacity_research.py | 82 +++++++++++++++++++ 3 files changed, 147 insertions(+), 7 deletions(-) diff --git a/docs/research/portfolio-capacity-bracket.md b/docs/research/portfolio-capacity-bracket.md index f4e5872..e463274 100644 --- a/docs/research/portfolio-capacity-bracket.md +++ b/docs/research/portfolio-capacity-bracket.md @@ -120,13 +120,32 @@ atomically and resume verifies a fingerprint over the implementation commit, this specification hash, snapshot SHA-256, cache key, arm definitions, costs, and cohort manifest. An authoritative run refuses a dirty worktree. +The loader reads only ticker ID/symbol and the OHLCV columns used by replay, so +snapshots created before SEC metadata added `tickers.cik`, `tickers.sic`, and +`tickers.sic_description` remain valid. Do not migrate or alter the research +snapshot: its original SHA-256 is part of the run fingerprint. + +macOS environment setup from the repository root (zsh): + + python3 -m venv .venv + ./.venv/bin/python -m pip install -e '.[dev]' + Preflight: - python scripts/run_portfolio_construction_matrix.py backtest_snapshots/research.sqlite --run-id prod505-capacity-bracket-daily-v1 --validate-only + ./.venv/bin/python scripts/run_portfolio_construction_matrix.py \ + backtest_snapshots/research.sqlite \ + --run-id prod505-capacity-bracket-daily-v1 \ + --workers auto \ + --resume \ + --validate-only Authoritative run: - python scripts/run_portfolio_construction_matrix.py backtest_snapshots/research.sqlite --run-id prod505-capacity-bracket-daily-v1 --workers auto --resume + ./.venv/bin/python scripts/run_portfolio_construction_matrix.py \ + backtest_snapshots/research.sqlite \ + --run-id prod505-capacity-bracket-daily-v1 \ + --workers auto \ + --resume Commit only the compact final JSON and Markdown reports. Raw curves, trades, candidate caches, and checkpoints remain ignored. diff --git a/scripts/run_portfolio_construction_matrix.py b/scripts/run_portfolio_construction_matrix.py index 84b133e..2df4277 100644 --- a/scripts/run_portfolio_construction_matrix.py +++ b/scripts/run_portfolio_construction_matrix.py @@ -210,6 +210,43 @@ def _worker_run_cell(cell: dict[str, Any]) -> dict[str, Any]: } +async def _fetch_snapshot_columns( + db: AsyncSession, + ticker_id: int, +) -> tuple | None: + '''Load only the stable OHLCV columns required by the research replay. + + Research snapshots can predate unrelated additions to the Ticker ORM model + (for example SEC CIK/SIC metadata). Keeping this query column-scoped avoids + requiring or mutating those newer application-schema fields. + ''' + from app.models.ohlcv import OHLCVRecord + + result = await db.execute( + select( + OHLCVRecord.date, + OHLCVRecord.open, + OHLCVRecord.high, + OHLCVRecord.low, + OHLCVRecord.close, + OHLCVRecord.volume, + ) + .where(OHLCVRecord.ticker_id == ticker_id) + .order_by(OHLCVRecord.date) + ) + rows = result.all() + if not rows: + return None + return ( + [row[0].toordinal() for row in rows], + [float(row[1]) for row in rows], + [float(row[2]) for row in rows], + [float(row[3]) for row in rows], + [float(row[4]) for row in rows], + [int(row[5]) for row in rows], + ) + + async def _load_snapshot( snapshot: Path, *, @@ -238,14 +275,16 @@ async def _load_snapshot( refresh=False, ) ticker_result = await db.execute( - select(Ticker).order_by(Ticker.symbol) + select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol) ) - symbols = [ - ticker.symbol for ticker in ticker_result.scalars().all() + ticker_rows = [ + (int(ticker_id), str(symbol)) + for ticker_id, symbol in ticker_result.all() ] + symbols = [symbol for _ticker_id, symbol in ticker_rows] prices: dict[str, tuple] = {} - for index, symbol in enumerate(symbols, 1): - columns = await bt._fetch_columns(db, symbol) + for index, (ticker_id, symbol) in enumerate(ticker_rows, 1): + columns = await _fetch_snapshot_columns(db, ticker_id) if columns is not None: prices[symbol] = columns if not quiet and index % 50 == 0: diff --git a/tests/unit/test_portfolio_capacity_research.py b/tests/unit/test_portfolio_capacity_research.py index ce81cf7..9b10368 100644 --- a/tests/unit/test_portfolio_capacity_research.py +++ b/tests/unit/test_portfolio_capacity_research.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +import sqlite3 from datetime import date, timedelta import pytest @@ -17,6 +19,7 @@ from scripts.portfolio_capacity_research import ( from scripts.run_portfolio_construction_matrix import ( _assert_clean_worktree, _checkpoint_state, + _load_snapshot, _markdown, _operational_summary, _worker_init, @@ -101,6 +104,85 @@ def test_new_simulator_option_defaults_match_explicit_defaults(): assert legacy == explicit +def test_load_snapshot_accepts_pre_sec_ticker_schema(tmp_path, monkeypatch): + snapshot = tmp_path / 'legacy-research.sqlite' + with sqlite3.connect(snapshot) as connection: + connection.executescript( + ''' + CREATE TABLE tickers ( + id INTEGER PRIMARY KEY, + symbol VARCHAR(10) NOT NULL UNIQUE, + name VARCHAR(120), + created_at DATETIME NOT NULL + ); + CREATE TABLE ohlcv_records ( + id INTEGER PRIMARY KEY, + ticker_id INTEGER NOT NULL, + date DATE NOT NULL, + open FLOAT NOT NULL, + high FLOAT NOT NULL, + low FLOAT NOT NULL, + close FLOAT NOT NULL, + volume BIGINT NOT NULL, + created_at DATETIME NOT NULL + ); + INSERT INTO tickers VALUES + (1, 'LEGACY', 'Legacy Co', '2024-01-01 00:00:00'); + INSERT INTO ohlcv_records VALUES + (1, 1, '2024-01-02', 100, 102, 99, 101, 1000000, + '2024-01-02 00:00:00'); + ''' + ) + + async def recommendation_config(_db): + return {} + + async def activation_config(_db): + return {'min_momentum_percentile': 80.0} + + async def exit_policy(_db): + return {'mode': 'atr_trailing', 'hold_days': 30, 'atr_multiplier': 3.0} + + async def benchmark_closes(_db, *, days, refresh): + assert days is None + assert refresh is False + return {date(2024, 1, 2): 100.0} + + monkeypatch.setattr( + 'app.services.recommendation_service.get_recommendation_config', + recommendation_config, + ) + monkeypatch.setattr( + 'app.services.admin_service.get_activation_config', + activation_config, + ) + monkeypatch.setattr( + 'app.services.paper_trade_service.get_exit_policy', + exit_policy, + ) + monkeypatch.setattr( + 'app.services.backtest_service._load_benchmark_closes_for_backtest', + benchmark_closes, + ) + + loaded = asyncio.run(_load_snapshot(snapshot, quiet=True)) + + assert loaded['symbols'] == ['LEGACY'] + assert loaded['prices']['LEGACY'] == ( + [date(2024, 1, 2).toordinal()], + [100.0], + [102.0], + [99.0], + [101.0], + [1_000_000], + ) + with sqlite3.connect(snapshot) as connection: + columns = { + row[1] for row in connection.execute('PRAGMA table_info(tickers)') + } + assert {'cik', 'sic', 'sic_description'}.isdisjoint(columns) + + def test_unbounded_count_and_effective_risk_floor(): start = date(2025, 1, 6) ords = [start.toordinal() + offset for offset in range(4)]