906 lines
28 KiB
Python
906 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import pickle
|
|
import sqlite3
|
|
from datetime import date, timedelta
|
|
|
|
import pytest
|
|
|
|
from app.services import backtest_service as bt
|
|
from scripts.portfolio_capacity_research import (
|
|
ANCHOR_YEARS,
|
|
RISK_FLOOR_ARMS,
|
|
aggregate_results,
|
|
bootstrap_median_interval,
|
|
build_cells,
|
|
build_cohort_manifest,
|
|
iqr,
|
|
summarize_simulation,
|
|
validate_cohort_manifest,
|
|
)
|
|
from scripts.run_portfolio_construction_matrix import (
|
|
CACHE_VERSION,
|
|
STUDIES,
|
|
_assert_clean_worktree,
|
|
_build_candidate_cache,
|
|
_checkpoint_state,
|
|
_construction_candidate_view,
|
|
_construction_universe_errors,
|
|
_json_hash,
|
|
_load_snapshot,
|
|
_markdown,
|
|
_operational_summary,
|
|
_risk_floor_markdown,
|
|
_worker_init,
|
|
_worker_run_cell,
|
|
_write_cell_checkpoint,
|
|
)
|
|
|
|
|
|
def _prices(ords: list[int], close: float = 100.0) -> tuple:
|
|
closes = [close] * len(ords)
|
|
return (
|
|
ords,
|
|
list(closes),
|
|
[value + 1.0 for value in closes],
|
|
[value - 1.0 for value in closes],
|
|
list(closes),
|
|
[1_000_000] * len(ords),
|
|
)
|
|
|
|
|
|
def _candidate(
|
|
symbol: str,
|
|
day: date,
|
|
*,
|
|
entry: float = 100.0,
|
|
stop: float = 80.0,
|
|
rank: float = 90.0,
|
|
) -> dict:
|
|
return {
|
|
'qualified': True,
|
|
'direction': 'long',
|
|
'symbol': symbol,
|
|
'date': day.isoformat(),
|
|
'entry': entry,
|
|
'stop': stop,
|
|
'target': entry + 100.0,
|
|
'momentum_percentile': rank,
|
|
'activation_momentum_percentile': rank,
|
|
'residual_high_vol_blend_80_20': rank,
|
|
}
|
|
|
|
|
|
def _business_days(start: date, end: date) -> list[date]:
|
|
days: list[date] = []
|
|
current = start
|
|
while current <= end:
|
|
if current.weekday() < 5:
|
|
days.append(current)
|
|
current += timedelta(days=1)
|
|
return days
|
|
|
|
|
|
def test_new_simulator_option_defaults_match_explicit_defaults():
|
|
start = date(2025, 1, 6)
|
|
ords = [start.toordinal() + offset for offset in range(8)]
|
|
prices = {'AAA': _prices(ords)}
|
|
candidates = [_candidate('AAA', start)]
|
|
|
|
legacy = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
3,
|
|
include_trades=True,
|
|
)
|
|
explicit = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
3,
|
|
max_positions=10,
|
|
min_initial_risk_fraction=None,
|
|
weekly_top_n_rebalance=False,
|
|
measurement_start_date=None,
|
|
hard_end_date=None,
|
|
include_capacity_diagnostics=False,
|
|
include_trades=True,
|
|
)
|
|
|
|
assert legacy == explicit
|
|
|
|
|
|
def test_load_snapshot_accepts_pre_sec_ticker_schema(tmp_path, monkeypatch):
|
|
snapshot = tmp_path / 'legacy-research.sqlite'
|
|
with sqlite3.connect(snapshot) as connection:
|
|
connection.executescript(
|
|
'''
|
|
CREATE TABLE tickers (
|
|
id INTEGER PRIMARY KEY,
|
|
symbol VARCHAR(10) NOT NULL UNIQUE,
|
|
name VARCHAR(120),
|
|
created_at DATETIME NOT NULL
|
|
);
|
|
CREATE TABLE ohlcv_records (
|
|
id INTEGER PRIMARY KEY,
|
|
ticker_id INTEGER NOT NULL,
|
|
date DATE NOT NULL,
|
|
open FLOAT NOT NULL,
|
|
high FLOAT NOT NULL,
|
|
low FLOAT NOT NULL,
|
|
close FLOAT NOT NULL,
|
|
volume BIGINT NOT NULL,
|
|
created_at DATETIME NOT NULL
|
|
);
|
|
CREATE TABLE research_rank_only (
|
|
symbol VARCHAR(10) PRIMARY KEY
|
|
);
|
|
INSERT INTO tickers VALUES
|
|
(1, 'LEGACY', 'Legacy Co', '2024-01-01 00:00:00'),
|
|
(2, 'RANK', 'Rank Only Co', '2024-01-01 00:00:00');
|
|
INSERT INTO ohlcv_records VALUES
|
|
(1, 1, '2024-01-02', 100, 102, 99, 101, 1000000,
|
|
'2024-01-02 00:00:00'),
|
|
(2, 2, '2024-01-02', 50, 51, 49, 50, 500000,
|
|
'2024-01-02 00:00:00');
|
|
INSERT INTO research_rank_only VALUES ('RANK');
|
|
'''
|
|
)
|
|
|
|
async def recommendation_config(_db):
|
|
return {}
|
|
|
|
async def activation_config(_db):
|
|
return {'min_momentum_percentile': 80.0}
|
|
|
|
async def exit_policy(_db):
|
|
return {'mode': 'atr_trailing', 'hold_days': 30, 'atr_multiplier': 3.0}
|
|
|
|
async def benchmark_closes(_db, *, days, refresh):
|
|
assert days is None
|
|
assert refresh is False
|
|
return {date(2024, 1, 2): 100.0}
|
|
|
|
monkeypatch.setattr(
|
|
'app.services.recommendation_service.get_recommendation_config',
|
|
recommendation_config,
|
|
)
|
|
monkeypatch.setattr(
|
|
'app.services.admin_service.get_activation_config',
|
|
activation_config,
|
|
)
|
|
monkeypatch.setattr(
|
|
'app.services.paper_trade_service.get_exit_policy',
|
|
exit_policy,
|
|
)
|
|
monkeypatch.setattr(
|
|
'app.services.backtest_service._load_benchmark_closes_for_backtest',
|
|
benchmark_closes,
|
|
)
|
|
|
|
loaded = asyncio.run(_load_snapshot(snapshot, quiet=True))
|
|
|
|
assert loaded['symbols'] == ['LEGACY', 'RANK']
|
|
assert loaded['construction_symbols'] == {'LEGACY'}
|
|
assert loaded['prices']['LEGACY'] == (
|
|
[date(2024, 1, 2).toordinal()],
|
|
[100.0],
|
|
[102.0],
|
|
[99.0],
|
|
[101.0],
|
|
[1_000_000],
|
|
)
|
|
assert loaded['prices']['RANK'][4] == [50.0]
|
|
assert loaded['construction_universe_manifest'][
|
|
'construction_ticker_rows'
|
|
] == 1
|
|
assert loaded['construction_universe_manifest']['rank_only_ticker_rows'] == 1
|
|
with sqlite3.connect(snapshot) as connection:
|
|
columns = {
|
|
row[1] for row in connection.execute('PRAGMA table_info(tickers)')
|
|
}
|
|
assert {'cik', 'sic', 'sic_description'}.isdisjoint(columns)
|
|
|
|
|
|
def test_construction_view_filters_rank_only_rows_without_rebuilding_cache():
|
|
manifest = {
|
|
'ranking_ticker_rows': 506,
|
|
'ranking_symbols_with_prices': 506,
|
|
'construction_ticker_rows': 505,
|
|
'construction_symbols_with_prices': 505,
|
|
'rank_only_ticker_rows': 1,
|
|
'rank_only_symbols_with_prices': 1,
|
|
'rank_only_unknown_symbols': 0,
|
|
}
|
|
cached = {
|
|
'key': {'version': 'existing-broad-cache'},
|
|
'qualified_candidates': [
|
|
{'symbol': 'PROD', 'date': '2025-01-02'},
|
|
{'symbol': 'RANK', 'date': '2025-01-02'},
|
|
],
|
|
'qualified_long_count': 2,
|
|
'daily_rank_map': {
|
|
('RANK', '2025-01-02'): {'strategy_rank': 99.0},
|
|
},
|
|
}
|
|
|
|
view = _construction_candidate_view(
|
|
cached,
|
|
{
|
|
'construction_symbols': {'PROD'},
|
|
'construction_universe_manifest': manifest,
|
|
},
|
|
)
|
|
|
|
assert [row['symbol'] for row in view['qualified_candidates']] == ['PROD']
|
|
assert view['raw_full_universe_qualified_long_count'] == 2
|
|
assert view['filtered_rank_only_qualified_long_count'] == 1
|
|
assert view['qualified_long_count'] == 1
|
|
assert ('RANK', '2025-01-02') in view['daily_rank_map']
|
|
assert len(cached['qualified_candidates']) == 2
|
|
|
|
|
|
def test_existing_broad_candidate_cache_key_remains_reusable(tmp_path, monkeypatch):
|
|
snapshot = tmp_path / 'research.sqlite'
|
|
snapshot.write_bytes(b'snapshot-placeholder')
|
|
cache_path = tmp_path / 'broad-cache.pkl'
|
|
snapshot_data = {
|
|
'recommendation_config': {'rr': 3.0},
|
|
'activation': {'min_momentum_percentile': 80.0},
|
|
'runtime_config': {'ranking_key': 'test'},
|
|
'universe_manifest': {
|
|
'ticker_rows': 4655,
|
|
'symbols_with_prices': 4654,
|
|
'symbols_sha256': 'symbols',
|
|
},
|
|
}
|
|
key = {
|
|
'version': CACHE_VERSION,
|
|
'snapshot': str(snapshot.resolve()),
|
|
'snapshot_sha256': 'snapshot-hash',
|
|
'cadence': 'daily',
|
|
'outcome_horizon_sessions': 0,
|
|
'recommendation_config_hash': _json_hash(
|
|
snapshot_data['recommendation_config']
|
|
),
|
|
'activation_hash': _json_hash(snapshot_data['activation']),
|
|
'runtime_config': snapshot_data['runtime_config'],
|
|
'universe_manifest': snapshot_data['universe_manifest'],
|
|
}
|
|
cached = {'key': key, 'qualified_candidates': [{'symbol': 'PROD'}]}
|
|
cache_path.write_bytes(pickle.dumps(cached))
|
|
monkeypatch.setattr(
|
|
bt,
|
|
'_replay_candidates_for_period',
|
|
lambda *_args: pytest.fail('existing cache should avoid replay'),
|
|
)
|
|
|
|
loaded = _build_candidate_cache(
|
|
snapshot_data,
|
|
snapshot=snapshot,
|
|
snapshot_sha256='snapshot-hash',
|
|
cache_path=cache_path,
|
|
workers=1,
|
|
quiet=True,
|
|
)
|
|
|
|
assert loaded == cached
|
|
|
|
|
|
def test_construction_universe_guard_rejects_leaked_broad_book():
|
|
valid = {
|
|
'ranking_ticker_rows': 4655,
|
|
'construction_ticker_rows': 506,
|
|
'construction_symbols_with_prices': 506,
|
|
'rank_only_ticker_rows': 4149,
|
|
'rank_only_unknown_symbols': 0,
|
|
}
|
|
assert _construction_universe_errors(valid) == []
|
|
|
|
leaked = {
|
|
**valid,
|
|
'construction_ticker_rows': 4655,
|
|
'construction_symbols_with_prices': 4654,
|
|
'rank_only_ticker_rows': 0,
|
|
}
|
|
errors = _construction_universe_errors(leaked)
|
|
assert any('450-600' in error for error in errors)
|
|
|
|
|
|
def test_unbounded_count_and_effective_risk_floor():
|
|
start = date(2025, 1, 6)
|
|
ords = [start.toordinal() + offset for offset in range(4)]
|
|
symbols = [f'S{index}' for index in range(25)]
|
|
prices = {symbol: _prices(ords) for symbol in symbols}
|
|
candidates = [
|
|
_candidate(symbol, start, stop=80.0, rank=100.0 - index)
|
|
for index, symbol in enumerate(symbols)
|
|
]
|
|
|
|
capped = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
max_positions=1,
|
|
hard_end_date=start + timedelta(days=4),
|
|
measurement_start_date=start,
|
|
include_capacity_diagnostics=True,
|
|
)
|
|
unbounded = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
max_positions=None,
|
|
min_initial_risk_fraction=0.005,
|
|
hard_end_date=start + timedelta(days=4),
|
|
measurement_start_date=start,
|
|
include_capacity_diagnostics=True,
|
|
)
|
|
|
|
assert capped is not None and unbounded is not None
|
|
assert capped['peak_positions'] == 1
|
|
assert capped['measurement_skipped_book_full'] == 24
|
|
assert unbounded['peak_positions'] > 1
|
|
assert unbounded['measurement_skipped_book_full'] == 0
|
|
assert unbounded['skipped_min_initial_risk'] > 0
|
|
assert unbounded['peak_positions'] == unbounded['trades']
|
|
|
|
|
|
def test_measurement_window_carries_state_but_excludes_pre_anchor_trade_ev():
|
|
start = date(2025, 1, 6)
|
|
anchor = start + timedelta(days=2)
|
|
hard_end = start + timedelta(days=7)
|
|
ords = [
|
|
start.toordinal() + offset
|
|
for offset in range((hard_end - start).days)
|
|
]
|
|
prices = {
|
|
'AAA': _prices(ords, 100.0),
|
|
'BBB': _prices(ords, 100.0),
|
|
}
|
|
candidates = [
|
|
_candidate('AAA', start),
|
|
_candidate('BBB', anchor + timedelta(days=1)),
|
|
]
|
|
|
|
sim = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
start_date=start,
|
|
end_date=hard_end,
|
|
measurement_start_date=anchor,
|
|
hard_end_date=hard_end,
|
|
include_curve=True,
|
|
include_trades=True,
|
|
)
|
|
|
|
assert sim is not None
|
|
assert sim['simulation_start_date'] == start.isoformat()
|
|
assert sim['start_date'] == anchor.isoformat()
|
|
assert sim['measurement_start_positions'] == 1
|
|
assert sim['trades'] == 1
|
|
assert [trade['symbol'] for trade in sim['trade_details']] == ['BBB']
|
|
assert sim['equity_curve'][0]['date'] == anchor.isoformat()
|
|
|
|
|
|
def test_weekly_top10_uses_current_rank_for_both_sides_not_entry_rank():
|
|
monday = date(2025, 1, 6)
|
|
friday = date(2025, 1, 10)
|
|
sessions = _business_days(monday, friday)
|
|
ords = [session.toordinal() for session in sessions]
|
|
prices = {
|
|
'AAA': _prices(ords),
|
|
'BBB': _prices(ords),
|
|
}
|
|
candidates = [
|
|
_candidate('AAA', monday, rank=99.0),
|
|
_candidate('BBB', friday, rank=10.0),
|
|
]
|
|
rank_map = {
|
|
('AAA', friday.isoformat()): {'strategy_rank': 10.0},
|
|
('BBB', friday.isoformat()): {'strategy_rank': 90.0},
|
|
}
|
|
|
|
sim = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
max_positions=1,
|
|
weekly_top_n_rebalance=True,
|
|
daily_rank_map=rank_map,
|
|
measurement_start_date=monday,
|
|
hard_end_date=friday + timedelta(days=1),
|
|
include_trades=True,
|
|
include_capacity_diagnostics=True,
|
|
)
|
|
|
|
assert sim is not None
|
|
assert [trade['symbol'] for trade in sim['trade_details']] == ['AAA', 'BBB']
|
|
assert sim['trade_details'][0]['reason'] == 'weekly_rebalance'
|
|
event = sim['weekly_rebalance_events'][0]
|
|
assert event['exited_symbols'] == ['AAA']
|
|
assert event['selected_entrant_symbols'] == ['BBB']
|
|
|
|
|
|
def test_weekly_top10_incumbent_wins_exact_current_rank_tie():
|
|
monday = date(2025, 1, 6)
|
|
friday = date(2025, 1, 10)
|
|
sessions = _business_days(monday, friday)
|
|
ords = [session.toordinal() for session in sessions]
|
|
prices = {
|
|
'AAA': _prices(ords),
|
|
'BBB': _prices(ords),
|
|
}
|
|
candidates = [
|
|
_candidate('AAA', monday, rank=10.0),
|
|
_candidate('BBB', friday, rank=99.0),
|
|
]
|
|
rank_map = {
|
|
('AAA', friday.isoformat()): {'strategy_rank': 80.0},
|
|
('BBB', friday.isoformat()): {'strategy_rank': 80.0},
|
|
}
|
|
|
|
sim = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
max_positions=1,
|
|
weekly_top_n_rebalance=True,
|
|
daily_rank_map=rank_map,
|
|
measurement_start_date=monday,
|
|
hard_end_date=friday + timedelta(days=1),
|
|
include_trades=True,
|
|
)
|
|
|
|
assert sim is not None
|
|
assert [trade['symbol'] for trade in sim['trade_details']] == ['AAA']
|
|
assert sim['trade_details'][0]['reason'] == 'open_at_end'
|
|
assert sim['weekly_rebalance_events'][0]['replacements'] == 0
|
|
|
|
|
|
def test_weekly_rebalance_exit_bypasses_cooldown_and_churn_is_counted():
|
|
first_monday = date(2025, 1, 6)
|
|
friday = date(2025, 1, 10)
|
|
next_monday = date(2025, 1, 13)
|
|
sessions = _business_days(first_monday, next_monday)
|
|
ords = [session.toordinal() for session in sessions]
|
|
prices = {
|
|
'AAA': _prices(ords),
|
|
'BBB': (
|
|
ords,
|
|
[100.0] * len(ords),
|
|
[101.0] * len(ords),
|
|
[99.0] * (len(ords) - 1) + [70.0],
|
|
[100.0] * len(ords),
|
|
[1_000_000] * len(ords),
|
|
),
|
|
}
|
|
candidates = [
|
|
_candidate('AAA', first_monday, rank=99.0),
|
|
_candidate('BBB', friday, rank=10.0),
|
|
_candidate('AAA', next_monday, rank=99.0),
|
|
]
|
|
rank_map = {
|
|
('AAA', friday.isoformat()): {'strategy_rank': 10.0},
|
|
('BBB', friday.isoformat()): {'strategy_rank': 90.0},
|
|
}
|
|
|
|
sim = bt._simulate_portfolio(
|
|
candidates,
|
|
prices,
|
|
None,
|
|
'hold',
|
|
30,
|
|
max_positions=1,
|
|
reentry_cooldown_sessions=5,
|
|
weekly_top_n_rebalance=True,
|
|
daily_rank_map=rank_map,
|
|
measurement_start_date=first_monday,
|
|
hard_end_date=next_monday + timedelta(days=1),
|
|
include_trades=True,
|
|
)
|
|
|
|
assert sim is not None
|
|
assert [trade['symbol'] for trade in sim['trade_details']] == [
|
|
'AAA',
|
|
'BBB',
|
|
'AAA',
|
|
]
|
|
assert sim['trade_details'][0]['reason'] == 'weekly_rebalance'
|
|
assert sim['rebalance_reentries_within_5_sessions'] == 1
|
|
assert sim['skipped_cooldown'] == 0
|
|
|
|
|
|
def test_cohort_manifest_realizes_seven_frozen_clusters():
|
|
sessions = _business_days(date(2016, 1, 4), date(2026, 7, 17))
|
|
manifest = build_cohort_manifest(sessions)
|
|
|
|
assert validate_cohort_manifest(manifest) == []
|
|
assert manifest['empty_cluster_count'] == 7
|
|
assert manifest['warm_cluster_count'] == 7
|
|
assert set(map(int, manifest['empty_cluster_counts'])) == set(ANCHOR_YEARS)
|
|
assert all(
|
|
int(count) >= 12 for count in manifest['warm_seed_counts'].values()
|
|
)
|
|
cells = build_cells(manifest)
|
|
assert len(cells) == (
|
|
len(manifest['empty_book']) + len(manifest['warm_book'])
|
|
) * 4 * 2
|
|
floor_cells = build_cells(manifest, arms=RISK_FLOOR_ARMS)
|
|
assert len(floor_cells) == (
|
|
len(manifest['empty_book']) + len(manifest['warm_book'])
|
|
) * 2 * 2
|
|
assert {row['arm_id'] for row in floor_cells} == {
|
|
'cap10_incumbent',
|
|
'cap10_min_risk_005',
|
|
}
|
|
|
|
|
|
def test_risk_floor_study_changes_only_the_effective_risk_floor():
|
|
control, treatment = RISK_FLOOR_ARMS
|
|
|
|
assert control['max_positions'] == treatment['max_positions'] == 10
|
|
assert (
|
|
control['weekly_top_n_rebalance']
|
|
== treatment['weekly_top_n_rebalance']
|
|
is False
|
|
)
|
|
assert control['min_initial_risk_fraction'] is None
|
|
assert treatment['min_initial_risk_fraction'] == 0.005
|
|
assert STUDIES['risk-floor-ab']['arms'] == RISK_FLOOR_ARMS
|
|
assert STUDIES['capacity-bracket']['arms'] != RISK_FLOOR_ARMS
|
|
|
|
|
|
def test_zero_outcome_horizon_extends_rank_replay_to_last_session(monkeypatch):
|
|
monkeypatch.setattr(bt, '_window_setups', lambda *_args, **_kwargs: [])
|
|
count = bt.MIN_LOOKBACK + bt.HORIZON
|
|
start = date(2025, 1, 1)
|
|
ords = [start.toordinal() + offset for offset in range(count)]
|
|
columns = _prices(ords)
|
|
|
|
legacy = bt._replay_candidates_for_period(
|
|
'AAA',
|
|
columns,
|
|
{},
|
|
{},
|
|
None,
|
|
date.min,
|
|
'daily',
|
|
True,
|
|
True,
|
|
)
|
|
zero_horizon = bt._replay_candidates_for_period(
|
|
'AAA',
|
|
columns,
|
|
{},
|
|
{},
|
|
None,
|
|
date.min,
|
|
'daily',
|
|
True,
|
|
True,
|
|
0,
|
|
)
|
|
|
|
assert len(zero_horizon) == len(legacy) + bt.HORIZON
|
|
assert zero_horizon[-1]['date'] == date.fromordinal(ords[-1]).isoformat()
|
|
|
|
|
|
def test_gain_to_pain_uses_all_monthly_returns_and_net_r():
|
|
sim = {
|
|
'measurement_start_equity': 100.0,
|
|
'trade_details': [
|
|
{
|
|
'net_r': 1.0,
|
|
'pnl': 10.0,
|
|
'shares': 1.0,
|
|
'entry': 100.0,
|
|
'fill': 110.0,
|
|
'transaction_cost': 0.0,
|
|
},
|
|
{
|
|
'net_r': -0.5,
|
|
'pnl': -5.0,
|
|
'shares': 1.0,
|
|
'entry': 100.0,
|
|
'fill': 95.0,
|
|
'transaction_cost': 0.0,
|
|
},
|
|
],
|
|
'equity_curve': [
|
|
{'date': '2025-01-31', 'equity': 110.0},
|
|
{'date': '2025-02-28', 'equity': 99.0},
|
|
],
|
|
'trades': 2,
|
|
'skipped_book_full': 0,
|
|
}
|
|
|
|
summary = summarize_simulation(sim)
|
|
|
|
assert summary['ev_net_r'] == pytest.approx(0.25)
|
|
assert summary['profit_factor'] == pytest.approx(2.0)
|
|
# Monthly returns are +10% and -10%; all-return numerator is zero.
|
|
assert summary['gain_to_pain'] == pytest.approx(0.0)
|
|
|
|
|
|
def test_simple_cluster_bootstrap_is_deterministic_and_not_a_gate():
|
|
first = bootstrap_median_interval(
|
|
[1, 2, 3, 4, 5, 6, 7],
|
|
seed_parts=('determinism',),
|
|
replicates=500,
|
|
)
|
|
second = bootstrap_median_interval(
|
|
[1, 2, 3, 4, 5, 6, 7],
|
|
seed_parts=('determinism',),
|
|
replicates=500,
|
|
)
|
|
|
|
assert first == second
|
|
assert first['point'] == 4
|
|
assert first['p05'] <= first['point'] <= first['p95']
|
|
|
|
|
|
def test_iqr_materializes_generator_before_both_quantiles():
|
|
assert iqr(value for value in (0.0, 1.0, 2.0, 3.0)) == pytest.approx(1.5)
|
|
|
|
|
|
def test_aggregate_reports_paired_years_and_separate_warm_iqrs():
|
|
cells: list[dict] = []
|
|
for cost in (0.1, 0.2):
|
|
for cluster in ANCHOR_YEARS:
|
|
for seed in range(3):
|
|
path_id = f'warm-{cluster}-{seed}'
|
|
for arm_id, shift in (
|
|
('cap10_incumbent', 0.0),
|
|
('cash_unbounded', 0.2),
|
|
('cap10_weekly_top10', 0.1),
|
|
('cap15_incumbent', 0.05),
|
|
):
|
|
cells.append({
|
|
'arm_id': arm_id,
|
|
'protocol': 'warm_book',
|
|
'path_id': path_id,
|
|
'cluster': cluster,
|
|
'cost_per_side_pct': cost,
|
|
'metrics': {
|
|
'ev_net_r': seed + shift,
|
|
'calmar': 1.0 + seed * 0.1 + shift,
|
|
'profit_factor': 1.5 + shift,
|
|
'gain_to_pain': 2.0 + shift,
|
|
'sortino': 1.0 + shift,
|
|
'cagr_pct': 10.0 + shift,
|
|
'max_drawdown_pct': 5.0,
|
|
'total_return_pct': 10.0 + shift,
|
|
'sharpe': 1.0 + shift,
|
|
},
|
|
})
|
|
for arm_id, shift in (
|
|
('cap10_incumbent', 0.0),
|
|
('cash_unbounded', 0.2),
|
|
('cap10_weekly_top10', 0.1),
|
|
('cap15_incumbent', 0.05),
|
|
):
|
|
cells.append({
|
|
'arm_id': arm_id,
|
|
'protocol': 'empty_book',
|
|
'path_id': f'empty-{cluster}',
|
|
'cluster': cluster,
|
|
'cost_per_side_pct': cost,
|
|
'metrics': {
|
|
'ev_net_r': 1.0 + shift,
|
|
'calmar': 2.0 + shift,
|
|
'profit_factor': 1.5 + shift,
|
|
'gain_to_pain': 2.0 + shift,
|
|
'sortino': 1.0 + shift,
|
|
'cagr_pct': 10.0 + shift,
|
|
'max_drawdown_pct': 5.0,
|
|
'total_return_pct': 10.0 + shift,
|
|
'sharpe': 1.0 + shift,
|
|
},
|
|
})
|
|
|
|
report = aggregate_results(cells)
|
|
|
|
cash_empty = next(
|
|
row
|
|
for row in report['paired_per_year']
|
|
if row['arm_id'] == 'cash_unbounded'
|
|
and row['protocol'] == 'empty_book'
|
|
and row['cost_per_side_pct'] == 0.1
|
|
)
|
|
assert cash_empty['headline']['ev_net_r']['paired_delta_median'] == pytest.approx(
|
|
0.2
|
|
)
|
|
cash_paths = next(
|
|
row
|
|
for row in report['paired_path_distributions']
|
|
if row['arm_id'] == 'cash_unbounded'
|
|
and row['protocol'] == 'empty_book'
|
|
and row['cost_per_side_pct'] == 0.1
|
|
)
|
|
assert cash_paths['metrics']['ev_net_r']['paired_delta_mean'] == pytest.approx(
|
|
0.2
|
|
)
|
|
assert cash_paths['metrics']['ev_net_r']['positive_fraction'] == 1.0
|
|
assert cash_paths['metrics']['ev_net_r']['identical_fraction'] == 0.0
|
|
cash_warm = next(
|
|
row
|
|
for row in report['warm_seed_dispersion']
|
|
if row['arm_id'] == 'cash_unbounded'
|
|
and row['cost_per_side_pct'] == 0.1
|
|
)
|
|
assert set(cash_warm['headline']) == {'ev_net_r', 'calmar'}
|
|
assert 'D' not in cash_warm
|
|
assert cash_warm['headline']['ev_net_r']['median_iqr_ratio'] == pytest.approx(
|
|
1.0
|
|
)
|
|
assert cash_warm['headline']['calmar']['median_iqr_ratio'] == pytest.approx(
|
|
1.0
|
|
)
|
|
assert cash_warm['headline']['ev_net_r']['bootstrap_90']['n'] == 7
|
|
markdown = _markdown({
|
|
'generated_at': '2026-08-05T00:00:00Z',
|
|
'analysis': report,
|
|
'operational_summary': _operational_summary(cells),
|
|
'validation': {
|
|
'construction_universe_manifest': {
|
|
'construction_symbols_with_prices': 506,
|
|
'rank_only_symbols_with_prices': 4148,
|
|
'ranking_symbols_with_prices': 4654,
|
|
},
|
|
'candidate_rank_coverage': {
|
|
'construction_qualified_longs': 5000,
|
|
'filtered_rank_only_qualified_longs': 137000,
|
|
},
|
|
},
|
|
})
|
|
assert 'ΔGain-to-Pain' in markdown
|
|
assert '0.10% per fill' in markdown
|
|
assert '0.20% per fill' in markdown
|
|
assert 'Tradable setup symbols with prices: 506.' in markdown
|
|
assert 'Rank-only qualified rows removed: 137000.' in markdown
|
|
assert 'formal promotion gate' in markdown
|
|
|
|
focused_cells = [
|
|
row
|
|
for row in cells
|
|
if row['arm_id'] == 'cap10_incumbent'
|
|
] + [
|
|
{
|
|
**row,
|
|
'arm_id': 'cap10_min_risk_005',
|
|
}
|
|
for row in cells
|
|
if row['arm_id'] == 'cash_unbounded'
|
|
]
|
|
focused_analysis = aggregate_results(
|
|
focused_cells,
|
|
arms=RISK_FLOOR_ARMS,
|
|
include_warm_dispersion=False,
|
|
)
|
|
assert focused_analysis['warm_seed_dispersion'] == []
|
|
focused_markdown = _risk_floor_markdown({
|
|
'generated_at': '2026-08-05T00:00:00Z',
|
|
'arms': list(RISK_FLOOR_ARMS),
|
|
'protocols': ['empty_book', 'warm_book'],
|
|
'costs_per_side_pct': [0.1, 0.2],
|
|
'analysis': focused_analysis,
|
|
'operational_summary': _operational_summary(
|
|
focused_cells,
|
|
arms=RISK_FLOOR_ARMS,
|
|
),
|
|
})
|
|
assert '# Effective initial-risk floor A/B' in focused_markdown
|
|
assert 'Mean dEV' in focused_markdown
|
|
assert 'Identical' in focused_markdown
|
|
assert 'Mean dGtP' in focused_markdown
|
|
assert 'Mean dCalmar/MAR' in focused_markdown
|
|
assert 'Floor rejects' in focused_markdown
|
|
assert 'not independent evidence' in focused_markdown
|
|
|
|
|
|
def test_synthetic_worker_matrix_covers_four_arms_protocols_and_costs(monkeypatch):
|
|
monkeypatch.setenv('BACKTEST_SNAPSHOT_OFFLINE', '0')
|
|
monkeypatch.setenv('BACKTEST_ALLOW_SPAWN', '0')
|
|
start = date(2025, 1, 6)
|
|
sessions = _business_days(start, date(2025, 1, 17))
|
|
ords = [session.toordinal() for session in sessions]
|
|
symbols = [f'S{index}' for index in range(12)]
|
|
prices = {symbol: _prices(ords) for symbol in symbols}
|
|
candidates = [
|
|
_candidate(symbol, start, rank=99.0 - index)
|
|
for index, symbol in enumerate(symbols[:11])
|
|
]
|
|
friday = date(2025, 1, 10)
|
|
candidates.append(_candidate('S11', friday, rank=99.0))
|
|
rank_map = {
|
|
(symbol, friday.isoformat()): {
|
|
'strategy_rank': 100.0 if symbol == 'S11' else float(index)
|
|
}
|
|
for index, symbol in enumerate(symbols)
|
|
}
|
|
_worker_init({
|
|
'qualified_candidates': candidates,
|
|
'daily_rank_map': rank_map,
|
|
'prices': prices,
|
|
'benchmark_closes': None,
|
|
'ranking_key': 'residual_high_vol_blend_80_20',
|
|
'exit_policy': 'hold',
|
|
'hold_days': 30,
|
|
'risk_per_trade': 0.01,
|
|
'atr_trail_multiplier': 3.0,
|
|
})
|
|
|
|
rows = []
|
|
for protocol, measurement_start in (
|
|
('empty_book', start),
|
|
('warm_book', date(2025, 1, 8)),
|
|
):
|
|
for cost in (0.1, 0.2):
|
|
for arm_id in (
|
|
'cap10_incumbent',
|
|
'cash_unbounded',
|
|
'cap10_weekly_top10',
|
|
'cap15_incumbent',
|
|
):
|
|
rows.append(_worker_run_cell({
|
|
'cell_id': f'{arm_id}|{protocol}|{cost}',
|
|
'arm_id': arm_id,
|
|
'protocol': protocol,
|
|
'path_id': f'{protocol}-synthetic',
|
|
'cluster': 2025,
|
|
'simulation_start': start.isoformat(),
|
|
'measurement_start': measurement_start.isoformat(),
|
|
'hard_end_exclusive': date(2025, 1, 14).isoformat(),
|
|
'cost_per_side_pct': cost,
|
|
}))
|
|
|
|
assert len(rows) == 16
|
|
assert {row['arm_id'] for row in rows} == {
|
|
'cap10_incumbent',
|
|
'cash_unbounded',
|
|
'cap10_weekly_top10',
|
|
'cap15_incumbent',
|
|
}
|
|
assert {row['protocol'] for row in rows} == {'empty_book', 'warm_book'}
|
|
assert {row['cost_per_side_pct'] for row in rows} == {0.1, 0.2}
|
|
assert all('ev_net_r' in row['metrics'] for row in rows)
|
|
|
|
|
|
def test_checkpoint_resume_rejects_fingerprint_mismatch(tmp_path):
|
|
checkpoint = tmp_path / 'checkpoint'
|
|
completed = _checkpoint_state(checkpoint, 'fingerprint-a', resume=False)
|
|
assert completed == {}
|
|
_write_cell_checkpoint(
|
|
checkpoint,
|
|
{'cell_id': 'one', 'metrics': {'ev_net_r': 1.0}},
|
|
)
|
|
resumed = _checkpoint_state(checkpoint, 'fingerprint-a', resume=True)
|
|
assert set(resumed) == {'one'}
|
|
with pytest.raises(SystemExit, match='fingerprint mismatch'):
|
|
_checkpoint_state(checkpoint, 'fingerprint-b', resume=True)
|
|
|
|
|
|
def test_dirty_worktree_guard(monkeypatch):
|
|
monkeypatch.setattr(
|
|
'scripts.run_portfolio_construction_matrix._git_output',
|
|
lambda *_args: ' M changed.py',
|
|
)
|
|
with pytest.raises(SystemExit, match='dirty worktree'):
|
|
_assert_clean_worktree()
|