dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: prepare effective risk floor ab

Browse Source
This commit is contained in:
dennisthiessen
2026-08-05 22:49:50 +02:00
parent aa6cd5cac4
commit 6f1ee450f1
5 changed files with 573 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_portfolio_construction_matrix.py
+269 -38
View File
@@ -1,8 +1,8 @@
'''Run the focused four-arm daily portfolio-capacity research matrix.
'''Run focused daily portfolio-construction research matrices.
The expensive point-in-time daily replay and full-universe ranks are cached
once. Empty-book monthly paths and warm-book weekly seeds are then evaluated
under cap 10, cap 15, cash-only unbounded, and weekly current-rank top 10.
once and reused by the frozen capacity bracket and its focused effective-risk
floor follow-up.
'''
from __future__ import annotations
@@ -37,6 +37,7 @@ from scripts.portfolio_capacity_research import ( # noqa: E402
BOOTSTRAP_REPLICATES,
BOOTSTRAP_SEED,
COSTS_PER_SIDE_PCT,
RISK_FLOOR_ARMS,
aggregate_results,
build_cells,
build_cohort_manifest,
@@ -54,13 +55,76 @@ MIN_PRODUCTION_UNIVERSE = 450
MAX_PRODUCTION_UNIVERSE = 600
SPEC_PATH = ROOT / 'docs' / 'research' / 'portfolio-capacity-bracket.md'
DEFAULT_RUN_ID = 'prod505-capacity-bracket-daily-v1'
RISK_FLOOR_SPEC_PATH = (
ROOT / 'docs' / 'research' / 'effective-risk-floor-ab.md'
)
STUDIES: dict[str, dict[str, Any]] = {
'capacity-bracket': {
'arms': ARMS,
'protocols': ('empty_book', 'warm_book'),
'include_warm_dispersion': True,
'runner_version': RUNNER_VERSION,
'spec_path': SPEC_PATH,
'default_run_id': DEFAULT_RUN_ID,
'research_question': (
'Bracket the economic cost of the binding ten-position cap and '
'test whether weekly current-rank selection beats arrival order.'
),
'decision_rule': (
'No formal promotion gate. Report paired annual medians, warm-seed '
'EV/Calmar IQR ratios, and simple bootstrap intervals as context.'
),
'motivation': {
'source': 'reports/research-matrix-phase-a.json a0_control full window',
'trades': 472,
'skipped_book_full': 519,
'blocked_fraction': 519 / (519 + 472),
'stale_claim_corrected': (
'The older weekly pre-gate-reset claim that cap 10 never bound '
'does not apply to the current daily configuration.'
),
},
},
'risk-floor-ab': {
'arms': RISK_FLOOR_ARMS,
'protocols': ('empty_book', 'warm_book'),
'include_warm_dispersion': False,
'runner_version': 'effective-risk-floor-ab-v1',
'spec_path': RISK_FLOOR_SPEC_PATH,
'default_run_id': 'prod505-effective-risk-floor-ab-daily-v1',
'research_question': (
'Estimate the isolated effect of rejecting entries whose effective '
'initial stop risk is below 0.5% of marked equity.'
),
'decision_rule': (
'No formal promotion gate. Attribute paired differences causally to '
'the floor, headline means and identical-path fractions beside '
'annual medians, and decide paper adoption after interpretation.'
),
'motivation': {
'source': (
'portfolio-construction-prod505-capacity-bracket-daily-v1 '
'post-run decomposition'
),
'finding': (
'The confounded floor arm improved EV most where the position '
'cap never bound; isolate the 0.5% floor at cap 10.'
),
},
},
}
_WORKER_CONTEXT: dict[str, Any] | None = None
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('snapshot', help='SQLite backtest snapshot')
parser.add_argument('--run-id', default=DEFAULT_RUN_ID)
parser.add_argument(
'--study',
choices=tuple(STUDIES),
default='capacity-bracket',
)
parser.add_argument('--run-id', default=None)
parser.add_argument(
'--workers',
default='auto',
@@ -235,7 +299,7 @@ def _worker_run_cell(cell: dict[str, Any]) -> dict[str, Any]:
from app.services import backtest_service as bt
context = _WORKER_CONTEXT
arm = ARM_BY_ID[str(cell['arm_id'])]
arm = context.get('arm_by_id', ARM_BY_ID)[str(cell['arm_id'])]
measurement_start = date.fromisoformat(str(cell['measurement_start']))
hard_end = date.fromisoformat(str(cell['hard_end_exclusive']))
sim = bt._simulate_portfolio(
@@ -600,6 +664,7 @@ def _checkpoint_state(
fingerprint: str,
*,
resume: bool,
runner_version: str = RUNNER_VERSION,
) -> dict[str, dict[str, Any]]:
manifest_path = checkpoint_dir / 'manifest.json'
if checkpoint_dir.exists() and not resume:
@@ -620,7 +685,7 @@ def _checkpoint_state(
_atomic_json(
manifest_path,
{
'runner_version': RUNNER_VERSION,
'runner_version': runner_version,
'fingerprint': fingerprint,
'created_at': datetime.now(timezone.utc).isoformat(),
},
@@ -650,9 +715,13 @@ def _fmt(value: Any, digits: int = 3) -> str:
return str(value)
def _operational_summary(cells: list[dict[str, Any]]) -> list[dict[str, Any]]:
def _operational_summary(
cells: list[dict[str, Any]],
*,
arms: tuple[dict[str, Any], ...] = ARMS,
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for arm in ARMS:
for arm in arms:
arm_cells = [
row
for row in cells
@@ -676,6 +745,16 @@ def _operational_summary(cells: list[dict[str, Any]]) -> list[dict[str, Any]]:
'median_avg_positions': median(
row['metrics'].get('avg_positions') for row in arm_cells
),
'median_avg_hold_days': median(
row['metrics'].get('avg_hold_days') for row in arm_cells
),
'median_avg_cash_pct': median(
row['metrics'].get('avg_cash_pct') for row in arm_cells
),
'median_avg_gross_exposure_pct': median(
row['metrics'].get('avg_gross_exposure_pct')
for row in arm_cells
),
'peak_positions': max(
(
int(row['metrics'].get('peak_positions') or 0)
@@ -865,19 +944,158 @@ def _markdown(report: dict[str, Any]) -> str:
return '\n'.join(lines)
def _risk_floor_markdown(report: dict[str, Any]) -> str:
treatment_id = str(report['arms'][1]['id'])
lines = [
'# Effective initial-risk floor A/B',
'',
'Generated: ' + str(report['generated_at']),
'',
'## Question',
'',
'Does rejecting an otherwise qualified cap-10 entry when actual initial '
'stop risk is below 0.5% of marked equity improve trade selection?',
'',
'The arms differ only by min_initial_risk_fraction=0.005. There is no '
'position-cap, ranking, exit, sizing-target, or execution confound.',
'',
'Use paired differences, not absolute profitability, because current '
'production membership is projected backward.',
'',
]
paired = report['analysis']['paired_per_year']
distributions = report['analysis']['paired_path_distributions']
for cost in report['costs_per_side_pct']:
for protocol in report['protocols']:
annual = next(
row for row in paired
if row['arm_id'] == treatment_id
and row['protocol'] == protocol
and float(row['cost_per_side_pct']) == float(cost)
)
path = next(
row for row in distributions
if row['arm_id'] == treatment_id
and row['protocol'] == protocol
and float(row['cost_per_side_pct']) == float(cost)
)
metrics = path['metrics']
ev = metrics['ev_net_r']
annual_ev = annual['headline']['ev_net_r']
ev_ci = annual_ev['bootstrap_90']
separator = chr(124)
lines.append(
'## ' + protocol.replace('_', ' ').title()
+ f' at {float(cost):.2f}% per fill'
)
lines.extend([
'',
separator + ' Paths ' + separator + ' Mean dEV '
+ separator + ' Median dEV ' + separator + ' P25 '
+ separator + ' P75 ' + separator + ' Positive '
+ separator + ' Identical ' + separator + ' Annual median '
+ separator + ' 90% context ' + separator,
separator.join(
['', '---:', '---:', '---:', '---:', '---:', '---:',
'---:', '---:', '---:', '']
),
separator + ' ' + str(ev['paired_paths']) + ' '
+ separator + ' ' + _fmt(ev['paired_delta_mean']) + ' '
+ separator + ' ' + _fmt(ev['paired_delta_median']) + ' '
+ separator + ' ' + _fmt(ev['paired_delta_p25']) + ' '
+ separator + ' ' + _fmt(ev['paired_delta_p75']) + ' '
+ separator + ' '
+ _fmt(ev['positive_fraction'] * 100.0, 1) + '% '
+ separator + ' '
+ _fmt(ev['identical_fraction'] * 100.0, 1) + '% '
+ separator + ' '
+ _fmt(annual_ev['paired_delta_median']) + ' '
+ separator + ' [' + _fmt(ev_ci['p05']) + ', '
+ _fmt(ev_ci['p95']) + '] '
+ separator,
'',
separator + ' Mean dPF ' + separator + ' Mean dGtP '
+ separator + ' Mean dSortino ' + separator
+ ' Mean dCalmar/MAR '
+ separator + ' Mean dCAGR pp ' + separator
+ ' Mean dMaxDD pp ' + separator,
separator.join(
['', '---:', '---:', '---:', '---:', '---:', '---:', '']
),
separator
+ ' ' + _fmt(
metrics['profit_factor']['paired_delta_mean']
) + ' '
+ separator
+ ' ' + _fmt(
metrics['gain_to_pain']['paired_delta_mean']
) + ' '
+ separator
+ ' ' + _fmt(metrics['sortino']['paired_delta_mean']) + ' '
+ separator
+ ' ' + _fmt(metrics['calmar']['paired_delta_mean']) + ' '
+ separator
+ ' ' + _fmt(metrics['cagr_pct']['paired_delta_mean']) + ' '
+ separator
+ ' ' + _fmt(
metrics['max_drawdown_pct']['paired_delta_mean']
) + ' '
+ separator,
'',
])
lines.extend([
'## Operations at 0.10% per fill',
'',
separator + ' Arm ' + separator + ' Trades ' + separator + ' Hold '
+ separator + ' Cash ' + separator + ' Gross ' + separator
+ ' Positions ' + separator + ' Floor rejects ' + separator,
separator.join(
['', '---', '---:', '---:', '---:', '---:', '---:', '---:', '']
),
])
for row in report['operational_summary']:
lines.append(
separator + ' ' + str(row['arm_id']) + ' '
+ separator + ' ' + _fmt(row['median_trades'], 1) + ' '
+ separator + ' ' + _fmt(row['median_avg_hold_days'], 1) + ' '
+ separator + ' ' + _fmt(row['median_avg_cash_pct'], 1) + '% '
+ separator
+ ' ' + _fmt(row['median_avg_gross_exposure_pct'], 1) + '% '
+ separator + ' ' + _fmt(row['median_avg_positions'], 2) + ' '
+ separator + ' ' + str(row['min_risk_rejections']) + ' '
+ separator
)
lines.extend([
'',
'Empty-book starts are primary. Warm-book paths are a state-carrying '
'replication over the same seven years, not independent evidence or an '
'initialization-dispersion test.',
'',
'The 90% intervals resample seven annual paired summaries. They are '
'descriptive context, not gates or population-confidence claims.',
])
return '\n'.join(lines)
async def _main() -> None:
args = _parse_args()
study = STUDIES[str(args.study)]
args.run_id = args.run_id or study['default_run_id']
arms = tuple(study['arms'])
protocols = tuple(study['protocols'])
runner_version = str(study['runner_version'])
spec_path = Path(study['spec_path'])
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f'Snapshot does not exist: {snapshot}')
if not SPEC_PATH.exists():
raise SystemExit(f'Frozen specification is missing: {SPEC_PATH}')
if not spec_path.exists():
raise SystemExit(f'Frozen specification is missing: {spec_path}')
os.environ['BACKTEST_SNAPSHOT_OFFLINE'] = '1'
os.environ['BACKTEST_ALLOW_SPAWN'] = '1'
workers = _worker_count(str(args.workers))
snapshot_sha256 = _sha256_file(snapshot)
specification_sha256 = _sha256_file(SPEC_PATH)
specification_sha256 = _sha256_file(spec_path)
snapshot_data = await _load_snapshot(snapshot, quiet=bool(args.quiet))
cache_path = (
@@ -906,9 +1124,14 @@ async def _main() -> None:
snapshot_data['construction_universe_manifest']
)
validation_errors = [*universe_errors, *cohort_errors]
cells = build_cells(cohort_manifest)
cells = build_cells(
cohort_manifest,
arms=arms,
protocols=protocols,
)
validation_payload = {
'runner_version': RUNNER_VERSION,
'study': args.study,
'runner_version': runner_version,
'snapshot': str(snapshot.resolve()),
'snapshot_sha256': snapshot_sha256,
'snapshot_sessions': {
@@ -939,6 +1162,8 @@ async def _main() -> None:
'empty_cluster_count': cohort_manifest['empty_cluster_count'],
'warm_cluster_count': cohort_manifest['warm_cluster_count'],
'expected_clusters': list(ANCHOR_YEARS),
'protocols': list(protocols),
'arms': [arm['id'] for arm in arms],
'matrix_cells': len(cells),
'cache_path': str(cache_path.resolve()),
'cache_key_hash': _json_hash(candidate_cache['key']),
@@ -958,7 +1183,8 @@ async def _main() -> None:
_assert_clean_worktree()
git_commit = _git_output('rev-parse', 'HEAD')
fingerprint_payload = {
'runner_version': RUNNER_VERSION,
'study': args.study,
'runner_version': runner_version,
'git_commit': git_commit,
'snapshot_sha256': snapshot_sha256,
'specification_sha256': specification_sha256,
@@ -969,7 +1195,9 @@ async def _main() -> None:
snapshot_data['construction_universe_manifest']
),
'cohort_manifest': cohort_manifest,
'arms': list(ARMS),
'arms': list(arms),
'protocols': list(protocols),
'include_warm_dispersion': bool(study['include_warm_dispersion']),
'costs_per_side_pct': list(COSTS_PER_SIDE_PCT),
'bootstrap': {
'replicates': BOOTSTRAP_REPLICATES,
@@ -983,12 +1211,13 @@ async def _main() -> None:
else ROOT
/ 'reports'
/ '.cache'
/ f'{args.run_id}-{RUNNER_VERSION}-checkpoint'
/ f'{args.run_id}-{runner_version}-checkpoint'
)
completed = _checkpoint_state(
checkpoint_dir,
fingerprint,
resume=bool(args.resume),
runner_version=runner_version,
)
expected_ids = {str(cell['cell_id']) for cell in cells}
unknown = set(completed) - expected_ids
@@ -1006,6 +1235,7 @@ async def _main() -> None:
)
worker_context = {
'arm_by_id': {arm['id']: arm for arm in arms},
'qualified_candidates': candidate_cache['qualified_candidates'],
'daily_rank_map': candidate_cache['daily_rank_map'],
'prices': snapshot_data['prices'],
@@ -1060,34 +1290,25 @@ async def _main() -> None:
completed.values(),
key=lambda row: str(row['cell_id']),
)
analysis = aggregate_results(result_cells)
analysis = aggregate_results(
result_cells,
arms=arms,
protocols=protocols,
include_warm_dispersion=bool(study['include_warm_dispersion']),
)
dependency_manifest = ROOT / 'pyproject.toml'
report: dict[str, Any] = {
'run_id': args.run_id,
'study': args.study,
'status': 'complete',
'generated_at': datetime.now(timezone.utc).isoformat(),
'research_question': (
'Bracket the economic cost of the binding ten-position cap and '
'test whether weekly current-rank selection beats arrival order.'
),
'decision_rule': (
'No formal promotion gate. Report paired annual medians, warm-seed '
'EV/Calmar IQR ratios, and simple bootstrap intervals as context.'
),
'research_question': study['research_question'],
'decision_rule': study['decision_rule'],
'survivorship_bias_caveat': (
'The current production universe is projected backward; construction '
'conclusions rely on paired relative comparisons, not absolute levels.'
),
'motivation': {
'source': 'reports/research-matrix-phase-a.json a0_control full window',
'trades': 472,
'skipped_book_full': 519,
'blocked_fraction': 519 / (519 + 472),
'stale_claim_corrected': (
'The older weekly pre-gate-reset claim that cap 10 never bound '
'does not apply to the current daily configuration.'
),
},
'motivation': dict(study['motivation']),
'fingerprint': fingerprint,
'fingerprint_payload': fingerprint_payload,
'environment': {
@@ -1117,12 +1338,17 @@ async def _main() -> None:
}
},
'cohort_manifest': cohort_manifest,
'arms': list(ARMS),
'arms': list(arms),
'protocols': list(protocols),
'include_warm_dispersion': bool(study['include_warm_dispersion']),
'costs_per_side_pct': list(COSTS_PER_SIDE_PCT),
'cell_count': len(result_cells),
'cells': result_cells,
'analysis': analysis,
'operational_summary': _operational_summary(result_cells),
'operational_summary': _operational_summary(
result_cells,
arms=arms,
),
}
out_path = (
Path(args.out)
@@ -1132,7 +1358,12 @@ async def _main() -> None:
/ f'portfolio-construction-{args.run_id}.json'
)
_atomic_json(out_path, report)
_atomic_text(out_path.with_suffix('.md'), _markdown(report))
markdown = (
_risk_floor_markdown(report)
if args.study == 'risk-floor-ab'
else _markdown(report)
)
_atomic_text(out_path.with_suffix('.md'), markdown)
if not args.quiet:
print(f'wrote {out_path}', flush=True)
print(f'wrote {out_path.with_suffix(".md")}', flush=True)