dennisthiessenandClaude Opus 5
83c0555e52
The v3 cutover run scored 2/4 corrections warned against v2's 3/4, which reads like a regression and is not one. Only 4 of the 11 detected corrections fall in the holdout, so recall is one event from a different headline -- and the event that flips is decided by threshold placement, not by what the score saw. "v3 without the credit sensor" catches 2025-02-21 at a *higher* threshold (35.5) than shipped v3 misses it at (32.3), because the alarm rule needs a rising edge and a lower threshold can fire outside the horizon then never reset below. Two caveats are now computed and surfaced rather than left for the reader to infer: - Holdout event count against MIN_EVENTS_FOR_CONFIDENCE. The summary sentence states how many of the detected corrections actually fall in the test period. - Warning-sensor coverage across the split. The score renormalises over what is available, so a training window predating a sensor's history freezes the threshold on a different construct than the holdout is measured against. At the cutover that is 39% of training sessions with all three sensors versus 100% of the test period, credit history beginning 2023-07-25. Restricting the threshold to sensor-matched training sessions was tested and rejected: those sessions are a calm recent stretch, so the threshold falls from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6/yr. It swaps a coverage bias for a regime-selection bias. The report states its limits instead. _warning_series now returns per-session sensor counts alongside the scores. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
116 lines
4.6 KiB
Python
116 lines
4.6 KiB
Python
"""Tests for v3 correction events, warning alarm episodes, and report caveats."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date, timedelta
|
|
|
|
from app.services.breadth_service import _breadth_from_closes, compute_divergence_series
|
|
from app.services.event_study_service import (
|
|
MIN_EVENTS_FOR_CONFIDENCE,
|
|
_percentile,
|
|
_reliability,
|
|
alarm_episodes,
|
|
detect_events,
|
|
evaluate_alarms,
|
|
)
|
|
|
|
|
|
def _days(count: int, start: date = date(2021, 1, 1)) -> list[date]:
|
|
return [start + timedelta(days=index) for index in range(count)]
|
|
|
|
|
|
def test_detect_events_uses_rising_edge_and_cooldown():
|
|
closes = [100.0] * 300 + [85.0] * 5 + [100.0] * 50 + [85.0] * 5
|
|
events = detect_events(closes, _days(len(closes)), threshold_pct=15.0, cooldown=40)
|
|
assert [event["index"] for event in events] == [300, 355]
|
|
|
|
|
|
def test_reliability_flags_a_thin_holdout():
|
|
"""2/4 must not read like a property of the score."""
|
|
dates = _days(100)
|
|
backing = dict.fromkeys(dates, 3)
|
|
|
|
thin = _reliability(dates, 70, backing, events_detected=11, events_in_holdout=4)
|
|
assert thin["underpowered"] is True
|
|
assert thin["events_detected"] == 11
|
|
assert thin["events_in_holdout"] == 4
|
|
assert thin["minimum_events"] == MIN_EVENTS_FOR_CONFIDENCE
|
|
|
|
ample = _reliability(dates, 70, backing, events_detected=20, events_in_holdout=12)
|
|
assert ample["underpowered"] is False
|
|
|
|
|
|
def test_reliability_flags_a_sensor_coverage_split():
|
|
"""The threshold must not be frozen on a different construct than it is tested on.
|
|
|
|
Credit history starts partway through the training window, so the score
|
|
renormalises over two sensors early and three later.
|
|
"""
|
|
dates = _days(100)
|
|
matched = dict.fromkeys(dates, 3)
|
|
assert _reliability(dates, 70, matched, 20, 12)["sensor_coverage_mismatch"] is False
|
|
|
|
# Training is 40% three-sensor; the holdout is entirely three-sensor.
|
|
split_backing = {d: (3 if index >= 42 else 2) for index, d in enumerate(dates)}
|
|
mismatched = _reliability(dates, 70, split_backing, 20, 12)
|
|
assert mismatched["sensor_coverage_mismatch"] is True
|
|
assert mismatched["train_full_sensor_share"] == 40.0
|
|
assert mismatched["holdout_full_sensor_share"] == 100.0
|
|
assert mismatched["sensors_expected"] == 3
|
|
|
|
|
|
def test_percentile_is_fixed_from_supplied_values():
|
|
values = [float(value) for value in range(0, 101, 10)]
|
|
assert _percentile(values, 50) == 50.0
|
|
assert _percentile(values, 80) == 80.0
|
|
assert _percentile([], 80) is None
|
|
|
|
|
|
def test_alarm_requires_upward_crossing_and_reset():
|
|
dates = _days(10)
|
|
values = [10, 70, 80, 75, 20, 70, 80, 20, 20, 70]
|
|
indicator = dict(zip(dates, values))
|
|
assert alarm_episodes(indicator, dates, threshold=60) == [1, 5, 9]
|
|
|
|
|
|
def test_holdout_start_does_not_invent_crossing_when_already_high():
|
|
dates = _days(6)
|
|
indicator = dict(zip(dates, [10, 70, 80, 80, 20, 70]))
|
|
assert alarm_episodes(indicator, dates, threshold=60, start_index=3) == [5]
|
|
|
|
|
|
def test_evaluate_alarms_counts_episodes_not_alarm_days():
|
|
dates = _days(100)
|
|
result = evaluate_alarms([10, 50, 80], [25, 70], dates, horizon=20)
|
|
assert result["events_warned"] == 2
|
|
assert result["events_missed"] == 0
|
|
assert result["false_alarms"] == 1
|
|
assert result["median_lead_days"] == 17.5
|
|
|
|
|
|
def test_breadth_from_fixed_closes_and_tapered_divergence():
|
|
dates = _days(10)
|
|
closes_by_symbol = {
|
|
"A": list(zip(dates, [1.0 + index for index in range(10)])),
|
|
"B": list(zip(dates, [10.0 - index for index in range(10)])),
|
|
"C": list(zip(dates, [5.0] * 10)),
|
|
}
|
|
breadth = _breadth_from_closes(closes_by_symbol, window=3, min_tickers=2)
|
|
assert breadth[dates[2]] == round(1 / 3 * 100, 2)
|
|
|
|
falling_breadth = {dates[index]: 80.0 - index * 3 for index in range(10)}
|
|
rising_benchmark = list(zip(dates, [100.0 + index for index in range(10)]))
|
|
divergence = compute_divergence_series(falling_breadth, rising_benchmark, lookback=3)
|
|
assert divergence[dates[-1]] > 0
|
|
|
|
# v3: breadth loss with price confirming it is still deterioration, scored at
|
|
# DIVERGENCE_CONFIRMED_FLOOR of the masked case rather than discarded. v2's
|
|
# hard gate zeroed this and left Warning at 0 through every selloff.
|
|
falling_benchmark = list(zip(dates, [100.0 - index for index in range(10)]))
|
|
confirmed = compute_divergence_series(falling_breadth, falling_benchmark, lookback=3)
|
|
assert 0 < confirmed[dates[-1]] < divergence[dates[-1]]
|
|
|
|
# Flat breadth is not deterioration regardless of price direction.
|
|
flat_breadth = {day: 60.0 for day in dates}
|
|
assert compute_divergence_series(flat_breadth, falling_benchmark, lookback=3)[dates[-1]] == 0
|