feat(risk-monitor): measure the rule that fires, and give fundamentals their own channel
The Warning study measured a fitted percentile crossing that nothing consumes. What reaches Telegram is a quadrant change: fixed 50/40 dividers, hysteresis, two-session confirmation, 3-day cooldown. Those thresholds are constants, not fits, so there is no training set to protect and all 11 detected corrections are evaluable instead of the 4 that fell in a holdout. Replaying it: 1/10 corrections, 0.9 false alarms/year. Random alarms at the same firing rate match or beat that in 65% of draws. The panel now carries ablations (does the quadrant machinery earn its place?), external baselines (does the score earn its complexity?), and that null, because a bare "2 of 4" was unreadable in either direction. Nothing in the alert path was retuned on the strength of it. Fundamentals become a third channel rather than a term in either score. v3 cut them arguing 12+8 of 100 points "could not change any published conclusion" -- true only when every technical sensor reads zero; weighted they moved the bar for the 40 divider from 40 to 25. But no fusion weight is measurable either: with ~10 events and no fundamental history, any weight is a policy preference presented as a measurement. So the read is a categorical state (supportive/neutral/adverse/ unknown) with an evidence grade, derived by fixed rules from stored facts, read by confluence. The LLM extracts and explains; it does not score. Absence stays absence throughout. `unknown` is unreachable by averaging, a stale or empty observation may display but never confirm, extraction failures map to `unknown` rather than `mixed`, and the study rows are coverage-matched and marked not-measurable until enough corrections are covered -- otherwise a fortnight of observations renders as 0/10 and reads as a failed test. Observations become a real time series (migration 033); they lived in a single overwritten settings slot, so no history existed to replay. Pre-rename snapshots are adapted rather than discarded. METHODOLOGY stays v4 -- no score changed -- so no reseed; STUDY_SCHEMA moves to 3 and discards the cached report. Post-deploy: re-run Event Study from Admin -> Jobs. The panel reads "not run yet" until then. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,10 +5,16 @@ different realized ranges -- Warning never exceeded 64.9 in the 408 calibration
|
||||
sessions, so a shared 60 left the whole upper half of that axis unreachable.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services import alert_service
|
||||
from app.services.alert_service import (
|
||||
CONFLUENCE_TYPE,
|
||||
FUND_TYPE,
|
||||
QUAD_X_DIV,
|
||||
QUAD_Y_DIV,
|
||||
_classify_quadrant,
|
||||
_collect_regime_fundamental,
|
||||
_parse_quadrant_log_key,
|
||||
_quadrant_log_key,
|
||||
)
|
||||
@@ -48,3 +54,140 @@ def test_quadrant_key_carries_basket_hash_and_parses_legacy_keys():
|
||||
assert _parse_quadrant_log_key(key) == ("abc123", "3", 32.4, 54.6)
|
||||
assert _parse_quadrant_log_key("3:32.4:54.6") == (None, "3", 32.4, 54.6)
|
||||
assert _parse_quadrant_log_key("3") == (None, "3", None, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fundamental-context and confluence alerts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _monitor(
|
||||
warning_score: float, state: str, *, coverage: float = 100.0, usable: bool = True
|
||||
) -> dict:
|
||||
return {
|
||||
"available": True,
|
||||
"warning": {"score": warning_score, "coverage": coverage},
|
||||
"fundamental_context": {
|
||||
"state": state,
|
||||
"evidence_quality": "complete" if usable else "stale",
|
||||
# The state survives going stale so the card can still show it, and
|
||||
# a failed extraction is fresh but knows nothing; `usable` is what
|
||||
# says whether it may still confirm anything.
|
||||
"available": usable,
|
||||
"usable": usable,
|
||||
},
|
||||
"data_quality": {"is_fresh": True},
|
||||
"quadrant_config": {"warning_divider": QUAD_Y_DIV},
|
||||
}
|
||||
|
||||
|
||||
class _LogSpyDB:
|
||||
"""Records what would be logged; returns a canned "last logged key"."""
|
||||
|
||||
def __init__(self, last: dict[str, str | None]) -> None:
|
||||
self.last = last
|
||||
self.logged: list[tuple[str, str]] = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched(monkeypatch):
|
||||
def apply(data: dict, last: dict[str, str | None]):
|
||||
db = _LogSpyDB(last)
|
||||
|
||||
async def fake_monitor(_db):
|
||||
return data
|
||||
|
||||
async def fake_last(_db, alert_type):
|
||||
return db.last.get(alert_type)
|
||||
|
||||
def fake_log(_db, alert_type, key, value=None):
|
||||
db.logged.append((alert_type, key))
|
||||
|
||||
import app.services.regime_monitor_service as rms
|
||||
|
||||
monkeypatch.setattr(rms, "get_regime_monitor", fake_monitor)
|
||||
monkeypatch.setattr(alert_service, "_last_logged_key", fake_last)
|
||||
monkeypatch.setattr(alert_service, "_log_alert", fake_log)
|
||||
return db
|
||||
|
||||
return apply
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_run_seeds_both_channels_without_alerting(patched):
|
||||
db = patched(_monitor(60.0, "adverse"), {FUND_TYPE: None, CONFLUENCE_TYPE: None})
|
||||
assert await _collect_regime_fundamental(db) == []
|
||||
assert dict(db.logged) == {FUND_TYPE: "adverse", CONFLUENCE_TYPE: "yes"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fundamental_change_and_confluence_are_separate_messages(patched):
|
||||
db = patched(_monitor(60.0, "adverse"), {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
out = await _collect_regime_fundamental(db)
|
||||
|
||||
assert [alert_type for alert_type, _, _ in out] == [FUND_TYPE, CONFLUENCE_TYPE]
|
||||
assert "neutral → adverse" in out[0][2]
|
||||
assert "Confluence" in out[1][2]
|
||||
# Neither message reports a fused score; they name which channel moved.
|
||||
assert "not a score" in out[0][2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_never_alerts(patched):
|
||||
"""Absence of evidence is not a change in the evidence."""
|
||||
db = patched(_monitor(60.0, "unknown"), {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
assert await _collect_regime_fundamental(db) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adverse_alone_is_not_confluence(patched):
|
||||
"""A calm tape with adverse fundamentals is a context change, not confluence."""
|
||||
db = patched(_monitor(10.0, "adverse"), {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
out = await _collect_regime_fundamental(db)
|
||||
assert [alert_type for alert_type, _, _ in out] == [FUND_TYPE]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_leaving_confluence_rebaselines_quietly(patched):
|
||||
db = patched(_monitor(10.0, "neutral"), {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "yes"})
|
||||
assert await _collect_regime_fundamental(db) == []
|
||||
assert (CONFLUENCE_TYPE, "no") in db.logged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_low_coverage_or_stale_inputs_stay_quiet(patched):
|
||||
thin = _monitor(60.0, "adverse", coverage=50.0)
|
||||
assert await _collect_regime_fundamental(
|
||||
patched(thin, {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
) == []
|
||||
|
||||
stale = _monitor(60.0, "adverse")
|
||||
stale["data_quality"]["is_fresh"] = False
|
||||
assert await _collect_regime_fundamental(
|
||||
patched(stale, {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stale_observation_cannot_confirm_a_new_crossing(patched):
|
||||
"""The state is kept for display, but it stops being evidence.
|
||||
|
||||
Without this, one adverse read corroborates every Warning crossing for the
|
||||
rest of time — the strongest claim the channel makes, from the data with the
|
||||
least right to make it.
|
||||
"""
|
||||
stale = _monitor(60.0, "adverse", usable=False)
|
||||
db = patched(stale, {FUND_TYPE: "adverse", CONFLUENCE_TYPE: "no"})
|
||||
assert await _collect_regime_fundamental(db) == []
|
||||
# It also rebaselines to "no", so recollecting the observation re-arms it.
|
||||
assert (CONFLUENCE_TYPE, "no") not in db.logged # already "no"; nothing to log
|
||||
|
||||
fresh = _monitor(60.0, "adverse", usable=True)
|
||||
db2 = patched(fresh, {FUND_TYPE: "adverse", CONFLUENCE_TYPE: "no"})
|
||||
out = await _collect_regime_fundamental(db2)
|
||||
assert [alert_type for alert_type, _, _ in out] == [CONFLUENCE_TYPE]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stale_state_change_does_not_alert(patched):
|
||||
db = patched(_monitor(10.0, "adverse", usable=False), {FUND_TYPE: "neutral", CONFLUENCE_TYPE: "no"})
|
||||
assert await _collect_regime_fundamental(db) == []
|
||||
|
||||
Reference in New Issue
Block a user