dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add split-safe fundamentals research protocol

Browse Source
This commit is contained in:
dennisthiessen
2026-07-23 17:27:28 +02:00
parent 7f944d718f
commit 34d6dda1ab
6 changed files with 273 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_fundamentals_macbook.sh
+20 -2
View File
@@ -5,6 +5,7 @@ ROOT="$(git rev-parse --show-toplevel)"
cd "$ROOT"
SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}"
PROTOCOL="${PROTOCOL:-split-safe}"
WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}"
if [[ "$WORKERS" -gt 1 ]]; then
WORKERS=$((WORKERS - 1))
@@ -21,17 +22,34 @@ if [[ ! -f "$SNAPSHOT" ]]; then
exit 1
fi
case "$PROTOCOL" in
split-safe)
PREFIX="fundamentals-splitsafe"
SCORE_CACHE="reports/.cache/fundamentals-splitsafe-scores.pkl"
;;
original)
PREFIX="fundamentals-overlay"
SCORE_CACHE="reports/.cache/fundamentals-scores.pkl"
;;
*)
echo "Unsupported PROTOCOL: $PROTOCOL (use split-safe or original)" >&2
exit 1
;;
esac
STAMP="$(date -u +%Y%m%d-%H%M%S)"
OUT="reports/fundamentals-overlay-${STAMP}.json"
OUT="reports/${PREFIX}-${STAMP}.json"
echo "Snapshot: $SNAPSHOT"
echo "Workers: $WORKERS"
echo "Protocol: $PROTOCOL"
echo "Output: $OUT"
"$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \
--protocol "$PROTOCOL" \
--workers "$WORKERS" \
--candidate-cache reports/.cache/fundamentals-candidates.pkl \
--fundamentals-cache reports/.cache/fundamentals-scores.pkl \
--fundamentals-cache "$SCORE_CACHE" \
--out "$OUT"
echo
scripts/run_fundamentals_research.py
+118 -44
View File
@@ -2,8 +2,9 @@
The production qualification gate is unchanged. The runner first measures
30-session factor IC, then reorders already-qualified candidates with quality,
growth, or balanced fundamental ranks at 10, 20, 30, and 40 percent weights.
It writes JSON, Markdown, CSV, and a portable ZIP bundle.
growth, or balanced fundamental ranks. It supports the original registered
matrix and a split-safe follow-up sensitivity. It writes JSON, Markdown, CSV,
and a portable ZIP bundle.
"""
from __future__ import annotations
@@ -37,26 +38,38 @@ if str(ROOT) not in sys.path:
CACHE_VERSION = "fundamentals-overlay-v1"
NY = ZoneInfo("America/New_York")
COMPOSITES = ("quality", "growth", "balanced")
ORIGINAL_PROTOCOL = "original"
SPLIT_SAFE_PROTOCOL = "split-safe"
WEIGHTS = (0.10, 0.20, 0.30, 0.40)
ARMS: tuple[dict[str, Any], ...] = (
{
"id": "control_w00",
"label": "Production 80/20 momentum-volatility rank",
"composite": None,
"weight": 0.0,
},
*tuple(
SPLIT_SAFE_WEIGHTS = (0.05, 0.10, 0.15)
def _arm_matrix(weights: tuple[float, ...]) -> tuple[dict[str, Any], ...]:
"""Build the control plus three bounded overlay families."""
return (
{
"id": f"{composite}_w{round(weight * 100):02d}",
"label": f"{composite.title()} overlay {round(weight * 100)}%",
"composite": composite,
"weight": weight,
}
for composite in COMPOSITES
for weight in WEIGHTS
),
)
"id": "control_w00",
"label": "Production 80/20 momentum-volatility rank",
"composite": None,
"weight": 0.0,
},
*tuple(
{
"id": f"{composite}_w{round(weight * 100):02d}",
"label": f"{composite.title()} overlay {round(weight * 100)}%",
"composite": composite,
"weight": weight,
}
for composite in COMPOSITES
for weight in weights
),
)
ARMS = _arm_matrix(WEIGHTS)
N_TRIALS = len(ARMS)
SPLIT_SAFE_ARMS = _arm_matrix(SPLIT_SAFE_WEIGHTS)
SPLIT_SAFE_N_TRIALS = len(SPLIT_SAFE_ARMS)
def _parse_args() -> argparse.Namespace:
@@ -74,6 +87,11 @@ def _parse_args() -> argparse.Namespace:
)
parser.add_argument("--train-end", default="2024-01-01")
parser.add_argument("--test-start", default="2025-01-01")
parser.add_argument(
"--protocol",
choices=(ORIGINAL_PROTOCOL, SPLIT_SAFE_PROTOCOL),
default=ORIGINAL_PROTOCOL,
)
parser.add_argument("--allow-spawn", action="store_true")
parser.add_argument("--quiet", action="store_true")
return parser.parse_args()
@@ -83,9 +101,14 @@ def _sqlite_url(path: Path) -> str:
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
def _default_out() -> Path:
def _default_out(protocol: str = ORIGINAL_PROTOCOL) -> Path:
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
return Path("reports") / f"fundamentals-overlay-{stamp}.json"
label = (
"fundamentals-splitsafe"
if protocol == SPLIT_SAFE_PROTOCOL
else "fundamentals-overlay"
)
return Path("reports") / f"{label}-{stamp}.json"
def _snapshot_hash(path: Path) -> str:
@@ -394,11 +417,15 @@ def _build_scores(
dates: set[date],
representatives: dict[str, str],
snapshots_by_cik: dict[str, list[Any]],
split_safe: bool,
args: argparse.Namespace,
) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]:
from app.services import fundamentals_derivation as derivation
from app.services import fundamentals_research as research
factor_polarity = (
research.SPLIT_SAFE_FACTOR_POLARITY if split_safe else research.FACTOR_POLARITY
)
ordered_dates = sorted(dates)
date_fingerprint = hashlib.sha256(
"|".join(value.isoformat() for value in ordered_dates).encode()
@@ -408,6 +435,7 @@ def _build_scores(
snapshot,
{
"kind": "point-in-time-scores",
"score_profile": SPLIT_SAFE_PROTOCOL if split_safe else ORIGINAL_PROTOCOL,
"date_fingerprint": date_fingerprint,
"availability": "accepted before signal-date midnight America/New_York",
},
@@ -446,12 +474,15 @@ def _build_scores(
current_features[cik] = research.raw_features(
derivation.derive(visible[cik])
)
scores = research.cross_section_scores(current_features)
scores = research.cross_section_scores(
current_features,
split_safe=split_safe,
)
scores_by_date[signal_date.isoformat()] = scores
coverage_rows.append(
{
key: sum(row.get(key) is not None for row in scores.values())
for key in (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
for key in (*factor_polarity, *research.COMPOSITE_KEYS)
}
)
if not args.quiet and date_index % 100 == 0:
@@ -469,13 +500,13 @@ def _factor_diagnostics(
representatives: dict[str, str],
prices: dict[str, tuple],
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
factor_keys: tuple[str, ...],
train_end: date,
test_start: date,
) -> dict[str, list[dict]]:
from app.services import backtest_service as bt
from app.services import fundamentals_research as research
signal_keys = (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
signal_keys = (*factor_keys, *COMPOSITES)
observations: list[dict[str, Any]] = []
for cik, symbol in representatives.items():
columns = prices[symbol]
@@ -533,6 +564,7 @@ def _attach_overlay_ranks(
candidates: list[dict],
ticker_rows: list[dict],
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
arms: tuple[dict[str, Any], ...],
) -> dict[str, Any]:
from app.services import backtest_service as bt
from app.services import fundamentals_research as research
@@ -550,7 +582,7 @@ def _attach_overlay_ranks(
if value is not None:
covered[composite] += 1
base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
for arm in ARMS:
for arm in arms:
if arm["composite"] is None:
continue
candidate[_ranking_key(arm)] = research.overlay_rank(
@@ -626,6 +658,7 @@ def _run_arm(
data: dict[str, Any],
train_end: date,
test_start: date,
n_trials: int,
) -> tuple[dict[str, Any], dict[str, set[str]]]:
from app.services import backtest_service as bt
from scripts import run_research_matrix as shared
@@ -691,7 +724,7 @@ def _run_arm(
dsr = bt.deflated_sharpe_ratio(
sim.get("sharpe"),
sim.get("sharpe_se"),
N_TRIALS,
n_trials,
n_returns=sim.get("n_returns"),
return_skew=sim.get("return_skew"),
return_kurtosis=sim.get("return_kurtosis"),
@@ -781,13 +814,20 @@ def _fmt(value: Any) -> str:
def _markdown(report: dict[str, Any]) -> str:
protocol_id = report.get("research_protocol", ORIGINAL_PROTOCOL)
title = (
"Split-safe fundamentals overlay sensitivity"
if protocol_id == SPLIT_SAFE_PROTOCOL
else "Point-in-time fundamentals overlay research"
)
lines = [
"# Point-in-time fundamentals overlay research",
f"# {title}",
"",
"Generated: {}".format(report.get("generated_at")),
"",
"## Protocol",
"",
"- Research protocol: **{}**.".format(protocol_id),
"- Train ends before **{}**.".format(report["splits"]["train_end"]),
"- Validation runs until **{}**.".format(report["splits"]["test_start"]),
"- Test starts at that date and is not used to select the arm.",
@@ -976,9 +1016,21 @@ def _write_outputs(report: dict[str, Any], out: Path, *, bundle: bool) -> None:
async def _main() -> None:
from app.services import fundamentals_research as research
args = _parse_args()
split_safe = args.protocol == SPLIT_SAFE_PROTOCOL
arms = SPLIT_SAFE_ARMS if split_safe else ARMS
n_trials = len(arms)
factor_keys = tuple(
(
research.SPLIT_SAFE_FACTOR_POLARITY
if split_safe
else research.FACTOR_POLARITY
).keys()
)
snapshot = Path(args.snapshot)
out = Path(args.out) if args.out else _default_out()
out = Path(args.out) if args.out else _default_out(args.protocol)
if not snapshot.exists():
raise SystemExit(f"snapshot not found: {snapshot}")
if args.workers < 1:
@@ -1010,21 +1062,45 @@ async def _main() -> None:
all_dates,
representatives,
data["snapshots_by_cik"],
split_safe,
args,
)
candidate_coverage = _attach_overlay_ranks(
candidates, data["ticker_rows"], scores_by_date
candidates, data["ticker_rows"], scores_by_date, arms
)
factor_ic = _factor_diagnostics(
representatives,
data["prices"],
scores_by_date,
factor_keys,
train_end,
test_start,
)
warnings = [
"Current tracked universe only: historical constituent membership and "
"delisted names are unavailable, so absolute results have survivorship bias.",
"Earnings surprise is excluded because the completed SUE study already "
"failed its promotion bar for this strategy.",
"The test window has already been observed; this follow-up is sensitivity "
"evidence and live paper performance remains the final out-of-sample check.",
]
if split_safe:
warnings.insert(
1,
"Diluted-EPS growth and share-count change are excluded because filing-time "
"values are not split-comparable without point-in-time split factors.",
)
else:
warnings.insert(
1,
"Historical valuation is excluded because split-adjusted bars cannot be "
"safely combined with filing-time EPS and shares without split factors.",
)
report: dict[str, Any] = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"research_protocol": args.protocol,
"git_commit": _git_commit(),
"snapshot": str(snapshot.resolve()),
"snapshot_sha256": _snapshot_hash(snapshot),
@@ -1039,9 +1115,10 @@ async def _main() -> None:
"test": f"entry date >= {test_start.isoformat()}",
},
},
"n_trials": N_TRIALS,
"pre_registered_arms": list(ARMS),
"n_trials": n_trials,
"pre_registered_arms": list(arms),
"protocol": {
"id": args.protocol,
"qualification": "unchanged production gate; rank overlay only",
"cadence": "daily",
"fill_mode": "close; production near-close proxy",
@@ -1051,22 +1128,17 @@ async def _main() -> None:
"conservative match for the daily pre-market SEC import"
),
"missing_fundamental_score": 50.0,
"factor_keys": list(factor_keys),
"split_sensitive_metrics_excluded": (
["eps_growth_yoy", "share_count_change_yoy"] if split_safe else []
),
"selection": (
"highest validation Sharpe among arms with train and validation "
"Sharpe not below control and validation drawdown within 2pp"
),
"production_mutation": False,
},
"warnings": [
"Current tracked universe only: historical constituent membership and "
"delisted names are unavailable, so absolute results have survivorship bias.",
"Historical valuation is excluded because split-adjusted bars cannot be "
"safely combined with filing-time EPS and shares without split factors.",
"Earnings surprise is excluded because the completed SUE study already "
"failed its promotion bar for this strategy.",
"The test window remains research evidence, not a pristine future sample; "
"live paper performance is still the final out-of-sample check.",
],
"warnings": warnings,
"data_audit": audit,
"strategy_config": {
"recommendation": data["config"],
@@ -1087,10 +1159,12 @@ async def _main() -> None:
trade_sets: dict[str, dict[str, set[str]]] = {}
control: dict[str, Any] | None = None
for arm in ARMS:
for arm in arms:
if not args.quiet:
print("running {}".format(arm["id"]), flush=True)
result, arm_trades = _run_arm(arm, candidates, data, train_end, test_start)
result, arm_trades = _run_arm(
arm, candidates, data, train_end, test_start, n_trials
)
trade_sets[str(arm["id"])] = arm_trades
if control is None:
control = result