diff --git a/app/services/fundamentals_research.py b/app/services/fundamentals_research.py index f2a6fde..c1fbdb9 100644 --- a/app/services/fundamentals_research.py +++ b/app/services/fundamentals_research.py @@ -30,6 +30,18 @@ QUALITY_FACTORS = ( "share_count_change_yoy", ) GROWTH_FACTORS = ("revenue_growth_yoy", "eps_growth_yoy") +SPLIT_SAFE_FACTOR_POLARITY: dict[str, bool] = { + "revenue_growth_yoy": True, + "operating_margin": True, + "fcf_margin": True, + "net_debt_to_ebitda": False, +} +SPLIT_SAFE_QUALITY_FACTORS = ( + "operating_margin", + "fcf_margin", + "net_debt_to_ebitda", +) +SPLIT_SAFE_GROWTH_FACTORS = ("revenue_growth_yoy",) COMPOSITE_KEYS = ("quality", "growth", "balanced") @@ -46,22 +58,29 @@ def cross_section_scores( features_by_issuer: Mapping[str, Mapping[str, Any]], *, min_cross_section: int = MIN_CROSS_SECTION, + split_safe: bool = False, ) -> dict[str, dict[str, float | None]]: """Return favorable factor ranks and composites for every issuer. - Quality needs two of four inputs; growth needs one of two. Balanced requires - both sub-scores and weights them equally, so quality's four inputs do not - mechanically dominate growth's two inputs. + The default reproduces the original registered experiment. ``split_safe`` + excludes diluted-EPS growth and share-count change because filing-time + values are not comparable across stock splits without point-in-time split + factors. Its quality score needs two of three remaining inputs and its + growth score is revenue growth. Balanced always weights the two sub-scores + equally. """ + factor_polarity = SPLIT_SAFE_FACTOR_POLARITY if split_safe else FACTOR_POLARITY + quality_factors = SPLIT_SAFE_QUALITY_FACTORS if split_safe else QUALITY_FACTORS + growth_factors = SPLIT_SAFE_GROWTH_FACTORS if split_safe else GROWTH_FACTORS result = { str(issuer): { - **{key: None for key in FACTOR_POLARITY}, + **{key: None for key in factor_polarity}, **{key: None for key in COMPOSITE_KEYS}, } for issuer in features_by_issuer } - for factor, higher_is_better in FACTOR_POLARITY.items(): + for factor, higher_is_better in factor_polarity.items(): values = { str(issuer): _finite_or_none(features.get(factor)) for issuer, features in features_by_issuer.items() @@ -75,8 +94,8 @@ def cross_section_scores( result[issuer][factor] = rank for scores in result.values(): - quality_values = _available(scores, QUALITY_FACTORS) - growth_values = _available(scores, GROWTH_FACTORS) + quality_values = _available(scores, quality_factors) + growth_values = _available(scores, growth_factors) if len(quality_values) >= 2: scores["quality"] = _mean(quality_values) if growth_values: diff --git a/docs/research/fundamentals-weight-backtest.md b/docs/research/fundamentals-weight-backtest.md index 6911b75..10956a5 100644 --- a/docs/research/fundamentals-weight-backtest.md +++ b/docs/research/fundamentals-weight-backtest.md @@ -1,6 +1,7 @@ # Point-in-time fundamentals weight backtest -Status: pre-registered local research. Running it does not change production. +Status: initial experiment completed; split-safe follow-up registered locally. +Running either protocol does not change production. ## Question @@ -10,7 +11,7 @@ position sizing, capacity, costs, ATR trail, and post-stop re-entry policy remai unchanged. This isolates the incremental value of fundamentals as a ranking overlay. -## Registered experiment +## Completed initial experiment The control is the production 80/20 residual-momentum / volatility rank. The runner tests three fundamental composites at weights 10%, 20%, 30%, and 40%: @@ -28,11 +29,38 @@ tracked universe. Missing composite scores are neutral at 50. The formula is: There are 13 registered portfolio trials including the control. That complete count is used by the Deflated Sharpe calculation. -Historical P/E and FCF yield are excluded. Stored Alpaca bars are split-adjusted, -while filing-time EPS and shares are not guaranteed to use today's split basis; -mixing them without point-in-time split factors can manufacture valuation moves. -Earnings surprise is also excluded because the completed Dolt SUE study already -failed its promotion bar for this strategy. +No overlay passed the registered train and validation requirements. Review also +found that filing-time diluted EPS and shares are not guaranteed to use the same +split basis across periods. That makes EPS growth and share-count change unsafe +for historical ranking without point-in-time split factors. The initial result +remains an auditable rejection of its registered arms, but it is not evidence +that split-safe fundamentals have no value. + +## Registered split-safe follow-up + +Run with `--protocol split-safe`. This is a smaller sensitivity experiment: + +- Quality: operating margin, FCF margin, and low net-debt/EBITDA. At least two + inputs must exist. +- Growth: revenue growth only. +- Balanced: equal weight to the quality and growth sub-scores. Both must exist. +- Overlay weights: 5%, 10%, and 15%. + +Diluted-EPS growth and share-count change are excluded completely: they are not +ranked, do not enter composites, and do not appear in factor-IC output. Historical +P/E and FCF yield remain excluded for the same split-basis reason. Earnings +surprise remains excluded because the completed Dolt SUE study failed its +promotion bar for this strategy. + +There are 10 registered portfolio trials including the control. The split-safe +report uses 10 in its Deflated Sharpe calculation. It has a separate score cache +and `fundamentals-splitsafe-*` output prefix, so it cannot be confused with or +silently reuse the initial experiment's scores. + +The test window has already been inspected during the initial experiment. Keep +the original date boundaries and development selection discipline, but treat the +follow-up as sensitivity evidence. Any promotion still requires forward paper +evidence. ## Point-in-time rule @@ -93,6 +121,8 @@ different path to the launcher. ## 3. Run it +The launcher now defaults to the registered `split-safe` follow-up: + ```bash ./scripts/run_fundamentals_macbook.sh \ backtest_snapshots/fundamentals-backtest.sqlite @@ -106,16 +136,24 @@ WORKERS=8 ./scripts/run_fundamentals_macbook.sh \ backtest_snapshots/fundamentals-backtest.sqlite ``` +To reproduce the completed initial matrix instead, opt in explicitly: + +```bash +PROTOCOL=original ./scripts/run_fundamentals_macbook.sh \ + backtest_snapshots/fundamentals-backtest.sqlite +``` + The first run builds two caches under `reports/.cache`: production candidate -replay and point-in-time fundamental scores. If interrupted, rerun the same -command; valid caches are reused. Cache keys include snapshot size and mtime, so -a new snapshot triggers a rebuild. +replay and protocol-specific point-in-time fundamental scores. If interrupted, +rerun the same command; valid caches are reused. Cache keys include the protocol, +snapshot size, and mtime, so neither a protocol switch nor a new snapshot can +reuse incompatible scores. ## 4. Bring the result back The final line names one ZIP such as: -`reports/fundamentals-overlay-20260723-180000.zip` +`reports/fundamentals-splitsafe-20260723-180000.zip` That ZIP contains: diff --git a/scripts/run_fundamentals_macbook.sh b/scripts/run_fundamentals_macbook.sh index b8bf5e1..97042c4 100755 --- a/scripts/run_fundamentals_macbook.sh +++ b/scripts/run_fundamentals_macbook.sh @@ -5,6 +5,7 @@ ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}" +PROTOCOL="${PROTOCOL:-split-safe}" WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}" if [[ "$WORKERS" -gt 1 ]]; then WORKERS=$((WORKERS - 1)) @@ -21,17 +22,34 @@ if [[ ! -f "$SNAPSHOT" ]]; then exit 1 fi +case "$PROTOCOL" in + split-safe) + PREFIX="fundamentals-splitsafe" + SCORE_CACHE="reports/.cache/fundamentals-splitsafe-scores.pkl" + ;; + original) + PREFIX="fundamentals-overlay" + SCORE_CACHE="reports/.cache/fundamentals-scores.pkl" + ;; + *) + echo "Unsupported PROTOCOL: $PROTOCOL (use split-safe or original)" >&2 + exit 1 + ;; +esac + STAMP="$(date -u +%Y%m%d-%H%M%S)" -OUT="reports/fundamentals-overlay-${STAMP}.json" +OUT="reports/${PREFIX}-${STAMP}.json" echo "Snapshot: $SNAPSHOT" echo "Workers: $WORKERS" +echo "Protocol: $PROTOCOL" echo "Output: $OUT" "$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \ + --protocol "$PROTOCOL" \ --workers "$WORKERS" \ --candidate-cache reports/.cache/fundamentals-candidates.pkl \ - --fundamentals-cache reports/.cache/fundamentals-scores.pkl \ + --fundamentals-cache "$SCORE_CACHE" \ --out "$OUT" echo diff --git a/scripts/run_fundamentals_research.py b/scripts/run_fundamentals_research.py index fe60425..a5095dc 100644 --- a/scripts/run_fundamentals_research.py +++ b/scripts/run_fundamentals_research.py @@ -2,8 +2,9 @@ The production qualification gate is unchanged. The runner first measures 30-session factor IC, then reorders already-qualified candidates with quality, -growth, or balanced fundamental ranks at 10, 20, 30, and 40 percent weights. -It writes JSON, Markdown, CSV, and a portable ZIP bundle. +growth, or balanced fundamental ranks. It supports the original registered +matrix and a split-safe follow-up sensitivity. It writes JSON, Markdown, CSV, +and a portable ZIP bundle. """ from __future__ import annotations @@ -37,26 +38,38 @@ if str(ROOT) not in sys.path: CACHE_VERSION = "fundamentals-overlay-v1" NY = ZoneInfo("America/New_York") COMPOSITES = ("quality", "growth", "balanced") +ORIGINAL_PROTOCOL = "original" +SPLIT_SAFE_PROTOCOL = "split-safe" WEIGHTS = (0.10, 0.20, 0.30, 0.40) -ARMS: tuple[dict[str, Any], ...] = ( - { - "id": "control_w00", - "label": "Production 80/20 momentum-volatility rank", - "composite": None, - "weight": 0.0, - }, - *tuple( +SPLIT_SAFE_WEIGHTS = (0.05, 0.10, 0.15) + + +def _arm_matrix(weights: tuple[float, ...]) -> tuple[dict[str, Any], ...]: + """Build the control plus three bounded overlay families.""" + return ( { - "id": f"{composite}_w{round(weight * 100):02d}", - "label": f"{composite.title()} overlay {round(weight * 100)}%", - "composite": composite, - "weight": weight, - } - for composite in COMPOSITES - for weight in WEIGHTS - ), -) + "id": "control_w00", + "label": "Production 80/20 momentum-volatility rank", + "composite": None, + "weight": 0.0, + }, + *tuple( + { + "id": f"{composite}_w{round(weight * 100):02d}", + "label": f"{composite.title()} overlay {round(weight * 100)}%", + "composite": composite, + "weight": weight, + } + for composite in COMPOSITES + for weight in weights + ), + ) + + +ARMS = _arm_matrix(WEIGHTS) N_TRIALS = len(ARMS) +SPLIT_SAFE_ARMS = _arm_matrix(SPLIT_SAFE_WEIGHTS) +SPLIT_SAFE_N_TRIALS = len(SPLIT_SAFE_ARMS) def _parse_args() -> argparse.Namespace: @@ -74,6 +87,11 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument("--train-end", default="2024-01-01") parser.add_argument("--test-start", default="2025-01-01") + parser.add_argument( + "--protocol", + choices=(ORIGINAL_PROTOCOL, SPLIT_SAFE_PROTOCOL), + default=ORIGINAL_PROTOCOL, + ) parser.add_argument("--allow-spawn", action="store_true") parser.add_argument("--quiet", action="store_true") return parser.parse_args() @@ -83,9 +101,14 @@ def _sqlite_url(path: Path) -> str: return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" -def _default_out() -> Path: +def _default_out(protocol: str = ORIGINAL_PROTOCOL) -> Path: stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - return Path("reports") / f"fundamentals-overlay-{stamp}.json" + label = ( + "fundamentals-splitsafe" + if protocol == SPLIT_SAFE_PROTOCOL + else "fundamentals-overlay" + ) + return Path("reports") / f"{label}-{stamp}.json" def _snapshot_hash(path: Path) -> str: @@ -394,11 +417,15 @@ def _build_scores( dates: set[date], representatives: dict[str, str], snapshots_by_cik: dict[str, list[Any]], + split_safe: bool, args: argparse.Namespace, ) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]: from app.services import fundamentals_derivation as derivation from app.services import fundamentals_research as research + factor_polarity = ( + research.SPLIT_SAFE_FACTOR_POLARITY if split_safe else research.FACTOR_POLARITY + ) ordered_dates = sorted(dates) date_fingerprint = hashlib.sha256( "|".join(value.isoformat() for value in ordered_dates).encode() @@ -408,6 +435,7 @@ def _build_scores( snapshot, { "kind": "point-in-time-scores", + "score_profile": SPLIT_SAFE_PROTOCOL if split_safe else ORIGINAL_PROTOCOL, "date_fingerprint": date_fingerprint, "availability": "accepted before signal-date midnight America/New_York", }, @@ -446,12 +474,15 @@ def _build_scores( current_features[cik] = research.raw_features( derivation.derive(visible[cik]) ) - scores = research.cross_section_scores(current_features) + scores = research.cross_section_scores( + current_features, + split_safe=split_safe, + ) scores_by_date[signal_date.isoformat()] = scores coverage_rows.append( { key: sum(row.get(key) is not None for row in scores.values()) - for key in (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS) + for key in (*factor_polarity, *research.COMPOSITE_KEYS) } ) if not args.quiet and date_index % 100 == 0: @@ -469,13 +500,13 @@ def _factor_diagnostics( representatives: dict[str, str], prices: dict[str, tuple], scores_by_date: dict[str, dict[str, dict[str, float | None]]], + factor_keys: tuple[str, ...], train_end: date, test_start: date, ) -> dict[str, list[dict]]: from app.services import backtest_service as bt - from app.services import fundamentals_research as research - signal_keys = (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS) + signal_keys = (*factor_keys, *COMPOSITES) observations: list[dict[str, Any]] = [] for cik, symbol in representatives.items(): columns = prices[symbol] @@ -533,6 +564,7 @@ def _attach_overlay_ranks( candidates: list[dict], ticker_rows: list[dict], scores_by_date: dict[str, dict[str, dict[str, float | None]]], + arms: tuple[dict[str, Any], ...], ) -> dict[str, Any]: from app.services import backtest_service as bt from app.services import fundamentals_research as research @@ -550,7 +582,7 @@ def _attach_overlay_ranks( if value is not None: covered[composite] += 1 base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY) - for arm in ARMS: + for arm in arms: if arm["composite"] is None: continue candidate[_ranking_key(arm)] = research.overlay_rank( @@ -626,6 +658,7 @@ def _run_arm( data: dict[str, Any], train_end: date, test_start: date, + n_trials: int, ) -> tuple[dict[str, Any], dict[str, set[str]]]: from app.services import backtest_service as bt from scripts import run_research_matrix as shared @@ -691,7 +724,7 @@ def _run_arm( dsr = bt.deflated_sharpe_ratio( sim.get("sharpe"), sim.get("sharpe_se"), - N_TRIALS, + n_trials, n_returns=sim.get("n_returns"), return_skew=sim.get("return_skew"), return_kurtosis=sim.get("return_kurtosis"), @@ -781,13 +814,20 @@ def _fmt(value: Any) -> str: def _markdown(report: dict[str, Any]) -> str: + protocol_id = report.get("research_protocol", ORIGINAL_PROTOCOL) + title = ( + "Split-safe fundamentals overlay sensitivity" + if protocol_id == SPLIT_SAFE_PROTOCOL + else "Point-in-time fundamentals overlay research" + ) lines = [ - "# Point-in-time fundamentals overlay research", + f"# {title}", "", "Generated: {}".format(report.get("generated_at")), "", "## Protocol", "", + "- Research protocol: **{}**.".format(protocol_id), "- Train ends before **{}**.".format(report["splits"]["train_end"]), "- Validation runs until **{}**.".format(report["splits"]["test_start"]), "- Test starts at that date and is not used to select the arm.", @@ -976,9 +1016,21 @@ def _write_outputs(report: dict[str, Any], out: Path, *, bundle: bool) -> None: async def _main() -> None: + from app.services import fundamentals_research as research + args = _parse_args() + split_safe = args.protocol == SPLIT_SAFE_PROTOCOL + arms = SPLIT_SAFE_ARMS if split_safe else ARMS + n_trials = len(arms) + factor_keys = tuple( + ( + research.SPLIT_SAFE_FACTOR_POLARITY + if split_safe + else research.FACTOR_POLARITY + ).keys() + ) snapshot = Path(args.snapshot) - out = Path(args.out) if args.out else _default_out() + out = Path(args.out) if args.out else _default_out(args.protocol) if not snapshot.exists(): raise SystemExit(f"snapshot not found: {snapshot}") if args.workers < 1: @@ -1010,21 +1062,45 @@ async def _main() -> None: all_dates, representatives, data["snapshots_by_cik"], + split_safe, args, ) candidate_coverage = _attach_overlay_ranks( - candidates, data["ticker_rows"], scores_by_date + candidates, data["ticker_rows"], scores_by_date, arms ) factor_ic = _factor_diagnostics( representatives, data["prices"], scores_by_date, + factor_keys, train_end, test_start, ) + warnings = [ + "Current tracked universe only: historical constituent membership and " + "delisted names are unavailable, so absolute results have survivorship bias.", + "Earnings surprise is excluded because the completed SUE study already " + "failed its promotion bar for this strategy.", + "The test window has already been observed; this follow-up is sensitivity " + "evidence and live paper performance remains the final out-of-sample check.", + ] + if split_safe: + warnings.insert( + 1, + "Diluted-EPS growth and share-count change are excluded because filing-time " + "values are not split-comparable without point-in-time split factors.", + ) + else: + warnings.insert( + 1, + "Historical valuation is excluded because split-adjusted bars cannot be " + "safely combined with filing-time EPS and shares without split factors.", + ) + report: dict[str, Any] = { "generated_at": datetime.now(timezone.utc).isoformat(), + "research_protocol": args.protocol, "git_commit": _git_commit(), "snapshot": str(snapshot.resolve()), "snapshot_sha256": _snapshot_hash(snapshot), @@ -1039,9 +1115,10 @@ async def _main() -> None: "test": f"entry date >= {test_start.isoformat()}", }, }, - "n_trials": N_TRIALS, - "pre_registered_arms": list(ARMS), + "n_trials": n_trials, + "pre_registered_arms": list(arms), "protocol": { + "id": args.protocol, "qualification": "unchanged production gate; rank overlay only", "cadence": "daily", "fill_mode": "close; production near-close proxy", @@ -1051,22 +1128,17 @@ async def _main() -> None: "conservative match for the daily pre-market SEC import" ), "missing_fundamental_score": 50.0, + "factor_keys": list(factor_keys), + "split_sensitive_metrics_excluded": ( + ["eps_growth_yoy", "share_count_change_yoy"] if split_safe else [] + ), "selection": ( "highest validation Sharpe among arms with train and validation " "Sharpe not below control and validation drawdown within 2pp" ), "production_mutation": False, }, - "warnings": [ - "Current tracked universe only: historical constituent membership and " - "delisted names are unavailable, so absolute results have survivorship bias.", - "Historical valuation is excluded because split-adjusted bars cannot be " - "safely combined with filing-time EPS and shares without split factors.", - "Earnings surprise is excluded because the completed SUE study already " - "failed its promotion bar for this strategy.", - "The test window remains research evidence, not a pristine future sample; " - "live paper performance is still the final out-of-sample check.", - ], + "warnings": warnings, "data_audit": audit, "strategy_config": { "recommendation": data["config"], @@ -1087,10 +1159,12 @@ async def _main() -> None: trade_sets: dict[str, dict[str, set[str]]] = {} control: dict[str, Any] | None = None - for arm in ARMS: + for arm in arms: if not args.quiet: print("running {}".format(arm["id"]), flush=True) - result, arm_trades = _run_arm(arm, candidates, data, train_end, test_start) + result, arm_trades = _run_arm( + arm, candidates, data, train_end, test_start, n_trials + ) trade_sets[str(arm["id"])] = arm_trades if control is None: control = result diff --git a/tests/unit/test_fundamentals_research.py b/tests/unit/test_fundamentals_research.py index c13dde2..7d7e563 100644 --- a/tests/unit/test_fundamentals_research.py +++ b/tests/unit/test_fundamentals_research.py @@ -51,6 +51,45 @@ def test_composites_use_equal_subgroup_weighting(): assert scores["3"]["balanced"] == 50.0 +def test_split_safe_composites_ignore_eps_and_share_count(): + def features(unsafe_multiplier: int): + return { + str(index): { + "operating_margin": index, + "fcf_margin": index, + "net_debt_to_ebitda": 6 - index, + "revenue_growth_yoy": index, + "eps_growth_yoy": unsafe_multiplier * (6 - index), + "share_count_change_yoy": unsafe_multiplier * index, + } + for index in range(1, 6) + } + + baseline = research.cross_section_scores(features(1), split_safe=True) + distorted = research.cross_section_scores(features(1_000_000), split_safe=True) + + assert distorted == baseline + assert baseline["5"]["quality"] == 100.0 + assert baseline["5"]["growth"] == 100.0 + assert baseline["5"]["balanced"] == 100.0 + assert "eps_growth_yoy" not in baseline["5"] + assert "share_count_change_yoy" not in baseline["5"] + + +def test_split_safe_quality_still_requires_two_comparable_inputs(): + features = { + str(index): { + "operating_margin": index, + "revenue_growth_yoy": index, + } + for index in range(1, 6) + } + scores = research.cross_section_scores(features, split_safe=True) + assert all(row["quality"] is None for row in scores.values()) + assert scores["5"]["growth"] == 100.0 + assert scores["5"]["balanced"] is None + + def test_overlay_uses_neutral_missing_score_and_validates_weight(): assert research.overlay_rank(90, None, 0.2) == 82.0 assert research.overlay_rank(90, 100, 0.2) == 92.0 diff --git a/tests/unit/test_fundamentals_research_runner.py b/tests/unit/test_fundamentals_research_runner.py index 6ce7853..16352e7 100644 --- a/tests/unit/test_fundamentals_research_runner.py +++ b/tests/unit/test_fundamentals_research_runner.py @@ -29,6 +29,27 @@ def test_arm_matrix_is_bounded_and_pre_registered(): } +def test_split_safe_matrix_is_smaller_and_trial_corrected(): + assert runner.SPLIT_SAFE_N_TRIALS == 10 + assert runner.SPLIT_SAFE_ARMS[0]["id"] == "control_w00" + assert {arm["weight"] for arm in runner.SPLIT_SAFE_ARMS[1:]} == { + 0.05, + 0.10, + 0.15, + } + assert {arm["composite"] for arm in runner.SPLIT_SAFE_ARMS[1:]} == { + "quality", + "growth", + "balanced", + } + + +def test_split_safe_output_has_a_distinct_name(): + assert runner._default_out(runner.SPLIT_SAFE_PROTOCOL).name.startswith( + "fundamentals-splitsafe-" + ) + + def test_development_grade_does_not_read_test_window(): control = { "windows": [