feat: add split-safe fundamentals research protocol
This commit is contained in:
@@ -30,6 +30,18 @@ QUALITY_FACTORS = (
|
|||||||
"share_count_change_yoy",
|
"share_count_change_yoy",
|
||||||
)
|
)
|
||||||
GROWTH_FACTORS = ("revenue_growth_yoy", "eps_growth_yoy")
|
GROWTH_FACTORS = ("revenue_growth_yoy", "eps_growth_yoy")
|
||||||
|
SPLIT_SAFE_FACTOR_POLARITY: dict[str, bool] = {
|
||||||
|
"revenue_growth_yoy": True,
|
||||||
|
"operating_margin": True,
|
||||||
|
"fcf_margin": True,
|
||||||
|
"net_debt_to_ebitda": False,
|
||||||
|
}
|
||||||
|
SPLIT_SAFE_QUALITY_FACTORS = (
|
||||||
|
"operating_margin",
|
||||||
|
"fcf_margin",
|
||||||
|
"net_debt_to_ebitda",
|
||||||
|
)
|
||||||
|
SPLIT_SAFE_GROWTH_FACTORS = ("revenue_growth_yoy",)
|
||||||
COMPOSITE_KEYS = ("quality", "growth", "balanced")
|
COMPOSITE_KEYS = ("quality", "growth", "balanced")
|
||||||
|
|
||||||
|
|
||||||
@@ -46,22 +58,29 @@ def cross_section_scores(
|
|||||||
features_by_issuer: Mapping[str, Mapping[str, Any]],
|
features_by_issuer: Mapping[str, Mapping[str, Any]],
|
||||||
*,
|
*,
|
||||||
min_cross_section: int = MIN_CROSS_SECTION,
|
min_cross_section: int = MIN_CROSS_SECTION,
|
||||||
|
split_safe: bool = False,
|
||||||
) -> dict[str, dict[str, float | None]]:
|
) -> dict[str, dict[str, float | None]]:
|
||||||
"""Return favorable factor ranks and composites for every issuer.
|
"""Return favorable factor ranks and composites for every issuer.
|
||||||
|
|
||||||
Quality needs two of four inputs; growth needs one of two. Balanced requires
|
The default reproduces the original registered experiment. ``split_safe``
|
||||||
both sub-scores and weights them equally, so quality's four inputs do not
|
excludes diluted-EPS growth and share-count change because filing-time
|
||||||
mechanically dominate growth's two inputs.
|
values are not comparable across stock splits without point-in-time split
|
||||||
|
factors. Its quality score needs two of three remaining inputs and its
|
||||||
|
growth score is revenue growth. Balanced always weights the two sub-scores
|
||||||
|
equally.
|
||||||
"""
|
"""
|
||||||
|
factor_polarity = SPLIT_SAFE_FACTOR_POLARITY if split_safe else FACTOR_POLARITY
|
||||||
|
quality_factors = SPLIT_SAFE_QUALITY_FACTORS if split_safe else QUALITY_FACTORS
|
||||||
|
growth_factors = SPLIT_SAFE_GROWTH_FACTORS if split_safe else GROWTH_FACTORS
|
||||||
result = {
|
result = {
|
||||||
str(issuer): {
|
str(issuer): {
|
||||||
**{key: None for key in FACTOR_POLARITY},
|
**{key: None for key in factor_polarity},
|
||||||
**{key: None for key in COMPOSITE_KEYS},
|
**{key: None for key in COMPOSITE_KEYS},
|
||||||
}
|
}
|
||||||
for issuer in features_by_issuer
|
for issuer in features_by_issuer
|
||||||
}
|
}
|
||||||
|
|
||||||
for factor, higher_is_better in FACTOR_POLARITY.items():
|
for factor, higher_is_better in factor_polarity.items():
|
||||||
values = {
|
values = {
|
||||||
str(issuer): _finite_or_none(features.get(factor))
|
str(issuer): _finite_or_none(features.get(factor))
|
||||||
for issuer, features in features_by_issuer.items()
|
for issuer, features in features_by_issuer.items()
|
||||||
@@ -75,8 +94,8 @@ def cross_section_scores(
|
|||||||
result[issuer][factor] = rank
|
result[issuer][factor] = rank
|
||||||
|
|
||||||
for scores in result.values():
|
for scores in result.values():
|
||||||
quality_values = _available(scores, QUALITY_FACTORS)
|
quality_values = _available(scores, quality_factors)
|
||||||
growth_values = _available(scores, GROWTH_FACTORS)
|
growth_values = _available(scores, growth_factors)
|
||||||
if len(quality_values) >= 2:
|
if len(quality_values) >= 2:
|
||||||
scores["quality"] = _mean(quality_values)
|
scores["quality"] = _mean(quality_values)
|
||||||
if growth_values:
|
if growth_values:
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# Point-in-time fundamentals weight backtest
|
# Point-in-time fundamentals weight backtest
|
||||||
|
|
||||||
Status: pre-registered local research. Running it does not change production.
|
Status: initial experiment completed; split-safe follow-up registered locally.
|
||||||
|
Running either protocol does not change production.
|
||||||
|
|
||||||
## Question
|
## Question
|
||||||
|
|
||||||
@@ -10,7 +11,7 @@ position sizing, capacity, costs, ATR trail, and post-stop re-entry policy remai
|
|||||||
unchanged. This isolates the incremental value of fundamentals as a ranking
|
unchanged. This isolates the incremental value of fundamentals as a ranking
|
||||||
overlay.
|
overlay.
|
||||||
|
|
||||||
## Registered experiment
|
## Completed initial experiment
|
||||||
|
|
||||||
The control is the production 80/20 residual-momentum / volatility rank. The
|
The control is the production 80/20 residual-momentum / volatility rank. The
|
||||||
runner tests three fundamental composites at weights 10%, 20%, 30%, and 40%:
|
runner tests three fundamental composites at weights 10%, 20%, 30%, and 40%:
|
||||||
@@ -28,11 +29,38 @@ tracked universe. Missing composite scores are neutral at 50. The formula is:
|
|||||||
There are 13 registered portfolio trials including the control. That complete
|
There are 13 registered portfolio trials including the control. That complete
|
||||||
count is used by the Deflated Sharpe calculation.
|
count is used by the Deflated Sharpe calculation.
|
||||||
|
|
||||||
Historical P/E and FCF yield are excluded. Stored Alpaca bars are split-adjusted,
|
No overlay passed the registered train and validation requirements. Review also
|
||||||
while filing-time EPS and shares are not guaranteed to use today's split basis;
|
found that filing-time diluted EPS and shares are not guaranteed to use the same
|
||||||
mixing them without point-in-time split factors can manufacture valuation moves.
|
split basis across periods. That makes EPS growth and share-count change unsafe
|
||||||
Earnings surprise is also excluded because the completed Dolt SUE study already
|
for historical ranking without point-in-time split factors. The initial result
|
||||||
failed its promotion bar for this strategy.
|
remains an auditable rejection of its registered arms, but it is not evidence
|
||||||
|
that split-safe fundamentals have no value.
|
||||||
|
|
||||||
|
## Registered split-safe follow-up
|
||||||
|
|
||||||
|
Run with `--protocol split-safe`. This is a smaller sensitivity experiment:
|
||||||
|
|
||||||
|
- Quality: operating margin, FCF margin, and low net-debt/EBITDA. At least two
|
||||||
|
inputs must exist.
|
||||||
|
- Growth: revenue growth only.
|
||||||
|
- Balanced: equal weight to the quality and growth sub-scores. Both must exist.
|
||||||
|
- Overlay weights: 5%, 10%, and 15%.
|
||||||
|
|
||||||
|
Diluted-EPS growth and share-count change are excluded completely: they are not
|
||||||
|
ranked, do not enter composites, and do not appear in factor-IC output. Historical
|
||||||
|
P/E and FCF yield remain excluded for the same split-basis reason. Earnings
|
||||||
|
surprise remains excluded because the completed Dolt SUE study failed its
|
||||||
|
promotion bar for this strategy.
|
||||||
|
|
||||||
|
There are 10 registered portfolio trials including the control. The split-safe
|
||||||
|
report uses 10 in its Deflated Sharpe calculation. It has a separate score cache
|
||||||
|
and `fundamentals-splitsafe-*` output prefix, so it cannot be confused with or
|
||||||
|
silently reuse the initial experiment's scores.
|
||||||
|
|
||||||
|
The test window has already been inspected during the initial experiment. Keep
|
||||||
|
the original date boundaries and development selection discipline, but treat the
|
||||||
|
follow-up as sensitivity evidence. Any promotion still requires forward paper
|
||||||
|
evidence.
|
||||||
|
|
||||||
## Point-in-time rule
|
## Point-in-time rule
|
||||||
|
|
||||||
@@ -93,6 +121,8 @@ different path to the launcher.
|
|||||||
|
|
||||||
## 3. Run it
|
## 3. Run it
|
||||||
|
|
||||||
|
The launcher now defaults to the registered `split-safe` follow-up:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./scripts/run_fundamentals_macbook.sh \
|
./scripts/run_fundamentals_macbook.sh \
|
||||||
backtest_snapshots/fundamentals-backtest.sqlite
|
backtest_snapshots/fundamentals-backtest.sqlite
|
||||||
@@ -106,16 +136,24 @@ WORKERS=8 ./scripts/run_fundamentals_macbook.sh \
|
|||||||
backtest_snapshots/fundamentals-backtest.sqlite
|
backtest_snapshots/fundamentals-backtest.sqlite
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To reproduce the completed initial matrix instead, opt in explicitly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PROTOCOL=original ./scripts/run_fundamentals_macbook.sh \
|
||||||
|
backtest_snapshots/fundamentals-backtest.sqlite
|
||||||
|
```
|
||||||
|
|
||||||
The first run builds two caches under `reports/.cache`: production candidate
|
The first run builds two caches under `reports/.cache`: production candidate
|
||||||
replay and point-in-time fundamental scores. If interrupted, rerun the same
|
replay and protocol-specific point-in-time fundamental scores. If interrupted,
|
||||||
command; valid caches are reused. Cache keys include snapshot size and mtime, so
|
rerun the same command; valid caches are reused. Cache keys include the protocol,
|
||||||
a new snapshot triggers a rebuild.
|
snapshot size, and mtime, so neither a protocol switch nor a new snapshot can
|
||||||
|
reuse incompatible scores.
|
||||||
|
|
||||||
## 4. Bring the result back
|
## 4. Bring the result back
|
||||||
|
|
||||||
The final line names one ZIP such as:
|
The final line names one ZIP such as:
|
||||||
|
|
||||||
`reports/fundamentals-overlay-20260723-180000.zip`
|
`reports/fundamentals-splitsafe-20260723-180000.zip`
|
||||||
|
|
||||||
That ZIP contains:
|
That ZIP contains:
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ ROOT="$(git rev-parse --show-toplevel)"
|
|||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}"
|
SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}"
|
||||||
|
PROTOCOL="${PROTOCOL:-split-safe}"
|
||||||
WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}"
|
WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}"
|
||||||
if [[ "$WORKERS" -gt 1 ]]; then
|
if [[ "$WORKERS" -gt 1 ]]; then
|
||||||
WORKERS=$((WORKERS - 1))
|
WORKERS=$((WORKERS - 1))
|
||||||
@@ -21,17 +22,34 @@ if [[ ! -f "$SNAPSHOT" ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
case "$PROTOCOL" in
|
||||||
|
split-safe)
|
||||||
|
PREFIX="fundamentals-splitsafe"
|
||||||
|
SCORE_CACHE="reports/.cache/fundamentals-splitsafe-scores.pkl"
|
||||||
|
;;
|
||||||
|
original)
|
||||||
|
PREFIX="fundamentals-overlay"
|
||||||
|
SCORE_CACHE="reports/.cache/fundamentals-scores.pkl"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unsupported PROTOCOL: $PROTOCOL (use split-safe or original)" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
||||||
OUT="reports/fundamentals-overlay-${STAMP}.json"
|
OUT="reports/${PREFIX}-${STAMP}.json"
|
||||||
|
|
||||||
echo "Snapshot: $SNAPSHOT"
|
echo "Snapshot: $SNAPSHOT"
|
||||||
echo "Workers: $WORKERS"
|
echo "Workers: $WORKERS"
|
||||||
|
echo "Protocol: $PROTOCOL"
|
||||||
echo "Output: $OUT"
|
echo "Output: $OUT"
|
||||||
|
|
||||||
"$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \
|
"$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \
|
||||||
|
--protocol "$PROTOCOL" \
|
||||||
--workers "$WORKERS" \
|
--workers "$WORKERS" \
|
||||||
--candidate-cache reports/.cache/fundamentals-candidates.pkl \
|
--candidate-cache reports/.cache/fundamentals-candidates.pkl \
|
||||||
--fundamentals-cache reports/.cache/fundamentals-scores.pkl \
|
--fundamentals-cache "$SCORE_CACHE" \
|
||||||
--out "$OUT"
|
--out "$OUT"
|
||||||
|
|
||||||
echo
|
echo
|
||||||
|
|||||||
@@ -2,8 +2,9 @@
|
|||||||
|
|
||||||
The production qualification gate is unchanged. The runner first measures
|
The production qualification gate is unchanged. The runner first measures
|
||||||
30-session factor IC, then reorders already-qualified candidates with quality,
|
30-session factor IC, then reorders already-qualified candidates with quality,
|
||||||
growth, or balanced fundamental ranks at 10, 20, 30, and 40 percent weights.
|
growth, or balanced fundamental ranks. It supports the original registered
|
||||||
It writes JSON, Markdown, CSV, and a portable ZIP bundle.
|
matrix and a split-safe follow-up sensitivity. It writes JSON, Markdown, CSV,
|
||||||
|
and a portable ZIP bundle.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -37,26 +38,38 @@ if str(ROOT) not in sys.path:
|
|||||||
CACHE_VERSION = "fundamentals-overlay-v1"
|
CACHE_VERSION = "fundamentals-overlay-v1"
|
||||||
NY = ZoneInfo("America/New_York")
|
NY = ZoneInfo("America/New_York")
|
||||||
COMPOSITES = ("quality", "growth", "balanced")
|
COMPOSITES = ("quality", "growth", "balanced")
|
||||||
|
ORIGINAL_PROTOCOL = "original"
|
||||||
|
SPLIT_SAFE_PROTOCOL = "split-safe"
|
||||||
WEIGHTS = (0.10, 0.20, 0.30, 0.40)
|
WEIGHTS = (0.10, 0.20, 0.30, 0.40)
|
||||||
ARMS: tuple[dict[str, Any], ...] = (
|
SPLIT_SAFE_WEIGHTS = (0.05, 0.10, 0.15)
|
||||||
{
|
|
||||||
"id": "control_w00",
|
|
||||||
"label": "Production 80/20 momentum-volatility rank",
|
def _arm_matrix(weights: tuple[float, ...]) -> tuple[dict[str, Any], ...]:
|
||||||
"composite": None,
|
"""Build the control plus three bounded overlay families."""
|
||||||
"weight": 0.0,
|
return (
|
||||||
},
|
|
||||||
*tuple(
|
|
||||||
{
|
{
|
||||||
"id": f"{composite}_w{round(weight * 100):02d}",
|
"id": "control_w00",
|
||||||
"label": f"{composite.title()} overlay {round(weight * 100)}%",
|
"label": "Production 80/20 momentum-volatility rank",
|
||||||
"composite": composite,
|
"composite": None,
|
||||||
"weight": weight,
|
"weight": 0.0,
|
||||||
}
|
},
|
||||||
for composite in COMPOSITES
|
*tuple(
|
||||||
for weight in WEIGHTS
|
{
|
||||||
),
|
"id": f"{composite}_w{round(weight * 100):02d}",
|
||||||
)
|
"label": f"{composite.title()} overlay {round(weight * 100)}%",
|
||||||
|
"composite": composite,
|
||||||
|
"weight": weight,
|
||||||
|
}
|
||||||
|
for composite in COMPOSITES
|
||||||
|
for weight in weights
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
ARMS = _arm_matrix(WEIGHTS)
|
||||||
N_TRIALS = len(ARMS)
|
N_TRIALS = len(ARMS)
|
||||||
|
SPLIT_SAFE_ARMS = _arm_matrix(SPLIT_SAFE_WEIGHTS)
|
||||||
|
SPLIT_SAFE_N_TRIALS = len(SPLIT_SAFE_ARMS)
|
||||||
|
|
||||||
|
|
||||||
def _parse_args() -> argparse.Namespace:
|
def _parse_args() -> argparse.Namespace:
|
||||||
@@ -74,6 +87,11 @@ def _parse_args() -> argparse.Namespace:
|
|||||||
)
|
)
|
||||||
parser.add_argument("--train-end", default="2024-01-01")
|
parser.add_argument("--train-end", default="2024-01-01")
|
||||||
parser.add_argument("--test-start", default="2025-01-01")
|
parser.add_argument("--test-start", default="2025-01-01")
|
||||||
|
parser.add_argument(
|
||||||
|
"--protocol",
|
||||||
|
choices=(ORIGINAL_PROTOCOL, SPLIT_SAFE_PROTOCOL),
|
||||||
|
default=ORIGINAL_PROTOCOL,
|
||||||
|
)
|
||||||
parser.add_argument("--allow-spawn", action="store_true")
|
parser.add_argument("--allow-spawn", action="store_true")
|
||||||
parser.add_argument("--quiet", action="store_true")
|
parser.add_argument("--quiet", action="store_true")
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
@@ -83,9 +101,14 @@ def _sqlite_url(path: Path) -> str:
|
|||||||
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
||||||
|
|
||||||
|
|
||||||
def _default_out() -> Path:
|
def _default_out(protocol: str = ORIGINAL_PROTOCOL) -> Path:
|
||||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||||
return Path("reports") / f"fundamentals-overlay-{stamp}.json"
|
label = (
|
||||||
|
"fundamentals-splitsafe"
|
||||||
|
if protocol == SPLIT_SAFE_PROTOCOL
|
||||||
|
else "fundamentals-overlay"
|
||||||
|
)
|
||||||
|
return Path("reports") / f"{label}-{stamp}.json"
|
||||||
|
|
||||||
|
|
||||||
def _snapshot_hash(path: Path) -> str:
|
def _snapshot_hash(path: Path) -> str:
|
||||||
@@ -394,11 +417,15 @@ def _build_scores(
|
|||||||
dates: set[date],
|
dates: set[date],
|
||||||
representatives: dict[str, str],
|
representatives: dict[str, str],
|
||||||
snapshots_by_cik: dict[str, list[Any]],
|
snapshots_by_cik: dict[str, list[Any]],
|
||||||
|
split_safe: bool,
|
||||||
args: argparse.Namespace,
|
args: argparse.Namespace,
|
||||||
) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]:
|
) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]:
|
||||||
from app.services import fundamentals_derivation as derivation
|
from app.services import fundamentals_derivation as derivation
|
||||||
from app.services import fundamentals_research as research
|
from app.services import fundamentals_research as research
|
||||||
|
|
||||||
|
factor_polarity = (
|
||||||
|
research.SPLIT_SAFE_FACTOR_POLARITY if split_safe else research.FACTOR_POLARITY
|
||||||
|
)
|
||||||
ordered_dates = sorted(dates)
|
ordered_dates = sorted(dates)
|
||||||
date_fingerprint = hashlib.sha256(
|
date_fingerprint = hashlib.sha256(
|
||||||
"|".join(value.isoformat() for value in ordered_dates).encode()
|
"|".join(value.isoformat() for value in ordered_dates).encode()
|
||||||
@@ -408,6 +435,7 @@ def _build_scores(
|
|||||||
snapshot,
|
snapshot,
|
||||||
{
|
{
|
||||||
"kind": "point-in-time-scores",
|
"kind": "point-in-time-scores",
|
||||||
|
"score_profile": SPLIT_SAFE_PROTOCOL if split_safe else ORIGINAL_PROTOCOL,
|
||||||
"date_fingerprint": date_fingerprint,
|
"date_fingerprint": date_fingerprint,
|
||||||
"availability": "accepted before signal-date midnight America/New_York",
|
"availability": "accepted before signal-date midnight America/New_York",
|
||||||
},
|
},
|
||||||
@@ -446,12 +474,15 @@ def _build_scores(
|
|||||||
current_features[cik] = research.raw_features(
|
current_features[cik] = research.raw_features(
|
||||||
derivation.derive(visible[cik])
|
derivation.derive(visible[cik])
|
||||||
)
|
)
|
||||||
scores = research.cross_section_scores(current_features)
|
scores = research.cross_section_scores(
|
||||||
|
current_features,
|
||||||
|
split_safe=split_safe,
|
||||||
|
)
|
||||||
scores_by_date[signal_date.isoformat()] = scores
|
scores_by_date[signal_date.isoformat()] = scores
|
||||||
coverage_rows.append(
|
coverage_rows.append(
|
||||||
{
|
{
|
||||||
key: sum(row.get(key) is not None for row in scores.values())
|
key: sum(row.get(key) is not None for row in scores.values())
|
||||||
for key in (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
|
for key in (*factor_polarity, *research.COMPOSITE_KEYS)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if not args.quiet and date_index % 100 == 0:
|
if not args.quiet and date_index % 100 == 0:
|
||||||
@@ -469,13 +500,13 @@ def _factor_diagnostics(
|
|||||||
representatives: dict[str, str],
|
representatives: dict[str, str],
|
||||||
prices: dict[str, tuple],
|
prices: dict[str, tuple],
|
||||||
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
||||||
|
factor_keys: tuple[str, ...],
|
||||||
train_end: date,
|
train_end: date,
|
||||||
test_start: date,
|
test_start: date,
|
||||||
) -> dict[str, list[dict]]:
|
) -> dict[str, list[dict]]:
|
||||||
from app.services import backtest_service as bt
|
from app.services import backtest_service as bt
|
||||||
from app.services import fundamentals_research as research
|
|
||||||
|
|
||||||
signal_keys = (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
|
signal_keys = (*factor_keys, *COMPOSITES)
|
||||||
observations: list[dict[str, Any]] = []
|
observations: list[dict[str, Any]] = []
|
||||||
for cik, symbol in representatives.items():
|
for cik, symbol in representatives.items():
|
||||||
columns = prices[symbol]
|
columns = prices[symbol]
|
||||||
@@ -533,6 +564,7 @@ def _attach_overlay_ranks(
|
|||||||
candidates: list[dict],
|
candidates: list[dict],
|
||||||
ticker_rows: list[dict],
|
ticker_rows: list[dict],
|
||||||
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
||||||
|
arms: tuple[dict[str, Any], ...],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
from app.services import backtest_service as bt
|
from app.services import backtest_service as bt
|
||||||
from app.services import fundamentals_research as research
|
from app.services import fundamentals_research as research
|
||||||
@@ -550,7 +582,7 @@ def _attach_overlay_ranks(
|
|||||||
if value is not None:
|
if value is not None:
|
||||||
covered[composite] += 1
|
covered[composite] += 1
|
||||||
base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
|
base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
|
||||||
for arm in ARMS:
|
for arm in arms:
|
||||||
if arm["composite"] is None:
|
if arm["composite"] is None:
|
||||||
continue
|
continue
|
||||||
candidate[_ranking_key(arm)] = research.overlay_rank(
|
candidate[_ranking_key(arm)] = research.overlay_rank(
|
||||||
@@ -626,6 +658,7 @@ def _run_arm(
|
|||||||
data: dict[str, Any],
|
data: dict[str, Any],
|
||||||
train_end: date,
|
train_end: date,
|
||||||
test_start: date,
|
test_start: date,
|
||||||
|
n_trials: int,
|
||||||
) -> tuple[dict[str, Any], dict[str, set[str]]]:
|
) -> tuple[dict[str, Any], dict[str, set[str]]]:
|
||||||
from app.services import backtest_service as bt
|
from app.services import backtest_service as bt
|
||||||
from scripts import run_research_matrix as shared
|
from scripts import run_research_matrix as shared
|
||||||
@@ -691,7 +724,7 @@ def _run_arm(
|
|||||||
dsr = bt.deflated_sharpe_ratio(
|
dsr = bt.deflated_sharpe_ratio(
|
||||||
sim.get("sharpe"),
|
sim.get("sharpe"),
|
||||||
sim.get("sharpe_se"),
|
sim.get("sharpe_se"),
|
||||||
N_TRIALS,
|
n_trials,
|
||||||
n_returns=sim.get("n_returns"),
|
n_returns=sim.get("n_returns"),
|
||||||
return_skew=sim.get("return_skew"),
|
return_skew=sim.get("return_skew"),
|
||||||
return_kurtosis=sim.get("return_kurtosis"),
|
return_kurtosis=sim.get("return_kurtosis"),
|
||||||
@@ -781,13 +814,20 @@ def _fmt(value: Any) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _markdown(report: dict[str, Any]) -> str:
|
def _markdown(report: dict[str, Any]) -> str:
|
||||||
|
protocol_id = report.get("research_protocol", ORIGINAL_PROTOCOL)
|
||||||
|
title = (
|
||||||
|
"Split-safe fundamentals overlay sensitivity"
|
||||||
|
if protocol_id == SPLIT_SAFE_PROTOCOL
|
||||||
|
else "Point-in-time fundamentals overlay research"
|
||||||
|
)
|
||||||
lines = [
|
lines = [
|
||||||
"# Point-in-time fundamentals overlay research",
|
f"# {title}",
|
||||||
"",
|
"",
|
||||||
"Generated: {}".format(report.get("generated_at")),
|
"Generated: {}".format(report.get("generated_at")),
|
||||||
"",
|
"",
|
||||||
"## Protocol",
|
"## Protocol",
|
||||||
"",
|
"",
|
||||||
|
"- Research protocol: **{}**.".format(protocol_id),
|
||||||
"- Train ends before **{}**.".format(report["splits"]["train_end"]),
|
"- Train ends before **{}**.".format(report["splits"]["train_end"]),
|
||||||
"- Validation runs until **{}**.".format(report["splits"]["test_start"]),
|
"- Validation runs until **{}**.".format(report["splits"]["test_start"]),
|
||||||
"- Test starts at that date and is not used to select the arm.",
|
"- Test starts at that date and is not used to select the arm.",
|
||||||
@@ -976,9 +1016,21 @@ def _write_outputs(report: dict[str, Any], out: Path, *, bundle: bool) -> None:
|
|||||||
|
|
||||||
|
|
||||||
async def _main() -> None:
|
async def _main() -> None:
|
||||||
|
from app.services import fundamentals_research as research
|
||||||
|
|
||||||
args = _parse_args()
|
args = _parse_args()
|
||||||
|
split_safe = args.protocol == SPLIT_SAFE_PROTOCOL
|
||||||
|
arms = SPLIT_SAFE_ARMS if split_safe else ARMS
|
||||||
|
n_trials = len(arms)
|
||||||
|
factor_keys = tuple(
|
||||||
|
(
|
||||||
|
research.SPLIT_SAFE_FACTOR_POLARITY
|
||||||
|
if split_safe
|
||||||
|
else research.FACTOR_POLARITY
|
||||||
|
).keys()
|
||||||
|
)
|
||||||
snapshot = Path(args.snapshot)
|
snapshot = Path(args.snapshot)
|
||||||
out = Path(args.out) if args.out else _default_out()
|
out = Path(args.out) if args.out else _default_out(args.protocol)
|
||||||
if not snapshot.exists():
|
if not snapshot.exists():
|
||||||
raise SystemExit(f"snapshot not found: {snapshot}")
|
raise SystemExit(f"snapshot not found: {snapshot}")
|
||||||
if args.workers < 1:
|
if args.workers < 1:
|
||||||
@@ -1010,21 +1062,45 @@ async def _main() -> None:
|
|||||||
all_dates,
|
all_dates,
|
||||||
representatives,
|
representatives,
|
||||||
data["snapshots_by_cik"],
|
data["snapshots_by_cik"],
|
||||||
|
split_safe,
|
||||||
args,
|
args,
|
||||||
)
|
)
|
||||||
candidate_coverage = _attach_overlay_ranks(
|
candidate_coverage = _attach_overlay_ranks(
|
||||||
candidates, data["ticker_rows"], scores_by_date
|
candidates, data["ticker_rows"], scores_by_date, arms
|
||||||
)
|
)
|
||||||
factor_ic = _factor_diagnostics(
|
factor_ic = _factor_diagnostics(
|
||||||
representatives,
|
representatives,
|
||||||
data["prices"],
|
data["prices"],
|
||||||
scores_by_date,
|
scores_by_date,
|
||||||
|
factor_keys,
|
||||||
train_end,
|
train_end,
|
||||||
test_start,
|
test_start,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
warnings = [
|
||||||
|
"Current tracked universe only: historical constituent membership and "
|
||||||
|
"delisted names are unavailable, so absolute results have survivorship bias.",
|
||||||
|
"Earnings surprise is excluded because the completed SUE study already "
|
||||||
|
"failed its promotion bar for this strategy.",
|
||||||
|
"The test window has already been observed; this follow-up is sensitivity "
|
||||||
|
"evidence and live paper performance remains the final out-of-sample check.",
|
||||||
|
]
|
||||||
|
if split_safe:
|
||||||
|
warnings.insert(
|
||||||
|
1,
|
||||||
|
"Diluted-EPS growth and share-count change are excluded because filing-time "
|
||||||
|
"values are not split-comparable without point-in-time split factors.",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
warnings.insert(
|
||||||
|
1,
|
||||||
|
"Historical valuation is excluded because split-adjusted bars cannot be "
|
||||||
|
"safely combined with filing-time EPS and shares without split factors.",
|
||||||
|
)
|
||||||
|
|
||||||
report: dict[str, Any] = {
|
report: dict[str, Any] = {
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"research_protocol": args.protocol,
|
||||||
"git_commit": _git_commit(),
|
"git_commit": _git_commit(),
|
||||||
"snapshot": str(snapshot.resolve()),
|
"snapshot": str(snapshot.resolve()),
|
||||||
"snapshot_sha256": _snapshot_hash(snapshot),
|
"snapshot_sha256": _snapshot_hash(snapshot),
|
||||||
@@ -1039,9 +1115,10 @@ async def _main() -> None:
|
|||||||
"test": f"entry date >= {test_start.isoformat()}",
|
"test": f"entry date >= {test_start.isoformat()}",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"n_trials": N_TRIALS,
|
"n_trials": n_trials,
|
||||||
"pre_registered_arms": list(ARMS),
|
"pre_registered_arms": list(arms),
|
||||||
"protocol": {
|
"protocol": {
|
||||||
|
"id": args.protocol,
|
||||||
"qualification": "unchanged production gate; rank overlay only",
|
"qualification": "unchanged production gate; rank overlay only",
|
||||||
"cadence": "daily",
|
"cadence": "daily",
|
||||||
"fill_mode": "close; production near-close proxy",
|
"fill_mode": "close; production near-close proxy",
|
||||||
@@ -1051,22 +1128,17 @@ async def _main() -> None:
|
|||||||
"conservative match for the daily pre-market SEC import"
|
"conservative match for the daily pre-market SEC import"
|
||||||
),
|
),
|
||||||
"missing_fundamental_score": 50.0,
|
"missing_fundamental_score": 50.0,
|
||||||
|
"factor_keys": list(factor_keys),
|
||||||
|
"split_sensitive_metrics_excluded": (
|
||||||
|
["eps_growth_yoy", "share_count_change_yoy"] if split_safe else []
|
||||||
|
),
|
||||||
"selection": (
|
"selection": (
|
||||||
"highest validation Sharpe among arms with train and validation "
|
"highest validation Sharpe among arms with train and validation "
|
||||||
"Sharpe not below control and validation drawdown within 2pp"
|
"Sharpe not below control and validation drawdown within 2pp"
|
||||||
),
|
),
|
||||||
"production_mutation": False,
|
"production_mutation": False,
|
||||||
},
|
},
|
||||||
"warnings": [
|
"warnings": warnings,
|
||||||
"Current tracked universe only: historical constituent membership and "
|
|
||||||
"delisted names are unavailable, so absolute results have survivorship bias.",
|
|
||||||
"Historical valuation is excluded because split-adjusted bars cannot be "
|
|
||||||
"safely combined with filing-time EPS and shares without split factors.",
|
|
||||||
"Earnings surprise is excluded because the completed SUE study already "
|
|
||||||
"failed its promotion bar for this strategy.",
|
|
||||||
"The test window remains research evidence, not a pristine future sample; "
|
|
||||||
"live paper performance is still the final out-of-sample check.",
|
|
||||||
],
|
|
||||||
"data_audit": audit,
|
"data_audit": audit,
|
||||||
"strategy_config": {
|
"strategy_config": {
|
||||||
"recommendation": data["config"],
|
"recommendation": data["config"],
|
||||||
@@ -1087,10 +1159,12 @@ async def _main() -> None:
|
|||||||
|
|
||||||
trade_sets: dict[str, dict[str, set[str]]] = {}
|
trade_sets: dict[str, dict[str, set[str]]] = {}
|
||||||
control: dict[str, Any] | None = None
|
control: dict[str, Any] | None = None
|
||||||
for arm in ARMS:
|
for arm in arms:
|
||||||
if not args.quiet:
|
if not args.quiet:
|
||||||
print("running {}".format(arm["id"]), flush=True)
|
print("running {}".format(arm["id"]), flush=True)
|
||||||
result, arm_trades = _run_arm(arm, candidates, data, train_end, test_start)
|
result, arm_trades = _run_arm(
|
||||||
|
arm, candidates, data, train_end, test_start, n_trials
|
||||||
|
)
|
||||||
trade_sets[str(arm["id"])] = arm_trades
|
trade_sets[str(arm["id"])] = arm_trades
|
||||||
if control is None:
|
if control is None:
|
||||||
control = result
|
control = result
|
||||||
|
|||||||
@@ -51,6 +51,45 @@ def test_composites_use_equal_subgroup_weighting():
|
|||||||
assert scores["3"]["balanced"] == 50.0
|
assert scores["3"]["balanced"] == 50.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_safe_composites_ignore_eps_and_share_count():
|
||||||
|
def features(unsafe_multiplier: int):
|
||||||
|
return {
|
||||||
|
str(index): {
|
||||||
|
"operating_margin": index,
|
||||||
|
"fcf_margin": index,
|
||||||
|
"net_debt_to_ebitda": 6 - index,
|
||||||
|
"revenue_growth_yoy": index,
|
||||||
|
"eps_growth_yoy": unsafe_multiplier * (6 - index),
|
||||||
|
"share_count_change_yoy": unsafe_multiplier * index,
|
||||||
|
}
|
||||||
|
for index in range(1, 6)
|
||||||
|
}
|
||||||
|
|
||||||
|
baseline = research.cross_section_scores(features(1), split_safe=True)
|
||||||
|
distorted = research.cross_section_scores(features(1_000_000), split_safe=True)
|
||||||
|
|
||||||
|
assert distorted == baseline
|
||||||
|
assert baseline["5"]["quality"] == 100.0
|
||||||
|
assert baseline["5"]["growth"] == 100.0
|
||||||
|
assert baseline["5"]["balanced"] == 100.0
|
||||||
|
assert "eps_growth_yoy" not in baseline["5"]
|
||||||
|
assert "share_count_change_yoy" not in baseline["5"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_safe_quality_still_requires_two_comparable_inputs():
|
||||||
|
features = {
|
||||||
|
str(index): {
|
||||||
|
"operating_margin": index,
|
||||||
|
"revenue_growth_yoy": index,
|
||||||
|
}
|
||||||
|
for index in range(1, 6)
|
||||||
|
}
|
||||||
|
scores = research.cross_section_scores(features, split_safe=True)
|
||||||
|
assert all(row["quality"] is None for row in scores.values())
|
||||||
|
assert scores["5"]["growth"] == 100.0
|
||||||
|
assert scores["5"]["balanced"] is None
|
||||||
|
|
||||||
|
|
||||||
def test_overlay_uses_neutral_missing_score_and_validates_weight():
|
def test_overlay_uses_neutral_missing_score_and_validates_weight():
|
||||||
assert research.overlay_rank(90, None, 0.2) == 82.0
|
assert research.overlay_rank(90, None, 0.2) == 82.0
|
||||||
assert research.overlay_rank(90, 100, 0.2) == 92.0
|
assert research.overlay_rank(90, 100, 0.2) == 92.0
|
||||||
|
|||||||
@@ -29,6 +29,27 @@ def test_arm_matrix_is_bounded_and_pre_registered():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_safe_matrix_is_smaller_and_trial_corrected():
|
||||||
|
assert runner.SPLIT_SAFE_N_TRIALS == 10
|
||||||
|
assert runner.SPLIT_SAFE_ARMS[0]["id"] == "control_w00"
|
||||||
|
assert {arm["weight"] for arm in runner.SPLIT_SAFE_ARMS[1:]} == {
|
||||||
|
0.05,
|
||||||
|
0.10,
|
||||||
|
0.15,
|
||||||
|
}
|
||||||
|
assert {arm["composite"] for arm in runner.SPLIT_SAFE_ARMS[1:]} == {
|
||||||
|
"quality",
|
||||||
|
"growth",
|
||||||
|
"balanced",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_safe_output_has_a_distinct_name():
|
||||||
|
assert runner._default_out(runner.SPLIT_SAFE_PROTOCOL).name.startswith(
|
||||||
|
"fundamentals-splitsafe-"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_development_grade_does_not_read_test_window():
|
def test_development_grade_does_not_read_test_window():
|
||||||
control = {
|
control = {
|
||||||
"windows": [
|
"windows": [
|
||||||
|
|||||||
Reference in New Issue
Block a user