1141 lines
41 KiB
Python
1141 lines
41 KiB
Python
"""Point-in-time fundamentals rank-overlay research.
|
|
|
|
The production qualification gate is unchanged. The runner first measures
|
|
30-session factor IC, then reorders already-qualified candidates with quality,
|
|
growth, or balanced fundamental ranks at 10, 20, 30, and 40 percent weights.
|
|
It writes JSON, Markdown, CSV, and a portable ZIP bundle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import pickle
|
|
import subprocess
|
|
import sys
|
|
import zipfile
|
|
from collections import defaultdict
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
from datetime import date, datetime, time, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
CACHE_VERSION = "fundamentals-overlay-v1"
|
|
NY = ZoneInfo("America/New_York")
|
|
COMPOSITES = ("quality", "growth", "balanced")
|
|
WEIGHTS = (0.10, 0.20, 0.30, 0.40)
|
|
ARMS: tuple[dict[str, Any], ...] = (
|
|
{
|
|
"id": "control_w00",
|
|
"label": "Production 80/20 momentum-volatility rank",
|
|
"composite": None,
|
|
"weight": 0.0,
|
|
},
|
|
*tuple(
|
|
{
|
|
"id": f"{composite}_w{round(weight * 100):02d}",
|
|
"label": f"{composite.title()} overlay {round(weight * 100)}%",
|
|
"composite": composite,
|
|
"weight": weight,
|
|
}
|
|
for composite in COMPOSITES
|
|
for weight in WEIGHTS
|
|
),
|
|
)
|
|
N_TRIALS = len(ARMS)
|
|
|
|
|
|
def _parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("snapshot")
|
|
parser.add_argument(
|
|
"--workers", type=int, default=max(1, multiprocessing.cpu_count() - 1)
|
|
)
|
|
parser.add_argument("--out", default=None)
|
|
parser.add_argument(
|
|
"--candidate-cache", default="reports/.cache/fundamentals-candidates.pkl"
|
|
)
|
|
parser.add_argument(
|
|
"--fundamentals-cache", default="reports/.cache/fundamentals-scores.pkl"
|
|
)
|
|
parser.add_argument("--train-end", default="2024-01-01")
|
|
parser.add_argument("--test-start", default="2025-01-01")
|
|
parser.add_argument("--allow-spawn", action="store_true")
|
|
parser.add_argument("--quiet", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def _sqlite_url(path: Path) -> str:
|
|
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
|
|
|
|
|
def _default_out() -> Path:
|
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
return Path("reports") / f"fundamentals-overlay-{stamp}.json"
|
|
|
|
|
|
def _snapshot_hash(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _cache_key(snapshot: Path, suffix: dict[str, Any]) -> dict[str, Any]:
|
|
stat = snapshot.stat()
|
|
return {
|
|
"version": CACHE_VERSION,
|
|
"snapshot": str(snapshot.resolve()),
|
|
"size": stat.st_size,
|
|
"mtime_ns": stat.st_mtime_ns,
|
|
**suffix,
|
|
}
|
|
|
|
|
|
def _load_cache(path: Path, key: dict[str, Any]) -> Any | None:
|
|
if not path.exists():
|
|
return None
|
|
with path.open("rb") as handle:
|
|
payload = pickle.load(handle) # noqa: S301 - trusted local cache
|
|
return payload.get("value") if payload.get("key") == key else None
|
|
|
|
|
|
def _save_cache(path: Path, key: dict[str, Any], value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("wb") as handle:
|
|
pickle.dump(
|
|
{"key": key, "value": value}, handle, protocol=pickle.HIGHEST_PROTOCOL
|
|
)
|
|
|
|
|
|
def _git_commit() -> str | None:
|
|
try:
|
|
return subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
check=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return None
|
|
|
|
|
|
async def _load_snapshot(snapshot: Path, quiet: bool) -> dict[str, Any]:
|
|
from app.models.earnings_event import EarningsEvent
|
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
|
from app.models.ohlcv import OHLCVRecord
|
|
from app.models.ticker import Ticker
|
|
from app.services import backtest_service as bt
|
|
from app.services.admin_service import get_activation_config
|
|
from app.services.paper_trade_service import get_exit_policy
|
|
from app.services.recommendation_service import get_recommendation_config
|
|
|
|
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
|
session_factory = async_sessionmaker(
|
|
engine, class_=AsyncSession, expire_on_commit=False
|
|
)
|
|
try:
|
|
async with session_factory() as db:
|
|
config = await get_recommendation_config(db)
|
|
activation = await get_activation_config(db)
|
|
exit_config = await get_exit_policy(db)
|
|
benchmark = await bt._load_benchmark_closes_for_backtest(
|
|
db, days=None, refresh=False
|
|
)
|
|
tickers = list(
|
|
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
|
)
|
|
snapshots = list(
|
|
(
|
|
await db.execute(
|
|
select(FundamentalSnapshot).order_by(
|
|
FundamentalSnapshot.cik,
|
|
FundamentalSnapshot.accepted_at,
|
|
)
|
|
)
|
|
).scalars()
|
|
)
|
|
earnings_count = int(
|
|
(
|
|
await db.execute(select(func.count()).select_from(EarningsEvent))
|
|
).scalar_one()
|
|
)
|
|
price_bounds = (
|
|
await db.execute(
|
|
select(
|
|
func.min(OHLCVRecord.date),
|
|
func.max(OHLCVRecord.date),
|
|
func.count(),
|
|
)
|
|
)
|
|
).one()
|
|
prices: dict[str, tuple] = {}
|
|
for index, ticker in enumerate(tickers, 1):
|
|
columns = await bt._fetch_columns(db, ticker.symbol)
|
|
if columns is not None:
|
|
prices[ticker.symbol] = columns
|
|
if not quiet and index % 50 == 0:
|
|
print(f"loaded prices {index}/{len(tickers)}", flush=True)
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
by_cik: dict[str, list[Any]] = defaultdict(list)
|
|
for row in snapshots:
|
|
by_cik[str(row.cik)].append(row)
|
|
ticker_rows = [
|
|
{
|
|
"symbol": ticker.symbol,
|
|
"cik": str(ticker.cik) if ticker.cik else None,
|
|
"sic": ticker.sic,
|
|
}
|
|
for ticker in tickers
|
|
]
|
|
audit = {
|
|
"tickers": len(tickers),
|
|
"tickers_with_prices": len(prices),
|
|
"tickers_with_cik": sum(row["cik"] is not None for row in ticker_rows),
|
|
"unique_ciks": len({row["cik"] for row in ticker_rows if row["cik"]}),
|
|
"fundamental_rows": len(snapshots),
|
|
"fundamental_ciks": len(by_cik),
|
|
"accepted_at_min": min((row.accepted_at for row in snapshots), default=None),
|
|
"accepted_at_max": max((row.accepted_at for row in snapshots), default=None),
|
|
"earnings_rows": earnings_count,
|
|
"price_date_min": price_bounds[0],
|
|
"price_date_max": price_bounds[1],
|
|
"price_rows": int(price_bounds[2] or 0),
|
|
}
|
|
return {
|
|
"config": config,
|
|
"activation": activation,
|
|
"exit_config": exit_config,
|
|
"benchmark": benchmark,
|
|
"ticker_rows": ticker_rows,
|
|
"prices": prices,
|
|
"snapshots_by_cik": dict(by_cik),
|
|
"audit": audit,
|
|
}
|
|
|
|
|
|
def _representatives(
|
|
ticker_rows: list[dict], prices: dict[str, tuple]
|
|
) -> dict[str, str]:
|
|
reps: dict[str, str] = {}
|
|
for row in ticker_rows:
|
|
cik = row.get("cik")
|
|
symbol = str(row["symbol"])
|
|
if not cik or symbol not in prices:
|
|
continue
|
|
if cik not in reps or symbol < reps[cik]:
|
|
reps[cik] = symbol
|
|
return reps
|
|
|
|
|
|
def _build_candidates(
|
|
snapshot: Path,
|
|
data: dict[str, Any],
|
|
args: argparse.Namespace,
|
|
) -> tuple[list[dict], int]:
|
|
from app.services import backtest_service as bt
|
|
from scripts import run_research_matrix as shared
|
|
|
|
cache_path = Path(args.candidate_cache)
|
|
key = _cache_key(snapshot, {"kind": "daily-production-candidates"})
|
|
cached = _load_cache(cache_path, key)
|
|
if cached is not None:
|
|
if not args.quiet:
|
|
print(f"loaded candidate cache {cache_path}", flush=True)
|
|
return list(cached["qualified"]), int(cached["entry_candidate_count"])
|
|
|
|
prices = data["prices"]
|
|
workers = max(1, min(args.workers, max(1, multiprocessing.cpu_count() - 1)))
|
|
replay_rows: list[dict] = []
|
|
replay_start = date(1900, 1, 1)
|
|
|
|
def replay_one(symbol: str, columns: tuple) -> list[dict]:
|
|
return bt._replay_candidates_for_period(
|
|
symbol,
|
|
columns,
|
|
data["config"],
|
|
data["activation"],
|
|
data["benchmark"],
|
|
replay_start,
|
|
"daily",
|
|
True,
|
|
True,
|
|
)
|
|
|
|
if workers == 1:
|
|
for index, (symbol, columns) in enumerate(prices.items(), 1):
|
|
replay_rows.extend(replay_one(symbol, columns))
|
|
if not args.quiet and index % 25 == 0:
|
|
print(f"replay {index}/{len(prices)}", flush=True)
|
|
else:
|
|
context = bt._mp_context() or multiprocessing.get_context("spawn")
|
|
with ProcessPoolExecutor(max_workers=workers, mp_context=context) as pool:
|
|
futures = {
|
|
pool.submit(
|
|
bt._replay_candidates_for_period,
|
|
symbol,
|
|
columns,
|
|
data["config"],
|
|
data["activation"],
|
|
data["benchmark"],
|
|
replay_start,
|
|
"daily",
|
|
True,
|
|
True,
|
|
): symbol
|
|
for symbol, columns in prices.items()
|
|
}
|
|
for index, future in enumerate(as_completed(futures), 1):
|
|
replay_rows.extend(future.result())
|
|
if not args.quiet and index % 25 == 0:
|
|
print(f"replay {index}/{len(futures)}", flush=True)
|
|
|
|
setups = [row for row in replay_rows if not row.get("_rank_only")]
|
|
observations = [row for row in replay_rows if row.get("_universe_rank_observation")]
|
|
ranks = shared._live_universe_rank_map(
|
|
observations,
|
|
data["benchmark"],
|
|
bt.STRATEGY_RANK_MOMENTUM_WEIGHT,
|
|
)
|
|
cutoff = float(data["activation"].get("min_momentum_percentile", 80.0))
|
|
qualified: list[dict] = []
|
|
for setup in setups:
|
|
if setup.get("direction") != "long":
|
|
continue
|
|
identity = (str(setup["symbol"]), str(setup["date"]))
|
|
rank = ranks.get(identity)
|
|
if rank is None:
|
|
continue
|
|
candidate = {
|
|
key: value
|
|
for key, value in setup.items()
|
|
if not key.startswith("_universe_")
|
|
}
|
|
candidate[bt.PRODUCTION_PERCENTILE_KEY] = rank["momentum_percentile"]
|
|
candidate[bt.VOL_PERCENTILE_KEY] = rank["volatility_percentile"]
|
|
candidate[bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY] = rank["strategy_rank"]
|
|
candidate["qualified"] = bt._momentum_qualifies(candidate, cutoff)
|
|
if candidate["qualified"]:
|
|
qualified.append(candidate)
|
|
|
|
if not qualified:
|
|
raise RuntimeError("no qualified long candidates after replay")
|
|
value = {"qualified": qualified, "entry_candidate_count": len(setups)}
|
|
_save_cache(cache_path, key, value)
|
|
if not args.quiet:
|
|
print(f"wrote candidate cache {cache_path}", flush=True)
|
|
return qualified, len(setups)
|
|
|
|
|
|
def _weekly_factor_dates(
|
|
representatives: dict[str, str], prices: dict[str, tuple]
|
|
) -> set[date]:
|
|
from app.services import backtest_service as bt
|
|
|
|
dates: set[date] = set()
|
|
for symbol in representatives.values():
|
|
columns = prices[symbol]
|
|
records = [
|
|
SimpleNamespace(date=date.fromordinal(int(value))) for value in columns[0]
|
|
]
|
|
for index in bt._weekly_asof_indices(records):
|
|
if index >= bt.MIN_LOOKBACK - 1 and index + bt.HORIZON < len(records):
|
|
dates.add(records[index].date)
|
|
return dates
|
|
|
|
|
|
def _utc(value: datetime) -> datetime:
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
|
|
|
|
def _coverage_summary(rows: list[dict[str, int]]) -> dict[str, Any]:
|
|
if not rows:
|
|
return {}
|
|
keys = sorted(rows[0])
|
|
result: dict[str, Any] = {"dates": len(rows)}
|
|
for key in keys:
|
|
values = sorted(row[key] for row in rows)
|
|
middle = len(values) // 2
|
|
median = (
|
|
values[middle]
|
|
if len(values) % 2
|
|
else (values[middle - 1] + values[middle]) / 2
|
|
)
|
|
result[key] = {
|
|
"min": values[0],
|
|
"median": median,
|
|
"max": values[-1],
|
|
}
|
|
return result
|
|
|
|
|
|
def _build_scores(
|
|
snapshot: Path,
|
|
dates: set[date],
|
|
representatives: dict[str, str],
|
|
snapshots_by_cik: dict[str, list[Any]],
|
|
args: argparse.Namespace,
|
|
) -> tuple[dict[str, dict[str, dict[str, float | None]]], dict[str, Any]]:
|
|
from app.services import fundamentals_derivation as derivation
|
|
from app.services import fundamentals_research as research
|
|
|
|
ordered_dates = sorted(dates)
|
|
date_fingerprint = hashlib.sha256(
|
|
"|".join(value.isoformat() for value in ordered_dates).encode()
|
|
).hexdigest()
|
|
cache_path = Path(args.fundamentals_cache)
|
|
key = _cache_key(
|
|
snapshot,
|
|
{
|
|
"kind": "point-in-time-scores",
|
|
"date_fingerprint": date_fingerprint,
|
|
"availability": "accepted before signal-date midnight America/New_York",
|
|
},
|
|
)
|
|
cached = _load_cache(cache_path, key)
|
|
if cached is not None:
|
|
if not args.quiet:
|
|
print(f"loaded fundamentals cache {cache_path}", flush=True)
|
|
return cached["scores"], cached["coverage"]
|
|
|
|
eligible_ciks = sorted(set(representatives) & set(snapshots_by_cik))
|
|
rows_by_cik = {
|
|
cik: sorted(snapshots_by_cik[cik], key=lambda row: _utc(row.accepted_at))
|
|
for cik in eligible_ciks
|
|
}
|
|
positions = {cik: 0 for cik in eligible_ciks}
|
|
visible = {cik: [] for cik in eligible_ciks}
|
|
current_features: dict[str, dict[str, float | None]] = {}
|
|
scores_by_date: dict[str, dict[str, dict[str, float | None]]] = {}
|
|
coverage_rows: list[dict[str, int]] = []
|
|
|
|
for date_index, signal_date in enumerate(ordered_dates, 1):
|
|
cutoff = datetime.combine(signal_date, time.min, tzinfo=NY).astimezone(
|
|
timezone.utc
|
|
)
|
|
for cik in eligible_ciks:
|
|
rows = rows_by_cik[cik]
|
|
position = positions[cik]
|
|
changed = False
|
|
while position < len(rows) and _utc(rows[position].accepted_at) <= cutoff:
|
|
visible[cik].append(rows[position])
|
|
position += 1
|
|
changed = True
|
|
positions[cik] = position
|
|
if changed:
|
|
current_features[cik] = research.raw_features(
|
|
derivation.derive(visible[cik])
|
|
)
|
|
scores = research.cross_section_scores(current_features)
|
|
scores_by_date[signal_date.isoformat()] = scores
|
|
coverage_rows.append(
|
|
{
|
|
key: sum(row.get(key) is not None for row in scores.values())
|
|
for key in (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
|
|
}
|
|
)
|
|
if not args.quiet and date_index % 100 == 0:
|
|
print(f"fundamentals dates {date_index}/{len(ordered_dates)}", flush=True)
|
|
|
|
coverage = _coverage_summary(coverage_rows)
|
|
value = {"scores": scores_by_date, "coverage": coverage}
|
|
_save_cache(cache_path, key, value)
|
|
if not args.quiet:
|
|
print(f"wrote fundamentals cache {cache_path}", flush=True)
|
|
return scores_by_date, coverage
|
|
|
|
|
|
def _factor_diagnostics(
|
|
representatives: dict[str, str],
|
|
prices: dict[str, tuple],
|
|
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
|
train_end: date,
|
|
test_start: date,
|
|
) -> dict[str, list[dict]]:
|
|
from app.services import backtest_service as bt
|
|
from app.services import fundamentals_research as research
|
|
|
|
signal_keys = (*research.FACTOR_POLARITY, *research.COMPOSITE_KEYS)
|
|
observations: list[dict[str, Any]] = []
|
|
for cik, symbol in representatives.items():
|
|
columns = prices[symbol]
|
|
ordinals, _opens, _highs, _lows, closes, _volumes = columns
|
|
records = [
|
|
SimpleNamespace(date=date.fromordinal(int(value))) for value in ordinals
|
|
]
|
|
for index in bt._weekly_asof_indices(records):
|
|
forward_index = index + bt.HORIZON
|
|
if index < bt.MIN_LOOKBACK - 1 or forward_index >= len(records):
|
|
continue
|
|
if closes[index] <= 0:
|
|
continue
|
|
signal_date = records[index].date
|
|
score = scores_by_date.get(signal_date.isoformat(), {}).get(cik, {})
|
|
forward = closes[forward_index] / closes[index] - 1.0
|
|
iso = signal_date.isocalendar()
|
|
for key in signal_keys:
|
|
value = score.get(key)
|
|
if value is not None:
|
|
observations.append(
|
|
{
|
|
"signal": key,
|
|
"date": signal_date,
|
|
"week": (iso.year, iso.week),
|
|
"value": float(value),
|
|
"forward": float(forward),
|
|
"symbol": symbol,
|
|
}
|
|
)
|
|
|
|
windows = {
|
|
"train": lambda value: value < train_end,
|
|
"validation": lambda value: train_end <= value < test_start,
|
|
"test": lambda value: value >= test_start,
|
|
"full": lambda _value: True,
|
|
}
|
|
result: dict[str, list[dict]] = {}
|
|
for window, predicate in windows.items():
|
|
collected: dict = defaultdict(lambda: defaultdict(list))
|
|
for row in observations:
|
|
if predicate(row["date"]):
|
|
collected[row["signal"]][row["week"]].append(
|
|
{
|
|
"val": row["value"],
|
|
"fwd": row["forward"],
|
|
"symbol": row["symbol"],
|
|
}
|
|
)
|
|
result[window] = bt._signal_evaluation(collected)
|
|
return result
|
|
|
|
|
|
def _attach_overlay_ranks(
|
|
candidates: list[dict],
|
|
ticker_rows: list[dict],
|
|
scores_by_date: dict[str, dict[str, dict[str, float | None]]],
|
|
) -> dict[str, Any]:
|
|
from app.services import backtest_service as bt
|
|
from app.services import fundamentals_research as research
|
|
|
|
symbol_to_cik = {
|
|
str(row["symbol"]): row.get("cik") for row in ticker_rows if row.get("cik")
|
|
}
|
|
covered = {composite: 0 for composite in COMPOSITES}
|
|
for candidate in candidates:
|
|
cik = symbol_to_cik.get(str(candidate["symbol"]))
|
|
score = scores_by_date.get(str(candidate["date"]), {}).get(cik, {})
|
|
for composite in COMPOSITES:
|
|
value = score.get(composite)
|
|
candidate[f"fund_{composite}"] = value
|
|
if value is not None:
|
|
covered[composite] += 1
|
|
base = candidate.get(bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
|
|
for arm in ARMS:
|
|
if arm["composite"] is None:
|
|
continue
|
|
candidate[_ranking_key(arm)] = research.overlay_rank(
|
|
base,
|
|
score.get(str(arm["composite"])),
|
|
float(arm["weight"]),
|
|
)
|
|
total = len(candidates)
|
|
return {
|
|
composite: {
|
|
"candidates": count,
|
|
"pct": round(count / total * 100.0, 2) if total else 0.0,
|
|
}
|
|
for composite, count in covered.items()
|
|
}
|
|
|
|
|
|
def _ranking_key(arm: dict[str, Any]) -> str:
|
|
from app.services import backtest_service as bt
|
|
|
|
if arm["composite"] is None:
|
|
return bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY
|
|
return "fund_overlay_{}_{:02d}".format(
|
|
arm["composite"], round(float(arm["weight"]) * 100)
|
|
)
|
|
|
|
|
|
def _window(arm: dict[str, Any], name: str) -> dict[str, Any] | None:
|
|
return next(
|
|
(row for row in arm.get("windows", []) if row.get("window") == name),
|
|
None,
|
|
)
|
|
|
|
|
|
def _trade_overlap(subject: set[str], control: set[str]) -> dict[str, Any]:
|
|
union = subject | control
|
|
return {
|
|
"overlap_pct": round(len(subject & control) / len(union) * 100.0, 2)
|
|
if union
|
|
else 100.0,
|
|
"added": len(subject - control),
|
|
"removed": len(control - subject),
|
|
}
|
|
|
|
|
|
def _winner_concentration(details: list[dict[str, Any]]) -> dict[str, Any]:
|
|
pnls = sorted(
|
|
(float(row["pnl"]) for row in details if row.get("pnl") is not None),
|
|
reverse=True,
|
|
)
|
|
rs = sorted(
|
|
(float(row["r"]) for row in details if row.get("r") is not None),
|
|
reverse=True,
|
|
)
|
|
top_pnl = sum(pnls[:5])
|
|
total_pnl = sum(pnls)
|
|
remaining_rs = rs[5:]
|
|
return {
|
|
"top5_pnl": round(top_pnl, 2) if pnls else None,
|
|
"net_pnl_ex_top5": round(total_pnl - top_pnl, 2) if pnls else None,
|
|
"top5_share_of_positive_net_pct": (
|
|
round(top_pnl / total_pnl * 100.0, 2) if total_pnl > 0 else None
|
|
),
|
|
"avg_r_ex_top5": (
|
|
round(sum(remaining_rs) / len(remaining_rs), 4) if remaining_rs else None
|
|
),
|
|
}
|
|
|
|
|
|
def _run_arm(
|
|
arm: dict[str, Any],
|
|
candidates: list[dict],
|
|
data: dict[str, Any],
|
|
train_end: date,
|
|
test_start: date,
|
|
) -> tuple[dict[str, Any], dict[str, set[str]]]:
|
|
from app.services import backtest_service as bt
|
|
from scripts import run_research_matrix as shared
|
|
|
|
strategy = next(
|
|
row for row in bt.PORTFOLIO_MONITOR_STRATEGIES if row.get("is_production")
|
|
)
|
|
entry = bt._entry_variant_config(str(strategy["entry_variant"]))
|
|
if entry is None:
|
|
raise RuntimeError("production entry configuration missing")
|
|
exit_config = data["exit_config"]
|
|
exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get(
|
|
str(exit_config.get("mode", "atr_trailing")), "atr_trail3"
|
|
)
|
|
hold_days = int(exit_config.get("hold_days", 30))
|
|
trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER))
|
|
ranking_key = _ranking_key(arm)
|
|
reentry = bt._make_gate_reset_reentry_fn(
|
|
candidates,
|
|
data["prices"],
|
|
cadence="daily",
|
|
ranking_key=ranking_key,
|
|
)
|
|
windows: list[dict[str, Any]] = []
|
|
trades_by_window: dict[str, set[str]] = {}
|
|
full_trade_details: list[dict[str, Any]] = []
|
|
for name, start, end in (
|
|
("train", None, train_end),
|
|
("validation", train_end, test_start),
|
|
("test", test_start, None),
|
|
("full", None, None),
|
|
):
|
|
sim = bt._simulate_portfolio(
|
|
candidates,
|
|
data["prices"],
|
|
data["benchmark"],
|
|
exit_policy,
|
|
hold_days,
|
|
ranking_key=ranking_key,
|
|
max_positions=int(entry["max_positions"]),
|
|
risk_per_trade=float(entry["risk_per_trade"]),
|
|
atr_trail_multiplier=trail,
|
|
post_stop_reentry_fn=reentry,
|
|
start_date=start,
|
|
end_date=end,
|
|
fill_mode=bt.FILL_MODE_CLOSE,
|
|
include_trades=True,
|
|
)
|
|
if sim is None:
|
|
windows.append({"window": name, "error": "no trades"})
|
|
trades_by_window[name] = set()
|
|
continue
|
|
shared._assert_calendar_truncation(sim, hold_days, bt.FILL_MODE_CLOSE)
|
|
details = sim.pop("trade_details", [])
|
|
sim["winner_concentration"] = _winner_concentration(details)
|
|
if name == "full":
|
|
full_trade_details = details
|
|
trades_by_window[name] = {
|
|
"{}:{}".format(row.get("symbol"), row.get("entry_date")) for row in details
|
|
}
|
|
for heavy in ("equity_curve", "benchmark_curve", "reentry_events"):
|
|
sim.pop(heavy, None)
|
|
dsr = bt.deflated_sharpe_ratio(
|
|
sim.get("sharpe"),
|
|
sim.get("sharpe_se"),
|
|
N_TRIALS,
|
|
n_returns=sim.get("n_returns"),
|
|
return_skew=sim.get("return_skew"),
|
|
return_kurtosis=sim.get("return_kurtosis"),
|
|
)
|
|
windows.append({"window": name, "dsr": dsr, **sim})
|
|
return (
|
|
{
|
|
"id": arm["id"],
|
|
"label": arm["label"],
|
|
"composite": arm["composite"],
|
|
"weight": arm["weight"],
|
|
"ranking_key": ranking_key,
|
|
"windows": windows,
|
|
"full_trade_details": full_trade_details,
|
|
},
|
|
trades_by_window,
|
|
)
|
|
|
|
|
|
def _development_grade(control: dict, arm: dict) -> dict[str, Any]:
|
|
control_train = _window(control, "train") or {}
|
|
control_validation = _window(control, "validation") or {}
|
|
arm_train = _window(arm, "train") or {}
|
|
arm_validation = _window(arm, "validation") or {}
|
|
required = (
|
|
control_train.get("sharpe"),
|
|
control_validation.get("sharpe"),
|
|
control_validation.get("max_drawdown_pct"),
|
|
arm_train.get("sharpe"),
|
|
arm_validation.get("sharpe"),
|
|
arm_validation.get("max_drawdown_pct"),
|
|
)
|
|
if any(value is None for value in required):
|
|
return {"pass": False, "reason": "missing train or validation statistic"}
|
|
checks = {
|
|
"train_sharpe_not_worse": arm_train["sharpe"] >= control_train["sharpe"],
|
|
"validation_sharpe_not_worse": (
|
|
arm_validation["sharpe"] >= control_validation["sharpe"]
|
|
),
|
|
"validation_drawdown_within_2pp": (
|
|
arm_validation["max_drawdown_pct"]
|
|
<= control_validation["max_drawdown_pct"] + 2.0
|
|
),
|
|
}
|
|
return {
|
|
"pass": all(checks.values()),
|
|
"checks": checks,
|
|
"train_sharpe_delta": round(arm_train["sharpe"] - control_train["sharpe"], 4),
|
|
"validation_sharpe_delta": round(
|
|
arm_validation["sharpe"] - control_validation["sharpe"], 4
|
|
),
|
|
}
|
|
|
|
|
|
def _final_check(control: dict, selected: dict | None) -> dict[str, Any] | None:
|
|
if selected is None:
|
|
return None
|
|
control_test = _window(control, "test") or {}
|
|
selected_test = _window(selected, "test") or {}
|
|
values = (
|
|
control_test.get("sharpe"),
|
|
control_test.get("max_drawdown_pct"),
|
|
selected_test.get("sharpe"),
|
|
selected_test.get("max_drawdown_pct"),
|
|
)
|
|
if any(value is None for value in values):
|
|
return {"pass": False, "reason": "missing test statistic"}
|
|
checks = {
|
|
"test_sharpe_not_worse": selected_test["sharpe"] >= control_test["sharpe"],
|
|
"test_drawdown_within_2pp": (
|
|
selected_test["max_drawdown_pct"] <= control_test["max_drawdown_pct"] + 2.0
|
|
),
|
|
}
|
|
return {
|
|
"arm_id": selected["id"],
|
|
"pass": all(checks.values()),
|
|
"checks": checks,
|
|
"test_sharpe_delta": round(selected_test["sharpe"] - control_test["sharpe"], 4),
|
|
"note": "Research evidence only; passing does not change production.",
|
|
}
|
|
|
|
|
|
def _fmt(value: Any) -> str:
|
|
if value is None:
|
|
return "—"
|
|
return f"{value:.4g}" if isinstance(value, float) else str(value)
|
|
|
|
|
|
def _markdown(report: dict[str, Any]) -> str:
|
|
lines = [
|
|
"# Point-in-time fundamentals overlay research",
|
|
"",
|
|
"Generated: {}".format(report.get("generated_at")),
|
|
"",
|
|
"## Protocol",
|
|
"",
|
|
"- Train ends before **{}**.".format(report["splits"]["train_end"]),
|
|
"- Validation runs until **{}**.".format(report["splits"]["test_start"]),
|
|
"- Test starts at that date and is not used to select the arm.",
|
|
"- Qualification is unchanged; fundamentals only reorder qualified longs.",
|
|
"- SEC filings become visible at midnight New York time after acceptance.",
|
|
"- Pre-registered portfolio trials for DSR: **{}**.".format(report["n_trials"]),
|
|
"",
|
|
"## Data warnings",
|
|
"",
|
|
]
|
|
lines.extend(f"- {warning}" for warning in report.get("warnings", []))
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Factor IC",
|
|
"",
|
|
"| window | signal | IC | t | positive | quintile spread | weeks | N |",
|
|
"|---|---|---:|---:|---:|---:|---:|---:|",
|
|
]
|
|
)
|
|
for window, rows in report.get("factor_ic", {}).items():
|
|
for row in rows:
|
|
lines.append(
|
|
"| {} | {} | {} | {} | {} | {} | {} | {} |".format(
|
|
window,
|
|
row.get("signal"),
|
|
_fmt(row.get("mean_ic")),
|
|
_fmt(row.get("ic_t_stat")),
|
|
_fmt(row.get("ic_positive_pct")),
|
|
_fmt(row.get("mean_quintile_spread")),
|
|
row.get("weeks"),
|
|
_fmt(row.get("avg_cross_section")),
|
|
)
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Portfolio arms",
|
|
"",
|
|
"| arm | window | Sharpe | SE | DSR | CAGR | MaxDD | Calmar | trades | overlap |",
|
|
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|",
|
|
]
|
|
)
|
|
for arm in report.get("arms", []):
|
|
for row in arm.get("windows", []):
|
|
overlap = row.get("selection_vs_control", {}).get("overlap_pct")
|
|
lines.append(
|
|
"| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |".format(
|
|
arm["id"],
|
|
row.get("window"),
|
|
_fmt(row.get("sharpe")),
|
|
_fmt(row.get("sharpe_se")),
|
|
_fmt(row.get("dsr")),
|
|
_fmt(row.get("cagr_pct")),
|
|
_fmt(row.get("max_drawdown_pct")),
|
|
_fmt(row.get("calmar")),
|
|
_fmt(row.get("trades")),
|
|
_fmt(overlap),
|
|
)
|
|
)
|
|
selection = report.get("development_selection")
|
|
lines.extend(["", "## Mechanical selection", ""])
|
|
if selection:
|
|
lines.append(
|
|
"- Development-selected arm: **{}**. ".format(selection["arm_id"])
|
|
+ "The test result is reported only as a final check."
|
|
)
|
|
lines.append("- Final check: `{}`".format(report.get("final_check")))
|
|
else:
|
|
lines.append("- No overlay passed the train + validation requirements.")
|
|
lines.extend(["", "Production remains unchanged pending human review.", ""])
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _write_outputs(report: dict[str, Any], out: Path, *, bundle: bool) -> None:
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8")
|
|
markdown_path = out.with_suffix(".md")
|
|
markdown_path.write_text(_markdown(report), encoding="utf-8")
|
|
|
|
arms_csv = out.with_name(f"{out.stem}-arms.csv")
|
|
with arms_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.writer(handle)
|
|
writer.writerow(
|
|
[
|
|
"arm",
|
|
"composite",
|
|
"weight",
|
|
"window",
|
|
"sharpe",
|
|
"sharpe_se",
|
|
"dsr",
|
|
"cagr_pct",
|
|
"max_drawdown_pct",
|
|
"calmar",
|
|
"trades",
|
|
"overlap_pct",
|
|
"top5_pnl_share_pct",
|
|
"avg_r_ex_top5",
|
|
]
|
|
)
|
|
for arm in report.get("arms", []):
|
|
for row in arm.get("windows", []):
|
|
writer.writerow(
|
|
[
|
|
arm["id"],
|
|
arm["composite"],
|
|
arm["weight"],
|
|
row.get("window"),
|
|
row.get("sharpe"),
|
|
row.get("sharpe_se"),
|
|
row.get("dsr"),
|
|
row.get("cagr_pct"),
|
|
row.get("max_drawdown_pct"),
|
|
row.get("calmar"),
|
|
row.get("trades"),
|
|
row.get("selection_vs_control", {}).get("overlap_pct"),
|
|
row.get("winner_concentration", {}).get(
|
|
"top5_share_of_positive_net_pct"
|
|
),
|
|
row.get("winner_concentration", {}).get("avg_r_ex_top5"),
|
|
]
|
|
)
|
|
|
|
factor_csv = out.with_name(f"{out.stem}-factor-ic.csv")
|
|
with factor_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.writer(handle)
|
|
writer.writerow(
|
|
[
|
|
"window",
|
|
"signal",
|
|
"mean_ic",
|
|
"ic_t_stat",
|
|
"ic_positive_pct",
|
|
"mean_quintile_spread",
|
|
"weeks",
|
|
"avg_cross_section",
|
|
"reliable",
|
|
]
|
|
)
|
|
for window, rows in report.get("factor_ic", {}).items():
|
|
for row in rows:
|
|
writer.writerow(
|
|
[
|
|
window,
|
|
row.get("signal"),
|
|
row.get("mean_ic"),
|
|
row.get("ic_t_stat"),
|
|
row.get("ic_positive_pct"),
|
|
row.get("mean_quintile_spread"),
|
|
row.get("weeks"),
|
|
row.get("avg_cross_section"),
|
|
row.get("reliable"),
|
|
]
|
|
)
|
|
|
|
trades_csv = out.with_name(f"{out.stem}-trades.csv")
|
|
trade_columns = [
|
|
"arm",
|
|
"symbol",
|
|
"entry_date",
|
|
"exit_date",
|
|
"r",
|
|
"pnl",
|
|
"reason",
|
|
"hold",
|
|
"entry",
|
|
"exit",
|
|
"risk_dollars",
|
|
]
|
|
with trades_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=trade_columns, extrasaction="ignore")
|
|
writer.writeheader()
|
|
for arm in report.get("arms", []):
|
|
for trade in arm.get("full_trade_details", []):
|
|
writer.writerow({"arm": arm["id"], **trade})
|
|
|
|
if bundle:
|
|
bundle_path = out.with_suffix(".zip")
|
|
with zipfile.ZipFile(bundle_path, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
for path in (out, markdown_path, arms_csv, factor_csv, trades_csv):
|
|
archive.write(path, arcname=path.name)
|
|
protocol = ROOT / "docs" / "research" / "fundamentals-weight-backtest.md"
|
|
if protocol.exists():
|
|
archive.write(protocol, arcname=protocol.name)
|
|
|
|
|
|
async def _main() -> None:
|
|
args = _parse_args()
|
|
snapshot = Path(args.snapshot)
|
|
out = Path(args.out) if args.out else _default_out()
|
|
if not snapshot.exists():
|
|
raise SystemExit(f"snapshot not found: {snapshot}")
|
|
if args.workers < 1:
|
|
raise SystemExit("--workers must be positive")
|
|
train_end = date.fromisoformat(args.train_end)
|
|
test_start = date.fromisoformat(args.test_start)
|
|
if train_end >= test_start:
|
|
raise SystemExit("--train-end must be earlier than --test-start")
|
|
|
|
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
|
if args.allow_spawn:
|
|
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
|
|
|
data = await _load_snapshot(snapshot, args.quiet)
|
|
audit = data["audit"]
|
|
if audit["fundamental_rows"] == 0 or audit["fundamental_ciks"] < 5:
|
|
raise SystemExit(
|
|
"snapshot lacks usable fundamental_snapshots; create a fresh export "
|
|
"with scripts/create_backtest_snapshot.py"
|
|
)
|
|
representatives = _representatives(data["ticker_rows"], data["prices"])
|
|
candidates, entry_candidate_count = _build_candidates(snapshot, data, args)
|
|
factor_dates = _weekly_factor_dates(representatives, data["prices"])
|
|
all_dates = factor_dates | {
|
|
date.fromisoformat(str(candidate["date"])) for candidate in candidates
|
|
}
|
|
scores_by_date, score_coverage = _build_scores(
|
|
snapshot,
|
|
all_dates,
|
|
representatives,
|
|
data["snapshots_by_cik"],
|
|
args,
|
|
)
|
|
candidate_coverage = _attach_overlay_ranks(
|
|
candidates, data["ticker_rows"], scores_by_date
|
|
)
|
|
factor_ic = _factor_diagnostics(
|
|
representatives,
|
|
data["prices"],
|
|
scores_by_date,
|
|
train_end,
|
|
test_start,
|
|
)
|
|
|
|
report: dict[str, Any] = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"git_commit": _git_commit(),
|
|
"snapshot": str(snapshot.resolve()),
|
|
"snapshot_sha256": _snapshot_hash(snapshot),
|
|
"splits": {
|
|
"train_end": train_end.isoformat(),
|
|
"test_start": test_start.isoformat(),
|
|
"windows": {
|
|
"train": f"entry date < {train_end.isoformat()}",
|
|
"validation": (
|
|
f"{train_end.isoformat()} <= entry date < {test_start.isoformat()}"
|
|
),
|
|
"test": f"entry date >= {test_start.isoformat()}",
|
|
},
|
|
},
|
|
"n_trials": N_TRIALS,
|
|
"pre_registered_arms": list(ARMS),
|
|
"protocol": {
|
|
"qualification": "unchanged production gate; rank overlay only",
|
|
"cadence": "daily",
|
|
"fill_mode": "close; production near-close proxy",
|
|
"horizon_sessions": 30,
|
|
"filing_availability": (
|
|
"accepted_at before signal-date midnight America/New_York; "
|
|
"conservative match for the daily pre-market SEC import"
|
|
),
|
|
"missing_fundamental_score": 50.0,
|
|
"selection": (
|
|
"highest validation Sharpe among arms with train and validation "
|
|
"Sharpe not below control and validation drawdown within 2pp"
|
|
),
|
|
"production_mutation": False,
|
|
},
|
|
"warnings": [
|
|
"Current tracked universe only: historical constituent membership and "
|
|
"delisted names are unavailable, so absolute results have survivorship bias.",
|
|
"Historical valuation is excluded because split-adjusted bars cannot be "
|
|
"safely combined with filing-time EPS and shares without split factors.",
|
|
"Earnings surprise is excluded because the completed SUE study already "
|
|
"failed its promotion bar for this strategy.",
|
|
"The test window remains research evidence, not a pristine future sample; "
|
|
"live paper performance is still the final out-of-sample check.",
|
|
],
|
|
"data_audit": audit,
|
|
"strategy_config": {
|
|
"recommendation": data["config"],
|
|
"activation": data["activation"],
|
|
"exit": data["exit_config"],
|
|
},
|
|
"score_cross_section_coverage": score_coverage,
|
|
"qualified_candidate_coverage": candidate_coverage,
|
|
"entry_candidate_count": entry_candidate_count,
|
|
"qualified_candidates": len(candidates),
|
|
"factor_ic": factor_ic,
|
|
"arms": [],
|
|
"development_grades": {},
|
|
"development_selection": None,
|
|
"final_check": None,
|
|
}
|
|
_write_outputs(report, out, bundle=False)
|
|
|
|
trade_sets: dict[str, dict[str, set[str]]] = {}
|
|
control: dict[str, Any] | None = None
|
|
for arm in ARMS:
|
|
if not args.quiet:
|
|
print("running {}".format(arm["id"]), flush=True)
|
|
result, arm_trades = _run_arm(arm, candidates, data, train_end, test_start)
|
|
trade_sets[str(arm["id"])] = arm_trades
|
|
if control is None:
|
|
control = result
|
|
else:
|
|
for row in result["windows"]:
|
|
name = str(row["window"])
|
|
row["selection_vs_control"] = _trade_overlap(
|
|
arm_trades.get(name, set()),
|
|
trade_sets["control_w00"].get(name, set()),
|
|
)
|
|
report["development_grades"][str(arm["id"])] = _development_grade(
|
|
control, result
|
|
)
|
|
report["arms"].append(result)
|
|
_write_outputs(report, out, bundle=False)
|
|
|
|
if control is None:
|
|
raise RuntimeError("control arm did not run")
|
|
eligible = [
|
|
arm
|
|
for arm in report["arms"][1:]
|
|
if report["development_grades"].get(arm["id"], {}).get("pass")
|
|
]
|
|
selected = max(
|
|
eligible,
|
|
key=lambda arm: (
|
|
float((_window(arm, "validation") or {}).get("sharpe") or -999.0),
|
|
-float(arm["weight"]),
|
|
),
|
|
default=None,
|
|
)
|
|
if selected is not None:
|
|
report["development_selection"] = {
|
|
"arm_id": selected["id"],
|
|
"chosen_without_test": True,
|
|
"validation_sharpe": (_window(selected, "validation") or {}).get("sharpe"),
|
|
}
|
|
report["final_check"] = _final_check(control, selected)
|
|
report["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
_write_outputs(report, out, bundle=True)
|
|
print(f"wrote {out}", flush=True)
|
|
bundle_path = out.with_suffix(".zip")
|
|
print("wrote {}".format(bundle_path), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(_main())
|