feat: add fundamentals weighting backtest research

app/services/fundamentals_derivation.py

@@ -20,7 +20,7 @@ Rules:
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from datetime import date
+from datetime import date, datetime, timezone
 from typing import Any, Iterable
 
 _FP_TO_Q = {"Q1": 1, "Q2": 2, "Q3": 3, "FY": 4}
@@ -30,7 +30,12 @@ TAPE_LEN = 4  # quarter-tape length
 
 # Duration (flow) fields differenced from YTD into discrete quarters + summed to TTM.
 _FLOW_FIELDS = (
-    "revenue", "net_income", "operating_income", "diluted_eps", "cfo", "capex",
+    "revenue",
+    "net_income",
+    "operating_income",
+    "diluted_eps",
+    "cfo",
+    "capex",
     "depreciation_amortization",
 )
 
@@ -44,7 +49,9 @@ class MetricPoint:
 @dataclass
 class MetricSeries:
     value: float | None = None
-    history: list[MetricPoint] = field(default_factory=list)  # oldest -> newest, <= TAPE_LEN
+    history: list[MetricPoint] = field(
+        default_factory=list
+    )  # oldest -> newest, <= TAPE_LEN
     period_end: date | None = None
     filed_date: date | None = None
 
@@ -82,7 +89,9 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals:
     result.ttm_diluted_eps = _ttm(discrete["diluted_eps"], *latest)
     ttm_cfo = _ttm(discrete["cfo"], *latest)
     ttm_capex = _ttm(discrete["capex"], *latest)
-    result.ttm_fcf = None if ttm_cfo is None or ttm_capex is None else ttm_cfo - ttm_capex
+    result.ttm_fcf = (
+        None if ttm_cfo is None or ttm_capex is None else ttm_cfo - ttm_capex
+    )
 
     # tape = the CONSECUTIVE run of up to TAPE_LEN quarters ending at the latest,
     # stopping at a gap — so trend text never compares non-adjacent periods.
@@ -90,7 +99,9 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals:
     result.metrics = {
         "revenue_growth_yoy": _yoy_growth_series(discrete["revenue"], selected, tape),
         "eps_growth_yoy": _yoy_growth_series(discrete["diluted_eps"], selected, tape),
-        "operating_margin": _margin_series(discrete["operating_income"], discrete["revenue"], selected, tape),
+        "operating_margin": _margin_series(
+            discrete["operating_income"], discrete["revenue"], selected, tape
+        ),
         "fcf_margin": _fcf_margin_series(discrete, selected, tape),
         "net_debt": _instant_series(selected, tape, _net_debt),
         "net_debt_to_ebitda": _leverage_series(selected, discrete, tape),
@@ -102,8 +113,27 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals:
     return result
 
 
+def derive_as_of(snapshots: Iterable[Any], as_of: datetime) -> DerivedFundamentals:
+    """Derive using only SEC filings accepted by the historical cutoff."""
+    cutoff = _utc_datetime(as_of)
+    visible = (
+        row
+        for row in snapshots
+        if (accepted := getattr(row, "accepted_at", None)) is not None
+        and _utc_datetime(accepted) <= cutoff
+    )
+    return derive(visible)
+
+
+def _utc_datetime(value: datetime) -> datetime:
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
+
+
 # -- period selection --------------------------------------------------------
 
+
 def _select_latest_per_period(snapshots: Iterable[Any]) -> dict[tuple[int, str], Any]:
     best: dict[tuple[int, str], Any] = {}
     for row in snapshots:
@@ -126,7 +156,9 @@ def _ordered_quarters(selected: dict[tuple[int, str], Any]) -> list[tuple[int, i
     return sorted((fy, _FP_TO_Q[fp]) for (fy, fp) in selected)
 
 
-def _consecutive_suffix(quarters: list[tuple[int, int]], n: int) -> list[tuple[int, int]]:
+def _consecutive_suffix(
+    quarters: list[tuple[int, int]], n: int
+) -> list[tuple[int, int]]:
     """The run of up to n quarters ending at the latest, walking back only through
     adjacent periods (stop at the first gap). Returned oldest -> newest."""
     if not quarters:
@@ -146,7 +178,10 @@ def _consecutive_suffix(quarters: list[tuple[int, int]], n: int) -> list[tuple[i
 
 # -- discrete + TTM ----------------------------------------------------------
 
-def _discrete_quarters(selected: dict[tuple[int, str], Any], field_name: str) -> dict[tuple[int, int], float]:
+
+def _discrete_quarters(
+    selected: dict[tuple[int, str], Any], field_name: str
+) -> dict[tuple[int, int], float]:
     out: dict[tuple[int, int], float] = {}
     for (fy, fp), row in selected.items():
         val = _discrete_value(selected, fy, fp, field_name)
@@ -189,6 +224,7 @@ def _pct_change(cur: float | None, prior: float | None) -> float | None:
 
 # -- per-metric series (value at latest + tape history) ----------------------
 
+
 def _period_end(selected, fy: int, q: int) -> date | None:
     row = selected.get((fy, _Q_TO_FP[q]))
     return row.period_end if row is not None else None
@@ -196,7 +232,7 @@ def _period_end(selected, fy: int, q: int) -> date | None:
 
 def _yoy_growth_series(dq, selected, tape) -> MetricSeries:
     pts = []
-    for (fy, q) in tape:
+    for fy, q in tape:
         cur, prior = _ttm(dq, fy, q), _ttm(dq, fy - 1, q)
         pts.append(MetricPoint(_period_end(selected, fy, q), _pct_change(cur, prior)))
     return _series(pts)
@@ -204,7 +240,7 @@ def _yoy_growth_series(dq, selected, tape) -> MetricSeries:
 
 def _margin_series(num_dq, den_dq, selected, tape) -> MetricSeries:
     pts = []
-    for (fy, q) in tape:
+    for fy, q in tape:
         num, den = _ttm(num_dq, fy, q), _ttm(den_dq, fy, q)
         val = None if num is None or not den else num / den * 100.0
         pts.append(MetricPoint(_period_end(selected, fy, q), val))
@@ -213,24 +249,38 @@ def _margin_series(num_dq, den_dq, selected, tape) -> MetricSeries:
 
 def _fcf_margin_series(discrete, selected, tape) -> MetricSeries:
     pts = []
-    for (fy, q) in tape:
+    for fy, q in tape:
-        cfo, capex, rev = _ttm(discrete["cfo"], fy, q), _ttm(discrete["capex"], fy, q), _ttm(discrete["revenue"], fy, q)
+        cfo, capex, rev = (
-        val = None if cfo is None or capex is None or not rev else (cfo - capex) / rev * 100.0
+            _ttm(discrete["cfo"], fy, q),
+            _ttm(discrete["capex"], fy, q),
+            _ttm(discrete["revenue"], fy, q),
+        )
+        val = (
+            None
+            if cfo is None or capex is None or not rev
+            else (cfo - capex) / rev * 100.0
+        )
         pts.append(MetricPoint(_period_end(selected, fy, q), val))
     return _series(pts)
 
 
 def _instant_series(selected, tape, fn) -> MetricSeries:
-    pts = [MetricPoint(_period_end(selected, fy, q), fn(selected.get((fy, _Q_TO_FP[q])))) for (fy, q) in tape]
+    pts = [
+        MetricPoint(_period_end(selected, fy, q), fn(selected.get((fy, _Q_TO_FP[q]))))
+        for (fy, q) in tape
+    ]
     return _series(pts)
 
 
 def _leverage_series(selected, discrete, tape) -> MetricSeries:
     pts = []
-    for (fy, q) in tape:
+    for fy, q in tape:
         row = selected.get((fy, _Q_TO_FP[q]))
         nd = _net_debt(row)
-        op, da = _ttm(discrete["operating_income"], fy, q), _ttm(discrete["depreciation_amortization"], fy, q)
+        op, da = (
+            _ttm(discrete["operating_income"], fy, q),
+            _ttm(discrete["depreciation_amortization"], fy, q),
+        )
         ebitda = None if op is None or da is None else op + da
         # Null when EBITDA <= 0: a negative denominator would flip polarity and a
         # "lower is better" read would rank a distressed issuer as favorable.
@@ -241,7 +291,7 @@ def _leverage_series(selected, discrete, tape) -> MetricSeries:
 
 def _share_change_series(selected, tape) -> MetricSeries:
     pts = []
-    for (fy, q) in tape:
+    for fy, q in tape:
         cur = _shares(selected.get((fy, _Q_TO_FP[q])))
         prior = _shares(selected.get((fy - 1, _Q_TO_FP[q])))
         pts.append(MetricPoint(_period_end(selected, fy, q), _pct_change(cur, prior)))

app/services/fundamentals_research.py

@@ -0,0 +1,157 @@
+"""Pure scoring helpers for point-in-time fundamentals research.
+
+The runner converts a CIK-deduplicated cross-section into favorable 0..100
+factor ranks and three deliberately small composites. Historical valuation is
+absent: stored bars are split-adjusted, while filing-time EPS and share counts
+are not guaranteed to be on today's split basis.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from typing import Any
+
+MIN_CROSS_SECTION = 5
+
+FACTOR_POLARITY: dict[str, bool] = {
+    "revenue_growth_yoy": True,
+    "eps_growth_yoy": True,
+    "operating_margin": True,
+    "fcf_margin": True,
+    "net_debt_to_ebitda": False,
+    "share_count_change_yoy": False,
+}
+
+QUALITY_FACTORS = (
+    "operating_margin",
+    "fcf_margin",
+    "net_debt_to_ebitda",
+    "share_count_change_yoy",
+)
+GROWTH_FACTORS = ("revenue_growth_yoy", "eps_growth_yoy")
+COMPOSITE_KEYS = ("quality", "growth", "balanced")
+
+
+def raw_features(derived: Any) -> dict[str, float | None]:
+    """Extract the six research-eligible values from derived fundamentals."""
+    metrics = getattr(derived, "metrics", {}) or {}
+    return {
+        key: _finite_or_none(getattr(metrics.get(key), "value", None))
+        for key in FACTOR_POLARITY
+    }
+
+
+def cross_section_scores(
+    features_by_issuer: Mapping[str, Mapping[str, Any]],
+    *,
+    min_cross_section: int = MIN_CROSS_SECTION,
+) -> dict[str, dict[str, float | None]]:
+    """Return favorable factor ranks and composites for every issuer.
+
+    Quality needs two of four inputs; growth needs one of two. Balanced requires
+    both sub-scores and weights them equally, so quality's four inputs do not
+    mechanically dominate growth's two inputs.
+    """
+    result = {
+        str(issuer): {
+            **{key: None for key in FACTOR_POLARITY},
+            **{key: None for key in COMPOSITE_KEYS},
+        }
+        for issuer in features_by_issuer
+    }
+
+    for factor, higher_is_better in FACTOR_POLARITY.items():
+        values = {
+            str(issuer): _finite_or_none(features.get(factor))
+            for issuer, features in features_by_issuer.items()
+        }
+        ranks = favorable_percentiles(
+            values,
+            higher_is_better=higher_is_better,
+            min_count=min_cross_section,
+        )
+        for issuer, rank in ranks.items():
+            result[issuer][factor] = rank
+
+    for scores in result.values():
+        quality_values = _available(scores, QUALITY_FACTORS)
+        growth_values = _available(scores, GROWTH_FACTORS)
+        if len(quality_values) >= 2:
+            scores["quality"] = _mean(quality_values)
+        if growth_values:
+            scores["growth"] = _mean(growth_values)
+        if scores["quality"] is not None and scores["growth"] is not None:
+            scores["balanced"] = _mean(
+                [float(scores["quality"]), float(scores["growth"])]
+            )
+    return result
+
+
+def favorable_percentiles(
+    values_by_issuer: Mapping[str, Any],
+    *,
+    higher_is_better: bool,
+    min_count: int = MIN_CROSS_SECTION,
+) -> dict[str, float | None]:
+    """Tie-aware favorable percentile for a deduplicated cross-section."""
+    valid = {
+        str(issuer): float(value)
+        for issuer, value in values_by_issuer.items()
+        if _finite_or_none(value) is not None
+    }
+    result: dict[str, float | None] = {str(issuer): None for issuer in values_by_issuer}
+    if len(valid) < min_count:
+        return result
+
+    for issuer, subject in valid.items():
+        others = [value for key, value in valid.items() if key != issuer]
+        if higher_is_better:
+            worse = sum(value < subject for value in others)
+        else:
+            worse = sum(value > subject for value in others)
+        tied = sum(value == subject for value in others)
+        result[issuer] = round(
+            (worse + 0.5 * tied) / len(others) * 100.0,
+            4,
+        )
+    return result
+
+
+def overlay_rank(
+    strategy_rank: Any,
+    fundamental_score: Any,
+    weight: float,
+    *,
+    missing_score: float = 50.0,
+) -> float | None:
+    """Blend production rank with fundamentals without changing the gate."""
+    base = _finite_or_none(strategy_rank)
+    if base is None:
+        return None
+    if not 0.0 <= weight <= 1.0:
+        raise ValueError("weight must be between 0 and 1")
+    score = _finite_or_none(fundamental_score)
+    if score is None:
+        score = float(missing_score)
+    return round((1.0 - weight) * base + weight * score, 4)
+
+
+def _available(scores: Mapping[str, Any], keys: tuple[str, ...]) -> list[float]:
+    return [
+        value for key in keys if (value := _finite_or_none(scores.get(key))) is not None
+    ]
+
+
+def _mean(values: list[float]) -> float:
+    return round(sum(values) / len(values), 4)
+
+
+def _finite_or_none(value: Any) -> float | None:
+    if (
+        isinstance(value, (int, float))
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+    ):
+        return float(value)
+    return None

docs/research/fundamentals-weight-backtest.md

@@ -0,0 +1,153 @@
+# Point-in-time fundamentals weight backtest
+
+Status: pre-registered local research. Running it does not change production.
+
+## Question
+
+Does reordering already-qualified long setups with SEC fundamentals improve the
+production book's risk-adjusted return? The qualification gate, execution model,
+position sizing, capacity, costs, ATR trail, and post-stop re-entry policy remain
+unchanged. This isolates the incremental value of fundamentals as a ranking
+overlay.
+
+## Registered experiment
+
+The control is the production 80/20 residual-momentum / volatility rank. The
+runner tests three fundamental composites at weights 10%, 20%, 30%, and 40%:
+
+- Quality: operating margin, FCF margin, low net-debt/EBITDA, and low dilution.
+  At least two inputs must exist.
+- Growth: revenue growth and diluted-EPS growth. At least one must exist.
+- Balanced: equal weight to the quality and growth sub-scores. Both must exist.
+
+Each raw metric is ranked favorably from 0 to 100 across the CIK-deduplicated
+tracked universe. Missing composite scores are neutral at 50. The formula is:
+
+`final rank = (1 - weight) * production rank + weight * fundamental rank`
+
+There are 13 registered portfolio trials including the control. That complete
+count is used by the Deflated Sharpe calculation.
+
+Historical P/E and FCF yield are excluded. Stored Alpaca bars are split-adjusted,
+while filing-time EPS and shares are not guaranteed to use today's split basis;
+mixing them without point-in-time split factors can manufacture valuation moves.
+Earnings surprise is also excluded because the completed Dolt SUE study already
+failed its promotion bar for this strategy.
+
+## Point-in-time rule
+
+Only SEC rows accepted before midnight America/New_York at the start of a signal
+date are visible. This is conservative relative to the daily pre-market SEC
+import and prevents same-day filings or later amendments from leaking backward.
+The derivation then selects the newest visible accession per fiscal period.
+
+Default windows are fixed before the first run:
+
+- Train: entry date before 2024-01-01
+- Validation: 2024-01-01 through 2024-12-31
+- Test: entry date on or after 2025-01-01
+
+Do not move these boundaries after seeing results. The test window is used only
+to check the single arm chosen from train and validation. Reports expose all
+registered rows for auditability and correct multiple-testing accounting.
+
+## 1. Create the portable snapshot
+
+Run this wherever the production PostgreSQL connection is already configured.
+The exporter copies prices, the safe strategy settings, SEC snapshots, and Dolt
+earnings rows. It does not copy credentials or unrelated system settings.
+
+Windows PowerShell:
+
+```powershell
+.venv\Scripts\python.exe scripts\create_backtest_snapshot.py `
+  --output backtest_snapshots\fundamentals-backtest.sqlite `
+  --force
+```
+
+Linux production host:
+
+```bash
+.venv/bin/python scripts/create_backtest_snapshot.py \
+  --output backtest_snapshots/fundamentals-backtest.sqlite \
+  --force
+```
+
+Copy only the SQLite file to the MacBook. `scp`, a local network share, or an
+encrypted USB drive are all fine. Do not copy `.env`.
+
+## 2. Prepare the MacBook
+
+Use the same Git commit as the machine that created the report. From the repo:
+
+```bash
+python3.11 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -e '.[dev]'
+chmod +x scripts/run_fundamentals_macbook.sh
+```
+
+Put the snapshot at `backtest_snapshots/fundamentals-backtest.sqlite`, or pass a
+different path to the launcher.
+
+## 3. Run it
+
+```bash
+./scripts/run_fundamentals_macbook.sh \
+  backtest_snapshots/fundamentals-backtest.sqlite
+```
+
+The launcher defaults to logical CPU count minus one. Override it if the laptop
+gets too warm or memory pressure rises:
+
+```bash
+WORKERS=8 ./scripts/run_fundamentals_macbook.sh \
+  backtest_snapshots/fundamentals-backtest.sqlite
+```
+
+The first run builds two caches under `reports/.cache`: production candidate
+replay and point-in-time fundamental scores. If interrupted, rerun the same
+command; valid caches are reused. Cache keys include snapshot size and mtime, so
+a new snapshot triggers a rebuild.
+
+## 4. Bring the result back
+
+The final line names one ZIP such as:
+
+`reports/fundamentals-overlay-20260723-180000.zip`
+
+That ZIP contains:
+
+- the complete JSON report and reproducibility metadata;
+- a readable Markdown summary;
+- portfolio-arm CSV;
+- factor-IC CSV;
+- full-period trade CSV for every arm, allowing winner-concentration checks;
+- this registered protocol.
+
+Copy the ZIP into this workspace or attach it in the conversation. The snapshot
+itself is not needed for the first evaluation unless a result looks inconsistent.
+
+## Evaluation order
+
+1. Data coverage and the accepted-at range.
+2. Individual factor IC: sign, magnitude, consistency, and cross-section size.
+3. Composite IC in train, validation, and test.
+4. Development selection made without the test window.
+5. Test Sharpe, CAGR, drawdown, yearly returns, trial-corrected DSR, and trade
+   overlap versus control.
+6. Sensitivity to a few dominant winners and whether the effect is economically
+   large enough to justify production complexity.
+
+The mechanical development bar requires train and validation Sharpe not below
+control and validation drawdown no more than two percentage points worse. A test
+pass is still research evidence, not automatic deployment.
+
+## Known limitation
+
+The snapshot contains today's tracked tickers, not historical constituent
+membership or delisted names. This creates survivorship bias. The same biased
+universe is used for control and overlays, so the local comparison is useful,
+but its absolute Sharpe or CAGR must not be presented as an unbiased live
+expectation. Forward paper performance remains the true out-of-sample check.

scripts/create_backtest_snapshot.py

@@ -1,9 +1,9 @@
-"""Create a minimal local SQLite snapshot for offline backtest research.
+"""Create a portable local SQLite snapshot for offline backtest research.
 
-Copies only the data required by app.services.backtest_service.run_backtest:
+Copies the data required by the production backtest and fundamentals research:
 tickers, OHLCV bars, SPY benchmark closes, and the activation / recommendation /
-paper-exit settings the run reads. Other system settings are intentionally
+paper-exit settings the run reads, immutable SEC snapshots, and Dolt earnings
-skipped to avoid copying secrets into local snapshot files.
+events. Other system settings are skipped to avoid copying secrets locally.
 """
 
 from __future__ import annotations
@@ -54,7 +54,9 @@ def _parse_args() -> argparse.Namespace:
         help="SQLite snapshot path to create.",
     )
     parser.add_argument("--batch-size", type=int, default=5000)
-    parser.add_argument("--force", action="store_true", help="Overwrite an existing snapshot file.")
+    parser.add_argument(
+        "--force", action="store_true", help="Overwrite an existing snapshot file."
+    )
     return parser.parse_args()
 
 
@@ -65,6 +67,7 @@ async def _copy_table(
     *,
     batch_size: int,
     where=None,
+    row_transform=None,
 ) -> int:
     table = model.__table__
     columns = list(table.columns)
@@ -87,6 +90,8 @@ async def _copy_table(
     stream = await source.stream(stmt.execution_options(yield_per=batch_size))
     async for partition in stream.partitions(batch_size):
         rows = [dict(row._mapping) for row in partition]
+        if row_transform is not None:
+            rows = [row_transform(row) for row in rows]
         if not rows:
             continue
         await dest.execute(insert(table), rows)
@@ -105,6 +110,8 @@ async def _main() -> None:
     from app.database import Base
     import app.models  # noqa: F401 - registers all metadata tables
     from app.models.benchmark_price import BenchmarkPrice
+    from app.models.earnings_event import EarningsEvent
+    from app.models.fundamental_snapshot import FundamentalSnapshot
     from app.models.ohlcv import OHLCVRecord
     from app.models.settings import SystemSetting
     from app.models.ticker import Ticker
@@ -123,8 +130,12 @@ async def _main() -> None:
         connect_args={"server_settings": {"default_transaction_read_only": "on"}},
     )
     dest_engine = create_async_engine(_sqlite_url(output))
-    SourceSession = async_sessionmaker(source_engine, class_=AsyncSession, expire_on_commit=False)
+    SourceSession = async_sessionmaker(
-    DestSession = async_sessionmaker(dest_engine, class_=AsyncSession, expire_on_commit=False)
+        source_engine, class_=AsyncSession, expire_on_commit=False
+    )
+    DestSession = async_sessionmaker(
+        dest_engine, class_=AsyncSession, expire_on_commit=False
+    )
 
     print(f"Source: {_hide_password(source_url)}")
     print(f"Snapshot: {output}")
@@ -135,7 +146,9 @@ async def _main() -> None:
 
         async with SourceSession() as source, DestSession() as dest:
             counts = {
-                "tickers": await _copy_table(source, dest, Ticker, batch_size=args.batch_size),
+                "tickers": await _copy_table(
+                    source, dest, Ticker, batch_size=args.batch_size
+                ),
                 "system_settings": await _copy_table(
                     source,
                     dest,
@@ -152,9 +165,30 @@ async def _main() -> None:
                         SystemSetting.key.like("paper_%"),
                     ),
                 ),
-                "benchmark_prices": await _copy_table(source, dest, BenchmarkPrice, batch_size=args.batch_size),
+                "benchmark_prices": await _copy_table(
-                "ohlcv_records": await _copy_table(source, dest, OHLCVRecord, batch_size=args.batch_size),
+                    source, dest, BenchmarkPrice, batch_size=args.batch_size
+                ),
+                "ohlcv_records": await _copy_table(
+                    source, dest, OHLCVRecord, batch_size=args.batch_size
+                ),
             }
+        # Import-run provenance is operational metadata, not a research input.
+        # Null it so the portable snapshot needs no data_import_runs rows.
+        async with SourceSession() as source, DestSession() as dest:
+            counts["fundamental_snapshots"] = await _copy_table(
+                source,
+                dest,
+                FundamentalSnapshot,
+                batch_size=args.batch_size,
+                row_transform=lambda row: {**row, "import_run_id": None},
+            )
+            counts["earnings_events"] = await _copy_table(
+                source,
+                dest,
+                EarningsEvent,
+                batch_size=args.batch_size,
+                row_transform=lambda row: {**row, "import_run_id": None},
+            )
     finally:
         await source_engine.dispose()
         await dest_engine.dispose()

scripts/run_fundamentals_macbook.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(git rev-parse --show-toplevel)"
+cd "$ROOT"
+
+SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}"
+WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}"
+if [[ "$WORKERS" -gt 1 ]]; then
+  WORKERS=$((WORKERS - 1))
+fi
+
+if [[ -x .venv/bin/python ]]; then
+  PYTHON=.venv/bin/python
+else
+  PYTHON="${PYTHON:-python3}"
+fi
+
+if [[ ! -f "$SNAPSHOT" ]]; then
+  echo "Snapshot not found: $SNAPSHOT" >&2
+  exit 1
+fi
+
+STAMP="$(date -u +%Y%m%d-%H%M%S)"
+OUT="reports/fundamentals-overlay-${STAMP}.json"
+
+echo "Snapshot: $SNAPSHOT"
+echo "Workers:  $WORKERS"
+echo "Output:   $OUT"
+
+"$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \
+  --workers "$WORKERS" \
+  --candidate-cache reports/.cache/fundamentals-candidates.pkl \
+  --fundamentals-cache reports/.cache/fundamentals-scores.pkl \
+  --out "$OUT"
+
+echo
+echo "Bring this file back for review: ${OUT%.json}.zip"

tests/unit/test_fundamentals_derivation.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import date, datetime, timezone
 
 import pytest
@@ -38,7 +38,9 @@ _ENDS = {  # period_end per (fy, quarter index 0..3)
 }
 
 
-def _year(fy, discretes: dict[str, list[float]], instants: dict[str, list] | None = None):
+def _year(
+    fy, discretes: dict[str, list[float]], instants: dict[str, list] | None = None
+):
     """Build 4 snapshot rows (Q1,Q2,Q3,FY) with YTD-cumulative flow fields from the
     given per-quarter discrete values; instants set as-is per quarter."""
     rows = []
@@ -55,22 +57,38 @@ def _year(fy, discretes: dict[str, list[float]], instants: dict[str, list] | Non
 def _two_years():
     rev25 = [100, 110, 120, 130]
     rev26 = [110, 121, 132, 143]  # +10% each quarter YoY
-    rows = _year(2025, {
+    rows = _year(
+        2025,
+        {
             "revenue": rev25,
             "operating_income": [x * 0.2 for x in rev25],
             "diluted_eps": [1.0, 1.1, 1.2, 1.3],
             "cfo": [x * 0.25 for x in rev25],
             "capex": [x * 0.05 for x in rev25],
             "depreciation_amortization": [x * 0.05 for x in rev25],
-    }, instants={"shares_outstanding": [1000, 1000, 1000, 1000], "cash_and_st_investments": [40] * 4, "total_debt": [140] * 4})
+        },
-    rows += _year(2026, {
+        instants={
+            "shares_outstanding": [1000, 1000, 1000, 1000],
+            "cash_and_st_investments": [40] * 4,
+            "total_debt": [140] * 4,
+        },
+    )
+    rows += _year(
+        2026,
+        {
             "revenue": rev26,
             "operating_income": [x * 0.2 for x in rev26],
             "diluted_eps": [1.1, 1.21, 1.32, 1.43],
             "cfo": [x * 0.25 for x in rev26],
             "capex": [x * 0.05 for x in rev26],
             "depreciation_amortization": [x * 0.05 for x in rev26],
-    }, instants={"shares_outstanding": [900, 900, 900, 900], "cash_and_st_investments": [50] * 4, "total_debt": [150] * 4})
+        },
+        instants={
+            "shares_outstanding": [900, 900, 900, 900],
+            "cash_and_st_investments": [50] * 4,
+            "total_debt": [150] * 4,
+        },
+    )
     return rows
 
 
@@ -99,7 +117,9 @@ def test_net_debt_leverage_and_share_dilution():
     # EBITDA TTM = TTM operating_income + TTM D&A; net_debt/ebitda
     op_ttm = 506 * 0.2  # 101.2
     da_ttm = 506 * 0.05  # 25.3
-    assert d.metrics["net_debt_to_ebitda"].value == pytest.approx(100.0 / (op_ttm + da_ttm), rel=1e-6)
+    assert d.metrics["net_debt_to_ebitda"].value == pytest.approx(
+        100.0 / (op_ttm + da_ttm), rel=1e-6
+    )
     # shares 900 vs 1000 a year earlier -> -10% (buyback)
     assert d.metrics["share_count_change_yoy"].value == pytest.approx(-10.0, abs=1e-6)
 
@@ -130,7 +150,9 @@ def test_net_debt_requires_both_components():
             r.total_debt = None
     d = fd.derive(rows)
     assert d.metrics["net_debt"].value is None
-    assert d.metrics["net_debt_to_ebitda"].value is None  # net debt null -> leverage null
+    assert (
+        d.metrics["net_debt_to_ebitda"].value is None
+    )  # net debt null -> leverage null
 
 
 def test_leverage_null_when_ebitda_nonpositive():
@@ -144,11 +166,19 @@ def test_leverage_null_when_ebitda_nonpositive():
 
 
 def test_tape_stops_at_a_gap():
-    rows = [r for r in _two_years() if not (r.fiscal_year == 2026 and r.fiscal_period == "Q1")]
+    rows = [
+        r
+        for r in _two_years()
+        if not (r.fiscal_year == 2026 and r.fiscal_period == "Q1")
+    ]
     d = fd.derive(rows)
     hist = d.metrics["operating_margin"].history
     # consecutive suffix ending at FY2026: Q2, Q3, FY (not compressed across the Q1 gap)
-    assert [p.period_end for p in hist] == [date(2026, 3, 31), date(2026, 6, 30), date(2026, 9, 30)]
+    assert [p.period_end for p in hist] == [
+        date(2026, 3, 31),
+        date(2026, 6, 30),
+        date(2026, 9, 30),
+    ]
 
 
 def test_yoy_growth_null_when_prior_nonpositive():
@@ -160,14 +190,48 @@ def test_yoy_growth_null_when_prior_nonpositive():
     assert d.metrics["eps_growth_yoy"].value is None  # loss->profit is not a %
 
 
+def test_derive_as_of_excludes_future_amendment():
+    rows = _two_years()
+    original = next(
+        row for row in rows if row.fiscal_year == 2026 and row.fiscal_period == "FY"
+    )
+    amendment = replace(
+        original,
+        accepted_at=datetime(2027, 1, 1, tzinfo=UTC),
+        revenue=999999,
+    )
+    before = fd.derive_as_of([*rows, amendment], datetime(2026, 12, 31, tzinfo=UTC))
+    after = fd.derive_as_of([*rows, amendment], datetime(2027, 1, 2, tzinfo=UTC))
+    assert before.metrics["revenue_growth_yoy"].value == pytest.approx(10.0)
+    assert after.metrics["revenue_growth_yoy"].value != pytest.approx(10.0)
+
+
+def test_derive_as_of_treats_sqlite_naive_acceptance_as_utc():
+    rows = _two_years()
+    rows[0].accepted_at = datetime(2025, 1, 1)
+    result = fd.derive_as_of(rows, datetime(2027, 1, 1, tzinfo=UTC))
+    assert result.latest_period_end == date(2026, 9, 30)
+
+
 def test_amendment_selection_newest_accepted_wins():
     rows = _two_years()
     # an amendment to FY2026 FY restates revenue YTD higher, accepted later
-    amended = Snap(2026, "FY", date(2026, 9, 30), date(2026, 11, 1),
+    amended = Snap(
-                   datetime(2027, 1, 1, tzinfo=UTC), revenue=999999,
+        2026,
-                   operating_income=100, diluted_eps=1.43, cfo=100, capex=10,
+        "FY",
-                   depreciation_amortization=25, shares_outstanding=900,
+        date(2026, 9, 30),
-                   cash_and_st_investments=50, total_debt=150)
+        date(2026, 11, 1),
+        datetime(2027, 1, 1, tzinfo=UTC),
+        revenue=999999,
+        operating_income=100,
+        diluted_eps=1.43,
+        cfo=100,
+        capex=10,
+        depreciation_amortization=25,
+        shares_outstanding=900,
+        cash_and_st_investments=50,
+        total_debt=150,
+    )
     d = fd.derive(rows + [amended])
     # Q4 revenue discrete now uses the amended YTD(FY)=999999 minus YTD(Q3)=363
     # so TTM/growth reflects the amendment, proving newest accepted_at won.

tests/unit/test_fundamentals_research.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import math
+
+import pytest
+
+from app.services import fundamentals_research as research
+
+
+def test_favorable_percentiles_are_tie_aware():
+    ranks = research.favorable_percentiles(
+        {"a": 3, "b": 3, "c": 3, "d": 3, "e": 3},
+        higher_is_better=True,
+    )
+    assert set(ranks.values()) == {50.0}
+
+
+def test_lower_is_better_flips_the_rank():
+    ranks = research.favorable_percentiles(
+        {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5},
+        higher_is_better=False,
+    )
+    assert ranks["a"] == 100.0
+    assert ranks["e"] == 0.0
+
+
+def test_invalid_and_thin_cross_sections_stay_null():
+    ranks = research.favorable_percentiles(
+        {"a": 1, "b": 2, "c": math.nan, "d": None, "e": 5},
+        higher_is_better=True,
+    )
+    assert all(value is None for value in ranks.values())
+
+
+def test_composites_use_equal_subgroup_weighting():
+    features = {
+        str(index): {
+            "operating_margin": index,
+            "fcf_margin": index,
+            "net_debt_to_ebitda": 6 - index,
+            "share_count_change_yoy": 6 - index,
+            "revenue_growth_yoy": index,
+            "eps_growth_yoy": index,
+        }
+        for index in range(1, 6)
+    }
+    scores = research.cross_section_scores(features)
+    assert scores["5"]["quality"] == 100.0
+    assert scores["5"]["growth"] == 100.0
+    assert scores["5"]["balanced"] == 100.0
+    assert scores["3"]["balanced"] == 50.0
+
+
+def test_overlay_uses_neutral_missing_score_and_validates_weight():
+    assert research.overlay_rank(90, None, 0.2) == 82.0
+    assert research.overlay_rank(90, 100, 0.2) == 92.0
+    with pytest.raises(ValueError, match="weight"):
+        research.overlay_rank(90, 50, 1.1)

tests/unit/test_fundamentals_research_runner.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import zipfile
+
+from scripts import run_fundamentals_research as runner
+
+
+def _window(name, sharpe, drawdown):
+    return {
+        "window": name,
+        "sharpe": sharpe,
+        "sharpe_se": 0.2,
+        "dsr": 0.8,
+        "cagr_pct": 10.0,
+        "max_drawdown_pct": drawdown,
+        "calmar": 1.0,
+        "trades": 20,
+    }
+
+
+def test_arm_matrix_is_bounded_and_pre_registered():
+    assert runner.N_TRIALS == 13
+    assert runner.ARMS[0]["id"] == "control_w00"
+    assert {arm["weight"] for arm in runner.ARMS[1:]} == {0.1, 0.2, 0.3, 0.4}
+    assert {arm["composite"] for arm in runner.ARMS[1:]} == {
+        "quality",
+        "growth",
+        "balanced",
+    }
+
+
+def test_development_grade_does_not_read_test_window():
+    control = {
+        "windows": [
+            _window("train", 1.0, 10.0),
+            _window("validation", 1.0, 10.0),
+            _window("test", 9.0, 1.0),
+        ]
+    }
+    arm = {
+        "windows": [
+            _window("train", 1.1, 10.0),
+            _window("validation", 1.2, 11.0),
+            _window("test", -9.0, 90.0),
+        ]
+    }
+    assert runner._development_grade(control, arm)["pass"] is True
+
+
+def test_output_bundle_is_self_contained(tmp_path):
+    report = {
+        "generated_at": "2026-07-23T00:00:00Z",
+        "splits": {"train_end": "2024-01-01", "test_start": "2025-01-01"},
+        "n_trials": 13,
+        "warnings": ["survivorship bias"],
+        "factor_ic": {"full": []},
+        "arms": [],
+        "development_selection": None,
+        "final_check": None,
+    }
+    output = tmp_path / "result.json"
+    runner._write_outputs(report, output, bundle=True)
+    with zipfile.ZipFile(output.with_suffix(".zip")) as archive:
+        names = set(archive.namelist())
+    assert {
+        "result.json",
+        "result.md",
+        "result-arms.csv",
+        "result-factor-ic.csv",
+        "result-trades.csv",
+    } <= names
+
+
+def test_winner_concentration_exposes_top_five_dependence():
+    details = [
+        {"pnl": value, "r": value / 10} for value in (100, 90, 80, 70, 60, -10, -20)
+    ]
+    result = runner._winner_concentration(details)
+    assert result["top5_pnl"] == 400
+    assert result["net_pnl_ex_top5"] == -30
+    assert result["avg_r_ex_top5"] == -1.5