feat: add fundamentals weighting backtest research

scripts/create_backtest_snapshot.py

@@ -1,9 +1,9 @@
-"""Create a minimal local SQLite snapshot for offline backtest research.
+"""Create a portable local SQLite snapshot for offline backtest research.
 
-Copies only the data required by app.services.backtest_service.run_backtest:
+Copies the data required by the production backtest and fundamentals research:
 tickers, OHLCV bars, SPY benchmark closes, and the activation / recommendation /
-paper-exit settings the run reads. Other system settings are intentionally
-skipped to avoid copying secrets into local snapshot files.
+paper-exit settings the run reads, immutable SEC snapshots, and Dolt earnings
+events. Other system settings are skipped to avoid copying secrets locally.
 """
 
 from __future__ import annotations
@@ -54,7 +54,9 @@ def _parse_args() -> argparse.Namespace:
         help="SQLite snapshot path to create.",
     )
     parser.add_argument("--batch-size", type=int, default=5000)
-    parser.add_argument("--force", action="store_true", help="Overwrite an existing snapshot file.")
+    parser.add_argument(
+        "--force", action="store_true", help="Overwrite an existing snapshot file."
+    )
     return parser.parse_args()
 
 
@@ -65,6 +67,7 @@ async def _copy_table(
     *,
     batch_size: int,
     where=None,
+    row_transform=None,
 ) -> int:
     table = model.__table__
     columns = list(table.columns)
@@ -87,6 +90,8 @@ async def _copy_table(
     stream = await source.stream(stmt.execution_options(yield_per=batch_size))
     async for partition in stream.partitions(batch_size):
         rows = [dict(row._mapping) for row in partition]
+        if row_transform is not None:
+            rows = [row_transform(row) for row in rows]
         if not rows:
             continue
         await dest.execute(insert(table), rows)
@@ -105,6 +110,8 @@ async def _main() -> None:
     from app.database import Base
     import app.models  # noqa: F401 - registers all metadata tables
     from app.models.benchmark_price import BenchmarkPrice
+    from app.models.earnings_event import EarningsEvent
+    from app.models.fundamental_snapshot import FundamentalSnapshot
     from app.models.ohlcv import OHLCVRecord
     from app.models.settings import SystemSetting
     from app.models.ticker import Ticker
@@ -123,8 +130,12 @@ async def _main() -> None:
         connect_args={"server_settings": {"default_transaction_read_only": "on"}},
     )
     dest_engine = create_async_engine(_sqlite_url(output))
-    SourceSession = async_sessionmaker(source_engine, class_=AsyncSession, expire_on_commit=False)
-    DestSession = async_sessionmaker(dest_engine, class_=AsyncSession, expire_on_commit=False)
+    SourceSession = async_sessionmaker(
+        source_engine, class_=AsyncSession, expire_on_commit=False
+    )
+    DestSession = async_sessionmaker(
+        dest_engine, class_=AsyncSession, expire_on_commit=False
+    )
 
     print(f"Source: {_hide_password(source_url)}")
     print(f"Snapshot: {output}")
@@ -135,7 +146,9 @@ async def _main() -> None:
 
         async with SourceSession() as source, DestSession() as dest:
             counts = {
-                "tickers": await _copy_table(source, dest, Ticker, batch_size=args.batch_size),
+                "tickers": await _copy_table(
+                    source, dest, Ticker, batch_size=args.batch_size
+                ),
                 "system_settings": await _copy_table(
                     source,
                     dest,
@@ -152,9 +165,30 @@ async def _main() -> None:
                         SystemSetting.key.like("paper_%"),
                     ),
                 ),
-                "benchmark_prices": await _copy_table(source, dest, BenchmarkPrice, batch_size=args.batch_size),
-                "ohlcv_records": await _copy_table(source, dest, OHLCVRecord, batch_size=args.batch_size),
+                "benchmark_prices": await _copy_table(
+                    source, dest, BenchmarkPrice, batch_size=args.batch_size
+                ),
+                "ohlcv_records": await _copy_table(
+                    source, dest, OHLCVRecord, batch_size=args.batch_size
+                ),
             }
+        # Import-run provenance is operational metadata, not a research input.
+        # Null it so the portable snapshot needs no data_import_runs rows.
+        async with SourceSession() as source, DestSession() as dest:
+            counts["fundamental_snapshots"] = await _copy_table(
+                source,
+                dest,
+                FundamentalSnapshot,
+                batch_size=args.batch_size,
+                row_transform=lambda row: {**row, "import_run_id": None},
+            )
+            counts["earnings_events"] = await _copy_table(
+                source,
+                dest,
+                EarningsEvent,
+                batch_size=args.batch_size,
+                row_transform=lambda row: {**row, "import_run_id": None},
+            )
     finally:
         await source_engine.dispose()
         await dest_engine.dispose()

scripts/run_fundamentals_macbook.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(git rev-parse --show-toplevel)"
+cd "$ROOT"
+
+SNAPSHOT="${1:-backtest_snapshots/fundamentals-backtest.sqlite}"
+WORKERS="${WORKERS:-$(sysctl -n hw.logicalcpu 2>/dev/null || echo 8)}"
+if [[ "$WORKERS" -gt 1 ]]; then
+  WORKERS=$((WORKERS - 1))
+fi
+
+if [[ -x .venv/bin/python ]]; then
+  PYTHON=.venv/bin/python
+else
+  PYTHON="${PYTHON:-python3}"
+fi
+
+if [[ ! -f "$SNAPSHOT" ]]; then
+  echo "Snapshot not found: $SNAPSHOT" >&2
+  exit 1
+fi
+
+STAMP="$(date -u +%Y%m%d-%H%M%S)"
+OUT="reports/fundamentals-overlay-${STAMP}.json"
+
+echo "Snapshot: $SNAPSHOT"
+echo "Workers:  $WORKERS"
+echo "Output:   $OUT"
+
+"$PYTHON" scripts/run_fundamentals_research.py "$SNAPSHOT" \
+  --workers "$WORKERS" \
+  --candidate-cache reports/.cache/fundamentals-candidates.pkl \
+  --fundamentals-cache reports/.cache/fundamentals-scores.pkl \
+  --out "$OUT"
+
+echo
+echo "Bring this file back for review: ${OUT%.json}.zip"