feat(dolt): A2 — DoltHub earnings importer (shadow ingestion)

A SourceImporter that ingests post-no-preference/earnings into earnings_events
for the tracked universe. Shadow by construction (nothing reads earnings_events
until A4).

- earnings_alignment.py: pure calendar<->EPS-history min-cost monotonic DP,
  reused from scripts/import_dolthub_earnings.py with identical constants (not
  extending that one-off script); symbol/session normalization; unit-tested
  against the pinned constants.
- dolt_client.py: async dolt CLI wrapper (pull / current_commit / query_csv via
  asyncio.create_subprocess_exec — never blocks the shared event loop) + disk
  guard before pull.
- dolt_earnings_importer.py: detect_revision = pull + HEAD hash; stage = query
  earnings_calendar + eps_history, dedup, align, map act_symbol->ticker_id
  (normalize both sides so dotted BRK.B joins); promote is destructive
  (delete future dolt_earnings rows + upsert; past never deleted) so validate is
  FAIL-CLOSED — blocks when the staged forward calendar is empty or has collapsed
  below 50% of what's loaded (the forward calendar is the acceptance gate).
- NOTICE: CC BY-SA 4.0 attribution; config: DOLT_BINARY / DOLT_DATA_DIR / etc.

Verified end-to-end against the real 1.68 GB clone (5 tickers: 133 events, 128
paired, forward calendar to 2026-08-26, BRK.B joined). Tests: 9 alignment + 7
importer + 1 skip-guarded real-clone smoke. Full suite 699 passed.

Remaining for A2: wire the daily ~02:30 ET shadow cron — deferred to pair with
the deploy-time dolt install + DOLT_DATA_DIR provisioning.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/unit/test_dolt_earnings_importer.py

@@ -0,0 +1,262 @@
+"""Integration tests for the DoltHub earnings importer, driven through the real
+import framework with a fake dolt client (no subprocess, no clone).
+
+Covers the load-bearing behaviors: symbol-normalized join to the tracked
+universe, calendar<->history pairing (matched → EPS, unmatched → null), the
+destructive-but-safe reschedule/cancel promotion, past rows never deleted, and
+the fail-closed forward-calendar gates that guard the destructive promote.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from datetime import date
+from pathlib import Path
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.database import Base
+import app.models  # noqa: F401
+from app.models.earnings_event import EarningsEvent
+from app.models.ticker import Ticker
+from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
+from app.services.dolt_earnings_importer import DoltEarningsImporter
+
+TODAY = date(2026, 7, 22)
+
+
+@pytest.fixture
+async def engine():
+    fd, path = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
+    async with eng.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    try:
+        yield eng
+    finally:
+        await eng.dispose()
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+
+
+def _factory(engine):
+    return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+class FakeDolt:
+    def __init__(self, calendar, history, commit="c1"):
+        self.calendar = calendar
+        self.history = history
+        self.commit = commit
+        self.pulled = False
+
+    async def pull(self, repo_dir, *, binary):
+        self.pulled = True
+
+    async def current_commit(self, repo_dir, *, binary):
+        return self.commit
+
+    async def query_csv(self, repo_dir, sql, *, binary):
+        if "earnings_calendar" in sql:
+            return self.calendar
+        if "eps_history" in sql:
+            return self.history
+        return []
+
+
+def _cal(symbol, d, when="After market close"):
+    return {"act_symbol": symbol, "date": d, "when": when}
+
+
+def _hist(symbol, pe, reported, estimate):
+    return {"act_symbol": symbol, "period_end_date": pe, "reported": str(reported), "estimate": str(estimate)}
+
+
+def _importer(fake, commit=None):
+    if commit:
+        fake.commit = commit
+    return DoltEarningsImporter(
+        repo_dir="unused", binary="unused", today=TODAY, do_pull=False, dolt=fake
+    )
+
+
+async def _seed_tickers(factory, symbols):
+    async with factory() as s:
+        for sym in symbols:
+            s.add(Ticker(symbol=sym))
+        await s.commit()
+    async with factory() as s:
+        return {sym: tid for tid, sym in (await s.execute(select(Ticker.id, Ticker.symbol))).all()}
+
+
+async def _events(factory):
+    async with factory() as s:
+        rows = (
+            await s.execute(select(EarningsEvent).order_by(EarningsEvent.announce_date))
+        ).scalars().all()
+        return list(rows)
+
+
+# ---------------------------------------------------------------------------
+
+
+async def test_stage_and_promote_basic(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL"])
+    fake = FakeDolt(
+        calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-01")],
+        history=[_hist("AAPL", "2026-03-31", 1.5, 1.4)],  # only the reported quarter
+    )
+    run = await run_import(_importer(fake), engine=engine)
+
+    assert run.status == STATUS_PROMOTED
+    assert run.source_max_date == date(2026, 8, 1)
+
+    events = await _events(factory)
+    assert len(events) == 2
+    past = next(e for e in events if e.announce_date == date(2026, 5, 1))
+    future = next(e for e in events if e.announce_date == date(2026, 8, 1))
+    # past announcement paired to the reported quarter
+    assert past.eps_actual == 1.5 and past.eps_estimate == 1.4
+    assert past.period_end == date(2026, 3, 31) and past.session == "amc"
+    assert past.import_run_id == run.id
+    # future announcement has no results yet → null EPS/period, session kept
+    assert future.eps_actual is None and future.period_end is None
+    assert future.session == "amc"
+
+
+async def test_symbol_normalisation_join(engine):
+    factory = _factory(engine)
+    ids = await _seed_tickers(factory, ["AAPL", "BRK.B"])
+    fake = FakeDolt(
+        calendar=[_cal("AAPL", "2026-08-01"), _cal("BRK.B", "2026-08-05")],  # dotted source symbol
+        history=[],
+    )
+    run = await run_import(_importer(fake), engine=engine)
+    assert run.status == STATUS_PROMOTED
+
+    events = await _events(factory)
+    mapped = {e.ticker_id for e in events}
+    assert mapped == {ids["AAPL"], ids["BRK.B"]}  # dotted BRK.B joined via normalization
+
+
+async def test_reschedule_moves_future_row(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL"])
+
+    fake1 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c1")
+    await run_import(_importer(fake1), engine=engine)
+
+    fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-08")], history=[], commit="c2")  # moved
+    run2 = await run_import(_importer(fake2), engine=engine)
+    assert run2.status == STATUS_PROMOTED
+
+    dates = {e.announce_date for e in await _events(factory)}
+    assert dates == {date(2026, 8, 8)}  # old future date gone, new one present
+
+
+async def test_cancellation_removes_future_row(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL"])
+
+    fake1 = FakeDolt(
+        calendar=[_cal("AAPL", "2026-08-01"), _cal("AAPL", "2026-08-15")], history=[], commit="c1"
+    )
+    await run_import(_importer(fake1), engine=engine)
+
+    fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-15")], history=[], commit="c2")  # 08-01 cancelled
+    await run_import(_importer(fake2), engine=engine)
+
+    dates = {e.announce_date for e in await _events(factory)}
+    assert dates == {date(2026, 8, 15)}
+
+
+async def test_past_row_never_deleted(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL"])
+
+    fake1 = FakeDolt(
+        calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-01")], history=[], commit="c1"
+    )
+    await run_import(_importer(fake1), engine=engine)
+
+    # Second import's calendar omits the past date but keeps a future one.
+    fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c2")
+    await run_import(_importer(fake2), engine=engine)
+
+    dates = {e.announce_date for e in await _events(factory)}
+    assert date(2026, 5, 1) in dates  # past result survived
+    assert date(2026, 8, 1) in dates
+
+
+async def test_validate_fails_when_no_future(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL"])
+    fake = FakeDolt(calendar=[_cal("AAPL", "2026-05-01")], history=[])  # only past
+
+    run = await run_import(_importer(fake), engine=engine)
+
+    assert run.status == STATUS_FAILED
+    assert "future" in (run.error_details or "")
+    assert len(await _events(factory)) == 0  # nothing promoted
+
+
+async def test_validate_fails_on_forward_collapse(engine):
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL", "MSFT", "NVDA", "AMZN"])
+
+    fake1 = FakeDolt(
+        calendar=[
+            _cal("AAPL", "2026-08-01"),
+            _cal("MSFT", "2026-08-02"),
+            _cal("NVDA", "2026-08-03"),
+            _cal("AMZN", "2026-08-04"),
+        ],
+        history=[],
+        commit="c1",
+    )
+    await run_import(_importer(fake1), engine=engine)
+    assert len([e for e in await _events(factory) if e.announce_date > TODAY]) == 4
+
+    # Only one future row now → 1 < 50% of 4 → fail-closed, no destructive wipe.
+    fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c2")
+    run2 = await run_import(_importer(fake2), engine=engine)
+
+    assert run2.status == STATUS_FAILED
+    assert "collapsed" in (run2.error_details or "")
+    assert len([e for e in await _events(factory) if e.announce_date > TODAY]) == 4  # preserved
+
+
+# --- Real-clone smoke test: exercises the actual dolt subprocess + parse + align
+# against the local clone. Skips in CI / anywhere the binary or clone is absent.
+
+_DOLT_BIN = os.environ.get("DOLT_BINARY") or shutil.which("dolt") or r"C:\Program Files\Dolt\bin\dolt.exe"
+_CLONE_DIR = Path("dolt-data/earnings")
+
+
+@pytest.mark.skipif(
+    not (Path(_DOLT_BIN).exists() and _CLONE_DIR.exists()),
+    reason="real dolt binary / earnings clone not available",
+)
+async def test_real_clone_smoke(engine):
+    from app.services import dolt_client
+
+    factory = _factory(engine)
+    await _seed_tickers(factory, ["AAPL", "MSFT", "JPM"])
+    imp = DoltEarningsImporter(
+        repo_dir=_CLONE_DIR, binary=_DOLT_BIN, today=date.today(), do_pull=False, dolt=dolt_client
+    )
+    run = await run_import(imp, engine=engine)
+
+    assert run.status == STATUS_PROMOTED
+    events = await _events(factory)
+    assert events, "no earnings parsed from the real clone"
+    assert any(e.announce_date > date.today() for e in events), "no forward calendar"
+    assert any(e.eps_actual is not None for e in events), "no calendar<->history pairing"

tests/unit/test_earnings_alignment.py

@@ -0,0 +1,104 @@
+"""Unit tests for the pure calendar<->EPS-history alignment.
+
+Anchored on the research script's exact constants (SKIP costs = 45, typical lag
+= 30, session penalty = 3, windows 90/14) so a silently changed constant fails
+here rather than quietly corrupting surprise-history pairing.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+
+from app.services import earnings_alignment as ea
+
+
+def test_normalise_symbol():
+    assert ea.normalise_symbol("bf.b ") == "BF-B"
+    assert ea.normalise_symbol(" aapl") == "AAPL"
+    assert ea.normalise_symbol(None) == ""
+
+
+def test_normalise_session():
+    assert ea.normalise_session("Before market open") == "bmo"
+    assert ea.normalise_session("After market close") == "amc"
+    assert ea.normalise_session("During market hours") == "unknown"
+    assert ea.normalise_session(None) == "unknown"
+    assert ea.normalise_session("") == "unknown"
+
+
+def test_safe_number():
+    assert ea.safe_number("1.5") == 1.5
+    assert ea.safe_number("") is None
+    assert ea.safe_number("not-a-number") is None
+    assert ea.safe_number("nan") is None  # non-finite rejected
+
+
+def test_constants_pinned():
+    assert ea.SKIP_EVENT_COST == 45.0
+    assert ea.SKIP_PERIOD_COST == 45.0
+    assert ea._TYPICAL_ANNOUNCE_LAG_DAYS == 30
+    assert ea._MISSING_SESSION_PENALTY == 3.0
+
+
+def test_match_cost_uses_pinned_lag_and_penalty():
+    period = {"period_end_date": date(2026, 3, 31)}
+    # delta == 30 (typical lag) → base cost 0; known session → no penalty
+    e_known = {"announce_date": date(2026, 4, 30), "session": "amc"}
+    assert ea.match_cost(e_known, period) == 0.0
+    # unknown session adds the penalty
+    e_unknown = {"announce_date": date(2026, 4, 30), "session": "unknown"}
+    assert ea.match_cost(e_unknown, period) == 3.0
+    # delta 45 → |45-30| == 15
+    e_far = {"announce_date": date(2026, 5, 15), "session": "amc"}
+    assert ea.match_cost(e_far, period) == 15.0
+
+
+def test_dedup_calendar_prefers_known_session():
+    rows = [
+        {"symbol": "AAPL", "announce_date": date(2026, 5, 1), "session": "unknown"},
+        {"symbol": "AAPL", "announce_date": date(2026, 5, 1), "session": "amc"},
+    ]
+    grouped, stats = ea.dedup_calendar(rows)
+    assert stats["duplicate_rows"] == 1
+    assert grouped["AAPL"][0]["session"] == "amc"
+
+
+def test_dedup_history_prefers_fuller_row():
+    rows = [
+        {"symbol": "AAPL", "period_end_date": date(2026, 3, 31), "eps_actual": 2.0, "eps_estimate": None},
+        {"symbol": "AAPL", "period_end_date": date(2026, 3, 31), "eps_actual": 2.0, "eps_estimate": 1.9},
+    ]
+    grouped, stats = ea.dedup_history(rows)
+    assert stats["duplicate_rows"] == 1
+    kept = grouped["AAPL"][0]
+    assert kept["eps_actual"] == 2.0 and kept["eps_estimate"] == 1.9
+
+
+def _events(*days):
+    return [{"announce_date": d, "session": "amc"} for d in days]
+
+
+def _periods(*days):
+    return [{"period_end_date": d, "eps_actual": 1.0, "eps_estimate": 0.9} for d in days]
+
+
+def test_align_matches_monotonic_pairs():
+    # two announcements ~30d after two quarter ends
+    events = _events(date(2026, 4, 30), date(2026, 7, 30))
+    periods = _periods(date(2026, 3, 31), date(2026, 6, 30))
+    matches, um_events, um_periods = ea.align_symbol(
+        events, periods, max_lag_days=90, max_lead_days=14
+    )
+    assert matches == [(0, 0), (1, 1)]
+    assert um_events == [] and um_periods == []
+
+
+def test_align_leaves_out_of_window_unmatched():
+    # announcement 200 days after the only period end → outside the 90d window
+    events = _events(date(2026, 10, 17))
+    periods = _periods(date(2026, 3, 31))
+    matches, um_events, um_periods = ea.align_symbol(
+        events, periods, max_lag_days=90, max_lead_days=14
+    )
+    assert matches == []
+    assert um_events == [0] and um_periods == [0]