feat(dolt): A2 — DoltHub earnings importer (shadow ingestion)

A SourceImporter that ingests post-no-preference/earnings into earnings_events for the tracked universe. Shadow by construction (nothing reads earnings_events until A4). - earnings_alignment.py: pure calendar<->EPS-history min-cost monotonic DP, reused from scripts/import_dolthub_earnings.py with identical constants (not extending that one-off script); symbol/session normalization; unit-tested against the pinned constants. - dolt_client.py: async dolt CLI wrapper (pull / current_commit / query_csv via asyncio.create_subprocess_exec — never blocks the shared event loop) + disk guard before pull. - dolt_earnings_importer.py: detect_revision = pull + HEAD hash; stage = query earnings_calendar + eps_history, dedup, align, map act_symbol->ticker_id (normalize both sides so dotted BRK.B joins); promote is destructive (delete future dolt_earnings rows + upsert; past never deleted) so validate is FAIL-CLOSED — blocks when the staged forward calendar is empty or has collapsed below 50% of what's loaded (the forward calendar is the acceptance gate). - NOTICE: CC BY-SA 4.0 attribution; config: DOLT_BINARY / DOLT_DATA_DIR / etc. Verified end-to-end against the real 1.68 GB clone (5 tickers: 133 events, 128 paired, forward calendar to 2026-08-26, BRK.B joined). Tests: 9 alignment + 7 importer + 1 skip-guarded real-clone smoke. Full suite 699 passed. Remaining for A2: wire the daily ~02:30 ET shadow cron — deferred to pair with the deploy-time dolt install + DOLT_DATA_DIR provisioning. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 10:55:27 +02:00
parent e5a62ca648
commit 54ae8ba153
8 changed files with 1017 additions and 0 deletions
@@ -0,0 +1,207 @@
+"""Pure calendar<->EPS-history alignment for the DoltHub earnings source.
+
+The earnings repo keeps the announcement calendar (`earnings_calendar`) and the
+reported/estimate EPS history (`eps_history`) in separate tables with no shared
+key — the calendar has announce dates, the history has period-end dates. This
+module reproduces the research importer's **minimum-cost monotonic alignment**
+(`scripts/import_dolthub_earnings.py`) as pure, DB-free, unit-testable functions
+so the production importer can reuse it without extending that one-off script.
+
+Constants and cost function are kept identical to the research script; the DP is
+what pairs each announcement with the quarter it reported, tolerating gaps on
+either side. Do not tune these without re-validating surprise-history pairing.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from datetime import date
+from typing import Any
+
+# Alignment costs — identical to scripts/import_dolthub_earnings.py.
+SKIP_EVENT_COST = 45.0
+SKIP_PERIOD_COST = 45.0
+_TYPICAL_ANNOUNCE_LAG_DAYS = 30  # announcements land ~a month after period end
+_MISSING_SESSION_PENALTY = 3.0
+
+# Session normalization → the three values the schema/API promise.
+_SESSION_ALIASES = {
+    "before market open": "bmo",
+    "before open": "bmo",
+    "bmo": "bmo",
+    "after market close": "amc",
+    "after close": "amc",
+    "amc": "amc",
+}
+
+
+def normalise_symbol(value: Any) -> str:
+    """Upper-case, trim, and map dots to dashes so the DoltHub `act_symbol`
+    (`BF.B`) and the app's `tickers.symbol` join after the same normalization."""
+    return str(value or "").strip().upper().replace(".", "-")
+
+
+def normalise_session(value: Any) -> str:
+    """Map the source `when` text to bmo | amc | unknown. Anything not clearly a
+    pre-open or post-close session (including 'during market hours' and blanks)
+    collapses to 'unknown' — the schema/API only promise those three."""
+    cleaned = str(value or "").strip().lower().replace("_", " ").replace("-", " ")
+    return _SESSION_ALIASES.get(cleaned, "unknown")
+
+
+def safe_number(value: Any) -> float | None:
+    if value is None or str(value).strip() == "":
+        return None
+    try:
+        result = float(value)
+    except (TypeError, ValueError):
+        return None
+    return result if math.isfinite(result) else None
+
+
+def dedup_calendar(
+    rows: list[dict[str, Any]],
+) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
+    """Collapse to one row per (symbol, announce_date), preferring a known
+    session over 'unknown'. Rows must be pre-parsed:
+    {symbol, announce_date: date, session}. Returns {symbol: [events sorted by
+    date]} and dedup stats."""
+    by_key: dict[tuple[str, date], dict[str, Any]] = {}
+    duplicate_rows = 0
+    restated_rows = 0
+    for row in rows:
+        key = (row["symbol"], row["announce_date"])
+        previous = by_key.get(key)
+        if previous is None:
+            by_key[key] = row
+            continue
+        duplicate_rows += 1
+        prev_known = previous["session"] != "unknown"
+        new_known = row["session"] != "unknown"
+        if prev_known and new_known and previous["session"] != row["session"]:
+            restated_rows += 1
+        # Prefer a row that carries a known session.
+        if new_known:
+            by_key[key] = row
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in by_key.values():
+        grouped[row["symbol"]].append(row)
+    for events in grouped.values():
+        events.sort(key=lambda item: item["announce_date"])
+    return grouped, {
+        "deduped_rows": len(by_key),
+        "duplicate_rows": duplicate_rows,
+        "restated_rows": restated_rows,
+    }
+
+
+def dedup_history(
+    rows: list[dict[str, Any]],
+) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
+    """Collapse to one row per (symbol, period_end_date), preferring the row with
+    more non-null EPS fields. Rows must be pre-parsed:
+    {symbol, period_end_date: date, eps_actual, eps_estimate}."""
+    fields = ("eps_actual", "eps_estimate")
+    by_key: dict[tuple[str, date], dict[str, Any]] = {}
+    duplicate_rows = 0
+    restated_rows = 0
+    for row in rows:
+        key = (row["symbol"], row["period_end_date"])
+        previous = by_key.get(key)
+        if previous is None:
+            by_key[key] = row
+            continue
+        duplicate_rows += 1
+        if any(
+            previous.get(f) is not None
+            and row.get(f) is not None
+            and previous[f] != row[f]
+            for f in fields
+        ):
+            restated_rows += 1
+        prev_score = sum(previous.get(f) is not None for f in fields)
+        new_score = sum(row.get(f) is not None for f in fields)
+        if new_score >= prev_score:
+            by_key[key] = row
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in by_key.values():
+        grouped[row["symbol"]].append(row)
+    for periods in grouped.values():
+        periods.sort(key=lambda item: item["period_end_date"])
+    return grouped, {
+        "deduped_rows": len(by_key),
+        "duplicate_rows": duplicate_rows,
+        "restated_rows": restated_rows,
+    }
+
+
+def match_cost(event: dict[str, Any], period: dict[str, Any]) -> float:
+    delta = (event["announce_date"] - period["period_end_date"]).days
+    penalty = _MISSING_SESSION_PENALTY if event.get("session") == "unknown" else 0.0
+    return float(abs(delta - _TYPICAL_ANNOUNCE_LAG_DAYS)) + penalty
+
+
+def align_symbol(
+    events: list[dict[str, Any]],
+    periods: list[dict[str, Any]],
+    *,
+    max_lag_days: int,
+    max_lead_days: int,
+) -> tuple[list[tuple[int, int]], list[int], list[int]]:
+    """Minimum-cost monotonic calendar-to-period alignment for one symbol.
+
+    Both lists must be sorted ascending (by announce_date / period_end_date). A
+    match is allowed only when ``-max_lead_days <= announce_date - period_end <=
+    max_lag_days``. Returns (matches, unmatched_event_indices,
+    unmatched_period_indices).
+    """
+    n_events = len(events)
+    n_periods = len(periods)
+    scores = [[0.0] * (n_periods + 1) for _ in range(n_events + 1)]
+    choices = [[""] * (n_periods + 1) for _ in range(n_events + 1)]
+    for e in range(n_events - 1, -1, -1):
+        scores[e][n_periods] = scores[e + 1][n_periods] + SKIP_EVENT_COST
+        choices[e][n_periods] = "event"
+    for p in range(n_periods - 1, -1, -1):
+        scores[n_events][p] = scores[n_events][p + 1] + SKIP_PERIOD_COST
+        choices[n_events][p] = "period"
+
+    for e in range(n_events - 1, -1, -1):
+        for p in range(n_periods - 1, -1, -1):
+            options = [
+                (scores[e + 1][p] + SKIP_EVENT_COST, 2, "event"),
+                (scores[e][p + 1] + SKIP_PERIOD_COST, 1, "period"),
+            ]
+            delta = (events[e]["announce_date"] - periods[p]["period_end_date"]).days
+            if -max_lead_days <= delta <= max_lag_days:
+                options.append(
+                    (scores[e + 1][p + 1] + match_cost(events[e], periods[p]), 0, "match")
+                )
+            score, _, choice = min(options)
+            scores[e][p] = score
+            choices[e][p] = choice
+
+    matches: list[tuple[int, int]] = []
+    unmatched_events: list[int] = []
+    unmatched_periods: list[int] = []
+    e = p = 0
+    while e < n_events or p < n_periods:
+        if e >= n_events:
+            unmatched_periods.extend(range(p, n_periods))
+            break
+        if p >= n_periods:
+            unmatched_events.extend(range(e, n_events))
+            break
+        choice = choices[e][p]
+        if choice == "match":
+            matches.append((e, p))
+            e += 1
+            p += 1
+        elif choice == "period":
+            unmatched_periods.append(p)
+            p += 1
+        else:
+            unmatched_events.append(e)
+            e += 1
+    return matches, unmatched_events, unmatched_periods