feat(dolt): A2 — DoltHub earnings importer (shadow ingestion)
A SourceImporter that ingests post-no-preference/earnings into earnings_events for the tracked universe. Shadow by construction (nothing reads earnings_events until A4). - earnings_alignment.py: pure calendar<->EPS-history min-cost monotonic DP, reused from scripts/import_dolthub_earnings.py with identical constants (not extending that one-off script); symbol/session normalization; unit-tested against the pinned constants. - dolt_client.py: async dolt CLI wrapper (pull / current_commit / query_csv via asyncio.create_subprocess_exec — never blocks the shared event loop) + disk guard before pull. - dolt_earnings_importer.py: detect_revision = pull + HEAD hash; stage = query earnings_calendar + eps_history, dedup, align, map act_symbol->ticker_id (normalize both sides so dotted BRK.B joins); promote is destructive (delete future dolt_earnings rows + upsert; past never deleted) so validate is FAIL-CLOSED — blocks when the staged forward calendar is empty or has collapsed below 50% of what's loaded (the forward calendar is the acceptance gate). - NOTICE: CC BY-SA 4.0 attribution; config: DOLT_BINARY / DOLT_DATA_DIR / etc. Verified end-to-end against the real 1.68 GB clone (5 tickers: 133 events, 128 paired, forward calendar to 2026-08-26, BRK.B joined). Tests: 9 alignment + 7 importer + 1 skip-guarded real-clone smoke. Full suite 699 passed. Remaining for A2: wire the daily ~02:30 ET shadow cron — deferred to pair with the deploy-time dolt install + DOLT_DATA_DIR provisioning. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -37,6 +37,16 @@ class Settings(BaseSettings):
|
||||
# Fundamentals Provider — Alpha Vantage (optional fallback)
|
||||
alpha_vantage_api_key: str = ""
|
||||
|
||||
# Dolt bulk-data — local clone of post-no-preference/earnings (workstream A).
|
||||
# dolt_binary: full path when not on PATH (dev/Windows install). dolt_data_dir
|
||||
# holds the clones; in production it MUST be outside the deploy tree (deploy is
|
||||
# rsync --delete) — set DOLT_DATA_DIR to a persistent path. The earnings clone
|
||||
# lives at <dolt_data_dir>/<dolt_earnings_subdir>.
|
||||
dolt_binary: str = "dolt"
|
||||
dolt_data_dir: str = "dolt-data"
|
||||
dolt_earnings_subdir: str = "earnings"
|
||||
dolt_min_free_disk_gb: float = 2.0
|
||||
|
||||
# Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
||||
# the volatility (P5) and credit-spread (F2) signals are reported as n/a.
|
||||
fred_api_key: str = ""
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Minimal async client for a local Dolt clone.
|
||||
|
||||
The application never runs a long-lived Dolt sql-server; it shells out to the
|
||||
`dolt` CLI against a persistent clone and reads results as CSV. Every call goes
|
||||
through ``asyncio.create_subprocess_exec`` because the scheduler shares one event
|
||||
loop with the API (`app/scheduler.py:73`) — a blocking `subprocess.run` here
|
||||
would stall request handling.
|
||||
|
||||
Production keeps the clone in ``DOLT_DATA_DIR`` outside the deploy tree; the
|
||||
binary path and data dir are configured (see ``app/config.py``). Read via
|
||||
``dolt sql -r csv``; refresh with ``pull`` and record the resulting commit hash
|
||||
as the import revision.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DoltError(RuntimeError):
|
||||
"""A dolt subprocess exited non-zero."""
|
||||
|
||||
|
||||
async def _run(binary: str, args: list[str], *, cwd: Path) -> str:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
binary,
|
||||
*args,
|
||||
cwd=str(cwd),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
raise DoltError(
|
||||
f"dolt {' '.join(args)} failed ({proc.returncode}): "
|
||||
f"{stderr.decode('utf-8', 'replace').strip()[:500]}"
|
||||
)
|
||||
return stdout.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def ensure_free_disk(path: Path, min_free_gb: float) -> None:
|
||||
"""Raise if free space at ``path`` is below the threshold (checked before a
|
||||
pull that could grow the clone). Uses the nearest existing ancestor so it
|
||||
works before the clone dir exists."""
|
||||
probe = path
|
||||
while not probe.exists() and probe.parent != probe:
|
||||
probe = probe.parent
|
||||
free_gb = shutil.disk_usage(probe).free / (1024**3)
|
||||
if free_gb < min_free_gb:
|
||||
raise DoltError(
|
||||
f"insufficient disk for dolt at {path}: {free_gb:.1f} GB free "
|
||||
f"< {min_free_gb:.1f} GB required"
|
||||
)
|
||||
|
||||
|
||||
async def pull(repo_dir: Path, *, binary: str) -> None:
|
||||
"""`dolt pull` the persistent clone to the latest upstream revision."""
|
||||
await _run(binary, ["pull"], cwd=repo_dir)
|
||||
|
||||
|
||||
async def current_commit(repo_dir: Path, *, binary: str) -> str:
|
||||
"""The HEAD commit hash of the clone — used as the import revision."""
|
||||
rows = await query_csv(
|
||||
repo_dir, "SELECT commit_hash FROM dolt_log ORDER BY date DESC LIMIT 1", binary=binary
|
||||
)
|
||||
if not rows or not rows[0].get("commit_hash"):
|
||||
raise DoltError("could not read HEAD commit hash from dolt_log")
|
||||
return rows[0]["commit_hash"]
|
||||
|
||||
|
||||
async def query_csv(repo_dir: Path, sql: str, *, binary: str) -> list[dict[str, str]]:
|
||||
"""Run a read query and parse the CSV result into a list of dict rows."""
|
||||
out = await _run(binary, ["sql", "-q", sql, "-r", "csv"], cwd=repo_dir)
|
||||
if not out.strip():
|
||||
return []
|
||||
return list(csv.DictReader(io.StringIO(out)))
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Production importer for the DoltHub post-no-preference/earnings calendar.
|
||||
|
||||
A ``SourceImporter`` (see ``app/services/data_import.py``) that pulls the local
|
||||
Dolt clone, aligns the announcement calendar to the EPS history with the pure DP
|
||||
in ``earnings_alignment`` (reused from the research script, not extending it),
|
||||
and writes ``earnings_events`` for the tracked universe.
|
||||
|
||||
Shadow by construction: nothing reads ``earnings_events`` until the API/panel
|
||||
lands (A4), so writing it does not touch production behavior.
|
||||
|
||||
**Promotion is destructive** — future-dated rows for this source are deleted and
|
||||
re-inserted every run so reschedules/cancellations never linger. The forward
|
||||
calendar is the project's acceptance gate, so ``validate`` is fail-closed: it
|
||||
blocks promotion when the staged future set is empty or has collapsed relative
|
||||
to what's already loaded.
|
||||
|
||||
Attribution: the earnings data is CC BY-SA 4.0 from post-no-preference/earnings.
|
||||
See the repo ``NOTICE``. Internal use only — no redistribution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, delete, func, select
|
||||
|
||||
from app.config import settings
|
||||
from app.database import insert_for_session
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import dolt_client, earnings_alignment
|
||||
from app.services.data_import import ValidationResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "dolt_earnings"
|
||||
|
||||
# Earliest announcement date to import (matches the research backfill window).
|
||||
WINDOW_START = date(2020, 1, 22)
|
||||
# Alignment tolerances (research defaults): an announcement may lead its period
|
||||
# end by up to 14 days or lag it by up to 90.
|
||||
MAX_LAG_DAYS = 90
|
||||
MAX_LEAD_DAYS = 14
|
||||
# Fail promotion if the staged forward calendar drops below this fraction of the
|
||||
# currently-loaded forward calendar (guards the destructive re-insert against a
|
||||
# partial parse / symbol-mapping regression).
|
||||
MIN_FUTURE_RATIO = 0.5
|
||||
|
||||
_CAL_SQL = (
|
||||
"SELECT act_symbol, `date`, `when` FROM earnings_calendar "
|
||||
f"WHERE `date` >= '{WINDOW_START.isoformat()}'"
|
||||
)
|
||||
_HIST_SQL = (
|
||||
"SELECT act_symbol, period_end_date, reported, estimate FROM eps_history "
|
||||
f"WHERE period_end_date >= '{(WINDOW_START.replace(year=WINDOW_START.year - 1)).isoformat()}'"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedEarnings:
|
||||
rows: list[dict[str, Any]]
|
||||
stats: dict[str, Any] = field(default_factory=dict)
|
||||
future_count: int = 0
|
||||
max_announce_date: date | None = None
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class DoltEarningsImporter:
|
||||
source = SOURCE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repo_dir: Path | str | None = None,
|
||||
binary: str | None = None,
|
||||
today: date | None = None,
|
||||
do_pull: bool = True,
|
||||
dolt: Any = dolt_client,
|
||||
) -> None:
|
||||
self.repo_dir = Path(
|
||||
repo_dir
|
||||
or (Path(settings.dolt_data_dir) / settings.dolt_earnings_subdir)
|
||||
)
|
||||
self.binary = binary or settings.dolt_binary
|
||||
self.today = today or _now().date()
|
||||
self.do_pull = do_pull
|
||||
self._dolt = dolt # injectable for tests
|
||||
|
||||
# -- SourceImporter protocol -------------------------------------------
|
||||
|
||||
async def detect_revision(self, db) -> str | None:
|
||||
if self.do_pull:
|
||||
dolt_client.ensure_free_disk(self.repo_dir, settings.dolt_min_free_disk_gb)
|
||||
await self._dolt.pull(self.repo_dir, binary=self.binary)
|
||||
return await self._dolt.current_commit(self.repo_dir, binary=self.binary)
|
||||
|
||||
async def stage(self, db) -> StagedEarnings:
|
||||
universe = await self._load_universe(db) # {normalised symbol: ticker_id}
|
||||
|
||||
cal_raw = await self._dolt.query_csv(self.repo_dir, _CAL_SQL, binary=self.binary)
|
||||
hist_raw = await self._dolt.query_csv(self.repo_dir, _HIST_SQL, binary=self.binary)
|
||||
_require_columns(cal_raw, {"act_symbol", "date", "when"}, "earnings_calendar")
|
||||
_require_columns(
|
||||
hist_raw, {"act_symbol", "period_end_date", "reported", "estimate"}, "eps_history"
|
||||
)
|
||||
|
||||
cal_parsed = _parse_calendar(cal_raw, universe)
|
||||
hist_parsed = _parse_history(hist_raw, universe)
|
||||
calendar, cal_stats = earnings_alignment.dedup_calendar(cal_parsed)
|
||||
history, hist_stats = earnings_alignment.dedup_history(hist_parsed)
|
||||
|
||||
period_lower = WINDOW_START.replace(year=WINDOW_START.year - 1)
|
||||
rows: list[dict[str, Any]] = []
|
||||
matched = unmatched = 0
|
||||
for symbol, events in calendar.items():
|
||||
ticker_id = universe[symbol]
|
||||
periods = [
|
||||
p for p in history.get(symbol, []) if p["period_end_date"] >= period_lower
|
||||
]
|
||||
matches, unmatched_events, _ = earnings_alignment.align_symbol(
|
||||
events, periods, max_lag_days=MAX_LAG_DAYS, max_lead_days=MAX_LEAD_DAYS
|
||||
)
|
||||
matched += len(matches)
|
||||
unmatched += len(unmatched_events)
|
||||
matched_by_event = {e: p for e, p in matches}
|
||||
for e_idx, event in enumerate(events):
|
||||
p_idx = matched_by_event.get(e_idx)
|
||||
period = periods[p_idx] if p_idx is not None else None
|
||||
rows.append(
|
||||
{
|
||||
"ticker_id": ticker_id,
|
||||
"symbol": symbol,
|
||||
"announce_date": event["announce_date"],
|
||||
"session": event["session"],
|
||||
"period_end": period["period_end_date"] if period else None,
|
||||
"eps_estimate": period["eps_estimate"] if period else None,
|
||||
"eps_actual": period["eps_actual"] if period else None,
|
||||
}
|
||||
)
|
||||
|
||||
future_rows = [r for r in rows if r["announce_date"] > self.today]
|
||||
tickers_with_future = {r["ticker_id"] for r in future_rows}
|
||||
stats = {
|
||||
"calendar": cal_stats,
|
||||
"eps_history": hist_stats,
|
||||
"universe_size": len(universe),
|
||||
"symbols_with_calendar": len(calendar),
|
||||
"matched_events": matched,
|
||||
"unmatched_events": unmatched,
|
||||
"tracked_tickers_with_future_date": len(tickers_with_future),
|
||||
}
|
||||
return StagedEarnings(
|
||||
rows=rows,
|
||||
stats=stats,
|
||||
future_count=len(future_rows),
|
||||
max_announce_date=max((r["announce_date"] for r in rows), default=None),
|
||||
)
|
||||
|
||||
async def validate(self, db, staged: StagedEarnings) -> ValidationResult:
|
||||
messages: list[str] = []
|
||||
|
||||
# Fail-closed forward-calendar protection (promote deletes+reinserts it).
|
||||
if staged.future_count == 0:
|
||||
messages.append("no future-dated earnings rows staged")
|
||||
current_future = await self._current_future_count(db)
|
||||
if current_future > 0 and staged.future_count < current_future * MIN_FUTURE_RATIO:
|
||||
messages.append(
|
||||
f"forward calendar collapsed: staged {staged.future_count} future rows "
|
||||
f"< {MIN_FUTURE_RATIO:.0%} of current {current_future}"
|
||||
)
|
||||
|
||||
keys = [(r["ticker_id"], r["announce_date"]) for r in staged.rows]
|
||||
if len(keys) != len(set(keys)):
|
||||
messages.append("duplicate (ticker_id, announce_date) in staged set")
|
||||
|
||||
summary = {
|
||||
**staged.stats,
|
||||
"staged_rows": len(staged.rows),
|
||||
"future_rows": staged.future_count,
|
||||
"current_future_rows": current_future,
|
||||
}
|
||||
return ValidationResult(
|
||||
ok=not messages,
|
||||
summary=summary,
|
||||
source_max_date=staged.max_announce_date,
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
async def promote(self, db, staged: StagedEarnings, run_id: int) -> dict[str, int]:
|
||||
# Rescheduling: drop this source's future rows, then upsert the staged
|
||||
# set. Past rows (results) are never deleted; moved/cancelled future
|
||||
# dates simply don't reappear.
|
||||
deleted = (
|
||||
await db.execute(
|
||||
delete(EarningsEvent).where(
|
||||
EarningsEvent.source == SOURCE,
|
||||
EarningsEvent.announce_date > self.today,
|
||||
)
|
||||
)
|
||||
).rowcount or 0
|
||||
|
||||
now = _now()
|
||||
for r in staged.rows:
|
||||
stmt = insert_for_session(db, EarningsEvent).values(
|
||||
ticker_id=r["ticker_id"],
|
||||
announce_date=r["announce_date"],
|
||||
session=r["session"],
|
||||
period_end=r["period_end"],
|
||||
eps_estimate=r["eps_estimate"],
|
||||
eps_actual=r["eps_actual"],
|
||||
source=SOURCE,
|
||||
import_run_id=run_id,
|
||||
created_at=now,
|
||||
)
|
||||
# Preserve a non-null prior EPS/period-end if a re-pairing comes back
|
||||
# null; prefer a known session over 'unknown'.
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["ticker_id", "announce_date"],
|
||||
set_={
|
||||
"session": case(
|
||||
(stmt.excluded.session != "unknown", stmt.excluded.session),
|
||||
else_=EarningsEvent.session,
|
||||
),
|
||||
"period_end": func.coalesce(
|
||||
stmt.excluded.period_end, EarningsEvent.period_end
|
||||
),
|
||||
"eps_estimate": func.coalesce(
|
||||
stmt.excluded.eps_estimate, EarningsEvent.eps_estimate
|
||||
),
|
||||
"eps_actual": func.coalesce(
|
||||
stmt.excluded.eps_actual, EarningsEvent.eps_actual
|
||||
),
|
||||
"source": stmt.excluded.source,
|
||||
"import_run_id": stmt.excluded.import_run_id,
|
||||
},
|
||||
)
|
||||
await db.execute(stmt)
|
||||
|
||||
return {"deleted_future": int(deleted), "upserted": len(staged.rows)}
|
||||
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _load_universe(self, db) -> dict[str, int]:
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all()
|
||||
return {
|
||||
earnings_alignment.normalise_symbol(symbol): tid
|
||||
for tid, symbol in rows
|
||||
if symbol
|
||||
}
|
||||
|
||||
async def _current_future_count(self, db) -> int:
|
||||
return (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(EarningsEvent)
|
||||
.where(
|
||||
EarningsEvent.source == SOURCE,
|
||||
EarningsEvent.announce_date > self.today,
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
|
||||
def _require_columns(rows: list[dict[str, str]], required: set[str], table: str) -> None:
|
||||
"""Upstream schema-change gate: a missing column stops the run (→ failed)."""
|
||||
if not rows:
|
||||
return
|
||||
present = set(rows[0].keys())
|
||||
missing = required - present
|
||||
if missing:
|
||||
raise ValueError(f"{table}: upstream schema change, missing columns {sorted(missing)}")
|
||||
|
||||
|
||||
def _parse_calendar(raw: list[dict[str, str]], universe: dict[str, int]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for row in raw:
|
||||
symbol = earnings_alignment.normalise_symbol(row.get("act_symbol"))
|
||||
raw_date = str(row.get("date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
announce_date = date.fromisoformat(raw_date)
|
||||
if announce_date < WINDOW_START:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"announce_date": announce_date,
|
||||
"session": earnings_alignment.normalise_session(row.get("when")),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _parse_history(raw: list[dict[str, str]], universe: dict[str, int]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for row in raw:
|
||||
symbol = earnings_alignment.normalise_symbol(row.get("act_symbol"))
|
||||
raw_date = str(row.get("period_end_date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"period_end_date": date.fromisoformat(raw_date),
|
||||
"eps_actual": earnings_alignment.safe_number(row.get("reported")),
|
||||
"eps_estimate": earnings_alignment.safe_number(row.get("estimate")),
|
||||
}
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Pure calendar<->EPS-history alignment for the DoltHub earnings source.
|
||||
|
||||
The earnings repo keeps the announcement calendar (`earnings_calendar`) and the
|
||||
reported/estimate EPS history (`eps_history`) in separate tables with no shared
|
||||
key — the calendar has announce dates, the history has period-end dates. This
|
||||
module reproduces the research importer's **minimum-cost monotonic alignment**
|
||||
(`scripts/import_dolthub_earnings.py`) as pure, DB-free, unit-testable functions
|
||||
so the production importer can reuse it without extending that one-off script.
|
||||
|
||||
Constants and cost function are kept identical to the research script; the DP is
|
||||
what pairs each announcement with the quarter it reported, tolerating gaps on
|
||||
either side. Do not tune these without re-validating surprise-history pairing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
# Alignment costs — identical to scripts/import_dolthub_earnings.py.
|
||||
SKIP_EVENT_COST = 45.0
|
||||
SKIP_PERIOD_COST = 45.0
|
||||
_TYPICAL_ANNOUNCE_LAG_DAYS = 30 # announcements land ~a month after period end
|
||||
_MISSING_SESSION_PENALTY = 3.0
|
||||
|
||||
# Session normalization → the three values the schema/API promise.
|
||||
_SESSION_ALIASES = {
|
||||
"before market open": "bmo",
|
||||
"before open": "bmo",
|
||||
"bmo": "bmo",
|
||||
"after market close": "amc",
|
||||
"after close": "amc",
|
||||
"amc": "amc",
|
||||
}
|
||||
|
||||
|
||||
def normalise_symbol(value: Any) -> str:
|
||||
"""Upper-case, trim, and map dots to dashes so the DoltHub `act_symbol`
|
||||
(`BF.B`) and the app's `tickers.symbol` join after the same normalization."""
|
||||
return str(value or "").strip().upper().replace(".", "-")
|
||||
|
||||
|
||||
def normalise_session(value: Any) -> str:
|
||||
"""Map the source `when` text to bmo | amc | unknown. Anything not clearly a
|
||||
pre-open or post-close session (including 'during market hours' and blanks)
|
||||
collapses to 'unknown' — the schema/API only promise those three."""
|
||||
cleaned = str(value or "").strip().lower().replace("_", " ").replace("-", " ")
|
||||
return _SESSION_ALIASES.get(cleaned, "unknown")
|
||||
|
||||
|
||||
def safe_number(value: Any) -> float | None:
|
||||
if value is None or str(value).strip() == "":
|
||||
return None
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return result if math.isfinite(result) else None
|
||||
|
||||
|
||||
def dedup_calendar(
|
||||
rows: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
"""Collapse to one row per (symbol, announce_date), preferring a known
|
||||
session over 'unknown'. Rows must be pre-parsed:
|
||||
{symbol, announce_date: date, session}. Returns {symbol: [events sorted by
|
||||
date]} and dedup stats."""
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
for row in rows:
|
||||
key = (row["symbol"], row["announce_date"])
|
||||
previous = by_key.get(key)
|
||||
if previous is None:
|
||||
by_key[key] = row
|
||||
continue
|
||||
duplicate_rows += 1
|
||||
prev_known = previous["session"] != "unknown"
|
||||
new_known = row["session"] != "unknown"
|
||||
if prev_known and new_known and previous["session"] != row["session"]:
|
||||
restated_rows += 1
|
||||
# Prefer a row that carries a known session.
|
||||
if new_known:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for events in grouped.values():
|
||||
events.sort(key=lambda item: item["announce_date"])
|
||||
return grouped, {
|
||||
"deduped_rows": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def dedup_history(
|
||||
rows: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
"""Collapse to one row per (symbol, period_end_date), preferring the row with
|
||||
more non-null EPS fields. Rows must be pre-parsed:
|
||||
{symbol, period_end_date: date, eps_actual, eps_estimate}."""
|
||||
fields = ("eps_actual", "eps_estimate")
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
for row in rows:
|
||||
key = (row["symbol"], row["period_end_date"])
|
||||
previous = by_key.get(key)
|
||||
if previous is None:
|
||||
by_key[key] = row
|
||||
continue
|
||||
duplicate_rows += 1
|
||||
if any(
|
||||
previous.get(f) is not None
|
||||
and row.get(f) is not None
|
||||
and previous[f] != row[f]
|
||||
for f in fields
|
||||
):
|
||||
restated_rows += 1
|
||||
prev_score = sum(previous.get(f) is not None for f in fields)
|
||||
new_score = sum(row.get(f) is not None for f in fields)
|
||||
if new_score >= prev_score:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for periods in grouped.values():
|
||||
periods.sort(key=lambda item: item["period_end_date"])
|
||||
return grouped, {
|
||||
"deduped_rows": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def match_cost(event: dict[str, Any], period: dict[str, Any]) -> float:
|
||||
delta = (event["announce_date"] - period["period_end_date"]).days
|
||||
penalty = _MISSING_SESSION_PENALTY if event.get("session") == "unknown" else 0.0
|
||||
return float(abs(delta - _TYPICAL_ANNOUNCE_LAG_DAYS)) + penalty
|
||||
|
||||
|
||||
def align_symbol(
|
||||
events: list[dict[str, Any]],
|
||||
periods: list[dict[str, Any]],
|
||||
*,
|
||||
max_lag_days: int,
|
||||
max_lead_days: int,
|
||||
) -> tuple[list[tuple[int, int]], list[int], list[int]]:
|
||||
"""Minimum-cost monotonic calendar-to-period alignment for one symbol.
|
||||
|
||||
Both lists must be sorted ascending (by announce_date / period_end_date). A
|
||||
match is allowed only when ``-max_lead_days <= announce_date - period_end <=
|
||||
max_lag_days``. Returns (matches, unmatched_event_indices,
|
||||
unmatched_period_indices).
|
||||
"""
|
||||
n_events = len(events)
|
||||
n_periods = len(periods)
|
||||
scores = [[0.0] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
choices = [[""] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
for e in range(n_events - 1, -1, -1):
|
||||
scores[e][n_periods] = scores[e + 1][n_periods] + SKIP_EVENT_COST
|
||||
choices[e][n_periods] = "event"
|
||||
for p in range(n_periods - 1, -1, -1):
|
||||
scores[n_events][p] = scores[n_events][p + 1] + SKIP_PERIOD_COST
|
||||
choices[n_events][p] = "period"
|
||||
|
||||
for e in range(n_events - 1, -1, -1):
|
||||
for p in range(n_periods - 1, -1, -1):
|
||||
options = [
|
||||
(scores[e + 1][p] + SKIP_EVENT_COST, 2, "event"),
|
||||
(scores[e][p + 1] + SKIP_PERIOD_COST, 1, "period"),
|
||||
]
|
||||
delta = (events[e]["announce_date"] - periods[p]["period_end_date"]).days
|
||||
if -max_lead_days <= delta <= max_lag_days:
|
||||
options.append(
|
||||
(scores[e + 1][p + 1] + match_cost(events[e], periods[p]), 0, "match")
|
||||
)
|
||||
score, _, choice = min(options)
|
||||
scores[e][p] = score
|
||||
choices[e][p] = choice
|
||||
|
||||
matches: list[tuple[int, int]] = []
|
||||
unmatched_events: list[int] = []
|
||||
unmatched_periods: list[int] = []
|
||||
e = p = 0
|
||||
while e < n_events or p < n_periods:
|
||||
if e >= n_events:
|
||||
unmatched_periods.extend(range(p, n_periods))
|
||||
break
|
||||
if p >= n_periods:
|
||||
unmatched_events.extend(range(e, n_events))
|
||||
break
|
||||
choice = choices[e][p]
|
||||
if choice == "match":
|
||||
matches.append((e, p))
|
||||
e += 1
|
||||
p += 1
|
||||
elif choice == "period":
|
||||
unmatched_periods.append(p)
|
||||
p += 1
|
||||
else:
|
||||
unmatched_events.append(e)
|
||||
e += 1
|
||||
return matches, unmatched_events, unmatched_periods
|
||||
Reference in New Issue
Block a user