dennisthiessenandClaude Opus 4.8
54ae8ba153
A SourceImporter that ingests post-no-preference/earnings into earnings_events for the tracked universe. Shadow by construction (nothing reads earnings_events until A4). - earnings_alignment.py: pure calendar<->EPS-history min-cost monotonic DP, reused from scripts/import_dolthub_earnings.py with identical constants (not extending that one-off script); symbol/session normalization; unit-tested against the pinned constants. - dolt_client.py: async dolt CLI wrapper (pull / current_commit / query_csv via asyncio.create_subprocess_exec — never blocks the shared event loop) + disk guard before pull. - dolt_earnings_importer.py: detect_revision = pull + HEAD hash; stage = query earnings_calendar + eps_history, dedup, align, map act_symbol->ticker_id (normalize both sides so dotted BRK.B joins); promote is destructive (delete future dolt_earnings rows + upsert; past never deleted) so validate is FAIL-CLOSED — blocks when the staged forward calendar is empty or has collapsed below 50% of what's loaded (the forward calendar is the acceptance gate). - NOTICE: CC BY-SA 4.0 attribution; config: DOLT_BINARY / DOLT_DATA_DIR / etc. Verified end-to-end against the real 1.68 GB clone (5 tickers: 133 events, 128 paired, forward calendar to 2026-08-26, BRK.B joined). Tests: 9 alignment + 7 importer + 1 skip-guarded real-clone smoke. Full suite 699 passed. Remaining for A2: wire the daily ~02:30 ET shadow cron — deferred to pair with the deploy-time dolt install + DOLT_DATA_DIR provisioning. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
84 lines
2.9 KiB
Python
84 lines
2.9 KiB
Python
"""Minimal async client for a local Dolt clone.
|
|
|
|
The application never runs a long-lived Dolt sql-server; it shells out to the
|
|
`dolt` CLI against a persistent clone and reads results as CSV. Every call goes
|
|
through ``asyncio.create_subprocess_exec`` because the scheduler shares one event
|
|
loop with the API (`app/scheduler.py:73`) — a blocking `subprocess.run` here
|
|
would stall request handling.
|
|
|
|
Production keeps the clone in ``DOLT_DATA_DIR`` outside the deploy tree; the
|
|
binary path and data dir are configured (see ``app/config.py``). Read via
|
|
``dolt sql -r csv``; refresh with ``pull`` and record the resulting commit hash
|
|
as the import revision.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import csv
|
|
import io
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DoltError(RuntimeError):
|
|
"""A dolt subprocess exited non-zero."""
|
|
|
|
|
|
async def _run(binary: str, args: list[str], *, cwd: Path) -> str:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
binary,
|
|
*args,
|
|
cwd=str(cwd),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await proc.communicate()
|
|
if proc.returncode != 0:
|
|
raise DoltError(
|
|
f"dolt {' '.join(args)} failed ({proc.returncode}): "
|
|
f"{stderr.decode('utf-8', 'replace').strip()[:500]}"
|
|
)
|
|
return stdout.decode("utf-8", "replace")
|
|
|
|
|
|
def ensure_free_disk(path: Path, min_free_gb: float) -> None:
|
|
"""Raise if free space at ``path`` is below the threshold (checked before a
|
|
pull that could grow the clone). Uses the nearest existing ancestor so it
|
|
works before the clone dir exists."""
|
|
probe = path
|
|
while not probe.exists() and probe.parent != probe:
|
|
probe = probe.parent
|
|
free_gb = shutil.disk_usage(probe).free / (1024**3)
|
|
if free_gb < min_free_gb:
|
|
raise DoltError(
|
|
f"insufficient disk for dolt at {path}: {free_gb:.1f} GB free "
|
|
f"< {min_free_gb:.1f} GB required"
|
|
)
|
|
|
|
|
|
async def pull(repo_dir: Path, *, binary: str) -> None:
|
|
"""`dolt pull` the persistent clone to the latest upstream revision."""
|
|
await _run(binary, ["pull"], cwd=repo_dir)
|
|
|
|
|
|
async def current_commit(repo_dir: Path, *, binary: str) -> str:
|
|
"""The HEAD commit hash of the clone — used as the import revision."""
|
|
rows = await query_csv(
|
|
repo_dir, "SELECT commit_hash FROM dolt_log ORDER BY date DESC LIMIT 1", binary=binary
|
|
)
|
|
if not rows or not rows[0].get("commit_hash"):
|
|
raise DoltError("could not read HEAD commit hash from dolt_log")
|
|
return rows[0]["commit_hash"]
|
|
|
|
|
|
async def query_csv(repo_dir: Path, sql: str, *, binary: str) -> list[dict[str, str]]:
|
|
"""Run a read query and parse the CSV result into a list of dict rows."""
|
|
out = await _run(binary, ["sql", "-q", sql, "-r", "csv"], cwd=repo_dir)
|
|
if not out.strip():
|
|
return []
|
|
return list(csv.DictReader(io.StringIO(out)))
|