feat(dolt): source-agnostic import-run framework (A1)

run_import + SourceImporter Protocol (detect_revision/stage/validate/promote) giving every bulk importer the plan's non-negotiables, KISS: - one run per source at a time — Postgres session-level advisory lock held on a single pinned engine.connect() so it survives the running-row and promotion commits; no-op on SQLite. - idempotent per revision — cheap detect_revision compared to the last promoted run; unchanged revision records a no_op with zero writes (no fetch). - staging (in-memory, no physical staging tables) → validate (read-only) → atomic promote + run-row flip in one transaction. - failed validation or mid-run exception marks the run failed, alerts via system_event_service, and leaves live tables untouched. Every attempt recorded in data_import_runs; conflicts summary in validation_json (no conflicts table). Concrete SEC/earnings importers land in later phases. Tests: 6 orchestration tests (no_op / promote / new-revision / failed-untouched / promote-exception-rollback) + deterministic advisory-key derivation. Full suite 680 passed. Advisory-lock mutual exclusion is PG-verify-pending (SQLite no-ops it — flagged, not covered). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 00:18:27 +02:00
parent 949cbbe7c0
commit febd741671
2 changed files with 456 additions and 0 deletions
@@ -0,0 +1,201 @@
+"""Orchestration tests for the source-agnostic import framework.
+
+These drive ``run_import`` with a fake importer to prove the framework's
+guarantees: idempotent no_op on an unchanged revision, atomic promotion on a
+new revision, and — the load-bearing one — a failed validation or a mid-run
+exception leaves the live tables untouched.
+
+The advisory-lock branch is a no-op on SQLite, so lock mutual-exclusion has NO
+coverage here (PG-verify-pending); only the deterministic key derivation is
+unit-tested. Per the SQLite StaticPool caveat we never share a connection: each
+test uses its own temp-file engine and seeds/asserts with short-lived sessions
+sequenced around the ``run_import`` call.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from datetime import date, datetime, timezone
+
+import pytest
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.database import Base
+import app.models  # noqa: F401  register models on Base.metadata
+from app.models.data_import_run import DataImportRun
+from app.models.fundamental_snapshot import FundamentalSnapshot
+from app.models.system_event import SystemEvent
+from app.services.data_import import (
+    STATUS_FAILED,
+    STATUS_NO_OP,
+    STATUS_PROMOTED,
+    ValidationResult,
+    _advisory_key,
+    run_import,
+)
+
+
+@pytest.fixture
+async def engine():
+    """A dedicated temp-file SQLite engine (independent connections, unlike the
+    shared in-memory test engine) so ``run_import`` can pin its own connection."""
+    fd, path = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
+    async with eng.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    try:
+        yield eng
+    finally:
+        await eng.dispose()
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+
+
+def _factory(engine) -> async_sessionmaker[AsyncSession]:
+    return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+class FakeImporter:
+    """Writes ``n_rows`` fundamental_snapshots on promote (CIK-keyed, no ticker
+    FK) so the live-table effect is easy to count."""
+
+    source = "sec_facts"
+
+    def __init__(self, revision, *, ok=True, n_rows=3, raise_in="none"):
+        self.revision = revision
+        self.ok = ok
+        self.n_rows = n_rows
+        self.raise_in = raise_in
+        self.staged_called = False
+        self.promoted = False
+
+    async def detect_revision(self, db):
+        if self.raise_in == "detect":
+            raise RuntimeError("boom-detect")
+        return self.revision
+
+    async def stage(self, db):
+        self.staged_called = True
+        if self.raise_in == "stage":
+            raise RuntimeError("boom-stage")
+        return list(range(self.n_rows))
+
+    async def validate(self, db, staged):
+        return ValidationResult(
+            ok=self.ok,
+            summary={"staged_rows": len(staged)},
+            source_max_date=date(2026, 7, 21),
+            messages=[] if self.ok else ["coverage below threshold"],
+        )
+
+    async def promote(self, db, staged):
+        if self.raise_in == "promote":
+            # write one row THEN raise, to prove rollback undoes partial writes
+            db.add(_snapshot(self.revision, 999))
+            raise RuntimeError("boom-promote")
+        for i in staged:
+            db.add(_snapshot(self.revision, i))
+        self.promoted = True
+        return {"fundamental_snapshots": len(staged)}
+
+
+def _snapshot(revision: str, i: int) -> FundamentalSnapshot:
+    return FundamentalSnapshot(
+        cik=f"{i:010d}",
+        accession=f"{revision}-{i:06d}",
+        form="10-Q",
+        filed_date=date(2026, 7, 1),
+        accepted_at=datetime(2026, 7, 1, tzinfo=timezone.utc),
+        period_end=date(2026, 6, 30),
+        fiscal_year=2026,
+        fiscal_period="Q2",
+        revenue=1000.0 + i,
+    )
+
+
+async def _count(factory, model) -> int:
+    async with factory() as s:
+        return (await s.execute(select(func.count()).select_from(model))).scalar_one()
+
+
+async def _runs(factory) -> list[DataImportRun]:
+    async with factory() as s:
+        return list(
+            (await s.execute(select(DataImportRun).order_by(DataImportRun.id))).scalars()
+        )
+
+
+# ---------------------------------------------------------------------------
+
+
+def test_advisory_key_deterministic_and_distinct():
+    assert _advisory_key("sec_facts") == _advisory_key("sec_facts")
+    assert _advisory_key("sec_facts") != _advisory_key("dolt_earnings")
+    for src in ("sec_facts", "dolt_earnings", "dolt_stocks"):
+        k = _advisory_key(src)
+        assert -(2**63) <= k < 2**63  # fits Postgres bigint
+
+
+async def test_promote_writes_and_records_run(engine):
+    factory = _factory(engine)
+    run = await run_import(FakeImporter("rev1", n_rows=4), engine=engine)
+
+    assert run is not None and run.status == STATUS_PROMOTED
+    assert run.revision == "rev1"
+    assert run.row_counts_json is not None and "fundamental_snapshots" in run.row_counts_json
+    assert run.source_max_date == date(2026, 7, 21)
+    assert run.completed_at is not None
+    assert await _count(factory, FundamentalSnapshot) == 4
+
+
+async def test_no_op_on_repeated_revision(engine):
+    factory = _factory(engine)
+    await run_import(FakeImporter("rev1", n_rows=4), engine=engine)
+
+    second = FakeImporter("rev1", n_rows=4)
+    run = await run_import(second, engine=engine)
+
+    assert run is not None and run.status == STATUS_NO_OP
+    assert second.staged_called is False  # never fetched
+    assert await _count(factory, FundamentalSnapshot) == 4  # unchanged
+
+    runs = await _runs(factory)
+    assert [r.status for r in runs] == [STATUS_PROMOTED, STATUS_NO_OP]
+
+
+async def test_new_revision_after_promote_stages_again(engine):
+    factory = _factory(engine)
+    await run_import(FakeImporter("rev1", n_rows=2), engine=engine)
+    run = await run_import(FakeImporter("rev2", n_rows=3), engine=engine)
+
+    assert run is not None and run.status == STATUS_PROMOTED
+    assert await _count(factory, FundamentalSnapshot) == 5  # 2 + 3
+
+
+async def test_failed_validation_leaves_data_untouched(engine):
+    factory = _factory(engine)
+    await run_import(FakeImporter("rev1", n_rows=3), engine=engine)  # baseline
+
+    run = await run_import(FakeImporter("rev2", ok=False, n_rows=5), engine=engine)
+
+    assert run is not None and run.status == STATUS_FAILED
+    assert "coverage" in (run.error_details or "")
+    assert await _count(factory, FundamentalSnapshot) == 3  # untouched
+    assert await _count(factory, SystemEvent) == 1  # alerted
+
+
+async def test_exception_in_promote_rolls_back(engine):
+    factory = _factory(engine)
+    await run_import(FakeImporter("rev1", n_rows=3), engine=engine)  # baseline
+
+    run = await run_import(FakeImporter("rev2", raise_in="promote"), engine=engine)
+
+    assert run is not None and run.status == STATUS_FAILED
+    assert "boom-promote" in (run.error_details or "")
+    assert await _count(factory, FundamentalSnapshot) == 3  # partial write rolled back
+    assert await _count(factory, SystemEvent) == 1