dennisthiessenandClaude Opus 4.8
5939be7b7f
Addresses the slice-1 review: 1. Resolution is now read-only (A1 transaction contract). resolve_ciks / fetch_sic_updates compute proposals and mutate nothing; a new apply_ticker_updates issues the writes, called only in promote — so a failed validation can't leak ticker changes on the framework's failure commit. 2. Only 404 means "missing". Added SecNotFoundError; daily_index / latest_index_date catch only that. 403, exhausted 429, 5xx, timeouts, and transport/parse errors now propagate instead of looking like "no index". 3. Fair-access enforced when opening a REAL client (transport=None): reject blank/placeholder/non-email User-Agent and sub-0.11s spacing. Mock transports skip it (tests use 0 spacing). 4. submissions(include_history=False) by default — only the one-time full backfill fetches the history shards; SIC/incremental work makes no extra requests. Plus: retry transient 5xx/network errors and honor Retry-After during the 1 GB backfill; compose_revision rejects a missing index date (no "None:..." revision). Re-verified live vs real SEC (fair-access validation passes, shard merge intact). Tests: 18 (added error propagation, read-only resolution, fair-access, recent-only submissions, reject-None revision). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
121 lines
4.5 KiB
Python
121 lines
4.5 KiB
Python
"""Tests for CIK/SIC resolution (read-only), apply-in-promote, and the
|
|
composite-revision fingerprint."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
import app.models # noqa: F401
|
|
from app.models.ticker import Ticker
|
|
from app.services import sec_universe as su
|
|
|
|
|
|
@pytest.fixture
|
|
async def factory():
|
|
fd, path = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
|
async with eng.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
try:
|
|
yield async_sessionmaker(eng, class_=AsyncSession, expire_on_commit=False)
|
|
finally:
|
|
await eng.dispose()
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
class FakeSecClient:
|
|
def __init__(self, tickers, submissions=None):
|
|
self._tickers = tickers
|
|
self._submissions = submissions or {}
|
|
|
|
async def company_tickers(self):
|
|
return dict(self._tickers)
|
|
|
|
async def submissions(self, cik, *, include_history=False):
|
|
return self._submissions[int(cik)]
|
|
|
|
|
|
async def test_resolve_ciks_is_read_only_and_proposes_updates(factory):
|
|
async with factory() as s:
|
|
for sym in ["AAPL", "GOOGL", "GOOG", "ZZZZ"]: # ZZZZ not in SEC
|
|
s.add(Ticker(symbol=sym))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient({"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044})
|
|
async with factory() as s:
|
|
resolved = await su.resolve_ciks(s, client)
|
|
assert not s.dirty and not s.new # NOTHING mutated during resolution
|
|
await s.rollback()
|
|
|
|
assert resolved.symbol_to_cik == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}
|
|
assert len(resolved.cik_updates) == 3 # AAPL, GOOGL, GOOG (ZZZZ unresolved)
|
|
assert set(resolved.cik_to_ticker_ids) == {320193, 1652044}
|
|
|
|
# Read-only really means the DB is untouched until apply.
|
|
async with factory() as s:
|
|
ciks = {t.symbol: t.cik for t in (await s.execute(select(Ticker))).scalars()}
|
|
assert all(v is None for v in ciks.values())
|
|
|
|
|
|
async def test_apply_ticker_updates_writes_cik_and_sic(factory):
|
|
async with factory() as s:
|
|
for sym in ["GOOGL", "GOOG"]:
|
|
s.add(Ticker(symbol=sym))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
{"GOOGL": 1652044, "GOOG": 1652044},
|
|
submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}},
|
|
)
|
|
async with factory() as s:
|
|
resolved = await su.resolve_ciks(s, client)
|
|
sic_updates = await su.fetch_sic_updates(client, resolved.cik_to_ticker_ids)
|
|
counts = await su.apply_ticker_updates(s, resolved, sic_updates)
|
|
await s.commit()
|
|
|
|
assert counts == {"cik_updates": 2, "sic_updates": 2}
|
|
async with factory() as s:
|
|
rows = {t.symbol: (t.cik, t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()}
|
|
assert rows["GOOGL"] == ("0001652044", "7370", "Services-Computer")
|
|
assert rows["GOOG"] == ("0001652044", "7370", "Services-Computer")
|
|
|
|
|
|
async def test_fetch_sic_updates_is_read_only(factory):
|
|
client = FakeSecClient({}, submissions={1: {"sic": "1", "sic_description": "x"}})
|
|
updates = await su.fetch_sic_updates(client, {1: [10, 11]})
|
|
assert updates == [(10, "1", "x"), (11, "1", "x")] # proposals only, no DB touched
|
|
|
|
|
|
def test_universe_fingerprint_changes_on_membership():
|
|
a = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019})
|
|
same = su.universe_fingerprint({"MSFT": 789019, "AAPL": 320193}) # order-independent
|
|
added = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810})
|
|
remapped = su.universe_fingerprint({"AAPL": 999, "MSFT": 789019})
|
|
assert a == same
|
|
assert a != added # new ticker forces a new revision
|
|
assert a != remapped # changed CIK mapping forces a new revision
|
|
|
|
|
|
def test_compose_revision_rejects_missing_index_date():
|
|
with pytest.raises(ValueError):
|
|
su.compose_revision(None, "abc", {"AAPL": 320193})
|
|
|
|
|
|
def test_compose_revision_and_index_hash():
|
|
rows = [{"cik": 320193, "accession": "a-1"}, {"cik": 66740, "accession": "b-2"}]
|
|
h1 = su.index_content_hash(rows)
|
|
h2 = su.index_content_hash(list(reversed(rows)))
|
|
assert h1 == h2 # order-independent
|
|
rev = su.compose_revision("2026-07-21", h1, {"AAPL": 320193})
|
|
assert rev.startswith("2026-07-21:") and rev.count(":") == 2
|