dennisthiessenandClaude Opus 4.8
cc67aebe61
First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
106 lines
3.6 KiB
Python
106 lines
3.6 KiB
Python
"""Tests for CIK/SIC resolution and the composite-revision fingerprint."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
import app.models # noqa: F401
|
|
from app.models.ticker import Ticker
|
|
from app.services import sec_universe as su
|
|
|
|
|
|
@pytest.fixture
|
|
async def factory():
|
|
fd, path = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
|
async with eng.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
try:
|
|
yield async_sessionmaker(eng, class_=AsyncSession, expire_on_commit=False)
|
|
finally:
|
|
await eng.dispose()
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
class FakeSecClient:
|
|
def __init__(self, tickers, submissions=None):
|
|
self._tickers = tickers
|
|
self._submissions = submissions or {}
|
|
|
|
async def company_tickers(self):
|
|
return dict(self._tickers)
|
|
|
|
async def submissions(self, cik):
|
|
return self._submissions[int(cik)]
|
|
|
|
|
|
async def test_resolve_ciks_sets_cik_and_returns_mapping(factory):
|
|
async with factory() as s:
|
|
for sym in ["AAPL", "GOOGL", "GOOG", "ZZZZ"]: # ZZZZ not in SEC
|
|
s.add(Ticker(symbol=sym))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient({"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044})
|
|
async with factory() as s:
|
|
mapping = await su.resolve_ciks(s, client)
|
|
await s.commit()
|
|
|
|
assert mapping == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}
|
|
async with factory() as s:
|
|
ciks = {t.symbol: t.cik for t in (await s.execute(select(Ticker))).scalars()}
|
|
assert ciks["AAPL"] == "0000320193"
|
|
assert ciks["GOOGL"] == ciks["GOOG"] == "0001652044" # multi-class share CIK
|
|
assert ciks["ZZZZ"] is None # unresolved stays null
|
|
|
|
|
|
async def test_refresh_sic_updates_all_tickers_of_a_cik(factory):
|
|
async with factory() as s:
|
|
for sym in ["GOOGL", "GOOG"]:
|
|
s.add(Ticker(symbol=sym, cik="0001652044"))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
{}, submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}}
|
|
)
|
|
async with factory() as s:
|
|
n = await su.refresh_sic(s, client, [1652044])
|
|
await s.commit()
|
|
|
|
assert n == 1
|
|
async with factory() as s:
|
|
rows = {t.symbol: (t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()}
|
|
assert rows["GOOGL"] == ("7370", "Services-Computer")
|
|
assert rows["GOOG"] == ("7370", "Services-Computer")
|
|
|
|
|
|
def test_universe_fingerprint_changes_on_membership():
|
|
a = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019})
|
|
same = su.universe_fingerprint({"MSFT": 789019, "AAPL": 320193}) # order-independent
|
|
added = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810})
|
|
remapped = su.universe_fingerprint({"AAPL": 999, "MSFT": 789019})
|
|
assert a == same
|
|
assert a != added # new ticker forces a new revision
|
|
assert a != remapped # changed CIK mapping forces a new revision
|
|
|
|
|
|
def test_compose_revision_and_index_hash():
|
|
rows = [
|
|
{"cik": 320193, "accession": "a-1"},
|
|
{"cik": 66740, "accession": "b-2"},
|
|
]
|
|
h1 = su.index_content_hash(rows)
|
|
h2 = su.index_content_hash(list(reversed(rows)))
|
|
assert h1 == h2 # order-independent
|
|
rev = su.compose_revision("2026-07-21", h1, {"AAPL": 320193})
|
|
assert rev.startswith("2026-07-21:") and rev.count(":") == 2
|