feat(sec): A3 slice 1 — SEC client + CIK/SIC resolution + composite revision
First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,159 @@
|
||||
"""Tests for the SEC client's parsing and fair-access behavior, via a mocked
|
||||
httpx transport (no network)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.services import sec_client as sc
|
||||
from app.services.sec_client import SecClient, SecError, SecForbiddenError
|
||||
|
||||
COMPANY_TICKERS = {
|
||||
"0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."},
|
||||
"1": {"cik_str": 1652044, "ticker": "GOOGL", "title": "Alphabet"},
|
||||
"2": {"cik_str": 1652044, "ticker": "GOOG", "title": "Alphabet"},
|
||||
"3": {"cik_str": 1067983, "ticker": "BRK-B", "title": "Berkshire"},
|
||||
}
|
||||
|
||||
SUBMISSIONS_BASE = {
|
||||
"cik": 320193,
|
||||
"name": "Apple Inc.",
|
||||
"sic": "3571",
|
||||
"sicDescription": "Electronic Computers",
|
||||
"fiscalYearEnd": "0926",
|
||||
"tickers": ["AAPL"],
|
||||
"filings": {
|
||||
"recent": {
|
||||
"accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"],
|
||||
"form": ["10-Q", "10-K", "8-K"],
|
||||
"reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"],
|
||||
"acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"],
|
||||
"isXBRL": [1, 1, 0],
|
||||
},
|
||||
"files": [{"name": "CIK0000320193-submissions-001.json", "filingFrom": "1994-01-26", "filingTo": "2015-05-27"}],
|
||||
},
|
||||
}
|
||||
|
||||
SUBMISSIONS_SHARD = {
|
||||
"accessionNumber": ["0000320193-94-000002"],
|
||||
"form": ["10-Q"],
|
||||
"reportDate": ["1993-12-31"],
|
||||
"acceptanceDateTime": ["1994-01-26T05:00:00.000Z"],
|
||||
"isXBRL": [0],
|
||||
}
|
||||
|
||||
FORM_IDX = """Description: Daily Index of EDGAR Dissemination Feed by Form Type
|
||||
|
||||
Form Type Company Name CIK Date Filed File Name
|
||||
-------------------------------------------------------------------------------
|
||||
10-K/A Starfighters Space, Inc. 1947016 20260721 edgar/data/1947016/0001062993-26-003746.txt
|
||||
10-Q 3M CO 66740 20260721 edgar/data/66740/0000066740-26-000246.txt
|
||||
8-K Some Corp 12345 20260721 edgar/data/12345/0000012345-26-000001.txt
|
||||
10-Q CALIX, INC 1406666 20260721 edgar/data/1406666/0001406666-26-000034.txt
|
||||
"""
|
||||
|
||||
DIR_JSON = {"directory": {"item": [
|
||||
{"name": "form.20260720.idx"}, {"name": "form.20260721.idx"}, {"name": "company.20260721.idx"},
|
||||
]}}
|
||||
|
||||
|
||||
def _handler(request: httpx.Request) -> httpx.Response:
|
||||
url = str(request.url)
|
||||
if url.endswith("company_tickers.json"):
|
||||
return httpx.Response(200, json=COMPANY_TICKERS)
|
||||
if url.endswith("submissions/CIK0000320193.json"):
|
||||
return httpx.Response(200, json=SUBMISSIONS_BASE)
|
||||
if url.endswith("CIK0000320193-submissions-001.json"):
|
||||
return httpx.Response(200, json=SUBMISSIONS_SHARD)
|
||||
if url.endswith("form.20260721.idx"):
|
||||
return httpx.Response(200, text=FORM_IDX)
|
||||
if url.endswith("QTR3/index.json"):
|
||||
return httpx.Response(200, json=DIR_JSON)
|
||||
return httpx.Response(404)
|
||||
|
||||
|
||||
def _client(**kw):
|
||||
return SecClient(transport=httpx.MockTransport(_handler), spacing_seconds=0, **kw)
|
||||
|
||||
|
||||
async def test_company_tickers_normalised_and_multiclass():
|
||||
async with _client() as c:
|
||||
m = await c.company_tickers()
|
||||
assert m["AAPL"] == 320193
|
||||
assert m["GOOGL"] == m["GOOG"] == 1652044 # multi-class share one CIK
|
||||
assert m["BRK-B"] == 1067983 # dash form
|
||||
|
||||
|
||||
async def test_submissions_merges_shards_and_filters_forms():
|
||||
async with _client() as c:
|
||||
sub = await c.submissions(320193)
|
||||
assert sub["sic"] == "3571" and sub["fiscal_year_end"] == "0926"
|
||||
accns = {f["accession"] for f in sub["filings"]}
|
||||
# 10-Q + 10-K from recent, 10-Q from the shard; the 8-K is filtered out
|
||||
assert accns == {"0000320193-26-000013", "0000320193-26-000006", "0000320193-94-000002"}
|
||||
older = next(f for f in sub["filings"] if f["accession"] == "0000320193-94-000002")
|
||||
assert older["report_date"] == "1993-12-31" and older["is_xbrl"] is False
|
||||
|
||||
|
||||
async def test_daily_index_parses_10kq_rows():
|
||||
async with _client() as c:
|
||||
rows = await c.daily_index(date(2026, 7, 21))
|
||||
forms = {r["form"] for r in rows}
|
||||
assert forms == {"10-K/A", "10-Q"} # 8-K excluded
|
||||
mmm = next(r for r in rows if r["cik"] == 66740)
|
||||
assert mmm["accession"] == "0000066740-26-000246"
|
||||
|
||||
|
||||
async def test_latest_index_date():
|
||||
async with _client() as c:
|
||||
d = await c.latest_index_date(today=date(2026, 7, 22))
|
||||
assert d == date(2026, 7, 21)
|
||||
|
||||
|
||||
async def test_403_raises_forbidden_no_retry():
|
||||
calls = {"n": 0}
|
||||
|
||||
def handler(request):
|
||||
calls["n"] += 1
|
||||
return httpx.Response(403)
|
||||
|
||||
async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
|
||||
with pytest.raises(SecForbiddenError):
|
||||
await c.get_json("https://data.sec.gov/x")
|
||||
assert calls["n"] == 1 # alert and stop, never retry-loop
|
||||
|
||||
|
||||
async def test_429_retries_then_succeeds(monkeypatch):
|
||||
async def _instant(_):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(sc.asyncio, "sleep", _instant) # no real backoff wait
|
||||
calls = {"n": 0}
|
||||
|
||||
def handler(request):
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 3:
|
||||
return httpx.Response(429)
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=5) as c:
|
||||
data = await c.get_json("https://data.sec.gov/y")
|
||||
assert data == {"ok": True} and calls["n"] == 3
|
||||
|
||||
|
||||
async def test_429_gives_up_after_max_retries(monkeypatch):
|
||||
async def _instant(_):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(sc.asyncio, "sleep", _instant)
|
||||
|
||||
def handler(request):
|
||||
return httpx.Response(429)
|
||||
|
||||
async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=2) as c:
|
||||
with pytest.raises(SecError):
|
||||
await c.get_json("https://data.sec.gov/z")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Tests for CIK/SIC resolution and the composite-revision fingerprint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from app.database import Base
|
||||
import app.models # noqa: F401
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import sec_universe as su
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def factory():
|
||||
fd, path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
||||
async with eng.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
try:
|
||||
yield async_sessionmaker(eng, class_=AsyncSession, expire_on_commit=False)
|
||||
finally:
|
||||
await eng.dispose()
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class FakeSecClient:
|
||||
def __init__(self, tickers, submissions=None):
|
||||
self._tickers = tickers
|
||||
self._submissions = submissions or {}
|
||||
|
||||
async def company_tickers(self):
|
||||
return dict(self._tickers)
|
||||
|
||||
async def submissions(self, cik):
|
||||
return self._submissions[int(cik)]
|
||||
|
||||
|
||||
async def test_resolve_ciks_sets_cik_and_returns_mapping(factory):
|
||||
async with factory() as s:
|
||||
for sym in ["AAPL", "GOOGL", "GOOG", "ZZZZ"]: # ZZZZ not in SEC
|
||||
s.add(Ticker(symbol=sym))
|
||||
await s.commit()
|
||||
|
||||
client = FakeSecClient({"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044})
|
||||
async with factory() as s:
|
||||
mapping = await su.resolve_ciks(s, client)
|
||||
await s.commit()
|
||||
|
||||
assert mapping == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}
|
||||
async with factory() as s:
|
||||
ciks = {t.symbol: t.cik for t in (await s.execute(select(Ticker))).scalars()}
|
||||
assert ciks["AAPL"] == "0000320193"
|
||||
assert ciks["GOOGL"] == ciks["GOOG"] == "0001652044" # multi-class share CIK
|
||||
assert ciks["ZZZZ"] is None # unresolved stays null
|
||||
|
||||
|
||||
async def test_refresh_sic_updates_all_tickers_of_a_cik(factory):
|
||||
async with factory() as s:
|
||||
for sym in ["GOOGL", "GOOG"]:
|
||||
s.add(Ticker(symbol=sym, cik="0001652044"))
|
||||
await s.commit()
|
||||
|
||||
client = FakeSecClient(
|
||||
{}, submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}}
|
||||
)
|
||||
async with factory() as s:
|
||||
n = await su.refresh_sic(s, client, [1652044])
|
||||
await s.commit()
|
||||
|
||||
assert n == 1
|
||||
async with factory() as s:
|
||||
rows = {t.symbol: (t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()}
|
||||
assert rows["GOOGL"] == ("7370", "Services-Computer")
|
||||
assert rows["GOOG"] == ("7370", "Services-Computer")
|
||||
|
||||
|
||||
def test_universe_fingerprint_changes_on_membership():
|
||||
a = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019})
|
||||
same = su.universe_fingerprint({"MSFT": 789019, "AAPL": 320193}) # order-independent
|
||||
added = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810})
|
||||
remapped = su.universe_fingerprint({"AAPL": 999, "MSFT": 789019})
|
||||
assert a == same
|
||||
assert a != added # new ticker forces a new revision
|
||||
assert a != remapped # changed CIK mapping forces a new revision
|
||||
|
||||
|
||||
def test_compose_revision_and_index_hash():
|
||||
rows = [
|
||||
{"cik": 320193, "accession": "a-1"},
|
||||
{"cik": 66740, "accession": "b-2"},
|
||||
]
|
||||
h1 = su.index_content_hash(rows)
|
||||
h2 = su.index_content_hash(list(reversed(rows)))
|
||||
assert h1 == h2 # order-independent
|
||||
rev = su.compose_revision("2026-07-21", h1, {"AAPL": 320193})
|
||||
assert rev.startswith("2026-07-21:") and rev.count(":") == 2
|
||||
Reference in New Issue
Block a user