fix(sec): A3 slice-1 review — read-only resolution, error propagation, fair-access
Addresses the slice-1 review: 1. Resolution is now read-only (A1 transaction contract). resolve_ciks / fetch_sic_updates compute proposals and mutate nothing; a new apply_ticker_updates issues the writes, called only in promote — so a failed validation can't leak ticker changes on the framework's failure commit. 2. Only 404 means "missing". Added SecNotFoundError; daily_index / latest_index_date catch only that. 403, exhausted 429, 5xx, timeouts, and transport/parse errors now propagate instead of looking like "no index". 3. Fair-access enforced when opening a REAL client (transport=None): reject blank/placeholder/non-email User-Agent and sub-0.11s spacing. Mock transports skip it (tests use 0 spacing). 4. submissions(include_history=False) by default — only the one-time full backfill fetches the history shards; SIC/incremental work makes no extra requests. Plus: retry transient 5xx/network errors and honor Retry-After during the 1 GB backfill; compose_revision rejects a missing index date (no "None:..." revision). Re-verified live vs real SEC (fair-access validation passes, shard merge intact). Tests: 18 (added error propagation, read-only resolution, fair-access, recent-only submissions, reject-None revision). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""Tests for CIK/SIC resolution and the composite-revision fingerprint."""
|
||||
"""Tests for CIK/SIC resolution (read-only), apply-in-promote, and the
|
||||
composite-revision fingerprint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -40,11 +41,11 @@ class FakeSecClient:
|
||||
async def company_tickers(self):
|
||||
return dict(self._tickers)
|
||||
|
||||
async def submissions(self, cik):
|
||||
async def submissions(self, cik, *, include_history=False):
|
||||
return self._submissions[int(cik)]
|
||||
|
||||
|
||||
async def test_resolve_ciks_sets_cik_and_returns_mapping(factory):
|
||||
async def test_resolve_ciks_is_read_only_and_proposes_updates(factory):
|
||||
async with factory() as s:
|
||||
for sym in ["AAPL", "GOOGL", "GOOG", "ZZZZ"]: # ZZZZ not in SEC
|
||||
s.add(Ticker(symbol=sym))
|
||||
@@ -52,35 +53,47 @@ async def test_resolve_ciks_sets_cik_and_returns_mapping(factory):
|
||||
|
||||
client = FakeSecClient({"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044})
|
||||
async with factory() as s:
|
||||
mapping = await su.resolve_ciks(s, client)
|
||||
await s.commit()
|
||||
resolved = await su.resolve_ciks(s, client)
|
||||
assert not s.dirty and not s.new # NOTHING mutated during resolution
|
||||
await s.rollback()
|
||||
|
||||
assert mapping == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}
|
||||
assert resolved.symbol_to_cik == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}
|
||||
assert len(resolved.cik_updates) == 3 # AAPL, GOOGL, GOOG (ZZZZ unresolved)
|
||||
assert set(resolved.cik_to_ticker_ids) == {320193, 1652044}
|
||||
|
||||
# Read-only really means the DB is untouched until apply.
|
||||
async with factory() as s:
|
||||
ciks = {t.symbol: t.cik for t in (await s.execute(select(Ticker))).scalars()}
|
||||
assert ciks["AAPL"] == "0000320193"
|
||||
assert ciks["GOOGL"] == ciks["GOOG"] == "0001652044" # multi-class share CIK
|
||||
assert ciks["ZZZZ"] is None # unresolved stays null
|
||||
assert all(v is None for v in ciks.values())
|
||||
|
||||
|
||||
async def test_refresh_sic_updates_all_tickers_of_a_cik(factory):
|
||||
async def test_apply_ticker_updates_writes_cik_and_sic(factory):
|
||||
async with factory() as s:
|
||||
for sym in ["GOOGL", "GOOG"]:
|
||||
s.add(Ticker(symbol=sym, cik="0001652044"))
|
||||
s.add(Ticker(symbol=sym))
|
||||
await s.commit()
|
||||
|
||||
client = FakeSecClient(
|
||||
{}, submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}}
|
||||
{"GOOGL": 1652044, "GOOG": 1652044},
|
||||
submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}},
|
||||
)
|
||||
async with factory() as s:
|
||||
n = await su.refresh_sic(s, client, [1652044])
|
||||
resolved = await su.resolve_ciks(s, client)
|
||||
sic_updates = await su.fetch_sic_updates(client, resolved.cik_to_ticker_ids)
|
||||
counts = await su.apply_ticker_updates(s, resolved, sic_updates)
|
||||
await s.commit()
|
||||
|
||||
assert n == 1
|
||||
assert counts == {"cik_updates": 2, "sic_updates": 2}
|
||||
async with factory() as s:
|
||||
rows = {t.symbol: (t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()}
|
||||
assert rows["GOOGL"] == ("7370", "Services-Computer")
|
||||
assert rows["GOOG"] == ("7370", "Services-Computer")
|
||||
rows = {t.symbol: (t.cik, t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()}
|
||||
assert rows["GOOGL"] == ("0001652044", "7370", "Services-Computer")
|
||||
assert rows["GOOG"] == ("0001652044", "7370", "Services-Computer")
|
||||
|
||||
|
||||
async def test_fetch_sic_updates_is_read_only(factory):
|
||||
client = FakeSecClient({}, submissions={1: {"sic": "1", "sic_description": "x"}})
|
||||
updates = await su.fetch_sic_updates(client, {1: [10, 11]})
|
||||
assert updates == [(10, "1", "x"), (11, "1", "x")] # proposals only, no DB touched
|
||||
|
||||
|
||||
def test_universe_fingerprint_changes_on_membership():
|
||||
@@ -93,11 +106,13 @@ def test_universe_fingerprint_changes_on_membership():
|
||||
assert a != remapped # changed CIK mapping forces a new revision
|
||||
|
||||
|
||||
def test_compose_revision_rejects_missing_index_date():
|
||||
with pytest.raises(ValueError):
|
||||
su.compose_revision(None, "abc", {"AAPL": 320193})
|
||||
|
||||
|
||||
def test_compose_revision_and_index_hash():
|
||||
rows = [
|
||||
{"cik": 320193, "accession": "a-1"},
|
||||
{"cik": 66740, "accession": "b-2"},
|
||||
]
|
||||
rows = [{"cik": 320193, "accession": "a-1"}, {"cik": 66740, "accession": "b-2"}]
|
||||
h1 = su.index_content_hash(rows)
|
||||
h2 = su.index_content_hash(list(reversed(rows)))
|
||||
assert h1 == h2 # order-independent
|
||||
|
||||
Reference in New Issue
Block a user