feat(sec): A3 slice 1 — SEC client + CIK/SIC resolution + composite revision

First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 14:15:56 +02:00
parent b1397fa82e
commit cc67aebe61
6 changed files with 657 additions and 0 deletions
@@ -0,0 +1,159 @@
+"""Tests for the SEC client's parsing and fair-access behavior, via a mocked
+httpx transport (no network)."""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+
+import httpx
+import pytest
+
+from app.services import sec_client as sc
+from app.services.sec_client import SecClient, SecError, SecForbiddenError
+
+COMPANY_TICKERS = {
+    "0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."},
+    "1": {"cik_str": 1652044, "ticker": "GOOGL", "title": "Alphabet"},
+    "2": {"cik_str": 1652044, "ticker": "GOOG", "title": "Alphabet"},
+    "3": {"cik_str": 1067983, "ticker": "BRK-B", "title": "Berkshire"},
+}
+
+SUBMISSIONS_BASE = {
+    "cik": 320193,
+    "name": "Apple Inc.",
+    "sic": "3571",
+    "sicDescription": "Electronic Computers",
+    "fiscalYearEnd": "0926",
+    "tickers": ["AAPL"],
+    "filings": {
+        "recent": {
+            "accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"],
+            "form": ["10-Q", "10-K", "8-K"],
+            "reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"],
+            "acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"],
+            "isXBRL": [1, 1, 0],
+        },
+        "files": [{"name": "CIK0000320193-submissions-001.json", "filingFrom": "1994-01-26", "filingTo": "2015-05-27"}],
+    },
+}
+
+SUBMISSIONS_SHARD = {
+    "accessionNumber": ["0000320193-94-000002"],
+    "form": ["10-Q"],
+    "reportDate": ["1993-12-31"],
+    "acceptanceDateTime": ["1994-01-26T05:00:00.000Z"],
+    "isXBRL": [0],
+}
+
+FORM_IDX = """Description:           Daily Index of EDGAR Dissemination Feed by Form Type
+
+      Form Type   Company Name    CIK    Date Filed  File Name
+-------------------------------------------------------------------------------
+   10-K/A        Starfighters Space, Inc.    1947016   20260721   edgar/data/1947016/0001062993-26-003746.txt
+   10-Q          3M CO                       66740     20260721   edgar/data/66740/0000066740-26-000246.txt
+   8-K           Some Corp                   12345     20260721   edgar/data/12345/0000012345-26-000001.txt
+   10-Q          CALIX, INC                  1406666   20260721   edgar/data/1406666/0001406666-26-000034.txt
+"""
+
+DIR_JSON = {"directory": {"item": [
+    {"name": "form.20260720.idx"}, {"name": "form.20260721.idx"}, {"name": "company.20260721.idx"},
+]}}
+
+
+def _handler(request: httpx.Request) -> httpx.Response:
+    url = str(request.url)
+    if url.endswith("company_tickers.json"):
+        return httpx.Response(200, json=COMPANY_TICKERS)
+    if url.endswith("submissions/CIK0000320193.json"):
+        return httpx.Response(200, json=SUBMISSIONS_BASE)
+    if url.endswith("CIK0000320193-submissions-001.json"):
+        return httpx.Response(200, json=SUBMISSIONS_SHARD)
+    if url.endswith("form.20260721.idx"):
+        return httpx.Response(200, text=FORM_IDX)
+    if url.endswith("QTR3/index.json"):
+        return httpx.Response(200, json=DIR_JSON)
+    return httpx.Response(404)
+
+
+def _client(**kw):
+    return SecClient(transport=httpx.MockTransport(_handler), spacing_seconds=0, **kw)
+
+
+async def test_company_tickers_normalised_and_multiclass():
+    async with _client() as c:
+        m = await c.company_tickers()
+    assert m["AAPL"] == 320193
+    assert m["GOOGL"] == m["GOOG"] == 1652044  # multi-class share one CIK
+    assert m["BRK-B"] == 1067983  # dash form
+
+
+async def test_submissions_merges_shards_and_filters_forms():
+    async with _client() as c:
+        sub = await c.submissions(320193)
+    assert sub["sic"] == "3571" and sub["fiscal_year_end"] == "0926"
+    accns = {f["accession"] for f in sub["filings"]}
+    # 10-Q + 10-K from recent, 10-Q from the shard; the 8-K is filtered out
+    assert accns == {"0000320193-26-000013", "0000320193-26-000006", "0000320193-94-000002"}
+    older = next(f for f in sub["filings"] if f["accession"] == "0000320193-94-000002")
+    assert older["report_date"] == "1993-12-31" and older["is_xbrl"] is False
+
+
+async def test_daily_index_parses_10kq_rows():
+    async with _client() as c:
+        rows = await c.daily_index(date(2026, 7, 21))
+    forms = {r["form"] for r in rows}
+    assert forms == {"10-K/A", "10-Q"}  # 8-K excluded
+    mmm = next(r for r in rows if r["cik"] == 66740)
+    assert mmm["accession"] == "0000066740-26-000246"
+
+
+async def test_latest_index_date():
+    async with _client() as c:
+        d = await c.latest_index_date(today=date(2026, 7, 22))
+    assert d == date(2026, 7, 21)
+
+
+async def test_403_raises_forbidden_no_retry():
+    calls = {"n": 0}
+
+    def handler(request):
+        calls["n"] += 1
+        return httpx.Response(403)
+
+    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
+        with pytest.raises(SecForbiddenError):
+            await c.get_json("https://data.sec.gov/x")
+    assert calls["n"] == 1  # alert and stop, never retry-loop
+
+
+async def test_429_retries_then_succeeds(monkeypatch):
+    async def _instant(_):
+        return None
+
+    monkeypatch.setattr(sc.asyncio, "sleep", _instant)  # no real backoff wait
+    calls = {"n": 0}
+
+    def handler(request):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            return httpx.Response(429)
+        return httpx.Response(200, json={"ok": True})
+
+    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=5) as c:
+        data = await c.get_json("https://data.sec.gov/y")
+    assert data == {"ok": True} and calls["n"] == 3
+
+
+async def test_429_gives_up_after_max_retries(monkeypatch):
+    async def _instant(_):
+        return None
+
+    monkeypatch.setattr(sc.asyncio, "sleep", _instant)
+
+    def handler(request):
+        return httpx.Response(429)
+
+    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=2) as c:
+        with pytest.raises(SecError):
+            await c.get_json("https://data.sec.gov/z")