From cc67aebe616f6c84161f93884a6df24a133083ba Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Wed, 22 Jul 2026 14:15:56 +0200 Subject: [PATCH] =?UTF-8?q?feat(sec):=20A3=20slice=201=20=E2=80=94=20SEC?= =?UTF-8?q?=20client=20+=20CIK/SIC=20resolution=20+=20composite=20revision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 --- .env.example | 8 + app/config.py | 9 + app/services/sec_client.py | 289 ++++++++++++++++++++++++++++++++ app/services/sec_universe.py | 87 ++++++++++ tests/unit/test_sec_client.py | 159 ++++++++++++++++++ tests/unit/test_sec_universe.py | 105 ++++++++++++ 6 files changed, 657 insertions(+) create mode 100644 app/services/sec_client.py create mode 100644 app/services/sec_universe.py create mode 100644 tests/unit/test_sec_client.py create mode 100644 tests/unit/test_sec_universe.py diff --git a/.env.example b/.env.example index 2324927..3092ca4 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,14 @@ DOLT_MIN_FREE_DISK_GB=5.0 # import connection + advisory lock. DOLT_COMMAND_TIMEOUT_SECONDS=600.0 +# SEC EDGAR (fundamentals, workstream A). SEC fair-access REQUIRES an identifying +# User-Agent with a REAL contact email — set it, or requests get 403'd. Stay well +# under 10 req/s (spacing below). +SEC_USER_AGENT=signal-platform/1.0 (contact: you@example.com) +SEC_REQUEST_SPACING_SECONDS=0.2 +SEC_MAX_RETRIES=4 +SEC_REQUEST_TIMEOUT_SECONDS=30.0 + # Regime Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html # Optional: without it the volatility (V1) and credit (C1) pillars show as n/a. FRED_API_KEY= diff --git a/app/config.py b/app/config.py index 70d219a..d07d69f 100644 --- a/app/config.py +++ b/app/config.py @@ -52,6 +52,15 @@ class Settings(BaseSettings): # connection + advisory lock indefinitely. dolt_command_timeout_seconds: float = 600.0 + # SEC EDGAR (workstream A, fundamentals). Fair-access policy REQUIRES an + # identifying User-Agent with a contact email — set a real one. Stay well + # under 10 req/s (spacing below); 403 means the UA/pattern is wrong → the + # client alerts and stops rather than retry-looping. + sec_user_agent: str = "signal-platform/1.0 (contact: set-a-real-email@example.com)" + sec_request_spacing_seconds: float = 0.2 + sec_max_retries: int = 4 + sec_request_timeout_seconds: float = 30.0 + # Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it # the volatility (P5) and credit-spread (F2) signals are reported as n/a. fred_api_key: str = "" diff --git a/app/services/sec_client.py b/app/services/sec_client.py new file mode 100644 index 0000000..b25cf79 --- /dev/null +++ b/app/services/sec_client.py @@ -0,0 +1,289 @@ +"""Async SEC EDGAR client for the fundamentals importer (workstream A). + +All access is batch (never at request time). This wraps the three SEC products +the A3 design uses — `company_tickers.json`, `submissions/`, `companyfacts/`, and +the daily filing index — behind one client that honors SEC's fair-access policy: + +- an identifying ``User-Agent`` with a contact email on every request (config); +- request spacing well under the 10 req/s limit; +- exponential backoff + retry on 429; +- **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a + 403 means the UA or request pattern is wrong and retrying won't fix it. + +Parsing lives here (index fixed-width, submissions pagination); DB writes and the +snapshot mapping live in the importer. No conditional GETs — the companyfacts +endpoint exposes no ETag/Last-Modified (verified), which is why the importer is +daily-index driven rather than polling archives. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from datetime import date, datetime +from pathlib import Path +from typing import Any + +import httpx + +from app.config import settings +from app.exceptions import ProviderError +from app.services.earnings_alignment import normalise_symbol + +logger = logging.getLogger(__name__) + +_WWW = "https://www.sec.gov" +_DATA = "https://data.sec.gov" + +# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py). +_CA = os.environ.get("SSL_CERT_FILE", "") +_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True + +_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"}) + + +class SecError(ProviderError): + """SEC request failed.""" + + +class SecForbiddenError(SecError): + """SEC returned 403 — User-Agent/pattern rejected. Alert and stop.""" + + +def cik10(cik: int | str) -> str: + """Zero-pad a CIK to the 10-digit form SEC URLs use (320193 -> 0000320193).""" + return str(int(cik)).zfill(10) + + +class SecClient: + """Fair-access SEC HTTP client. Use as ``async with SecClient() as c:``.""" + + def __init__( + self, + *, + user_agent: str | None = None, + spacing_seconds: float | None = None, + max_retries: int | None = None, + timeout: float | None = None, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: + self._ua = user_agent or settings.sec_user_agent + self._spacing = ( + spacing_seconds if spacing_seconds is not None else settings.sec_request_spacing_seconds + ) + self._max_retries = ( + max_retries if max_retries is not None else settings.sec_max_retries + ) + self._timeout = timeout if timeout is not None else settings.sec_request_timeout_seconds + self._transport = transport # injectable for tests + self._client: httpx.AsyncClient | None = None + self._lock = asyncio.Lock() + self._last_request = 0.0 + + async def __aenter__(self) -> "SecClient": + self._client = httpx.AsyncClient( + headers={"User-Agent": self._ua, "Accept-Encoding": "gzip, deflate"}, + timeout=self._timeout, + verify=_CA_VERIFY, + transport=self._transport, + ) + return self + + async def __aexit__(self, *exc) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + async def _throttle(self) -> None: + async with self._lock: + now = asyncio.get_event_loop().time() + wait = self._spacing - (now - self._last_request) + if wait > 0: + await asyncio.sleep(wait) + self._last_request = asyncio.get_event_loop().time() + + async def _get(self, url: str) -> httpx.Response: + assert self._client is not None, "use `async with SecClient()`" + attempt = 0 + while True: + await self._throttle() + resp = await self._client.get(url) + if resp.status_code == 403: + raise SecForbiddenError( + f"SEC 403 for {url} — User-Agent/pattern rejected; set a real " + "sec_user_agent contact email" + ) + if resp.status_code == 429: + attempt += 1 + if attempt > self._max_retries: + raise SecError(f"SEC 429 after {self._max_retries} retries: {url}") + backoff = min(2.0**attempt, 30.0) + logger.warning("SEC 429 for %s — backoff %.1fs (attempt %d)", url, backoff, attempt) + await asyncio.sleep(backoff) + continue + if resp.status_code >= 400: + raise SecError(f"SEC {resp.status_code} for {url}") + return resp + + async def get_json(self, url: str) -> Any: + return (await self._get(url)).json() + + async def get_text(self, url: str) -> str: + return (await self._get(url)).text + + # -- domain fetchers --------------------------------------------------- + + async def company_tickers(self) -> dict[str, int]: + """Map normalised ticker -> CIK (int). Multi-class tickers share a CIK.""" + data = await self.get_json(f"{_WWW}/files/company_tickers.json") + out: dict[str, int] = {} + for row in data.values(): + sym = normalise_symbol(row.get("ticker")) + if sym: + out[sym] = int(row["cik_str"]) + return out + + async def submissions(self, cik: int | str) -> dict[str, Any]: + """Issuer metadata + the FULL merged filing history. + + ``filings.recent`` caps at 1000; older accessions live in + ``filings.files[]`` shards. This merges them so backfill sees every + filing's reportDate / acceptanceDateTime / isXBRL. + """ + base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json") + filings = _rows_from_arrays(base["filings"]["recent"]) + for shard in base["filings"].get("files") or []: + shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}") + filings.extend(_rows_from_arrays(shard_data)) + return { + "cik": int(base["cik"]), + "name": base.get("name"), + "sic": base.get("sic"), + "sic_description": base.get("sicDescription"), + "fiscal_year_end": base.get("fiscalYearEnd"), + "tickers": base.get("tickers") or [], + "filings": filings, + } + + async def companyfacts(self, cik: int | str) -> dict[str, Any]: + """Raw companyfacts JSON ({cik, entityName, facts}).""" + return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json") + + async def latest_index_date(self, today: date | None = None) -> date | None: + """The most recent published daily-index date (drives the revision). Checks + the current quarter, falling back to the previous one at a quarter boundary.""" + today = today or date.today() + for year, qtr in _quarters_back(today, 2): + url = f"{_WWW}/Archives/edgar/daily-index/{year}/QTR{qtr}/index.json" + try: + idx = await self.get_json(url) + except SecError: + continue + dates = [ + d + for item in idx.get("directory", {}).get("item", []) + if (d := _index_file_date(item.get("name", ""))) is not None + and d <= today + ] + if dates: + return max(dates) + return None + + async def daily_index(self, day: date) -> list[dict[str, Any]]: + """Parse the daily form index into 10-K/10-Q(/A) rows for all issuers. + + Returns [{form, cik, accession, company}]. The caller filters to the + tracked universe. A missing index (weekend/holiday/not-yet-published) + returns [] rather than raising. + """ + qtr = (day.month - 1) // 3 + 1 + url = f"{_WWW}/Archives/edgar/daily-index/{day.year}/QTR{qtr}/form.{day:%Y%m%d}.idx" + try: + text = await self.get_text(url) + except SecError as exc: + logger.info("no daily index for %s (%s)", day, exc) + return [] + return _parse_form_index(text) + + +def _rows_from_arrays(arrays: dict[str, list]) -> list[dict[str, Any]]: + """Turn SEC's parallel-array filing block into row dicts (keeping only 10-K/10-Q + family filings — the ones that carry XBRL fundamentals).""" + forms = arrays.get("form", []) + out: list[dict[str, Any]] = [] + for i, form in enumerate(forms): + if form not in _FORMS_10: + continue + out.append( + { + "accession": arrays["accessionNumber"][i], + "form": form, + "report_date": arrays["reportDate"][i] or None, + "acceptance_datetime": arrays["acceptanceDateTime"][i] or None, + "is_xbrl": bool(arrays.get("isXBRL", [0] * len(forms))[i]), + } + ) + return out + + +def _parse_form_index(text: str) -> list[dict[str, Any]]: + """Parse a daily ``form.YYYYMMDD.idx`` (fixed columns: Form / Company / CIK / + Date Filed / File Name-with-accession).""" + rows: list[dict[str, Any]] = [] + started = False + for line in text.splitlines(): + if not started: + if set(line.strip()) == {"-"}: # the dashed separator row + started = True + continue + parts = line.split() + if len(parts) < 5: + continue + form = parts[0] + if form not in _FORMS_10: + continue + path = parts[-1] # edgar/data//.txt + cik = _cik_from_path(path) + accession = _accession_from_path(path) + if cik is None or accession is None: + continue + rows.append({"form": form, "cik": cik, "accession": accession, "path": path}) + return rows + + +def _index_file_date(name: str) -> date | None: + if name.startswith("form.") and name.endswith(".idx"): + try: + return datetime.strptime(name[5:13], "%Y%m%d").date() + except ValueError: + return None + return None + + +def _quarters_back(today: date, n: int) -> list[tuple[int, int]]: + """(year, quarter) for `today`'s quarter and the previous n-1, newest first.""" + q = (today.month - 1) // 3 + 1 + out = [] + y = today.year + for _ in range(n): + out.append((y, q)) + q -= 1 + if q == 0: + q = 4 + y -= 1 + return out + + +def _cik_from_path(path: str) -> int | None: + segs = path.split("/") + if len(segs) >= 3 and segs[2].isdigit(): + return int(segs[2]) + return None + + +def _accession_from_path(path: str) -> str | None: + stem = path.rsplit("/", 1)[-1] + if stem.endswith(".txt"): + stem = stem[:-4] + return stem or None diff --git a/app/services/sec_universe.py b/app/services/sec_universe.py new file mode 100644 index 0000000..15d4596 --- /dev/null +++ b/app/services/sec_universe.py @@ -0,0 +1,87 @@ +"""Tracked-universe CIK/SIC resolution and the SEC importer's composite revision. + +Resolves the app's tracked tickers to SEC issuers (CIK) and back-fills +``tickers.cik/sic/sic_description``. Also builds the **universe fingerprint** that +goes into the importer's composite revision, so that adding a ticker changes the +revision and forces a run instead of being ``no_op``'d away or starved waiting for +its issuer to file (A3 design, Decision 1 review fix). +""" + +from __future__ import annotations + +import hashlib +import logging +from typing import Iterable + +from sqlalchemy import select + +from app.models.ticker import Ticker +from app.services.earnings_alignment import normalise_symbol +from app.services.sec_client import SecClient + +logger = logging.getLogger(__name__) + + +async def resolve_ciks(db, client: SecClient) -> dict[str, int]: + """Resolve tracked tickers to CIKs via company_tickers.json and persist + ``tickers.cik`` where it changed. Returns {normalised symbol: cik} for the + tracked tickers that resolved (multi-class tickers share a CIK).""" + ticker_to_cik = await client.company_tickers() + rows = (await db.execute(select(Ticker))).scalars().all() + + resolved: dict[str, int] = {} + changed = 0 + for t in rows: + if not t.symbol: + continue + sym = normalise_symbol(t.symbol) + cik = ticker_to_cik.get(sym) + if cik is None: + continue # e.g. ADRs / non-SEC issuers — snapshots simply absent + resolved[sym] = cik + cik_str = f"{cik:010d}" + if t.cik != cik_str: + t.cik = cik_str + changed += 1 + logger.info("resolve_ciks: %d tracked resolved, %d cik updates", len(resolved), changed) + return resolved + + +async def refresh_sic(db, client: SecClient, ciks: Iterable[int]) -> int: + """Fetch submissions for the given CIKs and set sic/sic_description on every + tracked ticker sharing each CIK. Returns the number of CIKs refreshed. Callers + pass only the CIKs that need it (e.g. those still missing a SIC) to stay light.""" + refreshed = 0 + for cik in sorted(set(int(c) for c in ciks)): + sub = await client.submissions(cik) + cik_str = f"{cik:010d}" + rows = ( + await db.execute(select(Ticker).where(Ticker.cik == cik_str)) + ).scalars().all() + for t in rows: + t.sic = str(sub["sic"]) if sub.get("sic") else None + t.sic_description = sub.get("sic_description") + refreshed += 1 + return refreshed + + +def universe_fingerprint(symbol_to_cik: dict[str, int]) -> str: + """Stable short hash of the tracked symbol->CIK set. Changes whenever a ticker + is added/removed or its CIK mapping changes.""" + canonical = ";".join(f"{sym}:{cik}" for sym, cik in sorted(symbol_to_cik.items())) + return hashlib.blake2b(canonical.encode("utf-8"), digest_size=12).hexdigest() + + +def compose_revision( + index_date, index_content_hash: str, symbol_to_cik: dict[str, int] +) -> str: + """The importer's composite revision: latest processed index date + a hash of + the index content consumed this run + the universe fingerprint. Equal revision + across runs ⇒ nothing new to import ⇒ no_op.""" + return f"{index_date}:{index_content_hash}:{universe_fingerprint(symbol_to_cik)}" + + +def index_content_hash(index_rows: Iterable[dict]) -> str: + """Order-independent hash of the tracked index accessions consumed this run.""" + keys = sorted(f"{r['cik']}/{r['accession']}" for r in index_rows) + return hashlib.blake2b("|".join(keys).encode("utf-8"), digest_size=12).hexdigest() diff --git a/tests/unit/test_sec_client.py b/tests/unit/test_sec_client.py new file mode 100644 index 0000000..f98d538 --- /dev/null +++ b/tests/unit/test_sec_client.py @@ -0,0 +1,159 @@ +"""Tests for the SEC client's parsing and fair-access behavior, via a mocked +httpx transport (no network).""" + +from __future__ import annotations + +import json +from datetime import date + +import httpx +import pytest + +from app.services import sec_client as sc +from app.services.sec_client import SecClient, SecError, SecForbiddenError + +COMPANY_TICKERS = { + "0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."}, + "1": {"cik_str": 1652044, "ticker": "GOOGL", "title": "Alphabet"}, + "2": {"cik_str": 1652044, "ticker": "GOOG", "title": "Alphabet"}, + "3": {"cik_str": 1067983, "ticker": "BRK-B", "title": "Berkshire"}, +} + +SUBMISSIONS_BASE = { + "cik": 320193, + "name": "Apple Inc.", + "sic": "3571", + "sicDescription": "Electronic Computers", + "fiscalYearEnd": "0926", + "tickers": ["AAPL"], + "filings": { + "recent": { + "accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"], + "form": ["10-Q", "10-K", "8-K"], + "reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"], + "acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"], + "isXBRL": [1, 1, 0], + }, + "files": [{"name": "CIK0000320193-submissions-001.json", "filingFrom": "1994-01-26", "filingTo": "2015-05-27"}], + }, +} + +SUBMISSIONS_SHARD = { + "accessionNumber": ["0000320193-94-000002"], + "form": ["10-Q"], + "reportDate": ["1993-12-31"], + "acceptanceDateTime": ["1994-01-26T05:00:00.000Z"], + "isXBRL": [0], +} + +FORM_IDX = """Description: Daily Index of EDGAR Dissemination Feed by Form Type + + Form Type Company Name CIK Date Filed File Name +------------------------------------------------------------------------------- + 10-K/A Starfighters Space, Inc. 1947016 20260721 edgar/data/1947016/0001062993-26-003746.txt + 10-Q 3M CO 66740 20260721 edgar/data/66740/0000066740-26-000246.txt + 8-K Some Corp 12345 20260721 edgar/data/12345/0000012345-26-000001.txt + 10-Q CALIX, INC 1406666 20260721 edgar/data/1406666/0001406666-26-000034.txt +""" + +DIR_JSON = {"directory": {"item": [ + {"name": "form.20260720.idx"}, {"name": "form.20260721.idx"}, {"name": "company.20260721.idx"}, +]}} + + +def _handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("company_tickers.json"): + return httpx.Response(200, json=COMPANY_TICKERS) + if url.endswith("submissions/CIK0000320193.json"): + return httpx.Response(200, json=SUBMISSIONS_BASE) + if url.endswith("CIK0000320193-submissions-001.json"): + return httpx.Response(200, json=SUBMISSIONS_SHARD) + if url.endswith("form.20260721.idx"): + return httpx.Response(200, text=FORM_IDX) + if url.endswith("QTR3/index.json"): + return httpx.Response(200, json=DIR_JSON) + return httpx.Response(404) + + +def _client(**kw): + return SecClient(transport=httpx.MockTransport(_handler), spacing_seconds=0, **kw) + + +async def test_company_tickers_normalised_and_multiclass(): + async with _client() as c: + m = await c.company_tickers() + assert m["AAPL"] == 320193 + assert m["GOOGL"] == m["GOOG"] == 1652044 # multi-class share one CIK + assert m["BRK-B"] == 1067983 # dash form + + +async def test_submissions_merges_shards_and_filters_forms(): + async with _client() as c: + sub = await c.submissions(320193) + assert sub["sic"] == "3571" and sub["fiscal_year_end"] == "0926" + accns = {f["accession"] for f in sub["filings"]} + # 10-Q + 10-K from recent, 10-Q from the shard; the 8-K is filtered out + assert accns == {"0000320193-26-000013", "0000320193-26-000006", "0000320193-94-000002"} + older = next(f for f in sub["filings"] if f["accession"] == "0000320193-94-000002") + assert older["report_date"] == "1993-12-31" and older["is_xbrl"] is False + + +async def test_daily_index_parses_10kq_rows(): + async with _client() as c: + rows = await c.daily_index(date(2026, 7, 21)) + forms = {r["form"] for r in rows} + assert forms == {"10-K/A", "10-Q"} # 8-K excluded + mmm = next(r for r in rows if r["cik"] == 66740) + assert mmm["accession"] == "0000066740-26-000246" + + +async def test_latest_index_date(): + async with _client() as c: + d = await c.latest_index_date(today=date(2026, 7, 22)) + assert d == date(2026, 7, 21) + + +async def test_403_raises_forbidden_no_retry(): + calls = {"n": 0} + + def handler(request): + calls["n"] += 1 + return httpx.Response(403) + + async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c: + with pytest.raises(SecForbiddenError): + await c.get_json("https://data.sec.gov/x") + assert calls["n"] == 1 # alert and stop, never retry-loop + + +async def test_429_retries_then_succeeds(monkeypatch): + async def _instant(_): + return None + + monkeypatch.setattr(sc.asyncio, "sleep", _instant) # no real backoff wait + calls = {"n": 0} + + def handler(request): + calls["n"] += 1 + if calls["n"] < 3: + return httpx.Response(429) + return httpx.Response(200, json={"ok": True}) + + async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=5) as c: + data = await c.get_json("https://data.sec.gov/y") + assert data == {"ok": True} and calls["n"] == 3 + + +async def test_429_gives_up_after_max_retries(monkeypatch): + async def _instant(_): + return None + + monkeypatch.setattr(sc.asyncio, "sleep", _instant) + + def handler(request): + return httpx.Response(429) + + async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=2) as c: + with pytest.raises(SecError): + await c.get_json("https://data.sec.gov/z") diff --git a/tests/unit/test_sec_universe.py b/tests/unit/test_sec_universe.py new file mode 100644 index 0000000..69f85fb --- /dev/null +++ b/tests/unit/test_sec_universe.py @@ -0,0 +1,105 @@ +"""Tests for CIK/SIC resolution and the composite-revision fingerprint.""" + +from __future__ import annotations + +import os +import tempfile + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.database import Base +import app.models # noqa: F401 +from app.models.ticker import Ticker +from app.services import sec_universe as su + + +@pytest.fixture +async def factory(): + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + eng = create_async_engine(f"sqlite+aiosqlite:///{path}") + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + try: + yield async_sessionmaker(eng, class_=AsyncSession, expire_on_commit=False) + finally: + await eng.dispose() + try: + os.unlink(path) + except OSError: + pass + + +class FakeSecClient: + def __init__(self, tickers, submissions=None): + self._tickers = tickers + self._submissions = submissions or {} + + async def company_tickers(self): + return dict(self._tickers) + + async def submissions(self, cik): + return self._submissions[int(cik)] + + +async def test_resolve_ciks_sets_cik_and_returns_mapping(factory): + async with factory() as s: + for sym in ["AAPL", "GOOGL", "GOOG", "ZZZZ"]: # ZZZZ not in SEC + s.add(Ticker(symbol=sym)) + await s.commit() + + client = FakeSecClient({"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044}) + async with factory() as s: + mapping = await su.resolve_ciks(s, client) + await s.commit() + + assert mapping == {"AAPL": 320193, "GOOGL": 1652044, "GOOG": 1652044} + async with factory() as s: + ciks = {t.symbol: t.cik for t in (await s.execute(select(Ticker))).scalars()} + assert ciks["AAPL"] == "0000320193" + assert ciks["GOOGL"] == ciks["GOOG"] == "0001652044" # multi-class share CIK + assert ciks["ZZZZ"] is None # unresolved stays null + + +async def test_refresh_sic_updates_all_tickers_of_a_cik(factory): + async with factory() as s: + for sym in ["GOOGL", "GOOG"]: + s.add(Ticker(symbol=sym, cik="0001652044")) + await s.commit() + + client = FakeSecClient( + {}, submissions={1652044: {"sic": "7370", "sic_description": "Services-Computer"}} + ) + async with factory() as s: + n = await su.refresh_sic(s, client, [1652044]) + await s.commit() + + assert n == 1 + async with factory() as s: + rows = {t.symbol: (t.sic, t.sic_description) for t in (await s.execute(select(Ticker))).scalars()} + assert rows["GOOGL"] == ("7370", "Services-Computer") + assert rows["GOOG"] == ("7370", "Services-Computer") + + +def test_universe_fingerprint_changes_on_membership(): + a = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019}) + same = su.universe_fingerprint({"MSFT": 789019, "AAPL": 320193}) # order-independent + added = su.universe_fingerprint({"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810}) + remapped = su.universe_fingerprint({"AAPL": 999, "MSFT": 789019}) + assert a == same + assert a != added # new ticker forces a new revision + assert a != remapped # changed CIK mapping forces a new revision + + +def test_compose_revision_and_index_hash(): + rows = [ + {"cik": 320193, "accession": "a-1"}, + {"cik": 66740, "accession": "b-2"}, + ] + h1 = su.index_content_hash(rows) + h2 = su.index_content_hash(list(reversed(rows))) + assert h1 == h2 # order-independent + rev = su.compose_revision("2026-07-21", h1, {"AAPL": 320193}) + assert rev.startswith("2026-07-21:") and rev.count(":") == 2