"""Async SEC EDGAR client for the fundamentals importer (workstream A). All access is batch (never at request time). This wraps the three SEC products the A3 design uses — `company_tickers.json`, `submissions/`, `companyfacts/`, and the daily filing index — behind one client that honors SEC's fair-access policy: - an identifying ``User-Agent`` with a contact email on every request (config); - request spacing well under the 10 req/s limit; - exponential backoff + retry on 429; - **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a 403 means the UA or request pattern is wrong and retrying won't fix it. Parsing lives here (index fixed-width, submissions pagination); DB writes and the snapshot mapping live in the importer. No conditional GETs — the companyfacts endpoint exposes no ETag/Last-Modified (verified), which is why the importer is daily-index driven rather than polling archives. """ from __future__ import annotations import asyncio import logging import os from datetime import date, datetime from pathlib import Path from typing import Any import httpx from app.config import settings from app.exceptions import ProviderError from app.services.earnings_alignment import normalise_symbol logger = logging.getLogger(__name__) _WWW = "https://www.sec.gov" _DATA = "https://data.sec.gov" # Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py). _CA = os.environ.get("SSL_CERT_FILE", "") _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"}) class SecError(ProviderError): """SEC request failed.""" class SecForbiddenError(SecError): """SEC returned 403 — User-Agent/pattern rejected. Alert and stop.""" def cik10(cik: int | str) -> str: """Zero-pad a CIK to the 10-digit form SEC URLs use (320193 -> 0000320193).""" return str(int(cik)).zfill(10) class SecClient: """Fair-access SEC HTTP client. Use as ``async with SecClient() as c:``.""" def __init__( self, *, user_agent: str | None = None, spacing_seconds: float | None = None, max_retries: int | None = None, timeout: float | None = None, transport: httpx.AsyncBaseTransport | None = None, ) -> None: self._ua = user_agent or settings.sec_user_agent self._spacing = ( spacing_seconds if spacing_seconds is not None else settings.sec_request_spacing_seconds ) self._max_retries = ( max_retries if max_retries is not None else settings.sec_max_retries ) self._timeout = timeout if timeout is not None else settings.sec_request_timeout_seconds self._transport = transport # injectable for tests self._client: httpx.AsyncClient | None = None self._lock = asyncio.Lock() self._last_request = 0.0 async def __aenter__(self) -> "SecClient": self._client = httpx.AsyncClient( headers={"User-Agent": self._ua, "Accept-Encoding": "gzip, deflate"}, timeout=self._timeout, verify=_CA_VERIFY, transport=self._transport, ) return self async def __aexit__(self, *exc) -> None: if self._client is not None: await self._client.aclose() self._client = None async def _throttle(self) -> None: async with self._lock: now = asyncio.get_event_loop().time() wait = self._spacing - (now - self._last_request) if wait > 0: await asyncio.sleep(wait) self._last_request = asyncio.get_event_loop().time() async def _get(self, url: str) -> httpx.Response: assert self._client is not None, "use `async with SecClient()`" attempt = 0 while True: await self._throttle() resp = await self._client.get(url) if resp.status_code == 403: raise SecForbiddenError( f"SEC 403 for {url} — User-Agent/pattern rejected; set a real " "sec_user_agent contact email" ) if resp.status_code == 429: attempt += 1 if attempt > self._max_retries: raise SecError(f"SEC 429 after {self._max_retries} retries: {url}") backoff = min(2.0**attempt, 30.0) logger.warning("SEC 429 for %s — backoff %.1fs (attempt %d)", url, backoff, attempt) await asyncio.sleep(backoff) continue if resp.status_code >= 400: raise SecError(f"SEC {resp.status_code} for {url}") return resp async def get_json(self, url: str) -> Any: return (await self._get(url)).json() async def get_text(self, url: str) -> str: return (await self._get(url)).text # -- domain fetchers --------------------------------------------------- async def company_tickers(self) -> dict[str, int]: """Map normalised ticker -> CIK (int). Multi-class tickers share a CIK.""" data = await self.get_json(f"{_WWW}/files/company_tickers.json") out: dict[str, int] = {} for row in data.values(): sym = normalise_symbol(row.get("ticker")) if sym: out[sym] = int(row["cik_str"]) return out async def submissions(self, cik: int | str) -> dict[str, Any]: """Issuer metadata + the FULL merged filing history. ``filings.recent`` caps at 1000; older accessions live in ``filings.files[]`` shards. This merges them so backfill sees every filing's reportDate / acceptanceDateTime / isXBRL. """ base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json") filings = _rows_from_arrays(base["filings"]["recent"]) for shard in base["filings"].get("files") or []: shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}") filings.extend(_rows_from_arrays(shard_data)) return { "cik": int(base["cik"]), "name": base.get("name"), "sic": base.get("sic"), "sic_description": base.get("sicDescription"), "fiscal_year_end": base.get("fiscalYearEnd"), "tickers": base.get("tickers") or [], "filings": filings, } async def companyfacts(self, cik: int | str) -> dict[str, Any]: """Raw companyfacts JSON ({cik, entityName, facts}).""" return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json") async def latest_index_date(self, today: date | None = None) -> date | None: """The most recent published daily-index date (drives the revision). Checks the current quarter, falling back to the previous one at a quarter boundary.""" today = today or date.today() for year, qtr in _quarters_back(today, 2): url = f"{_WWW}/Archives/edgar/daily-index/{year}/QTR{qtr}/index.json" try: idx = await self.get_json(url) except SecError: continue dates = [ d for item in idx.get("directory", {}).get("item", []) if (d := _index_file_date(item.get("name", ""))) is not None and d <= today ] if dates: return max(dates) return None async def daily_index(self, day: date) -> list[dict[str, Any]]: """Parse the daily form index into 10-K/10-Q(/A) rows for all issuers. Returns [{form, cik, accession, company}]. The caller filters to the tracked universe. A missing index (weekend/holiday/not-yet-published) returns [] rather than raising. """ qtr = (day.month - 1) // 3 + 1 url = f"{_WWW}/Archives/edgar/daily-index/{day.year}/QTR{qtr}/form.{day:%Y%m%d}.idx" try: text = await self.get_text(url) except SecError as exc: logger.info("no daily index for %s (%s)", day, exc) return [] return _parse_form_index(text) def _rows_from_arrays(arrays: dict[str, list]) -> list[dict[str, Any]]: """Turn SEC's parallel-array filing block into row dicts (keeping only 10-K/10-Q family filings — the ones that carry XBRL fundamentals).""" forms = arrays.get("form", []) out: list[dict[str, Any]] = [] for i, form in enumerate(forms): if form not in _FORMS_10: continue out.append( { "accession": arrays["accessionNumber"][i], "form": form, "report_date": arrays["reportDate"][i] or None, "acceptance_datetime": arrays["acceptanceDateTime"][i] or None, "is_xbrl": bool(arrays.get("isXBRL", [0] * len(forms))[i]), } ) return out def _parse_form_index(text: str) -> list[dict[str, Any]]: """Parse a daily ``form.YYYYMMDD.idx`` (fixed columns: Form / Company / CIK / Date Filed / File Name-with-accession).""" rows: list[dict[str, Any]] = [] started = False for line in text.splitlines(): if not started: if set(line.strip()) == {"-"}: # the dashed separator row started = True continue parts = line.split() if len(parts) < 5: continue form = parts[0] if form not in _FORMS_10: continue path = parts[-1] # edgar/data//.txt cik = _cik_from_path(path) accession = _accession_from_path(path) if cik is None or accession is None: continue rows.append({"form": form, "cik": cik, "accession": accession, "path": path}) return rows def _index_file_date(name: str) -> date | None: if name.startswith("form.") and name.endswith(".idx"): try: return datetime.strptime(name[5:13], "%Y%m%d").date() except ValueError: return None return None def _quarters_back(today: date, n: int) -> list[tuple[int, int]]: """(year, quarter) for `today`'s quarter and the previous n-1, newest first.""" q = (today.month - 1) // 3 + 1 out = [] y = today.year for _ in range(n): out.append((y, q)) q -= 1 if q == 0: q = 4 y -= 1 return out def _cik_from_path(path: str) -> int | None: segs = path.split("/") if len(segs) >= 3 and segs[2].isdigit(): return int(segs[2]) return None def _accession_from_path(path: str) -> str | None: stem = path.rsplit("/", 1)[-1] if stem.endswith(".txt"): stem = stem[:-4] return stem or None