feat(sec): A3 slice 1 — SEC client + CIK/SIC resolution + composite revision
First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,289 @@
|
||||
"""Async SEC EDGAR client for the fundamentals importer (workstream A).
|
||||
|
||||
All access is batch (never at request time). This wraps the three SEC products
|
||||
the A3 design uses — `company_tickers.json`, `submissions/`, `companyfacts/`, and
|
||||
the daily filing index — behind one client that honors SEC's fair-access policy:
|
||||
|
||||
- an identifying ``User-Agent`` with a contact email on every request (config);
|
||||
- request spacing well under the 10 req/s limit;
|
||||
- exponential backoff + retry on 429;
|
||||
- **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a
|
||||
403 means the UA or request pattern is wrong and retrying won't fix it.
|
||||
|
||||
Parsing lives here (index fixed-width, submissions pagination); DB writes and the
|
||||
snapshot mapping live in the importer. No conditional GETs — the companyfacts
|
||||
endpoint exposes no ETag/Last-Modified (verified), which is why the importer is
|
||||
daily-index driven rather than polling archives.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.exceptions import ProviderError
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_WWW = "https://www.sec.gov"
|
||||
_DATA = "https://data.sec.gov"
|
||||
|
||||
# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py).
|
||||
_CA = os.environ.get("SSL_CERT_FILE", "")
|
||||
_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
|
||||
|
||||
_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
|
||||
|
||||
|
||||
class SecError(ProviderError):
|
||||
"""SEC request failed."""
|
||||
|
||||
|
||||
class SecForbiddenError(SecError):
|
||||
"""SEC returned 403 — User-Agent/pattern rejected. Alert and stop."""
|
||||
|
||||
|
||||
def cik10(cik: int | str) -> str:
|
||||
"""Zero-pad a CIK to the 10-digit form SEC URLs use (320193 -> 0000320193)."""
|
||||
return str(int(cik)).zfill(10)
|
||||
|
||||
|
||||
class SecClient:
|
||||
"""Fair-access SEC HTTP client. Use as ``async with SecClient() as c:``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
user_agent: str | None = None,
|
||||
spacing_seconds: float | None = None,
|
||||
max_retries: int | None = None,
|
||||
timeout: float | None = None,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
self._ua = user_agent or settings.sec_user_agent
|
||||
self._spacing = (
|
||||
spacing_seconds if spacing_seconds is not None else settings.sec_request_spacing_seconds
|
||||
)
|
||||
self._max_retries = (
|
||||
max_retries if max_retries is not None else settings.sec_max_retries
|
||||
)
|
||||
self._timeout = timeout if timeout is not None else settings.sec_request_timeout_seconds
|
||||
self._transport = transport # injectable for tests
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._last_request = 0.0
|
||||
|
||||
async def __aenter__(self) -> "SecClient":
|
||||
self._client = httpx.AsyncClient(
|
||||
headers={"User-Agent": self._ua, "Accept-Encoding": "gzip, deflate"},
|
||||
timeout=self._timeout,
|
||||
verify=_CA_VERIFY,
|
||||
transport=self._transport,
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _throttle(self) -> None:
|
||||
async with self._lock:
|
||||
now = asyncio.get_event_loop().time()
|
||||
wait = self._spacing - (now - self._last_request)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._last_request = asyncio.get_event_loop().time()
|
||||
|
||||
async def _get(self, url: str) -> httpx.Response:
|
||||
assert self._client is not None, "use `async with SecClient()`"
|
||||
attempt = 0
|
||||
while True:
|
||||
await self._throttle()
|
||||
resp = await self._client.get(url)
|
||||
if resp.status_code == 403:
|
||||
raise SecForbiddenError(
|
||||
f"SEC 403 for {url} — User-Agent/pattern rejected; set a real "
|
||||
"sec_user_agent contact email"
|
||||
)
|
||||
if resp.status_code == 429:
|
||||
attempt += 1
|
||||
if attempt > self._max_retries:
|
||||
raise SecError(f"SEC 429 after {self._max_retries} retries: {url}")
|
||||
backoff = min(2.0**attempt, 30.0)
|
||||
logger.warning("SEC 429 for %s — backoff %.1fs (attempt %d)", url, backoff, attempt)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
if resp.status_code >= 400:
|
||||
raise SecError(f"SEC {resp.status_code} for {url}")
|
||||
return resp
|
||||
|
||||
async def get_json(self, url: str) -> Any:
|
||||
return (await self._get(url)).json()
|
||||
|
||||
async def get_text(self, url: str) -> str:
|
||||
return (await self._get(url)).text
|
||||
|
||||
# -- domain fetchers ---------------------------------------------------
|
||||
|
||||
async def company_tickers(self) -> dict[str, int]:
|
||||
"""Map normalised ticker -> CIK (int). Multi-class tickers share a CIK."""
|
||||
data = await self.get_json(f"{_WWW}/files/company_tickers.json")
|
||||
out: dict[str, int] = {}
|
||||
for row in data.values():
|
||||
sym = normalise_symbol(row.get("ticker"))
|
||||
if sym:
|
||||
out[sym] = int(row["cik_str"])
|
||||
return out
|
||||
|
||||
async def submissions(self, cik: int | str) -> dict[str, Any]:
|
||||
"""Issuer metadata + the FULL merged filing history.
|
||||
|
||||
``filings.recent`` caps at 1000; older accessions live in
|
||||
``filings.files[]`` shards. This merges them so backfill sees every
|
||||
filing's reportDate / acceptanceDateTime / isXBRL.
|
||||
"""
|
||||
base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
|
||||
filings = _rows_from_arrays(base["filings"]["recent"])
|
||||
for shard in base["filings"].get("files") or []:
|
||||
shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}")
|
||||
filings.extend(_rows_from_arrays(shard_data))
|
||||
return {
|
||||
"cik": int(base["cik"]),
|
||||
"name": base.get("name"),
|
||||
"sic": base.get("sic"),
|
||||
"sic_description": base.get("sicDescription"),
|
||||
"fiscal_year_end": base.get("fiscalYearEnd"),
|
||||
"tickers": base.get("tickers") or [],
|
||||
"filings": filings,
|
||||
}
|
||||
|
||||
async def companyfacts(self, cik: int | str) -> dict[str, Any]:
|
||||
"""Raw companyfacts JSON ({cik, entityName, facts})."""
|
||||
return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")
|
||||
|
||||
async def latest_index_date(self, today: date | None = None) -> date | None:
|
||||
"""The most recent published daily-index date (drives the revision). Checks
|
||||
the current quarter, falling back to the previous one at a quarter boundary."""
|
||||
today = today or date.today()
|
||||
for year, qtr in _quarters_back(today, 2):
|
||||
url = f"{_WWW}/Archives/edgar/daily-index/{year}/QTR{qtr}/index.json"
|
||||
try:
|
||||
idx = await self.get_json(url)
|
||||
except SecError:
|
||||
continue
|
||||
dates = [
|
||||
d
|
||||
for item in idx.get("directory", {}).get("item", [])
|
||||
if (d := _index_file_date(item.get("name", ""))) is not None
|
||||
and d <= today
|
||||
]
|
||||
if dates:
|
||||
return max(dates)
|
||||
return None
|
||||
|
||||
async def daily_index(self, day: date) -> list[dict[str, Any]]:
|
||||
"""Parse the daily form index into 10-K/10-Q(/A) rows for all issuers.
|
||||
|
||||
Returns [{form, cik, accession, company}]. The caller filters to the
|
||||
tracked universe. A missing index (weekend/holiday/not-yet-published)
|
||||
returns [] rather than raising.
|
||||
"""
|
||||
qtr = (day.month - 1) // 3 + 1
|
||||
url = f"{_WWW}/Archives/edgar/daily-index/{day.year}/QTR{qtr}/form.{day:%Y%m%d}.idx"
|
||||
try:
|
||||
text = await self.get_text(url)
|
||||
except SecError as exc:
|
||||
logger.info("no daily index for %s (%s)", day, exc)
|
||||
return []
|
||||
return _parse_form_index(text)
|
||||
|
||||
|
||||
def _rows_from_arrays(arrays: dict[str, list]) -> list[dict[str, Any]]:
|
||||
"""Turn SEC's parallel-array filing block into row dicts (keeping only 10-K/10-Q
|
||||
family filings — the ones that carry XBRL fundamentals)."""
|
||||
forms = arrays.get("form", [])
|
||||
out: list[dict[str, Any]] = []
|
||||
for i, form in enumerate(forms):
|
||||
if form not in _FORMS_10:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"accession": arrays["accessionNumber"][i],
|
||||
"form": form,
|
||||
"report_date": arrays["reportDate"][i] or None,
|
||||
"acceptance_datetime": arrays["acceptanceDateTime"][i] or None,
|
||||
"is_xbrl": bool(arrays.get("isXBRL", [0] * len(forms))[i]),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _parse_form_index(text: str) -> list[dict[str, Any]]:
|
||||
"""Parse a daily ``form.YYYYMMDD.idx`` (fixed columns: Form / Company / CIK /
|
||||
Date Filed / File Name-with-accession)."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
started = False
|
||||
for line in text.splitlines():
|
||||
if not started:
|
||||
if set(line.strip()) == {"-"}: # the dashed separator row
|
||||
started = True
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
form = parts[0]
|
||||
if form not in _FORMS_10:
|
||||
continue
|
||||
path = parts[-1] # edgar/data/<cik>/<accession>.txt
|
||||
cik = _cik_from_path(path)
|
||||
accession = _accession_from_path(path)
|
||||
if cik is None or accession is None:
|
||||
continue
|
||||
rows.append({"form": form, "cik": cik, "accession": accession, "path": path})
|
||||
return rows
|
||||
|
||||
|
||||
def _index_file_date(name: str) -> date | None:
|
||||
if name.startswith("form.") and name.endswith(".idx"):
|
||||
try:
|
||||
return datetime.strptime(name[5:13], "%Y%m%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _quarters_back(today: date, n: int) -> list[tuple[int, int]]:
|
||||
"""(year, quarter) for `today`'s quarter and the previous n-1, newest first."""
|
||||
q = (today.month - 1) // 3 + 1
|
||||
out = []
|
||||
y = today.year
|
||||
for _ in range(n):
|
||||
out.append((y, q))
|
||||
q -= 1
|
||||
if q == 0:
|
||||
q = 4
|
||||
y -= 1
|
||||
return out
|
||||
|
||||
|
||||
def _cik_from_path(path: str) -> int | None:
|
||||
segs = path.split("/")
|
||||
if len(segs) >= 3 and segs[2].isdigit():
|
||||
return int(segs[2])
|
||||
return None
|
||||
|
||||
|
||||
def _accession_from_path(path: str) -> str | None:
|
||||
stem = path.rsplit("/", 1)[-1]
|
||||
if stem.endswith(".txt"):
|
||||
stem = stem[:-4]
|
||||
return stem or None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Tracked-universe CIK/SIC resolution and the SEC importer's composite revision.
|
||||
|
||||
Resolves the app's tracked tickers to SEC issuers (CIK) and back-fills
|
||||
``tickers.cik/sic/sic_description``. Also builds the **universe fingerprint** that
|
||||
goes into the importer's composite revision, so that adding a ticker changes the
|
||||
revision and forces a run instead of being ``no_op``'d away or starved waiting for
|
||||
its issuer to file (A3 design, Decision 1 review fix).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def resolve_ciks(db, client: SecClient) -> dict[str, int]:
|
||||
"""Resolve tracked tickers to CIKs via company_tickers.json and persist
|
||||
``tickers.cik`` where it changed. Returns {normalised symbol: cik} for the
|
||||
tracked tickers that resolved (multi-class tickers share a CIK)."""
|
||||
ticker_to_cik = await client.company_tickers()
|
||||
rows = (await db.execute(select(Ticker))).scalars().all()
|
||||
|
||||
resolved: dict[str, int] = {}
|
||||
changed = 0
|
||||
for t in rows:
|
||||
if not t.symbol:
|
||||
continue
|
||||
sym = normalise_symbol(t.symbol)
|
||||
cik = ticker_to_cik.get(sym)
|
||||
if cik is None:
|
||||
continue # e.g. ADRs / non-SEC issuers — snapshots simply absent
|
||||
resolved[sym] = cik
|
||||
cik_str = f"{cik:010d}"
|
||||
if t.cik != cik_str:
|
||||
t.cik = cik_str
|
||||
changed += 1
|
||||
logger.info("resolve_ciks: %d tracked resolved, %d cik updates", len(resolved), changed)
|
||||
return resolved
|
||||
|
||||
|
||||
async def refresh_sic(db, client: SecClient, ciks: Iterable[int]) -> int:
|
||||
"""Fetch submissions for the given CIKs and set sic/sic_description on every
|
||||
tracked ticker sharing each CIK. Returns the number of CIKs refreshed. Callers
|
||||
pass only the CIKs that need it (e.g. those still missing a SIC) to stay light."""
|
||||
refreshed = 0
|
||||
for cik in sorted(set(int(c) for c in ciks)):
|
||||
sub = await client.submissions(cik)
|
||||
cik_str = f"{cik:010d}"
|
||||
rows = (
|
||||
await db.execute(select(Ticker).where(Ticker.cik == cik_str))
|
||||
).scalars().all()
|
||||
for t in rows:
|
||||
t.sic = str(sub["sic"]) if sub.get("sic") else None
|
||||
t.sic_description = sub.get("sic_description")
|
||||
refreshed += 1
|
||||
return refreshed
|
||||
|
||||
|
||||
def universe_fingerprint(symbol_to_cik: dict[str, int]) -> str:
|
||||
"""Stable short hash of the tracked symbol->CIK set. Changes whenever a ticker
|
||||
is added/removed or its CIK mapping changes."""
|
||||
canonical = ";".join(f"{sym}:{cik}" for sym, cik in sorted(symbol_to_cik.items()))
|
||||
return hashlib.blake2b(canonical.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
|
||||
def compose_revision(
|
||||
index_date, index_content_hash: str, symbol_to_cik: dict[str, int]
|
||||
) -> str:
|
||||
"""The importer's composite revision: latest processed index date + a hash of
|
||||
the index content consumed this run + the universe fingerprint. Equal revision
|
||||
across runs ⇒ nothing new to import ⇒ no_op."""
|
||||
return f"{index_date}:{index_content_hash}:{universe_fingerprint(symbol_to_cik)}"
|
||||
|
||||
|
||||
def index_content_hash(index_rows: Iterable[dict]) -> str:
|
||||
"""Order-independent hash of the tracked index accessions consumed this run."""
|
||||
keys = sorted(f"{r['cik']}/{r['accession']}" for r in index_rows)
|
||||
return hashlib.blake2b("|".join(keys).encode("utf-8"), digest_size=12).hexdigest()
|
||||
Reference in New Issue
Block a user