feat(sec): A3 slice 1 — SEC client + CIK/SIC resolution + composite revision
First A3 implementation checkpoint (design: docs/dolt-sec-a3-design.md). - sec_client.py: async SEC EDGAR client honoring fair-access — identifying User-Agent (config), request spacing < 10 req/s, exponential backoff on 429, and 403 -> SecForbiddenError (alert and stop, never retry-loop). Fetchers: company_tickers (normalised, multi-class share CIK), submissions (merges the paginated filings.files shards so full history is visible), companyfacts, daily_index (fixed-width form.idx parse), latest_index_date. - sec_universe.py: resolve_ciks (tickers.cik backfill), refresh_sic (sic/sic_description), and the composite-revision pieces — universe_fingerprint (a new ticker changes the revision, so it's never no_op'd/starved), index_content_hash, compose_revision. - config + .env.example: SEC_USER_AGENT (must be a real contact email) + spacing / retries / timeout. Verified live against real SEC: AAPL->320193, GOOG==GOOGL, BRK-B resolved; submissions shard-merge proven (131 filings back to 1993); daily index parsed. Tests: 11 (mocked-transport parsing + 403/429 handling + resolution/fingerprint). Full suite 713 passed. Next slice: companyfacts -> snapshot parser + importer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
"""Tracked-universe CIK/SIC resolution and the SEC importer's composite revision.
|
||||
|
||||
Resolves the app's tracked tickers to SEC issuers (CIK) and back-fills
|
||||
``tickers.cik/sic/sic_description``. Also builds the **universe fingerprint** that
|
||||
goes into the importer's composite revision, so that adding a ticker changes the
|
||||
revision and forces a run instead of being ``no_op``'d away or starved waiting for
|
||||
its issuer to file (A3 design, Decision 1 review fix).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def resolve_ciks(db, client: SecClient) -> dict[str, int]:
|
||||
"""Resolve tracked tickers to CIKs via company_tickers.json and persist
|
||||
``tickers.cik`` where it changed. Returns {normalised symbol: cik} for the
|
||||
tracked tickers that resolved (multi-class tickers share a CIK)."""
|
||||
ticker_to_cik = await client.company_tickers()
|
||||
rows = (await db.execute(select(Ticker))).scalars().all()
|
||||
|
||||
resolved: dict[str, int] = {}
|
||||
changed = 0
|
||||
for t in rows:
|
||||
if not t.symbol:
|
||||
continue
|
||||
sym = normalise_symbol(t.symbol)
|
||||
cik = ticker_to_cik.get(sym)
|
||||
if cik is None:
|
||||
continue # e.g. ADRs / non-SEC issuers — snapshots simply absent
|
||||
resolved[sym] = cik
|
||||
cik_str = f"{cik:010d}"
|
||||
if t.cik != cik_str:
|
||||
t.cik = cik_str
|
||||
changed += 1
|
||||
logger.info("resolve_ciks: %d tracked resolved, %d cik updates", len(resolved), changed)
|
||||
return resolved
|
||||
|
||||
|
||||
async def refresh_sic(db, client: SecClient, ciks: Iterable[int]) -> int:
|
||||
"""Fetch submissions for the given CIKs and set sic/sic_description on every
|
||||
tracked ticker sharing each CIK. Returns the number of CIKs refreshed. Callers
|
||||
pass only the CIKs that need it (e.g. those still missing a SIC) to stay light."""
|
||||
refreshed = 0
|
||||
for cik in sorted(set(int(c) for c in ciks)):
|
||||
sub = await client.submissions(cik)
|
||||
cik_str = f"{cik:010d}"
|
||||
rows = (
|
||||
await db.execute(select(Ticker).where(Ticker.cik == cik_str))
|
||||
).scalars().all()
|
||||
for t in rows:
|
||||
t.sic = str(sub["sic"]) if sub.get("sic") else None
|
||||
t.sic_description = sub.get("sic_description")
|
||||
refreshed += 1
|
||||
return refreshed
|
||||
|
||||
|
||||
def universe_fingerprint(symbol_to_cik: dict[str, int]) -> str:
|
||||
"""Stable short hash of the tracked symbol->CIK set. Changes whenever a ticker
|
||||
is added/removed or its CIK mapping changes."""
|
||||
canonical = ";".join(f"{sym}:{cik}" for sym, cik in sorted(symbol_to_cik.items()))
|
||||
return hashlib.blake2b(canonical.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
|
||||
def compose_revision(
|
||||
index_date, index_content_hash: str, symbol_to_cik: dict[str, int]
|
||||
) -> str:
|
||||
"""The importer's composite revision: latest processed index date + a hash of
|
||||
the index content consumed this run + the universe fingerprint. Equal revision
|
||||
across runs ⇒ nothing new to import ⇒ no_op."""
|
||||
return f"{index_date}:{index_content_hash}:{universe_fingerprint(symbol_to_cik)}"
|
||||
|
||||
|
||||
def index_content_hash(index_rows: Iterable[dict]) -> str:
|
||||
"""Order-independent hash of the tracked index accessions consumed this run."""
|
||||
keys = sorted(f"{r['cik']}/{r['accession']}" for r in index_rows)
|
||||
return hashlib.blake2b("|".join(keys).encode("utf-8"), digest_size=12).hexdigest()
|
||||
Reference in New Issue
Block a user