dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sec): A3 slice-1 review — read-only resolution, error propagation, fair-access

Browse Source 
Addresses the slice-1 review:

1. Resolution is now read-only (A1 transaction contract). resolve_ciks /
   fetch_sic_updates compute proposals and mutate nothing; a new
   apply_ticker_updates issues the writes, called only in promote — so a failed
   validation can't leak ticker changes on the framework's failure commit.
2. Only 404 means "missing". Added SecNotFoundError; daily_index /
   latest_index_date catch only that. 403, exhausted 429, 5xx, timeouts, and
   transport/parse errors now propagate instead of looking like "no index".
3. Fair-access enforced when opening a REAL client (transport=None): reject
   blank/placeholder/non-email User-Agent and sub-0.11s spacing. Mock transports
   skip it (tests use 0 spacing).
4. submissions(include_history=False) by default — only the one-time full
   backfill fetches the history shards; SIC/incremental work makes no extra
   requests.

Plus: retry transient 5xx/network errors and honor Retry-After during the 1 GB
backfill; compose_revision rejects a missing index date (no "None:..." revision).

Re-verified live vs real SEC (fair-access validation passes, shard merge intact).
Tests: 18 (added error propagation, read-only resolution, fair-access, recent-only
submissions, reject-None revision).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
dennisthiessen
2026-07-22 14:49:33 +02:00
co-authored by Claude Opus 4.8
parent cc67aebe61
commit 5939be7b7f
4 changed files with 267 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/sec_client.py
+83 -22
View File
@@ -21,6 +21,7 @@ from __future__ import annotations
import asyncio
import logging
import os
import re
from datetime import date, datetime
from pathlib import Path
from typing import Any
@@ -44,13 +45,31 @@ _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
class SecError(ProviderError):
"""SEC request failed."""
"""SEC request failed (403, exhausted 429/5xx, timeout, transport, parse)."""
class SecForbiddenError(SecError):
"""SEC returned 403 — User-Agent/pattern rejected. Alert and stop."""
class SecNotFoundError(SecError):
"""SEC returned 404 — the resource does not exist (e.g. no index for a day).
The *only* error a caller may treat as 'missing' — every other SecError
(403, exhausted retries, 5xx, timeout) must propagate so a fetch failure is
never mistaken for an empty result."""
def _looks_like_contact_email(ua: str) -> bool:
if "example.com" in ua.lower() or "set-a-real-email" in ua.lower():
return False
return re.search(r"[^@\s]+@[^@\s]+\.[^@\s]+", ua) is not None
# SEC asks callers to stay well under 10 req/s; enforce a floor on real clients.
_MIN_PROD_SPACING = 0.11
def cik10(cik: int | str) -> str:
"""Zero-pad a CIK to the 10-digit form SEC URLs use (320193 -> 0000320193)."""
return str(int(cik)).zfill(10)
@@ -81,7 +100,24 @@ class SecClient:
self._lock = asyncio.Lock()
self._last_request = 0.0
def _validate_fair_access(self) -> None:
"""On a real (non-mocked) client, enforce SEC fair-access preconditions
so we can't accidentally hammer SEC or get 403'd: a genuine contact-email
User-Agent and a spacing floor. Mock transports skip this (tests use 0)."""
if not _looks_like_contact_email(self._ua):
raise SecError(
"sec_user_agent must contain a real contact email (got "
f"{self._ua!r}) — SEC fair-access requires it"
)
if self._spacing < _MIN_PROD_SPACING:
raise SecError(
f"sec_request_spacing_seconds {self._spacing} is below the "
f"{_MIN_PROD_SPACING}s fair-access floor"
)
async def __aenter__(self) -> "SecClient":
if self._transport is None:
self._validate_fair_access()
self._client = httpx.AsyncClient(
headers={"User-Agent": self._ua, "Accept-Encoding": "gzip, deflate"},
timeout=self._timeout,
@@ -108,22 +144,34 @@ class SecClient:
attempt = 0
while True:
await self._throttle()
resp = await self._client.get(url)
if resp.status_code == 403:
try:
resp = await self._client.get(url)
except (httpx.TimeoutException, httpx.TransportError) as exc:
attempt += 1
if attempt > self._max_retries:
raise SecError(f"SEC network error for {url}: {exc}") from exc
await asyncio.sleep(min(2.0**attempt, 30.0))
continue
code = resp.status_code
if code == 403:
raise SecForbiddenError(
f"SEC 403 for {url} — User-Agent/pattern rejected; set a real "
"sec_user_agent contact email"
)
if resp.status_code == 429:
if code == 404:
raise SecNotFoundError(f"SEC 404 for {url}")
# 429 and 5xx are transient — retry with backoff, honoring Retry-After.
if code == 429 or 500 <= code < 600:
attempt += 1
if attempt > self._max_retries:
raise SecError(f"SEC 429 after {self._max_retries} retries: {url}")
backoff = min(2.0**attempt, 30.0)
logger.warning("SEC 429 for %s — backoff %.1fs (attempt %d)", url, backoff, attempt)
await asyncio.sleep(backoff)
raise SecError(f"SEC {code} after {self._max_retries} retries: {url}")
delay = _retry_after_seconds(resp) or min(2.0**attempt, 30.0)
logger.warning("SEC %d for %s — backoff %.1fs (attempt %d)", code, url, delay, attempt)
await asyncio.sleep(delay)
continue
if resp.status_code >= 400:
raise SecError(f"SEC {resp.status_code} for {url}")
if code >= 400:
raise SecError(f"SEC {code} for {url}")
return resp
async def get_json(self, url: str) -> Any:
@@ -144,18 +192,20 @@ class SecClient:
out[sym] = int(row["cik_str"])
return out
async def submissions(self, cik: int | str) -> dict[str, Any]:
"""Issuer metadata + the FULL merged filing history.
async def submissions(self, cik: int | str, *, include_history: bool = False) -> dict[str, Any]:
"""Issuer metadata + filing list.
``filings.recent`` caps at 1000; older accessions live in
``filings.files[]`` shards. This merges them so backfill sees every
filing's reportDate / acceptanceDateTime / isXBRL.
``filings.files[]`` shards. Only ``include_history=True`` (the one-time
full backfill) fetches those shards — SIC refresh and incremental runs
use the recent list alone and make no extra requests.
"""
base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
filings = _rows_from_arrays(base["filings"]["recent"])
for shard in base["filings"].get("files") or []:
shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}")
filings.extend(_rows_from_arrays(shard_data))
if include_history:
for shard in base["filings"].get("files") or []:
shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}")
filings.extend(_rows_from_arrays(shard_data))
return {
"cik": int(base["cik"]),
"name": base.get("name"),
@@ -178,8 +228,8 @@ class SecClient:
url = f"{_WWW}/Archives/edgar/daily-index/{year}/QTR{qtr}/index.json"
try:
idx = await self.get_json(url)
except SecError:
continue
except SecNotFoundError:
continue # quarter dir absent — only 404 is "missing"
dates = [
d
for item in idx.get("directory", {}).get("item", [])
@@ -201,9 +251,9 @@ class SecClient:
url = f"{_WWW}/Archives/edgar/daily-index/{day.year}/QTR{qtr}/form.{day:%Y%m%d}.idx"
try:
text = await self.get_text(url)
except SecError as exc:
logger.info("no daily index for %s (%s)", day, exc)
return []
except SecNotFoundError:
logger.info("no daily index for %s (404)", day)
return [] # weekend/holiday/not-yet-published; other errors propagate
return _parse_form_index(text)
@@ -252,6 +302,17 @@ def _parse_form_index(text: str) -> list[dict[str, Any]]:
return rows
def _retry_after_seconds(resp: httpx.Response) -> float | None:
"""Parse a numeric-seconds Retry-After header (SEC uses seconds), capped."""
raw = resp.headers.get("Retry-After")
if not raw:
return None
try:
return min(float(raw), 60.0)
except (TypeError, ValueError):
return None
def _index_file_date(name: str) -> date | None:
if name.startswith("form.") and name.endswith(".idx"):
try: