signal-platform/app/services/sec_client.py

"""Async SEC EDGAR client for the fundamentals importer (workstream A).

All access is batch (never at request time). This wraps the three SEC products
the A3 design uses — `company_tickers.json`, `submissions/`, `companyfacts/`, and
the daily filing index — behind one client that honors SEC's fair-access policy:

- an identifying ``User-Agent`` with a contact email on every request (config);
- request spacing well under the 10 req/s limit;
- exponential backoff + retry on 429;
- **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a
  403 means the UA or request pattern is wrong and retrying won't fix it.

Parsing lives here (index fixed-width, submissions pagination); DB writes and the
snapshot mapping live in the importer. No conditional GETs — the companyfacts
endpoint exposes no ETag/Last-Modified (verified), which is why the importer is
daily-index driven rather than polling archives.
"""

from __future__ import annotations

import asyncio
import logging
import os
from datetime import date, datetime
from pathlib import Path
from typing import Any

import httpx

from app.config import settings
from app.exceptions import ProviderError
from app.services.earnings_alignment import normalise_symbol

logger = logging.getLogger(__name__)

_WWW = "https://www.sec.gov"
_DATA = "https://data.sec.gov"

# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py).
_CA = os.environ.get("SSL_CERT_FILE", "")
_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True

_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})


class SecError(ProviderError):
    """SEC request failed."""


class SecForbiddenError(SecError):
    """SEC returned 403 — User-Agent/pattern rejected. Alert and stop."""


def cik10(cik: int | str) -> str:
    """Zero-pad a CIK to the 10-digit form SEC URLs use (320193 -> 0000320193)."""
    return str(int(cik)).zfill(10)


class SecClient:
    """Fair-access SEC HTTP client. Use as ``async with SecClient() as c:``."""

    def __init__(
        self,
        *,
        user_agent: str | None = None,
        spacing_seconds: float | None = None,
        max_retries: int | None = None,
        timeout: float | None = None,
        transport: httpx.AsyncBaseTransport | None = None,
    ) -> None:
        self._ua = user_agent or settings.sec_user_agent
        self._spacing = (
            spacing_seconds if spacing_seconds is not None else settings.sec_request_spacing_seconds
        )
        self._max_retries = (
            max_retries if max_retries is not None else settings.sec_max_retries
        )
        self._timeout = timeout if timeout is not None else settings.sec_request_timeout_seconds
        self._transport = transport  # injectable for tests
        self._client: httpx.AsyncClient | None = None
        self._lock = asyncio.Lock()
        self._last_request = 0.0

    async def __aenter__(self) -> "SecClient":
        self._client = httpx.AsyncClient(
            headers={"User-Agent": self._ua, "Accept-Encoding": "gzip, deflate"},
            timeout=self._timeout,
            verify=_CA_VERIFY,
            transport=self._transport,
        )
        return self

    async def __aexit__(self, *exc) -> None:
        if self._client is not None:
            await self._client.aclose()
            self._client = None

    async def _throttle(self) -> None:
        async with self._lock:
            now = asyncio.get_event_loop().time()
            wait = self._spacing - (now - self._last_request)
            if wait > 0:
                await asyncio.sleep(wait)
            self._last_request = asyncio.get_event_loop().time()

    async def _get(self, url: str) -> httpx.Response:
        assert self._client is not None, "use `async with SecClient()`"
        attempt = 0
        while True:
            await self._throttle()
            resp = await self._client.get(url)
            if resp.status_code == 403:
                raise SecForbiddenError(
                    f"SEC 403 for {url} — User-Agent/pattern rejected; set a real "
                    "sec_user_agent contact email"
                )
            if resp.status_code == 429:
                attempt += 1
                if attempt > self._max_retries:
                    raise SecError(f"SEC 429 after {self._max_retries} retries: {url}")
                backoff = min(2.0**attempt, 30.0)
                logger.warning("SEC 429 for %s — backoff %.1fs (attempt %d)", url, backoff, attempt)
                await asyncio.sleep(backoff)
                continue
            if resp.status_code >= 400:
                raise SecError(f"SEC {resp.status_code} for {url}")
            return resp

    async def get_json(self, url: str) -> Any:
        return (await self._get(url)).json()

    async def get_text(self, url: str) -> str:
        return (await self._get(url)).text

    # -- domain fetchers ---------------------------------------------------

    async def company_tickers(self) -> dict[str, int]:
        """Map normalised ticker -> CIK (int). Multi-class tickers share a CIK."""
        data = await self.get_json(f"{_WWW}/files/company_tickers.json")
        out: dict[str, int] = {}
        for row in data.values():
            sym = normalise_symbol(row.get("ticker"))
            if sym:
                out[sym] = int(row["cik_str"])
        return out

    async def submissions(self, cik: int | str) -> dict[str, Any]:
        """Issuer metadata + the FULL merged filing history.

        ``filings.recent`` caps at 1000; older accessions live in
        ``filings.files[]`` shards. This merges them so backfill sees every
        filing's reportDate / acceptanceDateTime / isXBRL.
        """
        base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
        filings = _rows_from_arrays(base["filings"]["recent"])
        for shard in base["filings"].get("files") or []:
            shard_data = await self.get_json(f"{_DATA}/submissions/{shard['name']}")
            filings.extend(_rows_from_arrays(shard_data))
        return {
            "cik": int(base["cik"]),
            "name": base.get("name"),
            "sic": base.get("sic"),
            "sic_description": base.get("sicDescription"),
            "fiscal_year_end": base.get("fiscalYearEnd"),
            "tickers": base.get("tickers") or [],
            "filings": filings,
        }

    async def companyfacts(self, cik: int | str) -> dict[str, Any]:
        """Raw companyfacts JSON ({cik, entityName, facts})."""
        return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")

    async def latest_index_date(self, today: date | None = None) -> date | None:
        """The most recent published daily-index date (drives the revision). Checks
        the current quarter, falling back to the previous one at a quarter boundary."""
        today = today or date.today()
        for year, qtr in _quarters_back(today, 2):
            url = f"{_WWW}/Archives/edgar/daily-index/{year}/QTR{qtr}/index.json"
            try:
                idx = await self.get_json(url)
            except SecError:
                continue
            dates = [
                d
                for item in idx.get("directory", {}).get("item", [])
                if (d := _index_file_date(item.get("name", ""))) is not None
                and d <= today
            ]
            if dates:
                return max(dates)
        return None

    async def daily_index(self, day: date) -> list[dict[str, Any]]:
        """Parse the daily form index into 10-K/10-Q(/A) rows for all issuers.

        Returns [{form, cik, accession, company}]. The caller filters to the
        tracked universe. A missing index (weekend/holiday/not-yet-published)
        returns [] rather than raising.
        """
        qtr = (day.month - 1) // 3 + 1
        url = f"{_WWW}/Archives/edgar/daily-index/{day.year}/QTR{qtr}/form.{day:%Y%m%d}.idx"
        try:
            text = await self.get_text(url)
        except SecError as exc:
            logger.info("no daily index for %s (%s)", day, exc)
            return []
        return _parse_form_index(text)


def _rows_from_arrays(arrays: dict[str, list]) -> list[dict[str, Any]]:
    """Turn SEC's parallel-array filing block into row dicts (keeping only 10-K/10-Q
    family filings — the ones that carry XBRL fundamentals)."""
    forms = arrays.get("form", [])
    out: list[dict[str, Any]] = []
    for i, form in enumerate(forms):
        if form not in _FORMS_10:
            continue
        out.append(
            {
                "accession": arrays["accessionNumber"][i],
                "form": form,
                "report_date": arrays["reportDate"][i] or None,
                "acceptance_datetime": arrays["acceptanceDateTime"][i] or None,
                "is_xbrl": bool(arrays.get("isXBRL", [0] * len(forms))[i]),
            }
        )
    return out


def _parse_form_index(text: str) -> list[dict[str, Any]]:
    """Parse a daily ``form.YYYYMMDD.idx`` (fixed columns: Form / Company / CIK /
    Date Filed / File Name-with-accession)."""
    rows: list[dict[str, Any]] = []
    started = False
    for line in text.splitlines():
        if not started:
            if set(line.strip()) == {"-"}:  # the dashed separator row
                started = True
            continue
        parts = line.split()
        if len(parts) < 5:
            continue
        form = parts[0]
        if form not in _FORMS_10:
            continue
        path = parts[-1]  # edgar/data/<cik>/<accession>.txt
        cik = _cik_from_path(path)
        accession = _accession_from_path(path)
        if cik is None or accession is None:
            continue
        rows.append({"form": form, "cik": cik, "accession": accession, "path": path})
    return rows


def _index_file_date(name: str) -> date | None:
    if name.startswith("form.") and name.endswith(".idx"):
        try:
            return datetime.strptime(name[5:13], "%Y%m%d").date()
        except ValueError:
            return None
    return None


def _quarters_back(today: date, n: int) -> list[tuple[int, int]]:
    """(year, quarter) for `today`'s quarter and the previous n-1, newest first."""
    q = (today.month - 1) // 3 + 1
    out = []
    y = today.year
    for _ in range(n):
        out.append((y, q))
        q -= 1
        if q == 0:
            q = 4
            y -= 1
    return out


def _cik_from_path(path: str) -> int | None:
    segs = path.split("/")
    if len(segs) >= 3 and segs[2].isdigit():
        return int(segs[2])
    return None


def _accession_from_path(path: str) -> str | None:
    stem = path.rsplit("/", 1)[-1]
    if stem.endswith(".txt"):
        stem = stem[:-4]
    return stem or None