signal-platform/tests/unit/test_sec_client.py

"""Tests for the SEC client's parsing and fair-access behavior, via a mocked
httpx transport (no network)."""

from __future__ import annotations

import json
from datetime import date

import httpx
import pytest

from app.services import sec_client as sc
from app.services.sec_client import SecClient, SecError, SecForbiddenError

COMPANY_TICKERS = {
    "0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."},
    "1": {"cik_str": 1652044, "ticker": "GOOGL", "title": "Alphabet"},
    "2": {"cik_str": 1652044, "ticker": "GOOG", "title": "Alphabet"},
    "3": {"cik_str": 1067983, "ticker": "BRK-B", "title": "Berkshire"},
}

SUBMISSIONS_BASE = {
    "cik": 320193,
    "name": "Apple Inc.",
    "sic": "3571",
    "sicDescription": "Electronic Computers",
    "fiscalYearEnd": "0926",
    "tickers": ["AAPL"],
    "filings": {
        "recent": {
            "accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"],
            "form": ["10-Q", "10-K", "8-K"],
            "reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"],
            "filingDate": ["2026-05-01", "2026-01-30", "2026-04-02"],
            "acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"],
            "isXBRL": [1, 1, 0],
        },
        "files": [{"name": "CIK0000320193-submissions-001.json", "filingFrom": "1994-01-26", "filingTo": "2015-05-27"}],
    },
}

SUBMISSIONS_SHARD = {
    "accessionNumber": ["0000320193-94-000002"],
    "form": ["10-Q"],
    "reportDate": ["1993-12-31"],
    "filingDate": ["1994-01-26"],
    "acceptanceDateTime": ["1994-01-26T05:00:00.000Z"],
    "isXBRL": [0],
}

FORM_IDX = """Description:           Daily Index of EDGAR Dissemination Feed by Form Type

      Form Type   Company Name    CIK    Date Filed  File Name
-------------------------------------------------------------------------------
   10-K/A        Starfighters Space, Inc.    1947016   20260721   edgar/data/1947016/0001062993-26-003746.txt
   10-Q          3M CO                       66740     20260721   edgar/data/66740/0000066740-26-000246.txt
   8-K           Some Corp                   12345     20260721   edgar/data/12345/0000012345-26-000001.txt
   10-Q          CALIX, INC                  1406666   20260721   edgar/data/1406666/0001406666-26-000034.txt
"""

DIR_JSON = {"directory": {"item": [
    {"name": "form.20260720.idx"}, {"name": "form.20260721.idx"}, {"name": "company.20260721.idx"},
]}}


def _handler(request: httpx.Request) -> httpx.Response:
    url = str(request.url)
    if url.endswith("company_tickers.json"):
        return httpx.Response(200, json=COMPANY_TICKERS)
    if url.endswith("submissions/CIK0000320193.json"):
        return httpx.Response(200, json=SUBMISSIONS_BASE)
    if url.endswith("CIK0000320193-submissions-001.json"):
        return httpx.Response(200, json=SUBMISSIONS_SHARD)
    if url.endswith("form.20260721.idx"):
        return httpx.Response(200, text=FORM_IDX)
    if url.endswith("QTR3/index.json"):
        return httpx.Response(200, json=DIR_JSON)
    return httpx.Response(404)


def _client(**kw):
    return SecClient(transport=httpx.MockTransport(_handler), spacing_seconds=0, **kw)


async def test_company_tickers_normalised_and_multiclass():
    async with _client() as c:
        m = await c.company_tickers()
    assert m["AAPL"] == 320193
    assert m["GOOGL"] == m["GOOG"] == 1652044  # multi-class share one CIK
    assert m["BRK-B"] == 1067983  # dash form


async def test_submissions_history_merges_shards_and_filters_forms():
    async with _client() as c:
        sub = await c.submissions(320193, include_history=True)
    assert sub["sic"] == "3571" and sub["fiscal_year_end"] == "0926"
    accns = {f["accession"] for f in sub["filings"]}
    # 10-Q + 10-K from recent, 10-Q from the shard; the 8-K is filtered out
    assert accns == {"0000320193-26-000013", "0000320193-26-000006", "0000320193-94-000002"}
    older = next(f for f in sub["filings"] if f["accession"] == "0000320193-94-000002")
    assert older["report_date"] == "1993-12-31" and older["is_xbrl"] is False


async def test_submissions_recent_only_skips_shard_requests():
    seen = []

    def handler(request):
        seen.append(str(request.url))
        return _handler(request)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        sub = await c.submissions(320193)  # include_history defaults False
    assert not any("submissions-001" in u for u in seen)  # no shard fetch
    accns = {f["accession"] for f in sub["filings"]}
    assert accns == {"0000320193-26-000013", "0000320193-26-000006"}  # recent only


async def test_daily_index_parses_10kq_rows():
    async with _client() as c:
        rows = await c.daily_index(date(2026, 7, 21))
    forms = {r["form"] for r in rows}
    assert forms == {"10-K/A", "10-Q"}  # 8-K excluded
    mmm = next(r for r in rows if r["cik"] == 66740)
    assert mmm["accession"] == "0000066740-26-000246"


async def test_latest_index_date():
    async with _client() as c:
        d = await c.latest_index_date(today=date(2026, 7, 22))
    assert d == date(2026, 7, 21)


async def test_403_raises_forbidden_no_retry():
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        return httpx.Response(403)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        with pytest.raises(SecForbiddenError):
            await c.get_json("https://data.sec.gov/x")
    assert calls["n"] == 1  # alert and stop, never retry-loop


async def test_429_retries_then_succeeds(monkeypatch):
    async def _instant(_):
        return None

    monkeypatch.setattr(sc.asyncio, "sleep", _instant)  # no real backoff wait
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        if calls["n"] < 3:
            return httpx.Response(429)
        return httpx.Response(200, json={"ok": True})

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=5) as c:
        data = await c.get_json("https://data.sec.gov/y")
    assert data == {"ok": True} and calls["n"] == 3


async def test_429_gives_up_after_max_retries(monkeypatch):
    async def _instant(_):
        return None

    monkeypatch.setattr(sc.asyncio, "sleep", _instant)

    def handler(request):
        return httpx.Response(429)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=2) as c:
        with pytest.raises(SecError):
            await c.get_json("https://data.sec.gov/z")


def _status_client(status):
    def handler(request):
        return httpx.Response(status)
    # max_retries=0 so 5xx/429 raise immediately (no retry sleeps)
    return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=0)


async def test_index_methods_propagate_403():
    # A 403 must NOT be mistaken for "no index".
    async with _status_client(403) as c:
        with pytest.raises(sc.SecForbiddenError):
            await c.daily_index(date(2026, 7, 21))
    async with _status_client(403) as c:
        with pytest.raises(sc.SecForbiddenError):
            await c.latest_index_date(today=date(2026, 7, 22))


async def test_index_methods_propagate_500():
    async with _status_client(500) as c:
        with pytest.raises(SecError):
            await c.daily_index(date(2026, 7, 21))
    async with _status_client(500) as c:
        with pytest.raises(SecError):
            await c.latest_index_date(today=date(2026, 7, 22))


async def test_only_404_is_treated_as_missing():
    async with _status_client(404) as c:
        assert await c.daily_index(date(2026, 7, 21)) == []
        assert await c.latest_index_date(today=date(2026, 7, 22)) is None


# The two shapes a real SEC 403 takes (captured live 2026-07-30). /Archives is
# S3-backed with no ListBucket grant, so an ABSENT file comes back as S3's
# AccessDenied XML; a genuine fair-access rejection is the WAF interstitial.
S3_ACCESS_DENIED = (
    '<?xml version="1.0" encoding="UTF-8"?>'
    "<Error><Code>AccessDenied</Code><Message>Access Denied</Message>"
    "<RequestId>5AWQBRAEX3NPAPHB</RequestId><HostId>MHFbU0a3k0ER</HostId></Error>"
)
WAF_HTML = (
    "<!DOCTYPE html><html><head><title>SEC.gov | Your Request Originates from "
    "an Undeclared Automated Tool</title></head><body>...</body></html>"
)


def _forbidden(body: str, content_type: str) -> httpx.Response:
    return httpx.Response(
        403, content=body.encode(), headers={"Content-Type": content_type}
    )


async def test_archives_access_denied_is_absent_not_forbidden():
    # SEC publishes no daily index on weekends, and the bucket reports the absent
    # key as 403/AccessDenied. Treating that as fatal wedged the importer on the
    # first Saturday of an incremental walk (2026-07-25); it must read as "missing".
    def handler(request: httpx.Request) -> httpx.Response:
        url = str(request.url)
        if url.endswith("QTR2/index.json"):
            return httpx.Response(200, json={"directory": {"item": [{"name": "form.20260630.idx"}]}})
        return _forbidden(S3_ACCESS_DENIED, "application/xml")

    def client() -> SecClient:
        return SecClient(
            transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=0
        )

    async with client() as c:
        assert await c.daily_index(date(2026, 7, 25)) == []
    async with client() as c:
        # QTR3 absent → the previous-quarter fallback now actually fires.
        assert await c.latest_index_date(today=date(2026, 7, 22)) == date(2026, 6, 30)


async def test_archives_waf_rejection_stays_forbidden():
    # A real UA/pattern rejection is served for files that DO exist — never
    # downgrade it, or a blocked run would look like an empty index.
    def handler(request: httpx.Request) -> httpx.Response:
        return _forbidden(WAF_HTML, "text/html")

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        with pytest.raises(SecForbiddenError):
            await c.daily_index(date(2026, 7, 21))


async def test_access_denied_outside_archives_stays_forbidden():
    # The downgrade is gated on the Archives prefix; data.sec.gov is not S3-backed.
    def handler(request: httpx.Request) -> httpx.Response:
        return _forbidden(S3_ACCESS_DENIED, "application/xml")

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        with pytest.raises(SecForbiddenError):
            await c.companyfacts(320193)


async def test_fair_access_validation_on_real_client():
    # Placeholder email rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform (contact: you@example.com)"):
            pass
    # Non-email UA rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform"):
            pass
    # Valid UA but unsafe production spacing rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform real@person.io", spacing_seconds=0.0):
            pass
    # Valid UA + safe spacing opens fine.
    async with SecClient(user_agent="signal-platform real@person.io", spacing_seconds=0.2):
        pass