signal-platform/tests/unit/test_sec_client.py

"""Tests for the SEC client's parsing and fair-access behavior, via a mocked
httpx transport (no network)."""

from __future__ import annotations

import json
from datetime import date

import httpx
import pytest

from app.services import sec_client as sc
from app.services.sec_client import SecClient, SecError, SecForbiddenError

COMPANY_TICKERS = {
    "0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."},
    "1": {"cik_str": 1652044, "ticker": "GOOGL", "title": "Alphabet"},
    "2": {"cik_str": 1652044, "ticker": "GOOG", "title": "Alphabet"},
    "3": {"cik_str": 1067983, "ticker": "BRK-B", "title": "Berkshire"},
}

SUBMISSIONS_BASE = {
    "cik": 320193,
    "name": "Apple Inc.",
    "sic": "3571",
    "sicDescription": "Electronic Computers",
    "fiscalYearEnd": "0926",
    "tickers": ["AAPL"],
    "filings": {
        "recent": {
            "accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"],
            "form": ["10-Q", "10-K", "8-K"],
            "reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"],
            "filingDate": ["2026-05-01", "2026-01-30", "2026-04-02"],
            "acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"],
            "isXBRL": [1, 1, 0],
        },
        "files": [{"name": "CIK0000320193-submissions-001.json", "filingFrom": "1994-01-26", "filingTo": "2015-05-27"}],
    },
}

SUBMISSIONS_SHARD = {
    "accessionNumber": ["0000320193-94-000002"],
    "form": ["10-Q"],
    "reportDate": ["1993-12-31"],
    "filingDate": ["1994-01-26"],
    "acceptanceDateTime": ["1994-01-26T05:00:00.000Z"],
    "isXBRL": [0],
}

FORM_IDX = """Description:           Daily Index of EDGAR Dissemination Feed by Form Type

      Form Type   Company Name    CIK    Date Filed  File Name
-------------------------------------------------------------------------------
   10-K/A        Starfighters Space, Inc.    1947016   20260721   edgar/data/1947016/0001062993-26-003746.txt
   10-Q          3M CO                       66740     20260721   edgar/data/66740/0000066740-26-000246.txt
   8-K           Some Corp                   12345     20260721   edgar/data/12345/0000012345-26-000001.txt
   10-Q          CALIX, INC                  1406666   20260721   edgar/data/1406666/0001406666-26-000034.txt
"""

DIR_JSON = {"directory": {"item": [
    {"name": "form.20260720.idx"}, {"name": "form.20260721.idx"}, {"name": "company.20260721.idx"},
]}}


def _handler(request: httpx.Request) -> httpx.Response:
    url = str(request.url)
    if url.endswith("company_tickers.json"):
        return httpx.Response(200, json=COMPANY_TICKERS)
    if url.endswith("submissions/CIK0000320193.json"):
        return httpx.Response(200, json=SUBMISSIONS_BASE)
    if url.endswith("CIK0000320193-submissions-001.json"):
        return httpx.Response(200, json=SUBMISSIONS_SHARD)
    if url.endswith("form.20260721.idx"):
        return httpx.Response(200, text=FORM_IDX)
    if url.endswith("QTR3/index.json"):
        return httpx.Response(200, json=DIR_JSON)
    return httpx.Response(404)


def _client(**kw):
    return SecClient(transport=httpx.MockTransport(_handler), spacing_seconds=0, **kw)


async def test_company_tickers_normalised_and_multiclass():
    async with _client() as c:
        m = await c.company_tickers()
    assert m["AAPL"] == 320193
    assert m["GOOGL"] == m["GOOG"] == 1652044  # multi-class share one CIK
    assert m["BRK-B"] == 1067983  # dash form


async def test_submissions_history_merges_shards_and_filters_forms():
    async with _client() as c:
        sub = await c.submissions(320193, include_history=True)
    assert sub["sic"] == "3571" and sub["fiscal_year_end"] == "0926"
    accns = {f["accession"] for f in sub["filings"]}
    # 10-Q + 10-K from recent, 10-Q from the shard; the 8-K is filtered out
    assert accns == {"0000320193-26-000013", "0000320193-26-000006", "0000320193-94-000002"}
    older = next(f for f in sub["filings"] if f["accession"] == "0000320193-94-000002")
    assert older["report_date"] == "1993-12-31" and older["is_xbrl"] is False


async def test_submissions_recent_only_skips_shard_requests():
    seen = []

    def handler(request):
        seen.append(str(request.url))
        return _handler(request)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        sub = await c.submissions(320193)  # include_history defaults False
    assert not any("submissions-001" in u for u in seen)  # no shard fetch
    accns = {f["accession"] for f in sub["filings"]}
    assert accns == {"0000320193-26-000013", "0000320193-26-000006"}  # recent only


async def test_daily_index_parses_10kq_rows():
    async with _client() as c:
        rows = await c.daily_index(date(2026, 7, 21))
    forms = {r["form"] for r in rows}
    assert forms == {"10-K/A", "10-Q"}  # 8-K excluded
    mmm = next(r for r in rows if r["cik"] == 66740)
    assert mmm["accession"] == "0000066740-26-000246"


async def test_latest_index_date():
    async with _client() as c:
        d = await c.latest_index_date(today=date(2026, 7, 22))
    assert d == date(2026, 7, 21)


async def test_403_raises_forbidden_no_retry():
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        return httpx.Response(403)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
        with pytest.raises(SecForbiddenError):
            await c.get_json("https://data.sec.gov/x")
    assert calls["n"] == 1  # alert and stop, never retry-loop


async def test_429_retries_then_succeeds(monkeypatch):
    async def _instant(_):
        return None

    monkeypatch.setattr(sc.asyncio, "sleep", _instant)  # no real backoff wait
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        if calls["n"] < 3:
            return httpx.Response(429)
        return httpx.Response(200, json={"ok": True})

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=5) as c:
        data = await c.get_json("https://data.sec.gov/y")
    assert data == {"ok": True} and calls["n"] == 3


async def test_429_gives_up_after_max_retries(monkeypatch):
    async def _instant(_):
        return None

    monkeypatch.setattr(sc.asyncio, "sleep", _instant)

    def handler(request):
        return httpx.Response(429)

    async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=2) as c:
        with pytest.raises(SecError):
            await c.get_json("https://data.sec.gov/z")


def _status_client(status):
    def handler(request):
        return httpx.Response(status)
    # max_retries=0 so 5xx/429 raise immediately (no retry sleeps)
    return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=0)


async def test_index_methods_propagate_403():
    # A 403 must NOT be mistaken for "no index".
    async with _status_client(403) as c:
        with pytest.raises(sc.SecForbiddenError):
            await c.daily_index(date(2026, 7, 21))
    async with _status_client(403) as c:
        with pytest.raises(sc.SecForbiddenError):
            await c.latest_index_date(today=date(2026, 7, 22))


async def test_index_methods_propagate_500():
    async with _status_client(500) as c:
        with pytest.raises(SecError):
            await c.daily_index(date(2026, 7, 21))
    async with _status_client(500) as c:
        with pytest.raises(SecError):
            await c.latest_index_date(today=date(2026, 7, 22))


async def test_only_404_is_treated_as_missing():
    async with _status_client(404) as c:
        assert await c.daily_index(date(2026, 7, 21)) == []
        assert await c.latest_index_date(today=date(2026, 7, 22)) is None


async def test_fair_access_validation_on_real_client():
    # Placeholder email rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform (contact: you@example.com)"):
            pass
    # Non-email UA rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform"):
            pass
    # Valid UA but unsafe production spacing rejected.
    with pytest.raises(SecError):
        async with SecClient(user_agent="signal-platform real@person.io", spacing_seconds=0.0):
            pass
    # Valid UA + safe spacing opens fine.
    async with SecClient(user_agent="signal-platform real@person.io", spacing_seconds=0.2):
        pass