signal-platform/tests/unit/test_sec_facts_parser.py

"""Tests for the companyfacts -> snapshot parser, on a realistic Apple-shaped
fixture (the structure verified by live probe)."""

from __future__ import annotations

import os
from datetime import date, datetime, timezone

import pytest

from app.services.sec_facts_parser import (
    Fact,
    FilingMeta,
    _compose_cash,
    _compose_debt,
    _fiscal_context,
    _select_shares,
    parse_snapshots,
)

UTC = timezone.utc


def _dur(start, end, val, accn, fy=2026, fp="Q2"):
    return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-Q"}


def _inst(end, val, accn, fy=2026, fp="Q2"):
    return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-Q"}


COMPANYFACTS = {
    "cik": 320193,
    "facts": {
        "us-gaap": {
            "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
                _dur("2025-09-28", "2025-12-27", 143756, "A", fp="Q1"),   # Q1 discrete == YTD
                _dur("2025-09-28", "2026-03-28", 254940, "B"),            # Q2 YTD (181d)  <- want this
                _dur("2025-12-28", "2026-03-28", 111184, "B"),            # Q2 discrete (90d)
            ]}},
            "NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 40000, "B")]}},
            # only a discrete-length fact for Q2 -> must be null, not the discrete
            "OperatingIncomeLoss": {"units": {"USD": [_dur("2025-12-28", "2026-03-28", 30000, "B")]}},
            "EarningsPerShareDiluted": {"units": {"USD/shares": [_dur("2025-09-28", "2026-03-28", 2.55, "B")]}},
            "CashAndCashEquivalentsAtCarryingValue": {"units": {"USD": [_inst("2026-03-28", 30000, "B")]}},
            "MarketableSecuritiesCurrent": {"units": {"USD": [_inst("2026-03-28", 20000, "B")]}},
            "LongTermDebtNoncurrent": {"units": {"USD": [_inst("2026-03-28", 80000, "B")]}},
            "LongTermDebtCurrent": {"units": {"USD": [_inst("2026-03-28", 10000, "B")]}},
        },
        "dei": {
            "EntityCommonStockSharesOutstanding": {"units": {"shares": [
                {"end": "2026-04-17", "val": 14687356000, "fy": 2026, "fp": "Q2", "accn": "B", "form": "10-Q"},
            ]}},
        },
    },
}

FILINGS = {
    "A": FilingMeta(date(2025, 12, 27), date(2026, 1, 30), datetime(2026, 1, 30, 11, 1, tzinfo=UTC), "10-Q"),
    "B": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, 10, 1, tzinfo=UTC), "10-Q"),
}


def _by_accn(rows):
    return {r.accession: r for r in rows}


def test_parses_ytd_not_discrete_and_cover_date_shares():
    res = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
    assert not res.skipped_filings and not res.field_issues
    b = _by_accn(res.rows)["B"]

    assert (b.cik, b.fiscal_year, b.fiscal_period) == ("0000320193", 2026, "Q2")
    assert b.period_end == date(2026, 3, 28)
    assert b.period_start == date(2025, 9, 28)  # YTD start (fiscal-year start)
    assert b.revenue == 254940          # the 6-month YTD, NOT the 111184 discrete
    assert b.net_income == 40000
    assert b.operating_income is None   # only a discrete-length fact existed -> null
    assert b.diluted_eps == 2.55
    # cash + first-present ST investment (MarketableSecuritiesCurrent), each once
    assert b.cash_and_st_investments == 50000
    # long-term parts summed (no aggregate, no short-term)
    assert b.total_debt == 90000
    assert b.shares_outstanding == 14687356000
    assert b.shares_outstanding_date == date(2026, 4, 17)  # cover date != period_end
    assert b.filed_date == date(2026, 5, 1)
    assert b.accepted_at == datetime(2026, 5, 1, 10, 1, tzinfo=UTC)


def test_q1_discrete_is_the_ytd():
    res = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
    a = _by_accn(res.rows)["A"]
    assert a.fiscal_period == "Q1"
    assert a.revenue == 143756  # Q1 YTD == Q1 discrete
    assert a.period_start == date(2025, 9, 28)


def test_skips_filing_without_usable_period():
    cf = {
        "cik": 320193,
        "facts": {"us-gaap": {"NetIncomeLoss": {"units": {"USD": [
            {"start": "2025-09-28", "end": "2026-03-28", "val": 1, "fy": 2026, "fp": "H1", "accn": "X", "form": "10-Q"},
        ]}}}},
    }
    filings = {"X": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
    res = parse_snapshots(cf, filings, {"X"})
    assert res.rows == []
    assert res.skipped_filings == [{"accession": "X", "reason": "no usable period identity"}]


def test_missing_accession_is_skipped():
    res = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
    assert res.rows == []
    assert res.skipped_filings == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]


def test_debt_prefers_aggregate_over_parts():
    rd = date(2026, 3, 28)
    facts = [
        Fact("us-gaap", "LongTermDebt", "USD", None, rd, 95000, 2026, "Q2"),
        Fact("us-gaap", "LongTermDebtNoncurrent", "USD", None, rd, 80000, 2026, "Q2"),
        Fact("us-gaap", "LongTermDebtCurrent", "USD", None, rd, 10000, 2026, "Q2"),
        Fact("us-gaap", "CommercialPaper", "USD", None, rd, 5000, 2026, "Q2"),
    ]
    # aggregate (95000) used, parts ignored; + one short-term pick (5000)
    assert _compose_debt(facts, rd) == 100000


def test_cash_picks_one_st_investment_source():
    rd = date(2026, 3, 28)
    facts = [
        Fact("us-gaap", "CashAndCashEquivalentsAtCarryingValue", "USD", None, rd, 30000, 2026, "Q2"),
        Fact("us-gaap", "ShortTermInvestments", "USD", None, rd, 15000, 2026, "Q2"),
        Fact("us-gaap", "MarketableSecuritiesCurrent", "USD", None, rd, 20000, 2026, "Q2"),
    ]
    # ShortTermInvestments is first in priority -> 30000 + 15000 (not both ST tags)
    assert _compose_cash(facts, rd) == 45000


RD = date(2026, 3, 28)


def _dei(end, val):
    return Fact("dei", "EntityCommonStockSharesOutstanding", "shares", None, end, val, 2026, "Q2")


def _gaap_shares(end, val):
    return Fact("us-gaap", "CommonStockSharesOutstanding", "shares", None, end, val, 2026, "Q2")


def test_shares_prefers_dei_cover_page():
    facts = [_dei(date(2026, 4, 17), 100), _gaap_shares(RD, 999)]
    assert _select_shares(facts, RD) == (100.0, date(2026, 4, 17), False)


def test_shares_falls_back_to_usgaap_at_report_date():
    # Alphabet case: no dei fact; us-gaap current + a prior comparative.
    facts = [_gaap_shares(date(2025, 12, 31), 888), _gaap_shares(RD, 12116)]
    assert _select_shares(facts, RD) == (12116.0, RD, False)  # comparative excluded


def test_shares_conflict_returns_null_ambiguous():
    facts = [_dei(RD, 100), _dei(RD, 200)]  # two differing consolidated values
    assert _select_shares(facts, RD) == (None, None, True)


def test_shares_never_uses_weighted_average():
    facts = [Fact("us-gaap", "WeightedAverageNumberOfDilutedSharesOutstanding", "shares", None, RD, 5, 2026, "Q2")]
    assert _select_shares(facts, RD) == (None, None, False)  # not a shares source


def test_conflicting_fiscal_context_is_rejected():
    facts = [
        Fact("us-gaap", "Revenues", "USD", date(2025, 9, 28), RD, 1, 2026, "Q2"),
        Fact("us-gaap", "NetIncomeLoss", "USD", date(2025, 9, 28), RD, 2, 2025, "Q3"),
    ]  # 1-1 tie between two contexts at reportDate
    assert _fiscal_context(facts, RD) == (None, None)
    # a clear majority wins
    facts.append(Fact("us-gaap", "OperatingIncomeLoss", "USD", date(2025, 9, 28), RD, 3, 2026, "Q2"))
    assert _fiscal_context(facts, RD) == (2026, "Q2")


def test_conflicting_context_skips_row():
    cf = {"cik": 1, "facts": {"us-gaap": {
        "Revenues": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 1, "B", fp="Q2")]}},
        "NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 2, "B", fy=2025, fp="Q3")]}},
    }}}
    filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
    res = parse_snapshots(cf, filings, {"B"})
    assert res.rows == []
    assert res.skipped_filings == [{"accession": "B", "reason": "no usable period identity"}]


def test_foreign_taxonomy_and_malformed_facts_ignored():
    cf = {"cik": 1, "facts": {
        "us-gaap": {
            "Revenues": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 500, "B")]}},
            "NetIncomeLoss": {"units": {"USD": [
                {"start": "2025-09-28", "end": "2026-03-28", "val": None, "fy": 2026, "fp": "Q2", "accn": "B"},
            ]}},
            "OperatingIncomeLoss": {"units": {"USD": [
                {"start": "2025-09-28", "end": "2026-03-28", "val": float("nan"), "fy": 2026, "fp": "Q2", "accn": "B"},
            ]}},
        },
        "acme": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
            _dur("2025-09-28", "2026-03-28", 99999, "B"),  # custom taxonomy — must be ignored
        ]}}},
    }}
    filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
    res = parse_snapshots(cf, filings, {"B"})
    assert res.rows[0].revenue == 500          # us-gaap Revenues, not the acme concept
    assert res.rows[0].net_income is None      # val None ignored
    assert res.rows[0].operating_income is None  # NaN ignored


def test_ambiguous_shares_produces_row_plus_note():
    cf = {"cik": 1, "facts": {
        "us-gaap": {"Revenues": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 500, "B")]}}},
        "dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": [
            {"end": "2026-04-17", "val": 100, "fy": 2026, "fp": "Q2", "accn": "B"},
            {"end": "2026-04-17", "val": 200, "fy": 2026, "fp": "Q2", "accn": "B"},
        ]}}},
    }}
    filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
    res = parse_snapshots(cf, filings, {"B"})
    assert len(res.rows) == 1 and res.rows[0].shares_outstanding is None  # row kept, shares null
    assert res.field_issues == [{"accession": "B", "reason": "ambiguous shares outstanding"}]
    assert res.skipped_filings == []  # a field issue is NOT a skipped filing


# Opt-in live check against real Apple companyfacts. Skips unless SEC_LIVE=1 and a
# real SEC_USER_AGENT are set (network + fair-access contact email).
@pytest.mark.skipif(
    not (os.environ.get("SEC_LIVE") and os.environ.get("SEC_USER_AGENT")),
    reason="set SEC_LIVE=1 + SEC_USER_AGENT to run the live SEC parser check",
)
async def test_live_apple_parse_invariants():
    from app.services.sec_client import SecClient

    def _dt(s):
        return datetime.fromisoformat(s.replace("Z", "+00:00")) if s else None

    async with SecClient(user_agent=os.environ["SEC_USER_AGENT"]) as c:
        cf = await c.companyfacts(320193)
        sub = await c.submissions(320193, include_history=False)

    filings = {
        f["accession"]: FilingMeta(
            date.fromisoformat(f["report_date"]),
            date.fromisoformat(f["filing_date"]),
            _dt(f["acceptance_datetime"]),
            f["form"],
        )
        for f in sub["filings"]
        if f["report_date"] and f["filing_date"] and f["acceptance_datetime"]
    }
    rows = parse_snapshots(cf, filings, set(filings)).rows
    assert len(rows) > 20
    assert all(r.period_end and r.fiscal_year and r.fiscal_period for r in rows)
    # YTD revenue is non-decreasing within a fiscal year
    by_fy: dict[int, list] = {}
    for r in rows:
        if r.revenue is not None:
            by_fy.setdefault(r.fiscal_year, []).append((r.fiscal_period, r.revenue))
    order = {"Q1": 1, "Q2": 2, "Q3": 3, "FY": 4}
    for fy, series in by_fy.items():
        series.sort(key=lambda x: order[x[0]])
        vals = [v for _, v in series]
        assert vals == sorted(vals), f"YTD revenue not monotonic in FY{fy}: {series}"
    # shares cover-date differs from period_end
    latest = max(rows, key=lambda r: r.period_end)
    assert latest.shares_outstanding_date != latest.period_end