feat(sec): A3 slice 2a — companyfacts -> snapshot parser

Pure parser (no I/O/DB) turning one issuer's companyfacts + submissions filing metadata into per-accession snapshot rows for the filing's primary period. - Period identity from end == reportDate, never fy/fp (fy/fp is the filing's context; comparatives repeat it). - Duration facts stored as cumulative YTD: pick the fact whose span matches the fiscal-period-to-date length (Q1~3mo..FY~12mo) within tolerance; no YTD-length fact -> null (never a discrete masquerading as YTD). - Balance-sheet instants at end == reportDate; shares_outstanding is the dei cover-page fact whose own end (cover date) is stored in shares_outstanding_date. - Cash and debt composites are aggregate-first and mutually exclusive (each source tag counted at most once). - Carries filing_date through submissions rows (snapshot.filed_date). Verified on REAL Apple companyfacts: 44 snapshots, 0 skipped, YTD revenue 124.3B->219.7B->313.7B->416.2B across FY2025 (Q4 derives at read time), every shares_date is the cover date != period_end. Tests: 6 fixture + 1 skip-guarded live-invariants (monotonic YTD, cover-date shares). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 16:11:22 +02:00
parent 5939be7b7f
commit 7413de9301
4 changed files with 471 additions and 0 deletions
@@ -31,6 +31,7 @@ SUBMISSIONS_BASE = {
            "accessionNumber": ["0000320193-26-000013", "0000320193-26-000006", "0000320193-26-000099"],
            "form": ["10-Q", "10-K", "8-K"],
            "reportDate": ["2026-03-28", "2025-09-27", "2026-04-01"],
+            "filingDate": ["2026-05-01", "2026-01-30", "2026-04-02"],
            "acceptanceDateTime": ["2026-05-01T10:01:00.000Z", "2025-10-31T10:01:26.000Z", "2026-04-02T09:00:00.000Z"],
            "isXBRL": [1, 1, 0],
        },
@@ -42,6 +43,7 @@ SUBMISSIONS_SHARD = {
    "accessionNumber": ["0000320193-94-000002"],
    "form": ["10-Q"],
    "reportDate": ["1993-12-31"],
+    "filingDate": ["1994-01-26"],
    "acceptanceDateTime": ["1994-01-26T05:00:00.000Z"],
    "isXBRL": [0],
 }
@@ -0,0 +1,179 @@
+"""Tests for the companyfacts -> snapshot parser, on a realistic Apple-shaped
+fixture (the structure verified by live probe)."""
+
+from __future__ import annotations
+
+import os
+from datetime import date, datetime, timezone
+
+import pytest
+
+from app.services.sec_facts_parser import (
+    Fact,
+    FilingMeta,
+    _compose_cash,
+    _compose_debt,
+    parse_snapshots,
+)
+
+UTC = timezone.utc
+
+
+def _dur(start, end, val, accn, fy=2026, fp="Q2"):
+    return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-Q"}
+
+
+def _inst(end, val, accn, fy=2026, fp="Q2"):
+    return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-Q"}
+
+
+COMPANYFACTS = {
+    "cik": 320193,
+    "facts": {
+        "us-gaap": {
+            "RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
+                _dur("2025-09-28", "2025-12-27", 143756, "A", fp="Q1"),   # Q1 discrete == YTD
+                _dur("2025-09-28", "2026-03-28", 254940, "B"),            # Q2 YTD (181d)  <- want this
+                _dur("2025-12-28", "2026-03-28", 111184, "B"),            # Q2 discrete (90d)
+            ]}},
+            "NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 40000, "B")]}},
+            # only a discrete-length fact for Q2 -> must be null, not the discrete
+            "OperatingIncomeLoss": {"units": {"USD": [_dur("2025-12-28", "2026-03-28", 30000, "B")]}},
+            "EarningsPerShareDiluted": {"units": {"USD/shares": [_dur("2025-09-28", "2026-03-28", 2.55, "B")]}},
+            "CashAndCashEquivalentsAtCarryingValue": {"units": {"USD": [_inst("2026-03-28", 30000, "B")]}},
+            "MarketableSecuritiesCurrent": {"units": {"USD": [_inst("2026-03-28", 20000, "B")]}},
+            "LongTermDebtNoncurrent": {"units": {"USD": [_inst("2026-03-28", 80000, "B")]}},
+            "LongTermDebtCurrent": {"units": {"USD": [_inst("2026-03-28", 10000, "B")]}},
+        },
+        "dei": {
+            "EntityCommonStockSharesOutstanding": {"units": {"shares": [
+                {"end": "2026-04-17", "val": 14687356000, "fy": 2026, "fp": "Q2", "accn": "B", "form": "10-Q"},
+            ]}},
+        },
+    },
+}
+
+FILINGS = {
+    "A": FilingMeta(date(2025, 12, 27), date(2026, 1, 30), datetime(2026, 1, 30, 11, 1, tzinfo=UTC), "10-Q"),
+    "B": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, 10, 1, tzinfo=UTC), "10-Q"),
+}
+
+
+def _by_accn(rows):
+    return {r.accession: r for r in rows}
+
+
+def test_parses_ytd_not_discrete_and_cover_date_shares():
+    rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
+    assert not skips
+    b = _by_accn(rows)["B"]
+
+    assert (b.cik, b.fiscal_year, b.fiscal_period) == ("0000320193", 2026, "Q2")
+    assert b.period_end == date(2026, 3, 28)
+    assert b.period_start == date(2025, 9, 28)  # YTD start (fiscal-year start)
+    assert b.revenue == 254940          # the 6-month YTD, NOT the 111184 discrete
+    assert b.net_income == 40000
+    assert b.operating_income is None   # only a discrete-length fact existed -> null
+    assert b.diluted_eps == 2.55
+    # cash + first-present ST investment (MarketableSecuritiesCurrent), each once
+    assert b.cash_and_st_investments == 50000
+    # long-term parts summed (no aggregate, no short-term)
+    assert b.total_debt == 90000
+    assert b.shares_outstanding == 14687356000
+    assert b.shares_outstanding_date == date(2026, 4, 17)  # cover date != period_end
+    assert b.filed_date == date(2026, 5, 1)
+    assert b.accepted_at == datetime(2026, 5, 1, 10, 1, tzinfo=UTC)
+
+
+def test_q1_discrete_is_the_ytd():
+    rows, _ = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
+    a = _by_accn(rows)["A"]
+    assert a.fiscal_period == "Q1"
+    assert a.revenue == 143756  # Q1 YTD == Q1 discrete
+    assert a.period_start == date(2025, 9, 28)
+
+
+def test_skips_filing_without_usable_period():
+    cf = {
+        "cik": 320193,
+        "facts": {"us-gaap": {"NetIncomeLoss": {"units": {"USD": [
+            {"start": "2025-09-28", "end": "2026-03-28", "val": 1, "fy": 2026, "fp": "H1", "accn": "X", "form": "10-Q"},
+        ]}}}},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
+    rows, skips = parse_snapshots(cf, filings, {"X"})
+    assert rows == []
+    assert skips == [{"accession": "X", "reason": "no usable period identity"}]
+
+
+def test_missing_accession_is_skipped():
+    rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
+    assert rows == []
+    assert skips == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
+
+
+def test_debt_prefers_aggregate_over_parts():
+    rd = date(2026, 3, 28)
+    facts = [
+        Fact("us-gaap", "LongTermDebt", "USD", None, rd, 95000, 2026, "Q2"),
+        Fact("us-gaap", "LongTermDebtNoncurrent", "USD", None, rd, 80000, 2026, "Q2"),
+        Fact("us-gaap", "LongTermDebtCurrent", "USD", None, rd, 10000, 2026, "Q2"),
+        Fact("us-gaap", "CommercialPaper", "USD", None, rd, 5000, 2026, "Q2"),
+    ]
+    # aggregate (95000) used, parts ignored; + one short-term pick (5000)
+    assert _compose_debt(facts, rd) == 100000
+
+
+def test_cash_picks_one_st_investment_source():
+    rd = date(2026, 3, 28)
+    facts = [
+        Fact("us-gaap", "CashAndCashEquivalentsAtCarryingValue", "USD", None, rd, 30000, 2026, "Q2"),
+        Fact("us-gaap", "ShortTermInvestments", "USD", None, rd, 15000, 2026, "Q2"),
+        Fact("us-gaap", "MarketableSecuritiesCurrent", "USD", None, rd, 20000, 2026, "Q2"),
+    ]
+    # ShortTermInvestments is first in priority -> 30000 + 15000 (not both ST tags)
+    assert _compose_cash(facts, rd) == 45000
+
+
+# Opt-in live check against real Apple companyfacts. Skips unless SEC_LIVE=1 and a
+# real SEC_USER_AGENT are set (network + fair-access contact email).
+@pytest.mark.skipif(
+    not (os.environ.get("SEC_LIVE") and os.environ.get("SEC_USER_AGENT")),
+    reason="set SEC_LIVE=1 + SEC_USER_AGENT to run the live SEC parser check",
+)
+async def test_live_apple_parse_invariants():
+    from app.services.sec_client import SecClient
+
+    def _dt(s):
+        return datetime.fromisoformat(s.replace("Z", "+00:00")) if s else None
+
+    async with SecClient(user_agent=os.environ["SEC_USER_AGENT"]) as c:
+        cf = await c.companyfacts(320193)
+        sub = await c.submissions(320193, include_history=False)
+
+    filings = {
+        f["accession"]: FilingMeta(
+            date.fromisoformat(f["report_date"]),
+            date.fromisoformat(f["filing_date"]),
+            _dt(f["acceptance_datetime"]),
+            f["form"],
+        )
+        for f in sub["filings"]
+        if f["report_date"] and f["filing_date"] and f["acceptance_datetime"]
+    }
+    rows, _ = parse_snapshots(cf, filings, set(filings))
+    assert len(rows) > 20
+    assert all(r.period_end and r.fiscal_year and r.fiscal_period for r in rows)
+    # YTD revenue is non-decreasing within a fiscal year
+    by_fy: dict[int, list] = {}
+    for r in rows:
+        if r.revenue is not None:
+            by_fy.setdefault(r.fiscal_year, []).append((r.fiscal_period, r.revenue))
+    order = {"Q1": 1, "Q2": 2, "Q3": 3, "FY": 4}
+    for fy, series in by_fy.items():
+        series.sort(key=lambda x: order[x[0]])
+        vals = [v for _, v in series]
+        assert vals == sorted(vals), f"YTD revenue not monotonic in FY{fy}: {series}"
+    # shares cover-date differs from period_end
+    latest = max(rows, key=lambda r: r.period_end)
+    assert latest.shares_outstanding_date != latest.period_end