fix(sec): correct fundamentals derivation from SEC company facts

The A5 parity report surfaced coverage gaps and wrong values that all traced to the SEC facts parser and read-time derivation rather than to bad source data. Fixes, each validated by replaying the production parser + derivation against live company facts: - Period identity is derived from period_end against the issuer's fiscal calendar, not SEC's fy/fp fields, which collide (two period ends on one key, one silently discarded) and invert (a period sorting before one that precedes it) often enough to break the quarter chain. Recovers BXP, CRM, CRWD, FRT, MTD, NTAP, PPL, STX, WDAY. Fixed labels are internal ordering keys only (not in any API schema), so a filer whose year ends in early January shifting by one is harmless. - Revenue concept list gains RevenuesNetOfInterestExpense (banks) and the IncludingAssessedTax variant (REITs/consumer); EPS gains the continuing-ops variant (REG/FCX) and, last, basic EPS for a period tagging no diluted variant at all (PPL). All appended, so any issuer that already resolved keeps its concept. - YTD span tolerance 20 -> 25 days, covering 4-4-5 retail calendars whose 36-week YTD-Q3 (251-252d) previously missed by ~2 (COST, PEP, DPZ). - Amendment resolution is per field: a partial 10-K/A (Part III only, no financial facts) no longer blanks the period (DVN). - TTM diluted EPS is suppressed when a split contaminates the trailing window (BKNG's mixed-unit sum produced a P/E of 1.10 that clamped to a perfect fundamental sub-score). A post-filing split with no share-count evidence (KLAC) remains undetectable from this data. - Multi-class share fallback: weighted_avg_diluted_shares is captured and used for market cap when the cover-page count is absent (dimensional, so missing from company facts for META/CMCSA/CHTR/FOXA/NWSA/LEN). Within ~0.6% of the true count on controls; flagged shares_estimated in the API. BRK-B has no weighted-average fact either and stays unavailable. 820 unit tests pass; new tests confirmed to fail against the pre-fix code. Effect is inert until existing rows are reparsed (see reparse path). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 10:23:51 +02:00
parent 259001e419
commit 921f3d06fb
7 changed files with 626 additions and 21 deletions
@@ -270,3 +270,196 @@ async def test_live_apple_parse_invariants():
    # shares cover-date differs from period_end
    latest = max(rows, key=lambda r: r.period_end)
    assert latest.shares_outstanding_date != latest.period_end
+
+
+# -- revenue concept coverage (A5 parity findings) ---------------------------
+
+def _one_filing(concepts: dict, *, start: str, end: str, fp: str):
+    """A single 10-Q whose facts are the given {concept: value} at one YTD span."""
+    facts = {
+        name: {"units": {"USD": [_dur(start, end, val, "X", fp=fp)]}}
+        for name, val in concepts.items()
+    }
+    companyfacts = {"cik": 19617, "facts": {"us-gaap": facts}}
+    filings = {
+        "X": FilingMeta(
+            date.fromisoformat(end), date(2026, 5, 1), datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q"
+        )
+    }
+    return parse_snapshots(companyfacts, filings, {"X"})
+
+
+def test_revenue_reads_banks_total_revenue_tag():
+    # JPM/GS/WFC tag RevenuesNetOfInterestExpense in every 10-Q and never (or
+    # only annually) `Revenues` -- previously null, so revenue growth was too.
+    res = _one_filing(
+        {"RevenuesNetOfInterestExpense": 49836}, start="2026-01-01", end="2026-03-31", fp="Q1"
+    )
+    assert res.rows[0].revenue == 49836
+
+
+def test_revenue_reads_including_assessed_tax_variant():
+    # ARE/KHC tag only the Including variant.
+    res = _one_filing(
+        {"RevenueFromContractWithCustomerIncludingAssessedTax": 671},
+        start="2026-01-01", end="2026-03-31", fp="Q1",
+    )
+    assert res.rows[0].revenue == 671
+
+
+def test_revenue_concept_priority_is_unchanged_by_the_added_tags():
+    # The new entries are appended, so any issuer that already resolved keeps
+    # the same concept -- only issuers that resolved to nothing gain a value.
+    res = _one_filing(
+        {
+            "RevenueFromContractWithCustomerExcludingAssessedTax": 100,
+            "RevenueFromContractWithCustomerIncludingAssessedTax": 110,
+            "RevenuesNetOfInterestExpense": 120,
+            "Revenues": 130,
+        },
+        start="2026-01-01", end="2026-03-31", fp="Q1",
+    )
+    assert res.rows[0].revenue == 100
+
+
+def test_four_four_five_q3_ytd_span_is_accepted():
+    # A 12/12/12/16-week filer's YTD-Q3 is 36 weeks = 251 days (COST 2026 Q3),
+    # which missed the old 20-day tolerance around 273 by ~2 and dropped Q3
+    # every year -- breaking the quarter chain and nulling TTM and YoY.
+    res = _one_filing(
+        {"RevenueFromContractWithCustomerExcludingAssessedTax": 207431},
+        start="2025-09-01", end="2026-05-10", fp="Q3",
+    )
+    assert (date(2026, 5, 10) - date(2025, 9, 1)).days == 251
+    assert res.rows[0].revenue == 207431
+
+
+def test_eps_falls_back_to_continuing_operations_variant():
+    # REG tags only this variant on every filing; FCX tags it in its 10-K while
+    # using EarningsPerShareDiluted in its 10-Qs.
+    companyfacts = {
+        "cik": 910606,
+        "facts": {"us-gaap": {"IncomeLossFromContinuingOperationsPerDilutedShare": {
+            "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.81, "X", fp="Q1")]}
+        }}},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"})
+    assert res.rows[0].diluted_eps == 1.81
+
+
+def test_eps_concept_priority_is_unchanged_by_the_added_tag():
+    companyfacts = {
+        "cik": 831259,
+        "facts": {"us-gaap": {
+            "EarningsPerShareDiluted": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.61, "X", fp="Q1")]}},
+            "IncomeLossFromContinuingOperationsPerDilutedShare": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.75, "X", fp="Q1")]}},
+        }},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"})
+    assert res.rows[0].diluted_eps == 0.61
+
+
+# -- period identity from the fiscal calendar, not SEC's fy/fp ---------------
+
+from app.services.sec_facts_parser import _period_identity  # noqa: E402
+
+
+def _meta(end: str, form: str = "10-Q") -> FilingMeta:
+    d = date.fromisoformat(end)
+    return FilingMeta(d, d, datetime(d.year, d.month, d.day, tzinfo=UTC), form)
+
+
+def test_a_10q_is_never_labelled_fy():
+    # BXP: a 10-Q for period end 2026-03-31 carried fy/fp saying "2026 FY", which
+    # collided with the real annual row and measured a 90-day fact against the
+    # 365-day FY expectation.
+    fy, fp = _period_identity(_meta("2026-03-31"), "1231")
+    assert (fy, fp) == (2026, "Q1")
+
+
+def test_december_filer_years_do_not_collide():
+    # FRT: two 10-Ks, ending 2024-12-31 and 2025-12-31, both labelled "2024 FY".
+    assert _period_identity(_meta("2024-12-31", "10-K"), "1231") == (2024, "FY")
+    assert _period_identity(_meta("2025-12-31", "10-K"), "1231") == (2025, "FY")
+
+
+def test_january_year_end_groups_its_quarters(): 
+    # CRM/CRWD/WDAY: the year ending 2026-01-31 and its own quarters must share a
+    # fiscal year, and must not collide with the year ending 2025-01-31.
+    assert _period_identity(_meta("2026-01-31", "10-K"), "0131") == (2026, "FY")
+    assert _period_identity(_meta("2025-01-31", "10-K"), "0131") == (2025, "FY")
+    assert _period_identity(_meta("2025-04-30"), "0131") == (2026, "Q1")
+    assert _period_identity(_meta("2025-07-31"), "0131") == (2026, "Q2")
+    assert _period_identity(_meta("2025-10-31"), "0131") == (2026, "Q3")
+
+
+def test_mid_year_end_orders_correctly():
+    # STX: the year ending 2025-06-27 was labelled "2027 FY" and sorted after
+    # quarters that precede it.
+    assert _period_identity(_meta("2025-06-27", "10-K"), "0627") == (2025, "FY")
+    assert _period_identity(_meta("2025-10-03"), "0627") == (2026, "Q1")
+    assert _period_identity(_meta("2026-01-02"), "0627") == (2026, "Q2")
+    assert _period_identity(_meta("2026-04-03"), "0627") == (2026, "Q3")
+
+
+def test_four_four_five_quarters_place_correctly():
+    # COST: a 12/12/12/16-week year leaves Q3 112 days from the year end, not 91.
+    assert _period_identity(_meta("2025-11-23"), "0830") == (2026, "Q1")
+    assert _period_identity(_meta("2026-02-15"), "0830") == (2026, "Q2")
+    assert _period_identity(_meta("2026-05-10"), "0830") == (2026, "Q3")
+    assert _period_identity(_meta("2026-08-30", "10-K"), "0830") == (2026, "FY")
+
+
+def test_year_end_crossing_january_still_groups_one_year():
+    # DPZ (fiscalYearEnd 0102): the label shifts by one against Domino's own
+    # naming, which is fine -- a year and its quarters must simply agree.
+    year, _ = _period_identity(_meta("2025-12-28", "10-K"), "0102")
+    assert (year, "FY") == _period_identity(_meta("2025-12-28", "10-K"), "0102")
+    assert _period_identity(_meta("2025-03-23"), "0102") == (year, "Q1")
+    assert _period_identity(_meta("2025-06-15"), "0102") == (year, "Q2")
+    assert _period_identity(_meta("2025-09-07"), "0102") == (year, "Q3")
+
+
+def test_missing_fiscal_calendar_falls_back_to_filing_context():
+    assert _period_identity(_meta("2026-03-31"), None) == (None, None)
+    # ...and parse_snapshots then uses the fy/fp path, preserving old behaviour.
+    res = parse_snapshots(COMPANYFACTS, FILINGS, {"B"})
+    assert (res.rows[0].fiscal_year, res.rows[0].fiscal_period) == (2026, "Q2")
+
+
+
+def test_eps_falls_back_to_basic_only_when_no_diluted_variant_exists():
+    # PPL's 2026 Q1 tags no diluted EPS at all, only basic -- one missing period
+    # broke the quarter chain and nulled TTM.
+    companyfacts = {
+        "cik": 922224,
+        "facts": {"us-gaap": {"EarningsPerShareBasic": {
+            "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.60, "X", fp="Q1")]}
+        }}},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231")
+    assert res.rows[0].diluted_eps == 0.60
+
+
+def test_diluted_still_wins_over_basic_when_both_present():
+    companyfacts = {
+        "cik": 320193,
+        "facts": {"us-gaap": {
+            "EarningsPerShareDiluted": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.36, "X", fp="Q1")]}},
+            "EarningsPerShareBasic": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.40, "X", fp="Q1")]}},
+        }},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231")
+    assert res.rows[0].diluted_eps == 1.36