fix(sec): correct fundamentals derivation from SEC company facts

The A5 parity report surfaced coverage gaps and wrong values that all traced to the SEC facts parser and read-time derivation rather than to bad source data. Fixes, each validated by replaying the production parser + derivation against live company facts: - Period identity is derived from period_end against the issuer's fiscal calendar, not SEC's fy/fp fields, which collide (two period ends on one key, one silently discarded) and invert (a period sorting before one that precedes it) often enough to break the quarter chain. Recovers BXP, CRM, CRWD, FRT, MTD, NTAP, PPL, STX, WDAY. Fixed labels are internal ordering keys only (not in any API schema), so a filer whose year ends in early January shifting by one is harmless. - Revenue concept list gains RevenuesNetOfInterestExpense (banks) and the IncludingAssessedTax variant (REITs/consumer); EPS gains the continuing-ops variant (REG/FCX) and, last, basic EPS for a period tagging no diluted variant at all (PPL). All appended, so any issuer that already resolved keeps its concept. - YTD span tolerance 20 -> 25 days, covering 4-4-5 retail calendars whose 36-week YTD-Q3 (251-252d) previously missed by ~2 (COST, PEP, DPZ). - Amendment resolution is per field: a partial 10-K/A (Part III only, no financial facts) no longer blanks the period (DVN). - TTM diluted EPS is suppressed when a split contaminates the trailing window (BKNG's mixed-unit sum produced a P/E of 1.10 that clamped to a perfect fundamental sub-score). A post-filing split with no share-count evidence (KLAC) remains undetectable from this data. - Multi-class share fallback: weighted_avg_diluted_shares is captured and used for market cap when the cover-page count is absent (dimensional, so missing from company facts for META/CMCSA/CHTR/FOXA/NWSA/LEN). Within ~0.6% of the true count on controls; flagged shares_estimated in the API. BRK-B has no weighted-average fact either and stays unavailable. 820 unit tests pass; new tests confirmed to fail against the pre-fix code. Effect is inert until existing rows are reparsed (see reparse path). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 10:23:51 +02:00
parent 259001e419
commit 921f3d06fb
7 changed files with 626 additions and 21 deletions
@@ -29,6 +29,7 @@ class Snap:
    cash_and_st_investments: float | None = None
    total_debt: float | None = None
    shares_outstanding: float | None = None
+    weighted_avg_diluted_shares: float | None = None


 _FP = ["Q1", "Q2", "Q3", "FY"]
@@ -188,3 +189,112 @@ def test_amendment_selection_newest_accepted_wins():
    # Q4 revenue discrete now uses the amended YTD(FY)=999999 minus YTD(Q3)=363
    # so TTM/growth reflects the amendment, proving newest accepted_at won.
    assert d.metrics["revenue_growth_yoy"].value != pytest.approx(10.0, abs=1e-6)
+
+
+# -- partial amendments (A5 parity findings) ---------------------------------
+
+def test_partial_amendment_does_not_blank_the_period():
+    # DVN's FY2025 10-K/A carries no financial facts at the report date. Taking
+    # the newest accession wholesale nulled the period, and with it the quarter
+    # chain, TTM and YoY.
+    rows = _two_years()
+    part_iii_only = Snap(2026, "FY", date(2026, 9, 30), date(2026, 11, 1),
+                         datetime(2027, 1, 1, tzinfo=UTC))
+    baseline = fd.derive(rows)
+    d = fd.derive(rows + [part_iii_only])
+    assert d.ttm_diluted_eps == pytest.approx(baseline.ttm_diluted_eps)
+    assert d.metrics["revenue_growth_yoy"].value == pytest.approx(
+        baseline.metrics["revenue_growth_yoy"].value
+    )
+
+
+def test_amendment_restating_one_field_leaves_the_others_intact():
+    rows = _two_years()
+    revenue_only = Snap(2026, "FY", date(2026, 9, 30), date(2026, 11, 1),
+                        datetime(2027, 1, 1, tzinfo=UTC), revenue=999999)
+    baseline = fd.derive(rows)
+    d = fd.derive(rows + [revenue_only])
+    assert d.metrics["revenue_growth_yoy"].value != pytest.approx(
+        baseline.metrics["revenue_growth_yoy"].value
+    )
+    assert d.ttm_diluted_eps == pytest.approx(baseline.ttm_diluted_eps)  # fell back
+
+
+def test_same_key_row_for_a_different_period_is_never_merged():
+    # SEC labels two different year-ends with one fiscal_year for some filers
+    # (FRT, CRM). That is a mislabelled filing, not an amendment -- merging the
+    # two would silently blend fiscal years.
+    rows = _two_years()
+    mislabelled = Snap(2026, "FY", date(2027, 9, 30), date(2027, 11, 1),
+                       datetime(2027, 12, 1, tzinfo=UTC), revenue=999999)
+    selected = fd._select_latest_per_period(rows + [mislabelled])
+    assert selected[(2026, "FY")] is mislabelled
+
+
+# -- split safety for the TTM EPS scalar (A5 parity findings) ----------------
+
+def _split_rows():
+    """Two years where the share count jumps ~25x at the latest quarter, as
+    BKNG's did (31.7M -> 774.9M) when its split landed mid-window."""
+    rows = _two_years()
+    for row in rows:
+        if (row.fiscal_year, row.fiscal_period) == (2026, "FY"):
+            row.shares_outstanding = 25000.0  # vs 1000 a year earlier
+    return rows
+
+
+def test_split_suppresses_ttm_diluted_eps():
+    # TTM sums four quarters of per-share values; a split inside the window
+    # mixes units. Unguarded this produced BKNG's P/E of 1.10, which clamps to a
+    # *perfect* fundamental sub-score -- worse than having no value at all.
+    d = fd.derive(_split_rows())
+    assert d.ttm_diluted_eps is None
+    assert d.ttm_diluted_eps_caveat == fd.SPLIT_SENSITIVE_CAVEAT
+
+
+def test_ttm_diluted_eps_survives_when_no_split_is_suspected():
+    d = fd.derive(_two_years())
+    assert d.ttm_diluted_eps is not None
+    assert d.ttm_diluted_eps_caveat is None
+
+
+def test_split_guard_leaves_dollar_scalars_alone():
+    # Only per-share values are split-sensitive; FCF is in dollars.
+    baseline = fd.derive(_two_years())
+    d = fd.derive(_split_rows())
+    assert d.ttm_fcf == pytest.approx(baseline.ttm_fcf)
+
+
+# -- multi-class share-count fallback (A5 parity findings) -------------------
+
+def test_shares_fall_back_to_weighted_average_when_cover_page_count_is_absent():
+    # META/CMCSA/BRK-B/CHTR report the cover-page count per share class, which is
+    # dimensional and therefore absent from companyfacts -- silently removing
+    # market cap and FCF yield for some of the largest issuers.
+    rows = _two_years()
+    for row in rows:
+        row.shares_outstanding = None
+        row.weighted_avg_diluted_shares = 2_564_000_000.0
+    d = fd.derive(rows)
+    assert d.shares_outstanding == 2_564_000_000.0
+    assert d.shares_outstanding_estimated is True
+
+
+def test_point_in_time_share_count_is_preferred_and_not_flagged():
+    baseline = fd.derive(_two_years()).shares_outstanding
+    assert baseline is not None, "fixture should carry a cover-page count"
+    rows = _two_years()
+    for row in rows:
+        row.weighted_avg_diluted_shares = 1.0  # must lose to the real count
+    d = fd.derive(rows)
+    assert d.shares_outstanding == baseline
+    assert d.shares_outstanding_estimated is False
+
+
+def test_no_share_count_at_all_stays_none_and_unflagged():
+    rows = _two_years()
+    for row in rows:
+        row.shares_outstanding = None
+    d = fd.derive(rows)
+    assert d.shares_outstanding is None
+    assert d.shares_outstanding_estimated is False
@@ -270,3 +270,196 @@ async def test_live_apple_parse_invariants():
    # shares cover-date differs from period_end
    latest = max(rows, key=lambda r: r.period_end)
    assert latest.shares_outstanding_date != latest.period_end
+
+
+# -- revenue concept coverage (A5 parity findings) ---------------------------
+
+def _one_filing(concepts: dict, *, start: str, end: str, fp: str):
+    """A single 10-Q whose facts are the given {concept: value} at one YTD span."""
+    facts = {
+        name: {"units": {"USD": [_dur(start, end, val, "X", fp=fp)]}}
+        for name, val in concepts.items()
+    }
+    companyfacts = {"cik": 19617, "facts": {"us-gaap": facts}}
+    filings = {
+        "X": FilingMeta(
+            date.fromisoformat(end), date(2026, 5, 1), datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q"
+        )
+    }
+    return parse_snapshots(companyfacts, filings, {"X"})
+
+
+def test_revenue_reads_banks_total_revenue_tag():
+    # JPM/GS/WFC tag RevenuesNetOfInterestExpense in every 10-Q and never (or
+    # only annually) `Revenues` -- previously null, so revenue growth was too.
+    res = _one_filing(
+        {"RevenuesNetOfInterestExpense": 49836}, start="2026-01-01", end="2026-03-31", fp="Q1"
+    )
+    assert res.rows[0].revenue == 49836
+
+
+def test_revenue_reads_including_assessed_tax_variant():
+    # ARE/KHC tag only the Including variant.
+    res = _one_filing(
+        {"RevenueFromContractWithCustomerIncludingAssessedTax": 671},
+        start="2026-01-01", end="2026-03-31", fp="Q1",
+    )
+    assert res.rows[0].revenue == 671
+
+
+def test_revenue_concept_priority_is_unchanged_by_the_added_tags():
+    # The new entries are appended, so any issuer that already resolved keeps
+    # the same concept -- only issuers that resolved to nothing gain a value.
+    res = _one_filing(
+        {
+            "RevenueFromContractWithCustomerExcludingAssessedTax": 100,
+            "RevenueFromContractWithCustomerIncludingAssessedTax": 110,
+            "RevenuesNetOfInterestExpense": 120,
+            "Revenues": 130,
+        },
+        start="2026-01-01", end="2026-03-31", fp="Q1",
+    )
+    assert res.rows[0].revenue == 100
+
+
+def test_four_four_five_q3_ytd_span_is_accepted():
+    # A 12/12/12/16-week filer's YTD-Q3 is 36 weeks = 251 days (COST 2026 Q3),
+    # which missed the old 20-day tolerance around 273 by ~2 and dropped Q3
+    # every year -- breaking the quarter chain and nulling TTM and YoY.
+    res = _one_filing(
+        {"RevenueFromContractWithCustomerExcludingAssessedTax": 207431},
+        start="2025-09-01", end="2026-05-10", fp="Q3",
+    )
+    assert (date(2026, 5, 10) - date(2025, 9, 1)).days == 251
+    assert res.rows[0].revenue == 207431
+
+
+def test_eps_falls_back_to_continuing_operations_variant():
+    # REG tags only this variant on every filing; FCX tags it in its 10-K while
+    # using EarningsPerShareDiluted in its 10-Qs.
+    companyfacts = {
+        "cik": 910606,
+        "facts": {"us-gaap": {"IncomeLossFromContinuingOperationsPerDilutedShare": {
+            "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.81, "X", fp="Q1")]}
+        }}},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"})
+    assert res.rows[0].diluted_eps == 1.81
+
+
+def test_eps_concept_priority_is_unchanged_by_the_added_tag():
+    companyfacts = {
+        "cik": 831259,
+        "facts": {"us-gaap": {
+            "EarningsPerShareDiluted": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.61, "X", fp="Q1")]}},
+            "IncomeLossFromContinuingOperationsPerDilutedShare": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.75, "X", fp="Q1")]}},
+        }},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"})
+    assert res.rows[0].diluted_eps == 0.61
+
+
+# -- period identity from the fiscal calendar, not SEC's fy/fp ---------------
+
+from app.services.sec_facts_parser import _period_identity  # noqa: E402
+
+
+def _meta(end: str, form: str = "10-Q") -> FilingMeta:
+    d = date.fromisoformat(end)
+    return FilingMeta(d, d, datetime(d.year, d.month, d.day, tzinfo=UTC), form)
+
+
+def test_a_10q_is_never_labelled_fy():
+    # BXP: a 10-Q for period end 2026-03-31 carried fy/fp saying "2026 FY", which
+    # collided with the real annual row and measured a 90-day fact against the
+    # 365-day FY expectation.
+    fy, fp = _period_identity(_meta("2026-03-31"), "1231")
+    assert (fy, fp) == (2026, "Q1")
+
+
+def test_december_filer_years_do_not_collide():
+    # FRT: two 10-Ks, ending 2024-12-31 and 2025-12-31, both labelled "2024 FY".
+    assert _period_identity(_meta("2024-12-31", "10-K"), "1231") == (2024, "FY")
+    assert _period_identity(_meta("2025-12-31", "10-K"), "1231") == (2025, "FY")
+
+
+def test_january_year_end_groups_its_quarters(): 
+    # CRM/CRWD/WDAY: the year ending 2026-01-31 and its own quarters must share a
+    # fiscal year, and must not collide with the year ending 2025-01-31.
+    assert _period_identity(_meta("2026-01-31", "10-K"), "0131") == (2026, "FY")
+    assert _period_identity(_meta("2025-01-31", "10-K"), "0131") == (2025, "FY")
+    assert _period_identity(_meta("2025-04-30"), "0131") == (2026, "Q1")
+    assert _period_identity(_meta("2025-07-31"), "0131") == (2026, "Q2")
+    assert _period_identity(_meta("2025-10-31"), "0131") == (2026, "Q3")
+
+
+def test_mid_year_end_orders_correctly():
+    # STX: the year ending 2025-06-27 was labelled "2027 FY" and sorted after
+    # quarters that precede it.
+    assert _period_identity(_meta("2025-06-27", "10-K"), "0627") == (2025, "FY")
+    assert _period_identity(_meta("2025-10-03"), "0627") == (2026, "Q1")
+    assert _period_identity(_meta("2026-01-02"), "0627") == (2026, "Q2")
+    assert _period_identity(_meta("2026-04-03"), "0627") == (2026, "Q3")
+
+
+def test_four_four_five_quarters_place_correctly():
+    # COST: a 12/12/12/16-week year leaves Q3 112 days from the year end, not 91.
+    assert _period_identity(_meta("2025-11-23"), "0830") == (2026, "Q1")
+    assert _period_identity(_meta("2026-02-15"), "0830") == (2026, "Q2")
+    assert _period_identity(_meta("2026-05-10"), "0830") == (2026, "Q3")
+    assert _period_identity(_meta("2026-08-30", "10-K"), "0830") == (2026, "FY")
+
+
+def test_year_end_crossing_january_still_groups_one_year():
+    # DPZ (fiscalYearEnd 0102): the label shifts by one against Domino's own
+    # naming, which is fine -- a year and its quarters must simply agree.
+    year, _ = _period_identity(_meta("2025-12-28", "10-K"), "0102")
+    assert (year, "FY") == _period_identity(_meta("2025-12-28", "10-K"), "0102")
+    assert _period_identity(_meta("2025-03-23"), "0102") == (year, "Q1")
+    assert _period_identity(_meta("2025-06-15"), "0102") == (year, "Q2")
+    assert _period_identity(_meta("2025-09-07"), "0102") == (year, "Q3")
+
+
+def test_missing_fiscal_calendar_falls_back_to_filing_context():
+    assert _period_identity(_meta("2026-03-31"), None) == (None, None)
+    # ...and parse_snapshots then uses the fy/fp path, preserving old behaviour.
+    res = parse_snapshots(COMPANYFACTS, FILINGS, {"B"})
+    assert (res.rows[0].fiscal_year, res.rows[0].fiscal_period) == (2026, "Q2")
+
+
+
+def test_eps_falls_back_to_basic_only_when_no_diluted_variant_exists():
+    # PPL's 2026 Q1 tags no diluted EPS at all, only basic -- one missing period
+    # broke the quarter chain and nulled TTM.
+    companyfacts = {
+        "cik": 922224,
+        "facts": {"us-gaap": {"EarningsPerShareBasic": {
+            "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.60, "X", fp="Q1")]}
+        }}},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231")
+    assert res.rows[0].diluted_eps == 0.60
+
+
+def test_diluted_still_wins_over_basic_when_both_present():
+    companyfacts = {
+        "cik": 320193,
+        "facts": {"us-gaap": {
+            "EarningsPerShareDiluted": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.36, "X", fp="Q1")]}},
+            "EarningsPerShareBasic": {
+                "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.40, "X", fp="Q1")]}},
+        }},
+    }
+    filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1),
+                               datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")}
+    res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231")
+    assert res.rows[0].diluted_eps == 1.36