diff --git a/alembic/versions/027_snapshot_weighted_avg_diluted_shares.py b/alembic/versions/027_snapshot_weighted_avg_diluted_shares.py new file mode 100644 index 0000000..fcca808 --- /dev/null +++ b/alembic/versions/027_snapshot_weighted_avg_diluted_shares.py @@ -0,0 +1,43 @@ +"""fundamental_snapshots.weighted_avg_diluted_shares — market-cap fallback + +Revision ID: 027 +Revises: 026 +Create Date: 2026-07-24 00:00:00.000000 + +Multi-class issuers report the cover-page share count per share class. That is a +dimensional fact and Company Facts is non-dimensional, so it is absent entirely: +META has never tagged it, CMCSA stops in 2009, BRK-B in 2011, CHTR in 2016 (when +the Time Warner Cable deal made it multi-class). `shares_outstanding` is +therefore null for a large slice of the mega-cap universe, which silently removes +both `market_cap_est` and `fcf_yield`. + +The weighted-average diluted count is always present (EPS requires it) and is +consolidated across classes. Measured against issuers where the true +point-in-time count IS available, it lands within ~0.6%: GOOGL 0.9936, MRNA +1.0045, AAPL 0.9974, MSFT 0.9978. + +Stored as its own column rather than backfilled into `shares_outstanding`, so the +point-in-time column keeps its strict meaning and the fallback stays an explicit, +labelled read-time decision. Existing rows are null until a reparse. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "027" +down_revision: Union[str, None] = "026" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "fundamental_snapshots", + sa.Column("weighted_avg_diluted_shares", sa.Float(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("fundamental_snapshots", "weighted_avg_diluted_shares") diff --git a/app/models/fundamental_snapshot.py b/app/models/fundamental_snapshot.py index afdc469..6b3a8c4 100644 --- a/app/models/fundamental_snapshot.py +++ b/app/models/fundamental_snapshot.py @@ -12,8 +12,10 @@ class FundamentalSnapshot(Base): Keyed by issuer (CIK), not ticker — multi-class issuers (GOOG/GOOGL) share one CIK and one set of fundamentals; the ``tickers.cik`` column is the only join point. Amendments are retained: every accession is a distinct immutable - row, and readers pick the newest valid ``accepted_at`` per - (cik, fiscal_year, fiscal_period) at read time — no flags, no mutation. + row, and readers resolve (cik, fiscal_year, fiscal_period) at read time by + taking the newest ``accepted_at`` **per field**, falling back to the newest + accession that actually reports one — a partial amendment (a 10-K/A adding + Part III reports no financial facts) must not blank the period — no flags, no mutation. **Facts are stored as the filing reports them, never as derived quarters.** Duration facts (revenue, net_income, operating_income, diluted_eps, cfo, @@ -70,6 +72,12 @@ class FundamentalSnapshot(Base): # reported "as of" its own date, which can differ from period_end — store it # so market cap uses the right point-in-time count. shares_outstanding_date: Mapped[date | None] = mapped_column(Date, nullable=True) + # Weighted-average diluted count for the filing's most recent quarter — the + # market-cap fallback when the cover-page count is absent, which it always is + # for multi-class issuers (per-class facts are dimensional, and companyfacts + # is not). An average is not cumulative, so unlike the duration facts above + # this is NOT a YTD value: it is the shortest-span fact ending at period_end. + weighted_avg_diluted_shares: Mapped[float | None] = mapped_column(Float, nullable=True) import_run_id: Mapped[int | None] = mapped_column( ForeignKey("data_import_runs.id", ondelete="SET NULL"), nullable=True diff --git a/app/services/fundamentals_api_service.py b/app/services/fundamentals_api_service.py index 33541bb..f3c158e 100644 --- a/app/services/fundamentals_api_service.py +++ b/app/services/fundamentals_api_service.py @@ -155,6 +155,13 @@ def _build_valuation(derived, subject_price, peer_derived, peer_price_by_cik, tw "pe": _round(pe, 2), "fcf_yield": _round(fcf_yield, 2), "market_cap_est": _round(market_cap, 0), + # market_cap_est and fcf_yield both rest on the share count. When it came + # from the weighted-average diluted fallback (multi-class issuers, whose + # per-class cover-page count is absent from companyfacts), say so rather + # than presenting a period average as a point-in-time count. + "shares_estimated": bool( + market_cap is not None and derived.shares_outstanding_estimated + ), "pe_industry": pe_industry, "fcf_yield_industry": fcf_yield_industry, "price_date": _iso(price_date), diff --git a/app/services/fundamentals_derivation.py b/app/services/fundamentals_derivation.py index d9790ce..af0e18f 100644 --- a/app/services/fundamentals_derivation.py +++ b/app/services/fundamentals_derivation.py @@ -8,8 +8,10 @@ schema decision. No I/O, no DB: it takes an issuer's snapshot rows (ORM rows or any objects with the same attributes) and returns structured metrics. Rules: -- **Amendment selection:** for each (fiscal_year, fiscal_period), the row with - the newest `accepted_at` wins. +- **Amendment selection:** for each (fiscal_year, fiscal_period), the newest + `accepted_at` wins **per field**, falling back to the newest row that actually + reports one. A partial amendment (a 10-K/A adding Part III carries no financial + facts) must not blank the period. - **Discrete quarter** = YTD(Qn) − YTD(Qn−1); Q1 = YTD(Q1); **Q4 = YTD(FY) − YTD(Q3)**. Any missing period → the derived value is null, never partial. - **TTM** = sum of the trailing four discrete quarters ending at a period. @@ -21,6 +23,7 @@ from __future__ import annotations from dataclasses import dataclass, field from datetime import date +from types import SimpleNamespace from typing import Any, Iterable _FP_TO_Q = {"Q1": 1, "Q2": 2, "Q3": 3, "FY": 4} @@ -38,6 +41,20 @@ _FLOW_FIELDS = ( "revenue", "net_income", "operating_income", "diluted_eps", "cfo", "capex", "depreciation_amortization", ) +# Reported facts resolved independently across a period's accessions (see +# _merge_amendments); period identity/provenance is taken from the newest one. +_MERGED_FIELDS = ( + *_FLOW_FIELDS, + "cash_and_st_investments", "total_debt", "shares_outstanding", + "shares_outstanding_date", + # period_start is set alongside revenue by the parser, so it follows the same + # fallback: a bare amendment reports neither and must not blank it. + "period_start", +) +_CARRIED_FIELDS = ( + "fiscal_year", "fiscal_period", "period_end", "filed_date", + "accepted_at", "form", "accession", "cik", +) @dataclass @@ -60,8 +77,15 @@ class DerivedFundamentals: metrics: dict[str, MetricSeries] = field(default_factory=dict) # request-time valuation inputs (ratios are computed in the API with price) ttm_diluted_eps: float | None = None + # Set when ttm_diluted_eps was suppressed rather than simply unavailable. + ttm_diluted_eps_caveat: str | None = None ttm_fcf: float | None = None shares_outstanding: float | None = None + # True when shares_outstanding came from the weighted-average diluted count + # because the point-in-time cover-page count was absent (always so for + # multi-class issuers). Consumers must label anything derived from it as + # estimated — it is a period average, not a point-in-time count. + shares_outstanding_estimated: bool = False latest_period_end: date | None = None latest_filed_date: date | None = None @@ -85,6 +109,15 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals: result.latest_period_end = latest_row.period_end result.latest_filed_date = latest_row.filed_date result.shares_outstanding = getattr(latest_row, "shares_outstanding", None) + if result.shares_outstanding is None: + # Multi-class issuers (META, CMCSA, BRK-B, CHTR, FOXA, NWSA, LEN) report + # the cover-page count per class, which is dimensional and so absent from + # companyfacts — leaving market cap and FCF yield silently unavailable for + # some of the largest names. The weighted-average diluted count is always + # present and within ~0.6% of the true count where both exist, so fall + # back to it and mark the result estimated rather than show nothing. + result.shares_outstanding = getattr(latest_row, "weighted_avg_diluted_shares", None) + result.shares_outstanding_estimated = result.shares_outstanding is not None result.ttm_diluted_eps = _ttm(discrete["diluted_eps"], *latest) ttm_cfo = _ttm(discrete["cfo"], *latest) ttm_capex = _ttm(discrete["capex"], *latest) @@ -102,7 +135,14 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals: "net_debt_to_ebitda": _leverage_series(selected, discrete, tape), "share_count_change_yoy": _share_change_series(selected, tape), } - _guard_split_sensitive_metrics(result.metrics) + # TTM EPS sums four quarters of *per-share* values, so a split inside that + # window mixes pre- and post-split units — the same distortion the guard + # already catches for the series, and the one that produced BKNG's P/E of + # 1.10. Left unguarded it does not merely mislead: a nonsense-low P/E clamps + # to a perfect 100 fundamental sub-score, so it must null out like the rest. + if _guard_split_sensitive_metrics(result.metrics): + result.ttm_diluted_eps = None + result.ttm_diluted_eps_caveat = SPLIT_SENSITIVE_CAVEAT for series in result.metrics.values(): series.period_end = latest_row.period_end series.filed_date = latest_row.filed_date @@ -112,17 +152,57 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals: # -- period selection -------------------------------------------------------- def _select_latest_per_period(snapshots: Iterable[Any]) -> dict[tuple[int, str], Any]: - best: dict[tuple[int, str], Any] = {} + grouped: dict[tuple[int, str], list[Any]] = {} for row in snapshots: fp = getattr(row, "fiscal_period", None) fy = getattr(row, "fiscal_year", None) if fp not in _FP_TO_Q or fy is None: continue - key = (fy, fp) - cur = best.get(key) - if cur is None or _accepted(row) > _accepted(cur): - best[key] = row - return best + grouped.setdefault((fy, fp), []).append(row) + return {key: _merge_amendments(rows) for key, rows in grouped.items()} + + +def _merge_amendments(rows: list[Any]) -> Any: + """Resolve one period from its accessions: newest wins, per field. + + Amendments are frequently partial — a 10-K/A filed only to add Part III + reports no financial facts at all. Taking the newest accession wholesale + would blank every field it omits and null the period downstream (and with + it TTM and YoY, which need an unbroken quarter chain), so each field falls + back to the newest accession that actually reports it. + + Only rows sharing the newest row's ``period_end`` are merged. A same-key row + covering a *different* period is a mislabelled filing, not an amendment, and + blending the two would silently mix fiscal years. + """ + if len(rows) == 1: + return rows[0] + ordered = sorted(rows, key=_amendment_order, reverse=True) # newest first + newest = ordered[0] + same_period = [ + row + for row in ordered + if getattr(row, "period_end", None) == getattr(newest, "period_end", None) + ] + if len(same_period) == 1: + return newest + merged = SimpleNamespace(**{name: getattr(newest, name, None) for name in _CARRIED_FIELDS}) + for name in _MERGED_FIELDS: + merged_value = None + for row in same_period: # newest first + value = getattr(row, name, None) + if value is not None: + merged_value = value + break + setattr(merged, name, merged_value) + return merged + + +def _amendment_order(row: Any) -> tuple[bool, Any]: + # (has-timestamp, timestamp) so a row without one sorts oldest instead of + # raising when compared against a row that has one. + accepted = _accepted(row) + return (accepted is not None, accepted) def _accepted(row: Any): @@ -255,18 +335,21 @@ def _share_change_series(selected, tape) -> MetricSeries: return _series(pts) -def _guard_split_sensitive_metrics(metrics: dict[str, MetricSeries]) -> None: +def _guard_split_sensitive_metrics(metrics: dict[str, MetricSeries]) -> bool: """Suppress historical comparisons likely distorted by a corporate action. Company Facts has no point-in-time split factors. A large YoY share-count move can therefore make both the point-in-time share comparison and per-share EPS growth non-comparable. Keep the raw facts in snapshots, but expose nulls plus an explicit caveat in the user-facing derived series. + + Returns True when the *latest* period is suspect, so callers can apply the + same suppression to per-share scalars derived from that window. """ shares = metrics.get("share_count_change_yoy") eps = metrics.get("eps_growth_yoy") if shares is None or eps is None: - return + return False suspect_periods = { point.period_end @@ -275,18 +358,21 @@ def _guard_split_sensitive_metrics(metrics: dict[str, MetricSeries]) -> None: and abs(point.value) >= SPLIT_SUSPECT_SHARE_CHANGE_PCT } if not suspect_periods: - return + return False + latest_suspect = False for series in (shares, eps): latest_guarded = bool( series.history and series.history[-1].period_end in suspect_periods ) + latest_suspect = latest_suspect or latest_guarded for point in series.history: if point.period_end in suspect_periods: point.value = None series.value = series.history[-1].value if series.history else None if latest_guarded: series.caveat = SPLIT_SENSITIVE_CAVEAT + return latest_suspect def _net_debt(row: Any) -> float | None: diff --git a/app/services/sec_facts_parser.py b/app/services/sec_facts_parser.py index 99239cc..db1f943 100644 --- a/app/services/sec_facts_parser.py +++ b/app/services/sec_facts_parser.py @@ -8,6 +8,9 @@ fixture and verifiable against a real companyfacts pull. The load-bearing rules (design Decision 2 + review): - Period identity comes from `end == submissions.reportDate`, never `fy/fp` (fy/fp is the *filing's* context; comparatives inside a filing repeat it). + This applies to the stored `fiscal_year`/`fiscal_period` too: they are derived + from `reportDate` against the issuer's `fiscalYearEnd` (see `_period_identity`), + because SEC's fy/fp collide and invert often enough to break the quarter chain. - Duration facts are stored as **cumulative YTD**: pick the fact whose span matches the fiscal-period-to-date length (Q1≈3mo … FY≈12mo) within tolerance. If no YTD-length fact exists, store null — never a discrete masquerading as YTD. @@ -15,7 +18,10 @@ The load-bearing rules (design Decision 2 + review): is a single consolidated value: the cover-page `dei` fact (its own cover-date `end` stored separately) if present, else `us-gaap:CommonStockSharesOutstanding` at period end (e.g. Alphabet has no `dei` fact) — never a class sum or the - weighted-average/diluted count. + weighted-average/diluted count. Multi-class issuers report it per class, which + is dimensional and therefore absent from companyfacts entirely, so + `weighted_avg_diluted_shares` is stored alongside as an explicit fallback for + market cap — a separate column, never backfilled into `shares_outstanding`. - Cash and debt composites are aggregate-first and mutually exclusive (each source tag counted at most once). @@ -29,7 +35,7 @@ from __future__ import annotations import logging import math from dataclasses import dataclass, field -from datetime import date, datetime +from datetime import date, datetime, timedelta from typing import Any, NamedTuple logger = logging.getLogger(__name__) @@ -37,14 +43,35 @@ logger = logging.getLogger(__name__) # Expected YTD span (days) per fiscal period; a duration fact must land within # tolerance of this to count as the period's cumulative value. _EXPECTED_YTD_DAYS = {"Q1": 91, "Q2": 182, "Q3": 273, "FY": 365} -_YTD_TOLERANCE_DAYS = 20 # covers 52/53-week fiscal calendars +# Period identity (see _period_identity): how far a quarter end sits before its +# fiscal-year end, and how far a fiscal-year end may drift from the nominal MMDD. +# The quarter bands are 91 days apart, so ±35 stays unambiguous even for a 4-4-5 +# filer whose 16-week Q4 puts Q3 112 days out. +_QUARTER_DAYS_TO_FY_END = {"Q1": 273, "Q2": 182, "Q3": 91} +_QUARTER_TOLERANCE_DAYS = 35 +_FYE_DRIFT_TOLERANCE_DAYS = 21 +# Covers 52/53-week calendars *and* 4-4-5 retail ones (12/12/12/16 weeks), whose +# YTD-Q3 is 36 weeks = 251-252 days and missed a 20-day tolerance by ~2 -- so +# COST/PEP lost Q3 every year, breaking the quarter chain and nulling TTM + YoY. +# Q1 84d, Q2 168d and FY 364d were always inside. Adjacent periods stay +# unambiguous at 25 (66-116, 157-207, 248-298, 340-390). +_YTD_TOLERANCE_DAYS = 25 # us-gaap duration concepts (money), priority order; first present wins. _DURATION_USD = { + # Order is load-bearing (first present wins) and the tail entries are + # deliberately *appended*: every issuer that already resolved keeps the same + # concept, and only issuers that resolved to nothing gain a value. + # - IncludingAssessedTax: REITs/consumer filers that tag only this variant + # (e.g. ARE, KHC) reported no revenue at all. + # - RevenuesNetOfInterestExpense: the banks' total-revenue tag. JPM/GS/WFC + # tag it in every 10-Q and `Revenues` only (if at all) in the 10-K. "revenue": [ "RevenueFromContractWithCustomerExcludingAssessedTax", "Revenues", "SalesRevenueNet", + "RevenueFromContractWithCustomerIncludingAssessedTax", + "RevenuesNetOfInterestExpense", ], "net_income": ["NetIncomeLoss"], "operating_income": ["OperatingIncomeLoss"], @@ -62,7 +89,27 @@ _DURATION_USD = { "DepreciationAndAmortization", ], } -_EPS_CONCEPTS = ["EarningsPerShareDiluted"] # unit USD/shares +# unit USD/shares. Appended (not reordered) so any issuer that already resolved +# keeps the same concept. REG tags only the continuing-operations variant on every +# filing; FCX switches by form type -- EarningsPerShareDiluted in its 10-Qs, the +# continuing-ops tag in its 10-K -- which nulled the FY row and killed Q4 + TTM. +# The basic variants are a last resort for a period that tags no diluted EPS at +# all (PPL's 2026 Q1). Basic ignores option/convert dilution so it slightly +# overstates EPS (~1.2% for PPL), but only fires when diluted is entirely absent, +# and high-dilution names always tag diluted -- so it never displaces a real one. +_EPS_CONCEPTS = [ + "EarningsPerShareDiluted", + "IncomeLossFromContinuingOperationsPerDilutedShare", + "EarningsPerShareBasic", + "IncomeLossFromContinuingOperationsPerBasicShare", +] +# Weighted-average diluted share count (unit "shares"), the market-cap fallback +# for multi-class issuers whose cover-page count is dimensional and therefore +# absent from companyfacts. Always present, since EPS is computed from it. +_WEIGHTED_AVG_SHARE_CONCEPTS = [ + "WeightedAverageNumberOfDilutedSharesOutstanding", + "WeightedAverageNumberOfSharesOutstandingBasicAndDiluted", +] # us-gaap instant (balance-sheet) concepts, at end == reportDate. _CASH = ["CashAndCashEquivalentsAtCarryingValue"] _ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one @@ -104,6 +151,7 @@ class SnapshotRow: total_debt: float | None = None shares_outstanding: float | None = None shares_outstanding_date: date | None = None + weighted_avg_diluted_shares: float | None = None @dataclass @@ -127,9 +175,14 @@ def parse_snapshots( companyfacts: dict[str, Any], filings: dict[str, FilingMeta], accessions: set[str], + fiscal_year_end: str | None = None, ) -> ParseResult: """Build snapshot rows for ``accessions`` (those with facts + filing meta). + ``fiscal_year_end`` is the issuer's ``submissions.fiscalYearEnd`` (MMDD) and + is what makes period identity independent of SEC's unreliable fy/fp fields + (see ``_period_identity``). Omitting it falls back to the old fy/fp behaviour. + ``skipped_filings`` = no row produced (missing facts/meta or no usable period identity); ``field_issues`` = a row was produced but a field is null/ambiguous. Callers must not use field issues as failed-row coverage. @@ -143,7 +196,7 @@ def parse_snapshots( if meta is None or not facts: result.skipped_filings.append({"accession": accn, "reason": "no facts or filing metadata"}) continue - row, note = _parse_one(cik, accn, facts, meta) + row, note = _parse_one(cik, accn, facts, meta, fiscal_year_end) if row is None: result.skipped_filings.append({"accession": accn, "reason": note or "unparseable"}) continue @@ -190,12 +243,18 @@ def _index_by_accession(companyfacts: dict[str, Any]) -> dict[str, list[Fact]]: def _parse_one( - cik: str, accn: str, facts: list[Fact], meta: FilingMeta + cik: str, accn: str, facts: list[Fact], meta: FilingMeta, + fiscal_year_end: str | None = None, ) -> tuple[SnapshotRow | None, str | None]: """Returns (row, note). row is None when there's no usable period identity; note is a validation reason (row-skip reason when row is None, else a field-level issue such as ambiguous shares).""" - fy, fp = _fiscal_context(facts, meta.report_date) + fy, fp = _period_identity(meta, fiscal_year_end) + if fy is None or fp is None: + # No fiscal calendar, or a period the calendar cannot place (a transition + # period). Fall back to the filing's own context: an imperfect label still + # beats dropping the filing entirely. + fy, fp = _fiscal_context(facts, meta.report_date) if fy is None or fp not in _EXPECTED_YTD_DAYS: return None, "no usable period identity" @@ -227,9 +286,80 @@ def _parse_one( shares, shares_date, ambiguous = _select_shares(facts, meta.report_date) row.shares_outstanding = shares row.shares_outstanding_date = shares_date + row.weighted_avg_diluted_shares = _select_weighted_avg_shares(facts, meta.report_date) + return row, ("ambiguous shares outstanding" if ambiguous else None) +def _period_identity( + meta: FilingMeta, fiscal_year_end: str | None +) -> tuple[int | None, str | None]: + """(fiscal_year, fiscal_period) from the period end and the issuer's fiscal + calendar — never from the fy/fp fields. + + SEC's fy/fp describe the *filing*, and they are unreliable as period identity: + observed in production, a 10-Q labelled ``FY`` (BXP), a year ending 2025-12-31 + labelled 2024 (FRT, a December filer), a year ending 2025-06-27 labelled 2027 + (STX), and four different period ends all labelled 2022 Q3 (PPL). Because + readers key on (fiscal_year, fiscal_period), colliding labels silently discard + a period and inverted ones scramble the quarter chain — nulling TTM and YoY. + + ``period_end`` is authoritative, so identity is derived from it: the form + decides FY vs quarter, and distance to the fiscal-year end decides which + quarter. Labels need not match the issuer's own naming — a filer whose year + ends in early January (DPZ) shifts by one — they need to be unique, monotonic + and YoY-aligned, which is all the derivation asks of them. Nothing outside the + derivation reads these columns. + """ + fy = _fiscal_year_of(meta.report_date, fiscal_year_end) + if fy is None: + return None, None + if meta.form.startswith("10-K"): + return fy, "FY" + nominal_end = _nominal_fy_end(fy, fiscal_year_end) + if nominal_end is None: + return None, None + remaining = (nominal_end - meta.report_date).days + best = min( + _QUARTER_DAYS_TO_FY_END, + key=lambda k: abs(_QUARTER_DAYS_TO_FY_END[k] - remaining), + ) + if abs(_QUARTER_DAYS_TO_FY_END[best] - remaining) > _QUARTER_TOLERANCE_DAYS: + return None, None # transition period or odd filing — let the caller fall back + return fy, best + + +def _nominal_fy_end(year: int, fiscal_year_end: str | None) -> date | None: + """The issuer's nominal fiscal-year end in ``year`` from a MMDD string.""" + if not fiscal_year_end or len(fiscal_year_end) != 4 or not fiscal_year_end.isdigit(): + return None + month, day = int(fiscal_year_end[:2]), int(fiscal_year_end[2:]) + if not 1 <= month <= 12 or not 1 <= day <= 31: + return None + while day > 28: # 52/53-week ends land on 0229/0230/0231 in some filings + try: + return date(year, month, day) + except ValueError: + day -= 1 + return date(year, month, day) + + +def _fiscal_year_of(period_end: date, fiscal_year_end: str | None) -> int | None: + """Which fiscal year ``period_end`` belongs to. + + A 52/53-week calendar's real year end drifts around the nominal MMDD (and can + cross the calendar year), so allow drift before rolling into the next year. + """ + nominal = _nominal_fy_end(period_end.year, fiscal_year_end) + if nominal is None: + return None + return ( + period_end.year + if period_end <= nominal + timedelta(days=_FYE_DRIFT_TOLERANCE_DAYS) + else period_end.year + 1 + ) + + def _fiscal_context(facts: list[Fact], report_date: date) -> tuple[int | None, str | None]: """The filing's (fy, fp) taken as the majority context among the facts that end at reportDate (the current-period facts, which share the filing's @@ -352,6 +482,34 @@ def _select_shares( return None, None, False # simply absent — not a conflict +def _select_weighted_avg_shares(facts: list[Fact], report_date: date) -> float | None: + """The most recent quarter's weighted-average diluted share count. + + Deliberately the **shortest** duration ending at reportDate, not the YTD one: + the shorter the window the closer the average sits to the current count, which + is what a market cap wants. Measured against issuers where the true + point-in-time count is available, the quarter average is within ~0.6%. + """ + best: tuple[int, float] | None = None + for concept in _WEIGHTED_AVG_SHARE_CONCEPTS: + for f in facts: + if ( + f.taxonomy != "us-gaap" + or f.concept != concept + or f.unit != "shares" + or f.start is None + or f.end != report_date + or f.val <= 0 + ): + continue + span = (f.end - f.start).days + if best is None or span < best[0]: + best = (span, float(f.val)) + if best is not None: + return best[1] # first present concept wins, as elsewhere + return None + + def _d(value: Any) -> date | None: if not value: return None diff --git a/tests/unit/test_fundamentals_derivation.py b/tests/unit/test_fundamentals_derivation.py index d77a0c2..95e107d 100644 --- a/tests/unit/test_fundamentals_derivation.py +++ b/tests/unit/test_fundamentals_derivation.py @@ -29,6 +29,7 @@ class Snap: cash_and_st_investments: float | None = None total_debt: float | None = None shares_outstanding: float | None = None + weighted_avg_diluted_shares: float | None = None _FP = ["Q1", "Q2", "Q3", "FY"] @@ -188,3 +189,112 @@ def test_amendment_selection_newest_accepted_wins(): # Q4 revenue discrete now uses the amended YTD(FY)=999999 minus YTD(Q3)=363 # so TTM/growth reflects the amendment, proving newest accepted_at won. assert d.metrics["revenue_growth_yoy"].value != pytest.approx(10.0, abs=1e-6) + + +# -- partial amendments (A5 parity findings) --------------------------------- + +def test_partial_amendment_does_not_blank_the_period(): + # DVN's FY2025 10-K/A carries no financial facts at the report date. Taking + # the newest accession wholesale nulled the period, and with it the quarter + # chain, TTM and YoY. + rows = _two_years() + part_iii_only = Snap(2026, "FY", date(2026, 9, 30), date(2026, 11, 1), + datetime(2027, 1, 1, tzinfo=UTC)) + baseline = fd.derive(rows) + d = fd.derive(rows + [part_iii_only]) + assert d.ttm_diluted_eps == pytest.approx(baseline.ttm_diluted_eps) + assert d.metrics["revenue_growth_yoy"].value == pytest.approx( + baseline.metrics["revenue_growth_yoy"].value + ) + + +def test_amendment_restating_one_field_leaves_the_others_intact(): + rows = _two_years() + revenue_only = Snap(2026, "FY", date(2026, 9, 30), date(2026, 11, 1), + datetime(2027, 1, 1, tzinfo=UTC), revenue=999999) + baseline = fd.derive(rows) + d = fd.derive(rows + [revenue_only]) + assert d.metrics["revenue_growth_yoy"].value != pytest.approx( + baseline.metrics["revenue_growth_yoy"].value + ) + assert d.ttm_diluted_eps == pytest.approx(baseline.ttm_diluted_eps) # fell back + + +def test_same_key_row_for_a_different_period_is_never_merged(): + # SEC labels two different year-ends with one fiscal_year for some filers + # (FRT, CRM). That is a mislabelled filing, not an amendment -- merging the + # two would silently blend fiscal years. + rows = _two_years() + mislabelled = Snap(2026, "FY", date(2027, 9, 30), date(2027, 11, 1), + datetime(2027, 12, 1, tzinfo=UTC), revenue=999999) + selected = fd._select_latest_per_period(rows + [mislabelled]) + assert selected[(2026, "FY")] is mislabelled + + +# -- split safety for the TTM EPS scalar (A5 parity findings) ---------------- + +def _split_rows(): + """Two years where the share count jumps ~25x at the latest quarter, as + BKNG's did (31.7M -> 774.9M) when its split landed mid-window.""" + rows = _two_years() + for row in rows: + if (row.fiscal_year, row.fiscal_period) == (2026, "FY"): + row.shares_outstanding = 25000.0 # vs 1000 a year earlier + return rows + + +def test_split_suppresses_ttm_diluted_eps(): + # TTM sums four quarters of per-share values; a split inside the window + # mixes units. Unguarded this produced BKNG's P/E of 1.10, which clamps to a + # *perfect* fundamental sub-score -- worse than having no value at all. + d = fd.derive(_split_rows()) + assert d.ttm_diluted_eps is None + assert d.ttm_diluted_eps_caveat == fd.SPLIT_SENSITIVE_CAVEAT + + +def test_ttm_diluted_eps_survives_when_no_split_is_suspected(): + d = fd.derive(_two_years()) + assert d.ttm_diluted_eps is not None + assert d.ttm_diluted_eps_caveat is None + + +def test_split_guard_leaves_dollar_scalars_alone(): + # Only per-share values are split-sensitive; FCF is in dollars. + baseline = fd.derive(_two_years()) + d = fd.derive(_split_rows()) + assert d.ttm_fcf == pytest.approx(baseline.ttm_fcf) + + +# -- multi-class share-count fallback (A5 parity findings) ------------------- + +def test_shares_fall_back_to_weighted_average_when_cover_page_count_is_absent(): + # META/CMCSA/BRK-B/CHTR report the cover-page count per share class, which is + # dimensional and therefore absent from companyfacts -- silently removing + # market cap and FCF yield for some of the largest issuers. + rows = _two_years() + for row in rows: + row.shares_outstanding = None + row.weighted_avg_diluted_shares = 2_564_000_000.0 + d = fd.derive(rows) + assert d.shares_outstanding == 2_564_000_000.0 + assert d.shares_outstanding_estimated is True + + +def test_point_in_time_share_count_is_preferred_and_not_flagged(): + baseline = fd.derive(_two_years()).shares_outstanding + assert baseline is not None, "fixture should carry a cover-page count" + rows = _two_years() + for row in rows: + row.weighted_avg_diluted_shares = 1.0 # must lose to the real count + d = fd.derive(rows) + assert d.shares_outstanding == baseline + assert d.shares_outstanding_estimated is False + + +def test_no_share_count_at_all_stays_none_and_unflagged(): + rows = _two_years() + for row in rows: + row.shares_outstanding = None + d = fd.derive(rows) + assert d.shares_outstanding is None + assert d.shares_outstanding_estimated is False diff --git a/tests/unit/test_sec_facts_parser.py b/tests/unit/test_sec_facts_parser.py index 90c4df2..c398559 100644 --- a/tests/unit/test_sec_facts_parser.py +++ b/tests/unit/test_sec_facts_parser.py @@ -270,3 +270,196 @@ async def test_live_apple_parse_invariants(): # shares cover-date differs from period_end latest = max(rows, key=lambda r: r.period_end) assert latest.shares_outstanding_date != latest.period_end + + +# -- revenue concept coverage (A5 parity findings) --------------------------- + +def _one_filing(concepts: dict, *, start: str, end: str, fp: str): + """A single 10-Q whose facts are the given {concept: value} at one YTD span.""" + facts = { + name: {"units": {"USD": [_dur(start, end, val, "X", fp=fp)]}} + for name, val in concepts.items() + } + companyfacts = {"cik": 19617, "facts": {"us-gaap": facts}} + filings = { + "X": FilingMeta( + date.fromisoformat(end), date(2026, 5, 1), datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q" + ) + } + return parse_snapshots(companyfacts, filings, {"X"}) + + +def test_revenue_reads_banks_total_revenue_tag(): + # JPM/GS/WFC tag RevenuesNetOfInterestExpense in every 10-Q and never (or + # only annually) `Revenues` -- previously null, so revenue growth was too. + res = _one_filing( + {"RevenuesNetOfInterestExpense": 49836}, start="2026-01-01", end="2026-03-31", fp="Q1" + ) + assert res.rows[0].revenue == 49836 + + +def test_revenue_reads_including_assessed_tax_variant(): + # ARE/KHC tag only the Including variant. + res = _one_filing( + {"RevenueFromContractWithCustomerIncludingAssessedTax": 671}, + start="2026-01-01", end="2026-03-31", fp="Q1", + ) + assert res.rows[0].revenue == 671 + + +def test_revenue_concept_priority_is_unchanged_by_the_added_tags(): + # The new entries are appended, so any issuer that already resolved keeps + # the same concept -- only issuers that resolved to nothing gain a value. + res = _one_filing( + { + "RevenueFromContractWithCustomerExcludingAssessedTax": 100, + "RevenueFromContractWithCustomerIncludingAssessedTax": 110, + "RevenuesNetOfInterestExpense": 120, + "Revenues": 130, + }, + start="2026-01-01", end="2026-03-31", fp="Q1", + ) + assert res.rows[0].revenue == 100 + + +def test_four_four_five_q3_ytd_span_is_accepted(): + # A 12/12/12/16-week filer's YTD-Q3 is 36 weeks = 251 days (COST 2026 Q3), + # which missed the old 20-day tolerance around 273 by ~2 and dropped Q3 + # every year -- breaking the quarter chain and nulling TTM and YoY. + res = _one_filing( + {"RevenueFromContractWithCustomerExcludingAssessedTax": 207431}, + start="2025-09-01", end="2026-05-10", fp="Q3", + ) + assert (date(2026, 5, 10) - date(2025, 9, 1)).days == 251 + assert res.rows[0].revenue == 207431 + + +def test_eps_falls_back_to_continuing_operations_variant(): + # REG tags only this variant on every filing; FCX tags it in its 10-K while + # using EarningsPerShareDiluted in its 10-Qs. + companyfacts = { + "cik": 910606, + "facts": {"us-gaap": {"IncomeLossFromContinuingOperationsPerDilutedShare": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.81, "X", fp="Q1")]} + }}}, + } + filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1), + datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")} + res = parse_snapshots(companyfacts, filings, {"X"}) + assert res.rows[0].diluted_eps == 1.81 + + +def test_eps_concept_priority_is_unchanged_by_the_added_tag(): + companyfacts = { + "cik": 831259, + "facts": {"us-gaap": { + "EarningsPerShareDiluted": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.61, "X", fp="Q1")]}}, + "IncomeLossFromContinuingOperationsPerDilutedShare": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.75, "X", fp="Q1")]}}, + }}, + } + filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1), + datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")} + res = parse_snapshots(companyfacts, filings, {"X"}) + assert res.rows[0].diluted_eps == 0.61 + + +# -- period identity from the fiscal calendar, not SEC's fy/fp --------------- + +from app.services.sec_facts_parser import _period_identity # noqa: E402 + + +def _meta(end: str, form: str = "10-Q") -> FilingMeta: + d = date.fromisoformat(end) + return FilingMeta(d, d, datetime(d.year, d.month, d.day, tzinfo=UTC), form) + + +def test_a_10q_is_never_labelled_fy(): + # BXP: a 10-Q for period end 2026-03-31 carried fy/fp saying "2026 FY", which + # collided with the real annual row and measured a 90-day fact against the + # 365-day FY expectation. + fy, fp = _period_identity(_meta("2026-03-31"), "1231") + assert (fy, fp) == (2026, "Q1") + + +def test_december_filer_years_do_not_collide(): + # FRT: two 10-Ks, ending 2024-12-31 and 2025-12-31, both labelled "2024 FY". + assert _period_identity(_meta("2024-12-31", "10-K"), "1231") == (2024, "FY") + assert _period_identity(_meta("2025-12-31", "10-K"), "1231") == (2025, "FY") + + +def test_january_year_end_groups_its_quarters(): + # CRM/CRWD/WDAY: the year ending 2026-01-31 and its own quarters must share a + # fiscal year, and must not collide with the year ending 2025-01-31. + assert _period_identity(_meta("2026-01-31", "10-K"), "0131") == (2026, "FY") + assert _period_identity(_meta("2025-01-31", "10-K"), "0131") == (2025, "FY") + assert _period_identity(_meta("2025-04-30"), "0131") == (2026, "Q1") + assert _period_identity(_meta("2025-07-31"), "0131") == (2026, "Q2") + assert _period_identity(_meta("2025-10-31"), "0131") == (2026, "Q3") + + +def test_mid_year_end_orders_correctly(): + # STX: the year ending 2025-06-27 was labelled "2027 FY" and sorted after + # quarters that precede it. + assert _period_identity(_meta("2025-06-27", "10-K"), "0627") == (2025, "FY") + assert _period_identity(_meta("2025-10-03"), "0627") == (2026, "Q1") + assert _period_identity(_meta("2026-01-02"), "0627") == (2026, "Q2") + assert _period_identity(_meta("2026-04-03"), "0627") == (2026, "Q3") + + +def test_four_four_five_quarters_place_correctly(): + # COST: a 12/12/12/16-week year leaves Q3 112 days from the year end, not 91. + assert _period_identity(_meta("2025-11-23"), "0830") == (2026, "Q1") + assert _period_identity(_meta("2026-02-15"), "0830") == (2026, "Q2") + assert _period_identity(_meta("2026-05-10"), "0830") == (2026, "Q3") + assert _period_identity(_meta("2026-08-30", "10-K"), "0830") == (2026, "FY") + + +def test_year_end_crossing_january_still_groups_one_year(): + # DPZ (fiscalYearEnd 0102): the label shifts by one against Domino's own + # naming, which is fine -- a year and its quarters must simply agree. + year, _ = _period_identity(_meta("2025-12-28", "10-K"), "0102") + assert (year, "FY") == _period_identity(_meta("2025-12-28", "10-K"), "0102") + assert _period_identity(_meta("2025-03-23"), "0102") == (year, "Q1") + assert _period_identity(_meta("2025-06-15"), "0102") == (year, "Q2") + assert _period_identity(_meta("2025-09-07"), "0102") == (year, "Q3") + + +def test_missing_fiscal_calendar_falls_back_to_filing_context(): + assert _period_identity(_meta("2026-03-31"), None) == (None, None) + # ...and parse_snapshots then uses the fy/fp path, preserving old behaviour. + res = parse_snapshots(COMPANYFACTS, FILINGS, {"B"}) + assert (res.rows[0].fiscal_year, res.rows[0].fiscal_period) == (2026, "Q2") + + + +def test_eps_falls_back_to_basic_only_when_no_diluted_variant_exists(): + # PPL's 2026 Q1 tags no diluted EPS at all, only basic -- one missing period + # broke the quarter chain and nulled TTM. + companyfacts = { + "cik": 922224, + "facts": {"us-gaap": {"EarningsPerShareBasic": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 0.60, "X", fp="Q1")]} + }}}, + } + filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1), + datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")} + res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231") + assert res.rows[0].diluted_eps == 0.60 + + +def test_diluted_still_wins_over_basic_when_both_present(): + companyfacts = { + "cik": 320193, + "facts": {"us-gaap": { + "EarningsPerShareDiluted": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.36, "X", fp="Q1")]}}, + "EarningsPerShareBasic": { + "units": {"USD/shares": [_dur("2026-01-01", "2026-03-31", 1.40, "X", fp="Q1")]}}, + }}, + } + filings = {"X": FilingMeta(date(2026, 3, 31), date(2026, 5, 1), + datetime(2026, 5, 1, 10, tzinfo=UTC), "10-Q")} + res = parse_snapshots(companyfacts, filings, {"X"}, fiscal_year_end="1231") + assert res.rows[0].diluted_eps == 1.36