dennisthiessenandClaude Opus 5
8453b87290
total_debt read LongTermDebt, else LongTermDebtNoncurrent/Current, plus one of ShortTermBorrowings/CommercialPaper. That misses two whole tagging styles, and it feeds net_debt -> net_debt_to_ebitda -> the categorical leverage read, so the misses were not absences but confident wrong answers: Coca-Cola scored on 0.25bn of commercial paper against ~39bn of debt, Verizon on 21.78bn of current maturities against ~165bn, AT&T and Exxon produced no value at all against 134bn and 33bn tagged. Measured over 19 large caps and 14 REITs, 11 were wrong or absent and the rest are unchanged. Each concept's span is now respected. LongTermDebt already includes current maturities (Apple tags all three: 71.34 + 11.01 = 82.30), so only true short-term borrowing is added. LongTermDebtAndCapitalLeaseObligations — what KO, HD, T, XOM and CVX tag, and nothing read before — is noncurrent and takes a current complement, and DebtCurrent *is* that whole complement rather than an addition to it. The REIT branch needed disambiguating: NotesPayable is not the same line across issuers. MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn exactly, so there it is the total and adding the secured side double-counts; EQR tags it alongside a larger SecuredDebt, where it is only the unsecured component. UnsecuredDebt's presence separates them. A component alone is no longer reported as a total. Chevron tags full debt only in its 10-K, so its 10-Q carried 0.40bn of short-term borrowing; Boston Properties tags SecuredDebt 4.28bn against ~15bn real. net_debt needs both sides and yields nothing when either is missing, so None costs a leverage read where the fragment produced a confidently wrong one. Snapshots are immutable, so this corrects new filings only; stored history needs scripts/reparse_fundamentals.py, which cannot complete until the EQR/931182 collision is retired. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016Lo99z3jWqu9X9ueBU3Z3D
611 lines
26 KiB
Python
611 lines
26 KiB
Python
"""Pure parser: SEC companyfacts -> fundamental_snapshots rows.
|
|
|
|
Turns one issuer's `companyfacts` JSON (+ its submissions filing metadata) into
|
|
per-accession snapshot rows for the filing's **primary period**, following the
|
|
A3 design (docs/dolt-sec-a3-design.md). No I/O, no DB — unit-testable against a
|
|
fixture and verifiable against a real companyfacts pull.
|
|
|
|
The load-bearing rules (design Decision 2 + review):
|
|
- Period identity comes from `end == submissions.reportDate`, never `fy/fp`
|
|
(fy/fp is the *filing's* context; comparatives inside a filing repeat it).
|
|
This applies to the stored `fiscal_year`/`fiscal_period` too: they are derived
|
|
from `reportDate` against the issuer's `fiscalYearEnd` (see `_period_identity`),
|
|
because SEC's fy/fp collide and invert often enough to break the quarter chain.
|
|
- Duration facts are stored as **cumulative YTD**: pick the fact whose span
|
|
matches the fiscal-period-to-date length (Q1≈3mo … FY≈12mo) within tolerance.
|
|
If no YTD-length fact exists, store null — never a discrete masquerading as YTD.
|
|
- Balance-sheet instants are taken at `end == reportDate`. `shares_outstanding`
|
|
is a single consolidated value: the cover-page `dei` fact (its own cover-date
|
|
`end` stored separately) if present, else `us-gaap:CommonStockSharesOutstanding`
|
|
at period end (e.g. Alphabet has no `dei` fact) — never a class sum or the
|
|
weighted-average/diluted count. Multi-class issuers report it per class, which
|
|
is dimensional and therefore absent from companyfacts entirely, so
|
|
`weighted_avg_diluted_shares` is stored alongside as an explicit fallback for
|
|
market cap — a separate column, never backfilled into `shares_outstanding`.
|
|
- Cash and debt composites are aggregate-first and mutually exclusive (each
|
|
source tag counted at most once).
|
|
|
|
`parse_snapshots` separates `skipped_filings` (no usable row produced) from
|
|
`field_issues` (a row was produced but a field is null/ambiguous) — callers must
|
|
not treat field issues as missing coverage.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
from dataclasses import dataclass, field
|
|
from datetime import date, datetime, timedelta
|
|
from typing import Any, NamedTuple
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Expected YTD span (days) per fiscal period; a duration fact must land within
|
|
# tolerance of this to count as the period's cumulative value.
|
|
_EXPECTED_YTD_DAYS = {"Q1": 91, "Q2": 182, "Q3": 273, "FY": 365}
|
|
# Period identity (see _period_identity): how far a quarter end sits before its
|
|
# fiscal-year end, and how far a fiscal-year end may drift from the nominal MMDD.
|
|
# The quarter bands are 91 days apart, so ±35 stays unambiguous even for a 4-4-5
|
|
# filer whose 16-week Q4 puts Q3 112 days out.
|
|
_QUARTER_DAYS_TO_FY_END = {"Q1": 273, "Q2": 182, "Q3": 91}
|
|
_QUARTER_TOLERANCE_DAYS = 35
|
|
_FYE_DRIFT_TOLERANCE_DAYS = 21
|
|
# Covers 52/53-week calendars *and* 4-4-5 retail ones (12/12/12/16 weeks), whose
|
|
# YTD-Q3 is 36 weeks = 251-252 days and missed a 20-day tolerance by ~2 -- so
|
|
# COST/PEP lost Q3 every year, breaking the quarter chain and nulling TTM + YoY.
|
|
# Q1 84d, Q2 168d and FY 364d were always inside. Adjacent periods stay
|
|
# unambiguous at 25 (66-116, 157-207, 248-298, 340-390).
|
|
_YTD_TOLERANCE_DAYS = 25
|
|
|
|
# us-gaap duration concepts (money), priority order; first present wins.
|
|
_DURATION_USD = {
|
|
# Order is load-bearing (first present wins) and the tail entries are
|
|
# deliberately *appended*: every issuer that already resolved keeps the same
|
|
# concept, and only issuers that resolved to nothing gain a value.
|
|
# - IncludingAssessedTax: REITs/consumer filers that tag only this variant
|
|
# (e.g. ARE, KHC) reported no revenue at all.
|
|
# - RevenuesNetOfInterestExpense: the banks' total-revenue tag. JPM/GS/WFC
|
|
# tag it in every 10-Q and `Revenues` only (if at all) in the 10-K.
|
|
"revenue": [
|
|
"RevenueFromContractWithCustomerExcludingAssessedTax",
|
|
"Revenues",
|
|
"SalesRevenueNet",
|
|
"RevenueFromContractWithCustomerIncludingAssessedTax",
|
|
"RevenuesNetOfInterestExpense",
|
|
],
|
|
"net_income": ["NetIncomeLoss"],
|
|
"operating_income": ["OperatingIncomeLoss"],
|
|
"cfo": [
|
|
"NetCashProvidedByUsedInOperatingActivities",
|
|
"NetCashProvidedByUsedInOperatingActivitiesContinuingOperations",
|
|
],
|
|
"capex": [
|
|
"PaymentsToAcquirePropertyPlantAndEquipment",
|
|
"PaymentsToAcquireProductiveAssets",
|
|
],
|
|
"depreciation_amortization": [
|
|
"DepreciationDepletionAndAmortization",
|
|
"DepreciationAmortizationAndAccretionNet",
|
|
"DepreciationAndAmortization",
|
|
],
|
|
}
|
|
# unit USD/shares. Appended (not reordered) so any issuer that already resolved
|
|
# keeps the same concept. REG tags only the continuing-operations variant on every
|
|
# filing; FCX switches by form type -- EarningsPerShareDiluted in its 10-Qs, the
|
|
# continuing-ops tag in its 10-K -- which nulled the FY row and killed Q4 + TTM.
|
|
# The basic variants are a last resort for a period that tags no diluted EPS at
|
|
# all (PPL's 2026 Q1). Basic ignores option/convert dilution so it slightly
|
|
# overstates EPS (~1.2% for PPL), but only fires when diluted is entirely absent,
|
|
# and high-dilution names always tag diluted -- so it never displaces a real one.
|
|
_EPS_CONCEPTS = [
|
|
"EarningsPerShareDiluted",
|
|
"IncomeLossFromContinuingOperationsPerDilutedShare",
|
|
"EarningsPerShareBasic",
|
|
"IncomeLossFromContinuingOperationsPerBasicShare",
|
|
]
|
|
# Weighted-average diluted share count (unit "shares"), the market-cap fallback
|
|
# for multi-class issuers whose cover-page count is dimensional and therefore
|
|
# absent from companyfacts. Always present, since EPS is computed from it.
|
|
_WEIGHTED_AVG_SHARE_CONCEPTS = [
|
|
"WeightedAverageNumberOfDilutedSharesOutstanding",
|
|
"WeightedAverageNumberOfSharesOutstandingBasicAndDiluted",
|
|
]
|
|
# us-gaap instant (balance-sheet) concepts, at end == reportDate.
|
|
_CASH = ["CashAndCashEquivalentsAtCarryingValue"]
|
|
_ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one
|
|
# Debt is tagged in four mutually exclusive styles across large filers, and
|
|
# composing a total means knowing which span each concept covers (measured
|
|
# 2026-08 over a 20-issuer sample; the counts below are from it).
|
|
#
|
|
# ``LongTermDebt`` already spans current + noncurrent maturities — Apple tags all
|
|
# three and 71.34bn + 11.01bn = 82.30bn confirms it — so its complement is only
|
|
# genuinely short-term borrowing.
|
|
_LONG_TERM_DEBT_AGG = ["LongTermDebt"]
|
|
# Noncurrent-only balance-sheet lines, needing a current complement added.
|
|
# ``LongTermDebtAndCapitalLeaseObligations`` is what KO, HD, T, XOM and CVX tag
|
|
# and nothing read it before: AT&T reported no total_debt at all against 134bn
|
|
# tagged, and Coca-Cola reported 0.25bn of commercial paper against 39bn.
|
|
_LONG_TERM_DEBT_NONCURRENT = [
|
|
"LongTermDebtNoncurrent",
|
|
"LongTermDebtAndCapitalLeaseObligations",
|
|
]
|
|
_LONG_TERM_DEBT_CURRENT = ["LongTermDebtCurrent"]
|
|
# REITs that tag no aggregate at all, carrying a secured and an unsecured side
|
|
# instead. Both sides are required, because ``NotesPayable`` does not mean the
|
|
# same thing across issuers (measured 2026-08 over 14 REITs):
|
|
# - MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn
|
|
# exactly, so there it IS the total and adding SecuredDebt double-counts.
|
|
# - EQR/VMRK tags NotesPayable alongside a *larger* SecuredDebt (5.38bn vs
|
|
# 6.38bn in 2013), so there it is only the unsecured component.
|
|
# ``UnsecuredDebt`` is what separates them: where it is tagged it is the
|
|
# unambiguous unsecured side and NotesPayable is ignored; where it is absent,
|
|
# NotesPayable is that side. Requiring both sides is also what keeps this branch
|
|
# from inventing a total out of a fragment — Boston Properties tags SecuredDebt
|
|
# 4.28bn and nothing else against ~15bn of real debt, and Regency tags an
|
|
# UnsecuredDebt of 0.03bn that is a credit-line draw, not its 5bn of notes.
|
|
_SECURED_DEBT = ["SecuredDebt"]
|
|
_UNSECURED_DEBT = ["UnsecuredDebt", "NotesPayable"] # first present wins
|
|
# ``DebtCurrent`` spans short-term borrowing AND current maturities, so it is the
|
|
# whole current complement where present and must never be added alongside them.
|
|
_ALL_CURRENT_DEBT = ["DebtCurrent"]
|
|
_SHORT_TERM_DEBT = ["ShortTermBorrowings", "CommercialPaper"] # pick one
|
|
|
|
|
|
class Fact(NamedTuple):
|
|
taxonomy: str
|
|
concept: str
|
|
unit: str
|
|
start: date | None # None => instant
|
|
end: date
|
|
val: float
|
|
fy: int | None
|
|
fp: str | None
|
|
|
|
|
|
@dataclass
|
|
class SnapshotRow:
|
|
cik: str
|
|
accession: str
|
|
form: str
|
|
filed_date: date
|
|
accepted_at: datetime
|
|
period_end: date
|
|
fiscal_year: int
|
|
fiscal_period: str
|
|
period_start: date | None = None
|
|
revenue: float | None = None
|
|
net_income: float | None = None
|
|
operating_income: float | None = None
|
|
diluted_eps: float | None = None
|
|
cfo: float | None = None
|
|
capex: float | None = None
|
|
depreciation_amortization: float | None = None
|
|
cash_and_st_investments: float | None = None
|
|
total_debt: float | None = None
|
|
shares_outstanding: float | None = None
|
|
shares_outstanding_date: date | None = None
|
|
weighted_avg_diluted_shares: float | None = None
|
|
|
|
|
|
@dataclass
|
|
class FilingMeta:
|
|
report_date: date
|
|
filing_date: date
|
|
accepted_at: datetime
|
|
form: str
|
|
|
|
|
|
@dataclass
|
|
class ParseResult:
|
|
rows: list[SnapshotRow] = field(default_factory=list)
|
|
# accessions for which NO row was produced (no facts / no usable period).
|
|
skipped_filings: list[dict[str, str]] = field(default_factory=list)
|
|
# accessions with a row but a field-level warning (e.g. ambiguous shares).
|
|
field_issues: list[dict[str, str]] = field(default_factory=list)
|
|
|
|
|
|
def parse_snapshots(
|
|
companyfacts: dict[str, Any],
|
|
filings: dict[str, FilingMeta],
|
|
accessions: set[str],
|
|
fiscal_year_end: str | None = None,
|
|
) -> ParseResult:
|
|
"""Build snapshot rows for ``accessions`` (those with facts + filing meta).
|
|
|
|
``fiscal_year_end`` is the issuer's declared ``submissions.fiscalYearEnd``
|
|
(MMDD) and seeds period identity (see ``_period_identity``), making it
|
|
independent of SEC's unreliable fy/fp fields. It is only a hint: the issuer's
|
|
own 10-K period ends override it (see ``resolve_fiscal_year_end``). With
|
|
neither available, the old fy/fp behaviour is used.
|
|
|
|
``skipped_filings`` = no row produced (missing facts/meta or no usable period
|
|
identity); ``field_issues`` = a row was produced but a field is null/ambiguous.
|
|
Callers must not use field issues as failed-row coverage.
|
|
"""
|
|
cik = f"{int(companyfacts['cik']):010d}"
|
|
# The declared value is only a hint; the issuer's own 10-Ks are authoritative.
|
|
fiscal_year_end = resolve_fiscal_year_end(filings, fiscal_year_end)
|
|
by_accn = _index_by_accession(companyfacts)
|
|
result = ParseResult()
|
|
for accn in accessions:
|
|
meta = filings.get(accn)
|
|
facts = by_accn.get(accn)
|
|
if meta is None or not facts:
|
|
result.skipped_filings.append({"accession": accn, "reason": "no facts or filing metadata"})
|
|
continue
|
|
row, note = _parse_one(cik, accn, facts, meta, fiscal_year_end)
|
|
if row is None:
|
|
result.skipped_filings.append({"accession": accn, "reason": note or "unparseable"})
|
|
continue
|
|
result.rows.append(row)
|
|
if note:
|
|
result.field_issues.append({"accession": accn, "reason": note})
|
|
return result
|
|
|
|
|
|
def companyfacts_accessions(companyfacts: dict[str, Any]) -> set[str]:
|
|
"""Every accession that appears anywhere in a companyfacts payload — used by
|
|
the importer's index↔Company-Facts consistency gate."""
|
|
return set(_index_by_accession(companyfacts).keys())
|
|
|
|
|
|
def _index_by_accession(companyfacts: dict[str, Any]) -> dict[str, list[Fact]]:
|
|
"""One pass over companyfacts -> {accession: [Fact, ...]}."""
|
|
out: dict[str, list[Fact]] = {}
|
|
for taxonomy, concepts in companyfacts.get("facts", {}).items():
|
|
for concept, body in concepts.items():
|
|
for unit, facts in body.get("units", {}).items():
|
|
for f in facts:
|
|
accn = f.get("accn")
|
|
end = _d(f.get("end"))
|
|
val = f.get("val")
|
|
# Skip malformed facts so they can't be selected accidentally:
|
|
# every usable fact needs an accession, an end date, and a
|
|
# finite numeric value.
|
|
if not accn or end is None or not _finite(val):
|
|
continue
|
|
out.setdefault(accn, []).append(
|
|
Fact(
|
|
taxonomy=taxonomy,
|
|
concept=concept,
|
|
unit=unit,
|
|
start=_d(f.get("start")),
|
|
end=end,
|
|
val=val,
|
|
fy=f.get("fy"),
|
|
fp=f.get("fp"),
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
def _parse_one(
|
|
cik: str, accn: str, facts: list[Fact], meta: FilingMeta,
|
|
fiscal_year_end: str | None = None,
|
|
) -> tuple[SnapshotRow | None, str | None]:
|
|
"""Returns (row, note). row is None when there's no usable period identity;
|
|
note is a validation reason (row-skip reason when row is None, else a
|
|
field-level issue such as ambiguous shares)."""
|
|
fy, fp = _period_identity(meta, fiscal_year_end)
|
|
if fy is None or fp is None:
|
|
# No fiscal calendar, or a period the calendar cannot place (a transition
|
|
# period). Fall back to the filing's own context: an imperfect label still
|
|
# beats dropping the filing entirely.
|
|
fy, fp = _fiscal_context(facts, meta.report_date)
|
|
if fy is None or fp not in _EXPECTED_YTD_DAYS:
|
|
return None, "no usable period identity"
|
|
|
|
row = SnapshotRow(
|
|
cik=cik,
|
|
accession=accn,
|
|
form=meta.form,
|
|
filed_date=meta.filing_date,
|
|
accepted_at=meta.accepted_at,
|
|
period_end=meta.report_date,
|
|
fiscal_year=fy,
|
|
fiscal_period=fp,
|
|
)
|
|
|
|
# duration YTD facts (money) + EPS
|
|
for field_name, concepts in _DURATION_USD.items():
|
|
val, start = _select_ytd(facts, concepts, meta.report_date, fp, "USD")
|
|
setattr(row, field_name, val)
|
|
if field_name == "revenue" and start is not None:
|
|
row.period_start = start
|
|
eps, eps_start = _select_ytd(facts, _EPS_CONCEPTS, meta.report_date, fp, "USD/shares")
|
|
row.diluted_eps = eps
|
|
if row.period_start is None and eps_start is not None:
|
|
row.period_start = eps_start
|
|
|
|
# balance-sheet instants at reportDate
|
|
row.cash_and_st_investments = _compose_cash(facts, meta.report_date)
|
|
row.total_debt = _compose_debt(facts, meta.report_date)
|
|
shares, shares_date, ambiguous = _select_shares(facts, meta.report_date)
|
|
row.shares_outstanding = shares
|
|
row.shares_outstanding_date = shares_date
|
|
row.weighted_avg_diluted_shares = _select_weighted_avg_shares(facts, meta.report_date)
|
|
|
|
return row, ("ambiguous shares outstanding" if ambiguous else None)
|
|
|
|
|
|
def resolve_fiscal_year_end(
|
|
filings: dict[str, FilingMeta], declared: str | None
|
|
) -> str | None:
|
|
"""The issuer's fiscal-year-end MMDD, preferring its own 10-K period ends.
|
|
|
|
``submissions.fiscalYearEnd`` is *not* reliable: Franklin Resources (BEN)
|
|
declares 1231 while every one of its 10-Ks ends 09-30. Trusting it put BEN's
|
|
fiscal Q1 (Dec) 0 days from the claimed year end — matching no quarter band —
|
|
and labelled its fiscal Q2 (Mar) as Q1, colliding two periods on one key and
|
|
destroying the quarter chain.
|
|
|
|
A 10-K's reportDate **is** the fiscal year end by definition, so it wins
|
|
whenever one is available; the declared value is only a fallback for an issuer
|
|
with no annual filing in the set. The most recent 10-K is used, so an issuer
|
|
that changed its year end is measured against its current calendar.
|
|
"""
|
|
annual = [m.report_date for m in filings.values() if m.form.startswith("10-K")]
|
|
if annual:
|
|
latest = max(annual)
|
|
return f"{latest.month:02d}{latest.day:02d}"
|
|
return declared
|
|
|
|
|
|
def _period_identity(
|
|
meta: FilingMeta, fiscal_year_end: str | None
|
|
) -> tuple[int | None, str | None]:
|
|
"""(fiscal_year, fiscal_period) from the period end and the issuer's fiscal
|
|
calendar — never from the fy/fp fields.
|
|
|
|
SEC's fy/fp describe the *filing*, and they are unreliable as period identity:
|
|
observed in production, a 10-Q labelled ``FY`` (BXP), a year ending 2025-12-31
|
|
labelled 2024 (FRT, a December filer), a year ending 2025-06-27 labelled 2027
|
|
(STX), and four different period ends all labelled 2022 Q3 (PPL). Because
|
|
readers key on (fiscal_year, fiscal_period), colliding labels silently discard
|
|
a period and inverted ones scramble the quarter chain — nulling TTM and YoY.
|
|
|
|
``period_end`` is authoritative, so identity is derived from it: the form
|
|
decides FY vs quarter, and distance to the fiscal-year end decides which
|
|
quarter. Labels need not match the issuer's own naming — a filer whose year
|
|
ends in early January (DPZ) shifts by one — they need to be unique, monotonic
|
|
and YoY-aligned, which is all the derivation asks of them. Nothing outside the
|
|
derivation reads these columns.
|
|
|
|
Known limitation: ``fiscalYearEnd`` is the issuer's *current* calendar, so a
|
|
company that has changed its fiscal year end gets its historical periods
|
|
measured against the new one. The quarter tolerance shunts most of those to
|
|
the fy/fp fallback, and a same-key collision resolves newest-wins, so the
|
|
failure mode is a degraded old year rather than a scrambled current one.
|
|
"""
|
|
fy = _fiscal_year_of(meta.report_date, fiscal_year_end)
|
|
if fy is None:
|
|
return None, None
|
|
if meta.form.startswith("10-K"):
|
|
return fy, "FY"
|
|
nominal_end = _nominal_fy_end(fy, fiscal_year_end)
|
|
if nominal_end is None:
|
|
return None, None
|
|
remaining = (nominal_end - meta.report_date).days
|
|
best = min(
|
|
_QUARTER_DAYS_TO_FY_END,
|
|
key=lambda k: abs(_QUARTER_DAYS_TO_FY_END[k] - remaining),
|
|
)
|
|
if abs(_QUARTER_DAYS_TO_FY_END[best] - remaining) > _QUARTER_TOLERANCE_DAYS:
|
|
return None, None # transition period or odd filing — let the caller fall back
|
|
return fy, best
|
|
|
|
|
|
def _nominal_fy_end(year: int, fiscal_year_end: str | None) -> date | None:
|
|
"""The issuer's nominal fiscal-year end in ``year`` from a MMDD string."""
|
|
if not fiscal_year_end or len(fiscal_year_end) != 4 or not fiscal_year_end.isdigit():
|
|
return None
|
|
month, day = int(fiscal_year_end[:2]), int(fiscal_year_end[2:])
|
|
if not 1 <= month <= 12 or not 1 <= day <= 31:
|
|
return None
|
|
while day > 28: # 52/53-week ends land on 0229/0230/0231 in some filings
|
|
try:
|
|
return date(year, month, day)
|
|
except ValueError:
|
|
day -= 1
|
|
return date(year, month, day)
|
|
|
|
|
|
def _fiscal_year_of(period_end: date, fiscal_year_end: str | None) -> int | None:
|
|
"""Which fiscal year ``period_end`` belongs to.
|
|
|
|
A 52/53-week calendar's real year end drifts around the nominal MMDD (and can
|
|
cross the calendar year), so allow drift before rolling into the next year.
|
|
"""
|
|
nominal = _nominal_fy_end(period_end.year, fiscal_year_end)
|
|
if nominal is None:
|
|
return None
|
|
return (
|
|
period_end.year
|
|
if period_end <= nominal + timedelta(days=_FYE_DRIFT_TOLERANCE_DAYS)
|
|
else period_end.year + 1
|
|
)
|
|
|
|
|
|
def _fiscal_context(facts: list[Fact], report_date: date) -> tuple[int | None, str | None]:
|
|
"""The filing's (fy, fp) taken as the majority context among the facts that
|
|
end at reportDate (the current-period facts, which share the filing's
|
|
context). Reject a tie so a conflicting context is never chosen arbitrarily."""
|
|
counts: dict[tuple[int, str], int] = {}
|
|
for f in facts:
|
|
if f.end == report_date and f.fy is not None and f.fp:
|
|
counts[(f.fy, f.fp)] = counts.get((f.fy, f.fp), 0) + 1
|
|
if not counts:
|
|
return None, None
|
|
ranked = sorted(counts.items(), key=lambda kv: kv[1], reverse=True)
|
|
if len(ranked) > 1 and ranked[0][1] == ranked[1][1]:
|
|
return None, None # tie → conflicting contexts, reject
|
|
return ranked[0][0]
|
|
|
|
|
|
def _select_ytd(
|
|
facts: list[Fact], concepts: list[str], report_date: date, fp: str, unit: str
|
|
) -> tuple[float | None, date | None]:
|
|
"""First present concept whose duration fact ends at reportDate and whose span
|
|
matches the fiscal-period-to-date length. Returns (val, period_start)."""
|
|
expected = _EXPECTED_YTD_DAYS[fp]
|
|
for concept in concepts:
|
|
best: Fact | None = None
|
|
best_diff: int | None = None
|
|
for f in facts:
|
|
if (
|
|
f.taxonomy != "us-gaap"
|
|
or f.concept != concept
|
|
or f.unit != unit
|
|
or f.start is None
|
|
or f.end != report_date
|
|
):
|
|
continue
|
|
diff = abs((f.end - f.start).days - expected)
|
|
if diff <= _YTD_TOLERANCE_DAYS and (best_diff is None or diff < best_diff):
|
|
best, best_diff = f, diff
|
|
if best is not None:
|
|
return float(best.val), best.start
|
|
return None, None
|
|
|
|
|
|
def _select_instant(facts: list[Fact], concepts: list[str], report_date: date) -> float | None:
|
|
"""First present instant (balance-sheet) fact at end == reportDate, unit USD."""
|
|
for concept in concepts:
|
|
for f in facts:
|
|
if (
|
|
f.taxonomy == "us-gaap"
|
|
and f.concept == concept
|
|
and f.unit == "USD"
|
|
and f.start is None
|
|
and f.end == report_date
|
|
):
|
|
return float(f.val)
|
|
return None
|
|
|
|
|
|
def _compose_cash(facts: list[Fact], report_date: date) -> float | None:
|
|
cash = _select_instant(facts, _CASH, report_date)
|
|
st = _select_instant(facts, _ST_INVESTMENTS, report_date) # first present of the two
|
|
if cash is None and st is None:
|
|
return None
|
|
return (cash or 0.0) + (st or 0.0)
|
|
|
|
|
|
def _compose_debt(facts: list[Fact], report_date: date) -> float | None:
|
|
"""Total debt at ``report_date``, or None when no long-term component is found.
|
|
|
|
**A short-term component alone is never a total.** Chevron tags its full debt
|
|
only in the 10-K, so its 10-Q carries ``ShortTermBorrowings`` of 0.40bn and
|
|
nothing else; returning that as total debt reads as a near-unlevered issuer
|
|
carrying 50bn. Since ``_net_debt`` needs both sides and yields nothing when
|
|
either is missing, None costs a leverage read while the partial value
|
|
produces a confidently wrong one.
|
|
"""
|
|
# An aggregate spanning current + noncurrent: only true short-term is missing.
|
|
total = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date)
|
|
if total is not None:
|
|
return total + (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
|
|
|
noncurrent = _select_instant(facts, _LONG_TERM_DEBT_NONCURRENT, report_date)
|
|
if noncurrent is None:
|
|
secured = _select_instant(facts, _SECURED_DEBT, report_date)
|
|
unsecured = _select_instant(facts, _UNSECURED_DEBT, report_date)
|
|
if secured is None or unsecured is None:
|
|
return None # one side of a REIT's debt is not its total
|
|
noncurrent = secured + unsecured
|
|
|
|
current = _select_instant(facts, _ALL_CURRENT_DEBT, report_date)
|
|
if current is None:
|
|
current = (
|
|
(_select_instant(facts, _LONG_TERM_DEBT_CURRENT, report_date) or 0.0)
|
|
+ (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
|
)
|
|
return noncurrent + current
|
|
|
|
|
|
def _select_shares(
|
|
facts: list[Fact], report_date: date
|
|
) -> tuple[float | None, date | None, bool]:
|
|
"""Issuer-wide shares outstanding as a single consolidated value (never a
|
|
class sum — companyfacts is non-dimensional — and never weighted-average/
|
|
diluted). Returns (value, shares_date, ambiguous).
|
|
|
|
1. Prefer the `dei:EntityCommonStockSharesOutstanding` cover-page instant;
|
|
its own end is the shares date (cover date != period_end).
|
|
2. Else fall back to `us-gaap:CommonStockSharesOutstanding` at period end
|
|
(e.g. Alphabet has no dei fact); shares date = reportDate.
|
|
Conflicting values within the chosen source → (None, None, True) to be
|
|
counted in validation.
|
|
"""
|
|
dei = [
|
|
f
|
|
for f in facts
|
|
if f.taxonomy == "dei"
|
|
and f.concept == "EntityCommonStockSharesOutstanding"
|
|
and f.unit == "shares"
|
|
and f.start is None
|
|
]
|
|
if dei:
|
|
if len({f.val for f in dei}) > 1:
|
|
return None, None, True
|
|
best = max(dei, key=lambda f: f.end)
|
|
return float(best.val), best.end, False
|
|
|
|
gaap = [
|
|
f
|
|
for f in facts
|
|
if f.taxonomy == "us-gaap"
|
|
and f.concept == "CommonStockSharesOutstanding"
|
|
and f.unit == "shares"
|
|
and f.start is None
|
|
and f.end == report_date
|
|
]
|
|
if gaap:
|
|
if len({f.val for f in gaap}) > 1:
|
|
return None, None, True
|
|
return float(gaap[0].val), report_date, False
|
|
|
|
return None, None, False # simply absent — not a conflict
|
|
|
|
|
|
def _select_weighted_avg_shares(facts: list[Fact], report_date: date) -> float | None:
|
|
"""The most recent quarter's weighted-average diluted share count.
|
|
|
|
Deliberately the **shortest** duration ending at reportDate, not the YTD one:
|
|
the shorter the window the closer the average sits to the current count, which
|
|
is what a market cap wants. Measured against issuers where the true
|
|
point-in-time count is available, the quarter average is within ~0.6%.
|
|
"""
|
|
best: tuple[int, float] | None = None
|
|
for concept in _WEIGHTED_AVG_SHARE_CONCEPTS:
|
|
for f in facts:
|
|
if (
|
|
f.taxonomy != "us-gaap"
|
|
or f.concept != concept
|
|
or f.unit != "shares"
|
|
or f.start is None
|
|
or f.end != report_date
|
|
or f.val <= 0
|
|
):
|
|
continue
|
|
span = (f.end - f.start).days
|
|
if best is None or span < best[0]:
|
|
best = (span, float(f.val))
|
|
if best is not None:
|
|
return best[1] # first present concept wins, as elsewhere
|
|
return None
|
|
|
|
|
|
def _d(value: Any) -> date | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return date.fromisoformat(str(value)[:10])
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _finite(value: Any) -> bool:
|
|
"""True for a finite numeric value (rejects None, bool, strings, NaN/inf)."""
|
|
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|