diff --git a/app/services/sec_facts_parser.py b/app/services/sec_facts_parser.py index ae90258..7f1de5c 100644 --- a/app/services/sec_facts_parser.py +++ b/app/services/sec_facts_parser.py @@ -113,8 +113,41 @@ _WEIGHTED_AVG_SHARE_CONCEPTS = [ # us-gaap instant (balance-sheet) concepts, at end == reportDate. _CASH = ["CashAndCashEquivalentsAtCarryingValue"] _ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one +# Debt is tagged in four mutually exclusive styles across large filers, and +# composing a total means knowing which span each concept covers (measured +# 2026-08 over a 20-issuer sample; the counts below are from it). +# +# ``LongTermDebt`` already spans current + noncurrent maturities — Apple tags all +# three and 71.34bn + 11.01bn = 82.30bn confirms it — so its complement is only +# genuinely short-term borrowing. _LONG_TERM_DEBT_AGG = ["LongTermDebt"] -_LONG_TERM_DEBT_PARTS = ["LongTermDebtNoncurrent", "LongTermDebtCurrent"] +# Noncurrent-only balance-sheet lines, needing a current complement added. +# ``LongTermDebtAndCapitalLeaseObligations`` is what KO, HD, T, XOM and CVX tag +# and nothing read it before: AT&T reported no total_debt at all against 134bn +# tagged, and Coca-Cola reported 0.25bn of commercial paper against 39bn. +_LONG_TERM_DEBT_NONCURRENT = [ + "LongTermDebtNoncurrent", + "LongTermDebtAndCapitalLeaseObligations", +] +_LONG_TERM_DEBT_CURRENT = ["LongTermDebtCurrent"] +# REITs that tag no aggregate at all, carrying a secured and an unsecured side +# instead. Both sides are required, because ``NotesPayable`` does not mean the +# same thing across issuers (measured 2026-08 over 14 REITs): +# - MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn +# exactly, so there it IS the total and adding SecuredDebt double-counts. +# - EQR/VMRK tags NotesPayable alongside a *larger* SecuredDebt (5.38bn vs +# 6.38bn in 2013), so there it is only the unsecured component. +# ``UnsecuredDebt`` is what separates them: where it is tagged it is the +# unambiguous unsecured side and NotesPayable is ignored; where it is absent, +# NotesPayable is that side. Requiring both sides is also what keeps this branch +# from inventing a total out of a fragment — Boston Properties tags SecuredDebt +# 4.28bn and nothing else against ~15bn of real debt, and Regency tags an +# UnsecuredDebt of 0.03bn that is a credit-line draw, not its 5bn of notes. +_SECURED_DEBT = ["SecuredDebt"] +_UNSECURED_DEBT = ["UnsecuredDebt", "NotesPayable"] # first present wins +# ``DebtCurrent`` spans short-term borrowing AND current maturities, so it is the +# whole current complement where present and must never be added alongside them. +_ALL_CURRENT_DEBT = ["DebtCurrent"] _SHORT_TERM_DEBT = ["ShortTermBorrowings", "CommercialPaper"] # pick one @@ -459,15 +492,35 @@ def _compose_cash(facts: list[Fact], report_date: date) -> float | None: def _compose_debt(facts: list[Fact], report_date: date) -> float | None: - long_term = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date) - if long_term is None: - nc = _select_instant(facts, ["LongTermDebtNoncurrent"], report_date) - cur = _select_instant(facts, ["LongTermDebtCurrent"], report_date) - long_term = None if nc is None and cur is None else (nc or 0.0) + (cur or 0.0) - short_term = _select_instant(facts, _SHORT_TERM_DEBT, report_date) - if long_term is None and short_term is None: - return None - return (long_term or 0.0) + (short_term or 0.0) + """Total debt at ``report_date``, or None when no long-term component is found. + + **A short-term component alone is never a total.** Chevron tags its full debt + only in the 10-K, so its 10-Q carries ``ShortTermBorrowings`` of 0.40bn and + nothing else; returning that as total debt reads as a near-unlevered issuer + carrying 50bn. Since ``_net_debt`` needs both sides and yields nothing when + either is missing, None costs a leverage read while the partial value + produces a confidently wrong one. + """ + # An aggregate spanning current + noncurrent: only true short-term is missing. + total = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date) + if total is not None: + return total + (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0) + + noncurrent = _select_instant(facts, _LONG_TERM_DEBT_NONCURRENT, report_date) + if noncurrent is None: + secured = _select_instant(facts, _SECURED_DEBT, report_date) + unsecured = _select_instant(facts, _UNSECURED_DEBT, report_date) + if secured is None or unsecured is None: + return None # one side of a REIT's debt is not its total + noncurrent = secured + unsecured + + current = _select_instant(facts, _ALL_CURRENT_DEBT, report_date) + if current is None: + current = ( + (_select_instant(facts, _LONG_TERM_DEBT_CURRENT, report_date) or 0.0) + + (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0) + ) + return noncurrent + current def _select_shares( diff --git a/reports/sec-companyfacts-stale-20260821-findings.md b/reports/sec-companyfacts-stale-20260821-findings.md index ffda208..09fef39 100644 --- a/reports/sec-companyfacts-stale-20260821-findings.md +++ b/reports/sec-companyfacts-stale-20260821-findings.md @@ -148,3 +148,95 @@ roughly $10bn of debt. `_compose_debt` returns the short-term component alone wh every `_LONG_TERM_DEBT_AGG` concept **and** the `LongTermDebtNoncurrent`/`Current` pair miss — which is what happened here, and Q2 matched neither. Worth checking against a current REIT filer before trusting `total_debt` for that sector. + +--- + +## 3. Follow-ups from the two alerts above + +### 3a. The reprieve in (1) ended silently — now it doesn't + +The hand-off in section 1 is a **bounded** reprieve. It ends two ways, and neither +said anything: the issuer's stored filings age past `GAP_GATE_RECENT_FILING_DAYS` +(for the 43, their last good filings are late April, so ~2026-10-26), or a newer +filing gap arrives and the all-escalated condition fails. `filing_gap_aged` cannot +report either, because it only escalates gaps whose `escalated_at` is NULL and so +never fires twice for the same gap. + +`sec_filing_gaps.exempted_at` (migration `034`) makes the transition observable: set +quietly while the issuer is exempt, cleared when the exemption lapses, and the clear +is what raises `filing_gap_repaused`. Once per lapse, re-arming if the issuer's data +recovers and ages out again. A gap that was never exempt has no transition and stays +silent — it is simply still paused, which `filing_gap_aged` already said. + +The exemption rule itself is not duplicated: `fundamentals_quality_service.gap_exempt_ciks` +is now public and the importer alerts on membership changes in exactly the set the +gate reads. + +### 3b. `total_debt` was materially wrong for a third of large caps + +The EQR observation in section 2 was not a REIT edge case. Measured over 19 large +caps, the old composition — `LongTermDebt`, else `LongTermDebtNoncurrent`/`Current`, +plus one of `ShortTermBorrowings`/`CommercialPaper` — missed two whole tagging styles: + +| issuer | before | after | what was missed | +|---|---:|---:|---| +| T | None | 143.95b | `LongTermDebtAndCapitalLeaseObligations` | +| XOM | None | 47.66b | same | +| VZ | 21.78b | 165.23b | same (read only the current maturities) | +| KO | 0.25b | 39.31b | same (read only commercial paper) | +| HD | 3.50b | 48.33b | same | +| O | 1.40b | 26.53b | REIT parts (`NotesPayable` + `SecuredDebt`) | +| VMRK | 1.50b | 9.09b | same | +| CVX | 0.40b | **None** | partial suppressed — see below | +| PFE | 63.10b | 63.19b | `DebtCurrent` is the completer current side | +| 10 others | — | unchanged | already composed correctly | + +`total_debt` feeds `net_debt` → `net_debt_to_ebitda` → the peer percentile and the +categorical leverage read, so Coca-Cola at 0.25bn of debt was not a missing value — +it was a confident *"conservative leverage"* on an issuer carrying ~39bn. + +The composition now spans four mutually exclusive styles, with each concept's span +respected: `LongTermDebt` already includes current maturities (Apple tags all three +and 71.34 + 11.01 = 82.30 confirms it), `LongTermDebtAndCapitalLeaseObligations` is +noncurrent and needs a current complement, and `DebtCurrent` *is* that whole +complement rather than an addition to it. + +**A short-term component alone is no longer reported as a total.** Chevron tags full +debt only in its 10-K, so its 10-Q carries 0.40bn of short-term borrowing and nothing +else. `_net_debt` needs both sides and yields nothing when either is missing, so None +costs a leverage read where the partial value produced a confidently wrong one. + +The REIT branch needed disambiguating, because `NotesPayable` does not mean the same +thing across issuers (measured over 14 REITs): MAA tags `NotesPayable` 5.66bn = +`UnsecuredDebt` 5.30bn + `SecuredDebt` 0.36bn **exactly**, so there it is the total and +adding the secured side double-counts — while EQR tags it alongside a *larger* +`SecuredDebt` (5.38bn vs 6.38bn in 2013), where it is only the unsecured component. +`UnsecuredDebt`'s presence separates the two: where tagged it is the unambiguous +unsecured side and `NotesPayable` is ignored; where absent, `NotesPayable` is that +side. Both sides are required, which is also what stops the branch inventing a total +from a fragment. + +| REIT | before | after | | +|---|---:|---:|---| +| MAA | None | 5.66b | matches its own `NotesPayable` total exactly | +| KIM | None | 8.74b | | +| O / VMRK | 1.40b / 1.50b | 26.53b / 9.09b | | +| BXP | 0.75b | **None** | tagged only `SecuredDebt` + paper against ~15bn real debt | +| VTR | 0.27b | **None** | same shape | +| 8 others | — | unchanged | already composed correctly | + +Known limit: where EQR tags both the parts and the aggregate, the parts sum 2.6–12.2% +*below* it, so this branch approximates. It is last in line — any issuer tagging an +aggregate never reaches it — and the alternative there is no value at all. + +### Sequencing the history fix + +Snapshots are immutable, so **3b corrects new filings only**; every stored quarter +keeps its old `total_debt`. `scripts/reparse_fundamentals.py` exists for exactly this +("after a parser fix, keeping the stored row is preserving a stale cache"). + +**Retire the `EQR` ticker before reparsing.** A reparse backfills every tracked CIK, +so while both 0000906107 and 0000931182 are tracked, both stage the same two 2015 +accessions and the run fails validation on `duplicate accession in staged snapshots`. +That is a safe stop — nothing is written — but the reparse will not complete until the +collision is gone. diff --git a/tests/unit/test_sec_facts_parser.py b/tests/unit/test_sec_facts_parser.py index 569fc8a..4265246 100644 --- a/tests/unit/test_sec_facts_parser.py +++ b/tests/unit/test_sec_facts_parser.py @@ -538,3 +538,88 @@ def test_a_wrong_declared_year_end_no_longer_collides_two_periods(): keys = {(r.fiscal_year, r.fiscal_period) for r in res.rows} assert len(keys) == 2, f"periods collided on one key: {keys}" assert keys == {(2026, "Q1"), (2026, "Q2")} + + +# --- debt composition across the tagging styles large filers actually use ---- +# Values are the real shapes measured 2026-08; before this composition, seven of +# nineteen sampled large caps carried a materially wrong or absent total_debt. + +_RD = date(2026, 3, 28) + + +def _f(concept, val): + return Fact("us-gaap", concept, "USD", None, _RD, val, 2026, "Q2") + + +def test_debt_from_a_noncurrent_lease_aggregate_adds_its_current_side(): + """KO/HD/T/XOM/CVX tag LongTermDebtAndCapitalLeaseObligations, which nothing + read before — AT&T reported no debt at all against 134bn tagged.""" + facts = [_f("LongTermDebtAndCapitalLeaseObligations", 134_630), _f("DebtCurrent", 9_320)] + assert _compose_debt(facts, _RD) == 143_950 + + +def test_debt_current_is_the_whole_current_side_not_an_addition(): + """DebtCurrent already spans short-term borrowing AND current maturities, so + adding commercial paper on top would count it twice.""" + facts = [ + _f("LongTermDebtNoncurrent", 22_840), + _f("DebtCurrent", 11_300), + _f("LongTermDebtCurrent", 6_460), + _f("CommercialPaper", 4_840), + ] + assert _compose_debt(facts, _RD) == 34_140 + + +def test_debt_falls_back_to_the_split_current_parts(): + facts = [ + _f("LongTermDebtNoncurrent", 36_890), + _f("LongTermDebtCurrent", 3_900), + _f("ShortTermBorrowings", 10_670), + ] + assert _compose_debt(facts, _RD) == 51_460 + + +def test_notes_payable_is_the_unsecured_side_when_nothing_names_it(): + """Realty Income and VMRK tag a secured and an unsecured side, no aggregate.""" + facts = [_f("NotesPayable", 25_090), _f("SecuredDebt", 40), _f("CommercialPaper", 1_400)] + assert _compose_debt(facts, _RD) == 26_530 + + +def test_an_explicit_unsecured_side_wins_over_notes_payable(): + """MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn, so + NotesPayable is the total there and adding SecuredDebt to it double-counts. + Preferring the explicit unsecured side reproduces the total either way.""" + facts = [_f("NotesPayable", 5_660), _f("UnsecuredDebt", 5_300), _f("SecuredDebt", 360)] + assert _compose_debt(facts, _RD) == 5_660 + + +def test_one_side_of_a_reits_debt_is_not_a_total(): + """Boston Properties tags SecuredDebt 4.28bn and commercial paper against ~15bn + of real debt; Ventas the same shape. Composing from one side invents a total.""" + assert _compose_debt([_f("SecuredDebt", 4_280), _f("CommercialPaper", 750)], _RD) is None + assert _compose_debt([_f("UnsecuredDebt", 5_300)], _RD) is None + + +def test_current_maturities_alone_are_not_a_total(): + """LongTermDebtCurrent used to stand in for the whole long-term side, which + reports the slice due within a year as if it were the debt.""" + assert _compose_debt([_f("LongTermDebtCurrent", 6_460)], _RD) is None + + +def test_an_aggregate_beats_the_reit_parts(): + """AvalonBay tags all three; summing the parts would understate the total.""" + facts = [_f("LongTermDebt", 9_020), _f("SecuredDebt", 700), _f("UnsecuredDebt", 7_410), + _f("CommercialPaper", 920)] + assert _compose_debt(facts, _RD) == 9_940 + + +def test_a_short_term_only_filing_reports_no_total_at_all(): + """Chevron tags its full debt only in the 10-K, so a 10-Q carries 0.40bn of + short-term borrowing alone — reporting that as *total* debt reads as a + near-unlevered issuer carrying 50bn. None costs a leverage read; the partial + value produces a confidently wrong one.""" + assert _compose_debt([_f("ShortTermBorrowings", 401)], _RD) is None + + +def test_no_debt_facts_at_all_is_still_none(): + assert _compose_debt([_f("CashAndCashEquivalentsAtCarryingValue", 100)], _RD) is None