Compare commits
4
Commits
c97a067e0e
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8453b87290 | ||
|
|
83fe76c506 | ||
|
|
c15b51439e | ||
|
|
a13dbc9710 |
@@ -0,0 +1,41 @@
|
|||||||
|
"""Track when a filing gap stops pausing setups
|
||||||
|
|
||||||
|
Revision ID: 034
|
||||||
|
Revises: 033
|
||||||
|
Create Date: 2026-08-21 00:00:00.000000
|
||||||
|
|
||||||
|
An escalated gap stops pausing setups while the issuer's own fundamentals are
|
||||||
|
still recent (``GAP_GATE_RECENT_FILING_DAYS``). That reprieve is not permanent:
|
||||||
|
the stored filings age out, or a newer gap appears, and the pause returns —
|
||||||
|
silently, because ``filing_gap_aged`` only escalates gaps whose ``escalated_at``
|
||||||
|
is NULL and so never fires twice for the same gap.
|
||||||
|
|
||||||
|
``exempted_at`` is the state marker that makes the transition observable. It is
|
||||||
|
set (quietly) while the issuer is exempt and cleared when the exemption lapses,
|
||||||
|
which is when ``filing_gap_repaused`` fires — once per lapse, re-arming if the
|
||||||
|
issuer's data recovers and ages out again.
|
||||||
|
|
||||||
|
Nullable, and carrying no meaning of its own beyond that state: an existing gap
|
||||||
|
starts NULL and is stamped on the next import that finds it exempt.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "034"
|
||||||
|
down_revision: Union[str, None] = "033"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"sec_filing_gaps",
|
||||||
|
sa.Column("exempted_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("sec_filing_gaps", "exempted_at")
|
||||||
@@ -30,3 +30,8 @@ class SecFilingGap(Base):
|
|||||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
last_attempted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
last_attempted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
escalated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
escalated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
# Set while this gap's issuer is exempt from the setup pause (escalated, and
|
||||||
|
# its own fundamentals still recent — see fundamentals_quality_service).
|
||||||
|
# Cleared when the exemption lapses, which is the moment the pause silently
|
||||||
|
# comes back and the only moment worth alerting on.
|
||||||
|
exempted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import exists, func, select
|
from sqlalchemy import exists, func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
@@ -15,6 +17,23 @@ from app.models.ticker import Ticker
|
|||||||
|
|
||||||
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
||||||
|
|
||||||
|
# How recent the issuer's own newest filing must be for an *escalated* gap to
|
||||||
|
# stop pausing setups. A gap pauses an issuer until it is either resolved or
|
||||||
|
# superseded by a later ingested filing — which assumes the gap is temporary.
|
||||||
|
# It is not always: SEC's per-company Company-Facts files can go stale
|
||||||
|
# indefinitely (2026-08, 43 large caps whose Q2 10-Qs the frames API carried but
|
||||||
|
# whose companyfacts files never received), and since the supersede rule needs a
|
||||||
|
# *successfully ingested* later filing, a stale file also swallows the next
|
||||||
|
# quarter. The pause is then open-ended rather than seasonal.
|
||||||
|
#
|
||||||
|
# So the pause hands off to the alert: once `filing_gap_aged` has escalated a gap
|
||||||
|
# to an operator (`escalated_at`), the issuer resumes on the fundamentals it does
|
||||||
|
# have — provided those are recent. An issuer with nothing this fresh has no
|
||||||
|
# usable fundamentals at all and stays paused, which is the case the gate was
|
||||||
|
# built for. The retry queue is untouched: `active_gaps` still returns these, so
|
||||||
|
# the importer keeps retrying and a recovered filing still resolves normally.
|
||||||
|
GAP_GATE_RECENT_FILING_DAYS = 180
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class SetupQuality:
|
class SetupQuality:
|
||||||
@@ -51,6 +70,42 @@ async def active_gaps(
|
|||||||
return list((await db.execute(stmt)).scalars().all())
|
return list((await db.execute(stmt)).scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
async def gap_exempt_ciks(
|
||||||
|
db: AsyncSession, gaps: list[SecFilingGap]
|
||||||
|
) -> set[str]:
|
||||||
|
"""CIKs whose gaps have stopped pausing setups (see GAP_GATE_RECENT_FILING_DAYS).
|
||||||
|
|
||||||
|
Every one of a CIK's active gaps must be escalated: one fresh gap alongside an
|
||||||
|
old one still means a filing we might yet ingest, which is worth pausing for.
|
||||||
|
|
||||||
|
Public because the importer alerts on this exact transition (a CIK dropping
|
||||||
|
out of this set is a pause coming back on) and the rule must not exist twice.
|
||||||
|
"""
|
||||||
|
by_cik: dict[str, list[SecFilingGap]] = defaultdict(list)
|
||||||
|
for gap in gaps:
|
||||||
|
by_cik[gap.cik].append(gap)
|
||||||
|
escalated = {
|
||||||
|
cik
|
||||||
|
for cik, items in by_cik.items()
|
||||||
|
if all(gap.escalated_at is not None for gap in items)
|
||||||
|
}
|
||||||
|
if not escalated:
|
||||||
|
return set()
|
||||||
|
cutoff = (
|
||||||
|
datetime.now(timezone.utc) - timedelta(days=GAP_GATE_RECENT_FILING_DAYS)
|
||||||
|
).date()
|
||||||
|
rows = await db.execute(
|
||||||
|
select(FundamentalSnapshot.cik)
|
||||||
|
.where(
|
||||||
|
FundamentalSnapshot.cik.in_(escalated),
|
||||||
|
FundamentalSnapshot.form.in_(_SEC_FORMS),
|
||||||
|
FundamentalSnapshot.filed_date >= cutoff,
|
||||||
|
)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
return set(rows.scalars())
|
||||||
|
|
||||||
|
|
||||||
async def _latest_validation(db: AsyncSession) -> dict:
|
async def _latest_validation(db: AsyncSession) -> dict:
|
||||||
payload = (
|
payload = (
|
||||||
await db.execute(
|
await db.execute(
|
||||||
@@ -80,8 +135,12 @@ async def blocked_reasons_by_cik(
|
|||||||
if ciks is not None and not ciks:
|
if ciks is not None and not ciks:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
gaps = await active_gaps(db, ciks)
|
||||||
|
# Escalated gaps on issuers that still have recent fundamentals no longer
|
||||||
|
# pause setups, on either path below — the summary mirrors the same filings.
|
||||||
|
exempt = await gap_exempt_ciks(db, gaps)
|
||||||
reasons = {
|
reasons = {
|
||||||
gap.cik: "sec_filing_gap" for gap in await active_gaps(db, ciks)
|
gap.cik: "sec_filing_gap" for gap in gaps if gap.cik not in exempt
|
||||||
}
|
}
|
||||||
summary = await _latest_validation(db)
|
summary = await _latest_validation(db)
|
||||||
|
|
||||||
@@ -92,11 +151,11 @@ async def blocked_reasons_by_cik(
|
|||||||
# stay capped for audit readability. Detailed entries supply the reason.
|
# stay capped for audit readability. Detailed entries supply the reason.
|
||||||
for cik in summary.get("setup_blocked_ciks") or []:
|
for cik in summary.get("setup_blocked_ciks") or []:
|
||||||
normalized = str(cik) if cik else ""
|
normalized = str(cik) if cik else ""
|
||||||
if normalized and wanted(normalized):
|
if normalized and wanted(normalized) and normalized not in exempt:
|
||||||
reasons.setdefault(normalized, "sec_filing_gap")
|
reasons.setdefault(normalized, "sec_filing_gap")
|
||||||
for item in summary.get("missing_xbrl") or []:
|
for item in summary.get("missing_xbrl") or []:
|
||||||
normalized = str(item.get("cik") or "")
|
normalized = str(item.get("cik") or "")
|
||||||
if normalized and wanted(normalized):
|
if normalized and wanted(normalized) and normalized not in exempt:
|
||||||
reasons.setdefault(normalized, "sec_filing_gap")
|
reasons.setdefault(normalized, "sec_filing_gap")
|
||||||
for cik in summary.get("no_xbrl_ciks") or []:
|
for cik in summary.get("no_xbrl_ciks") or []:
|
||||||
normalized = str(cik) if cik else ""
|
normalized = str(cik) if cik else ""
|
||||||
|
|||||||
@@ -113,8 +113,41 @@ _WEIGHTED_AVG_SHARE_CONCEPTS = [
|
|||||||
# us-gaap instant (balance-sheet) concepts, at end == reportDate.
|
# us-gaap instant (balance-sheet) concepts, at end == reportDate.
|
||||||
_CASH = ["CashAndCashEquivalentsAtCarryingValue"]
|
_CASH = ["CashAndCashEquivalentsAtCarryingValue"]
|
||||||
_ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one
|
_ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one
|
||||||
|
# Debt is tagged in four mutually exclusive styles across large filers, and
|
||||||
|
# composing a total means knowing which span each concept covers (measured
|
||||||
|
# 2026-08 over a 20-issuer sample; the counts below are from it).
|
||||||
|
#
|
||||||
|
# ``LongTermDebt`` already spans current + noncurrent maturities — Apple tags all
|
||||||
|
# three and 71.34bn + 11.01bn = 82.30bn confirms it — so its complement is only
|
||||||
|
# genuinely short-term borrowing.
|
||||||
_LONG_TERM_DEBT_AGG = ["LongTermDebt"]
|
_LONG_TERM_DEBT_AGG = ["LongTermDebt"]
|
||||||
_LONG_TERM_DEBT_PARTS = ["LongTermDebtNoncurrent", "LongTermDebtCurrent"]
|
# Noncurrent-only balance-sheet lines, needing a current complement added.
|
||||||
|
# ``LongTermDebtAndCapitalLeaseObligations`` is what KO, HD, T, XOM and CVX tag
|
||||||
|
# and nothing read it before: AT&T reported no total_debt at all against 134bn
|
||||||
|
# tagged, and Coca-Cola reported 0.25bn of commercial paper against 39bn.
|
||||||
|
_LONG_TERM_DEBT_NONCURRENT = [
|
||||||
|
"LongTermDebtNoncurrent",
|
||||||
|
"LongTermDebtAndCapitalLeaseObligations",
|
||||||
|
]
|
||||||
|
_LONG_TERM_DEBT_CURRENT = ["LongTermDebtCurrent"]
|
||||||
|
# REITs that tag no aggregate at all, carrying a secured and an unsecured side
|
||||||
|
# instead. Both sides are required, because ``NotesPayable`` does not mean the
|
||||||
|
# same thing across issuers (measured 2026-08 over 14 REITs):
|
||||||
|
# - MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn
|
||||||
|
# exactly, so there it IS the total and adding SecuredDebt double-counts.
|
||||||
|
# - EQR/VMRK tags NotesPayable alongside a *larger* SecuredDebt (5.38bn vs
|
||||||
|
# 6.38bn in 2013), so there it is only the unsecured component.
|
||||||
|
# ``UnsecuredDebt`` is what separates them: where it is tagged it is the
|
||||||
|
# unambiguous unsecured side and NotesPayable is ignored; where it is absent,
|
||||||
|
# NotesPayable is that side. Requiring both sides is also what keeps this branch
|
||||||
|
# from inventing a total out of a fragment — Boston Properties tags SecuredDebt
|
||||||
|
# 4.28bn and nothing else against ~15bn of real debt, and Regency tags an
|
||||||
|
# UnsecuredDebt of 0.03bn that is a credit-line draw, not its 5bn of notes.
|
||||||
|
_SECURED_DEBT = ["SecuredDebt"]
|
||||||
|
_UNSECURED_DEBT = ["UnsecuredDebt", "NotesPayable"] # first present wins
|
||||||
|
# ``DebtCurrent`` spans short-term borrowing AND current maturities, so it is the
|
||||||
|
# whole current complement where present and must never be added alongside them.
|
||||||
|
_ALL_CURRENT_DEBT = ["DebtCurrent"]
|
||||||
_SHORT_TERM_DEBT = ["ShortTermBorrowings", "CommercialPaper"] # pick one
|
_SHORT_TERM_DEBT = ["ShortTermBorrowings", "CommercialPaper"] # pick one
|
||||||
|
|
||||||
|
|
||||||
@@ -459,15 +492,35 @@ def _compose_cash(facts: list[Fact], report_date: date) -> float | None:
|
|||||||
|
|
||||||
|
|
||||||
def _compose_debt(facts: list[Fact], report_date: date) -> float | None:
|
def _compose_debt(facts: list[Fact], report_date: date) -> float | None:
|
||||||
long_term = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date)
|
"""Total debt at ``report_date``, or None when no long-term component is found.
|
||||||
if long_term is None:
|
|
||||||
nc = _select_instant(facts, ["LongTermDebtNoncurrent"], report_date)
|
**A short-term component alone is never a total.** Chevron tags its full debt
|
||||||
cur = _select_instant(facts, ["LongTermDebtCurrent"], report_date)
|
only in the 10-K, so its 10-Q carries ``ShortTermBorrowings`` of 0.40bn and
|
||||||
long_term = None if nc is None and cur is None else (nc or 0.0) + (cur or 0.0)
|
nothing else; returning that as total debt reads as a near-unlevered issuer
|
||||||
short_term = _select_instant(facts, _SHORT_TERM_DEBT, report_date)
|
carrying 50bn. Since ``_net_debt`` needs both sides and yields nothing when
|
||||||
if long_term is None and short_term is None:
|
either is missing, None costs a leverage read while the partial value
|
||||||
return None
|
produces a confidently wrong one.
|
||||||
return (long_term or 0.0) + (short_term or 0.0)
|
"""
|
||||||
|
# An aggregate spanning current + noncurrent: only true short-term is missing.
|
||||||
|
total = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date)
|
||||||
|
if total is not None:
|
||||||
|
return total + (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
||||||
|
|
||||||
|
noncurrent = _select_instant(facts, _LONG_TERM_DEBT_NONCURRENT, report_date)
|
||||||
|
if noncurrent is None:
|
||||||
|
secured = _select_instant(facts, _SECURED_DEBT, report_date)
|
||||||
|
unsecured = _select_instant(facts, _UNSECURED_DEBT, report_date)
|
||||||
|
if secured is None or unsecured is None:
|
||||||
|
return None # one side of a REIT's debt is not its total
|
||||||
|
noncurrent = secured + unsecured
|
||||||
|
|
||||||
|
current = _select_instant(facts, _ALL_CURRENT_DEBT, report_date)
|
||||||
|
if current is None:
|
||||||
|
current = (
|
||||||
|
(_select_instant(facts, _LONG_TERM_DEBT_CURRENT, report_date) or 0.0)
|
||||||
|
+ (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
||||||
|
)
|
||||||
|
return noncurrent + current
|
||||||
|
|
||||||
|
|
||||||
def _select_shares(
|
def _select_shares(
|
||||||
|
|||||||
@@ -36,7 +36,11 @@ Guardrails (design + reviews):
|
|||||||
excluded from actionable setups until its filing is recovered.
|
excluded from actionable setups until its filing is recovered.
|
||||||
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
||||||
reports differing existing accessions, and applies ticker updates in the same
|
reports differing existing accessions, and applies ticker updates in the same
|
||||||
transaction.
|
transaction. A difference in ``cik`` **alone** is reported separately as an
|
||||||
|
``accession_cik_collision``: every fact matched, so two tracked CIKs are
|
||||||
|
claiming one filing and the fix is the universe, not the parser. It never
|
||||||
|
self-heals on its own — the losing CIK stores no row, so it is backfilled and
|
||||||
|
re-reported every run until its ticker is re-pointed or retired.
|
||||||
- ``reparse=True`` is the one exception to immutability, and it is deliberate:
|
- ``reparse=True`` is the one exception to immutability, and it is deliberate:
|
||||||
it restages every accession with the current parser and **rewrites** the rows
|
it restages every accession with the current parser and **rewrites** the rows
|
||||||
that now reconstruct differently. Immutability protects SEC's record (one row
|
that now reconstruct differently. Immutability protects SEC's record (one row
|
||||||
@@ -67,7 +71,7 @@ from app.services import sec_universe
|
|||||||
from app.services.data_import import STATUS_PROMOTED, ValidationResult
|
from app.services.data_import import STATUS_PROMOTED, ValidationResult
|
||||||
from app.services.sec_client import SecClient, SecError, cik10
|
from app.services.sec_client import SecClient, SecError, cik10
|
||||||
from app.services.sec_facts_parser import FilingMeta, SnapshotRow
|
from app.services.sec_facts_parser import FilingMeta, SnapshotRow
|
||||||
from app.services.sec_universe import ResolvedUniverse
|
from app.services.sec_universe import CIK_OVERRIDES_KEY, ResolvedUniverse
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -281,7 +285,18 @@ class SecFundamentalsImporter:
|
|||||||
if old is not None:
|
if old is not None:
|
||||||
fields = _diff_fields(row, old)
|
fields = _diff_fields(row, old)
|
||||||
if fields:
|
if fields:
|
||||||
staged.discrepancies.append({"accession": row.accession, "fields": fields})
|
# Carry both CIKs. promote() reads a bare ["cik"] as an
|
||||||
|
# attribution collision rather than a changed
|
||||||
|
# reconstruction, which holds only because _COMPARE_COLS
|
||||||
|
# spans every stored fact: a fact column added to the
|
||||||
|
# model but not to _SNAPSHOT_COLS would go uncompared and
|
||||||
|
# let a real difference through as a collision.
|
||||||
|
staged.discrepancies.append({
|
||||||
|
"accession": row.accession,
|
||||||
|
"fields": fields,
|
||||||
|
"cik": row.cik,
|
||||||
|
"stored_cik": old.cik,
|
||||||
|
})
|
||||||
return staged
|
return staged
|
||||||
|
|
||||||
async def _stage_issuer(
|
async def _stage_issuer(
|
||||||
@@ -555,8 +570,15 @@ class SecFundamentalsImporter:
|
|||||||
inserted = 0
|
inserted = 0
|
||||||
updated = 0
|
updated = 0
|
||||||
# Only accessions whose reconstruction actually changed are rewritten;
|
# Only accessions whose reconstruction actually changed are rewritten;
|
||||||
# an unchanged stored row is left completely alone.
|
# an unchanged stored row is left completely alone. A cik-only difference
|
||||||
changed = {d["accession"] for d in staged.discrepancies} if self.reparse else set()
|
# is excluded on purpose: the facts are identical there, so rewriting
|
||||||
|
# would re-stamp the filing onto the colliding co-registrant — taking it
|
||||||
|
# from the issuer that actually filed it, which no parser fix asks for.
|
||||||
|
changed = (
|
||||||
|
{d["accession"] for d in staged.discrepancies if d["fields"] != ["cik"]}
|
||||||
|
if self.reparse
|
||||||
|
else set()
|
||||||
|
)
|
||||||
for row in staged.rows:
|
for row in staged.rows:
|
||||||
if row.accession in staged.existing_accessions:
|
if row.accession in staged.existing_accessions:
|
||||||
if row.accession in changed:
|
if row.accession in changed:
|
||||||
@@ -641,10 +663,51 @@ class SecFundamentalsImporter:
|
|||||||
if gap["accession"] not in existing_gap_accessions
|
if gap["accession"] not in existing_gap_accessions
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Two tracked issuers claiming one filing is not a reconstruction change:
|
||||||
|
# every fact matched and only the CIK stamp differs, so re-parsing or
|
||||||
|
# reparsing fixes nothing — the universe resolution does. It is reported
|
||||||
|
# separately because it also does not self-heal: the loser of the
|
||||||
|
# collision never stores a row, so `_ciks_with_snapshots` never sees it,
|
||||||
|
# and it is full-history backfilled (and re-reported) on every run until
|
||||||
|
# a human re-points or retires the ticker. Observed 2026-08 for EQR,
|
||||||
|
# which SEC's own company_tickers.json maps to ERP Operating LP, the
|
||||||
|
# non-traded co-registrant of the issuer now trading as VMRK.
|
||||||
|
collisions = [d for d in staged.discrepancies if d["fields"] == ["cik"]]
|
||||||
|
if collisions:
|
||||||
|
named = ", ".join(
|
||||||
|
f"{d['accession']} (stored {d['stored_cik']}, parsed {d['cik']})"
|
||||||
|
for d in collisions[:10]
|
||||||
|
)
|
||||||
|
db.add(SystemEvent(
|
||||||
|
severity="warning",
|
||||||
|
source="sec_facts",
|
||||||
|
code="accession_cik_collision",
|
||||||
|
message=(
|
||||||
|
f"{len(collisions)} filing(s) are claimed by two tracked CIKs — "
|
||||||
|
"the reconstruction is identical, only the attribution differs, "
|
||||||
|
"so one of the two is a co-registrant the universe should not "
|
||||||
|
f"track. Re-point or retire the ticker (see {CIK_OVERRIDES_KEY}); "
|
||||||
|
f"this repeats every run until then: {named}"
|
||||||
|
)[:4000],
|
||||||
|
dedup_key=f"sec_facts:accession_cik_collision:{run_id}",
|
||||||
|
created_at=_now(),
|
||||||
|
))
|
||||||
|
|
||||||
# Warn (in-transaction, so it commits atomically with the promotion) when
|
# Warn (in-transaction, so it commits atomically with the promotion) when
|
||||||
# any existing accession reconstructed differently — kept immutable.
|
# any existing accession reconstructed differently — kept immutable.
|
||||||
if staged.discrepancies:
|
reconstruction_diffs = [
|
||||||
accns = ", ".join(d["accession"] for d in staged.discrepancies[:10])
|
d for d in staged.discrepancies if d["fields"] != ["cik"]
|
||||||
|
]
|
||||||
|
if reconstruction_diffs:
|
||||||
|
# Name the columns, not just the accession: "differs in revenue"
|
||||||
|
# (our numbers moved) and "differs in period_start" (the filing was
|
||||||
|
# re-placed in the calendar) need different responses, and the alert
|
||||||
|
# is where that call gets made. The fields are already computed for
|
||||||
|
# validation_json — they were simply dropped from the message.
|
||||||
|
accns = ", ".join(
|
||||||
|
f"{d['accession']} ({', '.join(d['fields'])})"
|
||||||
|
for d in reconstruction_diffs[:10]
|
||||||
|
)
|
||||||
disposition = (
|
disposition = (
|
||||||
f"REWRITTEN by reparse run {run_id}" if self.reparse else "kept immutable"
|
f"REWRITTEN by reparse run {run_id}" if self.reparse else "kept immutable"
|
||||||
)
|
)
|
||||||
@@ -653,7 +716,7 @@ class SecFundamentalsImporter:
|
|||||||
source="sec_facts",
|
source="sec_facts",
|
||||||
code="snapshot_reparse" if self.reparse else "snapshot_discrepancy",
|
code="snapshot_reparse" if self.reparse else "snapshot_discrepancy",
|
||||||
message=(
|
message=(
|
||||||
f"{len(staged.discrepancies)} stored accession(s) reconstructed "
|
f"{len(reconstruction_diffs)} stored accession(s) reconstructed "
|
||||||
f"differently; {disposition}: {accns}"
|
f"differently; {disposition}: {accns}"
|
||||||
)[:4000],
|
)[:4000],
|
||||||
dedup_key=f"sec_facts:discrepancy:{run_id}",
|
dedup_key=f"sec_facts:discrepancy:{run_id}",
|
||||||
@@ -713,6 +776,60 @@ class SecFundamentalsImporter:
|
|||||||
.values(escalated_at=now)
|
.values(escalated_at=now)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The escalation above fires once per gap, so nothing would report the
|
||||||
|
# *end* of the reprieve it grants. An escalated gap stops pausing setups
|
||||||
|
# while the issuer's own fundamentals are still recent, and that lapses
|
||||||
|
# on its own — the stored filings age past the window, or a newer gap
|
||||||
|
# appears — putting the pause back on with no alert anywhere. Track the
|
||||||
|
# exemption as state and alert on the transition, once per lapse.
|
||||||
|
current_gaps = await fundamentals_quality_service.active_gaps(db)
|
||||||
|
escalated_gaps = [g for g in current_gaps if g.escalated_at is not None]
|
||||||
|
if escalated_gaps:
|
||||||
|
exempt_ciks = await fundamentals_quality_service.gap_exempt_ciks(
|
||||||
|
db, escalated_gaps
|
||||||
|
)
|
||||||
|
newly_exempt = [
|
||||||
|
g for g in escalated_gaps
|
||||||
|
if g.cik in exempt_ciks and g.exempted_at is None
|
||||||
|
]
|
||||||
|
lapsed = [
|
||||||
|
g for g in escalated_gaps
|
||||||
|
if g.cik not in exempt_ciks and g.exempted_at is not None
|
||||||
|
]
|
||||||
|
if newly_exempt:
|
||||||
|
# Silent on purpose: filing_gap_aged already announced this gap,
|
||||||
|
# and setups resuming is the behaviour that alert describes.
|
||||||
|
await db.execute(
|
||||||
|
update(SecFilingGap)
|
||||||
|
.where(SecFilingGap.id.in_([g.id for g in newly_exempt]))
|
||||||
|
.values(exempted_at=now)
|
||||||
|
)
|
||||||
|
if lapsed:
|
||||||
|
named = ", ".join(
|
||||||
|
f"{gap.cik}/{gap.accession}" for gap in lapsed[:10]
|
||||||
|
)
|
||||||
|
db.add(SystemEvent(
|
||||||
|
severity="warning",
|
||||||
|
source="sec_facts",
|
||||||
|
code="filing_gap_repaused",
|
||||||
|
message=(
|
||||||
|
f"{len(lapsed)} SEC filing gap(s) pause setups again: the "
|
||||||
|
"issuer's own fundamentals have aged out of the "
|
||||||
|
f"{fundamentals_quality_service.GAP_GATE_RECENT_FILING_DAYS}"
|
||||||
|
"-day window, or a newer gap arrived, so there is nothing "
|
||||||
|
f"recent left to score on: {named}"
|
||||||
|
)[:4000],
|
||||||
|
dedup_key=f"sec_facts:filing_gap_repaused:{run_id}",
|
||||||
|
created_at=now,
|
||||||
|
))
|
||||||
|
# Cleared, not stamped: the issuer can recover and age out again,
|
||||||
|
# and each lapse is worth its own alert.
|
||||||
|
await db.execute(
|
||||||
|
update(SecFilingGap)
|
||||||
|
.where(SecFilingGap.id.in_([g.id for g in lapsed]))
|
||||||
|
.values(exempted_at=None)
|
||||||
|
)
|
||||||
|
|
||||||
# Recovered rows are real data from an unexpected place — record where they
|
# Recovered rows are real data from an unexpected place — record where they
|
||||||
# came from, so a wrong recovery is auditable rather than invisible.
|
# came from, so a wrong recovery is auditable rather than invisible.
|
||||||
if staged.recovered:
|
if staged.recovered:
|
||||||
|
|||||||
@@ -24,6 +24,17 @@ entries: the application scheduler owns both jobs.
|
|||||||
tickers are excluded from actionable setups until a snapshot is recovered or
|
tickers are excluded from actionable setups until a snapshot is recovered or
|
||||||
a later valid 10-K/10-Q supersedes the gap. Migration `028` materializes older
|
a later valid 10-K/10-Q supersedes the gap. Migration `028` materializes older
|
||||||
promoted gaps into this queue once, so setup reads never scan import history.
|
promoted gaps into this queue once, so setup reads never scan import history.
|
||||||
|
- A gap that survives 14 days raises `filing_gap_aged` and, from that point,
|
||||||
|
stops pausing setups **if** the issuer's own newest stored 10-K/10-Q is less
|
||||||
|
than `GAP_GATE_RECENT_FILING_DAYS` (180) old. This is the hand-off from pause
|
||||||
|
to alert, and it exists because the pause would otherwise be open-ended:
|
||||||
|
SEC's per-company Company-Facts files can go stale indefinitely (2026-08: 43
|
||||||
|
large caps whose Q2 10-Qs the `frames` API carried but whose
|
||||||
|
`companyfacts/CIK*.json` never received), and the supersede rule needs a
|
||||||
|
*successfully ingested* later filing, so a stale file swallows the next
|
||||||
|
quarter too. Retrying is unaffected — the gap stays queued and a recovered
|
||||||
|
filing still resolves it normally. An issuer with no filing that recent has no
|
||||||
|
usable fundamentals at all and stays paused.
|
||||||
|
|
||||||
The systemd service uses one application worker. The import framework also holds
|
The systemd service uses one application worker. The import framework also holds
|
||||||
a PostgreSQL advisory lock per source, so an overlapping manual/scheduled run is
|
a PostgreSQL advisory lock per source, so an overlapping manual/scheduled run is
|
||||||
|
|||||||
@@ -0,0 +1,242 @@
|
|||||||
|
# SEC fundamentals alerts, 2026-08-21
|
||||||
|
|
||||||
|
Two `sec_facts` warnings, investigated against live SEC data. Both originate in SEC's
|
||||||
|
own published data — a stale per-company Company-Facts file (1) and a stale
|
||||||
|
ticker→CIK mapping (2) — and neither is a parser defect: no stored fundamental value
|
||||||
|
is wrong. Every SEC-side probe below reproduces offline from public endpoints; the
|
||||||
|
four database facts used are quoted where they appear.
|
||||||
|
|
||||||
|
## 1. `filing_gap_aged` — 43 gaps, all `not_in_companyfacts`
|
||||||
|
|
||||||
|
**Root cause: SEC's per-company Company-Facts files are stale for these issuers,
|
||||||
|
while the same filings are present in SEC's own `frames` aggregation.**
|
||||||
|
|
||||||
|
All ten named filings are real 10-Qs filed 2026-07-28/29, present in the issuer's
|
||||||
|
`submissions` with `isXBRL=1`, with complete R-files and XBRL in the EDGAR archive
|
||||||
|
— and absent from `companyfacts/CIK*.json`:
|
||||||
|
|
||||||
|
| CIK | issuer | accession | filed | in `companyfacts` | newest fact in file |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 0000001800 | Abbott | 0001628280-26-050134 | 2026-07-28 | no | 2026-04-29 |
|
||||||
|
| 0000021344 | Coca-Cola | 0001628280-26-050503 | 2026-07-29 | no | 2026-04-30 |
|
||||||
|
| 0000024741 | Corning | 0000024741-26-000255 | 2026-07-29 | no | 2026-05-01 |
|
||||||
|
| 0000029989 | Omnicom | 0000029989-26-000019 | 2026-07-29 | no | 2026-04-29 |
|
||||||
|
| 0000037996 | Ford | 0000037996-26-000156 | 2026-07-29 | no | 2026-04-30 |
|
||||||
|
| 0000040533 | General Dynamics | 0000040533-26-000032 | 2026-07-29 | no | 2026-07-01 |
|
||||||
|
| 0000048898 | Hubbell | 0001628280-26-050405 | 2026-07-29 | no | 2026-06-04 |
|
||||||
|
| 0000049071 | Humana | 0000049071-26-000050 | 2026-07-29 | no | 2026-04-29 |
|
||||||
|
| 0000049196 | Huntington Bancshares | 0000049196-26-000066 | 2026-07-28 | no | 2026-04-30 |
|
||||||
|
| 0000062996 | Masco | 0000062996-26-000027 | 2026-07-29 | no | 2026-04-22 |
|
||||||
|
|
||||||
|
Ruled out, with evidence:
|
||||||
|
|
||||||
|
- **Not a global SEC outage.** Company Facts is current for other issuers filing the
|
||||||
|
same days — MSFT `0001193125-26-323660` @2026-07-29, AAPL @2026-07-31, P&G
|
||||||
|
@2026-08-04, Chevron @2026-08-06, JPMorgan @2026-08-20.
|
||||||
|
- **Not a CDN/cache artifact.** A cache-busted request with `Cache-Control: no-cache`
|
||||||
|
returns the identical stale 3.39 MB payload; the response carries no cache headers.
|
||||||
|
- **Not our filter.** The scan covers every taxonomy/concept/unit in the payload.
|
||||||
|
- **Not a metadata discriminator.** Gap and non-gap filings are identical on
|
||||||
|
`isXBRL`, `isInlineXBRL`, `reportDate`, `primaryDocDescription`.
|
||||||
|
- **SEC does have the facts.** `frames/us-gaap/Assets/USD/CY2026Q2I.json` lists
|
||||||
|
Abbott at exactly the missing accession `0001628280-26-050134`, and Coca-Cola and
|
||||||
|
Ford at theirs. The per-company endpoints are the degraded ones:
|
||||||
|
`companyconcept/CIK0000001800/us-gaap/Assets.json` returns `"units":{"USD":{}}`.
|
||||||
|
|
||||||
|
**Consequence, and why the gate changed.** Retrying `companyfacts` cannot recover
|
||||||
|
these — Abbott's file has been stale since April. And because `active_gaps`
|
||||||
|
supersedes a gap only on a *successfully ingested later* filing, a stale file also
|
||||||
|
swallows Q3: the pause was open-ended, not seasonal, on 43 large caps.
|
||||||
|
|
||||||
|
**Fix** (`app/services/fundamentals_quality_service.py`): once `filing_gap_aged` has
|
||||||
|
escalated a gap (`escalated_at`), it stops pausing setups **if** the issuer's own
|
||||||
|
newest stored 10-K/10-Q is under `GAP_GATE_RECENT_FILING_DAYS` (180) old. Pause hands
|
||||||
|
off to the alert; an issuer with nothing that recent stays paused. `active_gaps` is
|
||||||
|
deliberately untouched, so `_retry_backlog` keeps retrying and a recovered filing
|
||||||
|
still resolves normally. The bound is applied to the queue path *and* the
|
||||||
|
`validation_json` summary path, which mirrors the same filings — bounding only one
|
||||||
|
leaves the behaviour unchanged in production.
|
||||||
|
|
||||||
|
**This is a bounded reprieve, not a removal — know the two ways it ends.** Abbott's
|
||||||
|
newest ingested filing is `0001628280-26-028357`, filed 2026-04-29, so its recency
|
||||||
|
window closes around **2026-10-26**; most of the 43 sit on late-April filings and
|
||||||
|
turn back to paused within days of each other. That crossing is **silent**: the
|
||||||
|
importer escalates only gaps with `escalated_at IS NULL`, so `filing_gap_aged` does
|
||||||
|
not re-fire for a gap it has already reported. Separately, a Q3 10-Q that also fails
|
||||||
|
to ingest creates a *new* un-escalated gap on the same CIK, which re-pauses it at
|
||||||
|
once (that one does raise its own `filing_gap_aged` 14 days later). Whether the
|
||||||
|
silent re-block deserves a re-escalation signal is an open call, deliberately not
|
||||||
|
made here — "one actionable escalation rather than a daily warning" is the existing
|
||||||
|
design intent.
|
||||||
|
|
||||||
|
**Not done, with reasons.** A `frames`-backed recovery source was considered and
|
||||||
|
rejected: frames are calendar-aligned with a tolerance (off-fiscal filers drop out)
|
||||||
|
and carry one fact per issuer per period, so amendment/restatement semantics differ
|
||||||
|
from Company Facts — lossy as a snapshot source, not merely expensive. Parsing the
|
||||||
|
filing's own inline-XBRL instance is the authoritative alternative but is a new
|
||||||
|
subsystem (contexts, dimensions, unit refs) duplicating the parser's fact model.
|
||||||
|
|
||||||
|
## 2. `snapshot_discrepancy` — 0000906107-15-000012 / -000016
|
||||||
|
|
||||||
|
**Root cause: two tracked tickers claim the same filing, because SEC's
|
||||||
|
`company_tickers.json` still points the old symbol at a non-traded co-registrant.
|
||||||
|
No stored value is wrong and no reparse is warranted.**
|
||||||
|
|
||||||
|
CIK 0000906107 is **Vivmark Residential** (VMRK, formerly Equity Residential). Both
|
||||||
|
alerted accessions are **combined EQR + ERP Operating LP 10-Qs** — one accession, two
|
||||||
|
registrants (0000906107 and 0000931182) — the pattern behind the existing
|
||||||
|
co-registrant recovery path.
|
||||||
|
|
||||||
|
The stored rows are **byte-identical** to what the current parser reconstructs from
|
||||||
|
EQR's own Company Facts — every column, verified: `cik` (`0000906107`), `form`,
|
||||||
|
`filed_date`, `accepted_at`, both period dates, `fiscal_year`/`fiscal_period`,
|
||||||
|
`revenue`, `net_income`, `operating_income`, `diluted_eps`, `cfo`, the two nulls,
|
||||||
|
`cash_and_st_investments`, `total_debt` (340,900,000 / null),
|
||||||
|
`shares_outstanding`, `shares_outstanding_date`, `weighted_avg_diluted_shares`. Both
|
||||||
|
carry `import_run_id = 6`, and CIK 0000906107 holds all 69 of its filings across runs
|
||||||
|
6–30, so the issuer's own history is complete.
|
||||||
|
|
||||||
|
Run 63 (2026-08-19) recorded
|
||||||
|
`fields: ["cik"]` for both accessions, and the universe explains it:
|
||||||
|
|
||||||
|
```
|
||||||
|
tickers: VMRK -> 0000906107 (Vivmark Residential, ex-Equity Residential)
|
||||||
|
EQR -> 0000931182 (ERP Operating Ltd Partnership)
|
||||||
|
```
|
||||||
|
|
||||||
|
SEC's own `company_tickers.json` carries `{"cik_str": 931182, "ticker": "EQR",
|
||||||
|
"title": "ERP OPERATING LTD PARTNERSHIP"}` — after the rename, the old symbol stayed
|
||||||
|
attached to the **non-traded operating partnership**, the co-registrant on those
|
||||||
|
combined 10-Qs. `resolve_ciks` reads `active_only` tickers and follows SEC, so
|
||||||
|
0000931182 is tracked. Its Company Facts holds 7 accessions, exactly 2 of them
|
||||||
|
EQR-prefixed, so its backfill reconstructs exactly those two rows, stamps them
|
||||||
|
`cik=0000931182`, and collides with the rows already stored under 0000906107 —
|
||||||
|
identical in every fact, differing only in attribution.
|
||||||
|
|
||||||
|
It cannot self-heal. The collision loser never stores a row (the insert is skipped as
|
||||||
|
immutable), so `_ciks_with_snapshots` never sees 0000931182, and it is full-history
|
||||||
|
backfilled — refetching every submissions shard and its companyfacts — **on every
|
||||||
|
run**, re-raising the warning each time. `fundamental_snapshots` for 0000906107 holds
|
||||||
|
all 69 filings across runs 6–30, so the issuer's own history is complete and correct.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
|
||||||
|
**Code** (`sec_fundamentals_importer.py`): a `cik`-only difference is no longer
|
||||||
|
reported as a reconstruction discrepancy. It raises `accession_cik_collision`, naming
|
||||||
|
both CIKs and pointing at `sec_cik_overrides`, because the fix is the universe, not
|
||||||
|
the parser. The reparse path also excludes these from its rewrite set — rewriting a
|
||||||
|
cik-only difference would re-stamp the filing onto the co-registrant and take it from
|
||||||
|
the issuer that filed it. (A reparse run while both CIKs are tracked fails validation
|
||||||
|
on `duplicate accession in staged snapshots` instead, which is a safe stop.)
|
||||||
|
|
||||||
|
**Data — needs an operator, and the alert repeats daily until then.** `EQR` is a stale
|
||||||
|
symbol: the security now trades as `VMRK`, which is already tracked at the correct
|
||||||
|
CIK. Retiring the `EQR` ticker ends the loop. A `sec_cik_overrides` pin of
|
||||||
|
`EQR -> 906107` would silence the collision but leave two tickers on one security,
|
||||||
|
double-counting the issuer in scans — retirement is the right action.
|
||||||
|
|
||||||
|
**Not fixed, deliberately:** the permanent-backfill loop itself. A tracked CIK whose
|
||||||
|
only parseable filings belong to another CIK is re-backfilled every run; ending that
|
||||||
|
in code means teaching `_ciks_with_snapshots` about foreign-owned accessions, which is
|
||||||
|
more state for a condition that is now loudly and specifically reported.
|
||||||
|
|
||||||
|
### Separate observation: `total_debt` on this issuer looks wrong
|
||||||
|
|
||||||
|
Independent of the alert, and unchanged by any fix here: the parser reconstructs
|
||||||
|
`total_debt = 340,900,000` for EQR's 2015 Q1 and `null` for Q2, while the REIT carried
|
||||||
|
roughly $10bn of debt. `_compose_debt` returns the short-term component alone when
|
||||||
|
every `_LONG_TERM_DEBT_AGG` concept **and** the `LongTermDebtNoncurrent`/`Current`
|
||||||
|
pair miss — which is what happened here, and Q2 matched neither. Worth checking
|
||||||
|
against a current REIT filer before trusting `total_debt` for that sector.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Follow-ups from the two alerts above
|
||||||
|
|
||||||
|
### 3a. The reprieve in (1) ended silently — now it doesn't
|
||||||
|
|
||||||
|
The hand-off in section 1 is a **bounded** reprieve. It ends two ways, and neither
|
||||||
|
said anything: the issuer's stored filings age past `GAP_GATE_RECENT_FILING_DAYS`
|
||||||
|
(for the 43, their last good filings are late April, so ~2026-10-26), or a newer
|
||||||
|
filing gap arrives and the all-escalated condition fails. `filing_gap_aged` cannot
|
||||||
|
report either, because it only escalates gaps whose `escalated_at` is NULL and so
|
||||||
|
never fires twice for the same gap.
|
||||||
|
|
||||||
|
`sec_filing_gaps.exempted_at` (migration `034`) makes the transition observable: set
|
||||||
|
quietly while the issuer is exempt, cleared when the exemption lapses, and the clear
|
||||||
|
is what raises `filing_gap_repaused`. Once per lapse, re-arming if the issuer's data
|
||||||
|
recovers and ages out again. A gap that was never exempt has no transition and stays
|
||||||
|
silent — it is simply still paused, which `filing_gap_aged` already said.
|
||||||
|
|
||||||
|
The exemption rule itself is not duplicated: `fundamentals_quality_service.gap_exempt_ciks`
|
||||||
|
is now public and the importer alerts on membership changes in exactly the set the
|
||||||
|
gate reads.
|
||||||
|
|
||||||
|
### 3b. `total_debt` was materially wrong for a third of large caps
|
||||||
|
|
||||||
|
The EQR observation in section 2 was not a REIT edge case. Measured over 19 large
|
||||||
|
caps, the old composition — `LongTermDebt`, else `LongTermDebtNoncurrent`/`Current`,
|
||||||
|
plus one of `ShortTermBorrowings`/`CommercialPaper` — missed two whole tagging styles:
|
||||||
|
|
||||||
|
| issuer | before | after | what was missed |
|
||||||
|
|---|---:|---:|---|
|
||||||
|
| T | None | 143.95b | `LongTermDebtAndCapitalLeaseObligations` |
|
||||||
|
| XOM | None | 47.66b | same |
|
||||||
|
| VZ | 21.78b | 165.23b | same (read only the current maturities) |
|
||||||
|
| KO | 0.25b | 39.31b | same (read only commercial paper) |
|
||||||
|
| HD | 3.50b | 48.33b | same |
|
||||||
|
| O | 1.40b | 26.53b | REIT parts (`NotesPayable` + `SecuredDebt`) |
|
||||||
|
| VMRK | 1.50b | 9.09b | same |
|
||||||
|
| CVX | 0.40b | **None** | partial suppressed — see below |
|
||||||
|
| PFE | 63.10b | 63.19b | `DebtCurrent` is the completer current side |
|
||||||
|
| 10 others | — | unchanged | already composed correctly |
|
||||||
|
|
||||||
|
`total_debt` feeds `net_debt` → `net_debt_to_ebitda` → the peer percentile and the
|
||||||
|
categorical leverage read, so Coca-Cola at 0.25bn of debt was not a missing value —
|
||||||
|
it was a confident *"conservative leverage"* on an issuer carrying ~39bn.
|
||||||
|
|
||||||
|
The composition now spans four mutually exclusive styles, with each concept's span
|
||||||
|
respected: `LongTermDebt` already includes current maturities (Apple tags all three
|
||||||
|
and 71.34 + 11.01 = 82.30 confirms it), `LongTermDebtAndCapitalLeaseObligations` is
|
||||||
|
noncurrent and needs a current complement, and `DebtCurrent` *is* that whole
|
||||||
|
complement rather than an addition to it.
|
||||||
|
|
||||||
|
**A short-term component alone is no longer reported as a total.** Chevron tags full
|
||||||
|
debt only in its 10-K, so its 10-Q carries 0.40bn of short-term borrowing and nothing
|
||||||
|
else. `_net_debt` needs both sides and yields nothing when either is missing, so None
|
||||||
|
costs a leverage read where the partial value produced a confidently wrong one.
|
||||||
|
|
||||||
|
The REIT branch needed disambiguating, because `NotesPayable` does not mean the same
|
||||||
|
thing across issuers (measured over 14 REITs): MAA tags `NotesPayable` 5.66bn =
|
||||||
|
`UnsecuredDebt` 5.30bn + `SecuredDebt` 0.36bn **exactly**, so there it is the total and
|
||||||
|
adding the secured side double-counts — while EQR tags it alongside a *larger*
|
||||||
|
`SecuredDebt` (5.38bn vs 6.38bn in 2013), where it is only the unsecured component.
|
||||||
|
`UnsecuredDebt`'s presence separates the two: where tagged it is the unambiguous
|
||||||
|
unsecured side and `NotesPayable` is ignored; where absent, `NotesPayable` is that
|
||||||
|
side. Both sides are required, which is also what stops the branch inventing a total
|
||||||
|
from a fragment.
|
||||||
|
|
||||||
|
| REIT | before | after | |
|
||||||
|
|---|---:|---:|---|
|
||||||
|
| MAA | None | 5.66b | matches its own `NotesPayable` total exactly |
|
||||||
|
| KIM | None | 8.74b | |
|
||||||
|
| O / VMRK | 1.40b / 1.50b | 26.53b / 9.09b | |
|
||||||
|
| BXP | 0.75b | **None** | tagged only `SecuredDebt` + paper against ~15bn real debt |
|
||||||
|
| VTR | 0.27b | **None** | same shape |
|
||||||
|
| 8 others | — | unchanged | already composed correctly |
|
||||||
|
|
||||||
|
Known limit: where EQR tags both the parts and the aggregate, the parts sum 2.6–12.2%
|
||||||
|
*below* it, so this branch approximates. It is last in line — any issuer tagging an
|
||||||
|
aggregate never reaches it — and the alternative there is no value at all.
|
||||||
|
|
||||||
|
### Sequencing the history fix
|
||||||
|
|
||||||
|
Snapshots are immutable, so **3b corrects new filings only**; every stored quarter
|
||||||
|
keeps its old `total_debt`. `scripts/reparse_fundamentals.py` exists for exactly this
|
||||||
|
("after a parser fix, keeping the stored row is preserving a stale cache").
|
||||||
|
|
||||||
|
**Retire the `EQR` ticker before reparsing.** A reparse backfills every tracked CIK,
|
||||||
|
so while both 0000906107 and 0000931182 are tracked, both stage the same two 2015
|
||||||
|
accessions and the run fails validation on `duplicate accession in staged snapshots`.
|
||||||
|
That is a safe stop — nothing is written — but the reparse will not complete until the
|
||||||
|
collision is gone.
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
from app.models.data_import_run import DataImportRun
|
from app.models.data_import_run import DataImportRun
|
||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
@@ -139,3 +139,143 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session):
|
|||||||
assert await fundamentals_quality_service.ticker_is_eligible(
|
assert await fundamentals_quality_service.ticker_is_eligible(
|
||||||
db_session, ticker.id
|
db_session, ticker.id
|
||||||
) is False
|
) is False
|
||||||
|
|
||||||
|
|
||||||
|
def _escalated_gap(cik: str, *, escalated: bool = True) -> SecFilingGap:
|
||||||
|
first_seen = datetime.now(timezone.utc) - timedelta(days=24)
|
||||||
|
return SecFilingGap(
|
||||||
|
cik=cik,
|
||||||
|
accession=f"{cik}-STALE-Q",
|
||||||
|
form="10-Q",
|
||||||
|
index_date=(first_seen.date()),
|
||||||
|
reason="not_in_companyfacts",
|
||||||
|
first_seen_at=first_seen,
|
||||||
|
last_attempted_at=datetime.now(timezone.utc),
|
||||||
|
escalated_at=(
|
||||||
|
datetime.now(timezone.utc) - timedelta(days=10) if escalated else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _prior_quarter(cik: str, *, age_days: int) -> FundamentalSnapshot:
|
||||||
|
"""The issuer's last successfully ingested filing, older than the gap so it
|
||||||
|
cannot supersede it — exactly the production shape of a stale companyfacts
|
||||||
|
file: Q1 stored, Q2 missing."""
|
||||||
|
filed = date.today() - timedelta(days=age_days)
|
||||||
|
return FundamentalSnapshot(
|
||||||
|
cik=cik,
|
||||||
|
accession=f"{cik}-PRIOR-Q",
|
||||||
|
form="10-Q",
|
||||||
|
filed_date=filed,
|
||||||
|
accepted_at=datetime.now(timezone.utc) - timedelta(days=age_days),
|
||||||
|
period_end=filed,
|
||||||
|
fiscal_year=filed.year,
|
||||||
|
fiscal_period="Q1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_escalated_gap_stops_blocking_when_fundamentals_are_recent(db_session):
|
||||||
|
ticker = Ticker(symbol="STALEFACTS", cik="0000000046")
|
||||||
|
db_session.add(ticker)
|
||||||
|
db_session.add(_escalated_gap(ticker.cik))
|
||||||
|
await db_session.flush()
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == {
|
||||||
|
ticker.id
|
||||||
|
}
|
||||||
|
|
||||||
|
# The alert has run and the issuer still has last quarter to score on.
|
||||||
|
db_session.add(_prior_quarter(ticker.cik, age_days=120))
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set()
|
||||||
|
# ...but the filing is still queued, so the importer keeps retrying it.
|
||||||
|
assert len(await fundamentals_quality_service.active_gaps(db_session)) == 1
|
||||||
|
|
||||||
|
|
||||||
|
async def test_escalated_gap_keeps_blocking_when_fundamentals_are_stale(db_session):
|
||||||
|
ticker = Ticker(symbol="NOTHINGFRESH", cik="0000000047")
|
||||||
|
db_session.add_all([
|
||||||
|
ticker,
|
||||||
|
_escalated_gap(ticker.cik),
|
||||||
|
_prior_quarter(
|
||||||
|
ticker.cik,
|
||||||
|
age_days=fundamentals_quality_service.GAP_GATE_RECENT_FILING_DAYS + 30,
|
||||||
|
),
|
||||||
|
])
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == {
|
||||||
|
ticker.id
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_unescalated_gap_still_blocks_alongside_an_escalated_one(db_session):
|
||||||
|
ticker = Ticker(symbol="TWOGAPS", cik="0000000048")
|
||||||
|
fresh = datetime.now(timezone.utc)
|
||||||
|
db_session.add_all([
|
||||||
|
ticker,
|
||||||
|
_escalated_gap(ticker.cik),
|
||||||
|
SecFilingGap(
|
||||||
|
cik=ticker.cik,
|
||||||
|
accession="TWOGAPS-FRESH-Q",
|
||||||
|
form="10-Q",
|
||||||
|
index_date=date.today(),
|
||||||
|
reason="not_in_companyfacts",
|
||||||
|
first_seen_at=fresh,
|
||||||
|
last_attempted_at=fresh,
|
||||||
|
),
|
||||||
|
_prior_quarter(ticker.cik, age_days=120),
|
||||||
|
])
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == {
|
||||||
|
ticker.id
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_summary_path_does_not_reblock_an_exempt_cik(db_session):
|
||||||
|
"""The run summary mirrors the same filings as the queue — it must honour the
|
||||||
|
same hand-off, or the bound is inert in production."""
|
||||||
|
ticker = Ticker(symbol="MIRRORED", cik="0000000049")
|
||||||
|
db_session.add(ticker)
|
||||||
|
db_session.add(_escalated_gap(ticker.cik))
|
||||||
|
db_session.add(_prior_quarter(ticker.cik, age_days=120))
|
||||||
|
await db_session.flush()
|
||||||
|
db_session.add(
|
||||||
|
DataImportRun(
|
||||||
|
source="sec_facts",
|
||||||
|
status="promoted",
|
||||||
|
validation_json=json.dumps({
|
||||||
|
"setup_blocked_ciks": [ticker.cik],
|
||||||
|
"missing_xbrl": [
|
||||||
|
{"cik": ticker.cik, "accession": f"{ticker.cik}-STALE-Q"}
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
started_at=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_newer_gap_ends_the_exemption(db_session):
|
||||||
|
"""Production's second exit path: Q3 also fails to ingest, so an un-escalated
|
||||||
|
gap joins the escalated one and the issuer pauses again immediately."""
|
||||||
|
ticker = Ticker(symbol="NEWGAP", cik="0000000050")
|
||||||
|
db_session.add_all([
|
||||||
|
ticker, _escalated_gap(ticker.cik), _prior_quarter(ticker.cik, age_days=120)
|
||||||
|
])
|
||||||
|
await db_session.flush()
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set()
|
||||||
|
|
||||||
|
fresh = datetime.now(timezone.utc)
|
||||||
|
db_session.add(SecFilingGap(
|
||||||
|
cik=ticker.cik, accession="NEWGAP-Q3", form="10-Q", index_date=date.today(),
|
||||||
|
reason="not_in_companyfacts", first_seen_at=fresh, last_attempted_at=fresh,
|
||||||
|
))
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == {
|
||||||
|
ticker.id
|
||||||
|
}
|
||||||
|
|||||||
@@ -538,3 +538,88 @@ def test_a_wrong_declared_year_end_no_longer_collides_two_periods():
|
|||||||
keys = {(r.fiscal_year, r.fiscal_period) for r in res.rows}
|
keys = {(r.fiscal_year, r.fiscal_period) for r in res.rows}
|
||||||
assert len(keys) == 2, f"periods collided on one key: {keys}"
|
assert len(keys) == 2, f"periods collided on one key: {keys}"
|
||||||
assert keys == {(2026, "Q1"), (2026, "Q2")}
|
assert keys == {(2026, "Q1"), (2026, "Q2")}
|
||||||
|
|
||||||
|
|
||||||
|
# --- debt composition across the tagging styles large filers actually use ----
|
||||||
|
# Values are the real shapes measured 2026-08; before this composition, seven of
|
||||||
|
# nineteen sampled large caps carried a materially wrong or absent total_debt.
|
||||||
|
|
||||||
|
_RD = date(2026, 3, 28)
|
||||||
|
|
||||||
|
|
||||||
|
def _f(concept, val):
|
||||||
|
return Fact("us-gaap", concept, "USD", None, _RD, val, 2026, "Q2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_debt_from_a_noncurrent_lease_aggregate_adds_its_current_side():
|
||||||
|
"""KO/HD/T/XOM/CVX tag LongTermDebtAndCapitalLeaseObligations, which nothing
|
||||||
|
read before — AT&T reported no debt at all against 134bn tagged."""
|
||||||
|
facts = [_f("LongTermDebtAndCapitalLeaseObligations", 134_630), _f("DebtCurrent", 9_320)]
|
||||||
|
assert _compose_debt(facts, _RD) == 143_950
|
||||||
|
|
||||||
|
|
||||||
|
def test_debt_current_is_the_whole_current_side_not_an_addition():
|
||||||
|
"""DebtCurrent already spans short-term borrowing AND current maturities, so
|
||||||
|
adding commercial paper on top would count it twice."""
|
||||||
|
facts = [
|
||||||
|
_f("LongTermDebtNoncurrent", 22_840),
|
||||||
|
_f("DebtCurrent", 11_300),
|
||||||
|
_f("LongTermDebtCurrent", 6_460),
|
||||||
|
_f("CommercialPaper", 4_840),
|
||||||
|
]
|
||||||
|
assert _compose_debt(facts, _RD) == 34_140
|
||||||
|
|
||||||
|
|
||||||
|
def test_debt_falls_back_to_the_split_current_parts():
|
||||||
|
facts = [
|
||||||
|
_f("LongTermDebtNoncurrent", 36_890),
|
||||||
|
_f("LongTermDebtCurrent", 3_900),
|
||||||
|
_f("ShortTermBorrowings", 10_670),
|
||||||
|
]
|
||||||
|
assert _compose_debt(facts, _RD) == 51_460
|
||||||
|
|
||||||
|
|
||||||
|
def test_notes_payable_is_the_unsecured_side_when_nothing_names_it():
|
||||||
|
"""Realty Income and VMRK tag a secured and an unsecured side, no aggregate."""
|
||||||
|
facts = [_f("NotesPayable", 25_090), _f("SecuredDebt", 40), _f("CommercialPaper", 1_400)]
|
||||||
|
assert _compose_debt(facts, _RD) == 26_530
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_explicit_unsecured_side_wins_over_notes_payable():
|
||||||
|
"""MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn, so
|
||||||
|
NotesPayable is the total there and adding SecuredDebt to it double-counts.
|
||||||
|
Preferring the explicit unsecured side reproduces the total either way."""
|
||||||
|
facts = [_f("NotesPayable", 5_660), _f("UnsecuredDebt", 5_300), _f("SecuredDebt", 360)]
|
||||||
|
assert _compose_debt(facts, _RD) == 5_660
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_side_of_a_reits_debt_is_not_a_total():
|
||||||
|
"""Boston Properties tags SecuredDebt 4.28bn and commercial paper against ~15bn
|
||||||
|
of real debt; Ventas the same shape. Composing from one side invents a total."""
|
||||||
|
assert _compose_debt([_f("SecuredDebt", 4_280), _f("CommercialPaper", 750)], _RD) is None
|
||||||
|
assert _compose_debt([_f("UnsecuredDebt", 5_300)], _RD) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_current_maturities_alone_are_not_a_total():
|
||||||
|
"""LongTermDebtCurrent used to stand in for the whole long-term side, which
|
||||||
|
reports the slice due within a year as if it were the debt."""
|
||||||
|
assert _compose_debt([_f("LongTermDebtCurrent", 6_460)], _RD) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_aggregate_beats_the_reit_parts():
|
||||||
|
"""AvalonBay tags all three; summing the parts would understate the total."""
|
||||||
|
facts = [_f("LongTermDebt", 9_020), _f("SecuredDebt", 700), _f("UnsecuredDebt", 7_410),
|
||||||
|
_f("CommercialPaper", 920)]
|
||||||
|
assert _compose_debt(facts, _RD) == 9_940
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_short_term_only_filing_reports_no_total_at_all():
|
||||||
|
"""Chevron tags its full debt only in the 10-K, so a 10-Q carries 0.40bn of
|
||||||
|
short-term borrowing alone — reporting that as *total* debt reads as a
|
||||||
|
near-unlevered issuer carrying 50bn. None costs a leverage read; the partial
|
||||||
|
value produces a confidently wrong one."""
|
||||||
|
assert _compose_debt([_f("ShortTermBorrowings", 401)], _RD) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_debt_facts_at_all_is_still_none():
|
||||||
|
assert _compose_debt([_f("CashAndCashEquivalentsAtCarryingValue", 100)], _RD) is None
|
||||||
|
|||||||
@@ -950,6 +950,9 @@ async def test_discrepancy_in_shares_is_detected_and_reported(engine):
|
|||||||
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
||||||
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
||||||
assert len(events) == 1 and events[0].severity == "warning"
|
assert len(events) == 1 and events[0].severity == "warning"
|
||||||
|
# The alert has to say WHICH column moved: a differing cik is a co-registrant
|
||||||
|
# attribution, a differing revenue is our numbers changing.
|
||||||
|
assert "K (shares_outstanding, shares_outstanding_date)" in events[0].message
|
||||||
|
|
||||||
|
|
||||||
# --- reparse: rewriting rows a fixed parser reconstructs differently --------
|
# --- reparse: rewriting rows a fixed parser reconstructs differently --------
|
||||||
@@ -1283,3 +1286,181 @@ async def test_ceiling_promotes_queues_and_alerts_end_to_end(engine, monkeypatch
|
|||||||
assert len(events) == 1
|
assert len(events) == 1
|
||||||
assert events[0].severity == "warning"
|
assert events[0].severity == "warning"
|
||||||
assert "7 days" in events[0].message
|
assert "7 days" in events[0].message
|
||||||
|
|
||||||
|
|
||||||
|
# --- attribution collisions: two tracked CIKs claiming one filing ----------
|
||||||
|
|
||||||
|
# A REIT and its operating partnership co-file one 10-K, and SEC's
|
||||||
|
# company_tickers.json points the old symbol at the partnership (EQR ->
|
||||||
|
# ERP Operating LP) while the issuer itself trades under a new one (VMRK).
|
||||||
|
_COMBINED = [_filing("COMBINED-K", "10-K", "2025-12-31", "2026-02-13",
|
||||||
|
"2026-02-13T21:00:00.000Z")]
|
||||||
|
_CF_COMBINED = _rev("2025-01-01", "2025-12-31", 2900000, 2025, "FY", "COMBINED-K")
|
||||||
|
_SH_COMBINED = _shares("2026-02-01", 380000, "COMBINED-K", 2025, "FY")
|
||||||
|
|
||||||
|
|
||||||
|
def _reit_submissions(cik, tickers):
|
||||||
|
return {"cik": cik, "sic": "6798", "sic_description": "REIT",
|
||||||
|
"fiscal_year_end": "1231", "tickers": tickers, "filings": _COMBINED}
|
||||||
|
|
||||||
|
|
||||||
|
def _reit_client(tickers):
|
||||||
|
return FakeSecClient(
|
||||||
|
tickers=tickers,
|
||||||
|
companyfacts={
|
||||||
|
cik: _companyfacts([_CF_COMBINED], [_SH_COMBINED], cik=cik)
|
||||||
|
for cik in tickers.values()
|
||||||
|
},
|
||||||
|
submissions={
|
||||||
|
cik: _reit_submissions(cik, [sym]) for sym, cik in tickers.items()
|
||||||
|
},
|
||||||
|
latest_index=date(2026, 3, 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_cik_collision_is_reported_as_attribution_not_discrepancy(engine):
|
||||||
|
"""Only `cik` differs, so nothing was re-parsed differently — the universe
|
||||||
|
resolves a co-registrant it should not track, and the alert must say that."""
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["VMRK"])
|
||||||
|
run = await run_import(
|
||||||
|
_importer(_reit_client({"VMRK": 906107}), today=date(2026, 3, 2)), engine=engine
|
||||||
|
)
|
||||||
|
assert run.status == STATUS_PROMOTED
|
||||||
|
|
||||||
|
# The stale symbol is added, resolving to the partnership's CIK.
|
||||||
|
await _seed(factory, ["EQR"])
|
||||||
|
run = await run_import(
|
||||||
|
_importer(_reit_client({"VMRK": 906107, "EQR": 931182}), today=date(2026, 3, 2)),
|
||||||
|
engine=engine,
|
||||||
|
)
|
||||||
|
assert run.status == STATUS_PROMOTED
|
||||||
|
# Production's shape: a run-level incremental in which the untracked-until-now
|
||||||
|
# CIK is individually backfilled (run 63 recorded exactly this).
|
||||||
|
assert '"backfill": false' in (run.validation_json or "")
|
||||||
|
|
||||||
|
async with factory() as s:
|
||||||
|
rows = (await s.execute(select(FundamentalSnapshot))).scalars().all()
|
||||||
|
events = (await s.execute(select(SystemEvent))).scalars().all()
|
||||||
|
# The filing stays with the issuer that filed it, stored once.
|
||||||
|
assert [(r.accession, r.cik) for r in rows] == [("COMBINED-K", "0000906107")]
|
||||||
|
|
||||||
|
codes = {e.code for e in events}
|
||||||
|
assert "accession_cik_collision" in codes
|
||||||
|
assert "snapshot_discrepancy" not in codes # not a reconstruction change
|
||||||
|
collision = next(e for e in events if e.code == "accession_cik_collision")
|
||||||
|
assert "stored 0000906107, parsed 0000931182" in collision.message
|
||||||
|
assert "sec_cik_overrides" in collision.message # names the actual fix
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reparse_never_restamps_a_collision_onto_the_co_registrant(engine):
|
||||||
|
"""A reparse rewrites rows a fixed parser reconstructs differently. A cik-only
|
||||||
|
difference is not that: rewriting would hand the filing to the co-registrant."""
|
||||||
|
from app.services.sec_facts_parser import SnapshotRow
|
||||||
|
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["VMRK"])
|
||||||
|
assert (await run_import(
|
||||||
|
_importer(_reit_client({"VMRK": 906107}), today=date(2026, 3, 2)), engine=engine
|
||||||
|
)).status == STATUS_PROMOTED
|
||||||
|
|
||||||
|
importer = _importer(_reit_client({"VMRK": 906107}), today=date(2026, 3, 2))
|
||||||
|
importer.reparse = True
|
||||||
|
staged = StagedFundamentals(
|
||||||
|
resolved=ResolvedUniverse(),
|
||||||
|
rows=[SnapshotRow(
|
||||||
|
cik="0000931182", accession="COMBINED-K", form="10-K",
|
||||||
|
filed_date=date(2026, 2, 13),
|
||||||
|
accepted_at=datetime(2026, 2, 13, 21, tzinfo=timezone.utc),
|
||||||
|
period_end=date(2025, 12, 31), fiscal_year=2025, fiscal_period="FY",
|
||||||
|
)],
|
||||||
|
existing_accessions={"COMBINED-K"},
|
||||||
|
discrepancies=[{
|
||||||
|
"accession": "COMBINED-K", "fields": ["cik"],
|
||||||
|
"cik": "0000931182", "stored_cik": "0000906107",
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
async with _factory(engine)() as db:
|
||||||
|
counts = await importer.promote(db, staged, run_id=999)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert counts["updated"] == 0
|
||||||
|
async with factory() as s:
|
||||||
|
row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
||||||
|
assert row.cik == "0000906107" # still the issuer that filed it
|
||||||
|
|
||||||
|
|
||||||
|
# --- the reprieve ending: an exemption that lapses must not do so silently ---
|
||||||
|
|
||||||
|
def _stale_gap(cik, *, exempted: bool):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return SecFilingGap(
|
||||||
|
cik=cik, accession=f"{cik}-AGED-Q", form="10-Q",
|
||||||
|
index_date=(now - timedelta(days=30)).date(), reason="not_in_companyfacts",
|
||||||
|
first_seen_at=now - timedelta(days=30), last_attempted_at=now,
|
||||||
|
escalated_at=now - timedelta(days=16),
|
||||||
|
exempted_at=(now - timedelta(days=16)) if exempted else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot(cik, *, age_days):
|
||||||
|
filed = date.today() - timedelta(days=age_days)
|
||||||
|
return FundamentalSnapshot(
|
||||||
|
cik=cik, accession=f"{cik}-PRIOR", form="10-Q", filed_date=filed,
|
||||||
|
accepted_at=datetime.now(timezone.utc) - timedelta(days=age_days),
|
||||||
|
period_end=filed, fiscal_year=filed.year, fiscal_period="Q1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _promote_only(engine, seed):
|
||||||
|
"""Run promote() alone against seeded gap/snapshot state."""
|
||||||
|
factory = _factory(engine)
|
||||||
|
async with factory() as s:
|
||||||
|
for obj in seed:
|
||||||
|
s.add(obj)
|
||||||
|
await s.commit()
|
||||||
|
importer = _importer(FakeSecClient(
|
||||||
|
tickers={}, companyfacts={}, submissions={}, latest_index=date(2026, 3, 1)
|
||||||
|
))
|
||||||
|
async with factory() as db:
|
||||||
|
await importer.promote(db, StagedFundamentals(resolved=ResolvedUniverse()), run_id=77)
|
||||||
|
await db.commit()
|
||||||
|
async with factory() as s:
|
||||||
|
gaps = (await s.execute(select(SecFilingGap))).scalars().all()
|
||||||
|
events = (await s.execute(select(SystemEvent))).scalars().all()
|
||||||
|
return gaps, events
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_lapsed_exemption_raises_its_own_alert(engine):
|
||||||
|
"""filing_gap_aged fires once and never again, so nothing else would say the
|
||||||
|
pause came back when the issuer's own fundamentals aged out."""
|
||||||
|
cik = "0000000060"
|
||||||
|
gaps, events = await _promote_only(
|
||||||
|
engine, [_stale_gap(cik, exempted=True), _snapshot(cik, age_days=400)]
|
||||||
|
)
|
||||||
|
repaused = [e for e in events if e.code == "filing_gap_repaused"]
|
||||||
|
assert len(repaused) == 1
|
||||||
|
assert f"{cik}/{cik}-AGED-Q" in repaused[0].message
|
||||||
|
# Cleared, so a later recovery can re-arm and lapse again.
|
||||||
|
assert gaps[0].exempted_at is None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_an_exemption_taking_effect_is_stamped_silently(engine):
|
||||||
|
"""Setups resuming is what filing_gap_aged already described — stamping the
|
||||||
|
state must not raise a second alert for it."""
|
||||||
|
cik = "0000000061"
|
||||||
|
gaps, events = await _promote_only(
|
||||||
|
engine, [_stale_gap(cik, exempted=False), _snapshot(cik, age_days=120)]
|
||||||
|
)
|
||||||
|
assert [e.code for e in events if e.code.startswith("filing_gap")] == []
|
||||||
|
assert gaps[0].exempted_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_still_paused_gap_is_not_reported_as_lapsing(engine):
|
||||||
|
"""It never became exempt, so there is no transition to report."""
|
||||||
|
cik = "0000000062"
|
||||||
|
gaps, events = await _promote_only(
|
||||||
|
engine, [_stale_gap(cik, exempted=False), _snapshot(cik, age_days=400)]
|
||||||
|
)
|
||||||
|
assert [e for e in events if e.code == "filing_gap_repaused"] == []
|
||||||
|
assert gaps[0].exempted_at is None
|
||||||
|
|||||||
Reference in New Issue
Block a user