fix(sec): A3 slice-2b review — no index cap, full discrepancy + malformed gate
1. Removed the 45-day index-walk cap: it discarded the older part of a long outage while still advancing source_max_date, permanently losing filings. The walk now covers every unprocessed date (a large gap is one-time cost). 2. Discrepancy detection meets the immutability contract: it compares ALL source snapshot fields (not five), read-only during stage/validate, reports the differing accessions + fields in validation_json, and promote emits a warning system event (in-transaction) — never mutating the stored row. 3. Malformed companyfacts (missing facts/units structure) are recorded separately and FAIL validation, instead of silently degrading to skipped rows that the 50% backfill coverage floor could still pass. Also corrected the stale "sum share classes" / DEI-only wording in the snapshot model docstring and the A3 design doc to describe the us-gaap fallback. Tests: +4 regressions (>45-day gap loses nothing, newly-added issuer backfills without filing, malformed payload fails, shares discrepancy detected + evented). 23 passed, 1 skipped. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -50,9 +50,9 @@ def _shares(end, val, accn, fy, fp):
|
||||
return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
||||
|
||||
|
||||
def _companyfacts(rev_facts, share_facts):
|
||||
def _companyfacts(rev_facts, share_facts, cik=320193):
|
||||
return {
|
||||
"cik": 320193,
|
||||
"cik": cik,
|
||||
"facts": {
|
||||
"us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}},
|
||||
"dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}},
|
||||
@@ -269,36 +269,119 @@ async def test_failed_backfill_leaves_tickers_unwritten(engine):
|
||||
assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars())
|
||||
|
||||
|
||||
async def test_promote_conflict_reports_discrepancy_without_mutation(engine):
|
||||
from app.services.sec_facts_parser import SnapshotRow
|
||||
from app.services.sec_fundamentals_importer import StagedFundamentals
|
||||
from app.services.sec_universe import ResolvedUniverse
|
||||
|
||||
async def test_index_gap_over_45_days_loses_no_filings(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
backfill = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
await run_import(_importer(backfill), engine=engine)
|
||||
|
||||
# 74-day gap; the filing sits in the OLD part (>45d before latest).
|
||||
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "OLD")
|
||||
sh_q2 = _shares("2026-04-17", 14687, "OLD", 2026, "Q2")
|
||||
incr = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
||||
submissions={320193: _submissions(SUB_FILINGS + [
|
||||
_filing("OLD", "10-Q", "2026-03-28", "2026-02-10", "2026-02-10T10:01:00.000Z")])},
|
||||
latest_index=date(2026, 4, 15),
|
||||
daily={date(2026, 2, 10): [{"form": "10-Q", "cik": 320193, "accession": "OLD"}]},
|
||||
)
|
||||
run = await run_import(_importer(incr, today=date(2026, 4, 16)), engine=engine)
|
||||
assert run.status == STATUS_PROMOTED
|
||||
assert await _count(factory, FundamentalSnapshot) == 3 # the old-gap filing was NOT lost
|
||||
|
||||
|
||||
async def test_newly_added_issuer_backfills_without_filing(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
backfill = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
await run_import(_importer(backfill), engine=engine)
|
||||
|
||||
# MSFT added to the universe later; it did NOT file (not in the daily index).
|
||||
await _seed(factory, ["MSFT"])
|
||||
msft_rev = _rev("2024-07-01", "2025-06-30", 270000, 2025, "FY", "M")
|
||||
msft_sh = _shares("2025-07-15", 7400, "M", 2025, "FY")
|
||||
incr = FakeSecClient(
|
||||
tickers={"AAPL": 320193, "MSFT": 789019},
|
||||
companyfacts={
|
||||
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
||||
789019: _companyfacts([msft_rev], [msft_sh], cik=789019),
|
||||
},
|
||||
submissions={
|
||||
320193: _submissions(SUB_FILINGS),
|
||||
789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software",
|
||||
"filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]},
|
||||
},
|
||||
latest_index=date(2026, 2, 3),
|
||||
daily={}, # MSFT did not file
|
||||
)
|
||||
run = await run_import(_importer(incr, today=date(2026, 2, 4)), engine=engine)
|
||||
assert run.status == STATUS_PROMOTED
|
||||
async with factory() as s:
|
||||
msft = (await s.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.cik == "0000789019")
|
||||
)).scalars().all()
|
||||
assert len(msft) == 1 and msft[0].revenue == 270000 # full-history backfill despite no filing
|
||||
|
||||
|
||||
async def test_malformed_companyfacts_fails_validation(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL", "MSFT"])
|
||||
client = FakeSecClient(
|
||||
tickers={"AAPL": 320193, "MSFT": 789019},
|
||||
companyfacts={
|
||||
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
||||
789019: {"cik": 789019}, # malformed — no "facts" structure
|
||||
},
|
||||
submissions={
|
||||
320193: _submissions(SUB_FILINGS),
|
||||
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
||||
},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
run = await run_import(_importer(client), engine=engine)
|
||||
assert run.status == STATUS_FAILED
|
||||
assert "malformed" in (run.error_details or "")
|
||||
assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted
|
||||
|
||||
|
||||
async def test_discrepancy_in_shares_is_detected_and_reported(engine):
|
||||
from app.models.system_event import SystemEvent
|
||||
utc = timezone.utc
|
||||
async with factory() as s: # pre-existing immutable snapshot K (revenue 100, run 1)
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
# Pre-store accession K matching what the parser will produce EXCEPT shares.
|
||||
async with factory() as s:
|
||||
s.add(FundamentalSnapshot(
|
||||
cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
||||
accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
|
||||
fiscal_year=2025, fiscal_period="FY", revenue=100.0, import_run_id=1,
|
||||
created_at=datetime(2025, 10, 31, tzinfo=utc)))
|
||||
accepted_at=datetime(2025, 10, 31, 10, 1, 26, tzinfo=utc), period_start=date(2024, 9, 29),
|
||||
period_end=date(2025, 9, 27), fiscal_year=2025, fiscal_period="FY", revenue=416161.0,
|
||||
shares_outstanding=999.0, import_run_id=1, created_at=datetime(2025, 10, 31, tzinfo=utc)))
|
||||
await s.commit()
|
||||
|
||||
k_diff = SnapshotRow(cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
||||
accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
|
||||
fiscal_year=2025, fiscal_period="FY", revenue=999.0) # differs
|
||||
n_new = SnapshotRow(cik="0000320193", accession="N", form="10-Q", filed_date=date(2026, 1, 30),
|
||||
accepted_at=datetime(2026, 1, 30, tzinfo=utc), period_end=date(2025, 12, 27),
|
||||
fiscal_year=2026, fiscal_period="Q1", revenue=143.0)
|
||||
staged = StagedFundamentals(resolved=ResolvedUniverse(), rows=[k_diff, n_new])
|
||||
client = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # SH_K = 14776 != 999
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
run = await run_import(_importer(client), engine=engine)
|
||||
|
||||
imp = SecFundamentalsImporter(client_factory=lambda: None)
|
||||
async with factory() as s:
|
||||
counts = await imp.promote(s, staged, run_id=2)
|
||||
await s.commit()
|
||||
|
||||
assert counts["inserted"] == 1 and counts["discrepancies"] == 1
|
||||
assert run.status == STATUS_PROMOTED # a discrepancy is reported, not a failure
|
||||
assert '"discrepancy_count": 1' in (run.validation_json or "")
|
||||
assert "shares_outstanding" in (run.validation_json or "")
|
||||
async with factory() as s:
|
||||
k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one()
|
||||
assert k.revenue == 100.0 and k.import_run_id == 1 # immutable — not overwritten
|
||||
assert (await s.execute(select(func.count()).select_from(FundamentalSnapshot))).scalar_one() == 2
|
||||
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
||||
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
||||
assert len(events) == 1 and events[0].severity == "warning"
|
||||
|
||||
Reference in New Issue
Block a user