dennisthiessenandClaude Opus 5
ad3e47dc50
bootstrap_universe(prune_missing=True) ran DELETE FROM tickers, cascading OHLCV, setups, scores and S/R levels away for any symbol that had merely dropped out of the index. Migration 032's own comment predicted this. The survivorship audit measured what it costs: 2.09 Sharpe against 1.57 on universe composition alone. Migration 035 adds untracked_on, the sibling of delisted_on — delisted means the security stopped trading, untracked means a prune took it out of the scanner. Nothing is deleted either way. Migration 036 separates three more things that were still conflated: universe_memberships what the index actually contained, when tickers.untracked_on excluded from scanning by a prune tickers.scan_disabled_on the operator's own "stop scanning this" tickers.tracking_source who owns the row: a bootstrap, or an operator The operator flag is its own column because with one column serving both writers, a prune -> re-entry cycle would clear an explicit "off" — re-entry cannot tell who set it. Re-entry clears untracked_on and never touches scan_disabled_on. untracked_on is NOT current index membership. The scheduled sync runs prune_missing=False, so a symbol can have left the index and still be scanned; the two legitimately differ and nothing asserts they agree. Tracking ownership is sticky towards manual: prune touches index-owned rows only, and a bootstrap never converts manual to index — a hand-added symbol that joins the index would otherwise lose its protection the moment it left again, which is exactly when it is needed. An explicit add of an index-owned symbol claims it as manual without touching either off-switch. Membership is observed, never asserted. first_seen_on is the first live sighting; absent_since is the first observation that found the symbol GONE, so the interval is [first_seen_on, absent_since) — named for what it stores rather than "last_seen_on", which would have claimed the opposite. Intervals are append-only, so a symbol that leaves and returns has two and a point-in-time query for the gap correctly answers "not a member". members_on() implements the boundary rule once. A date before the first live sync returns empty: that means "we were not looking", not "the index was empty". A failed fetch drives nothing. The cache and seed fallbacks mean the live fetch failed, and the seed lists are ~44 symbols against a ~500-symbol universe — a regression test caught a seed fallback untracking 2 of 3 tickers. Recording, pruning and clearing prune flags all require a live source; only adding still runs, so a cold start can still bootstrap from seeds. Migration decisions: existing rows become manual (fail-safe — a wrong guess costs an un-pruned symbol, not one swept out of the scanner), and existing untracked_on values are copied into scan_disabled_on, since the original writer is ambiguous and re-entry clears the prune column. Backtest eligibility is unchanged. _load_backtest_symbols deliberately does not use active_only, so today's scanner state cannot erase yesterday's candidates, and untracking a symbol still cannot silence a coverage gap in the report. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HAH7vVtn7qbcn1EVheecHQ
428 lines
17 KiB
Python
428 lines
17 KiB
Python
"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation.
|
|
|
|
The behaviour under test is that a delisted symbol leaves the *live* path while
|
|
its rows stay put — deleting it instead is what makes the backtest universe
|
|
survivorship-biased, so retention is the point, not a side effect.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import AsyncGenerator
|
|
from datetime import date
|
|
|
|
import httpx
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
from app.models.ticker import Ticker
|
|
from app.services import ticker_service
|
|
from app.services.sec_client import SecClient
|
|
|
|
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
|
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def _setup_tables() -> AsyncGenerator[None, None]:
|
|
async with _engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
yield
|
|
async with _engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.drop_all)
|
|
|
|
|
|
@pytest.fixture
|
|
async def session() -> AsyncGenerator[AsyncSession, None]:
|
|
async with _session_factory() as s:
|
|
yield s
|
|
|
|
|
|
def _submissions(forms: list[str], dates: list[str]) -> dict:
|
|
return {
|
|
"cik": 712515,
|
|
"name": "ELECTRONIC ARTS INC.",
|
|
"filings": {
|
|
"recent": {
|
|
"form": forms,
|
|
"filingDate": dates,
|
|
"accessionNumber": [f"0001354457-26-{i:06d}" for i in range(len(forms))],
|
|
"primaryDocument": ["xslF25X02/primary_doc.xml"] * len(forms),
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _sec_client(payload: dict, security: str | None = "Common Stock") -> SecClient:
|
|
"""Mock submissions + the Form 25 primary document the class check reads."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("primary_doc.xml"):
|
|
if security is None:
|
|
return httpx.Response(404)
|
|
body = (
|
|
"<?xml version='1.0'?><notificationOfRemoval>"
|
|
f"<descriptionClassSecurity>{security}</descriptionClassSecurity>"
|
|
"</notificationOfRemoval>"
|
|
)
|
|
return httpx.Response(200, content=body.encode())
|
|
return httpx.Response(200, content=json.dumps(payload).encode())
|
|
|
|
return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
|
|
|
|
|
|
async def test_mark_delisted_is_idempotent(session: AsyncSession):
|
|
session.add(Ticker(symbol="EA"))
|
|
await session.commit()
|
|
|
|
assert await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 8, 4)
|
|
) is True
|
|
# A second call must not churn the row — the staleness path retries daily.
|
|
assert await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 9, 1)
|
|
) is False
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on == date(2026, 8, 4) # first date wins, not the retry
|
|
assert row.delisted_reason == ticker_service.REASON_MANUAL
|
|
|
|
|
|
async def test_clear_delisted_restores_the_symbol(session: AsyncSession):
|
|
session.add(Ticker(symbol="EA"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
assert await ticker_service.clear_delisted(session, "EA") is True
|
|
assert await ticker_service.clear_delisted(session, "EA") is False
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on is None and row.delisted_reason is None
|
|
|
|
|
|
async def test_active_only_filters_but_the_row_survives(session: AsyncSession):
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
active = (
|
|
await session.execute(ticker_service.active_only(select(Ticker.symbol)))
|
|
).scalars().all()
|
|
assert list(active) == ["AAPL"]
|
|
|
|
# The whole point: the row — and everything cascading off it — is still there.
|
|
everything = [t.symbol for t in await ticker_service.list_tickers(session)]
|
|
assert everything == ["AAPL", "EA"]
|
|
|
|
|
|
async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch):
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
|
|
marked = await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
)
|
|
# Removal is effective ten days after the 2026-08-04 filing, not on it.
|
|
assert marked == date(2026, 8, 14)
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_reason == ticker_service.REASON_FORM_25
|
|
|
|
|
|
async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch):
|
|
"""A halt or a rename files no Form 25 — those must keep warning, not retire."""
|
|
session.add(Ticker(symbol="SATS", cik="0000012345"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one()
|
|
assert row.delisted_on is None
|
|
|
|
|
|
async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession):
|
|
"""No CIK, no SEC lookup — must not raise, and must not mark."""
|
|
session.add(Ticker(symbol="ADRX"))
|
|
await session.commit()
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_delisting_filing_picks_the_newest_match():
|
|
client = _sec_client(
|
|
_submissions(
|
|
["25", "8-K", "25-NSE", "15-12B"],
|
|
["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"],
|
|
)
|
|
)
|
|
async with client as c:
|
|
found = await c.delisting_filing("0000712515")
|
|
assert found["form"] == "25-NSE"
|
|
assert found["filing_date"] == date(2026, 8, 4)
|
|
|
|
|
|
async def test_delisting_filing_returns_none_without_one():
|
|
client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"]))
|
|
async with client as c:
|
|
assert await c.delisting_filing("0000320193") is None
|
|
|
|
|
|
async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch):
|
|
"""A one-day gap is a weekend or a hiccup. Probing every stale symbol during a
|
|
market-data outage would be one SEC request per symbol per run."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
|
|
def _explode():
|
|
raise AssertionError("must not reach SEC before the stale threshold")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11)
|
|
) is None
|
|
# ...and no bars at all is an ingestion problem, not a delisting.
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=None, today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_sec_confirmation_upgrades_a_manual_mark(session: AsyncSession, monkeypatch):
|
|
"""An operator's estimated date is a guess; Form 25 carries the real one."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 11))
|
|
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25-NSE"], ["2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) == date(2026, 8, 14)
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on == date(2026, 8, 14)
|
|
assert row.delisted_reason == ticker_service.REASON_FORM_25
|
|
|
|
|
|
async def test_a_confirmed_row_is_never_reprobed(session: AsyncSession, monkeypatch):
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 8, 4),
|
|
reason=ticker_service.REASON_FORM_25,
|
|
)
|
|
|
|
def _explode():
|
|
raise AssertionError("a SEC-confirmed row must not cost another request")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
# Costs no request, and still reports the date so the caller knows this gap
|
|
# is explained and must not warn about it again.
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 9, 1)
|
|
) == date(2026, 8, 4)
|
|
|
|
|
|
async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession):
|
|
"""Covers the one statement where active_only wraps a compound select."""
|
|
from app.scheduler import _get_ohlcv_priority_tickers
|
|
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="MSFT")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
symbols = await _get_ohlcv_priority_tickers(session)
|
|
assert "EA" not in symbols
|
|
assert sorted(symbols) == ["AAPL", "MSFT"]
|
|
|
|
|
|
async def test_a_form_25_for_another_security_class_is_ignored(session: AsyncSession, monkeypatch):
|
|
"""Form 25 is per security class. An issuer delisting its notes, preferred or
|
|
warrants files one while the common keeps trading — retiring the ticker on
|
|
that would remove an actively traded symbol from every signal."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(
|
|
_submissions(["25-NSE"], ["2026-08-04"]),
|
|
security="6.25% Notes due 2030",
|
|
),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on is None
|
|
|
|
|
|
async def test_a_stale_historical_form_25_cannot_retire_a_symbol(session: AsyncSession, monkeypatch):
|
|
"""A 2019 filing for a long-gone class must not retire a symbol whose bars
|
|
ran until 2026 — and must certainly not stamp 2019 as the date."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25"], ["2019-03-01"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_form_15_alone_never_retires_a_symbol(session: AsyncSession, monkeypatch):
|
|
"""Form 15 ends a reporting obligation; it is not evidence trading stopped."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["15-12B", "15-12G"], ["2026-08-04", "2026-08-05"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_an_unreadable_form_25_fails_closed(session: AsyncSession, monkeypatch):
|
|
"""Pre-2009 filings have no primary_doc.xml. Unknown class must read as no."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25"], ["2026-08-04"]), security=None),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"description,expected",
|
|
[
|
|
("Common Stock", True),
|
|
("Class A Common Stock, $0.01 par value", True),
|
|
("Common Shares, no par value", True),
|
|
("6.25% Notes due 2030", False),
|
|
("7.5% Series B Cumulative Preferred Stock", False),
|
|
("Warrants to purchase Common Stock", False),
|
|
("Depositary Shares each representing 1/1000th interest", False),
|
|
("", False),
|
|
],
|
|
)
|
|
def test_common_stock_classification(description: str, expected: bool):
|
|
from app.services.sec_client import _is_common_stock
|
|
|
|
assert _is_common_stock(description) is expected
|
|
|
|
|
|
async def test_prune_keeps_every_row(session: AsyncSession, monkeypatch):
|
|
"""A prune destroys nothing at all now — not the delisted row it always
|
|
protected, and not the ex-constituent it used to delete outright."""
|
|
from app.services import ticker_universe_service as tus
|
|
|
|
session.add_all([
|
|
Ticker(symbol=s, tracking_source=ticker_service.SOURCE_INDEX)
|
|
for s in ("AAPL", "EA", "GONE")
|
|
])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
|
|
|
|
async def fake_fetch(db, universe):
|
|
# A live source label: pruning only acts on a successful fetch.
|
|
return ["AAPL"], "wikipedia_sp500"
|
|
|
|
monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch)
|
|
|
|
summary = await tus.bootstrap_universe(session, "sp500", prune_missing=True)
|
|
|
|
remaining = sorted(t.symbol for t in await ticker_service.list_tickers(session))
|
|
assert remaining == ["AAPL", "EA", "GONE"]
|
|
assert summary["deleted"] == 0
|
|
assert summary["untracked"] == 2
|
|
|
|
# Both leave the live path, by different routes: EA stopped trading, GONE
|
|
# merely left the index.
|
|
live = await session.execute(
|
|
ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol))
|
|
)
|
|
assert list(live.scalars().all()) == ["AAPL"]
|
|
|
|
|
|
async def test_a_future_effective_date_keeps_the_symbol_live(session: AsyncSession):
|
|
"""Form 25 is known ten days before removal takes effect. The symbol is still
|
|
trading in that window and must keep being scanned and ingested."""
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
|
|
|
|
def active(as_of: date) -> list[str]:
|
|
return ticker_service.active_only(select(Ticker.symbol), as_of=as_of)
|
|
|
|
before = (await session.execute(active(date(2026, 8, 11)))).scalars().all()
|
|
on_the_day = (await session.execute(active(date(2026, 8, 14)))).scalars().all()
|
|
after = (await session.execute(active(date(2026, 8, 15)))).scalars().all()
|
|
|
|
assert sorted(before) == ["AAPL", "EA"] # still trading
|
|
assert sorted(on_the_day) == ["AAPL"] # removal effective
|
|
assert sorted(after) == ["AAPL"]
|
|
|
|
|
|
async def test_the_pending_window_does_not_re_warn(session: AsyncSession, monkeypatch):
|
|
"""Between filing and effect the symbol is active but produces no bars. That
|
|
must not resurrect the daily staleness warning this flow exists to end."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25-NSE"], ["2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
first = await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
)
|
|
assert first == date(2026, 8, 14)
|
|
|
|
def _explode():
|
|
raise AssertionError("must not re-probe a confirmed row")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
# Every later run inside the window still reports the delisting, so the
|
|
# caller keeps emitting "delisted" rather than "no new bars".
|
|
for day in (date(2026, 8, 12), date(2026, 8, 13)):
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=day
|
|
) == date(2026, 8, 14)
|