dennisthiessenandClaude Opus 5
247a7889b9
active_only tested delisted_on IS NULL, so a symbol dropped out of signals the moment a Form 25 was detected — ten days before Rule 12d2-2 makes the removal effective, while it was demonstrably still trading. A manual future-dated mark behaved the same way. It now compares against the database's own date, so a pending delisting stays live until the day it takes effect. That exposes a second problem the fix would otherwise create. Trading typically stops before the ten-day delay expires, so across that window the symbol is correctly active yet produces no bars — and confirm_delisting returned None for an already-marked row, which would have fired the staleness warning daily for ten days, the exact noise this flow exists to remove. It now reports the known effective date on every path where the delisting is established, so the caller warns only about gaps that are still unexplained. CURRENT_DATE renders identically on postgres and sqlite, and the OR is parenthesized when callers chain further where clauses. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
417 lines
16 KiB
Python
417 lines
16 KiB
Python
"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation.
|
|
|
|
The behaviour under test is that a delisted symbol leaves the *live* path while
|
|
its rows stay put — deleting it instead is what makes the backtest universe
|
|
survivorship-biased, so retention is the point, not a side effect.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import AsyncGenerator
|
|
from datetime import date
|
|
|
|
import httpx
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
from app.models.ticker import Ticker
|
|
from app.services import ticker_service
|
|
from app.services.sec_client import SecClient
|
|
|
|
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
|
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def _setup_tables() -> AsyncGenerator[None, None]:
|
|
async with _engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
yield
|
|
async with _engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.drop_all)
|
|
|
|
|
|
@pytest.fixture
|
|
async def session() -> AsyncGenerator[AsyncSession, None]:
|
|
async with _session_factory() as s:
|
|
yield s
|
|
|
|
|
|
def _submissions(forms: list[str], dates: list[str]) -> dict:
|
|
return {
|
|
"cik": 712515,
|
|
"name": "ELECTRONIC ARTS INC.",
|
|
"filings": {
|
|
"recent": {
|
|
"form": forms,
|
|
"filingDate": dates,
|
|
"accessionNumber": [f"0001354457-26-{i:06d}" for i in range(len(forms))],
|
|
"primaryDocument": ["xslF25X02/primary_doc.xml"] * len(forms),
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _sec_client(payload: dict, security: str | None = "Common Stock") -> SecClient:
|
|
"""Mock submissions + the Form 25 primary document the class check reads."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("primary_doc.xml"):
|
|
if security is None:
|
|
return httpx.Response(404)
|
|
body = (
|
|
"<?xml version='1.0'?><notificationOfRemoval>"
|
|
f"<descriptionClassSecurity>{security}</descriptionClassSecurity>"
|
|
"</notificationOfRemoval>"
|
|
)
|
|
return httpx.Response(200, content=body.encode())
|
|
return httpx.Response(200, content=json.dumps(payload).encode())
|
|
|
|
return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
|
|
|
|
|
|
async def test_mark_delisted_is_idempotent(session: AsyncSession):
|
|
session.add(Ticker(symbol="EA"))
|
|
await session.commit()
|
|
|
|
assert await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 8, 4)
|
|
) is True
|
|
# A second call must not churn the row — the staleness path retries daily.
|
|
assert await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 9, 1)
|
|
) is False
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on == date(2026, 8, 4) # first date wins, not the retry
|
|
assert row.delisted_reason == ticker_service.REASON_MANUAL
|
|
|
|
|
|
async def test_clear_delisted_restores_the_symbol(session: AsyncSession):
|
|
session.add(Ticker(symbol="EA"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
assert await ticker_service.clear_delisted(session, "EA") is True
|
|
assert await ticker_service.clear_delisted(session, "EA") is False
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on is None and row.delisted_reason is None
|
|
|
|
|
|
async def test_active_only_filters_but_the_row_survives(session: AsyncSession):
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
active = (
|
|
await session.execute(ticker_service.active_only(select(Ticker.symbol)))
|
|
).scalars().all()
|
|
assert list(active) == ["AAPL"]
|
|
|
|
# The whole point: the row — and everything cascading off it — is still there.
|
|
everything = [t.symbol for t in await ticker_service.list_tickers(session)]
|
|
assert everything == ["AAPL", "EA"]
|
|
|
|
|
|
async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch):
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
|
|
marked = await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
)
|
|
# Removal is effective ten days after the 2026-08-04 filing, not on it.
|
|
assert marked == date(2026, 8, 14)
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_reason == ticker_service.REASON_FORM_25
|
|
|
|
|
|
async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch):
|
|
"""A halt or a rename files no Form 25 — those must keep warning, not retire."""
|
|
session.add(Ticker(symbol="SATS", cik="0000012345"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one()
|
|
assert row.delisted_on is None
|
|
|
|
|
|
async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession):
|
|
"""No CIK, no SEC lookup — must not raise, and must not mark."""
|
|
session.add(Ticker(symbol="ADRX"))
|
|
await session.commit()
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_delisting_filing_picks_the_newest_match():
|
|
client = _sec_client(
|
|
_submissions(
|
|
["25", "8-K", "25-NSE", "15-12B"],
|
|
["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"],
|
|
)
|
|
)
|
|
async with client as c:
|
|
found = await c.delisting_filing("0000712515")
|
|
assert found["form"] == "25-NSE"
|
|
assert found["filing_date"] == date(2026, 8, 4)
|
|
|
|
|
|
async def test_delisting_filing_returns_none_without_one():
|
|
client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"]))
|
|
async with client as c:
|
|
assert await c.delisting_filing("0000320193") is None
|
|
|
|
|
|
async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch):
|
|
"""A one-day gap is a weekend or a hiccup. Probing every stale symbol during a
|
|
market-data outage would be one SEC request per symbol per run."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
|
|
def _explode():
|
|
raise AssertionError("must not reach SEC before the stale threshold")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11)
|
|
) is None
|
|
# ...and no bars at all is an ingestion problem, not a delisting.
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=None, today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_sec_confirmation_upgrades_a_manual_mark(session: AsyncSession, monkeypatch):
|
|
"""An operator's estimated date is a guess; Form 25 carries the real one."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 11))
|
|
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25-NSE"], ["2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) == date(2026, 8, 14)
|
|
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on == date(2026, 8, 14)
|
|
assert row.delisted_reason == ticker_service.REASON_FORM_25
|
|
|
|
|
|
async def test_a_confirmed_row_is_never_reprobed(session: AsyncSession, monkeypatch):
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(
|
|
session, "EA", delisted_on=date(2026, 8, 4),
|
|
reason=ticker_service.REASON_FORM_25,
|
|
)
|
|
|
|
def _explode():
|
|
raise AssertionError("a SEC-confirmed row must not cost another request")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
# Costs no request, and still reports the date so the caller knows this gap
|
|
# is explained and must not warn about it again.
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 9, 1)
|
|
) == date(2026, 8, 4)
|
|
|
|
|
|
async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession):
|
|
"""Covers the one statement where active_only wraps a compound select."""
|
|
from app.scheduler import _get_ohlcv_priority_tickers
|
|
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="MSFT")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
|
|
|
symbols = await _get_ohlcv_priority_tickers(session)
|
|
assert "EA" not in symbols
|
|
assert sorted(symbols) == ["AAPL", "MSFT"]
|
|
|
|
|
|
async def test_a_form_25_for_another_security_class_is_ignored(session: AsyncSession, monkeypatch):
|
|
"""Form 25 is per security class. An issuer delisting its notes, preferred or
|
|
warrants files one while the common keeps trading — retiring the ticker on
|
|
that would remove an actively traded symbol from every signal."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(
|
|
_submissions(["25-NSE"], ["2026-08-04"]),
|
|
security="6.25% Notes due 2030",
|
|
),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
|
assert row.delisted_on is None
|
|
|
|
|
|
async def test_a_stale_historical_form_25_cannot_retire_a_symbol(session: AsyncSession, monkeypatch):
|
|
"""A 2019 filing for a long-gone class must not retire a symbol whose bars
|
|
ran until 2026 — and must certainly not stamp 2019 as the date."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25"], ["2019-03-01"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_form_15_alone_never_retires_a_symbol(session: AsyncSession, monkeypatch):
|
|
"""Form 15 ends a reporting obligation; it is not evidence trading stopped."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["15-12B", "15-12G"], ["2026-08-04", "2026-08-05"])),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
async def test_an_unreadable_form_25_fails_closed(session: AsyncSession, monkeypatch):
|
|
"""Pre-2009 filings have no primary_doc.xml. Unknown class must read as no."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25"], ["2026-08-04"]), security=None),
|
|
raising=False,
|
|
)
|
|
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"description,expected",
|
|
[
|
|
("Common Stock", True),
|
|
("Class A Common Stock, $0.01 par value", True),
|
|
("Common Shares, no par value", True),
|
|
("6.25% Notes due 2030", False),
|
|
("7.5% Series B Cumulative Preferred Stock", False),
|
|
("Warrants to purchase Common Stock", False),
|
|
("Depositary Shares each representing 1/1000th interest", False),
|
|
("", False),
|
|
],
|
|
)
|
|
def test_common_stock_classification(description: str, expected: bool):
|
|
from app.services.sec_client import _is_common_stock
|
|
|
|
assert _is_common_stock(description) is expected
|
|
|
|
|
|
async def test_prune_keeps_delisted_rows(session: AsyncSession, monkeypatch):
|
|
"""A prune must not destroy rows the delisting flow deliberately retained —
|
|
their price history is the whole reason those rows still exist."""
|
|
from app.services import ticker_universe_service as tus
|
|
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="GONE")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
|
|
|
|
async def fake_fetch(db, universe):
|
|
return ["AAPL"], "test"
|
|
|
|
monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch)
|
|
|
|
summary = await tus.bootstrap_universe(session, "sp500", prune_missing=True)
|
|
|
|
remaining = sorted(t.symbol for t in await ticker_service.list_tickers(session))
|
|
assert remaining == ["AAPL", "EA"] # GONE pruned, EA protected
|
|
assert summary["deleted"] == 1
|
|
assert summary["kept_delisted"] == ["EA"]
|
|
|
|
|
|
async def test_a_future_effective_date_keeps_the_symbol_live(session: AsyncSession):
|
|
"""Form 25 is known ten days before removal takes effect. The symbol is still
|
|
trading in that window and must keep being scanned and ingested."""
|
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
|
await session.commit()
|
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
|
|
|
|
def active(as_of: date) -> list[str]:
|
|
return ticker_service.active_only(select(Ticker.symbol), as_of=as_of)
|
|
|
|
before = (await session.execute(active(date(2026, 8, 11)))).scalars().all()
|
|
on_the_day = (await session.execute(active(date(2026, 8, 14)))).scalars().all()
|
|
after = (await session.execute(active(date(2026, 8, 15)))).scalars().all()
|
|
|
|
assert sorted(before) == ["AAPL", "EA"] # still trading
|
|
assert sorted(on_the_day) == ["AAPL"] # removal effective
|
|
assert sorted(after) == ["AAPL"]
|
|
|
|
|
|
async def test_the_pending_window_does_not_re_warn(session: AsyncSession, monkeypatch):
|
|
"""Between filing and effect the symbol is active but produces no bars. That
|
|
must not resurrect the daily staleness warning this flow exists to end."""
|
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
|
await session.commit()
|
|
monkeypatch.setattr(
|
|
ticker_service,
|
|
"_sec_client_factory",
|
|
lambda: _sec_client(_submissions(["25-NSE"], ["2026-08-04"])),
|
|
raising=False,
|
|
)
|
|
first = await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
|
)
|
|
assert first == date(2026, 8, 14)
|
|
|
|
def _explode():
|
|
raise AssertionError("must not re-probe a confirmed row")
|
|
|
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
|
# Every later run inside the window still reports the delisting, so the
|
|
# caller keeps emitting "delisted" rather than "no new bars".
|
|
for day in (date(2026, 8, 12), date(2026, 8, 13)):
|
|
assert await ticker_service.confirm_delisting(
|
|
session, "EA", last_bar=date(2026, 8, 4), today=day
|
|
) == date(2026, 8, 14)
|