feat(tickers): record delisting instead of deleting the symbol

Retiring a symbol meant delete_ticker or bootstrap_universe(prune_missing), both of which cascade through OHLCV, setups and scores. That destroys exactly the history four research documents already apologise for: today's tracked universe projected backward is survivorship-biased, and hard-deleting every delisted name is what causes it. Keeping the rows preserves the option to fix that — it does not fix it, which needs the replay to model a delisting as an exit event. tickers gains delisted_on / delisted_reason (migration 032). NULL means actively traded. The filter is opt-in via ticker_service.active_only rather than folded into a shared getter: the registry and admin views deliberately keep delisted rows so the delisting is visible, and a silent default would undo that. Applied to the live path only — scanner, momentum ranking, scoring, breadth, fundamentals candidates, SEC universe, earnings import, ingestion loops. run_backtest keeps them on purpose. Detection runs off OHLCV staleness, not off the SEC fundamentals import: that importer stalls for days on unrelated Company-Facts gaps and would take detection down with it. On a stale symbol the scheduler asks SEC for a Form 25/25-NSE/15 and retires it only on a hit, so a halt or a rename (SATS->ECHO) keeps the existing warning. The probe waits 3 stale days so a market-data outage cannot turn into one SEC request per symbol per run. Safe to automate because it is reversible: clear_delisted un-retires a false positive, where a delete had already taken the history. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-11 16:58:51 +02:00
parent 486fb500d1
commit 6501b7e9a0
15 changed files with 512 additions and 29 deletions
@@ -0,0 +1,183 @@
+"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation.
+
+The behaviour under test is that a delisted symbol leaves the *live* path while
+its rows stay put — deleting it instead is what makes the backtest universe
+survivorship-biased, so retention is the point, not a side effect.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncGenerator
+from datetime import date
+
+import httpx
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.database import Base
+from app.models.ticker import Ticker
+from app.services import ticker_service
+from app.services.sec_client import SecClient
+
+_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
+_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
+
+
+@pytest.fixture(autouse=True)
+async def _setup_tables() -> AsyncGenerator[None, None]:
+    async with _engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    yield
+    async with _engine.begin() as conn:
+        await conn.run_sync(Base.metadata.drop_all)
+
+
+@pytest.fixture
+async def session() -> AsyncGenerator[AsyncSession, None]:
+    async with _session_factory() as s:
+        yield s
+
+
+def _submissions(forms: list[str], dates: list[str]) -> dict:
+    return {
+        "cik": 712515,
+        "name": "ELECTRONIC ARTS INC.",
+        "filings": {"recent": {"form": forms, "filingDate": dates}},
+    }
+
+
+def _sec_client(payload: dict) -> SecClient:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, content=json.dumps(payload).encode())
+
+    return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
+
+
+async def test_mark_delisted_is_idempotent(session: AsyncSession):
+    session.add(Ticker(symbol="EA"))
+    await session.commit()
+
+    assert await ticker_service.mark_delisted(
+        session, "EA", delisted_on=date(2026, 8, 4)
+    ) is True
+    # A second call must not churn the row — the staleness path retries daily.
+    assert await ticker_service.mark_delisted(
+        session, "EA", delisted_on=date(2026, 9, 1)
+    ) is False
+
+    row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
+    assert row.delisted_on == date(2026, 8, 4)  # first date wins, not the retry
+    assert row.delisted_reason == ticker_service.REASON_MANUAL
+
+
+async def test_clear_delisted_restores_the_symbol(session: AsyncSession):
+    session.add(Ticker(symbol="EA"))
+    await session.commit()
+    await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
+
+    assert await ticker_service.clear_delisted(session, "EA") is True
+    assert await ticker_service.clear_delisted(session, "EA") is False
+
+    row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
+    assert row.delisted_on is None and row.delisted_reason is None
+
+
+async def test_active_only_filters_but_the_row_survives(session: AsyncSession):
+    session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
+    await session.commit()
+    await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
+
+    active = (
+        await session.execute(ticker_service.active_only(select(Ticker.symbol)))
+    ).scalars().all()
+    assert list(active) == ["AAPL"]
+
+    # The whole point: the row — and everything cascading off it — is still there.
+    everything = [t.symbol for t in await ticker_service.list_tickers(session)]
+    assert everything == ["AAPL", "EA"]
+
+
+async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch):
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])),
+        raising=False,
+    )
+
+    marked = await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    )
+    assert marked == date(2026, 8, 4)
+
+    row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
+    assert row.delisted_reason == ticker_service.REASON_FORM_25
+
+
+async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch):
+    """A halt or a rename files no Form 25 — those must keep warning, not retire."""
+    session.add(Ticker(symbol="SATS", cik="0000012345"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])),
+        raising=False,
+    )
+
+    assert await ticker_service.confirm_delisting(
+        session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+    row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one()
+    assert row.delisted_on is None
+
+
+async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession):
+    """No CIK, no SEC lookup — must not raise, and must not mark."""
+    session.add(Ticker(symbol="ADRX"))
+    await session.commit()
+    assert await ticker_service.confirm_delisting(
+        session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+
+
+async def test_delisting_filing_picks_the_newest_match():
+    client = _sec_client(
+        _submissions(
+            ["25", "8-K", "25-NSE", "15-12B"],
+            ["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"],
+        )
+    )
+    async with client as c:
+        found = await c.delisting_filing("0000712515")
+    assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)}
+
+
+async def test_delisting_filing_returns_none_without_one():
+    client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"]))
+    async with client as c:
+        assert await c.delisting_filing("0000320193") is None
+
+
+async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch):
+    """A one-day gap is a weekend or a hiccup. Probing every stale symbol during a
+    market-data outage would be one SEC request per symbol per run."""
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+
+    def _explode():
+        raise AssertionError("must not reach SEC before the stale threshold")
+
+    monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
+
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11)
+    ) is None
+    # ...and no bars at all is an ingestion problem, not a delisting.
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=None, today=date(2026, 8, 11)
+    ) is None