fix(tickers): close the delisting review findings

Detection could retire an actively traded symbol — silently, since it then
vanishes from every signal. Three causes:

- Form 25 is filed per security class. An issuer removing its notes, preferred
  or warrants files one while the common keeps trading. The filing's own
  descriptionClassSecurity distinguishes them, so the primary document is now
  fetched and read; anything not recognisably common equity is rejected, as is
  anything unreadable (pre-2009 filings have no primary_doc.xml). Fail closed.
- Form 15 ends a reporting obligation and is no evidence trading stopped. The
  whole family is dropped.
- A historical filing for a long-gone class could retire a symbol whose bars ran
  years later, stamping the old date. Filings before the last bar (less a 30-day
  lead for the exchange) are now ignored.

Rule 12d2-2 makes removal effective ten days after filing, so delisted_on is the
effective date rather than the filing date.

bootstrap_universe(prune_missing=True) still ran a cascading delete over
delisted rows, undoing the retention this branch exists for; it now skips them
and reports kept_delisted so the count is explicable.

clear_delisted had no route, which made "safe to automate because it is
reversible" false — reversal needed SQL. POST/DELETE /tickers/{symbol}/delisting
now mark and un-mark, giving an operator a non-destructive alternative to the
cascading DELETE that was the only option.

Shared-CIK siblings (GOOG/GOOGL) stay safe by construction: the probe is
per-symbol and gated on that symbol's own staleness, so a class that still
trades is never probed.

Not addressed: pruning a symbol merely dropped from the index still destroys its
history — the same survivorship problem in a different costume, needing a
tracked/membership state separate from delisting.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/routers/tickers.py

@@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.dependencies import get_db, require_access
 from app.models.user import User
 from app.schemas.common import APIEnvelope
-from app.schemas.ticker import TickerCreate, TickerResponse
+from app.schemas.ticker import TickerCreate, TickerDelistingUpdate, TickerResponse
 from app.services import ticker_service
 
 router = APIRouter(tags=["tickers"])
@@ -51,3 +51,35 @@ async def delete_ticker(
     """Delete a ticker and all associated data."""
     await ticker_service.delete_ticker(db, symbol)
     return APIEnvelope(status="success", data=None)
+
+
+@router.post("/tickers/{symbol}/delisting", response_model=APIEnvelope)
+async def mark_ticker_delisted(
+    symbol: str,
+    body: TickerDelistingUpdate,
+    _user: User = Depends(require_access),
+    db: AsyncSession = Depends(get_db),
+):
+    """Retire a symbol: excluded from signals, price history kept.
+
+    The non-destructive alternative to DELETE, which cascades the history away.
+    """
+    changed = await ticker_service.mark_delisted(
+        db, symbol, delisted_on=body.delisted_on, reason=ticker_service.REASON_MANUAL
+    )
+    return APIEnvelope(status="success", data={"changed": changed})
+
+
+@router.delete("/tickers/{symbol}/delisting", response_model=APIEnvelope)
+async def clear_ticker_delisting(
+    symbol: str,
+    _user: User = Depends(require_access),
+    db: AsyncSession = Depends(get_db),
+):
+    """Un-retire a symbol wrongly marked delisted.
+
+    Automatic marking is only defensible because this exists: a false positive
+    costs one row update rather than the price history a delete would take.
+    """
+    changed = await ticker_service.clear_delisted(db, symbol)
+    return APIEnvelope(status="success", data={"changed": changed})

app/schemas/ticker.py

@@ -21,3 +21,9 @@ class TickerResponse(BaseModel):
     delisted_reason: str | None = None
 
     model_config = {"from_attributes": True}
+
+
+class TickerDelistingUpdate(BaseModel):
+    delisted_on: date = Field(
+        ..., description="Effective date the symbol stopped trading"
+    )

app/services/sec_client.py

@@ -45,12 +45,32 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
 
 _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
 
-# Exchange delisting (25 / 25-NSE) and registration termination (15 family).
-# Their presence is SEC confirming a security stopped trading — see
-# ``SecClient.delisting_filing``.
-_DELISTING_FORMS = frozenset({
-    "25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G",
-})
+# Notification of removal from listing. "25" is issuer-filed, "25-NSE" exchange-
+# filed. The Form 15 family is deliberately absent: it ends a *reporting*
+# obligation and does not mean the security stopped trading.
+_DELISTING_FORMS = frozenset({"25", "25-NSE"})
+
+# ``descriptionClassSecurity`` is free text ("Common Stock", "Class A Common
+# Stock, $0.01 par value", "6.25% Notes due 2030", "Warrants", "Depositary
+# Shares"). Only a common-equity class means the ticker itself stopped trading.
+_NON_COMMON_CLASS = re.compile(
+    r"\b(note|bond|debenture|preferred|warrant|right|unit|depositary|"
+    r"subordinated|debt|trust)s?\b",
+    re.IGNORECASE,
+)
+
+
+def _is_common_stock(description: str) -> bool:
+    """Does this Form 25 security class describe common equity?
+
+    Requires an explicit common-stock match AND no debt/preferred/warrant marker,
+    so "Depositary Shares each representing 1/1000th of Preferred" cannot pass on
+    the word "shares" alone. Unrecognised text is rejected — a symbol is retired
+    on this answer, so ambiguity must not read as yes.
+    """
+    if _NON_COMMON_CLASS.search(description):
+        return False
+    return re.search(r"\bcommon\s+(stock|share)", description, re.IGNORECASE) is not None
 
 
 class SecError(ProviderError):
@@ -260,23 +280,40 @@ class SecClient:
             "filings": filings,
         }
 
-    async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None:
-        """Newest exchange-delisting / deregistration filing, or ``None``.
+    async def delisting_filing(
+        self, cik: int | str, *, not_before: date | None = None
+    ) -> dict[str, Any] | None:
+        """Newest Form 25 removing this issuer's COMMON stock from listing.
 
-        Form 25/25-NSE strikes a security from listing; Form 15 terminates the
-        registration. Either is SEC confirming the security stopped trading —
-        which is what separates a real delisting from a multi-day halt or a
-        ticker rename, neither of which files one. That precision is the reason
-        this can mark a symbol automatically.
+        Deliberately narrow, because the caller retires a symbol on the answer:
 
-        Reads ``filings.recent`` directly: ``submissions()`` keeps only the
-        10-K/10-Q family, so these forms never survive its parser.
+        - **Form 25 only.** The Form 15 family terminates a reporting obligation
+          (often just a class falling under the holder threshold) and is no
+          evidence that trading stopped.
+        - **Class-checked.** Form 25 is filed per security class — an issuer
+          delisting its notes, preferred, warrants or an ADR class while the
+          common keeps trading files one too. The filing's own
+          ``descriptionClassSecurity`` is what separates those, so the primary
+          document is fetched and read rather than trusting the form type.
+        - **``not_before``** rejects a historical filing for some long-gone
+          class. Without it a 2019 Form 25 would retire a symbol whose bars
+          stopped in 2026, and stamp 2019 as the date.
+
+        Anything unreadable — no primary document (pre-2009 filings have none),
+        malformed XML, unrecognised class — returns ``None``. Fail closed: the
+        caller keeps warning instead of retiring on a guess.
+
+        Reads ``filings.recent`` directly; ``submissions()`` keeps only the
+        10-K/10-Q family, so Form 25 never survives its parser.
         """
         base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
         arrays = (base.get("filings") or {}).get("recent") or {}
         forms = arrays.get("form") or []
         dates = arrays.get("filingDate") or []
-        best: dict[str, Any] | None = None
+        accessions = arrays.get("accessionNumber") or []
+        docs = arrays.get("primaryDocument") or []
+
+        candidates: list[tuple[date, str, str, str]] = []
         for i, form in enumerate(forms):
             if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]:
                 continue
@@ -284,9 +321,49 @@ class SecClient:
                 filed = date.fromisoformat(dates[i])
             except ValueError:
                 continue
-            if best is None or filed > best["filing_date"]:
-                best = {"form": form, "filing_date": filed}
-        return best
+            if not_before is not None and filed < not_before:
+                continue
+            if i >= len(accessions) or not accessions[i]:
+                continue
+            candidates.append((filed, form, accessions[i], docs[i] if i < len(docs) else ""))
+
+        for filed, form, accession, _doc in sorted(candidates, reverse=True):
+            security = await self._form25_security_class(cik, accession)
+            if security is None:
+                continue
+            if not _is_common_stock(security):
+                continue
+            return {
+                "form": form,
+                "filing_date": filed,
+                "security_class": security,
+            }
+        return None
+
+    async def _form25_security_class(
+        self, cik: int | str, accession: str
+    ) -> str | None:
+        """``descriptionClassSecurity`` from a Form 25's primary XML, or None.
+
+        The rendered ``primaryDocument`` is an XSL view of this file; the raw
+        ``primary_doc.xml`` beside it is the structured original.
+        """
+        folder = accession.replace("-", "")
+        url = (
+            f"{_WWW}/Archives/edgar/data/{int(cik)}/{folder}/primary_doc.xml"
+        )
+        try:
+            body = await self.get_text(url)
+        except SecNotFoundError:
+            return None
+        match = re.search(
+            r"<descriptionClassSecurity>(.*?)</descriptionClassSecurity>",
+            body,
+            re.IGNORECASE | re.DOTALL,
+        )
+        if match is None:
+            return None
+        return " ".join(match.group(1).split()) or None
 
     async def companyfacts(self, cik: int | str) -> dict[str, Any]:
         """Raw companyfacts JSON ({cik, entityName, facts})."""

app/services/ticker_service.py

@@ -2,7 +2,7 @@
 
 import logging
 import re
-from datetime import date
+from datetime import date, timedelta
 
 from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -21,6 +21,15 @@ REASON_MANUAL = "manual"     # an operator decided
 # universe at once; a real delisting is still stale days later.
 MIN_STALE_DAYS_BEFORE_PROBE = 3
 
+# Rule 12d2-2: a Form 25 removal takes effect ten days after filing, so the
+# filing date is not the date the security stopped trading.
+FORM_25_EFFECTIVE_DAYS = 10
+
+# How far before the last bar a Form 25 may be filed and still explain this gap.
+# An exchange can file shortly before trading actually stops; anything older
+# concerns a class that was already gone while the symbol kept printing bars.
+FILING_LOOKBACK_DAYS = 30
+
 
 def _sec_client_factory():
     """Build the SEC client for a delisting probe (patched in tests).
@@ -178,7 +187,12 @@ async def confirm_delisting(
 
     try:
         async with _sec_client_factory() as client:
-            filing = await client.delisting_filing(ticker.cik)
+            # Only a Form 25 filed around or after the last bar can explain THIS
+            # gap. An older one belongs to a class that stopped trading before
+            # the symbol was still printing bars, and must not retire it.
+            filing = await client.delisting_filing(
+                ticker.cik, not_before=last_bar - timedelta(days=FILING_LOOKBACK_DAYS)
+            )
     except SecError:
         # Never let a probe failure escalate a routine staleness warning.
         logger.warning("delisting probe failed for %s", normalised, exc_info=True)
@@ -186,10 +200,13 @@ async def confirm_delisting(
 
     if filing is None:
         return None
+    # Removal takes effect ten days after filing, so the filing date is not the
+    # date the symbol stopped trading.
+    effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS)
     if await mark_delisted(
-        db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25
+        db, normalised, delisted_on=effective, reason=REASON_FORM_25
     ):
-        return filing["filing_date"]
+        return effective
     return None
 
 

app/services/ticker_universe_service.py

@@ -357,8 +357,25 @@ async def bootstrap_universe(
         db.add(Ticker(symbol=symbol))
 
     deleted_count = 0
+    skipped_delisted: list[str] = []
     if symbols_to_delete:
-        result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(symbols_to_delete)))
+        # A delisted row was retained on purpose — its price history is exactly
+        # what a survivorship-honest backtest needs, and the delete cascades it
+        # away. Pruning must not undo that. (Pruning a symbol that is merely no
+        # longer an index constituent still destroys history; that needs a
+        # tracked/membership state separate from delisting.)
+        protected = (
+            await db.execute(
+                select(Ticker.symbol).where(
+                    Ticker.symbol.in_(symbols_to_delete),
+                    Ticker.delisted_on.is_not(None),
+                )
+            )
+        ).scalars().all()
+        skipped_delisted = sorted(protected)
+        deletable = [s for s in symbols_to_delete if s not in set(protected)]
+        if deletable:
+            result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(deletable)))
             deleted_count = int(result.rowcount or 0)
 
     await db.commit()
@@ -378,4 +395,8 @@ async def bootstrap_universe(
         "already_tracked": len(target_symbols & existing_symbols),
         "deleted": deleted_count,
         "added_symbols": symbols_to_add[:50],
+        # Delisted rows a prune declined to destroy, so the caller can see the
+        # count did not match what they asked to remove.
+        "kept_delisted": skipped_delisted[:50],
+        "kept_delisted_count": len(skipped_delisted),
     }

tests/unit/test_ticker_delisting.py

@@ -44,12 +44,30 @@ def _submissions(forms: list[str], dates: list[str]) -> dict:
     return {
         "cik": 712515,
         "name": "ELECTRONIC ARTS INC.",
-        "filings": {"recent": {"form": forms, "filingDate": dates}},
+        "filings": {
+            "recent": {
+                "form": forms,
+                "filingDate": dates,
+                "accessionNumber": [f"0001354457-26-{i:06d}" for i in range(len(forms))],
+                "primaryDocument": ["xslF25X02/primary_doc.xml"] * len(forms),
+            }
+        },
     }
 
 
-def _sec_client(payload: dict) -> SecClient:
+def _sec_client(payload: dict, security: str | None = "Common Stock") -> SecClient:
+    """Mock submissions + the Form 25 primary document the class check reads."""
+
     def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path.endswith("primary_doc.xml"):
+            if security is None:
+                return httpx.Response(404)
+            body = (
+                "<?xml version='1.0'?><notificationOfRemoval>"
+                f"<descriptionClassSecurity>{security}</descriptionClassSecurity>"
+                "</notificationOfRemoval>"
+            )
+            return httpx.Response(200, content=body.encode())
         return httpx.Response(200, content=json.dumps(payload).encode())
 
     return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
@@ -112,7 +130,8 @@ async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monke
     marked = await ticker_service.confirm_delisting(
         session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
     )
-    assert marked == date(2026, 8, 4)
+    # Removal is effective ten days after the 2026-08-04 filing, not on it.
+    assert marked == date(2026, 8, 14)
 
     row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
     assert row.delisted_reason == ticker_service.REASON_FORM_25
@@ -154,7 +173,8 @@ async def test_delisting_filing_picks_the_newest_match():
     )
     async with client as c:
         found = await c.delisting_filing("0000712515")
-    assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)}
+    assert found["form"] == "25-NSE"
+    assert found["filing_date"] == date(2026, 8, 4)
 
 
 async def test_delisting_filing_returns_none_without_one():
@@ -197,10 +217,10 @@ async def test_sec_confirmation_upgrades_a_manual_mark(session: AsyncSession, mo
     )
     assert await ticker_service.confirm_delisting(
         session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
-    ) == date(2026, 8, 4)
+    ) == date(2026, 8, 14)
 
     row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
-    assert row.delisted_on == date(2026, 8, 4)
+    assert row.delisted_on == date(2026, 8, 14)
     assert row.delisted_reason == ticker_service.REASON_FORM_25
 
 
@@ -232,3 +252,116 @@ async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession):
     symbols = await _get_ohlcv_priority_tickers(session)
     assert "EA" not in symbols
     assert sorted(symbols) == ["AAPL", "MSFT"]
+
+
+async def test_a_form_25_for_another_security_class_is_ignored(session: AsyncSession, monkeypatch):
+    """Form 25 is per security class. An issuer delisting its notes, preferred or
+    warrants files one while the common keeps trading — retiring the ticker on
+    that would remove an actively traded symbol from every signal."""
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(
+            _submissions(["25-NSE"], ["2026-08-04"]),
+            security="6.25% Notes due 2030",
+        ),
+        raising=False,
+    )
+
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+    row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
+    assert row.delisted_on is None
+
+
+async def test_a_stale_historical_form_25_cannot_retire_a_symbol(session: AsyncSession, monkeypatch):
+    """A 2019 filing for a long-gone class must not retire a symbol whose bars
+    ran until 2026 — and must certainly not stamp 2019 as the date."""
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(_submissions(["25"], ["2019-03-01"])),
+        raising=False,
+    )
+
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+
+
+async def test_form_15_alone_never_retires_a_symbol(session: AsyncSession, monkeypatch):
+    """Form 15 ends a reporting obligation; it is not evidence trading stopped."""
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(_submissions(["15-12B", "15-12G"], ["2026-08-04", "2026-08-05"])),
+        raising=False,
+    )
+
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+
+
+async def test_an_unreadable_form_25_fails_closed(session: AsyncSession, monkeypatch):
+    """Pre-2009 filings have no primary_doc.xml. Unknown class must read as no."""
+    session.add(Ticker(symbol="EA", cik="0000712515"))
+    await session.commit()
+    monkeypatch.setattr(
+        ticker_service,
+        "_sec_client_factory",
+        lambda: _sec_client(_submissions(["25"], ["2026-08-04"]), security=None),
+        raising=False,
+    )
+
+    assert await ticker_service.confirm_delisting(
+        session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
+    ) is None
+
+
+@pytest.mark.parametrize(
+    "description,expected",
+    [
+        ("Common Stock", True),
+        ("Class A Common Stock, $0.01 par value", True),
+        ("Common Shares, no par value", True),
+        ("6.25% Notes due 2030", False),
+        ("7.5% Series B Cumulative Preferred Stock", False),
+        ("Warrants to purchase Common Stock", False),
+        ("Depositary Shares each representing 1/1000th interest", False),
+        ("", False),
+    ],
+)
+def test_common_stock_classification(description: str, expected: bool):
+    from app.services.sec_client import _is_common_stock
+
+    assert _is_common_stock(description) is expected
+
+
+async def test_prune_keeps_delisted_rows(session: AsyncSession, monkeypatch):
+    """A prune must not destroy rows the delisting flow deliberately retained —
+    their price history is the whole reason those rows still exist."""
+    from app.services import ticker_universe_service as tus
+
+    session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="GONE")])
+    await session.commit()
+    await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
+
+    async def fake_fetch(db, universe):
+        return ["AAPL"], "test"
+
+    monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch)
+
+    summary = await tus.bootstrap_universe(session, "sp500", prune_missing=True)
+
+    remaining = sorted(t.symbol for t in await ticker_service.list_tickers(session))
+    assert remaining == ["AAPL", "EA"]  # GONE pruned, EA protected
+    assert summary["deleted"] == 1
+    assert summary["kept_delisted"] == ["EA"]