feat(tickers): record delisting instead of deleting the symbol

Retiring a symbol meant delete_ticker or bootstrap_universe(prune_missing),
both of which cascade through OHLCV, setups and scores. That destroys exactly
the history four research documents already apologise for: today's tracked
universe projected backward is survivorship-biased, and hard-deleting every
delisted name is what causes it. Keeping the rows preserves the option to fix
that — it does not fix it, which needs the replay to model a delisting as an
exit event.

tickers gains delisted_on / delisted_reason (migration 032). NULL means
actively traded.

The filter is opt-in via ticker_service.active_only rather than folded into a
shared getter: the registry and admin views deliberately keep delisted rows so
the delisting is visible, and a silent default would undo that. Applied to the
live path only — scanner, momentum ranking, scoring, breadth, fundamentals
candidates, SEC universe, earnings import, ingestion loops. run_backtest keeps
them on purpose.

Detection runs off OHLCV staleness, not off the SEC fundamentals import: that
importer stalls for days on unrelated Company-Facts gaps and would take
detection down with it. On a stale symbol the scheduler asks SEC for a Form
25/25-NSE/15 and retires it only on a hit, so a halt or a rename (SATS->ECHO)
keeps the existing warning. The probe waits 3 stale days so a market-data
outage cannot turn into one SEC request per symbol per run.

Safe to automate because it is reversible: clear_delisted un-retires a false
positive, where a delete had already taken the history.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/services/breadth_service.py

@@ -25,6 +25,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.models.ticker import Ticker
+from app.services import ticker_service
 from app.services.price_service import query_ohlcv
 
 logger = logging.getLogger(__name__)
@@ -112,7 +113,7 @@ def compute_divergence_series(
 async def _load_universe_closes(
     db: AsyncSession, symbols: list[str] | None = None
 ) -> dict[str, Series]:
-    stmt = select(Ticker).order_by(Ticker.symbol)
+    stmt = ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
     if symbols is not None:
         stmt = stmt.where(Ticker.symbol.in_(symbols))
     result = await db.execute(stmt)

app/services/dolt_earnings_importer.py

@@ -32,7 +32,7 @@ from app.config import settings
 from app.database import insert_for_session
 from app.models.earnings_event import EarningsEvent
 from app.models.ticker import Ticker
-from app.services import dolt_client, earnings_alignment
+from app.services import dolt_client, earnings_alignment, ticker_service
 from app.services.data_import import ValidationResult
 
 logger = logging.getLogger(__name__)
@@ -289,7 +289,11 @@ class DoltEarningsImporter:
     # -- helpers -----------------------------------------------------------
 
     async def _load_universe(self, db) -> dict[str, int]:
-        rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all()
+        rows = (
+            await db.execute(
+                ticker_service.active_only(select(Ticker.id, Ticker.symbol))
+            )
+        ).all()
         return {
             earnings_alignment.normalise_symbol(symbol): tid
             for tid, symbol in rows

app/services/fundamentals_candidate_service.py

@@ -22,6 +22,7 @@ from app.models.fundamental_snapshot import FundamentalSnapshot
 from app.models.ohlcv import OHLCVRecord
 from app.models.ticker import Ticker
 from app.services import fundamentals_derivation as deriv
+from app.services import ticker_service
 
 
 @dataclass(frozen=True)
@@ -46,7 +47,11 @@ async def build_candidates(
     """Derive current cache candidates using only already-stored data."""
     today = today or datetime.now(ZoneInfo("America/New_York")).date()
     tickers = list(
-        (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
+        (
+            await db.execute(
+                ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
+            )
+        ).scalars()
     )
     if not tickers:
         return []

app/services/momentum_service.py

@@ -18,6 +18,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.models.ticker import Ticker
+from app.services import ticker_service
 from app.services.price_service import query_ohlcv
 
 logger = logging.getLogger(__name__)
@@ -169,7 +170,9 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa
     before scanning; the research backtest ranked each weekly setup-candidate
     cross-section, so this is the deliberate production approximation.
     """
-    result = await db.execute(select(Ticker).order_by(Ticker.symbol))
+    result = await db.execute(
+        ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
+    )
     tickers = list(result.scalars().all())
 
     benchmark_closes = await _load_activation_benchmark(db)

app/services/rr_scanner_service.py

@@ -31,7 +31,7 @@ from app.services import fundamentals_quality_service, system_event_service
 from app.services.price_service import query_ohlcv
 from app.services.qualification import setup_qualifies
 from app.services.sr_service import detect_gate_target_ladder
-from app.services import settings_store
+from app.services import settings_store, ticker_service
 from app.services.trade_policy import (
     MANUAL_BOOK,
     SHADOW_BOOK,
@@ -735,7 +735,11 @@ async def scan_all_tickers(
     # Plain ids/strings, not Ticker instances: the rollbacks below expire any
     # ORM objects held across them, and touching an expired attribute afterwards
     # triggers sync lazy-loading, which raises on an AsyncSession.
-    result = await db.execute(select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol))
+    result = await db.execute(
+        ticker_service.active_only(
+            select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)
+        )
+    )
     ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()]
     total = len(ticker_rows)
 

app/services/scoring_service.py

@@ -20,7 +20,7 @@ from app.database import insert_for_session
 from app.exceptions import NotFoundError, ValidationError
 from app.models.score import CompositeScore, DimensionScore
 from app.models.ticker import Ticker
-from app.services import settings_store
+from app.services import settings_store, ticker_service
 
 logger = logging.getLogger(__name__)
 
@@ -883,7 +883,11 @@ async def get_rankings(db: AsyncSession) -> dict:
     Returns dict suitable for RankingResponse.
     """
     weights = await _get_weights(db)
-    tickers = (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars().all()
+    tickers = (
+        await db.execute(
+            ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
+        )
+    ).scalars().all()
 
     async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]:
         comps = {
@@ -947,7 +951,7 @@ async def update_weights(
     await _save_weights(db, full_weights)
 
     # Recompute all composite scores
-    result = await db.execute(select(Ticker))
+    result = await db.execute(ticker_service.active_only(select(Ticker)))
     tickers = list(result.scalars().all())
 
     for ticker in tickers:

app/services/sec_client.py

@@ -45,6 +45,13 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
 
 _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
 
+# Exchange delisting (25 / 25-NSE) and registration termination (15 family).
+# Their presence is SEC confirming a security stopped trading — see
+# ``SecClient.delisting_filing``.
+_DELISTING_FORMS = frozenset({
+    "25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G",
+})
+
 
 class SecError(ProviderError):
     """SEC request failed (403, exhausted 429/5xx, timeout, transport, parse)."""
@@ -253,6 +260,34 @@ class SecClient:
             "filings": filings,
         }
 
+    async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None:
+        """Newest exchange-delisting / deregistration filing, or ``None``.
+
+        Form 25/25-NSE strikes a security from listing; Form 15 terminates the
+        registration. Either is SEC confirming the security stopped trading —
+        which is what separates a real delisting from a multi-day halt or a
+        ticker rename, neither of which files one. That precision is the reason
+        this can mark a symbol automatically.
+
+        Reads ``filings.recent`` directly: ``submissions()`` keeps only the
+        10-K/10-Q family, so these forms never survive its parser.
+        """
+        base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
+        arrays = (base.get("filings") or {}).get("recent") or {}
+        forms = arrays.get("form") or []
+        dates = arrays.get("filingDate") or []
+        best: dict[str, Any] | None = None
+        for i, form in enumerate(forms):
+            if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]:
+                continue
+            try:
+                filed = date.fromisoformat(dates[i])
+            except ValueError:
+                continue
+            if best is None or filed > best["filing_date"]:
+                best = {"form": form, "filing_date": filed}
+        return best
+
     async def companyfacts(self, cik: int | str) -> dict[str, Any]:
         """Raw companyfacts JSON ({cik, entityName, facts})."""
         return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")

app/services/sec_universe.py

@@ -24,7 +24,7 @@ from typing import Iterable
 from sqlalchemy import select, update
 
 from app.models.ticker import Ticker
-from app.services import settings_store
+from app.services import settings_store, ticker_service
 from app.services.earnings_alignment import normalise_symbol
 from app.services.sec_client import SecClient
 
@@ -55,7 +55,11 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
     returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
     ticker_to_cik = await client.company_tickers()
     overrides = await cik_overrides(db)
-    rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
+    rows = (
+        await db.execute(
+            ticker_service.active_only(select(Ticker.id, Ticker.symbol, Ticker.cik))
+        )
+    ).all()
 
     result = ResolvedUniverse()
     for tid, symbol, current_cik in rows:

app/services/ticker_service.py

@@ -1,13 +1,48 @@
-"""Ticker Registry service: add, delete, and list tracked tickers."""
+"""Ticker Registry service: add, delete, list, and retire tracked tickers."""
 
+import logging
 import re
+from datetime import date
 
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.exceptions import DuplicateError, NotFoundError, ValidationError
 from app.models.ticker import Ticker
 
+logger = logging.getLogger(__name__)
+
+# Reasons a symbol may be marked delisted, narrowest first.
+REASON_FORM_25 = "form_25"   # SEC Form 25/25-NSE/15 confirmed the exchange exit
+REASON_MANUAL = "manual"     # an operator decided
+
+# How long a symbol must be without bars before we spend an SEC request asking
+# whether it delisted. Guards against a market-data outage probing the whole
+# universe at once; a real delisting is still stale days later.
+MIN_STALE_DAYS_BEFORE_PROBE = 3
+
+
+def _sec_client_factory():
+    """Build the SEC client for a delisting probe (patched in tests).
+
+    Imported lazily so the SEC/httpx stack stays off the import path of every
+    module that only wants ``active_only``.
+    """
+    from app.services.sec_client import SecClient
+
+    return SecClient()
+
+
+def active_only(stmt):
+    """Restrict a Ticker query to symbols that still trade.
+
+    Opt-in on purpose rather than folded into a shared getter: list and admin
+    views deliberately keep delisted rows so the delisting is *visible*, which a
+    silent default would undo. Apply this on the live signal path — scanning,
+    ranking, scoring, breadth, ingestion — and nowhere else.
+    """
+    return stmt.where(Ticker.delisted_on.is_(None))
+
 
 async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
     """Add a new ticker after validation.
@@ -52,6 +87,121 @@ async def delete_ticker(db: AsyncSession, symbol: str) -> None:
 
 
 async def list_tickers(db: AsyncSession) -> list[Ticker]:
-    """Return all tracked tickers sorted alphabetically by symbol."""
+    """Return all tracked tickers sorted alphabetically by symbol.
+
+    Delisted symbols are included and carry ``delisted_on`` — the registry is
+    where an operator needs to *see* that a symbol retired, not where it should
+    quietly disappear.
+    """
     result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
     return list(result.scalars().all())
+
+
+async def mark_delisted(
+    db: AsyncSession,
+    symbol: str,
+    *,
+    delisted_on: date,
+    reason: str = REASON_MANUAL,
+) -> bool:
+    """Record that a symbol stopped trading. True if this changed anything.
+
+    Idempotent: re-marking an already-delisted symbol is a no-op, so the
+    staleness path can call it on every run without churning the row or
+    re-emitting events.
+    """
+    normalised = symbol.strip().upper()
+    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
+    ticker = result.scalar_one_or_none()
+    if ticker is None:
+        raise NotFoundError(f"Ticker not found: {normalised}")
+    if ticker.delisted_on is not None:
+        return False
+
+    await db.execute(
+        update(Ticker)
+        .where(Ticker.id == ticker.id)
+        .values(delisted_on=delisted_on, delisted_reason=reason)
+    )
+    await db.commit()
+    logger.info(
+        "ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason
+    )
+    return True
+
+
+async def confirm_delisting(
+    db: AsyncSession,
+    symbol: str,
+    *,
+    last_bar: date | None,
+    today: date | None = None,
+) -> date | None:
+    """Ask SEC whether ``symbol`` actually delisted; mark it if so.
+
+    Called when OHLCV goes stale, because "no new bars" alone cannot tell a
+    delisting from a halt or a rename. Returns the effective date when this call
+    marked the symbol, else ``None`` — already-marked and unconfirmed both return
+    ``None``, so the caller keeps its existing alert for anything unproven.
+
+    Deliberately driven by staleness rather than by the SEC fundamentals import:
+    that importer stalls for days at a time on unrelated Company-Facts gaps, and
+    detection wired into it would stall with it.
+
+    The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays
+    stale forever, so the delay costs nothing, and it keeps a broad market-data
+    outage — where every tracked symbol reports stale at once — from turning into
+    one SEC request per symbol per run.
+    """
+    from app.services.sec_client import SecError
+
+    normalised = symbol.strip().upper()
+    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
+    ticker = result.scalar_one_or_none()
+    if ticker is None or ticker.delisted_on is not None or not ticker.cik:
+        return None
+    # No bars at all is an ingestion problem, not evidence of a delisting.
+    if last_bar is None:
+        return None
+    if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
+        return None
+
+    try:
+        async with _sec_client_factory() as client:
+            filing = await client.delisting_filing(ticker.cik)
+    except SecError:
+        # Never let a probe failure escalate a routine staleness warning.
+        logger.warning("delisting probe failed for %s", normalised, exc_info=True)
+        return None
+
+    if filing is None:
+        return None
+    if await mark_delisted(
+        db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25
+    ):
+        return filing["filing_date"]
+    return None
+
+
+async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
+    """Un-retire a symbol. True if it had been marked.
+
+    The counterpart that makes automatic marking acceptable: a false positive
+    costs one row update, where a delete would have cost the price history.
+    """
+    normalised = symbol.strip().upper()
+    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
+    ticker = result.scalar_one_or_none()
+    if ticker is None:
+        raise NotFoundError(f"Ticker not found: {normalised}")
+    if ticker.delisted_on is None:
+        return False
+
+    await db.execute(
+        update(Ticker)
+        .where(Ticker.id == ticker.id)
+        .values(delisted_on=None, delisted_reason=None)
+    )
+    await db.commit()
+    logger.info("ticker %s un-marked as delisted", normalised)
+    return True