feat(tickers): record delisting instead of deleting the symbol

Retiring a symbol meant delete_ticker or bootstrap_universe(prune_missing),
both of which cascade through OHLCV, setups and scores. That destroys exactly
the history four research documents already apologise for: today's tracked
universe projected backward is survivorship-biased, and hard-deleting every
delisted name is what causes it. Keeping the rows preserves the option to fix
that — it does not fix it, which needs the replay to model a delisting as an
exit event.

tickers gains delisted_on / delisted_reason (migration 032). NULL means
actively traded.

The filter is opt-in via ticker_service.active_only rather than folded into a
shared getter: the registry and admin views deliberately keep delisted rows so
the delisting is visible, and a silent default would undo that. Applied to the
live path only — scanner, momentum ranking, scoring, breadth, fundamentals
candidates, SEC universe, earnings import, ingestion loops. run_backtest keeps
them on purpose.

Detection runs off OHLCV staleness, not off the SEC fundamentals import: that
importer stalls for days on unrelated Company-Facts gaps and would take
detection down with it. On a stale symbol the scheduler asks SEC for a Form
25/25-NSE/15 and retires it only on a hit, so a halt or a rename (SATS->ECHO)
keeps the existing warning. The probe waits 3 stale days so a market-data
outage cannot turn into one SEC request per symbol per run.

Safe to automate because it is reversible: clear_delisted un-retires a false
positive, where a delete had already taken the history.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-11 16:58:51 +02:00
co-authored by Claude Opus 5
parent 486fb500d1
commit 6501b7e9a0
15 changed files with 512 additions and 29 deletions
+49
View File
@@ -0,0 +1,49 @@
"""Record delisting on tickers instead of deleting them
Revision ID: 032
Revises: 031
Create Date: 2026-08-11 00:00:00.000000
Until now the only way to retire a symbol was ``delete_ticker`` (or
``bootstrap_universe(prune_missing=True)``), both of which cascade through
OHLCV, setups and scores. That destroys exactly the history four research
documents already apologise for: today's tracked universe projected backward
is survivorship-biased, and hard-deleting every delisted name is what causes
it. Keeping the rows preserves the option to fix that later — it does not fix
it by itself, which needs the replay to model a delisting as an exit event.
``delisted_on`` is the effective date (from SEC Form 25/25-NSE/15 where we can
confirm it, else the day it was marked); ``delisted_reason`` is a short code
for how we learned. NULL in both means actively traded — the live signal path
filters on that, while list and admin views keep showing the row so the
delisting is visible rather than silently absent.
Nullable and reversible by design: clearing ``delisted_on`` un-retires a
symbol, which is what makes automatic marking safe where a delete would not be.
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "032"
down_revision: Union[str, None] = "031"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("tickers", sa.Column("delisted_on", sa.Date(), nullable=True))
op.add_column(
"tickers", sa.Column("delisted_reason", sa.String(length=32), nullable=True)
)
# The live path filters "actively traded" on every universe scan; the index
# keeps that predicate cheap as delisted rows accumulate.
op.create_index("ix_tickers_delisted_on", "tickers", ["delisted_on"])
def downgrade() -> None:
op.drop_index("ix_tickers_delisted_on", table_name="tickers")
op.drop_column("tickers", "delisted_reason")
op.drop_column("tickers", "delisted_on")
+9 -2
View File
@@ -1,6 +1,6 @@
from datetime import datetime from datetime import date, datetime
from sqlalchemy import String, DateTime from sqlalchemy import Date, String, DateTime
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base from app.database import Base
@@ -21,6 +21,13 @@ class Ticker(Base):
cik: Mapped[str | None] = mapped_column(String(10), nullable=True) cik: Mapped[str | None] = mapped_column(String(10), nullable=True)
sic: Mapped[str | None] = mapped_column(String(4), nullable=True) sic: Mapped[str | None] = mapped_column(String(4), nullable=True)
sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True) sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True)
# Delisting is recorded, never deleted: the rows carry the price history that
# makes a backtest less survivorship-biased, and a delete cascades it away.
# NULL == actively traded. The live signal path filters on this (see
# ticker_service.active_only); list/admin views keep the row and show it.
delisted_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True)
# How we learned: "form_25" (SEC confirmed), "manual" (operator).
delisted_reason: Mapped[str | None] = mapped_column(String(32), nullable=True)
created_at: Mapped[datetime] = mapped_column( created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.utcnow, nullable=False DateTime(timezone=True), default=datetime.utcnow, nullable=False
) )
+27 -2
View File
@@ -66,6 +66,7 @@ from app.services.event_study_service import run_and_store as run_event_study_an
from app.services.outcome_service import evaluate_pending_setups from app.services.outcome_service import evaluate_pending_setups
from app.services.rr_scanner_service import scan_all_tickers from app.services.rr_scanner_service import scan_all_tickers
from app.services.sentiment_provider_service import build_sentiment_provider from app.services.sentiment_provider_service import build_sentiment_provider
from app.services import ticker_service
from app.services.ticker_universe_service import bootstrap_universe from app.services.ticker_universe_service import bootstrap_universe
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -396,8 +397,10 @@ async def _is_job_enabled(db: AsyncSession, job_name: str) -> bool:
async def _get_all_tickers(db: AsyncSession) -> list[str]: async def _get_all_tickers(db: AsyncSession) -> list[str]:
"""Return all tracked ticker symbols sorted alphabetically.""" """Return all actively-traded ticker symbols sorted alphabetically."""
result = await db.execute(select(Ticker.symbol).order_by(Ticker.symbol)) result = await db.execute(
ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol))
)
return list(result.scalars().all()) return list(result.scalars().all())
@@ -412,8 +415,10 @@ async def _get_ohlcv_priority_tickers(db: AsyncSession) -> list[str]:
latest_date = func.max(OHLCVRecord.date) latest_date = func.max(OHLCVRecord.date)
missing_first = case((latest_date.is_(None), 0), else_=1) missing_first = case((latest_date.is_(None), 0), else_=1)
result = await db.execute( result = await db.execute(
ticker_service.active_only(
select(Ticker.symbol) select(Ticker.symbol)
.outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id) .outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id)
)
.group_by(Ticker.id, Ticker.symbol) .group_by(Ticker.id, Ticker.symbol)
.order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc()) .order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc())
) )
@@ -662,6 +667,26 @@ async def collect_ohlcv(
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol) _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested) _log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested)
if result.status == "stale": if result.status == "stale":
# "No new bars" cannot distinguish a delisting from a halt
# or a rename, so ask SEC before warning again. A confirmed
# delisting retires the symbol (keeping its history) and
# ends the alert; anything unproven keeps warning.
delisted_on = await ticker_service.confirm_delisting(
db, symbol, last_bar=result.last_date
)
if delisted_on is not None:
await _record_system_event(
severity="info",
source=job_name,
code="ticker_delisted",
message=(
f"{symbol} delisted on {delisted_on} (SEC Form 25/15). "
"Retired from signals; price history retained."
),
symbol=symbol,
dedup_key=f"ticker_delisted:{symbol}",
)
else:
await _record_system_event( await _record_system_event(
severity="warning", severity="warning",
source=job_name, source=job_name,
+6 -1
View File
@@ -1,6 +1,6 @@
"""Ticker request/response schemas.""" """Ticker request/response schemas."""
from datetime import datetime from datetime import date, datetime
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
@@ -14,5 +14,10 @@ class TickerResponse(BaseModel):
symbol: str symbol: str
name: str | None = None name: str | None = None
created_at: datetime created_at: datetime
# NULL == actively traded. Delisted symbols stay in the registry with their
# history and are excluded from signals — the date is what makes that
# visible instead of the row silently disappearing.
delisted_on: date | None = None
delisted_reason: str | None = None
model_config = {"from_attributes": True} model_config = {"from_attributes": True}
+2 -1
View File
@@ -25,6 +25,7 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import ticker_service
from app.services.price_service import query_ohlcv from app.services.price_service import query_ohlcv
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -112,7 +113,7 @@ def compute_divergence_series(
async def _load_universe_closes( async def _load_universe_closes(
db: AsyncSession, symbols: list[str] | None = None db: AsyncSession, symbols: list[str] | None = None
) -> dict[str, Series]: ) -> dict[str, Series]:
stmt = select(Ticker).order_by(Ticker.symbol) stmt = ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
if symbols is not None: if symbols is not None:
stmt = stmt.where(Ticker.symbol.in_(symbols)) stmt = stmt.where(Ticker.symbol.in_(symbols))
result = await db.execute(stmt) result = await db.execute(stmt)
+6 -2
View File
@@ -32,7 +32,7 @@ from app.config import settings
from app.database import insert_for_session from app.database import insert_for_session
from app.models.earnings_event import EarningsEvent from app.models.earnings_event import EarningsEvent
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import dolt_client, earnings_alignment from app.services import dolt_client, earnings_alignment, ticker_service
from app.services.data_import import ValidationResult from app.services.data_import import ValidationResult
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -289,7 +289,11 @@ class DoltEarningsImporter:
# -- helpers ----------------------------------------------------------- # -- helpers -----------------------------------------------------------
async def _load_universe(self, db) -> dict[str, int]: async def _load_universe(self, db) -> dict[str, int]:
rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all() rows = (
await db.execute(
ticker_service.active_only(select(Ticker.id, Ticker.symbol))
)
).all()
return { return {
earnings_alignment.normalise_symbol(symbol): tid earnings_alignment.normalise_symbol(symbol): tid
for tid, symbol in rows for tid, symbol in rows
@@ -22,6 +22,7 @@ from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.ohlcv import OHLCVRecord from app.models.ohlcv import OHLCVRecord
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import fundamentals_derivation as deriv from app.services import fundamentals_derivation as deriv
from app.services import ticker_service
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -46,7 +47,11 @@ async def build_candidates(
"""Derive current cache candidates using only already-stored data.""" """Derive current cache candidates using only already-stored data."""
today = today or datetime.now(ZoneInfo("America/New_York")).date() today = today or datetime.now(ZoneInfo("America/New_York")).date()
tickers = list( tickers = list(
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() (
await db.execute(
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
)
).scalars()
) )
if not tickers: if not tickers:
return [] return []
+4 -1
View File
@@ -18,6 +18,7 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import ticker_service
from app.services.price_service import query_ohlcv from app.services.price_service import query_ohlcv
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -169,7 +170,9 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa
before scanning; the research backtest ranked each weekly setup-candidate before scanning; the research backtest ranked each weekly setup-candidate
cross-section, so this is the deliberate production approximation. cross-section, so this is the deliberate production approximation.
""" """
result = await db.execute(select(Ticker).order_by(Ticker.symbol)) result = await db.execute(
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
)
tickers = list(result.scalars().all()) tickers = list(result.scalars().all())
benchmark_closes = await _load_activation_benchmark(db) benchmark_closes = await _load_activation_benchmark(db)
+6 -2
View File
@@ -31,7 +31,7 @@ from app.services import fundamentals_quality_service, system_event_service
from app.services.price_service import query_ohlcv from app.services.price_service import query_ohlcv
from app.services.qualification import setup_qualifies from app.services.qualification import setup_qualifies
from app.services.sr_service import detect_gate_target_ladder from app.services.sr_service import detect_gate_target_ladder
from app.services import settings_store from app.services import settings_store, ticker_service
from app.services.trade_policy import ( from app.services.trade_policy import (
MANUAL_BOOK, MANUAL_BOOK,
SHADOW_BOOK, SHADOW_BOOK,
@@ -735,7 +735,11 @@ async def scan_all_tickers(
# Plain ids/strings, not Ticker instances: the rollbacks below expire any # Plain ids/strings, not Ticker instances: the rollbacks below expire any
# ORM objects held across them, and touching an expired attribute afterwards # ORM objects held across them, and touching an expired attribute afterwards
# triggers sync lazy-loading, which raises on an AsyncSession. # triggers sync lazy-loading, which raises on an AsyncSession.
result = await db.execute(select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)) result = await db.execute(
ticker_service.active_only(
select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)
)
)
ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()] ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()]
total = len(ticker_rows) total = len(ticker_rows)
+7 -3
View File
@@ -20,7 +20,7 @@ from app.database import insert_for_session
from app.exceptions import NotFoundError, ValidationError from app.exceptions import NotFoundError, ValidationError
from app.models.score import CompositeScore, DimensionScore from app.models.score import CompositeScore, DimensionScore
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import settings_store from app.services import settings_store, ticker_service
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -883,7 +883,11 @@ async def get_rankings(db: AsyncSession) -> dict:
Returns dict suitable for RankingResponse. Returns dict suitable for RankingResponse.
""" """
weights = await _get_weights(db) weights = await _get_weights(db)
tickers = (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars().all() tickers = (
await db.execute(
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
)
).scalars().all()
async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]: async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]:
comps = { comps = {
@@ -947,7 +951,7 @@ async def update_weights(
await _save_weights(db, full_weights) await _save_weights(db, full_weights)
# Recompute all composite scores # Recompute all composite scores
result = await db.execute(select(Ticker)) result = await db.execute(ticker_service.active_only(select(Ticker)))
tickers = list(result.scalars().all()) tickers = list(result.scalars().all())
for ticker in tickers: for ticker in tickers:
+35
View File
@@ -45,6 +45,13 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"}) _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
# Exchange delisting (25 / 25-NSE) and registration termination (15 family).
# Their presence is SEC confirming a security stopped trading — see
# ``SecClient.delisting_filing``.
_DELISTING_FORMS = frozenset({
"25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G",
})
class SecError(ProviderError): class SecError(ProviderError):
"""SEC request failed (403, exhausted 429/5xx, timeout, transport, parse).""" """SEC request failed (403, exhausted 429/5xx, timeout, transport, parse)."""
@@ -253,6 +260,34 @@ class SecClient:
"filings": filings, "filings": filings,
} }
async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None:
"""Newest exchange-delisting / deregistration filing, or ``None``.
Form 25/25-NSE strikes a security from listing; Form 15 terminates the
registration. Either is SEC confirming the security stopped trading —
which is what separates a real delisting from a multi-day halt or a
ticker rename, neither of which files one. That precision is the reason
this can mark a symbol automatically.
Reads ``filings.recent`` directly: ``submissions()`` keeps only the
10-K/10-Q family, so these forms never survive its parser.
"""
base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
arrays = (base.get("filings") or {}).get("recent") or {}
forms = arrays.get("form") or []
dates = arrays.get("filingDate") or []
best: dict[str, Any] | None = None
for i, form in enumerate(forms):
if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]:
continue
try:
filed = date.fromisoformat(dates[i])
except ValueError:
continue
if best is None or filed > best["filing_date"]:
best = {"form": form, "filing_date": filed}
return best
async def companyfacts(self, cik: int | str) -> dict[str, Any]: async def companyfacts(self, cik: int | str) -> dict[str, Any]:
"""Raw companyfacts JSON ({cik, entityName, facts}).""" """Raw companyfacts JSON ({cik, entityName, facts})."""
return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json") return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")
+6 -2
View File
@@ -24,7 +24,7 @@ from typing import Iterable
from sqlalchemy import select, update from sqlalchemy import select, update
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.services import settings_store from app.services import settings_store, ticker_service
from app.services.earnings_alignment import normalise_symbol from app.services.earnings_alignment import normalise_symbol
from app.services.sec_client import SecClient from app.services.sec_client import SecClient
@@ -55,7 +55,11 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
returns the mapping + proposed `tickers.cik` writes; mutates nothing.""" returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
ticker_to_cik = await client.company_tickers() ticker_to_cik = await client.company_tickers()
overrides = await cik_overrides(db) overrides = await cik_overrides(db)
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all() rows = (
await db.execute(
ticker_service.active_only(select(Ticker.id, Ticker.symbol, Ticker.cik))
)
).all()
result = ResolvedUniverse() result = ResolvedUniverse()
for tid, symbol, current_cik in rows: for tid, symbol, current_cik in rows:
+153 -3
View File
@@ -1,13 +1,48 @@
"""Ticker Registry service: add, delete, and list tracked tickers.""" """Ticker Registry service: add, delete, list, and retire tracked tickers."""
import logging
import re import re
from datetime import date
from sqlalchemy import select from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.exceptions import DuplicateError, NotFoundError, ValidationError from app.exceptions import DuplicateError, NotFoundError, ValidationError
from app.models.ticker import Ticker from app.models.ticker import Ticker
logger = logging.getLogger(__name__)
# Reasons a symbol may be marked delisted, narrowest first.
REASON_FORM_25 = "form_25" # SEC Form 25/25-NSE/15 confirmed the exchange exit
REASON_MANUAL = "manual" # an operator decided
# How long a symbol must be without bars before we spend an SEC request asking
# whether it delisted. Guards against a market-data outage probing the whole
# universe at once; a real delisting is still stale days later.
MIN_STALE_DAYS_BEFORE_PROBE = 3
def _sec_client_factory():
"""Build the SEC client for a delisting probe (patched in tests).
Imported lazily so the SEC/httpx stack stays off the import path of every
module that only wants ``active_only``.
"""
from app.services.sec_client import SecClient
return SecClient()
def active_only(stmt):
"""Restrict a Ticker query to symbols that still trade.
Opt-in on purpose rather than folded into a shared getter: list and admin
views deliberately keep delisted rows so the delisting is *visible*, which a
silent default would undo. Apply this on the live signal path scanning,
ranking, scoring, breadth, ingestion and nowhere else.
"""
return stmt.where(Ticker.delisted_on.is_(None))
async def add_ticker(db: AsyncSession, symbol: str) -> Ticker: async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
"""Add a new ticker after validation. """Add a new ticker after validation.
@@ -52,6 +87,121 @@ async def delete_ticker(db: AsyncSession, symbol: str) -> None:
async def list_tickers(db: AsyncSession) -> list[Ticker]: async def list_tickers(db: AsyncSession) -> list[Ticker]:
"""Return all tracked tickers sorted alphabetically by symbol.""" """Return all tracked tickers sorted alphabetically by symbol.
Delisted symbols are included and carry ``delisted_on`` the registry is
where an operator needs to *see* that a symbol retired, not where it should
quietly disappear.
"""
result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc())) result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
return list(result.scalars().all()) return list(result.scalars().all())
async def mark_delisted(
db: AsyncSession,
symbol: str,
*,
delisted_on: date,
reason: str = REASON_MANUAL,
) -> bool:
"""Record that a symbol stopped trading. True if this changed anything.
Idempotent: re-marking an already-delisted symbol is a no-op, so the
staleness path can call it on every run without churning the row or
re-emitting events.
"""
normalised = symbol.strip().upper()
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
ticker = result.scalar_one_or_none()
if ticker is None:
raise NotFoundError(f"Ticker not found: {normalised}")
if ticker.delisted_on is not None:
return False
await db.execute(
update(Ticker)
.where(Ticker.id == ticker.id)
.values(delisted_on=delisted_on, delisted_reason=reason)
)
await db.commit()
logger.info(
"ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason
)
return True
async def confirm_delisting(
db: AsyncSession,
symbol: str,
*,
last_bar: date | None,
today: date | None = None,
) -> date | None:
"""Ask SEC whether ``symbol`` actually delisted; mark it if so.
Called when OHLCV goes stale, because "no new bars" alone cannot tell a
delisting from a halt or a rename. Returns the effective date when this call
marked the symbol, else ``None`` already-marked and unconfirmed both return
``None``, so the caller keeps its existing alert for anything unproven.
Deliberately driven by staleness rather than by the SEC fundamentals import:
that importer stalls for days at a time on unrelated Company-Facts gaps, and
detection wired into it would stall with it.
The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays
stale forever, so the delay costs nothing, and it keeps a broad market-data
outage where every tracked symbol reports stale at once from turning into
one SEC request per symbol per run.
"""
from app.services.sec_client import SecError
normalised = symbol.strip().upper()
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
ticker = result.scalar_one_or_none()
if ticker is None or ticker.delisted_on is not None or not ticker.cik:
return None
# No bars at all is an ingestion problem, not evidence of a delisting.
if last_bar is None:
return None
if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
return None
try:
async with _sec_client_factory() as client:
filing = await client.delisting_filing(ticker.cik)
except SecError:
# Never let a probe failure escalate a routine staleness warning.
logger.warning("delisting probe failed for %s", normalised, exc_info=True)
return None
if filing is None:
return None
if await mark_delisted(
db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25
):
return filing["filing_date"]
return None
async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
"""Un-retire a symbol. True if it had been marked.
The counterpart that makes automatic marking acceptable: a false positive
costs one row update, where a delete would have cost the price history.
"""
normalised = symbol.strip().upper()
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
ticker = result.scalar_one_or_none()
if ticker is None:
raise NotFoundError(f"Ticker not found: {normalised}")
if ticker.delisted_on is None:
return False
await db.execute(
update(Ticker)
.where(Ticker.id == ticker.id)
.values(delisted_on=None, delisted_reason=None)
)
await db.commit()
logger.info("ticker %s un-marked as delisted", normalised)
return True
+4
View File
@@ -858,6 +858,10 @@ export interface Ticker {
symbol: string; symbol: string;
name: string | null; name: string | null;
created_at: string; created_at: string;
/** Set once the symbol stopped trading: excluded from signals, history kept. */
delisted_on: string | null;
/** How the delisting was learned: "form_25" (SEC confirmed) | "manual". */
delisted_reason: string | null;
} }
// Admin // Admin
+183
View File
@@ -0,0 +1,183 @@
"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation.
The behaviour under test is that a delisted symbol leaves the *live* path while
its rows stay put deleting it instead is what makes the backtest universe
survivorship-biased, so retention is the point, not a side effect.
"""
from __future__ import annotations
import json
from collections.abc import AsyncGenerator
from datetime import date
import httpx
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.database import Base
from app.models.ticker import Ticker
from app.services import ticker_service
from app.services.sec_client import SecClient
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
@pytest.fixture(autouse=True)
async def _setup_tables() -> AsyncGenerator[None, None]:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.drop_all)
@pytest.fixture
async def session() -> AsyncGenerator[AsyncSession, None]:
async with _session_factory() as s:
yield s
def _submissions(forms: list[str], dates: list[str]) -> dict:
return {
"cik": 712515,
"name": "ELECTRONIC ARTS INC.",
"filings": {"recent": {"form": forms, "filingDate": dates}},
}
def _sec_client(payload: dict) -> SecClient:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=json.dumps(payload).encode())
return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
async def test_mark_delisted_is_idempotent(session: AsyncSession):
session.add(Ticker(symbol="EA"))
await session.commit()
assert await ticker_service.mark_delisted(
session, "EA", delisted_on=date(2026, 8, 4)
) is True
# A second call must not churn the row — the staleness path retries daily.
assert await ticker_service.mark_delisted(
session, "EA", delisted_on=date(2026, 9, 1)
) is False
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
assert row.delisted_on == date(2026, 8, 4) # first date wins, not the retry
assert row.delisted_reason == ticker_service.REASON_MANUAL
async def test_clear_delisted_restores_the_symbol(session: AsyncSession):
session.add(Ticker(symbol="EA"))
await session.commit()
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
assert await ticker_service.clear_delisted(session, "EA") is True
assert await ticker_service.clear_delisted(session, "EA") is False
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
assert row.delisted_on is None and row.delisted_reason is None
async def test_active_only_filters_but_the_row_survives(session: AsyncSession):
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
await session.commit()
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
active = (
await session.execute(ticker_service.active_only(select(Ticker.symbol)))
).scalars().all()
assert list(active) == ["AAPL"]
# The whole point: the row — and everything cascading off it — is still there.
everything = [t.symbol for t in await ticker_service.list_tickers(session)]
assert everything == ["AAPL", "EA"]
async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch):
session.add(Ticker(symbol="EA", cik="0000712515"))
await session.commit()
monkeypatch.setattr(
ticker_service,
"_sec_client_factory",
lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])),
raising=False,
)
marked = await ticker_service.confirm_delisting(
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
)
assert marked == date(2026, 8, 4)
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
assert row.delisted_reason == ticker_service.REASON_FORM_25
async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch):
"""A halt or a rename files no Form 25 — those must keep warning, not retire."""
session.add(Ticker(symbol="SATS", cik="0000012345"))
await session.commit()
monkeypatch.setattr(
ticker_service,
"_sec_client_factory",
lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])),
raising=False,
)
assert await ticker_service.confirm_delisting(
session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
) is None
row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one()
assert row.delisted_on is None
async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession):
"""No CIK, no SEC lookup — must not raise, and must not mark."""
session.add(Ticker(symbol="ADRX"))
await session.commit()
assert await ticker_service.confirm_delisting(
session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
) is None
async def test_delisting_filing_picks_the_newest_match():
client = _sec_client(
_submissions(
["25", "8-K", "25-NSE", "15-12B"],
["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"],
)
)
async with client as c:
found = await c.delisting_filing("0000712515")
assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)}
async def test_delisting_filing_returns_none_without_one():
client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"]))
async with client as c:
assert await c.delisting_filing("0000320193") is None
async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch):
"""A one-day gap is a weekend or a hiccup. Probing every stale symbol during a
market-data outage would be one SEC request per symbol per run."""
session.add(Ticker(symbol="EA", cik="0000712515"))
await session.commit()
def _explode():
raise AssertionError("must not reach SEC before the stale threshold")
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
assert await ticker_service.confirm_delisting(
session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11)
) is None
# ...and no bars at all is an ingestion problem, not a delisting.
assert await ticker_service.confirm_delisting(
session, "EA", last_bar=None, today=date(2026, 8, 11)
) is None