feat(tickers): record delisting instead of deleting the symbol
Retiring a symbol meant delete_ticker or bootstrap_universe(prune_missing), both of which cascade through OHLCV, setups and scores. That destroys exactly the history four research documents already apologise for: today's tracked universe projected backward is survivorship-biased, and hard-deleting every delisted name is what causes it. Keeping the rows preserves the option to fix that — it does not fix it, which needs the replay to model a delisting as an exit event. tickers gains delisted_on / delisted_reason (migration 032). NULL means actively traded. The filter is opt-in via ticker_service.active_only rather than folded into a shared getter: the registry and admin views deliberately keep delisted rows so the delisting is visible, and a silent default would undo that. Applied to the live path only — scanner, momentum ranking, scoring, breadth, fundamentals candidates, SEC universe, earnings import, ingestion loops. run_backtest keeps them on purpose. Detection runs off OHLCV staleness, not off the SEC fundamentals import: that importer stalls for days on unrelated Company-Facts gaps and would take detection down with it. On a stale symbol the scheduler asks SEC for a Form 25/25-NSE/15 and retires it only on a hit, so a halt or a rename (SATS->ECHO) keeps the existing warning. The probe waits 3 stale days so a market-data outage cannot turn into one SEC request per symbol per run. Safe to automate because it is reversible: clear_delisted un-retires a false positive, where a delete had already taken the history. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
"""Record delisting on tickers instead of deleting them
|
||||
|
||||
Revision ID: 032
|
||||
Revises: 031
|
||||
Create Date: 2026-08-11 00:00:00.000000
|
||||
|
||||
Until now the only way to retire a symbol was ``delete_ticker`` (or
|
||||
``bootstrap_universe(prune_missing=True)``), both of which cascade through
|
||||
OHLCV, setups and scores. That destroys exactly the history four research
|
||||
documents already apologise for: today's tracked universe projected backward
|
||||
is survivorship-biased, and hard-deleting every delisted name is what causes
|
||||
it. Keeping the rows preserves the option to fix that later — it does not fix
|
||||
it by itself, which needs the replay to model a delisting as an exit event.
|
||||
|
||||
``delisted_on`` is the effective date (from SEC Form 25/25-NSE/15 where we can
|
||||
confirm it, else the day it was marked); ``delisted_reason`` is a short code
|
||||
for how we learned. NULL in both means actively traded — the live signal path
|
||||
filters on that, while list and admin views keep showing the row so the
|
||||
delisting is visible rather than silently absent.
|
||||
|
||||
Nullable and reversible by design: clearing ``delisted_on`` un-retires a
|
||||
symbol, which is what makes automatic marking safe where a delete would not be.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "032"
|
||||
down_revision: Union[str, None] = "031"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("tickers", sa.Column("delisted_on", sa.Date(), nullable=True))
|
||||
op.add_column(
|
||||
"tickers", sa.Column("delisted_reason", sa.String(length=32), nullable=True)
|
||||
)
|
||||
# The live path filters "actively traded" on every universe scan; the index
|
||||
# keeps that predicate cheap as delisted rows accumulate.
|
||||
op.create_index("ix_tickers_delisted_on", "tickers", ["delisted_on"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_tickers_delisted_on", table_name="tickers")
|
||||
op.drop_column("tickers", "delisted_reason")
|
||||
op.drop_column("tickers", "delisted_on")
|
||||
@@ -1,6 +1,6 @@
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import String, DateTime
|
||||
from sqlalchemy import Date, String, DateTime
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
@@ -21,6 +21,13 @@ class Ticker(Base):
|
||||
cik: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
sic: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
# Delisting is recorded, never deleted: the rows carry the price history that
|
||||
# makes a backtest less survivorship-biased, and a delete cascades it away.
|
||||
# NULL == actively traded. The live signal path filters on this (see
|
||||
# ticker_service.active_only); list/admin views keep the row and show it.
|
||||
delisted_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True)
|
||||
# How we learned: "form_25" (SEC confirmed), "manual" (operator).
|
||||
delisted_reason: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
+27
-2
@@ -66,6 +66,7 @@ from app.services.event_study_service import run_and_store as run_event_study_an
|
||||
from app.services.outcome_service import evaluate_pending_setups
|
||||
from app.services.rr_scanner_service import scan_all_tickers
|
||||
from app.services.sentiment_provider_service import build_sentiment_provider
|
||||
from app.services import ticker_service
|
||||
from app.services.ticker_universe_service import bootstrap_universe
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -396,8 +397,10 @@ async def _is_job_enabled(db: AsyncSession, job_name: str) -> bool:
|
||||
|
||||
|
||||
async def _get_all_tickers(db: AsyncSession) -> list[str]:
|
||||
"""Return all tracked ticker symbols sorted alphabetically."""
|
||||
result = await db.execute(select(Ticker.symbol).order_by(Ticker.symbol))
|
||||
"""Return all actively-traded ticker symbols sorted alphabetically."""
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol))
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
@@ -412,8 +415,10 @@ async def _get_ohlcv_priority_tickers(db: AsyncSession) -> list[str]:
|
||||
latest_date = func.max(OHLCVRecord.date)
|
||||
missing_first = case((latest_date.is_(None), 0), else_=1)
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(
|
||||
select(Ticker.symbol)
|
||||
.outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id)
|
||||
)
|
||||
.group_by(Ticker.id, Ticker.symbol)
|
||||
.order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc())
|
||||
)
|
||||
@@ -662,6 +667,26 @@ async def collect_ohlcv(
|
||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
||||
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested)
|
||||
if result.status == "stale":
|
||||
# "No new bars" cannot distinguish a delisting from a halt
|
||||
# or a rename, so ask SEC before warning again. A confirmed
|
||||
# delisting retires the symbol (keeping its history) and
|
||||
# ends the alert; anything unproven keeps warning.
|
||||
delisted_on = await ticker_service.confirm_delisting(
|
||||
db, symbol, last_bar=result.last_date
|
||||
)
|
||||
if delisted_on is not None:
|
||||
await _record_system_event(
|
||||
severity="info",
|
||||
source=job_name,
|
||||
code="ticker_delisted",
|
||||
message=(
|
||||
f"{symbol} delisted on {delisted_on} (SEC Form 25/15). "
|
||||
"Retired from signals; price history retained."
|
||||
),
|
||||
symbol=symbol,
|
||||
dedup_key=f"ticker_delisted:{symbol}",
|
||||
)
|
||||
else:
|
||||
await _record_system_event(
|
||||
severity="warning",
|
||||
source=job_name,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Ticker request/response schemas."""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -14,5 +14,10 @@ class TickerResponse(BaseModel):
|
||||
symbol: str
|
||||
name: str | None = None
|
||||
created_at: datetime
|
||||
# NULL == actively traded. Delisted symbols stay in the registry with their
|
||||
# history and are excluded from signals — the date is what makes that
|
||||
# visible instead of the row silently disappearing.
|
||||
delisted_on: date | None = None
|
||||
delisted_reason: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
@@ -25,6 +25,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import ticker_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -112,7 +113,7 @@ def compute_divergence_series(
|
||||
async def _load_universe_closes(
|
||||
db: AsyncSession, symbols: list[str] | None = None
|
||||
) -> dict[str, Series]:
|
||||
stmt = select(Ticker).order_by(Ticker.symbol)
|
||||
stmt = ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
if symbols is not None:
|
||||
stmt = stmt.where(Ticker.symbol.in_(symbols))
|
||||
result = await db.execute(stmt)
|
||||
|
||||
@@ -32,7 +32,7 @@ from app.config import settings
|
||||
from app.database import insert_for_session
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import dolt_client, earnings_alignment
|
||||
from app.services import dolt_client, earnings_alignment, ticker_service
|
||||
from app.services.data_import import ValidationResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -289,7 +289,11 @@ class DoltEarningsImporter:
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _load_universe(self, db) -> dict[str, int]:
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all()
|
||||
rows = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker.id, Ticker.symbol))
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
earnings_alignment.normalise_symbol(symbol): tid
|
||||
for tid, symbol in rows
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import fundamentals_derivation as deriv
|
||||
from app.services import ticker_service
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -46,7 +47,11 @@ async def build_candidates(
|
||||
"""Derive current cache candidates using only already-stored data."""
|
||||
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
||||
tickers = list(
|
||||
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
||||
(
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
if not tickers:
|
||||
return []
|
||||
|
||||
@@ -18,6 +18,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import ticker_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -169,7 +170,9 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa
|
||||
before scanning; the research backtest ranked each weekly setup-candidate
|
||||
cross-section, so this is the deliberate production approximation.
|
||||
"""
|
||||
result = await db.execute(select(Ticker).order_by(Ticker.symbol))
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
tickers = list(result.scalars().all())
|
||||
|
||||
benchmark_closes = await _load_activation_benchmark(db)
|
||||
|
||||
@@ -31,7 +31,7 @@ from app.services import fundamentals_quality_service, system_event_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
from app.services.qualification import setup_qualifies
|
||||
from app.services.sr_service import detect_gate_target_ladder
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
from app.services.trade_policy import (
|
||||
MANUAL_BOOK,
|
||||
SHADOW_BOOK,
|
||||
@@ -735,7 +735,11 @@ async def scan_all_tickers(
|
||||
# Plain ids/strings, not Ticker instances: the rollbacks below expire any
|
||||
# ORM objects held across them, and touching an expired attribute afterwards
|
||||
# triggers sync lazy-loading, which raises on an AsyncSession.
|
||||
result = await db.execute(select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol))
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(
|
||||
select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)
|
||||
)
|
||||
)
|
||||
ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()]
|
||||
total = len(ticker_rows)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from app.database import insert_for_session
|
||||
from app.exceptions import NotFoundError, ValidationError
|
||||
from app.models.score import CompositeScore, DimensionScore
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -883,7 +883,11 @@ async def get_rankings(db: AsyncSession) -> dict:
|
||||
Returns dict suitable for RankingResponse.
|
||||
"""
|
||||
weights = await _get_weights(db)
|
||||
tickers = (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars().all()
|
||||
tickers = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]:
|
||||
comps = {
|
||||
@@ -947,7 +951,7 @@ async def update_weights(
|
||||
await _save_weights(db, full_weights)
|
||||
|
||||
# Recompute all composite scores
|
||||
result = await db.execute(select(Ticker))
|
||||
result = await db.execute(ticker_service.active_only(select(Ticker)))
|
||||
tickers = list(result.scalars().all())
|
||||
|
||||
for ticker in tickers:
|
||||
|
||||
@@ -45,6 +45,13 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
|
||||
|
||||
_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
|
||||
|
||||
# Exchange delisting (25 / 25-NSE) and registration termination (15 family).
|
||||
# Their presence is SEC confirming a security stopped trading — see
|
||||
# ``SecClient.delisting_filing``.
|
||||
_DELISTING_FORMS = frozenset({
|
||||
"25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G",
|
||||
})
|
||||
|
||||
|
||||
class SecError(ProviderError):
|
||||
"""SEC request failed (403, exhausted 429/5xx, timeout, transport, parse)."""
|
||||
@@ -253,6 +260,34 @@ class SecClient:
|
||||
"filings": filings,
|
||||
}
|
||||
|
||||
async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None:
|
||||
"""Newest exchange-delisting / deregistration filing, or ``None``.
|
||||
|
||||
Form 25/25-NSE strikes a security from listing; Form 15 terminates the
|
||||
registration. Either is SEC confirming the security stopped trading —
|
||||
which is what separates a real delisting from a multi-day halt or a
|
||||
ticker rename, neither of which files one. That precision is the reason
|
||||
this can mark a symbol automatically.
|
||||
|
||||
Reads ``filings.recent`` directly: ``submissions()`` keeps only the
|
||||
10-K/10-Q family, so these forms never survive its parser.
|
||||
"""
|
||||
base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
|
||||
arrays = (base.get("filings") or {}).get("recent") or {}
|
||||
forms = arrays.get("form") or []
|
||||
dates = arrays.get("filingDate") or []
|
||||
best: dict[str, Any] | None = None
|
||||
for i, form in enumerate(forms):
|
||||
if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]:
|
||||
continue
|
||||
try:
|
||||
filed = date.fromisoformat(dates[i])
|
||||
except ValueError:
|
||||
continue
|
||||
if best is None or filed > best["filing_date"]:
|
||||
best = {"form": form, "filing_date": filed}
|
||||
return best
|
||||
|
||||
async def companyfacts(self, cik: int | str) -> dict[str, Any]:
|
||||
"""Raw companyfacts JSON ({cik, entityName, facts})."""
|
||||
return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import Iterable
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
@@ -55,7 +55,11 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
||||
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
|
||||
ticker_to_cik = await client.company_tickers()
|
||||
overrides = await cik_overrides(db)
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
|
||||
rows = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker.id, Ticker.symbol, Ticker.cik))
|
||||
)
|
||||
).all()
|
||||
|
||||
result = ResolvedUniverse()
|
||||
for tid, symbol, current_cik in rows:
|
||||
|
||||
@@ -1,13 +1,48 @@
|
||||
"""Ticker Registry service: add, delete, and list tracked tickers."""
|
||||
"""Ticker Registry service: add, delete, list, and retire tracked tickers."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
||||
from app.models.ticker import Ticker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Reasons a symbol may be marked delisted, narrowest first.
|
||||
REASON_FORM_25 = "form_25" # SEC Form 25/25-NSE/15 confirmed the exchange exit
|
||||
REASON_MANUAL = "manual" # an operator decided
|
||||
|
||||
# How long a symbol must be without bars before we spend an SEC request asking
|
||||
# whether it delisted. Guards against a market-data outage probing the whole
|
||||
# universe at once; a real delisting is still stale days later.
|
||||
MIN_STALE_DAYS_BEFORE_PROBE = 3
|
||||
|
||||
|
||||
def _sec_client_factory():
|
||||
"""Build the SEC client for a delisting probe (patched in tests).
|
||||
|
||||
Imported lazily so the SEC/httpx stack stays off the import path of every
|
||||
module that only wants ``active_only``.
|
||||
"""
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
return SecClient()
|
||||
|
||||
|
||||
def active_only(stmt):
|
||||
"""Restrict a Ticker query to symbols that still trade.
|
||||
|
||||
Opt-in on purpose rather than folded into a shared getter: list and admin
|
||||
views deliberately keep delisted rows so the delisting is *visible*, which a
|
||||
silent default would undo. Apply this on the live signal path — scanning,
|
||||
ranking, scoring, breadth, ingestion — and nowhere else.
|
||||
"""
|
||||
return stmt.where(Ticker.delisted_on.is_(None))
|
||||
|
||||
|
||||
async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
||||
"""Add a new ticker after validation.
|
||||
@@ -52,6 +87,121 @@ async def delete_ticker(db: AsyncSession, symbol: str) -> None:
|
||||
|
||||
|
||||
async def list_tickers(db: AsyncSession) -> list[Ticker]:
|
||||
"""Return all tracked tickers sorted alphabetically by symbol."""
|
||||
"""Return all tracked tickers sorted alphabetically by symbol.
|
||||
|
||||
Delisted symbols are included and carry ``delisted_on`` — the registry is
|
||||
where an operator needs to *see* that a symbol retired, not where it should
|
||||
quietly disappear.
|
||||
"""
|
||||
result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def mark_delisted(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
*,
|
||||
delisted_on: date,
|
||||
reason: str = REASON_MANUAL,
|
||||
) -> bool:
|
||||
"""Record that a symbol stopped trading. True if this changed anything.
|
||||
|
||||
Idempotent: re-marking an already-delisted symbol is a no-op, so the
|
||||
staleness path can call it on every run without churning the row or
|
||||
re-emitting events.
|
||||
"""
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None:
|
||||
raise NotFoundError(f"Ticker not found: {normalised}")
|
||||
if ticker.delisted_on is not None:
|
||||
return False
|
||||
|
||||
await db.execute(
|
||||
update(Ticker)
|
||||
.where(Ticker.id == ticker.id)
|
||||
.values(delisted_on=delisted_on, delisted_reason=reason)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def confirm_delisting(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
*,
|
||||
last_bar: date | None,
|
||||
today: date | None = None,
|
||||
) -> date | None:
|
||||
"""Ask SEC whether ``symbol`` actually delisted; mark it if so.
|
||||
|
||||
Called when OHLCV goes stale, because "no new bars" alone cannot tell a
|
||||
delisting from a halt or a rename. Returns the effective date when this call
|
||||
marked the symbol, else ``None`` — already-marked and unconfirmed both return
|
||||
``None``, so the caller keeps its existing alert for anything unproven.
|
||||
|
||||
Deliberately driven by staleness rather than by the SEC fundamentals import:
|
||||
that importer stalls for days at a time on unrelated Company-Facts gaps, and
|
||||
detection wired into it would stall with it.
|
||||
|
||||
The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays
|
||||
stale forever, so the delay costs nothing, and it keeps a broad market-data
|
||||
outage — where every tracked symbol reports stale at once — from turning into
|
||||
one SEC request per symbol per run.
|
||||
"""
|
||||
from app.services.sec_client import SecError
|
||||
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None or ticker.delisted_on is not None or not ticker.cik:
|
||||
return None
|
||||
# No bars at all is an ingestion problem, not evidence of a delisting.
|
||||
if last_bar is None:
|
||||
return None
|
||||
if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
|
||||
return None
|
||||
|
||||
try:
|
||||
async with _sec_client_factory() as client:
|
||||
filing = await client.delisting_filing(ticker.cik)
|
||||
except SecError:
|
||||
# Never let a probe failure escalate a routine staleness warning.
|
||||
logger.warning("delisting probe failed for %s", normalised, exc_info=True)
|
||||
return None
|
||||
|
||||
if filing is None:
|
||||
return None
|
||||
if await mark_delisted(
|
||||
db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25
|
||||
):
|
||||
return filing["filing_date"]
|
||||
return None
|
||||
|
||||
|
||||
async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
|
||||
"""Un-retire a symbol. True if it had been marked.
|
||||
|
||||
The counterpart that makes automatic marking acceptable: a false positive
|
||||
costs one row update, where a delete would have cost the price history.
|
||||
"""
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None:
|
||||
raise NotFoundError(f"Ticker not found: {normalised}")
|
||||
if ticker.delisted_on is None:
|
||||
return False
|
||||
|
||||
await db.execute(
|
||||
update(Ticker)
|
||||
.where(Ticker.id == ticker.id)
|
||||
.values(delisted_on=None, delisted_reason=None)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info("ticker %s un-marked as delisted", normalised)
|
||||
return True
|
||||
|
||||
@@ -858,6 +858,10 @@ export interface Ticker {
|
||||
symbol: string;
|
||||
name: string | null;
|
||||
created_at: string;
|
||||
/** Set once the symbol stopped trading: excluded from signals, history kept. */
|
||||
delisted_on: string | null;
|
||||
/** How the delisting was learned: "form_25" (SEC confirmed) | "manual". */
|
||||
delisted_reason: string | null;
|
||||
}
|
||||
|
||||
// Admin
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation.
|
||||
|
||||
The behaviour under test is that a delisted symbol leaves the *live* path while
|
||||
its rows stay put — deleting it instead is what makes the backtest universe
|
||||
survivorship-biased, so retention is the point, not a side effect.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from app.database import Base
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import ticker_service
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _setup_tables() -> AsyncGenerator[None, None]:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def session() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with _session_factory() as s:
|
||||
yield s
|
||||
|
||||
|
||||
def _submissions(forms: list[str], dates: list[str]) -> dict:
|
||||
return {
|
||||
"cik": 712515,
|
||||
"name": "ELECTRONIC ARTS INC.",
|
||||
"filings": {"recent": {"form": forms, "filingDate": dates}},
|
||||
}
|
||||
|
||||
|
||||
def _sec_client(payload: dict) -> SecClient:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=json.dumps(payload).encode())
|
||||
|
||||
return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0)
|
||||
|
||||
|
||||
async def test_mark_delisted_is_idempotent(session: AsyncSession):
|
||||
session.add(Ticker(symbol="EA"))
|
||||
await session.commit()
|
||||
|
||||
assert await ticker_service.mark_delisted(
|
||||
session, "EA", delisted_on=date(2026, 8, 4)
|
||||
) is True
|
||||
# A second call must not churn the row — the staleness path retries daily.
|
||||
assert await ticker_service.mark_delisted(
|
||||
session, "EA", delisted_on=date(2026, 9, 1)
|
||||
) is False
|
||||
|
||||
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
||||
assert row.delisted_on == date(2026, 8, 4) # first date wins, not the retry
|
||||
assert row.delisted_reason == ticker_service.REASON_MANUAL
|
||||
|
||||
|
||||
async def test_clear_delisted_restores_the_symbol(session: AsyncSession):
|
||||
session.add(Ticker(symbol="EA"))
|
||||
await session.commit()
|
||||
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
||||
|
||||
assert await ticker_service.clear_delisted(session, "EA") is True
|
||||
assert await ticker_service.clear_delisted(session, "EA") is False
|
||||
|
||||
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
||||
assert row.delisted_on is None and row.delisted_reason is None
|
||||
|
||||
|
||||
async def test_active_only_filters_but_the_row_survives(session: AsyncSession):
|
||||
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
||||
await session.commit()
|
||||
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4))
|
||||
|
||||
active = (
|
||||
await session.execute(ticker_service.active_only(select(Ticker.symbol)))
|
||||
).scalars().all()
|
||||
assert list(active) == ["AAPL"]
|
||||
|
||||
# The whole point: the row — and everything cascading off it — is still there.
|
||||
everything = [t.symbol for t in await ticker_service.list_tickers(session)]
|
||||
assert everything == ["AAPL", "EA"]
|
||||
|
||||
|
||||
async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch):
|
||||
session.add(Ticker(symbol="EA", cik="0000712515"))
|
||||
await session.commit()
|
||||
monkeypatch.setattr(
|
||||
ticker_service,
|
||||
"_sec_client_factory",
|
||||
lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
marked = await ticker_service.confirm_delisting(
|
||||
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
||||
)
|
||||
assert marked == date(2026, 8, 4)
|
||||
|
||||
row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one()
|
||||
assert row.delisted_reason == ticker_service.REASON_FORM_25
|
||||
|
||||
|
||||
async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch):
|
||||
"""A halt or a rename files no Form 25 — those must keep warning, not retire."""
|
||||
session.add(Ticker(symbol="SATS", cik="0000012345"))
|
||||
await session.commit()
|
||||
monkeypatch.setattr(
|
||||
ticker_service,
|
||||
"_sec_client_factory",
|
||||
lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
assert await ticker_service.confirm_delisting(
|
||||
session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
||||
) is None
|
||||
row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one()
|
||||
assert row.delisted_on is None
|
||||
|
||||
|
||||
async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession):
|
||||
"""No CIK, no SEC lookup — must not raise, and must not mark."""
|
||||
session.add(Ticker(symbol="ADRX"))
|
||||
await session.commit()
|
||||
assert await ticker_service.confirm_delisting(
|
||||
session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
||||
) is None
|
||||
|
||||
|
||||
async def test_delisting_filing_picks_the_newest_match():
|
||||
client = _sec_client(
|
||||
_submissions(
|
||||
["25", "8-K", "25-NSE", "15-12B"],
|
||||
["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"],
|
||||
)
|
||||
)
|
||||
async with client as c:
|
||||
found = await c.delisting_filing("0000712515")
|
||||
assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)}
|
||||
|
||||
|
||||
async def test_delisting_filing_returns_none_without_one():
|
||||
client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"]))
|
||||
async with client as c:
|
||||
assert await c.delisting_filing("0000320193") is None
|
||||
|
||||
|
||||
async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch):
|
||||
"""A one-day gap is a weekend or a hiccup. Probing every stale symbol during a
|
||||
market-data outage would be one SEC request per symbol per run."""
|
||||
session.add(Ticker(symbol="EA", cik="0000712515"))
|
||||
await session.commit()
|
||||
|
||||
def _explode():
|
||||
raise AssertionError("must not reach SEC before the stale threshold")
|
||||
|
||||
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
||||
|
||||
assert await ticker_service.confirm_delisting(
|
||||
session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11)
|
||||
) is None
|
||||
# ...and no bars at all is an ingestion problem, not a delisting.
|
||||
assert await ticker_service.confirm_delisting(
|
||||
session, "EA", last_bar=None, today=date(2026, 8, 11)
|
||||
) is None
|
||||
Reference in New Issue
Block a user