From 6501b7e9a017be796fd4ddd57a881f418eccef1d Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Tue, 11 Aug 2026 14:30:04 +0200 Subject: [PATCH] feat(tickers): record delisting instead of deleting the symbol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retiring a symbol meant delete_ticker or bootstrap_universe(prune_missing), both of which cascade through OHLCV, setups and scores. That destroys exactly the history four research documents already apologise for: today's tracked universe projected backward is survivorship-biased, and hard-deleting every delisted name is what causes it. Keeping the rows preserves the option to fix that — it does not fix it, which needs the replay to model a delisting as an exit event. tickers gains delisted_on / delisted_reason (migration 032). NULL means actively traded. The filter is opt-in via ticker_service.active_only rather than folded into a shared getter: the registry and admin views deliberately keep delisted rows so the delisting is visible, and a silent default would undo that. Applied to the live path only — scanner, momentum ranking, scoring, breadth, fundamentals candidates, SEC universe, earnings import, ingestion loops. run_backtest keeps them on purpose. Detection runs off OHLCV staleness, not off the SEC fundamentals import: that importer stalls for days on unrelated Company-Facts gaps and would take detection down with it. On a stale symbol the scheduler asks SEC for a Form 25/25-NSE/15 and retires it only on a hit, so a halt or a rename (SATS->ECHO) keeps the existing warning. The probe waits 3 stale days so a market-data outage cannot turn into one SEC request per symbol per run. Safe to automate because it is reversible: clear_delisted un-retires a false positive, where a delete had already taken the history. Co-Authored-By: Claude Opus 5 --- alembic/versions/032_ticker_delisting.py | 49 +++++ app/models/ticker.py | 11 +- app/scheduler.py | 47 +++-- app/schemas/ticker.py | 7 +- app/services/breadth_service.py | 3 +- app/services/dolt_earnings_importer.py | 8 +- .../fundamentals_candidate_service.py | 7 +- app/services/momentum_service.py | 5 +- app/services/rr_scanner_service.py | 8 +- app/services/scoring_service.py | 10 +- app/services/sec_client.py | 35 ++++ app/services/sec_universe.py | 8 +- app/services/ticker_service.py | 156 ++++++++++++++- frontend/src/lib/types.ts | 4 + tests/unit/test_ticker_delisting.py | 183 ++++++++++++++++++ 15 files changed, 512 insertions(+), 29 deletions(-) create mode 100644 alembic/versions/032_ticker_delisting.py create mode 100644 tests/unit/test_ticker_delisting.py diff --git a/alembic/versions/032_ticker_delisting.py b/alembic/versions/032_ticker_delisting.py new file mode 100644 index 0000000..a19473e --- /dev/null +++ b/alembic/versions/032_ticker_delisting.py @@ -0,0 +1,49 @@ +"""Record delisting on tickers instead of deleting them + +Revision ID: 032 +Revises: 031 +Create Date: 2026-08-11 00:00:00.000000 + +Until now the only way to retire a symbol was ``delete_ticker`` (or +``bootstrap_universe(prune_missing=True)``), both of which cascade through +OHLCV, setups and scores. That destroys exactly the history four research +documents already apologise for: today's tracked universe projected backward +is survivorship-biased, and hard-deleting every delisted name is what causes +it. Keeping the rows preserves the option to fix that later — it does not fix +it by itself, which needs the replay to model a delisting as an exit event. + +``delisted_on`` is the effective date (from SEC Form 25/25-NSE/15 where we can +confirm it, else the day it was marked); ``delisted_reason`` is a short code +for how we learned. NULL in both means actively traded — the live signal path +filters on that, while list and admin views keep showing the row so the +delisting is visible rather than silently absent. + +Nullable and reversible by design: clearing ``delisted_on`` un-retires a +symbol, which is what makes automatic marking safe where a delete would not be. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "032" +down_revision: Union[str, None] = "031" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("tickers", sa.Column("delisted_on", sa.Date(), nullable=True)) + op.add_column( + "tickers", sa.Column("delisted_reason", sa.String(length=32), nullable=True) + ) + # The live path filters "actively traded" on every universe scan; the index + # keeps that predicate cheap as delisted rows accumulate. + op.create_index("ix_tickers_delisted_on", "tickers", ["delisted_on"]) + + +def downgrade() -> None: + op.drop_index("ix_tickers_delisted_on", table_name="tickers") + op.drop_column("tickers", "delisted_reason") + op.drop_column("tickers", "delisted_on") diff --git a/app/models/ticker.py b/app/models/ticker.py index 6f15177..0461d1a 100644 --- a/app/models/ticker.py +++ b/app/models/ticker.py @@ -1,6 +1,6 @@ -from datetime import datetime +from datetime import date, datetime -from sqlalchemy import String, DateTime +from sqlalchemy import Date, String, DateTime from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database import Base @@ -21,6 +21,13 @@ class Ticker(Base): cik: Mapped[str | None] = mapped_column(String(10), nullable=True) sic: Mapped[str | None] = mapped_column(String(4), nullable=True) sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True) + # Delisting is recorded, never deleted: the rows carry the price history that + # makes a backtest less survivorship-biased, and a delete cascades it away. + # NULL == actively traded. The live signal path filters on this (see + # ticker_service.active_only); list/admin views keep the row and show it. + delisted_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) + # How we learned: "form_25" (SEC confirmed), "manual" (operator). + delisted_reason: Mapped[str | None] = mapped_column(String(32), nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=datetime.utcnow, nullable=False ) diff --git a/app/scheduler.py b/app/scheduler.py index 6986747..a69c475 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -66,6 +66,7 @@ from app.services.event_study_service import run_and_store as run_event_study_an from app.services.outcome_service import evaluate_pending_setups from app.services.rr_scanner_service import scan_all_tickers from app.services.sentiment_provider_service import build_sentiment_provider +from app.services import ticker_service from app.services.ticker_universe_service import bootstrap_universe logger = logging.getLogger(__name__) @@ -396,8 +397,10 @@ async def _is_job_enabled(db: AsyncSession, job_name: str) -> bool: async def _get_all_tickers(db: AsyncSession) -> list[str]: - """Return all tracked ticker symbols sorted alphabetically.""" - result = await db.execute(select(Ticker.symbol).order_by(Ticker.symbol)) + """Return all actively-traded ticker symbols sorted alphabetically.""" + result = await db.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) return list(result.scalars().all()) @@ -412,8 +415,10 @@ async def _get_ohlcv_priority_tickers(db: AsyncSession) -> list[str]: latest_date = func.max(OHLCVRecord.date) missing_first = case((latest_date.is_(None), 0), else_=1) result = await db.execute( - select(Ticker.symbol) - .outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id) + ticker_service.active_only( + select(Ticker.symbol) + .outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id) + ) .group_by(Ticker.id, Ticker.symbol) .order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc()) ) @@ -662,14 +667,34 @@ async def collect_ohlcv( _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol) _log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested) if result.status == "stale": - await _record_system_event( - severity="warning", - source=job_name, - code="ohlcv_stale", - message=result.message or f"No new OHLCV bars for {symbol}", - symbol=symbol, - dedup_key=f"ohlcv_stale:{symbol}", + # "No new bars" cannot distinguish a delisting from a halt + # or a rename, so ask SEC before warning again. A confirmed + # delisting retires the symbol (keeping its history) and + # ends the alert; anything unproven keeps warning. + delisted_on = await ticker_service.confirm_delisting( + db, symbol, last_bar=result.last_date ) + if delisted_on is not None: + await _record_system_event( + severity="info", + source=job_name, + code="ticker_delisted", + message=( + f"{symbol} delisted on {delisted_on} (SEC Form 25/15). " + "Retired from signals; price history retained." + ), + symbol=symbol, + dedup_key=f"ticker_delisted:{symbol}", + ) + else: + await _record_system_event( + severity="warning", + source=job_name, + code="ohlcv_stale", + message=result.message or f"No new OHLCV bars for {symbol}", + symbol=symbol, + dedup_key=f"ohlcv_stale:{symbol}", + ) if result.status == "partial": # Rate limited — stop and resume next run _log_event(logging.WARNING, "rate_limited", job=job_name, ticker=symbol, processed=processed) diff --git a/app/schemas/ticker.py b/app/schemas/ticker.py index 966da06..4a79566 100644 --- a/app/schemas/ticker.py +++ b/app/schemas/ticker.py @@ -1,6 +1,6 @@ """Ticker request/response schemas.""" -from datetime import datetime +from datetime import date, datetime from pydantic import BaseModel, Field @@ -14,5 +14,10 @@ class TickerResponse(BaseModel): symbol: str name: str | None = None created_at: datetime + # NULL == actively traded. Delisted symbols stay in the registry with their + # history and are excluded from signals — the date is what makes that + # visible instead of the row silently disappearing. + delisted_on: date | None = None + delisted_reason: str | None = None model_config = {"from_attributes": True} diff --git a/app/services/breadth_service.py b/app/services/breadth_service.py index 6dd29bd..a472447 100644 --- a/app/services/breadth_service.py +++ b/app/services/breadth_service.py @@ -25,6 +25,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.ticker import Ticker +from app.services import ticker_service from app.services.price_service import query_ohlcv logger = logging.getLogger(__name__) @@ -112,7 +113,7 @@ def compute_divergence_series( async def _load_universe_closes( db: AsyncSession, symbols: list[str] | None = None ) -> dict[str, Series]: - stmt = select(Ticker).order_by(Ticker.symbol) + stmt = ticker_service.active_only(select(Ticker).order_by(Ticker.symbol)) if symbols is not None: stmt = stmt.where(Ticker.symbol.in_(symbols)) result = await db.execute(stmt) diff --git a/app/services/dolt_earnings_importer.py b/app/services/dolt_earnings_importer.py index d035c70..cb59486 100644 --- a/app/services/dolt_earnings_importer.py +++ b/app/services/dolt_earnings_importer.py @@ -32,7 +32,7 @@ from app.config import settings from app.database import insert_for_session from app.models.earnings_event import EarningsEvent from app.models.ticker import Ticker -from app.services import dolt_client, earnings_alignment +from app.services import dolt_client, earnings_alignment, ticker_service from app.services.data_import import ValidationResult logger = logging.getLogger(__name__) @@ -289,7 +289,11 @@ class DoltEarningsImporter: # -- helpers ----------------------------------------------------------- async def _load_universe(self, db) -> dict[str, int]: - rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all() + rows = ( + await db.execute( + ticker_service.active_only(select(Ticker.id, Ticker.symbol)) + ) + ).all() return { earnings_alignment.normalise_symbol(symbol): tid for tid, symbol in rows diff --git a/app/services/fundamentals_candidate_service.py b/app/services/fundamentals_candidate_service.py index 6ad0585..13fd01a 100644 --- a/app/services/fundamentals_candidate_service.py +++ b/app/services/fundamentals_candidate_service.py @@ -22,6 +22,7 @@ from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.ohlcv import OHLCVRecord from app.models.ticker import Ticker from app.services import fundamentals_derivation as deriv +from app.services import ticker_service @dataclass(frozen=True) @@ -46,7 +47,11 @@ async def build_candidates( """Derive current cache candidates using only already-stored data.""" today = today or datetime.now(ZoneInfo("America/New_York")).date() tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ( + await db.execute( + ticker_service.active_only(select(Ticker).order_by(Ticker.symbol)) + ) + ).scalars() ) if not tickers: return [] diff --git a/app/services/momentum_service.py b/app/services/momentum_service.py index 20758aa..764c345 100644 --- a/app/services/momentum_service.py +++ b/app/services/momentum_service.py @@ -18,6 +18,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.ticker import Ticker +from app.services import ticker_service from app.services.price_service import query_ohlcv logger = logging.getLogger(__name__) @@ -169,7 +170,9 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa before scanning; the research backtest ranked each weekly setup-candidate cross-section, so this is the deliberate production approximation. """ - result = await db.execute(select(Ticker).order_by(Ticker.symbol)) + result = await db.execute( + ticker_service.active_only(select(Ticker).order_by(Ticker.symbol)) + ) tickers = list(result.scalars().all()) benchmark_closes = await _load_activation_benchmark(db) diff --git a/app/services/rr_scanner_service.py b/app/services/rr_scanner_service.py index 012abbc..6c7d010 100644 --- a/app/services/rr_scanner_service.py +++ b/app/services/rr_scanner_service.py @@ -31,7 +31,7 @@ from app.services import fundamentals_quality_service, system_event_service from app.services.price_service import query_ohlcv from app.services.qualification import setup_qualifies from app.services.sr_service import detect_gate_target_ladder -from app.services import settings_store +from app.services import settings_store, ticker_service from app.services.trade_policy import ( MANUAL_BOOK, SHADOW_BOOK, @@ -735,7 +735,11 @@ async def scan_all_tickers( # Plain ids/strings, not Ticker instances: the rollbacks below expire any # ORM objects held across them, and touching an expired attribute afterwards # triggers sync lazy-loading, which raises on an AsyncSession. - result = await db.execute(select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)) + result = await db.execute( + ticker_service.active_only( + select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol) + ) + ) ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()] total = len(ticker_rows) diff --git a/app/services/scoring_service.py b/app/services/scoring_service.py index 90b01c0..1d5d0a6 100644 --- a/app/services/scoring_service.py +++ b/app/services/scoring_service.py @@ -20,7 +20,7 @@ from app.database import insert_for_session from app.exceptions import NotFoundError, ValidationError from app.models.score import CompositeScore, DimensionScore from app.models.ticker import Ticker -from app.services import settings_store +from app.services import settings_store, ticker_service logger = logging.getLogger(__name__) @@ -883,7 +883,11 @@ async def get_rankings(db: AsyncSession) -> dict: Returns dict suitable for RankingResponse. """ weights = await _get_weights(db) - tickers = (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars().all() + tickers = ( + await db.execute( + ticker_service.active_only(select(Ticker).order_by(Ticker.symbol)) + ) + ).scalars().all() async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]: comps = { @@ -947,7 +951,7 @@ async def update_weights( await _save_weights(db, full_weights) # Recompute all composite scores - result = await db.execute(select(Ticker)) + result = await db.execute(ticker_service.active_only(select(Ticker))) tickers = list(result.scalars().all()) for ticker in tickers: diff --git a/app/services/sec_client.py b/app/services/sec_client.py index 1f68bdd..91aa2c0 100644 --- a/app/services/sec_client.py +++ b/app/services/sec_client.py @@ -45,6 +45,13 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"}) +# Exchange delisting (25 / 25-NSE) and registration termination (15 family). +# Their presence is SEC confirming a security stopped trading — see +# ``SecClient.delisting_filing``. +_DELISTING_FORMS = frozenset({ + "25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G", +}) + class SecError(ProviderError): """SEC request failed (403, exhausted 429/5xx, timeout, transport, parse).""" @@ -253,6 +260,34 @@ class SecClient: "filings": filings, } + async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None: + """Newest exchange-delisting / deregistration filing, or ``None``. + + Form 25/25-NSE strikes a security from listing; Form 15 terminates the + registration. Either is SEC confirming the security stopped trading — + which is what separates a real delisting from a multi-day halt or a + ticker rename, neither of which files one. That precision is the reason + this can mark a symbol automatically. + + Reads ``filings.recent`` directly: ``submissions()`` keeps only the + 10-K/10-Q family, so these forms never survive its parser. + """ + base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json") + arrays = (base.get("filings") or {}).get("recent") or {} + forms = arrays.get("form") or [] + dates = arrays.get("filingDate") or [] + best: dict[str, Any] | None = None + for i, form in enumerate(forms): + if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]: + continue + try: + filed = date.fromisoformat(dates[i]) + except ValueError: + continue + if best is None or filed > best["filing_date"]: + best = {"form": form, "filing_date": filed} + return best + async def companyfacts(self, cik: int | str) -> dict[str, Any]: """Raw companyfacts JSON ({cik, entityName, facts}).""" return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json") diff --git a/app/services/sec_universe.py b/app/services/sec_universe.py index a81efde..4beff6c 100644 --- a/app/services/sec_universe.py +++ b/app/services/sec_universe.py @@ -24,7 +24,7 @@ from typing import Iterable from sqlalchemy import select, update from app.models.ticker import Ticker -from app.services import settings_store +from app.services import settings_store, ticker_service from app.services.earnings_alignment import normalise_symbol from app.services.sec_client import SecClient @@ -55,7 +55,11 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse: returns the mapping + proposed `tickers.cik` writes; mutates nothing.""" ticker_to_cik = await client.company_tickers() overrides = await cik_overrides(db) - rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all() + rows = ( + await db.execute( + ticker_service.active_only(select(Ticker.id, Ticker.symbol, Ticker.cik)) + ) + ).all() result = ResolvedUniverse() for tid, symbol, current_cik in rows: diff --git a/app/services/ticker_service.py b/app/services/ticker_service.py index e083253..6cf58d3 100644 --- a/app/services/ticker_service.py +++ b/app/services/ticker_service.py @@ -1,13 +1,48 @@ -"""Ticker Registry service: add, delete, and list tracked tickers.""" +"""Ticker Registry service: add, delete, list, and retire tracked tickers.""" +import logging import re +from datetime import date -from sqlalchemy import select +from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from app.exceptions import DuplicateError, NotFoundError, ValidationError from app.models.ticker import Ticker +logger = logging.getLogger(__name__) + +# Reasons a symbol may be marked delisted, narrowest first. +REASON_FORM_25 = "form_25" # SEC Form 25/25-NSE/15 confirmed the exchange exit +REASON_MANUAL = "manual" # an operator decided + +# How long a symbol must be without bars before we spend an SEC request asking +# whether it delisted. Guards against a market-data outage probing the whole +# universe at once; a real delisting is still stale days later. +MIN_STALE_DAYS_BEFORE_PROBE = 3 + + +def _sec_client_factory(): + """Build the SEC client for a delisting probe (patched in tests). + + Imported lazily so the SEC/httpx stack stays off the import path of every + module that only wants ``active_only``. + """ + from app.services.sec_client import SecClient + + return SecClient() + + +def active_only(stmt): + """Restrict a Ticker query to symbols that still trade. + + Opt-in on purpose rather than folded into a shared getter: list and admin + views deliberately keep delisted rows so the delisting is *visible*, which a + silent default would undo. Apply this on the live signal path — scanning, + ranking, scoring, breadth, ingestion — and nowhere else. + """ + return stmt.where(Ticker.delisted_on.is_(None)) + async def add_ticker(db: AsyncSession, symbol: str) -> Ticker: """Add a new ticker after validation. @@ -52,6 +87,121 @@ async def delete_ticker(db: AsyncSession, symbol: str) -> None: async def list_tickers(db: AsyncSession) -> list[Ticker]: - """Return all tracked tickers sorted alphabetically by symbol.""" + """Return all tracked tickers sorted alphabetically by symbol. + + Delisted symbols are included and carry ``delisted_on`` — the registry is + where an operator needs to *see* that a symbol retired, not where it should + quietly disappear. + """ result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc())) return list(result.scalars().all()) + + +async def mark_delisted( + db: AsyncSession, + symbol: str, + *, + delisted_on: date, + reason: str = REASON_MANUAL, +) -> bool: + """Record that a symbol stopped trading. True if this changed anything. + + Idempotent: re-marking an already-delisted symbol is a no-op, so the + staleness path can call it on every run without churning the row or + re-emitting events. + """ + normalised = symbol.strip().upper() + result = await db.execute(select(Ticker).where(Ticker.symbol == normalised)) + ticker = result.scalar_one_or_none() + if ticker is None: + raise NotFoundError(f"Ticker not found: {normalised}") + if ticker.delisted_on is not None: + return False + + await db.execute( + update(Ticker) + .where(Ticker.id == ticker.id) + .values(delisted_on=delisted_on, delisted_reason=reason) + ) + await db.commit() + logger.info( + "ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason + ) + return True + + +async def confirm_delisting( + db: AsyncSession, + symbol: str, + *, + last_bar: date | None, + today: date | None = None, +) -> date | None: + """Ask SEC whether ``symbol`` actually delisted; mark it if so. + + Called when OHLCV goes stale, because "no new bars" alone cannot tell a + delisting from a halt or a rename. Returns the effective date when this call + marked the symbol, else ``None`` — already-marked and unconfirmed both return + ``None``, so the caller keeps its existing alert for anything unproven. + + Deliberately driven by staleness rather than by the SEC fundamentals import: + that importer stalls for days at a time on unrelated Company-Facts gaps, and + detection wired into it would stall with it. + + The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays + stale forever, so the delay costs nothing, and it keeps a broad market-data + outage — where every tracked symbol reports stale at once — from turning into + one SEC request per symbol per run. + """ + from app.services.sec_client import SecError + + normalised = symbol.strip().upper() + result = await db.execute(select(Ticker).where(Ticker.symbol == normalised)) + ticker = result.scalar_one_or_none() + if ticker is None or ticker.delisted_on is not None or not ticker.cik: + return None + # No bars at all is an ingestion problem, not evidence of a delisting. + if last_bar is None: + return None + if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE: + return None + + try: + async with _sec_client_factory() as client: + filing = await client.delisting_filing(ticker.cik) + except SecError: + # Never let a probe failure escalate a routine staleness warning. + logger.warning("delisting probe failed for %s", normalised, exc_info=True) + return None + + if filing is None: + return None + if await mark_delisted( + db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25 + ): + return filing["filing_date"] + return None + + +async def clear_delisted(db: AsyncSession, symbol: str) -> bool: + """Un-retire a symbol. True if it had been marked. + + The counterpart that makes automatic marking acceptable: a false positive + costs one row update, where a delete would have cost the price history. + """ + normalised = symbol.strip().upper() + result = await db.execute(select(Ticker).where(Ticker.symbol == normalised)) + ticker = result.scalar_one_or_none() + if ticker is None: + raise NotFoundError(f"Ticker not found: {normalised}") + if ticker.delisted_on is None: + return False + + await db.execute( + update(Ticker) + .where(Ticker.id == ticker.id) + .values(delisted_on=None, delisted_reason=None) + ) + await db.commit() + logger.info("ticker %s un-marked as delisted", normalised) + return True diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index fa70a2a..8ab13c2 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -858,6 +858,10 @@ export interface Ticker { symbol: string; name: string | null; created_at: string; + /** Set once the symbol stopped trading: excluded from signals, history kept. */ + delisted_on: string | null; + /** How the delisting was learned: "form_25" (SEC confirmed) | "manual". */ + delisted_reason: string | null; } // Admin diff --git a/tests/unit/test_ticker_delisting.py b/tests/unit/test_ticker_delisting.py new file mode 100644 index 0000000..957c088 --- /dev/null +++ b/tests/unit/test_ticker_delisting.py @@ -0,0 +1,183 @@ +"""Delisting lifecycle: marking, the active_only filter, and SEC confirmation. + +The behaviour under test is that a delisted symbol leaves the *live* path while +its rows stay put — deleting it instead is what makes the backtest universe +survivorship-biased, so retention is the point, not a side effect. +""" + +from __future__ import annotations + +import json +from collections.abc import AsyncGenerator +from datetime import date + +import httpx +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.database import Base +from app.models.ticker import Ticker +from app.services import ticker_service +from app.services.sec_client import SecClient + +_engine = create_async_engine("sqlite+aiosqlite://", echo=False) +_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest.fixture(autouse=True) +async def _setup_tables() -> AsyncGenerator[None, None]: + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def session() -> AsyncGenerator[AsyncSession, None]: + async with _session_factory() as s: + yield s + + +def _submissions(forms: list[str], dates: list[str]) -> dict: + return { + "cik": 712515, + "name": "ELECTRONIC ARTS INC.", + "filings": {"recent": {"form": forms, "filingDate": dates}}, + } + + +def _sec_client(payload: dict) -> SecClient: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=json.dumps(payload).encode()) + + return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) + + +async def test_mark_delisted_is_idempotent(session: AsyncSession): + session.add(Ticker(symbol="EA")) + await session.commit() + + assert await ticker_service.mark_delisted( + session, "EA", delisted_on=date(2026, 8, 4) + ) is True + # A second call must not churn the row — the staleness path retries daily. + assert await ticker_service.mark_delisted( + session, "EA", delisted_on=date(2026, 9, 1) + ) is False + + row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() + assert row.delisted_on == date(2026, 8, 4) # first date wins, not the retry + assert row.delisted_reason == ticker_service.REASON_MANUAL + + +async def test_clear_delisted_restores_the_symbol(session: AsyncSession): + session.add(Ticker(symbol="EA")) + await session.commit() + await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4)) + + assert await ticker_service.clear_delisted(session, "EA") is True + assert await ticker_service.clear_delisted(session, "EA") is False + + row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() + assert row.delisted_on is None and row.delisted_reason is None + + +async def test_active_only_filters_but_the_row_survives(session: AsyncSession): + session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")]) + await session.commit() + await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 4)) + + active = ( + await session.execute(ticker_service.active_only(select(Ticker.symbol))) + ).scalars().all() + assert list(active) == ["AAPL"] + + # The whole point: the row — and everything cascading off it — is still there. + everything = [t.symbol for t in await ticker_service.list_tickers(session)] + assert everything == ["AAPL", "EA"] + + +async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monkeypatch): + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client(_submissions(["8-K", "25-NSE"], ["2026-07-01", "2026-08-04"])), + raising=False, + ) + + marked = await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) + assert marked == date(2026, 8, 4) + + row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() + assert row.delisted_reason == ticker_service.REASON_FORM_25 + + +async def test_confirm_delisting_leaves_a_halt_alone(session: AsyncSession, monkeypatch): + """A halt or a rename files no Form 25 — those must keep warning, not retire.""" + session.add(Ticker(symbol="SATS", cik="0000012345")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client(_submissions(["8-K", "10-Q"], ["2026-07-01", "2026-08-04"])), + raising=False, + ) + + assert await ticker_service.confirm_delisting( + session, "SATS", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + row = (await session.execute(select(Ticker).where(Ticker.symbol == "SATS"))).scalar_one() + assert row.delisted_on is None + + +async def test_confirm_delisting_skips_a_symbol_without_a_cik(session: AsyncSession): + """No CIK, no SEC lookup — must not raise, and must not mark.""" + session.add(Ticker(symbol="ADRX")) + await session.commit() + assert await ticker_service.confirm_delisting( + session, "ADRX", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + + +async def test_delisting_filing_picks_the_newest_match(): + client = _sec_client( + _submissions( + ["25", "8-K", "25-NSE", "15-12B"], + ["2024-01-02", "2026-08-01", "2026-08-04", "2025-05-05"], + ) + ) + async with client as c: + found = await c.delisting_filing("0000712515") + assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)} + + +async def test_delisting_filing_returns_none_without_one(): + client = _sec_client(_submissions(["10-K", "8-K"], ["2026-01-02", "2026-08-01"])) + async with client as c: + assert await c.delisting_filing("0000320193") is None + + +async def test_confirm_delisting_waits_before_spending_a_request(session: AsyncSession, monkeypatch): + """A one-day gap is a weekend or a hiccup. Probing every stale symbol during a + market-data outage would be one SEC request per symbol per run.""" + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + + def _explode(): + raise AssertionError("must not reach SEC before the stale threshold") + + monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False) + + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 10), today=date(2026, 8, 11) + ) is None + # ...and no bars at all is an ingestion problem, not a delisting. + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=None, today=date(2026, 8, 11) + ) is None