fix(tickers): honor the effective date instead of retiring on the mark
active_only tested delisted_on IS NULL, so a symbol dropped out of signals the moment a Form 25 was detected — ten days before Rule 12d2-2 makes the removal effective, while it was demonstrably still trading. A manual future-dated mark behaved the same way. It now compares against the database's own date, so a pending delisting stays live until the day it takes effect. That exposes a second problem the fix would otherwise create. Trading typically stops before the ten-day delay expires, so across that window the symbol is correctly active yet produces no bars — and confirm_delisting returned None for an already-marked row, which would have fired the staleness warning daily for ten days, the exact noise this flow exists to remove. It now reports the known effective date on every path where the delisting is established, so the caller warns only about gaps that are still unexplained. CURRENT_DATE renders identically on postgres and sqlite, and the OR is parenthesized when callers chain further where clauses. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@ import logging
|
|||||||
import re
|
import re
|
||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
|
||||||
from sqlalchemy import select, update
|
from sqlalchemy import func, or_, select, update
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
||||||
@@ -42,15 +42,23 @@ def _sec_client_factory():
|
|||||||
return SecClient()
|
return SecClient()
|
||||||
|
|
||||||
|
|
||||||
def active_only(stmt):
|
def active_only(stmt, *, as_of: date | None = None):
|
||||||
"""Restrict a Ticker query to symbols that still trade.
|
"""Restrict a Ticker query to symbols that still trade.
|
||||||
|
|
||||||
Opt-in on purpose rather than folded into a shared getter: list and admin
|
Opt-in on purpose rather than folded into a shared getter: list and admin
|
||||||
views deliberately keep delisted rows so the delisting is *visible*, which a
|
views deliberately keep delisted rows so the delisting is *visible*, which a
|
||||||
silent default would undo. Apply this on the live signal path — scanning,
|
silent default would undo. Apply this on the live signal path — scanning,
|
||||||
ranking, scoring, breadth, ingestion — and nowhere else.
|
ranking, scoring, breadth, ingestion — and nowhere else.
|
||||||
|
|
||||||
|
``delisted_on`` is an *effective* date, and a Form 25 is known ten days
|
||||||
|
before it takes effect, so a future date must not drop the symbol yet — it
|
||||||
|
is still trading and still worth scanning and ingesting. Compared in SQL
|
||||||
|
against the database's own date; ``as_of`` overrides it for tests.
|
||||||
"""
|
"""
|
||||||
return stmt.where(Ticker.delisted_on.is_(None))
|
cutoff = func.current_date() if as_of is None else as_of
|
||||||
|
return stmt.where(
|
||||||
|
or_(Ticker.delisted_on.is_(None), Ticker.delisted_on > cutoff)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
||||||
@@ -155,9 +163,16 @@ async def confirm_delisting(
|
|||||||
"""Ask SEC whether ``symbol`` actually delisted; mark it if so.
|
"""Ask SEC whether ``symbol`` actually delisted; mark it if so.
|
||||||
|
|
||||||
Called when OHLCV goes stale, because "no new bars" alone cannot tell a
|
Called when OHLCV goes stale, because "no new bars" alone cannot tell a
|
||||||
delisting from a halt or a rename. Returns the effective date when this call
|
delisting from a halt or a rename. Returns the effective date whenever the
|
||||||
marked the symbol, else ``None`` — already-marked and unconfirmed both return
|
symbol is known to have delisted — whether this call established that or an
|
||||||
``None``, so the caller keeps its existing alert for anything unproven.
|
earlier one did — and ``None`` while it remains unproven, so the caller warns
|
||||||
|
only about gaps that still have no explanation.
|
||||||
|
|
||||||
|
Returning the already-known date matters between filing and effect: trading
|
||||||
|
usually stops before the ten-day Rule 12d2-2 delay expires, so the symbol is
|
||||||
|
correctly still active (see ``active_only``) while producing no bars. Without
|
||||||
|
this the staleness warning would fire daily across that window — the exact
|
||||||
|
noise the delisting flow exists to remove.
|
||||||
|
|
||||||
Deliberately driven by staleness rather than by the SEC fundamentals import:
|
Deliberately driven by staleness rather than by the SEC fundamentals import:
|
||||||
that importer stalls for days at a time on unrelated Company-Facts gaps, and
|
that importer stalls for days at a time on unrelated Company-Facts gaps, and
|
||||||
@@ -173,17 +188,21 @@ async def confirm_delisting(
|
|||||||
normalised = symbol.strip().upper()
|
normalised = symbol.strip().upper()
|
||||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||||
ticker = result.scalar_one_or_none()
|
ticker = result.scalar_one_or_none()
|
||||||
if ticker is None or not ticker.cik:
|
if ticker is None:
|
||||||
return None
|
return None
|
||||||
# Already confirmed by SEC — nothing left to learn. A row an operator marked
|
known = ticker.delisted_on
|
||||||
# by hand is still worth probing: Form 25 upgrades the estimated date.
|
# Already confirmed by SEC — nothing left to learn, but the caller still
|
||||||
|
# needs the date to know this gap is explained. A row an operator marked by
|
||||||
|
# hand is worth probing: Form 25 upgrades the estimated date.
|
||||||
if ticker.delisted_reason == REASON_FORM_25:
|
if ticker.delisted_reason == REASON_FORM_25:
|
||||||
return None
|
return known
|
||||||
|
if not ticker.cik:
|
||||||
|
return known
|
||||||
# No bars at all is an ingestion problem, not evidence of a delisting.
|
# No bars at all is an ingestion problem, not evidence of a delisting.
|
||||||
if last_bar is None:
|
if last_bar is None:
|
||||||
return None
|
return known
|
||||||
if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
|
if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
|
||||||
return None
|
return known
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with _sec_client_factory() as client:
|
async with _sec_client_factory() as client:
|
||||||
@@ -196,10 +215,10 @@ async def confirm_delisting(
|
|||||||
except SecError:
|
except SecError:
|
||||||
# Never let a probe failure escalate a routine staleness warning.
|
# Never let a probe failure escalate a routine staleness warning.
|
||||||
logger.warning("delisting probe failed for %s", normalised, exc_info=True)
|
logger.warning("delisting probe failed for %s", normalised, exc_info=True)
|
||||||
return None
|
return known
|
||||||
|
|
||||||
if filing is None:
|
if filing is None:
|
||||||
return None
|
return known
|
||||||
# Removal takes effect ten days after filing, so the filing date is not the
|
# Removal takes effect ten days after filing, so the filing date is not the
|
||||||
# date the symbol stopped trading.
|
# date the symbol stopped trading.
|
||||||
effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS)
|
effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS)
|
||||||
@@ -207,7 +226,7 @@ async def confirm_delisting(
|
|||||||
db, normalised, delisted_on=effective, reason=REASON_FORM_25
|
db, normalised, delisted_on=effective, reason=REASON_FORM_25
|
||||||
):
|
):
|
||||||
return effective
|
return effective
|
||||||
return None
|
return known
|
||||||
|
|
||||||
|
|
||||||
async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
|
async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
|
||||||
|
|||||||
@@ -236,9 +236,11 @@ async def test_a_confirmed_row_is_never_reprobed(session: AsyncSession, monkeypa
|
|||||||
raise AssertionError("a SEC-confirmed row must not cost another request")
|
raise AssertionError("a SEC-confirmed row must not cost another request")
|
||||||
|
|
||||||
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
||||||
|
# Costs no request, and still reports the date so the caller knows this gap
|
||||||
|
# is explained and must not warn about it again.
|
||||||
assert await ticker_service.confirm_delisting(
|
assert await ticker_service.confirm_delisting(
|
||||||
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 9, 1)
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 9, 1)
|
||||||
) is None
|
) == date(2026, 8, 4)
|
||||||
|
|
||||||
|
|
||||||
async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession):
|
async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession):
|
||||||
@@ -365,3 +367,50 @@ async def test_prune_keeps_delisted_rows(session: AsyncSession, monkeypatch):
|
|||||||
assert remaining == ["AAPL", "EA"] # GONE pruned, EA protected
|
assert remaining == ["AAPL", "EA"] # GONE pruned, EA protected
|
||||||
assert summary["deleted"] == 1
|
assert summary["deleted"] == 1
|
||||||
assert summary["kept_delisted"] == ["EA"]
|
assert summary["kept_delisted"] == ["EA"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_future_effective_date_keeps_the_symbol_live(session: AsyncSession):
|
||||||
|
"""Form 25 is known ten days before removal takes effect. The symbol is still
|
||||||
|
trading in that window and must keep being scanned and ingested."""
|
||||||
|
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA")])
|
||||||
|
await session.commit()
|
||||||
|
await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14))
|
||||||
|
|
||||||
|
def active(as_of: date) -> list[str]:
|
||||||
|
return ticker_service.active_only(select(Ticker.symbol), as_of=as_of)
|
||||||
|
|
||||||
|
before = (await session.execute(active(date(2026, 8, 11)))).scalars().all()
|
||||||
|
on_the_day = (await session.execute(active(date(2026, 8, 14)))).scalars().all()
|
||||||
|
after = (await session.execute(active(date(2026, 8, 15)))).scalars().all()
|
||||||
|
|
||||||
|
assert sorted(before) == ["AAPL", "EA"] # still trading
|
||||||
|
assert sorted(on_the_day) == ["AAPL"] # removal effective
|
||||||
|
assert sorted(after) == ["AAPL"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_the_pending_window_does_not_re_warn(session: AsyncSession, monkeypatch):
|
||||||
|
"""Between filing and effect the symbol is active but produces no bars. That
|
||||||
|
must not resurrect the daily staleness warning this flow exists to end."""
|
||||||
|
session.add(Ticker(symbol="EA", cik="0000712515"))
|
||||||
|
await session.commit()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ticker_service,
|
||||||
|
"_sec_client_factory",
|
||||||
|
lambda: _sec_client(_submissions(["25-NSE"], ["2026-08-04"])),
|
||||||
|
raising=False,
|
||||||
|
)
|
||||||
|
first = await ticker_service.confirm_delisting(
|
||||||
|
session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11)
|
||||||
|
)
|
||||||
|
assert first == date(2026, 8, 14)
|
||||||
|
|
||||||
|
def _explode():
|
||||||
|
raise AssertionError("must not re-probe a confirmed row")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ticker_service, "_sec_client_factory", _explode, raising=False)
|
||||||
|
# Every later run inside the window still reports the delisting, so the
|
||||||
|
# caller keeps emitting "delisted" rather than "no new bars".
|
||||||
|
for day in (date(2026, 8, 12), date(2026, 8, 13)):
|
||||||
|
assert await ticker_service.confirm_delisting(
|
||||||
|
session, "EA", last_bar=date(2026, 8, 4), today=day
|
||||||
|
) == date(2026, 8, 14)
|
||||||
|
|||||||
Reference in New Issue
Block a user