diff --git a/app/routers/tickers.py b/app/routers/tickers.py index 74683ce..96d9027 100644 --- a/app/routers/tickers.py +++ b/app/routers/tickers.py @@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.dependencies import get_db, require_access from app.models.user import User from app.schemas.common import APIEnvelope -from app.schemas.ticker import TickerCreate, TickerResponse +from app.schemas.ticker import TickerCreate, TickerDelistingUpdate, TickerResponse from app.services import ticker_service router = APIRouter(tags=["tickers"]) @@ -51,3 +51,35 @@ async def delete_ticker( """Delete a ticker and all associated data.""" await ticker_service.delete_ticker(db, symbol) return APIEnvelope(status="success", data=None) + + +@router.post("/tickers/{symbol}/delisting", response_model=APIEnvelope) +async def mark_ticker_delisted( + symbol: str, + body: TickerDelistingUpdate, + _user: User = Depends(require_access), + db: AsyncSession = Depends(get_db), +): + """Retire a symbol: excluded from signals, price history kept. + + The non-destructive alternative to DELETE, which cascades the history away. + """ + changed = await ticker_service.mark_delisted( + db, symbol, delisted_on=body.delisted_on, reason=ticker_service.REASON_MANUAL + ) + return APIEnvelope(status="success", data={"changed": changed}) + + +@router.delete("/tickers/{symbol}/delisting", response_model=APIEnvelope) +async def clear_ticker_delisting( + symbol: str, + _user: User = Depends(require_access), + db: AsyncSession = Depends(get_db), +): + """Un-retire a symbol wrongly marked delisted. + + Automatic marking is only defensible because this exists: a false positive + costs one row update rather than the price history a delete would take. + """ + changed = await ticker_service.clear_delisted(db, symbol) + return APIEnvelope(status="success", data={"changed": changed}) diff --git a/app/schemas/ticker.py b/app/schemas/ticker.py index 4a79566..15530fa 100644 --- a/app/schemas/ticker.py +++ b/app/schemas/ticker.py @@ -21,3 +21,9 @@ class TickerResponse(BaseModel): delisted_reason: str | None = None model_config = {"from_attributes": True} + + +class TickerDelistingUpdate(BaseModel): + delisted_on: date = Field( + ..., description="Effective date the symbol stopped trading" + ) diff --git a/app/services/sec_client.py b/app/services/sec_client.py index 91aa2c0..19ba587 100644 --- a/app/services/sec_client.py +++ b/app/services/sec_client.py @@ -45,12 +45,32 @@ _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True _FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"}) -# Exchange delisting (25 / 25-NSE) and registration termination (15 family). -# Their presence is SEC confirming a security stopped trading — see -# ``SecClient.delisting_filing``. -_DELISTING_FORMS = frozenset({ - "25", "25-NSE", "15-12B", "15-12G", "15F-12B", "15F-12G", -}) +# Notification of removal from listing. "25" is issuer-filed, "25-NSE" exchange- +# filed. The Form 15 family is deliberately absent: it ends a *reporting* +# obligation and does not mean the security stopped trading. +_DELISTING_FORMS = frozenset({"25", "25-NSE"}) + +# ``descriptionClassSecurity`` is free text ("Common Stock", "Class A Common +# Stock, $0.01 par value", "6.25% Notes due 2030", "Warrants", "Depositary +# Shares"). Only a common-equity class means the ticker itself stopped trading. +_NON_COMMON_CLASS = re.compile( + r"\b(note|bond|debenture|preferred|warrant|right|unit|depositary|" + r"subordinated|debt|trust)s?\b", + re.IGNORECASE, +) + + +def _is_common_stock(description: str) -> bool: + """Does this Form 25 security class describe common equity? + + Requires an explicit common-stock match AND no debt/preferred/warrant marker, + so "Depositary Shares each representing 1/1000th of Preferred" cannot pass on + the word "shares" alone. Unrecognised text is rejected — a symbol is retired + on this answer, so ambiguity must not read as yes. + """ + if _NON_COMMON_CLASS.search(description): + return False + return re.search(r"\bcommon\s+(stock|share)", description, re.IGNORECASE) is not None class SecError(ProviderError): @@ -260,23 +280,40 @@ class SecClient: "filings": filings, } - async def delisting_filing(self, cik: int | str) -> dict[str, Any] | None: - """Newest exchange-delisting / deregistration filing, or ``None``. + async def delisting_filing( + self, cik: int | str, *, not_before: date | None = None + ) -> dict[str, Any] | None: + """Newest Form 25 removing this issuer's COMMON stock from listing. - Form 25/25-NSE strikes a security from listing; Form 15 terminates the - registration. Either is SEC confirming the security stopped trading — - which is what separates a real delisting from a multi-day halt or a - ticker rename, neither of which files one. That precision is the reason - this can mark a symbol automatically. + Deliberately narrow, because the caller retires a symbol on the answer: - Reads ``filings.recent`` directly: ``submissions()`` keeps only the - 10-K/10-Q family, so these forms never survive its parser. + - **Form 25 only.** The Form 15 family terminates a reporting obligation + (often just a class falling under the holder threshold) and is no + evidence that trading stopped. + - **Class-checked.** Form 25 is filed per security class — an issuer + delisting its notes, preferred, warrants or an ADR class while the + common keeps trading files one too. The filing's own + ``descriptionClassSecurity`` is what separates those, so the primary + document is fetched and read rather than trusting the form type. + - **``not_before``** rejects a historical filing for some long-gone + class. Without it a 2019 Form 25 would retire a symbol whose bars + stopped in 2026, and stamp 2019 as the date. + + Anything unreadable — no primary document (pre-2009 filings have none), + malformed XML, unrecognised class — returns ``None``. Fail closed: the + caller keeps warning instead of retiring on a guess. + + Reads ``filings.recent`` directly; ``submissions()`` keeps only the + 10-K/10-Q family, so Form 25 never survives its parser. """ base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json") arrays = (base.get("filings") or {}).get("recent") or {} forms = arrays.get("form") or [] dates = arrays.get("filingDate") or [] - best: dict[str, Any] | None = None + accessions = arrays.get("accessionNumber") or [] + docs = arrays.get("primaryDocument") or [] + + candidates: list[tuple[date, str, str, str]] = [] for i, form in enumerate(forms): if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]: continue @@ -284,9 +321,49 @@ class SecClient: filed = date.fromisoformat(dates[i]) except ValueError: continue - if best is None or filed > best["filing_date"]: - best = {"form": form, "filing_date": filed} - return best + if not_before is not None and filed < not_before: + continue + if i >= len(accessions) or not accessions[i]: + continue + candidates.append((filed, form, accessions[i], docs[i] if i < len(docs) else "")) + + for filed, form, accession, _doc in sorted(candidates, reverse=True): + security = await self._form25_security_class(cik, accession) + if security is None: + continue + if not _is_common_stock(security): + continue + return { + "form": form, + "filing_date": filed, + "security_class": security, + } + return None + + async def _form25_security_class( + self, cik: int | str, accession: str + ) -> str | None: + """``descriptionClassSecurity`` from a Form 25's primary XML, or None. + + The rendered ``primaryDocument`` is an XSL view of this file; the raw + ``primary_doc.xml`` beside it is the structured original. + """ + folder = accession.replace("-", "") + url = ( + f"{_WWW}/Archives/edgar/data/{int(cik)}/{folder}/primary_doc.xml" + ) + try: + body = await self.get_text(url) + except SecNotFoundError: + return None + match = re.search( + r"(.*?)", + body, + re.IGNORECASE | re.DOTALL, + ) + if match is None: + return None + return " ".join(match.group(1).split()) or None async def companyfacts(self, cik: int | str) -> dict[str, Any]: """Raw companyfacts JSON ({cik, entityName, facts}).""" diff --git a/app/services/ticker_service.py b/app/services/ticker_service.py index 9c58c15..db516f3 100644 --- a/app/services/ticker_service.py +++ b/app/services/ticker_service.py @@ -2,7 +2,7 @@ import logging import re -from datetime import date +from datetime import date, timedelta from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession @@ -21,6 +21,15 @@ REASON_MANUAL = "manual" # an operator decided # universe at once; a real delisting is still stale days later. MIN_STALE_DAYS_BEFORE_PROBE = 3 +# Rule 12d2-2: a Form 25 removal takes effect ten days after filing, so the +# filing date is not the date the security stopped trading. +FORM_25_EFFECTIVE_DAYS = 10 + +# How far before the last bar a Form 25 may be filed and still explain this gap. +# An exchange can file shortly before trading actually stops; anything older +# concerns a class that was already gone while the symbol kept printing bars. +FILING_LOOKBACK_DAYS = 30 + def _sec_client_factory(): """Build the SEC client for a delisting probe (patched in tests). @@ -178,7 +187,12 @@ async def confirm_delisting( try: async with _sec_client_factory() as client: - filing = await client.delisting_filing(ticker.cik) + # Only a Form 25 filed around or after the last bar can explain THIS + # gap. An older one belongs to a class that stopped trading before + # the symbol was still printing bars, and must not retire it. + filing = await client.delisting_filing( + ticker.cik, not_before=last_bar - timedelta(days=FILING_LOOKBACK_DAYS) + ) except SecError: # Never let a probe failure escalate a routine staleness warning. logger.warning("delisting probe failed for %s", normalised, exc_info=True) @@ -186,10 +200,13 @@ async def confirm_delisting( if filing is None: return None + # Removal takes effect ten days after filing, so the filing date is not the + # date the symbol stopped trading. + effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS) if await mark_delisted( - db, normalised, delisted_on=filing["filing_date"], reason=REASON_FORM_25 + db, normalised, delisted_on=effective, reason=REASON_FORM_25 ): - return filing["filing_date"] + return effective return None diff --git a/app/services/ticker_universe_service.py b/app/services/ticker_universe_service.py index b895c2a..a496da9 100644 --- a/app/services/ticker_universe_service.py +++ b/app/services/ticker_universe_service.py @@ -357,9 +357,26 @@ async def bootstrap_universe( db.add(Ticker(symbol=symbol)) deleted_count = 0 + skipped_delisted: list[str] = [] if symbols_to_delete: - result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(symbols_to_delete))) - deleted_count = int(result.rowcount or 0) + # A delisted row was retained on purpose — its price history is exactly + # what a survivorship-honest backtest needs, and the delete cascades it + # away. Pruning must not undo that. (Pruning a symbol that is merely no + # longer an index constituent still destroys history; that needs a + # tracked/membership state separate from delisting.) + protected = ( + await db.execute( + select(Ticker.symbol).where( + Ticker.symbol.in_(symbols_to_delete), + Ticker.delisted_on.is_not(None), + ) + ) + ).scalars().all() + skipped_delisted = sorted(protected) + deletable = [s for s in symbols_to_delete if s not in set(protected)] + if deletable: + result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(deletable))) + deleted_count = int(result.rowcount or 0) await db.commit() @@ -378,4 +395,8 @@ async def bootstrap_universe( "already_tracked": len(target_symbols & existing_symbols), "deleted": deleted_count, "added_symbols": symbols_to_add[:50], + # Delisted rows a prune declined to destroy, so the caller can see the + # count did not match what they asked to remove. + "kept_delisted": skipped_delisted[:50], + "kept_delisted_count": len(skipped_delisted), } diff --git a/tests/unit/test_ticker_delisting.py b/tests/unit/test_ticker_delisting.py index 08ce190..1263341 100644 --- a/tests/unit/test_ticker_delisting.py +++ b/tests/unit/test_ticker_delisting.py @@ -44,12 +44,30 @@ def _submissions(forms: list[str], dates: list[str]) -> dict: return { "cik": 712515, "name": "ELECTRONIC ARTS INC.", - "filings": {"recent": {"form": forms, "filingDate": dates}}, + "filings": { + "recent": { + "form": forms, + "filingDate": dates, + "accessionNumber": [f"0001354457-26-{i:06d}" for i in range(len(forms))], + "primaryDocument": ["xslF25X02/primary_doc.xml"] * len(forms), + } + }, } -def _sec_client(payload: dict) -> SecClient: +def _sec_client(payload: dict, security: str | None = "Common Stock") -> SecClient: + """Mock submissions + the Form 25 primary document the class check reads.""" + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("primary_doc.xml"): + if security is None: + return httpx.Response(404) + body = ( + "" + f"{security}" + "" + ) + return httpx.Response(200, content=body.encode()) return httpx.Response(200, content=json.dumps(payload).encode()) return SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) @@ -112,7 +130,8 @@ async def test_confirm_delisting_marks_on_a_form_25(session: AsyncSession, monke marked = await ticker_service.confirm_delisting( session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) ) - assert marked == date(2026, 8, 4) + # Removal is effective ten days after the 2026-08-04 filing, not on it. + assert marked == date(2026, 8, 14) row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() assert row.delisted_reason == ticker_service.REASON_FORM_25 @@ -154,7 +173,8 @@ async def test_delisting_filing_picks_the_newest_match(): ) async with client as c: found = await c.delisting_filing("0000712515") - assert found == {"form": "25-NSE", "filing_date": date(2026, 8, 4)} + assert found["form"] == "25-NSE" + assert found["filing_date"] == date(2026, 8, 4) async def test_delisting_filing_returns_none_without_one(): @@ -197,10 +217,10 @@ async def test_sec_confirmation_upgrades_a_manual_mark(session: AsyncSession, mo ) assert await ticker_service.confirm_delisting( session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) - ) == date(2026, 8, 4) + ) == date(2026, 8, 14) row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() - assert row.delisted_on == date(2026, 8, 4) + assert row.delisted_on == date(2026, 8, 14) assert row.delisted_reason == ticker_service.REASON_FORM_25 @@ -232,3 +252,116 @@ async def test_ohlcv_priority_ordering_skips_delisted(session: AsyncSession): symbols = await _get_ohlcv_priority_tickers(session) assert "EA" not in symbols assert sorted(symbols) == ["AAPL", "MSFT"] + + +async def test_a_form_25_for_another_security_class_is_ignored(session: AsyncSession, monkeypatch): + """Form 25 is per security class. An issuer delisting its notes, preferred or + warrants files one while the common keeps trading — retiring the ticker on + that would remove an actively traded symbol from every signal.""" + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client( + _submissions(["25-NSE"], ["2026-08-04"]), + security="6.25% Notes due 2030", + ), + raising=False, + ) + + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + row = (await session.execute(select(Ticker).where(Ticker.symbol == "EA"))).scalar_one() + assert row.delisted_on is None + + +async def test_a_stale_historical_form_25_cannot_retire_a_symbol(session: AsyncSession, monkeypatch): + """A 2019 filing for a long-gone class must not retire a symbol whose bars + ran until 2026 — and must certainly not stamp 2019 as the date.""" + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client(_submissions(["25"], ["2019-03-01"])), + raising=False, + ) + + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + + +async def test_form_15_alone_never_retires_a_symbol(session: AsyncSession, monkeypatch): + """Form 15 ends a reporting obligation; it is not evidence trading stopped.""" + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client(_submissions(["15-12B", "15-12G"], ["2026-08-04", "2026-08-05"])), + raising=False, + ) + + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + + +async def test_an_unreadable_form_25_fails_closed(session: AsyncSession, monkeypatch): + """Pre-2009 filings have no primary_doc.xml. Unknown class must read as no.""" + session.add(Ticker(symbol="EA", cik="0000712515")) + await session.commit() + monkeypatch.setattr( + ticker_service, + "_sec_client_factory", + lambda: _sec_client(_submissions(["25"], ["2026-08-04"]), security=None), + raising=False, + ) + + assert await ticker_service.confirm_delisting( + session, "EA", last_bar=date(2026, 8, 4), today=date(2026, 8, 11) + ) is None + + +@pytest.mark.parametrize( + "description,expected", + [ + ("Common Stock", True), + ("Class A Common Stock, $0.01 par value", True), + ("Common Shares, no par value", True), + ("6.25% Notes due 2030", False), + ("7.5% Series B Cumulative Preferred Stock", False), + ("Warrants to purchase Common Stock", False), + ("Depositary Shares each representing 1/1000th interest", False), + ("", False), + ], +) +def test_common_stock_classification(description: str, expected: bool): + from app.services.sec_client import _is_common_stock + + assert _is_common_stock(description) is expected + + +async def test_prune_keeps_delisted_rows(session: AsyncSession, monkeypatch): + """A prune must not destroy rows the delisting flow deliberately retained — + their price history is the whole reason those rows still exist.""" + from app.services import ticker_universe_service as tus + + session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="GONE")]) + await session.commit() + await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14)) + + async def fake_fetch(db, universe): + return ["AAPL"], "test" + + monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch) + + summary = await tus.bootstrap_universe(session, "sp500", prune_missing=True) + + remaining = sorted(t.symbol for t in await ticker_service.list_tickers(session)) + assert remaining == ["AAPL", "EA"] # GONE pruned, EA protected + assert summary["deleted"] == 1 + assert summary["kept_delisted"] == ["EA"]