signal-platform/app/services/ticker_service.py

"""Ticker Registry service: add, delete, list, and retire tracked tickers."""

import logging
import re
from datetime import date, timedelta

from sqlalchemy import func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession

from app.exceptions import DuplicateError, NotFoundError, ValidationError
from app.models.ticker import Ticker

logger = logging.getLogger(__name__)

# Reasons a symbol may be marked delisted, narrowest first.
REASON_FORM_25 = "form_25"   # SEC Form 25/25-NSE/15 confirmed the exchange exit
REASON_MANUAL = "manual"     # an operator decided

# Ticker.tracking_source — who owns this row's presence in the tracked set.
# Ownership is sticky towards manual: a bootstrap creates ``index`` rows and an
# operator's add claims a row as ``manual``, but nothing ever converts manual
# back. Only index-owned rows are prunable.
SOURCE_INDEX = "index"
SOURCE_MANUAL = "manual"

# How long a symbol must be without bars before we spend an SEC request asking
# whether it delisted. Guards against a market-data outage probing the whole
# universe at once; a real delisting is still stale days later.
MIN_STALE_DAYS_BEFORE_PROBE = 3

# Rule 12d2-2: a Form 25 removal takes effect ten days after filing, so the
# filing date is not the date the security stopped trading.
FORM_25_EFFECTIVE_DAYS = 10

# How far before the last bar a Form 25 may be filed and still explain this gap.
# An exchange can file shortly before trading actually stops; anything older
# concerns a class that was already gone while the symbol kept printing bars.
FILING_LOOKBACK_DAYS = 30


def _sec_client_factory():
    """Build the SEC client for a delisting probe (patched in tests).

    Imported lazily so the SEC/httpx stack stays off the import path of every
    module that only wants ``active_only``.
    """
    from app.services.sec_client import SecClient

    return SecClient()


def active_only(stmt, *, as_of: date | None = None):
    """Restrict a Ticker query to symbols in today's live signal path.

    Three independent ways to leave it, and they mean different things:

    ``delisted_on``       the security stopped trading
    ``untracked_on``      a prune excluded it from scanning
    ``scan_disabled_on``  an operator switched it off by hand

    Any one of them excludes a symbol here. None deletes anything — the rows and
    their history stay, which is what a survivorship-honest backtest needs.

    ``untracked_on`` is not "no longer an index constituent": the scheduled sync
    runs ``prune_missing=False``, so a symbol can have left the index and still
    be scanned. Membership lives in ``universe_memberships`` and is not consulted
    here — this stays a pure column filter so it costs nothing on the ~8 live
    call sites that use it.

    Opt-in on purpose rather than folded into a shared getter: list and admin
    views deliberately keep both kinds of row so the state is *visible*, which a
    silent default would undo. Apply this on the live signal path — scanning,
    ranking, scoring, breadth, ingestion — and nowhere else. In particular the
    backtest's universe query must NOT use it: today's membership would silently
    erase yesterday's candidates.

    ``delisted_on`` is an *effective* date, and a Form 25 is known ten days
    before it takes effect, so a future date must not drop the symbol yet — it
    is still trading and still worth scanning and ingesting. The other two read
    the same way, so ``as_of`` answers "what was in the scanner then?" for all
    three rather than only one. Compared in SQL against the database's own date;
    ``as_of`` overrides it for tests.
    """
    cutoff = func.current_date() if as_of is None else as_of
    return stmt.where(
        or_(Ticker.delisted_on.is_(None), Ticker.delisted_on > cutoff),
        or_(Ticker.untracked_on.is_(None), Ticker.untracked_on > cutoff),
        or_(Ticker.scan_disabled_on.is_(None), Ticker.scan_disabled_on > cutoff),
    )


async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
    """Add a new ticker after validation.

    Validates: non-empty, uppercase alphanumeric. Auto-uppercases input.
    Raises DuplicateError if symbol already tracked.
    """
    stripped = symbol.strip()
    if not stripped:
        raise ValidationError("Ticker symbol must not be empty or whitespace-only")

    normalised = stripped.upper()
    if not re.fullmatch(r"[A-Z0-9]+", normalised):
        raise ValidationError(
            f"Ticker symbol must be alphanumeric: {normalised}"
        )

    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    existing = result.scalar_one_or_none()
    if existing is not None:
        if existing.tracking_source == SOURCE_MANUAL:
            raise DuplicateError(f"Ticker already exists: {normalised}")
        # An explicit manual add of an index-owned symbol claims it: the operator
        # wants this stock kept regardless of the index, so it must stop being
        # prunable. Ownership only ever moves towards manual — a bootstrap never
        # moves it back, or the protection would evaporate the moment the symbol
        # rejoined the index.
        #
        # Deliberately does not touch scan_disabled_on or untracked_on: claiming
        # ownership is not the same as asking for it to be scanned, and silently
        # re-enabling something switched off by hand would undo a decision
        # nobody revisited. ``set_tracked`` is the explicit control for that.
        existing.tracking_source = SOURCE_MANUAL
        await db.commit()
        await db.refresh(existing)
        logger.info("ticker %s claimed as manually tracked", normalised)
        return existing

    ticker = Ticker(symbol=normalised, tracking_source=SOURCE_MANUAL)
    db.add(ticker)
    await db.commit()
    await db.refresh(ticker)
    return ticker


async def delete_ticker(db: AsyncSession, symbol: str) -> None:
    """Delete a ticker and cascade all associated data.

    Raises NotFoundError if the symbol is not tracked.
    """
    normalised = symbol.strip().upper()
    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    ticker = result.scalar_one_or_none()
    if ticker is None:
        raise NotFoundError(f"Ticker not found: {normalised}")

    await db.delete(ticker)
    await db.commit()


async def list_tickers(db: AsyncSession) -> list[Ticker]:
    """Return all tracked tickers sorted alphabetically by symbol.

    Delisted symbols are included and carry ``delisted_on`` — the registry is
    where an operator needs to *see* that a symbol retired, not where it should
    quietly disappear.
    """
    result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
    return list(result.scalars().all())


async def mark_delisted(
    db: AsyncSession,
    symbol: str,
    *,
    delisted_on: date,
    reason: str = REASON_MANUAL,
) -> bool:
    """Record that a symbol stopped trading. True if this changed anything.

    Idempotent, so the staleness path can call it every run without churning the
    row: re-marking is a no-op. The one exception is an SEC confirmation landing
    on a row an operator marked by hand — Form 25 carries the real effective
    date, so it replaces the operator's estimate. Nothing downgrades a confirmed
    row back to a manual one.
    """
    normalised = symbol.strip().upper()
    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    ticker = result.scalar_one_or_none()
    if ticker is None:
        raise NotFoundError(f"Ticker not found: {normalised}")
    if ticker.delisted_on is not None:
        upgrading = (
            reason == REASON_FORM_25 and ticker.delisted_reason != REASON_FORM_25
        )
        if not upgrading:
            return False

    await db.execute(
        update(Ticker)
        .where(Ticker.id == ticker.id)
        .values(delisted_on=delisted_on, delisted_reason=reason)
    )
    await db.commit()
    logger.info(
        "ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason
    )
    return True


async def confirm_delisting(
    db: AsyncSession,
    symbol: str,
    *,
    last_bar: date | None,
    today: date | None = None,
) -> date | None:
    """Ask SEC whether ``symbol`` actually delisted; mark it if so.

    Called when OHLCV goes stale, because "no new bars" alone cannot tell a
    delisting from a halt or a rename. Returns the effective date whenever the
    symbol is known to have delisted — whether this call established that or an
    earlier one did — and ``None`` while it remains unproven, so the caller warns
    only about gaps that still have no explanation.

    Returning the already-known date matters between filing and effect: trading
    usually stops before the ten-day Rule 12d2-2 delay expires, so the symbol is
    correctly still active (see ``active_only``) while producing no bars. Without
    this the staleness warning would fire daily across that window — the exact
    noise the delisting flow exists to remove.

    Deliberately driven by staleness rather than by the SEC fundamentals import:
    that importer stalls for days at a time on unrelated Company-Facts gaps, and
    detection wired into it would stall with it.

    The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays
    stale forever, so the delay costs nothing, and it keeps a broad market-data
    outage — where every tracked symbol reports stale at once — from turning into
    one SEC request per symbol per run.
    """
    from app.services.sec_client import SecError

    normalised = symbol.strip().upper()
    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    ticker = result.scalar_one_or_none()
    if ticker is None:
        return None
    known = ticker.delisted_on
    # Already confirmed by SEC — nothing left to learn, but the caller still
    # needs the date to know this gap is explained. A row an operator marked by
    # hand is worth probing: Form 25 upgrades the estimated date.
    if ticker.delisted_reason == REASON_FORM_25:
        return known
    if not ticker.cik:
        return known
    # No bars at all is an ingestion problem, not evidence of a delisting.
    if last_bar is None:
        return known
    if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
        return known

    try:
        async with _sec_client_factory() as client:
            # Only a Form 25 filed around or after the last bar can explain THIS
            # gap. An older one belongs to a class that stopped trading before
            # the symbol was still printing bars, and must not retire it.
            filing = await client.delisting_filing(
                ticker.cik, not_before=last_bar - timedelta(days=FILING_LOOKBACK_DAYS)
            )
    except SecError:
        # Never let a probe failure escalate a routine staleness warning.
        logger.warning("delisting probe failed for %s", normalised, exc_info=True)
        return known

    if filing is None:
        return known
    # Removal takes effect ten days after filing, so the filing date is not the
    # date the symbol stopped trading.
    effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS)
    if await mark_delisted(
        db, normalised, delisted_on=effective, reason=REASON_FORM_25
    ):
        return effective
    return known


async def set_tracked(db: AsyncSession, symbol: str, *, tracked: bool) -> bool:
    """Include or exclude one symbol from the live path. True if it changed.

    The per-symbol counterpart to ``bootstrap_universe(prune_missing=True)``,
    which can only act on the whole universe at once — and would sweep every
    manually added symbol along with it, since a manual add is by definition not
    in the fetched constituent list.

    Switching off is not delisting: the security still trades, we just stop
    scanning it. Use this for a symbol that cannot produce data (no provider
    coverage) or that an operator simply does not want scanned. The rows and
    their history stay either way, so a backtest still sees them — untracking is
    not a way to make a coverage gap disappear from the completeness report.

    Writes ``scan_disabled_on``, never ``untracked_on``: the operator's decision
    has to survive a prune -> re-entry cycle, which clears the prune flag.
    """
    normalised = symbol.strip().upper()
    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    ticker = result.scalar_one_or_none()
    if ticker is None:
        raise NotFoundError(f"Ticker not found: {normalised}")
    if (ticker.scan_disabled_on is None) == tracked:
        return False

    await db.execute(
        update(Ticker)
        .where(Ticker.id == ticker.id)
        # Stamped by the database for the same reason bootstrap_universe does:
        # active_only compares it against func.current_date().
        .values(scan_disabled_on=None if tracked else func.current_date())
    )
    await db.commit()
    logger.info(
        "ticker %s scanning %s", normalised, "enabled" if tracked else "disabled"
    )
    return True


async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
    """Un-retire a symbol. True if it had been marked.

    The counterpart that makes automatic marking acceptable: a false positive
    costs one row update, where a delete would have cost the price history.
    """
    normalised = symbol.strip().upper()
    result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
    ticker = result.scalar_one_or_none()
    if ticker is None:
        raise NotFoundError(f"Ticker not found: {normalised}")
    if ticker.delisted_on is None:
        return False

    await db.execute(
        update(Ticker)
        .where(Ticker.id == ticker.id)
        .values(delisted_on=None, delisted_reason=None)
    )
    await db.commit()
    logger.info("ticker %s un-marked as delisted", normalised)
    return True