diff --git a/alembic/versions/035_ticker_untracked.py b/alembic/versions/035_ticker_untracked.py new file mode 100644 index 0000000..e6885c8 --- /dev/null +++ b/alembic/versions/035_ticker_untracked.py @@ -0,0 +1,56 @@ +"""Separate scanner membership from retained history + +Revision ID: 035 +Revises: 034 +Create Date: 2026-09-07 00:00:00.000000 + +Migration 032 stopped ``delete_ticker`` and the delisting path from destroying +history, but left the other half of the problem open — its own comment said so: +"Pruning a symbol that is merely no longer an index constituent still destroys +history; that needs a tracked/membership state separate from delisting." + +``bootstrap_universe(prune_missing=True)`` still ran ``DELETE FROM tickers``, +cascading OHLCV, setups and scores away for any symbol that had merely dropped +out of the index. The survivorship audit measured what that costs: restoring 85 +removed constituents moved the same window from 2.09 Sharpe / 51.6% CAGR to +1.57 / 38.1%. A prune that deletes is how a universe silently becomes the +survivors. + +``untracked_on`` is the sibling of ``delisted_on``, and the distinction is the +point: + + delisted_on the security stopped trading + untracked_on it still trades; we stopped scanning it + +Both retain every row. NULL in both means "in the live path" — ``active_only`` +filters on both, so scanning, ranking, scoring, breadth and ingestion skip an +untracked name, while the backtest's universe query deliberately does not, so +today's membership cannot erase yesterday's candidates. + +Nullable and reversible: a symbol that returns to the index has ``untracked_on`` +cleared by the next bootstrap and rejoins the live path with its history intact. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "035" +down_revision: Union[str, None] = "034" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("tickers", sa.Column("untracked_on", sa.Date(), nullable=True)) + # Same reasoning as ix_tickers_delisted_on: the live path tests this + # predicate on every universe scan, and untracked rows accumulate. + op.create_index("ix_tickers_untracked_on", "tickers", ["untracked_on"]) + + +def downgrade() -> None: + # Dropping the column re-admits every untracked symbol to the live path. + # The rows and their history survive either way; that is the whole point. + op.drop_index("ix_tickers_untracked_on", table_name="tickers") + op.drop_column("tickers", "untracked_on") diff --git a/alembic/versions/036_universe_membership.py b/alembic/versions/036_universe_membership.py new file mode 100644 index 0000000..90a963c --- /dev/null +++ b/alembic/versions/036_universe_membership.py @@ -0,0 +1,116 @@ +"""Record universe membership, tracking ownership and operator preference + +Revision ID: 036 +Revises: 035 +Create Date: 2026-09-07 00:00:00.000000 + +Migration 035 separated "stopped trading" from "stopped scanning". This +separates three more things that were still conflated: + + universe_memberships which symbols the index actually contained, when + tickers.tracking_source who owns the row: a bootstrap, or an operator + tickers.scan_disabled_on the operator's own "stop scanning this" + +**Why the operator flag is a separate column.** ``untracked_on`` was written by +both the prune and ``set_tracked``. With one column a prune -> re-entry cycle +would clear an operator's explicit "off", because re-entry cannot tell which +writer set it. Splitting them makes the two survivable independently. + +**Existing "off" states are carried over, not reinterpreted.** Any row that +already has ``untracked_on`` set was written by one of those two callers and +there is no way to tell which. Copying the date into ``scan_disabled_on`` is the +conservative reading: a later sync clears ``untracked_on`` on re-entry but never +touches ``scan_disabled_on``, so a symbol someone deliberately switched off +cannot be reactivated by an automated run. + +**Existing rows default to ``manual``.** Provenance cannot be recovered from +``created_at``, and ``manual`` is the fail-safe value: manual rows are never +pruned, so a bad guess costs an un-pruned symbol rather than a symbol swept out +of the scanner. New rows created by a bootstrap are stamped ``index`` +explicitly. Nothing ever converts ``manual`` to ``index`` — a hand-added symbol +that later joins the index keeps its protection. + +Membership history starts at first observation. ``first_seen_on`` is the first +date a live fetch actually listed the symbol, never a reconstructed index +effective date; earlier membership is simply unknown. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "036" +down_revision: Union[str, None] = "035" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("tickers", sa.Column("scan_disabled_on", sa.Date(), nullable=True)) + op.create_index("ix_tickers_scan_disabled_on", "tickers", ["scan_disabled_on"]) + op.add_column( + "tickers", + sa.Column( + "tracking_source", + sa.String(length=16), + nullable=False, + server_default="manual", + ), + ) + + # Every pre-existing row is treated as manually tracked: unprunable until a + # bootstrap has had a chance to claim the ones it owns. + op.execute("UPDATE tickers SET tracking_source = 'manual'") + # Preserve any existing "off" so an automated sync cannot reactivate it. + op.execute( + "UPDATE tickers SET scan_disabled_on = untracked_on " + "WHERE untracked_on IS NOT NULL" + ) + + op.create_table( + "universe_memberships", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "ticker_id", + sa.Integer(), + sa.ForeignKey("tickers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("universe", sa.String(length=20), nullable=False), + sa.Column("first_seen_on", sa.Date(), nullable=False), + # NULL == still observed. Closing sets this and retains the row; a + # re-entry inserts a new one. + sa.Column("absent_since", sa.Date(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index( + "ix_universe_memberships_ticker_universe", + "universe_memberships", + ["ticker_id", "universe"], + ) + op.create_index( + "ix_universe_memberships_universe_seen", + "universe_memberships", + ["universe", "absent_since"], + ) + + +def downgrade() -> None: + # Membership history is observation-only and cannot be reconstructed once + # dropped; the ticker rows and their price history are untouched either way. + op.drop_index( + "ix_universe_memberships_universe_seen", table_name="universe_memberships" + ) + op.drop_index( + "ix_universe_memberships_ticker_universe", table_name="universe_memberships" + ) + op.drop_table("universe_memberships") + op.drop_column("tickers", "tracking_source") + op.drop_index("ix_tickers_scan_disabled_on", table_name="tickers") + op.drop_column("tickers", "scan_disabled_on") diff --git a/app/models/__init__.py b/app/models/__init__.py index 4628141..38128b7 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -18,6 +18,7 @@ from app.models.regime_fundamental_observation import RegimeFundamentalObservati from app.models.benchmark_price import BenchmarkPrice from app.models.signal_context_snapshot import SignalContextSnapshot from app.models.system_event import SystemEvent +from app.models.universe_membership import UniverseMembership from app.models.sec_filing_gap import SecFilingGap from app.models.job_run_state import JobRunState @@ -46,4 +47,5 @@ __all__ = [ "SystemEvent", "SecFilingGap", "JobRunState", + "UniverseMembership", ] diff --git a/app/models/ticker.py b/app/models/ticker.py index 0461d1a..ad14142 100644 --- a/app/models/ticker.py +++ b/app/models/ticker.py @@ -28,6 +28,32 @@ class Ticker(Base): delisted_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) # How we learned: "form_25" (SEC confirmed), "manual" (operator). delisted_reason: Mapped[str | None] = mapped_column(String(32), nullable=True) + # The sibling of delisted_on, and the distinction matters: delisted means the + # security stopped trading; untracked means it still trades and a prune took + # it out of the scanner. Both retain every row — the delete this replaced + # cascaded away exactly the history a survivorship-honest backtest needs. + # + # This records *exclusion by pruning*, NOT current index membership. The + # scheduled sync runs prune_missing=False, so a symbol can have left the + # index and still be scanned; the two states legitimately differ. Membership + # itself lives in universe_memberships. + # + # active_only filters on it; the backtest's universe query deliberately does + # not, so today's scanner state cannot erase yesterday's candidates. + untracked_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) + # The operator's own "stop scanning this", kept apart from the prune flag on + # purpose. If one column served both, a prune -> re-entry cycle would clear + # an explicit operator "off", because re-entry cannot tell which writer set + # it. Only set_tracked writes this; no bootstrap ever clears it. + scan_disabled_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) + # Who owns this row's tracking: "index" (created by a universe bootstrap) or + # "manual" (an operator added it). Pruning only ever touches index-owned + # rows. Ownership is sticky towards manual — a bootstrap must never convert + # manual to index, or a hand-added symbol that happens to join the index + # would lose its protection when it later leaves. + tracking_source: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="manual", default="manual" + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=datetime.utcnow, nullable=False ) @@ -43,3 +69,4 @@ class Ticker(Base): watchlist_entries = relationship("WatchlistEntry", back_populates="ticker", cascade="all, delete-orphan") ingestion_progress = relationship("IngestionProgress", back_populates="ticker", cascade="all, delete-orphan", uselist=False) earnings_events = relationship("EarningsEvent", back_populates="ticker", cascade="all, delete-orphan") + universe_memberships = relationship("UniverseMembership", back_populates="ticker", cascade="all, delete-orphan") diff --git a/app/models/universe_membership.py b/app/models/universe_membership.py new file mode 100644 index 0000000..87f93f9 --- /dev/null +++ b/app/models/universe_membership.py @@ -0,0 +1,59 @@ +from datetime import date, datetime + +from sqlalchemy import Date, DateTime, ForeignKey, Index, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base + + +class UniverseMembership(Base): + """One observed interval of a symbol's membership in an index universe. + + Append-only. A symbol that leaves and returns has two rows, so a + point-in-time query for the gap between them correctly answers "not a + member" — reopening the original row would erase that. + + **Observed, not asserted.** ``first_seen_on`` is the first date a live + universe fetch actually listed the symbol, not the date it joined the index. + Anything earlier is unknown, and this table never guesses: the removals list + reconstructed by hand in ``scripts/data/sp500_removals.json`` stays a + research artifact rather than a seed. + + Membership is independent of *tracking ownership* (``Ticker.tracking_source``) + and of whether the symbol is scanned. A manually added symbol that happens to + be an index constituent gets intervals like any other; being untracked or + scan-disabled does not close one. + """ + + __tablename__ = "universe_memberships" + __table_args__ = ( + Index("ix_universe_memberships_ticker_universe", "ticker_id", "universe"), + # "Who was a member of U at date D" — the point-in-time query. + Index("ix_universe_memberships_universe_seen", "universe", "absent_since"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + ticker_id: Mapped[int] = mapped_column( + ForeignKey("tickers.id", ondelete="CASCADE"), nullable=False + ) + # "sp500" | "nasdaq100" | "nasdaq_all" — a symbol can belong to several. + universe: Mapped[str] = mapped_column(String(20), nullable=False) + # The first observation that listed this symbol. INCLUSIVE lower bound. Not + # the date it joined the index — that is unknown and never guessed. + first_seen_on: Mapped[date] = mapped_column(Date, nullable=False) + # The first observation that found the symbol ABSENT — hence the name. It is + # *not* "last seen present": the symbol was present at every observation + # before this one and absent at this one, so it is an EXCLUSIVE upper bound. + # NULL means still present at the most recent observation. + # + # member on D <=> first_seen_on <= D < absent_since (or absent_since IS NULL) + # + # The real departure lies somewhere in (previous observation, absent_since]; + # with a daily sync that is a day, and with a missed week it is a week. The + # column records when we LOOKED, which is the only thing actually known. + absent_since: Mapped[date | None] = mapped_column(Date, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=datetime.utcnow, nullable=False + ) + + ticker = relationship("Ticker", back_populates="universe_memberships") diff --git a/app/routers/tickers.py b/app/routers/tickers.py index 96d9027..03ec92b 100644 --- a/app/routers/tickers.py +++ b/app/routers/tickers.py @@ -6,7 +6,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.dependencies import get_db, require_access from app.models.user import User from app.schemas.common import APIEnvelope -from app.schemas.ticker import TickerCreate, TickerDelistingUpdate, TickerResponse +from app.schemas.ticker import ( + TickerCreate, + TickerDelistingUpdate, + TickerResponse, + TickerTrackingUpdate, +) from app.services import ticker_service router = APIRouter(tags=["tickers"]) @@ -18,7 +23,23 @@ async def create_ticker( _user: User = Depends(require_access), db: AsyncSession = Depends(get_db), ): - """Add a new ticker to the registry.""" + """Add a ticker to the registry, or claim an existing index-owned one. + + Responses: + + - **New symbol** — created as manually tracked and returned. + - **Existing, index-owned** — 200 with the existing row, now marked manually + tracked. Not a 409: the operator is saying "keep this stock", and refusing + while leaving it prunable would defeat the protection they just asked for. + Ownership only ever moves towards manual, so a bootstrap cannot take it + back when the symbol next joins the index. + - **Existing, already manual** — 409 Duplicate. Nothing left to change. + + Claiming never clears ``scan_disabled_on`` or ``untracked_on``: wanting a + stock kept is not the same as asking for it to be scanned, and silently + re-enabling something switched off by hand would undo a decision nobody + revisited. ``POST /tickers/{symbol}/tracking`` is the explicit control. + """ ticker = await ticker_service.add_ticker(db, body.symbol) return APIEnvelope( status="success", @@ -70,6 +91,24 @@ async def mark_ticker_delisted( return APIEnvelope(status="success", data={"changed": changed}) +@router.post("/tickers/{symbol}/tracking", response_model=APIEnvelope) +async def set_ticker_tracking( + symbol: str, + body: TickerTrackingUpdate, + _user: User = Depends(require_access), + db: AsyncSession = Depends(get_db), +): + """Include or exclude one symbol from scanning, keeping its history. + + Untracking is not delisting — the security still trades, we just stop + scanning it. The case this exists for: a manually added symbol no provider + can serve, which would otherwise sit in the universe forever producing no + bars and flagging every backtest as incomplete. + """ + changed = await ticker_service.set_tracked(db, symbol, tracked=body.tracked) + return APIEnvelope(status="success", data={"changed": changed}) + + @router.delete("/tickers/{symbol}/delisting", response_model=APIEnvelope) async def clear_ticker_delisting( symbol: str, diff --git a/app/schemas/ticker.py b/app/schemas/ticker.py index 15530fa..a5e9889 100644 --- a/app/schemas/ticker.py +++ b/app/schemas/ticker.py @@ -27,3 +27,13 @@ class TickerDelistingUpdate(BaseModel): delisted_on: date = Field( ..., description="Effective date the symbol stopped trading" ) + + +class TickerTrackingUpdate(BaseModel): + tracked: bool = Field( + ..., + description=( + "True to scan this symbol, False to exclude it from the live path. " + "History is retained either way — this is not a delisting." + ), + ) diff --git a/app/services/ticker_service.py b/app/services/ticker_service.py index b5d9f60..9d3b0a8 100644 --- a/app/services/ticker_service.py +++ b/app/services/ticker_service.py @@ -16,6 +16,13 @@ logger = logging.getLogger(__name__) REASON_FORM_25 = "form_25" # SEC Form 25/25-NSE/15 confirmed the exchange exit REASON_MANUAL = "manual" # an operator decided +# Ticker.tracking_source — who owns this row's presence in the tracked set. +# Ownership is sticky towards manual: a bootstrap creates ``index`` rows and an +# operator's add claims a row as ``manual``, but nothing ever converts manual +# back. Only index-owned rows are prunable. +SOURCE_INDEX = "index" +SOURCE_MANUAL = "manual" + # How long a symbol must be without bars before we spend an SEC request asking # whether it delisted. Guards against a market-data outage probing the whole # universe at once; a real delisting is still stale days later. @@ -43,21 +50,42 @@ def _sec_client_factory(): def active_only(stmt, *, as_of: date | None = None): - """Restrict a Ticker query to symbols that still trade. + """Restrict a Ticker query to symbols in today's live signal path. + + Three independent ways to leave it, and they mean different things: + + ``delisted_on`` the security stopped trading + ``untracked_on`` a prune excluded it from scanning + ``scan_disabled_on`` an operator switched it off by hand + + Any one of them excludes a symbol here. None deletes anything — the rows and + their history stay, which is what a survivorship-honest backtest needs. + + ``untracked_on`` is not "no longer an index constituent": the scheduled sync + runs ``prune_missing=False``, so a symbol can have left the index and still + be scanned. Membership lives in ``universe_memberships`` and is not consulted + here — this stays a pure column filter so it costs nothing on the ~8 live + call sites that use it. Opt-in on purpose rather than folded into a shared getter: list and admin - views deliberately keep delisted rows so the delisting is *visible*, which a + views deliberately keep both kinds of row so the state is *visible*, which a silent default would undo. Apply this on the live signal path — scanning, - ranking, scoring, breadth, ingestion — and nowhere else. + ranking, scoring, breadth, ingestion — and nowhere else. In particular the + backtest's universe query must NOT use it: today's membership would silently + erase yesterday's candidates. ``delisted_on`` is an *effective* date, and a Form 25 is known ten days before it takes effect, so a future date must not drop the symbol yet — it - is still trading and still worth scanning and ingesting. Compared in SQL - against the database's own date; ``as_of`` overrides it for tests. + is still trading and still worth scanning and ingesting. The other two read + the same way, so ``as_of`` answers "what was in the scanner then?" for all + three rather than only one. Compared in SQL against the database's own date; + ``as_of`` overrides it for tests. """ cutoff = func.current_date() if as_of is None else as_of return stmt.where( - or_(Ticker.delisted_on.is_(None), Ticker.delisted_on > cutoff) + or_(Ticker.delisted_on.is_(None), Ticker.delisted_on > cutoff), + or_(Ticker.untracked_on.is_(None), Ticker.untracked_on > cutoff), + or_(Ticker.scan_disabled_on.is_(None), Ticker.scan_disabled_on > cutoff), ) @@ -78,10 +106,27 @@ async def add_ticker(db: AsyncSession, symbol: str) -> Ticker: ) result = await db.execute(select(Ticker).where(Ticker.symbol == normalised)) - if result.scalar_one_or_none() is not None: - raise DuplicateError(f"Ticker already exists: {normalised}") + existing = result.scalar_one_or_none() + if existing is not None: + if existing.tracking_source == SOURCE_MANUAL: + raise DuplicateError(f"Ticker already exists: {normalised}") + # An explicit manual add of an index-owned symbol claims it: the operator + # wants this stock kept regardless of the index, so it must stop being + # prunable. Ownership only ever moves towards manual — a bootstrap never + # moves it back, or the protection would evaporate the moment the symbol + # rejoined the index. + # + # Deliberately does not touch scan_disabled_on or untracked_on: claiming + # ownership is not the same as asking for it to be scanned, and silently + # re-enabling something switched off by hand would undo a decision + # nobody revisited. ``set_tracked`` is the explicit control for that. + existing.tracking_source = SOURCE_MANUAL + await db.commit() + await db.refresh(existing) + logger.info("ticker %s claimed as manually tracked", normalised) + return existing - ticker = Ticker(symbol=normalised) + ticker = Ticker(symbol=normalised, tracking_source=SOURCE_MANUAL) db.add(ticker) await db.commit() await db.refresh(ticker) @@ -229,6 +274,45 @@ async def confirm_delisting( return known +async def set_tracked(db: AsyncSession, symbol: str, *, tracked: bool) -> bool: + """Include or exclude one symbol from the live path. True if it changed. + + The per-symbol counterpart to ``bootstrap_universe(prune_missing=True)``, + which can only act on the whole universe at once — and would sweep every + manually added symbol along with it, since a manual add is by definition not + in the fetched constituent list. + + Switching off is not delisting: the security still trades, we just stop + scanning it. Use this for a symbol that cannot produce data (no provider + coverage) or that an operator simply does not want scanned. The rows and + their history stay either way, so a backtest still sees them — untracking is + not a way to make a coverage gap disappear from the completeness report. + + Writes ``scan_disabled_on``, never ``untracked_on``: the operator's decision + has to survive a prune -> re-entry cycle, which clears the prune flag. + """ + normalised = symbol.strip().upper() + result = await db.execute(select(Ticker).where(Ticker.symbol == normalised)) + ticker = result.scalar_one_or_none() + if ticker is None: + raise NotFoundError(f"Ticker not found: {normalised}") + if (ticker.scan_disabled_on is None) == tracked: + return False + + await db.execute( + update(Ticker) + .where(Ticker.id == ticker.id) + # Stamped by the database for the same reason bootstrap_universe does: + # active_only compares it against func.current_date(). + .values(scan_disabled_on=None if tracked else func.current_date()) + ) + await db.commit() + logger.info( + "ticker %s scanning %s", normalised, "enabled" if tracked else "disabled" + ) + return True + + async def clear_delisted(db: AsyncSession, symbol: str) -> bool: """Un-retire a symbol. True if it had been marked. diff --git a/app/services/ticker_universe_service.py b/app/services/ticker_universe_service.py index a496da9..cd2a12b 100644 --- a/app/services/ticker_universe_service.py +++ b/app/services/ticker_universe_service.py @@ -12,17 +12,18 @@ import logging import os import re from collections.abc import Iterable -from datetime import datetime, timezone +from datetime import date, datetime, timezone from pathlib import Path import httpx -from sqlalchemy import delete, select +from sqlalchemy import func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings from app.exceptions import ProviderError, ValidationError from app.models.ticker import Ticker -from app.services import settings_store +from app.models.universe_membership import UniverseMembership +from app.services import settings_store, ticker_service logger = logging.getLogger(__name__) @@ -333,6 +334,114 @@ async def backfill_ticker_names(db: AsyncSession, *, only_missing: bool = True) return {"updated": updated, "checked": len(targets), "unmatched": len(targets) - updated} +# Sources that represent an actual look at the index today. A ``cache`` or +# ``seed`` result means the live fetch FAILED — the seed lists are ~44 symbols, +# so treating one as an observation would close ~460 memberships on a network +# blip. Membership is only ever recorded from a live read. +LIVE_UNIVERSE_SOURCES = frozenset({ + "wikipedia_sp500", "wikipedia_nasdaq100", "nasdaq_trader", "public", +}) + + +async def members_on(db: AsyncSession, universe: str, as_of: date) -> list[str]: + """Which symbols were observed in *universe* on *as_of*, alphabetically. + + The point-in-time question item 3 exists to answer, with the interval + boundaries written down once as code rather than re-derived at each call + site: + + member on D <=> first_seen_on <= D < absent_since (or absent_since IS NULL) + + Lower bound inclusive, upper bound exclusive. ``absent_since`` is the first + observation that found the symbol *gone*, so on that date it was already + absent. + + **This is observation history, not index history.** A date before the first + successful live sync returns an empty list — that is "we were not looking", + not "the index was empty", and the caller has to know the difference. The + reconstructed removals in ``scripts/data/sp500_removals.json`` are research + data and deliberately do not seed this table. + """ + rows = await db.execute( + select(Ticker.symbol) + .join(UniverseMembership, UniverseMembership.ticker_id == Ticker.id) + .where( + UniverseMembership.universe == universe, + UniverseMembership.first_seen_on <= as_of, + or_( + UniverseMembership.absent_since.is_(None), + UniverseMembership.absent_since > as_of, + ), + ) + .order_by(Ticker.symbol) + ) + return list(rows.scalars().all()) + + +async def _observe_membership( + db: AsyncSession, universe: str, present: set[str] +) -> dict[str, int]: + """Record what a live universe fetch just saw. Returns opened/closed counts. + + Runs on every sync, not only when pruning: the scheduled job uses + ``prune_missing=False``, so membership recorded only on manual prunes would + be full of holes. Recording membership and deciding what to scan are + separate concerns, and this is the recording one — it never touches + ``untracked_on``, ``scan_disabled_on`` or ``tracking_source``, and it applies + to manually tracked symbols exactly as it does to index-owned ones. + + Intervals are append-only. A symbol that leaves and returns gets a second + row, so a point-in-time query for the gap answers "not a member"; reopening + the first row would erase that. + + Dates come from the database (``current_date``), matching every other date + this module writes. ``first_seen_on`` is the first *observation*, not an + index effective date — anything earlier is unknown and is not guessed. + """ + rows = (await db.execute( + select(Ticker.id, Ticker.symbol).where(Ticker.symbol.in_(present)) + )).all() + ticker_ids = {symbol: tid for tid, symbol in rows} + + open_rows = (await db.execute( + select(UniverseMembership.ticker_id, Ticker.symbol) + .join(Ticker, Ticker.id == UniverseMembership.ticker_id) + .where( + UniverseMembership.universe == universe, + UniverseMembership.absent_since.is_(None), + ) + )).all() + open_symbols = {symbol for _, symbol in open_rows} + + opened = 0 + for symbol in sorted(present - open_symbols): + ticker_id = ticker_ids.get(symbol) + if ticker_id is None: + continue + db.add(UniverseMembership( + ticker_id=ticker_id, + universe=universe, + first_seen_on=func.current_date(), + )) + opened += 1 + + departed_ids = [tid for tid, symbol in open_rows if symbol not in present] + closed = 0 + if departed_ids: + result = await db.execute( + update(UniverseMembership) + .where( + UniverseMembership.universe == universe, + UniverseMembership.absent_since.is_(None), + UniverseMembership.ticker_id.in_(departed_ids), + ) + .values(absent_since=func.current_date()) + ) + closed = int(result.rowcount or 0) + + return {"opened": opened, "closed": closed} + + async def bootstrap_universe( db: AsyncSession, universe: str, @@ -341,42 +450,98 @@ async def bootstrap_universe( ) -> dict[str, int | str]: """Upsert ticker universe into tracked tickers. - Returns summary counts for added/existing/deleted symbols. + ``prune_missing`` marks symbols no longer in the universe as untracked; it + never deletes them. "Not in today's scanner" and "not stored for historical + analysis" are different states, and conflating them is what makes a universe + quietly become the survivors. Symbols that have rejoined the universe have + the mark cleared and return to the live path with their history intact. + + Returns summary counts for added / already-tracked / untracked / retracked. """ normalised_universe = _validate_universe(universe) symbols, source = await fetch_universe_symbols(db, normalised_universe) + # A cache or seed result means the live fetch FAILED. The seed lists are ~44 + # symbols against a ~500-symbol universe, so treating one as the truth would + # untrack the entire index on a single network blip. Nothing that acts on + # "this symbol is/isn't in the index" may run on fallback data: not pruning, + # not clearing a prune flag, not recording membership. Adding is the one + # exception — it destroys nothing and is what lets a cold start bootstrap + # from seeds. + observed = source in LIVE_UNIVERSE_SOURCES + existing_rows = await db.execute(select(Ticker.symbol)) existing_symbols = set(existing_rows.scalars().all()) target_symbols = set(symbols) symbols_to_add = sorted(target_symbols - existing_symbols) - symbols_to_delete = sorted(existing_symbols - target_symbols) if prune_missing else [] + + # Only index-owned rows are prunable. A manual addition is by definition not + # in the constituent list, so without this filter every hand-added symbol + # would be swept by any prune. + prunable = set() if not (prune_missing and observed) else set(( + await db.execute( + select(Ticker.symbol).where( + Ticker.symbol.in_(existing_symbols - target_symbols), + Ticker.tracking_source == ticker_service.SOURCE_INDEX, + ) + ) + ).scalars().all()) + symbols_to_untrack = sorted(prunable) + kept_manual = ( + sorted((existing_symbols - target_symbols) - prunable) + if (prune_missing and observed) else [] + ) for symbol in symbols_to_add: - db.add(Ticker(symbol=symbol)) + db.add(Ticker(symbol=symbol, tracking_source=ticker_service.SOURCE_INDEX)) - deleted_count = 0 - skipped_delisted: list[str] = [] - if symbols_to_delete: - # A delisted row was retained on purpose — its price history is exactly - # what a survivorship-honest backtest needs, and the delete cascades it - # away. Pruning must not undo that. (Pruning a symbol that is merely no - # longer an index constituent still destroys history; that needs a - # tracked/membership state separate from delisting.) - protected = ( - await db.execute( - select(Ticker.symbol).where( - Ticker.symbol.in_(symbols_to_delete), - Ticker.delisted_on.is_not(None), - ) - ) - ).scalars().all() - skipped_delisted = sorted(protected) - deletable = [s for s in symbols_to_delete if s not in set(protected)] - if deletable: - result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(deletable))) - deleted_count = int(result.rowcount or 0) + # A symbol that has returned to the index must rejoin the live path. It is + # already in ``existing_symbols``, so it is neither added nor untracked here + # — without this it would keep the flag from its earlier removal and stay + # invisible to the scanner forever, with nothing to show why. + # + # Clears the prune flag ONLY. ``scan_disabled_on`` is the operator's own + # decision and no automated run may undo it: a symbol someone switched off + # by hand must stay off across a prune -> re-entry cycle. Nor does this touch + # ``tracking_source`` — a manual row that joins the index stays manual, or it + # would lose its protection the next time it left. + retracked_count = 0 + if observed: + retracked = await db.execute( + update(Ticker) + .where(Ticker.symbol.in_(target_symbols), Ticker.untracked_on.is_not(None)) + .values(untracked_on=None) + ) + retracked_count = int(retracked.rowcount or 0) + + untracked_count = 0 + if symbols_to_untrack: + # Marked, never deleted. The delete this replaced cascaded through + # OHLCV, setups and scores, which is precisely the history a + # survivorship-honest backtest needs — see migration 035. Rows already + # carrying a mark keep their original date rather than being re-stamped + # on every bootstrap. + # + # Stamped by the database, not date.today(): active_only compares this + # against func.current_date(), and a process in a timezone ahead of the + # database would otherwise write a date that reads as "effective + # tomorrow" and leave the symbol in the scanner for a day. Same clock on + # both sides by construction. + result = await db.execute( + update(Ticker) + .where(Ticker.symbol.in_(symbols_to_untrack), Ticker.untracked_on.is_(None)) + .values(untracked_on=func.current_date()) + ) + untracked_count = int(result.rowcount or 0) + + # Membership is observed on every live sync, independent of pruning and of + # who owns the row. A failed fetch must never close a membership. + membership = ( + await _observe_membership(db, normalised_universe, target_symbols) + if observed + else {"opened": 0, "closed": 0} + ) await db.commit() @@ -393,10 +558,29 @@ async def bootstrap_universe( "total_universe_symbols": len(symbols), "added": len(symbols_to_add), "already_tracked": len(target_symbols & existing_symbols), - "deleted": deleted_count, + # Always 0. Kept so a caller reading this key is told plainly that + # pruning no longer destroys rows, rather than seeing the old key vanish + # and assuming nothing happened — or worse, seeing a count under + # "deleted" that now means something else entirely. + "deleted": 0, + "untracked": untracked_count, + "untracked_symbols": symbols_to_untrack[:50], + # Symbols that rejoined the index and were returned to the live path. + "retracked": retracked_count, + # Absent from the universe but manually owned, so a prune left them + # alone. The operator asked for these by hand. + "kept_manual": kept_manual[:50], + "kept_manual_count": len(kept_manual), "added_symbols": symbols_to_add[:50], - # Delisted rows a prune declined to destroy, so the caller can see the - # count did not match what they asked to remove. - "kept_delisted": skipped_delisted[:50], - "kept_delisted_count": len(skipped_delisted), + # Membership observation is skipped entirely on a cache/seed fallback, + # so the caller can tell "nothing changed" from "we never looked". + "membership_observed": observed, + "membership_opened": membership["opened"], + "membership_closed": membership["closed"], + "prune_note": ( + "Pruning marks untracked_on and never deletes: the rows keep their " + "bars, setups and scores so historical backtests stay honest. " + "Untracked symbols leave the scanner, rankings and ingestion. " + "Manually added symbols are never pruned." + ), } diff --git a/docs/data-reliability.md b/docs/data-reliability.md new file mode 100644 index 0000000..8528706 --- /dev/null +++ b/docs/data-reliability.md @@ -0,0 +1,380 @@ +# Data reliability + +Three changes that make the platform's data trustworthy rather than faster or +more profitable. They came out of the [integrity +experiments](data-integrity-plan.md), which turned up two problems worth fixing +on their own: + +- Changing the stock universe **materially changes results** — Arm A 2.09 Sharpe + / 51.6% CAGR against Arm B 1.57 / 38.1% over the same window, from universe + composition alone ([survivorship audit](research/survivorship-audit.md)). +- **Missing data quietly produces a seemingly successful backtest.** Found while + running Step 3a: a snapshot whose schema predated migrations 026/032 made every + `_fetch_columns` call raise `no such column: tickers.cik`, both dispatch paths + caught the per-ticker exception and continued, and the run wrote a complete + report with zero candidates. Nothing in the report or the UI said the run had + lost its entire universe. + +| # | Change | Status | +|---|---|---| +| 1 | Make incomplete backtests visibly incomplete | **done** — 2026-09-07 | +| 2 | Preserve history when a stock leaves the tracked universe | **done** — 2026-09-07 | +| 3 | Record universe membership by effective date | **done** — 2026-09-07 | + +--- + +## 1. Make incomplete backtests visibly incomplete — done + +The replay loop is deliberately fault-tolerant: one unreadable ticker must not +abort a 500-symbol run. That is right for the run and wrong for its report, +which had no way to say how much of the universe it actually covered. A run that +lost every ticker and a run that honestly found no setups produced the same +report. + +### What the report now carries + +`run_backtest` builds a `_DataQuality` ledger +(`app/services/backtest_service.py`) and emits it as `report["data_quality"]`: + +```json +{ + "status": "failed", + "complete": false, + "reasons": ["no symbol was evaluated successfully", "506 symbols failed to load"], + "expected": 506, "loaded": 0, "evaluated": 0, + "no_data": 0, "no_data_symbols": [], "short_history": 0, + "failed": 506, + "failures_by_stage": {"fetch": 506}, + "failures": [{"symbol": "A", "stage": "fetch", + "error": "OperationalError: (sqlite3.OperationalError) no such column: tickers.cik"}], + "failures_truncated": true, + "benchmark": {"symbol": "SPY", "sessions": 1516, + "first": "2020-07-06", "last": "2026-07-17", + "momentum_basis": "residual", "error": null} +} +``` + +**"Complete" means the processing and coverage checks passed** — every expected +symbol loaded and evaluated, benchmark present. It does **not** mean the +underlying prices are proven correct. A complete run over silently wrong bars is +still complete; splits, adjustment drift and bad vendor prints are a different +problem, and this ledger must not be read as clearing them. The card says so on +the page, not only here. + +**The counters are self-checking.** Every universe symbol ends in exactly one of +`no_data`, `failures` or `evaluated`, and `as_report` verifies +`no_data + failed + evaluated == expected`. A ledger that stops balancing — +because a counter was missed when a code path was added — reports itself as a +failure rather than quietly under-counting. `short_history` is a *subset* of +`evaluated` (loaded, but fewer than `MIN_LOOKBACK + HORIZON` bars, so no outcome +can resolve), not a fourth bucket. + +**Benchmark availability is a separate axis.** Without SPY closes, residual 12-1 +momentum silently falls back to raw 12-1 — the gate stops being the production +gate. The block records the basis actually used, so the degradation is on the +report instead of only in a log line. + +### Three statuses, because severity differs + +| Status | Meaning | UI | +|---|---|---| +| `ok` | Every symbol evaluated, benchmark present | Normal | +| `degraded` | The strategy is intact but ran on less than the whole universe | Tiles shown, banner above | +| `failed` | The headline can't be read as the strategy it claims to be | **Tiles replaced by the callout** | + +`failed` means one of: nothing evaluated, an empty universe, an unbalanced +ledger, or a missing benchmark. Blanket suppression on a single missing ticker +would train the reader to ignore the banner, which is the failure mode this is +supposed to fix. + +The callout **replaces** the performance tiles rather than sitting above them. +A warning over ordinary-looking numbers is precisely how the zero-candidate run +would still have read as a quiet week. + +### Where it shows + +- `BacktestDataQualityCard` in the backtest panel, with the failure list behind a + disclosure. +- The "Last run" line now reads `505/506 tickers` — evaluated over expected, not + the universe size. "506 tickers" on a run that evaluated none was the + misleading part. +- `productionBaseRates` returns `null` for a failed run, so per-ticker base rates + quoted next to a live setup — which carry no caveat of their own — disappear + rather than being sourced from a broken replay. +- `EvaluationPanel` drops a failed run's expectancy as a drift baseline instead + of reporting "tracking" or "drift" against it. +- **A report with no `data_quality` block reads as unknown, not clean.** Old + cached reports predate the check; defaulting them to complete would reproduce + the exact bug being fixed. + +### Acceptance evidence + +Both runs used `scripts/run_backtest_snapshot.py`. + +| Snapshot | Result | +|---|---| +| `survivorship-a.sqlite` (healthy) | `degraded` · 505/506 evaluated · 202,765 candidates · one bar-less row (RHM) | +| the same file with `tickers.cik` dropped | `failed` · 0/506 evaluated · 506 fetch failures · 0 candidates | + +The broken-snapshot run is the one that matters: before this change it wrote a +report that looked like an ordinary run with no setups. + +The healthy run is not `ok`, and that turned out to be correct. It flagged +`no_data: 1` (RHM) and `short_history: 1` (SPCX, 14 bars), which an independent +bar-count query confirms. Neither is new — Step 2 already recorded RHM as inert +in `tr-rank-agreement.md`, which is why Step 3a generated candidates over 505 of +506 tickers. The ledger derives it independently, from the run rather than from +a hand-written note, which is the point: the same 506-vs-505 gap now appears on +the report instead of only in a research write-up someone remembered to make. + +`prod.sqlite` has the same bar-less RHM row, so **production runs will report +`degraded` too** until that universe row is cleared — which is item 2's +territory. That is deliberate: silencing a bar-less universe row would let a +wholesale import failure read as clean. The reason names the symbols so it is a +one-line fix rather than a standing nag. + +Tests: `tests/unit/test_backtest_data_quality.py` (10, covering each status, the +ledger identity, the failure cap, symbol naming and the benchmark axis) and four end-to-end +cases in `tests/unit/test_backtest_service.py`, including every-ticker-fails and +missing-benchmark. + +--- + +## 2. Preserve history when a stock leaves the tracked universe — done + +`bootstrap_universe(prune_missing=True)` ran `DELETE FROM tickers`, cascading +OHLCV, setups, scores and S/R levels away for any symbol that had merely dropped +out of the index. Migration 032's own comment predicted it: *"Pruning a symbol +that is merely no longer an index constituent still destroys history; that needs +a tracked/membership state separate from delisting."* + +### The two states, kept apart + +Migration 035 adds `Ticker.untracked_on`, the sibling of `delisted_on`: + +| Column | Meaning | +|---|---| +| `delisted_on` | The security stopped trading | +| `untracked_on` | It still trades; it left the index and we stopped scanning it | + +Neither deletes anything. `NULL` in both means "in today's live path". +`active_only` — the single opt-in choke point already applied to scanning, +ranking, scoring, breadth and ingestion — now filters on both, and honours +`as_of` for both, so a historical query can still ask what was in the scanner on +a given date. + +`prune_missing=True` now marks instead of deleting, and **clears the mark for +symbols that have rejoined the universe**. That last part is the bug this +refactor could have introduced: a returning name is already tracked, so it is +neither added nor untracked, and without an explicit clear it would keep the +flag from its earlier removal and stay invisible to the scanner forever. + +The `deleted` return key is now always `0` rather than repurposed — a caller +reading `deleted: 12` must not be told rows were destroyed when they were not. +`untracked`, `untracked_symbols` and `retracked` carry the real counts, and the +admin toast reads "untracked N" instead of "deleted N". + +### Historical eligibility stays explicit + +The backtest's universe query is now `_load_backtest_symbols`, a named function +whose docstring states that it deliberately does **not** use `active_only`. +Today's membership flag must not erase yesterday's candidates: a replay grades +what the strategy could have selected at each past date, so it must see symbols +that have since been delisted or dropped out. "Fixing the inconsistency" by +adding `active_only` there is exactly how survivorship bias gets reintroduced, +which is why a test asserts the opposite. + +The one path that still really deletes is `DELETE /tickers/{symbol}` — an +explicit operator action asking for exactly that. Pruning is not. + +### Untracking one symbol + +`POST /tickers/{symbol}/tracking` with `{"tracked": false}` excludes a single +symbol from the live path, mirroring the delisting pair (both are API-only; no +UI control exists for either). It exists because `prune_missing=True` can only +act on the whole universe, and a manually added symbol is by definition *not* in +the fetched constituent list — so a prune sweeps every manual addition along +with the genuine leavers. Reversible with `{"tracked": true}`. + +The case it was built for: **RHM**. Added by hand on 2026-06-28 out of interest +in the stock, it has produced zero rows in *every* ticker-linked table in the ten +weeks since — no bars, no ingestion progress, no scores — and the Alpaca name +backfill never resolved it. It is the only symbol in the universe that has +produced nothing at all (BK, CTRA and SATS also lack Alpaca names but carry full +history, so a missing name alone proves nothing). Rheinmetall is XETRA-listed, so +a plain US `RHM` is not something the OHLCV provider can serve. An attempt to +confirm this against Alpaca's asset API returned 401 for the AAPL control too, +so the *identity* is unverified — the justification is the persistent absence of +usable data, not a confirmed listing status or a delisting. + +**Untracking it does not clear the backtest's `degraded` flag, and must not.** +The backtest deliberately includes untracked symbols, so RHM stays in the +universe query, still returns no bars, and still counts in `no_data`. What +untracking buys is narrower and still worth having: RHM leaves the scanner, +ingestion and scoring, and the decision is recorded explicitly rather than +sitting as an unexplained gap. Making the warning go away would mean changing +historical eligibility to silence a real coverage gap — guarded by +`test_untracking_does_not_clear_a_data_quality_warning`. + +Note what is *not* solved: a manual addition is still indistinguishable from a +universe member, so a prune untracks it (non-destructively, and visibly via +`untracked_symbols`, but silently in the sense that nobody asked). Recording +*why* a symbol is tracked is item 3's job. + +### Tests + +`tests/unit/test_universe_membership.py` (12) covers the three behaviours +together: + +1. Pruning preserves the symbol's bars **and** its linked records (S/R levels + are checked too, so the assertion is about the cascade, not just one table). +2. The pruned symbol leaves `active_only` while staying visible in the registry. +3. `_load_backtest_symbols` still returns it. + +Plus re-tracking, the mark not being re-stamped on a second prune (which would +erase when the symbol actually left), and `as_of` working for both columns. +a row carrying *both* marks (it left the index, then delisted), and a +compilation guard asserting every `active_only` call site still receives both +predicates ANDed — including `scheduler.py`'s, which chains `group_by`/`order_by` +onto the result from outside. `tests/unit/test_migration_035.py` exercises the +migration's upgrade and downgrade directly, including that existing rows default +to NULL — an upgrade must not drop the whole universe out of the live path. + +One detail worth keeping: `untracked_on` is stamped by the **database** +(`func.current_date()`), not `date.today()`. `active_only` compares it against +`func.current_date()`, so a process in a timezone ahead of the database would +otherwise write a date that reads as "effective tomorrow" and leave the symbol +in the scanner for a day. Same clock on both sides by construction. + +## 3. Record universe membership by effective date — done + +Migration 036. Three things that were still conflated are now separate: + +| Concept | Where | Written by | +|---|---|---| +| Index membership, observed history | `universe_memberships` intervals | every **live** sync | +| Excluded from scanning by a prune | `Ticker.untracked_on` | prune only | +| Operator switched it off | `Ticker.scan_disabled_on` | `set_tracked` only | +| Tracking ownership | `Ticker.tracking_source` | `add_ticker` / bootstrap | + +**Why the operator flag is its own column.** With one column serving both +writers, a prune -> re-entry cycle would clear an explicit operator "off", +because re-entry cannot tell who set it. Re-entry clears `untracked_on` and +never touches `scan_disabled_on`. + +`active_only` now tests three columns and **stays join-free** — it is on ~8 live +call sites, and the compilation guard asserts all three predicates land, ANDed, +including at the call site that chains `group_by`/`order_by` on from outside. + +### `untracked_on` is not current membership + +It records *exclusion by pruning*. The scheduled sync runs +`prune_missing=False`, so a symbol can have left the index and still be scanned; +the two states legitimately differ, and nothing asserts they agree. + +### Tracking ownership is sticky towards manual + +`tracking_source` is `index` or `manual`. Prune touches only `index` rows, so +**manual additions survive index pruning** — the RHM case. + +A bootstrap **never** converts `manual` to `index`. That direction looks harmless +and is not: a hand-added symbol that happens to join the index would lose its +protection the moment it left again, which is exactly when it is needed. Only +new rows a bootstrap creates are stamped `index`; an explicit `add_ticker` on an +existing index-owned symbol claims it as `manual`. + +`POST /tickers` therefore has three responses: + +| Case | Response | +|---|---| +| New symbol | Created as manually tracked | +| Existing, index-owned | **200** with the existing row, now manually tracked | +| Existing, already manual | 409 Duplicate | + +Claiming is not a 409 because the operator is saying "keep this stock", and +refusing while leaving it prunable would defeat the protection they just asked +for. It deliberately does not touch `scan_disabled_on` or `untracked_on` — +wanting a stock kept is not the same as asking for it to be scanned, and +silently re-enabling something switched off by hand would undo a decision nobody +revisited. `POST /tickers/{symbol}/tracking` is the explicit control. + +### Membership is observed, never asserted + +`first_seen_on` is the first date a live fetch actually listed the symbol — not +an index effective date. Anything earlier is unknown and is not guessed; the +hand-reconstructed `scripts/data/sp500_removals.json` stays a research artifact. + +Intervals are append-only: departure sets `absent_since` and **retains the row**, +and a re-entry inserts a new one. A symbol that left in March and returned in +August has two intervals, so a point-in-time query for May correctly answers +"not a member" — reopening the original row would erase that. + +**Boundaries, stated once.** The column is named `absent_since`, not +`last_seen_on`, because it stores the first observation that found the symbol +*gone* — on that date it was already absent: + +``` +member on D <=> first_seen_on <= D < absent_since (or absent_since IS NULL) +``` + +Lower bound inclusive, upper bound exclusive. The real departure lies somewhere +in `(previous observation, absent_since]` — a day with a daily sync, a week if a +week was missed. The column records when we *looked*, which is the only thing +actually known. `members_on(db, universe, as_of)` implements exactly this, so +the rule lives in one place rather than being re-derived per call site. + +**A date before the first successful live sync returns an empty list.** That +means "we were not looking", not "the index was empty", and a caller has to know +the difference — answering with today's set would be precisely the survivorship +error this exists to end. + +Recording runs on **every live sync**, independent of pruning and of who owns +the row: the scheduled job uses `prune_missing=False`, so membership recorded +only on manual prunes would be full of holes. Manually owned symbols get +intervals like any other — membership and tracking ownership are independent +axes. Switching off scanning does not close a membership; the index still +contains the symbol, we simply stopped looking at it. + +**A failed fetch drives nothing at all.** `fetch_universe_symbols` falls back to +a cached snapshot and then to a built-in seed list of ~44 symbols. Nothing that +acts on "this symbol is or isn't in the index" may run on that data: not +recording membership, **not pruning, and not clearing a prune flag**. Skipping +only the membership write was half a safeguard — a seed fallback would still +have untracked the rest of the universe, which a regression test caught +(2 of 3 symbols untracked before the fix). Adding is the one exception: it +destroys nothing and is what lets a cold start bootstrap from seeds. The summary +reports `membership_observed: false` so a caller can tell "the index did not +change" from "we never looked". + +### Migration decisions + +- **Existing rows become `manual`.** Provenance cannot be recovered from + `created_at`, and manual is fail-safe: a wrong guess costs an un-pruned symbol + rather than one swept out of the scanner. Bootstraps claim the ones they own + as they create them. +- **Existing "off" states are carried into `scan_disabled_on`.** Whoever wrote an + existing `untracked_on` is ambiguous, and re-entry clears that column — so the + value is copied to the one no automated run touches. A symbol somebody + deliberately switched off cannot be reactivated by a sync. +- **No history is backfilled.** The table starts empty; history begins at the + first observation. + +### Tests + +`test_universe_membership.py` (19) and `test_universe_membership_intervals.py` +(14): manual additions survive a prune; a bootstrap never claims a manual row, +including across a join-then-leave cycle; an explicit add claims an index-owned +symbol without touching its switches; re-entry clears the prune flag but never +the operator's; a live sync opens intervals without pruning; departure closes and +re-entry appends; a cache/seed fallback records nothing, untracks nothing and +clears no prune flag; membership is recorded for manual rows; switching off +scanning does not close a membership; and the boundary rules — lower inclusive, +upper exclusive, gaps are not membership, per-universe scoping, and pre-recording +dates returning empty. `test_migration_036.py` (4) pins both migration judgement +calls and the downgrade. + +The fixtures pass a **live** source label deliberately: a made-up one would now +skip pruning entirely and make those tests vacuous. + +Backtest eligibility is unchanged — `_load_backtest_symbols` still returns every +symbol with stored history, and untracking still cannot silence a coverage gap. diff --git a/frontend/src/hooks/useAdmin.ts b/frontend/src/hooks/useAdmin.ts index 7d2d6b2..b47843c 100644 --- a/frontend/src/hooks/useAdmin.ts +++ b/frontend/src/hooks/useAdmin.ts @@ -274,7 +274,16 @@ export function useBootstrapTickers() { qc.invalidateQueries({ queryKey: ['admin', 'ticker-universe'] }); addToast( 'success', - `Bootstrap done (${result.source ?? 'unknown'}): +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}` + // "untracked", not "deleted": pruning leaves the rows and their history + // in place and only removes the symbol from the scanner. + `Bootstrap done (${result.source ?? 'unknown'}): +${result.added}, existing ${result.already_tracked}, untracked ${result.untracked ?? 0}` + + (result.retracked ? `, back in scanner ${result.retracked}` : '') + + (result.kept_manual_count ? `, kept manual ${result.kept_manual_count}` : '') + // A cache/seed fallback means the live fetch failed: say we never looked + // rather than letting it read as "the index did not change". + + (result.membership_observed === false + ? ' · membership not recorded (live fetch failed)' + : '') + (result.added_symbols?.length ? ` · added ${result.added_symbols.slice(0, 8).join(', ')}${result.added > 8 ? '…' : ''}` : ''), diff --git a/tests/unit/test_migration_035.py b/tests/unit/test_migration_035.py new file mode 100644 index 0000000..f2d0380 --- /dev/null +++ b/tests/unit/test_migration_035.py @@ -0,0 +1,86 @@ +"""Migration 035 adds the untracked flag and stays reversible. + +The downgrade matters more than usual here: dropping the column re-admits every +untracked symbol to the live path, and the rows survive either way. A migration +that could not roll back cleanly would make the prune change hard to undo on a +deploy that went wrong. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + + +def _load_migration_module(): + path = ( + Path(__file__).resolve().parents[2] + / "alembic" + / "versions" + / "035_ticker_untracked.py" + ) + spec = importlib.util.spec_from_file_location("migration_035", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _tickers_table(connection: sa.Connection) -> None: + connection.execute(sa.text( + """ + CREATE TABLE tickers ( + id INTEGER PRIMARY KEY, + symbol VARCHAR(10) NOT NULL, + delisted_on DATE + ) + """ + )) + connection.execute(sa.text( + "INSERT INTO tickers (id, symbol, delisted_on) VALUES (1, 'AAPL', NULL)" + )) + + +def _columns(connection: sa.Connection) -> set[str]: + return {row[1] for row in connection.execute(sa.text("PRAGMA table_info(tickers)"))} + + +def test_upgrade_adds_a_nullable_untracked_column_and_keeps_existing_rows(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + + assert "untracked_on" in _columns(connection) + # Existing rows default to NULL, i.e. still in the scanner — an upgrade + # must not silently drop the whole universe out of the live path. + rows = connection.execute( + sa.text("SELECT symbol, untracked_on FROM tickers") + ).all() + assert rows == [("AAPL", None)] + + +def test_downgrade_removes_the_column_without_touching_the_rows(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + connection.execute(sa.text( + "UPDATE tickers SET untracked_on = '2026-09-07' WHERE symbol = 'AAPL'" + )) + with Operations.context(MigrationContext.configure(connection)): + migration.downgrade() + + assert "untracked_on" not in _columns(connection) + # The ticker itself survives the rollback with its identity intact. + assert connection.execute(sa.text("SELECT symbol FROM tickers")).all() == [("AAPL",)] diff --git a/tests/unit/test_migration_036.py b/tests/unit/test_migration_036.py new file mode 100644 index 0000000..891bf17 --- /dev/null +++ b/tests/unit/test_migration_036.py @@ -0,0 +1,132 @@ +"""Migration 036: provenance, the operator switch, and membership intervals. + +The migration makes two irreversible-feeling judgement calls, and both need to +be pinned: + +1. **Existing rows become ``manual``.** Provenance cannot be recovered from + ``created_at``, and manual is the fail-safe guess — manual rows are never + pruned, so a wrong guess costs an un-pruned symbol rather than one swept out + of the scanner. +2. **Existing "off" states are carried into ``scan_disabled_on``.** Whoever set + ``untracked_on`` is ambiguous, and re-entry clears the prune flag. Copying it + is what stops an automated sync reactivating something switched off by hand. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + + +def _load_migration_module(): + path = ( + Path(__file__).resolve().parents[2] + / "alembic" + / "versions" + / "036_universe_membership.py" + ) + spec = importlib.util.spec_from_file_location("migration_036", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _tickers_table(connection: sa.Connection) -> None: + """The schema as migration 035 leaves it.""" + connection.execute(sa.text( + """ + CREATE TABLE tickers ( + id INTEGER PRIMARY KEY, + symbol VARCHAR(10) NOT NULL, + delisted_on DATE, + untracked_on DATE + ) + """ + )) + connection.execute(sa.text( + "INSERT INTO tickers (id, symbol, delisted_on, untracked_on) VALUES " + "(1, 'AAPL', NULL, NULL), " + "(2, 'OFF', NULL, '2026-08-01')" + )) + + +def _columns(connection: sa.Connection, table: str) -> set[str]: + return {row[1] for row in connection.execute(sa.text(f"PRAGMA table_info({table})"))} + + +def test_upgrade_defaults_existing_rows_to_manual(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + + rows = dict(connection.execute( + sa.text("SELECT symbol, tracking_source FROM tickers") + ).all()) + assert rows == {"AAPL": "manual", "OFF": "manual"} + + +def test_upgrade_carries_an_existing_off_state_into_the_operator_switch(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + + rows = dict(connection.execute( + sa.text("SELECT symbol, scan_disabled_on FROM tickers") + ).all()) + # The ambiguous existing "off" is preserved on the column no sync clears. + assert rows["OFF"] == "2026-08-01" + # A row that was never off stays on. + assert rows["AAPL"] is None + + +def test_upgrade_creates_the_membership_table_empty(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + + assert {"ticker_id", "universe", "first_seen_on", "absent_since"} <= _columns( + connection, "universe_memberships" + ) + # History starts at first observation — the migration invents none. + count = connection.execute( + sa.text("SELECT COUNT(*) FROM universe_memberships") + ).scalar_one() + assert count == 0 + + +def test_downgrade_removes_the_new_state_and_keeps_the_tickers(): + migration = _load_migration_module() + engine = sa.create_engine("sqlite://") + + with engine.begin() as connection: + _tickers_table(connection) + with Operations.context(MigrationContext.configure(connection)): + migration.upgrade() + with Operations.context(MigrationContext.configure(connection)): + migration.downgrade() + + columns = _columns(connection, "tickers") + assert "tracking_source" not in columns + assert "scan_disabled_on" not in columns + # 035's column and the rows themselves are untouched. + assert "untracked_on" in columns + assert connection.execute( + sa.text("SELECT symbol FROM tickers ORDER BY symbol") + ).all() == [("AAPL",), ("OFF",)] diff --git a/tests/unit/test_ticker_delisting.py b/tests/unit/test_ticker_delisting.py index b23b133..3491a51 100644 --- a/tests/unit/test_ticker_delisting.py +++ b/tests/unit/test_ticker_delisting.py @@ -347,26 +347,37 @@ def test_common_stock_classification(description: str, expected: bool): assert _is_common_stock(description) is expected -async def test_prune_keeps_delisted_rows(session: AsyncSession, monkeypatch): - """A prune must not destroy rows the delisting flow deliberately retained — - their price history is the whole reason those rows still exist.""" +async def test_prune_keeps_every_row(session: AsyncSession, monkeypatch): + """A prune destroys nothing at all now — not the delisted row it always + protected, and not the ex-constituent it used to delete outright.""" from app.services import ticker_universe_service as tus - session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="EA"), Ticker(symbol="GONE")]) + session.add_all([ + Ticker(symbol=s, tracking_source=ticker_service.SOURCE_INDEX) + for s in ("AAPL", "EA", "GONE") + ]) await session.commit() await ticker_service.mark_delisted(session, "EA", delisted_on=date(2026, 8, 14)) async def fake_fetch(db, universe): - return ["AAPL"], "test" + # A live source label: pruning only acts on a successful fetch. + return ["AAPL"], "wikipedia_sp500" monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch) summary = await tus.bootstrap_universe(session, "sp500", prune_missing=True) remaining = sorted(t.symbol for t in await ticker_service.list_tickers(session)) - assert remaining == ["AAPL", "EA"] # GONE pruned, EA protected - assert summary["deleted"] == 1 - assert summary["kept_delisted"] == ["EA"] + assert remaining == ["AAPL", "EA", "GONE"] + assert summary["deleted"] == 0 + assert summary["untracked"] == 2 + + # Both leave the live path, by different routes: EA stopped trading, GONE + # merely left the index. + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL"] async def test_a_future_effective_date_keeps_the_symbol_live(session: AsyncSession): diff --git a/tests/unit/test_ticker_universe_service.py b/tests/unit/test_ticker_universe_service.py index d3b3937..1741353 100644 --- a/tests/unit/test_ticker_universe_service.py +++ b/tests/unit/test_ticker_universe_service.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn from app.database import Base from app.models.settings import SystemSetting from app.models.ticker import Ticker -from app.services import ticker_universe_service +from app.services import ticker_service, ticker_universe_service from app.services.ticker_universe_service import _extract_wiki_symbols, _normalise_symbols _engine = create_async_engine("sqlite+aiosqlite://", echo=False) @@ -58,11 +58,17 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo @pytest.mark.asyncio async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession, monkeypatch: pytest.MonkeyPatch): - session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="MSFT"), Ticker(symbol="TSLA")]) + # Index-owned: a row defaults to manual, and manual rows are never pruned. + session.add_all([ + Ticker(symbol=s, tracking_source=ticker_service.SOURCE_INDEX) + for s in ("AAPL", "MSFT", "TSLA") + ]) await session.commit() async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]: - return ["AAPL", "MSFT"], "test" + # A live source label: a fallback result must not drive pruning, so a + # made-up label would make this test vacuous. + return ["AAPL", "MSFT"], "wikipedia_sp500" monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch) @@ -74,10 +80,18 @@ async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession, assert result["added"] == 0 assert result["already_tracked"] == 2 - assert result["deleted"] == 1 + assert result["untracked"] == 1 + assert result["untracked_symbols"] == ["TSLA"] + # Nothing is destroyed any more, and the key says so rather than vanishing. + assert result["deleted"] == 0 + # The row survives the prune; only its scanner membership changed. rows = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol.asc())) - assert list(rows.scalars().all()) == ["AAPL", "MSFT"] + assert list(rows.scalars().all()) == ["AAPL", "MSFT", "TSLA"] + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol.asc())) + ) + assert list(live.scalars().all()) == ["AAPL", "MSFT"] @pytest.mark.asyncio diff --git a/tests/unit/test_universe_membership.py b/tests/unit/test_universe_membership.py new file mode 100644 index 0000000..caaee2c --- /dev/null +++ b/tests/unit/test_universe_membership.py @@ -0,0 +1,477 @@ +"""Scanner membership and retained history are separate states. + +`bootstrap_universe(prune_missing=True)` used to run `DELETE FROM tickers`, +cascading OHLCV, setups and scores away for any symbol that had merely dropped +out of the index. The survivorship audit measured the cost of that: restoring 85 +removed constituents moved the same window from 2.09 Sharpe / 51.6% CAGR to 1.57 +/ 38.1%. + +Three properties have to hold together, and the third is the one nobody would +think to check: + +1. Pruning preserves the symbol's bars and its linked records. +2. The pruned symbol leaves today's scanner and rankings. +3. The backtest's universe query still returns it — today's membership flag must + not silently erase yesterday's candidates. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from datetime import date, timedelta + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.database import Base +from app.models.ohlcv import OHLCVRecord +from app.models.sr_level import SRLevel +from app.models.ticker import Ticker +from app.services import ticker_service +from app.services import ticker_universe_service as tus + +# Own engine, like the sibling delisting tests: these exercise schema-level +# behaviour (a new column, a cascade that must NOT fire) and want a clean slate. +_engine = create_async_engine("sqlite+aiosqlite://", echo=False) +_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest.fixture(autouse=True) +async def _setup_tables() -> AsyncGenerator[None, None]: + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def session() -> AsyncGenerator[AsyncSession, None]: + async with _session_factory() as s: + yield s + + +async def _seed(session: AsyncSession) -> None: + """Two index-owned constituents, one about to leave with history. + + ``tracking_source`` is explicit: a row defaults to ``manual``, and manual + rows are never pruned, so a fixture that omitted it would silently test + nothing. + """ + session.add_all([ + Ticker(symbol="AAPL", tracking_source=ticker_service.SOURCE_INDEX), + Ticker(symbol="LEAVER", tracking_source=ticker_service.SOURCE_INDEX), + ]) + await session.flush() + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + start = date(2026, 1, 5) + for i in range(10): + session.add(OHLCVRecord( + ticker_id=leaver.id, + date=start + timedelta(days=i), + open=100.0, high=101.0, low=99.0, close=100.5, volume=1_000_000, + )) + session.add(SRLevel( + ticker_id=leaver.id, price_level=95.0, type="support", strength=70, + detection_method="swing", + )) + await session.commit() + + +async def _prune_to(session: AsyncSession, monkeypatch, keep: list[str]) -> dict: + async def fake_fetch(db, universe): + # A LIVE source label: pruning and re-entry act only on a successful + # fetch, so a made-up label would silently make these tests vacuous. + return keep, "wikipedia_sp500" + + monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch) + return await tus.bootstrap_universe(session, "sp500", prune_missing=True) + + +async def test_pruning_preserves_bars_and_linked_records(session: AsyncSession, monkeypatch): + await _seed(session) + summary = await _prune_to(session, monkeypatch, ["AAPL"]) + + assert summary["untracked"] == 1 + assert summary["untracked_symbols"] == ["LEAVER"] + assert summary["deleted"] == 0 + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one_or_none() + assert leaver is not None, "the row itself must survive the prune" + # The database stamps this (see bootstrap_universe), and SQLite's + # CURRENT_DATE is UTC while date.today() is local — so allow the one-day + # window rather than making this fail around midnight. + assert leaver.untracked_on is not None + assert abs((leaver.untracked_on - date.today()).days) <= 1 + # Untracked is not delisted: the security still trades. + assert leaver.delisted_on is None + + bars = (await session.execute( + select(OHLCVRecord).where(OHLCVRecord.ticker_id == leaver.id) + )).scalars().all() + assert len(bars) == 10, "the delete used to cascade these away" + + levels = (await session.execute( + select(SRLevel).where(SRLevel.ticker_id == leaver.id) + )).scalars().all() + assert len(levels) == 1, "every cascade relationship must survive, not just bars" + + +async def test_a_pruned_symbol_leaves_the_live_path(session: AsyncSession, monkeypatch): + await _seed(session) + await _prune_to(session, monkeypatch, ["AAPL"]) + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL"] + + # ...while the registry still shows it, so the state is visible rather than + # silently absent. + everything = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol)) + assert list(everything.scalars().all()) == ["AAPL", "LEAVER"] + + +async def test_todays_membership_does_not_erase_yesterdays_candidates( + session: AsyncSession, monkeypatch +): + """The backtest replays stored history, so it must see every symbol that has + bars — including ones that have since left the index. This guards + ``backtest_service``'s bare universe query: "fixing the inconsistency" by + adding ``active_only`` there is exactly how survivorship bias gets + reintroduced. + """ + await _seed(session) + await _prune_to(session, monkeypatch, ["AAPL"]) + + from app.services import backtest_service as bt + + symbols = await bt._load_backtest_symbols(session) + assert "LEAVER" in symbols, ( + "the backtest universe must include untracked symbols — their bars are " + "exactly the history that keeps a replay survivorship-honest" + ) + assert symbols == ["AAPL", "LEAVER"] + + +async def test_a_symbol_returning_to_the_index_rejoins_the_scanner( + session: AsyncSession, monkeypatch +): + """Re-tracking is the failure this refactor could introduce: a returning + symbol is already tracked, so it is neither added nor untracked, and without + an explicit clear it would keep its old mark and stay invisible forever.""" + await _seed(session) + await _prune_to(session, monkeypatch, ["AAPL"]) + + summary = await _prune_to(session, monkeypatch, ["AAPL", "LEAVER"]) + assert summary["retracked"] == 1 + assert summary["added"] == 0, "the row was never deleted, so there is nothing to add" + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL", "LEAVER"] + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + assert leaver.untracked_on is None + bars = (await session.execute( + select(OHLCVRecord).where(OHLCVRecord.ticker_id == leaver.id) + )).scalars().all() + assert len(bars) == 10, "it rejoins with its history, not as a blank new row" + + +async def test_a_second_prune_does_not_restamp_the_date(session: AsyncSession, monkeypatch): + """The mark records when the symbol left, so a daily bootstrap must not keep + moving it forward — that would erase the only record of when it happened.""" + await _seed(session) + await _prune_to(session, monkeypatch, ["AAPL"]) + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + leaver.untracked_on = date(2026, 1, 15) + await session.commit() + + summary = await _prune_to(session, monkeypatch, ["AAPL"]) + assert summary["untracked"] == 0, "already-marked rows are not re-marked" + + await session.refresh(leaver) + assert leaver.untracked_on == date(2026, 1, 15) + + +@pytest.mark.parametrize("column", ["delisted_on", "untracked_on", "scan_disabled_on"]) +async def test_as_of_answers_what_was_live_then_for_both_columns( + session: AsyncSession, column: str +): + """``as_of`` exists so a historical query can ask what was live at a date. + An untracked clause that ignored it would break that for the new column.""" + session.add(Ticker(symbol="AAPL")) + session.add(Ticker(symbol="LEFT", **{column: date(2026, 6, 1)})) + await session.commit() + + before = await session.execute( + ticker_service.active_only(select(Ticker.symbol), as_of=date(2026, 5, 1)) + ) + assert sorted(before.scalars().all()) == ["AAPL", "LEFT"] + + after = await session.execute( + ticker_service.active_only(select(Ticker.symbol), as_of=date(2026, 7, 1)) + ) + assert sorted(after.scalars().all()) == ["AAPL"] + + +async def test_a_symbol_can_be_both_untracked_and_delisted(session: AsyncSession): + """The state a name reaches when it leaves the index and later stops + trading. The two clauses are independent, so this must not double-count or + resurrect it.""" + session.add(Ticker( + symbol="BOTH", untracked_on=date(2026, 3, 1), delisted_on=date(2026, 6, 1) + )) + session.add(Ticker(symbol="AAPL")) + await session.commit() + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol), as_of=date(2026, 7, 1)) + ) + assert sorted(live.scalars().all()) == ["AAPL"] + + # Before either date it was still in the live path. + early = await session.execute( + ticker_service.active_only(select(Ticker.symbol), as_of=date(2026, 1, 1)) + ) + assert sorted(early.scalars().all()) == ["AAPL", "BOTH"] + + # Between them: out of the scanner already, but not yet delisted. + between = await session.execute( + ticker_service.active_only(select(Ticker.symbol), as_of=date(2026, 4, 1)) + ) + assert sorted(between.scalars().all()) == ["AAPL"] + + +def test_every_active_only_call_site_carries_both_predicates(): + """``active_only`` grew from one ``.where()`` to two. Call sites that chain + ``group_by``/``order_by`` onto its result, or pass multi-column selects, + must still get both clauses ANDed — a silently dropped predicate would put + untracked symbols back in the live path with nothing to show for it. + """ + from sqlalchemy import case, func + from sqlalchemy.dialects import postgresql + + from app.models.ohlcv import OHLCVRecord + + latest = func.max(OHLCVRecord.date) + call_sites = { + # scheduler.py _get_all_tickers + "plain": select(Ticker.symbol).order_by(Ticker.symbol), + # sec_universe.py — multi-column select + "multi_column": select(Ticker.id, Ticker.symbol, Ticker.cik), + # rr_scanner_service.py — whole-entity select + "entity": select(Ticker).order_by(Ticker.symbol), + } + compiled = { + name: " ".join(str( + ticker_service.active_only(stmt).compile(dialect=postgresql.dialect()) + ).split()) + for name, stmt in call_sites.items() + } + # scheduler.py _get_ohlcv_priority_tickers chains OUTSIDE the call. + compiled["chained"] = " ".join(str( + ticker_service.active_only( + select(Ticker.symbol).outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id) + ) + .group_by(Ticker.id, Ticker.symbol) + .order_by(case((latest.is_(None), 0), else_=1).asc(), Ticker.symbol.asc()) + .compile(dialect=postgresql.dialect()) + ).split()) + + for name, sql in compiled.items(): + assert "tickers.delisted_on IS NULL" in sql, name + assert "tickers.untracked_on IS NULL" in sql, name + assert "tickers.scan_disabled_on IS NULL" in sql, name + # ANDed together, not collapsed into one OR that would let either alone + # re-admit the symbol. + assert ") AND (" in sql[sql.index("WHERE"):], name + + +async def test_a_symbol_can_be_untracked_individually(session: AsyncSession): + """The case RHM is: a manually added symbol no provider can serve. Without a + per-symbol control the only way to untrack it is a whole-universe prune, + which would sweep every other manual addition with it.""" + await _seed(session) + + changed = await ticker_service.set_tracked(session, "LEAVER", tracked=False) + assert changed is True + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL"] + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + # The operator's switch, not the prune flag: it has to survive a + # prune -> re-entry cycle, which clears untracked_on. + assert leaver.scan_disabled_on is not None + assert leaver.untracked_on is None + # Switching off is not delisting; the security still trades. + assert leaver.delisted_on is None + bars = (await session.execute( + select(OHLCVRecord).where(OHLCVRecord.ticker_id == leaver.id) + )).scalars().all() + assert len(bars) == 10 + + +async def test_setting_tracking_is_idempotent_and_reversible(session: AsyncSession): + await _seed(session) + + assert await ticker_service.set_tracked(session, "LEAVER", tracked=False) is True + assert await ticker_service.set_tracked(session, "LEAVER", tracked=False) is False + + assert await ticker_service.set_tracked(session, "LEAVER", tracked=True) is True + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL", "LEAVER"] + + +async def test_untracking_an_unknown_symbol_raises(session: AsyncSession): + from app.exceptions import NotFoundError + + with pytest.raises(NotFoundError): + await ticker_service.set_tracked(session, "NOPE", tracked=False) + + +async def test_untracking_does_not_clear_a_data_quality_warning(session: AsyncSession): + """Untracking is not a way to silence the completeness check. + + The backtest deliberately includes untracked symbols, so an untracked ticker + with no bars still counts in ``no_data`` and still reports ``degraded``. If + it did not, "untrack it" would become a way to make coverage gaps disappear + from the report — which is the exact failure the ledger exists to prevent. + """ + from app.services import backtest_service as bt + + session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="NOBARS")]) + await session.commit() + await ticker_service.set_tracked(session, "NOBARS", tracked=False) + + # Still in the backtest universe... + assert await bt._load_backtest_symbols(session) == ["AAPL", "NOBARS"] + # ...and still unreadable, so still a coverage gap. + assert await bt._fetch_columns(session, "NOBARS") is None + + quality = bt._DataQuality(expected=2, evaluated=1) + quality.no_data_symbols.append("NOBARS") + quality.record_benchmark({date(2026, 1, 5): 470.0}, None) + report = quality.as_report() + + assert report["status"] == "degraded" + assert report["no_data_symbols"] == ["NOBARS"] + assert "1 symbol had no stored bars (NOBARS)" in report["reasons"] + + +# --------------------------------------------------------------------------- +# Tracking ownership (item 3) +# --------------------------------------------------------------------------- + + +async def test_a_manual_addition_survives_index_pruning(session: AsyncSession, monkeypatch): + """The RHM case. A hand-added symbol is by definition absent from the + constituent list, so without the ownership filter every prune would sweep + it.""" + await _seed(session) + session.add(Ticker(symbol="MINE", tracking_source=ticker_service.SOURCE_MANUAL)) + await session.commit() + + summary = await _prune_to(session, monkeypatch, ["AAPL"]) + + assert summary["untracked_symbols"] == ["LEAVER"] + assert summary["kept_manual"] == ["MINE"] + assert summary["kept_manual_count"] == 1 + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL", "MINE"] + + +async def test_a_bootstrap_never_converts_a_manual_row_to_index_owned( + session: AsyncSession, monkeypatch +): + """Ownership is sticky towards manual. If a bootstrap could claim a manual + row just because the symbol joined the index, the protection would vanish + the moment it left again — which is exactly when it is needed.""" + await _seed(session) + session.add(Ticker(symbol="MINE", tracking_source=ticker_service.SOURCE_MANUAL)) + await session.commit() + + # MINE joins the index. + await _prune_to(session, monkeypatch, ["AAPL", "MINE"]) + mine = ( + await session.execute(select(Ticker).where(Ticker.symbol == "MINE")) + ).scalar_one() + assert mine.tracking_source == ticker_service.SOURCE_MANUAL + + # ...and later leaves it again. Still protected. + summary = await _prune_to(session, monkeypatch, ["AAPL"]) + assert summary["kept_manual"] == ["MINE"] + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL", "MINE"] + + +async def test_a_manual_add_claims_an_index_owned_symbol(session: AsyncSession): + """An explicit "I want this stock" has to stop it being prunable.""" + await _seed(session) + + claimed = await ticker_service.add_ticker(session, "leaver") + assert claimed.tracking_source == ticker_service.SOURCE_MANUAL + + # Claiming ownership is not the same as asking for it to be scanned, so the + # off-switches are left exactly as they were. + assert claimed.scan_disabled_on is None + assert claimed.untracked_on is None + + +async def test_adding_an_already_manual_symbol_still_raises(session: AsyncSession): + from app.exceptions import DuplicateError + + await _seed(session) + await ticker_service.add_ticker(session, "LEAVER") + with pytest.raises(DuplicateError): + await ticker_service.add_ticker(session, "LEAVER") + + +async def test_re_entry_clears_the_prune_flag_but_never_the_operator_switch( + session: AsyncSession, monkeypatch +): + """The conflation bug the extra column exists to prevent: one column serving + both writers means a prune -> re-entry cycle silently re-enables a symbol + somebody deliberately switched off.""" + await _seed(session) + await ticker_service.set_tracked(session, "LEAVER", tracked=False) + + await _prune_to(session, monkeypatch, ["AAPL"]) # leaves the index + summary = await _prune_to(session, monkeypatch, ["AAPL", "LEAVER"]) # returns + + assert summary["retracked"] == 1 + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + assert leaver.untracked_on is None, "the prune flag is cleared by re-entry" + assert leaver.scan_disabled_on is not None, "the operator's switch is not" + + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL"] diff --git a/tests/unit/test_universe_membership_intervals.py b/tests/unit/test_universe_membership_intervals.py new file mode 100644 index 0000000..3df15c8 --- /dev/null +++ b/tests/unit/test_universe_membership_intervals.py @@ -0,0 +1,284 @@ +"""Observed membership intervals. + +The table answers "what could the strategy actually have selected on date D?" +Three properties make that answer trustworthy: + +1. History starts at the first *observation*. Nothing invents an index effective + date for a symbol that was already tracked when recording began. +2. Intervals are append-only. Leaving and returning produces two rows, so a + point-in-time query for the gap correctly answers "not a member". +3. A failed universe fetch records nothing. The seed fallback is ~44 symbols, so + treating it as an observation would close ~460 memberships on a network blip. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from datetime import date + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.database import Base +from app.models.ticker import Ticker +from app.models.universe_membership import UniverseMembership +from app.services import ticker_service +from app.services import ticker_universe_service as tus + +_engine = create_async_engine("sqlite+aiosqlite://", echo=False) +_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest.fixture(autouse=True) +async def _setup_tables() -> AsyncGenerator[None, None]: + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield + async with _engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + + +@pytest.fixture +async def session() -> AsyncGenerator[AsyncSession, None]: + async with _session_factory() as s: + yield s + + +async def _sync(session, monkeypatch, symbols: list[str], *, source: str = "wikipedia_sp500", + prune: bool = False) -> dict: + async def fake_fetch(db, universe): + return symbols, source + + monkeypatch.setattr(tus, "fetch_universe_symbols", fake_fetch) + return await tus.bootstrap_universe(session, "sp500", prune_missing=prune) + + +async def _intervals(session, symbol: str) -> list[tuple]: + rows = (await session.execute( + select(UniverseMembership.first_seen_on, UniverseMembership.absent_since) + .join(Ticker, Ticker.id == UniverseMembership.ticker_id) + .where(Ticker.symbol == symbol) + .order_by(UniverseMembership.id) + )).all() + return [tuple(r) for r in rows] + + +async def test_a_live_sync_opens_intervals_without_pruning(session: AsyncSession, monkeypatch): + """The scheduled job runs prune_missing=False. If membership were only + recorded on prunes the history would be full of holes.""" + summary = await _sync(session, monkeypatch, ["AAPL", "MSFT"]) + + assert summary["membership_observed"] is True + assert summary["membership_opened"] == 2 + assert summary["membership_closed"] == 0 + assert summary["untracked"] == 0, "recording membership is not pruning" + + intervals = await _intervals(session, "AAPL") + assert len(intervals) == 1 + first_seen, last_seen = intervals[0] + assert last_seen is None, "an open interval means still observed" + assert abs((first_seen - date.today()).days) <= 1 + + +async def test_a_second_sync_does_not_duplicate_an_open_interval( + session: AsyncSession, monkeypatch +): + await _sync(session, monkeypatch, ["AAPL"]) + summary = await _sync(session, monkeypatch, ["AAPL"]) + + assert summary["membership_opened"] == 0 + assert len(await _intervals(session, "AAPL")) == 1 + + +async def test_leaving_closes_the_interval_and_returning_opens_a_new_one( + session: AsyncSession, monkeypatch +): + await _sync(session, monkeypatch, ["AAPL", "LEAVER"]) + closing = await _sync(session, monkeypatch, ["AAPL"]) + assert closing["membership_closed"] == 1 + + intervals = await _intervals(session, "LEAVER") + assert len(intervals) == 1 + assert intervals[0][1] is not None, "departure closes the interval" + + returning = await _sync(session, monkeypatch, ["AAPL", "LEAVER"]) + assert returning["membership_opened"] == 1 + + intervals = await _intervals(session, "LEAVER") + assert len(intervals) == 2, "re-entry appends; it does not reopen" + assert intervals[0][1] is not None, "the earlier interval keeps its end date" + assert intervals[1][1] is None + + +async def test_a_failed_fetch_records_nothing(session: AsyncSession, monkeypatch): + """A cache or seed result means the live fetch failed. The seed list is a + fraction of the real universe — closing memberships from it would rewrite + history on a network blip.""" + await _sync(session, monkeypatch, ["AAPL", "MSFT"]) + + for failed_source in ("cache", "seed"): + summary = await _sync(session, monkeypatch, ["AAPL"], source=failed_source) + assert summary["membership_observed"] is False + assert summary["membership_closed"] == 0 + assert summary["membership_opened"] == 0 + + # MSFT's membership is untouched: we never looked, so we know nothing new. + msft = await _intervals(session, "MSFT") + assert len(msft) == 1 + assert msft[0][1] is None, "still open — a failed fetch is not a departure" + + +async def test_membership_is_recorded_for_manual_rows_too(session: AsyncSession, monkeypatch): + """Membership and tracking ownership are independent axes: a manually owned + symbol that is genuinely in the index still gets intervals.""" + session.add(Ticker(symbol="MINE", tracking_source=ticker_service.SOURCE_MANUAL)) + await session.commit() + + await _sync(session, monkeypatch, ["AAPL", "MINE"]) + + mine = ( + await session.execute(select(Ticker).where(Ticker.symbol == "MINE")) + ).scalar_one() + assert mine.tracking_source == ticker_service.SOURCE_MANUAL + assert len(await _intervals(session, "MINE")) == 1 + + +async def test_switching_off_scanning_does_not_close_a_membership( + session: AsyncSession, monkeypatch +): + """The index still contains the symbol; we simply stopped looking at it. + Conflating those would corrupt the point-in-time answer.""" + await _sync(session, monkeypatch, ["AAPL", "MSFT"]) + await ticker_service.set_tracked(session, "MSFT", tracked=False) + + summary = await _sync(session, monkeypatch, ["AAPL", "MSFT"]) + assert summary["membership_closed"] == 0 + assert (await _intervals(session, "MSFT"))[0][1] is None + + +async def test_pruning_does_not_close_a_membership_by_itself( + session: AsyncSession, monkeypatch +): + """Closing happens because the symbol was absent from a live list, not + because a prune ran — the two just usually coincide.""" + await _sync(session, monkeypatch, ["AAPL", "LEAVER"]) + session.add(Ticker(symbol="MINE", tracking_source=ticker_service.SOURCE_MANUAL)) + await session.commit() + + summary = await _sync(session, monkeypatch, ["AAPL"], prune=True) + # LEAVER closed because it was absent; MINE never had an interval to close. + assert summary["membership_closed"] == 1 + assert await _intervals(session, "MINE") == [] + + +async def test_a_failed_fetch_does_not_untrack_anything(session: AsyncSession, monkeypatch): + """The seed fallback is ~44 symbols against a ~500-symbol universe. Pruning + from it would untrack the entire index on a single network blip — skipping + membership recording is only half the safeguard.""" + session.add_all([ + Ticker(symbol=s, tracking_source=ticker_service.SOURCE_INDEX) + for s in ("AAPL", "MSFT", "NVDA") + ]) + await session.commit() + + summary = await _sync(session, monkeypatch, ["AAPL"], source="seed", prune=True) + + assert summary["untracked"] == 0, "a failed fetch must not drive pruning" + live = await session.execute( + ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol)) + ) + assert list(live.scalars().all()) == ["AAPL", "MSFT", "NVDA"] + + +async def test_a_failed_fetch_does_not_clear_a_prune_flag(session: AsyncSession, monkeypatch): + """Re-entry is a claim about the index. A cache fallback is not evidence the + symbol rejoined it.""" + session.add_all([ + Ticker(symbol=s, tracking_source=ticker_service.SOURCE_INDEX) + for s in ("AAPL", "LEAVER") + ]) + await session.commit() + await _sync(session, monkeypatch, ["AAPL"], prune=True) # LEAVER untracked + + summary = await _sync(session, monkeypatch, ["AAPL", "LEAVER"], source="cache") + assert summary["retracked"] == 0, "stale data must not re-admit a symbol" + + leaver = ( + await session.execute(select(Ticker).where(Ticker.symbol == "LEAVER")) + ).scalar_one() + assert leaver.untracked_on is not None + + +# --------------------------------------------------------------------------- +# Interval boundaries, stated as executable rules +# --------------------------------------------------------------------------- + + +async def _interval(session, symbol: str, universe: str, first: date, absent: date | None): + """Write one interval directly, so boundary tests don't depend on today.""" + ticker = ( + await session.execute(select(Ticker).where(Ticker.symbol == symbol)) + ).scalar_one_or_none() + if ticker is None: + ticker = Ticker(symbol=symbol, tracking_source=ticker_service.SOURCE_INDEX) + session.add(ticker) + await session.flush() + session.add(UniverseMembership( + ticker_id=ticker.id, universe=universe, + first_seen_on=first, absent_since=absent, + )) + await session.commit() + + +async def test_membership_bounds_are_lower_inclusive_upper_exclusive(session: AsyncSession): + """``absent_since`` is the first observation that found the symbol GONE, so + on that date it is already absent. ``first_seen_on`` is the first that found + it present, so on that date it is already a member.""" + await _interval(session, "EDGE", "sp500", date(2026, 3, 10), date(2026, 8, 20)) + + async def members(d: date) -> list[str]: + return await tus.members_on(session, "sp500", d) + + assert await members(date(2026, 3, 9)) == [], "the day before first observation" + assert await members(date(2026, 3, 10)) == ["EDGE"], "lower bound is inclusive" + assert await members(date(2026, 8, 19)) == ["EDGE"], "still present the day before" + assert await members(date(2026, 8, 20)) == [], "upper bound is exclusive" + + +async def test_an_open_interval_has_no_upper_bound(session: AsyncSession): + await _interval(session, "OPEN", "sp500", date(2026, 3, 10), None) + assert await tus.members_on(session, "sp500", date(2030, 1, 1)) == ["OPEN"] + + +async def test_a_gap_between_two_intervals_is_not_membership(session: AsyncSession): + """The reason re-entry appends instead of reopening.""" + await _interval(session, "GAP", "sp500", date(2026, 1, 5), date(2026, 3, 2)) + await _interval(session, "GAP", "sp500", date(2026, 8, 3), None) + + assert await tus.members_on(session, "sp500", date(2026, 2, 1)) == ["GAP"] + assert await tus.members_on(session, "sp500", date(2026, 5, 1)) == [] + assert await tus.members_on(session, "sp500", date(2026, 9, 1)) == ["GAP"] + + +async def test_membership_is_scoped_per_universe(session: AsyncSession): + await _interval(session, "BOTH", "sp500", date(2026, 1, 5), None) + await _interval(session, "BOTH", "nasdaq100", date(2026, 6, 1), None) + await _interval(session, "SPONLY", "sp500", date(2026, 1, 5), None) + + assert await tus.members_on(session, "sp500", date(2026, 7, 1)) == ["BOTH", "SPONLY"] + assert await tus.members_on(session, "nasdaq100", date(2026, 7, 1)) == ["BOTH"] + # Before it joined the Nasdaq-100 it was only an S&P name. + assert await tus.members_on(session, "nasdaq100", date(2026, 5, 1)) == [] + + +async def test_before_recording_began_returns_empty_not_the_current_set( + session: AsyncSession, monkeypatch +): + """The honest limit of this table: a date before the first successful live + sync means "we were not looking", not "the index was empty". Answering with + today's set would be exactly the survivorship error item 3 exists to end.""" + await _sync(session, monkeypatch, ["AAPL", "MSFT"]) + + assert await tus.members_on(session, "sp500", date(2020, 1, 2)) == [] + assert await tus.members_on(session, "sp500", date.today()) == ["AAPL", "MSFT"]