Big refactoring

app/config.py

@@ -22,15 +22,24 @@ class Settings(BaseSettings):
     # Sentiment Provider — OpenAI
     openai_api_key: str = ""
     openai_model: str = "gpt-4o-mini"
+    openai_sentiment_batch_size: int = 5
 
     # Fundamentals Provider — Financial Modeling Prep
     fmp_api_key: str = ""
 
+    # Fundamentals Provider — Finnhub (optional fallback)
+    finnhub_api_key: str = ""
+
+    # Fundamentals Provider — Alpha Vantage (optional fallback)
+    alpha_vantage_api_key: str = ""
+
     # Scheduled Jobs
     data_collector_frequency: str = "daily"
     sentiment_poll_interval_minutes: int = 30
     fundamental_fetch_frequency: str = "daily"
     rr_scan_frequency: str = "daily"
+    fundamental_rate_limit_retries: int = 3
+    fundamental_rate_limit_backoff_seconds: int = 15
 
     # Scoring Defaults
     default_watchlist_auto_size: int = 10

app/models/trade_setup.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 
-from sqlalchemy import DateTime, Float, ForeignKey, String
+import json
+
+from sqlalchemy import DateTime, Float, ForeignKey, String, Text
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.database import Base
@@ -23,4 +25,34 @@ class TradeSetup(Base):
         DateTime(timezone=True), nullable=False
     )
 
+    confidence_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+    targets_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+    conflict_flags_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+    recommended_action: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    reasoning: Mapped[str | None] = mapped_column(Text, nullable=True)
+    risk_level: Mapped[str | None] = mapped_column(String(10), nullable=True)
+    actual_outcome: Mapped[str | None] = mapped_column(String(20), nullable=True)
+
     ticker = relationship("Ticker", back_populates="trade_setups")
+
+    @property
+    def targets(self) -> list[dict]:
+        if not self.targets_json:
+            return []
+        try:
+            parsed = json.loads(self.targets_json)
+        except (TypeError, ValueError):
+            return []
+        return parsed if isinstance(parsed, list) else []
+
+    @property
+    def conflict_flags(self) -> list[str]:
+        if not self.conflict_flags_json:
+            return []
+        try:
+            parsed = json.loads(self.conflict_flags_json)
+        except (TypeError, ValueError):
+            return []
+        if not isinstance(parsed, list):
+            return []
+        return [str(item) for item in parsed]

app/providers/fundamentals_chain.py

@@ -0,0 +1,253 @@
+"""Chained fundamentals provider with fallback adapters.
+
+Order:
+1) FMP (if configured)
+2) Finnhub (if configured)
+3) Alpha Vantage (if configured)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+
+from app.config import settings
+from app.exceptions import ProviderError, RateLimitError
+from app.providers.fmp import FMPFundamentalProvider
+from app.providers.protocol import FundamentalData, FundamentalProvider
+
+logger = logging.getLogger(__name__)
+
+_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
+if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
+    _CA_BUNDLE_PATH: str | bool = True
+else:
+    _CA_BUNDLE_PATH = _CA_BUNDLE
+
+
+def _safe_float(value: object) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+class FinnhubFundamentalProvider:
+    """Fundamentals provider backed by Finnhub free endpoints."""
+
+    def __init__(self, api_key: str) -> None:
+        if not api_key:
+            raise ProviderError("Finnhub API key is required")
+        self._api_key = api_key
+        self._base_url = "https://finnhub.io/api/v1"
+
+    async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
+        unavailable: dict[str, str] = {}
+
+        async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
+            profile_resp = await client.get(
+                f"{self._base_url}/stock/profile2",
+                params={"symbol": ticker, "token": self._api_key},
+            )
+            metric_resp = await client.get(
+                f"{self._base_url}/stock/metric",
+                params={"symbol": ticker, "metric": "all", "token": self._api_key},
+            )
+            earnings_resp = await client.get(
+                f"{self._base_url}/stock/earnings",
+                params={"symbol": ticker, "limit": 1, "token": self._api_key},
+            )
+
+        for resp, endpoint in (
+            (profile_resp, "profile2"),
+            (metric_resp, "stock/metric"),
+            (earnings_resp, "stock/earnings"),
+        ):
+            if resp.status_code == 429:
+                raise RateLimitError(f"Finnhub rate limit hit for {ticker} ({endpoint})")
+            if resp.status_code in (401, 403):
+                raise ProviderError(f"Finnhub access denied for {ticker} ({endpoint}): HTTP {resp.status_code}")
+            if resp.status_code != 200:
+                raise ProviderError(f"Finnhub error for {ticker} ({endpoint}): HTTP {resp.status_code}")
+
+        profile_payload = profile_resp.json() if profile_resp.text else {}
+        metric_payload = metric_resp.json() if metric_resp.text else {}
+        earnings_payload = earnings_resp.json() if earnings_resp.text else []
+
+        metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {}
+        market_cap = _safe_float((profile_payload or {}).get("marketCapitalization"))
+        pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual"))
+        revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y"))
+
+        earnings_surprise = None
+        if isinstance(earnings_payload, list) and earnings_payload:
+            first = earnings_payload[0] if isinstance(earnings_payload[0], dict) else {}
+            earnings_surprise = _safe_float(first.get("surprisePercent"))
+
+        if pe_ratio is None:
+            unavailable["pe_ratio"] = "not available from provider payload"
+        if revenue_growth is None:
+            unavailable["revenue_growth"] = "not available from provider payload"
+        if earnings_surprise is None:
+            unavailable["earnings_surprise"] = "not available from provider payload"
+        if market_cap is None:
+            unavailable["market_cap"] = "not available from provider payload"
+
+        return FundamentalData(
+            ticker=ticker,
+            pe_ratio=pe_ratio,
+            revenue_growth=revenue_growth,
+            earnings_surprise=earnings_surprise,
+            market_cap=market_cap,
+            fetched_at=datetime.now(timezone.utc),
+            unavailable_fields=unavailable,
+        )
+
+
+class AlphaVantageFundamentalProvider:
+    """Fundamentals provider backed by Alpha Vantage free endpoints."""
+
+    def __init__(self, api_key: str) -> None:
+        if not api_key:
+            raise ProviderError("Alpha Vantage API key is required")
+        self._api_key = api_key
+        self._base_url = "https://www.alphavantage.co/query"
+
+    async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
+        unavailable: dict[str, str] = {}
+
+        async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
+            overview_resp = await client.get(
+                self._base_url,
+                params={"function": "OVERVIEW", "symbol": ticker, "apikey": self._api_key},
+            )
+            earnings_resp = await client.get(
+                self._base_url,
+                params={"function": "EARNINGS", "symbol": ticker, "apikey": self._api_key},
+            )
+            income_resp = await client.get(
+                self._base_url,
+                params={"function": "INCOME_STATEMENT", "symbol": ticker, "apikey": self._api_key},
+            )
+
+        for resp, endpoint in (
+            (overview_resp, "OVERVIEW"),
+            (earnings_resp, "EARNINGS"),
+            (income_resp, "INCOME_STATEMENT"),
+        ):
+            if resp.status_code == 429:
+                raise RateLimitError(f"Alpha Vantage rate limit hit for {ticker} ({endpoint})")
+            if resp.status_code != 200:
+                raise ProviderError(f"Alpha Vantage error for {ticker} ({endpoint}): HTTP {resp.status_code}")
+
+        overview = overview_resp.json() if overview_resp.text else {}
+        earnings = earnings_resp.json() if earnings_resp.text else {}
+        income = income_resp.json() if income_resp.text else {}
+
+        if isinstance(overview, dict) and overview.get("Information"):
+            raise ProviderError(f"Alpha Vantage unavailable for {ticker}: {overview.get('Information')}")
+        if isinstance(overview, dict) and overview.get("Note"):
+            raise RateLimitError(f"Alpha Vantage rate limit for {ticker}: {overview.get('Note')}")
+
+        pe_ratio = _safe_float((overview or {}).get("PERatio"))
+        market_cap = _safe_float((overview or {}).get("MarketCapitalization"))
+
+        earnings_surprise = None
+        quarterly = earnings.get("quarterlyEarnings", []) if isinstance(earnings, dict) else []
+        if isinstance(quarterly, list) and quarterly:
+            first = quarterly[0] if isinstance(quarterly[0], dict) else {}
+            earnings_surprise = _safe_float(first.get("surprisePercentage"))
+
+        revenue_growth = None
+        annual = income.get("annualReports", []) if isinstance(income, dict) else []
+        if isinstance(annual, list) and len(annual) >= 2:
+            curr = _safe_float((annual[0] or {}).get("totalRevenue"))
+            prev = _safe_float((annual[1] or {}).get("totalRevenue"))
+            if curr is not None and prev not in (None, 0):
+                revenue_growth = ((curr - prev) / abs(prev)) * 100.0
+
+        if pe_ratio is None:
+            unavailable["pe_ratio"] = "not available from provider payload"
+        if revenue_growth is None:
+            unavailable["revenue_growth"] = "not available from provider payload"
+        if earnings_surprise is None:
+            unavailable["earnings_surprise"] = "not available from provider payload"
+        if market_cap is None:
+            unavailable["market_cap"] = "not available from provider payload"
+
+        return FundamentalData(
+            ticker=ticker,
+            pe_ratio=pe_ratio,
+            revenue_growth=revenue_growth,
+            earnings_surprise=earnings_surprise,
+            market_cap=market_cap,
+            fetched_at=datetime.now(timezone.utc),
+            unavailable_fields=unavailable,
+        )
+
+
+class ChainedFundamentalProvider:
+    """Try multiple fundamental providers in order until one succeeds."""
+
+    def __init__(self, providers: list[tuple[str, FundamentalProvider]]) -> None:
+        if not providers:
+            raise ProviderError("No fundamental providers configured")
+        self._providers = providers
+
+    async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
+        errors: list[str] = []
+
+        for provider_name, provider in self._providers:
+            try:
+                data = await provider.fetch_fundamentals(ticker)
+
+                has_any_metric = any(
+                    value is not None
+                    for value in (data.pe_ratio, data.revenue_growth, data.earnings_surprise, data.market_cap)
+                )
+                if not has_any_metric:
+                    errors.append(f"{provider_name}: no usable metrics returned")
+                    continue
+
+                unavailable = dict(data.unavailable_fields)
+                unavailable["provider"] = provider_name
+
+                return FundamentalData(
+                    ticker=data.ticker,
+                    pe_ratio=data.pe_ratio,
+                    revenue_growth=data.revenue_growth,
+                    earnings_surprise=data.earnings_surprise,
+                    market_cap=data.market_cap,
+                    fetched_at=data.fetched_at,
+                    unavailable_fields=unavailable,
+                )
+            except Exception as exc:
+                errors.append(f"{provider_name}: {type(exc).__name__}: {exc}")
+
+        attempts = "; ".join(errors[:6]) if errors else "no provider attempts"
+        raise ProviderError(f"All fundamentals providers failed for {ticker}. Attempts: {attempts}")
+
+
+def build_fundamental_provider_chain() -> FundamentalProvider:
+    providers: list[tuple[str, FundamentalProvider]] = []
+
+    if settings.fmp_api_key:
+        providers.append(("fmp", FMPFundamentalProvider(settings.fmp_api_key)))
+    if settings.finnhub_api_key:
+        providers.append(("finnhub", FinnhubFundamentalProvider(settings.finnhub_api_key)))
+    if settings.alpha_vantage_api_key:
+        providers.append(("alpha_vantage", AlphaVantageFundamentalProvider(settings.alpha_vantage_api_key)))
+
+    if not providers:
+        raise ProviderError(
+            "No fundamentals provider configured. Set one of FMP_API_KEY, FINNHUB_API_KEY, ALPHA_VANTAGE_API_KEY"
+        )
+
+    logger.info("Fundamentals provider chain configured: %s", [name for name, _ in providers])
+    return ChainedFundamentalProvider(providers)

app/providers/openai_sentiment.py

@@ -33,6 +33,24 @@ Rules:
 - reasoning should cite specific recent news or events you found
 """
 
+_SENTIMENT_BATCH_PROMPT = """\
+Search the web for the LATEST news, analyst opinions, and market developments \
+about each stock ticker from the past 24-48 hours.
+
+Tickers:
+{tickers_csv}
+
+Respond ONLY with a JSON array (no markdown, no extra text), one object per ticker:
+[{{"ticker":"AAPL","classification":"bullish|bearish|neutral","confidence":0-100,"reasoning":"brief explanation"}}]
+
+Rules:
+- Include every ticker exactly once
+- ticker must be uppercase symbol
+- classification must be exactly one of: bullish, bearish, neutral
+- confidence must be an integer from 0 to 100
+- reasoning should cite specific recent news or events you found
+"""
+
 VALID_CLASSIFICATIONS = {"bullish", "bearish", "neutral"}
 
 
@@ -49,6 +67,59 @@ class OpenAISentimentProvider:
         self._client = AsyncOpenAI(api_key=api_key, http_client=http_client)
         self._model = model
 
+    @staticmethod
+    def _extract_raw_text(response: object, ticker_context: str) -> str:
+        raw_text = ""
+        for item in response.output:
+            if item.type == "message" and item.content:
+                for block in item.content:
+                    if hasattr(block, "text") and block.text:
+                        raw_text = block.text
+                        break
+                if raw_text:
+                    break
+
+        if not raw_text:
+            raise ProviderError(f"No text output from OpenAI for {ticker_context}")
+
+        clean = raw_text.strip()
+        if clean.startswith("```"):
+            clean = clean.split("\n", 1)[1] if "\n" in clean else clean[3:]
+        if clean.endswith("```"):
+            clean = clean[:-3]
+        return clean.strip()
+
+    @staticmethod
+    def _normalize_single_result(parsed: dict, ticker: str, citations: list[dict[str, str]]) -> SentimentData:
+        classification = str(parsed.get("classification", "")).lower()
+        if classification not in VALID_CLASSIFICATIONS:
+            raise ProviderError(
+                f"Invalid classification '{classification}' from OpenAI for {ticker}"
+            )
+
+        confidence = int(parsed.get("confidence", 50))
+        confidence = max(0, min(100, confidence))
+        reasoning = str(parsed.get("reasoning", ""))
+
+        if reasoning:
+            logger.info(
+                "OpenAI sentiment for %s: %s (confidence=%d) — %s",
+                ticker,
+                classification,
+                confidence,
+                reasoning,
+            )
+
+        return SentimentData(
+            ticker=ticker,
+            classification=classification,
+            confidence=confidence,
+            source="openai",
+            timestamp=datetime.now(timezone.utc),
+            reasoning=reasoning,
+            citations=citations,
+        )
+
     async def fetch_sentiment(self, ticker: str) -> SentimentData:
         """Use the Responses API with web_search_preview to get live sentiment."""
         try:
@@ -58,48 +129,10 @@ class OpenAISentimentProvider:
                 instructions="You are a financial sentiment analyst. Always respond with valid JSON only, no markdown fences.",
                 input=_SENTIMENT_PROMPT.format(ticker=ticker),
             )
-
-            # Extract text from the ResponseOutputMessage in the output
-            raw_text = ""
-            for item in response.output:
-                if item.type == "message" and item.content:
-                    for block in item.content:
-                        if hasattr(block, "text") and block.text:
-                            raw_text = block.text
-                            break
-                    if raw_text:
-                        break
-
-            if not raw_text:
-                raise ProviderError(f"No text output from OpenAI for {ticker}")
-
-            raw_text = raw_text.strip()
-            logger.debug("OpenAI raw response for %s: %s", ticker, raw_text)
-
-            # Strip markdown fences if present
-            clean = raw_text
-            if clean.startswith("```"):
-                clean = clean.split("\n", 1)[1] if "\n" in clean else clean[3:]
-            if clean.endswith("```"):
-                clean = clean[:-3]
-            clean = clean.strip()
-
+            clean = self._extract_raw_text(response, ticker)
+            logger.debug("OpenAI raw response for %s: %s", ticker, clean)
             parsed = json.loads(clean)
 
-            classification = parsed.get("classification", "").lower()
-            if classification not in VALID_CLASSIFICATIONS:
-                raise ProviderError(
-                    f"Invalid classification '{classification}' from OpenAI for {ticker}"
-                )
-
-            confidence = int(parsed.get("confidence", 50))
-            confidence = max(0, min(100, confidence))
-
-            reasoning = parsed.get("reasoning", "")
-            if reasoning:
-                logger.info("OpenAI sentiment for %s: %s (confidence=%d) — %s",
-                            ticker, classification, confidence, reasoning)
-
             # Extract url_citation annotations from response output
             citations: list[dict[str, str]] = []
             for item in response.output:
@@ -112,19 +145,10 @@ class OpenAISentimentProvider:
                                         "url": getattr(annotation, "url", ""),
                                         "title": getattr(annotation, "title", ""),
                                     })
-
-            return SentimentData(
-                ticker=ticker,
-                classification=classification,
-                confidence=confidence,
-                source="openai",
-                timestamp=datetime.now(timezone.utc),
-                reasoning=reasoning,
-                citations=citations,
-            )
+            return self._normalize_single_result(parsed, ticker, citations)
 
         except json.JSONDecodeError as exc:
-            logger.error("Failed to parse OpenAI JSON for %s: %s — raw: %s", ticker, exc, raw_text)
+            logger.error("Failed to parse OpenAI JSON for %s: %s", ticker, exc)
             raise ProviderError(f"Invalid JSON from OpenAI for {ticker}") from exc
         except ProviderError:
             raise
@@ -134,3 +158,49 @@ class OpenAISentimentProvider:
                 raise RateLimitError(f"OpenAI rate limit hit for {ticker}") from exc
             logger.error("OpenAI provider error for %s: %s", ticker, exc)
             raise ProviderError(f"OpenAI provider error for {ticker}: {exc}") from exc
+
+    async def fetch_sentiment_batch(self, tickers: list[str]) -> dict[str, SentimentData]:
+        """Fetch sentiment for multiple tickers in one OpenAI request.
+
+        Returns a map keyed by uppercase ticker symbol. Invalid/missing rows are skipped.
+        """
+        normalized = [t.strip().upper() for t in tickers if t and t.strip()]
+        if not normalized:
+            return {}
+
+        ticker_context = ",".join(normalized)
+        try:
+            response = await self._client.responses.create(
+                model=self._model,
+                tools=[{"type": "web_search_preview"}],
+                instructions="You are a financial sentiment analyst. Always respond with valid JSON only, no markdown fences.",
+                input=_SENTIMENT_BATCH_PROMPT.format(tickers_csv=", ".join(normalized)),
+            )
+            clean = self._extract_raw_text(response, ticker_context)
+            logger.debug("OpenAI batch raw response for %s: %s", ticker_context, clean)
+            parsed = json.loads(clean)
+            if not isinstance(parsed, list):
+                raise ProviderError("Batch sentiment response must be a JSON array")
+
+            out: dict[str, SentimentData] = {}
+            requested = set(normalized)
+            for row in parsed:
+                if not isinstance(row, dict):
+                    continue
+                symbol = str(row.get("ticker", "")).strip().upper()
+                if symbol not in requested:
+                    continue
+                try:
+                    out[symbol] = self._normalize_single_result(row, symbol, citations=[])
+                except Exception:
+                    continue
+            return out
+        except json.JSONDecodeError as exc:
+            raise ProviderError(f"Invalid batch JSON from OpenAI for {ticker_context}") from exc
+        except ProviderError:
+            raise
+        except Exception as exc:
+            msg = str(exc).lower()
+            if "429" in msg or "rate" in msg or "quota" in msg:
+                raise RateLimitError(f"OpenAI rate limit hit for batch {ticker_context}") from exc
+            raise ProviderError(f"OpenAI batch provider error for {ticker_context}: {exc}") from exc

app/routers/admin.py

@@ -3,7 +3,7 @@
 All endpoints require admin role.
 """
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.dependencies import get_db, require_admin
@@ -12,13 +12,16 @@ from app.schemas.admin import (
     CreateUserRequest,
     DataCleanupRequest,
     JobToggle,
+    RecommendationConfigUpdate,
     PasswordReset,
     RegistrationToggle,
     SystemSettingUpdate,
+    TickerUniverseUpdate,
     UserManagement,
 )
 from app.schemas.common import APIEnvelope
 from app.services import admin_service
+from app.services import ticker_universe_service
 
 router = APIRouter(tags=["admin"])
 
@@ -123,6 +126,47 @@ async def list_settings(
     )
 
 
+@router.get("/admin/settings/recommendations", response_model=APIEnvelope)
+async def get_recommendation_settings(
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    config = await admin_service.get_recommendation_config(db)
+    return APIEnvelope(status="success", data=config)
+
+
+@router.put("/admin/settings/recommendations", response_model=APIEnvelope)
+async def update_recommendation_settings(
+    body: RecommendationConfigUpdate,
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    updated = await admin_service.update_recommendation_config(
+        db,
+        body.model_dump(exclude_unset=True),
+    )
+    return APIEnvelope(status="success", data=updated)
+
+
+@router.get("/admin/settings/ticker-universe", response_model=APIEnvelope)
+async def get_ticker_universe_setting(
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    data = await admin_service.get_ticker_universe_default(db)
+    return APIEnvelope(status="success", data=data)
+
+
+@router.put("/admin/settings/ticker-universe", response_model=APIEnvelope)
+async def update_ticker_universe_setting(
+    body: TickerUniverseUpdate,
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    data = await admin_service.update_ticker_universe_default(db, body.universe)
+    return APIEnvelope(status="success", data=data)
+
+
 @router.put("/admin/settings/{key}", response_model=APIEnvelope)
 async def update_setting(
     key: str,
@@ -138,6 +182,21 @@ async def update_setting(
     )
 
 
+@router.post("/admin/tickers/bootstrap", response_model=APIEnvelope)
+async def bootstrap_tickers(
+    universe: str = Query("sp500", pattern="^(sp500|nasdaq100|nasdaq_all)$"),
+    prune_missing: bool = Query(False),
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    result = await ticker_universe_service.bootstrap_universe(
+        db,
+        universe,
+        prune_missing=prune_missing,
+    )
+    return APIEnvelope(status="success", data=result)
+
+
 # ---------------------------------------------------------------------------
 # Data cleanup
 # ---------------------------------------------------------------------------
@@ -167,6 +226,15 @@ async def list_jobs(
     return APIEnvelope(status="success", data=jobs)
 
 
+@router.get("/admin/pipeline/readiness", response_model=APIEnvelope)
+async def get_pipeline_readiness(
+    _admin: User = Depends(require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    data = await admin_service.get_pipeline_readiness(db)
+    return APIEnvelope(status="success", data=data)
+
+
 @router.post("/admin/jobs/{job_name}/trigger", response_model=APIEnvelope)
 async def trigger_job(
     job_name: str,

app/routers/ingestion.py

@@ -18,10 +18,17 @@ from app.dependencies import get_db, require_access
 from app.exceptions import ProviderError
 from app.models.user import User
 from app.providers.alpaca import AlpacaOHLCVProvider
-from app.providers.fmp import FMPFundamentalProvider
+from app.providers.fundamentals_chain import build_fundamental_provider_chain
 from app.providers.openai_sentiment import OpenAISentimentProvider
+from app.services.rr_scanner_service import scan_ticker
 from app.schemas.common import APIEnvelope
-from app.services import fundamental_service, ingestion_service, sentiment_service
+from app.services import (
+    fundamental_service,
+    ingestion_service,
+    scoring_service,
+    sentiment_service,
+    sr_service,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -99,10 +106,10 @@ async def fetch_symbol(
         }
 
     # --- Fundamentals ---
-    if settings.fmp_api_key:
+    if settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key:
         try:
-            fmp_provider = FMPFundamentalProvider(settings.fmp_api_key)
-            fdata = await fmp_provider.fetch_fundamentals(symbol_upper)
+            fundamentals_provider = build_fundamental_provider_chain()
+            fdata = await fundamentals_provider.fetch_fundamentals(symbol_upper)
             await fundamental_service.store_fundamental(
                 db,
                 symbol=symbol_upper,
@@ -119,9 +126,50 @@ async def fetch_symbol(
     else:
         sources["fundamentals"] = {
             "status": "skipped",
-            "message": "FMP API key not configured",
+            "message": "No fundamentals provider key configured",
         }
 
+    # --- Derived pipeline: S/R levels ---
+    try:
+        levels = await sr_service.recalculate_sr_levels(db, symbol_upper)
+        sources["sr_levels"] = {
+            "status": "ok",
+            "count": len(levels),
+            "message": None,
+        }
+    except Exception as exc:
+        logger.error("S/R recalc failed for %s: %s", symbol_upper, exc)
+        sources["sr_levels"] = {"status": "error", "message": str(exc)}
+
+    # --- Derived pipeline: scores ---
+    try:
+        score_payload = await scoring_service.get_score(db, symbol_upper)
+        sources["scores"] = {
+            "status": "ok",
+            "composite_score": score_payload.get("composite_score"),
+            "missing_dimensions": score_payload.get("missing_dimensions", []),
+            "message": None,
+        }
+    except Exception as exc:
+        logger.error("Score recompute failed for %s: %s", symbol_upper, exc)
+        sources["scores"] = {"status": "error", "message": str(exc)}
+
+    # --- Derived pipeline: scanner ---
+    try:
+        setups = await scan_ticker(
+            db,
+            symbol_upper,
+            rr_threshold=settings.default_rr_threshold,
+        )
+        sources["scanner"] = {
+            "status": "ok",
+            "setups_found": len(setups),
+            "message": None,
+        }
+    except Exception as exc:
+        logger.error("Scanner run failed for %s: %s", symbol_upper, exc)
+        sources["scanner"] = {"status": "error", "message": str(exc)}
+
     # Always return success — per-source breakdown tells the full story
     return APIEnvelope(
         status="success",

app/routers/trades.py

@@ -5,8 +5,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.dependencies import get_db, require_access
 from app.schemas.common import APIEnvelope
-from app.schemas.trade_setup import TradeSetupResponse
-from app.services.rr_scanner_service import get_trade_setups
+from app.schemas.trade_setup import RecommendationSummaryResponse, TradeSetupResponse
+from app.services.rr_scanner_service import get_trade_setup_history, get_trade_setups
 
 router = APIRouter(tags=["trades"])
 
@@ -16,13 +16,73 @@ async def list_trade_setups(
     direction: str | None = Query(
         None, description="Filter by direction: long or short"
     ),
+    min_confidence: float | None = Query(
+        None, ge=0, le=100, description="Minimum confidence score"
+    ),
+    recommended_action: str | None = Query(
+        None,
+        description="Filter by action: LONG_HIGH, LONG_MODERATE, SHORT_HIGH, SHORT_MODERATE, NEUTRAL",
+    ),
     _user=Depends(require_access),
     db: AsyncSession = Depends(get_db),
 ) -> APIEnvelope:
-    """Get all trade setups sorted by R:R desc, secondary composite desc.
+    """Get latest trade setups with recommendation data."""
+    rows = await get_trade_setups(
+        db,
+        direction=direction,
+        min_confidence=min_confidence,
+        recommended_action=recommended_action,
+    )
+
+    data = []
+    for row in rows:
+        summary = RecommendationSummaryResponse(
+            action=row.get("recommended_action") or "NEUTRAL",
+            reasoning=row.get("reasoning"),
+            risk_level=row.get("risk_level"),
+            composite_score=row["composite_score"],
+        )
+        payload = {**row, "recommendation_summary": summary}
+        data.append(TradeSetupResponse(**payload).model_dump(mode="json"))
 
-    Optional direction filter (long/short).
-    """
-    rows = await get_trade_setups(db, direction=direction)
-    data = [TradeSetupResponse(**r).model_dump(mode="json") for r in rows]
+    return APIEnvelope(status="success", data=data)
+
+
+@router.get("/trades/{symbol}", response_model=APIEnvelope)
+async def get_ticker_trade_setups(
+    symbol: str,
+    _user=Depends(require_access),
+    db: AsyncSession = Depends(get_db),
+) -> APIEnvelope:
+    rows = await get_trade_setups(db, symbol=symbol)
+    data = []
+    for row in rows:
+        summary = RecommendationSummaryResponse(
+            action=row.get("recommended_action") or "NEUTRAL",
+            reasoning=row.get("reasoning"),
+            risk_level=row.get("risk_level"),
+            composite_score=row["composite_score"],
+        )
+        payload = {**row, "recommendation_summary": summary}
+        data.append(TradeSetupResponse(**payload).model_dump(mode="json"))
+    return APIEnvelope(status="success", data=data)
+
+
+@router.get("/trades/{symbol}/history", response_model=APIEnvelope)
+async def get_ticker_trade_history(
+    symbol: str,
+    _user=Depends(require_access),
+    db: AsyncSession = Depends(get_db),
+) -> APIEnvelope:
+    rows = await get_trade_setup_history(db, symbol=symbol)
+    data = []
+    for row in rows:
+        summary = RecommendationSummaryResponse(
+            action=row.get("recommended_action") or "NEUTRAL",
+            reasoning=row.get("reasoning"),
+            risk_level=row.get("risk_level"),
+            composite_score=row["composite_score"],
+        )
+        payload = {**row, "recommendation_summary": summary}
+        data.append(TradeSetupResponse(**payload).model_dump(mode="json"))
     return APIEnvelope(status="success", data=data)

app/scheduler.py

@@ -15,21 +15,27 @@ from __future__ import annotations
 
 import json
 import logging
-from datetime import date, timedelta
+import asyncio
+from datetime import date, datetime, timedelta, timezone
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from sqlalchemy import select
+from sqlalchemy import case, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import settings
 from app.database import async_session_factory
+from app.models.fundamental import FundamentalData
+from app.models.ohlcv import OHLCVRecord
 from app.models.settings import SystemSetting
+from app.models.sentiment import SentimentScore
 from app.models.ticker import Ticker
 from app.providers.alpaca import AlpacaOHLCVProvider
-from app.providers.fmp import FMPFundamentalProvider
+from app.providers.fundamentals_chain import build_fundamental_provider_chain
 from app.providers.openai_sentiment import OpenAISentimentProvider
+from app.providers.protocol import SentimentData
 from app.services import fundamental_service, ingestion_service, sentiment_service
 from app.services.rr_scanner_service import scan_all_tickers
+from app.services.ticker_universe_service import bootstrap_universe
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +49,64 @@ _last_successful: dict[str, str | None] = {
     "fundamental_collector": None,
 }
 
+_job_runtime: dict[str, dict[str, object]] = {
+    "data_collector": {
+        "running": False,
+        "status": "idle",
+        "processed": 0,
+        "total": None,
+        "progress_pct": None,
+        "current_ticker": None,
+        "started_at": None,
+        "finished_at": None,
+        "message": None,
+    },
+    "sentiment_collector": {
+        "running": False,
+        "status": "idle",
+        "processed": 0,
+        "total": None,
+        "progress_pct": None,
+        "current_ticker": None,
+        "started_at": None,
+        "finished_at": None,
+        "message": None,
+    },
+    "fundamental_collector": {
+        "running": False,
+        "status": "idle",
+        "processed": 0,
+        "total": None,
+        "progress_pct": None,
+        "current_ticker": None,
+        "started_at": None,
+        "finished_at": None,
+        "message": None,
+    },
+    "rr_scanner": {
+        "running": False,
+        "status": "idle",
+        "processed": 0,
+        "total": None,
+        "progress_pct": None,
+        "current_ticker": None,
+        "started_at": None,
+        "finished_at": None,
+        "message": None,
+    },
+    "ticker_universe_sync": {
+        "running": False,
+        "status": "idle",
+        "processed": 0,
+        "total": None,
+        "progress_pct": None,
+        "current_ticker": None,
+        "started_at": None,
+        "finished_at": None,
+        "message": None,
+    },
+}
+
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -62,6 +126,71 @@ def _log_job_error(job_name: str, ticker: str, error: Exception) -> None:
     )
 
 
+def _runtime_start(job_name: str, total: int | None = None, message: str | None = None) -> None:
+    now = datetime.now(timezone.utc).isoformat()
+    _job_runtime[job_name] = {
+        "running": True,
+        "status": "running",
+        "processed": 0,
+        "total": total,
+        "progress_pct": 0.0 if total and total > 0 else None,
+        "current_ticker": None,
+        "started_at": now,
+        "finished_at": None,
+        "message": message,
+    }
+
+
+def _runtime_progress(
+    job_name: str,
+    processed: int,
+    total: int | None,
+    current_ticker: str | None = None,
+    message: str | None = None,
+) -> None:
+    progress_pct: float | None = None
+    if total and total > 0:
+        progress_pct = round((processed / total) * 100.0, 1)
+    runtime = _job_runtime.get(job_name, {})
+    runtime.update({
+        "running": True,
+        "status": "running",
+        "processed": processed,
+        "total": total,
+        "progress_pct": progress_pct,
+        "current_ticker": current_ticker,
+        "message": message,
+    })
+    _job_runtime[job_name] = runtime
+
+
+def _runtime_finish(
+    job_name: str,
+    status: str,
+    processed: int,
+    total: int | None,
+    message: str | None = None,
+) -> None:
+    runtime = _job_runtime.get(job_name, {})
+    runtime.update({
+        "running": False,
+        "status": status,
+        "processed": processed,
+        "total": total,
+        "progress_pct": 100.0 if total and processed >= total else runtime.get("progress_pct"),
+        "current_ticker": None,
+        "finished_at": datetime.now(timezone.utc).isoformat(),
+        "message": message,
+    })
+    _job_runtime[job_name] = runtime
+
+
+def get_job_runtime_snapshot(job_name: str | None = None) -> dict[str, dict[str, object]] | dict[str, object]:
+    if job_name is not None:
+        return dict(_job_runtime.get(job_name, {}))
+    return {name: dict(meta) for name, meta in _job_runtime.items()}
+
+
 async def _is_job_enabled(db: AsyncSession, job_name: str) -> bool:
     """Check SystemSetting for job enabled state. Defaults to True."""
     key = f"job_{job_name}_enabled"
@@ -80,6 +209,61 @@ async def _get_all_tickers(db: AsyncSession) -> list[str]:
     return list(result.scalars().all())
 
 
+async def _get_ohlcv_priority_tickers(db: AsyncSession) -> list[str]:
+    """Return symbols prioritized for OHLCV collection.
+
+    Priority:
+      1) Tickers with no OHLCV bars
+      2) Tickers with data, oldest latest OHLCV date first
+      3) Alphabetical tiebreaker
+    """
+    latest_date = func.max(OHLCVRecord.date)
+    missing_first = case((latest_date.is_(None), 0), else_=1)
+    result = await db.execute(
+        select(Ticker.symbol)
+        .outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id)
+        .group_by(Ticker.id, Ticker.symbol)
+        .order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc())
+    )
+    return list(result.scalars().all())
+
+
+async def _get_sentiment_priority_tickers(db: AsyncSession) -> list[str]:
+    """Return symbols prioritized for sentiment collection.
+
+    Priority:
+      1) Tickers with no sentiment records
+      2) Tickers with records, oldest latest sentiment timestamp first
+      3) Alphabetical tiebreaker
+    """
+    latest_ts = func.max(SentimentScore.timestamp)
+    missing_first = case((latest_ts.is_(None), 0), else_=1)
+    result = await db.execute(
+        select(Ticker.symbol)
+        .outerjoin(SentimentScore, SentimentScore.ticker_id == Ticker.id)
+        .group_by(Ticker.id, Ticker.symbol)
+        .order_by(missing_first.asc(), latest_ts.asc(), Ticker.symbol.asc())
+    )
+    return list(result.scalars().all())
+
+
+async def _get_fundamental_priority_tickers(db: AsyncSession) -> list[str]:
+    """Return symbols prioritized for fundamentals refresh.
+
+    Priority:
+      1) Tickers with no fundamentals snapshot yet
+      2) Tickers with existing fundamentals, oldest fetched_at first
+      3) Alphabetical tiebreaker
+    """
+    missing_first = case((FundamentalData.fetched_at.is_(None), 0), else_=1)
+    result = await db.execute(
+        select(Ticker.symbol)
+        .outerjoin(FundamentalData, FundamentalData.ticker_id == Ticker.id)
+        .order_by(missing_first.asc(), FundamentalData.fetched_at.asc(), Ticker.symbol.asc())
+    )
+    return list(result.scalars().all())
+
+
 def _resume_tickers(symbols: list[str], job_name: str) -> list[str]:
     """Reorder tickers to resume after the last successful one (rate-limit resume).
 
@@ -94,6 +278,11 @@ def _resume_tickers(symbols: list[str], job_name: str) -> list[str]:
     return symbols[idx + 1:] + symbols[:idx + 1]
 
 
+def _chunked(symbols: list[str], chunk_size: int) -> list[list[str]]:
+    size = max(1, chunk_size)
+    return [symbols[i:i + size] for i in range(0, len(symbols), size)]
+
+
 # ---------------------------------------------------------------------------
 # Job: Data Collector (OHLCV)
 # ---------------------------------------------------------------------------
@@ -104,68 +293,84 @@ async def collect_ohlcv() -> None:
 
     Uses AlpacaOHLCVProvider. Processes each ticker independently.
     On rate limit, records last successful ticker for resume.
+        Start date is resolved by ingestion progress:
+            - existing ticker: resume from last_ingested_date + 1
+            - new ticker: backfill ~1 year by default
     """
     job_name = "data_collector"
     logger.info(json.dumps({"event": "job_start", "job": job_name}))
-
-    async with async_session_factory() as db:
-        if not await _is_job_enabled(db, job_name):
-            logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
-            return
-
-        symbols = await _get_all_tickers(db)
-        if not symbols:
-            logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
-            return
-
-    # Reorder for rate-limit resume
-    symbols = _resume_tickers(symbols, job_name)
-
-    # Build provider (skip if keys not configured)
-    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
-        logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "alpaca keys not configured"}))
-        return
+    _runtime_start(job_name)
+    processed = 0
+    total: int | None = None
 
     try:
-        provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
-    except Exception as exc:
-        logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
-        return
-
-    end_date = date.today()
-    start_date = end_date - timedelta(days=5)  # Fetch last 5 days to catch up
-    processed = 0
-
-    for symbol in symbols:
         async with async_session_factory() as db:
-            try:
-                result = await ingestion_service.fetch_and_ingest(
-                    db, provider, symbol, start_date=start_date, end_date=end_date,
-                )
-                _last_successful[job_name] = symbol
-                processed += 1
-                logger.info(json.dumps({
-                    "event": "ticker_collected",
-                    "job": job_name,
-                    "ticker": symbol,
-                    "status": result.status,
-                    "records": result.records_ingested,
-                }))
-                if result.status == "partial":
-                    # Rate limited — stop and resume next run
-                    logger.warning(json.dumps({
-                        "event": "rate_limited",
+            if not await _is_job_enabled(db, job_name):
+                logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
+                _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
+                return
+
+            symbols = await _get_ohlcv_priority_tickers(db)
+            if not symbols:
+                logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
+                _runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers")
+                return
+
+        total = len(symbols)
+        _runtime_progress(job_name, processed=0, total=total)
+
+        # Build provider (skip if keys not configured)
+        if not settings.alpaca_api_key or not settings.alpaca_api_secret:
+            logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "alpaca keys not configured"}))
+            _runtime_finish(job_name, "skipped", processed=0, total=total, message="Alpaca keys not configured")
+            return
+
+        try:
+            provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
+        except Exception as exc:
+            logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+            _runtime_finish(job_name, "error", processed=0, total=total, message=str(exc))
+            return
+
+        end_date = date.today()
+
+        for symbol in symbols:
+            _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+            async with async_session_factory() as db:
+                try:
+                    result = await ingestion_service.fetch_and_ingest(
+                        db, provider, symbol, start_date=None, end_date=end_date,
+                    )
+                    _last_successful[job_name] = symbol
+                    processed += 1
+                    _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+                    logger.info(json.dumps({
+                        "event": "ticker_collected",
                         "job": job_name,
                         "ticker": symbol,
-                        "processed": processed,
+                        "status": result.status,
+                        "records": result.records_ingested,
                     }))
-                    return
-            except Exception as exc:
-                _log_job_error(job_name, symbol, exc)
+                    if result.status == "partial":
+                        # Rate limited — stop and resume next run
+                        logger.warning(json.dumps({
+                            "event": "rate_limited",
+                            "job": job_name,
+                            "ticker": symbol,
+                            "processed": processed,
+                        }))
+                        _runtime_finish(job_name, "rate_limited", processed=processed, total=total, message=f"Rate limited at {symbol}")
+                        return
+                except Exception as exc:
+                    _log_job_error(job_name, symbol, exc)
 
-    # Reset resume pointer on full completion
-    _last_successful[job_name] = None
-    logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+        # Reset resume pointer on full completion
+        _last_successful[job_name] = None
+        logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+        _runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers")
+    except Exception as exc:
+        logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+        _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
 
 
 # ---------------------------------------------------------------------------
@@ -181,68 +386,119 @@ async def collect_sentiment() -> None:
     """
     job_name = "sentiment_collector"
     logger.info(json.dumps({"event": "job_start", "job": job_name}))
-
-    async with async_session_factory() as db:
-        if not await _is_job_enabled(db, job_name):
-            logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
-            return
-
-        symbols = await _get_all_tickers(db)
-        if not symbols:
-            logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
-            return
-
-    symbols = _resume_tickers(symbols, job_name)
-
-    if not settings.openai_api_key:
-        logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "openai key not configured"}))
-        return
+    _runtime_start(job_name)
+    processed = 0
+    total: int | None = None
 
     try:
-        provider = OpenAISentimentProvider(settings.openai_api_key, settings.openai_model)
+        async with async_session_factory() as db:
+            if not await _is_job_enabled(db, job_name):
+                logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
+                _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
+                return
+
+            symbols = await _get_sentiment_priority_tickers(db)
+            if not symbols:
+                logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
+                _runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers")
+                return
+
+        total = len(symbols)
+        _runtime_progress(job_name, processed=0, total=total)
+
+        if not settings.openai_api_key:
+            logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "openai key not configured"}))
+            _runtime_finish(job_name, "skipped", processed=0, total=total, message="OpenAI key not configured")
+            return
+
+        try:
+            provider = OpenAISentimentProvider(settings.openai_api_key, settings.openai_model)
+        except Exception as exc:
+            logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+            _runtime_finish(job_name, "error", processed=0, total=total, message=str(exc))
+            return
+
+        batch_size = max(1, settings.openai_sentiment_batch_size)
+        batches = _chunked(symbols, batch_size)
+
+        for batch in batches:
+            current_hint = batch[0] if len(batch) == 1 else f"{batch[0]} (+{len(batch) - 1})"
+            _runtime_progress(job_name, processed=processed, total=total, current_ticker=current_hint)
+
+            batch_results: dict[str, SentimentData] = {}
+            if len(batch) > 1 and hasattr(provider, "fetch_sentiment_batch"):
+                try:
+                    batch_results = await provider.fetch_sentiment_batch(batch)
+                except Exception as exc:
+                    msg = str(exc).lower()
+                    if "rate" in msg or "quota" in msg or "429" in msg:
+                        logger.warning(json.dumps({
+                            "event": "rate_limited",
+                            "job": job_name,
+                            "ticker": batch[0],
+                            "processed": processed,
+                        }))
+                        _runtime_finish(job_name, "rate_limited", processed=processed, total=total, message=f"Rate limited at {batch[0]}")
+                        return
+                    logger.warning(json.dumps({
+                        "event": "batch_fallback",
+                        "job": job_name,
+                        "batch": batch,
+                        "reason": str(exc),
+                    }))
+
+            for symbol in batch:
+                _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+                data = batch_results.get(symbol) if batch_results else None
+
+                if data is None:
+                    try:
+                        data = await provider.fetch_sentiment(symbol)
+                    except Exception as exc:
+                        msg = str(exc).lower()
+                        if "rate" in msg or "quota" in msg or "429" in msg:
+                            logger.warning(json.dumps({
+                                "event": "rate_limited",
+                                "job": job_name,
+                                "ticker": symbol,
+                                "processed": processed,
+                            }))
+                            _runtime_finish(job_name, "rate_limited", processed=processed, total=total, message=f"Rate limited at {symbol}")
+                            return
+                        _log_job_error(job_name, symbol, exc)
+                        continue
+
+                async with async_session_factory() as db:
+                    try:
+                        await sentiment_service.store_sentiment(
+                            db,
+                            symbol=symbol,
+                            classification=data.classification,
+                            confidence=data.confidence,
+                            source=data.source,
+                            timestamp=data.timestamp,
+                            reasoning=data.reasoning,
+                            citations=data.citations,
+                        )
+                        _last_successful[job_name] = symbol
+                        processed += 1
+                        _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+                        logger.info(json.dumps({
+                            "event": "ticker_collected",
+                            "job": job_name,
+                            "ticker": symbol,
+                            "classification": data.classification,
+                            "confidence": data.confidence,
+                        }))
+                    except Exception as exc:
+                        _log_job_error(job_name, symbol, exc)
+
+        _last_successful[job_name] = None
+        logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+        _runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers")
     except Exception as exc:
         logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
-        return
-
-    processed = 0
-
-    for symbol in symbols:
-        async with async_session_factory() as db:
-            try:
-                data = await provider.fetch_sentiment(symbol)
-                await sentiment_service.store_sentiment(
-                    db,
-                    symbol=symbol,
-                    classification=data.classification,
-                    confidence=data.confidence,
-                    source=data.source,
-                    timestamp=data.timestamp,
-                    reasoning=data.reasoning,
-                    citations=data.citations,
-                )
-                _last_successful[job_name] = symbol
-                processed += 1
-                logger.info(json.dumps({
-                    "event": "ticker_collected",
-                    "job": job_name,
-                    "ticker": symbol,
-                    "classification": data.classification,
-                    "confidence": data.confidence,
-                }))
-            except Exception as exc:
-                msg = str(exc).lower()
-                if "rate" in msg or "quota" in msg or "429" in msg:
-                    logger.warning(json.dumps({
-                        "event": "rate_limited",
-                        "job": job_name,
-                        "ticker": symbol,
-                        "processed": processed,
-                    }))
-                    return
-                _log_job_error(job_name, symbol, exc)
-
-    _last_successful[job_name] = None
-    logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+        _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
 
 
 # ---------------------------------------------------------------------------
@@ -258,65 +514,114 @@ async def collect_fundamentals() -> None:
     """
     job_name = "fundamental_collector"
     logger.info(json.dumps({"event": "job_start", "job": job_name}))
-
-    async with async_session_factory() as db:
-        if not await _is_job_enabled(db, job_name):
-            logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
-            return
-
-        symbols = await _get_all_tickers(db)
-        if not symbols:
-            logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
-            return
-
-    symbols = _resume_tickers(symbols, job_name)
-
-    if not settings.fmp_api_key:
-        logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "fmp key not configured"}))
-        return
+    _runtime_start(job_name)
+    processed = 0
+    total: int | None = None
 
     try:
-        provider = FMPFundamentalProvider(settings.fmp_api_key)
-    except Exception as exc:
-        logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
-        return
-
-    processed = 0
-
-    for symbol in symbols:
         async with async_session_factory() as db:
-            try:
-                data = await provider.fetch_fundamentals(symbol)
-                await fundamental_service.store_fundamental(
-                    db,
-                    symbol=symbol,
-                    pe_ratio=data.pe_ratio,
-                    revenue_growth=data.revenue_growth,
-                    earnings_surprise=data.earnings_surprise,
-                    market_cap=data.market_cap,
-                    unavailable_fields=data.unavailable_fields,
-                )
-                _last_successful[job_name] = symbol
-                processed += 1
-                logger.info(json.dumps({
-                    "event": "ticker_collected",
-                    "job": job_name,
-                    "ticker": symbol,
-                }))
-            except Exception as exc:
-                msg = str(exc).lower()
-                if "rate" in msg or "429" in msg:
-                    logger.warning(json.dumps({
-                        "event": "rate_limited",
+            if not await _is_job_enabled(db, job_name):
+                logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
+                _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
+                return
+
+            symbols = await _get_fundamental_priority_tickers(db)
+            if not symbols:
+                logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": 0}))
+                _runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers")
+                return
+
+        total = len(symbols)
+        _runtime_progress(job_name, processed=0, total=total)
+
+        if not (settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key):
+            logger.warning(json.dumps({"event": "job_skipped", "job": job_name, "reason": "no fundamentals provider keys configured"}))
+            _runtime_finish(job_name, "skipped", processed=0, total=total, message="No fundamentals provider keys configured")
+            return
+
+        try:
+            provider = build_fundamental_provider_chain()
+        except Exception as exc:
+            logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+            _runtime_finish(job_name, "error", processed=0, total=total, message=str(exc))
+            return
+
+        max_retries = max(0, settings.fundamental_rate_limit_retries)
+        base_backoff = max(1, settings.fundamental_rate_limit_backoff_seconds)
+
+        for symbol in symbols:
+            _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+            attempt = 0
+            while True:
+                try:
+                    data = await provider.fetch_fundamentals(symbol)
+                    async with async_session_factory() as db:
+                        await fundamental_service.store_fundamental(
+                            db,
+                            symbol=symbol,
+                            pe_ratio=data.pe_ratio,
+                            revenue_growth=data.revenue_growth,
+                            earnings_surprise=data.earnings_surprise,
+                            market_cap=data.market_cap,
+                            unavailable_fields=data.unavailable_fields,
+                        )
+                    _last_successful[job_name] = symbol
+                    processed += 1
+                    _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
+                    logger.info(json.dumps({
+                        "event": "ticker_collected",
                         "job": job_name,
                         "ticker": symbol,
-                        "processed": processed,
                     }))
-                    return
-                _log_job_error(job_name, symbol, exc)
+                    break
+                except Exception as exc:
+                    msg = str(exc).lower()
+                    if "rate" in msg or "429" in msg:
+                        if attempt < max_retries:
+                            wait_seconds = base_backoff * (2 ** attempt)
+                            attempt += 1
+                            logger.warning(json.dumps({
+                                "event": "rate_limited_retry",
+                                "job": job_name,
+                                "ticker": symbol,
+                                "attempt": attempt,
+                                "max_retries": max_retries,
+                                "wait_seconds": wait_seconds,
+                                "processed": processed,
+                            }))
+                            _runtime_progress(
+                                job_name,
+                                processed=processed,
+                                total=total,
+                                current_ticker=symbol,
+                                message=f"Rate-limited at {symbol}; retry {attempt}/{max_retries} in {wait_seconds}s",
+                            )
+                            await asyncio.sleep(wait_seconds)
+                            continue
 
-    _last_successful[job_name] = None
-    logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+                        logger.warning(json.dumps({
+                            "event": "rate_limited",
+                            "job": job_name,
+                            "ticker": symbol,
+                            "processed": processed,
+                        }))
+                        _runtime_finish(
+                            job_name,
+                            "rate_limited",
+                            processed=processed,
+                            total=total,
+                            message=f"Rate limited at {symbol} after {attempt} retries",
+                        )
+                        return
+                    _log_job_error(job_name, symbol, exc)
+                    break
+
+        _last_successful[job_name] = None
+        logger.info(json.dumps({"event": "job_complete", "job": job_name, "tickers": processed}))
+        _runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers")
+    except Exception as exc:
+        logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+        _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
 
 
 # ---------------------------------------------------------------------------
@@ -332,28 +637,90 @@ async def scan_rr() -> None:
     """
     job_name = "rr_scanner"
     logger.info(json.dumps({"event": "job_start", "job": job_name}))
+    _runtime_start(job_name)
+    processed = 0
+    total: int | None = None
 
-    async with async_session_factory() as db:
-        if not await _is_job_enabled(db, job_name):
-            logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
-            return
+    try:
+        async with async_session_factory() as db:
+            if not await _is_job_enabled(db, job_name):
+                logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
+                _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
+                return
 
-        try:
-            setups = await scan_all_tickers(
-                db, rr_threshold=settings.default_rr_threshold,
+            symbols = await _get_all_tickers(db)
+            total = len(symbols)
+            _runtime_progress(job_name, processed=0, total=total)
+
+            try:
+                setups = await scan_all_tickers(
+                    db, rr_threshold=settings.default_rr_threshold,
+                )
+                processed = total or 0
+                _runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Found {len(setups)} setups")
+                logger.info(json.dumps({
+                    "event": "job_complete",
+                    "job": job_name,
+                    "setups_found": len(setups),
+                }))
+            except Exception as exc:
+                _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
+                logger.error(json.dumps({
+                    "event": "job_error",
+                    "job": job_name,
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                }))
+    except Exception as exc:
+        logger.error(json.dumps({"event": "job_error", "job": job_name, "error_type": type(exc).__name__, "message": str(exc)}))
+        _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
+
+
+# ---------------------------------------------------------------------------
+# Job: Ticker Universe Sync
+# ---------------------------------------------------------------------------
+
+
+async def sync_ticker_universe() -> None:
+    """Sync tracked tickers from configured default universe.
+
+    Setting key: ticker_universe_default (sp500 | nasdaq100 | nasdaq_all)
+    """
+    job_name = "ticker_universe_sync"
+    logger.info(json.dumps({"event": "job_start", "job": job_name}))
+    _runtime_start(job_name, total=1)
+
+    try:
+        async with async_session_factory() as db:
+            if not await _is_job_enabled(db, job_name):
+                logger.info(json.dumps({"event": "job_skipped", "job": job_name, "reason": "disabled"}))
+                _runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
+                return
+
+            result = await db.execute(
+                select(SystemSetting).where(SystemSetting.key == "ticker_universe_default")
             )
-            logger.info(json.dumps({
-                "event": "job_complete",
-                "job": job_name,
-                "setups_found": len(setups),
-            }))
-        except Exception as exc:
-            logger.error(json.dumps({
-                "event": "job_error",
-                "job": job_name,
-                "error_type": type(exc).__name__,
-                "message": str(exc),
-            }))
+            setting = result.scalar_one_or_none()
+            universe = (setting.value if setting else "sp500").strip().lower()
+
+        async with async_session_factory() as db:
+            summary = await bootstrap_universe(db, universe, prune_missing=False)
+        _runtime_progress(job_name, processed=1, total=1)
+        _runtime_finish(job_name, "completed", processed=1, total=1, message=f"Synced {universe}")
+        logger.info(json.dumps({
+            "event": "job_complete",
+            "job": job_name,
+            "universe": universe,
+            "summary": summary,
+        }))
+    except Exception as exc:
+        _runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
+        logger.error(json.dumps({
+            "event": "job_error",
+            "job": job_name,
+            "error_type": type(exc).__name__,
+            "message": str(exc),
+        }))
 
 
 # ---------------------------------------------------------------------------
@@ -427,6 +794,16 @@ def configure_scheduler() -> None:
         replace_existing=True,
     )
 
+    # Universe Sync — nightly
+    scheduler.add_job(
+        sync_ticker_universe,
+        "interval",
+        hours=24,
+        id="ticker_universe_sync",
+        name="Ticker Universe Sync",
+        replace_existing=True,
+    )
+
     logger.info(
         json.dumps({
             "event": "scheduler_configured",
@@ -435,6 +812,7 @@ def configure_scheduler() -> None:
                 "sentiment_collector": {"minutes": settings.sentiment_poll_interval_minutes},
                 "fundamental_collector": fund_interval,
                 "rr_scanner": rr_interval,
+                "ticker_universe_sync": {"hours": 24},
             },
         })
     )

app/schemas/admin.py

@@ -1,5 +1,7 @@
 """Admin request/response schemas."""
 
+from typing import Literal
+
 from pydantic import BaseModel, Field
 
 
@@ -39,3 +41,18 @@ class DataCleanupRequest(BaseModel):
 class JobToggle(BaseModel):
     """Schema for enabling/disabling a scheduled job."""
     enabled: bool
+
+
+class RecommendationConfigUpdate(BaseModel):
+    high_confidence_threshold: float | None = Field(default=None, ge=0, le=100)
+    moderate_confidence_threshold: float | None = Field(default=None, ge=0, le=100)
+    confidence_diff_threshold: float | None = Field(default=None, ge=0, le=100)
+    signal_alignment_weight: float | None = Field(default=None, ge=0, le=1)
+    sr_strength_weight: float | None = Field(default=None, ge=0, le=1)
+    distance_penalty_factor: float | None = Field(default=None, ge=0, le=1)
+    momentum_technical_divergence_threshold: float | None = Field(default=None, ge=0, le=100)
+    fundamental_technical_divergence_threshold: float | None = Field(default=None, ge=0, le=100)
+
+
+class TickerUniverseUpdate(BaseModel):
+    universe: Literal["sp500", "nasdaq100", "nasdaq_all"]

app/schemas/trade_setup.py

@@ -4,7 +4,25 @@ from __future__ import annotations
 
 from datetime import datetime
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+
+
+class TradeTargetResponse(BaseModel):
+    price: float
+    distance_from_entry: float
+    distance_atr_multiple: float
+    rr_ratio: float
+    probability: float
+    classification: str
+    sr_level_id: int
+    sr_strength: float
+
+
+class RecommendationSummaryResponse(BaseModel):
+    action: str
+    reasoning: str | None
+    risk_level: str | None
+    composite_score: float
 
 
 class TradeSetupResponse(BaseModel):
@@ -19,3 +37,11 @@ class TradeSetupResponse(BaseModel):
     rr_ratio: float
     composite_score: float
     detected_at: datetime
+    confidence_score: float | None = None
+    targets: list[TradeTargetResponse] = Field(default_factory=list)
+    conflict_flags: list[str] = Field(default_factory=list)
+    recommended_action: str | None = None
+    reasoning: str | None = None
+    risk_level: str | None = None
+    actual_outcome: str | None = None
+    recommendation_summary: RecommendationSummaryResponse | None = None

app/services/admin_service.py

@@ -3,16 +3,34 @@
 from datetime import datetime, timedelta, timezone
 
 from passlib.hash import bcrypt
-from sqlalchemy import delete, select
+from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.exceptions import DuplicateError, NotFoundError, ValidationError
 from app.models.fundamental import FundamentalData
 from app.models.ohlcv import OHLCVRecord
+from app.models.score import CompositeScore, DimensionScore
 from app.models.sentiment import SentimentScore
+from app.models.sr_level import SRLevel
 from app.models.settings import SystemSetting
+from app.models.ticker import Ticker
+from app.models.trade_setup import TradeSetup
 from app.models.user import User
 
+RECOMMENDATION_CONFIG_DEFAULTS: dict[str, float] = {
+    "recommendation_high_confidence_threshold": 70.0,
+    "recommendation_moderate_confidence_threshold": 50.0,
+    "recommendation_confidence_diff_threshold": 20.0,
+    "recommendation_signal_alignment_weight": 0.15,
+    "recommendation_sr_strength_weight": 0.20,
+    "recommendation_distance_penalty_factor": 0.10,
+    "recommendation_momentum_technical_divergence_threshold": 30.0,
+    "recommendation_fundamental_technical_divergence_threshold": 40.0,
+}
+
+DEFAULT_TICKER_UNIVERSE = "sp500"
+SUPPORTED_TICKER_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
+
 
 # ---------------------------------------------------------------------------
 # User management
@@ -125,6 +143,67 @@ async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSettin
     return setting
 
 
+def _recommendation_public_to_storage_key(key: str) -> str:
+    return f"recommendation_{key}"
+
+
+async def get_recommendation_config(db: AsyncSession) -> dict[str, float]:
+    result = await db.execute(
+        select(SystemSetting).where(SystemSetting.key.like("recommendation_%"))
+    )
+    rows = result.scalars().all()
+
+    config = dict(RECOMMENDATION_CONFIG_DEFAULTS)
+    for row in rows:
+        try:
+            config[row.key] = float(row.value)
+        except (TypeError, ValueError):
+            continue
+
+    return {
+        "high_confidence_threshold": config["recommendation_high_confidence_threshold"],
+        "moderate_confidence_threshold": config["recommendation_moderate_confidence_threshold"],
+        "confidence_diff_threshold": config["recommendation_confidence_diff_threshold"],
+        "signal_alignment_weight": config["recommendation_signal_alignment_weight"],
+        "sr_strength_weight": config["recommendation_sr_strength_weight"],
+        "distance_penalty_factor": config["recommendation_distance_penalty_factor"],
+        "momentum_technical_divergence_threshold": config["recommendation_momentum_technical_divergence_threshold"],
+        "fundamental_technical_divergence_threshold": config["recommendation_fundamental_technical_divergence_threshold"],
+    }
+
+
+async def update_recommendation_config(
+    db: AsyncSession,
+    payload: dict[str, float],
+) -> dict[str, float]:
+    for public_key, public_value in payload.items():
+        storage_key = _recommendation_public_to_storage_key(public_key)
+        await update_setting(db, storage_key, str(public_value))
+
+    return await get_recommendation_config(db)
+
+
+async def get_ticker_universe_default(db: AsyncSession) -> dict[str, str]:
+    result = await db.execute(
+        select(SystemSetting).where(SystemSetting.key == "ticker_universe_default")
+    )
+    setting = result.scalar_one_or_none()
+    universe = setting.value if setting else DEFAULT_TICKER_UNIVERSE
+    if universe not in SUPPORTED_TICKER_UNIVERSES:
+        universe = DEFAULT_TICKER_UNIVERSE
+    return {"universe": universe}
+
+
+async def update_ticker_universe_default(db: AsyncSession, universe: str) -> dict[str, str]:
+    normalised = universe.strip().lower()
+    if normalised not in SUPPORTED_TICKER_UNIVERSES:
+        supported = ", ".join(sorted(SUPPORTED_TICKER_UNIVERSES))
+        raise ValidationError(f"Unsupported ticker universe '{universe}'. Supported: {supported}")
+
+    await update_setting(db, "ticker_universe_default", normalised)
+    return {"universe": normalised}
+
+
 # ---------------------------------------------------------------------------
 # Data cleanup
 # ---------------------------------------------------------------------------
@@ -160,23 +239,181 @@ async def cleanup_data(db: AsyncSession, older_than_days: int) -> dict[str, int]
     return counts
 
 
+async def get_pipeline_readiness(db: AsyncSession) -> list[dict]:
+    """Return per-ticker readiness snapshot for ingestion/scoring/scanner pipeline."""
+    tickers_result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
+    tickers = list(tickers_result.scalars().all())
+
+    if not tickers:
+        return []
+
+    ticker_ids = [ticker.id for ticker in tickers]
+
+    ohlcv_stats_result = await db.execute(
+        select(
+            OHLCVRecord.ticker_id,
+            func.count(OHLCVRecord.id),
+            func.max(OHLCVRecord.date),
+        )
+        .where(OHLCVRecord.ticker_id.in_(ticker_ids))
+        .group_by(OHLCVRecord.ticker_id)
+    )
+    ohlcv_stats = {
+        ticker_id: {
+            "bars": int(count or 0),
+            "last_date": max_date.isoformat() if max_date else None,
+        }
+        for ticker_id, count, max_date in ohlcv_stats_result.all()
+    }
+
+    dim_rows_result = await db.execute(
+        select(DimensionScore).where(DimensionScore.ticker_id.in_(ticker_ids))
+    )
+    dim_map_by_ticker: dict[int, dict[str, tuple[float | None, bool]]] = {}
+    for row in dim_rows_result.scalars().all():
+        dim_map_by_ticker.setdefault(row.ticker_id, {})[row.dimension] = (row.score, row.is_stale)
+
+    sr_counts_result = await db.execute(
+        select(SRLevel.ticker_id, func.count(SRLevel.id))
+        .where(SRLevel.ticker_id.in_(ticker_ids))
+        .group_by(SRLevel.ticker_id)
+    )
+    sr_counts = {ticker_id: int(count or 0) for ticker_id, count in sr_counts_result.all()}
+
+    sentiment_stats_result = await db.execute(
+        select(
+            SentimentScore.ticker_id,
+            func.count(SentimentScore.id),
+            func.max(SentimentScore.timestamp),
+        )
+        .where(SentimentScore.ticker_id.in_(ticker_ids))
+        .group_by(SentimentScore.ticker_id)
+    )
+    sentiment_stats = {
+        ticker_id: {
+            "count": int(count or 0),
+            "last_at": max_ts.isoformat() if max_ts else None,
+        }
+        for ticker_id, count, max_ts in sentiment_stats_result.all()
+    }
+
+    fundamentals_result = await db.execute(
+        select(FundamentalData.ticker_id, FundamentalData.fetched_at)
+        .where(FundamentalData.ticker_id.in_(ticker_ids))
+    )
+    fundamentals_map = {
+        ticker_id: fetched_at.isoformat() if fetched_at else None
+        for ticker_id, fetched_at in fundamentals_result.all()
+    }
+
+    composites_result = await db.execute(
+        select(CompositeScore.ticker_id, CompositeScore.is_stale)
+        .where(CompositeScore.ticker_id.in_(ticker_ids))
+    )
+    composites_map = {
+        ticker_id: is_stale
+        for ticker_id, is_stale in composites_result.all()
+    }
+
+    setup_counts_result = await db.execute(
+        select(TradeSetup.ticker_id, func.count(TradeSetup.id))
+        .where(TradeSetup.ticker_id.in_(ticker_ids))
+        .group_by(TradeSetup.ticker_id)
+    )
+    setup_counts = {ticker_id: int(count or 0) for ticker_id, count in setup_counts_result.all()}
+
+    readiness: list[dict] = []
+    for ticker in tickers:
+        ohlcv = ohlcv_stats.get(ticker.id, {"bars": 0, "last_date": None})
+        ohlcv_bars = int(ohlcv["bars"])
+        ohlcv_last_date = ohlcv["last_date"]
+
+        dim_map = dim_map_by_ticker.get(ticker.id, {})
+
+        sr_count = int(sr_counts.get(ticker.id, 0))
+
+        sentiment = sentiment_stats.get(ticker.id, {"count": 0, "last_at": None})
+        sentiment_count = int(sentiment["count"])
+        sentiment_last_at = sentiment["last_at"]
+
+        fundamentals_fetched_at = fundamentals_map.get(ticker.id)
+        has_fundamentals = ticker.id in fundamentals_map
+
+        has_composite = ticker.id in composites_map
+        composite_stale = composites_map.get(ticker.id)
+
+        setup_count = int(setup_counts.get(ticker.id, 0))
+
+        missing_reasons: list[str] = []
+        if ohlcv_bars < 30:
+            missing_reasons.append("insufficient_ohlcv_bars(<30)")
+        if "technical" not in dim_map or dim_map["technical"][0] is None:
+            missing_reasons.append("missing_technical")
+        if "momentum" not in dim_map or dim_map["momentum"][0] is None:
+            missing_reasons.append("missing_momentum")
+        if "sr_quality" not in dim_map or dim_map["sr_quality"][0] is None:
+            missing_reasons.append("missing_sr_quality")
+        if sentiment_count == 0:
+            missing_reasons.append("missing_sentiment")
+        if not has_fundamentals:
+            missing_reasons.append("missing_fundamentals")
+        if not has_composite:
+            missing_reasons.append("missing_composite")
+        if setup_count == 0:
+            missing_reasons.append("missing_trade_setup")
+
+        readiness.append(
+            {
+                "symbol": ticker.symbol,
+                "ohlcv_bars": ohlcv_bars,
+                "ohlcv_last_date": ohlcv_last_date,
+                "dimensions": {
+                    "technical": dim_map.get("technical", (None, True))[0],
+                    "sr_quality": dim_map.get("sr_quality", (None, True))[0],
+                    "sentiment": dim_map.get("sentiment", (None, True))[0],
+                    "fundamental": dim_map.get("fundamental", (None, True))[0],
+                    "momentum": dim_map.get("momentum", (None, True))[0],
+                },
+                "sentiment_count": sentiment_count,
+                "sentiment_last_at": sentiment_last_at,
+                "has_fundamentals": has_fundamentals,
+                "fundamentals_fetched_at": fundamentals_fetched_at,
+                "sr_level_count": sr_count,
+                "has_composite": has_composite,
+                "composite_stale": composite_stale,
+                "trade_setup_count": setup_count,
+                "missing_reasons": missing_reasons,
+                "ready_for_scanner": ohlcv_bars >= 15 and sr_count > 0,
+            }
+        )
+
+    return readiness
+
+
 # ---------------------------------------------------------------------------
 # Job control (placeholder — scheduler is Task 12.1)
 # ---------------------------------------------------------------------------
 
-VALID_JOB_NAMES = {"data_collector", "sentiment_collector", "fundamental_collector", "rr_scanner"}
+VALID_JOB_NAMES = {
+    "data_collector",
+    "sentiment_collector",
+    "fundamental_collector",
+    "rr_scanner",
+    "ticker_universe_sync",
+}
 
 JOB_LABELS = {
     "data_collector": "Data Collector (OHLCV)",
     "sentiment_collector": "Sentiment Collector",
     "fundamental_collector": "Fundamental Collector",
     "rr_scanner": "R:R Scanner",
+    "ticker_universe_sync": "Ticker Universe Sync",
 }
 
 
 async def list_jobs(db: AsyncSession) -> list[dict]:
     """Return status of all scheduled jobs."""
-    from app.scheduler import scheduler
+    from app.scheduler import get_job_runtime_snapshot, scheduler
 
     jobs_out = []
     for name in sorted(VALID_JOB_NAMES):
@@ -194,12 +431,23 @@ async def list_jobs(db: AsyncSession) -> list[dict]:
         if job and job.next_run_time:
             next_run = job.next_run_time.isoformat()
 
+        runtime = get_job_runtime_snapshot(name)
+
         jobs_out.append({
             "name": name,
             "label": JOB_LABELS.get(name, name),
             "enabled": enabled,
             "next_run_at": next_run,
             "registered": job is not None,
+            "running": bool(runtime.get("running", False)),
+            "runtime_status": runtime.get("status"),
+            "runtime_processed": runtime.get("processed"),
+            "runtime_total": runtime.get("total"),
+            "runtime_progress_pct": runtime.get("progress_pct"),
+            "runtime_current_ticker": runtime.get("current_ticker"),
+            "runtime_started_at": runtime.get("started_at"),
+            "runtime_finished_at": runtime.get("finished_at"),
+            "runtime_message": runtime.get("message"),
         })
 
     return jobs_out
@@ -213,7 +461,26 @@ async def trigger_job(db: AsyncSession, job_name: str) -> dict[str, str]:
     if job_name not in VALID_JOB_NAMES:
         raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
 
-    from app.scheduler import scheduler
+    from app.scheduler import get_job_runtime_snapshot, scheduler
+
+    runtime_target = get_job_runtime_snapshot(job_name)
+    if runtime_target.get("running"):
+        return {
+            "job": job_name,
+            "status": "busy",
+            "message": f"Job '{job_name}' is already running",
+        }
+
+    all_runtime = get_job_runtime_snapshot()
+    for running_name, runtime in all_runtime.items():
+        if running_name == job_name:
+            continue
+        if runtime.get("running"):
+            return {
+                "job": job_name,
+                "status": "blocked",
+                "message": f"Cannot trigger '{job_name}' while '{running_name}' is running",
+            }
 
     job = scheduler.get_job(job_name)
     if job is None:

app/services/ingestion_service.py

@@ -9,10 +9,11 @@ import logging
 from dataclasses import dataclass
 from datetime import date, timedelta
 
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.exceptions import NotFoundError, ProviderError, RateLimitError
+from app.models.ohlcv import OHLCVRecord
 from app.models.settings import IngestionProgress
 from app.models.ticker import Ticker
 from app.providers.protocol import MarketDataProvider
@@ -50,6 +51,13 @@ async def _get_progress(db: AsyncSession, ticker_id: int) -> IngestionProgress |
     return result.scalar_one_or_none()
 
 
+async def _get_ohlcv_bar_count(db: AsyncSession, ticker_id: int) -> int:
+    result = await db.execute(
+        select(func.count()).select_from(OHLCVRecord).where(OHLCVRecord.ticker_id == ticker_id)
+    )
+    return int(result.scalar() or 0)
+
+
 async def _update_progress(
     db: AsyncSession, ticker_id: int, last_date: date
 ) -> None:
@@ -84,10 +92,17 @@ async def fetch_and_ingest(
     if end_date is None:
         end_date = date.today()
 
-    # Resolve start_date: use progress resume or default to 1 year ago
+    # Resolve start_date: use progress resume or default to 1 year ago.
+    # If we have too little history, force a one-year backfill even if
+    # ingestion progress exists (upsert makes this safe and idempotent).
     if start_date is None:
         progress = await _get_progress(db, ticker.id)
-        if progress is not None:
+        bar_count = await _get_ohlcv_bar_count(db, ticker.id)
+        minimum_backfill_bars = 200
+
+        if bar_count < minimum_backfill_bars:
+            start_date = end_date - timedelta(days=365)
+        elif progress is not None:
             start_date = progress.last_ingested_date + timedelta(days=1)
         else:
             start_date = end_date - timedelta(days=365)

app/services/recommendation_service.py

@@ -0,0 +1,499 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.settings import SystemSetting
+from app.models.sr_level import SRLevel
+from app.models.ticker import Ticker
+from app.models.trade_setup import TradeSetup
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_RECOMMENDATION_CONFIG: dict[str, float] = {
+    "recommendation_high_confidence_threshold": 70.0,
+    "recommendation_moderate_confidence_threshold": 50.0,
+    "recommendation_confidence_diff_threshold": 20.0,
+    "recommendation_signal_alignment_weight": 0.15,
+    "recommendation_sr_strength_weight": 0.20,
+    "recommendation_distance_penalty_factor": 0.10,
+    "recommendation_momentum_technical_divergence_threshold": 30.0,
+    "recommendation_fundamental_technical_divergence_threshold": 40.0,
+}
+
+
+def _clamp(value: float, low: float, high: float) -> float:
+    return max(low, min(high, value))
+
+
+def _sentiment_value(sentiment_classification: str | None) -> str | None:
+    if sentiment_classification is None:
+        return None
+    return sentiment_classification.strip().lower()
+
+
+def check_signal_alignment(
+    direction: str,
+    dimension_scores: dict[str, float],
+    sentiment_classification: str | None,
+) -> tuple[bool, str]:
+    technical = float(dimension_scores.get("technical", 50.0))
+    momentum = float(dimension_scores.get("momentum", 50.0))
+    sentiment = _sentiment_value(sentiment_classification)
+
+    if direction == "long":
+        aligned_count = sum([
+            technical > 60,
+            momentum > 60,
+            sentiment == "bullish",
+        ])
+        if aligned_count >= 2:
+            return True, "Technical, momentum, and/or sentiment align with LONG direction."
+        return False, "Signals are mixed for LONG direction."
+
+    aligned_count = sum([
+        technical < 40,
+        momentum < 40,
+        sentiment == "bearish",
+    ])
+    if aligned_count >= 2:
+        return True, "Technical, momentum, and/or sentiment align with SHORT direction."
+    return False, "Signals are mixed for SHORT direction."
+
+
+class SignalConflictDetector:
+    def detect_conflicts(
+        self,
+        dimension_scores: dict[str, float],
+        sentiment_classification: str | None,
+        config: dict[str, float] | None = None,
+    ) -> list[str]:
+        cfg = config or DEFAULT_RECOMMENDATION_CONFIG
+        technical = float(dimension_scores.get("technical", 50.0))
+        momentum = float(dimension_scores.get("momentum", 50.0))
+        fundamental = float(dimension_scores.get("fundamental", 50.0))
+        sentiment = _sentiment_value(sentiment_classification)
+
+        mt_threshold = float(cfg.get("recommendation_momentum_technical_divergence_threshold", 30.0))
+        ft_threshold = float(cfg.get("recommendation_fundamental_technical_divergence_threshold", 40.0))
+
+        conflicts: list[str] = []
+
+        if sentiment == "bearish" and technical > 60:
+            conflicts.append(
+                f"sentiment-technical: Bearish sentiment conflicts with bullish technical ({technical:.0f})"
+            )
+        if sentiment == "bullish" and technical < 40:
+            conflicts.append(
+                f"sentiment-technical: Bullish sentiment conflicts with bearish technical ({technical:.0f})"
+            )
+
+        mt_diff = abs(momentum - technical)
+        if mt_diff > mt_threshold:
+            conflicts.append(
+                "momentum-technical: "
+                f"Momentum ({momentum:.0f}) diverges from technical ({technical:.0f}) by {mt_diff:.0f} points"
+            )
+
+        if sentiment == "bearish" and momentum > 60:
+            conflicts.append(
+                f"sentiment-momentum: Bearish sentiment conflicts with momentum ({momentum:.0f})"
+            )
+        if sentiment == "bullish" and momentum < 40:
+            conflicts.append(
+                f"sentiment-momentum: Bullish sentiment conflicts with momentum ({momentum:.0f})"
+            )
+
+        ft_diff = abs(fundamental - technical)
+        if ft_diff > ft_threshold:
+            conflicts.append(
+                "fundamental-technical: "
+                f"Fundamental ({fundamental:.0f}) diverges significantly from technical ({technical:.0f})"
+            )
+
+        return conflicts
+
+
+class DirectionAnalyzer:
+    def calculate_confidence(
+        self,
+        direction: str,
+        dimension_scores: dict[str, float],
+        sentiment_classification: str | None,
+        conflicts: list[str] | None = None,
+    ) -> float:
+        confidence = 50.0
+        technical = float(dimension_scores.get("technical", 50.0))
+        momentum = float(dimension_scores.get("momentum", 50.0))
+        fundamental = float(dimension_scores.get("fundamental", 50.0))
+        sentiment = _sentiment_value(sentiment_classification)
+
+        if direction == "long":
+            if technical > 70:
+                confidence += 25.0
+            elif technical > 60:
+                confidence += 15.0
+
+            if momentum > 70:
+                confidence += 20.0
+            elif momentum > 60:
+                confidence += 15.0
+
+            if sentiment == "bullish":
+                confidence += 15.0
+            elif sentiment == "neutral":
+                confidence += 5.0
+
+            if fundamental > 60:
+                confidence += 10.0
+        else:
+            if technical < 30:
+                confidence += 25.0
+            elif technical < 40:
+                confidence += 15.0
+
+            if momentum < 30:
+                confidence += 20.0
+            elif momentum < 40:
+                confidence += 15.0
+
+            if sentiment == "bearish":
+                confidence += 15.0
+            elif sentiment == "neutral":
+                confidence += 5.0
+
+            if fundamental < 40:
+                confidence += 10.0
+
+        for conflict in conflicts or []:
+            if "sentiment-technical" in conflict:
+                confidence -= 20.0
+            elif "momentum-technical" in conflict:
+                confidence -= 15.0
+            elif "sentiment-momentum" in conflict:
+                confidence -= 20.0
+            elif "fundamental-technical" in conflict:
+                confidence -= 10.0
+
+        return _clamp(confidence, 0.0, 100.0)
+
+
+class TargetGenerator:
+    def generate_targets(
+        self,
+        direction: str,
+        entry_price: float,
+        stop_loss: float,
+        sr_levels: list[SRLevel],
+        atr_value: float,
+    ) -> list[dict[str, Any]]:
+        if atr_value <= 0:
+            return []
+
+        risk = abs(entry_price - stop_loss)
+        if risk <= 0:
+            return []
+
+        candidates: list[dict[str, Any]] = []
+        atr_pct = atr_value / entry_price if entry_price > 0 else 0.0
+
+        max_atr_multiple: float | None = None
+        if atr_pct > 0.05:
+            max_atr_multiple = 10.0
+        elif atr_pct < 0.02:
+            max_atr_multiple = 3.0
+
+        for level in sr_levels:
+            is_candidate = False
+            if direction == "long":
+                is_candidate = level.type == "resistance" and level.price_level > entry_price
+            else:
+                is_candidate = level.type == "support" and level.price_level < entry_price
+
+            if not is_candidate:
+                continue
+
+            distance = abs(level.price_level - entry_price)
+            distance_atr_multiple = distance / atr_value
+
+            if distance_atr_multiple < 1.0:
+                continue
+            if max_atr_multiple is not None and distance_atr_multiple > max_atr_multiple:
+                continue
+
+            reward = abs(level.price_level - entry_price)
+            rr_ratio = reward / risk
+
+            norm_rr = min(rr_ratio / 10.0, 1.0)
+            norm_strength = _clamp(level.strength, 0, 100) / 100.0
+            norm_proximity = 1.0 - min(distance / entry_price, 1.0)
+            quality = 0.35 * norm_rr + 0.35 * norm_strength + 0.30 * norm_proximity
+
+            candidates.append(
+                {
+                    "price": float(level.price_level),
+                    "distance_from_entry": float(distance),
+                    "distance_atr_multiple": float(distance_atr_multiple),
+                    "rr_ratio": float(rr_ratio),
+                    "classification": "Moderate",
+                    "sr_level_id": int(level.id),
+                    "sr_strength": float(level.strength),
+                    "quality": float(quality),
+                }
+            )
+
+        candidates.sort(key=lambda row: row["quality"], reverse=True)
+        selected = candidates[:5]
+        selected.sort(key=lambda row: row["distance_from_entry"])
+
+        if not selected:
+            return []
+
+        n = len(selected)
+        for idx, target in enumerate(selected):
+            if n <= 2:
+                target["classification"] = "Conservative" if idx == 0 else "Aggressive"
+            elif idx <= 1:
+                target["classification"] = "Conservative"
+            elif idx >= n - 2:
+                target["classification"] = "Aggressive"
+            else:
+                target["classification"] = "Moderate"
+
+            target.pop("quality", None)
+
+        return selected
+
+
+class ProbabilityEstimator:
+    def estimate_probability(
+        self,
+        target: dict[str, Any],
+        dimension_scores: dict[str, float],
+        sentiment_classification: str | None,
+        direction: str,
+        config: dict[str, float],
+    ) -> float:
+        classification = str(target.get("classification", "Moderate"))
+        strength = float(target.get("sr_strength", 50.0))
+        atr_multiple = float(target.get("distance_atr_multiple", 1.0))
+
+        if classification == "Conservative":
+            base_prob = 70.0
+        elif classification == "Aggressive":
+            base_prob = 40.0
+        else:
+            base_prob = 55.0
+
+        if strength >= 80:
+            strength_adj = 15.0
+        elif strength >= 60:
+            strength_adj = 10.0
+        elif strength >= 40:
+            strength_adj = 5.0
+        else:
+            strength_adj = -10.0
+
+        technical = float(dimension_scores.get("technical", 50.0))
+        momentum = float(dimension_scores.get("momentum", 50.0))
+        sentiment = _sentiment_value(sentiment_classification)
+
+        alignment_adj = 0.0
+        if direction == "long":
+            if technical > 60 and (sentiment == "bullish" or momentum > 60):
+                alignment_adj = 15.0
+            elif technical < 40 or (sentiment == "bearish" and momentum < 40):
+                alignment_adj = -15.0
+        else:
+            if technical < 40 and (sentiment == "bearish" or momentum < 40):
+                alignment_adj = 15.0
+            elif technical > 60 or (sentiment == "bullish" and momentum > 60):
+                alignment_adj = -15.0
+
+        volatility_adj = 0.0
+        if atr_multiple > 5:
+            volatility_adj = 5.0
+        elif atr_multiple < 2:
+            volatility_adj = 5.0
+
+        signal_weight = float(config.get("recommendation_signal_alignment_weight", 0.15))
+        sr_weight = float(config.get("recommendation_sr_strength_weight", 0.20))
+        distance_penalty = float(config.get("recommendation_distance_penalty_factor", 0.10))
+
+        scaled_alignment_adj = alignment_adj * (signal_weight / 0.15)
+        scaled_strength_adj = strength_adj * (sr_weight / 0.20)
+        distance_adj = -distance_penalty * max(atr_multiple - 1.0, 0.0) * 2.0
+
+        probability = base_prob + scaled_strength_adj + scaled_alignment_adj + volatility_adj + distance_adj
+        probability = _clamp(probability, 10.0, 90.0)
+
+        if classification == "Conservative":
+            probability = max(probability, 61.0)
+        elif classification == "Moderate":
+            probability = _clamp(probability, 40.0, 70.0)
+        elif classification == "Aggressive":
+            probability = min(probability, 49.0)
+
+        return round(probability, 2)
+
+
+signal_conflict_detector = SignalConflictDetector()
+direction_analyzer = DirectionAnalyzer()
+target_generator = TargetGenerator()
+probability_estimator = ProbabilityEstimator()
+
+
+async def get_recommendation_config(db: AsyncSession) -> dict[str, float]:
+    result = await db.execute(
+        select(SystemSetting).where(SystemSetting.key.like("recommendation_%"))
+    )
+    rows = result.scalars().all()
+
+    config: dict[str, float] = dict(DEFAULT_RECOMMENDATION_CONFIG)
+    for setting in rows:
+        try:
+            config[setting.key] = float(setting.value)
+        except (TypeError, ValueError):
+            logger.warning("Invalid recommendation setting value for %s: %s", setting.key, setting.value)
+
+    return config
+
+
+def _risk_level_from_conflicts(conflicts: list[str]) -> str:
+    if not conflicts:
+        return "Low"
+    severe = [c for c in conflicts if "sentiment-technical" in c or "sentiment-momentum" in c]
+    if len(severe) >= 2 or len(conflicts) >= 3:
+        return "High"
+    return "Medium"
+
+
+def _choose_recommended_action(
+    long_confidence: float,
+    short_confidence: float,
+    config: dict[str, float],
+) -> str:
+    high = float(config.get("recommendation_high_confidence_threshold", 70.0))
+    moderate = float(config.get("recommendation_moderate_confidence_threshold", 50.0))
+    diff = float(config.get("recommendation_confidence_diff_threshold", 20.0))
+
+    if long_confidence >= high and (long_confidence - short_confidence) >= diff:
+        return "LONG_HIGH"
+    if short_confidence >= high and (short_confidence - long_confidence) >= diff:
+        return "SHORT_HIGH"
+    if long_confidence >= moderate and (long_confidence - short_confidence) >= diff:
+        return "LONG_MODERATE"
+    if short_confidence >= moderate and (short_confidence - long_confidence) >= diff:
+        return "SHORT_MODERATE"
+    return "NEUTRAL"
+
+
+def _build_reasoning(
+    direction: str,
+    confidence: float,
+    conflicts: list[str],
+    dimension_scores: dict[str, float],
+    sentiment_classification: str | None,
+    action: str,
+) -> str:
+    aligned, alignment_text = check_signal_alignment(
+        direction,
+        dimension_scores,
+        sentiment_classification,
+    )
+    sentiment = _sentiment_value(sentiment_classification) or "unknown"
+    technical = float(dimension_scores.get("technical", 50.0))
+    momentum = float(dimension_scores.get("momentum", 50.0))
+
+    direction_text = direction.upper()
+    alignment_summary = "aligned" if aligned else "mixed"
+    base = (
+        f"{direction_text} confidence {confidence:.1f}% with {alignment_summary} signals "
+        f"(technical={technical:.0f}, momentum={momentum:.0f}, sentiment={sentiment})."
+    )
+
+    if conflicts:
+        return (
+            f"{base} {alignment_text} Detected {len(conflicts)} conflict(s), "
+            f"so recommendation is risk-adjusted. Action={action}."
+        )
+
+    return f"{base} {alignment_text} No major conflicts detected. Action={action}."
+
+
+async def enhance_trade_setup(
+    db: AsyncSession,
+    ticker: Ticker,
+    setup: TradeSetup,
+    dimension_scores: dict[str, float],
+    sr_levels: list[SRLevel],
+    sentiment_classification: str | None,
+    atr_value: float,
+) -> TradeSetup:
+    config = await get_recommendation_config(db)
+
+    conflicts = signal_conflict_detector.detect_conflicts(
+        dimension_scores=dimension_scores,
+        sentiment_classification=sentiment_classification,
+        config=config,
+    )
+
+    long_confidence = direction_analyzer.calculate_confidence(
+        direction="long",
+        dimension_scores=dimension_scores,
+        sentiment_classification=sentiment_classification,
+        conflicts=conflicts,
+    )
+    short_confidence = direction_analyzer.calculate_confidence(
+        direction="short",
+        dimension_scores=dimension_scores,
+        sentiment_classification=sentiment_classification,
+        conflicts=conflicts,
+    )
+
+    direction = setup.direction.lower()
+    confidence = long_confidence if direction == "long" else short_confidence
+
+    targets = target_generator.generate_targets(
+        direction=direction,
+        entry_price=setup.entry_price,
+        stop_loss=setup.stop_loss,
+        sr_levels=sr_levels,
+        atr_value=atr_value,
+    )
+
+    for target in targets:
+        target["probability"] = probability_estimator.estimate_probability(
+            target=target,
+            dimension_scores=dimension_scores,
+            sentiment_classification=sentiment_classification,
+            direction=direction,
+            config=config,
+        )
+
+    if len(targets) < 3:
+        conflicts = [*conflicts, "target-availability: Fewer than 3 valid S/R targets available"]
+
+    action = _choose_recommended_action(long_confidence, short_confidence, config)
+    risk_level = _risk_level_from_conflicts(conflicts)
+
+    setup.confidence_score = round(confidence, 2)
+    setup.targets_json = json.dumps(targets)
+    setup.conflict_flags_json = json.dumps(conflicts)
+    setup.recommended_action = action
+    setup.reasoning = _build_reasoning(
+        direction=direction,
+        confidence=confidence,
+        conflicts=conflicts,
+        dimension_scores=dimension_scores,
+        sentiment_classification=sentiment_classification,
+        action=action,
+    )
+    setup.risk_level = risk_level
+
+    return setup

app/services/rr_scanner_service.py

@@ -3,24 +3,27 @@
 Scans tracked tickers for asymmetric risk-reward trade setups.
 Long: target = nearest SR above, stop = entry - ATR × multiplier.
 Short: target = nearest SR below, stop = entry + ATR × multiplier.
-Filters by configurable R:R threshold (default 3:1).
+Filters by configurable R:R threshold (default 1.5).
 """
 
 from __future__ import annotations
 
+import json
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import delete, select
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.exceptions import NotFoundError
-from app.models.score import CompositeScore
+from app.models.score import CompositeScore, DimensionScore
+from app.models.sentiment import SentimentScore
 from app.models.sr_level import SRLevel
 from app.models.ticker import Ticker
 from app.models.trade_setup import TradeSetup
 from app.services.indicator_service import _extract_ohlcv, compute_atr
 from app.services.price_service import query_ohlcv
+from app.services.recommendation_service import enhance_trade_setup
 
 logger = logging.getLogger(__name__)
 
@@ -45,70 +48,63 @@ def _compute_quality_score(
     w_proximity: float = 0.30,
     rr_cap: float = 10.0,
 ) -> float:
-    """Compute a quality score for a candidate S/R level.
-
-    Combines normalized R:R ratio, level strength, and proximity to entry
-    into a single 0–1 score using configurable weights.
-    """
+    """Compute a quality score for a candidate S/R level."""
     norm_rr = min(rr / rr_cap, 1.0)
     norm_strength = strength / 100.0
     norm_proximity = 1.0 - min(distance / entry_price, 1.0)
     return w_rr * norm_rr + w_strength * norm_strength + w_proximity * norm_proximity
 
 
+async def _get_dimension_scores(db: AsyncSession, ticker_id: int) -> dict[str, float]:
+    result = await db.execute(
+        select(DimensionScore).where(DimensionScore.ticker_id == ticker_id)
+    )
+    rows = result.scalars().all()
+    return {row.dimension: float(row.score) for row in rows}
+
+
+async def _get_latest_sentiment(db: AsyncSession, ticker_id: int) -> str | None:
+    result = await db.execute(
+        select(SentimentScore)
+        .where(SentimentScore.ticker_id == ticker_id)
+        .order_by(SentimentScore.timestamp.desc())
+        .limit(1)
+    )
+    row = result.scalar_one_or_none()
+    return row.classification if row else None
+
+
 async def scan_ticker(
     db: AsyncSession,
     symbol: str,
     rr_threshold: float = 1.5,
     atr_multiplier: float = 1.5,
 ) -> list[TradeSetup]:
-    """Scan a single ticker for trade setups meeting the R:R threshold.
-
-    1. Fetch OHLCV data and compute ATR.
-    2. Fetch SR levels.
-    3. Compute long and short setups.
-    4. Filter by R:R threshold.
-    5. Delete old setups for this ticker and persist new ones.
-
-    Returns list of persisted TradeSetup models.
-    """
+    """Scan a single ticker for trade setups meeting the R:R threshold."""
     ticker = await _get_ticker(db, symbol)
 
-    # Fetch OHLCV
     records = await query_ohlcv(db, symbol)
     if not records or len(records) < 15:
         logger.info(
             "Skipping %s: insufficient OHLCV data (%d bars, need 15+)",
             symbol, len(records),
         )
-        # Clear any stale setups
-        await db.execute(
-            delete(TradeSetup).where(TradeSetup.ticker_id == ticker.id)
-        )
         return []
 
     _, highs, lows, closes, _ = _extract_ohlcv(records)
     entry_price = closes[-1]
 
-    # Compute ATR
     try:
         atr_result = compute_atr(highs, lows, closes)
         atr_value = atr_result["atr"]
     except Exception:
         logger.info("Skipping %s: cannot compute ATR", symbol)
-        await db.execute(
-            delete(TradeSetup).where(TradeSetup.ticker_id == ticker.id)
-        )
         return []
 
     if atr_value <= 0:
         logger.info("Skipping %s: ATR is zero or negative", symbol)
-        await db.execute(
-            delete(TradeSetup).where(TradeSetup.ticker_id == ticker.id)
-        )
         return []
 
-    # Fetch SR levels from DB (already computed by sr_service)
     sr_result = await db.execute(
         select(SRLevel).where(SRLevel.ticker_id == ticker.id)
     )
@@ -116,9 +112,6 @@ async def scan_ticker(
 
     if not sr_levels:
         logger.info("Skipping %s: no SR levels available", symbol)
-        await db.execute(
-            delete(TradeSetup).where(TradeSetup.ticker_id == ticker.id)
-        )
         return []
 
     levels_above = sorted(
@@ -131,18 +124,18 @@ async def scan_ticker(
         reverse=True,
     )
 
-    # Get composite score for this ticker
     comp_result = await db.execute(
         select(CompositeScore).where(CompositeScore.ticker_id == ticker.id)
     )
     comp = comp_result.scalar_one_or_none()
     composite_score = comp.score if comp else 0.0
 
+    dimension_scores = await _get_dimension_scores(db, ticker.id)
+    sentiment_classification = await _get_latest_sentiment(db, ticker.id)
+
     now = datetime.now(timezone.utc)
     setups: list[TradeSetup] = []
 
-    # Long setup: target = nearest SR above, stop = entry - ATR × multiplier
-    # Check all resistance levels above and pick the one with the best quality score
     if levels_above:
         stop = entry_price - (atr_value * atr_multiplier)
         risk = entry_price - stop
@@ -152,15 +145,18 @@ async def scan_ticker(
             best_candidate_target = 0.0
             for lv in levels_above:
                 reward = lv.price_level - entry_price
-                if reward > 0:
-                    rr = reward / risk
-                    if rr >= rr_threshold:
-                        distance = lv.price_level - entry_price
-                        quality = _compute_quality_score(rr, lv.strength, distance, entry_price)
-                        if quality > best_quality:
-                            best_quality = quality
-                            best_candidate_rr = rr
-                            best_candidate_target = lv.price_level
+                if reward <= 0:
+                    continue
+                rr = reward / risk
+                if rr < rr_threshold:
+                    continue
+                distance = lv.price_level - entry_price
+                quality = _compute_quality_score(rr, lv.strength, distance, entry_price)
+                if quality > best_quality:
+                    best_quality = quality
+                    best_candidate_rr = rr
+                    best_candidate_target = lv.price_level
+
             if best_candidate_rr > 0:
                 setups.append(TradeSetup(
                     ticker_id=ticker.id,
@@ -173,8 +169,6 @@ async def scan_ticker(
                     detected_at=now,
                 ))
 
-    # Short setup: target = nearest SR below, stop = entry + ATR × multiplier
-    # Check all support levels below and pick the one with the best quality score
     if levels_below:
         stop = entry_price + (atr_value * atr_multiplier)
         risk = stop - entry_price
@@ -184,15 +178,18 @@ async def scan_ticker(
             best_candidate_target = 0.0
             for lv in levels_below:
                 reward = entry_price - lv.price_level
-                if reward > 0:
-                    rr = reward / risk
-                    if rr >= rr_threshold:
-                        distance = entry_price - lv.price_level
-                        quality = _compute_quality_score(rr, lv.strength, distance, entry_price)
-                        if quality > best_quality:
-                            best_quality = quality
-                            best_candidate_rr = rr
-                            best_candidate_target = lv.price_level
+                if reward <= 0:
+                    continue
+                rr = reward / risk
+                if rr < rr_threshold:
+                    continue
+                distance = entry_price - lv.price_level
+                quality = _compute_quality_score(rr, lv.strength, distance, entry_price)
+                if quality > best_quality:
+                    best_quality = quality
+                    best_candidate_rr = rr
+                    best_candidate_target = lv.price_level
+
             if best_candidate_rr > 0:
                 setups.append(TradeSetup(
                     ticker_id=ticker.id,
@@ -205,20 +202,32 @@ async def scan_ticker(
                     detected_at=now,
                 ))
 
-    # Delete old setups for this ticker, persist new ones
-    await db.execute(
-        delete(TradeSetup).where(TradeSetup.ticker_id == ticker.id)
-    )
+    enhanced_setups: list[TradeSetup] = []
     for setup in setups:
+        try:
+            enhanced = await enhance_trade_setup(
+                db=db,
+                ticker=ticker,
+                setup=setup,
+                dimension_scores=dimension_scores,
+                sr_levels=sr_levels,
+                sentiment_classification=sentiment_classification,
+                atr_value=atr_value,
+            )
+            enhanced_setups.append(enhanced)
+        except Exception:
+            logger.exception("Error enhancing setup for %s (%s)", ticker.symbol, setup.direction)
+            enhanced_setups.append(setup)
+
+    for setup in enhanced_setups:
         db.add(setup)
 
     await db.commit()
 
-    # Refresh to get IDs
-    for s in setups:
+    for s in enhanced_setups:
         await db.refresh(s)
 
-    return setups
+    return enhanced_setups
 
 
 async def scan_all_tickers(
@@ -226,11 +235,7 @@ async def scan_all_tickers(
     rr_threshold: float = 1.5,
     atr_multiplier: float = 1.5,
 ) -> list[TradeSetup]:
-    """Scan all tracked tickers for trade setups.
-
-    Processes each ticker independently — one failure doesn't stop others.
-    Returns all setups found across all tickers.
-    """
+    """Scan all tracked tickers for trade setups."""
     result = await db.execute(select(Ticker).order_by(Ticker.symbol))
     tickers = list(result.scalars().all())
 
@@ -250,38 +255,100 @@ async def scan_all_tickers(
 async def get_trade_setups(
     db: AsyncSession,
     direction: str | None = None,
+    min_confidence: float | None = None,
+    recommended_action: str | None = None,
+    symbol: str | None = None,
 ) -> list[dict]:
-    """Get all stored trade setups, optionally filtered by direction.
-
-    Returns dicts sorted by R:R desc, secondary composite desc.
-    Each dict includes the ticker symbol.
-    """
+    """Get latest stored trade setups, optionally filtered."""
     stmt = (
         select(TradeSetup, Ticker.symbol)
         .join(Ticker, TradeSetup.ticker_id == Ticker.id)
     )
     if direction is not None:
         stmt = stmt.where(TradeSetup.direction == direction.lower())
+    if symbol is not None:
+        stmt = stmt.where(Ticker.symbol == symbol.strip().upper())
+    if min_confidence is not None:
+        stmt = stmt.where(TradeSetup.confidence_score >= min_confidence)
+    if recommended_action is not None:
+        stmt = stmt.where(TradeSetup.recommended_action == recommended_action)
 
-    stmt = stmt.order_by(
-        TradeSetup.rr_ratio.desc(),
-        TradeSetup.composite_score.desc(),
-    )
+    stmt = stmt.order_by(TradeSetup.detected_at.desc(), TradeSetup.id.desc())
 
     result = await db.execute(stmt)
     rows = result.all()
 
-    return [
-        {
-            "id": setup.id,
-            "symbol": symbol,
-            "direction": setup.direction,
-            "entry_price": setup.entry_price,
-            "stop_loss": setup.stop_loss,
-            "target": setup.target,
-            "rr_ratio": setup.rr_ratio,
-            "composite_score": setup.composite_score,
-            "detected_at": setup.detected_at,
-        }
-        for setup, symbol in rows
-    ]
+    latest_by_key: dict[tuple[str, str], tuple[TradeSetup, str]] = {}
+    for setup, ticker_symbol in rows:
+        dedupe_key = (ticker_symbol, setup.direction)
+        if dedupe_key not in latest_by_key:
+            latest_by_key[dedupe_key] = (setup, ticker_symbol)
+
+    latest_rows = list(latest_by_key.values())
+    latest_rows.sort(
+        key=lambda row: (
+            row[0].confidence_score if row[0].confidence_score is not None else -1.0,
+            row[0].rr_ratio,
+            row[0].composite_score,
+        ),
+        reverse=True,
+    )
+
+    return [_trade_setup_to_dict(setup, ticker_symbol) for setup, ticker_symbol in latest_rows]
+
+
+async def get_trade_setup_history(
+    db: AsyncSession,
+    symbol: str,
+) -> list[dict]:
+    """Get full recommendation history for a symbol (newest first)."""
+    stmt = (
+        select(TradeSetup, Ticker.symbol)
+        .join(Ticker, TradeSetup.ticker_id == Ticker.id)
+        .where(Ticker.symbol == symbol.strip().upper())
+        .order_by(TradeSetup.detected_at.desc(), TradeSetup.id.desc())
+    )
+    result = await db.execute(stmt)
+    rows = result.all()
+
+    return [_trade_setup_to_dict(setup, ticker_symbol) for setup, ticker_symbol in rows]
+
+
+def _trade_setup_to_dict(setup: TradeSetup, symbol: str) -> dict:
+    targets: list[dict] = []
+    conflicts: list[str] = []
+
+    if setup.targets_json:
+        try:
+            parsed_targets = json.loads(setup.targets_json)
+            if isinstance(parsed_targets, list):
+                targets = parsed_targets
+        except (TypeError, ValueError):
+            targets = []
+
+    if setup.conflict_flags_json:
+        try:
+            parsed_conflicts = json.loads(setup.conflict_flags_json)
+            if isinstance(parsed_conflicts, list):
+                conflicts = [str(item) for item in parsed_conflicts]
+        except (TypeError, ValueError):
+            conflicts = []
+
+    return {
+        "id": setup.id,
+        "symbol": symbol,
+        "direction": setup.direction,
+        "entry_price": setup.entry_price,
+        "stop_loss": setup.stop_loss,
+        "target": setup.target,
+        "rr_ratio": setup.rr_ratio,
+        "composite_score": setup.composite_score,
+        "detected_at": setup.detected_at,
+        "confidence_score": setup.confidence_score,
+        "targets": targets,
+        "conflict_flags": conflicts,
+        "recommended_action": setup.recommended_action,
+        "reasoning": setup.reasoning,
+        "risk_level": setup.risk_level,
+        "actual_outcome": setup.actual_outcome,
+    }

app/services/ticker_universe_service.py

@@ -0,0 +1,405 @@
+"""Ticker universe discovery and bootstrap service.
+
+Provides a minimal, provider-backed way to populate tracked tickers from
+well-known universes (S&P 500, NASDAQ-100, NASDAQ All).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import settings
+from app.exceptions import ProviderError, ValidationError
+from app.models.settings import SystemSetting
+from app.models.ticker import Ticker
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
+_SYMBOL_PATTERN = re.compile(r"^[A-Z0-9-]{1,10}$")
+
+_SEED_UNIVERSES: dict[str, list[str]] = {
+    "sp500": [
+        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "BRK-B", "TSLA", "JPM",
+        "V", "MA", "UNH", "XOM", "LLY", "AVGO", "COST", "PG", "JNJ", "HD", "MRK", "BAC",
+        "ABBV", "PEP", "KO", "ADBE", "NFLX", "CRM", "CSCO", "WMT", "AMD", "TMO", "MCD",
+        "ORCL", "ACN", "CVX", "LIN", "DHR", "ABT", "QCOM", "TXN", "PM", "DIS", "INTU",
+    ],
+    "nasdaq100": [
+        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "TSLA", "AVGO", "COST",
+        "NFLX", "ADBE", "CSCO", "AMD", "INTU", "QCOM", "AMGN", "TXN", "INTC", "BKNG", "GILD",
+        "ISRG", "MDLZ", "ADP", "LRCX", "ADI", "PANW", "SNPS", "CDNS", "KLAC", "MELI", "MU",
+        "SBUX", "CSX", "REGN", "VRTX", "MAR", "MNST", "CTAS", "ASML", "PYPL", "AMAT", "NXPI",
+    ],
+    "nasdaq_all": [
+        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "TSLA", "AMD", "INTC", "QCOM", "CSCO",
+        "ADBE", "NFLX", "PYPL", "AMAT", "MU", "SBUX", "GILD", "INTU", "BKNG", "ADP", "CTAS",
+        "PANW", "SNPS", "CDNS", "LRCX", "KLAC", "MELI", "ASML", "REGN", "VRTX", "MDLZ", "AMGN",
+    ],
+}
+
+_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
+if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
+    _CA_BUNDLE_PATH: str | bool = True
+else:
+    _CA_BUNDLE_PATH = _CA_BUNDLE
+
+
+def _validate_universe(universe: str) -> str:
+    normalised = universe.strip().lower()
+    if normalised not in SUPPORTED_UNIVERSES:
+        supported = ", ".join(sorted(SUPPORTED_UNIVERSES))
+        raise ValidationError(f"Unsupported universe '{universe}'. Supported: {supported}")
+    return normalised
+
+
+def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
+    deduped: set[str] = set()
+    for raw_symbol in symbols:
+        symbol = raw_symbol.strip().upper().replace(".", "-")
+        if not symbol:
+            continue
+        if _SYMBOL_PATTERN.fullmatch(symbol) is None:
+            continue
+        deduped.add(symbol)
+    return sorted(deduped)
+
+
+def _extract_symbols_from_fmp_payload(payload: object) -> list[str]:
+    if not isinstance(payload, list):
+        return []
+
+    symbols: list[str] = []
+    for item in payload:
+        if not isinstance(item, dict):
+            continue
+        candidate = item.get("symbol") or item.get("ticker")
+        if isinstance(candidate, str):
+            symbols.append(candidate)
+    return symbols
+
+
+async def _try_fmp_urls(
+    client: httpx.AsyncClient,
+    urls: list[str],
+) -> tuple[list[str], list[str]]:
+    failures: list[str] = []
+    for url in urls:
+        endpoint = url.split("?")[0]
+        try:
+            response = await client.get(url)
+        except httpx.HTTPError as exc:
+            failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})")
+            continue
+
+        if response.status_code != 200:
+            failures.append(f"{endpoint}: HTTP {response.status_code}")
+            continue
+
+        try:
+            payload = response.json()
+        except ValueError:
+            failures.append(f"{endpoint}: invalid JSON payload")
+            continue
+
+        symbols = _extract_symbols_from_fmp_payload(payload)
+        if symbols:
+            return symbols, failures
+
+        failures.append(f"{endpoint}: empty/unsupported payload")
+
+    return [], failures
+
+
+async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
+    if not settings.fmp_api_key:
+        raise ValidationError(
+            "FMP API key is required for universe bootstrap (set FMP_API_KEY)"
+        )
+
+    api_key = settings.fmp_api_key
+    stable_base = "https://financialmodelingprep.com/stable"
+    legacy_base = "https://financialmodelingprep.com/api/v3"
+
+    stable_candidates: dict[str, list[str]] = {
+        "sp500": [
+            f"{stable_base}/sp500-constituent?apikey={api_key}",
+            f"{stable_base}/sp500-constituents?apikey={api_key}",
+        ],
+        "nasdaq100": [
+            f"{stable_base}/nasdaq-100-constituent?apikey={api_key}",
+            f"{stable_base}/nasdaq100-constituent?apikey={api_key}",
+            f"{stable_base}/nasdaq-100-constituents?apikey={api_key}",
+        ],
+        "nasdaq_all": [
+            f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
+            f"{stable_base}/available-traded/list?apikey={api_key}",
+        ],
+    }
+
+    legacy_candidates: dict[str, list[str]] = {
+        "sp500": [
+            f"{legacy_base}/sp500_constituent?apikey={api_key}",
+            f"{legacy_base}/sp500_constituent",
+        ],
+        "nasdaq100": [
+            f"{legacy_base}/nasdaq_constituent?apikey={api_key}",
+            f"{legacy_base}/nasdaq_constituent",
+        ],
+        "nasdaq_all": [
+            f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
+        ],
+    }
+
+    failures: list[str] = []
+    async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
+        stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe])
+        failures.extend(stable_failures)
+
+        if stable_symbols:
+            return stable_symbols
+
+        legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe])
+        failures.extend(legacy_failures)
+
+        if legacy_symbols:
+            return legacy_symbols
+
+    if failures:
+        reason = "; ".join(failures[:6])
+        logger.warning("FMP universe fetch failed for %s: %s", universe, reason)
+        raise ProviderError(
+            f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}"
+        )
+
+    raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
+
+
+async def _fetch_html_symbols(
+    client: httpx.AsyncClient,
+    url: str,
+    pattern: str,
+) -> tuple[list[str], str | None]:
+    try:
+        response = await client.get(url)
+    except httpx.HTTPError as exc:
+        return [], f"{url}: network error ({type(exc).__name__}: {exc})"
+
+    if response.status_code != 200:
+        return [], f"{url}: HTTP {response.status_code}"
+
+    matches = re.findall(pattern, response.text, flags=re.IGNORECASE)
+    if not matches:
+        return [], f"{url}: no symbols parsed"
+    return list(matches), None
+
+
+async def _fetch_nasdaq_trader_symbols(
+    client: httpx.AsyncClient,
+) -> tuple[list[str], str | None]:
+    url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
+    try:
+        response = await client.get(url)
+    except httpx.HTTPError as exc:
+        return [], f"{url}: network error ({type(exc).__name__}: {exc})"
+
+    if response.status_code != 200:
+        return [], f"{url}: HTTP {response.status_code}"
+
+    symbols: list[str] = []
+    for line in response.text.splitlines():
+        if not line or line.startswith("Symbol|") or line.startswith("File Creation Time"):
+            continue
+        parts = line.split("|")
+        if not parts:
+            continue
+        symbol = parts[0].strip()
+        test_issue = parts[6].strip() if len(parts) > 6 else "N"
+        if test_issue == "Y":
+            continue
+        symbols.append(symbol)
+
+    if not symbols:
+        return [], f"{url}: no symbols parsed"
+    return symbols, None
+
+
+async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str], list[str], str | None]:
+    failures: list[str] = []
+
+    sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
+    nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"
+    wiki_symbol_pattern = r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>"
+
+    async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
+        if universe == "sp500":
+            symbols, error = await _fetch_html_symbols(client, sp500_url, wiki_symbol_pattern)
+            if error:
+                failures.append(error)
+            else:
+                return symbols, failures, "wikipedia_sp500"
+
+        if universe == "nasdaq100":
+            symbols, error = await _fetch_html_symbols(client, nasdaq100_url, wiki_symbol_pattern)
+            if error:
+                failures.append(error)
+            else:
+                return symbols, failures, "wikipedia_nasdaq100"
+
+        if universe == "nasdaq_all":
+            symbols, error = await _fetch_nasdaq_trader_symbols(client)
+            if error:
+                failures.append(error)
+            else:
+                return symbols, failures, "nasdaq_trader"
+
+    return [], failures, None
+
+
+async def _read_cached_symbols(db: AsyncSession, universe: str) -> list[str]:
+    key = f"ticker_universe_cache_{universe}"
+    result = await db.execute(select(SystemSetting).where(SystemSetting.key == key))
+    setting = result.scalar_one_or_none()
+    if setting is None:
+        return []
+
+    try:
+        payload = json.loads(setting.value)
+    except (TypeError, ValueError):
+        return []
+
+    if isinstance(payload, dict):
+        symbols = payload.get("symbols", [])
+    elif isinstance(payload, list):
+        symbols = payload
+    else:
+        symbols = []
+
+    if not isinstance(symbols, list):
+        return []
+
+    return _normalise_symbols([str(symbol) for symbol in symbols])
+
+
+async def _write_cached_symbols(
+    db: AsyncSession,
+    universe: str,
+    symbols: list[str],
+    source: str,
+) -> None:
+    key = f"ticker_universe_cache_{universe}"
+    payload = {
+        "symbols": symbols,
+        "source": source,
+        "updated_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    result = await db.execute(select(SystemSetting).where(SystemSetting.key == key))
+    setting = result.scalar_one_or_none()
+    value = json.dumps(payload)
+
+    if setting is None:
+        db.add(SystemSetting(key=key, value=value))
+    else:
+        setting.value = value
+
+    await db.commit()
+
+
+async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
+    """Fetch and normalise symbols for a supported universe with fallbacks.
+
+    Fallback order:
+    1) Free public sources (Wikipedia/NASDAQ trader)
+    2) FMP endpoints (if available)
+    3) Cached snapshot in SystemSetting
+    4) Built-in seed symbols
+    """
+    normalised_universe = _validate_universe(universe)
+    failures: list[str] = []
+
+    public_symbols, public_failures, public_source = await _fetch_universe_symbols_from_public(normalised_universe)
+    failures.extend(public_failures)
+    cleaned_public = _normalise_symbols(public_symbols)
+    if cleaned_public:
+        await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
+        return cleaned_public
+
+    try:
+        fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
+        cleaned_fmp = _normalise_symbols(fmp_symbols)
+        if cleaned_fmp:
+            await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
+            return cleaned_fmp
+    except (ProviderError, ValidationError) as exc:
+        failures.append(str(exc))
+
+    cached_symbols = await _read_cached_symbols(db, normalised_universe)
+    if cached_symbols:
+        logger.warning(
+            "Using cached universe symbols for %s because live fetch failed: %s",
+            normalised_universe,
+            "; ".join(failures[:3]),
+        )
+        return cached_symbols
+
+    seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
+    if seed_symbols:
+        logger.warning(
+            "Using built-in seed symbols for %s because live/cache fetch failed: %s",
+            normalised_universe,
+            "; ".join(failures[:3]),
+        )
+        return seed_symbols
+
+    reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
+    raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")
+
+
+async def bootstrap_universe(
+    db: AsyncSession,
+    universe: str,
+    *,
+    prune_missing: bool = False,
+) -> dict[str, int | str]:
+    """Upsert ticker universe into tracked tickers.
+
+    Returns summary counts for added/existing/deleted symbols.
+    """
+    normalised_universe = _validate_universe(universe)
+    symbols = await fetch_universe_symbols(db, normalised_universe)
+
+    existing_rows = await db.execute(select(Ticker.symbol))
+    existing_symbols = set(existing_rows.scalars().all())
+    target_symbols = set(symbols)
+
+    symbols_to_add = sorted(target_symbols - existing_symbols)
+    symbols_to_delete = sorted(existing_symbols - target_symbols) if prune_missing else []
+
+    for symbol in symbols_to_add:
+        db.add(Ticker(symbol=symbol))
+
+    deleted_count = 0
+    if symbols_to_delete:
+        result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(symbols_to_delete)))
+        deleted_count = int(result.rowcount or 0)
+
+    await db.commit()
+
+    return {
+        "universe": normalised_universe,
+        "total_universe_symbols": len(symbols),
+        "added": len(symbols_to_add),
+        "already_tracked": len(target_symbols & existing_symbols),
+        "deleted": deleted_count,
+    }