From b6892d13fd87839ba50e28611d173a5a150e0860 Mon Sep 17 00:00:00 2001
From: Dennis Thiessen <dennis@thiessen.io>
Date: Sat, 18 Jul 2026 20:32:57 +0200
Subject: [PATCH] fix: resolve research universe without system_settings DB

Public/FMP/seed symbol lists no longer touch SystemSetting cache, so the
extender works offline on an empty in-memory session.
---
 scripts/extend_snapshot_universe.py | 74 ++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 22 deletions(-)

diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py
index e3585b8..6b0f28b 100644
--- a/scripts/extend_snapshot_universe.py
+++ b/scripts/extend_snapshot_universe.py
@@ -108,32 +108,62 @@ def _ensure_rank_only_table(conn) -> None:
 
 
 async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
-    """Return sorted unique symbols and source labels."""
-    from app.database import async_session_factory
-    from app.services.ticker_universe_service import fetch_universe_symbols
+    """Return sorted unique symbols and source labels.
+
+    Offline-safe: does **not** use production Postgres or SystemSetting cache
+    (those require a schema). Public sources first, then FMP, then seeds.
+    """
+    from app.services.ticker_universe_service import (
+        _SEED_UNIVERSES,
+        _fetch_universe_symbols_from_fmp,
+        _fetch_universe_symbols_from_public,
+        _normalise_symbols,
+    )
 
     sources: dict[str, str] = {}
     symbols: set[str] = set()
-    # Need a DB session for cache writes; use local async engine if configured,
-    # but public/FMP fetch works with any session. Prefer a throwaway sqlite.
-    from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
-    from sqlalchemy.ext.asyncio import AsyncSession
 
-    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
-    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
-    try:
-        async with Session() as db:
-            for universe in ("nasdaq_all", "sp500"):
-                try:
-                    syms, src = await fetch_universe_symbols(db, universe)
-                except Exception as exc:
-                    print(f"WARNING: universe {universe} failed: {exc}")
-                    continue
-                sources[universe] = src
-                symbols.update(syms)
-                print(f"  {universe}: {len(syms)} symbols (source={src})")
-    finally:
-        await engine.dispose()
+    for universe in ("nasdaq_all", "sp500"):
+        cleaned: list[str] = []
+        src = "none"
+
+        public_symbols, public_failures, public_source = (
+            await _fetch_universe_symbols_from_public(universe)
+        )
+        cleaned = _normalise_symbols(public_symbols)
+        if cleaned:
+            src = public_source or "public"
+        else:
+            if public_failures:
+                print(
+                    f"  WARNING: public fetch {universe}: "
+                    f"{'; '.join(public_failures[:3])}"
+                )
+            try:
+                fmp_symbols = await _fetch_universe_symbols_from_fmp(universe)
+                cleaned = _normalise_symbols(fmp_symbols)
+                if cleaned:
+                    src = "fmp"
+            except Exception as exc:
+                print(f"  WARNING: FMP fetch {universe}: {exc}")
+
+        if not cleaned:
+            cleaned = _normalise_symbols(_SEED_UNIVERSES.get(universe, []))
+            if cleaned:
+                src = "seed"
+                print(
+                    f"  WARNING: {universe} fell back to seed list "
+                    f"({len(cleaned)} symbols) — not full universe"
+                )
+
+        if not cleaned:
+            print(f"  WARNING: universe {universe} returned no symbols")
+            continue
+
+        sources[universe] = src
+        symbols.update(cleaned)
+        print(f"  {universe}: {len(cleaned)} symbols (source={src})")
+
     return sorted(symbols), sources