From 06cf054f60ba8f91b452a711a018f07e55eb1148 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 09:46:07 +0200 Subject: [PATCH] fix: bootstrap SSL/CA for research CLI on corporate MacBooks Extract app/ssl_bootstrap.py (shared with FastAPI main), wire it into research scripts, and teach run_tier1_macbook.sh to locate combined-ca-bundle.pem, certifi, optional USE_CORP_PROXY, plus --ssl-check diagnostics. --- app/main.py | 51 +-------- app/ssl_bootstrap.py | 136 +++++++++++++++++++++++ scripts/backfill_earnings_events.py | 4 + scripts/build_ticker_sector_map.py | 4 + scripts/extend_snapshot_universe.py | 4 + scripts/fetch_sector_etfs_to_snapshot.py | 4 + scripts/run_earnings_research.py | 4 + scripts/run_history_depth_research.py | 4 + scripts/run_sector_residual_research.py | 4 + scripts/run_tier1_macbook.sh | 94 +++++++++++++++- 10 files changed, 258 insertions(+), 51 deletions(-) create mode 100644 app/ssl_bootstrap.py diff --git a/app/main.py b/app/main.py index 9bc5f8f..c8064ad 100644 --- a/app/main.py +++ b/app/main.py @@ -3,56 +3,9 @@ # --------------------------------------------------------------------------- # SSL + proxy injection — MUST happen before any HTTP client imports # --------------------------------------------------------------------------- -import os as _os -import ssl as _ssl -from pathlib import Path as _Path +from app.ssl_bootstrap import bootstrap_ssl -_COMBINED_CERT = _Path(__file__).resolve().parent.parent / "combined-ca-bundle.pem" - -if _COMBINED_CERT.exists(): - _cert_path = str(_COMBINED_CERT) - # Env vars for libraries that respect them (requests, urllib3) - _os.environ["SSL_CERT_FILE"] = _cert_path - _os.environ["REQUESTS_CA_BUNDLE"] = _cert_path - _os.environ["CURL_CA_BUNDLE"] = _cert_path - - # Monkey-patch ssl.create_default_context so that ALL libraries - # (aiohttp, httpx, google-genai, alpaca-py, etc.) automatically - # use our combined CA bundle that includes the corporate root cert. - _original_create_default_context = _ssl.create_default_context - - def _patched_create_default_context( - purpose=_ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None - ): - ctx = _original_create_default_context( - purpose, cafile=cafile, capath=capath, cadata=cadata - ) - # Always load our combined bundle on top of whatever was loaded - ctx.load_verify_locations(cafile=_cert_path) - return ctx - - _ssl.create_default_context = _patched_create_default_context - - # Also patch aiohttp's cached SSL context objects directly, since - # aiohttp creates them at import time and may have already cached - # a context without our corporate CA bundle. - try: - import aiohttp.connector as _aio_conn - if hasattr(_aio_conn, '_SSL_CONTEXT_VERIFIED') and _aio_conn._SSL_CONTEXT_VERIFIED is not None: - _aio_conn._SSL_CONTEXT_VERIFIED.load_verify_locations(cafile=_cert_path) - if hasattr(_aio_conn, '_SSL_CONTEXT_UNVERIFIED') and _aio_conn._SSL_CONTEXT_UNVERIFIED is not None: - _aio_conn._SSL_CONTEXT_UNVERIFIED.load_verify_locations(cafile=_cert_path) - except ImportError: - pass - -# Corporate proxy — needed when Kiro spawns the process (no .zshrc sourced) -# Only enable this if explicitly requested via environment variable. -if _os.environ.get("USE_CORP_PROXY", "0") == "1": - _PROXY = "http://aproxy.corproot.net:8080" - _NO_PROXY = "corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com" - _os.environ.setdefault("HTTP_PROXY", _PROXY) - _os.environ.setdefault("HTTPS_PROXY", _PROXY) - _os.environ.setdefault("NO_PROXY", _NO_PROXY) +bootstrap_ssl() import logging import sys diff --git a/app/ssl_bootstrap.py b/app/ssl_bootstrap.py new file mode 100644 index 0000000..e3f2e4e --- /dev/null +++ b/app/ssl_bootstrap.py @@ -0,0 +1,136 @@ +"""TLS / corporate-proxy bootstrap for CLI scripts and the API. + +Must run **before** httpx / alpaca / aiohttp open connections. + +Resolution order for the CA bundle: +1. ``combined-ca-bundle.pem`` in the repo root (gitignored corporate bundle) +2. ``$HOME/combined-ca-bundle.pem`` (MacBook path used by existing tooling) +3. ``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` if already set and present +4. ``certifi.where()`` when the package is installed +5. System defaults (no patch) + +Optional corporate proxy (Swisscom-style) when ``USE_CORP_PROXY=1``. +""" + +from __future__ import annotations + +import os +import ssl +from pathlib import Path + +_BOOTSTRAPPED = False + + +def _candidate_ca_paths() -> list[Path]: + root = Path(__file__).resolve().parent.parent + home = Path.home() + env_paths = [ + os.environ.get("SSL_CERT_FILE", ""), + os.environ.get("REQUESTS_CA_BUNDLE", ""), + os.environ.get("CURL_CA_BUNDLE", ""), + ] + paths = [ + root / "combined-ca-bundle.pem", + home / "combined-ca-bundle.pem", + *[Path(p) for p in env_paths if p], + ] + try: + import certifi + + paths.append(Path(certifi.where())) + except Exception: + pass + return paths + + +def resolve_ca_bundle() -> str | None: + for path in _candidate_ca_paths(): + try: + if path.is_file() and path.stat().st_size > 0: + return str(path.resolve()) + except OSError: + continue + return None + + +def apply_corp_proxy_if_requested() -> None: + if os.environ.get("USE_CORP_PROXY", "0") != "1": + return + proxy = os.environ.get("CORP_HTTP_PROXY", "http://aproxy.corproot.net:8080") + no_proxy = os.environ.get( + "CORP_NO_PROXY", + "corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com", + ) + os.environ.setdefault("HTTP_PROXY", proxy) + os.environ.setdefault("HTTPS_PROXY", proxy) + os.environ.setdefault("NO_PROXY", no_proxy) + os.environ.setdefault("http_proxy", proxy) + os.environ.setdefault("https_proxy", proxy) + os.environ.setdefault("no_proxy", no_proxy) + + +def bootstrap_ssl(*, force: bool = False) -> str | None: + """Install CA env vars + patch ``ssl.create_default_context``. + + Returns the CA path used, or None if nothing was applied. + Safe to call multiple times. + """ + global _BOOTSTRAPPED + if _BOOTSTRAPPED and not force: + return os.environ.get("SSL_CERT_FILE") or None + + apply_corp_proxy_if_requested() + + cert_path = resolve_ca_bundle() + if not cert_path: + _BOOTSTRAPPED = True + return None + + os.environ["SSL_CERT_FILE"] = cert_path + os.environ["REQUESTS_CA_BUNDLE"] = cert_path + os.environ["CURL_CA_BUNDLE"] = cert_path + + original = ssl.create_default_context + + def _patched( + purpose=ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None + ): + ctx = original(purpose, cafile=cafile, capath=capath, cadata=cadata) + try: + ctx.load_verify_locations(cafile=cert_path) + except Exception: + pass + return ctx + + ssl.create_default_context = _patched # type: ignore[assignment] + + # aiohttp may cache SSL contexts at import time. + try: + import aiohttp.connector as aio_conn + + for attr in ("_SSL_CONTEXT_VERIFIED", "_SSL_CONTEXT_UNVERIFIED"): + ctx = getattr(aio_conn, attr, None) + if ctx is not None: + try: + ctx.load_verify_locations(cafile=cert_path) + except Exception: + pass + except ImportError: + pass + + _BOOTSTRAPPED = True + return cert_path + + +def ssl_status() -> dict: + """Diagnostic blob for research scripts / MacBook troubleshooting.""" + ca = resolve_ca_bundle() + return { + "ca_bundle": ca, + "ssl_cert_file_env": os.environ.get("SSL_CERT_FILE"), + "use_corp_proxy": os.environ.get("USE_CORP_PROXY", "0"), + "http_proxy": os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY"), + "candidates_exist": { + str(p): p.is_file() for p in _candidate_ca_paths()[:4] + }, + } diff --git a/scripts/backfill_earnings_events.py b/scripts/backfill_earnings_events.py index 971d555..665eabc 100644 --- a/scripts/backfill_earnings_events.py +++ b/scripts/backfill_earnings_events.py @@ -29,6 +29,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + FMP_STABLE = "https://financialmodelingprep.com/stable" DDL = """ CREATE TABLE IF NOT EXISTS earnings_events ( diff --git a/scripts/build_ticker_sector_map.py b/scripts/build_ticker_sector_map.py index f4a9ae9..e200b76 100644 --- a/scripts/build_ticker_sector_map.py +++ b/scripts/build_ticker_sector_map.py @@ -34,6 +34,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import ( # noqa: E402 DEFAULT_SECTOR_MAP_PATH, coverage_stats, diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py index 886d9e2..8f10b94 100644 --- a/scripts/extend_snapshot_universe.py +++ b/scripts/extend_snapshot_universe.py @@ -45,6 +45,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + def _parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) diff --git a/scripts/fetch_sector_etfs_to_snapshot.py b/scripts/fetch_sector_etfs_to_snapshot.py index db558e0..5968a03 100644 --- a/scripts/fetch_sector_etfs_to_snapshot.py +++ b/scripts/fetch_sector_etfs_to_snapshot.py @@ -27,6 +27,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import SECTOR_ETFS # noqa: E402 diff --git a/scripts/run_earnings_research.py b/scripts/run_earnings_research.py index 20c695f..1e12264 100644 --- a/scripts/run_earnings_research.py +++ b/scripts/run_earnings_research.py @@ -28,6 +28,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + IRON_IC_BAR = 0.03 MIN_RELIABLE = 12 SUE_CARRY_DAYS = 63 diff --git a/scripts/run_history_depth_research.py b/scripts/run_history_depth_research.py index 02bb229..cfe2285 100644 --- a/scripts/run_history_depth_research.py +++ b/scripts/run_history_depth_research.py @@ -35,6 +35,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + ERA_SPLIT = date(2021, 1, 1) SURVIVORSHIP_BANNER = ( "SURVIVORSHIP BIAS: today's constituents backfilled historically. " diff --git a/scripts/run_sector_residual_research.py b/scripts/run_sector_residual_research.py index 290987d..1f60590 100644 --- a/scripts/run_sector_residual_research.py +++ b/scripts/run_sector_residual_research.py @@ -40,6 +40,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import ( # noqa: E402 DEFAULT_SECTOR_MAP_PATH, SECTOR_ETFS, diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 5afbe20..1472cf4 100644 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -33,11 +33,21 @@ ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" FMP_LIMIT="${FMP_LIMIT:-250}" FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" +USE_CORP_PROXY="${USE_CORP_PROXY:-0}" -PHASE="depth" # depth | all | earnings | harness | coverage +PHASE="depth" # depth | all | earnings | harness | coverage | ssl-check usage() { - sed -n '2,22p' "$0" | sed 's/^# \?//' + sed -n '2,25p' "$0" | sed 's/^# \?//' + cat <<'EOF' + +SSL / network (corporate MacBook) + SSL errors usually mean the corp root CA is missing from Python. + 1) Put combined-ca-bundle.pem in the repo root OR $HOME + 2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem + 3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh + 4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check +EOF exit "${1:-0}" } @@ -48,6 +58,8 @@ while [[ $# -gt 0 ]]; do --harness-only) PHASE=harness; shift ;; --coverage-only) PHASE=coverage; shift ;; --depth) PHASE=depth; shift ;; + --ssl-check) PHASE=ssl; shift ;; + --corp-proxy) USE_CORP_PROXY=1; shift ;; --prod-snap) PROD_SNAP="$2"; shift 2 ;; --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; --history-days) HISTORY_DAYS="$2"; shift 2 ;; @@ -71,6 +83,80 @@ fi log() { printf '\n==> %s\n' "$*"; } die() { echo "ERROR: $*" >&2; exit 1; } +# --------------------------------------------------------------------------- +# TLS bootstrap — same corp CA path the FastAPI app uses +# --------------------------------------------------------------------------- +setup_ssl() { + export USE_CORP_PROXY + + # Prefer explicit env, then repo / home corporate bundle, then certifi. + if [[ -z "${SSL_CERT_FILE:-}" ]]; then + if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then + export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem" + elif [[ -f "$HOME/combined-ca-bundle.pem" ]]; then + export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem" + fi + fi + + if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then + export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" + export CURL_CA_BUNDLE="$SSL_CERT_FILE" + log "SSL CA bundle: $SSL_CERT_FILE" + else + # Fall back to certifi if installed + local certifi_path + certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)" + if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then + export SSL_CERT_FILE="$certifi_path" + export REQUESTS_CA_BUNDLE="$certifi_path" + export CURL_CA_BUNDLE="$certifi_path" + log "SSL CA bundle (certifi): $SSL_CERT_FILE" + else + log "WARNING: no CA bundle found — SSL may fail on corp networks" + log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/" + log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem" + fi + fi + + if [[ "$USE_CORP_PROXY" == "1" ]]; then + export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}" + export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}" + export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}" + export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY" + log "Corp proxy enabled: $HTTPS_PROXY" + fi + + # Ensure Python process sees the same bootstrap (patches ssl for alpaca-py). + export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}" +} + +ssl_check() { + setup_ssl + log "SSL diagnostic" + "$PYTHON" - <<'PY' +from app.ssl_bootstrap import bootstrap_ssl, ssl_status +import json +import urllib.request + +ca = bootstrap_ssl() +print(json.dumps(ssl_status(), indent=2)) +print("bootstrap_ssl ->", ca) +urls = [ + "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1", + "https://financialmodelingprep.com/stable/profile?symbol=AAPL", + "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM", +] +for url in urls: + try: + req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"}) + with urllib.request.urlopen(req, timeout=20) as resp: + print(f"OK {resp.status} {url[:60]}...") + except Exception as exc: + print(f"FAIL {type(exc).__name__}: {exc}") + print(f" {url[:80]}") +PY +} + need_file() { [[ -f "$1" ]] || die "missing $1" } @@ -138,8 +224,12 @@ run_harness() { } log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" +setup_ssl case "$PHASE" in + ssl) + ssl_check + ;; coverage) run_coverage ;;