dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: bootstrap SSL/CA for research CLI on corporate MacBooks

Browse Source 
Extract app/ssl_bootstrap.py (shared with FastAPI main), wire it into research
scripts, and teach run_tier1_macbook.sh to locate combined-ca-bundle.pem, certifi,
optional USE_CORP_PROXY, plus --ssl-check diagnostics.
This commit is contained in:
dennisthiessen
2026-07-19 09:46:07 +02:00
parent 32bf9c9297
commit 06cf054f60
10 changed files with 258 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
+2 -49
View File
@@ -3,56 +3,9 @@
# ---------------------------------------------------------------------------
# SSL + proxy injection — MUST happen before any HTTP client imports
# ---------------------------------------------------------------------------
import os as _os
import ssl as _ssl
from pathlib import Path as _Path
from app.ssl_bootstrap import bootstrap_ssl
_COMBINED_CERT = _Path(__file__).resolve().parent.parent / "combined-ca-bundle.pem"
if _COMBINED_CERT.exists():
_cert_path = str(_COMBINED_CERT)
# Env vars for libraries that respect them (requests, urllib3)
_os.environ["SSL_CERT_FILE"] = _cert_path
_os.environ["REQUESTS_CA_BUNDLE"] = _cert_path
_os.environ["CURL_CA_BUNDLE"] = _cert_path
# Monkey-patch ssl.create_default_context so that ALL libraries
# (aiohttp, httpx, google-genai, alpaca-py, etc.) automatically
# use our combined CA bundle that includes the corporate root cert.
_original_create_default_context = _ssl.create_default_context
def _patched_create_default_context(
purpose=_ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None
):
ctx = _original_create_default_context(
purpose, cafile=cafile, capath=capath, cadata=cadata
)
# Always load our combined bundle on top of whatever was loaded
ctx.load_verify_locations(cafile=_cert_path)
return ctx
_ssl.create_default_context = _patched_create_default_context
# Also patch aiohttp's cached SSL context objects directly, since
# aiohttp creates them at import time and may have already cached
# a context without our corporate CA bundle.
try:
import aiohttp.connector as _aio_conn
if hasattr(_aio_conn, '_SSL_CONTEXT_VERIFIED') and _aio_conn._SSL_CONTEXT_VERIFIED is not None:
_aio_conn._SSL_CONTEXT_VERIFIED.load_verify_locations(cafile=_cert_path)
if hasattr(_aio_conn, '_SSL_CONTEXT_UNVERIFIED') and _aio_conn._SSL_CONTEXT_UNVERIFIED is not None:
_aio_conn._SSL_CONTEXT_UNVERIFIED.load_verify_locations(cafile=_cert_path)
except ImportError:
pass
# Corporate proxy — needed when Kiro spawns the process (no .zshrc sourced)
# Only enable this if explicitly requested via environment variable.
if _os.environ.get("USE_CORP_PROXY", "0") == "1":
_PROXY = "http://aproxy.corproot.net:8080"
_NO_PROXY = "corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com"
_os.environ.setdefault("HTTP_PROXY", _PROXY)
_os.environ.setdefault("HTTPS_PROXY", _PROXY)
_os.environ.setdefault("NO_PROXY", _NO_PROXY)
bootstrap_ssl()
import logging
import sys
app/ssl_bootstrap.py
+136
View File
@@ -0,0 +1,136 @@
"""TLS / corporate-proxy bootstrap for CLI scripts and the API.
Must run **before** httpx / alpaca / aiohttp open connections.
Resolution order for the CA bundle:
1. ``combined-ca-bundle.pem`` in the repo root (gitignored corporate bundle)
2. ``$HOME/combined-ca-bundle.pem`` (MacBook path used by existing tooling)
3. ``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` if already set and present
4. ``certifi.where()`` when the package is installed
5. System defaults (no patch)
Optional corporate proxy (Swisscom-style) when ``USE_CORP_PROXY=1``.
"""
from __future__ import annotations
import os
import ssl
from pathlib import Path
_BOOTSTRAPPED = False
def _candidate_ca_paths() -> list[Path]:
root = Path(__file__).resolve().parent.parent
home = Path.home()
env_paths = [
os.environ.get("SSL_CERT_FILE", ""),
os.environ.get("REQUESTS_CA_BUNDLE", ""),
os.environ.get("CURL_CA_BUNDLE", ""),
]
paths = [
root / "combined-ca-bundle.pem",
home / "combined-ca-bundle.pem",
*[Path(p) for p in env_paths if p],
]
try:
import certifi
paths.append(Path(certifi.where()))
except Exception:
pass
return paths
def resolve_ca_bundle() -> str | None:
for path in _candidate_ca_paths():
try:
if path.is_file() and path.stat().st_size > 0:
return str(path.resolve())
except OSError:
continue
return None
def apply_corp_proxy_if_requested() -> None:
if os.environ.get("USE_CORP_PROXY", "0") != "1":
return
proxy = os.environ.get("CORP_HTTP_PROXY", "http://aproxy.corproot.net:8080")
no_proxy = os.environ.get(
"CORP_NO_PROXY",
"corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com",
)
os.environ.setdefault("HTTP_PROXY", proxy)
os.environ.setdefault("HTTPS_PROXY", proxy)
os.environ.setdefault("NO_PROXY", no_proxy)
os.environ.setdefault("http_proxy", proxy)
os.environ.setdefault("https_proxy", proxy)
os.environ.setdefault("no_proxy", no_proxy)
def bootstrap_ssl(*, force: bool = False) -> str | None:
"""Install CA env vars + patch ``ssl.create_default_context``.
Returns the CA path used, or None if nothing was applied.
Safe to call multiple times.
"""
global _BOOTSTRAPPED
if _BOOTSTRAPPED and not force:
return os.environ.get("SSL_CERT_FILE") or None
apply_corp_proxy_if_requested()
cert_path = resolve_ca_bundle()
if not cert_path:
_BOOTSTRAPPED = True
return None
os.environ["SSL_CERT_FILE"] = cert_path
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
os.environ["CURL_CA_BUNDLE"] = cert_path
original = ssl.create_default_context
def _patched(
purpose=ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None
):
ctx = original(purpose, cafile=cafile, capath=capath, cadata=cadata)
try:
ctx.load_verify_locations(cafile=cert_path)
except Exception:
pass
return ctx
ssl.create_default_context = _patched # type: ignore[assignment]
# aiohttp may cache SSL contexts at import time.
try:
import aiohttp.connector as aio_conn
for attr in ("_SSL_CONTEXT_VERIFIED", "_SSL_CONTEXT_UNVERIFIED"):
ctx = getattr(aio_conn, attr, None)
if ctx is not None:
try:
ctx.load_verify_locations(cafile=cert_path)
except Exception:
pass
except ImportError:
pass
_BOOTSTRAPPED = True
return cert_path
def ssl_status() -> dict:
"""Diagnostic blob for research scripts / MacBook troubleshooting."""
ca = resolve_ca_bundle()
return {
"ca_bundle": ca,
"ssl_cert_file_env": os.environ.get("SSL_CERT_FILE"),
"use_corp_proxy": os.environ.get("USE_CORP_PROXY", "0"),
"http_proxy": os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY"),
"candidates_exist": {
str(p): p.is_file() for p in _candidate_ca_paths()[:4]
},
}
scripts/backfill_earnings_events.py
+4
View File
@@ -29,6 +29,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
FMP_STABLE = "https://financialmodelingprep.com/stable"
DDL = """
CREATE TABLE IF NOT EXISTS earnings_events (
scripts/build_ticker_sector_map.py
+4
View File
@@ -34,6 +34,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
from app.services.sector_map import ( # noqa: E402
DEFAULT_SECTOR_MAP_PATH,
coverage_stats,
scripts/extend_snapshot_universe.py
+4
View File
@@ -45,6 +45,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
scripts/fetch_sector_etfs_to_snapshot.py
+4
View File
@@ -27,6 +27,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
from app.services.sector_map import SECTOR_ETFS # noqa: E402
scripts/run_earnings_research.py
+4
View File
@@ -28,6 +28,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
IRON_IC_BAR = 0.03
MIN_RELIABLE = 12
SUE_CARRY_DAYS = 63
scripts/run_history_depth_research.py
+4
View File
@@ -35,6 +35,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
ERA_SPLIT = date(2021, 1, 1)
SURVIVORSHIP_BANNER = (
"SURVIVORSHIP BIAS: today's constituents backfilled historically. "
scripts/run_sector_residual_research.py
+4
View File
@@ -40,6 +40,10 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
bootstrap_ssl()
from app.services.sector_map import ( # noqa: E402
DEFAULT_SECTOR_MAP_PATH,
SECTOR_ETFS,
scripts/run_tier1_macbook.sh
+92 -2
View File
@@ -33,11 +33,21 @@ ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}"
FMP_LIMIT="${FMP_LIMIT:-250}"
FMP_SLEEP="${FMP_SLEEP:-0.35}"
PYTHON="${PYTHON:-python3}"
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"
PHASE="depth" # depth | all | earnings | harness | coverage
PHASE="depth" # depth | all | earnings | harness | coverage | ssl-check
usage() {
sed -n '2,22p' "$0" | sed 's/^# \?//'
sed -n '2,25p' "$0" | sed 's/^# \?//'
cat <<'EOF'
SSL / network (corporate MacBook)
SSL errors usually mean the corp root CA is missing from Python.
1) Put combined-ca-bundle.pem in the repo root OR $HOME
2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem
3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh
4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check
EOF
exit "${1:-0}"
}
@@ -48,6 +58,8 @@ while [[ $# -gt 0 ]]; do
--harness-only) PHASE=harness; shift ;;
--coverage-only) PHASE=coverage; shift ;;
--depth) PHASE=depth; shift ;;
--ssl-check) PHASE=ssl; shift ;;
--corp-proxy) USE_CORP_PROXY=1; shift ;;
--prod-snap) PROD_SNAP="$2"; shift 2 ;;
--research-snap) RESEARCH_SNAP="$2"; shift 2 ;;
--history-days) HISTORY_DAYS="$2"; shift 2 ;;
@@ -71,6 +83,80 @@ fi
log() { printf '\n==> %s\n' "$*"; }
die() { echo "ERROR: $*" >&2; exit 1; }
# ---------------------------------------------------------------------------
# TLS bootstrap — same corp CA path the FastAPI app uses
# ---------------------------------------------------------------------------
setup_ssl() {
export USE_CORP_PROXY
# Prefer explicit env, then repo / home corporate bundle, then certifi.
if [[ -z "${SSL_CERT_FILE:-}" ]]; then
if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then
export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem"
elif [[ -f "$HOME/combined-ca-bundle.pem" ]]; then
export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem"
fi
fi
if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
export CURL_CA_BUNDLE="$SSL_CERT_FILE"
log "SSL CA bundle: $SSL_CERT_FILE"
else
# Fall back to certifi if installed
local certifi_path
certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)"
if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then
export SSL_CERT_FILE="$certifi_path"
export REQUESTS_CA_BUNDLE="$certifi_path"
export CURL_CA_BUNDLE="$certifi_path"
log "SSL CA bundle (certifi): $SSL_CERT_FILE"
else
log "WARNING: no CA bundle found — SSL may fail on corp networks"
log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/"
log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem"
fi
fi
if [[ "$USE_CORP_PROXY" == "1" ]]; then
export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}"
export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}"
export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}"
export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY"
log "Corp proxy enabled: $HTTPS_PROXY"
fi
# Ensure Python process sees the same bootstrap (patches ssl for alpaca-py).
export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}"
}
ssl_check() {
setup_ssl
log "SSL diagnostic"
"$PYTHON" - <<'PY'
from app.ssl_bootstrap import bootstrap_ssl, ssl_status
import json
import urllib.request
ca = bootstrap_ssl()
print(json.dumps(ssl_status(), indent=2))
print("bootstrap_ssl ->", ca)
urls = [
"https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
"https://financialmodelingprep.com/stable/profile?symbol=AAPL",
"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM",
]
for url in urls:
try:
req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"})
with urllib.request.urlopen(req, timeout=20) as resp:
print(f"OK {resp.status} {url[:60]}...")
except Exception as exc:
print(f"FAIL {type(exc).__name__}: {exc}")
print(f" {url[:80]}")
PY
}
need_file() {
[[ -f "$1" ]] || die "missing $1"
}
@@ -138,8 +224,12 @@ run_harness() {
}
log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS"
setup_ssl
case "$PHASE" in
ssl)
ssl_check
;;
coverage)
run_coverage
;;