signal-platform/scripts/run_tier1_macbook.sh

#!/usr/bin/env bash
# Tier-1 alpha research runner for a high-CPU MacBook (local only).
#
# Prerequisites
#   - git checkout research/earnings-gap-and-sue (or later research branch)
#   - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume;
#     optional ALPHA_VANTAGE_* as earnings fallback
#   - Python venv with project deps installed
#   - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild)
#
# Usage
#   chmod +x scripts/run_tier1_macbook.sh
#   ./scripts/run_tier1_macbook.sh                  # coverage + deep rebuild + harness
#   ./scripts/run_tier1_macbook.sh --all            # earnings resume + full depth pipeline
#   ./scripts/run_tier1_macbook.sh --earnings-only  # multi-day FMP backfill + re-run 2a/2b
#   ./scripts/run_tier1_macbook.sh --harness-only   # skip rebuild; race-guard + IC only
#   ./scripts/run_tier1_macbook.sh --coverage-only  # bars-per-year probe only
#   ./scripts/run_tier1_macbook.sh --sector-resid-deep  # deepen shallow + ONE masked grade
#   ./scripts/run_tier1_macbook.sh --prod-book-matrix   # 4-arm universe×horizon book matrix
#
# Does NOT touch production Postgres, scheduler, gates, or prod config.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"

# --- defaults (override via flags or env) ---
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}"
HISTORY_DAYS="${HISTORY_DAYS:-5000}"
MIN_BARS="${MIN_BARS:-260}"
WORKERS="${WORKERS:-8}"
ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}"
FMP_LIMIT="${FMP_LIMIT:-250}"
FMP_SLEEP="${FMP_SLEEP:-0.35}"
PYTHON="${PYTHON:-python3}"
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"

PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book

usage() {
  sed -n '2,25p' "$0" | sed 's/^# \?//'
  cat <<'EOF'

SSL / network (corporate MacBook)
  SSL errors usually mean the corp root CA is missing from Python.
  1) Put combined-ca-bundle.pem in the repo root OR $HOME
  2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem
  3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh
  4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check
EOF
  exit "${1:-0}"
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --all) PHASE=all; shift ;;
    --earnings-only) PHASE=earnings; shift ;;
    --harness-only) PHASE=harness; shift ;;
    --coverage-only) PHASE=coverage; shift ;;
    --depth) PHASE=depth; shift ;;
    --ssl-check) PHASE=ssl; shift ;;
    --sector-resid-deep) PHASE=sector_resid_deep; shift ;;
    --prod-book-matrix) PHASE=prod_book; shift ;;
    --corp-proxy) USE_CORP_PROXY=1; shift ;;
    --prod-snap) PROD_SNAP="$2"; shift 2 ;;
    --research-snap) RESEARCH_SNAP="$2"; shift 2 ;;
    --history-days) HISTORY_DAYS="$2"; shift 2 ;;
    --workers) WORKERS="$2"; shift 2 ;;
    --fmp-limit) FMP_LIMIT="$2"; shift 2 ;;
    --python) PYTHON="$2"; shift 2 ;;
    -h|--help) usage 0 ;;
    *) echo "Unknown flag: $1" >&2; usage 1 ;;
  esac
done

if [[ -x .venv/bin/python ]]; then
  PYTHON=".venv/bin/python"
elif command -v "$PYTHON" >/dev/null 2>&1; then
  :
else
  echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2
  exit 1
fi

log() { printf '\n==> %s\n' "$*"; }
die() { echo "ERROR: $*" >&2; exit 1; }

# ---------------------------------------------------------------------------
# TLS bootstrap — same corp CA path the FastAPI app uses
# ---------------------------------------------------------------------------
setup_ssl() {
  export USE_CORP_PROXY

  # Prefer explicit env, then repo / home corporate bundle, then certifi.
  if [[ -z "${SSL_CERT_FILE:-}" ]]; then
    if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then
      export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem"
    elif [[ -f "$HOME/combined-ca-bundle.pem" ]]; then
      export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem"
    fi
  fi

  if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then
    export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
    export CURL_CA_BUNDLE="$SSL_CERT_FILE"
    log "SSL CA bundle: $SSL_CERT_FILE"
  else
    # Fall back to certifi if installed
    local certifi_path
    certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)"
    if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then
      export SSL_CERT_FILE="$certifi_path"
      export REQUESTS_CA_BUNDLE="$certifi_path"
      export CURL_CA_BUNDLE="$certifi_path"
      log "SSL CA bundle (certifi): $SSL_CERT_FILE"
    else
      log "WARNING: no CA bundle found — SSL may fail on corp networks"
      log "  Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/"
      log "  Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem"
    fi
  fi

  if [[ "$USE_CORP_PROXY" == "1" ]]; then
    export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}"
    export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}"
    export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}"
    export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY"
    log "Corp proxy enabled: $HTTPS_PROXY"
  fi

  # Ensure Python process sees the same bootstrap (patches ssl for alpaca-py).
  export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}"
}

ssl_check() {
  setup_ssl
  log "SSL diagnostic"
  "$PYTHON" - <<'PY'
from app.ssl_bootstrap import bootstrap_ssl, ssl_status
import json
import urllib.request

ca = bootstrap_ssl()
print(json.dumps(ssl_status(), indent=2))
print("bootstrap_ssl ->", ca)
urls = [
    "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
    "https://financialmodelingprep.com/stable/profile?symbol=AAPL",
    "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM",
]
for url in urls:
    try:
        req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"})
        with urllib.request.urlopen(req, timeout=20) as resp:
            print(f"OK  {resp.status} {url[:60]}...")
    except Exception as exc:
        print(f"FAIL {type(exc).__name__}: {exc}")
        print(f"     {url[:80]}")
PY
}

need_file() {
  [[ -f "$1" ]] || die "missing $1"
}

require_prod() {
  need_file "$PROD_SNAP"
}

run_earnings() {
  require_prod
  log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)"
  "$PYTHON" scripts/backfill_earnings_events.py \
    --snapshot "$PROD_SNAP" \
    --provider fmp \
    --force-symbol \
    --limit "$FMP_LIMIT" \
    --sleep "$FMP_SLEEP"

  log "Earnings research 2a+2b (report only; no filters shipped)"
  "$PYTHON" scripts/run_earnings_research.py \
    --snapshot "$PROD_SNAP" \
    --workers "$WORKERS" \
    --allow-spawn
}

run_coverage() {
  require_prod
  log "Coverage probe (bars per year) on $PROD_SNAP"
  "$PYTHON" scripts/run_history_depth_research.py \
    --phase coverage \
    --snapshot "$PROD_SNAP"
}

run_rebuild() {
  require_prod
  log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)"
  log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels"
  "$PYTHON" scripts/extend_snapshot_universe.py \
    --source "$PROD_SNAP" \
    --output "$RESEARCH_SNAP" \
    --force-copy \
    --history-days "$HISTORY_DAYS" \
    --min-bars "$MIN_BARS" \
    --sleep "$ALPACA_SLEEP"

  log "Refresh SPY + 11 sector ETFs on research snapshot"
  "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
    --snapshot "$RESEARCH_SNAP" \
    --history-days "$HISTORY_DAYS"

  log "Also deepen sector ETFs on prod snapshot (for local A/B parity)"
  "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
    --snapshot "$PROD_SNAP" \
    --history-days "$HISTORY_DAYS"
}

run_harness() {
  need_file "$RESEARCH_SNAP"
  log "Race-guard + full signal harness + era split on $RESEARCH_SNAP"
  "$PYTHON" scripts/run_history_depth_research.py \
    --phase harness \
    --snapshot "$RESEARCH_SNAP" \
    --workers "$WORKERS" \
    --allow-spawn
}

run_sector_resid_deep() {
  need_file "$RESEARCH_SNAP"
  log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade"
  log "Pre-registered PASS/FAIL only — thread ends after this run"
  "$PYTHON" scripts/run_sector_resid_deep_test.py \
    --snapshot "$RESEARCH_SNAP" \
    --history-days "$HISTORY_DAYS" \
    --sleep "$ALPACA_SLEEP" \
    --workers "$WORKERS" \
    --allow-spawn
}

run_prod_book_matrix() {
  need_file "$RESEARCH_SNAP"
  log "Production book × universe × horizon (4 arms, strategy unchanged)"
  "$PYTHON" scripts/run_prod_book_universe_matrix.py \
    --snapshot "$RESEARCH_SNAP" \
    --workers "$WORKERS" \
    --allow-spawn \
    --candidate-cache reports/.cache/prod-book-universe-cands.pkl
}

log "cwd=$ROOT  python=$PYTHON  phase=$PHASE  workers=$WORKERS"
setup_ssl

case "$PHASE" in
  ssl)
    ssl_check
    ;;
  sector_resid_deep)
    run_sector_resid_deep
    ;;
  prod_book)
    run_prod_book_matrix
    ;;
  coverage)
    run_coverage
    ;;
  earnings)
    run_earnings
    ;;
  harness)
    run_harness
    ;;
  depth)
    run_coverage
    run_rebuild
    run_harness
    ;;
  all)
    run_earnings
    run_coverage
    run_rebuild
    run_harness
    ;;
  *)
    die "unknown phase $PHASE"
    ;;
esac

log "Done. Check reports/ and docs/research/history-depth-extension.md"
log "Commit reports on this machine if they look good, or copy them back to Windows."