dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: clean up closed Tier-1 scaffolding from branch

Browse Source 
Drop intermediate history-depth reports, sector-residual runners/map/code hooks
(evidence stays in final reports + docs), and slim MacBook helper to ssl/earnings/
prod-book-matrix only. SSL bootstrap and archived research conclusions retained.
This commit is contained in:
dennisthiessen
2026-07-19 14:41:52 +02:00
parent 1c38a94dd0
commit bb8aa655a1
22 changed files with 88 additions and 5450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_tier1_macbook.sh
+46 -198
View File
@@ -1,99 +1,70 @@
#!/usr/bin/env bash
# Tier-1 alpha research runner for a high-CPU MacBook (local only).
# Research helpers for a high-CPU MacBook (local only).
#
# Prerequisites
# - git checkout research/earnings-gap-and-sue (or later research branch)
# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume;
# optional ALPHA_VANTAGE_* as earnings fallback
# - Python venv with project deps installed
# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild)
# Kept after Tier-1 cleanup:
# --ssl-check diagnose corporate CA / proxy
# --earnings-only resume FMP earnings backfill + 2a/2b (parked)
# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix
#
# Prerequisites: git checkout research branch, .env, deep research.sqlite for
# book matrix, combined-ca-bundle.pem or certifi when on corp network.
#
# Usage
# chmod +x scripts/run_tier1_macbook.sh
# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness
# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline
# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b
# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only
# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only
# ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade
# ./scripts/run_tier1_macbook.sh --prod-book-matrix # 4-arm universe×horizon book matrix
#
# Does NOT touch production Postgres, scheduler, gates, or prod config.
# ./scripts/run_tier1_macbook.sh --ssl-check
# ./scripts/run_tier1_macbook.sh --prod-book-matrix
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
# --- defaults (override via flags or env) ---
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}"
HISTORY_DAYS="${HISTORY_DAYS:-5000}"
MIN_BARS="${MIN_BARS:-260}"
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
WORKERS="${WORKERS:-8}"
ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}"
FMP_LIMIT="${FMP_LIMIT:-250}"
FMP_SLEEP="${FMP_SLEEP:-0.35}"
PYTHON="${PYTHON:-python3}"
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"
PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book
PHASE=""
usage() {
sed -n '2,25p' "$0" | sed 's/^# \?//'
cat <<'EOF'
SSL / network (corporate MacBook)
SSL errors usually mean the corp root CA is missing from Python.
1) Put combined-ca-bundle.pem in the repo root OR $HOME
2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem
3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh
4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check
EOF
sed -n '2,16p' "$0" | sed 's/^# \?//'
exit "${1:-0}"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--all) PHASE=all; shift ;;
--earnings-only) PHASE=earnings; shift ;;
--harness-only) PHASE=harness; shift ;;
--coverage-only) PHASE=coverage; shift ;;
--depth) PHASE=depth; shift ;;
--ssl-check) PHASE=ssl; shift ;;
--sector-resid-deep) PHASE=sector_resid_deep; shift ;;
--earnings-only) PHASE=earnings; shift ;;
--prod-book-matrix) PHASE=prod_book; shift ;;
--corp-proxy) USE_CORP_PROXY=1; shift ;;
--prod-snap) PROD_SNAP="$2"; shift 2 ;;
--research-snap) RESEARCH_SNAP="$2"; shift 2 ;;
--history-days) HISTORY_DAYS="$2"; shift 2 ;;
--workers) WORKERS="$2"; shift 2 ;;
--fmp-limit) FMP_LIMIT="$2"; shift 2 ;;
--python) PYTHON="$2"; shift 2 ;;
-h|--help) usage 0 ;;
*) echo "Unknown flag: $1" >&2; usage 1 ;;
esac
done
if [[ -z "$PHASE" ]]; then
echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2
usage 1
fi
if [[ -x .venv/bin/python ]]; then
PYTHON=".venv/bin/python"
elif command -v "$PYTHON" >/dev/null 2>&1; then
:
else
echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2
echo "ERROR: no Python found" >&2
exit 1
fi
log() { printf '\n==> %s\n' "$*"; }
die() { echo "ERROR: $*" >&2; exit 1; }
need_file() { [[ -f "$1" ]] || die "missing $1"; }
# ---------------------------------------------------------------------------
# TLS bootstrap — same corp CA path the FastAPI app uses
# ---------------------------------------------------------------------------
setup_ssl() {
export USE_CORP_PROXY
# Prefer explicit env, then repo / home corporate bundle, then certifi.
if [[ -z "${SSL_CERT_FILE:-}" ]]; then
if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then
export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem"
@@ -101,191 +72,68 @@ setup_ssl() {
export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem"
fi
fi
if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
export CURL_CA_BUNDLE="$SSL_CERT_FILE"
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" CURL_CA_BUNDLE="$SSL_CERT_FILE"
log "SSL CA bundle: $SSL_CERT_FILE"
else
# Fall back to certifi if installed
local certifi_path
certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)"
if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then
export SSL_CERT_FILE="$certifi_path"
export REQUESTS_CA_BUNDLE="$certifi_path"
export CURL_CA_BUNDLE="$certifi_path"
export SSL_CERT_FILE="$certifi_path" REQUESTS_CA_BUNDLE="$certifi_path" CURL_CA_BUNDLE="$certifi_path"
log "SSL CA bundle (certifi): $SSL_CERT_FILE"
else
log "WARNING: no CA bundle found — SSL may fail on corp networks"
log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/"
log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem"
fi
fi
if [[ "$USE_CORP_PROXY" == "1" ]]; then
export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}"
export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}"
export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}"
export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost}"
export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY"
log "Corp proxy enabled: $HTTPS_PROXY"
fi
# Ensure Python process sees the same bootstrap (patches ssl for alpaca-py).
export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}"
}
ssl_check() {
setup_ssl
log "SSL diagnostic"
"$PYTHON" - <<'PY'
from app.ssl_bootstrap import bootstrap_ssl, ssl_status
import json
import urllib.request
ca = bootstrap_ssl()
import json, urllib.request
print(json.dumps(ssl_status(), indent=2))
print("bootstrap_ssl ->", ca)
urls = [
print("bootstrap ->", bootstrap_ssl())
for url in (
"https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
"https://financialmodelingprep.com/stable/profile?symbol=AAPL",
"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM",
]
for url in urls:
):
try:
req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"})
req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"})
with urllib.request.urlopen(req, timeout=20) as resp:
print(f"OK {resp.status} {url[:60]}...")
print(f"OK {resp.status} {url[:60]}")
except Exception as exc:
print(f"FAIL {type(exc).__name__}: {exc}")
print(f" {url[:80]}")
PY
}
need_file() {
[[ -f "$1" ]] || die "missing $1"
}
require_prod() {
need_file "$PROD_SNAP"
}
run_earnings() {
require_prod
log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)"
"$PYTHON" scripts/backfill_earnings_events.py \
--snapshot "$PROD_SNAP" \
--provider fmp \
--force-symbol \
--limit "$FMP_LIMIT" \
--sleep "$FMP_SLEEP"
log "Earnings research 2a+2b (report only; no filters shipped)"
"$PYTHON" scripts/run_earnings_research.py \
--snapshot "$PROD_SNAP" \
--workers "$WORKERS" \
--allow-spawn
}
run_coverage() {
require_prod
log "Coverage probe (bars per year) on $PROD_SNAP"
"$PYTHON" scripts/run_history_depth_research.py \
--phase coverage \
--snapshot "$PROD_SNAP"
}
run_rebuild() {
require_prod
log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)"
log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels"
"$PYTHON" scripts/extend_snapshot_universe.py \
--source "$PROD_SNAP" \
--output "$RESEARCH_SNAP" \
--force-copy \
--history-days "$HISTORY_DAYS" \
--min-bars "$MIN_BARS" \
--sleep "$ALPACA_SLEEP"
log "Refresh SPY + 11 sector ETFs on research snapshot"
"$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
--snapshot "$RESEARCH_SNAP" \
--history-days "$HISTORY_DAYS"
log "Also deepen sector ETFs on prod snapshot (for local A/B parity)"
"$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
--snapshot "$PROD_SNAP" \
--history-days "$HISTORY_DAYS"
}
run_harness() {
need_file "$RESEARCH_SNAP"
log "Race-guard + full signal harness + era split on $RESEARCH_SNAP"
"$PYTHON" scripts/run_history_depth_research.py \
--phase harness \
--snapshot "$RESEARCH_SNAP" \
--workers "$WORKERS" \
--allow-spawn
}
run_sector_resid_deep() {
need_file "$RESEARCH_SNAP"
log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade"
log "Pre-registered PASS/FAIL only — thread ends after this run"
"$PYTHON" scripts/run_sector_resid_deep_test.py \
--snapshot "$RESEARCH_SNAP" \
--history-days "$HISTORY_DAYS" \
--sleep "$ALPACA_SLEEP" \
--workers "$WORKERS" \
--allow-spawn
}
run_prod_book_matrix() {
need_file "$RESEARCH_SNAP"
log "Production book × universe × horizon (4 arms, strategy unchanged)"
"$PYTHON" scripts/run_prod_book_universe_matrix.py \
--snapshot "$RESEARCH_SNAP" \
--workers "$WORKERS" \
--allow-spawn \
--candidate-cache reports/.cache/prod-book-universe-cands.pkl
}
log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS"
setup_ssl
case "$PHASE" in
ssl)
ssl_check
;;
sector_resid_deep)
run_sector_resid_deep
ssl) ssl_check ;;
earnings)
need_file "$PROD_SNAP"
log "Earnings backfill + research (parked experiment)"
"$PYTHON" scripts/backfill_earnings_events.py \
--snapshot "$PROD_SNAP" --provider fmp --force-symbol \
--limit "$FMP_LIMIT" --sleep "$FMP_SLEEP"
"$PYTHON" scripts/run_earnings_research.py \
--snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn
;;
prod_book)
run_prod_book_matrix
;;
coverage)
run_coverage
;;
earnings)
run_earnings
;;
harness)
run_harness
;;
depth)
run_coverage
run_rebuild
run_harness
;;
all)
run_earnings
run_coverage
run_rebuild
run_harness
;;
*)
die "unknown phase $PHASE"
need_file "$RESEARCH_SNAP"
log "Production book universe × horizon matrix"
"$PYTHON" scripts/run_prod_book_universe_matrix.py \
--snapshot "$RESEARCH_SNAP" --workers "$WORKERS" --allow-spawn \
--candidate-cache reports/.cache/prod-book-universe-cands.pkl
;;
*) die "unknown phase $PHASE" ;;
esac
log "Done. Check reports/ and docs/research/history-depth-extension.md"
log "Commit reports on this machine if they look good, or copy them back to Windows."
log "Done."