From 32bf9c9297a8019cdce4ccc9f139f4d412c954af Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 09:34:49 +0200 Subject: [PATCH] research: bundle MacBook tier-1 pipeline into one bash script scripts/run_tier1_macbook.sh wraps earnings resume, coverage probe, deep snapshot rebuild, sector ETF refresh, and history-depth harness with phase flags. --- docs/research/history-depth-extension.md | 50 +++---- scripts/run_tier1_macbook.sh | 169 +++++++++++++++++++++++ 2 files changed, 187 insertions(+), 32 deletions(-) create mode 100644 scripts/run_tier1_macbook.sh diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 6ffcc16..e25fb92 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -45,45 +45,31 @@ the 2018 vol shock and full 2020 crash (where the feed allows). ## MacBook runbook +Prefer the bundled script (one entry point): + ```bash -# 0. Repo + env -git fetch origin -git checkout research/earnings-gap-and-sue # or history-depth branch once pushed -# ensure .env has ALPACA_* (and FMP if resuming earnings) +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local -# 1. (Optional) finish earnings backfill first — multi-day free tier -python scripts/backfill_earnings_events.py \ - --snapshot backtest_snapshots/prod.sqlite \ - --provider fmp --force-symbol --limit 250 --sleep 0.35 +chmod +x scripts/run_tier1_macbook.sh -# 2. Coverage probe (before long rebuild) -python scripts/run_history_depth_research.py --phase coverage \ - --snapshot backtest_snapshots/prod.sqlite +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh -# 3. Full deep rebuild of research.sqlite (LONG — Alpaca per symbol) -# Clears prior completion manifest; writes complete=true only at end. -python scripts/extend_snapshot_universe.py \ - --source backtest_snapshots/prod.sqlite \ - --output backtest_snapshots/research.sqlite \ - --force-copy \ - --history-days 5000 \ - --min-bars 260 \ - --sleep 0.15 +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only -# 4. Also refresh SPY + sector ETFs to the same depth on BOTH snapshots -python scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot backtest_snapshots/research.sqlite --history-days 5000 -python scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot backtest_snapshots/prod.sqlite --history-days 5000 - -# 5. Harness + era split (after race guard passes) -python scripts/run_history_depth_research.py --phase harness \ - --snapshot backtest_snapshots/research.sqlite \ - --workers 8 --allow-spawn - -# 6. Copy reports/ + docs/research/history-depth-extension.md results back +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 ``` +Then commit `reports/` + updated research docs, or copy them back to Windows. + --- ## Data provenance diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh new file mode 100644 index 0000000..5afbe20 --- /dev/null +++ b/scripts/run_tier1_macbook.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Tier-1 alpha research runner for a high-CPU MacBook (local only). +# +# Prerequisites +# - git checkout research/earnings-gap-and-sue (or later research branch) +# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume; +# optional ALPHA_VANTAGE_* as earnings fallback +# - Python venv with project deps installed +# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild) +# +# Usage +# chmod +x scripts/run_tier1_macbook.sh +# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness +# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline +# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b +# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only +# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only +# +# Does NOT touch production Postgres, scheduler, gates, or prod config. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# --- defaults (override via flags or env) --- +PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" +RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}" +HISTORY_DAYS="${HISTORY_DAYS:-5000}" +MIN_BARS="${MIN_BARS:-260}" +WORKERS="${WORKERS:-8}" +ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" +FMP_LIMIT="${FMP_LIMIT:-250}" +FMP_SLEEP="${FMP_SLEEP:-0.35}" +PYTHON="${PYTHON:-python3}" + +PHASE="depth" # depth | all | earnings | harness | coverage + +usage() { + sed -n '2,22p' "$0" | sed 's/^# \?//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --all) PHASE=all; shift ;; + --earnings-only) PHASE=earnings; shift ;; + --harness-only) PHASE=harness; shift ;; + --coverage-only) PHASE=coverage; shift ;; + --depth) PHASE=depth; shift ;; + --prod-snap) PROD_SNAP="$2"; shift 2 ;; + --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; + --history-days) HISTORY_DAYS="$2"; shift 2 ;; + --workers) WORKERS="$2"; shift 2 ;; + --fmp-limit) FMP_LIMIT="$2"; shift 2 ;; + --python) PYTHON="$2"; shift 2 ;; + -h|--help) usage 0 ;; + *) echo "Unknown flag: $1" >&2; usage 1 ;; + esac +done + +if [[ -x .venv/bin/python ]]; then + PYTHON=".venv/bin/python" +elif command -v "$PYTHON" >/dev/null 2>&1; then + : +else + echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2 + exit 1 +fi + +log() { printf '\n==> %s\n' "$*"; } +die() { echo "ERROR: $*" >&2; exit 1; } + +need_file() { + [[ -f "$1" ]] || die "missing $1" +} + +require_prod() { + need_file "$PROD_SNAP" +} + +run_earnings() { + require_prod + log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)" + "$PYTHON" scripts/backfill_earnings_events.py \ + --snapshot "$PROD_SNAP" \ + --provider fmp \ + --force-symbol \ + --limit "$FMP_LIMIT" \ + --sleep "$FMP_SLEEP" + + log "Earnings research 2a+2b (report only; no filters shipped)" + "$PYTHON" scripts/run_earnings_research.py \ + --snapshot "$PROD_SNAP" \ + --workers "$WORKERS" \ + --allow-spawn +} + +run_coverage() { + require_prod + log "Coverage probe (bars per year) on $PROD_SNAP" + "$PYTHON" scripts/run_history_depth_research.py \ + --phase coverage \ + --snapshot "$PROD_SNAP" +} + +run_rebuild() { + require_prod + log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)" + log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels" + "$PYTHON" scripts/extend_snapshot_universe.py \ + --source "$PROD_SNAP" \ + --output "$RESEARCH_SNAP" \ + --force-copy \ + --history-days "$HISTORY_DAYS" \ + --min-bars "$MIN_BARS" \ + --sleep "$ALPACA_SLEEP" + + log "Refresh SPY + 11 sector ETFs on research snapshot" + "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot "$RESEARCH_SNAP" \ + --history-days "$HISTORY_DAYS" + + log "Also deepen sector ETFs on prod snapshot (for local A/B parity)" + "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot "$PROD_SNAP" \ + --history-days "$HISTORY_DAYS" +} + +run_harness() { + need_file "$RESEARCH_SNAP" + log "Race-guard + full signal harness + era split on $RESEARCH_SNAP" + "$PYTHON" scripts/run_history_depth_research.py \ + --phase harness \ + --snapshot "$RESEARCH_SNAP" \ + --workers "$WORKERS" \ + --allow-spawn +} + +log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" + +case "$PHASE" in + coverage) + run_coverage + ;; + earnings) + run_earnings + ;; + harness) + run_harness + ;; + depth) + run_coverage + run_rebuild + run_harness + ;; + all) + run_earnings + run_coverage + run_rebuild + run_harness + ;; + *) + die "unknown phase $PHASE" + ;; +esac + +log "Done. Check reports/ and docs/research/history-depth-extension.md" +log "Commit reports on this machine if they look good, or copy them back to Windows."