The A5 cutover has been on and observed in production, so SEC Company Facts + DoltHub earnings are already the live source for `fundamental_data`. This removes everything the legacy path still occupied. Gone: the three providers and their config/env keys; the weekly `fundamental_collector` job; the cutover toggle (SEC + Dolt is now the unconditional path, so `off` can no longer silently freeze scoring inputs); the A5 parity report, whose deltas became structurally zero once the candidate builder started writing the table it compared against; and the FMP tier of universe bootstrap. Two behavioral notes: - Disabling **SEC Fundamentals Import** now stops the SEC network fetch only. The local cache refresh moved outside the job-enable check, because candidates also derive from daily closes and earnings events — freezing those on an ingestion pause would stale scoring with no fallback left to recover from. - `/ingestion/fetch?sources=fundamentals` still accepts the key and reports `skipped`; there is no per-ticker fetch any more. Migration 029 does not blanket-delete the leftover settings rows. Migrations run before the service restart, and pre-A6 code reads an absent `job_*_enabled` row as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe values (hidden in Admin) and only the inert three are deleted. Removing the provider keys from the production `.env` is the matching rollout step. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
141 lines
4.7 KiB
Bash
Executable File
141 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# One-time production provisioning for the shadow fundamentals sources.
|
|
# Run as root to install; run with --check as the deploy user for a read-only
|
|
# preflight. Version upgrades are intentional code changes, never "latest".
|
|
DOLT_VERSION="2.2.0"
|
|
DOLT_BINARY="${DOLT_BINARY:-/usr/local/bin/dolt}"
|
|
DOLT_DATA_DIR="${DOLT_DATA_DIR:-/var/lib/signal-platform/dolt}"
|
|
DOLT_EARNINGS_SUBDIR="${DOLT_EARNINGS_SUBDIR:-earnings}"
|
|
APP_USER="${APP_USER:-deploy}"
|
|
APP_GROUP="${APP_GROUP:-deploy}"
|
|
ENV_FILE="${ENV_FILE:-/opt/signalplatform/.env}"
|
|
MIN_FREE_GB="${DOLT_MIN_FREE_DISK_GB:-5}"
|
|
EARNINGS_DIR="${DOLT_DATA_DIR}/${DOLT_EARNINGS_SUBDIR}"
|
|
DOLT_IDENTITY_NAME="${DOLT_IDENTITY_NAME:-Signal Platform}"
|
|
DOLT_IDENTITY_EMAIL="${DOLT_IDENTITY_EMAIL:-signal-platform@localhost}"
|
|
|
|
fail() {
|
|
echo "ERROR: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
version_ok() {
|
|
local output
|
|
output="$("$DOLT_BINARY" version 2>/dev/null || true)"
|
|
grep -Eq "(^|[[:space:]])v?${DOLT_VERSION}([[:space:]]|$)" <<<"$output"
|
|
}
|
|
|
|
as_app_user() {
|
|
if [[ "$(id -un)" == "$APP_USER" ]]; then
|
|
"$@"
|
|
else
|
|
command -v runuser >/dev/null 2>&1 || fail "runuser is required"
|
|
runuser -u "$APP_USER" -- "$@"
|
|
fi
|
|
}
|
|
|
|
repo_command() {
|
|
(
|
|
cd "$EARNINGS_DIR"
|
|
as_app_user "$@"
|
|
)
|
|
}
|
|
|
|
repo_config_value() {
|
|
repo_command "$DOLT_BINARY" config --get "$1"
|
|
}
|
|
|
|
configure_identity() {
|
|
local name email
|
|
name="$(repo_config_value user.name 2>/dev/null || true)"
|
|
email="$(repo_config_value user.email 2>/dev/null || true)"
|
|
if [[ -z "$name" ]]; then
|
|
repo_command "$DOLT_BINARY" config --local --add user.name "$DOLT_IDENTITY_NAME"
|
|
fi
|
|
if [[ -z "$email" ]]; then
|
|
repo_command "$DOLT_BINARY" config --local --add user.email "$DOLT_IDENTITY_EMAIL"
|
|
fi
|
|
}
|
|
|
|
check_free_space() {
|
|
local available_kb
|
|
available_kb="$(df -Pk "$DOLT_DATA_DIR" | awk 'NR == 2 {print $4}')"
|
|
[[ "$available_kb" =~ ^[0-9]+$ ]] || fail "could not read free space for $DOLT_DATA_DIR"
|
|
if ! awk -v available="$available_kb" -v minimum_gb="$MIN_FREE_GB" \
|
|
'BEGIN { exit !(available >= minimum_gb * 1024 * 1024) }'; then
|
|
fail "$DOLT_DATA_DIR has less than ${MIN_FREE_GB} GB free"
|
|
fi
|
|
}
|
|
|
|
check_env() {
|
|
[[ -f "$ENV_FILE" ]] || fail "missing environment file: $ENV_FILE"
|
|
grep -Fqx "DOLT_BINARY=$DOLT_BINARY" "$ENV_FILE" \
|
|
|| fail "set DOLT_BINARY=$DOLT_BINARY in $ENV_FILE"
|
|
grep -Fqx "DOLT_DATA_DIR=$DOLT_DATA_DIR" "$ENV_FILE" \
|
|
|| fail "set DOLT_DATA_DIR=$DOLT_DATA_DIR in $ENV_FILE"
|
|
grep -Fqx "DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR" "$ENV_FILE" \
|
|
|| fail "set DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR in $ENV_FILE"
|
|
grep -Eq '^SEC_USER_AGENT=.*@.*' "$ENV_FILE" \
|
|
|| fail "SEC_USER_AGENT in $ENV_FILE must contain a real contact email"
|
|
}
|
|
|
|
check_all() {
|
|
local identity_name identity_email
|
|
id "$APP_USER" >/dev/null 2>&1 || fail "missing service user: $APP_USER"
|
|
[[ -x "$DOLT_BINARY" ]] || fail "missing Dolt binary: $DOLT_BINARY"
|
|
version_ok || fail "expected Dolt $DOLT_VERSION at $DOLT_BINARY"
|
|
[[ -d "$EARNINGS_DIR/.dolt" ]] \
|
|
|| fail "missing earnings clone: $EARNINGS_DIR"
|
|
if [[ "$(id -un)" == "$APP_USER" ]]; then
|
|
[[ -r "$EARNINGS_DIR/.dolt" ]] \
|
|
|| fail "earnings clone is not readable by $APP_USER"
|
|
elif command -v runuser >/dev/null 2>&1; then
|
|
runuser -u "$APP_USER" -- test -r "$EARNINGS_DIR/.dolt" \
|
|
|| fail "earnings clone is not readable by $APP_USER"
|
|
else
|
|
fail "run --check as $APP_USER (or install runuser)"
|
|
fi
|
|
identity_name="$(repo_config_value user.name 2>/dev/null || true)"
|
|
identity_email="$(repo_config_value user.email 2>/dev/null || true)"
|
|
[[ -n "$identity_name" ]] || fail "missing Dolt user.name for $EARNINGS_DIR"
|
|
[[ -n "$identity_email" ]] || fail "missing Dolt user.email for $EARNINGS_DIR"
|
|
check_free_space
|
|
check_env
|
|
echo "OK: Dolt $DOLT_VERSION and earnings clone are provisioned"
|
|
}
|
|
|
|
if [[ "${1:-}" == "--check" ]]; then
|
|
check_all
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$EUID" -eq 0 ]] || fail "run provisioning as root (or use --check)"
|
|
command -v curl >/dev/null 2>&1 || fail "curl is required"
|
|
command -v runuser >/dev/null 2>&1 || fail "runuser is required"
|
|
id "$APP_USER" >/dev/null 2>&1 || fail "missing service user: $APP_USER"
|
|
|
|
if ! version_ok; then
|
|
installer="$(mktemp)"
|
|
trap 'rm -f "$installer"' EXIT
|
|
curl -fsSL \
|
|
"https://github.com/dolthub/dolt/releases/download/v${DOLT_VERSION}/install.sh" \
|
|
-o "$installer"
|
|
bash "$installer"
|
|
fi
|
|
version_ok || fail "Dolt $DOLT_VERSION installation failed"
|
|
|
|
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$DOLT_DATA_DIR"
|
|
check_free_space
|
|
|
|
if [[ ! -d "$EARNINGS_DIR/.dolt" ]]; then
|
|
[[ ! -e "$EARNINGS_DIR" ]] \
|
|
|| fail "$EARNINGS_DIR exists but is not a Dolt clone"
|
|
runuser -u "$APP_USER" -- \
|
|
"$DOLT_BINARY" clone post-no-preference/earnings "$EARNINGS_DIR"
|
|
fi
|
|
|
|
configure_identity
|
|
check_all
|