dennisthiessenandClaude Opus 5
083c9dbf7c
Groundwork for the Admin -> Jobs cleanup. Three sources of truth collapse into
app/job_catalog.py, which imports nothing from app so both the scheduler and
admin_service can import it at module level (admin_service otherwise has to
import the scheduler inside functions to dodge a cycle).
PIPELINE_MEMBERS is now DERIVED from the four pipeline step lists instead of
being a literal set in admin_service duplicating four lists in scheduler.py with
nothing asserting they agreed. A test pins that the derivation reproduces the
previous hand-maintained 9 names exactly, so this is behaviour-preserving.
Deletes the private _JOB_NAMES list, which held 16 of the 19 jobs:
benchmark_collector, outcome_evaluator and shadow_book had no runtime row, and
so no "last run" line in the panel, until their first run in a given process.
_job_runtime is now seeded from the catalog, and a test pins the invariant.
Next-run is decided by category rather than by reading a timestamp. A pipeline
step has no schedule of its own, so it reports its parent's ("next via Morning
Pipeline in 3h") instead of nothing; a manual job says manual_only rather than
rendering a date. This also fixes a real bug: triggering a paused job set
next_run_time=now, APScheduler re-armed the 520-week backstop behind it, and the
panel displayed "next run in ~87600h". Two independent guards -- the category
rule, plus _visible_next_run dropping anything past a year -- and an APScheduler
listener that re-pauses steps and manual jobs once their run finishes. The
listener is registered at module level because configure_scheduler is called
more than once and add_listener does not deduplicate.
Migrates backtest and ticker_universe_sync from interval to cron (Sun 03:00 ET
and 01:00 ET). configure_scheduler calls remove_all_jobs() on every startup, so
an interval countdown restarts each deploy -- a 168h backtest needed a week of
uninterrupted uptime to fire even once. The codebase already documented this
pitfall as the reason cron was adopted; these two were never migrated. Both are
now editable in Admin -> Schedule.
Also: list_jobs went from one settings query per job (19) to one for all of
them, and data_backfill is hidden from the listing while staying registered and
API-triggerable.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
796 lines
31 KiB
Python
796 lines
31 KiB
Python
"""Admin service: user management, system settings, data cleanup, job control."""
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from passlib.hash import bcrypt
|
|
from sqlalchemy import delete, func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app import job_catalog
|
|
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
|
from app.models.fundamental import FundamentalData
|
|
from app.models.ohlcv import OHLCVRecord
|
|
from app.models.score import CompositeScore, DimensionScore
|
|
from app.models.sentiment import SentimentScore
|
|
from app.models.sr_level import SRLevel
|
|
from app.models.settings import SystemSetting
|
|
from app.models.ticker import Ticker
|
|
from app.models.trade_setup import TradeSetup
|
|
from app.models.user import User
|
|
from app.services import settings_store
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
RECOMMENDATION_CONFIG_DEFAULTS: dict[str, float] = {
|
|
"recommendation_high_confidence_threshold": 70.0,
|
|
"recommendation_moderate_confidence_threshold": 50.0,
|
|
"recommendation_confidence_diff_threshold": 20.0,
|
|
"recommendation_signal_alignment_weight": 0.15,
|
|
"recommendation_sr_strength_weight": 0.20,
|
|
"recommendation_momentum_technical_divergence_threshold": 30.0,
|
|
"recommendation_fundamental_technical_divergence_threshold": 40.0,
|
|
}
|
|
|
|
DEFAULT_TICKER_UNIVERSE = "sp500"
|
|
SUPPORTED_TICKER_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
|
|
|
|
# Activation gate: what counts as a signal worth acting on. Used by the
|
|
# Dashboard's "Qualified" metric, the Signals "Qualified only" view, and the
|
|
# Track Record's qualified stats. The outcome evaluator deliberately ignores
|
|
# these — every setup is evaluated so the gate itself can be validated.
|
|
#
|
|
# The core selection is residual cross-sectional 12-1 momentum (top percentile
|
|
# of the universe, long-only). R:R and confidence are floors; high-conviction /
|
|
# clean-read are optional tighteners (off by default).
|
|
_ACTIVATION_FLOAT_KEYS: dict[str, str] = {
|
|
"min_momentum_percentile": "activation_min_momentum_percentile",
|
|
"min_rr": "activation_min_rr",
|
|
"min_confidence": "activation_min_confidence",
|
|
}
|
|
_ACTIVATION_BOOL_KEYS: dict[str, str] = {
|
|
"require_high_conviction": "activation_require_high_conviction",
|
|
"exclude_conflicts": "activation_exclude_conflicts",
|
|
"exclude_neutral": "activation_exclude_neutral",
|
|
}
|
|
ACTIVATION_DEFAULTS: dict[str, float | bool] = {
|
|
"min_momentum_percentile": 80.0,
|
|
# Production floor from the 2026-07-12 min_rr sweep (in-sample and OOS peak).
|
|
# 1.2 was the old code default and the trough next to the spike — do not restore.
|
|
"min_rr": 2.0,
|
|
# 0 = off. The July 2026 gate ablation showed the confidence floor added
|
|
# nothing (identical net/trade with it removed, under both exit models)
|
|
# while cutting ~25% of qualified trades.
|
|
"min_confidence": 0.0,
|
|
"require_high_conviction": False,
|
|
"exclude_conflicts": False,
|
|
# On by default: a NEUTRAL ("no clear setup") recommendation isn't an
|
|
# actionable signal, so it shouldn't qualify or be crowned a top pick.
|
|
"exclude_neutral": True,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# User management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def list_users(db: AsyncSession) -> list[User]:
|
|
"""Return all users ordered by id."""
|
|
result = await db.execute(select(User).order_by(User.id))
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def create_user(
|
|
db: AsyncSession,
|
|
username: str,
|
|
password: str,
|
|
role: str = "user",
|
|
has_access: bool = False,
|
|
) -> User:
|
|
"""Create a new user account (admin action)."""
|
|
result = await db.execute(select(User).where(User.username == username))
|
|
if result.scalar_one_or_none() is not None:
|
|
raise DuplicateError(f"Username already exists: {username}")
|
|
|
|
user = User(
|
|
username=username,
|
|
password_hash=bcrypt.hash(password),
|
|
role=role,
|
|
has_access=has_access,
|
|
)
|
|
db.add(user)
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
async def set_user_access(db: AsyncSession, user_id: int, has_access: bool) -> User:
|
|
"""Grant or revoke API access for a user."""
|
|
result = await db.execute(select(User).where(User.id == user_id))
|
|
user = result.scalar_one_or_none()
|
|
if user is None:
|
|
raise NotFoundError(f"User not found: {user_id}")
|
|
|
|
user.has_access = has_access
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
async def reset_password(db: AsyncSession, user_id: int, new_password: str) -> User:
|
|
"""Reset a user's password."""
|
|
result = await db.execute(select(User).where(User.id == user_id))
|
|
user = result.scalar_one_or_none()
|
|
if user is None:
|
|
raise NotFoundError(f"User not found: {user_id}")
|
|
|
|
user.password_hash = bcrypt.hash(new_password)
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Registration toggle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def toggle_registration(db: AsyncSession, enabled: bool) -> SystemSetting:
|
|
"""Enable or disable user registration via SystemSetting."""
|
|
setting = await settings_store.upsert_setting(db, "registration_enabled", str(enabled).lower())
|
|
await db.commit()
|
|
await db.refresh(setting)
|
|
return setting
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# System settings CRUD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def list_settings(db: AsyncSession) -> list[SystemSetting]:
|
|
"""Return all system settings."""
|
|
result = await db.execute(select(SystemSetting).order_by(SystemSetting.key))
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSetting:
|
|
"""Create or update a system setting."""
|
|
setting = await settings_store.upsert_setting(db, key, value)
|
|
await db.commit()
|
|
await db.refresh(setting)
|
|
return setting
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Activation thresholds
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_activation_config(db: AsyncSession) -> dict[str, float | bool]:
|
|
"""Return the activation gate config with public keys."""
|
|
result = await db.execute(
|
|
select(SystemSetting).where(SystemSetting.key.like("activation_%"))
|
|
)
|
|
stored = {s.key: s.value for s in result.scalars().all()}
|
|
|
|
config: dict[str, float | bool] = dict(ACTIVATION_DEFAULTS)
|
|
for public_key, storage_key in _ACTIVATION_FLOAT_KEYS.items():
|
|
if storage_key in stored:
|
|
try:
|
|
config[public_key] = float(stored[storage_key])
|
|
except (TypeError, ValueError):
|
|
pass
|
|
for public_key, storage_key in _ACTIVATION_BOOL_KEYS.items():
|
|
if storage_key in stored:
|
|
config[public_key] = str(stored[storage_key]).strip().lower() == "true"
|
|
return config
|
|
|
|
|
|
async def update_activation_config(
|
|
db: AsyncSession, updates: dict[str, float | bool]
|
|
) -> dict[str, float | bool]:
|
|
"""Update the activation gate. Accepts public keys; only supplied keys change."""
|
|
if "min_momentum_percentile" in updates and not 0 <= updates["min_momentum_percentile"] <= 100:
|
|
raise ValidationError("min_momentum_percentile must be between 0 and 100")
|
|
if "min_rr" in updates and updates["min_rr"] < 0:
|
|
raise ValidationError("min_rr must be >= 0")
|
|
if "min_confidence" in updates and not 0 <= updates["min_confidence"] <= 100:
|
|
raise ValidationError("min_confidence must be between 0 and 100")
|
|
|
|
for public_key, storage_key in _ACTIVATION_FLOAT_KEYS.items():
|
|
if public_key in updates and updates[public_key] is not None:
|
|
await update_setting(db, storage_key, str(float(updates[public_key])))
|
|
for public_key, storage_key in _ACTIVATION_BOOL_KEYS.items():
|
|
if public_key in updates and updates[public_key] is not None:
|
|
await update_setting(db, storage_key, "true" if updates[public_key] else "false")
|
|
|
|
return await get_activation_config(db)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Performance window + shadow book
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_performance_config(db: AsyncSession) -> dict:
|
|
"""Start date for the Performance comparison ('' = all history)."""
|
|
from app.services.paper_trade_service import KEY_PERFORMANCE_START
|
|
|
|
return {"start_date": await settings_store.get_value(db, KEY_PERFORMANCE_START, "") or ""}
|
|
|
|
|
|
async def update_performance_config(db: AsyncSession, updates: dict) -> dict:
|
|
"""Set (or clear) the performance start date. Empty string means all history."""
|
|
from datetime import date as _date
|
|
|
|
from app.services.paper_trade_service import KEY_PERFORMANCE_START
|
|
|
|
if "start_date" in updates:
|
|
raw = (updates.get("start_date") or "").strip()
|
|
if raw:
|
|
try:
|
|
_date.fromisoformat(raw)
|
|
except ValueError as exc:
|
|
raise ValidationError("start_date must be an ISO date (YYYY-MM-DD)") from exc
|
|
await update_setting(db, KEY_PERFORMANCE_START, raw)
|
|
return await get_performance_config(db)
|
|
|
|
|
|
async def get_shadow_book_config(db: AsyncSession) -> dict:
|
|
"""Shadow book switch + sizing, with the validated defaults filled in."""
|
|
from app.services import shadow_book_service
|
|
|
|
config = await shadow_book_service.get_config(db)
|
|
config["enabled"] = await shadow_book_service.is_enabled(db)
|
|
return config
|
|
|
|
|
|
async def update_shadow_book_config(db: AsyncSession, updates: dict) -> dict:
|
|
"""Update the shadow book. Enabling it starts automatic live entries."""
|
|
from app.services import shadow_book_service
|
|
|
|
if "enabled" in updates:
|
|
await update_setting(
|
|
db, shadow_book_service.KEY_ENABLED, "true" if updates["enabled"] else "false"
|
|
)
|
|
for key, storage_key in (
|
|
("capacity", shadow_book_service.KEY_CAPACITY),
|
|
("risk_pct", shadow_book_service.KEY_RISK_PCT),
|
|
("start_equity", shadow_book_service.KEY_START_EQUITY),
|
|
):
|
|
if key in updates:
|
|
await update_setting(db, storage_key, str(updates[key]))
|
|
return await get_shadow_book_config(db)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pipeline schedule (cron)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_schedule_config(db: AsyncSession) -> dict[str, str]:
|
|
"""Cron schedule for the daily/intraday pipelines and fundamentals."""
|
|
from app.scheduler import load_schedule_config
|
|
|
|
return await load_schedule_config(db)
|
|
|
|
|
|
async def update_schedule_config(
|
|
db: AsyncSession, updates: dict[str, str]
|
|
) -> dict[str, str]:
|
|
"""Validate, persist, and apply cron schedule changes to the running scheduler."""
|
|
from app.scheduler import (
|
|
SCHEDULE_DEFAULTS,
|
|
load_schedule_config,
|
|
reschedule_jobs,
|
|
validate_cron,
|
|
)
|
|
|
|
current = await load_schedule_config(db)
|
|
tz = (updates.get("schedule_timezone") or current["schedule_timezone"]).strip()
|
|
|
|
for key, value in updates.items():
|
|
if key not in SCHEDULE_DEFAULTS:
|
|
raise ValidationError(f"Unknown schedule key: {key}")
|
|
if key == "schedule_timezone":
|
|
# Validate the timezone against an existing cron expression.
|
|
try:
|
|
validate_cron(current["schedule_daily_pipeline_cron"], value)
|
|
except Exception as exc:
|
|
raise ValidationError(f"Invalid timezone: {value}") from exc
|
|
else:
|
|
try:
|
|
validate_cron(value, tz)
|
|
except Exception as exc:
|
|
raise ValidationError(f"Invalid cron for {key}: {value!r}") from exc
|
|
|
|
for key, value in updates.items():
|
|
await update_setting(db, key, str(value).strip())
|
|
|
|
new_config = await load_schedule_config(db)
|
|
try:
|
|
reschedule_jobs(new_config)
|
|
except Exception:
|
|
# Scheduler may not be running (e.g. unit tests) — the config is saved
|
|
# regardless and applied on next startup.
|
|
logger.warning("Could not reschedule jobs after config update", exc_info=True)
|
|
return new_config
|
|
|
|
|
|
def _recommendation_public_to_storage_key(key: str) -> str:
|
|
return f"recommendation_{key}"
|
|
|
|
|
|
async def get_recommendation_config(db: AsyncSession) -> dict[str, float]:
|
|
result = await db.execute(
|
|
select(SystemSetting).where(SystemSetting.key.like("recommendation_%"))
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
config = dict(RECOMMENDATION_CONFIG_DEFAULTS)
|
|
for row in rows:
|
|
try:
|
|
config[row.key] = float(row.value)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return {
|
|
"high_confidence_threshold": config["recommendation_high_confidence_threshold"],
|
|
"moderate_confidence_threshold": config["recommendation_moderate_confidence_threshold"],
|
|
"confidence_diff_threshold": config["recommendation_confidence_diff_threshold"],
|
|
"signal_alignment_weight": config["recommendation_signal_alignment_weight"],
|
|
"sr_strength_weight": config["recommendation_sr_strength_weight"],
|
|
"momentum_technical_divergence_threshold": config["recommendation_momentum_technical_divergence_threshold"],
|
|
"fundamental_technical_divergence_threshold": config["recommendation_fundamental_technical_divergence_threshold"],
|
|
}
|
|
|
|
|
|
async def update_recommendation_config(
|
|
db: AsyncSession,
|
|
payload: dict[str, float],
|
|
) -> dict[str, float]:
|
|
for public_key, public_value in payload.items():
|
|
storage_key = _recommendation_public_to_storage_key(public_key)
|
|
await update_setting(db, storage_key, str(public_value))
|
|
|
|
return await get_recommendation_config(db)
|
|
|
|
|
|
async def get_ticker_universe_default(db: AsyncSession) -> dict[str, str]:
|
|
setting = await settings_store.get_setting(db, "ticker_universe_default")
|
|
universe = setting.value if setting else DEFAULT_TICKER_UNIVERSE
|
|
if universe not in SUPPORTED_TICKER_UNIVERSES:
|
|
universe = DEFAULT_TICKER_UNIVERSE
|
|
return {"universe": universe}
|
|
|
|
|
|
async def update_ticker_universe_default(db: AsyncSession, universe: str) -> dict[str, str]:
|
|
normalised = universe.strip().lower()
|
|
if normalised not in SUPPORTED_TICKER_UNIVERSES:
|
|
supported = ", ".join(sorted(SUPPORTED_TICKER_UNIVERSES))
|
|
raise ValidationError(f"Unsupported ticker universe '{universe}'. Supported: {supported}")
|
|
|
|
await update_setting(db, "ticker_universe_default", normalised)
|
|
return {"universe": normalised}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data cleanup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def cleanup_data(db: AsyncSession, older_than_days: int) -> dict:
|
|
"""Delete OHLCV, sentiment, and fundamental records older than N days.
|
|
|
|
Preserves tickers, users, and latest scores. After OHLCV pruning, rebuilds
|
|
Structural S/R for every ticker so chart levels match the remaining history.
|
|
|
|
Returns deleted-row counts plus S/R refresh outcomes. A per-ticker S/R
|
|
failure rolls the session back (so later tickers still run) and is listed
|
|
in ``sr_refresh_failures`` rather than aborting the whole cleanup.
|
|
"""
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=older_than_days)
|
|
counts: dict = {}
|
|
|
|
# OHLCV — date column is a date, compare with cutoff date
|
|
result = await db.execute(
|
|
delete(OHLCVRecord).where(OHLCVRecord.date < cutoff.date())
|
|
)
|
|
counts["ohlcv"] = result.rowcount # type: ignore[assignment]
|
|
|
|
# Sentiment — timestamp is datetime
|
|
result = await db.execute(
|
|
delete(SentimentScore).where(SentimentScore.timestamp < cutoff)
|
|
)
|
|
counts["sentiment"] = result.rowcount # type: ignore[assignment]
|
|
|
|
# Fundamentals — fetched_at is datetime
|
|
result = await db.execute(
|
|
delete(FundamentalData).where(FundamentalData.fetched_at < cutoff)
|
|
)
|
|
counts["fundamentals"] = result.rowcount # type: ignore[assignment]
|
|
|
|
await db.commit()
|
|
|
|
counts["sr_refresh_ok"] = 0
|
|
counts["sr_refresh_failed"] = 0
|
|
counts["sr_refresh_failures"] = []
|
|
|
|
# Structural S/R is derived from OHLCV; recompute after history shrinks.
|
|
if counts["ohlcv"]:
|
|
from app.services.sr_service import recalculate_sr_levels
|
|
|
|
symbols = list(
|
|
(await db.execute(select(Ticker.symbol).order_by(Ticker.symbol))).scalars().all()
|
|
)
|
|
for symbol in symbols:
|
|
try:
|
|
await recalculate_sr_levels(db, symbol)
|
|
counts["sr_refresh_ok"] += 1
|
|
except Exception as exc:
|
|
logger.exception("S/R refresh after cleanup failed for %s", symbol)
|
|
try:
|
|
await db.rollback()
|
|
except Exception:
|
|
logger.exception(
|
|
"Session rollback after S/R cleanup failure also failed for %s",
|
|
symbol,
|
|
)
|
|
counts["sr_refresh_failed"] += 1
|
|
counts["sr_refresh_failures"].append(
|
|
{"symbol": symbol, "error": f"{type(exc).__name__}: {exc}"}
|
|
)
|
|
|
|
return counts
|
|
|
|
|
|
async def reset_trade_setups(db: AsyncSession) -> dict[str, int]:
|
|
"""Delete all trade setups, wiping the track record for a fresh start.
|
|
|
|
Stats are derived from evaluated trade setups, so this resets the Track
|
|
Record to zero. Live setups regenerate on the next R:R scan. Used after
|
|
material changes to scoring / setup generation, when historical outcomes no
|
|
longer reflect current logic.
|
|
"""
|
|
result = await db.execute(delete(TradeSetup))
|
|
await db.commit()
|
|
return {"trade_setups": result.rowcount} # type: ignore[attr-defined]
|
|
|
|
|
|
async def get_pipeline_readiness(db: AsyncSession) -> list[dict]:
|
|
"""Return per-ticker readiness snapshot for ingestion/scoring/scanner pipeline."""
|
|
tickers_result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
|
|
tickers = list(tickers_result.scalars().all())
|
|
|
|
if not tickers:
|
|
return []
|
|
|
|
ticker_ids = [ticker.id for ticker in tickers]
|
|
|
|
ohlcv_stats_result = await db.execute(
|
|
select(
|
|
OHLCVRecord.ticker_id,
|
|
func.count(OHLCVRecord.id),
|
|
func.max(OHLCVRecord.date),
|
|
)
|
|
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
|
.group_by(OHLCVRecord.ticker_id)
|
|
)
|
|
ohlcv_stats = {
|
|
ticker_id: {
|
|
"bars": int(count or 0),
|
|
"last_date": max_date.isoformat() if max_date else None,
|
|
}
|
|
for ticker_id, count, max_date in ohlcv_stats_result.all()
|
|
}
|
|
|
|
dim_rows_result = await db.execute(
|
|
select(DimensionScore).where(DimensionScore.ticker_id.in_(ticker_ids))
|
|
)
|
|
dim_map_by_ticker: dict[int, dict[str, tuple[float | None, bool]]] = {}
|
|
for row in dim_rows_result.scalars().all():
|
|
dim_map_by_ticker.setdefault(row.ticker_id, {})[row.dimension] = (row.score, row.is_stale)
|
|
|
|
sr_counts_result = await db.execute(
|
|
select(SRLevel.ticker_id, func.count(SRLevel.id))
|
|
.where(SRLevel.ticker_id.in_(ticker_ids))
|
|
.group_by(SRLevel.ticker_id)
|
|
)
|
|
sr_counts = {ticker_id: int(count or 0) for ticker_id, count in sr_counts_result.all()}
|
|
|
|
sentiment_stats_result = await db.execute(
|
|
select(
|
|
SentimentScore.ticker_id,
|
|
func.count(SentimentScore.id),
|
|
func.max(SentimentScore.timestamp),
|
|
)
|
|
.where(SentimentScore.ticker_id.in_(ticker_ids))
|
|
.group_by(SentimentScore.ticker_id)
|
|
)
|
|
sentiment_stats = {
|
|
ticker_id: {
|
|
"count": int(count or 0),
|
|
"last_at": max_ts.isoformat() if max_ts else None,
|
|
}
|
|
for ticker_id, count, max_ts in sentiment_stats_result.all()
|
|
}
|
|
|
|
fundamentals_result = await db.execute(
|
|
select(FundamentalData.ticker_id, FundamentalData.fetched_at)
|
|
.where(FundamentalData.ticker_id.in_(ticker_ids))
|
|
)
|
|
fundamentals_map = {
|
|
ticker_id: fetched_at.isoformat() if fetched_at else None
|
|
for ticker_id, fetched_at in fundamentals_result.all()
|
|
}
|
|
|
|
composites_result = await db.execute(
|
|
select(CompositeScore.ticker_id, CompositeScore.is_stale)
|
|
.where(CompositeScore.ticker_id.in_(ticker_ids))
|
|
)
|
|
composites_map = {
|
|
ticker_id: is_stale
|
|
for ticker_id, is_stale in composites_result.all()
|
|
}
|
|
|
|
setup_counts_result = await db.execute(
|
|
select(TradeSetup.ticker_id, func.count(TradeSetup.id))
|
|
.where(TradeSetup.ticker_id.in_(ticker_ids))
|
|
.group_by(TradeSetup.ticker_id)
|
|
)
|
|
setup_counts = {ticker_id: int(count or 0) for ticker_id, count in setup_counts_result.all()}
|
|
|
|
readiness: list[dict] = []
|
|
for ticker in tickers:
|
|
ohlcv = ohlcv_stats.get(ticker.id, {"bars": 0, "last_date": None})
|
|
ohlcv_bars = int(ohlcv["bars"])
|
|
ohlcv_last_date = ohlcv["last_date"]
|
|
|
|
dim_map = dim_map_by_ticker.get(ticker.id, {})
|
|
|
|
sr_count = int(sr_counts.get(ticker.id, 0))
|
|
|
|
sentiment = sentiment_stats.get(ticker.id, {"count": 0, "last_at": None})
|
|
sentiment_count = int(sentiment["count"])
|
|
sentiment_last_at = sentiment["last_at"]
|
|
|
|
fundamentals_fetched_at = fundamentals_map.get(ticker.id)
|
|
has_fundamentals = ticker.id in fundamentals_map
|
|
|
|
has_composite = ticker.id in composites_map
|
|
composite_stale = composites_map.get(ticker.id)
|
|
|
|
setup_count = int(setup_counts.get(ticker.id, 0))
|
|
|
|
missing_reasons: list[str] = []
|
|
if ohlcv_bars < 30:
|
|
missing_reasons.append("insufficient_ohlcv_bars(<30)")
|
|
if "technical" not in dim_map or dim_map["technical"][0] is None:
|
|
missing_reasons.append("missing_technical")
|
|
if "momentum" not in dim_map or dim_map["momentum"][0] is None:
|
|
missing_reasons.append("missing_momentum")
|
|
if "sr_quality" not in dim_map or dim_map["sr_quality"][0] is None:
|
|
missing_reasons.append("missing_sr_quality")
|
|
if sentiment_count == 0:
|
|
missing_reasons.append("missing_sentiment")
|
|
if not has_fundamentals:
|
|
missing_reasons.append("missing_fundamentals")
|
|
if not has_composite:
|
|
missing_reasons.append("missing_composite")
|
|
if setup_count == 0:
|
|
missing_reasons.append("missing_trade_setup")
|
|
|
|
readiness.append(
|
|
{
|
|
"symbol": ticker.symbol,
|
|
"ohlcv_bars": ohlcv_bars,
|
|
"ohlcv_last_date": ohlcv_last_date,
|
|
"dimensions": {
|
|
"technical": dim_map.get("technical", (None, True))[0],
|
|
"sr_quality": dim_map.get("sr_quality", (None, True))[0],
|
|
"sentiment": dim_map.get("sentiment", (None, True))[0],
|
|
"fundamental": dim_map.get("fundamental", (None, True))[0],
|
|
"momentum": dim_map.get("momentum", (None, True))[0],
|
|
},
|
|
"sentiment_count": sentiment_count,
|
|
"sentiment_last_at": sentiment_last_at,
|
|
"has_fundamentals": has_fundamentals,
|
|
"fundamentals_fetched_at": fundamentals_fetched_at,
|
|
"sr_level_count": sr_count,
|
|
"has_composite": has_composite,
|
|
"composite_stale": composite_stale,
|
|
"trade_setup_count": setup_count,
|
|
"missing_reasons": missing_reasons,
|
|
"ready_for_scanner": ohlcv_bars >= 15 and sr_count > 0,
|
|
}
|
|
)
|
|
|
|
return readiness
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Job control (placeholder — scheduler is Task 12.1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Job identity, labels and pipeline membership now live in app.job_catalog, which
|
|
# derives PIPELINE_MEMBERS from the pipeline step lists instead of restating them.
|
|
# Re-exported here because callers (routers, tests) import them from this module.
|
|
VALID_JOB_NAMES = job_catalog.VALID_JOB_NAMES
|
|
JOB_LABELS = job_catalog.JOB_LABELS
|
|
PIPELINE_MEMBERS = job_catalog.PIPELINE_MEMBERS
|
|
|
|
# Anything further out than this is a parked backstop, not a schedule: pipeline
|
|
# steps and manual jobs are registered on a 520-week interval, and triggering one
|
|
# re-arms it. Belt-and-braces behind the category rule in _next_run_fields.
|
|
_NEXT_RUN_HORIZON_DAYS = 365
|
|
|
|
|
|
def _visible_next_run(next_run: datetime | None) -> datetime | None:
|
|
"""Drop a next-run that is really the parked backstop."""
|
|
if next_run is None:
|
|
return None
|
|
horizon = datetime.now(next_run.tzinfo) + timedelta(days=_NEXT_RUN_HORIZON_DAYS)
|
|
return None if next_run > horizon else next_run
|
|
|
|
|
|
def _own_next_run(scheduler, name: str) -> datetime | None:
|
|
# getattr: APScheduler only sets next_run_time once the scheduler is running,
|
|
# so a job registered but not yet started has no such attribute at all.
|
|
job = scheduler.get_job(name)
|
|
return _visible_next_run(getattr(job, "next_run_time", None)) if job else None
|
|
|
|
|
|
def _next_run_fields(scheduler, name: str, enabled_map: dict[str, bool]) -> dict:
|
|
"""Where this job's next run comes from, decided by category not by clock.
|
|
|
|
A pipeline step has no meaningful schedule of its own, so reporting one is
|
|
the bug: its parent's timer is the answer. Manual jobs have no answer at all,
|
|
and saying so beats rendering a parked backstop as a date.
|
|
"""
|
|
category = job_catalog.JOB_CATEGORY.get(name)
|
|
if category == job_catalog.CATEGORY_STEP:
|
|
parents = job_catalog.PIPELINES_BY_MEMBER.get(name, ())
|
|
soonest: datetime | None = None
|
|
via: str | None = None
|
|
for parent in parents:
|
|
if not enabled_map.get(parent, True):
|
|
continue
|
|
candidate = _own_next_run(scheduler, parent)
|
|
if candidate is not None and (soonest is None or candidate < soonest):
|
|
soonest, via = candidate, parent
|
|
return {
|
|
"next_run_at": None,
|
|
"next_run_source": "via_pipeline",
|
|
"via_next_run_at": soonest.isoformat() if soonest else None,
|
|
"via_next_run_job": via,
|
|
}
|
|
if category == job_catalog.CATEGORY_MANUAL:
|
|
return {
|
|
"next_run_at": None,
|
|
"next_run_source": "manual_only",
|
|
"via_next_run_at": None,
|
|
"via_next_run_job": None,
|
|
}
|
|
own = _own_next_run(scheduler, name)
|
|
return {
|
|
"next_run_at": own.isoformat() if own else None,
|
|
"next_run_source": "own_schedule",
|
|
"via_next_run_at": None,
|
|
"via_next_run_job": None,
|
|
}
|
|
|
|
|
|
async def list_jobs(db: AsyncSession) -> list[dict]:
|
|
"""Return status of all scheduled jobs, grouped and ordered by category."""
|
|
from app.scheduler import get_job_runtime_snapshot, scheduler
|
|
|
|
visible = sorted(VALID_JOB_NAMES - job_catalog.HIDDEN_JOBS, key=job_catalog.sort_order)
|
|
# One query for every flag instead of one per job. Parents are read too, since
|
|
# a step reports its parent's next run only while that parent is enabled.
|
|
flags = await settings_store.get_map(
|
|
db, [f"job_{name}_enabled" for name in VALID_JOB_NAMES]
|
|
)
|
|
enabled_map = {
|
|
name: flags.get(f"job_{name}_enabled", "true") == "true"
|
|
for name in VALID_JOB_NAMES
|
|
}
|
|
|
|
jobs_out = []
|
|
for name in visible:
|
|
job = scheduler.get_job(name)
|
|
runtime = get_job_runtime_snapshot(name)
|
|
|
|
jobs_out.append({
|
|
"name": name,
|
|
"label": JOB_LABELS.get(name, name),
|
|
"enabled": enabled_map.get(name, True),
|
|
"category": job_catalog.JOB_CATEGORY.get(name),
|
|
"sort_order": job_catalog.sort_order(name),
|
|
# Parent pipelines for a step; the steps themselves for a pipeline.
|
|
"pipelines": list(job_catalog.PIPELINES_BY_MEMBER.get(name, ())),
|
|
"steps": [step for step, _ in job_catalog.PIPELINE_STEPS.get(name, ())],
|
|
"registered": job is not None,
|
|
"running": bool(runtime.get("running", False)),
|
|
# runtime_* are strictly live in-memory state. Persisted history is
|
|
# reported separately as last_run_*, so a stale error cannot pin the
|
|
# status chip or the rate-limit banner.
|
|
"runtime_status": runtime.get("status"),
|
|
"runtime_processed": runtime.get("processed"),
|
|
"runtime_total": runtime.get("total"),
|
|
"runtime_progress_pct": runtime.get("progress_pct"),
|
|
"runtime_current_ticker": runtime.get("current_ticker"),
|
|
"runtime_started_at": runtime.get("started_at"),
|
|
"runtime_finished_at": runtime.get("finished_at"),
|
|
"runtime_message": runtime.get("message"),
|
|
**_next_run_fields(scheduler, name, enabled_map),
|
|
})
|
|
|
|
return jobs_out
|
|
|
|
|
|
async def trigger_job(
|
|
db: AsyncSession,
|
|
job_name: str,
|
|
*,
|
|
target_model: str | None = None,
|
|
cadence: str | None = None,
|
|
) -> dict[str, str]:
|
|
"""Trigger a manual job run via the scheduler.
|
|
|
|
Runs the job immediately (in addition to its regular schedule).
|
|
"""
|
|
if job_name not in VALID_JOB_NAMES:
|
|
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
|
if target_model is not None and job_name != "backtest":
|
|
raise ValidationError("target_model is supported only for the backtest job")
|
|
if cadence is not None and job_name != "backtest":
|
|
raise ValidationError("cadence is supported only for the backtest job")
|
|
|
|
from app.scheduler import get_job_runtime_snapshot, scheduler
|
|
|
|
runtime_target = get_job_runtime_snapshot(job_name)
|
|
if runtime_target.get("running"):
|
|
return {
|
|
"job": job_name,
|
|
"status": "busy",
|
|
"message": f"Job '{job_name}' is already running",
|
|
}
|
|
|
|
all_runtime = get_job_runtime_snapshot()
|
|
for running_name, runtime in all_runtime.items():
|
|
if running_name == job_name:
|
|
continue
|
|
if runtime.get("running"):
|
|
return {
|
|
"job": job_name,
|
|
"status": "blocked",
|
|
"message": f"Cannot trigger '{job_name}' while '{running_name}' is running",
|
|
}
|
|
|
|
job = scheduler.get_job(job_name)
|
|
if job is None:
|
|
return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"}
|
|
|
|
if job_name == "backtest":
|
|
from app.scheduler import queue_backtest_options
|
|
|
|
target_model, cadence = queue_backtest_options(target_model, cadence)
|
|
|
|
job.modify(next_run_time=None) # Reset, then trigger immediately
|
|
from datetime import datetime, timezone
|
|
job.modify(next_run_time=datetime.now(timezone.utc))
|
|
|
|
result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
|
|
if target_model is not None:
|
|
result["target_model"] = target_model
|
|
if cadence is not None:
|
|
result["cadence"] = cadence
|
|
return result
|
|
|
|
|
|
async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSetting:
|
|
"""Enable or disable a scheduled job by storing state in SystemSetting.
|
|
|
|
Actual scheduler integration happens in Task 12.1.
|
|
"""
|
|
if job_name not in VALID_JOB_NAMES:
|
|
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
|
|
|
key = f"job_{job_name}_enabled"
|
|
return await update_setting(db, key, str(enabled).lower())
|