dennisthiessenandClaude Opus 5
22bee28ac7
Job outcomes lived only in scheduler._job_runtime, an in-memory dict. Every
deploy wiped it, so Admin -> Jobs could report "Active" with no indication a job
had ever run or how it ended -- which is the main thing that page is for.
New job_run_state table (migration 031): one row per job, upserted on job_name.
Deliberately not history -- system_events already grows unbounded with no
retention job, and a second append-only operational table would repeat that
debt. Adding history later is purely additive.
Written from two hooks, NOT from _runtime_finish. That looked cheapest (one
function, ~40 call sites) but unit tests invoke job coroutines directly, so it
would fire detached DB writes at the real session factory throughout the suite,
and there is no testing flag to guard on.
- An APScheduler EVENT_JOB_EXECUTED/ERROR listener covers everything the
scheduler fires, including manual triggers. Its detached task is held in a
module-level set (a bare create_task result can be collected mid-flight) and
drained in the app lifespan before engine.dispose().
- _run_pipeline persists directly, and must: pipeline steps are plain
coroutine calls that emit no scheduler events, so the listener cannot see
them. The step persist sits AFTER the except that swallows step errors --
inside it, exactly the failed runs worth seeing would be skipped. The
orchestrator persists in the finally, and the disabled early-return persists
too, or "skipped" is silently dropped.
_persist_job_run never raises: a persistence failure must not break an otherwise
successful pipeline.
The API reports this as last_run_* and leaves runtime_* meaning strictly live
in-memory state. Reusing runtime_status would have been a regression, not a
no-op: JobControls drives the status chip from it (a job that errored eight days
ago would read "Last run error" forever instead of "Active") and picks the
rate-limit banner from it (a week-old rate limit would pin the banner
permanently). Tests pin the split.
The table starts empty; each job fills its row the next time it finishes. No
backfill from system_events, which records only warning/error outcomes under a
different status vocabulary and would invent successes that never happened.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
806 lines
32 KiB
Python
806 lines
32 KiB
Python
"""Admin service: user management, system settings, data cleanup, job control."""
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from passlib.hash import bcrypt
|
|
from sqlalchemy import delete, func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app import job_catalog
|
|
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
|
from app.models.fundamental import FundamentalData
|
|
from app.models.ohlcv import OHLCVRecord
|
|
from app.models.score import CompositeScore, DimensionScore
|
|
from app.models.sentiment import SentimentScore
|
|
from app.models.sr_level import SRLevel
|
|
from app.models.settings import SystemSetting
|
|
from app.models.ticker import Ticker
|
|
from app.models.trade_setup import TradeSetup
|
|
from app.models.user import User
|
|
from app.services import job_run_store, settings_store
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
RECOMMENDATION_CONFIG_DEFAULTS: dict[str, float] = {
|
|
"recommendation_high_confidence_threshold": 70.0,
|
|
"recommendation_moderate_confidence_threshold": 50.0,
|
|
"recommendation_confidence_diff_threshold": 20.0,
|
|
"recommendation_signal_alignment_weight": 0.15,
|
|
"recommendation_sr_strength_weight": 0.20,
|
|
"recommendation_momentum_technical_divergence_threshold": 30.0,
|
|
"recommendation_fundamental_technical_divergence_threshold": 40.0,
|
|
}
|
|
|
|
DEFAULT_TICKER_UNIVERSE = "sp500"
|
|
SUPPORTED_TICKER_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
|
|
|
|
# Activation gate: what counts as a signal worth acting on. Used by the
|
|
# Dashboard's "Qualified" metric, the Signals "Qualified only" view, and the
|
|
# Track Record's qualified stats. The outcome evaluator deliberately ignores
|
|
# these — every setup is evaluated so the gate itself can be validated.
|
|
#
|
|
# The core selection is residual cross-sectional 12-1 momentum (top percentile
|
|
# of the universe, long-only). R:R and confidence are floors; high-conviction /
|
|
# clean-read are optional tighteners (off by default).
|
|
_ACTIVATION_FLOAT_KEYS: dict[str, str] = {
|
|
"min_momentum_percentile": "activation_min_momentum_percentile",
|
|
"min_rr": "activation_min_rr",
|
|
"min_confidence": "activation_min_confidence",
|
|
}
|
|
_ACTIVATION_BOOL_KEYS: dict[str, str] = {
|
|
"require_high_conviction": "activation_require_high_conviction",
|
|
"exclude_conflicts": "activation_exclude_conflicts",
|
|
"exclude_neutral": "activation_exclude_neutral",
|
|
}
|
|
ACTIVATION_DEFAULTS: dict[str, float | bool] = {
|
|
"min_momentum_percentile": 80.0,
|
|
# Production floor from the 2026-07-12 min_rr sweep (in-sample and OOS peak).
|
|
# 1.2 was the old code default and the trough next to the spike — do not restore.
|
|
"min_rr": 2.0,
|
|
# 0 = off. The July 2026 gate ablation showed the confidence floor added
|
|
# nothing (identical net/trade with it removed, under both exit models)
|
|
# while cutting ~25% of qualified trades.
|
|
"min_confidence": 0.0,
|
|
"require_high_conviction": False,
|
|
"exclude_conflicts": False,
|
|
# On by default: a NEUTRAL ("no clear setup") recommendation isn't an
|
|
# actionable signal, so it shouldn't qualify or be crowned a top pick.
|
|
"exclude_neutral": True,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# User management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def list_users(db: AsyncSession) -> list[User]:
|
|
"""Return all users ordered by id."""
|
|
result = await db.execute(select(User).order_by(User.id))
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def create_user(
|
|
db: AsyncSession,
|
|
username: str,
|
|
password: str,
|
|
role: str = "user",
|
|
has_access: bool = False,
|
|
) -> User:
|
|
"""Create a new user account (admin action)."""
|
|
result = await db.execute(select(User).where(User.username == username))
|
|
if result.scalar_one_or_none() is not None:
|
|
raise DuplicateError(f"Username already exists: {username}")
|
|
|
|
user = User(
|
|
username=username,
|
|
password_hash=bcrypt.hash(password),
|
|
role=role,
|
|
has_access=has_access,
|
|
)
|
|
db.add(user)
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
async def set_user_access(db: AsyncSession, user_id: int, has_access: bool) -> User:
|
|
"""Grant or revoke API access for a user."""
|
|
result = await db.execute(select(User).where(User.id == user_id))
|
|
user = result.scalar_one_or_none()
|
|
if user is None:
|
|
raise NotFoundError(f"User not found: {user_id}")
|
|
|
|
user.has_access = has_access
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
async def reset_password(db: AsyncSession, user_id: int, new_password: str) -> User:
|
|
"""Reset a user's password."""
|
|
result = await db.execute(select(User).where(User.id == user_id))
|
|
user = result.scalar_one_or_none()
|
|
if user is None:
|
|
raise NotFoundError(f"User not found: {user_id}")
|
|
|
|
user.password_hash = bcrypt.hash(new_password)
|
|
await db.commit()
|
|
await db.refresh(user)
|
|
return user
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Registration toggle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def toggle_registration(db: AsyncSession, enabled: bool) -> SystemSetting:
|
|
"""Enable or disable user registration via SystemSetting."""
|
|
setting = await settings_store.upsert_setting(db, "registration_enabled", str(enabled).lower())
|
|
await db.commit()
|
|
await db.refresh(setting)
|
|
return setting
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# System settings CRUD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def list_settings(db: AsyncSession) -> list[SystemSetting]:
|
|
"""Return all system settings."""
|
|
result = await db.execute(select(SystemSetting).order_by(SystemSetting.key))
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSetting:
|
|
"""Create or update a system setting."""
|
|
setting = await settings_store.upsert_setting(db, key, value)
|
|
await db.commit()
|
|
await db.refresh(setting)
|
|
return setting
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Activation thresholds
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_activation_config(db: AsyncSession) -> dict[str, float | bool]:
|
|
"""Return the activation gate config with public keys."""
|
|
result = await db.execute(
|
|
select(SystemSetting).where(SystemSetting.key.like("activation_%"))
|
|
)
|
|
stored = {s.key: s.value for s in result.scalars().all()}
|
|
|
|
config: dict[str, float | bool] = dict(ACTIVATION_DEFAULTS)
|
|
for public_key, storage_key in _ACTIVATION_FLOAT_KEYS.items():
|
|
if storage_key in stored:
|
|
try:
|
|
config[public_key] = float(stored[storage_key])
|
|
except (TypeError, ValueError):
|
|
pass
|
|
for public_key, storage_key in _ACTIVATION_BOOL_KEYS.items():
|
|
if storage_key in stored:
|
|
config[public_key] = str(stored[storage_key]).strip().lower() == "true"
|
|
return config
|
|
|
|
|
|
async def update_activation_config(
|
|
db: AsyncSession, updates: dict[str, float | bool]
|
|
) -> dict[str, float | bool]:
|
|
"""Update the activation gate. Accepts public keys; only supplied keys change."""
|
|
if "min_momentum_percentile" in updates and not 0 <= updates["min_momentum_percentile"] <= 100:
|
|
raise ValidationError("min_momentum_percentile must be between 0 and 100")
|
|
if "min_rr" in updates and updates["min_rr"] < 0:
|
|
raise ValidationError("min_rr must be >= 0")
|
|
if "min_confidence" in updates and not 0 <= updates["min_confidence"] <= 100:
|
|
raise ValidationError("min_confidence must be between 0 and 100")
|
|
|
|
for public_key, storage_key in _ACTIVATION_FLOAT_KEYS.items():
|
|
if public_key in updates and updates[public_key] is not None:
|
|
await update_setting(db, storage_key, str(float(updates[public_key])))
|
|
for public_key, storage_key in _ACTIVATION_BOOL_KEYS.items():
|
|
if public_key in updates and updates[public_key] is not None:
|
|
await update_setting(db, storage_key, "true" if updates[public_key] else "false")
|
|
|
|
return await get_activation_config(db)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Performance window + shadow book
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_performance_config(db: AsyncSession) -> dict:
|
|
"""Start date for the Performance comparison ('' = all history)."""
|
|
from app.services.paper_trade_service import KEY_PERFORMANCE_START
|
|
|
|
return {"start_date": await settings_store.get_value(db, KEY_PERFORMANCE_START, "") or ""}
|
|
|
|
|
|
async def update_performance_config(db: AsyncSession, updates: dict) -> dict:
|
|
"""Set (or clear) the performance start date. Empty string means all history."""
|
|
from datetime import date as _date
|
|
|
|
from app.services.paper_trade_service import KEY_PERFORMANCE_START
|
|
|
|
if "start_date" in updates:
|
|
raw = (updates.get("start_date") or "").strip()
|
|
if raw:
|
|
try:
|
|
_date.fromisoformat(raw)
|
|
except ValueError as exc:
|
|
raise ValidationError("start_date must be an ISO date (YYYY-MM-DD)") from exc
|
|
await update_setting(db, KEY_PERFORMANCE_START, raw)
|
|
return await get_performance_config(db)
|
|
|
|
|
|
async def get_shadow_book_config(db: AsyncSession) -> dict:
|
|
"""Shadow book switch + sizing, with the validated defaults filled in."""
|
|
from app.services import shadow_book_service
|
|
|
|
config = await shadow_book_service.get_config(db)
|
|
config["enabled"] = await shadow_book_service.is_enabled(db)
|
|
return config
|
|
|
|
|
|
async def update_shadow_book_config(db: AsyncSession, updates: dict) -> dict:
|
|
"""Update the shadow book. Enabling it starts automatic live entries."""
|
|
from app.services import shadow_book_service
|
|
|
|
if "enabled" in updates:
|
|
await update_setting(
|
|
db, shadow_book_service.KEY_ENABLED, "true" if updates["enabled"] else "false"
|
|
)
|
|
for key, storage_key in (
|
|
("capacity", shadow_book_service.KEY_CAPACITY),
|
|
("risk_pct", shadow_book_service.KEY_RISK_PCT),
|
|
("start_equity", shadow_book_service.KEY_START_EQUITY),
|
|
):
|
|
if key in updates:
|
|
await update_setting(db, storage_key, str(updates[key]))
|
|
return await get_shadow_book_config(db)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pipeline schedule (cron)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_schedule_config(db: AsyncSession) -> dict[str, str]:
|
|
"""Cron schedule for the daily/intraday pipelines and fundamentals."""
|
|
from app.scheduler import load_schedule_config
|
|
|
|
return await load_schedule_config(db)
|
|
|
|
|
|
async def update_schedule_config(
|
|
db: AsyncSession, updates: dict[str, str]
|
|
) -> dict[str, str]:
|
|
"""Validate, persist, and apply cron schedule changes to the running scheduler."""
|
|
from app.scheduler import (
|
|
SCHEDULE_DEFAULTS,
|
|
load_schedule_config,
|
|
reschedule_jobs,
|
|
validate_cron,
|
|
)
|
|
|
|
current = await load_schedule_config(db)
|
|
tz = (updates.get("schedule_timezone") or current["schedule_timezone"]).strip()
|
|
|
|
for key, value in updates.items():
|
|
if key not in SCHEDULE_DEFAULTS:
|
|
raise ValidationError(f"Unknown schedule key: {key}")
|
|
if key == "schedule_timezone":
|
|
# Validate the timezone against an existing cron expression.
|
|
try:
|
|
validate_cron(current["schedule_daily_pipeline_cron"], value)
|
|
except Exception as exc:
|
|
raise ValidationError(f"Invalid timezone: {value}") from exc
|
|
else:
|
|
try:
|
|
validate_cron(value, tz)
|
|
except Exception as exc:
|
|
raise ValidationError(f"Invalid cron for {key}: {value!r}") from exc
|
|
|
|
for key, value in updates.items():
|
|
await update_setting(db, key, str(value).strip())
|
|
|
|
new_config = await load_schedule_config(db)
|
|
try:
|
|
reschedule_jobs(new_config)
|
|
except Exception:
|
|
# Scheduler may not be running (e.g. unit tests) — the config is saved
|
|
# regardless and applied on next startup.
|
|
logger.warning("Could not reschedule jobs after config update", exc_info=True)
|
|
return new_config
|
|
|
|
|
|
def _recommendation_public_to_storage_key(key: str) -> str:
|
|
return f"recommendation_{key}"
|
|
|
|
|
|
async def get_recommendation_config(db: AsyncSession) -> dict[str, float]:
|
|
result = await db.execute(
|
|
select(SystemSetting).where(SystemSetting.key.like("recommendation_%"))
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
config = dict(RECOMMENDATION_CONFIG_DEFAULTS)
|
|
for row in rows:
|
|
try:
|
|
config[row.key] = float(row.value)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
return {
|
|
"high_confidence_threshold": config["recommendation_high_confidence_threshold"],
|
|
"moderate_confidence_threshold": config["recommendation_moderate_confidence_threshold"],
|
|
"confidence_diff_threshold": config["recommendation_confidence_diff_threshold"],
|
|
"signal_alignment_weight": config["recommendation_signal_alignment_weight"],
|
|
"sr_strength_weight": config["recommendation_sr_strength_weight"],
|
|
"momentum_technical_divergence_threshold": config["recommendation_momentum_technical_divergence_threshold"],
|
|
"fundamental_technical_divergence_threshold": config["recommendation_fundamental_technical_divergence_threshold"],
|
|
}
|
|
|
|
|
|
async def update_recommendation_config(
|
|
db: AsyncSession,
|
|
payload: dict[str, float],
|
|
) -> dict[str, float]:
|
|
for public_key, public_value in payload.items():
|
|
storage_key = _recommendation_public_to_storage_key(public_key)
|
|
await update_setting(db, storage_key, str(public_value))
|
|
|
|
return await get_recommendation_config(db)
|
|
|
|
|
|
async def get_ticker_universe_default(db: AsyncSession) -> dict[str, str]:
|
|
setting = await settings_store.get_setting(db, "ticker_universe_default")
|
|
universe = setting.value if setting else DEFAULT_TICKER_UNIVERSE
|
|
if universe not in SUPPORTED_TICKER_UNIVERSES:
|
|
universe = DEFAULT_TICKER_UNIVERSE
|
|
return {"universe": universe}
|
|
|
|
|
|
async def update_ticker_universe_default(db: AsyncSession, universe: str) -> dict[str, str]:
|
|
normalised = universe.strip().lower()
|
|
if normalised not in SUPPORTED_TICKER_UNIVERSES:
|
|
supported = ", ".join(sorted(SUPPORTED_TICKER_UNIVERSES))
|
|
raise ValidationError(f"Unsupported ticker universe '{universe}'. Supported: {supported}")
|
|
|
|
await update_setting(db, "ticker_universe_default", normalised)
|
|
return {"universe": normalised}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data cleanup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def cleanup_data(db: AsyncSession, older_than_days: int) -> dict:
|
|
"""Delete OHLCV, sentiment, and fundamental records older than N days.
|
|
|
|
Preserves tickers, users, and latest scores. After OHLCV pruning, rebuilds
|
|
Structural S/R for every ticker so chart levels match the remaining history.
|
|
|
|
Returns deleted-row counts plus S/R refresh outcomes. A per-ticker S/R
|
|
failure rolls the session back (so later tickers still run) and is listed
|
|
in ``sr_refresh_failures`` rather than aborting the whole cleanup.
|
|
"""
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=older_than_days)
|
|
counts: dict = {}
|
|
|
|
# OHLCV — date column is a date, compare with cutoff date
|
|
result = await db.execute(
|
|
delete(OHLCVRecord).where(OHLCVRecord.date < cutoff.date())
|
|
)
|
|
counts["ohlcv"] = result.rowcount # type: ignore[assignment]
|
|
|
|
# Sentiment — timestamp is datetime
|
|
result = await db.execute(
|
|
delete(SentimentScore).where(SentimentScore.timestamp < cutoff)
|
|
)
|
|
counts["sentiment"] = result.rowcount # type: ignore[assignment]
|
|
|
|
# Fundamentals — fetched_at is datetime
|
|
result = await db.execute(
|
|
delete(FundamentalData).where(FundamentalData.fetched_at < cutoff)
|
|
)
|
|
counts["fundamentals"] = result.rowcount # type: ignore[assignment]
|
|
|
|
await db.commit()
|
|
|
|
counts["sr_refresh_ok"] = 0
|
|
counts["sr_refresh_failed"] = 0
|
|
counts["sr_refresh_failures"] = []
|
|
|
|
# Structural S/R is derived from OHLCV; recompute after history shrinks.
|
|
if counts["ohlcv"]:
|
|
from app.services.sr_service import recalculate_sr_levels
|
|
|
|
symbols = list(
|
|
(await db.execute(select(Ticker.symbol).order_by(Ticker.symbol))).scalars().all()
|
|
)
|
|
for symbol in symbols:
|
|
try:
|
|
await recalculate_sr_levels(db, symbol)
|
|
counts["sr_refresh_ok"] += 1
|
|
except Exception as exc:
|
|
logger.exception("S/R refresh after cleanup failed for %s", symbol)
|
|
try:
|
|
await db.rollback()
|
|
except Exception:
|
|
logger.exception(
|
|
"Session rollback after S/R cleanup failure also failed for %s",
|
|
symbol,
|
|
)
|
|
counts["sr_refresh_failed"] += 1
|
|
counts["sr_refresh_failures"].append(
|
|
{"symbol": symbol, "error": f"{type(exc).__name__}: {exc}"}
|
|
)
|
|
|
|
return counts
|
|
|
|
|
|
async def reset_trade_setups(db: AsyncSession) -> dict[str, int]:
|
|
"""Delete all trade setups, wiping the track record for a fresh start.
|
|
|
|
Stats are derived from evaluated trade setups, so this resets the Track
|
|
Record to zero. Live setups regenerate on the next R:R scan. Used after
|
|
material changes to scoring / setup generation, when historical outcomes no
|
|
longer reflect current logic.
|
|
"""
|
|
result = await db.execute(delete(TradeSetup))
|
|
await db.commit()
|
|
return {"trade_setups": result.rowcount} # type: ignore[attr-defined]
|
|
|
|
|
|
async def get_pipeline_readiness(db: AsyncSession) -> list[dict]:
|
|
"""Return per-ticker readiness snapshot for ingestion/scoring/scanner pipeline."""
|
|
tickers_result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
|
|
tickers = list(tickers_result.scalars().all())
|
|
|
|
if not tickers:
|
|
return []
|
|
|
|
ticker_ids = [ticker.id for ticker in tickers]
|
|
|
|
ohlcv_stats_result = await db.execute(
|
|
select(
|
|
OHLCVRecord.ticker_id,
|
|
func.count(OHLCVRecord.id),
|
|
func.max(OHLCVRecord.date),
|
|
)
|
|
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
|
.group_by(OHLCVRecord.ticker_id)
|
|
)
|
|
ohlcv_stats = {
|
|
ticker_id: {
|
|
"bars": int(count or 0),
|
|
"last_date": max_date.isoformat() if max_date else None,
|
|
}
|
|
for ticker_id, count, max_date in ohlcv_stats_result.all()
|
|
}
|
|
|
|
dim_rows_result = await db.execute(
|
|
select(DimensionScore).where(DimensionScore.ticker_id.in_(ticker_ids))
|
|
)
|
|
dim_map_by_ticker: dict[int, dict[str, tuple[float | None, bool]]] = {}
|
|
for row in dim_rows_result.scalars().all():
|
|
dim_map_by_ticker.setdefault(row.ticker_id, {})[row.dimension] = (row.score, row.is_stale)
|
|
|
|
sr_counts_result = await db.execute(
|
|
select(SRLevel.ticker_id, func.count(SRLevel.id))
|
|
.where(SRLevel.ticker_id.in_(ticker_ids))
|
|
.group_by(SRLevel.ticker_id)
|
|
)
|
|
sr_counts = {ticker_id: int(count or 0) for ticker_id, count in sr_counts_result.all()}
|
|
|
|
sentiment_stats_result = await db.execute(
|
|
select(
|
|
SentimentScore.ticker_id,
|
|
func.count(SentimentScore.id),
|
|
func.max(SentimentScore.timestamp),
|
|
)
|
|
.where(SentimentScore.ticker_id.in_(ticker_ids))
|
|
.group_by(SentimentScore.ticker_id)
|
|
)
|
|
sentiment_stats = {
|
|
ticker_id: {
|
|
"count": int(count or 0),
|
|
"last_at": max_ts.isoformat() if max_ts else None,
|
|
}
|
|
for ticker_id, count, max_ts in sentiment_stats_result.all()
|
|
}
|
|
|
|
fundamentals_result = await db.execute(
|
|
select(FundamentalData.ticker_id, FundamentalData.fetched_at)
|
|
.where(FundamentalData.ticker_id.in_(ticker_ids))
|
|
)
|
|
fundamentals_map = {
|
|
ticker_id: fetched_at.isoformat() if fetched_at else None
|
|
for ticker_id, fetched_at in fundamentals_result.all()
|
|
}
|
|
|
|
composites_result = await db.execute(
|
|
select(CompositeScore.ticker_id, CompositeScore.is_stale)
|
|
.where(CompositeScore.ticker_id.in_(ticker_ids))
|
|
)
|
|
composites_map = {
|
|
ticker_id: is_stale
|
|
for ticker_id, is_stale in composites_result.all()
|
|
}
|
|
|
|
setup_counts_result = await db.execute(
|
|
select(TradeSetup.ticker_id, func.count(TradeSetup.id))
|
|
.where(TradeSetup.ticker_id.in_(ticker_ids))
|
|
.group_by(TradeSetup.ticker_id)
|
|
)
|
|
setup_counts = {ticker_id: int(count or 0) for ticker_id, count in setup_counts_result.all()}
|
|
|
|
readiness: list[dict] = []
|
|
for ticker in tickers:
|
|
ohlcv = ohlcv_stats.get(ticker.id, {"bars": 0, "last_date": None})
|
|
ohlcv_bars = int(ohlcv["bars"])
|
|
ohlcv_last_date = ohlcv["last_date"]
|
|
|
|
dim_map = dim_map_by_ticker.get(ticker.id, {})
|
|
|
|
sr_count = int(sr_counts.get(ticker.id, 0))
|
|
|
|
sentiment = sentiment_stats.get(ticker.id, {"count": 0, "last_at": None})
|
|
sentiment_count = int(sentiment["count"])
|
|
sentiment_last_at = sentiment["last_at"]
|
|
|
|
fundamentals_fetched_at = fundamentals_map.get(ticker.id)
|
|
has_fundamentals = ticker.id in fundamentals_map
|
|
|
|
has_composite = ticker.id in composites_map
|
|
composite_stale = composites_map.get(ticker.id)
|
|
|
|
setup_count = int(setup_counts.get(ticker.id, 0))
|
|
|
|
missing_reasons: list[str] = []
|
|
if ohlcv_bars < 30:
|
|
missing_reasons.append("insufficient_ohlcv_bars(<30)")
|
|
if "technical" not in dim_map or dim_map["technical"][0] is None:
|
|
missing_reasons.append("missing_technical")
|
|
if "momentum" not in dim_map or dim_map["momentum"][0] is None:
|
|
missing_reasons.append("missing_momentum")
|
|
if "sr_quality" not in dim_map or dim_map["sr_quality"][0] is None:
|
|
missing_reasons.append("missing_sr_quality")
|
|
if sentiment_count == 0:
|
|
missing_reasons.append("missing_sentiment")
|
|
if not has_fundamentals:
|
|
missing_reasons.append("missing_fundamentals")
|
|
if not has_composite:
|
|
missing_reasons.append("missing_composite")
|
|
if setup_count == 0:
|
|
missing_reasons.append("missing_trade_setup")
|
|
|
|
readiness.append(
|
|
{
|
|
"symbol": ticker.symbol,
|
|
"ohlcv_bars": ohlcv_bars,
|
|
"ohlcv_last_date": ohlcv_last_date,
|
|
"dimensions": {
|
|
"technical": dim_map.get("technical", (None, True))[0],
|
|
"sr_quality": dim_map.get("sr_quality", (None, True))[0],
|
|
"sentiment": dim_map.get("sentiment", (None, True))[0],
|
|
"fundamental": dim_map.get("fundamental", (None, True))[0],
|
|
"momentum": dim_map.get("momentum", (None, True))[0],
|
|
},
|
|
"sentiment_count": sentiment_count,
|
|
"sentiment_last_at": sentiment_last_at,
|
|
"has_fundamentals": has_fundamentals,
|
|
"fundamentals_fetched_at": fundamentals_fetched_at,
|
|
"sr_level_count": sr_count,
|
|
"has_composite": has_composite,
|
|
"composite_stale": composite_stale,
|
|
"trade_setup_count": setup_count,
|
|
"missing_reasons": missing_reasons,
|
|
"ready_for_scanner": ohlcv_bars >= 15 and sr_count > 0,
|
|
}
|
|
)
|
|
|
|
return readiness
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Job control (placeholder — scheduler is Task 12.1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Job identity, labels and pipeline membership now live in app.job_catalog, which
|
|
# derives PIPELINE_MEMBERS from the pipeline step lists instead of restating them.
|
|
# Re-exported here because callers (routers, tests) import them from this module.
|
|
VALID_JOB_NAMES = job_catalog.VALID_JOB_NAMES
|
|
JOB_LABELS = job_catalog.JOB_LABELS
|
|
PIPELINE_MEMBERS = job_catalog.PIPELINE_MEMBERS
|
|
|
|
# Anything further out than this is a parked backstop, not a schedule: pipeline
|
|
# steps and manual jobs are registered on a 520-week interval, and triggering one
|
|
# re-arms it. Belt-and-braces behind the category rule in _next_run_fields.
|
|
_NEXT_RUN_HORIZON_DAYS = 365
|
|
|
|
|
|
def _visible_next_run(next_run: datetime | None) -> datetime | None:
|
|
"""Drop a next-run that is really the parked backstop."""
|
|
if next_run is None:
|
|
return None
|
|
horizon = datetime.now(next_run.tzinfo) + timedelta(days=_NEXT_RUN_HORIZON_DAYS)
|
|
return None if next_run > horizon else next_run
|
|
|
|
|
|
def _own_next_run(scheduler, name: str) -> datetime | None:
|
|
# getattr: APScheduler only sets next_run_time once the scheduler is running,
|
|
# so a job registered but not yet started has no such attribute at all.
|
|
job = scheduler.get_job(name)
|
|
return _visible_next_run(getattr(job, "next_run_time", None)) if job else None
|
|
|
|
|
|
def _next_run_fields(scheduler, name: str, enabled_map: dict[str, bool]) -> dict:
|
|
"""Where this job's next run comes from, decided by category not by clock.
|
|
|
|
A pipeline step has no meaningful schedule of its own, so reporting one is
|
|
the bug: its parent's timer is the answer. Manual jobs have no answer at all,
|
|
and saying so beats rendering a parked backstop as a date.
|
|
"""
|
|
category = job_catalog.JOB_CATEGORY.get(name)
|
|
if category == job_catalog.CATEGORY_STEP:
|
|
parents = job_catalog.PIPELINES_BY_MEMBER.get(name, ())
|
|
soonest: datetime | None = None
|
|
via: str | None = None
|
|
for parent in parents:
|
|
if not enabled_map.get(parent, True):
|
|
continue
|
|
candidate = _own_next_run(scheduler, parent)
|
|
if candidate is not None and (soonest is None or candidate < soonest):
|
|
soonest, via = candidate, parent
|
|
return {
|
|
"next_run_at": None,
|
|
"next_run_source": "via_pipeline",
|
|
"via_next_run_at": soonest.isoformat() if soonest else None,
|
|
"via_next_run_job": via,
|
|
}
|
|
if category == job_catalog.CATEGORY_MANUAL:
|
|
return {
|
|
"next_run_at": None,
|
|
"next_run_source": "manual_only",
|
|
"via_next_run_at": None,
|
|
"via_next_run_job": None,
|
|
}
|
|
own = _own_next_run(scheduler, name)
|
|
return {
|
|
"next_run_at": own.isoformat() if own else None,
|
|
"next_run_source": "own_schedule",
|
|
"via_next_run_at": None,
|
|
"via_next_run_job": None,
|
|
}
|
|
|
|
|
|
async def list_jobs(db: AsyncSession) -> list[dict]:
|
|
"""Return status of all scheduled jobs, grouped and ordered by category."""
|
|
from app.scheduler import get_job_runtime_snapshot, scheduler
|
|
|
|
visible = sorted(VALID_JOB_NAMES - job_catalog.HIDDEN_JOBS, key=job_catalog.sort_order)
|
|
# One query for every flag instead of one per job. Parents are read too, since
|
|
# a step reports its parent's next run only while that parent is enabled.
|
|
flags = await settings_store.get_map(
|
|
db, [f"job_{name}_enabled" for name in VALID_JOB_NAMES]
|
|
)
|
|
enabled_map = {
|
|
name: flags.get(f"job_{name}_enabled", "true") == "true"
|
|
for name in VALID_JOB_NAMES
|
|
}
|
|
last_runs = await job_run_store.get_map(db, visible)
|
|
|
|
jobs_out = []
|
|
for name in visible:
|
|
job = scheduler.get_job(name)
|
|
runtime = get_job_runtime_snapshot(name)
|
|
last = last_runs.get(name)
|
|
|
|
jobs_out.append({
|
|
"name": name,
|
|
"label": JOB_LABELS.get(name, name),
|
|
"enabled": enabled_map.get(name, True),
|
|
"category": job_catalog.JOB_CATEGORY.get(name),
|
|
"sort_order": job_catalog.sort_order(name),
|
|
# Parent pipelines for a step; the steps themselves for a pipeline.
|
|
"pipelines": list(job_catalog.PIPELINES_BY_MEMBER.get(name, ())),
|
|
"steps": [step for step, _ in job_catalog.PIPELINE_STEPS.get(name, ())],
|
|
"registered": job is not None,
|
|
"running": bool(runtime.get("running", False)),
|
|
# runtime_* are strictly live in-memory state. Persisted history is
|
|
# reported separately as last_run_*, so a stale error cannot pin the
|
|
# status chip or the rate-limit banner.
|
|
"runtime_status": runtime.get("status"),
|
|
"runtime_processed": runtime.get("processed"),
|
|
"runtime_total": runtime.get("total"),
|
|
"runtime_progress_pct": runtime.get("progress_pct"),
|
|
"runtime_current_ticker": runtime.get("current_ticker"),
|
|
"runtime_started_at": runtime.get("started_at"),
|
|
"runtime_finished_at": runtime.get("finished_at"),
|
|
"runtime_message": runtime.get("message"),
|
|
# Survives restarts, unlike runtime_*. Reported separately so the
|
|
# status chip keeps meaning "state now" rather than "last outcome,
|
|
# forever" -- an error a week ago must not read as Inactive today.
|
|
"last_run_at": last.finished_at.isoformat() if last else None,
|
|
"last_run_status": last.status if last else None,
|
|
"last_run_message": last.message if last else None,
|
|
"last_run_processed": last.processed if last else None,
|
|
"last_run_total": last.total if last else None,
|
|
**_next_run_fields(scheduler, name, enabled_map),
|
|
})
|
|
|
|
return jobs_out
|
|
|
|
|
|
async def trigger_job(
|
|
db: AsyncSession,
|
|
job_name: str,
|
|
*,
|
|
target_model: str | None = None,
|
|
cadence: str | None = None,
|
|
) -> dict[str, str]:
|
|
"""Trigger a manual job run via the scheduler.
|
|
|
|
Runs the job immediately (in addition to its regular schedule).
|
|
"""
|
|
if job_name not in VALID_JOB_NAMES:
|
|
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
|
if target_model is not None and job_name != "backtest":
|
|
raise ValidationError("target_model is supported only for the backtest job")
|
|
if cadence is not None and job_name != "backtest":
|
|
raise ValidationError("cadence is supported only for the backtest job")
|
|
|
|
from app.scheduler import get_job_runtime_snapshot, scheduler
|
|
|
|
runtime_target = get_job_runtime_snapshot(job_name)
|
|
if runtime_target.get("running"):
|
|
return {
|
|
"job": job_name,
|
|
"status": "busy",
|
|
"message": f"Job '{job_name}' is already running",
|
|
}
|
|
|
|
all_runtime = get_job_runtime_snapshot()
|
|
for running_name, runtime in all_runtime.items():
|
|
if running_name == job_name:
|
|
continue
|
|
if runtime.get("running"):
|
|
return {
|
|
"job": job_name,
|
|
"status": "blocked",
|
|
"message": f"Cannot trigger '{job_name}' while '{running_name}' is running",
|
|
}
|
|
|
|
job = scheduler.get_job(job_name)
|
|
if job is None:
|
|
return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"}
|
|
|
|
if job_name == "backtest":
|
|
from app.scheduler import queue_backtest_options
|
|
|
|
target_model, cadence = queue_backtest_options(target_model, cadence)
|
|
|
|
job.modify(next_run_time=None) # Reset, then trigger immediately
|
|
from datetime import datetime, timezone
|
|
job.modify(next_run_time=datetime.now(timezone.utc))
|
|
|
|
result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
|
|
if target_model is not None:
|
|
result["target_model"] = target_model
|
|
if cadence is not None:
|
|
result["cadence"] = cadence
|
|
return result
|
|
|
|
|
|
async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSetting:
|
|
"""Enable or disable a scheduled job by storing state in SystemSetting.
|
|
|
|
Actual scheduler integration happens in Task 12.1.
|
|
"""
|
|
if job_name not in VALID_JOB_NAMES:
|
|
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
|
|
|
|
key = f"job_{job_name}_enabled"
|
|
return await update_setting(db, key, str(enabled).lower())
|