refactor(jobs): derive job topology from one catalog, make next-run coherent
Groundwork for the Admin -> Jobs cleanup. Three sources of truth collapse into
app/job_catalog.py, which imports nothing from app so both the scheduler and
admin_service can import it at module level (admin_service otherwise has to
import the scheduler inside functions to dodge a cycle).
PIPELINE_MEMBERS is now DERIVED from the four pipeline step lists instead of
being a literal set in admin_service duplicating four lists in scheduler.py with
nothing asserting they agreed. A test pins that the derivation reproduces the
previous hand-maintained 9 names exactly, so this is behaviour-preserving.
Deletes the private _JOB_NAMES list, which held 16 of the 19 jobs:
benchmark_collector, outcome_evaluator and shadow_book had no runtime row, and
so no "last run" line in the panel, until their first run in a given process.
_job_runtime is now seeded from the catalog, and a test pins the invariant.
Next-run is decided by category rather than by reading a timestamp. A pipeline
step has no schedule of its own, so it reports its parent's ("next via Morning
Pipeline in 3h") instead of nothing; a manual job says manual_only rather than
rendering a date. This also fixes a real bug: triggering a paused job set
next_run_time=now, APScheduler re-armed the 520-week backstop behind it, and the
panel displayed "next run in ~87600h". Two independent guards -- the category
rule, plus _visible_next_run dropping anything past a year -- and an APScheduler
listener that re-pauses steps and manual jobs once their run finishes. The
listener is registered at module level because configure_scheduler is called
more than once and add_listener does not deduplicate.
Migrates backtest and ticker_universe_sync from interval to cron (Sun 03:00 ET
and 01:00 ET). configure_scheduler calls remove_all_jobs() on every startup, so
an interval countdown restarts each deploy -- a 168h backtest needed a week of
uninterrupted uptime to fire even once. The codebase already documented this
pitfall as the reason cron was adopted; these two were never migrated. Both are
now editable in Admin -> Schedule.
Also: list_jobs went from one settings query per job (19) to one for all of
them, and data_backfill is hidden from the listing while staying registered and
API-triggerable.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+60
-72
@@ -18,11 +18,13 @@ import logging
|
||||
import asyncio
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from apscheduler.events import EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import job_catalog
|
||||
from app.config import settings
|
||||
from app.database import async_session_factory
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
@@ -84,6 +86,34 @@ scheduler = AsyncIOScheduler(
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _repause_after_manual_run(event: object) -> None:
|
||||
"""Re-pause a job that only ever runs on demand, once its run finishes.
|
||||
|
||||
Pipeline steps and manual jobs are registered with a 520-week interval and
|
||||
``next_run_time=None`` as a backstop. Triggering one sets next_run_time=now,
|
||||
and APScheduler then re-arms that backstop -- so Admin → Jobs would show a
|
||||
"next run" ten years out. Guarding on category means the six cron jobs and
|
||||
the real interval jobs are never touched.
|
||||
|
||||
Registered at module level, not inside ``configure_scheduler``: that function
|
||||
is called more than once (idempotency test) and ``add_listener`` does not
|
||||
deduplicate.
|
||||
"""
|
||||
job_id = getattr(event, "job_id", None)
|
||||
if job_catalog.JOB_CATEGORY.get(job_id) not in (
|
||||
job_catalog.CATEGORY_STEP,
|
||||
job_catalog.CATEGORY_MANUAL,
|
||||
):
|
||||
return
|
||||
try:
|
||||
scheduler.modify_job(job_id, next_run_time=None)
|
||||
except Exception: # job gone, scheduler stopped — nothing to re-pause
|
||||
logger.debug("Could not re-pause %s after its run", job_id, exc_info=True)
|
||||
|
||||
|
||||
scheduler.add_listener(_repause_after_manual_run, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
|
||||
|
||||
# Track last successful ticker per job for rate-limit resume
|
||||
_last_successful: dict[str, str | None] = {
|
||||
"data_collector": None,
|
||||
@@ -91,26 +121,10 @@ _last_successful: dict[str, str | None] = {
|
||||
"sentiment_collector": None,
|
||||
}
|
||||
|
||||
# Jobs whose per-run progress is surfaced to Admin → Jobs. (outcome_evaluator is
|
||||
# created lazily on first run via _runtime_start.)
|
||||
_JOB_NAMES = [
|
||||
"data_collector",
|
||||
"data_backfill",
|
||||
"sentiment_collector",
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"rr_scanner",
|
||||
"ticker_universe_sync",
|
||||
"alerts",
|
||||
"market_regime",
|
||||
"regime_monitor",
|
||||
"event_study",
|
||||
"backtest",
|
||||
"daily_pipeline", # morning: OHLCV/sentiment/regime — no qualifying scan
|
||||
"near_close_pipeline", # OHLCV fetch → R:R scan → Telegram alerts
|
||||
"after_close_pipeline", # OHLCV fetch → outcome eval (final bar)
|
||||
"intraday_pipeline",
|
||||
]
|
||||
# Seeded from the catalog rather than a private list. The old literal held 16 of
|
||||
# the 19 jobs -- benchmark_collector, outcome_evaluator and shadow_book were
|
||||
# missing, so they had no runtime row (and so no "last run" line in Admin → Jobs)
|
||||
# until their first run in a given process.
|
||||
|
||||
|
||||
def _idle_runtime() -> dict[str, object]:
|
||||
@@ -127,7 +141,9 @@ def _idle_runtime() -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
_job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES}
|
||||
_job_runtime: dict[str, dict[str, object]] = {
|
||||
name: _idle_runtime() for name in sorted(job_catalog.VALID_JOB_NAMES)
|
||||
}
|
||||
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
||||
_next_backtest_cadence = DEFAULT_BACKTEST_CADENCE
|
||||
|
||||
@@ -1300,54 +1316,14 @@ async def sync_ticker_universe() -> None:
|
||||
# the intraday partial one (covers a long weekend / holiday gap).
|
||||
_FINAL_REFETCH_DAYS = 5
|
||||
|
||||
_DAILY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("benchmark_collector", "collect_benchmark"),
|
||||
("sentiment_collector", "collect_sentiment"),
|
||||
("market_regime", "compute_market_regime"),
|
||||
# Observational only — display/alerts; not trade selection.
|
||||
("regime_monitor", "compute_regime_monitor"),
|
||||
# Alerts after regime so quadrant changes reach Telegram in the morning.
|
||||
# Dispatcher is change-driven; quiet days stay quiet. Setup alerts still
|
||||
# fire on the near-close pipeline after the qualifying scan.
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# Near-close (~15:30 ET Mon–Fri): refresh in-progress day-t bars (incremental
|
||||
# ingestion overlaps the latest stored session), then the only daily
|
||||
# qualifying R:R scan, then Telegram immediately so manual fills can still hit
|
||||
# MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan
|
||||
# may see ~15:15 prices — immaterial for a 12-1 momentum signal.
|
||||
#
|
||||
# US early-close days (~3/year, 13:00 ET close): this job runs post-close and
|
||||
# entries behave like stale_close (still acceptable per execution-recovery matrix).
|
||||
# No exchange calendar dependency.
|
||||
_NEAR_CLOSE_PIPELINE_STEPS = [
|
||||
# Must land today's in-progress bar (~20 min behind live), or the scan falls
|
||||
# back to the previous close and execution degrades to the stale_close floor.
|
||||
("data_collector", "collect_ohlcv_for_scan"),
|
||||
("rr_scanner", "scan_rr"),
|
||||
# Straight after the scan so shadow entries mark at the same near-close
|
||||
# prices the discretionary book is looking at.
|
||||
("shadow_book", "run_shadow_book"),
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# After close (~16:45 ET Mon–Fri): fresh OHLCV fetch so outcomes resolve on the
|
||||
# final bar, not the near-close partial bar, then outcome/paper close.
|
||||
_AFTER_CLOSE_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv_final"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
|
||||
# Intraday (light): keep prices current and resolve outcomes through the day,
|
||||
# without the expensive scan/sentiment. The dashboard recomputes live R:R from
|
||||
# the latest price, so refreshing OHLCV is enough to stop prices lagging; the
|
||||
# outcome step also closes paper trades that hit their stop/target intraday.
|
||||
_INTRADAY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
# Step lists live in app.job_catalog so the runner, the admin API's pipeline
|
||||
# membership and the UI's grouping all read one definition. Re-exported here
|
||||
# under their original names: _run_pipeline and the scheduler_configured log
|
||||
# payload refer to them directly.
|
||||
_DAILY_PIPELINE_STEPS = job_catalog._DAILY_PIPELINE_STEPS
|
||||
_NEAR_CLOSE_PIPELINE_STEPS = job_catalog._NEAR_CLOSE_PIPELINE_STEPS
|
||||
_AFTER_CLOSE_PIPELINE_STEPS = job_catalog._AFTER_CLOSE_PIPELINE_STEPS
|
||||
_INTRADAY_PIPELINE_STEPS = job_catalog._INTRADAY_PIPELINE_STEPS
|
||||
|
||||
# Warn if near-close fetch+scan+alert drifts past this — entries leave the close
|
||||
# and the stale_close floor quietly becomes the ceiling.
|
||||
@@ -1486,6 +1462,12 @@ SCHEDULE_DEFAULTS: dict[str, str] = {
|
||||
"schedule_after_close_pipeline_cron": "45 16 * * mon-fri",
|
||||
# Hourly mid-session price + outcome (10:00–15:00 ET Mon–Fri).
|
||||
"schedule_intraday_pipeline_cron": "0 10-15 * * mon-fri",
|
||||
# Both were interval jobs until 2026-08-08 and hit exactly the pitfall
|
||||
# described above: configure_scheduler calls remove_all_jobs() on every
|
||||
# startup, so an interval countdown restarts from zero each deploy. A 168h
|
||||
# backtest needed a week of uninterrupted uptime to fire even once.
|
||||
"schedule_backtest_cron": "0 3 * * sun",
|
||||
"schedule_ticker_universe_cron": "0 1 * * *",
|
||||
}
|
||||
|
||||
# job id -> schedule setting key
|
||||
@@ -1496,6 +1478,8 @@ _CRON_JOBS: dict[str, str] = {
|
||||
"near_close_pipeline": "schedule_near_close_pipeline_cron",
|
||||
"after_close_pipeline": "schedule_after_close_pipeline_cron",
|
||||
"intraday_pipeline": "schedule_intraday_pipeline_cron",
|
||||
"backtest": "schedule_backtest_cron",
|
||||
"ticker_universe_sync": "schedule_ticker_universe_cron",
|
||||
}
|
||||
|
||||
|
||||
@@ -1630,9 +1614,12 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
id="intraday_pipeline", name="Intraday Pipeline", replace_existing=True,
|
||||
)
|
||||
|
||||
# Independent interval jobs (own cadence, no ordering dependency)
|
||||
# Independent jobs (own cadence, no ordering dependency). Cron, not interval,
|
||||
# for the reason documented at SCHEDULE_DEFAULTS: an interval countdown
|
||||
# restarts on every deploy, so these could be deferred indefinitely.
|
||||
scheduler.add_job(
|
||||
sync_ticker_universe, "interval", hours=24,
|
||||
sync_ticker_universe,
|
||||
_cron_trigger(cfg["schedule_ticker_universe_cron"], tz, "schedule_ticker_universe_cron"),
|
||||
id="ticker_universe_sync", name="Ticker Universe Sync", replace_existing=True,
|
||||
)
|
||||
# Alerts auto-fire only via near_close_pipeline (scan → alert before MOC).
|
||||
@@ -1643,7 +1630,8 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
replace_existing=True, next_run_time=None,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_backtest_job, "interval", hours=168,
|
||||
run_backtest_job,
|
||||
_cron_trigger(cfg["schedule_backtest_cron"], tz, "schedule_backtest_cron"),
|
||||
id="backtest", name="Backtest", replace_existing=True,
|
||||
)
|
||||
# Deep history backfill: manual only (never auto-fires); triggered from
|
||||
|
||||
Reference in New Issue
Block a user