"""Job topology: names, labels, pipeline membership, categories, ordering. The single source of truth for *what the jobs are*, as opposed to how they run. It deliberately imports nothing from ``app`` so both ``app.scheduler`` and ``app.services.admin_service`` can import it at module level -- admin_service otherwise has to do ``from app.scheduler import ...`` inside functions to dodge a cycle. The pipeline step lists live here rather than in the scheduler because three separate things need them and used to keep private copies: the runner, the ``PIPELINE_MEMBERS`` set the admin API reports, and the UI's grouping. Steps are ``(step_name, coroutine_name)``; ``_run_pipeline`` resolves the coroutine late out of the scheduler's own globals, so nothing here depends on those functions existing. """ from __future__ import annotations # --------------------------------------------------------------------------- # Pipelines # --------------------------------------------------------------------------- _DAILY_PIPELINE_STEPS = [ ("data_collector", "collect_ohlcv"), ("benchmark_collector", "collect_benchmark"), ("sentiment_collector", "collect_sentiment"), ("market_regime", "compute_market_regime"), # Observational only — display/alerts; not trade selection. ("regime_monitor", "compute_regime_monitor"), # Alerts after regime so quadrant changes reach Telegram in the morning. # Dispatcher is change-driven; quiet days stay quiet. Setup alerts still # fire on the near-close pipeline after the qualifying scan. ("alerts", "dispatch_alerts_job"), ] # Near-close (~15:30 ET Mon–Fri): refresh in-progress day-t bars (incremental # ingestion overlaps the latest stored session), then the only daily # qualifying R:R scan, then Telegram immediately so manual fills can still hit # MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan # may see ~15:15 prices — immaterial for a 12-1 momentum signal. # # US early-close days (~3/year, 13:00 ET close): this job runs post-close and # entries behave like stale_close (still acceptable per execution-recovery matrix). # No exchange calendar dependency. _NEAR_CLOSE_PIPELINE_STEPS = [ # Must land today's in-progress bar (~20 min behind live), or the scan falls # back to the previous close and execution degrades to the stale_close floor. ("data_collector", "collect_ohlcv_for_scan"), ("rr_scanner", "scan_rr"), # Straight after the scan so shadow entries mark at the same near-close # prices the discretionary book is looking at. ("shadow_book", "run_shadow_book"), ("alerts", "dispatch_alerts_job"), ] # After close (~16:45 ET Mon–Fri): fresh OHLCV fetch so outcomes resolve on the # final bar, not the near-close partial bar, then outcome/paper close. _AFTER_CLOSE_PIPELINE_STEPS = [ ("data_collector", "collect_ohlcv_final"), ("outcome_evaluator", "evaluate_outcomes"), ] # Intraday (light): keep prices current and resolve outcomes through the day, # without the expensive scan/sentiment. The dashboard recomputes live R:R from # the latest price, so refreshing OHLCV is enough to stop prices lagging; the # outcome step also closes paper trades that hit their stop/target intraday. _INTRADAY_PIPELINE_STEPS = [ ("data_collector", "collect_ohlcv"), ("outcome_evaluator", "evaluate_outcomes"), ] # Ordered by trading day, not alphabetically: this is the sequence an operator # reads down the page, and it drives the UI's ordering too. PIPELINE_STEPS: dict[str, list[tuple[str, str]]] = { "daily_pipeline": _DAILY_PIPELINE_STEPS, "intraday_pipeline": _INTRADAY_PIPELINE_STEPS, "near_close_pipeline": _NEAR_CLOSE_PIPELINE_STEPS, "after_close_pipeline": _AFTER_CLOSE_PIPELINE_STEPS, } # Derived, never hand-maintained: this used to be a literal set in admin_service # duplicating the four lists above from another module, with nothing asserting # the two agreed. PIPELINE_MEMBERS: frozenset[str] = frozenset( step for steps in PIPELINE_STEPS.values() for step, _ in steps ) def _pipelines_by_member() -> dict[str, tuple[str, ...]]: """Member -> the orchestrators that run it, in trading-day order. Membership is many-to-many: data_collector runs in all four pipelines (via three different coroutines), alerts and outcome_evaluator in two each. """ out: dict[str, list[str]] = {} for pipeline, steps in PIPELINE_STEPS.items(): for step, _ in steps: bucket = out.setdefault(step, []) if pipeline not in bucket: bucket.append(pipeline) return {member: tuple(pipelines) for member, pipelines in out.items()} PIPELINES_BY_MEMBER: dict[str, tuple[str, ...]] = _pipelines_by_member() # --------------------------------------------------------------------------- # Job identity # --------------------------------------------------------------------------- # Orchestrators, in trading-day order. PIPELINE_JOBS: tuple[str, ...] = tuple(PIPELINE_STEPS) # Own timer, independent of any pipeline. SCHEDULED_JOBS: tuple[str, ...] = ( "dolt_earnings_import", "sec_fundamentals_import", "ticker_universe_sync", "backtest", ) # Registered but never auto-fired; run only when a human asks. MANUAL_JOBS: tuple[str, ...] = ("event_study", "data_backfill") # Steps in the order an operator meets them across the trading day, so the UI # reads as a sequence rather than an alphabetical jumble. PIPELINE_STEP_JOBS: tuple[str, ...] = tuple( dict.fromkeys(step for steps in PIPELINE_STEPS.values() for step, _ in steps) ) VALID_JOB_NAMES: frozenset[str] = frozenset( PIPELINE_JOBS + PIPELINE_STEP_JOBS + SCHEDULED_JOBS + MANUAL_JOBS ) JOB_LABELS: dict[str, str] = { "data_collector": "Data Collector (OHLCV)", "data_backfill": "Data Backfill (deep history)", "benchmark_collector": "Benchmark Collector", "sentiment_collector": "Sentiment Collector", "dolt_earnings_import": "Dolt Earnings Import", "sec_fundamentals_import": "SEC Fundamentals Import", "rr_scanner": "R:R Scanner", "ticker_universe_sync": "Ticker Universe Sync", "outcome_evaluator": "Outcome Evaluator", "alerts": "Alerts Dispatcher", # Keys are persisted job ids and must not change; these are display only. "market_regime": "Market Trend (SPY)", "regime_monitor": "AI/Tech Risk Monitor", "event_study": "Event Study", "backtest": "Backtest", "daily_pipeline": "Morning Pipeline", "near_close_pipeline": "Near-Close Pipeline (scan+alert)", "after_close_pipeline": "After-Close Pipeline (outcome)", "intraday_pipeline": "Intraday Pipeline", "shadow_book": "Shadow Book (auto-traded strategy)", } CATEGORY_PIPELINE = "pipeline" CATEGORY_STEP = "pipeline_step" CATEGORY_SCHEDULED = "scheduled" CATEGORY_MANUAL = "manual" # Order the sections appear in. CATEGORY_ORDER: tuple[str, ...] = ( CATEGORY_PIPELINE, CATEGORY_STEP, CATEGORY_SCHEDULED, CATEGORY_MANUAL, ) CATEGORY_LABELS: dict[str, str] = { CATEGORY_PIPELINE: "Pipelines", CATEGORY_STEP: "Pipeline steps", CATEGORY_SCHEDULED: "Standalone scheduled", CATEGORY_MANUAL: "Manual only", } _CATEGORY_MEMBERS: dict[str, tuple[str, ...]] = { CATEGORY_PIPELINE: PIPELINE_JOBS, CATEGORY_STEP: PIPELINE_STEP_JOBS, CATEGORY_SCHEDULED: SCHEDULED_JOBS, CATEGORY_MANUAL: MANUAL_JOBS, } JOB_CATEGORY: dict[str, str] = { name: category for category, names in _CATEGORY_MEMBERS.items() for name in names } # Registered and triggerable through the API, but kept out of Admin → Jobs. # data_backfill's only capability beyond collect_ohlcv (which already backfills # full history for *new* tickers) is re-deepening *existing* ones after # ohlcv_history_days is raised -- a rare one-off, not something to scan past # every time you open the page. HIDDEN_JOBS: frozenset[str] = frozenset({"data_backfill"}) _SORT_INDEX: dict[str, tuple[int, int]] = { name: (CATEGORY_ORDER.index(category), position) for category, names in _CATEGORY_MEMBERS.items() for position, name in enumerate(names) } def sort_order(job_name: str) -> tuple[int, int]: """(category rank, position within category). Unknown jobs sort last.""" return _SORT_INDEX.get(job_name, (len(CATEGORY_ORDER), 0))