refactor(jobs): derive job topology from one catalog, make next-run coherent

Groundwork for the Admin -> Jobs cleanup. Three sources of truth collapse into
app/job_catalog.py, which imports nothing from app so both the scheduler and
admin_service can import it at module level (admin_service otherwise has to
import the scheduler inside functions to dodge a cycle).

PIPELINE_MEMBERS is now DERIVED from the four pipeline step lists instead of
being a literal set in admin_service duplicating four lists in scheduler.py with
nothing asserting they agreed. A test pins that the derivation reproduces the
previous hand-maintained 9 names exactly, so this is behaviour-preserving.

Deletes the private _JOB_NAMES list, which held 16 of the 19 jobs:
benchmark_collector, outcome_evaluator and shadow_book had no runtime row, and
so no "last run" line in the panel, until their first run in a given process.
_job_runtime is now seeded from the catalog, and a test pins the invariant.

Next-run is decided by category rather than by reading a timestamp. A pipeline
step has no schedule of its own, so it reports its parent's ("next via Morning
Pipeline in 3h") instead of nothing; a manual job says manual_only rather than
rendering a date. This also fixes a real bug: triggering a paused job set
next_run_time=now, APScheduler re-armed the 520-week backstop behind it, and the
panel displayed "next run in ~87600h". Two independent guards -- the category
rule, plus _visible_next_run dropping anything past a year -- and an APScheduler
listener that re-pauses steps and manual jobs once their run finishes. The
listener is registered at module level because configure_scheduler is called
more than once and add_listener does not deduplicate.

Migrates backtest and ticker_universe_sync from interval to cron (Sun 03:00 ET
and 01:00 ET). configure_scheduler calls remove_all_jobs() on every startup, so
an interval countdown restarts each deploy -- a 168h backtest needed a week of
uninterrupted uptime to fire even once. The codebase already documented this
pitfall as the reason cron was adopted; these two were never migrated. Both are
now editable in Admin -> Schedule.

Also: list_jobs went from one settings query per job (19) to one for all of
them, and data_backfill is hidden from the listing while staying registered and
API-triggerable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/unit/test_scheduler.py

@@ -1,16 +1,19 @@
 """Unit tests for app.scheduler module."""
 
 import asyncio
+from datetime import datetime, timezone
 from types import SimpleNamespace
 
 import pytest
 
+from app import job_catalog
 from app.scheduler import (
     _DAILY_PIPELINE_STEPS,
     _NEAR_CLOSE_PIPELINE_STEPS,
     _consume_backtest_options,
     _consume_backtest_target_model,
     _parse_frequency,
+    _repause_after_manual_run,
     _resume_tickers,
     _last_successful,
     _run_source_import,
@@ -112,60 +115,119 @@ class TestResumeTickers:
 
 class TestConfigureScheduler:
     def test_configure_adds_all_jobs(self):
-        # Remove any existing jobs first
+        # Derived from the catalog, not a fourth hand-maintained copy of the
+        # job list: a job added to the catalog but never registered now fails
+        # here instead of silently rendering "Not registered" in the admin UI.
         scheduler.remove_all_jobs()
         configure_scheduler()
-        jobs = scheduler.get_jobs()
-        job_ids = {j.id for j in jobs}
-        assert job_ids == {
-            "data_collector",
-            "data_backfill",
-            "benchmark_collector",
-            "sentiment_collector",
-            "dolt_earnings_import",
-            "sec_fundamentals_import",
-            "rr_scanner",
-            "shadow_book",
-            "ticker_universe_sync",
-            "outcome_evaluator",
-            "alerts",
-            "market_regime",
-            "regime_monitor",
-            "event_study",
-            "backtest",
-            "daily_pipeline",
-            "near_close_pipeline",
-            "after_close_pipeline",
-            "intraday_pipeline",
-        }
+        assert {j.id for j in scheduler.get_jobs()} == set(job_catalog.VALID_JOB_NAMES)
 
     def test_configure_is_idempotent(self):
         scheduler.remove_all_jobs()
         configure_scheduler()
         configure_scheduler()  # Should replace, not duplicate
         job_ids = [j.id for j in scheduler.get_jobs()]
-        # Each ID should appear exactly once
-        assert sorted(job_ids) == sorted([
-            "after_close_pipeline",
-            "alerts",
-            "backtest",
-            "benchmark_collector",
-            "daily_pipeline",
-            "intraday_pipeline",
+        assert sorted(job_ids) == sorted(job_catalog.VALID_JOB_NAMES)
+
+    def test_independent_jobs_use_cron_not_interval(self):
+        """Interval countdowns restart on every deploy, so a weekly interval on a
+        frequently-redeployed box can defer forever. Both standalone jobs were
+        migrated to cron; this pins them there."""
+        scheduler.remove_all_jobs()
+        configure_scheduler()
+        for job_id in ("backtest", "ticker_universe_sync"):
+            trigger = type(scheduler.get_job(job_id).trigger).__name__
+            assert trigger == "CronTrigger", f"{job_id} regressed to {trigger}"
+
+
+class TestJobCatalog:
+    def test_pipeline_members_are_derived_from_step_lists(self):
+        derived = {
+            step
+            for steps in job_catalog.PIPELINE_STEPS.values()
+            for step, _ in steps
+        }
+        assert job_catalog.PIPELINE_MEMBERS == derived
+        # ...and reproduces the set that used to be maintained by hand, so the
+        # derivation is behaviour-preserving rather than merely self-consistent.
+        assert job_catalog.PIPELINE_MEMBERS == {
             "data_collector",
-            "data_backfill",
-            "dolt_earnings_import",
-            "sec_fundamentals_import",
-            "market_regime",
-            "near_close_pipeline",
-            "regime_monitor",
-            "event_study",
-            "outcome_evaluator",
-            "rr_scanner",
+            "benchmark_collector",
             "sentiment_collector",
+            "rr_scanner",
             "shadow_book",
-            "ticker_universe_sync",
-        ])
+            "outcome_evaluator",
+            "alerts",
+            "market_regime",
+            "regime_monitor",
+        }
+
+    def test_categories_partition_every_job_exactly_once(self):
+        buckets = [
+            job_catalog.PIPELINE_JOBS,
+            job_catalog.PIPELINE_STEP_JOBS,
+            job_catalog.SCHEDULED_JOBS,
+            job_catalog.MANUAL_JOBS,
+        ]
+        flat = [name for bucket in buckets for name in bucket]
+        assert len(flat) == len(set(flat)), "a job is in two categories"
+        assert set(flat) == set(job_catalog.VALID_JOB_NAMES)
+        assert all(name in job_catalog.JOB_CATEGORY for name in flat)
+
+    def test_every_job_has_a_label_and_a_unique_sort_order(self):
+        names = job_catalog.VALID_JOB_NAMES
+        assert set(job_catalog.JOB_LABELS) == set(names)
+        assert len({job_catalog.sort_order(n) for n in names}) == len(names)
+
+    def test_multi_pipeline_members_report_every_parent(self):
+        """Membership is many-to-many — the reason the UI groups into sections
+        rather than nesting steps under one parent."""
+        by_member = job_catalog.PIPELINES_BY_MEMBER
+        assert set(by_member["data_collector"]) == set(job_catalog.PIPELINE_JOBS)
+        assert set(by_member["alerts"]) == {"daily_pipeline", "near_close_pipeline"}
+        assert set(by_member["outcome_evaluator"]) == {
+            "intraday_pipeline",
+            "after_close_pipeline",
+        }
+        assert "backtest" not in by_member
+
+    def test_every_job_has_a_runtime_row_before_it_first_runs(self):
+        """The old private _JOB_NAMES list held 16 of 19, so three jobs showed no
+        last-run line until their first run in a given process."""
+        assert set(get_job_runtime_snapshot()) == set(job_catalog.VALID_JOB_NAMES)
+
+
+class TestRepauseListener:
+    def _configured(self):
+        scheduler.remove_all_jobs()
+        configure_scheduler()
+
+    def test_manual_job_is_repaused_after_running(self):
+        """Triggering a paused job re-arms its 520-week backstop, which used to
+        surface as a "next run in ~87600h"."""
+        self._configured()
+        scheduler.modify_job("event_study", next_run_time=datetime.now(timezone.utc))
+        _repause_after_manual_run(SimpleNamespace(job_id="event_study"))
+        assert scheduler.get_job("event_study").next_run_time is None
+
+    def test_pipeline_step_is_repaused_after_running(self):
+        self._configured()
+        scheduler.modify_job("rr_scanner", next_run_time=datetime.now(timezone.utc))
+        _repause_after_manual_run(SimpleNamespace(job_id="rr_scanner"))
+        assert scheduler.get_job("rr_scanner").next_run_time is None
+
+    def test_cron_jobs_are_left_alone(self):
+        # Set an explicit next run first: an unstarted scheduler leaves the
+        # attribute unset, so comparing None to None would prove nothing.
+        self._configured()
+        due = datetime.now(timezone.utc)
+        scheduler.modify_job("daily_pipeline", next_run_time=due)
+        _repause_after_manual_run(SimpleNamespace(job_id="daily_pipeline"))
+        assert scheduler.get_job("daily_pipeline").next_run_time == due
+
+    def test_unknown_job_is_ignored(self):
+        self._configured()
+        _repause_after_manual_run(SimpleNamespace(job_id="not_a_job"))
 
 
 class _SessionContext: