redesign activation gate to expected value + make pipelines cron-configurable

Diagnosing "no qualified signals for 5 days": setups were generated but none
qualified. The gate required BOTH a high min_rr (2.0) AND a high
min_target_probability (60), which became contradictory after the Jun-15
probability recalibration — probability already embeds R:R via the 1/(rr+1) ruin
term, so high-R:R targets are inherently low-probability and nothing cleared both.

Gate is now expected value (R): p*rr - (1-p) from the primary target's
probability. R:R and confidence stay as floors; high-conviction / exclude-conflicts
/ min-target-probability become optional tighteners (default off). Defaults:
min_expected_value=0.15, min_rr=1.2, min_confidence=55. EV is only enforced when
computable. Migration 009 clears stored activation_* rows so the new defaults
apply. Backtest sweeps min_expected_value instead of target probability.

Scheduling: pipelines are now cron-configurable in Admin -> Jobs. daily_pipeline
(full, default 0 7 * * *) plus a new light intraday_pipeline (OHLCV + outcome eval,
default hourly US session) that keeps prices/live-R:R current without setup churn.
Fundamentals on its own early weekly cron. Timezone configurable (default
Europe/Berlin). Moving interval->CronTrigger also fixes the restart-deferral bug
where an interval job's countdown resets on every process restart.

319 backend unit tests pass; frontend tsc clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/services/admin_service.py

@@ -1,5 +1,6 @@
 """Admin service: user management, system settings, data cleanup, job control."""
 
+import logging
 from datetime import datetime, timedelta, timezone
 
 from passlib.hash import bcrypt
@@ -17,6 +18,8 @@ from app.models.ticker import Ticker
 from app.models.trade_setup import TradeSetup
 from app.models.user import User
 
+logger = logging.getLogger(__name__)
+
 RECOMMENDATION_CONFIG_DEFAULTS: dict[str, float] = {
     "recommendation_high_confidence_threshold": 70.0,
     "recommendation_moderate_confidence_threshold": 50.0,
@@ -35,10 +38,12 @@ SUPPORTED_TICKER_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
 # Track Record's qualified stats. The outcome evaluator deliberately ignores
 # these — every setup is evaluated so the gate itself can be validated.
 #
-# Beyond raw R:R and confidence, the gate demands conviction: a high-conviction
-# action (LONG_HIGH / SHORT_HIGH), a clean read (risk Low / no conflicts), and a
-# probable primary target.
+# The core test is expected value (in R): probability-weighted asymmetry, so a
+# fat-but-improbable target and a likely-but-thin one are both rejected. R:R and
+# confidence are floors; high-conviction / clean-read / target-probability are
+# optional tighteners (off by default — turn on to be more selective).
 _ACTIVATION_FLOAT_KEYS: dict[str, str] = {
+    "min_expected_value": "activation_min_expected_value",
     "min_rr": "activation_min_rr",
     "min_confidence": "activation_min_confidence",
     "min_target_probability": "activation_min_target_probability",
@@ -48,11 +53,12 @@ _ACTIVATION_BOOL_KEYS: dict[str, str] = {
     "exclude_conflicts": "activation_exclude_conflicts",
 }
 ACTIVATION_DEFAULTS: dict[str, float | bool] = {
-    "min_rr": 2.0,
-    "min_confidence": 70.0,
-    "min_target_probability": 60.0,
-    "require_high_conviction": True,
-    "exclude_conflicts": True,
+    "min_expected_value": 0.15,
+    "min_rr": 1.2,
+    "min_confidence": 55.0,
+    "min_target_probability": 0.0,
+    "require_high_conviction": False,
+    "exclude_conflicts": False,
 }
 
 
@@ -195,6 +201,8 @@ async def update_activation_config(
     db: AsyncSession, updates: dict[str, float | bool]
 ) -> dict[str, float | bool]:
     """Update the activation gate. Accepts public keys; only supplied keys change."""
+    if "min_expected_value" in updates and not -1.0 <= updates["min_expected_value"] <= 10.0:
+        raise ValidationError("min_expected_value must be between -1 and 10 (R units)")
     if "min_rr" in updates and updates["min_rr"] < 0:
         raise ValidationError("min_rr must be >= 0")
     if "min_confidence" in updates and not 0 <= updates["min_confidence"] <= 100:
@@ -212,6 +220,59 @@ async def update_activation_config(
     return await get_activation_config(db)
 
 
+# ---------------------------------------------------------------------------
+# Pipeline schedule (cron)
+# ---------------------------------------------------------------------------
+
+async def get_schedule_config(db: AsyncSession) -> dict[str, str]:
+    """Cron schedule for the daily/intraday pipelines and fundamentals."""
+    from app.scheduler import load_schedule_config
+
+    return await load_schedule_config(db)
+
+
+async def update_schedule_config(
+    db: AsyncSession, updates: dict[str, str]
+) -> dict[str, str]:
+    """Validate, persist, and apply cron schedule changes to the running scheduler."""
+    from app.scheduler import (
+        SCHEDULE_DEFAULTS,
+        load_schedule_config,
+        reschedule_jobs,
+        validate_cron,
+    )
+
+    current = await load_schedule_config(db)
+    tz = (updates.get("schedule_timezone") or current["schedule_timezone"]).strip()
+
+    for key, value in updates.items():
+        if key not in SCHEDULE_DEFAULTS:
+            raise ValidationError(f"Unknown schedule key: {key}")
+        if key == "schedule_timezone":
+            # Validate the timezone against an existing cron expression.
+            try:
+                validate_cron(current["schedule_daily_pipeline_cron"], value)
+            except Exception as exc:
+                raise ValidationError(f"Invalid timezone: {value}") from exc
+        else:
+            try:
+                validate_cron(value, tz)
+            except Exception as exc:
+                raise ValidationError(f"Invalid cron for {key}: {value!r}") from exc
+
+    for key, value in updates.items():
+        await update_setting(db, key, str(value).strip())
+
+    new_config = await load_schedule_config(db)
+    try:
+        reschedule_jobs(new_config)
+    except Exception:
+        # Scheduler may not be running (e.g. unit tests) — the config is saved
+        # regardless and applied on next startup.
+        logger.warning("Could not reschedule jobs after config update", exc_info=True)
+    return new_config
+
+
 def _recommendation_public_to_storage_key(key: str) -> str:
     return f"recommendation_{key}"
 
@@ -486,6 +547,7 @@ VALID_JOB_NAMES = {
     "market_regime",
     "backtest",
     "daily_pipeline",
+    "intraday_pipeline",
 }
 
 JOB_LABELS = {
@@ -499,6 +561,7 @@ JOB_LABELS = {
     "market_regime": "Market Regime",
     "backtest": "Backtest",
     "daily_pipeline": "Daily Pipeline",
+    "intraday_pipeline": "Intraday Pipeline",
 }
 
 # Jobs driven by the daily_pipeline (in order) rather than their own timer.

app/services/backtest_service.py

@@ -36,7 +36,11 @@ from app.services.outcome_service import (
     evaluate_setup_against_bars,
 )
 from app.services.price_service import query_ohlcv
-from app.services.qualification import best_target_probability, setup_qualifies
+from app.services.qualification import (
+    best_target_probability,
+    expected_value_r,
+    setup_qualifies,
+)
 from app.services.recommendation_service import (
     _choose_recommended_action,
     _classify_by_probability,
@@ -131,6 +135,10 @@ def _window_setups(
         primary = _select_primary_target(targets)
         if primary is None:
             continue
+        # Flag the primary so qualification's EV uses the primary target's
+        # probability (matching production's enhance_trade_setup).
+        for t in targets:
+            t["is_primary"] = t is primary
         per_dir[direction] = {"stop": stop, "targets": targets, "primary": primary}
 
     available = set(per_dir.keys())
@@ -160,12 +168,13 @@ def _window_setups(
             stop_loss=stop,
             entry_price=entry,
         )
-        # meets_core = clears every gate EXCEPT target probability, so the report
-        # can sweep the min_target_probability threshold without re-replaying.
-        core_config = {**activation, "min_target_probability": 0.0}
+        # meets_core = clears every gate EXCEPT the expected-value floor, so the
+        # report can sweep the min_expected_value threshold without re-replaying.
+        core_config = {**activation, "min_expected_value": float("-inf")}
         meets_core = setup_qualifies(setup_ns, core_config)
+        ev = expected_value_r(setup_ns)
         best_prob = best_target_probability(setup_ns)
-        min_tp = float(activation.get("min_target_probability", 0.0))
+        min_ev = float(activation.get("min_expected_value", 0.0))
         out.append({
             "direction": direction,
             "entry": entry,
@@ -175,10 +184,11 @@ def _window_setups(
             "confidence": confidences[direction],
             "primary_prob": float(primary["probability"]),
             "best_prob": best_prob,
+            "ev": ev,
             "meets_core": meets_core,
             "action": action,
             "risk_level": risk_level,
-            "qualified": meets_core and best_prob >= min_tp,
+            "qualified": meets_core and ev is not None and ev >= min_ev,
         })
     return out
 
@@ -216,6 +226,7 @@ def _replay_ticker(symbol: str, records: list, config: dict, activation: dict) -
                 "confidence": s["confidence"],
                 "primary_prob": s["primary_prob"],
                 "best_prob": s["best_prob"],
+                "ev": s["ev"],
                 "meets_core": s["meets_core"],
                 "qualified": s["qualified"],
                 "outcome": outcome,
@@ -288,14 +299,17 @@ async def run_backtest(
     longs = [c for c in qualified if c["direction"] == "long"]
     shorts = [c for c in qualified if c["direction"] == "short"]
 
-    # Threshold sweep: re-apply the gate at several min_target_probability values
+    # Threshold sweep: re-apply the gate at several min_expected_value values
     # (holding the other conditions fixed) so the trade-off between how many
     # setups qualify and their expectancy is visible without re-replaying.
-    current_min_tp = float(activation.get("min_target_probability", 60.0))
+    current_min_ev = float(activation.get("min_expected_value", 0.15))
     sweep = []
-    for threshold in (60, 55, 50, 45, 40, 35, 30):
-        cands = [c for c in candidates if c["meets_core"] and c["best_prob"] >= threshold]
-        sweep.append({"min_target_probability": threshold, **_bucket_stats(cands)})
+    for threshold in (0.4, 0.3, 0.25, 0.2, 0.15, 0.1, 0.05, 0.0):
+        cands = [
+            c for c in candidates
+            if c["meets_core"] and c["ev"] is not None and c["ev"] >= threshold
+        ]
+        sweep.append({"min_expected_value": threshold, **_bucket_stats(cands)})
 
     return {
         "generated_at": datetime.now(timezone.utc).isoformat(),
@@ -310,7 +324,7 @@ async def run_backtest(
             "long": _bucket_stats(longs),
             "short": _bucket_stats(shorts),
         },
-        "min_target_probability": current_min_tp,
+        "min_expected_value": current_min_ev,
         "sweep": sweep,
         "calibration": _calibration(candidates),
         "note": (

app/services/qualification.py

@@ -1,9 +1,11 @@
 """Shared definition of a 'qualified' (actionable) trade setup.
 
 A single predicate, driven by the admin activation config, used by the
-performance stats (server) and mirrored on the frontend. Beyond raw R:R and
-confidence, an actionable setup must show genuine conviction: a high-conviction
-recommended action, a clean (conflict-free) read, and a probable primary target.
+performance stats (server) and mirrored on the frontend. The core gate is
+expected value (in R): a setup must promise positive, probability-weighted
+asymmetry, not just a fat-but-improbable target or a likely-but-thin one. R:R
+and confidence remain as floors, and conviction/conflict/target-probability
+survive as optional tighteners (off by default).
 """
 
 from __future__ import annotations
@@ -20,6 +22,37 @@ def best_target_probability(setup: Any) -> float:
     return max(probs, default=0.0)
 
 
+def primary_target_probability(setup: Any) -> float | None:
+    """Probability of the starred primary target (the one the headline R:R refers
+    to). Falls back to the best target's probability when none is flagged primary,
+    and None when there are no targets at all (probability unknowable).
+    """
+    targets = getattr(setup, "targets", None) or []
+    primary = next(
+        (t for t in targets if isinstance(t, dict) and t.get("is_primary")), None
+    )
+    if primary is not None:
+        return float(primary.get("probability", 0.0))
+    probs = [float(t.get("probability", 0.0)) for t in targets if isinstance(t, dict)]
+    return max(probs) if probs else None
+
+
+def expected_value_r(setup: Any) -> float | None:
+    """Expected value per unit of risk, in R: ``p·(R:R) − (1 − p)``.
+
+    ``p`` is the primary target's hit probability. This single number captures
+    "is this worth taking": it rewards both a good payoff ratio and a likely
+    target, so a fat-but-improbable target can't outrank a solid, probable one —
+    and a high R:R no longer fights a high probability the way the old separate
+    gates did. Returns None when no target probability is known.
+    """
+    p = primary_target_probability(setup)
+    if p is None:
+        return None
+    p = p / 100.0
+    return p * setup.rr_ratio - (1.0 - p)
+
+
 def live_risk_reward(setup: Any, current_price: float) -> float | None:
     """R:R recomputed from the CURRENT price, not the (possibly stale) entry.
 
@@ -43,6 +76,11 @@ def setup_qualifies(setup: Any, config: dict) -> bool:
 
     ``setup`` is duck-typed: any object exposing rr_ratio, confidence_score,
     recommended_action, risk_level and a ``targets`` list of dicts.
+
+    Gate order: R:R floor → freshness (live R:R) → confidence floor → expected
+    value (the core test) → optional conviction / conflict / target-probability
+    tighteners. ``min_expected_value`` defaults to -inf for callers that pass a
+    legacy config without the key, so they behave exactly as before.
     """
     if setup.rr_ratio < config["min_rr"]:
         return False
@@ -56,6 +94,13 @@ def setup_qualifies(setup: Any, config: dict) -> bool:
             return False
     if (setup.confidence_score or 0.0) < config["min_confidence"]:
         return False
+    # Expected value (R): the core gate. Only enforced when computable — setups
+    # without target probabilities (e.g. legacy historical rows) defer to the
+    # R:R + confidence floors above rather than being silently dropped.
+    min_ev = float(config.get("min_expected_value", float("-inf")))
+    ev = expected_value_r(setup)
+    if ev is not None and ev < min_ev:
+        return False
     if config.get("require_high_conviction"):
         if (setup.recommended_action or "") not in HIGH_CONVICTION_ACTIONS:
             return False