fix: match shadow book to its pipeline's scan by run id, not timestamp

A manually triggered rr_scanner and the scheduled near-close pipeline are
separate APScheduler jobs; max_instances=1 serialises a job only against
itself, so they can overlap. A manual scan starting just before the
pipeline can finish just after it began and overwrite the scan markers.
Its completion timestamp is then later than the pipeline start, so the
previous 'completed >= pipeline_start' check accepted its batch as though
it were the pipeline's own -- exactly when the pipeline's scan may have
failed.

Replace the timestamp comparison with an exact run-id match. A new
pipeline_run module holds a per-task run-id contextvar (separate module so
the scanner and scheduler import it without a cycle). _run_pipeline binds a
fresh id per invocation; scan_all_tickers stamps that id -- or a fresh one
when run standalone -- into the scan markers, written with started/completed
in a single commit. The shadow step requires the stored run id to equal its
pipeline's id exactly, so a concurrent manual scan (its own id) or a failed
pipeline scan (a prior run's id) can never be mistaken for it. Direct Admin
triggers have no pipeline context and keep the freshness fallback.

Known residual: the id match governs whether shadow proceeds; setup
selection remains detected_at >= scan start, so a fully per-run setup
isolation would need a run_id column on trade_setups (not required here).

Tests cover the reported race (manual scan finishing last is refused), a
failed pipeline scan, the id-match accept path, and contextvar propagation
and non-leakage across tasks.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

app/services/pipeline_run.py

@@ -0,0 +1,47 @@
+"""Per-invocation identity for pipeline runs.
+
+A pipeline invocation stamps a unique run id into the task context. The scan it
+runs records that id alongside its completion markers, and the shadow book
+requires an *exact* match before acting on the scan's batch.
+
+This is what timestamp comparison cannot provide. A manually triggered scan and
+the scheduled near-close pipeline are separate APScheduler jobs, and
+``max_instances=1`` only serialises a job against itself — not two different
+jobs. So a manual scan can start just before the pipeline and finish just after
+it began, leaving a completion timestamp later than the pipeline's start even
+though its batch is unrelated. Matching on a run id generated by the pipeline,
+and stamped only by the scan running inside that pipeline, removes the ambiguity.
+
+Lives in its own module so the scheduler (which sets the id), the scanner (which
+stamps it), and the shadow book (which checks it) can all import it without an
+import cycle.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import uuid
+
+_run_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+    "pipeline_run_id", default=None
+)
+
+
+def new_run_id() -> str:
+    """A fresh, collision-free run id."""
+    return uuid.uuid4().hex
+
+
+def current() -> str | None:
+    """Run id of the pipeline invocation on the current task, if any."""
+    return _run_id.get()
+
+
+def bind(run_id: str) -> contextvars.Token:
+    """Set the current run id; pass the returned token to ``release``."""
+    return _run_id.set(run_id)
+
+
+def release(token: contextvars.Token) -> None:
+    """Restore the previous run id (call in a finally)."""
+    _run_id.reset(token)

app/services/rr_scanner_service.py

@@ -47,12 +47,15 @@ from app.services.recommendation_service import (
 
 logger = logging.getLogger(__name__)
 
-# Boundary of the most recent *successful* scan. Written only when
-# scan_all_tickers completes, so a consumer can tell a scan actually ran this
-# pipeline pass (freshness of COMPLETED) and which setups belong to it
-# (detected_at >= STARTED). The shadow book relies on both.
+# Boundary of the most recent *successful* scan. Written together, only when
+# scan_all_tickers completes: STARTED bounds which setups belong to the run
+# (detected_at >= STARTED), COMPLETED gives its freshness, and RUN_ID identifies
+# the pipeline invocation that produced it (or a fresh id for a manual scan).
+# The shadow book matches RUN_ID exactly rather than trusting timestamps, so a
+# concurrent manual scan cannot be mistaken for the pipeline's own.
 KEY_LAST_SCAN_STARTED = "last_scan_run_started_at"
 KEY_LAST_SCAN_COMPLETED = "last_scan_run_completed_at"
+KEY_LAST_SCAN_RUN_ID = "last_scan_run_id"
 
 STRATEGY_VERSION = "residual_highvol_80_20_atr_trail3_v1"
 
@@ -820,17 +823,22 @@ async def scan_all_tickers(
     if progress_callback is not None and total:
         progress_callback(total, total, "")
 
-    # Record the run boundary only now that the scan has completed. The shadow
-    # book refuses to trade unless COMPLETED is fresh (proving a scan ran in this
-    # pipeline pass, not a prior session) and selects only setups from this run
-    # (detected_at >= STARTED). Written after the loop so a hard failure above
-    # leaves the previous, now-stale, marker in place.
+    # Record the run boundary only now that the scan has completed, stamped with
+    # the run id of the pipeline this scan ran inside (or a fresh id when run
+    # standalone — a manual scan then can never match a pipeline's expected id).
+    # All three markers are written in one commit so a reader sees a consistent
+    # (started, completed, run_id) triple, and a hard failure above leaves the
+    # previous, now-superseded, markers in place.
+    from app.services import pipeline_run
+
+    run_id = pipeline_run.current() or pipeline_run.new_run_id()
     await settings_store.upsert_setting(
         db, KEY_LAST_SCAN_STARTED, gate_observation_started_at.isoformat()
     )
     await settings_store.upsert_setting(
         db, KEY_LAST_SCAN_COMPLETED, datetime.now(timezone.utc).isoformat()
     )
+    await settings_store.upsert_setting(db, KEY_LAST_SCAN_RUN_ID, run_id)
     await db.commit()
 
     return all_setups

app/services/shadow_book_service.py

@@ -173,24 +173,22 @@ async def _last_scan_start(
     db: AsyncSession,
     *,
     now: datetime,
-    require_scan_after: datetime | None = None,
+    expected_run_id: str | None = None,
 ) -> datetime | None:
-    """Start of the last successful scan, if it is the one we may act on.
+    """Start of the scan we may act on, or None if there is none.
 
-    Returns None — "no scan to act on" — unless the scanner's COMPLETED marker
-    passes the applicable check:
+    * ``expected_run_id`` set (pipeline step): the stored run id must match it
+      exactly. This is the airtight guarantee — a scan that was disabled or
+      failed in *this* pipeline never stamped this id, and a concurrent manual
+      scan (a separate APScheduler job, not serialised against the pipeline)
+      stamps its own id even when it finishes last, so neither can be mistaken
+      for the pipeline's own scan. Timestamp order alone cannot tell them apart.
+    * ``expected_run_id`` None (direct Admin trigger): fall back to the freshness
+      window. There is no pipeline scan to bind to, so acting on a recent scan is
+      the operator's explicit choice.
 
-    * ``require_scan_after`` set (pipeline step): the scan must have completed
-      at/after the pipeline began. This is the airtight guarantee — a scan that
-      was disabled or failed in *this* pipeline leaves the marker at a previous
-      run, and a fresh but earlier manual scan completed before the pipeline
-      started, so neither can stand in for the pipeline's own scan.
-    * ``require_scan_after`` None (direct Admin trigger): fall back to the
-      freshness window. There is no pipeline scan to bind to, so acting on a
-      recent scan is the operator's explicit choice.
-
-    Either way the returned value is the scan's start, the lower bound for the
-    setups belonging to that run.
+    The three markers are written in one commit, so the returned STARTED belongs
+    to the same run as the matched RUN_ID and correctly bounds its setups.
     """
     from app.services import rr_scanner_service as rr
 
@@ -198,10 +196,11 @@ async def _last_scan_start(
     completed = _parse_dt(
         await settings_store.get_value(db, rr.KEY_LAST_SCAN_COMPLETED)
     )
+    run_id = await settings_store.get_value(db, rr.KEY_LAST_SCAN_RUN_ID)
     if started is None or completed is None:
         return None
-    if require_scan_after is not None:
-        if completed < require_scan_after:
+    if expected_run_id is not None:
+        if not run_id or run_id != expected_run_id:
             return None
     elif now - completed > MAX_SCAN_AGE:
         return None
@@ -222,7 +221,7 @@ async def _todays_qualified_setups(
     config: dict,
     *,
     now: datetime,
-    require_scan_after: datetime | None = None,
+    expected_run_id: str | None = None,
 ) -> list[TradeSetup]:
     """Long-only qualified setups from the scan that just ran, best rank first.
 
@@ -241,7 +240,7 @@ async def _todays_qualified_setups(
     4. Qualify, then rank by ``strategy_rank`` (unranked sort last).
     """
     run_start = await _last_scan_start(
-        db, now=now, require_scan_after=require_scan_after
+        db, now=now, expected_run_id=expected_run_id
     )
     if run_start is None:
         return []
@@ -275,7 +274,7 @@ async def open_shadow_positions(
     *,
     activation_config: dict,
     opened_at: datetime | None = None,
-    require_scan_after: datetime | None = None,
+    expected_run_id: str | None = None,
 ) -> dict:
     """Fill free capacity with the top-ranked qualified setups.
 
@@ -283,9 +282,10 @@ async def open_shadow_positions(
     skip anything already held or locked out by post-stop gate-reset, and stop
     at capacity. Returns a summary for the job log.
 
-    ``require_scan_after`` binds this run to a scan that completed at/after that
-    instant (the pipeline's start), so a stale or manual scan cannot substitute
-    for a scan that failed in this pipeline pass. See ``_last_scan_start``.
+    ``expected_run_id`` binds this run to the scan that stamped that exact id
+    (the pipeline's own scan), so a scan that failed in this pipeline — or a
+    concurrent manual scan that finished last — cannot substitute for it. See
+    ``_last_scan_start``.
     """
     summary = {
         "opened": 0,
@@ -312,7 +312,7 @@ async def open_shadow_positions(
     timestamp = opened_at or datetime.now(timezone.utc)
 
     candidates = await _todays_qualified_setups(
-        db, activation_config, now=timestamp, require_scan_after=require_scan_after
+        db, activation_config, now=timestamp, expected_run_id=expected_run_id
     )
     for setup in candidates:
         if free_slots <= 0: