dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize GTL and retire S/R research harness

Browse Source
This commit is contained in:
dennisthiessen
2026-07-13 17:58:04 +02:00
parent 9d362bd568
commit bee5a5ce89
35 changed files with 374 additions and 5385779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/routers/admin.py
+8 -2
View File
@@ -13,6 +13,7 @@ from app.schemas.admin import (
AlertConfigUpdate,
CreateUserRequest,
DataCleanupRequest,
JobTriggerRequest,
JobToggle,
RecommendationConfigUpdate,
ScheduleConfigUpdate,
@@ -376,11 +377,16 @@ async def get_pipeline_readiness(
@router.post("/admin/jobs/{job_name}/trigger", response_model=APIEnvelope)
async def trigger_job(
job_name: str,
body: JobTriggerRequest | None = None,
_admin: User = Depends(require_admin),
db: AsyncSession = Depends(get_db),
):
"""Trigger a manual job run (placeholder)."""
result = await admin_service.trigger_job(db, job_name)
"""Trigger a manual job run, optionally with one-run parameters."""
result = await admin_service.trigger_job(
db,
job_name,
target_model=body.target_model if body is not None else None,
)
return APIEnvelope(status="success", data=result)
app/scheduler.py
+44 -4
View File
@@ -35,7 +35,12 @@ from app.providers.fundamentals_chain import build_fundamental_provider_chain
from app.providers.protocol import SentimentData
from app.services import fundamental_service, ingestion_service, sentiment_service, settings_store
from app.services.alert_service import dispatch_alerts
from app.services.backtest_service import run_and_store as run_backtest_and_store
from app.services.backtest_service import (
BACKTEST_TARGET_MODELS,
PRODUCTION_GTL_TARGET_MODEL,
run_and_store as run_backtest_and_store,
validate_backtest_target_model,
)
from app.services.benchmark_service import refresh_benchmark_prices
from app.services.market_regime_service import update_market_regime
from app.services.regime_monitor_service import update_regime_monitor
@@ -106,6 +111,7 @@ def _idle_runtime() -> dict[str, object]:
_job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES}
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
# ---------------------------------------------------------------------------
@@ -113,6 +119,26 @@ _job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in
# ---------------------------------------------------------------------------
def queue_backtest_target_model(target_model: str | None) -> str:
"""Select the model for the next manual backtest run only.
Scheduled runs and subsequent manual runs return to the production GTL.
"""
global _next_backtest_target_model
selected = validate_backtest_target_model(
target_model or PRODUCTION_GTL_TARGET_MODEL
)
_next_backtest_target_model = selected
return selected
def _consume_backtest_target_model() -> str:
global _next_backtest_target_model
selected = _next_backtest_target_model
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
return selected
def _log_event(level: int, event: str, **fields: object) -> None:
"""Emit a structured JSON log line: {"event": ..., **fields}."""
logger.log(level, json.dumps({"event": event, **fields}))
@@ -939,7 +965,13 @@ async def compute_regime_monitor() -> None:
async def run_backtest_job() -> None:
"""Replay the price-derived engine over history and cache the report."""
job_name = "backtest"
_log_event(logging.INFO, "job_start", job=job_name)
target_model = _consume_backtest_target_model()
_log_event(
logging.INFO,
"job_start",
job=job_name,
target_model=target_model,
)
_runtime_start(job_name)
def _on_progress(done: int, count: int, symbol: str) -> None:
@@ -952,12 +984,20 @@ async def run_backtest_job() -> None:
_runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
return
report = await run_backtest_and_store(db, _on_progress)
report = await run_backtest_and_store(
db,
_on_progress,
target_model=target_model,
)
_runtime_finish(
job_name, "completed",
processed=report.get("tickers", 0), total=report.get("tickers", 0),
message=f"{report.get('candidates', 0)} setups, {report.get('qualified', 0)} qualified",
message=(
f"{BACKTEST_TARGET_MODELS[target_model]}: "
f"{report.get('candidates', 0)} setups, "
f"{report.get('qualified', 0)} qualified"
),
)
_log_event(logging.INFO, "job_complete", job=job_name, candidates=report.get("candidates"))
except Exception as exc:
app/schemas/admin.py
+5
View File
@@ -43,6 +43,11 @@ class JobToggle(BaseModel):
enabled: bool
class JobTriggerRequest(BaseModel):
"""Optional parameters for a one-time manual job run."""
target_model: Literal["production_gtl", "structural_sr"] | None = None
class RecommendationConfigUpdate(BaseModel):
high_confidence_threshold: float | None = Field(default=None, ge=0, le=100)
moderate_confidence_threshold: float | None = Field(default=None, ge=0, le=100)
app/services/admin_service.py
+17 -2
View File
@@ -602,13 +602,20 @@ async def list_jobs(db: AsyncSession) -> list[dict]:
return jobs_out
async def trigger_job(db: AsyncSession, job_name: str) -> dict[str, str]:
async def trigger_job(
db: AsyncSession,
job_name: str,
*,
target_model: str | None = None,
) -> dict[str, str]:
"""Trigger a manual job run via the scheduler.
Runs the job immediately (in addition to its regular schedule).
"""
if job_name not in VALID_JOB_NAMES:
raise ValidationError(f"Unknown job: {job_name}. Valid jobs: {', '.join(sorted(VALID_JOB_NAMES))}")
if target_model is not None and job_name != "backtest":
raise ValidationError("target_model is supported only for the backtest job")
from app.scheduler import get_job_runtime_snapshot, scheduler
@@ -635,11 +642,19 @@ async def trigger_job(db: AsyncSession, job_name: str) -> dict[str, str]:
if job is None:
return {"job": job_name, "status": "not_found", "message": f"Job '{job_name}' is not registered in the scheduler"}
if job_name == "backtest":
from app.scheduler import queue_backtest_target_model
target_model = queue_backtest_target_model(target_model)
job.modify(next_run_time=None) # Reset, then trigger immediately
from datetime import datetime, timezone
job.modify(next_run_time=datetime.now(timezone.utc))
return {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
result = {"job": job_name, "status": "triggered", "message": f"Job '{job_name}' triggered for immediate execution"}
if target_model is not None:
result["target_model"] = target_model
return result
async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSetting:
app/services/backtest_service.py
+61 -427
View File
@@ -33,7 +33,7 @@ import statistics
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import ProcessPoolExecutor
from datetime import date, datetime, timedelta, timezone
from datetime import date, datetime, timezone
from types import SimpleNamespace
from typing import Any
@@ -82,12 +82,7 @@ from app.services.scoring_service import (
compute_momentum_from_closes,
compute_technical_from_arrays,
)
from app.services.sr_service import (
MAX_LEVELS,
detect_gate_target_ladder,
detect_sr_levels,
detect_sr_levels_legacy,
)
from app.services.sr_service import detect_gate_target_ladder, detect_sr_levels
logger = logging.getLogger(__name__)
@@ -97,23 +92,11 @@ STEP_DAYS = 5 # weekly cadence (≈ 5 trading days)
MIN_LOOKBACK = 60 # bars needed before D for indicators (EMA cross needs 51)
HORIZON = 30 # trading days to resolve an outcome (matches the evaluator)
ATR_MULTIPLIER = 1.5
RANGE_FACTOR_LOOKBACK = 504
RANGE_FACTOR_MIN_LOG = 1.0 # approximately a 2.7x high/low span
STRUCTURAL_OVERLAY_VARIANT = "production_structural_overlay"
STRUCTURAL_OVERLAY_SOURCE_VARIANT = (
"rewrite_range504_structural_legacy_primary"
)
STRUCTURAL_OVERLAY_WEIGHT = 0.05
STRUCTURAL_OVERLAY_SCORE_KEY = "structural_overlay_95_5_score"
EXPLICIT_TARGET_LADDER_VARIANT = "explicit_target_ladder"
RANGE_RESIDUAL_VARIANTS = {
"rewrite_range504_structural_legacy_primary",
"rewrite_range504_structural_primary2",
}
RANGE_FACTOR_VARIANTS = {
"production_range504",
"rewrite_range504_legacy_primary",
*RANGE_RESIDUAL_VARIANTS,
PRODUCTION_GTL_TARGET_MODEL = "production_gtl"
STRUCTURAL_SR_TARGET_MODEL = "structural_sr"
BACKTEST_TARGET_MODELS = {
PRODUCTION_GTL_TARGET_MODEL: "Live GTL (production)",
STRUCTURAL_SR_TARGET_MODEL: "Structural S/R (comparison)",
}
# Cross-sectional signal evaluation (factor IC). Each candidate signal is a
@@ -153,128 +136,13 @@ def _wrap_levels(level_dicts: list[dict]) -> list[Any]:
]
SR_RESEARCH_VARIANTS = {
"production_control",
"rr_aligned_control",
"rewrite",
"soft_zones",
"confirmed_rounds",
"gate_v2",
"rewrite_legacy_primary",
"soft_zones_legacy_primary",
"confirmed_rounds_legacy_primary",
"gate_v2_legacy_primary",
"legacy_geometry_neutral",
"legacy_pivots_only",
"legacy_traffic_grid_only",
"legacy_range_grid_touch",
"legacy_range_grid_neutral",
EXPLICIT_TARGET_LADDER_VARIANT,
STRUCTURAL_OVERLAY_VARIANT,
*RANGE_FACTOR_VARIANTS,
}
def _sr_research_variant() -> str:
"""Backtest target policy; research arms require an explicit override.
The unconfigured online/scheduled backtest must replay the same transient
Gate Target Ladder as the live scanner. Clean Structural S/R remains an
opt-in research arm here because it is intentionally chart/alert structure,
not the production gate's target source.
"""
value = os.getenv(
"BACKTEST_SR_VARIANT",
EXPLICIT_TARGET_LADDER_VARIANT,
).strip().lower()
if value not in SR_RESEARCH_VARIANTS:
allowed = ", ".join(sorted(SR_RESEARCH_VARIANTS))
raise ValueError(f"Unknown BACKTEST_SR_VARIANT={value!r}; expected one of {allowed}")
return value
def _sr_detector_variant(sr_variant: str) -> str:
"""Map factor-gated research arms to the detector they hold fixed."""
if sr_variant in {"production_range504", STRUCTURAL_OVERLAY_VARIANT}:
return "production_control"
if (
sr_variant == "rewrite_range504_legacy_primary"
or sr_variant in RANGE_RESIDUAL_VARIANTS
):
return "rewrite"
return sr_variant.removesuffix("_legacy_primary")
def _range_504_log(highs: list[float], lows: list[float]) -> float:
"""Multiplicative high/low range over the last two trading years."""
window_highs = highs[-RANGE_FACTOR_LOOKBACK:]
window_lows = lows[-RANGE_FACTOR_LOOKBACK:]
if not window_highs or not window_lows:
return 0.0
high = max(window_highs)
low = min(window_lows)
if high <= 0 or low <= 0 or high < low:
return 0.0
return math.log(high / low)
def _range_factor_allows(sr_variant: str, range_504_log: float) -> bool:
"""Apply the explicit range factor only in its diagnostic arms."""
return (
sr_variant not in RANGE_FACTOR_VARIANTS
or range_504_log >= RANGE_FACTOR_MIN_LOG
)
def _primary_min_rr_for_variant(sr_variant: str, activation: dict) -> float:
"""Preserve the deployed selector except in explicit primary-2 research."""
if (
sr_variant in {
"production_control",
"production_range504",
EXPLICIT_TARGET_LADDER_VARIANT,
STRUCTURAL_OVERLAY_VARIANT,
}
or sr_variant.endswith("_legacy_primary")
or sr_variant.startswith("legacy_")
):
return 1.5
return float(activation.get("min_rr", 0.0))
def _apply_zone_strength_variant(zone_levels: list[Any], sr_variant: str) -> list[Any]:
"""Apply post-cluster research controls without changing zone geometry."""
if sr_variant in {"legacy_geometry_neutral", "legacy_range_grid_neutral"}:
# Neutrality must be enforced after the shared zone cluster: its legacy
# sum mode can otherwise turn two 50-strength constituents back into a
# 100-strength target and silently invalidate the ablation.
for level in zone_levels:
level.strength = 50
return zone_levels
def _backtest_entry_bounds() -> tuple[date | None, date | None]:
"""Optional research-only entry bounds used to protect validation data."""
parsed: list[date | None] = []
for key in ("BACKTEST_ENTRY_START", "BACKTEST_ENTRY_END"):
raw = os.getenv(key, "").strip()
if not raw:
parsed.append(None)
continue
try:
parsed.append(date.fromisoformat(raw))
except ValueError as exc:
raise ValueError(f"{key} must be YYYY-MM-DD, got {raw!r}") from exc
start, end = parsed
if start is not None and end is not None and start > end:
raise ValueError("BACKTEST_ENTRY_START must be on or before BACKTEST_ENTRY_END")
return start, end
def _sr_audit_enabled() -> bool:
return os.getenv("BACKTEST_SR_AUDIT", "").strip().lower() in {
"1", "true", "yes", "on",
}
def validate_backtest_target_model(value: str) -> str:
"""Validate the small, user-facing set of supported backtest target models."""
normalized = value.strip().lower()
if normalized not in BACKTEST_TARGET_MODELS:
allowed = ", ".join(BACKTEST_TARGET_MODELS)
raise ValueError(f"Unknown backtest target model {value!r}; expected one of {allowed}")
return normalized
def _atr_target_fallback_k() -> float | None:
@@ -346,7 +214,7 @@ def _window_setups(
config: dict,
activation: dict,
*,
sr_variant: str | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> list[dict]:
"""Rebuild the setup(s) at the last bar of ``window_records`` (the as-of date),
using only those bars. Returns one dict per tradeable direction."""
@@ -373,58 +241,20 @@ def _window_setups(
if atr <= 0:
return []
sr_variant = sr_variant or _sr_research_variant()
detector_variant = _sr_detector_variant(sr_variant)
range_504_log = _range_504_log(highs, lows)
if sr_variant == "legacy_geometry_neutral":
detected_levels = detect_sr_levels_legacy(
highs, lows, closes, volumes, neutral_strength=True
)
elif sr_variant == "legacy_pivots_only":
detected_levels = detect_sr_levels_legacy(
highs, lows, closes, volumes, include_volume_profile=False
)
elif sr_variant == "legacy_traffic_grid_only":
detected_levels = detect_sr_levels_legacy(
highs, lows, closes, volumes, include_pivots=False
)
elif sr_variant == EXPLICIT_TARGET_LADDER_VARIANT:
target_model = validate_backtest_target_model(target_model)
if target_model == PRODUCTION_GTL_TARGET_MODEL:
detected_levels = detect_gate_target_ladder(
highs,
lows,
closes,
)
elif sr_variant in {"legacy_range_grid_touch", "legacy_range_grid_neutral"}:
detected_levels = detect_sr_levels_legacy(
highs,
lows,
closes,
volumes,
include_pivots=False,
neutral_strength=sr_variant == "legacy_range_grid_neutral",
explicit_range_grid=True,
)
elif detector_variant in {"production_control", "rr_aligned_control"}:
detected_levels = detect_sr_levels_legacy(highs, lows, closes, volumes)
else:
detector_cap = 0 if detector_variant == "gate_v2" else MAX_LEVELS
detected_levels = detect_sr_levels(
highs, lows, closes, volumes, max_levels=detector_cap
)
detected_levels = detect_sr_levels(highs, lows, closes, volumes)
sr_levels = _wrap_levels(detected_levels)
if not sr_levels:
return []
gate_levels = _gate_eligible_levels(
sr_levels,
confirmed_rounds_only=detector_variant in {"confirmed_rounds", "gate_v2"},
exclude_standalone_rounds=sr_variant in RANGE_RESIDUAL_VARIANTS,
)
zone_strength_mode = (
"soft"
if detector_variant in {"soft_zones", "confirmed_rounds", "gate_v2"}
else "sum"
)
gate_levels = _gate_eligible_levels(sr_levels)
technical = (compute_technical_from_arrays(highs, lows, closes, volumes)[0]) or 50.0
momentum = (compute_momentum_from_closes(closes)[0]) or 50.0
@@ -443,9 +273,8 @@ def _window_setups(
zone_levels = _zone_representative_levels(
gate_levels,
entry,
strength_mode=zone_strength_mode,
strength_mode="sum",
)
zone_levels = _apply_zone_strength_variant(zone_levels, sr_variant)
targets = target_generator.generate_targets(direction, entry, stop, zone_levels, atr)
if not targets:
fallback_k = _atr_target_fallback_k()
@@ -462,10 +291,9 @@ def _window_setups(
# Collapse duplicate floor-pinned lottery targets (parity with
# enhance_trade_setup).
targets = _prune_floor_pinned_targets(targets)
primary_min_rr = _primary_min_rr_for_variant(sr_variant, activation)
primary = _select_primary_target(
targets,
min_rr=primary_min_rr,
min_rr=1.5,
)
if primary is None:
continue
@@ -507,10 +335,6 @@ def _window_setups(
# week are known. run_backtest ranks momentum and finalizes `qualified`.
core_config = {**activation, "min_momentum_percentile": 0.0}
meets_core = setup_qualifies(setup_ns, core_config)
meets_core = meets_core and _range_factor_allows(
sr_variant,
range_504_log,
)
best_prob = best_target_probability(setup_ns)
out.append({
"direction": direction,
@@ -525,7 +349,7 @@ def _window_setups(
"meets_core": meets_core,
"action": action,
"risk_level": risk_level,
"sr_variant": sr_variant,
"target_model": target_model,
"primary_sources": list(primary.get("sr_sources") or []),
"primary_strength": float(primary.get("sr_strength", 0.0)),
"primary_rejection_count": int(
@@ -537,62 +361,10 @@ def _window_setups(
),
"raw_level_count": len(sr_levels),
"gate_level_count": len(gate_levels),
"range_504_log": range_504_log,
"range_504_ratio": math.exp(range_504_log),
"range_factor_pass": range_504_log >= RANGE_FACTOR_MIN_LOG,
})
return out
def _structural_overlay_window_setups(
window_records: list,
config: dict,
activation: dict,
) -> list[dict]:
"""Production setups tagged by the clean structural/range candidate.
The returned population and setup geometry remain production-identical.
The clean detector is only a point-in-time feature, so this arm can test a
ranking overlay without silently changing admission breadth or targets.
"""
production = _window_setups(
window_records,
config,
activation,
sr_variant="production_control",
)
if not production:
return []
structural = _window_setups(
window_records,
config,
activation,
sr_variant=STRUCTURAL_OVERLAY_SOURCE_VARIANT,
)
structural_by_direction = {row["direction"]: row for row in structural}
tagged: list[dict] = []
for production_row in production:
row = dict(production_row)
structural_row = structural_by_direction.get(row["direction"])
row["sr_variant"] = STRUCTURAL_OVERLAY_VARIANT
row["structural_overlay_pass"] = bool(
structural_row and structural_row.get("meets_core")
)
row["structural_overlay_rr"] = (
float(structural_row["rr"]) if structural_row is not None else None
)
row["structural_overlay_sources"] = (
list(structural_row.get("primary_sources") or [])
if structural_row is not None else []
)
row["structural_overlay_gate_level_count"] = (
int(structural_row.get("gate_level_count", 0) or 0)
if structural_row is not None else 0
)
tagged.append(row)
return tagged
def _stop_fill_r(direction: str, entry: float, stop: float, bar) -> float:
"""Realized R when the stop is hit on ``bar``: filled at the stop, or at the
bar's open when price gapped through it — so a gap can lose more than −1R,
@@ -672,6 +444,7 @@ def _replay_ticker(
config: dict,
activation: dict,
benchmark_closes: dict[date, float] | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> list[dict]:
"""Walk one ticker's history weekly, building setups and their realized outcomes."""
candidates: list[dict] = []
@@ -679,14 +452,7 @@ def _replay_ticker(
if n < MIN_LOOKBACK + HORIZON:
return candidates
entry_start, entry_end = _backtest_entry_bounds()
sr_variant = _sr_research_variant()
for i in range(MIN_LOOKBACK - 1, n - HORIZON, STEP_DAYS):
as_of = records[i].date
if entry_start is not None and as_of < entry_start:
continue
if entry_end is not None and as_of > entry_end:
continue
window = records[: i + 1]
forward = records[i + 1 :]
forward_bars = [Bar(date=r.date, high=r.high, low=r.low) for r in forward]
@@ -697,15 +463,11 @@ def _replay_ticker(
)
vol_6m = _realized_vol_6m(closes, len(window) - 1)
setups = (
_structural_overlay_window_setups(window, config, activation)
if sr_variant == STRUCTURAL_OVERLAY_VARIANT
else _window_setups(
window,
config,
activation,
sr_variant=sr_variant,
)
setups = _window_setups(
window,
config,
activation,
target_model=target_model,
)
for s in setups:
outcome, outcome_date = evaluate_setup_against_bars(
@@ -755,7 +517,7 @@ def _replay_ticker(
# every candidate looks NEUTRAL and the ablation rows collapse.
"action": s["action"],
"risk_level": s["risk_level"],
"sr_variant": s["sr_variant"],
"target_model": s["target_model"],
"primary_sources": s["primary_sources"],
"primary_strength": s["primary_strength"],
"primary_rejection_count": s["primary_rejection_count"],
@@ -763,15 +525,6 @@ def _replay_ticker(
"primary_distance_atr": s["primary_distance_atr"],
"raw_level_count": s["raw_level_count"],
"gate_level_count": s["gate_level_count"],
"range_504_log": s["range_504_log"],
"range_504_ratio": s["range_504_ratio"],
"range_factor_pass": s["range_factor_pass"],
"structural_overlay_pass": s.get("structural_overlay_pass"),
"structural_overlay_rr": s.get("structural_overlay_rr"),
"structural_overlay_sources": s.get("structural_overlay_sources"),
"structural_overlay_gate_level_count": s.get(
"structural_overlay_gate_level_count"
),
"outcome": outcome,
"target_hit": target_hit,
"realized_r": realized_r,
@@ -832,8 +585,8 @@ def _robustness_stats(net_rs: list[float]) -> dict:
}
def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
"""Compact evidence audit for the active local S/R research arm."""
def _target_model_diagnostics(candidates: list[dict], target_model: str) -> dict:
"""Compact target-source diagnostics for the selected supported model."""
source_counts: dict[str, int] = defaultdict(int)
round_only = 0
strengths: list[float] = []
@@ -841,9 +594,6 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
rejections: list[int] = []
raw_counts: list[int] = []
gate_counts: list[int] = []
range_logs: list[float] = []
overlay_rows = 0
overlay_pass = 0
for cand in candidates:
sources = list(cand.get("primary_sources") or [])
for source in sources:
@@ -855,16 +605,13 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
rejections.append(int(cand.get("primary_rejection_count", 0) or 0))
raw_counts.append(int(cand.get("raw_level_count", 0) or 0))
gate_counts.append(int(cand.get("gate_level_count", 0) or 0))
range_logs.append(float(cand.get("range_504_log", 0.0) or 0.0))
if cand.get("structural_overlay_pass") is not None:
overlay_rows += 1
overlay_pass += int(bool(cand["structural_overlay_pass"]))
def avg(values: list[float] | list[int]) -> float | None:
return round(sum(values) / len(values), 3) if values else None
return {
"variant": _sr_research_variant(),
"target_model": target_model,
"target_model_label": BACKTEST_TARGET_MODELS[target_model],
"candidate_count": len(candidates),
"primary_source_counts": dict(sorted(source_counts.items())),
"primary_round_only": round_only,
@@ -874,80 +621,9 @@ def _sr_variant_diagnostics(candidates: list[dict]) -> dict:
"avg_primary_rejection_count": avg(rejections),
"avg_raw_level_count": avg(raw_counts),
"avg_gate_level_count": avg(gate_counts),
"avg_range_504_log": avg(range_logs),
"range_factor_pass": sum(
1 for value in range_logs if value >= RANGE_FACTOR_MIN_LOG
),
"structural_overlay_rows": overlay_rows,
"structural_overlay_pass": overlay_pass,
"structural_overlay_weight": (
STRUCTURAL_OVERLAY_WEIGHT if overlay_rows else None
),
}
def _sr_candidate_audit(candidates: list[dict], min_percentile: float) -> list[dict] | None:
"""Candidate-level audit for paired S/R variant comparisons.
Limit the sidecar population to the long momentum slice that could reach
production qualification. This keeps reports reviewable while retaining
gate failures, additions, removals, and portfolio-relevant near misses.
"""
if not _sr_audit_enabled():
return None
rows: list[dict] = []
for cand in candidates:
percentile = cand.get(PRODUCTION_PERCENTILE_KEY)
if cand.get("direction") != "long" or percentile is None:
continue
if float(percentile) < min_percentile:
continue
rows.append({
"symbol": cand["symbol"],
"date": cand["date"],
"direction": cand["direction"],
"qualified": bool(cand.get("qualified")),
"meets_core": bool(cand.get("meets_core")),
"momentum_percentile": round(float(percentile), 6),
"strategy_rank": round(float(cand.get(RESIDUAL_HIGH_VOL_BLEND_KEY, 0.0) or 0.0), 6),
"production_rank": round(
float(cand.get(RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, 0.0) or 0.0),
6,
),
"structural_overlay_pass": cand.get("structural_overlay_pass"),
"structural_overlay_score": (
round(float(cand[STRUCTURAL_OVERLAY_SCORE_KEY]), 6)
if cand.get(STRUCTURAL_OVERLAY_SCORE_KEY) is not None else None
),
"structural_overlay_rr": (
round(float(cand["structural_overlay_rr"]), 6)
if cand.get("structural_overlay_rr") is not None else None
),
"structural_overlay_sources": list(
cand.get("structural_overlay_sources") or []
),
"structural_overlay_gate_level_count": int(
cand.get("structural_overlay_gate_level_count", 0) or 0
),
"rr": round(float(cand.get("rr", 0.0)), 6),
"primary_prob": round(float(cand.get("primary_prob", 0.0)), 6),
"primary_sources": list(cand.get("primary_sources") or []),
"primary_strength": round(float(cand.get("primary_strength", 0.0)), 3),
"primary_rejection_count": int(cand.get("primary_rejection_count", 0) or 0),
"primary_distance_atr": round(float(cand.get("primary_distance_atr", 0.0)), 6),
"raw_level_count": int(cand.get("raw_level_count", 0) or 0),
"gate_level_count": int(cand.get("gate_level_count", 0) or 0),
"range_504_log": round(float(cand.get("range_504_log", 0.0)), 6),
"range_504_ratio": round(float(cand.get("range_504_ratio", 1.0)), 6),
"range_factor_pass": bool(cand.get("range_factor_pass")),
"outcome": cand.get("outcome"),
"net_r": round(float(cand.get("realized_r", 0.0)) - _cost_r(cand), 6),
"hold30_r": round(float((cand.get("time_r") or {}).get(30, 0.0)), 6),
})
rows.sort(key=lambda row: (row["date"], row["symbol"], row["direction"]))
return rows
# The fixed take-profit and trailing-stop sweeps were retired 2026-07: swept
# TPs never found an interior optimum (momentum's edge lives in the right tail)
# and wide trails converged to the hold-to-horizon exit, so the time-exit sweep
@@ -1280,6 +956,7 @@ def _replay_and_signals(
config: dict,
activation: dict,
benchmark_closes: dict[date, float] | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> tuple[list[dict], dict]:
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can
run in a worker process. Takes primitive column arrays (cheap to pickle),
@@ -1292,7 +969,14 @@ def _replay_and_signals(
for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes)
]
return (
_replay_ticker(symbol, bars, config, activation, benchmark_closes),
_replay_ticker(
symbol,
bars,
config,
activation,
benchmark_closes,
target_model,
),
_signal_series(bars, benchmark_closes),
)
@@ -1464,23 +1148,6 @@ def _assign_residual_high_vol_blend(candidates: list[dict]) -> None:
)
def _assign_structural_overlay_score(candidates: list[dict]) -> None:
"""Conservative rank nudge for production setups confirmed by clean S/R."""
for cand in candidates:
if cand.get("structural_overlay_pass") is None:
cand[STRUCTURAL_OVERLAY_SCORE_KEY] = None
continue
production_rank = cand.get(RESIDUAL_HIGH_VOL_BLEND_80_20_KEY)
if production_rank is None:
cand[STRUCTURAL_OVERLAY_SCORE_KEY] = None
continue
structural_score = 100.0 if cand["structural_overlay_pass"] else 0.0
cand[STRUCTURAL_OVERLAY_SCORE_KEY] = (
float(production_rank) * (1.0 - STRUCTURAL_OVERLAY_WEIGHT)
+ structural_score * STRUCTURAL_OVERLAY_WEIGHT
)
def _momentum_qualifies(cand: dict, threshold: float) -> bool:
"""Whether a candidate clears the floors (meets_core) and the momentum gate.
Threshold 0 disables the momentum gate (floors only). The gate is long-only:
@@ -1641,17 +1308,9 @@ def _simulate_portfolio(
qualified_fn = _default_qualified
entries_by_ord: dict[int, list[dict]] = defaultdict(list)
configured_start, configured_end = _backtest_entry_bounds()
effective_start = start_date if start_date is not None else configured_start
start_ord = effective_start.toordinal() if effective_start is not None else None
if end_date is not None:
# Explicit simulator/holdout end dates are exclusive split boundaries.
end_ord = end_date.toordinal()
elif configured_end is not None:
# BACKTEST_ENTRY_END is documented and applied as an inclusive bound.
end_ord = (configured_end + timedelta(days=1)).toordinal()
else:
end_ord = None
start_ord = start_date.toordinal() if start_date is not None else None
# Explicit simulator/holdout end dates are exclusive split boundaries.
end_ord = end_date.toordinal() if end_date is not None else None
for c in candidates:
if not qualified_fn(c) or c.get("direction") != "long":
continue
@@ -2333,7 +1992,6 @@ PORTFOLIO_MONITOR_LOOKBACKS: tuple[dict, ...] = (
)
PRODUCTION_PORTFOLIO_STRATEGY = "residual80_highvol80_20_atr3"
STRUCTURAL_OVERLAY_PORTFOLIO_STRATEGY = "production_structural_overlay5_atr3"
PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
{
"strategy": "legacy_residual80_hold",
@@ -2368,22 +2026,7 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
def _portfolio_monitor_strategies() -> tuple[dict, ...]:
"""Add the frozen overlay only inside its explicit research arm."""
if _sr_research_variant() != STRUCTURAL_OVERLAY_VARIANT:
return PORTFOLIO_MONITOR_STRATEGIES
return PORTFOLIO_MONITOR_STRATEGIES + ({
"strategy": STRUCTURAL_OVERLAY_PORTFOLIO_STRATEGY,
"label": "Research: production gate + 5% clean-structure rank overlay",
"description": (
"Production-qualified universe and live exit, ranked by 95% current "
"80/20 strategy rank plus 5% clean structural/range confirmation."
),
"entry_variant": "residual80_highvol_blend80_20_fixed10",
"exit_policy": "atr_trail3",
"ranking_key": STRUCTURAL_OVERLAY_SCORE_KEY,
"use_live_config": True,
"is_production": False,
},)
return PORTFOLIO_MONITOR_STRATEGIES
def _entry_variant_config(variant: str) -> dict | None:
@@ -3132,8 +2775,11 @@ def _build_recommendation(report: dict) -> dict:
async def run_backtest(
db: AsyncSession,
progress_cb: Callable[[int, int, str], None] | None = None,
*,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> dict:
"""Replay every ticker and aggregate the Phase-1 reports for the current config."""
target_model = validate_backtest_target_model(target_model)
config = await get_recommendation_config(db)
activation = await get_activation_config(db)
@@ -3198,6 +2844,7 @@ async def run_backtest(
futures.append(loop.run_in_executor(
pool, _replay_and_signals, ticker.symbol, columns, config, activation,
benchmark_closes,
target_model,
))
for result in await asyncio.gather(*futures, return_exceptions=True):
if isinstance(result, Exception):
@@ -3219,6 +2866,7 @@ async def run_backtest(
_merge(await asyncio.to_thread(
_replay_and_signals, ticker.symbol, columns, config, activation,
benchmark_closes,
target_model,
))
except Exception:
logger.exception("Backtest replay failed for %s", ticker.symbol)
@@ -3235,7 +2883,6 @@ async def run_backtest(
_assign_activation_momentum_percentiles(candidates)
_assign_residual_low_vol_blend(candidates)
_assign_residual_high_vol_blend(candidates)
_assign_structural_overlay_score(candidates)
current_min_pct = float(activation.get("min_momentum_percentile", 80.0))
for c in candidates:
c["qualified"] = _momentum_qualifies(c, current_min_pct)
@@ -3334,26 +2981,9 @@ async def run_backtest(
"horizon_days": HORIZON,
"min_lookback": MIN_LOOKBACK,
"cost_per_side_pct": round(COST_PER_SIDE * 100, 3),
"sr_variant": _sr_research_variant(),
"range_factor_lookback": RANGE_FACTOR_LOOKBACK,
"range_factor_min_log": RANGE_FACTOR_MIN_LOG,
"range_factor_min_ratio": round(math.exp(RANGE_FACTOR_MIN_LOG), 4),
"structural_overlay_weight": (
STRUCTURAL_OVERLAY_WEIGHT
if _sr_research_variant() == STRUCTURAL_OVERLAY_VARIANT else None
),
"structural_overlay_source_variant": (
STRUCTURAL_OVERLAY_SOURCE_VARIANT
if _sr_research_variant() == STRUCTURAL_OVERLAY_VARIANT else None
),
"entry_start": (
_backtest_entry_bounds()[0].isoformat()
if _backtest_entry_bounds()[0] is not None else None
),
"entry_end": (
_backtest_entry_bounds()[1].isoformat()
if _backtest_entry_bounds()[1] is not None else None
),
"target_model": target_model,
"target_model_label": BACKTEST_TARGET_MODELS[target_model],
"is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL,
},
"activation": activation,
"overall_qualified": _bucket_stats(qualified),
@@ -3417,8 +3047,10 @@ async def run_backtest(
"portfolio_monitor": portfolio_monitor_report,
"holdout": holdout_report,
"min_rr_sweep": min_rr_sweep_report,
"sr_variant_diagnostics": _sr_variant_diagnostics(candidates),
"sr_candidate_audit": _sr_candidate_audit(candidates, current_min_pct),
"target_model_diagnostics": _target_model_diagnostics(
candidates,
target_model,
),
"signal_eval": _signal_evaluation(collected),
"signal_eval_note": (
"Cross-sectional rank-IC of price-only signals vs the forward "
@@ -3446,9 +3078,11 @@ async def run_backtest(
async def run_and_store(
db: AsyncSession,
progress_cb: Callable[[int, int, str], None] | None = None,
*,
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
) -> dict:
"""Run the backtest and cache the report in a SystemSetting. Job entrypoint."""
report = await run_backtest(db, progress_cb)
report = await run_backtest(db, progress_cb, target_model=target_model)
await update_setting(db, KEY_REPORT, json.dumps(report))
return report
app/services/sr_service.py
+23 -103
View File
@@ -358,55 +358,13 @@ def _extract_candidate_levels(
return candidates
def _legacy_volume_profile_nodes(
highs: list[float],
lows: list[float],
closes: list[float],
volumes: list[int],
num_bins: int = 20,
) -> list[float]:
"""Reproduce the deployed pre-rewrite HVN/LVN grid for a control arm.
This intentionally retains the old span-volume double counting. It exists
only so a local research report can prove that its control still reproduces
the frozen production baseline while the live detector is being rewritten.
"""
if len(closes) < 20:
raise ValidationError(
f"Volume Profile requires at least 20 bars, got {len(closes)}"
)
price_min = min(lows)
price_max = max(highs)
if price_max == price_min:
price_max = price_min + 1.0
bin_width = (price_max - price_min) / num_bins
bins: list[float] = [0.0] * num_bins
prices = [price_min + (i + 0.5) * bin_width for i in range(num_bins)]
for i in range(len(closes)):
for b in range(num_bins):
low_edge = price_min + b * bin_width
high_edge = low_edge + bin_width
if highs[i] >= low_edge and lows[i] <= high_edge:
bins[b] += volumes[i]
average = sum(bins) / num_bins
hvn = [round(prices[i], 4) for i in range(num_bins) if bins[i] > average]
lvn = [round(prices[i], 4) for i in range(num_bins) if bins[i] < average]
return hvn + lvn
def _legacy_range_grid_nodes(
def _gate_target_range_centers(
highs: list[float],
lows: list[float],
closes: list[float],
num_bins: int = 20,
) -> list[float]:
"""Return every deployed grid center without the irrelevant volume pass.
The legacy profile returns both its above-average (HVN) and below-average
(LVN) bins. Their union is therefore the complete range grid except for the
rare bin whose traffic equals the average exactly. This explicit helper
isolates that geometry from the misleading volume-profile label.
"""
"""Return the evenly spaced price proposals used by the production GTL."""
if len(closes) < 20:
raise ValidationError(
f"Range grid requires at least 20 bars, got {len(closes)}"
@@ -422,49 +380,38 @@ def _legacy_range_grid_nodes(
]
def detect_sr_levels_legacy(
def detect_gate_target_ladder(
highs: list[float],
lows: list[float],
closes: list[float],
volumes: list[int],
tolerance: float = DEFAULT_TOLERANCE,
*,
include_volume_profile: bool = True,
include_pivots: bool = True,
neutral_strength: bool = False,
explicit_range_grid: bool = False,
) -> list[dict]:
"""Deployed detector plus source-isolation controls for local research.
"""Build the scanner's internal, volume-free target proposal ladder.
The defaults remain the exact production control. Keyword-only switches
allow a research arm to remove one source or neutralize strength without
copying or subtly changing the legacy implementation.
This is intentionally not human-facing support/resistance. It builds the
production gate's broad 20-bin range grid, adds unfiltered pivots, scores
historical price traffic, and merges nearby proposals. The returned levels
are transient and must not be persisted as chart S/R.
"""
if not closes:
return []
candidates: list[tuple[float, str]] = []
if include_volume_profile:
try:
nodes = (
_legacy_range_grid_nodes(highs, lows, closes)
if explicit_range_grid
else _legacy_volume_profile_nodes(highs, lows, closes, volumes)
)
method = "range_grid" if explicit_range_grid else "volume_profile"
for price in nodes:
candidates.append((float(price), method))
except ValidationError:
pass
if include_pivots:
try:
pivots = compute_pivot_points(highs, lows, closes)
candidates.extend(
(float(price), "pivot_point")
for price in pivots.get("swing_highs", []) + pivots.get("swing_lows", [])
)
except ValidationError:
pass
try:
candidates.extend(
(float(price), "range_grid")
for price in _gate_target_range_centers(highs, lows, closes)
)
except ValidationError:
pass
try:
pivots = compute_pivot_points(highs, lows, closes)
candidates.extend(
(float(price), "pivot_point")
for price in pivots.get("swing_highs", []) + pivots.get("swing_lows", [])
)
except ValidationError:
pass
if not candidates:
return []
@@ -515,37 +462,10 @@ def detect_sr_levels_legacy(
)
_tag_levels(merged, closes[-1])
if neutral_strength:
for level in merged:
level["strength"] = 50
merged.sort(key=lambda row: row["strength"], reverse=True)
return merged
def detect_gate_target_ladder(
highs: list[float],
lows: list[float],
closes: list[float],
tolerance: float = DEFAULT_TOLERANCE,
) -> list[dict]:
"""Build the scanner's internal, volume-free target proposal ladder.
This is intentionally not human-facing support/resistance. It preserves
the production gate's broad 20-bin range grid, unfiltered pivots, touch
strength, and merge geometry without performing or claiming a volume
profile calculation. The returned levels are transient and must not be
persisted as chart S/R.
"""
return detect_sr_levels_legacy(
highs,
lows,
closes,
[0] * len(closes),
tolerance,
explicit_range_grid=True,
)
def _merge_levels(
levels: list[dict],
tolerance: float = DEFAULT_TOLERANCE,