Compare commits
72
Commits
3df36a9bfb
...
main
+4
-18
@@ -18,16 +18,9 @@ OPENAI_API_KEY=
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
OPENAI_SENTIMENT_BATCH_SIZE=5
|
||||
|
||||
# Fundamentals Provider — Financial Modeling Prep
|
||||
FMP_API_KEY=
|
||||
|
||||
# Fundamentals Provider — Finnhub (optional fallback)
|
||||
FINNHUB_API_KEY=
|
||||
|
||||
# Fundamentals Provider — Alpha Vantage (optional fallback)
|
||||
ALPHA_VANTAGE_API_KEY=
|
||||
|
||||
# Dolt bulk data — local clone of post-no-preference/earnings (workstream A).
|
||||
# Dolt bulk data — local clone of post-no-preference/earnings. Together with the
|
||||
# SEC EDGAR block below this is the ONLY fundamentals source; there is no
|
||||
# provider-API fallback.
|
||||
# DOLT_BINARY: path to the dolt CLI (set the full path in dev if it's not on PATH,
|
||||
# e.g. Windows: C:\Program Files\Dolt\bin\dolt.exe). DOLT_DATA_DIR holds the
|
||||
# clones; in PRODUCTION it MUST be outside the deploy tree (deploy is
|
||||
@@ -52,21 +45,14 @@ SEC_REQUEST_SPACING_SECONDS=0.2
|
||||
SEC_MAX_RETRIES=4
|
||||
SEC_REQUEST_TIMEOUT_SECONDS=30.0
|
||||
|
||||
# A5 read-only parity report archive. In production keep this outside the
|
||||
# rsync deployment tree, e.g. /var/lib/signal-platform/reports/fundamentals-parity.
|
||||
FUNDAMENTALS_PARITY_REPORT_DIR=reports/fundamentals-parity
|
||||
|
||||
# Regime Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||
# AI/Tech Risk Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||
# Optional: without it the volatility (V1) and credit (C1) pillars show as n/a.
|
||||
FRED_API_KEY=
|
||||
|
||||
# Scheduled Jobs
|
||||
DATA_COLLECTOR_FREQUENCY=daily
|
||||
SENTIMENT_POLL_INTERVAL_MINUTES=30
|
||||
FUNDAMENTAL_FETCH_FREQUENCY=daily
|
||||
RR_SCAN_FREQUENCY=daily
|
||||
FUNDAMENTAL_RATE_LIMIT_RETRIES=3
|
||||
FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS=15
|
||||
|
||||
# Scoring Defaults
|
||||
DEFAULT_WATCHLIST_AUTO_SIZE=10
|
||||
|
||||
@@ -38,7 +38,10 @@ jobs:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
- run: pip install ruff
|
||||
- run: ruff check app/
|
||||
# Whole repo, not just app/: tests/ and scripts/ drifted to 11 findings
|
||||
# while unchecked. Rules are pinned in pyproject.toml, so the unpinned
|
||||
# ruff above cannot change what this enforces.
|
||||
- run: ruff check .
|
||||
|
||||
test:
|
||||
needs: lint
|
||||
|
||||
@@ -54,3 +54,6 @@ reports/.cache/
|
||||
# Runtime A5 parity bundles are generated on the production server. Research
|
||||
# conclusions belong in docs/research, not as an ever-growing artifact archive.
|
||||
reports/fundamentals-parity/
|
||||
|
||||
# Calibration harness raw-pull cache (Alpaca/FRED); regenerable, not a record.
|
||||
.calib-cache/
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Investing-signal platform for US equities. It runs one strategy, and it is a boring one:
|
||||
|
||||
> **A long-only cross-sectional momentum book.** Buy the top quintile by beta-adjusted 12-1 month momentum, tilt toward higher volatility, hold at most 10 names, cut at 1.5× ATR, then trail at 3× ATR for up to 30 trading days. After an initial-stop exit, re-enter only after the gate has failed and subsequently qualified again.
|
||||
> **A long-only cross-sectional momentum book.** Buy the top quintile by beta-adjusted 12-1 month momentum, tilt toward higher volatility, hold at most 15 names, cut at 1.5× ATR, then trail at 3× ATR for up to 30 trading days. After an initial-stop exit, re-enter only after the gate has failed and subsequently qualified again.
|
||||
|
||||
**Philosophy:** don't predict price — rank it. The edge is *relative* strength across the universe, and the discipline is in the exit: cut losers fast, let winners run until the trail catches them.
|
||||
|
||||
@@ -31,7 +31,7 @@ flowchart TD
|
||||
Q -->|no| SKIP
|
||||
Q -->|yes| RANK["Rank by production score<br/>80% momentum %ile<br/>+ 20% volatility %ile"]
|
||||
|
||||
RANK --> BOOK{"Room in the book?<br/>max 10 positions"}
|
||||
RANK --> BOOK{"Room in the book?<br/>max 15 positions"}
|
||||
BOOK -->|no| WAIT["Wait for a slot"]
|
||||
BOOK -->|yes| OPEN["OPEN — size at 1% account risk"]
|
||||
|
||||
@@ -131,16 +131,17 @@ indicators.
|
||||
|
||||
**Morning** (~02:00 ET) — data and display only, **no** qualifying R:R scan:
|
||||
|
||||
1. **OHLCV** — latest daily bars (Alpaca); new tickers backfill ~5 years.
|
||||
1. **OHLCV** — latest daily bars (Alpaca) plus the SPY benchmark; new tickers backfill ~5 years. A symbol whose bars have been stale for 3 days is probed against SEC for a Form 25/25-NSE/15 and **retired** on a hit (history kept — see *Delisting*).
|
||||
2. **Sentiment** — stale names that matter (top-pick feeders, watchlist, open paper, discovery net). Display context only; the activation gate is price-only.
|
||||
3. **Market Regime** + **Regime Monitor** — breadth/trend and the v2 risk thermometer; feed no trades.
|
||||
4. **Telegram alerts** — change-driven (regime-quadrant etc.); quiet days stay quiet. Setup alerts still fire on the near-close pipeline after the scan.
|
||||
3. **Market Trend (SPY)** + **AI/Tech Risk Monitor** — the SPY trend guard and the v4 risk thermometer; feed no trades.
|
||||
4. **Telegram alerts** — change-driven (risk-quadrant etc.); quiet days stay quiet. Setup alerts still fire on the near-close pipeline after the scan.
|
||||
|
||||
**Near-close** (~15:30 ET Mon–Fri) — the only full-universe qualifying observation:
|
||||
|
||||
1. **OHLCV fetch** — refresh the in-progress day-t bar (same path as intraday).
|
||||
2. **R:R Scan** — Structural S/R, scores, Gate Target Ladder setups, residual 12‑1 + 80/20 rank. Advances post-stop gate-reset transitions; failed scans never count.
|
||||
3. **Telegram alerts** — chained immediately so manual MOC fills can still hit ~15:50/15:55.
|
||||
3. **Shadow book** — opt-in automated book; opens top-ranked qualified setups up to capacity at the same near-close prices. Only accepts a scan from this same pipeline run.
|
||||
4. **Telegram alerts** — chained immediately so manual MOC fills can still hit ~15:50/15:55.
|
||||
|
||||
**After close** (~16:45 ET Mon–Fri):
|
||||
|
||||
@@ -155,7 +156,39 @@ Hourly mid-session (Mon–Fri ~10:00–15:00 ET): only **OHLCV → Outcome Eval*
|
||||
|
||||
### Other jobs
|
||||
|
||||
Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs).
|
||||
Dolt earnings import (daily 02:30 ET) · SEC fundamentals import (daily 04:00 ET, also refreshes the fundamentals cache scoring reads) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs).
|
||||
|
||||
The SEC import defers a run rather than writing partial data when a filing's XBRL
|
||||
hasn't landed. Two bounds keep that from compounding: `MISSING_XBRL_RETRY_DAYS`
|
||||
caps how long *one* filing blocks promotion, and `PROMOTION_CEILING_DAYS` (7)
|
||||
caps how long the import as a whole can stay deferred — past the ceiling every
|
||||
unresolved filing is aged out in place so `promote()` queues it as a gap row,
|
||||
`source_max_date` advances, and the import self-heals. A `deferred_stale` alert
|
||||
inside that window is normal and clears on its own; check `source_max_date` in
|
||||
`data_import_runs` before diagnosing a wedge.
|
||||
|
||||
### Delisting, not deletion
|
||||
|
||||
Retiring a symbol used to mean `delete_ticker` or a pruning universe bootstrap,
|
||||
both of which cascade through OHLCV, setups and scores. That destroys exactly the
|
||||
history four research documents apologise for: today's tracked universe projected
|
||||
backward is survivorship-biased, and hard-deleting every delisted name is what
|
||||
causes it. Keeping the rows preserves the option to fix that later (it does not
|
||||
fix it — the replay still has to model a delisting as an exit event).
|
||||
|
||||
`tickers` therefore carries `delisted_on` / `delisted_reason` (migration 032);
|
||||
`NULL` means actively traded. The filter is **opt-in** via
|
||||
`ticker_service.active_only`, applied to the live path only — scanner, momentum
|
||||
ranking, scoring, breadth, fundamentals candidates, SEC universe, earnings import,
|
||||
ingestion. The registry and admin views deliberately keep delisted rows visible,
|
||||
and `run_backtest` keeps them on purpose. Detection runs off OHLCV staleness
|
||||
(not the SEC fundamentals import, which stalls for days on unrelated Company-Facts
|
||||
gaps) and retires only on a Form 25/25-NSE/15 hit, so a halt or a rename keeps the
|
||||
existing warning instead. `delisted_on` is the *effective* date — Rule 12d2-2
|
||||
makes a Form 25 removal take effect ten days after filing, so a symbol filed today
|
||||
keeps trading (and keeps qualifying) until that date. It is safe to automate
|
||||
because it is reversible: `clear_delisted` un-retires a false positive, where a
|
||||
delete had already taken the history.
|
||||
|
||||
### From score to "top pick"
|
||||
|
||||
@@ -166,6 +199,33 @@ Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe s
|
||||
|
||||
**What the R:R and reach-probability in step 3 actually are.** They are *gate inputs*, computed from a Gate Target Ladder proposal the trade will never exit at — they exist to filter setups, not to forecast the trade you're about to take. A setup with "R:R 2.4:1, 34% reach probability" is not a claim that you'll make 2.4R with 34% probability; it's a claim that this setup cleared the screen. What actually happens to a trade is in the exit box of the diagram above, and on the "what usually happens" panel in the UI. Conflating the two is the single easiest way to misread this app.
|
||||
|
||||
### Two books: shadow (automated) and discretionary (manual)
|
||||
|
||||
The platform keeps **two** paper books, and the difference between them is the
|
||||
whole point.
|
||||
|
||||
| Book | Who selects | What it measures |
|
||||
|---|---|---|
|
||||
| **Shadow book** (`app/services/shadow_book_service.py`) | The machine — top-ranked qualified setups up to capacity, every near-close scan | The **strategy**, faithfully |
|
||||
| **Discretionary book** | You, by clicking "paper trade" on a setup | The strategy **plus** your discretion and availability |
|
||||
|
||||
The manual book only ever contains trades the user chose to take, inside a ~20
|
||||
minute window, on days they were around. The backtest that validated this
|
||||
strategy does none of that, which makes the manual record unusable on its own as
|
||||
out-of-sample evidence. The shadow book closes that gap: it mirrors
|
||||
`_simulate_portfolio`'s selection rule exactly, orders on the *stored*
|
||||
`strategy_rank` the scanner already wrote (so the two cannot drift apart) and
|
||||
shares the manual book's exit policy — the only difference between the books is
|
||||
*which* qualified setups get taken.
|
||||
|
||||
It runs as a step of the near-close pipeline, straight after the scan so entries
|
||||
mark at the same near-close prices, and it only accepts a scan from the same
|
||||
pipeline run. It is **opt-in** (`shadow_book_enabled`, with capacity, risk % and
|
||||
starting equity under **Admin → Settings → Performance & Shadow Book**) because it
|
||||
writes live trades. The **Dashboard**'s performance chart plots shadow vs
|
||||
discretionary vs SPY; *Signals → Paper Trades* still shows the discretionary book
|
||||
only.
|
||||
|
||||
## Strategy Status — What's Validated and What Isn't
|
||||
|
||||
**Read this before touching scoring, gating, or setup logic.** The platform measures itself — a weekly-replay backtest plus a factor rank-IC harness (`app/services/backtest_service.py`) — and the verdicts below come from those reports (latest run July 2026, ~5 years of OHLCV), not from opinion.
|
||||
@@ -176,7 +236,8 @@ Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe s
|
||||
|---|---|---|
|
||||
| **Residual 12-1 cross-sectional momentum** (the activation gate, long-only) | **Production gate — in-sample edge** | Promoted July 2026 after the portfolio variant beat raw 80 on CAGR, Sharpe and drawdown. Raw 12-1 remains a fallback only when benchmark data is unavailable |
|
||||
| **3× ATR trailing exit** (+ 1.5× ATR initial stop, 30-day max hold) | **Production exit — best Sharpe of every exit tested** | Beat hold / SMA50 / 20-day-low / technical-40 and both take-profit variants (July 2026) |
|
||||
| **Post-stop gate reset** | **Production re-entry policy** | The initial stop always closes; the ticker must later fail the daily gate and subsequently qualify again. At the production capacity of 10: Sharpe 1.67 → 1.77, CAGR 45.2% → 48.3%, DD 24.3% → 21.6% versus immediate re-entry. [Full study](docs/research/post-stop-reentry.md) |
|
||||
| **Post-stop gate reset** | **Production re-entry policy** | The initial stop always closes; the ticker must later fail the daily gate and subsequently qualify again. At the then-production capacity of 10: Sharpe 1.67 → 1.77, CAGR 45.2% → 48.3%, DD 24.3% → 21.6% versus immediate re-entry. Capacity has since been raised to 15 — see the open question under the re-entry section. [Full study](docs/research/post-stop-reentry.md) |
|
||||
| **Book capacity 15** (raised from 10, 2026-08-05) | **Production sizing** | The focused daily capacity bracket found the count cap was binding and cost real compounding: +1.075pp CAGR paired, 51 paths better / 2 worse, drawdown unchanged. Cash plus the 20% notional cap saturates the book near 12, so the cap no longer binds. [Findings](docs/research/portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||
| **Structural S/R** | **Human-facing context only — not a gate and not an exit** | Clean, capped zones are persisted for charts and alerts. The scanner deliberately does not read them. |
|
||||
| **Gate Target Ladder** | **Gate input only — not market structure and not an exit** | Volume-free range grid + pivots preserves the useful legacy screening behavior exactly: 1,086/1,086 qualified setups retained and identical Sharpe 2.03 / CAGR 50.0% / DD 21.4% / 321 trades. The exit never reads its target. [Full write-up](docs/research/sr-levels-and-exits.md#explicit-gate-target-ladder) |
|
||||
| Composite score + 5 dimensions | **Display/ranking only** | Sub-scores are hand-built heuristics; none has a measured IC. Note: the "momentum" *dimension* is 5/20-day ROC — NOT the validated 12-1 factor (that lives in `momentum_service`) |
|
||||
@@ -187,7 +248,7 @@ Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe s
|
||||
| Gate target as a take-profit (tested July 2026) | **Rejected** | Sharpe 2.04 → 1.47, CAGR halved. Win rate *rose* — it truncates the right tail where the edge lives |
|
||||
| "Clear-air" gate relaxation (tested July 2026) | **Rejected — failed out-of-sample** | Strictly better in-sample (Sharpe 2.07 / CAGR 62.3% / DD 20.1%), then lost on a real train/test split (Sharpe 2.78 → 2.45). A cautionary tale: nested lookbacks are not OOS |
|
||||
|
||||
Caveats on the momentum result: in-sample, roughly one market regime, costs/slippage approximated at 0.1% per side, and residual momentum still needs SPY benchmark history to compute. The **out-of-sample proof is the forward paper-trade record**: Signals → Track Record compares live qualified expectancy against the backtest.
|
||||
Caveats on the momentum result: in-sample, roughly one market regime, costs/slippage approximated at 0.1% per side, and residual momentum still needs SPY benchmark history to compute. The **out-of-sample proof is the forward record of the shadow book** — the automated twin that takes every top-ranked qualified setup, with no discretion or availability mixed in. The Dashboard chart tracks it against the discretionary book and SPY; *Signals → Backtest* is what it is being compared against.
|
||||
|
||||
### Daily post-stop re-entry decision (2026-07-17)
|
||||
|
||||
@@ -200,7 +261,9 @@ The production policy is **normal gate reset**, evaluated with daily setup oppor
|
||||
| Strict gate reset (live timing analogue) | 342.7% | 44.8% | -23.4% | 1.68 | 471 |
|
||||
| Fixed five-session cooldown | 250.8% | 36.6% | -22.2% | 1.47 | 473 |
|
||||
|
||||
In the disjoint 2025+ book, gate reset also beat immediate re-entry (Sharpe 1.66 vs 1.55; CAGR 41.8% vs 39.3%) and the fixed five-session rule (Sharpe 1.43; CAGR 32.7%). Its lead over both survived costs of 0.2% and 0.3% per side. The result is capacity-specific: cooldown 5 won at capacity 5, while immediate had slightly higher return and Sharpe at capacity 15. Production uses capacity 10, so that is the portfolio for which this decision is valid.
|
||||
In the disjoint 2025+ book, gate reset also beat immediate re-entry (Sharpe 1.66 vs 1.55; CAGR 41.8% vs 39.3%) and the fixed five-session rule (Sharpe 1.43; CAGR 32.7%). Its lead over both survived costs of 0.2% and 0.3% per side. The result is capacity-specific: cooldown 5 won at capacity 5, while immediate had slightly higher return and Sharpe at capacity 15.
|
||||
|
||||
> **Open question (since 2026-08-05).** This study was run — and gate reset promoted — at capacity 10. Production capacity was subsequently raised to 15, which is the one capacity in the matrix where *immediate* re-entry edged ahead. The re-entry policy is therefore currently running outside the portfolio it was validated on. Nothing else changed, and the two arms differed only modestly, but the matrix should be rerun at capacity 15 before treating gate reset as settled. Until then, keep gate reset (the incumbent) rather than switching on an untested read.
|
||||
|
||||
Those promotion numbers belong to the selected normal-reset study arm. Under the **pre-cutover** morning-scan scheduler (scan always before any outcome eval), live first-observation timing matched the stricter `strict_gate_reset` analogue (full-period Sharpe 1.68 / CAGR 44.8% / DD 23.4%). After the **near-close cutover** (2026-07), stops closed by earlier same-day intraday evals can receive a same-day fail observation at ~15:30 ET — moving live behavior **toward** the promoted `gate_reset` arm. Requalification still requires a later America/New_York trading date than the failure (`trade_policy` distinct-day guard). Full definitions and all nine policy arms: [docs/research/post-stop-reentry.md](docs/research/post-stop-reentry.md); execution evidence: [docs/research/execution-recovery.md](docs/research/execution-recovery.md).
|
||||
|
||||
@@ -208,7 +271,7 @@ Those promotion numbers belong to the selected normal-reset study arm. Under the
|
||||
|
||||
### Historical weekly production baseline (pre gate-reset)
|
||||
|
||||
Use this as the historical ranking/exit regression guardrail, not as a return promise or the current re-entry-policy result. This run predates the post-stop gate reset and uses weekly entry replay, so its portfolio headline is not directly comparable with the daily matrix above. Backtest run: local production SQLite snapshot, 506 tickers, weekly cadence, 30-trading-day horizon, 2022-06-28 → 2026-07-02, 0.1% per-side costs, price-only SPY benchmark. Numbers below are the 2026-07-11 run (`reports/backtest-20260711-prod-baseline.json`) — measured *after* the primary-target probability floor shipped, which pruned lottery-target setups (1,428 → 1,089 qualified) and lifted Sharpe on all three promotion contenders.
|
||||
Use this as the historical ranking/exit regression guardrail, not as a return promise or the current re-entry-policy result. This run predates the post-stop gate reset **and the 2026-08-05 capacity raise to 15**, and uses weekly entry replay, so its portfolio headline is not directly comparable with the daily matrix above. Backtest run: local production SQLite snapshot, 506 tickers, weekly cadence, 30-trading-day horizon, 2022-06-28 → 2026-07-02, 0.1% per-side costs, price-only SPY benchmark. Numbers below are the 2026-07-11 run (`reports/backtest-20260711-prod-baseline.json`) — measured *after* the primary-target probability floor shipped, which pruned lottery-target setups (1,428 → 1,089 qualified) and lifted Sharpe on all three promotion contenders.
|
||||
|
||||
| Item | Historical weekly baseline |
|
||||
|---|---|
|
||||
@@ -248,16 +311,16 @@ Parity guard (July 2026): the portfolio monitor's **Production** row replays the
|
||||
|
||||
### Tuned and confirmed — do not retest without new data (July 2026)
|
||||
|
||||
A systematic single-variable sweep (offline prod snapshot, production gate/rank/exit, 2022-06 → 2026-07 plus disjoint 2022–23 / 2024–26 folds) confirmed **every** production setting. Retesting these against the same ~4-year snapshot is wasted compute and invites overfitting; revisit only with meaningfully new data (longer history or broader universe).
|
||||
A systematic single-variable sweep (offline prod snapshot, production gate/rank/exit, 2022-06 → 2026-07 plus disjoint 2022–23 / 2024–26 folds) confirmed every production setting **except book size**, which a later focused bracket reversed (see the row below). Retesting these against the same ~4-year snapshot is wasted compute and invites overfitting; revisit only with meaningfully new data (longer history or broader universe) — or, as with capacity, a demonstrably better measurement lens.
|
||||
|
||||
| Knob tested | Verdict | Evidence |
|
||||
|---|---|---|
|
||||
| ATR trail multiple {1.5–4.0} | **Keep 3.0** | Return+Sharpe peak; ≤2.0 whipsaws out the momentum right tail; ≥2.5 is a plateau |
|
||||
| SPY 200d-MA regime overlay (block entries / go flat) | **Reject** | Halves return (315%→138%) with zero drawdown benefit — the ATR trail already manages downside, and the filter blocks the recovery-phase entries that make the money |
|
||||
| Momentum lookback: 6-1, 3-1, 12-7 (Novy-Marx), composites | **Keep residual 12-1** | 6-1/3-1 rank-IC ≈ 0; 12-7 IC 0.045 / t 1.58 — weaker than residual 12-1 (0.055 / t 1.98) |
|
||||
| Selection cutoff {70, 75, 85, 90} × book size {10, 15, 20} | **Keep 80 × 10** | Monotonically worse in both directions from 80; the 10-slot cap never binds (<10 concurrent) |
|
||||
| Selection cutoff {70, 75, 85, 90} × book size {10, 15, 20} | **Keep cutoff 80; book size raised to 15 (2026-08-05)** | The cutoff is monotonically worse in both directions from 80. The book-size half of this row was **reversed**: the weekly replay's "the 10-slot cap never binds" read came from EV per trade, which is the wrong lens for anything that changes trade *count*. The focused daily bracket found cap 10 *was* binding and cost +1.075pp CAGR; at 15 the cap never bound in any cell (max observed 12 concurrent, zero full-book skips) |
|
||||
| Position sizing: equal-weight, inverse-vol, risk-% sweep | **Keep 1% fixed-fractional** | See the inverse-vol warning below |
|
||||
| Post-stop re-entry: immediate, fixed 2–5 sessions, gate resets, confirmation filters | **Keep normal gate reset for the 10-position production book** | Sharpe 1.77 vs 1.67 immediate and 1.47 cooldown 5; rerun before changing portfolio capacity |
|
||||
| Post-stop re-entry: immediate, fixed 2–5 sessions, gate resets, confirmation filters | **Keep normal gate reset** — but measured at capacity 10, and capacity is now 15 | Sharpe 1.77 vs 1.67 immediate and 1.47 cooldown 5. The "rerun before changing portfolio capacity" caveat is now outstanding — see the open question above |
|
||||
| FIP path-smoothness as an in-book tie-breaker/filter | **Reject** (but see the lead below) | Non-monotonic across FIP quintiles within the qualified set; either half of a median split underperforms the full book — thinning the entry stream costs more compounding than the tilt returns |
|
||||
|
||||
Two findings future sessions must not re-litigate:
|
||||
@@ -270,23 +333,24 @@ Two findings future sessions must not re-litigate:
|
||||
A signal earns its way into selection **only** through the factor harness:
|
||||
|
||||
1. Add it as a point-in-time function of past bars in `_signal_values()` (`backtest_service.py`).
|
||||
2. Run the backtest (Admin → Jobs, or the weekly run) and read the **Signal edge** table (Signals → Track Record).
|
||||
2. Run the backtest (Admin → Jobs, or the weekly run) and read the report's `signal_eval` section. This one is **local-report only** — the deployed Backtest tab does not render it (see *Reading a local backtest report* below).
|
||||
3. Wire it into the gate or ranking **only if** |mean IC| ≳ 0.03 with a consistent sign and `reliable: true` (≥ 12 non-overlapping windows).
|
||||
|
||||
Corollaries: never let an unvalidated score gate setups; the outcome evaluator must keep scoring **all** setups (unqualified ones are the control group); LLM output stays display-only in the quant path.
|
||||
|
||||
### Highest-value next experiments (in order)
|
||||
|
||||
> Check **[docs/research/](docs/research/README.md)** first — 12 strategy ideas have already been tested and rejected, including the obvious ones (take-profit exits, regime overlays, inverse-vol sizing, shorts).
|
||||
> Check **[docs/research/](docs/research/README.md)** first — 13 strategy ideas have already been tested and rejected, including the obvious ones (take-profit exits, regime overlays, inverse-vol sizing, shorts, sector-residual momentum).
|
||||
|
||||
1. **Forward monitor the promoted strategy** — the production UI now behaves like a portfolio monitor for the current strategy, with selectable lookbacks and SPY comparison. Forward paper-trade months are the only evidence the snapshot cannot provide; the July 2026 tuning pass closed every in-sample lead. (Trailing-stop sensitivity and the max-15 capacity check are done — see the tuning table above.)
|
||||
1. **Forward monitor the promoted strategy** — *Signals → Backtest* behaves like a portfolio monitor for the current strategy, with selectable lookbacks and SPY comparison, and the Dashboard chart carries the forward record. Forward months of the **shadow book** are the only evidence the snapshot cannot provide; the July 2026 tuning pass closed every in-sample lead. (Trailing-stop sensitivity and the capacity bracket are done — capacity was raised to 15.)
|
||||
2. **Signal context snapshots** — accumulate point-in-time composite/sentiment/fundamental context for every new setup so the discretionary overlay can be tested forward-only.
|
||||
3. **Breadth is no longer free leverage** — Phase B found residual-mom t-stat *fell* on liquid-1500 vs the 505-name fingerprint (0.055/1.98 → 0.029/1.33). Any breadth book must clear a pre-registered baseline arm before fip tilts mean anything. (Deeper history was considered and declined.)
|
||||
|
||||
## Key Use Cases
|
||||
|
||||
- **Find today's best long setup.** On the **Dashboard**, the *Top Setups* table lists residual-gated qualified setups ranked by the production 80/20 residual/high-vol score, with the #1 flagged "Top pick". Each row opens the ticker page for its chart, Structural S/R, Gate Target Ladder targets and entry/stop.
|
||||
- **Track a trade you took.** Mark a setup as a **paper trade**: it's marked-to-market against the latest close, auto-closed by the active exit policy (default: 3x ATR trail with a 30-trading-day max hold), and its sentiment stays fresh while open. *Signals → Track Record* shows the realized edge.
|
||||
- **Track a trade you took.** Mark a setup as a **paper trade**: it's marked-to-market against the latest close, auto-closed by the active exit policy (default: 3x ATR trail with a 30-trading-day max hold), and its sentiment stays fresh while open. *Signals → Paper Trades* shows the realized edge of your discretionary book; the Dashboard chart puts it next to the automated shadow book and SPY.
|
||||
- **Ask whether the strategy is worth trading at all.** *Signals → Backtest* replays the promoted strategy over history — portfolio monitor vs SPY over selectable lookbacks, headline risk-adjusted metrics (Sharpe, Sortino, Gain-to-Pain, dollar profit factor) and the report's own recommendation — with the live-outcome evaluation panel underneath it.
|
||||
|
||||
## Stack
|
||||
|
||||
@@ -301,13 +365,13 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m
|
||||
| Charts | Canvas 2D candlestick chart with S/R overlays |
|
||||
| Routing | React Router v6 (SPA) |
|
||||
| HTTP | Axios with JWT interceptor |
|
||||
| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); Fundamentals chain: FMP → Finnhub → Alpha Vantage; FRED (regime); Telegram (alerts) |
|
||||
| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); SEC EDGAR Company Facts + DoltHub earnings (fundamentals, bulk import); FRED (regime); Telegram (alerts) |
|
||||
|
||||
## Features
|
||||
|
||||
### Backend
|
||||
- Ticker registry with full cascade delete
|
||||
- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint
|
||||
- Ticker registry with reversible delisting (history preserved) plus an explicit cascade delete
|
||||
- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint — free public sources (Wikipedia / NASDAQ Trader), then the cached snapshot, then a built-in seed list. The seeds are representative, not complete, so a *fresh* install bootstrapped while the public source is unreachable gets a partial universe; a warm instance falls through to its cache.
|
||||
- OHLCV price storage with upsert and validation
|
||||
- Technical indicators: ADX, EMA, RSI, ATR, Volume Profile, Pivot Points, EMA Cross
|
||||
- Structural Support/Resistance detection with rejection/recency strength, ATR-adaptive merging and a hard cap; persisted for charts and alerts
|
||||
@@ -319,7 +383,9 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m
|
||||
- Activation gate — qualifies setups on a residual-momentum percentile floor (the actual selection), a headline gate-target R:R floor (prod: 2.0) and a 20% primary-target reach-probability floor (validated long-only edge)
|
||||
- Recommendation layer — directional confidence, conflict detection, per-target reach-probability
|
||||
- Paper trading — take a setup, mark-to-market vs. latest close, auto-close per the exit policy (default: 3x ATR trail with a 30-trading-day max hold; time / percent-trailing / target-stop selectable), realized track record + outcome evaluation
|
||||
- Market-regime guard + observational State/Warning monitor (fixed-basket breadth, VIX, credit, PIT fundamentals) with a manual chronological correction study
|
||||
- Shadow book — opt-in automated twin of the backtest's selection rule (top-ranked qualified setups up to capacity, every near-close scan), sharing the manual book's exit policy; the honest forward out-of-sample record
|
||||
- System events — structured job/import/data warnings with acknowledgement, surfaced in Admin and deduplicated for alerting
|
||||
- Market-regime guard + observational State/Warning monitor (fixed-basket breadth, VIX, credit level + impulse) with a manual chronological correction study
|
||||
- Telegram alerts (e.g. regime-quadrant changes)
|
||||
- User-curated watchlist (cap: 20), enriched with composite score, R:R and S/R summary
|
||||
- JWT auth with admin role, configurable registration, user access control
|
||||
@@ -337,7 +403,10 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m
|
||||
- Ticker detail page: chart, scores, sentiment breakdown, fundamentals, technical indicators, S/R table
|
||||
- Rankings table with configurable dimension weights
|
||||
- Trade scanner showing detected R:R setups
|
||||
- Admin page: user management, job status with live indicators, enable/disable toggles, data cleanup, system settings
|
||||
- Backtest tab: portfolio monitor vs SPY over selectable lookbacks, headline risk-adjusted tiles (Sharpe, Sortino, Gain-to-Pain, dollar profit factor), the report's recommendation card, and a live-outcome evaluation panel
|
||||
- Dashboard performance chart: cumulative shadow book vs discretionary book vs SPY since the configured start date
|
||||
- Paper Trades tab: open/closed discretionary trades with realized R and P&L tiles
|
||||
- Admin page: user management, job status with live indicators, enable/disable toggles, pipeline readiness, system-event log, ticker management, data cleanup, system settings
|
||||
- Protected routes with JWT auth, admin-only sections
|
||||
- Responsive layout with mobile navigation
|
||||
- Toast notifications for async operations
|
||||
@@ -348,14 +417,14 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m
|
||||
|---|---|---|
|
||||
| `/login` | Login | Public |
|
||||
| `/register` | Register | Public (when enabled) |
|
||||
| `/` | Dashboard — top setups, open trades, regime (default) | Authenticated |
|
||||
| `/` | Dashboard — top setups, open trades, regime, shadow-vs-manual-vs-SPY performance chart (default) | Authenticated |
|
||||
| `/market` | Market — watchlist + rankings tabs | Authenticated |
|
||||
| `/signals` | Signals — scanner + track record tabs | Authenticated |
|
||||
| `/regime` | Market Regime | Authenticated |
|
||||
| `/signals` | Signals — Setups / Paper Trades / Backtest tabs | Authenticated |
|
||||
| `/regime` | AI/Tech Risk Monitor | Authenticated |
|
||||
| `/ticker/:symbol` | Ticker Detail | Authenticated |
|
||||
| `/admin` | Admin Panel | Admin only |
|
||||
|
||||
Legacy routes redirect: `/watchlist` → `/market`, `/rankings` → `/market?tab=rankings`, `/scanner` → `/signals`, `/performance` → `/signals?tab=track`.
|
||||
Legacy routes redirect: `/watchlist` → `/market`, `/rankings` → `/market?tab=rankings`, `/scanner` → `/signals`, `/performance` → `/signals?tab=track` (the Paper Trades tab — `track` stays its slug so the old link keeps working).
|
||||
|
||||
## API Endpoints
|
||||
|
||||
@@ -365,7 +434,7 @@ All under `/api/v1/`. Interactive docs at `/docs` (Swagger) and `/redoc`.
|
||||
|---|---|
|
||||
| Health | `GET /health` |
|
||||
| Auth | `POST /auth/register`, `POST /auth/login` |
|
||||
| Tickers | `POST /tickers`, `GET /tickers`, `DELETE /tickers/{symbol}` |
|
||||
| Tickers | `POST /tickers`, `GET /tickers`, `DELETE /tickers/{symbol}`, `POST /tickers/{symbol}/delisting`, `DELETE /tickers/{symbol}/delisting` |
|
||||
| OHLCV | `POST /ohlcv`, `GET /ohlcv/{symbol}` |
|
||||
| Ingestion | `POST /ingestion/fetch/{symbol}` |
|
||||
| Indicators | `GET /indicators/{symbol}/{type}`, `GET /indicators/{symbol}/ema-cross` |
|
||||
@@ -375,11 +444,11 @@ All under `/api/v1/`. Interactive docs at `/docs` (Swagger) and `/redoc`.
|
||||
| Fundamentals | `GET /fundamentals/{symbol}` |
|
||||
| Scores | `GET /scores/{symbol}`, `GET /rankings`, `PUT /scores/weights` |
|
||||
| Trades | `GET /trades`, `GET /trades/{symbol}`, `GET /trades/{symbol}/history`, `GET /trades/activation`, `GET /trades/performance` |
|
||||
| Paper Trades | `GET /paper-trades`, `POST /paper-trades`, `POST /paper-trades/{id}/close` |
|
||||
| Market / Regime | `GET /market/regime`, `GET /regime/monitor`, `GET/PUT /regime/config`, `GET /regime/history`, `GET /regime/event-study`, `GET/PUT /regime/fundamentals`, `GET /backtest/report` |
|
||||
| Paper Trades | `GET /paper-trades`, `POST /paper-trades`, `POST /paper-trades/{id}/close`, `GET /paper-trades/equity-curve`, `GET /paper-trades/performance` (shadow vs manual vs SPY), `GET/PUT /paper-trades/exit-policy` |
|
||||
| Market / Regime | `GET /market/regime`, `GET /regime/monitor`, `GET/PUT /regime/config`, `GET /regime/history`, `GET /regime/event-study`, `GET/PUT /regime/fundamentals`, `POST /regime/fundamentals/refresh`, `GET /backtest/report` |
|
||||
| Jobs | `GET /jobs/running` |
|
||||
| Watchlist | `GET /watchlist`, `POST /watchlist/{symbol}`, `DELETE /watchlist/{symbol}` |
|
||||
| Admin | `GET /admin/users`, `POST /admin/users`, `PUT /admin/users/{id}/access`, `PUT /admin/users/{id}/password`, `PUT /admin/settings/registration`, `GET /admin/settings`, `PUT /admin/settings/{key}`, `GET/PUT /admin/settings/recommendations`, `GET/PUT /admin/settings/ticker-universe`, `POST /admin/tickers/bootstrap`, `POST /admin/data/cleanup`, `GET /admin/jobs`, `POST /admin/jobs/{name}/trigger`, `PUT /admin/jobs/{name}/toggle`, `GET /admin/pipeline/readiness` |
|
||||
| Admin | `GET /admin/users`, `POST /admin/users`, `PUT /admin/users/{id}/access`, `PUT /admin/users/{id}/password`, `PUT /admin/settings/registration`, `GET /admin/settings`, `PUT /admin/settings/{key}`, `GET/PUT /admin/settings/{recommendations,activation,schedule,performance,shadow-book,sentiment,alerts,ticker-universe}`, `POST /admin/settings/{sentiment,alerts}/test`, `POST /admin/tickers/bootstrap`, `POST /admin/tickers/backfill-names`, `POST /admin/data/cleanup`, `POST /admin/track-record/reset`, `GET /admin/jobs`, `POST /admin/jobs/{name}/trigger`, `PUT /admin/jobs/{name}/toggle`, `GET /admin/pipeline/readiness`, `GET /admin/system-events`, `GET /admin/system-events/summary`, `POST /admin/system-events/acknowledge` |
|
||||
|
||||
## Development Setup
|
||||
|
||||
@@ -439,8 +508,8 @@ npm run preview # Preview the production build locally
|
||||
# Backend tests (in-memory SQLite — no PostgreSQL needed)
|
||||
pytest tests/ -v
|
||||
|
||||
# Frontend: there is no test suite — `npm test` calls vitest, which is not
|
||||
# installed. The frontend check is the full TypeScript build:
|
||||
# Frontend: there is no test suite and no `test` script at all. The frontend
|
||||
# check is the full TypeScript build:
|
||||
cd frontend
|
||||
npm run build
|
||||
```
|
||||
@@ -524,10 +593,10 @@ the [full research record](docs/research/sr-levels-and-exits.md#gtl-tuning-matri
|
||||
|
||||
### Reading a local backtest report
|
||||
|
||||
The deployed **Signals → Track Record** page is deliberately trimmed to validation
|
||||
(portfolio monitor vs SPY, realized paper trades) and how-to-trade. The
|
||||
strategy-tuning tables that used to live there now live **only** in the local
|
||||
report — inspect these `reports/backtest-<timestamp>.json` sections and produce the
|
||||
The deployed **Signals → Backtest** tab is deliberately trimmed to validation
|
||||
(portfolio monitor vs SPY, headline metrics, the report's recommendation, and the
|
||||
live-outcome evaluation panel). The strategy-tuning tables that used to live there
|
||||
now live **only** in the local report — inspect these `reports/backtest-<timestamp>.json` sections and produce the
|
||||
matching decision. Every change still goes through the factor harness first (see
|
||||
**The iron rule for strategy changes** above).
|
||||
|
||||
@@ -564,8 +633,11 @@ Research-only flags, all off by default (the default report is byte-identical to
|
||||
| `BACKTEST_ATR_TARGET_FALLBACK=k` | Synthesizes a k×ATR target where S/R offers none |
|
||||
| `BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1` | Restricts that fallback to setups with genuinely no structure ahead |
|
||||
|
||||
`recommendation` is the one section surfaced on the deployed page ("What this
|
||||
backtest recommends"); everything else in this table is intentionally local-only.
|
||||
`portfolio_monitor` and `recommendation` are the sections surfaced on the deployed
|
||||
Backtest tab (the monitor chart/tiles and "What this backtest recommends"; the
|
||||
recommendation is rebuilt on read, so it always matches the lookback on screen and
|
||||
flags one it was not computed on). Everything else in this table is intentionally
|
||||
local-only.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
@@ -583,20 +655,25 @@ Configure in `.env` (copy from `.env.example`):
|
||||
| `OPENAI_API_KEY` | For sentiment (OpenAI path) | — | OpenAI API key |
|
||||
| `OPENAI_MODEL` | No | `gpt-4o-mini` | OpenAI model name |
|
||||
| `OPENAI_SENTIMENT_BATCH_SIZE` | No | `5` | Micro-batch size for sentiment collector |
|
||||
| `FMP_API_KEY` | Optional (fundamentals) | — | Financial Modeling Prep API key (first provider in chain) |
|
||||
| `FINNHUB_API_KEY` | Optional (fundamentals) | — | Finnhub API key (fallback provider) |
|
||||
| `ALPHA_VANTAGE_API_KEY` | Optional (fundamentals) | — | Alpha Vantage API key (fallback provider) |
|
||||
| `FRED_API_KEY` | Optional (regime) | — | FRED key for the regime monitor (VIX, credit spreads) |
|
||||
| `DEEPSEEK_API_KEY` / `XAI_API_KEY` | For sentiment (those paths) | — | Alternative pluggable sentiment providers |
|
||||
| `SEC_USER_AGENT` | **For fundamentals** | placeholder | SEC EDGAR requires a real `name (contact: email)` UA — the shipped default is a placeholder and SEC will throttle/refuse it |
|
||||
| `SEC_REQUEST_SPACING_SECONDS` | No | `0.2` | Politeness delay between SEC requests |
|
||||
| `SEC_MAX_RETRIES` / `SEC_REQUEST_TIMEOUT_SECONDS` | No | `4` / `30` | SEC client retry and timeout budget |
|
||||
| `DOLT_BINARY` | For earnings import | `dolt` | Path to the `dolt` executable |
|
||||
| `DOLT_DATA_DIR` / `DOLT_EARNINGS_SUBDIR` | No | `dolt-data` / `earnings` | Local Dolt clone location |
|
||||
| `DOLT_MIN_FREE_DISK_GB` | No | `5.0` | Refuse to clone/pull below this free space |
|
||||
| `DOLT_COMMAND_TIMEOUT_SECONDS` | No | `600` | Per-command Dolt timeout |
|
||||
| `FRED_API_KEY` | Optional (risk monitor) | — | FRED key for the AI/Tech risk monitor (VIX, credit spreads) |
|
||||
| `TELEGRAM_BOT_TOKEN` | Optional (alerts) | — | Telegram bot token for alerts (can also be set in Admin) |
|
||||
| `TELEGRAM_CHAT_ID` | Optional (alerts) | — | Telegram chat id for alerts |
|
||||
| `DATA_COLLECTOR_FREQUENCY` | No | `daily` | OHLCV collection schedule (legacy — see note below) |
|
||||
| `SENTIMENT_POLL_INTERVAL_MINUTES` | No | `30` | Sentiment polling interval |
|
||||
| `FUNDAMENTAL_FETCH_FREQUENCY` | No | `weekly` | Fundamentals fetch cadence |
|
||||
| `RR_SCAN_FREQUENCY` | No | `daily` | R:R scanner schedule |
|
||||
| `FUNDAMENTAL_RATE_LIMIT_RETRIES` | No | `3` | Retries per ticker on fundamentals rate-limit |
|
||||
| `FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS` | No | `15` | Base backoff seconds for fundamentals retry (exponential) |
|
||||
| `DEFAULT_WATCHLIST_AUTO_SIZE` | No | `10` | Auto-watchlist size |
|
||||
| `DEFAULT_RR_THRESHOLD` | No | `1.5` | Minimum R:R ratio for setups |
|
||||
| `OHLCV_HISTORY_DAYS` | No | `1825` | Backfill depth for new tickers (~5 years) |
|
||||
| `OUTCOME_EVALUATION_MAX_BARS` | No | `30` | Bars the outcome evaluator resolves a setup over |
|
||||
| `BACKTEST_WORKERS` | No | `4` | Worker processes for the scheduled backtest |
|
||||
| `DB_POOL_SIZE` | No | `5` | Database connection pool size |
|
||||
| `LOG_LEVEL` | No | `INFO` | Logging level |
|
||||
|
||||
@@ -689,7 +766,9 @@ app/
|
||||
├── exceptions.py # Exception hierarchy
|
||||
├── middleware.py # Global error handler → JSON envelope
|
||||
├── cache.py # LRU cache with per-ticker invalidation
|
||||
├── ssl_bootstrap.py # TLS trust-store bootstrap for outbound calls
|
||||
├── scheduler.py # APScheduler job definitions
|
||||
├── job_catalog.py # Single source of truth for job names + pipeline step lists
|
||||
├── models/ # SQLAlchemy ORM models
|
||||
├── schemas/ # Pydantic request/response schemas
|
||||
├── services/ # Business logic layer
|
||||
@@ -709,9 +788,11 @@ frontend/
|
||||
│ ├── admin/ # User table, job controls, settings, data cleanup
|
||||
│ ├── auth/ # Protected route wrapper
|
||||
│ ├── charts/ # Canvas candlestick chart
|
||||
│ ├── dashboard/ # Top setups, open trades, shadow-vs-manual performance chart
|
||||
│ ├── layout/ # App shell, sidebar, mobile nav
|
||||
│ ├── rankings/ # Rankings table, weights form
|
||||
│ ├── scanner/ # Trade table
|
||||
│ ├── signals/ # Setups / Paper Trades / Backtest panels
|
||||
│ ├── ticker/ # Sentiment panel, fundamentals, indicators, S/R overlay
|
||||
│ ├── ui/ # Badge, toast, skeleton, score card, confirm dialog
|
||||
│ └── watchlist/ # Watchlist table, add ticker form
|
||||
@@ -722,16 +803,26 @@ frontend/
|
||||
└── styles/ # Global CSS with glassmorphism classes
|
||||
|
||||
docs/
|
||||
├── dolt-integration-plan.md # Design record for the Dolt/SEC fundamentals workstream
|
||||
├── dolt-sec-a3-design.md
|
||||
├── fundamentals-deployment.md
|
||||
└── research/ # Experiment log: what was tested, the result, the decision
|
||||
├── README.md # Overview — start here before proposing a strategy change
|
||||
└── sr-levels-and-exits.md
|
||||
├── sr-levels-and-exits.md
|
||||
├── post-stop-reentry.md
|
||||
├── portfolio-capacity-bracket*.md
|
||||
├── execution-recovery.md
|
||||
├── fip-breadth-ic.md
|
||||
├── regime-monitor-v3.md / -v4.md
|
||||
└── … # 16 documents total
|
||||
|
||||
reports/ # Committed backtest reports (JSON) + compare_reports.py
|
||||
|
||||
deploy/
|
||||
├── nginx.conf # Reverse proxy + static file serving
|
||||
├── setup_db.sh # Idempotent DB setup script
|
||||
└── stock-data-backend.service # systemd unit
|
||||
├── provision_fundamentals.sh # Server-side Dolt/SEC fundamentals provisioning
|
||||
└── signalplatform.service # systemd unit
|
||||
|
||||
tests/
|
||||
├── conftest.py # Fixtures, strategies, test DB
|
||||
@@ -749,9 +840,11 @@ Context for whoever — human or AI — continues this work. The owner pushes st
|
||||
- **Live scan and backtest share the same pure functions.** The backtest replays production logic through DB-free functions (`compute_technical_from_arrays`, `compute_momentum_from_closes`, `detect_sr_levels`, `detect_gate_target_ladder`, the recommendation helpers). New strategy logic must stay in pure functions consumed by both paths, or the backtest stops measuring what production actually does.
|
||||
- **Keep the two price-level models separate.** `detect_sr_levels` produces persisted Structural S/R for charts and alerts. `detect_gate_target_ladder` produces transient screening proposals and must never be persisted or presented as market structure. The scanner must not read `SRLevel` rows for target generation.
|
||||
- **The Gate Target Ladder target is a gate input, never an exit.** `_atr_trailing_close()` does not take it as a parameter, and it must stay that way — take-profit exits were tested and halve CAGR. Any UI or alert that implies the trade exits at the target is a bug ([research](docs/research/sr-levels-and-exits.md#explicit-gate-target-ladder)).
|
||||
- **The outcome evaluator evaluates ALL setups**, not just qualified ones — unqualified setups are the control group that makes the Track Record meaningful.
|
||||
- **The outcome evaluator evaluates ALL setups**, not just qualified ones — unqualified setups are the control group that makes the realized-outcome record meaningful.
|
||||
- **`SystemSetting` access goes through `app/services/settings_store.py`** — don't query the model directly.
|
||||
- **Time-series data gets a real table** (see `benchmark_prices`, `regime_snapshots`); `SystemSetting` JSON is only for config and cached reports.
|
||||
- **The shadow book must stay parity-clean.** It orders on the *stored* `strategy_rank` the scanner wrote and mirrors `_simulate_portfolio`'s selection rule; it accepts only a scan from its own pipeline run. Recomputing its ranking, or letting it consume a stale/manual scan, turns the forward OOS record back into an approximation.
|
||||
- **Delisted tickers are retired, never deleted.** Live paths opt into `ticker_service.active_only`; the registry, admin views and `run_backtest` deliberately still see them. Deleting a symbol takes the history that a survivorship-bias fix would need.
|
||||
- **Discretionary overlay data is forward-only.** `signal_context_snapshots` captures composite/dimension/sentiment/fundamental context for new setups. Do not approximate historical sentiment/fundamental snapshots from today's data.
|
||||
- Style: surgical changes, minimal new files; extend existing services rather than adding parallel ones.
|
||||
|
||||
@@ -768,11 +861,14 @@ Context for whoever — human or AI — continues this work. The owner pushes st
|
||||
| Backtest + factor rank-IC harness ("Signal edge") | `app/services/backtest_service.py` |
|
||||
| Outcome resolution (target/stop/expired/ambiguous) | `app/services/outcome_service.py` |
|
||||
| Paper trades + time/trailing/target auto-exit | `app/services/paper_trade_service.py` |
|
||||
| Shadow book (automated twin of the backtest's selection) | `app/services/shadow_book_service.py` |
|
||||
| Re-entry locks / distinct-day guard / book identities | `app/services/trade_policy.py` |
|
||||
| Ticker registry, delisting + `active_only` filter | `app/services/ticker_service.py` |
|
||||
| Point-in-time setup context snapshots | `app/models/signal_context_snapshot.py` + `app/services/rr_scanner_service.py` |
|
||||
| Structural S/R detection, Gate Target Ladder & zone clustering | `app/services/sr_service.py` |
|
||||
| **Research log — what's been tested and rejected** | **`docs/research/`** |
|
||||
| SPY benchmark for residual momentum + paper-trade alpha | `app/services/benchmark_service.py` |
|
||||
| Pipelines & job registration | `app/scheduler.py` |
|
||||
| Pipelines & job registration | `app/scheduler.py` (step lists and job names in `app/job_catalog.py`) |
|
||||
|
||||
### Verifying changes
|
||||
|
||||
@@ -781,7 +877,7 @@ pytest tests/ -q # backend; in-memory SQLite, no Postgres needed
|
||||
cd frontend && npm run build # full tsc check — this IS the frontend "test"
|
||||
```
|
||||
|
||||
- `npm test` in `frontend/` is dead (vitest isn't installed; there are no frontend test files). Use `npm run build`.
|
||||
- There is no `npm test` in `frontend/` — no test script, no test files. `npm run build` (`tsc -b && vite build`) is the frontend check.
|
||||
- Backend tests that exercise services which `commit()` need a plain session fixture, not the rolling-back `db_session` — copy the pattern in `tests/unit/test_rr_scanner_integration.py`.
|
||||
- `ruff` reports ~11 pre-existing errors in old test files; those are not regressions.
|
||||
|
||||
@@ -798,6 +894,6 @@ Practical consequences:
|
||||
|
||||
### Roadmap (agreed June 2026)
|
||||
|
||||
1. **Forward paper-test the momentum book** — the out-of-sample proof the backtest can't give. Watch Signals → Track Record (live vs backtest).
|
||||
1. **Forward paper-test the momentum book** — the out-of-sample proof the backtest can't give. Watch the Dashboard chart (shadow book vs discretionary vs SPY) against Signals → Backtest.
|
||||
2. **Full IBKR integration** — read real positions, overlay entries/stops on charts, alert on holdings' score deterioration. (Paper trading, the lighter alternative, is done.)
|
||||
3. Strategy experiments in the order listed under **Strategy Status** above — each one goes through the factor harness first.
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""SEC filing retry queue and setup-quality gate
|
||||
|
||||
Revision ID: 028
|
||||
Revises: 027
|
||||
Create Date: 2026-08-03 00:00:00.000000
|
||||
"""
|
||||
from datetime import date, datetime, timezone
|
||||
import json
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "028"
|
||||
down_revision: Union[str, None] = "027"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"sec_filing_gaps",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("cik", sa.String(length=10), nullable=False),
|
||||
sa.Column("accession", sa.String(length=25), nullable=False),
|
||||
sa.Column("form", sa.String(length=12), nullable=True),
|
||||
sa.Column("index_date", sa.Date(), nullable=True),
|
||||
sa.Column("reason", sa.String(length=64), nullable=False),
|
||||
sa.Column("coregistrant_ciks_json", sa.Text(), nullable=True),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_attempted_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("escalated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.UniqueConstraint("accession", name="uq_sec_filing_gaps_accession"),
|
||||
)
|
||||
op.create_index("ix_sec_filing_gaps_cik", "sec_filing_gaps", ["cik"])
|
||||
_backfill_retry_queue()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_sec_filing_gaps_cik", table_name="sec_filing_gaps")
|
||||
op.drop_table("sec_filing_gaps")
|
||||
|
||||
|
||||
def _as_date(value) -> date | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _backfill_retry_queue() -> None:
|
||||
"""Materialize pre-queue promoted gaps once; runtime never scans history."""
|
||||
bind = op.get_bind()
|
||||
runs = sa.table(
|
||||
"data_import_runs",
|
||||
sa.column("source", sa.String()),
|
||||
sa.column("status", sa.String()),
|
||||
sa.column("validation_json", sa.Text()),
|
||||
sa.column("source_max_date", sa.Date()),
|
||||
sa.column("started_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
snapshots = sa.table(
|
||||
"fundamental_snapshots",
|
||||
sa.column("cik", sa.String()),
|
||||
sa.column("accession", sa.String()),
|
||||
sa.column("filed_date", sa.Date()),
|
||||
)
|
||||
gaps = sa.table(
|
||||
"sec_filing_gaps",
|
||||
sa.column("cik", sa.String()),
|
||||
sa.column("accession", sa.String()),
|
||||
sa.column("form", sa.String()),
|
||||
sa.column("index_date", sa.Date()),
|
||||
sa.column("reason", sa.String()),
|
||||
sa.column("coregistrant_ciks_json", sa.Text()),
|
||||
sa.column("first_seen_at", sa.DateTime(timezone=True)),
|
||||
sa.column("last_attempted_at", sa.DateTime(timezone=True)),
|
||||
sa.column("escalated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
snapshot_rows = bind.execute(
|
||||
sa.select(snapshots.c.cik, snapshots.c.accession, snapshots.c.filed_date)
|
||||
).all()
|
||||
resolved_accessions = {row.accession for row in snapshot_rows}
|
||||
latest_filed_by_cik: dict[str, date] = {}
|
||||
for row in snapshot_rows:
|
||||
if row.filed_date is not None:
|
||||
current = latest_filed_by_cik.get(row.cik)
|
||||
if current is None or row.filed_date > current:
|
||||
latest_filed_by_cik[row.cik] = row.filed_date
|
||||
|
||||
audit_rows = bind.execute(
|
||||
sa.select(
|
||||
runs.c.validation_json,
|
||||
runs.c.source_max_date,
|
||||
runs.c.started_at,
|
||||
).where(
|
||||
runs.c.source == "sec_facts",
|
||||
runs.c.status == "promoted",
|
||||
runs.c.validation_json.is_not(None),
|
||||
)
|
||||
).all()
|
||||
now = datetime.now(timezone.utc)
|
||||
candidates: dict[str, dict] = {}
|
||||
for audit in audit_rows:
|
||||
try:
|
||||
summary = json.loads(audit.validation_json)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not isinstance(summary, dict):
|
||||
continue
|
||||
for item in summary.get("missing_xbrl") or []:
|
||||
accession = item.get("accession")
|
||||
raw_cik = item.get("cik")
|
||||
if not accession or raw_cik is None or accession in resolved_accessions:
|
||||
continue
|
||||
cik = str(raw_cik).zfill(10)
|
||||
index_date = _as_date(item.get("index_date")) or _as_date(
|
||||
audit.source_max_date
|
||||
)
|
||||
later_filed = latest_filed_by_cik.get(cik)
|
||||
if index_date is not None and later_filed is not None and later_filed > index_date:
|
||||
continue
|
||||
first_seen = audit.started_at or now
|
||||
existing = candidates.get(accession)
|
||||
if existing is not None and existing["first_seen_at"] <= first_seen:
|
||||
continue
|
||||
candidates[accession] = {
|
||||
"cik": cik,
|
||||
"accession": accession,
|
||||
"form": item.get("form"),
|
||||
"index_date": index_date,
|
||||
"reason": item.get("reason") or "not_in_companyfacts",
|
||||
"coregistrant_ciks_json": json.dumps(item.get("coregistrants") or []),
|
||||
"first_seen_at": first_seen,
|
||||
"last_attempted_at": first_seen,
|
||||
"escalated_at": None,
|
||||
}
|
||||
if candidates:
|
||||
op.bulk_insert(gaps, list(candidates.values()))
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Retire the legacy fundamentals settings (A6)
|
||||
|
||||
Revision ID: 029
|
||||
Revises: 028
|
||||
Create Date: 2026-08-07 00:00:00.000000
|
||||
|
||||
A6 removed the FMP/Finnhub/Alpha Vantage providers, the weekly
|
||||
``fundamental_collector`` job and the A5 parity report. Five SystemSetting rows
|
||||
are left over. They are NOT all deleted, because the deploy runs migrations
|
||||
before restarting the service: for a short window — and for the whole of any
|
||||
rollback — pre-A6 code is still live, and it reads absent rows permissively
|
||||
(cutover absent -> disabled; ``job_<name>_enabled`` absent -> enabled). Deleting
|
||||
both would hand a rolled-back process a re-armed legacy collector writing over
|
||||
the SEC/Dolt cache.
|
||||
|
||||
So the two rows that carry behavior become tombstones pinned to the safe value,
|
||||
and only the inert ones are deleted. The tombstones are dropped in a later
|
||||
release once the rollback window has closed; ``SettingsForm`` hides them
|
||||
meanwhile.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "029"
|
||||
down_revision: Union[str, None] = "028"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
# Behavior-bearing under pre-A6 code -> pin to the safe value, keep the row.
|
||||
_TOMBSTONES: dict[str, str] = {
|
||||
"fundamental_data_sec_dolt_cutover_enabled": "true",
|
||||
"job_fundamental_collector_enabled": "false",
|
||||
}
|
||||
|
||||
# Inert either way: an absent cron falls back to a default for a job that no
|
||||
# longer registers, and the parity report never wrote anything.
|
||||
_OBSOLETE: tuple[str, ...] = (
|
||||
"schedule_fundamentals_cron",
|
||||
"schedule_fundamentals_parity_cron",
|
||||
"job_fundamentals_parity_report_enabled",
|
||||
)
|
||||
|
||||
_settings = sa.table(
|
||||
"system_settings",
|
||||
sa.column("id", sa.Integer),
|
||||
sa.column("key", sa.String),
|
||||
sa.column("value", sa.Text),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
now = sa.func.now()
|
||||
|
||||
for key, pinned in _TOMBSTONES.items():
|
||||
row = conn.execute(
|
||||
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||
).fetchone()
|
||||
old_value = row[0] if row is not None else None
|
||||
print(f"a6_tombstone {key}: {old_value!r} -> {pinned!r}", flush=True)
|
||||
if row is None:
|
||||
conn.execute(
|
||||
sa.insert(_settings).values(key=key, value=pinned, updated_at=now)
|
||||
)
|
||||
elif old_value != pinned:
|
||||
conn.execute(
|
||||
sa.update(_settings)
|
||||
.where(_settings.c.key == key)
|
||||
.values(value=pinned, updated_at=now)
|
||||
)
|
||||
|
||||
# Print the value before deleting — a bare DELETE cannot be undone from the
|
||||
# migration output.
|
||||
for key in _OBSOLETE:
|
||||
row = conn.execute(
|
||||
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
print(f"a6_delete {key}: absent", flush=True)
|
||||
continue
|
||||
print(f"a6_delete {key}: {row[0]!r}", flush=True)
|
||||
conn.execute(sa.delete(_settings).where(_settings.c.key == key))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op.
|
||||
|
||||
The deleted rows configured jobs this revision's code no longer registers,
|
||||
and the tombstones already hold the values pre-A6 code needs. Recreating
|
||||
them would restore nothing useful; the printed values above cover recovery.
|
||||
"""
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Drop the A6 rollback tombstones
|
||||
|
||||
Revision ID: 030
|
||||
Revises: 029
|
||||
Create Date: 2026-08-07 00:00:00.000000
|
||||
|
||||
Migration ``029`` kept two SystemSetting rows alive as rollback tombstones,
|
||||
pinned to the values a pre-A6 process needed to behave safely. A6 is deployed
|
||||
and healthy, and the provider keys are gone from the production ``.env`` — which
|
||||
makes the legacy collector inert regardless of any settings row — so the
|
||||
tombstones have no remaining job.
|
||||
|
||||
Nothing in the current codebase reads either key.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "030"
|
||||
down_revision: Union[str, None] = "029"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
# The safe values 029 pinned. Kept here so downgrade restores real protection
|
||||
# rather than leaving a rolled-back process reading absent rows permissively.
|
||||
_TOMBSTONES: dict[str, str] = {
|
||||
"fundamental_data_sec_dolt_cutover_enabled": "true",
|
||||
"job_fundamental_collector_enabled": "false",
|
||||
}
|
||||
|
||||
_settings = sa.table(
|
||||
"system_settings",
|
||||
sa.column("id", sa.Integer),
|
||||
sa.column("key", sa.String),
|
||||
sa.column("value", sa.Text),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for key in _TOMBSTONES:
|
||||
row = conn.execute(
|
||||
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
print(f"a6_tombstone_drop {key}: absent", flush=True)
|
||||
continue
|
||||
print(f"a6_tombstone_drop {key}: {row[0]!r}", flush=True)
|
||||
conn.execute(sa.delete(_settings).where(_settings.c.key == key))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Restore the tombstones at their safe values.
|
||||
|
||||
Unlike 029's no-op downgrade, this one is meaningful: going back past this
|
||||
revision implies going back toward code that still reads these keys.
|
||||
"""
|
||||
conn = op.get_bind()
|
||||
now = sa.func.now()
|
||||
for key, pinned in _TOMBSTONES.items():
|
||||
exists = conn.execute(
|
||||
sa.select(_settings.c.id).where(_settings.c.key == key)
|
||||
).fetchone()
|
||||
if exists is None:
|
||||
conn.execute(
|
||||
sa.insert(_settings).values(key=key, value=pinned, updated_at=now)
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Durable last-run state per scheduled job
|
||||
|
||||
Revision ID: 031
|
||||
Revises: 030
|
||||
Create Date: 2026-08-08 00:00:00.000000
|
||||
|
||||
Job run state lived only in an in-memory dict in ``app.scheduler``, so every
|
||||
process restart wiped it. Admin → Jobs could then only report "Active" with no
|
||||
indication of whether a job had ever run, or how it ended — which is exactly
|
||||
the information an operator opens that page for.
|
||||
|
||||
One row per job, upserted on ``job_name``. Not history: ``system_events``
|
||||
already grows unbounded with no retention job, and a second append-only
|
||||
operational table would repeat that debt.
|
||||
|
||||
The table starts empty; each job populates its row the next time it finishes.
|
||||
No backfill from ``system_events`` — that table only records warning/error
|
||||
outcomes and uses a different status vocabulary, so seeding from it would
|
||||
invent successful runs that never happened.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "031"
|
||||
down_revision: Union[str, None] = "030"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"job_run_state",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("job_name", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("processed", sa.Integer(), nullable=True),
|
||||
sa.Column("total", sa.Integer(), nullable=True),
|
||||
sa.Column("message", sa.Text(), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("job_name", name="uq_job_run_state_job_name"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("job_run_state")
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Record delisting on tickers instead of deleting them
|
||||
|
||||
Revision ID: 032
|
||||
Revises: 031
|
||||
Create Date: 2026-08-11 00:00:00.000000
|
||||
|
||||
Until now the only way to retire a symbol was ``delete_ticker`` (or
|
||||
``bootstrap_universe(prune_missing=True)``), both of which cascade through
|
||||
OHLCV, setups and scores. That destroys exactly the history four research
|
||||
documents already apologise for: today's tracked universe projected backward
|
||||
is survivorship-biased, and hard-deleting every delisted name is what causes
|
||||
it. Keeping the rows preserves the option to fix that later — it does not fix
|
||||
it by itself, which needs the replay to model a delisting as an exit event.
|
||||
|
||||
``delisted_on`` is the effective date (from SEC Form 25/25-NSE/15 where we can
|
||||
confirm it, else the day it was marked); ``delisted_reason`` is a short code
|
||||
for how we learned. NULL in both means actively traded — the live signal path
|
||||
filters on that, while list and admin views keep showing the row so the
|
||||
delisting is visible rather than silently absent.
|
||||
|
||||
Nullable and reversible by design: clearing ``delisted_on`` un-retires a
|
||||
symbol, which is what makes automatic marking safe where a delete would not be.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "032"
|
||||
down_revision: Union[str, None] = "031"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("tickers", sa.Column("delisted_on", sa.Date(), nullable=True))
|
||||
op.add_column(
|
||||
"tickers", sa.Column("delisted_reason", sa.String(length=32), nullable=True)
|
||||
)
|
||||
# The live path filters "actively traded" on every universe scan; the index
|
||||
# keeps that predicate cheap as delisted rows accumulate.
|
||||
op.create_index("ix_tickers_delisted_on", "tickers", ["delisted_on"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_tickers_delisted_on", table_name="tickers")
|
||||
op.drop_column("tickers", "delisted_reason")
|
||||
op.drop_column("tickers", "delisted_on")
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Point-in-time history for the sourced fundamental observation
|
||||
|
||||
Revision ID: 033
|
||||
Revises: 032
|
||||
Create Date: 2026-08-12 00:00:00.000000
|
||||
|
||||
The hyperscaler capex / "good news, stock down" read lived in a single
|
||||
``SystemSetting`` slot, so each refresh overwrote the last and no history
|
||||
existed. The read is now a categorical channel reported alongside State and
|
||||
Warning (never a term in either), and a channel with no history cannot be
|
||||
replayed: a snapshot rebuild would record every historical session as if nothing
|
||||
had ever been observed, and the event study could not measure the channel at all.
|
||||
|
||||
Keyed on ``effective_date`` (the session the observation becomes usable on,
|
||||
normally the next weekday) rather than ``fetched_at``, because that is the gate
|
||||
that stops a rebuild stamping today's reading onto historical rows.
|
||||
|
||||
The table starts empty. ``update_regime_monitor`` records the currently stored
|
||||
observation on its next run, so a deployment does not lose the live reading —
|
||||
but genuine history does not exist and cannot be invented here. Backfilling it
|
||||
from the SEC capex line and earnings-date reactions is separate work; until then
|
||||
every historical session reads ``unknown``, which is the honest value rather than
|
||||
a guessed one.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "033"
|
||||
down_revision: Union[str, None] = "032"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"regime_fundamental_observations",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("effective_date", sa.Date(), nullable=False),
|
||||
sa.Column("f1_score", sa.Float(), nullable=True),
|
||||
sa.Column("f3_score", sa.Float(), nullable=True),
|
||||
sa.Column("capex_json", sa.Text(), nullable=False),
|
||||
sa.Column("good_news_stock_down", sa.String(length=10), nullable=False),
|
||||
sa.Column("reasoning", sa.Text(), nullable=True),
|
||||
sa.Column("source", sa.String(length=30), nullable=False),
|
||||
sa.Column("fetched_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
# One unique index, not a unique constraint plus a plain index: the model
|
||||
# declares `unique=True, index=True`, which SQLAlchemy renders as exactly
|
||||
# this. The constraint-plus-index pairing worked but left a redundant second
|
||||
# index on the column and a permanent metadata diff for autogenerate to keep
|
||||
# trying to reconcile. Matches RegimeSnapshot.date, the sibling table.
|
||||
op.create_index(
|
||||
"ix_regime_fundamental_observations_effective_date",
|
||||
"regime_fundamental_observations",
|
||||
["effective_date"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_regime_fundamental_observations_effective_date",
|
||||
table_name="regime_fundamental_observations",
|
||||
)
|
||||
op.drop_table("regime_fundamental_observations")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Track when a filing gap stops pausing setups
|
||||
|
||||
Revision ID: 034
|
||||
Revises: 033
|
||||
Create Date: 2026-08-21 00:00:00.000000
|
||||
|
||||
An escalated gap stops pausing setups while the issuer's own fundamentals are
|
||||
still recent (``GAP_GATE_RECENT_FILING_DAYS``). That reprieve is not permanent:
|
||||
the stored filings age out, or a newer gap appears, and the pause returns —
|
||||
silently, because ``filing_gap_aged`` only escalates gaps whose ``escalated_at``
|
||||
is NULL and so never fires twice for the same gap.
|
||||
|
||||
``exempted_at`` is the state marker that makes the transition observable. It is
|
||||
set (quietly) while the issuer is exempt and cleared when the exemption lapses,
|
||||
which is when ``filing_gap_repaused`` fires — once per lapse, re-arming if the
|
||||
issuer's data recovers and ages out again.
|
||||
|
||||
Nullable, and carrying no meaning of its own beyond that state: an existing gap
|
||||
starts NULL and is stamped on the next import that finds it exempt.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "034"
|
||||
down_revision: Union[str, None] = "033"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"sec_filing_gaps",
|
||||
sa.Column("exempted_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("sec_filing_gaps", "exempted_at")
|
||||
+1
-21
@@ -28,15 +28,6 @@ class Settings(BaseSettings):
|
||||
deepseek_api_key: str = ""
|
||||
xai_api_key: str = ""
|
||||
|
||||
# Fundamentals Provider — Financial Modeling Prep
|
||||
fmp_api_key: str = ""
|
||||
|
||||
# Fundamentals Provider — Finnhub (optional fallback)
|
||||
finnhub_api_key: str = ""
|
||||
|
||||
# Fundamentals Provider — Alpha Vantage (optional fallback)
|
||||
alpha_vantage_api_key: str = ""
|
||||
|
||||
# Dolt bulk-data — local clone of post-no-preference/earnings (workstream A).
|
||||
# dolt_binary: full path when not on PATH (dev/Windows install). dolt_data_dir
|
||||
# holds the clones; in production it MUST be outside the deploy tree (deploy is
|
||||
@@ -61,11 +52,7 @@ class Settings(BaseSettings):
|
||||
sec_max_retries: int = 4
|
||||
sec_request_timeout_seconds: float = 30.0
|
||||
|
||||
# A5 read-only comparison artifacts. Production must keep this outside the
|
||||
# rsync deployment tree so the 5-7 day review window survives deploys.
|
||||
fundamentals_parity_report_dir: str = "reports/fundamentals-parity"
|
||||
|
||||
# Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
||||
# AI/Tech Risk Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
||||
# the volatility (P5) and credit-spread (F2) signals are reported as n/a.
|
||||
fred_api_key: str = ""
|
||||
|
||||
@@ -86,15 +73,8 @@ class Settings(BaseSettings):
|
||||
# the score window is 7 days).
|
||||
sentiment_fresh_hours: int = 120
|
||||
sentiment_top_composite: int = 30
|
||||
fundamental_fetch_frequency: str = "weekly" # quarterly-ish data; weekly conserves API quota
|
||||
rr_scan_frequency: str = "daily" # legacy label; qualifying scan is cron near-close
|
||||
# alerts_frequency removed: alerts fire only via morning + near-close pipelines
|
||||
fundamental_rate_limit_retries: int = 3
|
||||
fundamental_rate_limit_backoff_seconds: int = 15
|
||||
# Pause between tickers in the bulk fundamentals job. Free tiers throttle
|
||||
# hard (Finnhub ~60 calls/min, ~3 calls/ticker → ~3s/ticker); without
|
||||
# spacing the job bursts straight into 429s. 0 disables.
|
||||
fundamental_request_spacing_seconds: float = 3.0
|
||||
|
||||
# Scoring Defaults
|
||||
default_watchlist_auto_size: int = 10
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Job topology: names, labels, pipeline membership, categories, ordering.
|
||||
|
||||
The single source of truth for *what the jobs are*, as opposed to how they run.
|
||||
It deliberately imports nothing from ``app`` so both ``app.scheduler`` and
|
||||
``app.services.admin_service`` can import it at module level -- admin_service
|
||||
otherwise has to do ``from app.scheduler import ...`` inside functions to dodge a
|
||||
cycle.
|
||||
|
||||
The pipeline step lists live here rather than in the scheduler because three
|
||||
separate things need them and used to keep private copies: the runner, the
|
||||
``PIPELINE_MEMBERS`` set the admin API reports, and the UI's grouping. Steps are
|
||||
``(step_name, coroutine_name)``; ``_run_pipeline`` resolves the coroutine late
|
||||
out of the scheduler's own globals, so nothing here depends on those functions
|
||||
existing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipelines
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DAILY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("benchmark_collector", "collect_benchmark"),
|
||||
("sentiment_collector", "collect_sentiment"),
|
||||
("market_regime", "compute_market_regime"),
|
||||
# Observational only — display/alerts; not trade selection.
|
||||
("regime_monitor", "compute_regime_monitor"),
|
||||
# Alerts after regime so quadrant changes reach Telegram in the morning.
|
||||
# Dispatcher is change-driven; quiet days stay quiet. Setup alerts still
|
||||
# fire on the near-close pipeline after the qualifying scan.
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# Near-close (~15:30 ET Mon–Fri): refresh in-progress day-t bars (incremental
|
||||
# ingestion overlaps the latest stored session), then the only daily
|
||||
# qualifying R:R scan, then Telegram immediately so manual fills can still hit
|
||||
# MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan
|
||||
# may see ~15:15 prices — immaterial for a 12-1 momentum signal.
|
||||
#
|
||||
# US early-close days (~3/year, 13:00 ET close): this job runs post-close and
|
||||
# entries behave like stale_close (still acceptable per execution-recovery matrix).
|
||||
# No exchange calendar dependency.
|
||||
_NEAR_CLOSE_PIPELINE_STEPS = [
|
||||
# Must land today's in-progress bar (~20 min behind live), or the scan falls
|
||||
# back to the previous close and execution degrades to the stale_close floor.
|
||||
("data_collector", "collect_ohlcv_for_scan"),
|
||||
("rr_scanner", "scan_rr"),
|
||||
# Straight after the scan so shadow entries mark at the same near-close
|
||||
# prices the discretionary book is looking at.
|
||||
("shadow_book", "run_shadow_book"),
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# After close (~16:45 ET Mon–Fri): fresh OHLCV fetch so outcomes resolve on the
|
||||
# final bar, not the near-close partial bar, then outcome/paper close.
|
||||
_AFTER_CLOSE_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv_final"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
|
||||
# Intraday (light): keep prices current and resolve outcomes through the day,
|
||||
# without the expensive scan/sentiment. The dashboard recomputes live R:R from
|
||||
# the latest price, so refreshing OHLCV is enough to stop prices lagging; the
|
||||
# outcome step also closes paper trades that hit their stop/target intraday.
|
||||
_INTRADAY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
|
||||
# Ordered by trading day, not alphabetically: this is the sequence an operator
|
||||
# reads down the page, and it drives the UI's ordering too.
|
||||
PIPELINE_STEPS: dict[str, list[tuple[str, str]]] = {
|
||||
"daily_pipeline": _DAILY_PIPELINE_STEPS,
|
||||
"intraday_pipeline": _INTRADAY_PIPELINE_STEPS,
|
||||
"near_close_pipeline": _NEAR_CLOSE_PIPELINE_STEPS,
|
||||
"after_close_pipeline": _AFTER_CLOSE_PIPELINE_STEPS,
|
||||
}
|
||||
|
||||
# Derived, never hand-maintained: this used to be a literal set in admin_service
|
||||
# duplicating the four lists above from another module, with nothing asserting
|
||||
# the two agreed.
|
||||
PIPELINE_MEMBERS: frozenset[str] = frozenset(
|
||||
step for steps in PIPELINE_STEPS.values() for step, _ in steps
|
||||
)
|
||||
|
||||
|
||||
def _pipelines_by_member() -> dict[str, tuple[str, ...]]:
|
||||
"""Member -> the orchestrators that run it, in trading-day order.
|
||||
|
||||
Membership is many-to-many: data_collector runs in all four pipelines (via
|
||||
three different coroutines), alerts and outcome_evaluator in two each.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for pipeline, steps in PIPELINE_STEPS.items():
|
||||
for step, _ in steps:
|
||||
bucket = out.setdefault(step, [])
|
||||
if pipeline not in bucket:
|
||||
bucket.append(pipeline)
|
||||
return {member: tuple(pipelines) for member, pipelines in out.items()}
|
||||
|
||||
|
||||
PIPELINES_BY_MEMBER: dict[str, tuple[str, ...]] = _pipelines_by_member()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job identity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Orchestrators, in trading-day order.
|
||||
PIPELINE_JOBS: tuple[str, ...] = tuple(PIPELINE_STEPS)
|
||||
|
||||
# Own timer, independent of any pipeline.
|
||||
SCHEDULED_JOBS: tuple[str, ...] = (
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"ticker_universe_sync",
|
||||
"backtest",
|
||||
)
|
||||
|
||||
# Registered but never auto-fired; run only when a human asks.
|
||||
MANUAL_JOBS: tuple[str, ...] = ("event_study", "data_backfill")
|
||||
|
||||
# Steps in the order an operator meets them across the trading day, so the UI
|
||||
# reads as a sequence rather than an alphabetical jumble.
|
||||
PIPELINE_STEP_JOBS: tuple[str, ...] = tuple(
|
||||
dict.fromkeys(step for steps in PIPELINE_STEPS.values() for step, _ in steps)
|
||||
)
|
||||
|
||||
VALID_JOB_NAMES: frozenset[str] = frozenset(
|
||||
PIPELINE_JOBS + PIPELINE_STEP_JOBS + SCHEDULED_JOBS + MANUAL_JOBS
|
||||
)
|
||||
|
||||
JOB_LABELS: dict[str, str] = {
|
||||
"data_collector": "Data Collector (OHLCV)",
|
||||
"data_backfill": "Data Backfill (deep history)",
|
||||
"benchmark_collector": "Benchmark Collector",
|
||||
"sentiment_collector": "Sentiment Collector",
|
||||
"dolt_earnings_import": "Dolt Earnings Import",
|
||||
"sec_fundamentals_import": "SEC Fundamentals Import",
|
||||
"rr_scanner": "R:R Scanner",
|
||||
"ticker_universe_sync": "Ticker Universe Sync",
|
||||
"outcome_evaluator": "Outcome Evaluator",
|
||||
"alerts": "Alerts Dispatcher",
|
||||
# Keys are persisted job ids and must not change; these are display only.
|
||||
"market_regime": "Market Trend (SPY)",
|
||||
"regime_monitor": "AI/Tech Risk Monitor",
|
||||
"event_study": "Event Study",
|
||||
"backtest": "Backtest",
|
||||
"daily_pipeline": "Morning Pipeline",
|
||||
"near_close_pipeline": "Near-Close Pipeline (scan+alert)",
|
||||
"after_close_pipeline": "After-Close Pipeline (outcome)",
|
||||
"intraday_pipeline": "Intraday Pipeline",
|
||||
"shadow_book": "Shadow Book (auto-traded strategy)",
|
||||
}
|
||||
|
||||
CATEGORY_PIPELINE = "pipeline"
|
||||
CATEGORY_STEP = "pipeline_step"
|
||||
CATEGORY_SCHEDULED = "scheduled"
|
||||
CATEGORY_MANUAL = "manual"
|
||||
|
||||
# Order the sections appear in.
|
||||
CATEGORY_ORDER: tuple[str, ...] = (
|
||||
CATEGORY_PIPELINE,
|
||||
CATEGORY_STEP,
|
||||
CATEGORY_SCHEDULED,
|
||||
CATEGORY_MANUAL,
|
||||
)
|
||||
|
||||
CATEGORY_LABELS: dict[str, str] = {
|
||||
CATEGORY_PIPELINE: "Pipelines",
|
||||
CATEGORY_STEP: "Pipeline steps",
|
||||
CATEGORY_SCHEDULED: "Standalone scheduled",
|
||||
CATEGORY_MANUAL: "Manual only",
|
||||
}
|
||||
|
||||
_CATEGORY_MEMBERS: dict[str, tuple[str, ...]] = {
|
||||
CATEGORY_PIPELINE: PIPELINE_JOBS,
|
||||
CATEGORY_STEP: PIPELINE_STEP_JOBS,
|
||||
CATEGORY_SCHEDULED: SCHEDULED_JOBS,
|
||||
CATEGORY_MANUAL: MANUAL_JOBS,
|
||||
}
|
||||
|
||||
JOB_CATEGORY: dict[str, str] = {
|
||||
name: category
|
||||
for category, names in _CATEGORY_MEMBERS.items()
|
||||
for name in names
|
||||
}
|
||||
|
||||
# Registered and triggerable through the API, but kept out of Admin → Jobs.
|
||||
# data_backfill's only capability beyond collect_ohlcv (which already backfills
|
||||
# full history for *new* tickers) is re-deepening *existing* ones after
|
||||
# ohlcv_history_days is raised -- a rare one-off, not something to scan past
|
||||
# every time you open the page.
|
||||
HIDDEN_JOBS: frozenset[str] = frozenset({"data_backfill"})
|
||||
|
||||
_SORT_INDEX: dict[str, tuple[int, int]] = {
|
||||
name: (CATEGORY_ORDER.index(category), position)
|
||||
for category, names in _CATEGORY_MEMBERS.items()
|
||||
for position, name in enumerate(names)
|
||||
}
|
||||
|
||||
|
||||
def sort_order(job_name: str) -> tuple[int, int]:
|
||||
"""(category rank, position within category). Unknown jobs sort last."""
|
||||
return _SORT_INDEX.get(job_name, (len(CATEGORY_ORDER), 0))
|
||||
+9
-1
@@ -21,7 +21,12 @@ from app.config import settings
|
||||
from app.database import async_session_factory, engine
|
||||
from app.middleware import register_exception_handlers
|
||||
from app.models.user import User
|
||||
from app.scheduler import configure_scheduler, load_schedule_config, scheduler
|
||||
from app.scheduler import (
|
||||
configure_scheduler,
|
||||
flush_job_run_persists,
|
||||
load_schedule_config,
|
||||
scheduler,
|
||||
)
|
||||
from app.routers.admin import router as admin_router
|
||||
from app.routers.auth import router as auth_router
|
||||
from app.routers.health import router as health_router
|
||||
@@ -91,6 +96,9 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
scheduler.shutdown(wait=False)
|
||||
logger.info("Scheduler stopped")
|
||||
# Drain detached last-run writes before the engine goes away, or a job that
|
||||
# finished during shutdown loses the row it just wrote.
|
||||
await flush_job_run_persists()
|
||||
await engine.dispose()
|
||||
logger.info("Shutting down")
|
||||
|
||||
|
||||
@@ -14,9 +14,12 @@ from app.models.settings import SystemSetting, IngestionProgress
|
||||
from app.models.alert import AlertLog
|
||||
from app.models.paper_trade import PaperTrade
|
||||
from app.models.regime_snapshot import RegimeSnapshot
|
||||
from app.models.regime_fundamental_observation import RegimeFundamentalObservation
|
||||
from app.models.benchmark_price import BenchmarkPrice
|
||||
from app.models.signal_context_snapshot import SignalContextSnapshot
|
||||
from app.models.system_event import SystemEvent
|
||||
from app.models.sec_filing_gap import SecFilingGap
|
||||
from app.models.job_run_state import JobRunState
|
||||
|
||||
__all__ = [
|
||||
"Ticker",
|
||||
@@ -37,7 +40,10 @@ __all__ = [
|
||||
"AlertLog",
|
||||
"PaperTrade",
|
||||
"RegimeSnapshot",
|
||||
"RegimeFundamentalObservation",
|
||||
"BenchmarkPrice",
|
||||
"SignalContextSnapshot",
|
||||
"SystemEvent",
|
||||
"SecFilingGap",
|
||||
"JobRunState",
|
||||
]
|
||||
|
||||
@@ -10,7 +10,8 @@ class DataImportRun(Base):
|
||||
"""One row per bulk-import attempt (SEC facts / Dolt earnings / Dolt stocks).
|
||||
|
||||
Lean audit record for the batch import framework: every attempt is logged,
|
||||
whether it promoted, was a ``no_op`` (unchanged revision), or ``failed``.
|
||||
whether it promoted, was a ``no_op`` (unchanged revision), was ``deferred``
|
||||
for an expected retry, or ``failed``.
|
||||
``row_counts`` and ``validation`` hold JSON strings (repo convention — see
|
||||
``fundamental_data.unavailable_fields_json``), not JSONB; the validation
|
||||
blob carries reconciliation/discrepancy summaries so no separate conflicts
|
||||
@@ -28,7 +29,7 @@ class DataImportRun(Base):
|
||||
source: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# Dolt commit hash, or SEC archive SHA-256. Null until known.
|
||||
revision: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# running | validated | promoted | no_op | failed
|
||||
# running | validated | promoted | no_op | deferred | failed
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
source_max_date: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
row_counts_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -39,4 +40,5 @@ class DataImportRun(Base):
|
||||
completed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
# Failure detail, or the non-error reason when status is deferred.
|
||||
error_details: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class JobRunState(Base):
|
||||
"""How each scheduled job last finished. One row per job, overwritten.
|
||||
|
||||
The scheduler's ``_job_runtime`` dict is the live view and is deliberately
|
||||
in-memory, but it is also wiped by every process restart -- so after a deploy
|
||||
Admin → Jobs could only say "Active" with no indication of whether a job had
|
||||
ever run. This is the durable half.
|
||||
|
||||
Deliberately not history: ``system_events`` already grows without a reaper,
|
||||
and a second append-only operational table would repeat that. Rows are
|
||||
upserted on ``job_name``; adding history later is purely additive.
|
||||
"""
|
||||
|
||||
__tablename__ = "job_run_state"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
|
||||
# Scheduler vocabulary: completed | skipped | error | rate_limited | deferred.
|
||||
# Distinct from data_import_runs' statuses, which is one reason this is its
|
||||
# own table rather than a widened column there.
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
processed: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
from datetime import date as date_type
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Date, DateTime, Float, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class RegimeFundamentalObservation(Base):
|
||||
"""Point-in-time record of the sourced hyperscaler capex / earnings read.
|
||||
|
||||
One row per ``effective_date`` (unique, upserted). Before this table the
|
||||
observation lived in a single ``SystemSetting`` slot, so every refresh
|
||||
overwrote the previous one and no history existed at all — which made the
|
||||
read impossible to replay, impossible to backtest, and meant a snapshot
|
||||
rebuild could only ever score historical sessions as if nothing had been
|
||||
observed.
|
||||
|
||||
The read is a categorical channel reported beside State and Warning, never a
|
||||
term in either, so this series is not a scoring input. It is the record that
|
||||
makes the channel replayable at all -- and the only route to eventually
|
||||
testing whether it improves prediction conditional on Warning, which is the
|
||||
one thing that could justify combining the channels later.
|
||||
|
||||
``effective_date`` rather than ``fetched_at`` is the key: it is the session
|
||||
the observation becomes usable on (normally the next weekday), and the gate
|
||||
that stops a rebuild stamping today's reading onto historical rows.
|
||||
"""
|
||||
|
||||
__tablename__ = "regime_fundamental_observations"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
effective_date: Mapped[date_type] = mapped_column(
|
||||
Date, nullable=False, unique=True, index=True
|
||||
)
|
||||
f1_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
f3_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
capex_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
good_news_stock_down: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
reasoning: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
source: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -8,7 +8,7 @@ from app.database import Base
|
||||
|
||||
|
||||
class RegimeSnapshot(Base):
|
||||
"""Daily point-in-time snapshot of the AI/Tech Regime Monitor.
|
||||
"""Daily point-in-time snapshot of the AI/Tech Risk Monitor.
|
||||
|
||||
One row per calendar date (unique). ``breakdown_json`` holds the full
|
||||
``breakdown_json`` is authoritative for v2 State, Warning, source dates,
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import Date, DateTime, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class SecFilingGap(Base):
|
||||
"""Active SEC filing that could not yet be reconstructed.
|
||||
|
||||
Rows form a small retry queue. Successful snapshot ingestion deletes the
|
||||
matching row; a later valid filing supersedes it. While a current row remains,
|
||||
tickers mapped to its CIK are not eligible for actionable trade setups.
|
||||
"""
|
||||
|
||||
__tablename__ = "sec_filing_gaps"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("accession", name="uq_sec_filing_gaps_accession"),
|
||||
Index("ix_sec_filing_gaps_cik", "cik"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cik: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
accession: Mapped[str] = mapped_column(String(25), nullable=False)
|
||||
form: Mapped[str | None] = mapped_column(String(12), nullable=True)
|
||||
index_date: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
reason: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
coregistrant_ciks_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
last_attempted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
escalated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
# Set while this gap's issuer is exempt from the setup pause (escalated, and
|
||||
# its own fundamentals still recent — see fundamentals_quality_service).
|
||||
# Cleared when the exemption lapses, which is the moment the pause silently
|
||||
# comes back and the only moment worth alerting on.
|
||||
exempted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
@@ -1,6 +1,6 @@
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import String, DateTime
|
||||
from sqlalchemy import Date, String, DateTime
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
@@ -21,6 +21,13 @@ class Ticker(Base):
|
||||
cik: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
sic: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
# Delisting is recorded, never deleted: the rows carry the price history that
|
||||
# makes a backtest less survivorship-biased, and a delete cascades it away.
|
||||
# NULL == actively traded. The live signal path filters on this (see
|
||||
# ticker_service.active_only); list/admin views keep the row and show it.
|
||||
delisted_on: Mapped[date | None] = mapped_column(Date, nullable=True, index=True)
|
||||
# How we learned: "form_25" (SEC confirmed), "manual" (operator).
|
||||
delisted_reason: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
"""Financial Modeling Prep (FMP) fundamentals provider using httpx.
|
||||
|
||||
Uses the stable API endpoints (https://financialmodelingprep.com/stable/)
|
||||
which replaced the legacy /api/v3/ endpoints deprecated in Aug 2025.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from app.exceptions import ProviderError, RateLimitError
|
||||
from app.providers.protocol import FundamentalData
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FMP_STABLE_URL = "https://financialmodelingprep.com/stable"
|
||||
|
||||
# Resolve CA bundle for explicit httpx verify
|
||||
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
|
||||
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
|
||||
_CA_BUNDLE_PATH: str | bool = True # use system default
|
||||
else:
|
||||
_CA_BUNDLE_PATH = _CA_BUNDLE
|
||||
|
||||
|
||||
class FMPFundamentalProvider:
|
||||
"""Fetches fundamental data from Financial Modeling Prep REST API."""
|
||||
|
||||
def __init__(self, api_key: str) -> None:
|
||||
if not api_key:
|
||||
raise ProviderError("FMP API key is required")
|
||||
self._api_key = api_key
|
||||
|
||||
# Mapping from FMP endpoint name to the FundamentalData field it populates
|
||||
_ENDPOINT_FIELD_MAP: dict[str, str] = {
|
||||
"ratios-ttm": "pe_ratio",
|
||||
"financial-growth": "revenue_growth",
|
||||
"earnings": "earnings_surprise",
|
||||
}
|
||||
|
||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
||||
"""Fetch P/E, revenue growth, earnings surprise, and market cap.
|
||||
|
||||
Fetches from multiple stable endpoints. If a supplementary endpoint
|
||||
(ratios, growth, earnings) returns 402 (paid tier), we gracefully
|
||||
degrade and return partial data rather than failing entirely, and
|
||||
record the affected field in ``unavailable_fields``.
|
||||
"""
|
||||
try:
|
||||
endpoints_402: set[str] = set()
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
||||
params = {"symbol": ticker, "apikey": self._api_key}
|
||||
|
||||
# Profile is the primary source — must succeed
|
||||
profile = await self._fetch_json(client, "profile", params, ticker)
|
||||
|
||||
# Supplementary sources — degrade gracefully on 402
|
||||
ratios, was_402 = await self._fetch_json_optional(client, "ratios-ttm", params, ticker)
|
||||
if was_402:
|
||||
endpoints_402.add("ratios-ttm")
|
||||
|
||||
growth, was_402 = await self._fetch_json_optional(client, "financial-growth", params, ticker)
|
||||
if was_402:
|
||||
endpoints_402.add("financial-growth")
|
||||
|
||||
earnings, was_402 = await self._fetch_json_optional(client, "earnings", params, ticker)
|
||||
if was_402:
|
||||
endpoints_402.add("earnings")
|
||||
|
||||
pe_ratio = self._safe_float(ratios.get("priceToEarningsRatioTTM"))
|
||||
revenue_growth = self._safe_float(growth.get("revenueGrowth"))
|
||||
market_cap = self._safe_float(profile.get("marketCap"))
|
||||
earnings_surprise = self._compute_earnings_surprise(earnings)
|
||||
|
||||
# Build unavailable_fields from 402 endpoints
|
||||
unavailable_fields: dict[str, str] = {
|
||||
self._ENDPOINT_FIELD_MAP[ep]: "requires paid plan"
|
||||
for ep in endpoints_402
|
||||
if ep in self._ENDPOINT_FIELD_MAP
|
||||
}
|
||||
|
||||
return FundamentalData(
|
||||
ticker=ticker,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
fetched_at=datetime.now(timezone.utc),
|
||||
unavailable_fields=unavailable_fields,
|
||||
)
|
||||
|
||||
except (ProviderError, RateLimitError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("FMP provider error for %s: %s", ticker, exc)
|
||||
raise ProviderError(f"FMP provider error for {ticker}: {exc}") from exc
|
||||
|
||||
async def _fetch_json(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
endpoint: str,
|
||||
params: dict,
|
||||
ticker: str,
|
||||
) -> dict:
|
||||
"""Fetch a stable endpoint and return the first item (or empty dict)."""
|
||||
url = f"{_FMP_STABLE_URL}/{endpoint}"
|
||||
resp = await client.get(url, params=params)
|
||||
self._check_response(resp, ticker, endpoint)
|
||||
data = resp.json()
|
||||
if isinstance(data, list):
|
||||
return data[0] if data else {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
async def _fetch_json_optional(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
endpoint: str,
|
||||
params: dict,
|
||||
ticker: str,
|
||||
) -> tuple[dict, bool]:
|
||||
"""Fetch a stable endpoint, returning ``({}, True)`` on 402 (paid tier).
|
||||
|
||||
Returns a tuple of (data_dict, was_402) so callers can track which
|
||||
endpoints required a paid plan.
|
||||
"""
|
||||
url = f"{_FMP_STABLE_URL}/{endpoint}"
|
||||
resp = await client.get(url, params=params)
|
||||
if resp.status_code == 402:
|
||||
logger.warning("FMP %s requires paid plan — skipping for %s", endpoint, ticker)
|
||||
return {}, True
|
||||
self._check_response(resp, ticker, endpoint)
|
||||
data = resp.json()
|
||||
if isinstance(data, list):
|
||||
return (data[0] if data else {}, False)
|
||||
return (data if isinstance(data, dict) else {}, False)
|
||||
|
||||
def _compute_earnings_surprise(self, earnings_data: dict) -> float | None:
|
||||
"""Compute earnings surprise % from the most recent actual vs estimated EPS."""
|
||||
actual = self._safe_float(earnings_data.get("epsActual"))
|
||||
estimated = self._safe_float(earnings_data.get("epsEstimated"))
|
||||
if actual is None or estimated is None or estimated == 0:
|
||||
return None
|
||||
return ((actual - estimated) / abs(estimated)) * 100
|
||||
|
||||
def _check_response(
|
||||
self, resp: httpx.Response, ticker: str, endpoint: str
|
||||
) -> None:
|
||||
"""Raise appropriate errors for non-200 responses."""
|
||||
if resp.status_code == 429:
|
||||
raise RateLimitError(f"FMP rate limit hit for {ticker} ({endpoint})")
|
||||
if resp.status_code == 403:
|
||||
raise ProviderError(
|
||||
f"FMP {endpoint} access denied for {ticker}: HTTP 403 — check API key validity and plan tier"
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise ProviderError(
|
||||
f"FMP {endpoint} error for {ticker}: HTTP {resp.status_code}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _safe_float(value: object) -> float | None:
|
||||
"""Convert a value to float, returning None on failure."""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -1,354 +0,0 @@
|
||||
"""Chained fundamentals provider with fallback adapters.
|
||||
|
||||
Order:
|
||||
1) FMP (if configured)
|
||||
2) Finnhub (if configured)
|
||||
3) Alpha Vantage (if configured)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.exceptions import ProviderError, RateLimitError
|
||||
from app.providers.fmp import FMPFundamentalProvider
|
||||
from app.providers.protocol import FundamentalData, FundamentalProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
|
||||
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
|
||||
_CA_BUNDLE_PATH: str | bool = True
|
||||
else:
|
||||
_CA_BUNDLE_PATH = _CA_BUNDLE
|
||||
|
||||
|
||||
def _safe_float(value: object) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _to_api_symbol(symbol: str) -> str:
|
||||
"""Convert internal symbol format (BRK-B) to API format (BRK.B).
|
||||
|
||||
Finnhub and Alpha Vantage use dot-separated share class notation.
|
||||
"""
|
||||
return symbol.replace("-", ".")
|
||||
|
||||
|
||||
class FinnhubFundamentalProvider:
|
||||
"""Fundamentals provider backed by Finnhub free endpoints."""
|
||||
|
||||
def __init__(self, api_key: str) -> None:
|
||||
if not api_key:
|
||||
raise ProviderError("Finnhub API key is required")
|
||||
self._api_key = api_key
|
||||
self._base_url = "https://finnhub.io/api/v1"
|
||||
|
||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
||||
unavailable: dict[str, str] = {}
|
||||
api_symbol = _to_api_symbol(ticker)
|
||||
|
||||
today = date.today()
|
||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
||||
profile_resp = await client.get(
|
||||
f"{self._base_url}/stock/profile2",
|
||||
params={"symbol": api_symbol, "token": self._api_key},
|
||||
)
|
||||
metric_resp = await client.get(
|
||||
f"{self._base_url}/stock/metric",
|
||||
params={"symbol": api_symbol, "metric": "all", "token": self._api_key},
|
||||
)
|
||||
earnings_resp = await client.get(
|
||||
f"{self._base_url}/stock/earnings",
|
||||
params={"symbol": api_symbol, "limit": 1, "token": self._api_key},
|
||||
)
|
||||
calendar_resp = await client.get(
|
||||
f"{self._base_url}/calendar/earnings",
|
||||
params={
|
||||
"symbol": api_symbol,
|
||||
"from": today.isoformat(),
|
||||
"to": (today + timedelta(days=120)).isoformat(),
|
||||
"token": self._api_key,
|
||||
},
|
||||
)
|
||||
|
||||
for resp, endpoint in (
|
||||
(profile_resp, "profile2"),
|
||||
(metric_resp, "stock/metric"),
|
||||
(earnings_resp, "stock/earnings"),
|
||||
(calendar_resp, "calendar/earnings"),
|
||||
):
|
||||
if resp.status_code == 429:
|
||||
raise RateLimitError(f"Finnhub rate limit hit for {ticker} ({endpoint})")
|
||||
if resp.status_code in (401, 403):
|
||||
raise ProviderError(f"Finnhub access denied for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
||||
if resp.status_code != 200:
|
||||
raise ProviderError(f"Finnhub error for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
||||
|
||||
profile_payload = profile_resp.json() if profile_resp.text else {}
|
||||
metric_payload = metric_resp.json() if metric_resp.text else {}
|
||||
earnings_payload = earnings_resp.json() if earnings_resp.text else []
|
||||
|
||||
metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {}
|
||||
# Finnhub profile2 marketCapitalization is in millions of USD.
|
||||
# Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage.
|
||||
market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization"))
|
||||
market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None
|
||||
pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual"))
|
||||
revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y"))
|
||||
|
||||
earnings_surprise = None
|
||||
if isinstance(earnings_payload, list) and earnings_payload:
|
||||
first = earnings_payload[0] if isinstance(earnings_payload[0], dict) else {}
|
||||
earnings_surprise = _safe_float(first.get("surprisePercent"))
|
||||
|
||||
next_earnings_date = self._next_earnings(calendar_resp)
|
||||
|
||||
if pe_ratio is None:
|
||||
unavailable["pe_ratio"] = "not available from provider payload"
|
||||
if revenue_growth is None:
|
||||
unavailable["revenue_growth"] = "not available from provider payload"
|
||||
if earnings_surprise is None:
|
||||
unavailable["earnings_surprise"] = "not available from provider payload"
|
||||
if market_cap is None:
|
||||
unavailable["market_cap"] = "not available from provider payload"
|
||||
|
||||
return FundamentalData(
|
||||
ticker=ticker,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
fetched_at=datetime.now(timezone.utc),
|
||||
next_earnings_date=next_earnings_date,
|
||||
unavailable_fields=unavailable,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _next_earnings(resp: httpx.Response) -> date | None:
|
||||
"""Earliest upcoming earnings date from Finnhub's calendar payload."""
|
||||
try:
|
||||
payload = resp.json() if resp.text else {}
|
||||
except ValueError:
|
||||
return None
|
||||
entries = payload.get("earningsCalendar", []) if isinstance(payload, dict) else []
|
||||
dates: list[date] = []
|
||||
today = date.today()
|
||||
for entry in entries if isinstance(entries, list) else []:
|
||||
raw = entry.get("date") if isinstance(entry, dict) else None
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
parsed = date.fromisoformat(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if parsed >= today:
|
||||
dates.append(parsed)
|
||||
return min(dates) if dates else None
|
||||
|
||||
|
||||
class AlphaVantageFundamentalProvider:
|
||||
"""Fundamentals provider backed by Alpha Vantage free endpoints."""
|
||||
|
||||
def __init__(self, api_key: str) -> None:
|
||||
if not api_key:
|
||||
raise ProviderError("Alpha Vantage API key is required")
|
||||
self._api_key = api_key
|
||||
self._base_url = "https://www.alphavantage.co/query"
|
||||
|
||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
||||
unavailable: dict[str, str] = {}
|
||||
api_symbol = _to_api_symbol(ticker)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
||||
overview_resp = await client.get(
|
||||
self._base_url,
|
||||
params={"function": "OVERVIEW", "symbol": api_symbol, "apikey": self._api_key},
|
||||
)
|
||||
earnings_resp = await client.get(
|
||||
self._base_url,
|
||||
params={"function": "EARNINGS", "symbol": api_symbol, "apikey": self._api_key},
|
||||
)
|
||||
income_resp = await client.get(
|
||||
self._base_url,
|
||||
params={"function": "INCOME_STATEMENT", "symbol": api_symbol, "apikey": self._api_key},
|
||||
)
|
||||
|
||||
for resp, endpoint in (
|
||||
(overview_resp, "OVERVIEW"),
|
||||
(earnings_resp, "EARNINGS"),
|
||||
(income_resp, "INCOME_STATEMENT"),
|
||||
):
|
||||
if resp.status_code == 429:
|
||||
raise RateLimitError(f"Alpha Vantage rate limit hit for {ticker} ({endpoint})")
|
||||
if resp.status_code != 200:
|
||||
raise ProviderError(f"Alpha Vantage error for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
||||
|
||||
overview = overview_resp.json() if overview_resp.text else {}
|
||||
earnings = earnings_resp.json() if earnings_resp.text else {}
|
||||
income = income_resp.json() if income_resp.text else {}
|
||||
|
||||
if isinstance(overview, dict) and overview.get("Information"):
|
||||
raise ProviderError(f"Alpha Vantage unavailable for {ticker}: {overview.get('Information')}")
|
||||
if isinstance(overview, dict) and overview.get("Note"):
|
||||
raise RateLimitError(f"Alpha Vantage rate limit for {ticker}: {overview.get('Note')}")
|
||||
|
||||
pe_ratio = _safe_float((overview or {}).get("PERatio"))
|
||||
market_cap = _safe_float((overview or {}).get("MarketCapitalization"))
|
||||
|
||||
earnings_surprise = None
|
||||
quarterly = earnings.get("quarterlyEarnings", []) if isinstance(earnings, dict) else []
|
||||
if isinstance(quarterly, list) and quarterly:
|
||||
first = quarterly[0] if isinstance(quarterly[0], dict) else {}
|
||||
earnings_surprise = _safe_float(first.get("surprisePercentage"))
|
||||
|
||||
revenue_growth = None
|
||||
annual = income.get("annualReports", []) if isinstance(income, dict) else []
|
||||
if isinstance(annual, list) and len(annual) >= 2:
|
||||
curr = _safe_float((annual[0] or {}).get("totalRevenue"))
|
||||
prev = _safe_float((annual[1] or {}).get("totalRevenue"))
|
||||
if curr is not None and prev not in (None, 0):
|
||||
revenue_growth = ((curr - prev) / abs(prev)) * 100.0
|
||||
|
||||
if pe_ratio is None:
|
||||
unavailable["pe_ratio"] = "not available from provider payload"
|
||||
if revenue_growth is None:
|
||||
unavailable["revenue_growth"] = "not available from provider payload"
|
||||
if earnings_surprise is None:
|
||||
unavailable["earnings_surprise"] = "not available from provider payload"
|
||||
if market_cap is None:
|
||||
unavailable["market_cap"] = "not available from provider payload"
|
||||
|
||||
return FundamentalData(
|
||||
ticker=ticker,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
fetched_at=datetime.now(timezone.utc),
|
||||
unavailable_fields=unavailable,
|
||||
)
|
||||
|
||||
|
||||
_FUNDAMENTAL_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise", "market_cap")
|
||||
|
||||
|
||||
class ChainedFundamentalProvider:
|
||||
"""Merge fundamentals across providers, filling gaps from later sources.
|
||||
|
||||
A single provider rarely covers everything on free tiers — FMP's free plan,
|
||||
for example, returns only market cap (the ratios/growth/earnings endpoints
|
||||
402). Rather than stop at the first provider with *any* field, we take each
|
||||
field from the first provider that supplies it, so FMP's market cap is
|
||||
combined with Finnhub's P/E and earnings surprise.
|
||||
"""
|
||||
|
||||
def __init__(self, providers: list[tuple[str, FundamentalProvider]]) -> None:
|
||||
if not providers:
|
||||
raise ProviderError("No fundamental providers configured")
|
||||
self._providers = providers
|
||||
|
||||
async def fetch_fundamentals(self, ticker: str, allow_partial: bool = False) -> FundamentalData:
|
||||
"""Merge fundamentals across providers.
|
||||
|
||||
``allow_partial`` controls behaviour when a fallback provider is *rate
|
||||
limited* and we end up with missing fields. By default we raise
|
||||
RateLimitError so the caller (the bulk collector) can back off and retry
|
||||
the ticker once the window frees — otherwise a transient 429 on Finnhub
|
||||
would be silently stored as market-cap-only. Pass ``allow_partial=True``
|
||||
(manual single fetches, or the collector's final give-up attempt) to
|
||||
accept whatever was gathered instead of raising.
|
||||
"""
|
||||
merged: dict[str, float | None] = {f: None for f in _FUNDAMENTAL_FIELDS}
|
||||
field_source: dict[str, str] = {}
|
||||
errors: list[str] = []
|
||||
rate_limited = False
|
||||
next_earnings_date = None
|
||||
|
||||
for provider_name, provider in self._providers:
|
||||
if all(merged[f] is not None for f in _FUNDAMENTAL_FIELDS) and next_earnings_date:
|
||||
break
|
||||
try:
|
||||
data = await provider.fetch_fundamentals(ticker)
|
||||
except RateLimitError as exc:
|
||||
rate_limited = True
|
||||
errors.append(f"{provider_name}: RateLimitError: {exc}")
|
||||
continue
|
||||
except Exception as exc:
|
||||
errors.append(f"{provider_name}: {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
|
||||
if next_earnings_date is None and data.next_earnings_date is not None:
|
||||
next_earnings_date = data.next_earnings_date
|
||||
|
||||
for field in _FUNDAMENTAL_FIELDS:
|
||||
if merged[field] is None:
|
||||
value = getattr(data, field)
|
||||
if value is not None:
|
||||
merged[field] = value
|
||||
field_source[field] = provider_name
|
||||
|
||||
missing = [f for f in _FUNDAMENTAL_FIELDS if merged[f] is None]
|
||||
|
||||
# A rate limit left data incomplete: signal it (unless partial is OK) so
|
||||
# the collector backs off rather than persisting a degraded record.
|
||||
if rate_limited and missing and not allow_partial:
|
||||
attempts = "; ".join(errors[:6])
|
||||
raise RateLimitError(
|
||||
f"Fundamentals incomplete for {ticker} due to provider rate limits "
|
||||
f"(missing {', '.join(missing)}). Attempts: {attempts}"
|
||||
)
|
||||
|
||||
if all(merged[f] is None for f in _FUNDAMENTAL_FIELDS):
|
||||
attempts = "; ".join(errors[:6]) if errors else "no usable metrics from any provider"
|
||||
raise ProviderError(f"All fundamentals providers failed for {ticker}. Attempts: {attempts}")
|
||||
|
||||
unavailable: dict[str, str] = {
|
||||
field: "not available from any configured provider"
|
||||
for field in _FUNDAMENTAL_FIELDS
|
||||
if merged[field] is None
|
||||
}
|
||||
# Record which provider supplied each field for transparency.
|
||||
for field, src in field_source.items():
|
||||
unavailable[f"source_{field}"] = src
|
||||
|
||||
return FundamentalData(
|
||||
ticker=ticker,
|
||||
pe_ratio=merged["pe_ratio"],
|
||||
revenue_growth=merged["revenue_growth"],
|
||||
earnings_surprise=merged["earnings_surprise"],
|
||||
market_cap=merged["market_cap"],
|
||||
fetched_at=datetime.now(timezone.utc),
|
||||
next_earnings_date=next_earnings_date,
|
||||
unavailable_fields=unavailable,
|
||||
)
|
||||
|
||||
|
||||
def build_fundamental_provider_chain() -> FundamentalProvider:
|
||||
providers: list[tuple[str, FundamentalProvider]] = []
|
||||
|
||||
if settings.fmp_api_key:
|
||||
providers.append(("fmp", FMPFundamentalProvider(settings.fmp_api_key)))
|
||||
if settings.finnhub_api_key:
|
||||
providers.append(("finnhub", FinnhubFundamentalProvider(settings.finnhub_api_key)))
|
||||
if settings.alpha_vantage_api_key:
|
||||
providers.append(("alpha_vantage", AlphaVantageFundamentalProvider(settings.alpha_vantage_api_key)))
|
||||
|
||||
if not providers:
|
||||
raise ProviderError(
|
||||
"No fundamentals provider configured. Set one of FMP_API_KEY, FINNHUB_API_KEY, ALPHA_VANTAGE_API_KEY"
|
||||
)
|
||||
|
||||
logger.info("Fundamentals provider chain configured: %s", [name for name, _ in providers])
|
||||
return ChainedFundamentalProvider(providers)
|
||||
@@ -44,20 +44,6 @@ class SentimentData:
|
||||
recommendation: str | None = None # "buy" | "hold" | "avoid" — actionable LLM view
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FundamentalData:
|
||||
"""Fundamental metrics returned by fundamental providers."""
|
||||
|
||||
ticker: str
|
||||
pe_ratio: float | None
|
||||
revenue_growth: float | None
|
||||
earnings_surprise: float | None
|
||||
market_cap: float | None
|
||||
fetched_at: datetime
|
||||
next_earnings_date: date | None = None
|
||||
unavailable_fields: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider Protocols
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -81,9 +67,5 @@ class SentimentProvider(Protocol):
|
||||
...
|
||||
|
||||
|
||||
class FundamentalProvider(Protocol):
|
||||
"""Protocol for fundamental data providers."""
|
||||
|
||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
||||
"""Fetch fundamental data for a ticker."""
|
||||
...
|
||||
# No fundamentals provider protocol: since A6 fundamentals come only from the
|
||||
# batch SEC/Dolt imports, never from a request-time provider call.
|
||||
|
||||
@@ -453,36 +453,6 @@ async def toggle_job(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_report(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Latest read-only A5 source/score comparison, or null before first run."""
|
||||
return APIEnvelope(
|
||||
status="success", data=admin_service.get_fundamentals_parity_report()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity/csv", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_csv(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Latest flattened A5 report for an authenticated browser download."""
|
||||
artifact = admin_service.get_fundamentals_parity_csv()
|
||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
||||
return APIEnvelope(status="success", data=data)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity/json", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_json(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Canonical A5 JSON artifact for an authenticated browser download."""
|
||||
artifact = admin_service.get_fundamentals_parity_json()
|
||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
||||
return APIEnvelope(status="success", data=data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System events (operational warnings / errors)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.schemas.common import APIEnvelope
|
||||
from app.schemas.fundamental import FundamentalResponse
|
||||
from app.services.fundamental_service import get_fundamental
|
||||
from app.services.fundamentals_api_service import build_fundamentals_v1
|
||||
from app.services import fundamentals_quality_service
|
||||
|
||||
router = APIRouter(tags=["fundamentals"])
|
||||
|
||||
@@ -34,6 +35,7 @@ async def read_fundamentals(
|
||||
"""Get latest fundamental data for a symbol (legacy fields + additive v1)."""
|
||||
record = await get_fundamental(db, symbol)
|
||||
v1 = await build_fundamentals_v1(db, symbol)
|
||||
quality = await fundamentals_quality_service.ticker_quality(db, symbol)
|
||||
|
||||
legacy: dict = {}
|
||||
if record is not None:
|
||||
@@ -47,5 +49,12 @@ async def read_fundamentals(
|
||||
unavailable_fields=_parse_unavailable_fields(record.unavailable_fields_json),
|
||||
)
|
||||
|
||||
data = FundamentalResponse(symbol=symbol.strip().upper(), **legacy, **v1)
|
||||
data = FundamentalResponse(
|
||||
symbol=symbol.strip().upper(),
|
||||
setup_eligible=quality.eligible,
|
||||
setup_block_code=quality.code,
|
||||
setup_block_reason=quality.message,
|
||||
**legacy,
|
||||
**v1,
|
||||
)
|
||||
return APIEnvelope(status="success", data=data.model_dump())
|
||||
|
||||
@@ -23,7 +23,6 @@ from app.models.sr_level import SRLevel
|
||||
from app.models.ticker import Ticker
|
||||
from app.models.user import User
|
||||
from app.providers.alpaca import AlpacaOHLCVProvider
|
||||
from app.providers.fundamentals_chain import build_fundamental_provider_chain
|
||||
from app.services.rr_scanner_service import (
|
||||
resolve_activation_ranks_for_symbol,
|
||||
scan_ticker,
|
||||
@@ -31,7 +30,6 @@ from app.services.rr_scanner_service import (
|
||||
from app.services.sentiment_provider_service import build_sentiment_provider
|
||||
from app.schemas.common import APIEnvelope
|
||||
from app.services import (
|
||||
fundamental_service,
|
||||
ingestion_service,
|
||||
scoring_service,
|
||||
sentiment_service,
|
||||
@@ -185,34 +183,14 @@ async def fetch_symbol(
|
||||
sources_out["sentiment"] = {"status": "error", "message": str(exc)}
|
||||
|
||||
# --- Fundamentals ---
|
||||
# No per-ticker fetch exists any more: fundamental_data is rebuilt for the
|
||||
# whole universe by the nightly SEC + Dolt imports, from local PostgreSQL.
|
||||
# The source key is still accepted so older clients get a truthful answer.
|
||||
if "fundamentals" in requested:
|
||||
if settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key:
|
||||
try:
|
||||
fundamentals_provider = build_fundamental_provider_chain()
|
||||
# Manual single fetch: take whatever we can get (a lone 429 on a
|
||||
# fallback shouldn't fail the whole refresh).
|
||||
fdata = await fundamentals_provider.fetch_fundamentals(
|
||||
symbol_upper, allow_partial=True
|
||||
)
|
||||
await fundamental_service.store_fundamental(
|
||||
db,
|
||||
symbol=symbol_upper,
|
||||
pe_ratio=fdata.pe_ratio,
|
||||
revenue_growth=fdata.revenue_growth,
|
||||
earnings_surprise=fdata.earnings_surprise,
|
||||
market_cap=fdata.market_cap,
|
||||
next_earnings_date=fdata.next_earnings_date,
|
||||
unavailable_fields=fdata.unavailable_fields,
|
||||
)
|
||||
sources_out["fundamentals"] = {"status": "ok", "message": None}
|
||||
except Exception as exc:
|
||||
logger.error("Fundamentals fetch failed for %s: %s", symbol_upper, exc)
|
||||
sources_out["fundamentals"] = {"status": "error", "message": str(exc)}
|
||||
else:
|
||||
sources_out["fundamentals"] = {
|
||||
"status": "skipped",
|
||||
"message": "No fundamentals provider key configured",
|
||||
}
|
||||
sources_out["fundamentals"] = {
|
||||
"status": "skipped",
|
||||
"message": "Fundamentals refresh nightly from the SEC + Dolt imports",
|
||||
}
|
||||
|
||||
# --- Derived pipeline: S/R levels (free, always) ---
|
||||
try:
|
||||
|
||||
+33
-1
@@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.dependencies import get_db, require_access
|
||||
from app.models.user import User
|
||||
from app.schemas.common import APIEnvelope
|
||||
from app.schemas.ticker import TickerCreate, TickerResponse
|
||||
from app.schemas.ticker import TickerCreate, TickerDelistingUpdate, TickerResponse
|
||||
from app.services import ticker_service
|
||||
|
||||
router = APIRouter(tags=["tickers"])
|
||||
@@ -51,3 +51,35 @@ async def delete_ticker(
|
||||
"""Delete a ticker and all associated data."""
|
||||
await ticker_service.delete_ticker(db, symbol)
|
||||
return APIEnvelope(status="success", data=None)
|
||||
|
||||
|
||||
@router.post("/tickers/{symbol}/delisting", response_model=APIEnvelope)
|
||||
async def mark_ticker_delisted(
|
||||
symbol: str,
|
||||
body: TickerDelistingUpdate,
|
||||
_user: User = Depends(require_access),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Retire a symbol: excluded from signals, price history kept.
|
||||
|
||||
The non-destructive alternative to DELETE, which cascades the history away.
|
||||
"""
|
||||
changed = await ticker_service.mark_delisted(
|
||||
db, symbol, delisted_on=body.delisted_on, reason=ticker_service.REASON_MANUAL
|
||||
)
|
||||
return APIEnvelope(status="success", data={"changed": changed})
|
||||
|
||||
|
||||
@router.delete("/tickers/{symbol}/delisting", response_model=APIEnvelope)
|
||||
async def clear_ticker_delisting(
|
||||
symbol: str,
|
||||
_user: User = Depends(require_access),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Un-retire a symbol wrongly marked delisted.
|
||||
|
||||
Automatic marking is only defensible because this exists: a false positive
|
||||
costs one row update rather than the price history a delete would take.
|
||||
"""
|
||||
changed = await ticker_service.clear_delisted(db, symbol)
|
||||
return APIEnvelope(status="success", data={"changed": changed})
|
||||
|
||||
+294
-305
@@ -1,9 +1,9 @@
|
||||
"""APScheduler job definitions and FastAPI lifespan integration.
|
||||
|
||||
Defines four scheduled jobs:
|
||||
Defines the scheduled jobs, among them:
|
||||
- Data Collector (OHLCV fetch for all tickers)
|
||||
- Sentiment Collector (sentiment for all tickers)
|
||||
- Fundamental Collector (fundamentals for all tickers)
|
||||
- Dolt Earnings / SEC Fundamentals imports (bulk fundamentals sources)
|
||||
- R:R Scanner (trade setup scan for all tickers)
|
||||
|
||||
Each job processes tickers independently, logs errors as structured JSON,
|
||||
@@ -18,31 +18,36 @@ import logging
|
||||
import asyncio
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from apscheduler.events import EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import job_catalog
|
||||
from app.config import settings
|
||||
from app.database import async_session_factory
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.sentiment import SentimentScore
|
||||
from app.models.ticker import Ticker
|
||||
from app.exceptions import ProviderError
|
||||
from app.providers.alpaca import AlpacaOHLCVProvider
|
||||
from app.providers.fundamentals_chain import build_fundamental_provider_chain
|
||||
from app.providers.protocol import SentimentData
|
||||
from app.services import job_run_store
|
||||
from app.services import (
|
||||
fundamental_service,
|
||||
ingestion_service,
|
||||
pipeline_run,
|
||||
sentiment_service,
|
||||
settings_store,
|
||||
shadow_book_service,
|
||||
fundamentals_parity_service,
|
||||
fundamental_data_refresh_service,
|
||||
)
|
||||
from app.services.data_import import (
|
||||
STATUS_DEFERRED,
|
||||
STATUS_FAILED,
|
||||
SourceImporter,
|
||||
run_import,
|
||||
)
|
||||
from app.services.data_import import STATUS_FAILED, SourceImporter, run_import
|
||||
from app.services.dolt_earnings_importer import DoltEarningsImporter
|
||||
from app.services.sec_fundamentals_importer import SecFundamentalsImporter
|
||||
from app.services.alert_service import dispatch_alerts
|
||||
@@ -61,6 +66,7 @@ from app.services.event_study_service import run_and_store as run_event_study_an
|
||||
from app.services.outcome_service import evaluate_pending_setups
|
||||
from app.services.rr_scanner_service import scan_all_tickers
|
||||
from app.services.sentiment_provider_service import build_sentiment_provider
|
||||
from app.services import ticker_service
|
||||
from app.services.ticker_universe_service import bootstrap_universe
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -82,36 +88,58 @@ scheduler = AsyncIOScheduler(
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _on_job_finished(event: object) -> None:
|
||||
"""Persist the run, then re-pause the job if it only runs on demand.
|
||||
|
||||
Covers every job APScheduler fires itself, including manual triggers.
|
||||
Pipeline *steps* are invoked as plain coroutines and emit no events, so
|
||||
``_run_pipeline`` persists those directly.
|
||||
"""
|
||||
job_id = getattr(event, "job_id", None)
|
||||
if job_id:
|
||||
_schedule_persist(job_id)
|
||||
_repause_after_manual_run(event)
|
||||
|
||||
|
||||
def _repause_after_manual_run(event: object) -> None:
|
||||
"""Re-pause a job that only ever runs on demand, once its run finishes.
|
||||
|
||||
Pipeline steps and manual jobs are registered with a 520-week interval and
|
||||
``next_run_time=None`` as a backstop. Triggering one sets next_run_time=now,
|
||||
and APScheduler then re-arms that backstop -- so Admin → Jobs would show a
|
||||
"next run" ten years out. Guarding on category means the six cron jobs and
|
||||
the real interval jobs are never touched.
|
||||
|
||||
Registered at module level, not inside ``configure_scheduler``: that function
|
||||
is called more than once (idempotency test) and ``add_listener`` does not
|
||||
deduplicate.
|
||||
"""
|
||||
job_id = getattr(event, "job_id", None)
|
||||
if job_catalog.JOB_CATEGORY.get(job_id) not in (
|
||||
job_catalog.CATEGORY_STEP,
|
||||
job_catalog.CATEGORY_MANUAL,
|
||||
):
|
||||
return
|
||||
try:
|
||||
scheduler.modify_job(job_id, next_run_time=None)
|
||||
except Exception: # job gone, scheduler stopped — nothing to re-pause
|
||||
logger.debug("Could not re-pause %s after its run", job_id, exc_info=True)
|
||||
|
||||
|
||||
scheduler.add_listener(_on_job_finished, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
|
||||
|
||||
# Track last successful ticker per job for rate-limit resume
|
||||
_last_successful: dict[str, str | None] = {
|
||||
"data_collector": None,
|
||||
"data_backfill": None,
|
||||
"sentiment_collector": None,
|
||||
"fundamental_collector": None,
|
||||
}
|
||||
|
||||
# Jobs whose per-run progress is surfaced to Admin → Jobs. (outcome_evaluator is
|
||||
# created lazily on first run via _runtime_start.)
|
||||
_JOB_NAMES = [
|
||||
"data_collector",
|
||||
"data_backfill",
|
||||
"sentiment_collector",
|
||||
"fundamental_collector",
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"fundamentals_parity_report",
|
||||
"rr_scanner",
|
||||
"ticker_universe_sync",
|
||||
"alerts",
|
||||
"market_regime",
|
||||
"regime_monitor",
|
||||
"event_study",
|
||||
"backtest",
|
||||
"daily_pipeline", # morning: OHLCV/sentiment/regime — no qualifying scan
|
||||
"near_close_pipeline", # OHLCV fetch → R:R scan → Telegram alerts
|
||||
"after_close_pipeline", # OHLCV fetch → outcome eval (final bar)
|
||||
"intraday_pipeline",
|
||||
]
|
||||
# Seeded from the catalog rather than a private list. The old literal held 16 of
|
||||
# the 19 jobs -- benchmark_collector, outcome_evaluator and shadow_book were
|
||||
# missing, so they had no runtime row (and so no "last run" line in Admin → Jobs)
|
||||
# until their first run in a given process.
|
||||
|
||||
|
||||
def _idle_runtime() -> dict[str, object]:
|
||||
@@ -128,7 +156,9 @@ def _idle_runtime() -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
_job_runtime: dict[str, dict[str, object]] = {name: _idle_runtime() for name in _JOB_NAMES}
|
||||
_job_runtime: dict[str, dict[str, object]] = {
|
||||
name: _idle_runtime() for name in sorted(job_catalog.VALID_JOB_NAMES)
|
||||
}
|
||||
_next_backtest_target_model = PRODUCTION_GTL_TARGET_MODEL
|
||||
_next_backtest_cadence = DEFAULT_BACKTEST_CADENCE
|
||||
|
||||
@@ -255,7 +285,14 @@ def _runtime_finish(
|
||||
processed: int,
|
||||
total: int | None,
|
||||
message: str | None = None,
|
||||
emit_event: bool = True,
|
||||
) -> None:
|
||||
"""Finalize a job's runtime row, optionally raising a durable event.
|
||||
|
||||
``emit_event=False`` is for a *re-finalize* that only rewords an outcome an
|
||||
earlier call already reported. The dedup key includes the message, so a
|
||||
reworded error would otherwise land in Admin → System Events twice.
|
||||
"""
|
||||
runtime = _job_runtime.get(job_name, {})
|
||||
runtime.update({
|
||||
"running": False,
|
||||
@@ -269,7 +306,7 @@ def _runtime_finish(
|
||||
})
|
||||
_job_runtime[job_name] = runtime
|
||||
# Durable event for error / rate-limit finishes (badge + Admin → Jobs panel).
|
||||
if status in ("error", "rate_limited"):
|
||||
if emit_event and status in ("error", "rate_limited"):
|
||||
severity = "error" if status == "error" else "warning"
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -286,6 +323,67 @@ def _runtime_finish(
|
||||
pass
|
||||
|
||||
|
||||
async def _persist_job_run(job_name: str) -> None:
|
||||
"""Write a job's finished runtime row to the durable last-run table.
|
||||
|
||||
Never raises: a persistence failure must not break the pipeline that was
|
||||
otherwise successful. The in-memory row stays authoritative for live state.
|
||||
"""
|
||||
runtime = _job_runtime.get(job_name)
|
||||
if not runtime or runtime.get("running") or not runtime.get("finished_at"):
|
||||
return
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
await job_run_store.record_finish(db, job_name, runtime)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
logger.exception("Could not persist last-run state for %s", job_name)
|
||||
|
||||
|
||||
# Detached persists are kept referenced: a bare create_task result can be
|
||||
# garbage-collected mid-flight, and the shutdown drain needs something to await.
|
||||
_persist_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def _schedule_persist(job_name: str) -> None:
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(_persist_job_run(job_name))
|
||||
except RuntimeError: # no loop (sync context / tests) — nothing to persist
|
||||
return
|
||||
_persist_tasks.add(task)
|
||||
task.add_done_callback(_persist_tasks.discard)
|
||||
|
||||
|
||||
async def flush_job_run_persists(timeout: float = 5.0, settle: float = 0.05) -> None:
|
||||
"""Drain last-run writes, including ones queued while we are draining.
|
||||
|
||||
``scheduler.shutdown(wait=False)`` returns before APScheduler has dispatched
|
||||
its job-completion events, and those events are what create persist tasks. A
|
||||
single snapshot of the set therefore misses writes still to be queued, and
|
||||
``engine.dispose()`` could then close the pool underneath them. So: give the
|
||||
loop a moment for pending callbacks to land, then keep draining until the
|
||||
set stays empty or the deadline passes.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
# Bounded settle so callbacks dispatched by shutdown get to queue their work
|
||||
# before the first emptiness check decides there is nothing to wait for.
|
||||
await asyncio.sleep(min(settle, timeout))
|
||||
while True:
|
||||
pending = {task for task in _persist_tasks if not task.done()}
|
||||
if not pending:
|
||||
return
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
"Timed out draining %d last-run write(s); some may be lost", len(pending)
|
||||
)
|
||||
return
|
||||
await asyncio.wait(pending, timeout=remaining)
|
||||
# Loop rather than return: a completion callback may have queued another.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
def get_job_runtime_snapshot(job_name: str | None = None) -> dict[str, dict[str, object]] | dict[str, object]:
|
||||
if job_name is not None:
|
||||
return dict(_job_runtime.get(job_name, {}))
|
||||
@@ -299,8 +397,10 @@ async def _is_job_enabled(db: AsyncSession, job_name: str) -> bool:
|
||||
|
||||
|
||||
async def _get_all_tickers(db: AsyncSession) -> list[str]:
|
||||
"""Return all tracked ticker symbols sorted alphabetically."""
|
||||
result = await db.execute(select(Ticker.symbol).order_by(Ticker.symbol))
|
||||
"""Return all actively-traded ticker symbols sorted alphabetically."""
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(select(Ticker.symbol).order_by(Ticker.symbol))
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
@@ -315,8 +415,10 @@ async def _get_ohlcv_priority_tickers(db: AsyncSession) -> list[str]:
|
||||
latest_date = func.max(OHLCVRecord.date)
|
||||
missing_first = case((latest_date.is_(None), 0), else_=1)
|
||||
result = await db.execute(
|
||||
select(Ticker.symbol)
|
||||
.outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id)
|
||||
ticker_service.active_only(
|
||||
select(Ticker.symbol)
|
||||
.outerjoin(OHLCVRecord, OHLCVRecord.ticker_id == Ticker.id)
|
||||
)
|
||||
.group_by(Ticker.id, Ticker.symbol)
|
||||
.order_by(missing_first.asc(), latest_date.asc(), Ticker.symbol.asc())
|
||||
)
|
||||
@@ -460,23 +562,6 @@ async def _get_sentiment_priority_tickers(db: AsyncSession) -> list[str]:
|
||||
return priority_syms + filler_syms
|
||||
|
||||
|
||||
async def _get_fundamental_priority_tickers(db: AsyncSession) -> list[str]:
|
||||
"""Return symbols prioritized for fundamentals refresh.
|
||||
|
||||
Priority:
|
||||
1) Tickers with no fundamentals snapshot yet
|
||||
2) Tickers with existing fundamentals, oldest fetched_at first
|
||||
3) Alphabetical tiebreaker
|
||||
"""
|
||||
missing_first = case((FundamentalData.fetched_at.is_(None), 0), else_=1)
|
||||
result = await db.execute(
|
||||
select(Ticker.symbol)
|
||||
.outerjoin(FundamentalData, FundamentalData.ticker_id == Ticker.id)
|
||||
.order_by(missing_first.asc(), FundamentalData.fetched_at.asc(), Ticker.symbol.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
def _resume_tickers(symbols: list[str], job_name: str) -> list[str]:
|
||||
"""Reorder tickers to resume after the last successful one (rate-limit resume).
|
||||
|
||||
@@ -506,13 +591,14 @@ async def collect_ohlcv(
|
||||
job_name: str = "data_collector",
|
||||
*,
|
||||
refetch_days: int = 0,
|
||||
refresh_sr: bool = True,
|
||||
) -> None:
|
||||
"""Fetch latest daily OHLCV for all tracked tickers.
|
||||
|
||||
Uses AlpacaOHLCVProvider. Processes each ticker independently.
|
||||
On rate limit, records last successful ticker for resume.
|
||||
Start date is resolved by ingestion progress:
|
||||
- existing ticker: resume from last_ingested_date + 1
|
||||
- existing ticker: overlap last_ingested_date so partial bars refresh
|
||||
- new ticker: backfill the configured history window
|
||||
|
||||
``full_backfill`` forces every ticker to re-fetch the full
|
||||
@@ -574,20 +660,41 @@ async def collect_ohlcv(
|
||||
try:
|
||||
result = await ingestion_service.fetch_and_ingest(
|
||||
db, provider, symbol, start_date=backfill_start, end_date=end_date,
|
||||
refresh_sr=refresh_sr,
|
||||
)
|
||||
_last_successful[job_name] = symbol
|
||||
processed += 1
|
||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
||||
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested)
|
||||
if result.status == "stale":
|
||||
await _record_system_event(
|
||||
severity="warning",
|
||||
source=job_name,
|
||||
code="ohlcv_stale",
|
||||
message=result.message or f"No new OHLCV bars for {symbol}",
|
||||
symbol=symbol,
|
||||
dedup_key=f"ohlcv_stale:{symbol}",
|
||||
# "No new bars" cannot distinguish a delisting from a halt
|
||||
# or a rename, so ask SEC before warning again. A confirmed
|
||||
# delisting retires the symbol (keeping its history) and
|
||||
# ends the alert; anything unproven keeps warning.
|
||||
delisted_on = await ticker_service.confirm_delisting(
|
||||
db, symbol, last_bar=result.last_date
|
||||
)
|
||||
if delisted_on is not None:
|
||||
await _record_system_event(
|
||||
severity="info",
|
||||
source=job_name,
|
||||
code="ticker_delisted",
|
||||
message=(
|
||||
f"{symbol} delisted on {delisted_on} (SEC Form 25/15). "
|
||||
"Retired from signals; price history retained."
|
||||
),
|
||||
symbol=symbol,
|
||||
dedup_key=f"ticker_delisted:{symbol}",
|
||||
)
|
||||
else:
|
||||
await _record_system_event(
|
||||
severity="warning",
|
||||
source=job_name,
|
||||
code="ohlcv_stale",
|
||||
message=result.message or f"No new OHLCV bars for {symbol}",
|
||||
symbol=symbol,
|
||||
dedup_key=f"ohlcv_stale:{symbol}",
|
||||
)
|
||||
if result.status == "partial":
|
||||
# Rate limited — stop and resume next run
|
||||
_log_event(logging.WARNING, "rate_limited", job=job_name, ticker=symbol, processed=processed)
|
||||
@@ -613,6 +720,11 @@ async def collect_ohlcv(
|
||||
_runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
|
||||
|
||||
|
||||
async def collect_ohlcv_for_scan() -> None:
|
||||
"""Near-close fetch; the scanner immediately rebuilds S/R per ticker."""
|
||||
await collect_ohlcv(refresh_sr=False)
|
||||
|
||||
|
||||
async def backfill_ohlcv() -> None:
|
||||
"""Deep historical backfill: re-fetch the full ``settings.ohlcv_history_days``
|
||||
window for every ticker, ignoring incremental resume.
|
||||
@@ -652,7 +764,7 @@ async def run_shadow_book() -> None:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
||||
return
|
||||
return False
|
||||
if not await shadow_book_service.is_enabled(db):
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="not enabled in settings")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Not enabled")
|
||||
@@ -803,129 +915,17 @@ async def collect_sentiment() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job: Fundamental Collector
|
||||
# Jobs: bulk fundamentals source imports
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def collect_fundamentals() -> None:
|
||||
"""Fetch fundamentals for all tracked tickers via FMP.
|
||||
async def _run_source_import(job_name: str, importer: SourceImporter) -> bool:
|
||||
"""Run an importer and return whether its scheduled job was enabled.
|
||||
|
||||
Processes each ticker independently. On rate limit, records last
|
||||
successful ticker for resume.
|
||||
The SEC wrapper uses the return value only to word its runtime message: its
|
||||
local cache step runs after deferred, failed, no-op, promoted, source-locked
|
||||
and disabled attempts alike.
|
||||
"""
|
||||
job_name = "fundamental_collector"
|
||||
_log_event(logging.INFO, "job_start", job=job_name)
|
||||
_runtime_start(job_name)
|
||||
processed = 0
|
||||
total: int | None = None
|
||||
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
|
||||
return
|
||||
|
||||
symbols = await _get_fundamental_priority_tickers(db)
|
||||
if not symbols:
|
||||
_log_event(logging.INFO, "job_complete", job=job_name, tickers=0)
|
||||
_runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers")
|
||||
return
|
||||
|
||||
total = len(symbols)
|
||||
_runtime_progress(job_name, processed=0, total=total)
|
||||
|
||||
if not (settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key):
|
||||
_log_event(logging.WARNING, "job_skipped", job=job_name, reason="no fundamentals provider keys configured")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=total, message="No fundamentals provider keys configured")
|
||||
return
|
||||
|
||||
try:
|
||||
provider = build_fundamental_provider_chain()
|
||||
except Exception as exc:
|
||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
||||
_runtime_finish(job_name, "error", processed=0, total=total, message=str(exc))
|
||||
return
|
||||
|
||||
max_retries = max(0, settings.fundamental_rate_limit_retries)
|
||||
base_backoff = max(1, settings.fundamental_rate_limit_backoff_seconds)
|
||||
spacing = max(0.0, settings.fundamental_request_spacing_seconds)
|
||||
|
||||
async def _store(symbol: str, data) -> None:
|
||||
async with async_session_factory() as db:
|
||||
await fundamental_service.store_fundamental(
|
||||
db,
|
||||
symbol=symbol,
|
||||
pe_ratio=data.pe_ratio,
|
||||
revenue_growth=data.revenue_growth,
|
||||
earnings_surprise=data.earnings_surprise,
|
||||
market_cap=data.market_cap,
|
||||
next_earnings_date=data.next_earnings_date,
|
||||
unavailable_fields=data.unavailable_fields,
|
||||
)
|
||||
|
||||
for symbol in symbols:
|
||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
data = await provider.fetch_fundamentals(symbol)
|
||||
await _store(symbol, data)
|
||||
_last_successful[job_name] = symbol
|
||||
processed += 1
|
||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
||||
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol)
|
||||
break
|
||||
except Exception as exc:
|
||||
msg = str(exc).lower()
|
||||
if "rate" in msg or "429" in msg:
|
||||
if attempt < max_retries:
|
||||
wait_seconds = base_backoff * (2 ** attempt)
|
||||
attempt += 1
|
||||
_log_event(logging.WARNING, "rate_limited_retry", job=job_name, ticker=symbol, attempt=attempt, max_retries=max_retries, wait_seconds=wait_seconds, processed=processed)
|
||||
_runtime_progress(
|
||||
job_name,
|
||||
processed=processed,
|
||||
total=total,
|
||||
current_ticker=symbol,
|
||||
message=f"Rate-limited at {symbol}; retry {attempt}/{max_retries} in {wait_seconds}s",
|
||||
)
|
||||
await asyncio.sleep(wait_seconds)
|
||||
continue
|
||||
|
||||
# Retries exhausted: store whatever partial data we can
|
||||
# still get (e.g. FMP market cap) and move on, rather than
|
||||
# aborting the whole run and leaving every later ticker
|
||||
# untouched.
|
||||
_log_event(logging.WARNING, "rate_limited_partial", job=job_name, ticker=symbol, processed=processed)
|
||||
try:
|
||||
data = await provider.fetch_fundamentals(symbol, allow_partial=True)
|
||||
await _store(symbol, data)
|
||||
processed += 1
|
||||
except Exception as exc2:
|
||||
_log_job_error(job_name, symbol, exc2)
|
||||
break
|
||||
_log_job_error(job_name, symbol, exc)
|
||||
break
|
||||
|
||||
if spacing:
|
||||
await asyncio.sleep(spacing)
|
||||
|
||||
_last_successful[job_name] = None
|
||||
_log_event(logging.INFO, "job_complete", job=job_name, tickers=processed)
|
||||
_runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers")
|
||||
except Exception as exc:
|
||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
||||
_runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Jobs: shadow fundamentals sources
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
||||
"""Run one source importer and surface its audit result in Admin → Jobs."""
|
||||
_log_event(logging.INFO, "job_start", job=job_name)
|
||||
_runtime_start(job_name, total=1)
|
||||
|
||||
@@ -934,22 +934,27 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
||||
return
|
||||
return False
|
||||
|
||||
run = await run_import(importer)
|
||||
if run is None:
|
||||
message = "Another import for this source is already running"
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="source_locked")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message=message)
|
||||
return
|
||||
return True
|
||||
|
||||
revision = f" · {run.revision[:12]}" if run.revision else ""
|
||||
message = f"{run.status}{revision}"
|
||||
if run.status == STATUS_DEFERRED:
|
||||
message = run.error_details or message
|
||||
_log_event(logging.INFO, "job_deferred", job=job_name, message=message)
|
||||
_runtime_finish(job_name, "deferred", processed=0, total=1, message=message)
|
||||
return True
|
||||
if run.status == STATUS_FAILED:
|
||||
message = run.error_details or message
|
||||
_log_event(logging.ERROR, "job_error", job=job_name, message=message)
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
||||
return
|
||||
return True
|
||||
|
||||
_log_event(
|
||||
logging.INFO,
|
||||
@@ -959,6 +964,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
||||
revision=run.revision,
|
||||
)
|
||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
||||
return True
|
||||
except asyncio.CancelledError:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
||||
raise
|
||||
@@ -971,59 +977,79 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
||||
message=str(exc),
|
||||
)
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
||||
return True
|
||||
|
||||
|
||||
async def run_dolt_earnings_import() -> None:
|
||||
"""Pull and import the Dolt earnings calendar/results feed in shadow."""
|
||||
await _run_shadow_import("dolt_earnings_import", DoltEarningsImporter())
|
||||
"""Pull and import the Dolt earnings calendar/results feed."""
|
||||
await _run_source_import("dolt_earnings_import", DoltEarningsImporter())
|
||||
|
||||
|
||||
async def run_sec_fundamentals_import() -> None:
|
||||
"""Import tracked-universe SEC facts in shadow."""
|
||||
await _run_shadow_import("sec_fundamentals_import", SecFundamentalsImporter())
|
||||
"""Import SEC facts, then refresh the local compat cache.
|
||||
|
||||
The refresh is deliberately independent of the network import: it reads only
|
||||
stored snapshots, earnings events and closes, so it runs identically when SEC
|
||||
is unavailable, unchanged, or owned by another import — and also when the
|
||||
job's ingestion is switched off in Admin → Jobs. Disabling the job stops
|
||||
SEC network access, not the cache; prices and earnings move daily even when
|
||||
no filing does, and `fundamental_data` feeds scoring.
|
||||
"""
|
||||
job_name = "sec_fundamentals_import"
|
||||
import_ran = await _run_source_import(job_name, SecFundamentalsImporter())
|
||||
|
||||
async def run_fundamentals_parity_report() -> None:
|
||||
"""Generate the A5 comparison bundle without mutating live fundamentals/scores."""
|
||||
job_name = "fundamentals_parity_report"
|
||||
_log_event(logging.INFO, "job_start", job=job_name)
|
||||
_runtime_start(job_name, total=1)
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_runtime_finish(
|
||||
job_name, "skipped", processed=0, total=1, message="Disabled"
|
||||
)
|
||||
return
|
||||
report, artifacts = await fundamentals_parity_service.generate_and_store(
|
||||
db, settings.fundamentals_parity_report_dir
|
||||
)
|
||||
summary = report["summary"]
|
||||
message = (
|
||||
f"{summary['universe_count']} tickers · "
|
||||
f"{summary['fundamental_score_material_changes']} material score changes"
|
||||
)
|
||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
||||
_log_event(
|
||||
logging.INFO,
|
||||
"job_complete",
|
||||
job=job_name,
|
||||
generated_at=report["generated_at"],
|
||||
json_path=artifacts["json"],
|
||||
csv_path=artifacts["csv"],
|
||||
)
|
||||
summary = await fundamental_data_refresh_service.refresh(db)
|
||||
except asyncio.CancelledError:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
||||
_runtime_finish(
|
||||
job_name, "error", processed=0, total=1, message="Cancelled"
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
||||
message = f"Local fundamental_data refresh failed: {exc}"
|
||||
_log_event(
|
||||
logging.ERROR,
|
||||
"job_error",
|
||||
"fundamental_data_refresh_error",
|
||||
job=job_name,
|
||||
error_type=type(exc).__name__,
|
||||
message=str(exc),
|
||||
)
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
||||
return
|
||||
|
||||
_log_event(
|
||||
logging.INFO,
|
||||
"fundamental_data_refresh_complete",
|
||||
job=job_name,
|
||||
**summary,
|
||||
)
|
||||
cache_message = (
|
||||
f"cache {summary['refreshed']} · "
|
||||
f"{summary['score_inputs_changed']} score inputs changed"
|
||||
)
|
||||
# Every outcome carries the cache summary — including deferred, failed and
|
||||
# source-locked ones. The import status is what varies; the refresh always
|
||||
# happened, and Admin → Jobs is the only place an operator sees that.
|
||||
#
|
||||
# This only rewords what _run_source_import already finalized, so it must not
|
||||
# emit a second durable event: the dedup key includes the message, and a
|
||||
# failure would otherwise show up twice in Admin → System Events.
|
||||
runtime = get_job_runtime_snapshot(job_name)
|
||||
if import_ran:
|
||||
status = str(runtime.get("status") or "completed")
|
||||
import_message = runtime.get("message") or "import completed"
|
||||
processed = 1 if status == "completed" else 0
|
||||
else:
|
||||
status, import_message, processed = "completed", "Import disabled", 1
|
||||
_runtime_finish(
|
||||
job_name,
|
||||
status,
|
||||
processed=processed,
|
||||
total=1,
|
||||
message=f"{import_message} · {cache_message}",
|
||||
emit_event=False,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1147,7 +1173,7 @@ async def dispatch_alerts_job() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job: Market Regime
|
||||
# Job: Market Trend (SPY)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1206,7 +1232,7 @@ async def collect_benchmark() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job: Regime Monitor
|
||||
# Job: AI/Tech Risk Monitor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1325,8 +1351,11 @@ async def run_event_study_job() -> None:
|
||||
report = await run_event_study_and_store(db)
|
||||
|
||||
_runtime_progress(job_name, processed=1, total=1)
|
||||
shipped = report.get("shipped") or {}
|
||||
if report.get("available"):
|
||||
metrics = report.get("metrics") or {}
|
||||
# The shipped quadrant rule is the headline; the fitted-threshold
|
||||
# variant lives under report["fitted"] and is not what fires.
|
||||
metrics = shipped.get("metrics") or {}
|
||||
msg = (
|
||||
f"{metrics.get('events_warned', 0)}/{metrics.get('events', 0)} warned, "
|
||||
f"{metrics.get('false_alarms_per_year', 0)} false alarms/year"
|
||||
@@ -1334,7 +1363,10 @@ async def run_event_study_job() -> None:
|
||||
else:
|
||||
msg = report.get("reason", "no data")
|
||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=msg)
|
||||
_log_event(logging.INFO, "job_complete", job=job_name, events=len(report.get("events", [])))
|
||||
_log_event(
|
||||
logging.INFO, "job_complete", job=job_name,
|
||||
events=len(shipped.get("events") or []),
|
||||
)
|
||||
except Exception as exc:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
||||
@@ -1390,54 +1422,14 @@ async def sync_ticker_universe() -> None:
|
||||
# the intraday partial one (covers a long weekend / holiday gap).
|
||||
_FINAL_REFETCH_DAYS = 5
|
||||
|
||||
_DAILY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("benchmark_collector", "collect_benchmark"),
|
||||
("sentiment_collector", "collect_sentiment"),
|
||||
("market_regime", "compute_market_regime"),
|
||||
# Observational only — display/alerts; not trade selection.
|
||||
("regime_monitor", "compute_regime_monitor"),
|
||||
# Alerts after regime so quadrant changes reach Telegram in the morning.
|
||||
# Dispatcher is change-driven; quiet days stay quiet. Setup alerts still
|
||||
# fire on the near-close pipeline after the qualifying scan.
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# Near-close (~15:30 ET Mon–Fri): refresh in-progress day-t bars (already how
|
||||
# the intraday pipeline keeps the dashboard live), then the only daily
|
||||
# qualifying R:R scan, then Telegram immediately so manual fills can still hit
|
||||
# MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan
|
||||
# may see ~15:15 prices — immaterial for a 12-1 momentum signal.
|
||||
#
|
||||
# US early-close days (~3/year, 13:00 ET close): this job runs post-close and
|
||||
# entries behave like stale_close (still acceptable per execution-recovery matrix).
|
||||
# No exchange calendar dependency.
|
||||
_NEAR_CLOSE_PIPELINE_STEPS = [
|
||||
# Must land today's in-progress bar (~20 min behind live), or the scan falls
|
||||
# back to the previous close and execution degrades to the stale_close floor.
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("rr_scanner", "scan_rr"),
|
||||
# Straight after the scan so shadow entries mark at the same near-close
|
||||
# prices the discretionary book is looking at.
|
||||
("shadow_book", "run_shadow_book"),
|
||||
("alerts", "dispatch_alerts_job"),
|
||||
]
|
||||
|
||||
# After close (~16:45 ET Mon–Fri): fresh OHLCV fetch so outcomes resolve on the
|
||||
# final bar, not the near-close partial bar, then outcome/paper close.
|
||||
_AFTER_CLOSE_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv_final"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
|
||||
# Intraday (light): keep prices current and resolve outcomes through the day,
|
||||
# without the expensive scan/sentiment. The dashboard recomputes live R:R from
|
||||
# the latest price, so refreshing OHLCV is enough to stop prices lagging; the
|
||||
# outcome step also closes paper trades that hit their stop/target intraday.
|
||||
_INTRADAY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("outcome_evaluator", "evaluate_outcomes"),
|
||||
]
|
||||
# Step lists live in app.job_catalog so the runner, the admin API's pipeline
|
||||
# membership and the UI's grouping all read one definition. Re-exported here
|
||||
# under their original names: _run_pipeline and the scheduler_configured log
|
||||
# payload refer to them directly.
|
||||
_DAILY_PIPELINE_STEPS = job_catalog._DAILY_PIPELINE_STEPS
|
||||
_NEAR_CLOSE_PIPELINE_STEPS = job_catalog._NEAR_CLOSE_PIPELINE_STEPS
|
||||
_AFTER_CLOSE_PIPELINE_STEPS = job_catalog._AFTER_CLOSE_PIPELINE_STEPS
|
||||
_INTRADAY_PIPELINE_STEPS = job_catalog._INTRADAY_PIPELINE_STEPS
|
||||
|
||||
# Warn if near-close fetch+scan+alert drifts past this — entries leave the close
|
||||
# and the stale_close floor quietly becomes the ceiling.
|
||||
@@ -1460,6 +1452,7 @@ async def _run_pipeline(job_name: str, steps: list[tuple[str, str]]) -> None:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||
_runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
|
||||
await _persist_job_run(job_name)
|
||||
return
|
||||
|
||||
total = len(steps)
|
||||
@@ -1475,6 +1468,11 @@ async def _run_pipeline(job_name: str, steps: list[tuple[str, str]]) -> None:
|
||||
await funcs[func_name]()
|
||||
except Exception:
|
||||
logger.exception("%s step %s failed", job_name, step_name)
|
||||
# Outside the except on purpose: the step's own _runtime_finish has
|
||||
# already recorded its outcome, so persisting here captures failures
|
||||
# too. Steps are plain coroutine calls and fire no scheduler events,
|
||||
# so the listener cannot see them -- this is their only write path.
|
||||
await _persist_job_run(step_name)
|
||||
done += 1
|
||||
_runtime_finish(job_name, "completed", processed=done, total=total, message="Pipeline complete")
|
||||
_log_event(logging.INFO, "job_complete", job=job_name)
|
||||
@@ -1483,10 +1481,11 @@ async def _run_pipeline(job_name: str, steps: list[tuple[str, str]]) -> None:
|
||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
||||
finally:
|
||||
pipeline_run.release(token)
|
||||
await _persist_job_run(job_name)
|
||||
|
||||
|
||||
async def run_daily_pipeline() -> None:
|
||||
"""Morning flow: OHLCV → benchmark → sentiment → market regime (no scan)."""
|
||||
"""Morning flow: OHLCV → benchmark → sentiment → trend/risk (no scan)."""
|
||||
await _run_pipeline("daily_pipeline", _DAILY_PIPELINE_STEPS)
|
||||
|
||||
|
||||
@@ -1566,18 +1565,22 @@ SCHEDULE_DEFAULTS: dict[str, str] = {
|
||||
"schedule_timezone": "America/New_York",
|
||||
# Morning data/display refresh (no qualifying R:R scan).
|
||||
"schedule_daily_pipeline_cron": "0 2 * * *",
|
||||
# Shadow source imports. They never write legacy fundamental_data before A5.
|
||||
# Bulk source imports. The SEC job also refreshes the fundamental_data compat
|
||||
# cache that scoring reads — locally, from stored snapshots/earnings/closes.
|
||||
"schedule_dolt_earnings_cron": "30 2 * * *",
|
||||
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
||||
"schedule_fundamentals_parity_cron": "30 5 * * *",
|
||||
# Fetch in-progress bars → scan → Telegram (manual MOC window).
|
||||
"schedule_near_close_pipeline_cron": "30 15 * * mon-fri",
|
||||
# Fetch final bars → outcome eval (must not run on the partial near-close bar).
|
||||
"schedule_after_close_pipeline_cron": "45 16 * * mon-fri",
|
||||
# Hourly mid-session price + outcome (10:00–15:00 ET Mon–Fri).
|
||||
"schedule_intraday_pipeline_cron": "0 10-15 * * mon-fri",
|
||||
# Weekly fundamentals early Monday NY.
|
||||
"schedule_fundamentals_cron": "0 1 * * mon",
|
||||
# Both were interval jobs until 2026-08-08 and hit exactly the pitfall
|
||||
# described above: configure_scheduler calls remove_all_jobs() on every
|
||||
# startup, so an interval countdown restarts from zero each deploy. A 168h
|
||||
# backtest needed a week of uninterrupted uptime to fire even once.
|
||||
"schedule_backtest_cron": "0 3 * * sun",
|
||||
"schedule_ticker_universe_cron": "0 1 * * *",
|
||||
}
|
||||
|
||||
# job id -> schedule setting key
|
||||
@@ -1585,11 +1588,11 @@ _CRON_JOBS: dict[str, str] = {
|
||||
"daily_pipeline": "schedule_daily_pipeline_cron",
|
||||
"dolt_earnings_import": "schedule_dolt_earnings_cron",
|
||||
"sec_fundamentals_import": "schedule_sec_fundamentals_cron",
|
||||
"fundamentals_parity_report": "schedule_fundamentals_parity_cron",
|
||||
"near_close_pipeline": "schedule_near_close_pipeline_cron",
|
||||
"after_close_pipeline": "schedule_after_close_pipeline_cron",
|
||||
"intraday_pipeline": "schedule_intraday_pipeline_cron",
|
||||
"fundamental_collector": "schedule_fundamentals_cron",
|
||||
"backtest": "schedule_backtest_cron",
|
||||
"ticker_universe_sync": "schedule_ticker_universe_cron",
|
||||
}
|
||||
|
||||
|
||||
@@ -1655,8 +1658,12 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
(scan_rr, "rr_scanner", "R:R Scanner"),
|
||||
(run_shadow_book, "shadow_book", "Shadow Book (auto-traded strategy)"),
|
||||
(evaluate_outcomes, "outcome_evaluator", "Outcome Evaluator"),
|
||||
(compute_market_regime, "market_regime", "Market Regime"),
|
||||
(compute_regime_monitor, "regime_monitor", "Regime Monitor"),
|
||||
# Labels only -- the ids are persisted (pipeline steps, cron config, run
|
||||
# history), so they stay. "Market Regime"/"Regime Monitor" read as the
|
||||
# same job and had it backwards besides: the SPY guard is the one that
|
||||
# changes what a setup shows, while the monitor is observational.
|
||||
(compute_market_regime, "market_regime", "Market Trend (SPY)"),
|
||||
(compute_regime_monitor, "regime_monitor", "AI/Tech Risk Monitor"),
|
||||
]
|
||||
for fn, job_id, job_name in _members:
|
||||
scheduler.add_job(
|
||||
@@ -1678,7 +1685,7 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
"schedule_dolt_earnings_cron",
|
||||
),
|
||||
id="dolt_earnings_import",
|
||||
name="Dolt Earnings Import (shadow)",
|
||||
name="Dolt Earnings Import",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
@@ -1689,18 +1696,7 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
"schedule_sec_fundamentals_cron",
|
||||
),
|
||||
id="sec_fundamentals_import",
|
||||
name="SEC Fundamentals Import (shadow)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_fundamentals_parity_report,
|
||||
_cron_trigger(
|
||||
cfg["schedule_fundamentals_parity_cron"],
|
||||
tz,
|
||||
"schedule_fundamentals_parity_cron",
|
||||
),
|
||||
id="fundamentals_parity_report",
|
||||
name="Fundamentals Parity Report (read-only)",
|
||||
name="SEC Fundamentals Import",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
@@ -1730,17 +1726,13 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
_cron_trigger(cfg["schedule_intraday_pipeline_cron"], tz, "schedule_intraday_pipeline_cron"),
|
||||
id="intraday_pipeline", name="Intraday Pipeline", replace_existing=True,
|
||||
)
|
||||
# Fundamentals — quarterly-ish data; weekly by default (conserves API quota).
|
||||
# Its own early cron so the slow, rate-limited fetch finishes before the day.
|
||||
scheduler.add_job(
|
||||
collect_fundamentals,
|
||||
_cron_trigger(cfg["schedule_fundamentals_cron"], tz, "schedule_fundamentals_cron"),
|
||||
id="fundamental_collector", name="Fundamental Collector", replace_existing=True,
|
||||
)
|
||||
|
||||
# Independent interval jobs (own cadence, no ordering dependency)
|
||||
# Independent jobs (own cadence, no ordering dependency). Cron, not interval,
|
||||
# for the reason documented at SCHEDULE_DEFAULTS: an interval countdown
|
||||
# restarts on every deploy, so these could be deferred indefinitely.
|
||||
scheduler.add_job(
|
||||
sync_ticker_universe, "interval", hours=24,
|
||||
sync_ticker_universe,
|
||||
_cron_trigger(cfg["schedule_ticker_universe_cron"], tz, "schedule_ticker_universe_cron"),
|
||||
id="ticker_universe_sync", name="Ticker Universe Sync", replace_existing=True,
|
||||
)
|
||||
# Alerts auto-fire only via near_close_pipeline (scan → alert before MOC).
|
||||
@@ -1751,7 +1743,8 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
replace_existing=True, next_run_time=None,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_backtest_job, "interval", hours=168,
|
||||
run_backtest_job,
|
||||
_cron_trigger(cfg["schedule_backtest_cron"], tz, "schedule_backtest_cron"),
|
||||
id="backtest", name="Backtest", replace_existing=True,
|
||||
)
|
||||
# Deep history backfill: manual only (never auto-fires); triggered from
|
||||
@@ -1778,9 +1771,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
},
|
||||
dolt_earnings_import={"cron": cfg["schedule_dolt_earnings_cron"]},
|
||||
sec_fundamentals_import={"cron": cfg["schedule_sec_fundamentals_cron"]},
|
||||
fundamentals_parity_report={
|
||||
"cron": cfg["schedule_fundamentals_parity_cron"]
|
||||
},
|
||||
near_close_pipeline={
|
||||
"cron": cfg["schedule_near_close_pipeline_cron"],
|
||||
"steps": [name for name, _ in _NEAR_CLOSE_PIPELINE_STEPS],
|
||||
@@ -1793,7 +1783,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
"cron": cfg["schedule_intraday_pipeline_cron"],
|
||||
"steps": [name for name, _ in _INTRADAY_PIPELINE_STEPS],
|
||||
},
|
||||
fundamental_collector={"cron": cfg["schedule_fundamentals_cron"]},
|
||||
independent=["ticker_universe_sync", "backtest"],
|
||||
manual_only=["alerts", "data_backfill", "event_study"],
|
||||
)
|
||||
|
||||
@@ -80,11 +80,11 @@ class ScheduleConfigUpdate(BaseModel):
|
||||
schedule_daily_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_dolt_earnings_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_sec_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_fundamentals_parity_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_near_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_after_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_intraday_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_backtest_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_ticker_universe_cron: str | None = Field(default=None, max_length=120)
|
||||
|
||||
|
||||
class PerformanceConfigUpdate(BaseModel):
|
||||
|
||||
@@ -91,3 +91,6 @@ class FundamentalResponse(BaseModel):
|
||||
metrics: list[MetricItem] | None = None
|
||||
valuation: Valuation | None = None
|
||||
reads: FundamentalsReads | None = None
|
||||
setup_eligible: bool = True
|
||||
setup_block_code: str | None = None
|
||||
setup_block_reason: str | None = None
|
||||
|
||||
@@ -53,3 +53,7 @@ class PaperTradeResponse(BaseModel):
|
||||
# when the trailing exit policy is active.
|
||||
trailing_stop: float | None = None
|
||||
trailing_distance_pct: float | None = None
|
||||
# Trading sessions represented by post-entry OHLCV bars. These are populated
|
||||
# only while the active exit policy has a max-hold rule.
|
||||
sessions_held: int | None = None
|
||||
sessions_remaining: int | None = None
|
||||
|
||||
+12
-1
@@ -1,6 +1,6 @@
|
||||
"""Ticker request/response schemas."""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -14,5 +14,16 @@ class TickerResponse(BaseModel):
|
||||
symbol: str
|
||||
name: str | None = None
|
||||
created_at: datetime
|
||||
# NULL == actively traded. Delisted symbols stay in the registry with their
|
||||
# history and are excluded from signals — the date is what makes that
|
||||
# visible instead of the row silently disappearing.
|
||||
delisted_on: date | None = None
|
||||
delisted_reason: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class TickerDelistingUpdate(BaseModel):
|
||||
delisted_on: date = Field(
|
||||
..., description="Effective date the symbol stopped trading"
|
||||
)
|
||||
|
||||
+99
-100
@@ -7,6 +7,7 @@ from passlib.hash import bcrypt
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import job_catalog
|
||||
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
@@ -17,7 +18,7 @@ from app.models.settings import SystemSetting
|
||||
from app.models.ticker import Ticker
|
||||
from app.models.trade_setup import TradeSetup
|
||||
from app.models.user import User
|
||||
from app.services import settings_store
|
||||
from app.services import job_run_store, settings_store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -606,94 +607,110 @@ async def get_pipeline_readiness(db: AsyncSession) -> list[dict]:
|
||||
# Job control (placeholder — scheduler is Task 12.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_JOB_NAMES = {
|
||||
"data_collector",
|
||||
"data_backfill",
|
||||
"benchmark_collector",
|
||||
"sentiment_collector",
|
||||
"fundamental_collector",
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"fundamentals_parity_report",
|
||||
"rr_scanner",
|
||||
"ticker_universe_sync",
|
||||
"outcome_evaluator",
|
||||
"alerts",
|
||||
"market_regime",
|
||||
"regime_monitor",
|
||||
"event_study",
|
||||
"backtest",
|
||||
"daily_pipeline",
|
||||
"near_close_pipeline",
|
||||
"after_close_pipeline",
|
||||
"intraday_pipeline",
|
||||
"shadow_book",
|
||||
}
|
||||
# Job identity, labels and pipeline membership now live in app.job_catalog, which
|
||||
# derives PIPELINE_MEMBERS from the pipeline step lists instead of restating them.
|
||||
# Re-exported here because callers (routers, tests) import them from this module.
|
||||
VALID_JOB_NAMES = job_catalog.VALID_JOB_NAMES
|
||||
JOB_LABELS = job_catalog.JOB_LABELS
|
||||
PIPELINE_MEMBERS = job_catalog.PIPELINE_MEMBERS
|
||||
|
||||
JOB_LABELS = {
|
||||
"data_collector": "Data Collector (OHLCV)",
|
||||
"data_backfill": "Data Backfill (deep history)",
|
||||
"benchmark_collector": "Benchmark Collector",
|
||||
"sentiment_collector": "Sentiment Collector",
|
||||
"fundamental_collector": "Fundamental Collector",
|
||||
"dolt_earnings_import": "Dolt Earnings Import (shadow)",
|
||||
"sec_fundamentals_import": "SEC Fundamentals Import (shadow)",
|
||||
"fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
|
||||
"rr_scanner": "R:R Scanner",
|
||||
"ticker_universe_sync": "Ticker Universe Sync",
|
||||
"outcome_evaluator": "Outcome Evaluator",
|
||||
"alerts": "Alerts Dispatcher",
|
||||
"market_regime": "Market Regime",
|
||||
"regime_monitor": "Regime Monitor",
|
||||
"event_study": "Event Study",
|
||||
"backtest": "Backtest",
|
||||
"daily_pipeline": "Morning Pipeline",
|
||||
"near_close_pipeline": "Near-Close Pipeline (scan+alert)",
|
||||
"after_close_pipeline": "After-Close Pipeline (outcome)",
|
||||
"intraday_pipeline": "Intraday Pipeline",
|
||||
"shadow_book": "Shadow Book (auto-traded strategy)",
|
||||
}
|
||||
# Anything further out than this is a parked backstop, not a schedule: pipeline
|
||||
# steps and manual jobs are registered on a 520-week interval, and triggering one
|
||||
# re-arms it. Belt-and-braces behind the category rule in _next_run_fields.
|
||||
_NEXT_RUN_HORIZON_DAYS = 365
|
||||
|
||||
# Jobs driven by a pipeline (in order) rather than their own auto timer.
|
||||
PIPELINE_MEMBERS = {
|
||||
"data_collector",
|
||||
"benchmark_collector",
|
||||
"sentiment_collector",
|
||||
"rr_scanner",
|
||||
"outcome_evaluator",
|
||||
"alerts",
|
||||
"market_regime",
|
||||
"regime_monitor",
|
||||
"shadow_book",
|
||||
}
|
||||
|
||||
def _visible_next_run(next_run: datetime | None) -> datetime | None:
|
||||
"""Drop a next-run that is really the parked backstop."""
|
||||
if next_run is None:
|
||||
return None
|
||||
horizon = datetime.now(next_run.tzinfo) + timedelta(days=_NEXT_RUN_HORIZON_DAYS)
|
||||
return None if next_run > horizon else next_run
|
||||
|
||||
|
||||
def _own_next_run(scheduler, name: str) -> datetime | None:
|
||||
# getattr: APScheduler only sets next_run_time once the scheduler is running,
|
||||
# so a job registered but not yet started has no such attribute at all.
|
||||
job = scheduler.get_job(name)
|
||||
return _visible_next_run(getattr(job, "next_run_time", None)) if job else None
|
||||
|
||||
|
||||
def _next_run_fields(scheduler, name: str, enabled_map: dict[str, bool]) -> dict:
|
||||
"""Where this job's next run comes from, decided by category not by clock.
|
||||
|
||||
A pipeline step has no meaningful schedule of its own, so reporting one is
|
||||
the bug: its parent's timer is the answer. Manual jobs have no answer at all,
|
||||
and saying so beats rendering a parked backstop as a date.
|
||||
"""
|
||||
category = job_catalog.JOB_CATEGORY.get(name)
|
||||
if category == job_catalog.CATEGORY_STEP:
|
||||
parents = job_catalog.PIPELINES_BY_MEMBER.get(name, ())
|
||||
soonest: datetime | None = None
|
||||
via: str | None = None
|
||||
for parent in parents:
|
||||
if not enabled_map.get(parent, True):
|
||||
continue
|
||||
candidate = _own_next_run(scheduler, parent)
|
||||
if candidate is not None and (soonest is None or candidate < soonest):
|
||||
soonest, via = candidate, parent
|
||||
return {
|
||||
"next_run_at": None,
|
||||
"next_run_source": "via_pipeline",
|
||||
"via_next_run_at": soonest.isoformat() if soonest else None,
|
||||
"via_next_run_job": via,
|
||||
}
|
||||
if category == job_catalog.CATEGORY_MANUAL:
|
||||
return {
|
||||
"next_run_at": None,
|
||||
"next_run_source": "manual_only",
|
||||
"via_next_run_at": None,
|
||||
"via_next_run_job": None,
|
||||
}
|
||||
own = _own_next_run(scheduler, name)
|
||||
return {
|
||||
"next_run_at": own.isoformat() if own else None,
|
||||
"next_run_source": "own_schedule",
|
||||
"via_next_run_at": None,
|
||||
"via_next_run_job": None,
|
||||
}
|
||||
|
||||
|
||||
async def list_jobs(db: AsyncSession) -> list[dict]:
|
||||
"""Return status of all scheduled jobs."""
|
||||
"""Return status of all scheduled jobs, grouped and ordered by category."""
|
||||
from app.scheduler import get_job_runtime_snapshot, scheduler
|
||||
|
||||
visible = sorted(VALID_JOB_NAMES - job_catalog.HIDDEN_JOBS, key=job_catalog.sort_order)
|
||||
# One query for every flag instead of one per job. Parents are read too, since
|
||||
# a step reports its parent's next run only while that parent is enabled.
|
||||
flags = await settings_store.get_map(
|
||||
db, [f"job_{name}_enabled" for name in VALID_JOB_NAMES]
|
||||
)
|
||||
enabled_map = {
|
||||
name: flags.get(f"job_{name}_enabled", "true") == "true"
|
||||
for name in VALID_JOB_NAMES
|
||||
}
|
||||
last_runs = await job_run_store.get_map(db, visible)
|
||||
|
||||
jobs_out = []
|
||||
for name in sorted(VALID_JOB_NAMES):
|
||||
# Check enabled setting
|
||||
setting = await settings_store.get_setting(db, f"job_{name}_enabled")
|
||||
enabled = setting.value == "true" if setting else True # default enabled
|
||||
|
||||
# Get scheduler job info
|
||||
for name in visible:
|
||||
job = scheduler.get_job(name)
|
||||
next_run = None
|
||||
if job and job.next_run_time:
|
||||
next_run = job.next_run_time.isoformat()
|
||||
|
||||
runtime = get_job_runtime_snapshot(name)
|
||||
last = last_runs.get(name)
|
||||
|
||||
jobs_out.append({
|
||||
"name": name,
|
||||
"label": JOB_LABELS.get(name, name),
|
||||
"enabled": enabled,
|
||||
"next_run_at": next_run,
|
||||
"via_pipeline": name in PIPELINE_MEMBERS,
|
||||
"enabled": enabled_map.get(name, True),
|
||||
"category": job_catalog.JOB_CATEGORY.get(name),
|
||||
"sort_order": job_catalog.sort_order(name),
|
||||
# Parent pipelines for a step; the steps themselves for a pipeline.
|
||||
"pipelines": list(job_catalog.PIPELINES_BY_MEMBER.get(name, ())),
|
||||
"steps": [step for step, _ in job_catalog.PIPELINE_STEPS.get(name, ())],
|
||||
"registered": job is not None,
|
||||
"running": bool(runtime.get("running", False)),
|
||||
# runtime_* are strictly live in-memory state. Persisted history is
|
||||
# reported separately as last_run_*, so a stale error cannot pin the
|
||||
# status chip or the rate-limit banner.
|
||||
"runtime_status": runtime.get("status"),
|
||||
"runtime_processed": runtime.get("processed"),
|
||||
"runtime_total": runtime.get("total"),
|
||||
@@ -702,6 +719,15 @@ async def list_jobs(db: AsyncSession) -> list[dict]:
|
||||
"runtime_started_at": runtime.get("started_at"),
|
||||
"runtime_finished_at": runtime.get("finished_at"),
|
||||
"runtime_message": runtime.get("message"),
|
||||
# Survives restarts, unlike runtime_*. Reported separately so the
|
||||
# status chip keeps meaning "state now" rather than "last outcome,
|
||||
# forever" -- an error a week ago must not read as Inactive today.
|
||||
"last_run_at": last.finished_at.isoformat() if last else None,
|
||||
"last_run_status": last.status if last else None,
|
||||
"last_run_message": last.message if last else None,
|
||||
"last_run_processed": last.processed if last else None,
|
||||
"last_run_total": last.total if last else None,
|
||||
**_next_run_fields(scheduler, name, enabled_map),
|
||||
})
|
||||
|
||||
return jobs_out
|
||||
@@ -777,30 +803,3 @@ async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSe
|
||||
|
||||
key = f"job_{job_name}_enabled"
|
||||
return await update_setting(db, key, str(enabled).lower())
|
||||
|
||||
|
||||
def get_fundamentals_parity_report() -> dict | None:
|
||||
"""Return the latest compact A5 summary, if the job has run."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest
|
||||
|
||||
report = load_latest(settings.fundamentals_parity_report_dir)
|
||||
if report is not None:
|
||||
report.pop("rows", None) # full per-ticker data is download-only
|
||||
return report
|
||||
|
||||
|
||||
def get_fundamentals_parity_csv() -> tuple[str, str] | None:
|
||||
"""Return the latest A5 CSV filename and content for authenticated download."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest_csv
|
||||
|
||||
return load_latest_csv(settings.fundamentals_parity_report_dir)
|
||||
|
||||
|
||||
def get_fundamentals_parity_json() -> tuple[str, str] | None:
|
||||
"""Return the canonical A5 JSON artifact for authenticated download."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest_json
|
||||
|
||||
return load_latest_json(settings.fundamentals_parity_report_dir)
|
||||
|
||||
@@ -97,8 +97,16 @@ SIGNAL_BUNDLE_MAX_CHARS = 3900 # Telegram limit is 4096; keep room for HTML par
|
||||
# Hysteresis (a deadband around each divider) stops a point sitting on a boundary
|
||||
# from flip-flopping; the cooldown caps how often a genuine change can re-alert.
|
||||
QUAD_TYPE = "regime_quadrant"
|
||||
QUAD_X_DIV = 60.0 # v2 State divider (backend response is authoritative)
|
||||
QUAD_Y_DIV = 60.0 # v2 Warning divider
|
||||
# The fundamental channel gets its own alerts rather than shifting a score:
|
||||
# "the context changed" and "both channels are elevated" are different facts from
|
||||
# "the market axes moved", and fusing them into one number would destroy exactly
|
||||
# the information an operator uses to decide how much the alert is worth.
|
||||
FUND_TYPE = "regime_fundamental"
|
||||
CONFLUENCE_TYPE = "regime_confluence"
|
||||
# States that count as fundamental risk for the confluence test.
|
||||
FUND_ADVERSE = "adverse"
|
||||
QUAD_X_DIV = 50.0 # v3 State divider (backend response is authoritative)
|
||||
QUAD_Y_DIV = 40.0 # v3 Warning divider; the axes have different ranges
|
||||
QUAD_MARGIN = 5.0 # half-width of the hysteresis deadband around each divider
|
||||
QUAD_COOLDOWN_DAYS = 3 # min days between quadrant-change alerts
|
||||
QUAD_LABELS = {
|
||||
@@ -859,16 +867,111 @@ async def _collect_regime_quadrant(db: AsyncSession) -> list[tuple[str, str]]:
|
||||
)
|
||||
else:
|
||||
metrics = f"State {x:.0f} · Warning {y:.0f}"
|
||||
# The fundamental channel is reported, never added in: this alert is about
|
||||
# the two market axes, and the context is stated beside them so a reader can
|
||||
# judge confluence themselves rather than being handed a fused number.
|
||||
context = data.get("fundamental_context") or {}
|
||||
context_line = (
|
||||
f"fundamentals: {context.get('state', 'unknown')} "
|
||||
f"({context.get('evidence_quality', 'unavailable')})\n"
|
||||
)
|
||||
text = (
|
||||
f"🧭 <b>Regime quadrant change</b>\n"
|
||||
f"🧭 <b>AI/Tech risk quadrant change</b>\n"
|
||||
f"{QUAD_LABELS.get(prev, prev)} → {QUAD_LABELS.get(new_q, new_q)}\n"
|
||||
f"{metrics}\n"
|
||||
f"{context_line}"
|
||||
f"coverage: state {state.get('coverage'):.0f}% / warning {warning.get('coverage'):.0f}%\n"
|
||||
f"<i>Risk thermometer - not a trade signal.</i>"
|
||||
)
|
||||
return [(_quadrant_log_key(new_q, x, y, basket_hash), text)]
|
||||
|
||||
|
||||
async def _last_logged_key(db: AsyncSession, alert_type: str) -> str | None:
|
||||
"""Most recent logged key for a type, our baseline for change detection."""
|
||||
result = await db.execute(
|
||||
select(AlertLog.dedup_key)
|
||||
.where(AlertLog.alert_type == alert_type)
|
||||
.order_by(AlertLog.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.first()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
async def _collect_regime_fundamental(db: AsyncSession) -> list[tuple[str, str, str]]:
|
||||
"""Fundamental-context changes and market/fundamental confluence.
|
||||
|
||||
Two triggers, deliberately separate from the quadrant alert and from each
|
||||
other, because they answer different questions: *what the evidence says* and
|
||||
*whether both channels agree*. Neither is derived by moving a score.
|
||||
|
||||
``unknown`` never alerts. An absence of evidence is not a change in the
|
||||
evidence, and alerting on it would train the reader to ignore the channel.
|
||||
Both seed silently on first run, exactly as the quadrant alert does.
|
||||
"""
|
||||
from app.services.regime_monitor_service import get_regime_monitor
|
||||
|
||||
data = await get_regime_monitor(db)
|
||||
if not data.get("available"):
|
||||
return []
|
||||
warning = data.get("warning") or {}
|
||||
context = data.get("fundamental_context") or {}
|
||||
state = str(context.get("state") or "unknown")
|
||||
# `usable`, not `available`: the state is deliberately preserved past its
|
||||
# staleness horizon so the card can keep showing the last thing observed, and
|
||||
# an observation whose extraction failed is fresh but knows nothing. Neither
|
||||
# may confirm anything — without this gate a months-old adverse read silently
|
||||
# corroborates every new Warning crossing forever, which is the strongest
|
||||
# claim this channel makes and the one it has least right to make.
|
||||
usable = bool(context.get("usable"))
|
||||
score = warning.get("score")
|
||||
|
||||
quality = data.get("data_quality") or {}
|
||||
if not quality.get("is_fresh") or float(warning.get("coverage") or 0) < 75:
|
||||
return []
|
||||
|
||||
quadrant_cfg = data.get("quadrant_config") or {}
|
||||
y_div = float(quadrant_cfg.get("warning_divider", QUAD_Y_DIV))
|
||||
warning_elevated = score is not None and float(score) >= y_div
|
||||
|
||||
out: list[tuple[str, str, str]] = []
|
||||
|
||||
previous_state = await _last_logged_key(db, FUND_TYPE)
|
||||
if previous_state is None:
|
||||
_log_alert(db, FUND_TYPE, state) # seed
|
||||
elif previous_state != state and state != "unknown" and usable:
|
||||
effective = context.get("effective_date")
|
||||
out.append((
|
||||
FUND_TYPE,
|
||||
state,
|
||||
f"📋 <b>Fundamental context changed</b>\n"
|
||||
f"{previous_state} → {state}\n"
|
||||
f"evidence: {context.get('evidence_quality', 'unavailable')}"
|
||||
+ (f" · effective {effective}" if effective else "")
|
||||
+ "\n<i>Context channel — not a score, not a trade signal.</i>",
|
||||
))
|
||||
|
||||
confluence = "yes" if (warning_elevated and state == FUND_ADVERSE and usable) else "no"
|
||||
previous_confluence = await _last_logged_key(db, CONFLUENCE_TYPE)
|
||||
if previous_confluence is None:
|
||||
_log_alert(db, CONFLUENCE_TYPE, confluence) # seed
|
||||
elif previous_confluence != confluence and confluence == "yes":
|
||||
out.append((
|
||||
CONFLUENCE_TYPE,
|
||||
confluence,
|
||||
f"⚠️ <b>Confluence: market and fundamental risk both elevated</b>\n"
|
||||
f"Warning {float(score):.0f} (≥ {y_div:.0f}) with fundamentals {state}\n"
|
||||
f"evidence: {context.get('evidence_quality', 'unavailable')}\n"
|
||||
f"<i>Highest attention. Still a thermometer — not a trade signal.</i>",
|
||||
))
|
||||
elif previous_confluence != confluence:
|
||||
# Falling out of confluence is a state change worth recording as the new
|
||||
# baseline, but not worth a message.
|
||||
_log_alert(db, CONFLUENCE_TYPE, confluence)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -961,6 +1064,11 @@ async def dispatch_alerts(db: AsyncSession) -> dict:
|
||||
# cooldown/hysteresis handled in the collector (like score drops)
|
||||
for key, text in await _collect_regime_quadrant(db):
|
||||
outgoing.append((QUAD_TYPE, key, text))
|
||||
# Deliberately three separate messages off one toggle, not one fused
|
||||
# signal: the market axes and the fundamental channel are different kinds
|
||||
# of evidence, and an operator needs to know which one moved.
|
||||
for alert_type, key, text in await _collect_regime_fundamental(db):
|
||||
outgoing.append((alert_type, key, text))
|
||||
|
||||
if cfg["trade_closed"]:
|
||||
for key, text, pnl_usd in await _collect_closed_trades(db):
|
||||
|
||||
@@ -1460,6 +1460,21 @@ def _mp_context():
|
||||
return None
|
||||
|
||||
|
||||
async def _rollback_quietly(db: AsyncSession, context: str) -> None:
|
||||
"""Discard a failed unit of work so later statements on this session survive.
|
||||
|
||||
Every DB call in ``run_backtest`` is best-effort — one unreadable ticker must
|
||||
not abort the whole replay. But swallowing the exception alone leaves asyncpg
|
||||
in "current transaction is aborted": every later statement then fails the same
|
||||
way until the first unguarded one (the report write) surfaces it as the job
|
||||
error, long after the real cause. Same guard as ``price_service``.
|
||||
"""
|
||||
try:
|
||||
await db.rollback()
|
||||
except Exception:
|
||||
logger.exception("Session rollback after %s also failed", context)
|
||||
|
||||
|
||||
async def _fetch_columns(db: AsyncSession, symbol: str) -> tuple | None:
|
||||
"""Read one ticker's OHLCV and detach it to primitive column arrays in the
|
||||
event loop (safe ORM access), ready to hand to a worker. None if no data."""
|
||||
@@ -1701,7 +1716,14 @@ def _gate_ablation(candidates: list[dict], activation: dict, threshold: float) -
|
||||
# the QUALIFIED setups at their detection close, best momentum first while
|
||||
# slots and cash allow.
|
||||
SIM_STARTING_CAPITAL = 10_000.0
|
||||
SIM_MAX_POSITIONS = 10
|
||||
# Headroom, not a target: the count cap should never bind. The capacity study
|
||||
# (reports/portfolio-construction-prod505-capacity-bracket-daily-v1) showed a book
|
||||
# that never hits the count cap earns +1.1pp CAGR over the old 10 (51 cohorts of 175
|
||||
# better, 2 worse) at unchanged drawdown, because the blocked entries were as good as
|
||||
# the taken ones — capacity costs trade COUNT, not trade quality. The real ceiling is
|
||||
# cash plus SIM_NOTIONAL_CAP, which saturates the book near 12 positions, so 15/20/None
|
||||
# are the same experiment. Judge any future change here on CAGR, never on EV per trade.
|
||||
SIM_MAX_POSITIONS = 15
|
||||
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
|
||||
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
|
||||
_EULER_MASCHERONI = 0.5772156649015329
|
||||
@@ -2603,6 +2625,19 @@ def _simulate_portfolio(
|
||||
diag = sharpe_diagnostics(rets)
|
||||
sharpe = diag["sharpe"]
|
||||
|
||||
# Sortino: the same numerator as Sharpe over downside deviation about a zero
|
||||
# target. The denominator divides by len(rets) — the full-sample lower partial
|
||||
# moment — NOT by the count of down days, which would shrink the denominator
|
||||
# and inflate the ratio. n >= 3 matches sharpe_diagnostics so the two appear
|
||||
# together or not at all. No down days is +inf, reported as None.
|
||||
sortino = None
|
||||
downside = [r for r in rets if r < 0.0]
|
||||
if len(rets) >= 3 and downside:
|
||||
mean_ret = sum(rets) / len(rets)
|
||||
dd = math.sqrt(sum(r * r for r in downside) / len(rets))
|
||||
if dd > 0:
|
||||
sortino = round(mean_ret / dd * math.sqrt(252.0), 2)
|
||||
|
||||
# Per-calendar-year returns off the equity curve — shows whether every year
|
||||
# contributed or one exceptional stretch carried the result.
|
||||
yearly: list[dict] = []
|
||||
@@ -2628,8 +2663,40 @@ def _simulate_portfolio(
|
||||
),
|
||||
})
|
||||
|
||||
# Gain-to-Pain off the same curve, on MONTHLY returns: Schwager's ratio is
|
||||
# defined monthly and the daily variant is not comparable to published
|
||||
# figures. Distinct loop variables from the yearly pass above — that one exits
|
||||
# with last_eq at final equity, so reusing its names silently corrupts the
|
||||
# first month. The monthly series itself is not emitted: 36-120 floats per
|
||||
# strategy per lookback would bloat the single stored report blob.
|
||||
monthly: list[float] = []
|
||||
month_start_eq = curve[0][1]
|
||||
month_last_eq = curve[0][1]
|
||||
cur_month = date.fromordinal(curve[0][0]).replace(day=1)
|
||||
for o, eq in curve:
|
||||
m = date.fromordinal(o).replace(day=1)
|
||||
if m != cur_month:
|
||||
if month_start_eq > 0:
|
||||
monthly.append(month_last_eq / month_start_eq - 1.0)
|
||||
cur_month = m
|
||||
month_start_eq = month_last_eq
|
||||
month_last_eq = eq
|
||||
if month_start_eq > 0:
|
||||
monthly.append(month_last_eq / month_start_eq - 1.0)
|
||||
|
||||
# Schwager: SUM OF ALL monthly returns over the absolute sum of the negative
|
||||
# ones. Not sum(positive)/|sum(negative)| — that is profit-factor-shaped and
|
||||
# sits exactly 1.0 higher for every input, since sum(all) = sum(pos) - |sum(neg)|.
|
||||
monthly_pain = -sum(r for r in monthly if r < 0.0)
|
||||
gain_to_pain = round(sum(monthly) / monthly_pain, 2) if monthly_pain > 0 else None
|
||||
|
||||
pnls = [t["pnl"] for t in trades]
|
||||
wins = sum(1 for p in pnls if p > 0)
|
||||
# Dollar-based, over closed-trade P&L. Distinct from the R-based profit_factor
|
||||
# in _robustness_stats; the two never share an object.
|
||||
gross_win = sum(p for p in pnls if p > 0)
|
||||
gross_loss = -sum(p for p in pnls if p < 0)
|
||||
profit_factor = round(gross_win / gross_loss, 2) if gross_loss > 0 else None
|
||||
reason_counts = {
|
||||
reason: sum(1 for t in trades if t["reason"] == reason)
|
||||
for reason in sorted({t["reason"] for t in trades})
|
||||
@@ -2684,7 +2751,13 @@ def _simulate_portfolio(
|
||||
"total_return_pct": round(total_return_pct, 1),
|
||||
"cagr_pct": round(cagr_pct, 1) if cagr_pct is not None else None,
|
||||
"max_drawdown_pct": round(max_dd_pct, 1),
|
||||
# calmar IS MAR here (CAGR / max drawdown) — one field, two names.
|
||||
"calmar": round(calmar, 2) if calmar is not None else None,
|
||||
# Emitted unconditionally even when None: the UI treats an ABSENT key as
|
||||
# "report predates these metrics", so presence is a contract.
|
||||
"sortino": sortino,
|
||||
"gain_to_pain": gain_to_pain,
|
||||
"profit_factor": profit_factor,
|
||||
"sharpe": sharpe,
|
||||
"sharpe_se": diag["sharpe_se"],
|
||||
"psr": diag["psr"],
|
||||
@@ -3871,40 +3944,11 @@ def _build_recommendation(report: dict) -> dict:
|
||||
})
|
||||
|
||||
q = report.get("overall_qualified") or {}
|
||||
target_net = q.get("net_avg_r")
|
||||
|
||||
# Legacy diagnostic: target/stop race vs the best fixed hold.
|
||||
time_rows = [r for r in report.get("time_exit_sweep") or [] if r.get("net_avg_r") is not None]
|
||||
best_hold = max(time_rows, key=lambda r: r["net_avg_r"], default=None)
|
||||
sim_rows = {
|
||||
p.get("policy"): p
|
||||
for p in (report.get("portfolio_sim") or {}).get("policies", [])
|
||||
}
|
||||
hold_sim = sim_rows.get("hold")
|
||||
if best_hold is not None and target_net is not None:
|
||||
if best_hold["net_avg_r"] > target_net + _EXIT_SWITCH_THRESHOLD:
|
||||
text = (
|
||||
f"Legacy exit diagnostic: hold {best_hold['hold_days']} trading days with the initial stop "
|
||||
f"({best_hold['net_avg_r']:+.2f}R net/trade vs {target_net:+.2f}R for the S/R target exit)."
|
||||
)
|
||||
target_sim = sim_rows.get("target")
|
||||
if (
|
||||
hold_sim is not None and target_sim is not None
|
||||
and hold_sim.get("cagr_pct") is not None and target_sim.get("cagr_pct") is not None
|
||||
):
|
||||
text += (
|
||||
f" The simulated book agrees: {hold_sim['cagr_pct']:+.1f}% vs "
|
||||
f"{target_sim['cagr_pct']:+.1f}% CAGR at similar drawdown."
|
||||
)
|
||||
items.append({"topic": "exit", "text": text})
|
||||
else:
|
||||
items.append({
|
||||
"topic": "exit",
|
||||
"text": (
|
||||
f"Legacy exit diagnostic: keep the S/R target exit ({target_net:+.2f}R net/trade) — "
|
||||
"no fixed hold beats it by a meaningful margin."
|
||||
),
|
||||
})
|
||||
# Nothing here reads time_exit_sweep any more. The hold-vs-target comparison
|
||||
# is not reported (both are exits the production book replaced, so choosing
|
||||
# between them cannot lead to an action), and the robustness check below no
|
||||
# longer picks its basis from them either.
|
||||
|
||||
# Gate floors, judged under the hold exit (the ablation's Hold column).
|
||||
ablation = {r["variant"]: r for r in report.get("gate_ablation") or []}
|
||||
@@ -3952,33 +3996,32 @@ def _build_recommendation(report: dict) -> dict:
|
||||
),
|
||||
})
|
||||
|
||||
# Book vs benchmark.
|
||||
book = hold_sim or sim_rows.get("target")
|
||||
if book is not None and book.get("spy_return_pct") is not None:
|
||||
edge = book["total_return_pct"] - book["spy_return_pct"]
|
||||
# Book vs benchmark — read from the SAME production monitor row the page
|
||||
# shows in its tiles. It used to read the hold/target policy sim, so the
|
||||
# recommendation quoted a different portfolio return than the tile directly
|
||||
# above it, against an identical SPY figure. Those policies are legacy
|
||||
# diagnostics; the production book is the ATR trail.
|
||||
if production_row is not None and production_row.get("spy_return_pct") is not None:
|
||||
edge = production_row["total_return_pct"] - production_row["spy_return_pct"]
|
||||
verdict = "beats" if edge > 0 else "LAGS"
|
||||
items.append({
|
||||
"topic": "benchmark",
|
||||
"text": (
|
||||
f"Book vs SPY: {verdict} buy-and-hold by {edge:+.1f} points "
|
||||
f"({book['total_return_pct']:+.1f}% vs {book['spy_return_pct']:+.1f}%), "
|
||||
f"max drawdown −{book['max_drawdown_pct']:.1f}%."
|
||||
f"({production_row['total_return_pct']:+.1f}% vs "
|
||||
f"{production_row['spy_return_pct']:+.1f}%)."
|
||||
),
|
||||
})
|
||||
|
||||
# Robustness: does the edge survive without the biggest winners? Judged on
|
||||
# the RECOMMENDED exit — outlier dependence under an exit we'd abandon
|
||||
# would be the wrong warning.
|
||||
hold_recommended = (
|
||||
best_hold is not None and target_net is not None
|
||||
and best_hold["net_avg_r"] > target_net + _EXIT_SWITCH_THRESHOLD
|
||||
)
|
||||
if hold_recommended and best_hold.get("net_avg_r_ex_top5") is not None:
|
||||
trimmed = best_hold["net_avg_r_ex_top5"]
|
||||
basis = f"under the recommended {best_hold['hold_days']}d hold"
|
||||
else:
|
||||
trimmed = q.get("net_avg_r_ex_top5")
|
||||
basis = "under the S/R target exit"
|
||||
# Robustness: does the edge survive without the biggest winners?
|
||||
#
|
||||
# There is no ATR-trail equivalent of this number in the report — the only
|
||||
# ex-top-5% figure is the gate-level target/stop grading. So it is reported
|
||||
# on that basis and SAYS SO, rather than being dressed up as a verdict on the
|
||||
# production book. It used to pick between "the recommended Nd hold" and "the
|
||||
# S/R target exit", naming a rejected exit as recommended.
|
||||
trimmed = q.get("net_avg_r_ex_top5")
|
||||
basis = "gate-level grading, not the production ATR-trail book"
|
||||
if trimmed is not None:
|
||||
if trimmed > 0:
|
||||
items.append({
|
||||
@@ -3999,20 +4042,20 @@ def _build_recommendation(report: dict) -> dict:
|
||||
),
|
||||
})
|
||||
|
||||
if headline is None and hold_recommended:
|
||||
cagr_note = (
|
||||
f" (~{hold_sim['cagr_pct']:.0f}% CAGR simulated)"
|
||||
if hold_sim is not None and hold_sim.get("cagr_pct") is not None
|
||||
else ""
|
||||
)
|
||||
headline = (
|
||||
f"Trade the qualified list long-only; hold {best_hold['hold_days']} trading days "
|
||||
f"with the initial ATR stop{cagr_note}."
|
||||
)
|
||||
# No fallback headline. It used to recommend the fixed-hold exit whenever the
|
||||
# portfolio monitor was missing, which meant a report without a production
|
||||
# row advised an exit the production book had already replaced. A report that
|
||||
# cannot describe the production baseline states no baseline.
|
||||
|
||||
return {
|
||||
"headline": headline,
|
||||
"items": items,
|
||||
# Which monitor row every production/benchmark figure above was read
|
||||
# from. The page defaults its lookback selector to this, so the tiles and
|
||||
# the recommendation cannot open on different windows — they used to,
|
||||
# because this preferred "all" while the UI defaulted to "3y".
|
||||
"basis_lookback": (production_row or {}).get("lookback"),
|
||||
"basis_lookback_label": (production_row or {}).get("lookback_label"),
|
||||
"note": "Derived from this report's numbers on every run — the advice flips if the data does.",
|
||||
}
|
||||
|
||||
@@ -4030,9 +4073,12 @@ async def run_backtest(
|
||||
config = await get_recommendation_config(db)
|
||||
activation = await get_activation_config(db)
|
||||
|
||||
result = await db.execute(select(Ticker).order_by(Ticker.symbol))
|
||||
tickers = list(result.scalars().all())
|
||||
total = len(tickers)
|
||||
# Plain strings, not Ticker instances: the rollbacks below expire any ORM
|
||||
# objects held across them, and touching an expired attribute afterwards
|
||||
# triggers sync lazy-loading, which raises on an AsyncSession.
|
||||
result = await db.execute(select(Ticker.symbol).order_by(Ticker.symbol))
|
||||
symbols = list(result.scalars().all())
|
||||
total = len(symbols)
|
||||
rank_only_symbols = await _load_research_rank_only_symbols(db)
|
||||
if rank_only_symbols:
|
||||
logger.info(json.dumps({
|
||||
@@ -4056,6 +4102,7 @@ async def run_backtest(
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Benchmark load for residual momentum failed")
|
||||
await _rollback_quietly(db, "benchmark load")
|
||||
|
||||
def _merge(result: tuple[list[dict], dict]) -> None:
|
||||
cands, series = result
|
||||
@@ -4087,26 +4134,27 @@ async def run_backtest(
|
||||
done = 0
|
||||
with pool:
|
||||
for start in range(0, total, chunk):
|
||||
batch = tickers[start : start + chunk]
|
||||
batch = symbols[start : start + chunk]
|
||||
futures = []
|
||||
for ticker in batch:
|
||||
for symbol in batch:
|
||||
try:
|
||||
columns = await _fetch_columns(db, ticker.symbol)
|
||||
columns = await _fetch_columns(db, symbol)
|
||||
except Exception:
|
||||
logger.exception("Backtest fetch failed for %s", ticker.symbol)
|
||||
logger.exception("Backtest fetch failed for %s", symbol)
|
||||
await _rollback_quietly(db, f"fetch for {symbol}")
|
||||
continue
|
||||
if columns is not None:
|
||||
futures.append(loop.run_in_executor(
|
||||
pool,
|
||||
_replay_and_signals,
|
||||
ticker.symbol,
|
||||
symbol,
|
||||
columns,
|
||||
config,
|
||||
activation,
|
||||
benchmark_closes,
|
||||
target_model,
|
||||
cadence,
|
||||
ticker.symbol in rank_only_symbols,
|
||||
symbol in rank_only_symbols,
|
||||
))
|
||||
for result in await asyncio.gather(*futures, return_exceptions=True):
|
||||
if isinstance(result, Exception):
|
||||
@@ -4119,25 +4167,26 @@ async def run_backtest(
|
||||
else:
|
||||
# Sequential fallback (Windows / 1 worker): run each replay in a worker
|
||||
# thread so the event loop — and the API server — stays responsive.
|
||||
for index, ticker in enumerate(tickers):
|
||||
for index, symbol in enumerate(symbols):
|
||||
if progress_cb is not None:
|
||||
progress_cb(index, total, ticker.symbol)
|
||||
progress_cb(index, total, symbol)
|
||||
try:
|
||||
columns = await _fetch_columns(db, ticker.symbol)
|
||||
columns = await _fetch_columns(db, symbol)
|
||||
if columns is not None:
|
||||
_merge(await asyncio.to_thread(
|
||||
_replay_and_signals,
|
||||
ticker.symbol,
|
||||
symbol,
|
||||
columns,
|
||||
config,
|
||||
activation,
|
||||
benchmark_closes,
|
||||
target_model,
|
||||
cadence,
|
||||
ticker.symbol in rank_only_symbols,
|
||||
symbol in rank_only_symbols,
|
||||
))
|
||||
except Exception:
|
||||
logger.exception("Backtest replay failed for %s", ticker.symbol)
|
||||
logger.exception("Backtest replay failed for %s", symbol)
|
||||
await _rollback_quietly(db, f"replay for {symbol}")
|
||||
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(total, total, "")
|
||||
@@ -4202,6 +4251,7 @@ async def run_backtest(
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Benchmark load for the portfolio sim failed")
|
||||
await _rollback_quietly(db, "portfolio-sim benchmark load")
|
||||
|
||||
for policy in ("target", "hold"):
|
||||
sim = _simulate_portfolio(
|
||||
@@ -4222,6 +4272,7 @@ async def run_backtest(
|
||||
live_exit_policy = await get_exit_policy(db)
|
||||
except Exception:
|
||||
logger.exception("Live exit policy load failed; monitor uses defaults")
|
||||
await _rollback_quietly(db, "exit policy load")
|
||||
portfolio_monitor_report = _portfolio_monitor(
|
||||
candidates, price_columns, spy_closes, hold_horizon,
|
||||
live_exit_policy=live_exit_policy,
|
||||
@@ -4241,6 +4292,11 @@ async def run_backtest(
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Portfolio simulation failed")
|
||||
# Catches the price_columns fetch loop, which has no handler of its
|
||||
# own. The inner handlers above may already have rolled back; a
|
||||
# rollback on a clean session is a no-op, so this stays safe as the
|
||||
# backstop for whichever DB call actually failed.
|
||||
await _rollback_quietly(db, "portfolio simulation")
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
@@ -4380,11 +4436,38 @@ async def run_and_store(
|
||||
|
||||
|
||||
async def get_backtest_report(db: AsyncSession) -> dict | None:
|
||||
"""Return the last cached backtest report, or None if never run."""
|
||||
"""Return the last cached backtest report, or None if never run.
|
||||
|
||||
The recommendation is **re-derived from the cached report** rather than
|
||||
served as stored. It is a pure function of the numbers already in the
|
||||
report — the payload's own note says it is derived from them on every run —
|
||||
so recomputing costs nothing and keeps one class of bug out:
|
||||
|
||||
A report cached by an older build carries that build's recommendation. After
|
||||
a change to how the recommendation is sourced, the page would keep showing
|
||||
the old one — quoting the legacy policy book, naming a rejected exit as
|
||||
"recommended", and omitting ``basis_lookback``, which in turn let the
|
||||
lookback selector default somewhere else. The result was the exact
|
||||
tiles-disagree-with-recommendation contradiction this rebuild exists to
|
||||
prevent, silently, until the next scheduled run happened to overwrite it.
|
||||
|
||||
Re-deriving means a corrected recommendation appears on the first page load
|
||||
after deploy instead of after the next backtest.
|
||||
"""
|
||||
setting = await settings_store.get_setting(db, KEY_REPORT)
|
||||
if setting is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(setting.value)
|
||||
report = json.loads(setting.value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not isinstance(report, dict):
|
||||
return None
|
||||
try:
|
||||
report["recommendation"] = _build_recommendation(report)
|
||||
except Exception:
|
||||
# Fail closed: drop it rather than fall back to the stored one, which is
|
||||
# precisely the stale derivation this rebuild is here to replace.
|
||||
logger.exception("Could not rebuild the backtest recommendation; omitting it")
|
||||
report.pop("recommendation", None)
|
||||
return report
|
||||
|
||||
@@ -25,6 +25,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import ticker_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -72,15 +73,25 @@ def _breadth_from_closes(
|
||||
return _breadth_with_counts(closes_by_symbol, window, min_tickers)[0]
|
||||
|
||||
|
||||
# Breadth deterioration counts fully when price masks it (true divergence, the
|
||||
# dangerous pre-top case) and at CONFIRMED_FLOOR when price falls with it.
|
||||
# v2 used a hard ``price_ret >= 0`` cliff, which zeroed the sensor during every
|
||||
# decline -- so on 2026-07-24, with the basket shedding 10 percentage points
|
||||
# above their 200-DMA in 20 sessions, Warning read exactly 0. Breadth *level*
|
||||
# lives in State but breadth *velocity* appears nowhere else, so partial credit
|
||||
# here is not double counting.
|
||||
DIVERGENCE_CONFIRMED_FLOOR = 0.35
|
||||
DIVERGENCE_TAPER_PCT = 3.0
|
||||
|
||||
|
||||
def compute_divergence_series(
|
||||
breadth: dict[date, float], benchmark_closes: Series, lookback: int = 20
|
||||
) -> dict[date, float]:
|
||||
"""Early-warning score (0-100, high = fragile) per date.
|
||||
|
||||
This is deliberately a pure divergence: it is positive only when benchmark
|
||||
price holds/rises while breadth falls. Absolute low breadth belongs in the
|
||||
State score, so it is not counted again here. A 20 percentage-point breadth
|
||||
deterioration maps to 100.
|
||||
A 20 percentage-point breadth deterioration maps to 100 when the benchmark
|
||||
is flat or rising, tapering to ``DIVERGENCE_CONFIRMED_FLOOR`` of that once
|
||||
the benchmark is down ``DIVERGENCE_TAPER_PCT`` or more over the window.
|
||||
"""
|
||||
bench = {d: c for d, c in benchmark_closes}
|
||||
common = sorted(d for d in bench if d in breadth)
|
||||
@@ -93,15 +104,16 @@ def compute_divergence_series(
|
||||
price_ret = (bench[d] / price_past - 1.0) * 100.0 # %
|
||||
breadth_chg = breadth[d] - breadth[d0] # percentage points
|
||||
deterioration = max(0.0, -breadth_chg)
|
||||
score = deterioration * 5.0 if price_ret >= 0 else 0.0
|
||||
out[d] = max(0.0, min(100.0, round(score, 2)))
|
||||
taper = max(0.0, min(1.0, (price_ret + DIVERGENCE_TAPER_PCT) / DIVERGENCE_TAPER_PCT))
|
||||
gate = DIVERGENCE_CONFIRMED_FLOOR + (1.0 - DIVERGENCE_CONFIRMED_FLOOR) * taper
|
||||
out[d] = max(0.0, min(100.0, round(deterioration * 5.0 * gate, 2)))
|
||||
return out
|
||||
|
||||
|
||||
async def _load_universe_closes(
|
||||
db: AsyncSession, symbols: list[str] | None = None
|
||||
) -> dict[str, Series]:
|
||||
stmt = select(Ticker).order_by(Ticker.symbol)
|
||||
stmt = ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
if symbols is not None:
|
||||
stmt = stmt.where(Ticker.symbol.in_(symbols))
|
||||
result = await db.execute(stmt)
|
||||
@@ -137,11 +149,3 @@ async def compute_breadth_details(
|
||||
"""Breadth values plus the qualifying-member count for snapshot metadata."""
|
||||
closes_by_symbol = await _load_universe_closes(db, symbols)
|
||||
return _breadth_with_counts(closes_by_symbol, window, min_tickers)
|
||||
|
||||
|
||||
async def compute_breadth_today(db: AsyncSession) -> float | None:
|
||||
"""Latest breadth reading (thin wrapper, for future live use)."""
|
||||
series = await compute_breadth_series(db)
|
||||
if not series:
|
||||
return None
|
||||
return series[max(series)]
|
||||
|
||||
@@ -30,10 +30,10 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy import exists, select, text
|
||||
from sqlalchemy.engine import Engine # noqa: F401 (typing only)
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
|
||||
|
||||
@@ -48,6 +48,7 @@ STATUS_RUNNING = "running"
|
||||
STATUS_VALIDATED = "validated"
|
||||
STATUS_PROMOTED = "promoted"
|
||||
STATUS_NO_OP = "no_op"
|
||||
STATUS_DEFERRED = "deferred"
|
||||
STATUS_FAILED = "failed"
|
||||
|
||||
_MAX_ERROR_LEN = 4000
|
||||
@@ -68,6 +69,12 @@ class ValidationResult:
|
||||
summary: dict[str, Any] = field(default_factory=dict)
|
||||
source_max_date: date | None = None
|
||||
messages: list[str] = field(default_factory=list)
|
||||
# Expected source-side lag: retry without an immediate error alert. Sources
|
||||
# can bound the quiet period with deferred_alert_after_days. Only meaningful
|
||||
# when ok=False.
|
||||
retryable: bool = False
|
||||
deferred_alert_after_days: int | None = None
|
||||
deferred_alert_messages: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
@@ -123,19 +130,46 @@ async def _last_promoted_revision(db: AsyncSession, source: str) -> str | None:
|
||||
return row.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _promotion_state_since(
|
||||
db: AsyncSession, source: str, cutoff: datetime
|
||||
) -> str:
|
||||
promoted = (
|
||||
DataImportRun.source == source,
|
||||
DataImportRun.status == STATUS_PROMOTED,
|
||||
)
|
||||
ever, recent = (
|
||||
await db.execute(
|
||||
select(
|
||||
exists().where(*promoted),
|
||||
exists().where(*promoted, DataImportRun.started_at >= cutoff),
|
||||
)
|
||||
)
|
||||
).one()
|
||||
return "recent" if recent else "stale" if ever else "never"
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
async def _alert(db: AsyncSession, source: str, code: str, messages: list[str]) -> None:
|
||||
async def _alert(
|
||||
db: AsyncSession,
|
||||
source: str,
|
||||
code: str,
|
||||
messages: list[str],
|
||||
*,
|
||||
severity: str = "error",
|
||||
dedup_hours: int = 24,
|
||||
) -> None:
|
||||
try:
|
||||
await system_event_service.log_event(
|
||||
db,
|
||||
severity="error",
|
||||
severity=severity,
|
||||
source="data_import",
|
||||
code=f"{source}_{code}",
|
||||
message=(("; ".join(messages)) or code)[:_MAX_ERROR_LEN],
|
||||
dedup_key=f"data_import:{source}:{code}",
|
||||
dedup_hours=dedup_hours,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — alerting must never mask the real outcome
|
||||
logger.exception("Failed to emit data_import alert %s/%s", source, code)
|
||||
@@ -149,7 +183,8 @@ async def run_import(
|
||||
) -> DataImportRun | None:
|
||||
"""Run one import for ``importer``.
|
||||
|
||||
Returns the recorded ``DataImportRun`` (promoted / no_op / failed), or None
|
||||
Returns the recorded ``DataImportRun`` (promoted / no_op / deferred /
|
||||
failed), or None
|
||||
when the per-source advisory lock is already held (another run is active).
|
||||
|
||||
``force`` runs even when the revision is unchanged. The revision tracks the
|
||||
@@ -208,9 +243,46 @@ async def run_import(
|
||||
run.validation_json = json.dumps(result.summary, default=str)
|
||||
|
||||
if not result.ok:
|
||||
run.status = STATUS_FAILED
|
||||
run.error_details = ("; ".join(result.messages))[:_MAX_ERROR_LEN]
|
||||
run.completed_at = _now()
|
||||
if result.retryable:
|
||||
run.status = STATUS_DEFERRED
|
||||
await session.commit()
|
||||
alert_days = result.deferred_alert_after_days
|
||||
if alert_days is not None:
|
||||
alert_days = max(1, alert_days)
|
||||
cutoff = run.started_at - timedelta(days=alert_days)
|
||||
promotion_state = await _promotion_state_since(
|
||||
session, source, cutoff
|
||||
)
|
||||
if promotion_state != "recent":
|
||||
history = (
|
||||
f"{source} import has never promoted successfully"
|
||||
if promotion_state == "never"
|
||||
else f"{source} import has not promoted successfully "
|
||||
f"within {alert_days} day(s)"
|
||||
)
|
||||
await _alert(
|
||||
session,
|
||||
source,
|
||||
"deferred_stale",
|
||||
[
|
||||
f"{history}; import remains deferred",
|
||||
*result.deferred_alert_messages,
|
||||
f"Current deferral: "
|
||||
f"{run.error_details or 'validation deferred'}",
|
||||
],
|
||||
severity="warning",
|
||||
dedup_hours=alert_days * 24,
|
||||
)
|
||||
logger.info(
|
||||
"data_import %s: deferred for retry: %s",
|
||||
source,
|
||||
result.messages,
|
||||
)
|
||||
return run
|
||||
|
||||
run.status = STATUS_FAILED
|
||||
await session.commit()
|
||||
await _alert(session, source, "validation_failed", result.messages)
|
||||
logger.warning(
|
||||
|
||||
@@ -32,7 +32,7 @@ from app.config import settings
|
||||
from app.database import insert_for_session
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import dolt_client, earnings_alignment
|
||||
from app.services import dolt_client, earnings_alignment, ticker_service
|
||||
from app.services.data_import import ValidationResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -289,7 +289,11 @@ class DoltEarningsImporter:
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _load_universe(self, db) -> dict[str, int]:
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol))).all()
|
||||
rows = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker.id, Ticker.symbol))
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
earnings_alignment.normalise_symbol(symbol): tid
|
||||
for tid, symbol in rows
|
||||
|
||||
@@ -1,15 +1,48 @@
|
||||
"""Compact chronological validation for the Regime Monitor warning score.
|
||||
"""Chronological validation for the AI/Tech Risk Monitor warning score.
|
||||
|
||||
The study calls its outcome a 10% correction, uses the first 70% of sessions to
|
||||
freeze an 80th-percentile warning threshold, and reports alarm episodes only on
|
||||
the final 30%. It is still labelled exploratory while the fixed breadth basket
|
||||
is reconstructed before its freeze date.
|
||||
The outcome is a 10% correction in the leader, never a regime break. Two rules
|
||||
are measured against it, and they answer different questions:
|
||||
|
||||
* **shipped** -- the quadrant-change rule that actually reaches Telegram
|
||||
(``alert_service._collect_regime_quadrant``). Its thresholds are fixed
|
||||
constants chosen by scenario arithmetic, so nothing is fitted, so there is no
|
||||
training set to protect and the whole sample is evaluable. This is the
|
||||
headline.
|
||||
* **fitted** -- the original study: an 80th-percentile Warning threshold frozen
|
||||
on the first 70% of sessions and measured on the last 30%. Kept because it is
|
||||
what the methodology document reports, and because a fitted threshold is a
|
||||
genuinely different question -- but it is measured on the four corrections that
|
||||
happen to fall in the holdout, which is too few to read as a property of the
|
||||
score.
|
||||
|
||||
Both are scored by the same ``evaluate_alarms`` harness, alongside ablations
|
||||
(does the quadrant machinery earn its place?), external baselines (does the
|
||||
score earn its complexity?), and a random-alarm null (is any of this better than
|
||||
chance?). Without those rows a bare "2 of 4" is unreadable in either direction.
|
||||
|
||||
The fundamental channel is compared, never fused. It appears as its own rule
|
||||
(transitions into an adverse state), as a confluence gate (a market crossing kept
|
||||
only when the state agrees), and as a market-only comparator over the identical
|
||||
window -- because with ~10 correction events and almost no fundamental history,
|
||||
any weight that combined it with the market axes would be a policy preference
|
||||
presented as a measurement.
|
||||
|
||||
Those three rows are **coverage-matched**: scored only on the sessions where the
|
||||
channel had usable context and on the corrections whose warning horizon fell
|
||||
inside it, and marked ``measurable: false`` until enough corrections are covered.
|
||||
A fundamental rule scores zero whether it is wrong or merely absent, so scoring
|
||||
it over the market rows' full sample would turn a fortnight of observations into
|
||||
a 0/10 that reads as a failed test.
|
||||
|
||||
Still labelled exploratory while the fixed breadth basket is reconstructed
|
||||
before its freeze date.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -17,17 +50,51 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.services import breadth_service, settings_store
|
||||
from app.services import regime_monitor_service as rms
|
||||
from app.services.admin_service import update_setting
|
||||
from app.services.alert_service import (
|
||||
QUAD_COOLDOWN_DAYS,
|
||||
QUAD_MARGIN,
|
||||
QUAD_X_DIV,
|
||||
QUAD_Y_DIV,
|
||||
_classify_quadrant,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KEY_REPORT = "regime_event_study"
|
||||
|
||||
# Report shape, independent of METHODOLOGY. A cached report from an older shape
|
||||
# parses fine and reports the current methodology, so without this check the
|
||||
# panel would render a report missing half its blocks. Bumping discards the cache
|
||||
# the way a methodology change does -- and it is the *only* thing that does so
|
||||
# here, because the fundamental-channel rework left METHODOLOGY on v4 (the scores
|
||||
# did not change), so the methodology check cannot catch a stale report.
|
||||
STUDY_SCHEMA = 3
|
||||
|
||||
EVENT_THRESHOLD_PCT = 10.0
|
||||
EVENT_COOLDOWN_DAYS = 40
|
||||
DRAWDOWN_LOOKBACK = 252
|
||||
HORIZON_DAYS = 20
|
||||
WARN_PERCENTILE = 80.0
|
||||
TRAIN_FRACTION = 0.70
|
||||
# Below this many holdout corrections, recall is one event away from a very
|
||||
# different headline and should not be read as a property of the score.
|
||||
MIN_EVENTS_FOR_CONFIDENCE = 8
|
||||
SENSOR_MISMATCH_TOLERANCE = 0.10
|
||||
|
||||
# _collect_regime_quadrant confirms against get_regime_history(db, days=14), so a
|
||||
# prior session older than that window is not available to confirm with.
|
||||
QUAD_HISTORY_DAYS = 14
|
||||
# Quadrants with Warning above its divider: "1" early warning, "2" active stress.
|
||||
WARNING_QUADRANTS = ("1", "2")
|
||||
STRESS_QUADRANT = ("2",)
|
||||
|
||||
# Draws for the random-alarm null. Seeded, because a cached report that moves
|
||||
# on re-run for RNG reasons is worse than no report.
|
||||
NULL_DRAWS = 2000
|
||||
NULL_SEED = 20260812
|
||||
|
||||
BASELINE_SMA_WINDOW = 50
|
||||
BASELINE_VIX_LEVEL = 20.0
|
||||
|
||||
|
||||
def _median(values: list[float]) -> float | None:
|
||||
@@ -144,37 +211,516 @@ def evaluate_alarms(
|
||||
}
|
||||
|
||||
|
||||
def _warning_series(
|
||||
def _score_rule(
|
||||
alarm_indices: list[int],
|
||||
event_indices: list[int],
|
||||
dates: list[date],
|
||||
horizon: int,
|
||||
sessions: int,
|
||||
) -> dict:
|
||||
"""``evaluate_alarms`` plus the annualised false-alarm rate for one rule.
|
||||
|
||||
The rate is ``None`` when the rule had no eligible sessions. Dividing by a
|
||||
tiny floor instead produced 5e9 alarms/year for a coverage-matched rule with
|
||||
an empty window -- a number that means "undefined" while looking like a
|
||||
measurement, which is the failure mode this whole panel is built to avoid.
|
||||
"""
|
||||
metrics = evaluate_alarms(alarm_indices, event_indices, dates, horizon)
|
||||
metrics["false_alarms_per_year"] = (
|
||||
round(metrics["false_alarms"] / (sessions / 252.0), 2) if sessions > 0 else None
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The shipped rule
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _axis_rows(
|
||||
prices: dict[str, rms.Series],
|
||||
breadth_divergence: dict[date, float],
|
||||
vix_series: rms.Series | None,
|
||||
oas_series: rms.Series | None,
|
||||
breadth_series: rms.Series | None,
|
||||
divergence_series: rms.Series | None,
|
||||
dates: list[date],
|
||||
config: dict,
|
||||
) -> dict[date, float]:
|
||||
"""Technical Warning score used historically (fundamentals have no PIT history)."""
|
||||
tickers = config["tickers"]
|
||||
smh_full = prices.get(tickers["leaders"][0], [])
|
||||
spy_full = prices.get(tickers["market"], [])
|
||||
out: dict[date, float] = {}
|
||||
observations: list[dict] | None = None,
|
||||
) -> dict[date, dict]:
|
||||
"""State and Warning per session, from the function that writes snapshots.
|
||||
|
||||
Calling ``_compute_index`` rather than re-deriving the two axes is the same
|
||||
anti-drift argument that produced ``warning_sensor_scores``: the v2 study
|
||||
re-derived Warning by hand and would have kept measuring the old construct
|
||||
through a scoring change. State has no such shared helper, so the whole
|
||||
snapshot builder is the shared definition.
|
||||
|
||||
``observations`` is the point-in-time fundamental series. It does not enter
|
||||
either score -- the fundamental channel is categorical and read by confluence
|
||||
-- but the per-session ``fundamental_state`` it produces is what the
|
||||
confluence rule below is measured on, so it has to be the same series
|
||||
production reports from. Every variant in this module reads its Warning from
|
||||
these rows, so there is no second derivation to fall out of step.
|
||||
"""
|
||||
rows: dict[date, dict] = {}
|
||||
for session in dates:
|
||||
divergence = breadth_divergence.get(session)
|
||||
relative = rms.p4_relative_strength(
|
||||
rms._closes_asof(smh_full, session),
|
||||
rms._closes_asof(spy_full, session),
|
||||
snapshot = rms._compute_index(
|
||||
prices,
|
||||
vix_series,
|
||||
oas_series,
|
||||
{},
|
||||
config,
|
||||
session,
|
||||
breadth_series=breadth_series,
|
||||
divergence_series=divergence_series,
|
||||
observations=observations or [],
|
||||
)
|
||||
values: list[tuple[float, float]] = []
|
||||
if divergence is not None:
|
||||
values.append((divergence, rms.WARNING_WEIGHTS["breadth_divergence"]))
|
||||
if relative is not None:
|
||||
values.append((relative, rms.WARNING_WEIGHTS["relative_strength"]))
|
||||
if values:
|
||||
out[session] = round(
|
||||
sum(value * weight for value, weight in values)
|
||||
/ sum(weight for _, weight in values),
|
||||
2,
|
||||
)
|
||||
state = snapshot["state"]
|
||||
warning = snapshot["warning"]
|
||||
rows[session] = {
|
||||
"state": state.get("score"),
|
||||
"warning": warning.get("score"),
|
||||
"fundamental_state": (snapshot.get("fundamental_context") or {}).get("state"),
|
||||
# `usable`, not `available`: a stale observation keeps its state for
|
||||
# display but stops counting as evidence, and an observation whose
|
||||
# extraction failed on everything is fresh but knows nothing. Either
|
||||
# one counted here would inflate the covered window with sessions the
|
||||
# channel could not have contributed to.
|
||||
"fundamental_usable": bool(
|
||||
(snapshot.get("fundamental_context") or {}).get("usable")
|
||||
),
|
||||
"state_coverage": state.get("coverage") or 0.0,
|
||||
"warning_coverage": warning.get("coverage") or 0.0,
|
||||
# The score renormalises over available sensors, so a session backed
|
||||
# by two is not drawn from the same distribution as one backed by
|
||||
# three, and a frozen threshold assumes it is.
|
||||
"warning_sensors": len(warning.get("available_pillars") or []),
|
||||
"inputs_fresh": bool((snapshot.get("data_quality") or {}).get("inputs_fresh")),
|
||||
}
|
||||
return rows
|
||||
|
||||
|
||||
def _publishable(row: dict | None) -> bool:
|
||||
"""What ``get_regime_history`` leaves for the alert to confirm against.
|
||||
|
||||
Deliberately not freshness-gated: ``_collect_regime_quadrant`` checks
|
||||
``is_fresh`` on today's live reading only, while the prior session comes from
|
||||
stored history where the only filter is a published band on both axes.
|
||||
"""
|
||||
return (
|
||||
row is not None
|
||||
and row["state"] is not None
|
||||
and row["warning"] is not None
|
||||
and row["state_coverage"] >= rms.MIN_COVERAGE
|
||||
and row["warning_coverage"] >= rms.MIN_COVERAGE
|
||||
)
|
||||
|
||||
|
||||
def _prior_publishable(
|
||||
rows: dict[date, dict], dates: list[date], index: int, history_days: int
|
||||
) -> dict | None:
|
||||
"""``valid[-2]``: the previous published session inside the 14-day window.
|
||||
|
||||
The monitor writes today's snapshot before the alert step runs
|
||||
(``job_catalog._DAILY_PIPELINE_STEPS``), so ``valid[-1]`` is today and this
|
||||
is genuinely the prior session rather than t-2.
|
||||
"""
|
||||
cutoff = dates[index] - timedelta(days=history_days)
|
||||
for position in range(index - 1, -1, -1):
|
||||
if dates[position] < cutoff:
|
||||
return None
|
||||
candidate = rows.get(dates[position])
|
||||
if _publishable(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def replay_quadrant_changes(
|
||||
rows: dict[date, dict],
|
||||
dates: list[date],
|
||||
state_divider: float = QUAD_X_DIV,
|
||||
warning_divider: float = QUAD_Y_DIV,
|
||||
margin: float = QUAD_MARGIN,
|
||||
cooldown_days: int = QUAD_COOLDOWN_DAYS,
|
||||
history_days: int = QUAD_HISTORY_DAYS,
|
||||
) -> list[dict]:
|
||||
"""Every quadrant change the shipped alert would have sent, in order.
|
||||
|
||||
A faithful replay of ``_collect_regime_quadrant``, including three details a
|
||||
state machine written from first principles gets wrong:
|
||||
|
||||
* the prior session is classified against the *current baseline*, not against
|
||||
its own predecessor, so confirmation asks "did yesterday already look like
|
||||
this change" rather than "did yesterday change too";
|
||||
* the baseline advances only when an alert actually fires, so a change that
|
||||
fails confirmation or cooldown is re-evaluated against the old quadrant on
|
||||
the next session rather than being forgotten;
|
||||
* one cooldown is shared by every quadrant change, so a 3->4 alert can
|
||||
swallow a 4->2 alert three days later.
|
||||
|
||||
Returns the fires themselves rather than alarm indices, because which
|
||||
transitions count as a *warning* is the caller's question: entering
|
||||
Warning-high territory and entering both-high territory are different rules
|
||||
over the same replay.
|
||||
"""
|
||||
fires: list[dict] = []
|
||||
baseline: str | None = None
|
||||
baseline_date: date | None = None
|
||||
|
||||
for index, session in enumerate(dates):
|
||||
row = rows.get(session)
|
||||
if not _publishable(row) or not row["inputs_fresh"]:
|
||||
continue
|
||||
x, y = float(row["state"]), float(row["warning"])
|
||||
|
||||
if baseline is None: # seeds silently, exactly as a fresh install does
|
||||
baseline = _classify_quadrant(x, y, None, margin, state_divider, warning_divider)
|
||||
baseline_date = session
|
||||
continue
|
||||
|
||||
new_quadrant = _classify_quadrant(x, y, baseline, margin, state_divider, warning_divider)
|
||||
if new_quadrant == baseline:
|
||||
continue
|
||||
|
||||
prior = _prior_publishable(rows, dates, index, history_days)
|
||||
if prior is None:
|
||||
continue
|
||||
prior_quadrant = _classify_quadrant(
|
||||
float(prior["state"]), float(prior["warning"]),
|
||||
baseline, margin, state_divider, warning_divider,
|
||||
)
|
||||
if prior_quadrant != new_quadrant:
|
||||
continue
|
||||
|
||||
if baseline_date is not None and (session - baseline_date).days < cooldown_days:
|
||||
continue
|
||||
|
||||
fires.append({
|
||||
"index": index,
|
||||
"date": session.isoformat(),
|
||||
"from": baseline,
|
||||
"to": new_quadrant,
|
||||
"state": x,
|
||||
"warning": y,
|
||||
})
|
||||
baseline, baseline_date = new_quadrant, session
|
||||
|
||||
return fires
|
||||
|
||||
|
||||
def entry_alarms(fires: list[dict], entry: tuple[str, ...]) -> list[int]:
|
||||
"""Fires that *enter* the given quadrant set from outside it."""
|
||||
return [f["index"] for f in fires if f["to"] in entry and f["from"] not in entry]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ablations, baselines, null
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _usable_adverse(rows: dict[date, dict], session: date) -> bool:
|
||||
"""Adverse *and* still within its staleness horizon.
|
||||
|
||||
Both callers need this pair, and neither may use the state alone: the state
|
||||
survives going stale so the card can show it, which would otherwise let a
|
||||
months-old read confirm crossings indefinitely.
|
||||
"""
|
||||
row = rows.get(session) or {}
|
||||
return row.get("fundamental_state") == "adverse" and bool(row.get("fundamental_usable"))
|
||||
|
||||
|
||||
def adverse_episodes(
|
||||
rows: dict[date, dict], dates: list[date], start_index: int
|
||||
) -> list[int]:
|
||||
"""Sessions where the fundamental state *becomes* usably adverse.
|
||||
|
||||
The market rules alarm on a rising-edge crossing; a categorical state has no
|
||||
crossing, so its analogue is the transition into ``adverse``. That keeps the
|
||||
row comparable with every other row in the table rather than counting every
|
||||
day the state happens to sit there.
|
||||
"""
|
||||
alarms: list[int] = []
|
||||
was_adverse = start_index > 0 and _usable_adverse(rows, dates[start_index - 1])
|
||||
for index in range(start_index, len(dates)):
|
||||
if dates[index] not in rows:
|
||||
continue
|
||||
adverse = _usable_adverse(rows, dates[index])
|
||||
if adverse and not was_adverse:
|
||||
alarms.append(index)
|
||||
was_adverse = adverse
|
||||
return alarms
|
||||
|
||||
|
||||
def confluence_episodes(
|
||||
warning_alarms: list[int], rows: dict[date, dict], dates: list[date]
|
||||
) -> list[int]:
|
||||
"""Warning crossings that happen while the fundamental state is usably adverse.
|
||||
|
||||
Deliberately gated on the market crossing rather than on either channel
|
||||
moving: it preserves the rising-edge semantics every other row uses, so the
|
||||
column measures "does requiring fundamental agreement help?" instead of a
|
||||
differently-shaped rule that cannot be compared with the others.
|
||||
"""
|
||||
return [index for index in warning_alarms if _usable_adverse(rows, dates[index])]
|
||||
|
||||
|
||||
def covered_events(
|
||||
event_indices: list[int],
|
||||
rows: dict[date, dict],
|
||||
dates: list[date],
|
||||
horizon: int,
|
||||
) -> list[int]:
|
||||
"""Corrections a fundamental rule actually had a chance to warn about.
|
||||
|
||||
An alarm counts only if it fires in ``[event - horizon, event - 1]``, so a
|
||||
correction is *coverable* only if the channel had usable context somewhere in
|
||||
that window. Scoring these rules against every correction instead would make
|
||||
one day of observation render as 0/10 -- an untested rule reported as a
|
||||
failed one, which is the exact mistake the ``measurable`` flag exists to
|
||||
prevent for the empty-table case.
|
||||
"""
|
||||
covered: list[int] = []
|
||||
for event_index in event_indices:
|
||||
window = range(max(0, event_index - horizon), event_index)
|
||||
if any(
|
||||
bool((rows.get(dates[index]) or {}).get("fundamental_usable"))
|
||||
for index in window
|
||||
):
|
||||
covered.append(event_index)
|
||||
return covered
|
||||
|
||||
|
||||
def eligible_sessions(
|
||||
rows: dict[date, dict], dates: list[date], start_index: int
|
||||
) -> int:
|
||||
"""Sessions a fundamental rule could have fired on, for the FA/year rate.
|
||||
|
||||
Annualising over the whole window instead would divide a rule's false alarms
|
||||
by years in which it was structurally incapable of firing, reporting a
|
||||
flattering rate that means nothing.
|
||||
"""
|
||||
return sum(
|
||||
1
|
||||
for session in dates[start_index:]
|
||||
if bool((rows.get(session) or {}).get("fundamental_usable"))
|
||||
)
|
||||
|
||||
|
||||
def below_average_series(
|
||||
series: rms.Series, window: int = BASELINE_SMA_WINDOW
|
||||
) -> dict[date, float]:
|
||||
"""100 while the close sits under its ``window``-session average, else 0."""
|
||||
out: dict[date, float] = {}
|
||||
closes = [value for _, value in series]
|
||||
for index, (session, close) in enumerate(series):
|
||||
if index + 1 < window:
|
||||
continue
|
||||
average = sum(closes[index + 1 - window: index + 1]) / window
|
||||
out[session] = 100.0 if close < average else 0.0
|
||||
return out
|
||||
|
||||
|
||||
def _null_model(
|
||||
alarm_count: int,
|
||||
event_indices: list[int],
|
||||
dates: list[date],
|
||||
horizon: int,
|
||||
start_index: int,
|
||||
observed_warned: int,
|
||||
draws: int = NULL_DRAWS,
|
||||
seed: int = NULL_SEED,
|
||||
) -> dict | None:
|
||||
"""Recall from alarms scattered at random over the same evaluable sessions.
|
||||
|
||||
Drawn only from sessions a real rule could have fired on: over the whole
|
||||
sample the null would be diluted by warm-up sessions and would understate
|
||||
what chance achieves. That matters here -- with ~11 events and a 20-session
|
||||
horizon, a sixth of the sample already sits inside a hit window.
|
||||
|
||||
Corrections cluster, and uniform placement does not, so this is the floor
|
||||
rather than the bar: an alarm process that clusters would beat it for
|
||||
reasons that have nothing to do with foresight.
|
||||
"""
|
||||
population = range(start_index, len(dates))
|
||||
if alarm_count <= 0 or not event_indices or alarm_count > len(population):
|
||||
return None
|
||||
rng = random.Random(seed)
|
||||
recalls: list[int] = []
|
||||
for _ in range(draws):
|
||||
picks = sorted(rng.sample(population, alarm_count))
|
||||
recalls.append(evaluate_alarms(picks, event_indices, dates, horizon)["events_warned"])
|
||||
mean = sum(recalls) / len(recalls)
|
||||
variance = sum((value - mean) ** 2 for value in recalls) / len(recalls)
|
||||
return {
|
||||
"draws": draws,
|
||||
"alarms_per_draw": alarm_count,
|
||||
"events": len(event_indices),
|
||||
"mean_warned": round(mean, 2),
|
||||
"sd_warned": round(variance ** 0.5, 2),
|
||||
"observed_warned": observed_warned,
|
||||
"p_at_least_observed": round(
|
||||
sum(1 for value in recalls if value >= observed_warned) / len(recalls), 3
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _reliability(
|
||||
dates: list[date],
|
||||
split: int,
|
||||
backing: dict[date, int],
|
||||
events_detected: int,
|
||||
events_in_holdout: int,
|
||||
) -> dict:
|
||||
"""How far the *fitted* variant's headline metrics can be trusted.
|
||||
|
||||
Two things repeatedly invite over-reading it:
|
||||
|
||||
* The holdout carries only the corrections that fall in the last 30% of the
|
||||
sample. A "2/4" is one event away from "3/4", and in practice the events
|
||||
that flip are decided by where the frozen threshold happens to land rather
|
||||
than by whether the score saw anything.
|
||||
* The score renormalises over available sensors, so a training window that
|
||||
predates a sensor's history freezes a threshold on a different construct
|
||||
than the holdout is measured against.
|
||||
|
||||
Neither applies to the shipped rule, whose thresholds are fixed constants --
|
||||
but the second one does not vanish, it relocates: see ``_era_split``.
|
||||
"""
|
||||
expected = len(rms.WARNING_WEIGHTS)
|
||||
train = [backing[d] for d in dates[:split] if d in backing]
|
||||
holdout = [backing[d] for d in dates[split:] if d in backing]
|
||||
train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0
|
||||
holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0
|
||||
return {
|
||||
"events_detected": events_detected,
|
||||
"events_in_holdout": events_in_holdout,
|
||||
"minimum_events": MIN_EVENTS_FOR_CONFIDENCE,
|
||||
"underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE,
|
||||
"sensors_expected": expected,
|
||||
"train_full_sensor_share": round(train_full * 100, 1),
|
||||
"holdout_full_sensor_share": round(holdout_full * 100, 1),
|
||||
"sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE,
|
||||
}
|
||||
|
||||
|
||||
def _era_split(
|
||||
alarms: list[int],
|
||||
event_indices: list[int],
|
||||
dates: list[date],
|
||||
horizon: int,
|
||||
start_index: int,
|
||||
credit_from: date | None,
|
||||
) -> dict | None:
|
||||
"""Shipped-rule metrics either side of the credit sensor's first session.
|
||||
|
||||
Dropping the fitted threshold makes the whole sample evaluable, which is the
|
||||
point -- but most of the extra events sit before 2023-08, where W3 does not
|
||||
exist and Warning renormalises to ``(W1*45 + W2*30)/75``. The fixed 40
|
||||
divider is then applied to a different construct than it was reasoned about,
|
||||
so the coverage caveat does not disappear with the split; it relocates from
|
||||
the threshold to the score. Reporting the two eras separately is what keeps
|
||||
the fuller sample from being a differently misleading headline.
|
||||
|
||||
The pre-credit era is close to a "Warning without W3" ablation on real
|
||||
sessions -- and a clean one, because the fundamental channel is not a term in
|
||||
Warning at all, so the two eras differ by W3 and nothing else. That stays
|
||||
true however much fundamental history accumulates.
|
||||
|
||||
Alarms and events are assigned to eras by index, so an alarm days before the
|
||||
boundary that matched an event days after it lands in the earlier era. With
|
||||
the eras years long and the events sparse, that costs nothing.
|
||||
"""
|
||||
if credit_from is None:
|
||||
return None
|
||||
boundary = next(
|
||||
(index for index, session in enumerate(dates) if session >= credit_from), None
|
||||
)
|
||||
if boundary is None or boundary <= start_index or boundary >= len(dates):
|
||||
return None
|
||||
|
||||
def slice_metrics(low: int, high: int) -> dict:
|
||||
sessions = max(0, high - low)
|
||||
metrics = _score_rule(
|
||||
[a for a in alarms if low <= a < high],
|
||||
[e for e in event_indices if low <= e < high],
|
||||
dates, horizon, sessions,
|
||||
)
|
||||
metrics.pop("per_event", None)
|
||||
metrics["sessions"] = sessions
|
||||
return metrics
|
||||
|
||||
return {
|
||||
"credit_from": credit_from.isoformat(),
|
||||
"pre_credit": {
|
||||
"label": "W1+W2 only",
|
||||
"start": dates[start_index].isoformat(),
|
||||
"end": dates[boundary - 1].isoformat(),
|
||||
**slice_metrics(start_index, boundary),
|
||||
},
|
||||
"full_coverage": {
|
||||
"label": "all three sensors",
|
||||
"start": dates[boundary].isoformat(),
|
||||
"end": dates[-1].isoformat(),
|
||||
**slice_metrics(boundary, len(dates)),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _warning_from_rows(
|
||||
rows: dict[date, dict], dates: list[date]
|
||||
) -> tuple[dict[date, float], dict[date, int]]:
|
||||
"""Published Warning per session plus how many sensors backed it.
|
||||
|
||||
Read off ``_axis_rows`` rather than recomputed. v2 re-derived Warning by hand
|
||||
from ``WARNING_WEIGHTS`` and would have kept measuring the old construct
|
||||
after a scoring change; a second derivation here would have done the same to
|
||||
any later change to how Warning is assembled -- silently, in the fitted
|
||||
variant and the ``warning_bare`` ablation, while the shipped replay moved on
|
||||
without it.
|
||||
"""
|
||||
out: dict[date, float] = {}
|
||||
backing: dict[date, int] = {}
|
||||
for session in dates:
|
||||
row = rows.get(session)
|
||||
if row is None or row["warning"] is None:
|
||||
continue
|
||||
out[session] = float(row["warning"])
|
||||
backing[session] = int(row["warning_sensors"])
|
||||
return out, backing
|
||||
|
||||
|
||||
def _rule_row(
|
||||
rule_id: str,
|
||||
label: str,
|
||||
kind: str,
|
||||
note: str,
|
||||
alarms: list[int],
|
||||
event_indices: list[int],
|
||||
dates: list[date],
|
||||
horizon: int,
|
||||
sessions: int,
|
||||
measurable: bool = True,
|
||||
) -> dict:
|
||||
"""One comparison row.
|
||||
|
||||
``measurable=False`` marks a rule whose *input* is too thin to have been
|
||||
tested, not one that failed. A fundamental rule scores 0/N whether it is
|
||||
wrong or merely absent, and a 0/N sitting in this table would read as
|
||||
tested-and-failed -- the same false precision the whole restructure exists to
|
||||
remove. It stays false until the channel has covered
|
||||
``MIN_EVENTS_FOR_CONFIDENCE`` corrections, because a 1/1 or 0/2 over a
|
||||
two-week exposure is not a result either.
|
||||
"""
|
||||
metrics = _score_rule(alarms, event_indices, dates, horizon, sessions)
|
||||
metrics.pop("per_event", None)
|
||||
return {
|
||||
"id": rule_id,
|
||||
"label": label,
|
||||
"kind": kind,
|
||||
"note": note,
|
||||
"measurable": measurable,
|
||||
**metrics,
|
||||
}
|
||||
|
||||
|
||||
async def run_event_study(
|
||||
db: AsyncSession,
|
||||
threshold_pct: float = EVENT_THRESHOLD_PCT,
|
||||
@@ -195,41 +741,203 @@ async def run_event_study(
|
||||
db, config["breadth_basket"], window=200, min_tickers=20
|
||||
)
|
||||
divergence = breadth_service.compute_divergence_series(breadth, benchmark)
|
||||
warning = _warning_series(prices, divergence, dates, config)
|
||||
oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end)
|
||||
# State needs volatility, which the Warning-only study never fetched.
|
||||
vix_series = await rms._fetch_fred_series("VIXCLS", start, end)
|
||||
# The point-in-time fundamental series. It is not in either score; it drives
|
||||
# the categorical channel the confluence rule below is measured on.
|
||||
observations = await rms.get_fundamental_observations(db)
|
||||
# The credit sensor cannot reach back as far as the price history does (the
|
||||
# upstream series is capped at ~3 years), so the earlier part of the sample
|
||||
# scores on W1+W2 alone via renormalisation. Report where W3 starts rather
|
||||
# than letting the threshold quietly straddle two sensor sets.
|
||||
credit_from = oas_series[0][0] if oas_series else None
|
||||
|
||||
all_events = detect_events(closes, dates, threshold_pct)
|
||||
all_event_indices = [event["index"] for event in all_events]
|
||||
|
||||
# --- one pass; every rule below reads its Warning from these rows ----
|
||||
rows = _axis_rows(
|
||||
prices,
|
||||
vix_series,
|
||||
oas_series,
|
||||
rms._mapping_series(breadth),
|
||||
rms._mapping_series(divergence),
|
||||
dates,
|
||||
config,
|
||||
observations,
|
||||
)
|
||||
warning, backing = _warning_from_rows(rows, dates)
|
||||
fires = replay_quadrant_changes(rows, dates)
|
||||
# Nothing can alarm before the baseline seeds, so every rule is measured from
|
||||
# the same session and the comparison stays like-for-like.
|
||||
seeded = next(
|
||||
(
|
||||
index
|
||||
for index, session in enumerate(dates)
|
||||
if _publishable(rows.get(session)) and rows[session]["inputs_fresh"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if seeded is None:
|
||||
return {"available": False, "reason": "no session with publishable coverage"}
|
||||
evaluable_start = seeded + 1
|
||||
evaluable_sessions = max(1, len(dates) - evaluable_start)
|
||||
evaluable_events = [index for index in all_event_indices if index >= evaluable_start]
|
||||
|
||||
warning_alarms = entry_alarms(fires, WARNING_QUADRANTS)
|
||||
shipped_metrics = _score_rule(
|
||||
warning_alarms, evaluable_events, dates, horizon, evaluable_sessions
|
||||
)
|
||||
shipped_events = shipped_metrics.pop("per_event")
|
||||
|
||||
# --- the fitted variant, kept for continuity -------------------------
|
||||
split = max(1, min(len(dates) - 1, int(len(dates) * TRAIN_FRACTION)))
|
||||
train_values = [warning[d] for d in dates[:split] if d in warning]
|
||||
warn_threshold = _percentile(train_values, WARN_PERCENTILE)
|
||||
if warn_threshold is None:
|
||||
return {"available": False, "reason": "insufficient warning history"}
|
||||
|
||||
all_events = detect_events(closes, dates, threshold_pct)
|
||||
holdout_events = [event["index"] for event in all_events if event["index"] >= split]
|
||||
alarms = alarm_episodes(warning, dates, warn_threshold, start_index=split)
|
||||
metrics = evaluate_alarms(alarms, holdout_events, dates, horizon)
|
||||
holdout_events = [index for index in all_event_indices if index >= split]
|
||||
fitted_alarms = alarm_episodes(warning, dates, warn_threshold, start_index=split)
|
||||
holdout_sessions = max(1, len(dates) - split)
|
||||
metrics["false_alarms_per_year"] = round(
|
||||
metrics["false_alarms"] / (holdout_sessions / 252.0), 2
|
||||
fitted_metrics = _score_rule(
|
||||
fitted_alarms, holdout_events, dates, horizon, holdout_sessions
|
||||
)
|
||||
fitted_events = fitted_metrics.pop("per_event")
|
||||
reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events))
|
||||
|
||||
# --- ablations and baselines, all on fixed thresholds ----------------
|
||||
# Fitted thresholds are deliberately excluded here: a threshold fitted on the
|
||||
# full sample would have lookahead the shipped rule does not, and one fitted
|
||||
# on a training split could only be scored on the four holdout events. Fixed
|
||||
# constants keep every row on the same events over the same sessions.
|
||||
state_series = {
|
||||
session: row["state"] for session, row in rows.items() if row["state"] is not None
|
||||
}
|
||||
vix_indicator = {
|
||||
session: value
|
||||
for session in dates
|
||||
if (value := rms._value_asof(vix_series, session)) is not None
|
||||
}
|
||||
# The fundamental channel is categorical and never enters a score, so it is
|
||||
# compared as its own rule and as a confluence gate rather than tuned as a
|
||||
# weight. With an empty observation series both are unmeasurable, and say so.
|
||||
fundamental_alarms = adverse_episodes(rows, dates, evaluable_start)
|
||||
confluence_alarms = confluence_episodes(warning_alarms, rows, dates)
|
||||
# Coverage-matched denominators. These rules only existed on the sessions the
|
||||
# channel had usable context, so scoring them over the whole window would
|
||||
# report an exposure they never had -- and one day of coverage would render
|
||||
# as 0/10.
|
||||
fundamental_events = covered_events(evaluable_events, rows, dates, horizon)
|
||||
fundamental_sessions = eligible_sessions(rows, dates, evaluable_start)
|
||||
fundamental_measurable = len(fundamental_events) >= MIN_EVENTS_FOR_CONFIDENCE
|
||||
comparison = [
|
||||
_rule_row(
|
||||
"fundamental_adverse", "Fundamental context turns adverse", "fundamental",
|
||||
"The third channel on its own: transitions into an adverse capex / "
|
||||
"earnings-reaction state, with no market input at all.",
|
||||
fundamental_alarms, fundamental_events, dates, horizon, fundamental_sessions,
|
||||
measurable=fundamental_measurable,
|
||||
),
|
||||
_rule_row(
|
||||
"confluence", "Confluence: Warning crossing while adverse", "fundamental",
|
||||
"The shipped market crossing, kept only when the fundamental channel "
|
||||
"agrees. Answers whether requiring agreement buys precision, at what "
|
||||
"cost in recall.",
|
||||
confluence_alarms, fundamental_events, dates, horizon, fundamental_sessions,
|
||||
measurable=fundamental_measurable,
|
||||
),
|
||||
_rule_row(
|
||||
"market_over_covered", "Quadrant alert, covered window only", "fundamental",
|
||||
"The shipped market rule scored on exactly the events, sessions and "
|
||||
"alarms the two rows above were scored on. Without it, any difference "
|
||||
"between them and the headline could be the window rather than the "
|
||||
"channel.",
|
||||
# Alarms are restricted to the covered window too: counting crossings
|
||||
# that fired when the channel had no context would compare the market
|
||||
# rule's full exposure against the channel's partial one.
|
||||
[
|
||||
index
|
||||
for index in warning_alarms
|
||||
if index >= evaluable_start
|
||||
and bool((rows.get(dates[index]) or {}).get("fundamental_usable"))
|
||||
],
|
||||
fundamental_events, dates, horizon, fundamental_sessions,
|
||||
measurable=fundamental_measurable,
|
||||
),
|
||||
_rule_row(
|
||||
"quadrant_stress_entry", "Quadrant alert, both axes high", "ablation",
|
||||
"The same replay, recording only entries into the both-high quadrant. "
|
||||
"State is coincident by construction, so requiring it should convert "
|
||||
"leads into confirmations.",
|
||||
entry_alarms(fires, STRESS_QUADRANT),
|
||||
evaluable_events, dates, horizon, evaluable_sessions,
|
||||
),
|
||||
_rule_row(
|
||||
"warning_bare", f"Warning >= {QUAD_Y_DIV:.0f} (bare crossing)", "ablation",
|
||||
"The shipped divider with none of the quadrant machinery: no State "
|
||||
"condition, no hysteresis, no confirmation, no cooldown.",
|
||||
alarm_episodes(warning, dates, QUAD_Y_DIV, start_index=evaluable_start),
|
||||
evaluable_events, dates, horizon, evaluable_sessions,
|
||||
),
|
||||
_rule_row(
|
||||
"state_bare", f"State >= {QUAD_X_DIV:.0f} (bare crossing)", "ablation",
|
||||
"The coincident axis alone. State measures stress that has already "
|
||||
"arrived, so a competitive lead here would be surprising.",
|
||||
alarm_episodes(state_series, dates, QUAD_X_DIV, start_index=evaluable_start),
|
||||
evaluable_events, dates, horizon, evaluable_sessions,
|
||||
),
|
||||
_rule_row(
|
||||
"smh_below_50dma", f"{leader} below its {BASELINE_SMA_WINDOW}-DMA", "baseline",
|
||||
"The crudest possible trend rule, and free.",
|
||||
alarm_episodes(
|
||||
below_average_series(benchmark, BASELINE_SMA_WINDOW), dates,
|
||||
50.0, start_index=evaluable_start,
|
||||
),
|
||||
evaluable_events, dates, horizon, evaluable_sessions,
|
||||
),
|
||||
_rule_row(
|
||||
"vix_level", f"VIX >= {BASELINE_VIX_LEVEL:.0f}", "baseline",
|
||||
"The market's own risk gauge, unweighted and unmodelled.",
|
||||
alarm_episodes(
|
||||
vix_indicator, dates, BASELINE_VIX_LEVEL, start_index=evaluable_start
|
||||
),
|
||||
evaluable_events, dates, horizon, evaluable_sessions,
|
||||
),
|
||||
]
|
||||
|
||||
null_model = _null_model(
|
||||
len(warning_alarms), evaluable_events, dates, horizon,
|
||||
evaluable_start, shipped_metrics["events_warned"],
|
||||
# Passed rather than defaulted: a default argument binds the constant at
|
||||
# import, so overriding it (in tests) would silently do nothing.
|
||||
draws=NULL_DRAWS, seed=NULL_SEED,
|
||||
)
|
||||
eras = _era_split(
|
||||
warning_alarms, evaluable_events, dates, horizon, evaluable_start, credit_from
|
||||
)
|
||||
basket_asof = date.fromisoformat(config["basket_asof"])
|
||||
retrospective = dates[split] < basket_asof
|
||||
retrospective = dates[evaluable_start] < basket_asof
|
||||
evaluation = "exploratory" if retrospective else "holdout"
|
||||
lead_text = (
|
||||
f"median lead {metrics['median_lead_days']:.0f} sessions"
|
||||
if metrics["median_lead_days"] is not None
|
||||
f"median lead {shipped_metrics['median_lead_days']:.0f} sessions"
|
||||
if shipped_metrics["median_lead_days"] is not None
|
||||
else "no successful warning lead"
|
||||
)
|
||||
summary = (
|
||||
f"{evaluation.capitalize()} chronological test: warning episodes preceded "
|
||||
f"{metrics['events_warned']}/{metrics['events']} 10% corrections; "
|
||||
f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} "
|
||||
f"false alarms/year, {lead_text}."
|
||||
f"{evaluation.capitalize()} replay of the shipped quadrant alert over "
|
||||
f"{evaluable_sessions} sessions: it entered Warning-high territory ahead of "
|
||||
f"{shipped_metrics['events_warned']} of {shipped_metrics['events']} 10% "
|
||||
f"corrections, with {shipped_metrics['false_alarms_per_year']:.1f} false "
|
||||
f"alarms/year and {lead_text}. Its dividers are fixed constants rather than "
|
||||
f"fitted, so there is no training split and every detected correction is "
|
||||
f"evaluable — compare it against the ablations and baselines below before "
|
||||
f"reading the ratio as good or bad."
|
||||
)
|
||||
per_event = metrics.pop("per_event")
|
||||
|
||||
report = {
|
||||
"available": True,
|
||||
"schema": STUDY_SCHEMA,
|
||||
"methodology": rms.METHODOLOGY,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"evaluation": evaluation,
|
||||
@@ -240,22 +948,69 @@ async def run_event_study(
|
||||
"event_threshold_pct": threshold_pct,
|
||||
"event_cooldown_days": EVENT_COOLDOWN_DAYS,
|
||||
"horizon_days": horizon,
|
||||
"train_fraction": TRAIN_FRACTION,
|
||||
"warn_percentile": WARN_PERCENTILE,
|
||||
"warn_threshold": round(warn_threshold, 1),
|
||||
"credit_sensor_from": credit_from.isoformat() if credit_from else None,
|
||||
"basket_hash": rms._basket_hash(config["breadth_basket"]),
|
||||
"basket_asof": config["basket_asof"],
|
||||
},
|
||||
# The channel's actual exposure, which is what its rows are scored on.
|
||||
# The series starts empty -- the observation lived in a single
|
||||
# overwritten settings slot until 2026-08-12 -- and it accumulates one
|
||||
# observation at a time, so for a long while these rows are unmeasurable
|
||||
# rather than unsuccessful. Stating the exposure is what stops the table
|
||||
# inventing a failed result out of a thin one.
|
||||
"fundamental_coverage": {
|
||||
"observations": len(observations),
|
||||
"sessions_eligible": fundamental_sessions,
|
||||
"evaluable_sessions": evaluable_sessions,
|
||||
"events_covered": len(fundamental_events),
|
||||
"events_evaluable": len(evaluable_events),
|
||||
"minimum_events": MIN_EVENTS_FOR_CONFIDENCE,
|
||||
"measurable": fundamental_measurable,
|
||||
},
|
||||
"sample": {
|
||||
"start": dates[0].isoformat(),
|
||||
"end": dates[-1].isoformat(),
|
||||
"train_end": dates[split - 1].isoformat(),
|
||||
"test_start": dates[split].isoformat(),
|
||||
"sessions": len(dates),
|
||||
"holdout_sessions": holdout_sessions,
|
||||
# Not "test_start": the shipped rule fits nothing, so this is where
|
||||
# the baseline seeds and every rule becomes measurable, not where a
|
||||
# holdout begins. The fitted variant's split lives under "fitted".
|
||||
"evaluable_from": dates[evaluable_start].isoformat(),
|
||||
"evaluable_sessions": evaluable_sessions,
|
||||
"events_detected": len(all_events),
|
||||
"events_evaluable": len(evaluable_events),
|
||||
},
|
||||
"metrics": metrics,
|
||||
"events": per_event,
|
||||
"shipped": {
|
||||
"rule": {
|
||||
"state_divider": QUAD_X_DIV,
|
||||
"warning_divider": QUAD_Y_DIV,
|
||||
"margin": QUAD_MARGIN,
|
||||
"confirm_sessions": 2,
|
||||
"cooldown_days": QUAD_COOLDOWN_DAYS,
|
||||
"entry": "Warning-high quadrant (early warning or active stress)",
|
||||
},
|
||||
"metrics": shipped_metrics,
|
||||
"events": shipped_events,
|
||||
"quadrant_changes": len(fires),
|
||||
"fires": fires,
|
||||
"by_era": eras,
|
||||
},
|
||||
"comparison": comparison,
|
||||
"null_model": null_model,
|
||||
"fitted": {
|
||||
"params": {
|
||||
"train_fraction": TRAIN_FRACTION,
|
||||
"warn_percentile": WARN_PERCENTILE,
|
||||
"warn_threshold": round(warn_threshold, 1),
|
||||
},
|
||||
"sample": {
|
||||
"train_end": dates[split - 1].isoformat(),
|
||||
"test_start": dates[split].isoformat(),
|
||||
"holdout_sessions": holdout_sessions,
|
||||
},
|
||||
"metrics": fitted_metrics,
|
||||
"events": fitted_events,
|
||||
},
|
||||
"reliability": reliability,
|
||||
"recent_breadth": [
|
||||
{"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)}
|
||||
for d in dates[-90:]
|
||||
@@ -265,9 +1020,15 @@ async def run_event_study(
|
||||
logger.info(json.dumps({
|
||||
"event": "regime_event_study_complete",
|
||||
"evaluation": evaluation,
|
||||
"events": metrics["events"],
|
||||
"warned": metrics["events_warned"],
|
||||
"false_alarms_per_year": metrics["false_alarms_per_year"],
|
||||
"shipped_events": shipped_metrics["events"],
|
||||
"shipped_warned": shipped_metrics["events_warned"],
|
||||
"shipped_false_alarms_per_year": shipped_metrics["false_alarms_per_year"],
|
||||
"quadrant_changes": len(fires),
|
||||
"fitted_events": fitted_metrics["events"],
|
||||
"fitted_warned": fitted_metrics["events_warned"],
|
||||
"null_p_at_least_observed": (null_model or {}).get("p_at_least_observed"),
|
||||
"underpowered": reliability["underpowered"],
|
||||
"sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"],
|
||||
}))
|
||||
return report
|
||||
|
||||
@@ -286,4 +1047,8 @@ async def get_event_study_report(db: AsyncSession) -> dict | None:
|
||||
report = json.loads(setting.value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return report if report.get("methodology") == rms.METHODOLOGY else None
|
||||
if report.get("methodology") != rms.METHODOLOGY:
|
||||
return None
|
||||
# A pre-replay report parses fine and carries the current methodology, so the
|
||||
# shape has to be checked separately or the panel renders a headline-less v4.
|
||||
return report if report.get("schema") == STUDY_SCHEMA else None
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Refresh the fundamentals compat cache from local SEC/Dolt bulk data.
|
||||
|
||||
``fundamental_data`` is the table scoring reads. This is its only writer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import insert_for_session
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.score import CompositeScore, DimensionScore
|
||||
from app.services import fundamentals_candidate_service
|
||||
|
||||
_SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
||||
|
||||
|
||||
async def refresh(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
today: date | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Replace every ticker's compat-cache row in one database transaction.
|
||||
|
||||
Candidate values are assembled before the first write and use only local
|
||||
PostgreSQL tables. A failure rolls the whole refresh back. Only changes to
|
||||
the three scoring inputs invalidate cached scores; market cap and the next
|
||||
earnings date are display-only.
|
||||
"""
|
||||
refreshed_at = now or datetime.now(timezone.utc)
|
||||
candidates = await fundamentals_candidate_service.build_candidates(
|
||||
db, today=today
|
||||
)
|
||||
ticker_ids = [candidate.ticker_id for candidate in candidates]
|
||||
existing = await _existing_by_ticker(db, ticker_ids)
|
||||
changed_ids = {
|
||||
candidate.ticker_id
|
||||
for candidate in candidates
|
||||
if _score_inputs_changed(existing.get(candidate.ticker_id), candidate)
|
||||
}
|
||||
|
||||
for candidate in candidates:
|
||||
unavailable_json = json.dumps(
|
||||
candidate.unavailable_fields, sort_keys=True
|
||||
)
|
||||
stmt = insert_for_session(db, FundamentalData).values(
|
||||
ticker_id=candidate.ticker_id,
|
||||
pe_ratio=candidate.pe_ratio,
|
||||
revenue_growth=candidate.revenue_growth,
|
||||
earnings_surprise=candidate.earnings_surprise,
|
||||
market_cap=candidate.market_cap,
|
||||
next_earnings_date=candidate.next_earnings_date,
|
||||
fetched_at=refreshed_at,
|
||||
unavailable_fields_json=unavailable_json,
|
||||
)
|
||||
await db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["ticker_id"],
|
||||
set_={
|
||||
"pe_ratio": stmt.excluded.pe_ratio,
|
||||
"revenue_growth": stmt.excluded.revenue_growth,
|
||||
"earnings_surprise": stmt.excluded.earnings_surprise,
|
||||
"market_cap": stmt.excluded.market_cap,
|
||||
"next_earnings_date": stmt.excluded.next_earnings_date,
|
||||
"fetched_at": stmt.excluded.fetched_at,
|
||||
"unavailable_fields_json": (
|
||||
stmt.excluded.unavailable_fields_json
|
||||
),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
dimension_ids = await _fundamental_dimension_ids(db, changed_ids)
|
||||
composite_ids = await _composite_ids(db, changed_ids)
|
||||
if dimension_ids:
|
||||
await db.execute(
|
||||
update(DimensionScore)
|
||||
.where(DimensionScore.ticker_id.in_(dimension_ids))
|
||||
.values(is_stale=True)
|
||||
)
|
||||
if composite_ids:
|
||||
await db.execute(
|
||||
update(CompositeScore)
|
||||
.where(CompositeScore.ticker_id.in_(composite_ids))
|
||||
.values(is_stale=True)
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"refreshed": len(candidates),
|
||||
"score_inputs_changed": len(changed_ids),
|
||||
"dimension_scores_staled": len(dimension_ids),
|
||||
"composite_scores_staled": len(composite_ids),
|
||||
}
|
||||
|
||||
|
||||
async def _existing_by_ticker(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, FundamentalData]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalData).where(
|
||||
FundamentalData.ticker_id.in_(ticker_ids)
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
return {row.ticker_id: row for row in rows}
|
||||
|
||||
|
||||
async def _fundamental_dimension_ids(
|
||||
db: AsyncSession, ticker_ids: set[int]
|
||||
) -> set[int]:
|
||||
if not ticker_ids:
|
||||
return set()
|
||||
rows = await db.execute(
|
||||
select(DimensionScore.ticker_id).where(
|
||||
DimensionScore.ticker_id.in_(ticker_ids),
|
||||
DimensionScore.dimension == "fundamental",
|
||||
)
|
||||
)
|
||||
return set(rows.scalars())
|
||||
|
||||
|
||||
async def _composite_ids(
|
||||
db: AsyncSession, ticker_ids: set[int]
|
||||
) -> set[int]:
|
||||
if not ticker_ids:
|
||||
return set()
|
||||
rows = await db.execute(
|
||||
select(CompositeScore.ticker_id).where(
|
||||
CompositeScore.ticker_id.in_(ticker_ids)
|
||||
)
|
||||
)
|
||||
return set(rows.scalars())
|
||||
|
||||
|
||||
def _score_inputs_changed(
|
||||
existing: FundamentalData | None,
|
||||
candidate: fundamentals_candidate_service.CandidateFundamentals,
|
||||
) -> bool:
|
||||
if existing is None:
|
||||
return True
|
||||
return any(
|
||||
getattr(existing, field) != getattr(candidate, field)
|
||||
for field in _SCORE_FIELDS
|
||||
)
|
||||
@@ -1,22 +1,19 @@
|
||||
"""Fundamental data service.
|
||||
"""Fundamental data read access.
|
||||
|
||||
Stores fundamental data (P/E, revenue growth, earnings surprise, market cap)
|
||||
and marks the fundamental dimension score as stale on new data.
|
||||
``fundamental_data`` is the compat cache scoring reads. It is written solely by
|
||||
``fundamental_data_refresh_service`` from SEC snapshots, Dolt earnings events and
|
||||
stored closes; nothing fetches it per ticker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import insert_for_session
|
||||
from app.exceptions import NotFoundError
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.score import DimensionScore
|
||||
from app.models.ticker import Ticker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,65 +29,6 @@ async def _get_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
||||
return ticker
|
||||
|
||||
|
||||
async def store_fundamental(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
pe_ratio: float | None = None,
|
||||
revenue_growth: float | None = None,
|
||||
earnings_surprise: float | None = None,
|
||||
market_cap: float | None = None,
|
||||
next_earnings_date=None,
|
||||
unavailable_fields: dict[str, str] | None = None,
|
||||
) -> FundamentalData:
|
||||
"""Store or update fundamental data for a ticker.
|
||||
|
||||
Keeps a single latest snapshot per ticker. On new data, marks the
|
||||
fundamental dimension score as stale (if one exists).
|
||||
"""
|
||||
ticker = await _get_ticker(db, symbol)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
unavailable_fields_json = json.dumps(unavailable_fields or {})
|
||||
|
||||
stmt = insert_for_session(db, FundamentalData).values(
|
||||
ticker_id=ticker.id,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
next_earnings_date=next_earnings_date,
|
||||
fetched_at=now,
|
||||
unavailable_fields_json=unavailable_fields_json,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["ticker_id"],
|
||||
set_={
|
||||
"pe_ratio": stmt.excluded.pe_ratio,
|
||||
"revenue_growth": stmt.excluded.revenue_growth,
|
||||
"earnings_surprise": stmt.excluded.earnings_surprise,
|
||||
"market_cap": stmt.excluded.market_cap,
|
||||
"next_earnings_date": stmt.excluded.next_earnings_date,
|
||||
"fetched_at": stmt.excluded.fetched_at,
|
||||
"unavailable_fields_json": stmt.excluded.unavailable_fields_json,
|
||||
},
|
||||
).returning(FundamentalData)
|
||||
record = (await db.execute(stmt)).scalar_one()
|
||||
|
||||
# Mark fundamental dimension score as stale if it exists
|
||||
# TODO: Use DimensionScore service when built
|
||||
await db.execute(
|
||||
update(DimensionScore)
|
||||
.where(
|
||||
DimensionScore.ticker_id == ticker.id,
|
||||
DimensionScore.dimension == "fundamental",
|
||||
)
|
||||
.values(is_stale=True)
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
return record
|
||||
|
||||
|
||||
async def get_fundamental(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Local SEC/Dolt candidate values for the fundamentals compat cache.
|
||||
|
||||
This is the read path behind the ``fundamental_data`` refresh. It never contacts
|
||||
SEC or Dolt: every input comes from PostgreSQL, so price- and earnings-driven
|
||||
values can still refresh when an upstream import is unchanged or unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import fundamentals_derivation as deriv
|
||||
from app.services import ticker_service
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CandidateFundamentals:
|
||||
ticker_id: int
|
||||
symbol: str
|
||||
cik: str | None
|
||||
pe_ratio: float | None
|
||||
revenue_growth: float | None
|
||||
earnings_surprise: float | None
|
||||
market_cap: float | None
|
||||
next_earnings_date: date | None
|
||||
price_date: date | None
|
||||
unavailable_fields: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
async def build_candidates(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> list[CandidateFundamentals]:
|
||||
"""Derive current cache candidates using only already-stored data."""
|
||||
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
||||
tickers = list(
|
||||
(
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
if not tickers:
|
||||
return []
|
||||
|
||||
ticker_ids = [ticker.id for ticker in tickers]
|
||||
ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
|
||||
derived_by_cik = await _derived_by_cik(db, ciks)
|
||||
closes_by_ticker = await _latest_closes(db, ticker_ids)
|
||||
surprise_by_ticker, next_by_ticker = await _earnings_values(
|
||||
db, ticker_ids, today
|
||||
)
|
||||
|
||||
out: list[CandidateFundamentals] = []
|
||||
for ticker in tickers:
|
||||
derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
|
||||
close = closes_by_ticker.get(ticker.id)
|
||||
price = close[0] if close is not None else None
|
||||
price_date = close[1] if close is not None else None
|
||||
growth_series = (
|
||||
derived.metrics.get("revenue_growth_yoy")
|
||||
if derived is not None
|
||||
else None
|
||||
)
|
||||
|
||||
pe_ratio = (
|
||||
_pe(price, derived.ttm_diluted_eps)
|
||||
if derived is not None
|
||||
else None
|
||||
)
|
||||
revenue_growth = (
|
||||
float(growth_series.value)
|
||||
if growth_series is not None and _finite(growth_series.value)
|
||||
else None
|
||||
)
|
||||
earnings_surprise = surprise_by_ticker.get(ticker.id)
|
||||
market_cap = (
|
||||
_market_cap(price, derived.shares_outstanding)
|
||||
if derived is not None
|
||||
else None
|
||||
)
|
||||
next_earnings_date = next_by_ticker.get(ticker.id)
|
||||
|
||||
out.append(
|
||||
CandidateFundamentals(
|
||||
ticker_id=ticker.id,
|
||||
symbol=ticker.symbol,
|
||||
cik=ticker.cik,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
next_earnings_date=next_earnings_date,
|
||||
price_date=price_date,
|
||||
unavailable_fields=_availability_metadata(
|
||||
derived=derived,
|
||||
price=price,
|
||||
pe_ratio=pe_ratio,
|
||||
revenue_growth=revenue_growth,
|
||||
earnings_surprise=earnings_surprise,
|
||||
market_cap=market_cap,
|
||||
next_earnings_date=next_earnings_date,
|
||||
),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def _derived_by_cik(
|
||||
db: AsyncSession, ciks: list[str]
|
||||
) -> dict[str, deriv.DerivedFundamentals]:
|
||||
if not ciks:
|
||||
return {}
|
||||
grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
|
||||
)
|
||||
).scalars()
|
||||
for row in rows:
|
||||
grouped[row.cik].append(row)
|
||||
return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
|
||||
|
||||
|
||||
async def _latest_closes(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, tuple[float, date]]:
|
||||
latest = (
|
||||
select(
|
||||
OHLCVRecord.ticker_id,
|
||||
func.max(OHLCVRecord.date).label("max_date"),
|
||||
)
|
||||
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
||||
.group_by(OHLCVRecord.ticker_id)
|
||||
.subquery()
|
||||
)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
OHLCVRecord.ticker_id,
|
||||
OHLCVRecord.close,
|
||||
OHLCVRecord.date,
|
||||
).join(
|
||||
latest,
|
||||
(OHLCVRecord.ticker_id == latest.c.ticker_id)
|
||||
& (OHLCVRecord.date == latest.c.max_date),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
ticker_id: (float(close), close_date)
|
||||
for ticker_id, close, close_date in rows
|
||||
if _finite(close)
|
||||
}
|
||||
|
||||
|
||||
async def _earnings_values(
|
||||
db: AsyncSession,
|
||||
ticker_ids: list[int],
|
||||
today: date,
|
||||
) -> tuple[dict[int, float], dict[int, date]]:
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(EarningsEvent)
|
||||
.where(EarningsEvent.ticker_id.in_(ticker_ids))
|
||||
.order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
|
||||
)
|
||||
).scalars()
|
||||
surprises: dict[int, float] = {}
|
||||
upcoming: dict[int, date] = {}
|
||||
for row in rows:
|
||||
if row.announce_date >= today:
|
||||
current = upcoming.get(row.ticker_id)
|
||||
if current is None or row.announce_date < current:
|
||||
upcoming[row.ticker_id] = row.announce_date
|
||||
continue
|
||||
if row.ticker_id in surprises:
|
||||
continue
|
||||
surprise = _surprise(row.eps_estimate, row.eps_actual)
|
||||
if surprise is not None:
|
||||
surprises[row.ticker_id] = surprise
|
||||
return surprises, upcoming
|
||||
|
||||
|
||||
def _availability_metadata(
|
||||
*,
|
||||
derived: deriv.DerivedFundamentals | None,
|
||||
price: float | None,
|
||||
pe_ratio: float | None,
|
||||
revenue_growth: float | None,
|
||||
earnings_surprise: float | None,
|
||||
market_cap: float | None,
|
||||
next_earnings_date: date | None,
|
||||
) -> dict[str, str]:
|
||||
metadata: dict[str, str] = {}
|
||||
|
||||
if pe_ratio is not None:
|
||||
metadata["source_pe_ratio"] = "sec_facts+ohlcv_records"
|
||||
elif derived is None or derived.latest_period_end is None:
|
||||
metadata["pe_ratio"] = "no SEC fundamental snapshots"
|
||||
elif not _finite(price) or price <= 0:
|
||||
metadata["pe_ratio"] = "no usable PostgreSQL close"
|
||||
elif derived.ttm_diluted_eps_caveat:
|
||||
metadata["pe_ratio"] = derived.ttm_diluted_eps_caveat
|
||||
else:
|
||||
metadata["pe_ratio"] = "no positive SEC-derived TTM diluted EPS"
|
||||
|
||||
if revenue_growth is not None:
|
||||
metadata["source_revenue_growth"] = "sec_facts"
|
||||
else:
|
||||
metadata["revenue_growth"] = "SEC-derived TTM revenue growth unavailable"
|
||||
|
||||
if earnings_surprise is not None:
|
||||
metadata["source_earnings_surprise"] = "dolt_earnings"
|
||||
else:
|
||||
metadata["earnings_surprise"] = (
|
||||
"no completed earnings event with actual and nonzero estimate"
|
||||
)
|
||||
|
||||
if market_cap is not None:
|
||||
metadata["source_market_cap"] = "sec_facts+ohlcv_records"
|
||||
if derived is not None and derived.shares_outstanding_estimated:
|
||||
metadata["market_cap_estimated"] = (
|
||||
"shares use the SEC weighted-average diluted fallback"
|
||||
)
|
||||
elif derived is None or derived.latest_period_end is None:
|
||||
metadata["market_cap"] = "no SEC fundamental snapshots"
|
||||
elif not _finite(price) or price <= 0:
|
||||
metadata["market_cap"] = "no usable PostgreSQL close"
|
||||
else:
|
||||
metadata["market_cap"] = "SEC-derived shares outstanding unavailable"
|
||||
|
||||
if next_earnings_date is not None:
|
||||
metadata["source_next_earnings_date"] = "dolt_earnings"
|
||||
else:
|
||||
metadata["next_earnings_date"] = "no upcoming earnings event"
|
||||
return metadata
|
||||
|
||||
|
||||
def _surprise(
|
||||
estimate: float | None,
|
||||
actual: float | None,
|
||||
) -> float | None:
|
||||
if not _finite(estimate) or not _finite(actual) or estimate == 0:
|
||||
return None
|
||||
return (float(actual) - float(estimate)) / abs(float(estimate)) * 100.0
|
||||
|
||||
|
||||
def _pe(price: float | None, ttm_eps: float | None) -> float | None:
|
||||
if (
|
||||
not _finite(price)
|
||||
or price <= 0
|
||||
or not _finite(ttm_eps)
|
||||
or ttm_eps <= 0
|
||||
):
|
||||
return None
|
||||
return float(price) / float(ttm_eps)
|
||||
|
||||
|
||||
def _market_cap(
|
||||
price: float | None,
|
||||
shares_outstanding: float | None,
|
||||
) -> float | None:
|
||||
if (
|
||||
not _finite(price)
|
||||
or price <= 0
|
||||
or not _finite(shares_outstanding)
|
||||
or shares_outstanding <= 0
|
||||
):
|
||||
return None
|
||||
return float(price) * float(shares_outstanding)
|
||||
|
||||
|
||||
def _finite(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(value)
|
||||
)
|
||||
@@ -1,598 +0,0 @@
|
||||
"""Read-only A5 comparison of legacy and SEC/Dolt fundamental inputs.
|
||||
|
||||
The report deliberately does not write ``fundamental_data`` or score tables.
|
||||
It reconstructs the current legacy and candidate fundamental scores, projects
|
||||
their composite-score/rank effect with the active weights, and archives a
|
||||
timestamped JSON + CSV bundle for explicit human approval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import fundamentals_derivation as deriv
|
||||
|
||||
REPORT_VERSION = 1
|
||||
APPROVAL_STATUS = "pending_explicit_approval"
|
||||
FIELD_KEYS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
||||
MIN_SCORE_METRICS = 2
|
||||
|
||||
# Materiality is a review aid, never an automatic cutover verdict. Definition
|
||||
# changes remain visible even when a delta falls inside these bands.
|
||||
FIELD_TOLERANCES = {
|
||||
"pe_ratio": {"absolute": 1.0, "relative_pct": 10.0},
|
||||
"revenue_growth": {"absolute": 2.0, "relative_pct": None},
|
||||
"earnings_surprise": {"absolute": 2.0, "relative_pct": None},
|
||||
}
|
||||
DEFINITION_NOTES = {
|
||||
"pe_ratio": (
|
||||
"Legacy provider P/E convention versus latest close divided by "
|
||||
"SEC-derived TTM diluted EPS."
|
||||
),
|
||||
"revenue_growth": (
|
||||
"Legacy provider growth convention versus SEC-derived TTM revenue YoY."
|
||||
),
|
||||
"earnings_surprise": (
|
||||
"Legacy provider latest surprise versus latest completed Dolt earnings "
|
||||
"event with actual and estimate."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def fundamental_score(
|
||||
pe_ratio: float | None,
|
||||
revenue_growth: float | None,
|
||||
earnings_surprise: float | None,
|
||||
) -> float | None:
|
||||
"""Match the production fundamental-dimension formula without persistence."""
|
||||
scores: list[float] = []
|
||||
if _finite(pe_ratio) and pe_ratio > 0:
|
||||
scores.append(max(0.0, min(100.0, 100.0 - (pe_ratio - 15.0) * (100.0 / 30.0))))
|
||||
if _finite(revenue_growth):
|
||||
scores.append(max(0.0, min(100.0, 50.0 + revenue_growth * 2.5)))
|
||||
if _finite(earnings_surprise):
|
||||
scores.append(max(0.0, min(100.0, 50.0 + earnings_surprise * 5.0)))
|
||||
return sum(scores) / len(scores) if len(scores) >= MIN_SCORE_METRICS else None
|
||||
|
||||
|
||||
async def build_report(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
today: date | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a point-in-time parity report from one database session."""
|
||||
generated_at = generated_at or datetime.now(timezone.utc)
|
||||
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
||||
|
||||
# A report must not mix rows from before and after a concurrent import
|
||||
# promotion. The scheduled job provides a fresh session, so establish the
|
||||
# production snapshot before its first query and have Postgres enforce the
|
||||
# no-write contract as well. SQLite tests retain their normal transaction.
|
||||
if db.get_bind().dialect.name == "postgresql":
|
||||
connection = await db.connection(
|
||||
execution_options={"isolation_level": "REPEATABLE READ"}
|
||||
)
|
||||
await connection.execute(text("SET TRANSACTION READ ONLY"))
|
||||
|
||||
tickers = list((await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars())
|
||||
ticker_ids = [ticker.id for ticker in tickers]
|
||||
ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
|
||||
|
||||
legacy_by_ticker = await _legacy_values(db, ticker_ids)
|
||||
derived_by_cik = await _derived_by_cik(db, ciks)
|
||||
closes_by_ticker = await _latest_closes(db, ticker_ids)
|
||||
surprise_by_ticker = await _latest_surprises(db, ticker_ids, today)
|
||||
source_runs = await _source_runs(db)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for ticker in tickers:
|
||||
legacy = legacy_by_ticker.get(ticker.id)
|
||||
derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
|
||||
close = closes_by_ticker.get(ticker.id)
|
||||
candidate_pe = (
|
||||
_pe(close[0], derived.ttm_diluted_eps)
|
||||
if close is not None and derived is not None
|
||||
else None
|
||||
)
|
||||
growth_series = (
|
||||
derived.metrics.get("revenue_growth_yoy") if derived is not None else None
|
||||
)
|
||||
candidate = {
|
||||
"pe_ratio": candidate_pe,
|
||||
"revenue_growth": growth_series.value if growth_series else None,
|
||||
"earnings_surprise": surprise_by_ticker.get(ticker.id),
|
||||
}
|
||||
legacy_values = {
|
||||
"pe_ratio": legacy.pe_ratio if legacy else None,
|
||||
"revenue_growth": legacy.revenue_growth if legacy else None,
|
||||
"earnings_surprise": legacy.earnings_surprise if legacy else None,
|
||||
}
|
||||
fields = {
|
||||
key: _field_comparison(key, legacy_values[key], candidate[key])
|
||||
for key in FIELD_KEYS
|
||||
}
|
||||
legacy_score = fundamental_score(**legacy_values)
|
||||
candidate_score = fundamental_score(**candidate)
|
||||
rows.append(
|
||||
{
|
||||
"symbol": ticker.symbol,
|
||||
"cik": ticker.cik,
|
||||
"legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
|
||||
"price_date": _iso(close[1]) if close else None,
|
||||
"fields": fields,
|
||||
"scores": {
|
||||
"legacy_fundamental": _round(legacy_score),
|
||||
"candidate_fundamental": _round(candidate_score),
|
||||
"fundamental_delta": _delta(legacy_score, candidate_score),
|
||||
"legacy_fundamental_rank": None,
|
||||
"candidate_fundamental_rank": None,
|
||||
"fundamental_rank_change": None,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
_attach_ranks(rows, "legacy_fundamental", "legacy_fundamental_rank")
|
||||
_attach_ranks(rows, "candidate_fundamental", "candidate_fundamental_rank")
|
||||
for row in rows:
|
||||
scores = row["scores"]
|
||||
scores["fundamental_rank_change"] = _rank_change(
|
||||
scores["legacy_fundamental_rank"], scores["candidate_fundamental_rank"]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_version": REPORT_VERSION,
|
||||
"generated_at": generated_at.isoformat(),
|
||||
"as_of_date": today.isoformat(),
|
||||
"approval_status": APPROVAL_STATUS,
|
||||
"read_only": True,
|
||||
"fundamental_score_formula": (
|
||||
"Equal-weighted mean of 2+ available sub-scores: P/E = "
|
||||
"clamp(100-(pe-15)*(100/30)); revenue growth = "
|
||||
"clamp(50+growth*2.5); earnings surprise = "
|
||||
"clamp(50+surprise*5)."
|
||||
),
|
||||
"source_runs": source_runs,
|
||||
"definition_notes": DEFINITION_NOTES,
|
||||
"materiality_notes": {
|
||||
"fields": FIELD_TOLERANCES,
|
||||
"fundamental_score_absolute": 5.0,
|
||||
"automatic_cutover": False,
|
||||
},
|
||||
"summary": _summary(rows),
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
def store_report(report: dict[str, Any], report_dir: str | Path) -> dict[str, str]:
|
||||
"""Atomically archive JSON/CSV artifacts and update the latest manifest."""
|
||||
directory = Path(report_dir).expanduser().resolve()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
stamp = _artifact_stamp(report["generated_at"])
|
||||
json_name = f"fundamentals-parity-{stamp}.json"
|
||||
csv_name = f"fundamentals-parity-{stamp}.csv"
|
||||
json_path = directory / json_name
|
||||
csv_path = directory / csv_name
|
||||
|
||||
_atomic_write(json_path, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
_atomic_write(csv_path, report_csv(report))
|
||||
manifest = {
|
||||
"generated_at": report["generated_at"],
|
||||
"json_file": json_name,
|
||||
"csv_file": csv_name,
|
||||
}
|
||||
_atomic_write(
|
||||
directory / "latest.json",
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
)
|
||||
return {
|
||||
"json": str(json_path),
|
||||
"csv": str(csv_path),
|
||||
"manifest": str(directory / "latest.json"),
|
||||
}
|
||||
|
||||
|
||||
async def generate_and_store(
|
||||
db: AsyncSession,
|
||||
report_dir: str | Path,
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
today: date | None = None,
|
||||
) -> tuple[dict[str, Any], dict[str, str]]:
|
||||
report = await build_report(db, generated_at=generated_at, today=today)
|
||||
return report, store_report(report, report_dir)
|
||||
|
||||
|
||||
def load_latest(report_dir: str | Path) -> dict[str, Any] | None:
|
||||
manifest = _load_manifest(report_dir)
|
||||
if manifest is None:
|
||||
return None
|
||||
try:
|
||||
path = _manifest_artifact(report_dir, manifest, "json_file")
|
||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
return loaded if isinstance(loaded, dict) else None
|
||||
|
||||
|
||||
def load_latest_csv(report_dir: str | Path) -> tuple[str, str] | None:
|
||||
return _load_latest_text_artifact(report_dir, "csv_file")
|
||||
|
||||
|
||||
def load_latest_json(report_dir: str | Path) -> tuple[str, str] | None:
|
||||
return _load_latest_text_artifact(report_dir, "json_file")
|
||||
|
||||
|
||||
def _load_latest_text_artifact(
|
||||
report_dir: str | Path, manifest_key: str
|
||||
) -> tuple[str, str] | None:
|
||||
manifest = _load_manifest(report_dir)
|
||||
if manifest is None:
|
||||
return None
|
||||
try:
|
||||
path = _manifest_artifact(report_dir, manifest, manifest_key)
|
||||
return path.name, path.read_text(encoding="utf-8")
|
||||
except (OSError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def report_csv(report: dict[str, Any]) -> str:
|
||||
output = io.StringIO(newline="")
|
||||
columns = [
|
||||
"symbol",
|
||||
"cik",
|
||||
"legacy_fetched_at",
|
||||
"price_date",
|
||||
*(
|
||||
f"{field}_{suffix}"
|
||||
for field in FIELD_KEYS
|
||||
for suffix in ("legacy", "candidate", "absolute_delta", "relative_delta_pct", "material")
|
||||
),
|
||||
"legacy_fundamental",
|
||||
"candidate_fundamental",
|
||||
"fundamental_delta",
|
||||
"legacy_fundamental_rank",
|
||||
"candidate_fundamental_rank",
|
||||
"fundamental_rank_change",
|
||||
]
|
||||
writer = csv.DictWriter(output, fieldnames=columns)
|
||||
writer.writeheader()
|
||||
for row in report.get("rows", []):
|
||||
flat = {
|
||||
"symbol": row["symbol"],
|
||||
"cik": row.get("cik"),
|
||||
"legacy_fetched_at": row.get("legacy_fetched_at"),
|
||||
"price_date": row.get("price_date"),
|
||||
**row["scores"],
|
||||
}
|
||||
for field in FIELD_KEYS:
|
||||
comparison = row["fields"][field]
|
||||
for suffix in (
|
||||
"legacy",
|
||||
"candidate",
|
||||
"absolute_delta",
|
||||
"relative_delta_pct",
|
||||
"material",
|
||||
):
|
||||
flat[f"{field}_{suffix}"] = comparison.get(suffix)
|
||||
writer.writerow(flat)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
async def _legacy_values(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, FundamentalData]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalData).where(FundamentalData.ticker_id.in_(ticker_ids))
|
||||
)
|
||||
).scalars()
|
||||
return {row.ticker_id: row for row in rows}
|
||||
|
||||
|
||||
async def _derived_by_cik(
|
||||
db: AsyncSession, ciks: list[str]
|
||||
) -> dict[str, deriv.DerivedFundamentals]:
|
||||
if not ciks:
|
||||
return {}
|
||||
grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
|
||||
)
|
||||
).scalars()
|
||||
for row in rows:
|
||||
grouped[row.cik].append(row)
|
||||
return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
|
||||
|
||||
|
||||
async def _latest_closes(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, tuple[float, date]]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
latest = (
|
||||
select(OHLCVRecord.ticker_id, func.max(OHLCVRecord.date).label("max_date"))
|
||||
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
||||
.group_by(OHLCVRecord.ticker_id)
|
||||
.subquery()
|
||||
)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(OHLCVRecord.ticker_id, OHLCVRecord.close, OHLCVRecord.date).join(
|
||||
latest,
|
||||
(OHLCVRecord.ticker_id == latest.c.ticker_id)
|
||||
& (OHLCVRecord.date == latest.c.max_date),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
ticker_id: (float(close), close_date)
|
||||
for ticker_id, close, close_date in rows
|
||||
if _finite(close)
|
||||
}
|
||||
|
||||
|
||||
async def _latest_surprises(
|
||||
db: AsyncSession, ticker_ids: list[int], today: date
|
||||
) -> dict[int, float]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(EarningsEvent)
|
||||
.where(
|
||||
EarningsEvent.ticker_id.in_(ticker_ids),
|
||||
EarningsEvent.announce_date < today,
|
||||
)
|
||||
.order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
|
||||
)
|
||||
).scalars()
|
||||
out: dict[int, float] = {}
|
||||
for row in rows:
|
||||
if row.ticker_id in out:
|
||||
continue
|
||||
surprise = _surprise(row.eps_estimate, row.eps_actual)
|
||||
if surprise is not None:
|
||||
out[row.ticker_id] = surprise
|
||||
return out
|
||||
|
||||
|
||||
async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
|
||||
sources = ("sec_facts", "dolt_earnings")
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(DataImportRun)
|
||||
.where(
|
||||
DataImportRun.source.in_(sources),
|
||||
DataImportRun.status.in_(("promoted", "no_op")),
|
||||
)
|
||||
.order_by(DataImportRun.id.desc())
|
||||
)
|
||||
).scalars()
|
||||
latest: dict[str, dict[str, Any] | None] = {source: None for source in sources}
|
||||
for row in rows:
|
||||
if latest[row.source] is None:
|
||||
latest[row.source] = {
|
||||
"run_id": row.id,
|
||||
"status": row.status,
|
||||
"revision": row.revision,
|
||||
"source_max_date": _iso(row.source_max_date),
|
||||
"completed_at": _iso(row.completed_at),
|
||||
}
|
||||
return latest
|
||||
|
||||
|
||||
def _field_comparison(
|
||||
key: str, legacy: float | None, candidate: float | None
|
||||
) -> dict[str, Any]:
|
||||
legacy = float(legacy) if _finite(legacy) else None
|
||||
candidate = float(candidate) if _finite(candidate) else None
|
||||
absolute = _delta(legacy, candidate)
|
||||
relative = (
|
||||
None
|
||||
if absolute is None or legacy in (None, 0)
|
||||
else round(absolute / abs(legacy) * 100.0, 4)
|
||||
)
|
||||
tolerance = FIELD_TOLERANCES[key]
|
||||
material = False
|
||||
if absolute is not None:
|
||||
material = abs(absolute) > tolerance["absolute"]
|
||||
relative_limit = tolerance["relative_pct"]
|
||||
if relative_limit is not None:
|
||||
material = material and relative is not None and abs(relative) > relative_limit
|
||||
return {
|
||||
"legacy": _round(legacy),
|
||||
"candidate": _round(candidate),
|
||||
"absolute_delta": absolute,
|
||||
"relative_delta_pct": relative,
|
||||
"material": material,
|
||||
"definition_changed": True,
|
||||
}
|
||||
|
||||
|
||||
def _attach_ranks(rows: list[dict[str, Any]], value_key: str, rank_key: str) -> None:
|
||||
values = [
|
||||
row["scores"][value_key]
|
||||
for row in rows
|
||||
if _finite(row["scores"][value_key])
|
||||
]
|
||||
for row in rows:
|
||||
value = row["scores"][value_key]
|
||||
row["scores"][rank_key] = (
|
||||
1 + sum(other > value for other in values) if _finite(value) else None
|
||||
)
|
||||
|
||||
|
||||
def _summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
field_stats = {}
|
||||
for key in FIELD_KEYS:
|
||||
comparisons = [row["fields"][key] for row in rows]
|
||||
deltas = [
|
||||
abs(item["absolute_delta"])
|
||||
for item in comparisons
|
||||
if item["absolute_delta"] is not None
|
||||
]
|
||||
field_stats[key] = {
|
||||
"legacy_available": sum(item["legacy"] is not None for item in comparisons),
|
||||
"candidate_available": sum(
|
||||
item["candidate"] is not None for item in comparisons
|
||||
),
|
||||
"both_available": len(deltas),
|
||||
"material_differences": sum(item["material"] for item in comparisons),
|
||||
"median_absolute_delta": _round(statistics.median(deltas) if deltas else None),
|
||||
"p95_absolute_delta": _round(_percentile(deltas, 0.95)),
|
||||
"max_absolute_delta": _round(max(deltas) if deltas else None),
|
||||
}
|
||||
|
||||
fundamental_deltas = _score_deltas(rows, "fundamental_delta")
|
||||
changed_rows = sorted(
|
||||
(
|
||||
{
|
||||
"symbol": row["symbol"],
|
||||
"fundamental_delta": row["scores"]["fundamental_delta"],
|
||||
"fundamental_rank_change": row["scores"]["fundamental_rank_change"],
|
||||
}
|
||||
for row in rows
|
||||
if row["scores"]["fundamental_delta"] is not None
|
||||
),
|
||||
key=lambda item: (
|
||||
abs(item["fundamental_delta"] or 0),
|
||||
),
|
||||
reverse=True,
|
||||
)[:20]
|
||||
return {
|
||||
"universe_count": len(rows),
|
||||
"legacy_fundamental_score_available": _count_score(
|
||||
rows, "legacy_fundamental"
|
||||
),
|
||||
"candidate_fundamental_score_available": _count_score(
|
||||
rows, "candidate_fundamental"
|
||||
),
|
||||
"fundamental_scores_compared": len(fundamental_deltas),
|
||||
"fundamental_score_material_changes": sum(
|
||||
abs(delta) > 5.0 for delta in fundamental_deltas
|
||||
),
|
||||
"fundamental_rank_changes": _rank_change_count(
|
||||
rows, "fundamental_rank_change"
|
||||
),
|
||||
"field_stats": field_stats,
|
||||
"largest_changes": changed_rows,
|
||||
}
|
||||
|
||||
|
||||
def _score_deltas(rows: Iterable[dict[str, Any]], key: str) -> list[float]:
|
||||
return [
|
||||
row["scores"][key]
|
||||
for row in rows
|
||||
if row["scores"][key] is not None
|
||||
]
|
||||
|
||||
|
||||
def _count_score(rows: Iterable[dict[str, Any]], key: str) -> int:
|
||||
return sum(row["scores"][key] is not None for row in rows)
|
||||
|
||||
|
||||
def _rank_change_count(rows: Iterable[dict[str, Any]], key: str) -> int:
|
||||
return sum(
|
||||
row["scores"][key] not in (None, 0)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
|
||||
def _rank_change(legacy: int | None, candidate: int | None) -> int | None:
|
||||
# Positive means the candidate improved its rank.
|
||||
return legacy - candidate if legacy is not None and candidate is not None else None
|
||||
|
||||
|
||||
def _surprise(estimate: float | None, actual: float | None) -> float | None:
|
||||
if not _finite(estimate) or not _finite(actual) or estimate == 0:
|
||||
return None
|
||||
return (actual - estimate) / abs(estimate) * 100.0
|
||||
|
||||
|
||||
def _pe(price: float | None, ttm_eps: float | None) -> float | None:
|
||||
if not _finite(price) or price <= 0 or not _finite(ttm_eps) or ttm_eps <= 0:
|
||||
return None
|
||||
return price / ttm_eps
|
||||
|
||||
|
||||
def _delta(legacy: float | None, candidate: float | None) -> float | None:
|
||||
if not _finite(legacy) or not _finite(candidate):
|
||||
return None
|
||||
return round(candidate - legacy, 4)
|
||||
|
||||
|
||||
def _round(value: float | None, digits: int = 4) -> float | None:
|
||||
return round(float(value), digits) if _finite(value) else None
|
||||
|
||||
|
||||
def _percentile(values: list[float], quantile: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
index = max(0, math.ceil(quantile * len(ordered)) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _finite(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(value)
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
return value.isoformat() if value is not None else None
|
||||
|
||||
|
||||
def _artifact_stamp(raw: str) -> str:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
return parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
|
||||
|
||||
def _atomic_write(path: Path, content: str) -> None:
|
||||
temp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
temp.write_text(content, encoding="utf-8", newline="")
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def _load_manifest(report_dir: str | Path) -> dict[str, Any] | None:
|
||||
path = Path(report_dir).expanduser().resolve() / "latest.json"
|
||||
try:
|
||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
return loaded if isinstance(loaded, dict) else None
|
||||
|
||||
|
||||
def _manifest_artifact(
|
||||
report_dir: str | Path, manifest: dict[str, Any], key: str
|
||||
) -> Path:
|
||||
directory = Path(report_dir).expanduser().resolve()
|
||||
name = Path(str(manifest.get(key, ""))).name
|
||||
if not name:
|
||||
raise ValueError(f"Latest parity manifest has no {key}")
|
||||
return directory / name
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Actionability gate for incomplete SEC fundamentals."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import exists, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.sec_filing_gap import SecFilingGap
|
||||
from app.models.ticker import Ticker
|
||||
|
||||
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
||||
|
||||
# How recent the issuer's own newest filing must be for an *escalated* gap to
|
||||
# stop pausing setups. A gap pauses an issuer until it is either resolved or
|
||||
# superseded by a later ingested filing — which assumes the gap is temporary.
|
||||
# It is not always: SEC's per-company Company-Facts files can go stale
|
||||
# indefinitely (2026-08, 43 large caps whose Q2 10-Qs the frames API carried but
|
||||
# whose companyfacts files never received), and since the supersede rule needs a
|
||||
# *successfully ingested* later filing, a stale file also swallows the next
|
||||
# quarter. The pause is then open-ended rather than seasonal.
|
||||
#
|
||||
# So the pause hands off to the alert: once `filing_gap_aged` has escalated a gap
|
||||
# to an operator (`escalated_at`), the issuer resumes on the fundamentals it does
|
||||
# have — provided those are recent. An issuer with nothing this fresh has no
|
||||
# usable fundamentals at all and stays paused, which is the case the gate was
|
||||
# built for. The retry queue is untouched: `active_gaps` still returns these, so
|
||||
# the importer keeps retrying and a recovered filing still resolves normally.
|
||||
GAP_GATE_RECENT_FILING_DAYS = 180
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SetupQuality:
|
||||
eligible: bool
|
||||
code: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
async def active_gaps(
|
||||
db: AsyncSession,
|
||||
ciks: set[str] | None = None,
|
||||
) -> list[SecFilingGap]:
|
||||
"""Unresolved gaps that have not been superseded by a later filing."""
|
||||
matching_snapshot = exists().where(
|
||||
FundamentalSnapshot.accession == SecFilingGap.accession
|
||||
)
|
||||
gap_date = func.coalesce(
|
||||
SecFilingGap.index_date,
|
||||
func.date(SecFilingGap.first_seen_at),
|
||||
)
|
||||
later_snapshot = exists().where(
|
||||
FundamentalSnapshot.cik == SecFilingGap.cik,
|
||||
FundamentalSnapshot.form.in_(_SEC_FORMS),
|
||||
FundamentalSnapshot.filed_date > gap_date,
|
||||
)
|
||||
stmt = select(SecFilingGap).where(
|
||||
~matching_snapshot,
|
||||
~later_snapshot,
|
||||
)
|
||||
if ciks is not None:
|
||||
if not ciks:
|
||||
return []
|
||||
stmt = stmt.where(SecFilingGap.cik.in_(ciks))
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def gap_exempt_ciks(
|
||||
db: AsyncSession, gaps: list[SecFilingGap]
|
||||
) -> set[str]:
|
||||
"""CIKs whose gaps have stopped pausing setups (see GAP_GATE_RECENT_FILING_DAYS).
|
||||
|
||||
Every one of a CIK's active gaps must be escalated: one fresh gap alongside an
|
||||
old one still means a filing we might yet ingest, which is worth pausing for.
|
||||
|
||||
Public because the importer alerts on this exact transition (a CIK dropping
|
||||
out of this set is a pause coming back on) and the rule must not exist twice.
|
||||
"""
|
||||
by_cik: dict[str, list[SecFilingGap]] = defaultdict(list)
|
||||
for gap in gaps:
|
||||
by_cik[gap.cik].append(gap)
|
||||
escalated = {
|
||||
cik
|
||||
for cik, items in by_cik.items()
|
||||
if all(gap.escalated_at is not None for gap in items)
|
||||
}
|
||||
if not escalated:
|
||||
return set()
|
||||
cutoff = (
|
||||
datetime.now(timezone.utc) - timedelta(days=GAP_GATE_RECENT_FILING_DAYS)
|
||||
).date()
|
||||
rows = await db.execute(
|
||||
select(FundamentalSnapshot.cik)
|
||||
.where(
|
||||
FundamentalSnapshot.cik.in_(escalated),
|
||||
FundamentalSnapshot.form.in_(_SEC_FORMS),
|
||||
FundamentalSnapshot.filed_date >= cutoff,
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
return set(rows.scalars())
|
||||
|
||||
|
||||
async def _latest_validation(db: AsyncSession) -> dict:
|
||||
payload = (
|
||||
await db.execute(
|
||||
select(DataImportRun.validation_json)
|
||||
.where(
|
||||
DataImportRun.source == "sec_facts",
|
||||
DataImportRun.validation_json.is_not(None),
|
||||
)
|
||||
.order_by(DataImportRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not payload:
|
||||
return {}
|
||||
try:
|
||||
summary = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return summary if isinstance(summary, dict) else {}
|
||||
|
||||
|
||||
async def blocked_reasons_by_cik(
|
||||
db: AsyncSession,
|
||||
ciks: set[str] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Current SEC blocker code by CIK; no historical audit scan."""
|
||||
if ciks is not None and not ciks:
|
||||
return {}
|
||||
|
||||
gaps = await active_gaps(db, ciks)
|
||||
# Escalated gaps on issuers that still have recent fundamentals no longer
|
||||
# pause setups, on either path below — the summary mirrors the same filings.
|
||||
exempt = await gap_exempt_ciks(db, gaps)
|
||||
reasons = {
|
||||
gap.cik: "sec_filing_gap" for gap in gaps if gap.cik not in exempt
|
||||
}
|
||||
summary = await _latest_validation(db)
|
||||
|
||||
def wanted(cik: str) -> bool:
|
||||
return ciks is None or cik in ciks
|
||||
|
||||
# New summaries carry the complete compact CIK set while the detailed lists
|
||||
# stay capped for audit readability. Detailed entries supply the reason.
|
||||
for cik in summary.get("setup_blocked_ciks") or []:
|
||||
normalized = str(cik) if cik else ""
|
||||
if normalized and wanted(normalized) and normalized not in exempt:
|
||||
reasons.setdefault(normalized, "sec_filing_gap")
|
||||
for item in summary.get("missing_xbrl") or []:
|
||||
normalized = str(item.get("cik") or "")
|
||||
if normalized and wanted(normalized) and normalized not in exempt:
|
||||
reasons.setdefault(normalized, "sec_filing_gap")
|
||||
for cik in summary.get("no_xbrl_ciks") or []:
|
||||
normalized = str(cik) if cik else ""
|
||||
if normalized and wanted(normalized):
|
||||
reasons[normalized] = "no_xbrl_filings"
|
||||
for item in summary.get("no_xbrl_filings") or []:
|
||||
normalized = str(item.get("cik") or "")
|
||||
if normalized and wanted(normalized):
|
||||
reasons[normalized] = "no_xbrl_filings"
|
||||
return reasons
|
||||
|
||||
|
||||
async def blocked_ciks(db: AsyncSession) -> set[str]:
|
||||
return set(await blocked_reasons_by_cik(db))
|
||||
|
||||
|
||||
async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
|
||||
ciks = await blocked_ciks(db)
|
||||
if not ciks:
|
||||
return set()
|
||||
rows = await db.execute(select(Ticker.id).where(Ticker.cik.in_(ciks)))
|
||||
return {int(ticker_id) for ticker_id in rows.scalars()}
|
||||
|
||||
|
||||
async def ticker_quality(db: AsyncSession, symbol: str) -> SetupQuality:
|
||||
ticker = (
|
||||
await db.execute(
|
||||
select(Ticker).where(Ticker.symbol == symbol.strip().upper())
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if ticker is None or not ticker.cik:
|
||||
return SetupQuality(eligible=True)
|
||||
reason = (await blocked_reasons_by_cik(db, {ticker.cik})).get(ticker.cik)
|
||||
if reason == "no_xbrl_filings":
|
||||
return SetupQuality(
|
||||
eligible=False,
|
||||
code=reason,
|
||||
message=(
|
||||
"No SEC 10-K/10-Q is available for this registrant, so new setups "
|
||||
"are paused. New registrants clear automatically after their first "
|
||||
"filing; a successor shell needs an SEC CIK override."
|
||||
),
|
||||
)
|
||||
if reason:
|
||||
return SetupQuality(
|
||||
eligible=False,
|
||||
code=reason,
|
||||
message=(
|
||||
"A recent SEC filing is still being reconciled, so new setups are "
|
||||
"paused. The scheduled fundamentals import retries it automatically."
|
||||
),
|
||||
)
|
||||
return SetupQuality(eligible=True)
|
||||
|
||||
|
||||
async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool:
|
||||
cik = (
|
||||
await db.execute(select(Ticker.cik).where(Ticker.id == ticker_id))
|
||||
).scalar_one_or_none()
|
||||
if not cik:
|
||||
return True
|
||||
return cik not in await blocked_reasons_by_cik(db, {cik})
|
||||
@@ -100,6 +100,8 @@ async def fetch_and_ingest(
|
||||
symbol: str,
|
||||
start_date: date | None = None,
|
||||
end_date: date | None = None,
|
||||
*,
|
||||
refresh_sr: bool = True,
|
||||
) -> IngestionResult:
|
||||
"""Fetch OHLCV data from provider and upsert into Price Store.
|
||||
|
||||
@@ -129,7 +131,12 @@ async def fetch_and_ingest(
|
||||
if bar_count < minimum_backfill_bars:
|
||||
start_date = backfill_start
|
||||
elif progress is not None:
|
||||
start_date = progress.last_ingested_date + timedelta(days=1)
|
||||
# Re-fetch the latest stored session so an in-progress daily bar can
|
||||
# be overwritten as the market moves. Starting one day later makes
|
||||
# every subsequent intraday, near-close, and manual refresh skip
|
||||
# today's bar once the first partial snapshot has been stored.
|
||||
# The price-store upsert keeps this one-session overlap idempotent.
|
||||
start_date = progress.last_ingested_date
|
||||
else:
|
||||
start_date = backfill_start
|
||||
|
||||
@@ -239,7 +246,7 @@ async def fetch_and_ingest(
|
||||
ticker.symbol,
|
||||
ingested_count,
|
||||
)
|
||||
if ingested_count > 0:
|
||||
if ingested_count > 0 and refresh_sr:
|
||||
await _refresh_structural_sr(db, ticker.symbol)
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
@@ -249,9 +256,28 @@ async def fetch_and_ingest(
|
||||
message=f"Rate limited. Ingested {ingested_count} records. Resume available.",
|
||||
)
|
||||
|
||||
if ingested_count > 0:
|
||||
if ingested_count > 0 and refresh_sr:
|
||||
await _refresh_structural_sr(db, ticker.symbol)
|
||||
|
||||
# Incremental fetches deliberately overlap the latest stored session so an
|
||||
# in-progress bar can be updated. A halted/delisted symbol can therefore
|
||||
# return one old bar forever; non-empty no longer means fresh. Judge stale
|
||||
# state from the newest stored session after the upserts instead.
|
||||
latest = await _get_latest_ohlcv_date(db, ticker.id)
|
||||
gap_days = (end_date - latest).days if latest is not None else None
|
||||
if gap_days is not None and gap_days > _STALE_OHLCV_GAP_DAYS:
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
records_ingested=ingested_count,
|
||||
last_date=latest,
|
||||
status="stale",
|
||||
message=(
|
||||
f"No new bars since {latest.isoformat()} ({gap_days}d gap). "
|
||||
"The symbol may be halted, delisted, or renamed under a new ticker — "
|
||||
"check the listing and add/fetch the current symbol if it changed."
|
||||
),
|
||||
)
|
||||
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
records_ingested=ingested_count,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Single source for JobRunState reads/writes.
|
||||
|
||||
Mirrors ``settings_store``: ``record_finish`` never commits — the caller owns
|
||||
the transaction — and reads are batched so the admin listing stays one query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.job_run_state import JobRunState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _as_datetime(value: object) -> datetime | None:
|
||||
"""Runtime snapshots carry ISO strings; the column wants a datetime."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def get_map(db: AsyncSession, job_names: Iterable[str]) -> dict[str, JobRunState]:
|
||||
"""Return {job_name: row} for the given jobs that have ever finished.
|
||||
|
||||
``populate_existing`` because rows are written by core upserts, which leave
|
||||
any previously-loaded ORM instance in the identity map stale.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(JobRunState)
|
||||
.where(JobRunState.job_name.in_(list(job_names)))
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return {row.job_name: row for row in result.scalars().all()}
|
||||
|
||||
|
||||
def _insert_for(db: AsyncSession):
|
||||
"""ON CONFLICT is dialect-specific; prod is Postgres, tests are SQLite."""
|
||||
dialect = db.get_bind().dialect.name
|
||||
return pg_insert if dialect == "postgresql" else sqlite_insert
|
||||
|
||||
|
||||
async def record_finish(db: AsyncSession, job_name: str, runtime: dict) -> None:
|
||||
"""Upsert the last-run row from a scheduler runtime snapshot.
|
||||
|
||||
Atomic, and newer-wins. Select-then-insert loses races that really happen
|
||||
here: pipelines are separate scheduler jobs that can overlap, and they share
|
||||
step ids -- data_collector belongs to all four. Two of them finishing that
|
||||
step together would both see no row and both insert, and the loser's
|
||||
IntegrityError is swallowed by the caller, so the run silently vanishes.
|
||||
|
||||
The ``where`` guard is the other half: without it a slower pipeline
|
||||
finishing an *older* run last would rewind finished_at and the status with
|
||||
it, so the panel would report a stale outcome as the latest one.
|
||||
"""
|
||||
finished_at = _as_datetime(runtime.get("finished_at")) or datetime.now(timezone.utc)
|
||||
message = runtime.get("message")
|
||||
now = datetime.now(timezone.utc)
|
||||
values = {
|
||||
"job_name": job_name,
|
||||
"status": str(runtime.get("status") or "completed"),
|
||||
"started_at": _as_datetime(runtime.get("started_at")),
|
||||
"finished_at": finished_at,
|
||||
"processed": runtime.get("processed"),
|
||||
"total": runtime.get("total"),
|
||||
"message": str(message)[:4000] if message else None,
|
||||
# Set explicitly: the model's onupdate hook does not fire for a core
|
||||
# INSERT ... ON CONFLICT DO UPDATE.
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
statement = _insert_for(db)(JobRunState).values(**values)
|
||||
await db.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=[JobRunState.job_name],
|
||||
set_={key: statement.excluded[key] for key in values if key != "job_name"},
|
||||
where=JobRunState.finished_at < statement.excluded.finished_at,
|
||||
)
|
||||
)
|
||||
@@ -18,6 +18,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import ticker_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -169,7 +170,9 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa
|
||||
before scanning; the research backtest ranked each weekly setup-candidate
|
||||
cross-section, so this is the deliberate production approximation.
|
||||
"""
|
||||
result = await db.execute(select(Ticker).order_by(Ticker.symbol))
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
tickers = list(result.scalars().all())
|
||||
|
||||
benchmark_closes = await _load_activation_benchmark(db)
|
||||
|
||||
@@ -352,6 +352,7 @@ def _to_dict(
|
||||
current_price: float | None,
|
||||
benchmark_closes: dict[date, float] | None = None,
|
||||
trailing: tuple[float, float | None] | None = None,
|
||||
holding_sessions: tuple[int, int] | None = None,
|
||||
) -> dict:
|
||||
# For open trades, mark to market; for closed, the realized exit price.
|
||||
ref = current_price if trade.status == "open" else trade.close_price
|
||||
@@ -395,6 +396,8 @@ def _to_dict(
|
||||
"fill_mode": trade.fill_mode,
|
||||
"trailing_stop": trailing[0] if trailing else None,
|
||||
"trailing_distance_pct": trailing[1] if trailing else None,
|
||||
"sessions_held": holding_sessions[0] if holding_sessions else None,
|
||||
"sessions_remaining": holding_sessions[1] if holding_sessions else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -435,6 +438,35 @@ async def list_trades(
|
||||
# Current trailing-stop level + distance for open trades (when a trailing
|
||||
# policy is active).
|
||||
policy = await get_exit_policy(db)
|
||||
holding_sessions: dict[int, tuple[int, int]] = {}
|
||||
if policy["mode"] in ("time", "atr_trailing"):
|
||||
hold_days = int(policy["hold_days"])
|
||||
open_trades = [trade for trade, _ in rows if trade.status == "open"]
|
||||
if open_trades:
|
||||
ticker_ids = {trade.ticker_id for trade in open_trades}
|
||||
earliest_opened = min(trade.opened_at.date() for trade in open_trades)
|
||||
session_rows = (
|
||||
await db.execute(
|
||||
select(OHLCVRecord.ticker_id, OHLCVRecord.date)
|
||||
.where(
|
||||
OHLCVRecord.ticker_id.in_(ticker_ids),
|
||||
OHLCVRecord.date > earliest_opened,
|
||||
)
|
||||
.order_by(OHLCVRecord.ticker_id, OHLCVRecord.date)
|
||||
)
|
||||
).all()
|
||||
dates_by_ticker: dict[int, list[date]] = {}
|
||||
for ticker_id, session_date in session_rows:
|
||||
dates_by_ticker.setdefault(int(ticker_id), []).append(session_date)
|
||||
for trade in open_trades:
|
||||
dates = dates_by_ticker.get(trade.ticker_id, [])
|
||||
held = len(dates) - bisect.bisect_right(
|
||||
dates, trade.opened_at.date()
|
||||
)
|
||||
# Do not clamp: a policy shortened below the current holding
|
||||
# period must remain visible as overdue until the exit pass runs.
|
||||
holding_sessions[trade.id] = (held, hold_days - held)
|
||||
|
||||
trailing_info: dict[int, tuple[float, float | None]] = {}
|
||||
if policy["mode"] == "trailing":
|
||||
trail_frac = policy["trailing_pct"] / 100.0
|
||||
@@ -483,7 +515,14 @@ async def list_trades(
|
||||
trailing_info[t.id] = (level, dist)
|
||||
|
||||
return [
|
||||
_to_dict(t, sym, prices.get(t.ticker_id), benchmark_closes, trailing_info.get(t.id))
|
||||
_to_dict(
|
||||
t,
|
||||
sym,
|
||||
prices.get(t.ticker_id),
|
||||
benchmark_closes,
|
||||
trailing_info.get(t.id),
|
||||
holding_sessions.get(t.id),
|
||||
)
|
||||
for t, sym in rows
|
||||
]
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -27,10 +27,11 @@ from app.models.signal_context_snapshot import SignalContextSnapshot
|
||||
from app.models.ticker import Ticker
|
||||
from app.models.trade_setup import TradeSetup
|
||||
from app.services.indicator_service import _extract_ohlcv, compute_atr
|
||||
from app.services import fundamentals_quality_service, system_event_service
|
||||
from app.services.price_service import query_ohlcv
|
||||
from app.services.qualification import setup_qualifies
|
||||
from app.services.sr_service import detect_gate_target_ladder
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
from app.services.trade_policy import (
|
||||
MANUAL_BOOK,
|
||||
SHADOW_BOOK,
|
||||
@@ -526,6 +527,7 @@ async def scan_ticker(
|
||||
primary_min_rr: float | None = None,
|
||||
gate_levels_override: list[Any] | None = None,
|
||||
scan_run_id: str | None = None,
|
||||
fundamentals_eligible: bool | None = None,
|
||||
) -> list[TradeSetup]:
|
||||
"""Scan a single ticker for trade setups meeting the R:R threshold.
|
||||
|
||||
@@ -542,6 +544,17 @@ async def scan_ticker(
|
||||
"""
|
||||
ticker = await _get_ticker(db, symbol)
|
||||
|
||||
if fundamentals_eligible is None:
|
||||
fundamentals_eligible = await fundamentals_quality_service.ticker_is_eligible(
|
||||
db, ticker.id
|
||||
)
|
||||
if not fundamentals_eligible:
|
||||
logger.info(
|
||||
"Skipping %s: unresolved or unavailable SEC fundamentals",
|
||||
ticker.symbol,
|
||||
)
|
||||
return []
|
||||
|
||||
if primary_min_rr is None:
|
||||
primary_min_rr = PRIMARY_TARGET_MIN_RR
|
||||
|
||||
@@ -722,10 +735,37 @@ async def scan_all_tickers(
|
||||
# Plain ids/strings, not Ticker instances: the rollbacks below expire any
|
||||
# ORM objects held across them, and touching an expired attribute afterwards
|
||||
# triggers sync lazy-loading, which raises on an AsyncSession.
|
||||
result = await db.execute(select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol))
|
||||
result = await db.execute(
|
||||
ticker_service.active_only(
|
||||
select(Ticker.id, Ticker.symbol).order_by(Ticker.symbol)
|
||||
)
|
||||
)
|
||||
ticker_rows = [(int(ticker_id), symbol) for ticker_id, symbol in result.all()]
|
||||
total = len(ticker_rows)
|
||||
|
||||
# Data-quality failures are not weak signals: they make a ticker ineligible.
|
||||
# Resolve once for the universe scan and pass the decision into scan_ticker.
|
||||
try:
|
||||
fundamentals_blocked_ids = (
|
||||
await fundamentals_quality_service.blocked_ticker_ids(db)
|
||||
)
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
logger.exception(
|
||||
"Could not resolve fundamentals quality; blocking this scan closed"
|
||||
)
|
||||
await system_event_service.log_event_standalone(
|
||||
severity="error",
|
||||
source="rr_scanner",
|
||||
code="fundamentals_quality_unavailable",
|
||||
message=(
|
||||
"The fundamentals quality gate could not be evaluated; the "
|
||||
"universe scan was blocked to avoid issuing unchecked setups."
|
||||
),
|
||||
dedup_key="rr_scanner:fundamentals_quality_unavailable",
|
||||
)
|
||||
fundamentals_blocked_ids = {ticker_id for ticker_id, _ in ticker_rows}
|
||||
|
||||
# Gate-reset observations must use the same runtime activation settings as
|
||||
# the live setup list. If the config cannot be loaded, scan normally but do
|
||||
# not mutate reset state from an evaluation whose rules are unknown.
|
||||
@@ -765,6 +805,12 @@ async def scan_all_tickers(
|
||||
for index, (ticker_id, symbol) in enumerate(ticker_rows):
|
||||
if progress_callback is not None:
|
||||
progress_callback(index, total, symbol)
|
||||
if ticker_id in fundamentals_blocked_ids:
|
||||
logger.info(
|
||||
"Skipping %s: unresolved or unavailable SEC fundamentals",
|
||||
symbol,
|
||||
)
|
||||
continue
|
||||
# Refresh Structural S/R once, then scores. get_sr_levels is read-only;
|
||||
# without this recalculate the score path would see yesterday's zones.
|
||||
# A refresh failure still scans the ticker: qualification re-gates on
|
||||
@@ -795,6 +841,7 @@ async def scan_all_tickers(
|
||||
volatility_percentile=(ranks.get(symbol) or {}).get("volatility_percentile"),
|
||||
primary_min_rr=PRIMARY_TARGET_MIN_RR,
|
||||
scan_run_id=scan_run_id,
|
||||
fundamentals_eligible=True,
|
||||
)
|
||||
all_setups.extend(setups)
|
||||
if activation is not None:
|
||||
@@ -882,6 +929,26 @@ async def get_trade_setups(
|
||||
stmt = stmt.where(TradeSetup.recommended_action == recommended_action)
|
||||
excluded_ticker_ids: set[int] = set()
|
||||
reentry_gate_locks: dict[int, datetime] = {}
|
||||
try:
|
||||
excluded_ticker_ids.update(
|
||||
await fundamentals_quality_service.blocked_ticker_ids(db)
|
||||
)
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
logger.exception(
|
||||
"Could not resolve fundamentals quality; hiding actionable setups"
|
||||
)
|
||||
await system_event_service.log_event_standalone(
|
||||
severity="error",
|
||||
source="rr_scanner",
|
||||
code="fundamentals_quality_unavailable",
|
||||
message=(
|
||||
"The fundamentals quality gate could not be evaluated; actionable "
|
||||
"setups were hidden until the metadata check recovers."
|
||||
),
|
||||
dedup_key="rr_scanner:fundamentals_quality_unavailable",
|
||||
)
|
||||
return []
|
||||
if exclude_open_trade_tickers:
|
||||
# Manual book only. The shadow book holds the *top-ranked* names by
|
||||
# construction, so letting its positions hide setups would leave the
|
||||
|
||||
@@ -20,7 +20,7 @@ from app.database import insert_for_session
|
||||
from app.exceptions import NotFoundError, ValidationError
|
||||
from app.models.score import CompositeScore, DimensionScore
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -497,8 +497,8 @@ async def _compute_fundamental_score(
|
||||
"reason": "Earnings surprise data not available",
|
||||
})
|
||||
|
||||
# Require at least two real metrics — a single available metric (e.g. only
|
||||
# market cap is free on FMP) does not make a meaningful fundamental score.
|
||||
# Require at least two real metrics — a single available metric (e.g. an
|
||||
# issuer with only a market cap) does not make a meaningful fundamental score.
|
||||
MIN_METRICS = 2
|
||||
if len(scores) < MIN_METRICS:
|
||||
unavailable.append({
|
||||
@@ -883,7 +883,11 @@ async def get_rankings(db: AsyncSession) -> dict:
|
||||
Returns dict suitable for RankingResponse.
|
||||
"""
|
||||
weights = await _get_weights(db)
|
||||
tickers = (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars().all()
|
||||
tickers = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker).order_by(Ticker.symbol))
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
async def _load_scores() -> tuple[dict[int, CompositeScore], dict[int, dict[str, DimensionScore]]]:
|
||||
comps = {
|
||||
@@ -947,7 +951,7 @@ async def update_weights(
|
||||
await _save_weights(db, full_weights)
|
||||
|
||||
# Recompute all composite scores
|
||||
result = await db.execute(select(Ticker))
|
||||
result = await db.execute(ticker_service.active_only(select(Ticker)))
|
||||
tickers = list(result.scalars().all())
|
||||
|
||||
for ticker in tickers:
|
||||
|
||||
+165
-6
@@ -8,7 +8,9 @@ the daily filing index — behind one client that honors SEC's fair-access polic
|
||||
- request spacing well under the 10 req/s limit;
|
||||
- exponential backoff + retry on 429;
|
||||
- **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a
|
||||
403 means the UA or request pattern is wrong and retrying won't fix it.
|
||||
403 means the UA or request pattern is wrong and retrying won't fix it. The one
|
||||
exception is S3's ``AccessDenied`` on an ``/Archives/`` path, which is how the
|
||||
bucket reports an absent file (``_is_absent_archive_key``).
|
||||
|
||||
Parsing lives here (index fixed-width, submissions pagination); DB writes and the
|
||||
snapshot mapping live in the importer. No conditional GETs — the companyfacts
|
||||
@@ -37,12 +39,39 @@ logger = logging.getLogger(__name__)
|
||||
_WWW = "https://www.sec.gov"
|
||||
_DATA = "https://data.sec.gov"
|
||||
|
||||
# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py).
|
||||
# Resolve CA bundle for explicit httpx verify (matches app/providers/alpaca.py).
|
||||
_CA = os.environ.get("SSL_CERT_FILE", "")
|
||||
_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
|
||||
|
||||
_FORMS_10 = frozenset({"10-K", "10-Q", "10-K/A", "10-Q/A"})
|
||||
|
||||
# Notification of removal from listing. "25" is issuer-filed, "25-NSE" exchange-
|
||||
# filed. The Form 15 family is deliberately absent: it ends a *reporting*
|
||||
# obligation and does not mean the security stopped trading.
|
||||
_DELISTING_FORMS = frozenset({"25", "25-NSE"})
|
||||
|
||||
# ``descriptionClassSecurity`` is free text ("Common Stock", "Class A Common
|
||||
# Stock, $0.01 par value", "6.25% Notes due 2030", "Warrants", "Depositary
|
||||
# Shares"). Only a common-equity class means the ticker itself stopped trading.
|
||||
_NON_COMMON_CLASS = re.compile(
|
||||
r"\b(note|bond|debenture|preferred|warrant|right|unit|depositary|"
|
||||
r"subordinated|debt|trust)s?\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_common_stock(description: str) -> bool:
|
||||
"""Does this Form 25 security class describe common equity?
|
||||
|
||||
Requires an explicit common-stock match AND no debt/preferred/warrant marker,
|
||||
so "Depositary Shares each representing 1/1000th of Preferred" cannot pass on
|
||||
the word "shares" alone. Unrecognised text is rejected — a symbol is retired
|
||||
on this answer, so ambiguity must not read as yes.
|
||||
"""
|
||||
if _NON_COMMON_CLASS.search(description):
|
||||
return False
|
||||
return re.search(r"\bcommon\s+(stock|share)", description, re.IGNORECASE) is not None
|
||||
|
||||
|
||||
class SecError(ProviderError):
|
||||
"""SEC request failed (403, exhausted 429/5xx, timeout, transport, parse)."""
|
||||
@@ -53,11 +82,44 @@ class SecForbiddenError(SecError):
|
||||
|
||||
|
||||
class SecNotFoundError(SecError):
|
||||
"""SEC returned 404 — the resource does not exist (e.g. no index for a day).
|
||||
"""The resource does not exist (e.g. no daily index published for a day).
|
||||
|
||||
The *only* error a caller may treat as 'missing' — every other SecError
|
||||
(403, exhausted retries, 5xx, timeout) must propagate so a fetch failure is
|
||||
never mistaken for an empty result."""
|
||||
(fair-access rejection, exhausted retries, 5xx, timeout) must propagate so a
|
||||
fetch failure is never mistaken for an empty result.
|
||||
|
||||
Raised for a 404, and for the one 403 that also means "absent": see
|
||||
``_is_absent_archive_key``."""
|
||||
|
||||
|
||||
def _is_absent_archive_key(url: str, resp: httpx.Response) -> bool:
|
||||
"""True when a 403 means "this file does not exist", not "you are blocked".
|
||||
|
||||
``www.sec.gov/Archives`` is served straight out of an S3 bucket that grants
|
||||
no ``s3:ListBucket``, so a missing key cannot be answered with 404 — S3
|
||||
returns **403 with its ``AccessDenied`` XML** instead. SEC publishes a daily
|
||||
index only for business days, so every weekend and market holiday inside an
|
||||
incremental walk lands on exactly this response (verified 2026-07-30:
|
||||
``form.20260725.idx``, a Saturday, 403s while the Friday and Monday files
|
||||
return 200 on the same User-Agent).
|
||||
|
||||
A genuine fair-access rejection is distinguishable and must stay fatal: it is
|
||||
SEC's WAF interstitial — ``text/html``, "Your Request Originates from an
|
||||
Undeclared Automated Tool" — and it is returned for files that *do* exist,
|
||||
on any path. Hence the narrow gate: the Archives prefix plus S3's own error
|
||||
document. Nothing else may be downgraded to "missing"."""
|
||||
try:
|
||||
parsed = httpx.URL(url)
|
||||
except (TypeError, ValueError): # pragma: no cover — url comes from us
|
||||
return False
|
||||
if parsed.host != "www.sec.gov" or not parsed.path.startswith("/Archives/"):
|
||||
return False
|
||||
if "xml" not in resp.headers.get("Content-Type", "").lower():
|
||||
return False
|
||||
try:
|
||||
return "<Code>AccessDenied</Code>" in resp.text
|
||||
except (UnicodeDecodeError, httpx.HTTPError): # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
def _looks_like_contact_email(ua: str) -> bool:
|
||||
@@ -155,6 +217,8 @@ class SecClient:
|
||||
|
||||
code = resp.status_code
|
||||
if code == 403:
|
||||
if _is_absent_archive_key(url, resp):
|
||||
raise SecNotFoundError(f"SEC 403/AccessDenied (absent) for {url}")
|
||||
raise SecForbiddenError(
|
||||
f"SEC 403 for {url} — User-Agent/pattern rejected; set a real "
|
||||
"sec_user_agent contact email"
|
||||
@@ -216,6 +280,91 @@ class SecClient:
|
||||
"filings": filings,
|
||||
}
|
||||
|
||||
async def delisting_filing(
|
||||
self, cik: int | str, *, not_before: date | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
"""Newest Form 25 removing this issuer's COMMON stock from listing.
|
||||
|
||||
Deliberately narrow, because the caller retires a symbol on the answer:
|
||||
|
||||
- **Form 25 only.** The Form 15 family terminates a reporting obligation
|
||||
(often just a class falling under the holder threshold) and is no
|
||||
evidence that trading stopped.
|
||||
- **Class-checked.** Form 25 is filed per security class — an issuer
|
||||
delisting its notes, preferred, warrants or an ADR class while the
|
||||
common keeps trading files one too. The filing's own
|
||||
``descriptionClassSecurity`` is what separates those, so the primary
|
||||
document is fetched and read rather than trusting the form type.
|
||||
- **``not_before``** rejects a historical filing for some long-gone
|
||||
class. Without it a 2019 Form 25 would retire a symbol whose bars
|
||||
stopped in 2026, and stamp 2019 as the date.
|
||||
|
||||
Anything unreadable — no primary document (pre-2009 filings have none),
|
||||
malformed XML, unrecognised class — returns ``None``. Fail closed: the
|
||||
caller keeps warning instead of retiring on a guess.
|
||||
|
||||
Reads ``filings.recent`` directly; ``submissions()`` keeps only the
|
||||
10-K/10-Q family, so Form 25 never survives its parser.
|
||||
"""
|
||||
base = await self.get_json(f"{_DATA}/submissions/CIK{cik10(cik)}.json")
|
||||
arrays = (base.get("filings") or {}).get("recent") or {}
|
||||
forms = arrays.get("form") or []
|
||||
dates = arrays.get("filingDate") or []
|
||||
accessions = arrays.get("accessionNumber") or []
|
||||
docs = arrays.get("primaryDocument") or []
|
||||
|
||||
candidates: list[tuple[date, str, str, str]] = []
|
||||
for i, form in enumerate(forms):
|
||||
if form not in _DELISTING_FORMS or i >= len(dates) or not dates[i]:
|
||||
continue
|
||||
try:
|
||||
filed = date.fromisoformat(dates[i])
|
||||
except ValueError:
|
||||
continue
|
||||
if not_before is not None and filed < not_before:
|
||||
continue
|
||||
if i >= len(accessions) or not accessions[i]:
|
||||
continue
|
||||
candidates.append((filed, form, accessions[i], docs[i] if i < len(docs) else ""))
|
||||
|
||||
for filed, form, accession, _doc in sorted(candidates, reverse=True):
|
||||
security = await self._form25_security_class(cik, accession)
|
||||
if security is None:
|
||||
continue
|
||||
if not _is_common_stock(security):
|
||||
continue
|
||||
return {
|
||||
"form": form,
|
||||
"filing_date": filed,
|
||||
"security_class": security,
|
||||
}
|
||||
return None
|
||||
|
||||
async def _form25_security_class(
|
||||
self, cik: int | str, accession: str
|
||||
) -> str | None:
|
||||
"""``descriptionClassSecurity`` from a Form 25's primary XML, or None.
|
||||
|
||||
The rendered ``primaryDocument`` is an XSL view of this file; the raw
|
||||
``primary_doc.xml`` beside it is the structured original.
|
||||
"""
|
||||
folder = accession.replace("-", "")
|
||||
url = (
|
||||
f"{_WWW}/Archives/edgar/data/{int(cik)}/{folder}/primary_doc.xml"
|
||||
)
|
||||
try:
|
||||
body = await self.get_text(url)
|
||||
except SecNotFoundError:
|
||||
return None
|
||||
match = re.search(
|
||||
r"<descriptionClassSecurity>(.*?)</descriptionClassSecurity>",
|
||||
body,
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if match is None:
|
||||
return None
|
||||
return " ".join(match.group(1).split()) or None
|
||||
|
||||
async def companyfacts(self, cik: int | str) -> dict[str, Any]:
|
||||
"""Raw companyfacts JSON ({cik, entityName, facts})."""
|
||||
return await self.get_json(f"{_DATA}/api/xbrl/companyfacts/CIK{cik10(cik)}.json")
|
||||
@@ -252,7 +401,17 @@ class SecClient:
|
||||
try:
|
||||
text = await self.get_text(url)
|
||||
except SecNotFoundError:
|
||||
logger.info("no daily index for %s (404)", day)
|
||||
# Absent on a weekend is routine (SEC publishes business days only); on a
|
||||
# weekday it is either a market holiday or something worth a look — a SEC
|
||||
# hiccup, or a rejection page misread as absent, would otherwise let the
|
||||
# importer advance past real filings silently. Log-level only, no alert:
|
||||
# cheaper than carrying a holiday calendar just to stay quiet ~10 days/yr.
|
||||
logger.log(
|
||||
logging.INFO if day.weekday() >= 5 else logging.WARNING,
|
||||
"no daily index published for %s (%s)",
|
||||
day,
|
||||
f"{day:%a}",
|
||||
)
|
||||
return [] # weekend/holiday/not-yet-published; other errors propagate
|
||||
return _parse_form_index(text)
|
||||
|
||||
|
||||
@@ -113,8 +113,41 @@ _WEIGHTED_AVG_SHARE_CONCEPTS = [
|
||||
# us-gaap instant (balance-sheet) concepts, at end == reportDate.
|
||||
_CASH = ["CashAndCashEquivalentsAtCarryingValue"]
|
||||
_ST_INVESTMENTS = ["ShortTermInvestments", "MarketableSecuritiesCurrent"] # pick one
|
||||
# Debt is tagged in four mutually exclusive styles across large filers, and
|
||||
# composing a total means knowing which span each concept covers (measured
|
||||
# 2026-08 over a 20-issuer sample; the counts below are from it).
|
||||
#
|
||||
# ``LongTermDebt`` already spans current + noncurrent maturities — Apple tags all
|
||||
# three and 71.34bn + 11.01bn = 82.30bn confirms it — so its complement is only
|
||||
# genuinely short-term borrowing.
|
||||
_LONG_TERM_DEBT_AGG = ["LongTermDebt"]
|
||||
_LONG_TERM_DEBT_PARTS = ["LongTermDebtNoncurrent", "LongTermDebtCurrent"]
|
||||
# Noncurrent-only balance-sheet lines, needing a current complement added.
|
||||
# ``LongTermDebtAndCapitalLeaseObligations`` is what KO, HD, T, XOM and CVX tag
|
||||
# and nothing read it before: AT&T reported no total_debt at all against 134bn
|
||||
# tagged, and Coca-Cola reported 0.25bn of commercial paper against 39bn.
|
||||
_LONG_TERM_DEBT_NONCURRENT = [
|
||||
"LongTermDebtNoncurrent",
|
||||
"LongTermDebtAndCapitalLeaseObligations",
|
||||
]
|
||||
_LONG_TERM_DEBT_CURRENT = ["LongTermDebtCurrent"]
|
||||
# REITs that tag no aggregate at all, carrying a secured and an unsecured side
|
||||
# instead. Both sides are required, because ``NotesPayable`` does not mean the
|
||||
# same thing across issuers (measured 2026-08 over 14 REITs):
|
||||
# - MAA tags NotesPayable 5.66bn = UnsecuredDebt 5.30bn + SecuredDebt 0.36bn
|
||||
# exactly, so there it IS the total and adding SecuredDebt double-counts.
|
||||
# - EQR/VMRK tags NotesPayable alongside a *larger* SecuredDebt (5.38bn vs
|
||||
# 6.38bn in 2013), so there it is only the unsecured component.
|
||||
# ``UnsecuredDebt`` is what separates them: where it is tagged it is the
|
||||
# unambiguous unsecured side and NotesPayable is ignored; where it is absent,
|
||||
# NotesPayable is that side. Requiring both sides is also what keeps this branch
|
||||
# from inventing a total out of a fragment — Boston Properties tags SecuredDebt
|
||||
# 4.28bn and nothing else against ~15bn of real debt, and Regency tags an
|
||||
# UnsecuredDebt of 0.03bn that is a credit-line draw, not its 5bn of notes.
|
||||
_SECURED_DEBT = ["SecuredDebt"]
|
||||
_UNSECURED_DEBT = ["UnsecuredDebt", "NotesPayable"] # first present wins
|
||||
# ``DebtCurrent`` spans short-term borrowing AND current maturities, so it is the
|
||||
# whole current complement where present and must never be added alongside them.
|
||||
_ALL_CURRENT_DEBT = ["DebtCurrent"]
|
||||
_SHORT_TERM_DEBT = ["ShortTermBorrowings", "CommercialPaper"] # pick one
|
||||
|
||||
|
||||
@@ -459,15 +492,35 @@ def _compose_cash(facts: list[Fact], report_date: date) -> float | None:
|
||||
|
||||
|
||||
def _compose_debt(facts: list[Fact], report_date: date) -> float | None:
|
||||
long_term = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date)
|
||||
if long_term is None:
|
||||
nc = _select_instant(facts, ["LongTermDebtNoncurrent"], report_date)
|
||||
cur = _select_instant(facts, ["LongTermDebtCurrent"], report_date)
|
||||
long_term = None if nc is None and cur is None else (nc or 0.0) + (cur or 0.0)
|
||||
short_term = _select_instant(facts, _SHORT_TERM_DEBT, report_date)
|
||||
if long_term is None and short_term is None:
|
||||
return None
|
||||
return (long_term or 0.0) + (short_term or 0.0)
|
||||
"""Total debt at ``report_date``, or None when no long-term component is found.
|
||||
|
||||
**A short-term component alone is never a total.** Chevron tags its full debt
|
||||
only in the 10-K, so its 10-Q carries ``ShortTermBorrowings`` of 0.40bn and
|
||||
nothing else; returning that as total debt reads as a near-unlevered issuer
|
||||
carrying 50bn. Since ``_net_debt`` needs both sides and yields nothing when
|
||||
either is missing, None costs a leverage read while the partial value
|
||||
produces a confidently wrong one.
|
||||
"""
|
||||
# An aggregate spanning current + noncurrent: only true short-term is missing.
|
||||
total = _select_instant(facts, _LONG_TERM_DEBT_AGG, report_date)
|
||||
if total is not None:
|
||||
return total + (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
||||
|
||||
noncurrent = _select_instant(facts, _LONG_TERM_DEBT_NONCURRENT, report_date)
|
||||
if noncurrent is None:
|
||||
secured = _select_instant(facts, _SECURED_DEBT, report_date)
|
||||
unsecured = _select_instant(facts, _UNSECURED_DEBT, report_date)
|
||||
if secured is None or unsecured is None:
|
||||
return None # one side of a REIT's debt is not its total
|
||||
noncurrent = secured + unsecured
|
||||
|
||||
current = _select_instant(facts, _ALL_CURRENT_DEBT, report_date)
|
||||
if current is None:
|
||||
current = (
|
||||
(_select_instant(facts, _LONG_TERM_DEBT_CURRENT, report_date) or 0.0)
|
||||
+ (_select_instant(facts, _SHORT_TERM_DEBT, report_date) or 0.0)
|
||||
)
|
||||
return noncurrent + current
|
||||
|
||||
|
||||
def _select_shares(
|
||||
|
||||
@@ -18,9 +18,29 @@ Guardrails (design + reviews):
|
||||
a tracked XBRL index accession missing from Company Facts fails the run (the two
|
||||
are separate SEC products that can lag) so we retry rather than record a
|
||||
null/partial snapshot. Non-XBRL amendments are skipped with a recorded reason.
|
||||
A failure here blocks every later run (``source_max_date`` only advances on a
|
||||
promoted run), so it names the offending filings in the alert and separates the
|
||||
causes — ``not_in_companyfacts`` (facts lag) vs ``not_in_submissions`` (the
|
||||
index row is absent from the issuer's own filing list, which no retry fixes).
|
||||
- **Co-registrant recovery**, because "missing from Company Facts" is often not
|
||||
missing at all: SEC files some combined parent/subsidiary filings' XBRL under
|
||||
the co-registrant's CIK, so the ticker-carrying parent's own file never gets
|
||||
that accession. The daily index lists every co-registrant of an accession, so
|
||||
the facts are found there and re-stamped to the real filer — guarded by a
|
||||
share-count continuity check so a subsidiary's standalone numbers can never be
|
||||
stored as the parent's. Confirmed 2026-07-27 (NEE via FPL, DOW via Dow Chemical)
|
||||
and it is not transient: an NEE filing misattributed in 2014 is still misfiled.
|
||||
- **Bounded blocking.** Anything still unresolvable after ``MISSING_XBRL_RETRY_DAYS``
|
||||
stops failing the whole import and enters a durable retry queue. The scheduled
|
||||
importer retries queued accessions automatically, while the affected issuer is
|
||||
excluded from actionable setups until its filing is recovered.
|
||||
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
||||
reports differing existing accessions, and applies ticker updates in the same
|
||||
transaction.
|
||||
transaction. A difference in ``cik`` **alone** is reported separately as an
|
||||
``accession_cik_collision``: every fact matched, so two tracked CIKs are
|
||||
claiming one filing and the fix is the universe, not the parser. It never
|
||||
self-heals on its own — the losing CIK stores no row, so it is backfilled and
|
||||
re-reported every run until its ticker is re-pointed or retired.
|
||||
- ``reparse=True`` is the one exception to immutability, and it is deliberate:
|
||||
it restages every accession with the current parser and **rewrites** the rows
|
||||
that now reconstruct differently. Immutability protects SEC's record (one row
|
||||
@@ -31,24 +51,27 @@ Guardrails (design + reviews):
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import delete, exists, select, update
|
||||
|
||||
from app.database import insert_for_session
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.sec_filing_gap import SecFilingGap
|
||||
from app.models.system_event import SystemEvent
|
||||
from app.services import fundamentals_quality_service
|
||||
from app.services import sec_facts_parser as parser
|
||||
from app.services import sec_universe
|
||||
from app.services.data_import import STATUS_PROMOTED, ValidationResult
|
||||
from app.services.sec_client import SecClient, SecError, cik10
|
||||
from app.services.sec_facts_parser import FilingMeta, SnapshotRow
|
||||
from app.services.sec_universe import ResolvedUniverse
|
||||
from app.services.sec_universe import CIK_OVERRIDES_KEY, ResolvedUniverse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -57,6 +80,37 @@ _XBRL_FORMS = {"10-K", "10-Q", "10-K/A", "10-Q/A"}
|
||||
# On the one-time backfill, require this fraction of tracked issuers to yield at
|
||||
# least one snapshot (guards a broken fetch/parse from promoting a hollow table).
|
||||
MIN_BACKFILL_COVERAGE = 0.5
|
||||
# How long an index accession may stay unresolvable before the run stops failing
|
||||
# on it. Genuine index↔facts lag clears within a day (a weekend stretches it to
|
||||
# three); past that it is misfiled, not late, and blocking forever costs more
|
||||
# than the missing filing does — see the unresolved-filing guardrail below.
|
||||
MISSING_XBRL_RETRY_DAYS = 3
|
||||
|
||||
# Aggregate ceiling on deferral. MISSING_XBRL_RETRY_DAYS bounds how long ONE
|
||||
# filing blocks; it does not bound how long the import as a whole can stay
|
||||
# deferred. Those differ because a blocking filing is only queued by promote(),
|
||||
# which a deferred run never reaches — so during a rolling supply of
|
||||
# unresolvable filings (earnings season, when SEC's Company-Facts aggregation is
|
||||
# furthest behind) each new arrival restarts the clock before the previous one
|
||||
# clears, and nothing is written at all: not the good rows, not the gap rows
|
||||
# that would stop those filings blocking again.
|
||||
#
|
||||
# Once promotions have been stale this long, every unresolved filing is treated
|
||||
# as past the window. promote() then queues them all (see the _past_retry_window
|
||||
# call there), source_max_date advances, and _missing() forces queued rows
|
||||
# aged-out on later runs so they never block again — the import self-heals
|
||||
# through the paths that already exist.
|
||||
#
|
||||
# Well above MISSING_XBRL_RETRY_DAYS so ordinary overlapping blocks never trip
|
||||
# it. Affected symbols stay barred from setups either way: setup_blocked_ciks is
|
||||
# built from every missing filing regardless of window.
|
||||
PROMOTION_CEILING_DAYS = 7
|
||||
FILING_GAP_ESCALATE_DAYS = 14
|
||||
# Share-count band a co-registrant-recovered row must land in, relative to the
|
||||
# issuer's own last snapshot. Wide enough for buybacks/issuance, nowhere near
|
||||
# wide enough to let a subsidiary shell's token float through (see _shares_continuous).
|
||||
RECOVERY_SHARES_MIN = 0.5
|
||||
RECOVERY_SHARES_MAX = 2.0
|
||||
|
||||
_SNAPSHOT_COLS = (
|
||||
"cik", "accession", "form", "filed_date", "accepted_at", "period_start",
|
||||
@@ -78,7 +132,11 @@ class StagedFundamentals:
|
||||
skipped_filings: list[dict[str, str]] = field(default_factory=list)
|
||||
field_issues: list[dict[str, str]] = field(default_factory=list)
|
||||
skipped_non_xbrl: list[dict[str, str]] = field(default_factory=list)
|
||||
missing_xbrl: list[dict[str, str]] = field(default_factory=list)
|
||||
# Index rows we could not resolve to Company Facts, with a per-row reason
|
||||
# (not_in_companyfacts | not_in_submissions | ...) — see _missing().
|
||||
missing_xbrl: list[dict[str, Any]] = field(default_factory=list)
|
||||
# Accessions parsed out of a co-registrant's Company Facts file.
|
||||
recovered: list[dict[str, Any]] = field(default_factory=list)
|
||||
invalid_payloads: list[dict[str, str]] = field(default_factory=list)
|
||||
existing_accessions: set[str] = field(default_factory=set)
|
||||
# Tracked issuers whose registrant has NO XBRL 10-K/10-Q at all: they can
|
||||
@@ -117,8 +175,15 @@ class SecFundamentalsImporter:
|
||||
# cached by detect_revision, consumed by stage:
|
||||
self._resolved: ResolvedUniverse | None = None
|
||||
self._index_rows: list[dict[str, Any]] = []
|
||||
# accession -> the OTHER CIKs the daily index lists it under (co-registrants
|
||||
# of a combined filing). Only populated for accessions a tracked issuer filed.
|
||||
self._coregistrants: dict[str, list[int]] = {}
|
||||
self._retry_rows: list[dict[str, Any]] = []
|
||||
self._latest_index_date: date | None = None
|
||||
self._backfill = False
|
||||
# Set by validate() when the aggregate ceiling forced the block open;
|
||||
# read by promote() to alert that it did.
|
||||
self._ceiling_tripped: dict[str, Any] | None = None
|
||||
|
||||
# -- SourceImporter protocol -------------------------------------------
|
||||
|
||||
@@ -132,6 +197,7 @@ class SecFundamentalsImporter:
|
||||
# Reparse needs every accession restaged, not just those filed since
|
||||
# the last run — the facts a fixed parser now accepts were never
|
||||
# stored, so a reparse cannot be served from the database.
|
||||
self._coregistrants = {}
|
||||
if last_processed is None or self.reparse:
|
||||
self._backfill = True
|
||||
self._index_rows = []
|
||||
@@ -141,9 +207,19 @@ class SecFundamentalsImporter:
|
||||
client, last_processed, self._latest_index_date
|
||||
)
|
||||
content = sec_universe.index_content_hash(self._index_rows)
|
||||
return sec_universe.compose_revision(
|
||||
revision = sec_universe.compose_revision(
|
||||
self._latest_index_date, content, self._resolved.symbol_to_cik
|
||||
)
|
||||
self._retry_rows = []
|
||||
if not self._backfill:
|
||||
self._retry_rows = await self._retry_backlog(
|
||||
db,
|
||||
set(self._resolved.cik_to_ticker_ids),
|
||||
)
|
||||
# Company Facts can change while the daily index revision stays fixed.
|
||||
# Returning None deliberately bypasses the framework's no-op gate so a
|
||||
# scheduled run retries every active gap.
|
||||
return None if self._retry_rows else revision
|
||||
|
||||
async def stage(self, db) -> StagedFundamentals:
|
||||
assert self._resolved is not None, "detect_revision must run first"
|
||||
@@ -151,10 +227,32 @@ class SecFundamentalsImporter:
|
||||
staged = StagedFundamentals(resolved=resolved, backfill=self._backfill)
|
||||
|
||||
cik_to_tids = resolved.cik_to_ticker_ids
|
||||
filed_by_cik: dict[int, list[str]] = defaultdict(list)
|
||||
# Whole index rows (not bare accessions): form + index date are what make
|
||||
# an unresolvable filing diagnosable without re-walking the index by hand.
|
||||
filed_by_cik: dict[int, list[dict[str, Any]]] = defaultdict(list)
|
||||
for r in self._index_rows:
|
||||
if r["cik"] in cik_to_tids:
|
||||
filed_by_cik[r["cik"]].append(r["accession"])
|
||||
filed_by_cik[r["cik"]].append(r)
|
||||
|
||||
# Promoted-around filings live in a small durable retry queue, including
|
||||
# the one-time migration backfill. Merge them into normal incremental
|
||||
# work so the scheduled importer heals them without operator action.
|
||||
if not self._backfill:
|
||||
seen = {
|
||||
(int(cik), row["accession"])
|
||||
for cik, rows in filed_by_cik.items()
|
||||
for row in rows
|
||||
}
|
||||
for row in self._retry_rows:
|
||||
cik = int(row["cik"])
|
||||
key = (cik, row["accession"])
|
||||
if key in seen:
|
||||
continue
|
||||
filed_by_cik[cik].append(row)
|
||||
seen.add(key)
|
||||
coregistrants = [int(value) for value in row.get("coregistrants") or []]
|
||||
if coregistrants:
|
||||
self._coregistrants[row["accession"]] = coregistrants
|
||||
|
||||
existing = await self._ciks_with_snapshots(db, set(cik_to_tids))
|
||||
if self._backfill:
|
||||
@@ -165,10 +263,15 @@ class SecFundamentalsImporter:
|
||||
backfill_ciks = {c for c in cik_to_tids if c not in existing}
|
||||
incremental_ciks = set(filed_by_cik) - backfill_ciks
|
||||
|
||||
# Continuity reference for co-registrant recovery, read once up front.
|
||||
last_shares = await self._last_shares_outstanding(db, set(cik_to_tids))
|
||||
|
||||
async with self._client_factory() as client:
|
||||
for cik in sorted(backfill_ciks | incremental_ciks):
|
||||
is_backfill = cik in backfill_ciks
|
||||
await self._stage_issuer(client, cik, is_backfill, filed_by_cik, staged)
|
||||
await self._stage_issuer(
|
||||
client, cik, is_backfill, filed_by_cik, staged, last_shares
|
||||
)
|
||||
|
||||
# Read-only discrepancy detection: an accession we reconstructed that is
|
||||
# already stored, differing in ANY source field (immutable → report in
|
||||
@@ -182,10 +285,23 @@ class SecFundamentalsImporter:
|
||||
if old is not None:
|
||||
fields = _diff_fields(row, old)
|
||||
if fields:
|
||||
staged.discrepancies.append({"accession": row.accession, "fields": fields})
|
||||
# Carry both CIKs. promote() reads a bare ["cik"] as an
|
||||
# attribution collision rather than a changed
|
||||
# reconstruction, which holds only because _COMPARE_COLS
|
||||
# spans every stored fact: a fact column added to the
|
||||
# model but not to _SNAPSHOT_COLS would go uncompared and
|
||||
# let a real difference through as a collision.
|
||||
staged.discrepancies.append({
|
||||
"accession": row.accession,
|
||||
"fields": fields,
|
||||
"cik": row.cik,
|
||||
"stored_cik": old.cik,
|
||||
})
|
||||
return staged
|
||||
|
||||
async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None:
|
||||
async def _stage_issuer(
|
||||
self, client, cik, is_backfill, filed_by_cik, staged, last_shares
|
||||
) -> None:
|
||||
cf = await client.companyfacts(cik)
|
||||
bad = _companyfacts_structure_error(cf)
|
||||
if bad is not None:
|
||||
@@ -201,31 +317,85 @@ class SecFundamentalsImporter:
|
||||
{"cik": cik10(cik), "name": sub.get("name"), "tickers": sub.get("tickers")}
|
||||
)
|
||||
|
||||
fiscal_year_end = sub.get("fiscal_year_end")
|
||||
recovered_rows: list[SnapshotRow] = []
|
||||
index_rows = {
|
||||
row["accession"]: row for row in filed_by_cik.get(cik, [])
|
||||
}
|
||||
if is_backfill:
|
||||
accns = set(xbrl_meta)
|
||||
else:
|
||||
present = parser.companyfacts_accessions(cf)
|
||||
accns = set()
|
||||
for accn in filed_by_cik.get(cik, []):
|
||||
for index_row in filed_by_cik.get(cik, []):
|
||||
accn = index_row["accession"]
|
||||
if accn in nonxbrl:
|
||||
staged.skipped_non_xbrl.append({"cik": cik10(cik), "accession": accn})
|
||||
elif accn in xbrl_meta and accn in present:
|
||||
elif accn not in xbrl_meta:
|
||||
# The daily index lists it but the issuer's own filing list does
|
||||
# not (submissions lagging the index, no usable period metadata,
|
||||
# or a co-registrant filing). NOT a Company-Facts lag — separate
|
||||
# cause, separate fix, so it gets its own reason.
|
||||
staged.missing_xbrl.append(
|
||||
_missing(
|
||||
cik,
|
||||
index_row,
|
||||
"not_in_submissions",
|
||||
self.today,
|
||||
self._coregistrants.get(accn),
|
||||
)
|
||||
)
|
||||
elif accn in present:
|
||||
accns.add(accn)
|
||||
else:
|
||||
# XBRL (or unknown) filing not yet in Company Facts → the products
|
||||
# have lagged; fail+retry rather than record nothing for it.
|
||||
staged.missing_xbrl.append({"cik": cik10(cik), "accession": accn})
|
||||
# Filed, XBRL, but absent from this issuer's Company Facts. Try
|
||||
# the co-registrant file before treating it as missing data.
|
||||
row, source_cik = await self._recover_from_coregistrant(
|
||||
client, cik, accn, xbrl_meta, fiscal_year_end,
|
||||
last_shares.get(cik10(cik)), staged,
|
||||
)
|
||||
if row is not None:
|
||||
recovered_rows.append(row)
|
||||
staged.recovered.append({
|
||||
"cik": cik10(cik),
|
||||
"accession": accn,
|
||||
"source_cik": source_cik,
|
||||
"form": index_row.get("form"),
|
||||
})
|
||||
else:
|
||||
staged.missing_xbrl.append(_missing(
|
||||
cik, index_row,
|
||||
# Found, but it did not look like this issuer's own
|
||||
# numbers — say so; it is not the same as absent.
|
||||
"coregistrant_facts_rejected" if source_cik
|
||||
else "not_in_companyfacts",
|
||||
self.today,
|
||||
self._coregistrants.get(accn),
|
||||
))
|
||||
|
||||
# fiscalYearEnd (MMDD) is what lets the parser derive period identity from
|
||||
# reportDate instead of SEC's unreliable fy/fp fields.
|
||||
result = parser.parse_snapshots(
|
||||
cf, xbrl_meta, accns, fiscal_year_end=sub.get("fiscal_year_end")
|
||||
)
|
||||
result = parser.parse_snapshots(cf, xbrl_meta, accns, fiscal_year_end=fiscal_year_end)
|
||||
for skipped in result.skipped_filings:
|
||||
index_row = index_rows.get(skipped["accession"])
|
||||
if index_row is not None:
|
||||
# Facts are present but our parser cannot construct a snapshot.
|
||||
# A new index row keeps the normal grace period before promotion;
|
||||
# a row already read from the queue retains its _retry_queue marker
|
||||
# so later imports promote and retry without wedging the index.
|
||||
staged.missing_xbrl.append(_missing(
|
||||
cik,
|
||||
index_row,
|
||||
"parser_unusable",
|
||||
self.today,
|
||||
self._coregistrants.get(skipped["accession"]),
|
||||
))
|
||||
staged.rows.extend(result.rows)
|
||||
staged.rows.extend(recovered_rows)
|
||||
staged.skipped_filings.extend(result.skipped_filings)
|
||||
staged.field_issues.extend(result.field_issues)
|
||||
staged.issuers_fetched += 1
|
||||
if result.rows:
|
||||
if result.rows or recovered_rows:
|
||||
staged.issuers_with_rows += 1
|
||||
|
||||
# SIC proposal for this issuer's tickers (read-only; applied in promote).
|
||||
@@ -234,14 +404,84 @@ class SecFundamentalsImporter:
|
||||
for tid in staged.resolved.cik_to_ticker_ids.get(cik, []):
|
||||
staged.sic_updates.append((tid, sic, desc))
|
||||
|
||||
async def _recover_from_coregistrant(
|
||||
self, client, cik: int, accn: str, xbrl_meta, fiscal_year_end, reference, staged,
|
||||
) -> tuple[SnapshotRow | None, str | None]:
|
||||
"""Look for ``accn``'s facts in a co-registrant's Company Facts file.
|
||||
|
||||
SEC sometimes files a combined parent/subsidiary filing's XBRL under the
|
||||
co-registrant's CIK rather than the filer's — the ticker-carrying parent's
|
||||
own file simply never gets that accession. Verified 2026-07-27 for NEE
|
||||
(facts under Florida Power & Light) and DOW (under Dow Chemical); an NEE
|
||||
filing misattributed the same way in **2014** is still misattributed, so
|
||||
this does not self-correct and no amount of retrying recovers it.
|
||||
|
||||
Returns ``(row, source_cik)`` on success, ``(None, source_cik)`` when the
|
||||
facts were found but rejected by the continuity guard, ``(None, None)``
|
||||
when no co-registrant has them.
|
||||
|
||||
Incremental path only: the co-registrant map comes from the daily index,
|
||||
which a backfill/reparse does not walk. A reparse therefore recovers a
|
||||
filing only once SEC re-files it under the filer's own CIK.
|
||||
"""
|
||||
for co in self._coregistrants.get(accn, []):
|
||||
try:
|
||||
cf_co = await client.companyfacts(co)
|
||||
except SecError:
|
||||
continue # a co-registrant shell often has no facts file at all
|
||||
if _companyfacts_structure_error(cf_co) is not None:
|
||||
continue
|
||||
result = parser.parse_snapshots(
|
||||
cf_co, xbrl_meta, {accn}, fiscal_year_end=fiscal_year_end
|
||||
)
|
||||
if not result.rows:
|
||||
continue
|
||||
row = result.rows[0]
|
||||
if not _shares_continuous(row.shares_outstanding, reference):
|
||||
return None, cik10(co)
|
||||
# A recovered row is the one most worth flagging, so its parser caveats
|
||||
# travel with it rather than being dropped on the way out.
|
||||
staged.field_issues.extend(result.field_issues)
|
||||
# parse_snapshots stamps the CIK of the payload it read — re-stamp to
|
||||
# the issuer that actually filed, or the row lands under the shell.
|
||||
return replace(row, cik=cik10(cik)), cik10(co)
|
||||
return None, None
|
||||
|
||||
async def validate(self, db, staged: StagedFundamentals) -> ValidationResult:
|
||||
messages: list[str] = []
|
||||
|
||||
# Consistency gate — before any write.
|
||||
if staged.missing_xbrl:
|
||||
# Consistency gate — before any write. Only filings still inside the retry
|
||||
# window block: a failure here stops every later run too (source_max_date
|
||||
# advances on promotion alone), so blocking forever on a filing SEC has
|
||||
# misfiled would cost far more than the one filing it withholds. Older
|
||||
# ones are carried by promote() as a warning instead. The message names
|
||||
# the filings: "which ones" has to be in the alert itself, not merely
|
||||
# reconstructible by re-walking the index.
|
||||
blocking = _within_retry_window(staged.missing_xbrl)
|
||||
aged_out = _past_retry_window(staged.missing_xbrl)
|
||||
|
||||
# ...unless promotions have been stale past the aggregate ceiling, in
|
||||
# which case the deferral has cost more than the filings it withholds.
|
||||
# Ageing them here (not just locally) is deliberate: promote() re-derives
|
||||
# the queue from the same list, so this is what gets them queued.
|
||||
self._ceiling_tripped = None
|
||||
if blocking and db is not None and await self._promotions_stale(db):
|
||||
for item in staged.missing_xbrl:
|
||||
item["age_days"] = max(
|
||||
item.get("age_days", 0), MISSING_XBRL_RETRY_DAYS + 1
|
||||
)
|
||||
self._ceiling_tripped = {
|
||||
"forced": len(blocking),
|
||||
"unresolved": len(staged.missing_xbrl),
|
||||
}
|
||||
blocking = _within_retry_window(staged.missing_xbrl)
|
||||
aged_out = _past_retry_window(staged.missing_xbrl)
|
||||
|
||||
if blocking:
|
||||
messages.append(
|
||||
f"{len(staged.missing_xbrl)} tracked XBRL filing(s) not yet in "
|
||||
"Company Facts (index/facts lag) — retry"
|
||||
f"{len(blocking)} tracked XBRL filing(s) unresolved within the "
|
||||
f"{MISSING_XBRL_RETRY_DAYS}-day retry window "
|
||||
f"({_reason_counts(blocking)}) — retry: {_missing_detail(blocking)}"
|
||||
)
|
||||
# Malformed companyfacts payloads must fail, not degrade to skipped rows.
|
||||
if staged.invalid_payloads:
|
||||
@@ -272,7 +512,25 @@ class SecFundamentalsImporter:
|
||||
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
|
||||
"no_xbrl_filings": staged.no_xbrl_filings[:50],
|
||||
"no_xbrl_filings_count": len(staged.no_xbrl_filings),
|
||||
"missing_xbrl": len(staged.missing_xbrl),
|
||||
"no_xbrl_ciks": sorted({
|
||||
str(item["cik"])
|
||||
for item in staged.no_xbrl_filings
|
||||
if item.get("cik")
|
||||
}),
|
||||
"missing_xbrl": staged.missing_xbrl[:50],
|
||||
"missing_xbrl_count": len(staged.missing_xbrl),
|
||||
"missing_xbrl_blocking": len(blocking),
|
||||
# Present only when the aggregate ceiling forced this run through, so
|
||||
# a promoted run that carries known-unresolved filings says so.
|
||||
"promotion_ceiling_tripped": self._ceiling_tripped,
|
||||
"recovered_from_coregistrant": staged.recovered[:50],
|
||||
"recovered_count": len(staged.recovered),
|
||||
# Complete compact gate input; detailed audit lists above stay capped.
|
||||
"setup_blocked_ciks": sorted({
|
||||
str(item["cik"])
|
||||
for item in [*staged.missing_xbrl, *staged.no_xbrl_filings]
|
||||
if item.get("cik")
|
||||
}),
|
||||
"invalid_payloads": staged.invalid_payloads,
|
||||
"cik_updates": len(staged.resolved.cik_updates),
|
||||
# differing existing accessions (immutable — kept, reported here)
|
||||
@@ -284,14 +542,43 @@ class SecFundamentalsImporter:
|
||||
summary=summary,
|
||||
source_max_date=self._latest_index_date,
|
||||
messages=messages,
|
||||
# Company-Facts absence is usually publication lag, but can also be a
|
||||
# permanent co-registrant misfile that the daily index did not expose.
|
||||
# Defer quietly at first; the framework warns if promotions stay stale.
|
||||
retryable=(
|
||||
len(messages) == 1
|
||||
and bool(blocking)
|
||||
and all(
|
||||
m.get("reason") in {"not_in_companyfacts", "parser_unusable"}
|
||||
for m in blocking
|
||||
)
|
||||
),
|
||||
deferred_alert_after_days=MISSING_XBRL_RETRY_DAYS,
|
||||
deferred_alert_messages=(
|
||||
[
|
||||
f"{len(aged_out)} tracked SEC filing(s) remain unresolved past "
|
||||
f"the {MISSING_XBRL_RETRY_DAYS}-day retry window. They will "
|
||||
f"enter automatic retry and block affected symbols from setups: "
|
||||
f"{_missing_detail(aged_out)}"
|
||||
]
|
||||
if aged_out
|
||||
else []
|
||||
),
|
||||
)
|
||||
|
||||
async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]:
|
||||
inserted = 0
|
||||
updated = 0
|
||||
# Only accessions whose reconstruction actually changed are rewritten;
|
||||
# an unchanged stored row is left completely alone.
|
||||
changed = {d["accession"] for d in staged.discrepancies} if self.reparse else set()
|
||||
# an unchanged stored row is left completely alone. A cik-only difference
|
||||
# is excluded on purpose: the facts are identical there, so rewriting
|
||||
# would re-stamp the filing onto the colliding co-registrant — taking it
|
||||
# from the issuer that actually filed it, which no parser fix asks for.
|
||||
changed = (
|
||||
{d["accession"] for d in staged.discrepancies if d["fields"] != ["cik"]}
|
||||
if self.reparse
|
||||
else set()
|
||||
)
|
||||
for row in staged.rows:
|
||||
if row.accession in staged.existing_accessions:
|
||||
if row.accession in changed:
|
||||
@@ -313,10 +600,114 @@ class SecFundamentalsImporter:
|
||||
await db.execute(stmt)
|
||||
inserted += 1
|
||||
|
||||
# Synchronize the retry queue in the snapshot-promotion transaction.
|
||||
existing_gaps = (await db.execute(select(SecFilingGap))).scalars().all()
|
||||
existing_gap_accessions = {gap.accession for gap in existing_gaps}
|
||||
resolved_accessions = {row.accession for row in staged.rows}
|
||||
# A filing now classified non-XBRL can never yield a snapshot and is no
|
||||
# longer a fundamentals completeness gap.
|
||||
resolved_accessions.update(
|
||||
item["accession"] for item in staged.skipped_non_xbrl
|
||||
)
|
||||
queue_resolved = 0
|
||||
if resolved_accessions:
|
||||
result = await db.execute(
|
||||
delete(SecFilingGap).where(
|
||||
SecFilingGap.accession.in_(resolved_accessions)
|
||||
)
|
||||
)
|
||||
queue_resolved = int(result.rowcount or 0)
|
||||
|
||||
now = _now()
|
||||
tolerated = _past_retry_window(staged.missing_xbrl)
|
||||
for gap in tolerated:
|
||||
stmt = insert_for_session(db, SecFilingGap).values(
|
||||
cik=gap["cik"],
|
||||
accession=gap["accession"],
|
||||
form=gap.get("form"),
|
||||
index_date=gap.get("index_date"),
|
||||
reason=gap["reason"],
|
||||
coregistrant_ciks_json=json.dumps(gap.get("coregistrants") or []),
|
||||
first_seen_at=now,
|
||||
last_attempted_at=now,
|
||||
)
|
||||
await db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["accession"],
|
||||
set_={
|
||||
"cik": stmt.excluded.cik,
|
||||
"form": stmt.excluded.form,
|
||||
"index_date": stmt.excluded.index_date,
|
||||
"reason": stmt.excluded.reason,
|
||||
"coregistrant_ciks_json": stmt.excluded.coregistrant_ciks_json,
|
||||
"last_attempted_at": stmt.excluded.last_attempted_at,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Remove gaps made irrelevant by a later valid 10-K/10-Q. Quality reads
|
||||
# already ignore them; physical cleanup keeps the queue small.
|
||||
active_ids = {gap.id for gap in await fundamentals_quality_service.active_gaps(db)}
|
||||
obsolete_ids = {
|
||||
gap.id for gap in existing_gaps
|
||||
if gap.id not in active_ids and gap.accession not in resolved_accessions
|
||||
}
|
||||
if obsolete_ids:
|
||||
result = await db.execute(
|
||||
delete(SecFilingGap).where(SecFilingGap.id.in_(obsolete_ids))
|
||||
)
|
||||
queue_resolved += int(result.rowcount or 0)
|
||||
|
||||
newly_queued = [
|
||||
gap for gap in tolerated
|
||||
if gap["accession"] not in existing_gap_accessions
|
||||
]
|
||||
|
||||
# Two tracked issuers claiming one filing is not a reconstruction change:
|
||||
# every fact matched and only the CIK stamp differs, so re-parsing or
|
||||
# reparsing fixes nothing — the universe resolution does. It is reported
|
||||
# separately because it also does not self-heal: the loser of the
|
||||
# collision never stores a row, so `_ciks_with_snapshots` never sees it,
|
||||
# and it is full-history backfilled (and re-reported) on every run until
|
||||
# a human re-points or retires the ticker. Observed 2026-08 for EQR,
|
||||
# which SEC's own company_tickers.json maps to ERP Operating LP, the
|
||||
# non-traded co-registrant of the issuer now trading as VMRK.
|
||||
collisions = [d for d in staged.discrepancies if d["fields"] == ["cik"]]
|
||||
if collisions:
|
||||
named = ", ".join(
|
||||
f"{d['accession']} (stored {d['stored_cik']}, parsed {d['cik']})"
|
||||
for d in collisions[:10]
|
||||
)
|
||||
db.add(SystemEvent(
|
||||
severity="warning",
|
||||
source="sec_facts",
|
||||
code="accession_cik_collision",
|
||||
message=(
|
||||
f"{len(collisions)} filing(s) are claimed by two tracked CIKs — "
|
||||
"the reconstruction is identical, only the attribution differs, "
|
||||
"so one of the two is a co-registrant the universe should not "
|
||||
f"track. Re-point or retire the ticker (see {CIK_OVERRIDES_KEY}); "
|
||||
f"this repeats every run until then: {named}"
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:accession_cik_collision:{run_id}",
|
||||
created_at=_now(),
|
||||
))
|
||||
|
||||
# Warn (in-transaction, so it commits atomically with the promotion) when
|
||||
# any existing accession reconstructed differently — kept immutable.
|
||||
if staged.discrepancies:
|
||||
accns = ", ".join(d["accession"] for d in staged.discrepancies[:10])
|
||||
reconstruction_diffs = [
|
||||
d for d in staged.discrepancies if d["fields"] != ["cik"]
|
||||
]
|
||||
if reconstruction_diffs:
|
||||
# Name the columns, not just the accession: "differs in revenue"
|
||||
# (our numbers moved) and "differs in period_start" (the filing was
|
||||
# re-placed in the calendar) need different responses, and the alert
|
||||
# is where that call gets made. The fields are already computed for
|
||||
# validation_json — they were simply dropped from the message.
|
||||
accns = ", ".join(
|
||||
f"{d['accession']} ({', '.join(d['fields'])})"
|
||||
for d in reconstruction_diffs[:10]
|
||||
)
|
||||
disposition = (
|
||||
f"REWRITTEN by reparse run {run_id}" if self.reparse else "kept immutable"
|
||||
)
|
||||
@@ -325,35 +716,168 @@ class SecFundamentalsImporter:
|
||||
source="sec_facts",
|
||||
code="snapshot_reparse" if self.reparse else "snapshot_discrepancy",
|
||||
message=(
|
||||
f"{len(staged.discrepancies)} stored accession(s) reconstructed "
|
||||
f"{len(reconstruction_diffs)} stored accession(s) reconstructed "
|
||||
f"differently; {disposition}: {accns}"
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:discrepancy:{run_id}",
|
||||
created_at=_now(),
|
||||
))
|
||||
|
||||
# A tracked issuer whose registrant has no XBRL filings can never produce a
|
||||
# snapshot, and it is restaged on every run forever. That is a resolution
|
||||
# problem, not missing data, and it is silent without this.
|
||||
if staged.no_xbrl_filings:
|
||||
# A ceiling-forced promotion is the safety valve firing — it must be
|
||||
# visible, or the import silently starts carrying known-unresolved
|
||||
# filings. The affected symbols stay barred from setups regardless.
|
||||
if self._ceiling_tripped:
|
||||
db.add(SystemEvent(
|
||||
severity="warning",
|
||||
source="sec_facts",
|
||||
code="promotion_ceiling_forced",
|
||||
message=(
|
||||
f"Promoted with {self._ceiling_tripped['unresolved']} unresolved "
|
||||
f"filing(s) — {self._ceiling_tripped['forced']} still inside the "
|
||||
f"{MISSING_XBRL_RETRY_DAYS}-day retry window — because nothing had "
|
||||
f"promoted in {PROMOTION_CEILING_DAYS} days. They are queued for "
|
||||
"retry and their symbols remain blocked from setups."
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:promotion_ceiling_forced:{run_id}",
|
||||
created_at=now,
|
||||
))
|
||||
|
||||
# Persistent current gaps get one actionable escalation rather than a
|
||||
# daily warning. The nullable marker makes this durable and noise-free.
|
||||
escalation_cutoff = now - timedelta(days=FILING_GAP_ESCALATE_DAYS)
|
||||
aged_gaps = (
|
||||
await db.execute(
|
||||
select(SecFilingGap).where(
|
||||
SecFilingGap.first_seen_at <= escalation_cutoff,
|
||||
SecFilingGap.escalated_at.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
if aged_gaps:
|
||||
named = ", ".join(
|
||||
f"{e['cik']} ({e.get('name') or '?'})" for e in staged.no_xbrl_filings[:10]
|
||||
f"{gap.cik}/{gap.accession} ({gap.reason})"
|
||||
for gap in aged_gaps[:10]
|
||||
)
|
||||
db.add(SystemEvent(
|
||||
severity="warning",
|
||||
source="sec_facts",
|
||||
code="no_xbrl_filings",
|
||||
code="filing_gap_aged",
|
||||
message=(
|
||||
f"{len(staged.no_xbrl_filings)} tracked issuer(s) resolved to a "
|
||||
f"registrant with no XBRL 10-K/10-Q. Either a successor shell "
|
||||
f"(pin the real filer via the '{sec_universe.CIK_OVERRIDES_KEY}' "
|
||||
f"setting) or a new registrant that has not filed its first "
|
||||
f"10-K/10-Q yet, which needs nothing and clears itself: {named}"
|
||||
f"{len(aged_gaps)} SEC filing gap(s) remain unresolved after "
|
||||
f"{FILING_GAP_ESCALATE_DAYS} days; affected setups remain paused. "
|
||||
f"Review the filing/CIK mapping or parser: {named}"
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:no_xbrl_filings:{run_id}",
|
||||
dedup_key=f"sec_facts:filing_gap_aged:{run_id}",
|
||||
created_at=now,
|
||||
))
|
||||
await db.execute(
|
||||
update(SecFilingGap)
|
||||
.where(SecFilingGap.id.in_([gap.id for gap in aged_gaps]))
|
||||
.values(escalated_at=now)
|
||||
)
|
||||
|
||||
# The escalation above fires once per gap, so nothing would report the
|
||||
# *end* of the reprieve it grants. An escalated gap stops pausing setups
|
||||
# while the issuer's own fundamentals are still recent, and that lapses
|
||||
# on its own — the stored filings age past the window, or a newer gap
|
||||
# appears — putting the pause back on with no alert anywhere. Track the
|
||||
# exemption as state and alert on the transition, once per lapse.
|
||||
current_gaps = await fundamentals_quality_service.active_gaps(db)
|
||||
escalated_gaps = [g for g in current_gaps if g.escalated_at is not None]
|
||||
if escalated_gaps:
|
||||
exempt_ciks = await fundamentals_quality_service.gap_exempt_ciks(
|
||||
db, escalated_gaps
|
||||
)
|
||||
newly_exempt = [
|
||||
g for g in escalated_gaps
|
||||
if g.cik in exempt_ciks and g.exempted_at is None
|
||||
]
|
||||
lapsed = [
|
||||
g for g in escalated_gaps
|
||||
if g.cik not in exempt_ciks and g.exempted_at is not None
|
||||
]
|
||||
if newly_exempt:
|
||||
# Silent on purpose: filing_gap_aged already announced this gap,
|
||||
# and setups resuming is the behaviour that alert describes.
|
||||
await db.execute(
|
||||
update(SecFilingGap)
|
||||
.where(SecFilingGap.id.in_([g.id for g in newly_exempt]))
|
||||
.values(exempted_at=now)
|
||||
)
|
||||
if lapsed:
|
||||
named = ", ".join(
|
||||
f"{gap.cik}/{gap.accession}" for gap in lapsed[:10]
|
||||
)
|
||||
db.add(SystemEvent(
|
||||
severity="warning",
|
||||
source="sec_facts",
|
||||
code="filing_gap_repaused",
|
||||
message=(
|
||||
f"{len(lapsed)} SEC filing gap(s) pause setups again: the "
|
||||
"issuer's own fundamentals have aged out of the "
|
||||
f"{fundamentals_quality_service.GAP_GATE_RECENT_FILING_DAYS}"
|
||||
"-day window, or a newer gap arrived, so there is nothing "
|
||||
f"recent left to score on: {named}"
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:filing_gap_repaused:{run_id}",
|
||||
created_at=now,
|
||||
))
|
||||
# Cleared, not stamped: the issuer can recover and age out again,
|
||||
# and each lapse is worth its own alert.
|
||||
await db.execute(
|
||||
update(SecFilingGap)
|
||||
.where(SecFilingGap.id.in_([g.id for g in lapsed]))
|
||||
.values(exempted_at=None)
|
||||
)
|
||||
|
||||
# Recovered rows are real data from an unexpected place — record where they
|
||||
# came from, so a wrong recovery is auditable rather than invisible.
|
||||
if staged.recovered:
|
||||
named = ", ".join(
|
||||
f"{r['accession']} <- CIK {r['source_cik']}" for r in staged.recovered[:10]
|
||||
)
|
||||
logger.info(
|
||||
"sec_facts: recovered %d filing(s) from co-registrants: %s",
|
||||
len(staged.recovered),
|
||||
named,
|
||||
)
|
||||
|
||||
# One warning when a gap first enters automatic retry. Repeating it every
|
||||
# day adds noise; the queue remains the durable actionable state.
|
||||
if newly_queued:
|
||||
symbols_by_cik: dict[str, list[str]] = defaultdict(list)
|
||||
for symbol, cik in staged.resolved.symbol_to_cik.items():
|
||||
symbols_by_cik[cik10(cik)].append(symbol)
|
||||
named = ", ".join(
|
||||
f"{'/'.join(symbols_by_cik.get(gap['cik'], [])) or gap['cik']}"
|
||||
f"/{gap['accession']}"
|
||||
for gap in newly_queued[:10]
|
||||
)
|
||||
db.add(SystemEvent(
|
||||
severity="warning",
|
||||
source="sec_facts",
|
||||
code="unresolved_filing",
|
||||
message=(
|
||||
f"{len(newly_queued)} filing(s) entered automatic SEC retry. "
|
||||
f"Affected symbols are blocked from new actionable setups until "
|
||||
f"their filing is recovered: {named}"
|
||||
)[:4000],
|
||||
dedup_key=f"sec_facts:unresolved_filing:{run_id}",
|
||||
created_at=_now(),
|
||||
))
|
||||
|
||||
# A new registrant may have no XBRL filing yet. Keep it out of actionable
|
||||
# setups, but log it instead of raising a recurring operator warning.
|
||||
if staged.no_xbrl_filings:
|
||||
named = ", ".join(
|
||||
f"{e['cik']} ({e.get('name') or '?'})" for e in staged.no_xbrl_filings[:10]
|
||||
)
|
||||
logger.info(
|
||||
"sec_facts: %d registrant(s) have no XBRL history yet: %s",
|
||||
len(staged.no_xbrl_filings),
|
||||
named,
|
||||
)
|
||||
|
||||
ticker_counts = await sec_universe.apply_ticker_updates(
|
||||
db, staged.resolved, staged.sic_updates
|
||||
)
|
||||
@@ -362,11 +886,89 @@ class SecFundamentalsImporter:
|
||||
"updated": updated,
|
||||
"existing_unchanged": len(staged.existing_accessions) - updated,
|
||||
"discrepancies": len(staged.discrepancies),
|
||||
"retry_queue_added": len(newly_queued),
|
||||
"retry_queue_resolved": queue_resolved,
|
||||
**ticker_counts,
|
||||
}
|
||||
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _retry_backlog(
|
||||
self,
|
||||
db,
|
||||
tracked_ciks: set[int],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Active typed gaps; migration 028 owns historical bootstrap."""
|
||||
if not tracked_ciks:
|
||||
return []
|
||||
tracked = {cik10(cik) for cik in tracked_ciks}
|
||||
candidates: dict[str, dict[str, Any]] = {}
|
||||
|
||||
queued = await fundamentals_quality_service.active_gaps(db, tracked)
|
||||
for gap in queued:
|
||||
try:
|
||||
coregistrants = json.loads(gap.coregistrant_ciks_json or "[]")
|
||||
except (TypeError, ValueError):
|
||||
coregistrants = []
|
||||
candidates[gap.accession] = {
|
||||
"cik": gap.cik,
|
||||
"accession": gap.accession,
|
||||
"form": gap.form,
|
||||
"index_date": gap.index_date,
|
||||
"reason": gap.reason,
|
||||
"coregistrants": coregistrants,
|
||||
"_retry_queue": True,
|
||||
}
|
||||
|
||||
if not candidates:
|
||||
return []
|
||||
resolved = set(
|
||||
(
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot.accession).where(
|
||||
FundamentalSnapshot.accession.in_(list(candidates))
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
return [
|
||||
item
|
||||
for accession, item in candidates.items()
|
||||
if accession not in resolved
|
||||
]
|
||||
|
||||
async def _promotions_stale(self, db) -> bool:
|
||||
"""Has nothing promoted within ``PROMOTION_CEILING_DAYS``?
|
||||
|
||||
Only true for a source that HAS promoted before. A never-promoted import
|
||||
is initial setup, not a wedge: forcing its first promotion through would
|
||||
mask a misconfiguration rather than recover from a transient SEC gap.
|
||||
|
||||
Measured from ``self.today`` rather than the wall clock, so the ceiling
|
||||
honors the same injected date that ages the filings it releases.
|
||||
"""
|
||||
cutoff = datetime.combine(
|
||||
self.today - timedelta(days=PROMOTION_CEILING_DAYS),
|
||||
time.min,
|
||||
tzinfo=timezone.utc,
|
||||
)
|
||||
ever, recent = (
|
||||
await db.execute(
|
||||
select(
|
||||
exists().where(
|
||||
DataImportRun.source == SOURCE,
|
||||
DataImportRun.status == STATUS_PROMOTED,
|
||||
),
|
||||
exists().where(
|
||||
DataImportRun.source == SOURCE,
|
||||
DataImportRun.status == STATUS_PROMOTED,
|
||||
DataImportRun.started_at >= cutoff,
|
||||
),
|
||||
)
|
||||
)
|
||||
).one()
|
||||
return bool(ever) and not bool(recent)
|
||||
|
||||
async def _last_processed_index_date(self, db) -> date | None:
|
||||
return (
|
||||
await db.execute(
|
||||
@@ -390,9 +992,25 @@ class SecFundamentalsImporter:
|
||||
rows: list[dict[str, Any]] = []
|
||||
day = last_processed + timedelta(days=1)
|
||||
while day <= latest:
|
||||
# Group the whole day first: a combined filing is listed once per
|
||||
# co-registrant CIK, and those sibling CIKs are the only pointer to
|
||||
# where SEC may have put the XBRL (see _recover_from_coregistrant).
|
||||
by_accession: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for r in await client.daily_index(day):
|
||||
if r["form"] in _XBRL_FORMS and r["cik"] in tracked:
|
||||
rows.append(r)
|
||||
if r["form"] in _XBRL_FORMS:
|
||||
by_accession[r["accession"]].append(r)
|
||||
for accession, group in by_accession.items():
|
||||
filers = {r["cik"] for r in group}
|
||||
tracked_filers = filers & tracked
|
||||
if not tracked_filers:
|
||||
continue
|
||||
siblings = sorted(filers - tracked_filers)
|
||||
if siblings:
|
||||
self._coregistrants[accession] = siblings
|
||||
for r in group:
|
||||
if r["cik"] in tracked_filers:
|
||||
r["index_date"] = day # not hashed (revision uses cik/accession)
|
||||
rows.append(r)
|
||||
day += timedelta(days=1)
|
||||
return rows
|
||||
|
||||
@@ -409,6 +1027,30 @@ class SecFundamentalsImporter:
|
||||
).scalars().all()
|
||||
return {int(c) for c in found}
|
||||
|
||||
async def _last_shares_outstanding(self, db, ciks: set[int]) -> dict[str, float]:
|
||||
"""Latest known shares outstanding per tracked issuer — the continuity
|
||||
reference co-registrant recovery is checked against."""
|
||||
if not ciks:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot.cik, FundamentalSnapshot.shares_outstanding)
|
||||
.where(
|
||||
FundamentalSnapshot.cik.in_([cik10(c) for c in ciks]),
|
||||
FundamentalSnapshot.shares_outstanding.is_not(None),
|
||||
)
|
||||
# Last write per cik wins, so the sort must be total: an amendment
|
||||
# and its original share a period_end, and an undefined tie there
|
||||
# would make recovery non-deterministic across runs and dialects.
|
||||
.order_by(
|
||||
FundamentalSnapshot.period_end,
|
||||
FundamentalSnapshot.filed_date,
|
||||
FundamentalSnapshot.accession,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {cik: float(shares) for cik, shares in rows}
|
||||
|
||||
async def _existing_by_accession(self, db, accessions: list[str]) -> dict[str, FundamentalSnapshot]:
|
||||
if not accessions:
|
||||
return {}
|
||||
@@ -435,6 +1077,73 @@ def _companyfacts_structure_error(cf: Any) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _missing(
|
||||
cik: int,
|
||||
row: dict[str, Any],
|
||||
reason: str,
|
||||
today: date,
|
||||
coregistrants: list[int] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""One unresolvable index row, carrying everything needed to look the filing
|
||||
up by hand (EDGAR accession + the index date it was seen on) and to decide
|
||||
whether it is still young enough to be worth blocking on."""
|
||||
index_date = row.get("index_date")
|
||||
age_days = (
|
||||
(today - index_date).days if isinstance(index_date, date) else 0
|
||||
)
|
||||
if row.get("_retry_queue"):
|
||||
age_days = max(age_days, MISSING_XBRL_RETRY_DAYS + 1)
|
||||
return {
|
||||
"cik": cik10(cik),
|
||||
"accession": row["accession"],
|
||||
"form": row.get("form"),
|
||||
"index_date": index_date,
|
||||
# A newly observed row without a date blocks safely. A durable queue row
|
||||
# has already passed the bounded window and is forced aged-out above.
|
||||
"age_days": age_days,
|
||||
"reason": reason,
|
||||
"coregistrants": list(coregistrants or []),
|
||||
}
|
||||
|
||||
|
||||
def _within_retry_window(missing: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [m for m in missing if m.get("age_days", 0) <= MISSING_XBRL_RETRY_DAYS]
|
||||
|
||||
|
||||
def _past_retry_window(missing: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [m for m in missing if m.get("age_days", 0) > MISSING_XBRL_RETRY_DAYS]
|
||||
|
||||
|
||||
def _shares_continuous(shares: float | None, reference: float | None) -> bool:
|
||||
"""Does a co-registrant-recovered share count look like this issuer's own?
|
||||
|
||||
The failure worth preventing is storing a subsidiary's standalone facts as the
|
||||
parent's. A co-registrant shell holds a token float — Florida Power & Light
|
||||
against NextEra's 2.09bn shares — so any sane band separates them while still
|
||||
tolerating buybacks and issuance. With no history to compare against (a newly
|
||||
tracked issuer) or no share count at all, recovery is refused, not guessed.
|
||||
"""
|
||||
if not shares or not reference:
|
||||
return False
|
||||
return RECOVERY_SHARES_MIN <= shares / reference <= RECOVERY_SHARES_MAX
|
||||
|
||||
|
||||
def _reason_counts(missing: list[dict[str, Any]]) -> str:
|
||||
counts = Counter(m["reason"] for m in missing)
|
||||
return ", ".join(f"{reason}={n}" for reason, n in sorted(counts.items()))
|
||||
|
||||
|
||||
def _missing_detail(missing: list[dict[str, Any]], limit: int = 10) -> str:
|
||||
detail = ", ".join(
|
||||
f"{m['cik']}/{m['accession']} {m.get('form') or '?'} "
|
||||
f"[{m.get('index_date') or '?'}] {m['reason']}"
|
||||
for m in missing[:limit]
|
||||
)
|
||||
if len(missing) > limit:
|
||||
detail += f", +{len(missing) - limit} more"
|
||||
return detail
|
||||
|
||||
|
||||
def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]:
|
||||
"""(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only
|
||||
includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata."""
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import Iterable
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import settings_store
|
||||
from app.services import settings_store, ticker_service
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
@@ -55,7 +55,11 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
||||
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
|
||||
ticker_to_cik = await client.company_tickers()
|
||||
overrides = await cik_overrides(db)
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
|
||||
rows = (
|
||||
await db.execute(
|
||||
ticker_service.active_only(select(Ticker.id, Ticker.symbol, Ticker.cik))
|
||||
)
|
||||
).all()
|
||||
|
||||
result = ResolvedUniverse()
|
||||
for tid, symbol, current_cik in rows:
|
||||
|
||||
@@ -40,10 +40,12 @@ KEY_CAPACITY = "shadow_book_capacity"
|
||||
KEY_RISK_PCT = "shadow_book_risk_pct"
|
||||
KEY_START_EQUITY = "shadow_book_start_equity"
|
||||
|
||||
# Matches the validated configuration: 10-position book, 1% fixed-fractional
|
||||
# risk. Start equity is only a sizing base — comparisons are drawn in percent
|
||||
# and R-multiples, never in raw currency.
|
||||
DEFAULT_CAPACITY = 10
|
||||
# Matches the validated configuration: 1% fixed-fractional risk, and a count cap
|
||||
# set as headroom rather than a target — see backtest_service.SIM_MAX_POSITIONS,
|
||||
# which this must track. NOTIONAL_CAP below saturates the book near 12 positions,
|
||||
# so the count cap should simply never bind. Start equity is only a sizing base —
|
||||
# comparisons are drawn in percent and R-multiples, never in raw currency.
|
||||
DEFAULT_CAPACITY = 15
|
||||
DEFAULT_RISK_PCT = 1.0
|
||||
DEFAULT_START_EQUITY = 100_000.0
|
||||
|
||||
|
||||
@@ -1,13 +1,65 @@
|
||||
"""Ticker Registry service: add, delete, and list tracked tickers."""
|
||||
"""Ticker Registry service: add, delete, list, and retire tracked tickers."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.exceptions import DuplicateError, NotFoundError, ValidationError
|
||||
from app.models.ticker import Ticker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Reasons a symbol may be marked delisted, narrowest first.
|
||||
REASON_FORM_25 = "form_25" # SEC Form 25/25-NSE/15 confirmed the exchange exit
|
||||
REASON_MANUAL = "manual" # an operator decided
|
||||
|
||||
# How long a symbol must be without bars before we spend an SEC request asking
|
||||
# whether it delisted. Guards against a market-data outage probing the whole
|
||||
# universe at once; a real delisting is still stale days later.
|
||||
MIN_STALE_DAYS_BEFORE_PROBE = 3
|
||||
|
||||
# Rule 12d2-2: a Form 25 removal takes effect ten days after filing, so the
|
||||
# filing date is not the date the security stopped trading.
|
||||
FORM_25_EFFECTIVE_DAYS = 10
|
||||
|
||||
# How far before the last bar a Form 25 may be filed and still explain this gap.
|
||||
# An exchange can file shortly before trading actually stops; anything older
|
||||
# concerns a class that was already gone while the symbol kept printing bars.
|
||||
FILING_LOOKBACK_DAYS = 30
|
||||
|
||||
|
||||
def _sec_client_factory():
|
||||
"""Build the SEC client for a delisting probe (patched in tests).
|
||||
|
||||
Imported lazily so the SEC/httpx stack stays off the import path of every
|
||||
module that only wants ``active_only``.
|
||||
"""
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
return SecClient()
|
||||
|
||||
|
||||
def active_only(stmt, *, as_of: date | None = None):
|
||||
"""Restrict a Ticker query to symbols that still trade.
|
||||
|
||||
Opt-in on purpose rather than folded into a shared getter: list and admin
|
||||
views deliberately keep delisted rows so the delisting is *visible*, which a
|
||||
silent default would undo. Apply this on the live signal path — scanning,
|
||||
ranking, scoring, breadth, ingestion — and nowhere else.
|
||||
|
||||
``delisted_on`` is an *effective* date, and a Form 25 is known ten days
|
||||
before it takes effect, so a future date must not drop the symbol yet — it
|
||||
is still trading and still worth scanning and ingesting. Compared in SQL
|
||||
against the database's own date; ``as_of`` overrides it for tests.
|
||||
"""
|
||||
cutoff = func.current_date() if as_of is None else as_of
|
||||
return stmt.where(
|
||||
or_(Ticker.delisted_on.is_(None), Ticker.delisted_on > cutoff)
|
||||
)
|
||||
|
||||
|
||||
async def add_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
||||
"""Add a new ticker after validation.
|
||||
@@ -52,6 +104,150 @@ async def delete_ticker(db: AsyncSession, symbol: str) -> None:
|
||||
|
||||
|
||||
async def list_tickers(db: AsyncSession) -> list[Ticker]:
|
||||
"""Return all tracked tickers sorted alphabetically by symbol."""
|
||||
"""Return all tracked tickers sorted alphabetically by symbol.
|
||||
|
||||
Delisted symbols are included and carry ``delisted_on`` — the registry is
|
||||
where an operator needs to *see* that a symbol retired, not where it should
|
||||
quietly disappear.
|
||||
"""
|
||||
result = await db.execute(select(Ticker).order_by(Ticker.symbol.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def mark_delisted(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
*,
|
||||
delisted_on: date,
|
||||
reason: str = REASON_MANUAL,
|
||||
) -> bool:
|
||||
"""Record that a symbol stopped trading. True if this changed anything.
|
||||
|
||||
Idempotent, so the staleness path can call it every run without churning the
|
||||
row: re-marking is a no-op. The one exception is an SEC confirmation landing
|
||||
on a row an operator marked by hand — Form 25 carries the real effective
|
||||
date, so it replaces the operator's estimate. Nothing downgrades a confirmed
|
||||
row back to a manual one.
|
||||
"""
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None:
|
||||
raise NotFoundError(f"Ticker not found: {normalised}")
|
||||
if ticker.delisted_on is not None:
|
||||
upgrading = (
|
||||
reason == REASON_FORM_25 and ticker.delisted_reason != REASON_FORM_25
|
||||
)
|
||||
if not upgrading:
|
||||
return False
|
||||
|
||||
await db.execute(
|
||||
update(Ticker)
|
||||
.where(Ticker.id == ticker.id)
|
||||
.values(delisted_on=delisted_on, delisted_reason=reason)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"ticker %s marked delisted on %s (%s)", normalised, delisted_on, reason
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def confirm_delisting(
|
||||
db: AsyncSession,
|
||||
symbol: str,
|
||||
*,
|
||||
last_bar: date | None,
|
||||
today: date | None = None,
|
||||
) -> date | None:
|
||||
"""Ask SEC whether ``symbol`` actually delisted; mark it if so.
|
||||
|
||||
Called when OHLCV goes stale, because "no new bars" alone cannot tell a
|
||||
delisting from a halt or a rename. Returns the effective date whenever the
|
||||
symbol is known to have delisted — whether this call established that or an
|
||||
earlier one did — and ``None`` while it remains unproven, so the caller warns
|
||||
only about gaps that still have no explanation.
|
||||
|
||||
Returning the already-known date matters between filing and effect: trading
|
||||
usually stops before the ten-day Rule 12d2-2 delay expires, so the symbol is
|
||||
correctly still active (see ``active_only``) while producing no bars. Without
|
||||
this the staleness warning would fire daily across that window — the exact
|
||||
noise the delisting flow exists to remove.
|
||||
|
||||
Deliberately driven by staleness rather than by the SEC fundamentals import:
|
||||
that importer stalls for days at a time on unrelated Company-Facts gaps, and
|
||||
detection wired into it would stall with it.
|
||||
|
||||
The probe waits for ``MIN_STALE_DAYS_BEFORE_PROBE``. A delisted symbol stays
|
||||
stale forever, so the delay costs nothing, and it keeps a broad market-data
|
||||
outage — where every tracked symbol reports stale at once — from turning into
|
||||
one SEC request per symbol per run.
|
||||
"""
|
||||
from app.services.sec_client import SecError
|
||||
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None:
|
||||
return None
|
||||
known = ticker.delisted_on
|
||||
# Already confirmed by SEC — nothing left to learn, but the caller still
|
||||
# needs the date to know this gap is explained. A row an operator marked by
|
||||
# hand is worth probing: Form 25 upgrades the estimated date.
|
||||
if ticker.delisted_reason == REASON_FORM_25:
|
||||
return known
|
||||
if not ticker.cik:
|
||||
return known
|
||||
# No bars at all is an ingestion problem, not evidence of a delisting.
|
||||
if last_bar is None:
|
||||
return known
|
||||
if ((today or date.today()) - last_bar).days < MIN_STALE_DAYS_BEFORE_PROBE:
|
||||
return known
|
||||
|
||||
try:
|
||||
async with _sec_client_factory() as client:
|
||||
# Only a Form 25 filed around or after the last bar can explain THIS
|
||||
# gap. An older one belongs to a class that stopped trading before
|
||||
# the symbol was still printing bars, and must not retire it.
|
||||
filing = await client.delisting_filing(
|
||||
ticker.cik, not_before=last_bar - timedelta(days=FILING_LOOKBACK_DAYS)
|
||||
)
|
||||
except SecError:
|
||||
# Never let a probe failure escalate a routine staleness warning.
|
||||
logger.warning("delisting probe failed for %s", normalised, exc_info=True)
|
||||
return known
|
||||
|
||||
if filing is None:
|
||||
return known
|
||||
# Removal takes effect ten days after filing, so the filing date is not the
|
||||
# date the symbol stopped trading.
|
||||
effective = filing["filing_date"] + timedelta(days=FORM_25_EFFECTIVE_DAYS)
|
||||
if await mark_delisted(
|
||||
db, normalised, delisted_on=effective, reason=REASON_FORM_25
|
||||
):
|
||||
return effective
|
||||
return known
|
||||
|
||||
|
||||
async def clear_delisted(db: AsyncSession, symbol: str) -> bool:
|
||||
"""Un-retire a symbol. True if it had been marked.
|
||||
|
||||
The counterpart that makes automatic marking acceptable: a false positive
|
||||
costs one row update, where a delete would have cost the price history.
|
||||
"""
|
||||
normalised = symbol.strip().upper()
|
||||
result = await db.execute(select(Ticker).where(Ticker.symbol == normalised))
|
||||
ticker = result.scalar_one_or_none()
|
||||
if ticker is None:
|
||||
raise NotFoundError(f"Ticker not found: {normalised}")
|
||||
if ticker.delisted_on is None:
|
||||
return False
|
||||
|
||||
await db.execute(
|
||||
update(Ticker)
|
||||
.where(Ticker.id == ticker.id)
|
||||
.values(delisted_on=None, delisted_reason=None)
|
||||
)
|
||||
await db.commit()
|
||||
logger.info("ticker %s un-marked as delisted", normalised)
|
||||
return True
|
||||
|
||||
@@ -113,116 +113,6 @@ def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
|
||||
return sorted(deduped)
|
||||
|
||||
|
||||
def _extract_symbols_from_fmp_payload(payload: object) -> list[str]:
|
||||
if not isinstance(payload, list):
|
||||
return []
|
||||
|
||||
symbols: list[str] = []
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
candidate = item.get("symbol") or item.get("ticker")
|
||||
if isinstance(candidate, str):
|
||||
symbols.append(candidate)
|
||||
return symbols
|
||||
|
||||
|
||||
async def _try_fmp_urls(
|
||||
client: httpx.AsyncClient,
|
||||
urls: list[str],
|
||||
) -> tuple[list[str], list[str]]:
|
||||
failures: list[str] = []
|
||||
for url in urls:
|
||||
endpoint = url.split("?")[0]
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})")
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
failures.append(f"{endpoint}: HTTP {response.status_code}")
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
failures.append(f"{endpoint}: invalid JSON payload")
|
||||
continue
|
||||
|
||||
symbols = _extract_symbols_from_fmp_payload(payload)
|
||||
if symbols:
|
||||
return symbols, failures
|
||||
|
||||
failures.append(f"{endpoint}: empty/unsupported payload")
|
||||
|
||||
return [], failures
|
||||
|
||||
|
||||
async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
|
||||
if not settings.fmp_api_key:
|
||||
raise ValidationError(
|
||||
"FMP API key is required for universe bootstrap (set FMP_API_KEY)"
|
||||
)
|
||||
|
||||
api_key = settings.fmp_api_key
|
||||
stable_base = "https://financialmodelingprep.com/stable"
|
||||
legacy_base = "https://financialmodelingprep.com/api/v3"
|
||||
|
||||
stable_candidates: dict[str, list[str]] = {
|
||||
"sp500": [
|
||||
f"{stable_base}/sp500-constituent?apikey={api_key}",
|
||||
f"{stable_base}/sp500-constituents?apikey={api_key}",
|
||||
],
|
||||
"nasdaq100": [
|
||||
f"{stable_base}/nasdaq-100-constituent?apikey={api_key}",
|
||||
f"{stable_base}/nasdaq100-constituent?apikey={api_key}",
|
||||
f"{stable_base}/nasdaq-100-constituents?apikey={api_key}",
|
||||
],
|
||||
"nasdaq_all": [
|
||||
f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
||||
f"{stable_base}/available-traded/list?apikey={api_key}",
|
||||
],
|
||||
}
|
||||
|
||||
legacy_candidates: dict[str, list[str]] = {
|
||||
"sp500": [
|
||||
f"{legacy_base}/sp500_constituent?apikey={api_key}",
|
||||
f"{legacy_base}/sp500_constituent",
|
||||
],
|
||||
"nasdaq100": [
|
||||
f"{legacy_base}/nasdaq_constituent?apikey={api_key}",
|
||||
f"{legacy_base}/nasdaq_constituent",
|
||||
],
|
||||
"nasdaq_all": [
|
||||
f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
||||
],
|
||||
}
|
||||
|
||||
failures: list[str] = []
|
||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
||||
stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe])
|
||||
failures.extend(stable_failures)
|
||||
|
||||
if stable_symbols:
|
||||
return stable_symbols
|
||||
|
||||
legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe])
|
||||
failures.extend(legacy_failures)
|
||||
|
||||
if legacy_symbols:
|
||||
return legacy_symbols
|
||||
|
||||
if failures:
|
||||
reason = "; ".join(failures[:6])
|
||||
logger.warning("FMP universe fetch failed for %s: %s", universe, reason)
|
||||
raise ProviderError(
|
||||
f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}"
|
||||
)
|
||||
|
||||
raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
|
||||
|
||||
|
||||
async def _fetch_wiki_constituent_symbols(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
@@ -351,13 +241,16 @@ async def fetch_universe_symbols(
|
||||
|
||||
Fallback order:
|
||||
1) Free public sources (Wikipedia/NASDAQ trader)
|
||||
2) FMP endpoints (if available)
|
||||
3) Cached snapshot in SystemSetting
|
||||
4) Built-in seed symbols
|
||||
2) Cached snapshot in SystemSetting
|
||||
3) Built-in seed symbols
|
||||
|
||||
Returns ``(symbols, source_label)`` so bootstrap UI can show where the
|
||||
list came from (important when Wikipedia/FMP fail and a stale cache still
|
||||
lists BK instead of BNY).
|
||||
list came from (important when the public source fails and a stale cache
|
||||
still lists BK instead of BNY).
|
||||
|
||||
The seeds are representative, not complete, so a *fresh* install whose
|
||||
public source is down bootstraps a partial universe. A warm instance is
|
||||
unaffected — it falls through to its cached snapshot.
|
||||
"""
|
||||
normalised_universe = _validate_universe(universe)
|
||||
failures: list[str] = []
|
||||
@@ -369,15 +262,6 @@ async def fetch_universe_symbols(
|
||||
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
|
||||
return cleaned_public, public_source or "public"
|
||||
|
||||
try:
|
||||
fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
|
||||
cleaned_fmp = _normalise_symbols(fmp_symbols)
|
||||
if cleaned_fmp:
|
||||
await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
|
||||
return cleaned_fmp, "fmp"
|
||||
except (ProviderError, ValidationError) as exc:
|
||||
failures.append(str(exc))
|
||||
|
||||
cached_symbols = await _read_cached_symbols(db, normalised_universe)
|
||||
if cached_symbols:
|
||||
logger.warning(
|
||||
@@ -473,9 +357,26 @@ async def bootstrap_universe(
|
||||
db.add(Ticker(symbol=symbol))
|
||||
|
||||
deleted_count = 0
|
||||
skipped_delisted: list[str] = []
|
||||
if symbols_to_delete:
|
||||
result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(symbols_to_delete)))
|
||||
deleted_count = int(result.rowcount or 0)
|
||||
# A delisted row was retained on purpose — its price history is exactly
|
||||
# what a survivorship-honest backtest needs, and the delete cascades it
|
||||
# away. Pruning must not undo that. (Pruning a symbol that is merely no
|
||||
# longer an index constituent still destroys history; that needs a
|
||||
# tracked/membership state separate from delisting.)
|
||||
protected = (
|
||||
await db.execute(
|
||||
select(Ticker.symbol).where(
|
||||
Ticker.symbol.in_(symbols_to_delete),
|
||||
Ticker.delisted_on.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
skipped_delisted = sorted(protected)
|
||||
deletable = [s for s in symbols_to_delete if s not in set(protected)]
|
||||
if deletable:
|
||||
result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(deletable)))
|
||||
deleted_count = int(result.rowcount or 0)
|
||||
|
||||
await db.commit()
|
||||
|
||||
@@ -494,4 +395,8 @@ async def bootstrap_universe(
|
||||
"already_tracked": len(target_symbols & existing_symbols),
|
||||
"deleted": deleted_count,
|
||||
"added_symbols": symbols_to_add[:50],
|
||||
# Delisted rows a prune declined to destroy, so the caller can see the
|
||||
# count did not match what they asked to remove.
|
||||
"kept_delisted": skipped_delisted[:50],
|
||||
"kept_delisted_count": len(skipped_delisted),
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ MIN_FREE_GB="${DOLT_MIN_FREE_DISK_GB:-5}"
|
||||
EARNINGS_DIR="${DOLT_DATA_DIR}/${DOLT_EARNINGS_SUBDIR}"
|
||||
DOLT_IDENTITY_NAME="${DOLT_IDENTITY_NAME:-Signal Platform}"
|
||||
DOLT_IDENTITY_EMAIL="${DOLT_IDENTITY_EMAIL:-signal-platform@localhost}"
|
||||
FUNDAMENTALS_PARITY_REPORT_DIR="${FUNDAMENTALS_PARITY_REPORT_DIR:-/var/lib/signal-platform/reports/fundamentals-parity}"
|
||||
|
||||
fail() {
|
||||
echo "ERROR: $*" >&2
|
||||
@@ -80,8 +79,6 @@ check_env() {
|
||||
|| fail "set DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR in $ENV_FILE"
|
||||
grep -Eq '^SEC_USER_AGENT=.*@.*' "$ENV_FILE" \
|
||||
|| fail "SEC_USER_AGENT in $ENV_FILE must contain a real contact email"
|
||||
grep -Fqx "FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR" "$ENV_FILE" \
|
||||
|| fail "set FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR in $ENV_FILE"
|
||||
}
|
||||
|
||||
check_all() {
|
||||
@@ -104,15 +101,6 @@ check_all() {
|
||||
identity_email="$(repo_config_value user.email 2>/dev/null || true)"
|
||||
[[ -n "$identity_name" ]] || fail "missing Dolt user.name for $EARNINGS_DIR"
|
||||
[[ -n "$identity_email" ]] || fail "missing Dolt user.email for $EARNINGS_DIR"
|
||||
[[ -d "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \
|
||||
|| fail "missing parity report directory: $FUNDAMENTALS_PARITY_REPORT_DIR"
|
||||
if [[ "$(id -un)" == "$APP_USER" ]]; then
|
||||
[[ -w "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \
|
||||
|| fail "parity report directory is not writable by $APP_USER"
|
||||
else
|
||||
runuser -u "$APP_USER" -- test -w "$FUNDAMENTALS_PARITY_REPORT_DIR" \
|
||||
|| fail "parity report directory is not writable by $APP_USER"
|
||||
fi
|
||||
check_free_space
|
||||
check_env
|
||||
echo "OK: Dolt $DOLT_VERSION and earnings clone are provisioned"
|
||||
@@ -139,7 +127,6 @@ fi
|
||||
version_ok || fail "Dolt $DOLT_VERSION installation failed"
|
||||
|
||||
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$DOLT_DATA_DIR"
|
||||
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$FUNDAMENTALS_PARITY_REPORT_DIR"
|
||||
check_free_space
|
||||
|
||||
if [[ ! -d "$EARNINGS_DIR/.dolt" ]]; then
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
# Dolt bulk-data integration — implementation plan
|
||||
|
||||
Status: approved 2026-07-21, revised through four review rounds; direction: KISS
|
||||
backend, UI value first. Hand-off document for the implementing agent;
|
||||
self-contained.
|
||||
Status: **workstream A complete and deployed** (A0–A6, last step 2026-08-07);
|
||||
**workstream B dropped 2026-08-07** — see § Why B was dropped. Approved 2026-07-21,
|
||||
revised through five review rounds; direction: KISS backend, UI value first.
|
||||
Originally a hand-off document for the implementing agent; now the design record.
|
||||
Current operations live in `docs/fundamentals-deployment.md`.
|
||||
|
||||
## Objective
|
||||
|
||||
Replace the free-tier fundamentals APIs (FMP, Finnhub, Alpha Vantage) with bulk
|
||||
data: SEC Company Facts for fundamentals, the DoltHub earnings repo for the
|
||||
earnings calendar/history, and — later, independently — the DoltHub stocks repo for
|
||||
historical OHLCV. PostgreSQL stays the production system of record.
|
||||
data: SEC Company Facts for fundamentals and the DoltHub earnings repo for the
|
||||
earnings calendar/history. PostgreSQL stays the production system of record.
|
||||
(A third source — the DoltHub stocks repo for historical OHLCV — was planned as
|
||||
workstream B and dropped; Alpaca remains the price source.)
|
||||
|
||||
**Delivery order: two independent workstreams.**
|
||||
|
||||
@@ -17,9 +20,9 @@ historical OHLCV. PostgreSQL stays the production system of record.
|
||||
FundamentalsPanel + decommission FMP/Finnhub/Alpha Vantage. Valuation uses the
|
||||
existing Alpaca closes already in `ohlcv_records`. This alone achieves the goal
|
||||
(killing the quota-limited APIs) and delivers all the UI value.
|
||||
- **Workstream B (later, optional until needed):** replace historical OHLCV with
|
||||
the Dolt stocks repo. The most complex machinery (4.7 GB clone, split
|
||||
adjustment, source-bar table, reconciliation) lives here and blocks nothing in A.
|
||||
- **Workstream B — DROPPED 2026-08-07, see below.** Would have replaced historical
|
||||
OHLCV with the Dolt stocks repo. Its design is retained further down as a record,
|
||||
not as a backlog item.
|
||||
|
||||
**Guiding principle: KISS.** Plain daily importers with staging and atomic
|
||||
promotion — no forensic replay, no permanent archive store, no conflict tables, no
|
||||
@@ -70,8 +73,9 @@ notes (retain a CC BY-SA 4.0 reference + attribution to `post-no-preference/earn
|
||||
and a note of the transformations applied — e.g. in a repo `NOTICE`/attribution file
|
||||
and the importer module); **no public API, bulk export, or redistribution** of the
|
||||
data; re-review licensing before any public or commercial access. The
|
||||
`post-no-preference/stocks` repo (workstream B) is **not** covered here and will be
|
||||
reviewed separately if B begins.
|
||||
`post-no-preference/stocks` repo (workstream B) is **not** covered here. B was
|
||||
dropped before any licensing review, so that repo has never been assessed — any
|
||||
future use of it starts that review from scratch.
|
||||
|
||||
## Schema
|
||||
|
||||
@@ -119,7 +123,9 @@ reviewed separately if B begins.
|
||||
cache, repopulated by the daily SEC job — but only after the phase-A5 parity
|
||||
gate.
|
||||
|
||||
**Migration 027 (workstream B, written when B starts):**
|
||||
**Migration 027 (workstream B — NEVER WRITTEN; B was dropped, and `027` was
|
||||
subsequently used for `fundamental_snapshots.weighted_avg_diluted_shares`). The
|
||||
design below is a record only:**
|
||||
|
||||
- `ohlcv_source_bars` — source-truth bar table, required because `ohlcv_records`
|
||||
allows one row per (ticker_id, date) (`app/models/ohlcv.py:12`) and Alpaca
|
||||
@@ -219,7 +225,7 @@ Workstream A:
|
||||
**The new API valuation object is not stored anywhere** — it is computed at
|
||||
request time (below). No valuation cache or table exists.
|
||||
|
||||
Workstream B:
|
||||
Workstream B (dropped — never built):
|
||||
|
||||
- Dolt OHLCV+splits pull/import: `0 2 * * tue-sat` ET. If source_max_date is not
|
||||
fresh, retry hourly until ~06:00, then give up quietly. After a successful
|
||||
@@ -427,18 +433,58 @@ workstream B — Alpaca remains the price source throughout.
|
||||
fundamental-score/ranking changes, require explicit approval. Definition
|
||||
changes (e.g. TTM vs provider convention) called out, not averaged away.
|
||||
**Status 2026-07-24: the gate has been exercised and the evidence supports
|
||||
approval** — see the handoff section below. What remains of A5 is the
|
||||
activation itself: implementing step (c) and flipping it on.
|
||||
- A6. Remove FMP/Finnhub/Alpha Vantage; keep monitoring + manual fallback.
|
||||
approval** — see the handoff section below. Step (c) is implemented behind the
|
||||
default-off `fundamental_data_sec_dolt_cutover_enabled` SystemSetting; the
|
||||
remaining production action is flipping that switch on and observing it.
|
||||
- A6. **DONE 2026-08-07.** FMP/Finnhub/Alpha Vantage removed, along with the
|
||||
weekly `fundamental_collector` job, the A5 cutover toggle (SEC+Dolt is now the
|
||||
unconditional path) and the parity report. Migration `029` tombstoned the two
|
||||
behavior-bearing settings rows for the rollback window and `030` dropped them
|
||||
once the deploy was confirmed healthy; the archived parity bundles stay as the
|
||||
A5 evidence trail.
|
||||
|
||||
**Workstream B (independent, start when wanted):**
|
||||
**Workstream B — DROPPED 2026-08-07.** The phases below are recorded for anyone
|
||||
who revisits the decision; none of them are scheduled work.
|
||||
|
||||
- B0. Stocks clone (~4.7 GB) provisioned; migration 027.
|
||||
- B1. OHLCV + split adjustment in shadow (writes `ohlcv_source_bars` only; Alpaca
|
||||
keeps owning `ohlcv_records`); historical backfill.
|
||||
- B2. Reconciliation window (≥ 2 weeks) vs Alpaca; review validation summaries.
|
||||
- B3. Promote Dolt as historical OHLCV source (canonical rebuilt from raw source
|
||||
bars + splits); morning pipeline → 03:00.
|
||||
- ~~B0. Stocks clone (~4.7 GB) provisioned; migration 027.~~
|
||||
- ~~B1. OHLCV + split adjustment in shadow (writes `ohlcv_source_bars` only; Alpaca
|
||||
keeps owning `ohlcv_records`); historical backfill.~~
|
||||
- ~~B2. Reconciliation window (≥ 2 weeks) vs Alpaca; review validation summaries.~~
|
||||
- ~~B3. Promote Dolt as historical OHLCV source (canonical rebuilt from raw source
|
||||
bars + splits); morning pipeline → 03:00.~~
|
||||
|
||||
### Why B was dropped
|
||||
|
||||
Reviewed after A6 shipped. Four reasons, in order of weight:
|
||||
|
||||
1. **Its motivation no longer exists.** B was scoped inside a plan whose goal was
|
||||
killing the quota-limited free-tier APIs. Alpaca was never one of them, and the
|
||||
plan always said so (§ Decommissioning: "Alpaca remains the price source
|
||||
throughout"). A6 achieved the goal. What remained was swapping one working
|
||||
price source for another.
|
||||
2. **Its only concrete benefit is reachable far more cheaply.** The prize was
|
||||
`corporate_actions`, the documented fix for the KLAC-class post-filing split
|
||||
(TTM EPS pre-split vs a post-split price → P/E 6.19 instead of ~13, invisible to
|
||||
snapshots). That needs *split events*, not 4.7 GB of bars — and the Alpaca SDK
|
||||
already in the venv exposes them via
|
||||
`alpaca.data.historical.corporate_actions.CorporateActionsClient.get_corporate_actions`
|
||||
with `CorporateActionsRequest` / `CorporateActionsType`. See the follow-up below.
|
||||
3. **The benefit is small.** Fundamentals carry 20% of the composite, P/E is one of
|
||||
three fundamental inputs, and only names that split between their last 10-Q and
|
||||
today are affected — a handful at a time, self-correcting at the next filing.
|
||||
4. **B would add a risk the current setup does not carry.** By design a newly
|
||||
published split rewrites a symbol's entire adjusted history. A backtest↔prod
|
||||
parity guard exists precisely because changed history invalidates comparisons;
|
||||
B makes history mutable as a routine event. It also needs its own license
|
||||
review — the A0 CC BY-SA decision covers only `post-no-preference/earnings`.
|
||||
|
||||
**Optional follow-up, not scheduled:** a small `corporate_actions` table populated
|
||||
from Alpaca, used to null or correct P/E when a split post-dates the newest
|
||||
snapshot. Roughly a day's work; captures essentially all of B's value with no
|
||||
clone, no `ohlcv_source_bars`, no split-adjustment pipeline and no reconciliation
|
||||
window. Worth doing only if the wart starts costing something — it has been visible
|
||||
and harmless since July 2026. Note that migration numbering has moved on: head is
|
||||
`030`, so any such table would be `031+`, not the `027` named below.
|
||||
|
||||
## Test plan
|
||||
|
||||
@@ -463,8 +509,8 @@ workstream B — Alpaca remains the price source throughout.
|
||||
falls back from P/E to FCF yield for the valuation segment when P/E is null.
|
||||
- Peer comparison disappears below 5 peer issuers; favorable-percentile direction
|
||||
correct for both polarities.
|
||||
- Workstream B: split-adjusted OHLCV matches Alpaca on representative normal /
|
||||
split / reverse-split symbols.
|
||||
- ~~Workstream B: split-adjusted OHLCV matches Alpaca on representative normal /
|
||||
split / reverse-split symbols.~~ (dropped)
|
||||
- UI states: positive, adverse, neutral, insufficient history, insufficient
|
||||
peers; mobile layout; non-color accessibility.
|
||||
- Unit, integration, scheduler and frontend suites pass.
|
||||
@@ -491,26 +537,32 @@ Post-fix: candidate scores 504 of 511 vs legacy's 507 (gap = PSKY/Q new registra
|
||||
FITB, all explained); revenue-growth agreement 0.0038 median abs delta where both exist.
|
||||
Dennis reviewed the evidence 2026-07-24 and directed proceeding to cutover.
|
||||
|
||||
**Task 1 — A5 activation (implement step (c) above, ~line 207).** The post-activation
|
||||
local refresh of `fundamental_data` does not exist yet. Per the spec: `pe_ratio` and
|
||||
`market_cap` from newest valid snapshots × latest PostgreSQL close, `revenue_growth`
|
||||
from snapshots, `earnings_surprise`/`next_earnings_date` from `earnings_events`; mark
|
||||
affected cached fundamental scores stale; must run identically when SEC is unreachable.
|
||||
Implementation notes from the parity work: consume `fundamentals_derivation.derive()`
|
||||
outputs, NOT raw snapshot fields — that path carries the split guard (`ttm_diluted_eps`
|
||||
nulls when contaminated, with `ttm_diluted_eps_caveat`) and the multi-class share
|
||||
fallback (`shares_outstanding` + `shares_outstanding_estimated`). Activation should be
|
||||
an explicit switch (SystemSetting, like `sec_cik_overrides`), default off.
|
||||
**Task 1 — A5 activation: DONE.** Implemented 2026-07-24, switched on and observed
|
||||
in production, and made unconditional by A6 (2026-08-07) — there is no longer a
|
||||
switch, an Admin card, or a weekly legacy collector to skip. The local refresh of
|
||||
`fundamental_data` derives `pe_ratio` and `market_cap` from newest valid snapshots ×
|
||||
latest PostgreSQL close, `revenue_growth` from snapshots, and
|
||||
`earnings_surprise`/`next_earnings_date` from `earnings_events`; it marks affected
|
||||
cached fundamental scores stale and runs identically when SEC is unreachable.
|
||||
It consumes `fundamentals_derivation.derive()` outputs, NOT raw snapshot fields —
|
||||
that path carries the split guard (`ttm_diluted_eps` nulls when contaminated, with
|
||||
`ttm_diluted_eps_caveat`) and the multi-class share fallback (`shares_outstanding` +
|
||||
`shares_outstanding_estimated`). See `docs/fundamentals-deployment.md` for current
|
||||
operations and rollback.
|
||||
|
||||
**Task 2 — A6 decommissioning.** After a short observation window: remove
|
||||
FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual
|
||||
fallback. Gated by the acceptance criteria above — especially forward-calendar
|
||||
timeliness from `dolt_earnings` (its `source_max_date` ran ~5 weeks ahead as of
|
||||
2026-07-23, which passes).
|
||||
**Task 2 — A6 decommissioning: DONE 2026-08-07.** The cutover ran on and was
|
||||
observed in production, so the legacy providers, their config/env keys, the weekly
|
||||
collector job and the parity report were all removed. Two consequences to carry:
|
||||
(1) `fundamental_data` now has no provider fallback — recovery is restore-from-backup;
|
||||
(2) disabling **SEC Fundamentals Import** stops the SEC fetch only, because the local
|
||||
cache refresh was deliberately moved outside the job-enable check. No follow-ups
|
||||
remain: migration `030` dropped the tombstone rows after the deploy was verified.
|
||||
|
||||
**Known caveats to carry (documented in the findings report, not bugs to fix):**
|
||||
- KLAC-class post-filing splits: P/E wrong until the next 10-Q; undetectable from
|
||||
snapshots. Workstream B's `corporate_actions` table is the natural future fix.
|
||||
snapshots. Still open and still harmless. The fix, if ever wanted, is a small
|
||||
`corporate_actions` table fed from Alpaca — **not** workstream B, which was
|
||||
dropped; see § Why B was dropped.
|
||||
- BRK-B: no share count exists anywhere in companyfacts → no market cap, correctly.
|
||||
- FITB: unscored (split guard + no taggable revenue) — the one name that lost its
|
||||
score relative to legacy; composite renormalises.
|
||||
@@ -523,7 +575,7 @@ timeliness from `dolt_earnings` (its `source_max_date` ran ~5 weeks ahead as of
|
||||
|
||||
## Deferred (explicitly, until a concrete need appears)
|
||||
|
||||
- Workstream B itself is deferred relative to A and blocks nothing in A.
|
||||
- Workstream B: **dropped** 2026-08-07, not deferred — see § Why B was dropped.
|
||||
- Exact byte-level source replay of historical imports; permanent archive store.
|
||||
- Point-in-time backtest enforcement (`accepted_at` is stored now; derivation and
|
||||
backtest visibility rules are built only when fundamentals enter
|
||||
|
||||
@@ -226,7 +226,12 @@ primary period (safe — a filing's own context is correct for its current perio
|
||||
|
||||
Identifying `User-Agent` with contact email on every request; well under 10 req/s
|
||||
with spacing; exponential backoff on 429; **403 → alert and stop, never
|
||||
retry-loop**. New config: `sec_user_agent`, `sec_request_spacing_seconds`,
|
||||
retry-loop** — with one carved-out exception: `www.sec.gov/Archives` is served
|
||||
from an S3 bucket without a `ListBucket` grant, so an **absent** file 403s with
|
||||
S3's `AccessDenied` XML rather than 404 (every weekend/holiday daily index does
|
||||
this). That one shape is read as "missing"; a real rejection is the WAF's
|
||||
`text/html` "Undeclared Automated Tool" page and still stops the run.
|
||||
New config: `sec_user_agent`, `sec_request_spacing_seconds`,
|
||||
`sec_max_retries`. Keep only the last ~2 fetched artifacts on disk for debugging
|
||||
(reproducibility is the normalized Postgres rows, per the plan).
|
||||
|
||||
|
||||
@@ -1,18 +1,40 @@
|
||||
# Fundamentals production deployment
|
||||
|
||||
This is the one-time production setup for the Dolt earnings and SEC fundamentals
|
||||
imports. Both imports remain shadow inputs until the separate A5 scoring-cutover
|
||||
approval. Do not add OS cron entries: the application scheduler owns both jobs.
|
||||
imports. Since A6 (2026-08) these are the *only* fundamentals sources — the
|
||||
FMP/Finnhub/Alpha Vantage providers, the weekly legacy collector and the A5 parity
|
||||
report are gone, and the cache write path is unconditional. Do not add OS cron
|
||||
entries: the application scheduler owns both jobs.
|
||||
|
||||
## What the deployment adds
|
||||
|
||||
- `Dolt Earnings Import (shadow)` runs daily at 02:30 America/New_York.
|
||||
- `SEC Fundamentals Import (shadow)` runs daily at 04:00 America/New_York.
|
||||
- `Fundamentals Parity Report (read-only)` runs daily at 05:30 America/New_York.
|
||||
- `Dolt Earnings Import` runs daily at 02:30 America/New_York.
|
||||
- `SEC Fundamentals Import` runs daily at 04:00 America/New_York, then refreshes
|
||||
`fundamental_data` — the compat cache scoring reads — from stored snapshots,
|
||||
earnings events and closes.
|
||||
- Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs.
|
||||
**Disabling the SEC job stops its SEC network fetch only**; the local cache
|
||||
refresh still runs, because prices and earnings move daily even when no filing
|
||||
does.
|
||||
- Cron expressions are editable in Admin → Schedule.
|
||||
- Every attempt is recorded in `data_import_runs`; failures also create a system
|
||||
event. A failed validation does not promote partial data.
|
||||
- An SEC filing still missing after the short publication-lag window enters
|
||||
`sec_filing_gaps`. The daily importer retries it automatically; affected
|
||||
tickers are excluded from actionable setups until a snapshot is recovered or
|
||||
a later valid 10-K/10-Q supersedes the gap. Migration `028` materializes older
|
||||
promoted gaps into this queue once, so setup reads never scan import history.
|
||||
- A gap that survives 14 days raises `filing_gap_aged` and, from that point,
|
||||
stops pausing setups **if** the issuer's own newest stored 10-K/10-Q is less
|
||||
than `GAP_GATE_RECENT_FILING_DAYS` (180) old. This is the hand-off from pause
|
||||
to alert, and it exists because the pause would otherwise be open-ended:
|
||||
SEC's per-company Company-Facts files can go stale indefinitely (2026-08: 43
|
||||
large caps whose Q2 10-Qs the `frames` API carried but whose
|
||||
`companyfacts/CIK*.json` never received), and the supersede rule needs a
|
||||
*successfully ingested* later filing, so a stale file swallows the next
|
||||
quarter too. Retrying is unaffected — the gap stays queued and a recovered
|
||||
filing still resolves it normally. An issuer with no filing that recent has no
|
||||
usable fundamentals at all and stays paused.
|
||||
|
||||
The systemd service uses one application worker. The import framework also holds
|
||||
a PostgreSQL advisory lock per source, so an overlapping manual/scheduled run is
|
||||
@@ -29,14 +51,16 @@ DOLT_EARNINGS_SUBDIR=earnings
|
||||
DOLT_MIN_FREE_DISK_GB=5.0
|
||||
SEC_USER_AGENT=signal-platform/1.0 (contact: real-address@example.com)
|
||||
SEC_REQUEST_SPACING_SECONDS=0.2
|
||||
FUNDAMENTALS_PARITY_REPORT_DIR=/var/lib/signal-platform/reports/fundamentals-parity
|
||||
```
|
||||
|
||||
Use a real monitored contact address. Keep at least 5 GB free at the Dolt data
|
||||
path; 8–10 GB gives comfortable growth headroom. The data directory must stay
|
||||
outside `/opt/signalplatform`, because deployments use `rsync --delete` there.
|
||||
The parity-report directory is also persistent and owned by the service user;
|
||||
its small timestamped JSON/CSV bundles form the temporary A5 review trail.
|
||||
|
||||
`FMP_API_KEY`, `FINNHUB_API_KEY` and `ALPHA_VANTAGE_API_KEY` must be **removed**
|
||||
from this file. Nothing reads them any more, and leaving them installed is the
|
||||
one thing that would let a rolled-back pre-A6 process resume the legacy
|
||||
collector and overwrite the SEC/Dolt cache.
|
||||
|
||||
## One-time provisioning
|
||||
|
||||
@@ -73,37 +97,18 @@ a reviewed change to `DOLT_VERSION`, followed by the same provision/check flow.
|
||||
|
||||
In Admin → Jobs, wait until no other job is running, then:
|
||||
|
||||
1. Trigger **Dolt Earnings Import (shadow)**. Expect `completed` with import
|
||||
1. Trigger **Dolt Earnings Import**. Expect `completed` with import
|
||||
status `promoted`; a repeat without an upstream change should report `no_op`.
|
||||
2. Trigger **SEC Fundamentals Import (shadow)**. The first run performs the
|
||||
2. Trigger **SEC Fundamentals Import**. The first run performs the
|
||||
tracked-universe history backfill and can take materially longer than a daily
|
||||
incremental run. Expect `completed` with import status `promoted`.
|
||||
3. Check Admin → System Events. There should be no new import error.
|
||||
4. Confirm the next-run times correspond to 02:30 and 04:00 New York time.
|
||||
5. Open several ticker pages and confirm the fundamentals panel has populated
|
||||
data and still handles partial/missing issuers cleanly.
|
||||
data and still handles partial/missing issuers cleanly. A ticker held by the
|
||||
quality gate should show **New setups paused** with the specific SEC reason.
|
||||
|
||||
## A5 parity observation window
|
||||
|
||||
After both shadow imports are healthy, trigger **Fundamentals Parity Report
|
||||
(read-only)** once in Admin → Jobs. The **A5 Fundamentals Parity** card above
|
||||
the jobs shows the latest coverage/delta summary and provides authenticated JSON
|
||||
and CSV downloads. The canonical server-side bundles are archived at:
|
||||
|
||||
```text
|
||||
/var/lib/signal-platform/reports/fundamentals-parity/
|
||||
```
|
||||
|
||||
The scheduler then generates one report daily at 05:30 New York time, after the
|
||||
02:30 Dolt and 04:00 SEC jobs. Review 5–7 consecutive reports before making the
|
||||
cutover decision. A report never writes `fundamental_data`, dimension/composite
|
||||
scores, rankings, qualification state, or an approval flag. Materiality bands
|
||||
only highlight rows for review; A5 still requires explicit approval.
|
||||
|
||||
Each bundle contains legacy and candidate P/E, revenue growth, and earnings
|
||||
surprise; definition notes; source revisions and price dates; recomputed legacy
|
||||
and candidate fundamental scores; and per-universe fundamental-rank changes.
|
||||
Definition changes remain explicit even when numeric deltas are small.
|
||||
## Verification
|
||||
|
||||
Optional database verification:
|
||||
|
||||
@@ -154,19 +159,68 @@ Expect `OK: source lock is busy`. This is the remaining live-PostgreSQL
|
||||
mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory
|
||||
locks. A second Admin trigger should independently report the job as busy.
|
||||
|
||||
## The fundamentals cache
|
||||
|
||||
`fundamental_data` is the compat cache scoring reads. The SEC Fundamentals
|
||||
Import rebuilds it every run from data already in PostgreSQL: newest valid
|
||||
snapshots x latest close for `pe_ratio` and `market_cap`, snapshots alone for
|
||||
`revenue_growth`, and `earnings_events` for `earnings_surprise` and
|
||||
`next_earnings_date`. It therefore also runs after an SEC network/validation
|
||||
failure, a `no_op`, a source-lock skip, or with the job disabled — no network
|
||||
access is involved. The job message appends the cache row count and the changed
|
||||
score-input count.
|
||||
|
||||
A refresh marks affected fundamental and composite score caches stale. The
|
||||
normal 15:30 near-close scanner recomputes them before using the rankings; until
|
||||
then, reads truthfully expose the stale state.
|
||||
|
||||
Verify the refreshed rows:
|
||||
|
||||
```sql
|
||||
SELECT count(*) AS rows,
|
||||
max(fetched_at) AS refreshed_at,
|
||||
count(pe_ratio) AS pe_available,
|
||||
count(revenue_growth) AS growth_available,
|
||||
count(earnings_surprise) AS surprise_available,
|
||||
count(next_earnings_date) AS next_date_available
|
||||
FROM fundamental_data;
|
||||
|
||||
SELECT dimension, is_stale, count(*)
|
||||
FROM dimension_scores
|
||||
WHERE dimension = 'fundamental'
|
||||
GROUP BY dimension, is_stale;
|
||||
```
|
||||
|
||||
## Failure and rollback
|
||||
|
||||
- Disable the failing shadow job in Admin → Jobs. This stops scheduled imports
|
||||
without changing existing data or the legacy scoring path.
|
||||
- **There is no provider fallback any more, and no Admin switch that freezes the
|
||||
cache.** Disabling **SEC Fundamentals Import** stops SEC network access only;
|
||||
the 04:00 job still rebuilds `fundamental_data` from the stored snapshots,
|
||||
earnings events and closes.
|
||||
- Restoring `fundamental_data` from the PostgreSQL backup is therefore a
|
||||
*temporary* fix on its own: if the bad values come from the snapshots or from
|
||||
the derivation code, the next scheduled run reproduces them. Fix the cause —
|
||||
restore or repair `fundamental_snapshots` / `earnings_events`, or revert the
|
||||
parser change and re-run `scripts/reparse_fundamentals.py --apply`.
|
||||
- To genuinely freeze the cache while you work, stop the service
|
||||
(`sudo systemctl stop signalplatform.service`) — that stops the scheduler with
|
||||
it. There is no finer-grained control, by design: a silently frozen scoring
|
||||
input is worse than an obvious outage.
|
||||
- Disable a failing source-import job in Admin → Jobs when SEC network access
|
||||
itself must stop. Existing promoted snapshots and events remain available, and
|
||||
the job's runtime message still reports the cache result.
|
||||
- Inspect the job runtime, latest `data_import_runs.validation_json`, service
|
||||
logs, and Admin → System Events before retrying.
|
||||
- `unresolved_filing` is emitted once when a filing enters automatic retry. It
|
||||
does not require a server command. If the gap is still current after 14 days,
|
||||
`filing_gap_aged` is emitted once with the CIK, accession, and parser/mapping
|
||||
reason. A later valid 10-K/10-Q retires the gap even when the original SEC
|
||||
accession never becomes usable.
|
||||
- Successful co-registrant recovery is logged without a warning. New registrants
|
||||
with no XBRL history are also logged quietly, but their ticker page explains
|
||||
that setups remain paused and that successor shells may need `sec_cik_overrides`.
|
||||
- Re-run `sudo -u deploy bash ./deploy/provision_fundamentals.sh --check` for
|
||||
binary, clone, permission, disk, or environment failures.
|
||||
- The Dolt clone is a reproducible cache and does not need a bespoke backup.
|
||||
PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import
|
||||
audit rows) must remain covered by the normal production database backup.
|
||||
- Do not proceed to A5 while either shadow feed is unhealthy or the parity gate
|
||||
has not received explicit approval.
|
||||
- If report generation fails, inspect Admin → System Events and verify
|
||||
`FUNDAMENTALS_PARITY_REPORT_DIR` exists and is writable by `deploy`. Existing
|
||||
reports and all live data remain untouched.
|
||||
|
||||
+30
-2
@@ -25,7 +25,7 @@ score, Structural S/R, the Gate Target Ladder, sentiment, fundamentals) is
|
||||
| 1.5× ATR initial stop | Real exit | Cuts losers fast |
|
||||
| 3× ATR trailing stop, 30-day max hold | Real exit | Best Sharpe of every exit tested |
|
||||
| Post-stop normal gate reset | Re-entry policy | Stop always closes; a later gate failure and subsequent fresh qualification define the next signal episode. The selected study arm reached Sharpe 1.77 / CAGR 48.3% at capacity 10; live scan-before-outcome timing is stricter (Sharpe 1.68 / CAGR 44.8% analogue). [Full study](post-stop-reentry.md) |
|
||||
| Max 10 concurrent positions, 1% risk per trade | Sizing | Cap never binds in practice |
|
||||
| Max **15** concurrent positions, 1% risk per trade | Sizing | Raised from 10 (2026-08-05) so the count cap never binds: +1.075pp CAGR paired, 51 paths better / 2 worse, drawdown unchanged. Cash plus the 20% notional cap saturates the book near 12. [Findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||
| Structural S/R | Human-facing product context | Clean, capped zones for charts and alerts; not read by the scanner |
|
||||
| Gate Target Ladder | Screening machinery | Volume-free transient proposals preserve the production candidate set exactly; never an exit |
|
||||
|
||||
@@ -61,7 +61,7 @@ invites overfitting.
|
||||
|---|---|
|
||||
| ATR trail multiple {1.5–4.0} | **Keep 3.0** — ≤2.0 whipsaws out the right tail; ≥2.5 is a plateau |
|
||||
| Momentum lookback (6-1, 3-1, 12-7 Novy-Marx, composites) | **Keep residual 12-1** — the others have IC ≈ 0 or weaker t-stats |
|
||||
| Selection cutoff {70…90} × book size {10, 15, 20} | **Keep 80 × 10** — monotonically worse in both directions |
|
||||
| Selection cutoff {70…90} × book size {10, 15, 20} | **Keep cutoff 80; book size now 15** — the focused daily bracket found cap 15 worth +1.075pp CAGR (the weekly replay's contrary reading was EV-per-trade). Weekly rank replacement hurt. [Findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||
| Position sizing (equal-weight, inverse-vol, risk-% sweep) | **Keep 1% fixed-fractional** |
|
||||
| Primary-target probability floor | **Keep 20%** — pruned lottery targets, 1,428 → 1,089 qualified, lifted Sharpe |
|
||||
| Primary-target R:R selector | **Keep 1.5** — target choice is intentionally independent of the later 2.0 activation floor |
|
||||
@@ -146,6 +146,7 @@ knobs.
|
||||
| **Broader universe** | Composition changes factor signs (fip tug-of-war); vol-tilt on breadth is only a **directional hypothesis** (auth. −0.048 / t −1.36) | Any prod broaden must re-validate 80/20 tilt; offline research only; research.sqlite requires completion manifest |
|
||||
| **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships |
|
||||
| **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR |
|
||||
| **Minimum effective-risk floor** | ⛔ CLOSED NEGATIVE, not run. The floor lifts EV/trade (+0.032) and PF (+0.073) *by deleting trades* — 11.4 fewer per path, never one more — and costs **−0.753pp CAGR**, −0.047 Sharpe, −0.051 Calmar | Do not run the A/B; its EV-based pass rule would have shipped it. [Withdrawn specification](effective-risk-floor-ab.md) / [findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||
|
||||
---
|
||||
|
||||
@@ -197,4 +198,31 @@ qualification. The [daily re-entry matrix](post-stop-reentry.md) supports this
|
||||
for the current 10-position book, but not as a universal rule for other
|
||||
portfolio capacities.
|
||||
|
||||
Capacity is closed **positive**: the count cap was raised 10 → 15 so it no longer
|
||||
binds, worth **+1.075pp CAGR** paired across 175 paths (51 better, 2 worse) at
|
||||
unchanged drawdown. Fifteen is headroom, not a target — cap15 peaked at 12 with
|
||||
zero full-book skips, so cash plus the 20% notional cap is the real ceiling.
|
||||
|
||||
An earlier reading of this run concluded "keep cap 10, added only 0.0018 R/trade."
|
||||
That was **EV per trade**, which is the wrong metric for a treatment that changes
|
||||
trade *count*: flat EV/trade means the blocked entries were as good as the taken
|
||||
ones, so refusing them cost their whole contribution to return. Weekly
|
||||
current-rank replacement remains rejected (−0.043 EV R, 24% churn). The 0.5%
|
||||
effective-risk-floor A/B is **closed negative** without being run — it costs
|
||||
0.75pp of CAGR while raising EV/trade, and its frozen pass rule would have shipped
|
||||
it. See the [frozen specification](portfolio-capacity-bracket.md) and the
|
||||
[capacity findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||
|
||||
The next real evidence is **forward**, not backward: the live paper-trade record.
|
||||
|
||||
## AI/Tech Risk Monitor
|
||||
|
||||
An observational risk thermometer (State + Warning) shown on the Risk page. It
|
||||
gates nothing — no entries, exits, sizing or ranking — so it is not a strategy
|
||||
document, but its calibration follows the same rules as one.
|
||||
|
||||
- [Methodology, v4](regime-monitor-v4.md) — sensors, weights, bands, and the
|
||||
reasoning behind each cut from v2 onward.
|
||||
- Reproduce any number in it with `scripts/run_regime_monitor_calibration.py`,
|
||||
which replays the series offline and refuses to report unless it first
|
||||
reproduces the published v2 and v3 figures.
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# Effective initial-risk floor A/B - frozen specification
|
||||
|
||||
> ## ⛔ CLOSED 2026-08-05 — NEGATIVE. DO NOT RUN.
|
||||
>
|
||||
> This A/B was never executed because the capacity-bracket run already contains
|
||||
> it. `cap15_incumbent` (peak 12, zero blocked, no floor) and `cash_unbounded`
|
||||
> (peak 12, floor) have the same effective capacity and differ essentially only
|
||||
> by `min_initial_risk_fraction`. Paired over 175 paths, the 0.5% floor gives
|
||||
> **EV/trade +0.032 and profit factor +0.073, but CAGR −0.753pp, total return
|
||||
> −0.765pp, Sharpe −0.047, Calmar −0.051**, and it removes 11.4 trades per path
|
||||
> while never adding one (174 worse / 0 better).
|
||||
>
|
||||
> **The pass rule below is unsafe.** It promotes on paired EV, and the floor
|
||||
> raises EV per trade *precisely by deleting trades* that were net positive
|
||||
> contributors — so this specification would have shipped a change costing
|
||||
> 0.75pp of CAGR. Any successor study must decide on CAGR/total return and treat
|
||||
> EV per trade as a diagnostic.
|
||||
>
|
||||
> See [portfolio-capacity-bracket-findings.md](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||
> Retained as a record of what was specified and why it was withdrawn.
|
||||
|
||||
Date frozen: 2026-08-05
|
||||
Branch: research/portfolio-capacity-rebalancing (deleted; tag `research/portfolio-capacity-final`)
|
||||
Runner: scripts/run_portfolio_construction_matrix.py (not on main; see tag)
|
||||
Study ID: risk-floor-ab
|
||||
|
||||
## Question
|
||||
|
||||
Does rejecting an otherwise qualified cap-10 entry when its actual initial
|
||||
stop-risk after cash and notional sizing is below 0.5% of marked equity improve
|
||||
trade selection?
|
||||
|
||||
The completed capacity bracket cannot answer this. Its cash_unbounded arm
|
||||
removed the count cap and applied the 0.5% floor simultaneously. In the 70 paths
|
||||
where the control cap never bound, that arm still raised mean EV from 0.328 to
|
||||
0.399 R and profit factor from 1.60 to 1.75 while trades fell about 8% and
|
||||
exposure stayed nearly flat. Capacity was a no-op in those paths, so the floor
|
||||
is the plausible cause, but the prior arm remains confounded.
|
||||
|
||||
This A/B changes only the floor. It has no formal promotion gate and does not
|
||||
automatically change production.
|
||||
|
||||
## Frozen arms
|
||||
|
||||
1. cap10_incumbent: current production-style cap-10 control, with no minimum
|
||||
effective-risk floor.
|
||||
2. cap10_min_risk_005: the same cap-10 strategy, rejecting an entry only when
|
||||
actual initial stop-risk after cash/notional sizing is below 0.5% of marked
|
||||
equity.
|
||||
|
||||
Both arms have max_positions=10, weekly replacement disabled, 1% target risk
|
||||
per trade, and identical admission ordering. The only differing simulator
|
||||
argument is min_initial_risk_fraction: None versus 0.005.
|
||||
|
||||
All other settings remain the frozen daily Phase A control: current production
|
||||
construction universe, full-universe residual-momentum/low-volatility 80/20
|
||||
rank, threshold 80, normal gate-reset re-entry, close fills, 3x ATR trail,
|
||||
30-session maximum hold, 20% per-position notional ceiling, no leverage, and
|
||||
costs of 0.10% and 0.20% per fill.
|
||||
|
||||
Every priced symbol contributes to the daily cross-sectional rank. Rank-only
|
||||
symbols cannot submit trades. Validation retains the 450-600-symbol production
|
||||
construction guardrail and the legacy-snapshot column-scoped loader.
|
||||
|
||||
## Frozen cohorts
|
||||
|
||||
Reuse the completed bracket's point-in-time daily candidate/rank cache and
|
||||
cohort manifest:
|
||||
|
||||
- Empty book: first eligible session of each month in 2019-2025, with 504 prior
|
||||
scoring sessions and 252 measurement sessions. This is the primary start-date
|
||||
evidence.
|
||||
- Warm book: weekly seeds 63-126 sessions before each 2019-2025 annual anchor,
|
||||
with state carried into the same 252-session measurement window. This is a
|
||||
state-carrying replication, not independent evidence.
|
||||
|
||||
The expected realization is 78 empty-book paths, 97 warm paths, seven annual
|
||||
clusters in each protocol, two costs, two arms, and 700 cells.
|
||||
|
||||
Do not use warm-seed IQR as evidence. Six of seven completed-bracket anchors
|
||||
were structurally degenerate because fractional sizing is scale invariant and
|
||||
the 30-session maximum hold washed out books before anchors. The 2023 exception
|
||||
shows that state carrying itself works.
|
||||
|
||||
## Reporting and interpretation
|
||||
|
||||
For every protocol and cost, pair identical paths. Report:
|
||||
|
||||
- mean, median, P25, and P75 paired net-EV changes in R;
|
||||
- positive-path and bit-identical-path fractions;
|
||||
- the median paired delta within each year and the median across seven years;
|
||||
- simple 90% cluster-bootstrap context for EV and Calmar, with no CI gate;
|
||||
- mean paired PF, Gain-to-Pain, Sortino, Calmar/MAR, CAGR, maximum drawdown,
|
||||
total return, and Sharpe changes;
|
||||
- trades, floor rejections, holding time, cash, gross exposure, average/peak
|
||||
positions, turnover, and costs.
|
||||
|
||||
Means and identical-path fractions must appear beside medians so inert cohorts
|
||||
cannot turn a left- or right-skewed treatment into a misleading zero headline.
|
||||
For these 252-session windows, the implementation's full-window Calmar is CAGR
|
||||
divided by maximum drawdown, the same numeric definition commonly called MAR;
|
||||
do not present the duplicate label as a second independent metric.
|
||||
|
||||
Today's production membership is projected backward. Use paired differences
|
||||
for the treatment conclusion; absolute profitability remains descriptive and
|
||||
survivorship-biased. Empty and warm protocols cover the same seven market years
|
||||
and must not be interpreted as independent replications.
|
||||
|
||||
Interpretation is deliberately simple:
|
||||
|
||||
- a positive result means the isolated floor improves the paired EV
|
||||
distribution without an economically important loss of total-return or
|
||||
drawdown quality;
|
||||
- a negative result closes the floor;
|
||||
- mixed EV/portfolio-quality results are reported as a trade-off, not forced
|
||||
through a composite score.
|
||||
|
||||
## Reproducibility and macOS execution
|
||||
|
||||
The authoritative run refuses a dirty worktree. Its fingerprint includes the
|
||||
implementation commit, this specification hash, snapshot hash, candidate-cache
|
||||
key, construction view, cohort manifest, arm definitions, costs, and study
|
||||
version. Cells checkpoint atomically and --resume verifies the fingerprint.
|
||||
|
||||
From the repository root on macOS:
|
||||
|
||||
python3 -m venv .venv
|
||||
./.venv/bin/python -m pip install -e '.[dev]'
|
||||
|
||||
Preflight, reusing the completed bracket's candidate/rank cache:
|
||||
|
||||
./.venv/bin/python scripts/run_portfolio_construction_matrix.py + backtest_snapshots/research.sqlite + --study risk-floor-ab + --run-id prod505-effective-risk-floor-ab-daily-v1 + --candidate-cache reports/.cache/prod505-capacity-bracket-daily-v1-candidates.pkl + --workers 8 + --resume + --validate-only
|
||||
|
||||
Authoritative run:
|
||||
|
||||
./.venv/bin/python scripts/run_portfolio_construction_matrix.py + backtest_snapshots/research.sqlite + --study risk-floor-ab + --run-id prod505-effective-risk-floor-ab-daily-v1 + --candidate-cache reports/.cache/prod505-capacity-bracket-daily-v1-candidates.pkl + --workers 8 + --resume
|
||||
|
||||
On an M2 Pro, eight workers is the explicit high-utilization setting. Use six
|
||||
instead on a memory-constrained machine; auto intentionally caps itself at six.
|
||||
Changing worker count does not change the fingerprint or results.
|
||||
|
||||
Commit only the compact final JSON and Markdown reports. Candidate caches,
|
||||
checkpoints, raw curves, and trade ledgers remain ignored.
|
||||
@@ -28,6 +28,18 @@ Mechanics guards confirmed before reading results: calendar truncation asserted
|
||||
| **Validation** | **1.68** | **0.72** | **41.6%** | **20.9%** | **1.99** | **239** |
|
||||
| Full (close-fill) | 1.77 | 0.50 | 48.3% | 21.6% | 2.23 | 472 |
|
||||
|
||||
**Capacity correction (2026-08-05):** the full close-fill control also records
|
||||
skipped_book_full = 519 versus 472 admitted trades, so the ten-slot book
|
||||
refuses 52.4% of admitted+blocked qualified opportunities. The older weekly
|
||||
claim that the cap never bound is stale and does not apply to this daily
|
||||
gate-reset configuration. Capacity was isolated in the
|
||||
[focused bracket study](portfolio-capacity-bracket.md) and **resolved: the count
|
||||
cap was raised 10 → 15 so it no longer binds (+1.075pp CAGR paired, 51 paths
|
||||
better / 2 worse, drawdown unchanged).** Note that the blocked *count* was a poor
|
||||
guide in both directions — one path had 244 blocked entries and relieving all of
|
||||
them moved CAGR by −0.1pp. See the
|
||||
[findings correction](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||
|
||||
Validation SE ≈ 0.72 — almost no arm clears a 1-SE delta.
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
# Portfolio-capacity bracket — findings
|
||||
|
||||
Date interpreted: 2026-08-05
|
||||
|
||||
Status: **SUPERSEDED IN PART — see [Correction](#correction-2026-08-05-ev-per-trade-was-the-wrong-lens)
|
||||
at the foot of this document before acting on anything here.** Weekly replacement
|
||||
is closed as a negative result and that still holds. The capacity decision below
|
||||
("keep cap 10") and the recommendation to run the effective-risk-floor A/B were
|
||||
both reached on EV per trade and are **reversed** by the correction: the count cap
|
||||
was raised so it no longer binds, and the floor A/B is closed as negative.
|
||||
|
||||
> The runner (`scripts/run_portfolio_construction_matrix.py`), the research
|
||||
> simulator hooks, and the study's unit tests were deliberately not merged to
|
||||
> main. They live at tag `research/portfolio-capacity-final`.
|
||||
|
||||
This document interprets the frozen v2 run without modifying its generated
|
||||
outputs:
|
||||
|
||||
- result commit: `24482c6`;
|
||||
- simulation source commit: `6fc82ae8574de9104c83273e018391e75a5f8ac6`;
|
||||
- frozen specification SHA-256:
|
||||
`f1e37783cf6d157ecc827d48211fa45da16f0a0ac19cd23686b3902d347a1898`;
|
||||
- JSON SHA-256:
|
||||
`2435875667097db7416a0d96f412db81d2f2d09ba053748c9f2cfb8a0cba4417`;
|
||||
- Markdown SHA-256:
|
||||
`dc3f5de25eb0a156ce51d0025c90e04ac0977e9502dec47bcf1b25bdcf609c81`.
|
||||
|
||||
The run completed 78 empty-book paths, 97 warm-seed paths, seven annual
|
||||
clusters under both protocols, two cost levels, four arms, and 1,400 cells with
|
||||
no validation errors. The construction universe was 505 priced tradable
|
||||
symbols plus 4,149 priced rank-only symbols.
|
||||
|
||||
## Capacity is economically free
|
||||
|
||||
The clean capacity treatment is `cap15_incumbent`: it changes no sizing or
|
||||
admission rule. Its cap never bound in any cell (maximum observed position count
|
||||
12; zero full-book skips), so it absorbed every opportunity blocked by cap 10.
|
||||
|
||||
At 0.10% per fill, split the 175 paths by whether the paired control recorded
|
||||
any `skipped_book_full`. Values below are mean paired changes in net EV per
|
||||
trade, in R:
|
||||
|
||||
| Arm | Cap never bound (n=70) | Cap did bind (n=105) |
|
||||
|---|---:|---:|
|
||||
| `cap15_incumbent` | +0.0000 | +0.0018 |
|
||||
| `cash_unbounded` | +0.0714 | +0.0077 |
|
||||
| `cap10_weekly_top10` | -0.0246 | -0.0426 |
|
||||
|
||||
The exact zero for cap15 in the never-bound stratum is also a harness validity
|
||||
check: when the treatment cannot act, results are identical. Where it does act,
|
||||
giving the strategy every slot it requested adds only 0.0018 R/trade. The old
|
||||
519-blocked-versus-472-admitted count was true, but it did not imply that the
|
||||
blocked opportunities were economically valuable.
|
||||
|
||||
Decision: **keep the production cap at 10.** Do not remove it or raise it in the
|
||||
expectation of additional edge.
|
||||
|
||||
## The positive arm measured the risk floor
|
||||
|
||||
`cash_unbounded` combined two treatments: no count cap and a 0.5% minimum
|
||||
effective initial-risk fraction. Its EV effect is roughly nine times larger in
|
||||
the 70 paths where the control cap never bound, so capacity cannot explain the
|
||||
improvement.
|
||||
|
||||
Within that never-bound stratum:
|
||||
|
||||
| Measure | Control | `cash_unbounded` |
|
||||
|---|---:|---:|
|
||||
| Mean trades | 75.7 | 69.9 |
|
||||
| Mean cash | 27.8% | 28.2% |
|
||||
| Mean gross exposure | 72.2% | 71.8% |
|
||||
| Mean hold | 15.4 sessions | 15.6 sessions |
|
||||
| Mean EV | +0.328 R | +0.399 R |
|
||||
| Mean profit factor | 1.60 | 1.75 |
|
||||
|
||||
The floor removes about 8% of fills while leaving exposure and holding time
|
||||
nearly unchanged. This is selection, not general de-risking: candidates that
|
||||
available sizing compresses below half the intended risk are worse on average.
|
||||
The report records repeated reject attempts, not the rejected candidates'
|
||||
ranks, so whether the effect is rank-mediated remains unknown.
|
||||
|
||||
Next research: one single-variable A/B, `cap10_incumbent` versus cap 10 with
|
||||
`min_initial_risk_fraction=0.005`, with every other rule unchanged. Do not call
|
||||
the current `cash_unbounded` result causal evidence for that floor until this
|
||||
confound-free comparison is run.
|
||||
|
||||
## Weekly replacement hurts
|
||||
|
||||
Median paired deltas read zero because enough cohorts are inert. The distribution
|
||||
is not neutral:
|
||||
|
||||
| Protocol | Mean ΔEV | P25 ΔEV | Identical paths |
|
||||
|---|---:|---:|---:|
|
||||
| Empty book | -0.0360 R | -0.0817 R | 27/78 (34.6%) |
|
||||
| Warm book | -0.0348 R | -0.1582 R | 14/97 (14.4%) |
|
||||
|
||||
The arm made 2,170 replacements and 529 same-symbol re-entries within ten
|
||||
sessions, so 24% of replacements were associated with short-horizon churn.
|
||||
|
||||
Decision: **reject weekly top-10 replacement.** Future reports should show mean
|
||||
paired effects and identical-path fractions beside medians whenever treatments
|
||||
are inert in a material share of cohorts.
|
||||
|
||||
## Warm dispersion was mostly structurally degenerate
|
||||
|
||||
For six of seven anchors, control EV IQR is numerical zero (approximately
|
||||
`1e-16`) and Calmar IQR is exactly zero. The displayed ratio `1.000` is therefore
|
||||
mostly the implementation's zero-over-zero convention, not evidence of equal
|
||||
nonzero dispersion.
|
||||
|
||||
Two mechanics cause convergence: sizing and notional limits are fractions of
|
||||
equity, making R and ratio metrics scale-invariant; and the 30-session maximum
|
||||
hold is shorter than the 63-session minimum seed offset, allowing initial books
|
||||
to wash out before the anchor.
|
||||
|
||||
The exception is 2023. Control measurement-start positions vary from 6 to 9,
|
||||
EV IQR is 0.0274 R, and Calmar IQR is 0.2675. The protocol therefore carries
|
||||
state correctly, but its chosen offsets usually erase the initialization effect
|
||||
it was intended to measure.
|
||||
|
||||
Future initialization studies should use seed offsets shorter than maximum hold,
|
||||
approximately 5–25 sessions. The current empty-book cohorts remain the primary
|
||||
start-date evidence, but they necessarily mix initialization with market regime.
|
||||
|
||||
## Final decisions
|
||||
|
||||
1. ~~Keep cap 10; its measured opportunity cost is negligible.~~ **REVERSED —
|
||||
see the correction below.**
|
||||
2. Reject weekly rank replacement. *(Stands.)*
|
||||
3. Do not interpret the `cash_unbounded` improvement as a capacity effect.
|
||||
*(Stands — and it is not a floor effect worth having either; see below.)*
|
||||
4. ~~Run only the focused cap-10 effective-risk-floor A/B next.~~ **REVERSED —
|
||||
that A/B is answered and negative; do not run it.**
|
||||
5. Report means, inert fractions, and absolute dispersion beside medians and
|
||||
ratios in future sparse-treatment studies. *(Stands, and see below — the
|
||||
metric itself matters as much as the summary statistic.)*
|
||||
|
||||
## Correction 2026-08-05: EV per trade was the wrong lens
|
||||
|
||||
Everything above judged the arms on **mean paired net EV per trade**. That is the
|
||||
wrong metric for any treatment that changes how many trades the book takes.
|
||||
Capacity does not change trade *quality*; it changes trade *count*. A flat EV/trade
|
||||
delta therefore does not mean "no benefit" — it means the blocked entries were
|
||||
**just as good** as the taken ones, so refusing them cost their entire
|
||||
contribution to return. Re-running the same paired comparison on CAGR inverts two
|
||||
conclusions.
|
||||
|
||||
### Capacity: raise the cap (reverses decision 1)
|
||||
|
||||
`cap15_incumbent` versus `cap10_incumbent`, paired, all 175 paths, 0.10% per fill:
|
||||
|
||||
| Metric | Mean Δ | Worse / better |
|
||||
|---|---:|---:|
|
||||
| Trades | +1.00 | **0 / 76** (never fewer) |
|
||||
| **CAGR pp** | **+1.075** | 2 / 51 |
|
||||
| Total return pp | +1.079 | 1 / 51 |
|
||||
| Max drawdown pp | +0.007 | 1 / 2 |
|
||||
| Calmar | +0.062 | **1 / 51** |
|
||||
| Sharpe | +0.022 | 10 / 28 |
|
||||
| Net EV R/trade | +0.001 | 47 / 29 |
|
||||
|
||||
Restricted to the 105 paths where the cap actually bound: **+1.791pp CAGR**.
|
||||
|
||||
The honest tail: exactly one path was materially hurt — `empty-2023-04`, CAGR
|
||||
87.2 → 81.2 (−6.0pp), drawdown 13.0 → 14.3, from two extra trades. Second-worst
|
||||
was −0.1pp. The best paths (+6.6/+6.7/+6.9pp) came with *identical* drawdown. Best
|
||||
and worst magnitudes are symmetric at roughly ±6pp, but the frequency is 51:1.
|
||||
|
||||
Blocked count is not lost value in either direction: `empty-2021-05` had **244**
|
||||
blocked entries under cap 10, and relieving every one of them moved CAGR by
|
||||
−0.1pp.
|
||||
|
||||
**Shipped:** `SIM_MAX_POSITIONS` and `shadow_book_service.DEFAULT_CAPACITY` raised
|
||||
10 → 15. Fifteen is headroom, not a target — cap15 peaked at 12 with zero
|
||||
full-book skips, so cash plus the 20% notional cap is the real ceiling and
|
||||
15/20/None are the same experiment.
|
||||
|
||||
### Effective-risk floor: closed negative (reverses decision 4)
|
||||
|
||||
The floor A/B does not need running — this study already contains it.
|
||||
`cap15_incumbent` (peak 12, zero blocked, no floor) and `cash_unbounded` (peak 12,
|
||||
floor) have the same effective capacity and differ essentially only by
|
||||
`min_initial_risk_fraction`. Paired, n=175, 0.10% per fill, floor minus no-floor:
|
||||
|
||||
| Metric | Mean Δ | Worse / better |
|
||||
|---|---:|---:|
|
||||
| Net EV R/trade | **+0.032** | 53 / 121 |
|
||||
| Profit factor | **+0.073** | 46 / 128 |
|
||||
| Trades | **−11.4** | **174 / 0** (never adds one) |
|
||||
| **CAGR pp** | **−0.753** | 105 / 68 |
|
||||
| Total return pp | −0.765 | 105 / 68 |
|
||||
| Sharpe | −0.047 | 108 / 65 |
|
||||
| Calmar | −0.051 | 103 / 71 |
|
||||
| Max drawdown pp | +0.333 (worse) | — |
|
||||
|
||||
The same trap, mirrored: the floor raises per-trade quality *precisely by deleting
|
||||
trades*, and the deleted trades were net positive contributors. The frozen
|
||||
specification in [effective-risk-floor-ab.md](effective-risk-floor-ab.md) would
|
||||
have passed it on paired EV and shipped a change costing 0.75pp of CAGR.
|
||||
|
||||
Genuinely open, low priority: 0.005 clearly over-cuts, but the sizing code's real
|
||||
floor is a **$1** minimum, which is no floor at all. Whether something near 0.001
|
||||
strips true dust without cutting real trades is untested, and only worth revisiting
|
||||
if live broker order minimums force it.
|
||||
|
||||
### Start-date sensitivity is real but not a capacity artifact
|
||||
|
||||
Within-year spread of EV across monthly start dates is ~0.672 R and is
|
||||
*identical* for `cap10` (0.672), `cap15` (0.672) and `cash_unbounded` (0.677). It
|
||||
is small-sample noise — roughly 84 trades per 252-session window drawn from a
|
||||
fat-tailed R distribution gives an EV standard error near 0.15–0.25 R — not a
|
||||
queueing artifact. No construction policy reduces it.
|
||||
|
||||
### Rule for future studies
|
||||
|
||||
Choose the metric from the treatment's mechanism before reading any table. If a
|
||||
treatment changes trade count, CAGR and total return are the decision metrics and
|
||||
EV per trade is a diagnostic. The generated report's headline tables lead with
|
||||
ΔEV net R, which is what made this error easy to make twice.
|
||||
@@ -0,0 +1,169 @@
|
||||
# Portfolio-capacity bracket — frozen specification
|
||||
|
||||
Date frozen: 2026-08-05
|
||||
Branch: research/portfolio-capacity-rebalancing
|
||||
Runner: scripts/run_portfolio_construction_matrix.py
|
||||
|
||||
## Question and motivation
|
||||
|
||||
The daily Phase A production control (a0_control: close fill, 30-session
|
||||
maximum hold, 1% fixed-fractional risk, no correlation or volatility overlay)
|
||||
recorded 472 trades and 519 otherwise qualified entries rejected because the
|
||||
ten-position book was full. The blocked share is 519 / (519 + 472) = 52.4%.
|
||||
The book is therefore materially arrival-order constrained.
|
||||
|
||||
This supersedes the older statement that the ten-slot cap never bound. That
|
||||
statement came from a shorter, weekly, pre-gate-reset replay and is not evidence
|
||||
about the current daily strategy.
|
||||
|
||||
The study brackets the value of capacity before tuning replacement details. It
|
||||
does not contain a formal promotion rule or automatically change production.
|
||||
Because the current ~505-name production membership is projected backward,
|
||||
paired arm-versus-control differences are the primary evidence. Absolute
|
||||
profitability is descriptive and survivorship-biased.
|
||||
|
||||
Implementation correction: the first completed v1 artifact at commit `23fe39f`
|
||||
incorrectly allowed the snapshot's broad rank-only universe to submit trades.
|
||||
That artifact is invalid, is removed from the branch, and must not be used for
|
||||
strategy conclusions. Runner v2 fixes the construction/ranking partition below.
|
||||
|
||||
## Frozen arms
|
||||
|
||||
1. **cap10_incumbent:** exact production-style cap-10 control, no displacement.
|
||||
2. **cash_unbounded:** no position-count cap; cash/no leverage and the existing
|
||||
20% per-position notional ceiling remain. Reject an entry if actual initial
|
||||
stop-risk after cash/notional sizing is below 0.5% of marked equity.
|
||||
3. **cap10_weekly_top10:** on the final trading session of each ISO week, rank
|
||||
holdings plus fresh same-day qualified entrants and retain the top ten.
|
||||
4. **cap15_incumbent:** cap 15, no displacement.
|
||||
|
||||
All arms use the frozen Phase A control configuration: daily candidate replay,
|
||||
live-like full-universe residual-momentum/low-volatility 80/20 rank, activation
|
||||
threshold 80, normal gate-reset re-entry, close fill, 3×ATR trail, 30-session
|
||||
maximum hold, 1% risk, and costs of 0.10% and 0.20% per fill.
|
||||
|
||||
Every priced symbol contributes to the daily cross-sectional rank. Only symbols
|
||||
not listed in the snapshot's `research_rank_only` side table may submit trade
|
||||
setups to any arm. The resulting construction universe must contain 450-600
|
||||
symbols (expected approximately 505); validation fails outside that frozen
|
||||
guardrail or when the side table references unknown ticker symbols.
|
||||
|
||||
The daily replay uses zero outcome horizon: setup and rank observations continue
|
||||
through the snapshot's last session because portfolio simulation, unlike outcome
|
||||
grading, does not require 30 future bars.
|
||||
|
||||
Control-parity note: a direct main-versus-branch comparison found identical
|
||||
total return, CAGR, maximum drawdown, and Sharpe. The branch intentionally
|
||||
changes only the first calendar year's `yearly_returns` convention: it starts
|
||||
from initial capital rather than equity after the first session, so day-one
|
||||
entry costs are now charged to year one. Older reports can therefore show a
|
||||
different first-year contextual return without a strategy-performance
|
||||
regression. New trade-detail and measurement-start fields are additive.
|
||||
|
||||
### Weekly-selection mechanics
|
||||
|
||||
- Ordinary exits run before entries/rebalancing.
|
||||
- Open slots may still fill from daily qualified entries during the week.
|
||||
- On the final ISO-week session, current holdings and that day's fresh qualified
|
||||
entrants use the full-universe strategy_rank for that same date.
|
||||
- Stored entry-day rank is never used.
|
||||
- Holdings with missing current rank/data are protected and consume a slot;
|
||||
entrants missing rank are ineligible.
|
||||
- Incumbents win exact rank ties; symbol is the deterministic final tie-breaker.
|
||||
- Rebalance exits pay costs and bypass cooldown/post-stop state.
|
||||
- Report entrant-pool sizes, replacements, turnover, and same-symbol re-entry
|
||||
within 5/10/20 sessions.
|
||||
|
||||
## Frozen cohorts
|
||||
|
||||
research.sqlite is expected to cover 2016-01-04 through 2026-07-17. Residual
|
||||
momentum requires 252 benchmark sessions. Empty-book starts additionally require
|
||||
504 prior scoring sessions and 252 forward measurement sessions.
|
||||
|
||||
- **Empty book:** first eligible session of each month, approximately January
|
||||
2019 through July 2025; start with no positions and measure 252 sessions.
|
||||
- **Warm book:** first session of each year 2019–2025 is the measurement anchor.
|
||||
Seed the portfolio on the first session of every ISO week falling 63–126
|
||||
trading sessions before the anchor, carry all positions and gate-reset state
|
||||
forward, and measure the same 252-session anchor window.
|
||||
|
||||
Warm portfolio returns reset to marked equity immediately before the anchor
|
||||
session. P&L after the anchor from carried positions belongs to portfolio
|
||||
returns, while trade EV includes only entries on or after the anchor. Remaining
|
||||
positions liquidate at the last measurement close with costs.
|
||||
|
||||
The validate-only mode must print realized cohort counts and fail unless both
|
||||
protocols contain the seven annual clusters 2019–2025 and every warm anchor has
|
||||
at least 12 seeds. It must also print ranking, rank-only, and tradable symbol
|
||||
counts plus the raw, removed, and retained qualified-long counts.
|
||||
|
||||
## Reporting
|
||||
|
||||
Primary reported measures:
|
||||
|
||||
- net EV per trade in R, with costs and actual initial stop-risk dollars;
|
||||
- Calmar (CAGR / max drawdown);
|
||||
- profit factor on net trade R;
|
||||
- Gain-to-Pain (sum of all monthly returns / absolute sum of negative months);
|
||||
- Sortino using daily returns and zero target.
|
||||
|
||||
Also report total return/CAGR, maximum drawdown, Sharpe, win rate, time
|
||||
underwater, exposure, cash, average/peak positions, sessions at capacity,
|
||||
turnover, costs, qualified/admitted/blocked opportunities, and minimum-risk
|
||||
rejections.
|
||||
|
||||
For each arm/protocol/cost/metric, pair identical paths with cap10_incumbent,
|
||||
take the median paired delta within each start year or annual anchor, show all
|
||||
seven cluster values, and headline their median.
|
||||
|
||||
Initialization dispersion is reported separately for EV and Calmar: calculate
|
||||
the seed-path IQR within each warm anchor, divide by the paired control IQR, show
|
||||
all seven ratios, and headline their median. Do not combine them into a composite.
|
||||
|
||||
For context only, run a deterministic 10,000-replicate cluster bootstrap over
|
||||
the seven paired annual summaries and report the central 90% percentile interval
|
||||
for median EV and Calmar deltas and warm IQR ratios. These intervals are not
|
||||
promotion gates, independent-population confidence claims, or formal inference.
|
||||
|
||||
## Reproducibility and execution
|
||||
|
||||
Candidate replay/ranks cache under reports/.cache; each matrix cell checkpoints
|
||||
atomically and resume verifies a fingerprint over the implementation commit,
|
||||
this specification hash, snapshot SHA-256, cache key, arm definitions, costs,
|
||||
and cohort manifest. An authoritative run refuses a dirty worktree.
|
||||
|
||||
The existing v1 candidate/rank cache is intentionally reusable: its
|
||||
full-universe current-day ranks are correct. Runner v2 derives a fingerprinted
|
||||
construction view by removing qualified rows whose symbols are rank-only. V2
|
||||
uses a versioned checkpoint directory, so invalid v1 portfolio cells are never
|
||||
resumed and the expensive daily rank replay does not need to run again.
|
||||
|
||||
The loader reads only ticker ID/symbol and the OHLCV columns used by replay, so
|
||||
snapshots created before SEC metadata added `tickers.cik`, `tickers.sic`, and
|
||||
`tickers.sic_description` remain valid. Do not migrate or alter the research
|
||||
snapshot: its original SHA-256 is part of the run fingerprint.
|
||||
|
||||
macOS environment setup from the repository root (zsh):
|
||||
|
||||
python3 -m venv .venv
|
||||
./.venv/bin/python -m pip install -e '.[dev]'
|
||||
|
||||
Preflight:
|
||||
|
||||
./.venv/bin/python scripts/run_portfolio_construction_matrix.py \
|
||||
backtest_snapshots/research.sqlite \
|
||||
--run-id prod505-capacity-bracket-daily-v1 \
|
||||
--workers auto \
|
||||
--resume \
|
||||
--validate-only
|
||||
|
||||
Authoritative run:
|
||||
|
||||
./.venv/bin/python scripts/run_portfolio_construction_matrix.py \
|
||||
backtest_snapshots/research.sqlite \
|
||||
--run-id prod505-capacity-bracket-daily-v1 \
|
||||
--workers auto \
|
||||
--resume
|
||||
|
||||
Commit only the compact final JSON and Markdown reports. Raw curves, trades,
|
||||
candidate caches, and checkpoints remain ignored.
|
||||
@@ -1,75 +0,0 @@
|
||||
# Regime Monitor v2 methodology
|
||||
|
||||
The Regime Monitor is an observational AI/Tech risk thermometer. It does not
|
||||
gate entries, exits, position size, ranking, or alerts about individual setups.
|
||||
|
||||
## Outputs
|
||||
|
||||
**State** measures current structural stress:
|
||||
|
||||
- Price structure, 40%: `max(P1, P2, P3)`, so the correlated 200-DMA, death-cross,
|
||||
and drawdown readings receive one capped vote.
|
||||
- Fixed-basket breadth level, 25%.
|
||||
- HY option-adjusted credit spread, 20%.
|
||||
- VIX level, 15%.
|
||||
|
||||
**Warning** measures deterioration and divergence:
|
||||
|
||||
- Fixed-basket breadth divergence while SMH holds/rises, 50%.
|
||||
- 60-session SMH/SPY relative-strength deterioration, 30%.
|
||||
- Hyperscaler capex cuts, 12%.
|
||||
- Good-news-stock-down earnings reactions, 8%.
|
||||
|
||||
Combined, RSP/SPY (former F4), and the NVDA canary (former P6) do not enter v2.
|
||||
|
||||
## Scale and missing data
|
||||
|
||||
Zero means ordinary/healthy, and only stress contributes positively. Automated
|
||||
capex `raising`/`holding` and no good-news-stock-down pattern map to zero;
|
||||
`mixed`, unknown, and stale observations are unavailable rather than neutral 50.
|
||||
Manual observations use the same categories: each hyperscaler is marked
|
||||
`raising`, `holding`, `cutting`, or `unknown`, while the earnings reaction is
|
||||
`yes`, `no`, or `mixed`. F1 is derived from the share of at least three known
|
||||
hyperscalers marked `cutting`; arbitrary numeric overrides are not accepted.
|
||||
|
||||
Scores renormalize over available fixed weights, but a band is published only at
|
||||
75% or greater coverage. Trend deltas are suppressed when the participating
|
||||
pillar set changes. Bands are stable `<30`, watch `<60`, elevated `<80`, and
|
||||
breaking `>=80`.
|
||||
|
||||
Credit uses named HY OAS anchors (3.5 mild, 5.0 elevated, 7.0 stressed) for 70%
|
||||
of its score and a ten-year upper-tail percentile for 30%.
|
||||
|
||||
## Point-in-time record
|
||||
|
||||
The first v2 run rebuilds the latest 400 trading sessions with sufficient sensor
|
||||
warm-up. Routine runs thereafter insert/update only the latest trading date.
|
||||
Fundamental observations have an effective date (normally the next session after
|
||||
collection) and are never replayed backward. The history API and main chart show
|
||||
only snapshots marked `methodology: v2`.
|
||||
|
||||
Each snapshot stores the fixed basket symbols, hash, and freeze date. Reconstructed
|
||||
history before that freeze date is retrospective/exploratory; readings after it
|
||||
form the forward record.
|
||||
|
||||
The automatic 400-session rebuild is intentionally one-shot: it runs only when
|
||||
no v2 snapshot exists. If an initial seed used partial data or the wrong basket,
|
||||
the operational reseed procedure is to remove the v2 snapshot rows and run the
|
||||
Regime Monitor job again. There is no routine force-rebuild flag.
|
||||
|
||||
## Warning study
|
||||
|
||||
The study calls the outcome a **10% correction**, not a regime break. The first
|
||||
70% of sessions freezes the 80th-percentile warning threshold; alarm episodes are
|
||||
measured on the final 30%. An alarm requires an upward crossing and another alarm
|
||||
requires a reset below the threshold. The report exposes warned/missed events,
|
||||
false alarms per year, median lead, sample dates, event count, report date, and
|
||||
whether the result is exploratory or a true forward holdout. UI claims are
|
||||
generated from that report; no performance sentence is hard-coded.
|
||||
|
||||
## Operator rule
|
||||
|
||||
Quadrant alerts default off for new/reset configurations. When enabled they
|
||||
require fresh inputs, at least 75% coverage on both axes, two consecutive daily
|
||||
confirmations, hysteresis, and cooldown. Every alert states: **Risk thermometer —
|
||||
not a trade signal.**
|
||||
@@ -0,0 +1,6 @@
|
||||
# Moved
|
||||
|
||||
The methodology doc now lives at [regime-monitor-v4.md](regime-monitor-v4.md).
|
||||
|
||||
v3's text is in git history (`git log --follow docs/research/regime-monitor-v4.md`).
|
||||
This stub exists because commit messages up to 2026-08-08 cite the old path.
|
||||
@@ -0,0 +1,837 @@
|
||||
# AI/Tech Risk Monitor v4 methodology
|
||||
|
||||
Named "Regime Monitor" until 2026-08-07; the filename's `regime` stem, the
|
||||
`regime_monitor` job id, the `/regime` route and the `METHODOLOGY`/snapshot
|
||||
fields keep the old word, because those are persisted or externally linked.
|
||||
|
||||
The AI/Tech Risk Monitor is an observational risk thermometer. It does not
|
||||
gate entries, exits, position size, ranking, or alerts about individual setups.
|
||||
|
||||
**v4 supersedes v3** (2026-08-08). Unlike v3, whose calibration was ad-hoc and
|
||||
never landed, every number below is reproducible:
|
||||
|
||||
```
|
||||
.venv/Scripts/python.exe scripts/run_regime_monitor_calibration.py --methodology v2_reconstruction,v2_reconstruction_oas400,v3,v4,v4-vix-only,v4-p1-only --cache-dir .calib-cache
|
||||
```
|
||||
|
||||
`v3` and `v4` are mandatory — the row-wise `state_v4 <= state_v3` invariant is
|
||||
a hard gate and needs both — and the replayed **start** date is asserted
|
||||
against the published window. The session *count* alone proves nothing, since
|
||||
the harness slices the tail of the price series to whatever was asked for.
|
||||
|
||||
The harness replays the 408 sessions ending 2026-07-24 from the live inputs
|
||||
(Alpaca for all 33 symbols, FRED for VIX and HY OAS) with no database, and
|
||||
reproduces the published v2 and v3 figures before it will emit anything:
|
||||
|
||||
| figure | published | replayed |
|
||||
|---|---|---|
|
||||
| v2 State avg | 22.6 | 22.68 |
|
||||
| v2 State p80 | 35.1 | **35.1** |
|
||||
| v2 State max | 91.2 | **91.2** |
|
||||
| v2 P3 pegged | 39 | **39** |
|
||||
| v2 W1 live | 108 | **108** |
|
||||
| v3 State max | 87.4 | **87.4** |
|
||||
| v3 band shares | 73.3 / 15.0 / 8.3 / 3.4 | 73.0 / 15.4 / 8.1 / 3.4 |
|
||||
|
||||
It refuses to emit a band recommendation, and exits non-zero, unless every hard
|
||||
gate passes — 33 symbols fetched with full warm-up, the whole basket on every
|
||||
session, the calendar anchors, 100% coverage on every row, and a row-wise
|
||||
`state_v4 <= state_v3` invariant. Reading a calibration result out of a run whose
|
||||
pipeline did not validate is meant to be structurally impossible.
|
||||
|
||||
## The fundamental channel (2026-08-12)
|
||||
|
||||
The monitor has **three channels**, not two scores with a decoration:
|
||||
|
||||
- **State** — current observable technical stress (price, breadth, credit, volatility).
|
||||
- **Warning** — observable deterioration that may precede stress (breadth
|
||||
divergence, relative strength, credit impulse).
|
||||
- **Fundamental context** — a categorical state (`supportive` / `neutral` /
|
||||
`adverse` / `unknown`) with an `evidence_quality` grade.
|
||||
|
||||
The third is **never a term in the other two**. They are read together by
|
||||
confluence:
|
||||
|
||||
| Warning | Fundamentals | Reading |
|
||||
|---|---|---|
|
||||
| Calm | Supportive/neutral | Normal |
|
||||
| Elevated | Supportive/neutral | Technical warning, not fundamentally confirmed |
|
||||
| Calm | Adverse | Fundamental concern; tape has not confirmed |
|
||||
| Elevated | Adverse | Confluence — highest attention |
|
||||
|
||||
`METHODOLOGY` stays **v4**: no score changed, so partitioning the history API and
|
||||
discarding the event study cache would be churn. `STUDY_SCHEMA` moved to 3
|
||||
instead, and is now the only thing that discards a stale report.
|
||||
|
||||
### Why the read is a channel and not a weight
|
||||
|
||||
Two things are true at once, and only this shape honours both.
|
||||
|
||||
**v3's reason for removing fundamentals from the score was wrong.** Not stale —
|
||||
wrong. v3 argued that F1 (capex) and F3 (good-news-stock-down), carrying 12 + 8
|
||||
of 100 Warning points, "could not change any published conclusion" because pegged
|
||||
they produced a Warning of exactly 20.0, below the alarm threshold. That
|
||||
arithmetic holds only when *every* technical sensor reads exactly zero, which is
|
||||
the one case that never matters. Warning is a weighted average, so the sensors
|
||||
add:
|
||||
|
||||
| technical Warning | without fundamentals | with them pegged | delta |
|
||||
|---|---|---|---|
|
||||
| 0 | 0.0 | 20.0 | +20.0 |
|
||||
| 20 | 20.0 | 36.0 | +16.0 |
|
||||
| 25 | 25.0 | **40.0** | +15.0 |
|
||||
| 35 | 35.0 | **48.0** | +13.0 |
|
||||
| 50 | 50.0 | 60.0 | +10.0 |
|
||||
| 80 | 80.0 | 84.0 | +4.0 |
|
||||
|
||||
Pegged fundamentals lowered the technical Warning needed to reach the 40 quadrant
|
||||
divider from 40 to 25. That is a 15-point shift in where the alert fires, which
|
||||
is emphatically a changed conclusion. The v3 section below is kept as written,
|
||||
with this correction attached, because its reasoning is cited elsewhere in this
|
||||
file and a silent overwrite would hide that the error was ever made.
|
||||
|
||||
**But no weight is measurable either.** A weighted modifier was built and
|
||||
reverted: 0–25 points added onto the technical Warning, sized so a maxed-out read
|
||||
carried a calm tape over the 40 divider on its own. Nothing could justify the 25.
|
||||
With ~10 correction events and essentially no fundamental history, any fusion
|
||||
weight is a policy preference presented as a measurement — and the debate it
|
||||
invites ("does the read deserve 10%, 20%, 30%?") has no evidence that can settle
|
||||
it. Adding a slow categorical judgement to a fast continuous score also
|
||||
manufactures precision by summing unlike things, and it forces a missing
|
||||
observation to silently redistribute its weight onto the technical sensors, which
|
||||
is the opposite of leaving it unknown.
|
||||
|
||||
So: the read gets a channel, not a coefficient. Both facts survive — the v3
|
||||
removal was badly argued *and* no weight is defensible — because "report it
|
||||
separately" is the only design that neither buries the observation nor invents a
|
||||
number for it.
|
||||
|
||||
### Derivation
|
||||
|
||||
Deterministic, from the stored categorical facts. The LLM is an **extraction and
|
||||
explanation layer**: it finds the capex guidance, classifies it, and cites it.
|
||||
Fixed rules turn those facts into a state, so the same observation always yields
|
||||
the same category.
|
||||
|
||||
`capex_signal`: any `cutting` → adverse; else any `holding` → neutral; else all
|
||||
known `raising` → supportive; nothing known → unknown.
|
||||
`reaction_signal`: `yes` → adverse, `mixed` → neutral, `no` → supportive,
|
||||
`unknown` → unknown.
|
||||
|
||||
`mixed` and `unknown` are different reaction states and were merged until
|
||||
2026-08-13. A failed LLM parse fell back to `mixed`, so an extraction error
|
||||
became *neutral evidence* — an observation of normality manufactured out of a
|
||||
bug. `mixed` now means an observed mixed reaction; anything unreadable, missing
|
||||
or unattempted is `unknown` and contributes nothing.
|
||||
|
||||
Combined by precedence, never by averaging: **any adverse read carries**; both
|
||||
unknown → unknown; every observed signal supportive → supportive; otherwise
|
||||
neutral.
|
||||
|
||||
`unknown` is deliberately unreachable by combination. Averaging would let two
|
||||
`cutting` reads and two `unknown` ones land on "neutral", presenting missing
|
||||
evidence as evidence of normality — the same conflation `current_observation`
|
||||
already refuses between "no observation" and "an observation of zero". Two cuts
|
||||
and two unknowns read **adverse with `evidence_quality: partial`**.
|
||||
|
||||
`evidence_quality` is ordered by what an operator needs first: `unavailable`
|
||||
(nothing collected) → `stale` (past `fundamental_staleness_days`) → `manual`
|
||||
(hand override) → `complete` / `partial`.
|
||||
|
||||
### Presentation and alerts
|
||||
|
||||
The Path view colours each dot by the fundamental state recorded that day; the
|
||||
axes are untouched, because context is confluence information rather than a
|
||||
position on either axis. The card leads with the state and evidence grade.
|
||||
|
||||
Alerts stay **separate**, off one toggle:
|
||||
|
||||
- quadrant change — the market axes moved (existing);
|
||||
- `regime_fundamental` — the context changed, e.g. neutral → adverse;
|
||||
- `regime_confluence` — Warning elevated *and* fundamentals adverse.
|
||||
|
||||
`unknown` never alerts: an absence of evidence is not a change in the evidence,
|
||||
and alerting on it would train the reader to ignore the channel. Both new
|
||||
triggers seed silently on first run, as the quadrant alert does.
|
||||
|
||||
### The observation is now a real time series
|
||||
|
||||
`regime_fundamental_observations` (migration 033), one row per `effective_date`,
|
||||
upserted. Before this it lived in a single `SystemSetting` slot that every
|
||||
refresh overwrote, so no history existed at all — which made the read impossible
|
||||
to replay, impossible to backtest, and meant a rebuild recorded every historical
|
||||
session as if nothing had been observed. `update_regime_monitor` carries the
|
||||
pre-existing single-slot observation into the series on its next run.
|
||||
|
||||
### What this does not establish
|
||||
|
||||
The table starts empty and fills one observation at a time, so the fundamental
|
||||
rows are **untested, not failed**. Two things enforce that rather than one:
|
||||
|
||||
- they are **coverage-matched** — scored only on sessions where the channel had
|
||||
usable context and on corrections whose warning horizon fell inside it, with a
|
||||
market-only comparator over the identical window so any difference between them
|
||||
is the channel and not the window;
|
||||
- `measurable` stays false until `MIN_EVENTS_FOR_CONFIDENCE` corrections are
|
||||
covered, and the panel prints "insufficient exposure" rather than a ratio.
|
||||
|
||||
Without the first, one day of coverage would render as 0/10 — recreating, one
|
||||
observation later, exactly the tested-versus-unavailable confusion the flag was
|
||||
added to prevent. The market rows are unchanged, and the 1/10 shipped-rule figure
|
||||
remains a verdict on the technical sensors and the alert machinery alone.
|
||||
|
||||
The rationale for expecting the read to matter is the operator's: hyperscaler
|
||||
capex is the demand side of the entire AI trade, and good earnings being sold is
|
||||
a classic late-cycle tell. Both are plausible. Neither is measured here, and this
|
||||
file's convention is that published numbers are reproducible.
|
||||
|
||||
**The path forward is accumulation, then a test — in that order.** Once enough
|
||||
point-in-time observations exist, test whether the state improves prediction
|
||||
*conditional on* Warning. If it does, a fitted and calibrated model has something
|
||||
to fit; until then there is nothing to calibrate against. Backfilling would get
|
||||
there faster: capex direction is derivable from the 10-Q/10-K capex line, which
|
||||
the SEC fundamentals import already carries, and "good news, stock down" from
|
||||
earnings dates plus next-day returns, which the Dolt earnings import already
|
||||
carries. That last one is worth computing deterministically rather than asking
|
||||
the LLM to judge, for the same reason the state derivation is rule-based.
|
||||
|
||||
## What changed in v4
|
||||
|
||||
**V1 stopped saturating at VIX 30.** `(vix - 15) / 15` reached 100 at VIX 30 —
|
||||
the same defect v3 had *just* removed from P3, left in place one sensor over. VIX
|
||||
30 is a bad week, 50 is a crisis and 82 was March 2020, and all three scored
|
||||
identically. In the calibration window this flattened five distinct April-2025
|
||||
prints (52.33, 46.98, 45.31, 40.72, 38.57) into a single 100. It pegged on 14 of
|
||||
408 sessions; under the anchors below, none.
|
||||
|
||||
**The trend break is graded by depth, not a yes/no.** `_under_200` returned a
|
||||
bare 0/100, so P1 printed 100 the moment SMH and QQQ were both under their
|
||||
average — and because the price pillar takes `max(P1, P2, P3)`, that pinned the
|
||||
pillar and stopped P3's anchored ladder resolving anything for the whole of a
|
||||
selloff. It pegged on 46 of 408 sessions; now none. A 2% break reads ~30 where it
|
||||
used to read 100.
|
||||
|
||||
`max()` was **kept**. The defect was the step function feeding it, not the vote
|
||||
itself, and v3's "one capped vote for correlated reads" rationale still holds.
|
||||
The `P1_SCORE_CAP` fallback drafted during design was to fire if P1 became the
|
||||
sole price argmax on **more than 80% of sessions with State ≥ 40** — i.e. if it
|
||||
had quietly become a second drawdown sensor. Measured on that population: 47
|
||||
qualifying sessions, P1 sole argmax on **17 of them (36.2%)**, against P2's 16
|
||||
and P3's 14. Well under the threshold, so the cap is not shipped.
|
||||
|
||||
**The top State band moved 80 → 65.** See Calibration; this is the one change
|
||||
that is about the band rather than a sensor.
|
||||
|
||||
**Scope.** All three are State-side. `WARNING_BANDS`, `WARNING_WEIGHTS`,
|
||||
`QUADRANT_WARNING_DIVIDER` and the event study's frozen threshold are untouched.
|
||||
`QUADRANT_STATE_DIVIDER` stays 50 because only `breaking` moved.
|
||||
|
||||
## What changed in v3
|
||||
|
||||
**Fundamentals left the score.** F1 (capex) and F3 (good-news-stock-down)
|
||||
carried 12 + 8 of 100 Warning points. Pegged at maximum stress they produced a
|
||||
Warning of exactly 20.0 — below the event study's 25.3 alarm threshold, and
|
||||
still inside the "stable" band. The sourced observation could not change any
|
||||
published conclusion, so refreshing it looked like it did nothing. They are now
|
||||
a qualitative overlay reported beside the scores. Capex also stopped scoring
|
||||
`raising` and `holding` identically at 0: `holding` is the deceleration case and
|
||||
now scores 50, so a boom no longer reads the same as a stall.
|
||||
|
||||
> **Corrected 2026-08-12.** The claim in this paragraph is false. "Pegged
|
||||
> they produced a Warning of exactly 20.0" describes only the case where every
|
||||
> technical sensor reads zero; Warning is a weighted average, so in the general
|
||||
> case those 20 points added +10 to +20 and moved the technical score needed to
|
||||
> reach the 40 quadrant divider from 40 to 25. The observation was removed for
|
||||
> being *underweighted*, on reasoning that mistook a corner case for the whole
|
||||
> range. See "The fundamental channel" above for what replaced it — a separate
|
||||
> categorical channel, not a restored weight. The capex `holding` rescale in the second half
|
||||
> of this paragraph stands and is still live.
|
||||
|
||||
**The drawdown sensor stopped saturating.** v2 used `dd_pct * 5`, reaching 100 at
|
||||
a 20% drawdown — the 90th percentile of the observed distribution. 39 of 408
|
||||
sessions sat at exactly 100 with no resolution left, and the price pillar showed
|
||||
the top band on 13.5% of sessions. v3 uses named anchors with headroom past the
|
||||
observed 36% maximum, and blends leader/confirm 2:1 as P1 and P2 already did
|
||||
instead of taking `max()`. P3's realized share of State falls from 65% to 40%,
|
||||
matching its nominal weight.
|
||||
|
||||
**Warning gained a sensor with range.** The HY OAS *level* is pinned at zero
|
||||
below the 3.5 mild anchor (2.77 at the cutover), so credit contributed nothing
|
||||
in a calm tape. Its 20-session rate of change still does, and spread widening is
|
||||
a classic lead.
|
||||
|
||||
**The credit percentile leg was removed.** Its reference window silently shrank
|
||||
from 10 years to 3 when ICE restricted the upstream series in April 2026, after
|
||||
which it scored 20 points of stress at a spread the same sensor's anchors call
|
||||
"mild". See Calibration below.
|
||||
|
||||
**Breadth loss counts during declines.** v2's divergence gate was
|
||||
`price_ret >= 0`, so the sensor zeroed during every selloff. On 2026-07-24 the
|
||||
basket shed 10 points of participation in 20 sessions while SMH fell 11.9% and
|
||||
Warning printed exactly 0. v3 tapers to a floor instead: deterioration counts
|
||||
fully when price masks it (true divergence, the dangerous pre-top case) and at
|
||||
35% when price confirms it. Breadth *level* lives in State, but breadth
|
||||
*velocity* appears nowhere else, so this is not double counting.
|
||||
|
||||
**Bands are per axis.** v2 Warning never exceeded 64.9 in 408 sessions while
|
||||
State reached 91.2, yet both used 30/60/80 with quadrant dividers at 60. The
|
||||
upper half of the Warning axis was unreachable.
|
||||
|
||||
## Outputs
|
||||
|
||||
**State** — current structural stress:
|
||||
|
||||
- Price structure, 40%: `max(P1, P2, P3)`, one capped vote for correlated reads.
|
||||
- Fixed-basket breadth level, 25%.
|
||||
- HY option-adjusted credit spread level, 20%.
|
||||
- VIX level, 15%.
|
||||
|
||||
**Warning** — deterioration and divergence:
|
||||
|
||||
- Fixed-basket breadth divergence, 45%.
|
||||
- 60-session SMH/SPY relative-strength deterioration, 30%.
|
||||
- HY OAS 20-session widening, 25%.
|
||||
|
||||
**Fundamental context** — a categorical third channel, not a term in either
|
||||
score. See "The fundamental channel" above.
|
||||
|
||||
Combined, RSP/SPY (former F4), and the NVDA canary (former P6) do not enter v3
|
||||
or v4.
|
||||
|
||||
## Calibration
|
||||
|
||||
### Interpolated sensor tables
|
||||
|
||||
All three are `(x, stress score)` pairs read by `_interpolate`, flat outside the
|
||||
first and last anchor.
|
||||
|
||||
| sensor | anchors |
|
||||
|---|---|
|
||||
| P3 drawdown (% below the 52w high) | 0→0, 4→10, 8→25, 16→50, 28→78, 40→100 |
|
||||
| **P1 trend break** (% below the 200-DMA) | 0→**20**, 3→35, 8→55, 15→75, 25→100 |
|
||||
| **V1 volatility** (VIX level) | 15→0, 20→20, 25→38, 30→55, 40→80, 55→100 |
|
||||
|
||||
P1's floor of 20 at the crossing is deliberate: the break itself is a genuine
|
||||
binary event and deserves a floor; only the depth past it is graded. P1 is
|
||||
calibrated to sit alongside P3 rather than swamp it — the 200-DMA lags, so a 20%
|
||||
drawdown typically coincides with ~10% below the average, where P1 reads ~61
|
||||
against P3's ~59.
|
||||
|
||||
V1 reaches full scale at 55 rather than at 2020's ~82: anchoring the top at a
|
||||
once-in-a-generation print would make VIX 50 — a genuine crisis — read only ~70.
|
||||
The anchors encode the long-run distribution as constants, the same argument the
|
||||
credit level uses. Unlike P1 and V1, whose slopes ease off monotonically, P3's do
|
||||
not (2.5, 3.75, 3.125, 2.33, 1.83) — its gentle onset is intentional and the
|
||||
monotone-slope test excludes it.
|
||||
|
||||
Credit impulse is relative (+35% over 20 sessions = 100) rather than absolute,
|
||||
because +0.5pp means something very different at an OAS of 2.7 than at 8.0.
|
||||
|
||||
### Bands
|
||||
|
||||
Round, meaning-anchored numbers, **not** percentile fits — those would drift on
|
||||
every rebuild and silently rewrite what past snapshots meant.
|
||||
|
||||
**Why `breaking` moved 80 → 65.** With credit calm, `f2_credit_spreads` returns
|
||||
`0.0` (not `None`), so it keeps its full 20 points pinned at zero. Price, breadth
|
||||
and volatility at *literal maximum* therefore sum to:
|
||||
|
||||
(100×40 + 100×25 + 0×20 + 100×15) / 100 = 80.0 exactly
|
||||
|
||||
`band_for` uses `>=`, so v3's top band was reachable only by touching its floor
|
||||
to the decimal, with nothing above it. The band was fit on v2, when credit's
|
||||
since-removed percentile leg still contributed regularly; the sensor is not
|
||||
wrong — a calm-credit selloff genuinely *is* less stressed than one with credit
|
||||
contagion — the threshold was stale.
|
||||
|
||||
Chosen by scenario arithmetic on unchanged weights (`_scenarios` in the harness
|
||||
computes these, so they are machine-checked, not prose):
|
||||
|
||||
| scenario | price | breadth | C1 | V1 | State |
|
||||
|---|---|---|---|---|---|
|
||||
| Ordinary tape (3% dd, breadth 65%, VIX 16, OAS 2.8) | 7.5 | 0 | 0 | 4.0 | **3.6** |
|
||||
| 10% correction, calm credit (2% below, breadth 35%, VIX 24) | 31.2 | 62.5 | 0 | 34.4 | **33.3** |
|
||||
| **2022-style drawdown, calm credit, no death cross** | 90.8 | 100 | 0 | 60.0 | **70.3** |
|
||||
| **same, with death cross** (P2 pegged) | 100 | 100 | 0 | 60.0 | **74.0** |
|
||||
| Credit event on top (OAS 6.0, VIX 45) | 100 | 100 | 75.0 | 86.7 | **93.0** |
|
||||
| March 2020 (everything pegged) | 100 | 100 | 100 | 100 | **100** |
|
||||
|
||||
Rows 3 and 4 are the case this monitor exists to measure, and they must print
|
||||
`breaking`. At 80 they do not. **65** clears them under either P2 assumption,
|
||||
which matters because P2 is set by the 50/200-DMA gap and no drawdown figure
|
||||
implies it; 70 would have left 0.33 points of headroom in row 3, reproducing the
|
||||
defect being fixed.
|
||||
|
||||
Realized shares, **reported not fitted**, over the 408 sessions to 2026-07-24:
|
||||
|
||||
| Axis | stable | watch | elevated | breaking | thresholds |
|
||||
|------|--------|-------|----------|----------|------------|
|
||||
| State (v4) | 78.9% | 13.0% | 4.7% | **3.4%** | 20 / 50 / **65** |
|
||||
| Warning | 69.4% | 19.6% | 7.6% | 3.4% | 20 / 40 / 60 |
|
||||
|
||||
The v4 `breaking` share lands on 3.4% — the same as v3's — having been chosen by
|
||||
scenario reasoning rather than aimed at that number. Sensitivity: 60 gives 5.1%,
|
||||
70 gives 1.2%.
|
||||
|
||||
Quadrant dividers sit at each axis's watch/elevated boundary: State 50,
|
||||
Warning 40. Only `breaking` moved in v4, so the dividers and every alert
|
||||
threshold are unchanged. `test_quadrant_dividers_match_the_band_boundaries` now
|
||||
enforces that relationship, which nothing did before.
|
||||
|
||||
Scores renormalize over available fixed weights, but a band is published only at
|
||||
75% or greater coverage. Trend deltas are suppressed when the participating
|
||||
pillar set changes. Zero means ordinary/healthy; only stress contributes.
|
||||
|
||||
Credit level is the named HY OAS anchors alone: 3.5 mild, 5.0 elevated, 7.0
|
||||
stressed, linear between, and nothing else. v2 blended those anchors at 70% with
|
||||
a 30% upper-tail percentile over a nominally 10-year window.
|
||||
|
||||
That leg was removed rather than repaired. ICE restricted FRED to a rolling
|
||||
3-year window for `BAMLH0A0HYM2` in April 2026 — the series metadata states it
|
||||
outright ("Starting in April 2026, this series will only include 3 years of
|
||||
observations"), and an unbounded request returns the same 795 observations as a
|
||||
30-year one. The v2 percentile therefore ranked the current spread against three
|
||||
uniformly tight years (range 2.59–4.61 over the calibration window), which made
|
||||
it fire early and saturate absurdly: at an OAS of 3.50 — the level the anchors
|
||||
call *mild*, scoring zero stress — the blended sensor read 20.1, and the
|
||||
percentile leg pegged at 100 by an OAS of 4.5. Across the 408 sessions it
|
||||
roughly tripled the credit sensor's average (2.70 vs 1.00) and more than doubled
|
||||
its nonzero days (60 vs 27).
|
||||
|
||||
The anchors already encode the long-run distribution as constants, so the
|
||||
percentile was a second, noisier estimate of the same thing. What it was
|
||||
genuinely reaching for — "unusual versus recent history" — is now W3 on the
|
||||
Warning axis, computed as a rate of change, which is where deterioration
|
||||
belongs. Removing it moved State's average by −0.4 and its maximum by −3.8, left
|
||||
Warning bit-identical, and did not shift any band threshold.
|
||||
|
||||
A long-history alternative (`BAA10Y`, Fed-published, 7,712 observations back to
|
||||
1997) was considered and rejected: ranking an HY spread against investment-grade
|
||||
history is not a coherent statistic, and it would rescue a leg that is redundant
|
||||
anyway.
|
||||
|
||||
Every snapshot now records `data_quality.credit_history_days` and
|
||||
`vix_history_days`. This defect was invisible for roughly three months because
|
||||
nothing asserted the window the code claimed; the spans make a future upstream
|
||||
truncation show up in the record instead of quietly reshaping a sensor.
|
||||
|
||||
**Survivorship caveat.** The basket was frozen 2026-07-15 but the calibration
|
||||
window reaches back to 2024, so names were partly selected for having done well.
|
||||
Every distribution above inherits that bias. It is the same bias v2 carried, so
|
||||
the v2/v3 comparison is like-for-like, but the absolute band shares are
|
||||
optimistic.
|
||||
|
||||
**Which OAS window the published v2 figures used.** v2 requested 13 years of HY
|
||||
OAS and sliced `HY_OAS_REFERENCE_YEARS = 10.0` per session; ICE serves only ~3
|
||||
years (778 observations from 2023-08-08), so the effective window was that. But
|
||||
production v2 also fetched only 400 *calendar* days at one point — the bug fixed
|
||||
2026-08-07 — and whether the published numbers predate that was not recoverable
|
||||
from the text. Settled by replay rather than assumed: the
|
||||
`v2_reconstruction_oas400` variant truncates the OAS **source series** to 400
|
||||
days (patching the per-session window cannot simulate data that was simply
|
||||
absent) and yields avg 26.54, p80 42.52, max **100.00**, against published
|
||||
22.6 / 35.1 / 91.2. Full coverage reproduces all three. So the published figures
|
||||
correspond to the untruncated fetch.
|
||||
|
||||
**The top VIX anchors are exercised, not just asserted.** The window contains a
|
||||
52.33 close (2025-04-08), so the 40 → 80 → 55 → 100 segment is fed by real data
|
||||
rather than justified from long-run history alone.
|
||||
|
||||
## Point-in-time record
|
||||
|
||||
The first run under a new `METHODOLOGY` rebuilds every session inside
|
||||
`REBUILD_LOOKBACK_DAYS` — 672 calendar days, roughly 464 trading sessions;
|
||||
routine runs thereafter insert/update only the latest trading date. The bound is
|
||||
in calendar days rather than a session count because the binding constraint is
|
||||
the OAS fetch: each replayed row needs W3's lookback inside
|
||||
`HY_OAS_WINDOW_DAYS`, so replaying further back would recreate the credit gap a
|
||||
reseed exists to close. The history API and main chart show only snapshots matching
|
||||
the current methodology, so a bump reseeds the series rather than splicing two
|
||||
formulas into one line.
|
||||
|
||||
The fundamental channel keeps its effective date (normally the next session after
|
||||
collection) and is never replayed backward, so a rebuild cannot stamp today's
|
||||
observation onto historical snapshots. Since the observations became a real
|
||||
series (`regime_fundamental_observations`, migration 033), the effective-date
|
||||
lookup *is* the gate: a replayed session gets whichever observation was live on
|
||||
it, and sessions before the first one read `unknown`.
|
||||
|
||||
Two functions, deliberately: `fundamental_context` is the **record** and keeps
|
||||
the gate — it runs for every replayed date during a rebuild, so it must never
|
||||
grow a bypass flag. `current_observation` is the **live reading** behind
|
||||
`fundamental_live`, and *reports* the effective date instead of blanking the
|
||||
content.
|
||||
|
||||
Until 2026-08-07 the live reading called the gated function, so a just-collected
|
||||
observation stayed hidden until the next weekday — three days over a weekend —
|
||||
and refreshing appeared to do nothing. That was the opposite of what this section
|
||||
already claimed. Showing it early cannot leak into a published score, because
|
||||
nothing in the channel is scored.
|
||||
|
||||
`current_observation` gates on `observed` (a non-null `fetched_at`, the one field
|
||||
every path writing real content stamps). Without it, the default override —
|
||||
`unknown` for every hyperscaler and, since 2026-08-13, `unknown` for the reaction
|
||||
— was reported as a live observation with `available: true`, so the card
|
||||
presented placeholders as a collected reading. Those are the absence of an
|
||||
observation, not an observation of absence. `fundamental_context` never had this
|
||||
problem: no observation means no effective date, which means `pending`, which
|
||||
already blanks the content.
|
||||
|
||||
**`usable` is what may confirm; `available` is only what to display.** Three
|
||||
distinct things, and collapsing any two of them is a bug:
|
||||
|
||||
- `state` — the last thing observed. Survives going stale, so the card can show it.
|
||||
- `available` — *timing*: there is an effective, non-stale record to display.
|
||||
- `usable` — *content*: available **and** the observation actually determined
|
||||
something (`state != "unknown"`).
|
||||
|
||||
The confluence alert and all three coverage-matched study rules gate on `usable`.
|
||||
Gating on `available` instead has two failure modes, and both were live at some
|
||||
point in this design:
|
||||
|
||||
1. a reading past `fundamental_staleness_days` would corroborate every Warning
|
||||
crossing indefinitely — the strongest claim this channel makes, from the data
|
||||
with the least right to make it;
|
||||
2. an LLM run that failed to extract anything produces a perfectly fresh
|
||||
observation that knows nothing. Counting it as exposure means repeated
|
||||
extraction failures slowly accumulate coverage until the fundamental rows flip
|
||||
to a *measurable* 0/8 — a failed result published for a channel that never saw
|
||||
a thing, which is precisely what coverage-matching exists to prevent.
|
||||
|
||||
**Pre-rename snapshots are adapted, not discarded.** The channel was stored as
|
||||
`fundamental_overlay` until 2026-08-12. The rename shipped without a methodology
|
||||
bump — no score changed — so those rows are still served and were never reseeded.
|
||||
Reading only the new key would have turned every one of them into `unknown`,
|
||||
silently dropping real recorded evidence: historical Path colours, and exposure
|
||||
the event study can legitimately count. `_parse_snapshot` derives the channel
|
||||
from a legacy overlay's own stored facts (its capex map supplies the basket, so
|
||||
the derivation uses the names observed at the time rather than today's config).
|
||||
Normalising there rather than at each call site means no reader can receive an
|
||||
un-adapted row. Delete only after a reseed has rewritten the whole window.
|
||||
|
||||
**The blob and the series row are one transaction.** They are the same
|
||||
observation seen by the live card and by the point-in-time replay; committing
|
||||
them separately leaves a window where a failure publishes one and not the other,
|
||||
and the two then disagree permanently with nothing to detect it. Both writers use
|
||||
`settings_store.upsert_setting` (which does not commit) plus a single commit;
|
||||
`record_fundamental_observation` deliberately takes no commit of its own so
|
||||
`update_regime_monitor` keeps its own transaction boundary.
|
||||
|
||||
Each snapshot stores the fixed basket symbols, hash, and freeze date.
|
||||
Reconstructed history before that freeze date is retrospective/exploratory.
|
||||
|
||||
## Presentation
|
||||
|
||||
The page is deliberately thin: two gauges, one chart card, one pillar table, the
|
||||
overlay, and a provenance strip. Time and Path are two projections of the same
|
||||
snapshot series and share one card and one query key — they were previously two
|
||||
panels, which read as two datasets. Methodology rationale lives in this document,
|
||||
not on the page; page text is limited to what changes how the reader interprets
|
||||
today's number. The quadrant dividers rendered in Path view come from
|
||||
`quadrant_config` and are the same constants the alert path consumes
|
||||
(`alert_service`), so the chart cannot drift from what actually fires.
|
||||
|
||||
## Warning study
|
||||
|
||||
The study calls the outcome a **10% correction**, not a regime break. It measures
|
||||
two rules against that outcome, plus enough context to tell whether either number
|
||||
is any good.
|
||||
|
||||
A cached report is discarded when its methodology no longer matches *or* when
|
||||
`STUDY_SCHEMA` moves, so the panel reverts to "not run yet" rather than showing
|
||||
stale numbers or a report missing half its blocks. **Re-run the Event Study job
|
||||
after a methodology cutover or a schema bump.**
|
||||
|
||||
### The headline is the rule that actually fires
|
||||
|
||||
Until 2026-08-12 the study measured a bare rising-edge crossing of an
|
||||
80th-percentile threshold fitted on the first 70% of sessions. **Nothing consumes
|
||||
that rule.** What reaches Telegram is `_collect_regime_quadrant`: a quadrant
|
||||
change with State ≥ 50 and Warning ≥ 40 as fixed dividers, a ±5 hysteresis
|
||||
deadband, a two-session confirmation, a 3-day cooldown, and a 75% coverage gate
|
||||
on both axes. The two differ on every one of those axes, including the threshold
|
||||
itself (a fitted ~32 against a shipped 40).
|
||||
|
||||
`replay_quadrant_changes` replays the shipped state machine over the whole
|
||||
sample. Three details are reproduced rather than cleaned up, because a state
|
||||
machine written from first principles gets each of them wrong:
|
||||
|
||||
- the prior session is classified against the **current baseline**, not against
|
||||
its own predecessor, so confirmation asks "did yesterday already look like this
|
||||
change" rather than "did yesterday change too";
|
||||
- the baseline advances only when an alert actually fires, so a change blocked by
|
||||
confirmation or cooldown is re-evaluated against the old quadrant next session;
|
||||
- one cooldown is shared by every quadrant change, so a 3→4 alert can swallow a
|
||||
4→2 alert three days later.
|
||||
|
||||
Two consequences worth stating. The alarm is dated at the **confirmation**, not
|
||||
at the first crossing, which costs one session of lead by construction. And the
|
||||
rule alerts on changes in both directions, so the replay's exits are recorded but
|
||||
filtered out by `entry_alarms` — only entering a Warning-high quadrant is a
|
||||
warning about anything.
|
||||
|
||||
The replay reuses `_compute_index` rather than re-deriving the axes. That is the
|
||||
same anti-drift argument that produced `warning_sensor_scores`: the v2 study
|
||||
re-derived Warning by hand and would have kept measuring the old construct
|
||||
through a scoring change. State has no equivalent shared helper, so the snapshot
|
||||
builder itself is the shared definition.
|
||||
|
||||
**Nothing is fitted, so nothing needs protecting from a training set.** There is
|
||||
no split, and every detected correction is evaluable instead of the four that
|
||||
happen to land in the last 30%. The `underpowered` and "threshold frozen on a
|
||||
different construct" caveats do not apply to this variant.
|
||||
|
||||
### Reading the result
|
||||
|
||||
A bare "2 of 4" is unreadable in either direction, so the report scores four more
|
||||
rules through the same `evaluate_alarms` harness over the same events and
|
||||
sessions, and adds a null. All use fixed thresholds — a threshold fitted on the
|
||||
full sample would have lookahead the shipped rule does not, and one fitted on a
|
||||
split could only be scored on the holdout events.
|
||||
|
||||
| kind | rules | the question |
|
||||
|---|---|---|
|
||||
| ablation | Warning ≥ 40 bare, State ≥ 50 bare | does the quadrant machinery earn its place? |
|
||||
| baseline | leader below its 50-DMA, VIX ≥ 20 | does the score earn its complexity? |
|
||||
| null | K random alarms at the observed firing rate | is any of this better than chance? |
|
||||
|
||||
The two kinds must not be read as one list. If a baseline matches the score, the
|
||||
composite is not earning its complexity and that is the finding — it does not
|
||||
mean the monitor is worthless, since State and Warning exist to be *read*, but it
|
||||
caps how much further calibration is justified. If the bare Warning crossing
|
||||
beats the shipped rule, the machinery (not the sensor) is what is costing recall.
|
||||
|
||||
The null draws only from sessions a rule could actually have fired on. Over the
|
||||
whole sample it would be diluted by warm-up sessions and would understate what
|
||||
chance achieves — which matters, because with ~11 events and a 20-session horizon
|
||||
roughly a sixth of the sample already sits inside a hit window. It is seeded, so
|
||||
a re-run cannot move the report. Corrections cluster and uniform placement does
|
||||
not, so it is the **floor, not the bar**: an alarm process that clustered would
|
||||
beat it for reasons unrelated to foresight.
|
||||
|
||||
### First result (2026-08-12): the shipped rule is not distinguishable from chance
|
||||
|
||||
Replayed over 2021-07-14 → 2026-08-12. The 200-DMA warm-up means the baseline
|
||||
only seeds on 2022-05-26, so 1056 of 1276 sessions are evaluable and 10 of the 11
|
||||
detected corrections fall inside them.
|
||||
|
||||
| rule | kind | warned | FA/yr | median lead |
|
||||
|---|---|---|---|---|
|
||||
| **Quadrant alert (shipped)** | | **1/10** | **0.9** | 19d |
|
||||
| Quadrant alert, both axes high | ablation | 0/10 | 0.9 | — |
|
||||
| Warning ≥ 40, bare crossing | ablation | 3/10 | 4.8 | 20d |
|
||||
| State ≥ 50, bare crossing | ablation | 0/10 | 0.7 | — |
|
||||
| SMH below its 50-DMA | baseline | 7/10 | 6.7 | 8d |
|
||||
| VIX ≥ 20 | baseline | 4/10 | 7.2 | 9.5d |
|
||||
| Random alarms, same firing rate | null | 0.9 ± 0.8 | — | — |
|
||||
|
||||
**P(chance ≥ 1/10) = 0.65.** Alarms scattered at random over the same sessions at
|
||||
the rule's own firing rate match or beat it two times in three. Whatever the
|
||||
score knows, this rule is not transmitting it.
|
||||
|
||||
Three readings, in order of how much they should change:
|
||||
|
||||
**The machinery costs more than it protects.** The bare Warning crossing catches
|
||||
3 with a 20-session lead; wrapping it in the quadrant rule drops that to 1. The
|
||||
State condition is the largest single cost — requiring both axes high catches
|
||||
nothing at all, which is what a coincident axis gating a leading one predicts.
|
||||
Hysteresis, the two-session confirmation and the shared cooldown between them
|
||||
take the rest, and the cooldown is shared across *every* quadrant change, so
|
||||
exits consume the budget that entries need. Only 5 of the 15 replayed changes are
|
||||
Warning-high entries.
|
||||
|
||||
**The crude baselines beat everything on recall, at a price.** SMH below its
|
||||
50-DMA catches 7 of 10 — but at 6.7 false alarms a year against the shipped
|
||||
rule's 0.9. That is a 7× recall improvement for 7× the noise, so it is not a
|
||||
clean dominance and this table cannot settle it; the missing axis is what a false
|
||||
alarm actually costs, which nothing here measures. What it does settle is that
|
||||
the composite is not buying recall the 50-DMA does not already have.
|
||||
|
||||
**The 0.9 false alarms/year is not the achievement it looks like.** A rule that
|
||||
almost never fires has few false alarms by construction. Read the two columns
|
||||
together or not at all.
|
||||
|
||||
Recorded from an offline replay (live Alpaca + FRED, no database, breadth
|
||||
computed from the same Alpaca closes rather than the stored universe). The job in
|
||||
Admin → Jobs is the canonical path and reads breadth from the DB, so re-run it to
|
||||
confirm these figures before treating them as the record.
|
||||
|
||||
**This is a verdict on the market channels only.** The fundamental and confluence
|
||||
rows in the same table are marked `measurable: false` and print "not measurable"
|
||||
rather than a ratio: with an empty observation series they never fire, and a 0/10
|
||||
sitting in a comparison column would read as tested-and-failed. `false` here means
|
||||
the input does not exist yet, not that the rule lost.
|
||||
|
||||
(The figures above were also produced under a briefly-built weighted modifier and
|
||||
came back bit-identical, which is what confirmed the modifier was inert over the
|
||||
whole window — the numbers depend on the technical sensors alone either way.)
|
||||
|
||||
**Not acted on.** Nothing in the alert path was changed on the strength of this.
|
||||
The obvious candidates — dropping the State condition from the entry test,
|
||||
separating the entry and exit cooldowns, or lowering the Warning divider — are
|
||||
threshold changes to a live alerting rule and want their own decision.
|
||||
|
||||
### The coverage gap relocates, it does not close
|
||||
|
||||
Dropping the fitted threshold makes the whole sample evaluable, but most of the
|
||||
extra events predate 2023-08. W3 does not exist there, so Warning renormalises to
|
||||
`(W1×45 + W2×30)/75` and the fixed 40 divider is applied to a different construct
|
||||
than it was reasoned about. The report therefore splits shipped-rule metrics at
|
||||
the credit sensor's first session and the panel states both, because replacing
|
||||
one misleading headline with a differently misleading one would be no gain.
|
||||
|
||||
Convenient side effect: the pre-credit era *is* the "Warning without W3"
|
||||
ablation, measured on real sessions rather than simulated ones, so that ablation
|
||||
is not run separately.
|
||||
|
||||
Alarms and events are assigned to eras by index, so an alarm days before the
|
||||
boundary matching an event days after it lands in the earlier era. With the eras
|
||||
years long and the events sparse, that costs nothing.
|
||||
|
||||
### The fitted variant, kept for continuity
|
||||
|
||||
The 70/30 percentile study is still computed and still reported, collapsed, with
|
||||
its `reliability` block intact — it is a genuinely different question, and it is
|
||||
what earlier revisions of this document report. Its caveats stand:
|
||||
|
||||
**The holdout is thin.** The study detects 11 corrections across 5 years but the
|
||||
70/30 split leaves only 4 in the test period. Recall is one event away from a
|
||||
materially different headline, and in practice the event that flips is decided by
|
||||
where the frozen threshold happens to land rather than by whether the score saw
|
||||
anything. The v3 cutover run illustrates it: v3 scored 2/4 against v2's 3/4, but
|
||||
"v3 without the credit sensor" scores 3/4 at a *higher* threshold (35.5) than
|
||||
shipped v3 misses it at (32.3) — because the alarm rule needs a rising edge, and a
|
||||
lower threshold can mean the alarm already fired outside the 20-session horizon
|
||||
and never reset below. Below `MIN_EVENTS_FOR_CONFIDENCE` holdout events the
|
||||
report says so explicitly.
|
||||
|
||||
Some events carry no information at all for comparison: in that run every
|
||||
variant caught 2026-03-06, every variant missed 2026-06-05, and every variant
|
||||
"caught" 2025-11-20 with a 1-session lead, which is coincident rather than a
|
||||
warning. The headline recall does not currently discount those; a minimum-lead
|
||||
rule is the obvious next change and has not been made.
|
||||
|
||||
**Sensor coverage straddles the split.** The score renormalises over available
|
||||
sensors, so a training window predating a sensor's history freezes the threshold
|
||||
on a different construct than the holdout is measured against. At the v3 cutover
|
||||
only 39% of training sessions had all three Warning sensors versus 100% of the
|
||||
test period, because credit history begins 2023-07-25.
|
||||
|
||||
Restricting the threshold to sensor-matched training sessions was tried and is
|
||||
*not* the fix: those sessions are a calm recent stretch, so the threshold drops
|
||||
from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6 per year. It trades a
|
||||
coverage bias for a regime-selection bias. The honest position is that a fitted
|
||||
threshold is hypersensitive to window choice at this sample size — which is the
|
||||
strongest argument for making the unfitted shipped rule the headline.
|
||||
|
||||
### Considered and not done
|
||||
|
||||
**An ETF credit proxy (HYG/IEF) to extend W3 back over the whole sample.** It
|
||||
would trade "two sensors versus three" for "proxy sensor versus real sensor" —
|
||||
still a construct straddle, but no longer flagged by the coverage split. This is
|
||||
the same objection that rejected `BAA10Y` as a percentile reference. If ever
|
||||
revisited, check the impulse correlation on the three years of real-OAS overlap
|
||||
first and report it as a sensitivity, never as the headline.
|
||||
|
||||
**A depth sweep (5%/7%/15% corrections) for more events.** `EVENT_COOLDOWN_DAYS`
|
||||
is 40, so at shallower thresholds re-triggers inside a single decline merge or
|
||||
drop and the denominator moves for cooldown reasons rather than market ones.
|
||||
|
||||
## Resolved in v4 (raised 2026-08-07, shipped 2026-08-08)
|
||||
|
||||
The three questions this section used to hold are now answered. Kept here
|
||||
because the reasoning that resolved them is not obvious from the code.
|
||||
|
||||
**1. `breaking` had zero headroom — resolved by moving the band, not the sensor.**
|
||||
`f2_credit_spreads` returns `0.0`, not `None`, below the 3.5 mild anchor, so
|
||||
credit stays *available* at weight 20 and is pinned at zero on roughly 93% of
|
||||
sessions rather than being renormalized out. Price + breadth + volatility at
|
||||
literal maximum therefore summed to exactly 80.0 — v3's threshold, to the
|
||||
decimal.
|
||||
|
||||
The sensor is **deliberately unchanged**. A calm-credit selloff genuinely is less
|
||||
stressed than one with credit contagion, so scoring it lower is correct; what was
|
||||
stale was `STATE_BANDS`, fit on v2 while credit's since-removed percentile leg
|
||||
still contributed. Making credit `None` when calm was considered and rejected: it
|
||||
would leave State on 80% coverage, which still publishes, but consumes the whole
|
||||
buffer — any *second* missing pillar would then suppress the band, and the 7d/30d
|
||||
trend deltas would null out every time OAS crossed 3.5, because `_delta`
|
||||
suppresses on a change of participating pillars. See Calibration for the
|
||||
scenario arithmetic behind 65.
|
||||
|
||||
**2. V1 saturated at VIX 30 — resolved with an anchor table.** See "What changed
|
||||
in v4".
|
||||
|
||||
**3. `max(P1, P2, P3)` defeated P3's anchoring — resolved by grading `_under_200`,
|
||||
keeping `max()`.** The `max` was deliberate ("one capped vote for correlated
|
||||
reads") and survives; the binary step feeding it was the defect.
|
||||
|
||||
**Its limit, stated precisely.** `_death_cross` is `clamp(-gap_pct * 20)`, so P2
|
||||
pegs at a −5% 50/200-DMA gap — routine in a real downtrend. In a *deep* selloff
|
||||
the price pillar therefore still reaches 100 via P2 even with P1 graded. What v4
|
||||
repairs is the shallow-to-moderate break, which is where resolution was most
|
||||
obviously missing: a 10% correction 2% below the average now scores 31 where v3
|
||||
scored 100. It would be wrong to claim "the price pillar no longer pegs".
|
||||
|
||||
P2 did not peg once in the 408-session calibration window, so this is a property
|
||||
of the sensor rather than an observed problem. Grading P2 the same way is the
|
||||
natural next item if it starts binding; the replay reports a P2-pegged census
|
||||
alongside P3 and V1 so the evidence accumulates.
|
||||
|
||||
## Fixed 2026-08-07: the OAS fetch window did not cover a rebuild
|
||||
|
||||
`HY_OAS_WINDOW_DAYS` was 400 **calendar** days, but a rebuild replays
|
||||
`leader_series[-REBUILD_SESSIONS:]` — 400 **trading** sessions, about 579
|
||||
calendar days. The oldest ~180 calendar days of any rebuild therefore got no OAS
|
||||
data at all, so `f2_credit_spreads` and `w3_credit_impulse` both returned `None`.
|
||||
Verified: State then lands at 80% coverage and Warning at exactly 75.0% —
|
||||
`MIN_COVERAGE` — so **both still publish bands**. The rebuilt series would look
|
||||
homogeneous while its oldest rows had been scored without credit, the tell being
|
||||
a null `data_quality.credit_history_days` on exactly those rows.
|
||||
|
||||
The window is now 700 days: it must cover the oldest replayed date (~579) plus
|
||||
W3's lookback and slack, while staying under ICE's ~3-year cap so FRED still
|
||||
honours the request. This required **no methodology bump** — C1 reads
|
||||
`oas_values[-1]` and W3 reads `oas_values[-21]`, both indexed from the end, so
|
||||
widening only prepends older observations and every live score is bit-identical.
|
||||
Confirmed by evaluating both windows against a varying synthetic series: today's
|
||||
C1/W3 match exactly, while the oldest rebuild row goes from `None`/`None` to real
|
||||
values.
|
||||
|
||||
Expect `credit_history_days` on new snapshots to rise from ~400 to ~700. That is
|
||||
the widened request, not new upstream history — and it makes the chip a better
|
||||
truncation canary, since a 700-day request returning ~1095 days' worth is now
|
||||
the visible ceiling.
|
||||
|
||||
**Widening the window alone does not repair stored history.** Routine runs
|
||||
recompute only the latest trading date, and `rebuilding` was keyed on "no v3
|
||||
snapshot exists at all" — which is false once the cutover has run — so every row
|
||||
already written would have kept its credit gap indefinitely. `SENSOR_REVISION`
|
||||
fixes that: it is stamped into each snapshot, snapshots predating it read as 1,
|
||||
and a stored revision below the current one triggers exactly one reseed.
|
||||
|
||||
It is deliberately not `METHODOLOGY`. That constant partitions the history API
|
||||
and discards the cached event study; neither is warranted here, because the study
|
||||
recomputes its Warning series from source (`_warning_series` calls
|
||||
`warning_sensor_scores` against freshly fetched prices and OAS) rather than
|
||||
reading snapshots, so a reseed cannot stale it.
|
||||
|
||||
The reseed is bounded by `REBUILD_LOOKBACK_DAYS` in calendar days rather than a
|
||||
session count, because the binding constraint is the OAS fetch: each replayed row
|
||||
needs W3's 20-business-day lookback inside `HY_OAS_WINDOW_DAYS`. At 672 days the
|
||||
replay reaches ~464 sessions, W3's oldest requirement lands exactly on the first
|
||||
fetched OAS day, and the ~400-session series the v3 cutover wrote is fully
|
||||
covered. A test asserts that relationship so the two constants cannot drift into
|
||||
recreating the gap.
|
||||
|
||||
The fix was sequenced deliberately: acting on items 1–3 above bumped
|
||||
`METHODOLOGY`, which fires `rebuilding`, which would have baked the credit-less
|
||||
rows into the fresh series. Fixing the window first meant the v4 reseed replayed
|
||||
a clean window; doing it the other way round would have meant reseeding twice.
|
||||
|
||||
## Operator rule
|
||||
|
||||
Quadrant alerts default off for new/reset configurations. When enabled they
|
||||
require fresh inputs, at least 75% coverage on both axes, two consecutive daily
|
||||
confirmations, hysteresis, and cooldown. Every alert states: **Risk thermometer —
|
||||
not a trade signal.**
|
||||
@@ -1,18 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en" class="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>FundamentalsPanel harness</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Instrument+Sans:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
</head>
|
||||
<body class="bg-[#0a0b11] text-gray-100 font-sans">
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/dev/harness.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
+23
-54
@@ -207,14 +207,28 @@ export function backfillTickerNames() {
|
||||
}
|
||||
|
||||
// Jobs
|
||||
export type JobCategory = 'pipeline' | 'pipeline_step' | 'scheduled' | 'manual';
|
||||
export type NextRunSource = 'own_schedule' | 'via_pipeline' | 'manual_only';
|
||||
|
||||
export interface JobStatus {
|
||||
name: string;
|
||||
label: string;
|
||||
enabled: boolean;
|
||||
next_run_at: string | null;
|
||||
via_pipeline?: boolean;
|
||||
registered: boolean;
|
||||
category?: JobCategory;
|
||||
/** Server-assigned ordering; the payload already arrives grouped by it. */
|
||||
sort_order?: [number, number];
|
||||
/** Parent pipelines for a step. Many-to-many: data_collector runs in all four. */
|
||||
pipelines?: string[];
|
||||
/** Step names, for a pipeline row. */
|
||||
steps?: string[];
|
||||
next_run_at: string | null;
|
||||
next_run_source?: NextRunSource;
|
||||
/** For a step: the soonest enabled parent's next run, and which parent. */
|
||||
via_next_run_at?: string | null;
|
||||
via_next_run_job?: string | null;
|
||||
running?: boolean;
|
||||
/** runtime_* is live, in-memory state only — it resets when the app restarts. */
|
||||
runtime_status?: string | null;
|
||||
runtime_processed?: number | null;
|
||||
runtime_total?: number | null;
|
||||
@@ -223,6 +237,13 @@ export interface JobStatus {
|
||||
runtime_started_at?: string | null;
|
||||
runtime_finished_at?: string | null;
|
||||
runtime_message?: string | null;
|
||||
/** last_run_* is persisted and survives restarts. Kept separate from
|
||||
* runtime_* so a stale error cannot pin the status chip or the banner. */
|
||||
last_run_at?: string | null;
|
||||
last_run_status?: string | null;
|
||||
last_run_message?: string | null;
|
||||
last_run_processed?: number | null;
|
||||
last_run_total?: number | null;
|
||||
}
|
||||
|
||||
export interface TriggerJobResponse {
|
||||
@@ -233,40 +254,6 @@ export interface TriggerJobResponse {
|
||||
cadence?: BacktestCadence;
|
||||
}
|
||||
|
||||
export interface ParityFieldStats {
|
||||
legacy_available: number;
|
||||
candidate_available: number;
|
||||
both_available: number;
|
||||
material_differences: number;
|
||||
median_absolute_delta: number | null;
|
||||
p95_absolute_delta: number | null;
|
||||
max_absolute_delta: number | null;
|
||||
}
|
||||
|
||||
export interface FundamentalsParityReport {
|
||||
report_version: number;
|
||||
generated_at: string;
|
||||
as_of_date: string;
|
||||
approval_status: string;
|
||||
read_only: boolean;
|
||||
summary: {
|
||||
universe_count: number;
|
||||
legacy_fundamental_score_available: number;
|
||||
candidate_fundamental_score_available: number;
|
||||
fundamental_scores_compared: number;
|
||||
fundamental_score_material_changes: number;
|
||||
fundamental_rank_changes: number;
|
||||
field_stats: Record<string, ParityFieldStats>;
|
||||
};
|
||||
source_runs: Record<string, {
|
||||
run_id: number;
|
||||
status: string;
|
||||
revision: string | null;
|
||||
source_max_date: string | null;
|
||||
completed_at: string | null;
|
||||
} | null>;
|
||||
}
|
||||
|
||||
export type BacktestTargetModel = 'production_gtl' | 'structural_sr';
|
||||
export type BacktestCadence = 'weekly' | 'daily';
|
||||
|
||||
@@ -293,24 +280,6 @@ export function triggerJob(
|
||||
.then((r) => r.data);
|
||||
}
|
||||
|
||||
export function getFundamentalsParityReport() {
|
||||
return apiClient
|
||||
.get<FundamentalsParityReport | null>('admin/fundamentals-parity')
|
||||
.then((r) => r.data);
|
||||
}
|
||||
|
||||
export function getFundamentalsParityCsv() {
|
||||
return apiClient
|
||||
.get<{ filename: string; content: string } | null>('admin/fundamentals-parity/csv')
|
||||
.then((r) => r.data);
|
||||
}
|
||||
|
||||
export function getFundamentalsParityJson() {
|
||||
return apiClient
|
||||
.get<{ filename: string; content: string } | null>('admin/fundamentals-parity/json')
|
||||
.then((r) => r.data);
|
||||
}
|
||||
|
||||
// System events (operational warnings / errors)
|
||||
export interface SystemEvent {
|
||||
id: number;
|
||||
|
||||
@@ -6,7 +6,7 @@ import { useAuthStore } from '../stores/authStore';
|
||||
* Typed error class for API errors, providing structured error handling
|
||||
* across the application.
|
||||
*/
|
||||
export class ApiError extends Error {
|
||||
class ApiError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
|
||||
@@ -14,7 +14,7 @@ export interface FetchDataResult {
|
||||
}
|
||||
|
||||
/** Provider sources that cost an API call/quota. */
|
||||
export type FetchSource = 'ohlcv' | 'sentiment' | 'fundamentals';
|
||||
export type FetchSource = 'ohlcv' | 'sentiment';
|
||||
/** Source selector: omit → fetch all; array → those providers; 'recompute' → derived only (free). */
|
||||
export type FetchSelector = FetchSource[] | 'recompute';
|
||||
|
||||
|
||||
@@ -34,10 +34,6 @@ export interface EquityPoint {
|
||||
benchmark_pnl: number;
|
||||
}
|
||||
|
||||
export function getEquityCurve() {
|
||||
return apiClient.get<EquityPoint[]>('paper-trades/equity-curve').then((r) => r.data);
|
||||
}
|
||||
|
||||
export interface PerfPoint {
|
||||
date: string;
|
||||
manual_pnl: number;
|
||||
|
||||
@@ -21,7 +21,7 @@ const TRIGGERS: { key: TriggerKey; label: string; hint: string }[] = [
|
||||
{ key: 'sr_proximity_enabled', label: 'Watchlist S/R proximity', hint: 'a watched ticker nears a strong support/resistance' },
|
||||
{ key: 'score_drop_enabled', label: 'Score deterioration', hint: 'a watched ticker’s composite drops sharply' },
|
||||
{ key: 'digest_enabled', label: 'Daily digest', hint: 'end-of-day summary incl. open trades + trailing stops' },
|
||||
{ key: 'regime_quadrant_enabled', label: 'Regime quadrant change', hint: 'the regime monitor shifts quadrant (hysteresis + cooldown)' },
|
||||
{ key: 'regime_quadrant_enabled', label: 'Risk quadrant change', hint: 'the AI/Tech risk monitor shifts quadrant (hysteresis + cooldown)' },
|
||||
{ key: 'trade_closed_enabled', label: 'Trade closed', hint: 'a paper trade auto-closes (trailing/target/stop) — incl. losses' },
|
||||
];
|
||||
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
getFundamentalsParityCsv,
|
||||
getFundamentalsParityJson,
|
||||
} from '../../api/admin';
|
||||
import { useFundamentalsParityReport } from '../../hooks/useAdmin';
|
||||
import { SkeletonTable } from '../ui/Skeleton';
|
||||
|
||||
const FIELD_LABELS: Record<string, string> = {
|
||||
pe_ratio: 'P/E',
|
||||
revenue_growth: 'Revenue growth',
|
||||
earnings_surprise: 'Earnings surprise',
|
||||
};
|
||||
|
||||
function downloadText(filename: string, content: string, type: string) {
|
||||
const blob = new Blob([content], { type });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const anchor = document.createElement('a');
|
||||
anchor.href = url;
|
||||
anchor.download = filename;
|
||||
anchor.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
export function FundamentalsParityPanel() {
|
||||
const { data: report, isLoading, isError, error } = useFundamentalsParityReport();
|
||||
const [downloading, setDownloading] = useState(false);
|
||||
|
||||
if (isLoading) return <SkeletonTable rows={2} cols={4} />;
|
||||
if (isError) {
|
||||
return <p className="text-sm text-red-400">{(error as Error).message}</p>;
|
||||
}
|
||||
|
||||
if (!report) {
|
||||
return (
|
||||
<div className="glass p-5">
|
||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
No report yet. Trigger “Fundamentals Parity Report (read-only)” below.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const summary = report.summary;
|
||||
const generated = new Date(report.generated_at).toLocaleString();
|
||||
|
||||
async function downloadCsv() {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const artifact = await getFundamentalsParityCsv();
|
||||
if (artifact) downloadText(artifact.filename, artifact.content, 'text/csv;charset=utf-8');
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadJson() {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const artifact = await getFundamentalsParityJson();
|
||||
if (artifact) downloadText(artifact.filename, artifact.content, 'application/json;charset=utf-8');
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="glass p-5 space-y-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
||||
<span className="rounded-full border border-amber-400/20 bg-amber-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-amber-300">
|
||||
approval pending
|
||||
</span>
|
||||
<span className="rounded-full border border-cyan-400/20 bg-cyan-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-cyan-300">
|
||||
read-only
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Generated {generated} · as of {report.as_of_date} · {summary.universe_count} tracked tickers
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white"
|
||||
onClick={downloadJson}
|
||||
disabled={downloading}
|
||||
>
|
||||
Download JSON
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white disabled:opacity-50"
|
||||
onClick={downloadCsv}
|
||||
disabled={downloading}
|
||||
>
|
||||
{downloading ? 'Preparing…' : 'Download CSV'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<Summary label="Candidate score coverage" value={`${summary.candidate_fundamental_score_available}/${summary.universe_count}`} />
|
||||
<Summary label="Scores compared" value={summary.fundamental_scores_compared} />
|
||||
<Summary label="Material score moves" value={summary.fundamental_score_material_changes} />
|
||||
<Summary label="Fundamental rank moves" value={summary.fundamental_rank_changes} />
|
||||
</div>
|
||||
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-left text-xs">
|
||||
<thead className="text-[10px] uppercase tracking-wider text-gray-500">
|
||||
<tr>
|
||||
<th className="pb-2 pr-4 font-medium">Field</th>
|
||||
<th className="pb-2 px-3 font-medium">Legacy</th>
|
||||
<th className="pb-2 px-3 font-medium">Candidate</th>
|
||||
<th className="pb-2 px-3 font-medium">Compared</th>
|
||||
<th className="pb-2 px-3 font-medium">Material</th>
|
||||
<th className="pb-2 pl-3 font-medium">Median |Δ|</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-white/[0.06] text-gray-300">
|
||||
{Object.entries(summary.field_stats).map(([key, stats]) => (
|
||||
<tr key={key}>
|
||||
<td className="py-2.5 pr-4">{FIELD_LABELS[key] ?? key}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.legacy_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.candidate_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.both_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.material_differences}</td>
|
||||
<td className="py-2.5 pl-3 num">
|
||||
{stats.median_absolute_delta == null ? 'n/a' : stats.median_absolute_delta.toFixed(2)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<p className="text-[11px] leading-relaxed text-gray-500">
|
||||
Materiality bands highlight review candidates only. They do not approve a cutover or write fundamentals,
|
||||
scores, rankings, or qualification state.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Summary({ label, value }: { label: string; value: string | number }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-white/[0.07] bg-white/[0.025] px-3 py-2.5">
|
||||
<div className="text-[10px] uppercase tracking-wider text-gray-500">{label}</div>
|
||||
<div className="mt-1 num text-lg text-gray-200">{value}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useJobs, useToggleJob, useTriggerJob } from '../../hooks/useAdmin';
|
||||
import type { JobCategory, JobStatus } from '../../api/admin';
|
||||
import { SkeletonTable } from '../ui/Skeleton';
|
||||
|
||||
function formatNextRun(iso: string | null): string {
|
||||
@@ -10,7 +11,8 @@ function formatNextRun(iso: string | null): string {
|
||||
const mins = Math.round(diffMs / 60_000);
|
||||
if (mins < 60) return `in ${mins}m`;
|
||||
const hrs = Math.round(mins / 60);
|
||||
return `in ${hrs}h`;
|
||||
if (hrs < 48) return `in ${hrs}h`;
|
||||
return `in ${Math.round(hrs / 24)}d`;
|
||||
}
|
||||
|
||||
function formatAgo(iso: string | null | undefined): string {
|
||||
@@ -25,23 +27,270 @@ function formatAgo(iso: string | null | undefined): string {
|
||||
|
||||
function lastRunColor(status: string | null | undefined): string {
|
||||
if (status === 'error') return 'text-red-300';
|
||||
if (status === 'rate_limited') return 'text-amber-300';
|
||||
if (status === 'rate_limited' || status === 'deferred') return 'text-amber-300';
|
||||
return 'text-gray-500';
|
||||
}
|
||||
|
||||
/** The four kinds of job, in the order the API already sorts them. A job whose
|
||||
* category the client does not recognise still renders, under "Other" — better
|
||||
* a stray section than a job that silently vanishes from the admin page. */
|
||||
const SECTIONS: { key: JobCategory; title: string; hint: string }[] = [
|
||||
{
|
||||
key: 'pipeline',
|
||||
title: 'Pipelines',
|
||||
hint: 'own schedule · run their steps in order',
|
||||
},
|
||||
{
|
||||
key: 'pipeline_step',
|
||||
title: 'Pipeline steps',
|
||||
hint: 'no timer of their own · still triggerable individually',
|
||||
},
|
||||
{
|
||||
key: 'scheduled',
|
||||
title: 'Standalone scheduled',
|
||||
hint: 'own schedule · independent of any pipeline',
|
||||
},
|
||||
{ key: 'manual', title: 'Manual only', hint: 'never fires on its own' },
|
||||
];
|
||||
|
||||
/** One consistent answer per job: its own timer, its parent's, or "manual only".
|
||||
* A step has no schedule of its own, so reporting one was the original bug. */
|
||||
function NextRun({ job, labels }: { job: JobStatus; labels: Record<string, string> }) {
|
||||
const muted = 'text-[11px] text-gray-500';
|
||||
if (job.next_run_source === 'manual_only') {
|
||||
return <span className={muted}>manual only</span>;
|
||||
}
|
||||
if (job.next_run_source === 'via_pipeline') {
|
||||
if (!job.via_next_run_at || !job.via_next_run_job) {
|
||||
return <span className={muted}>runs via pipeline</span>;
|
||||
}
|
||||
return (
|
||||
<span className={muted}>
|
||||
Next via {labels[job.via_next_run_job] ?? job.via_next_run_job}{' '}
|
||||
{formatNextRun(job.via_next_run_at)}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (!job.next_run_at) return null;
|
||||
return <span className={muted}>Next run {formatNextRun(job.next_run_at)}</span>;
|
||||
}
|
||||
|
||||
/** Membership, shown rather than nested: a step can belong to several pipelines
|
||||
* (data_collector is in all four), so duplicating rows under each parent would
|
||||
* render Trigger buttons that are not distinct actions. */
|
||||
function Membership({ job, labels }: { job: JobStatus; labels: Record<string, string> }) {
|
||||
const name = (id: string) => labels[id] ?? id;
|
||||
if (job.category === 'pipeline' && job.steps?.length) {
|
||||
return (
|
||||
<div className="mt-1 text-[11px] leading-relaxed text-gray-600">
|
||||
{job.steps.map(name).join(' → ')}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (job.category === 'pipeline_step' && job.pipelines?.length) {
|
||||
return (
|
||||
<div className="mt-1 text-[11px] leading-relaxed text-gray-600">
|
||||
runs in: {job.pipelines.map(name).join(', ')}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
interface JobCardProps {
|
||||
job: JobStatus;
|
||||
labels: Record<string, string>;
|
||||
anyJobRunning: boolean;
|
||||
runningJobLabel?: string;
|
||||
onToggle: (job: JobStatus) => void;
|
||||
onTrigger: (job: JobStatus) => void;
|
||||
togglePending: boolean;
|
||||
triggerPending: boolean;
|
||||
}
|
||||
|
||||
function JobCard({
|
||||
job,
|
||||
labels,
|
||||
anyJobRunning,
|
||||
runningJobLabel,
|
||||
onToggle,
|
||||
onTrigger,
|
||||
togglePending,
|
||||
triggerPending,
|
||||
}: JobCardProps) {
|
||||
return (
|
||||
<div className="glass p-4 glass-hover">
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
<div className="flex items-center gap-3">
|
||||
{/* Status dot */}
|
||||
<span
|
||||
className={`inline-block h-2.5 w-2.5 rounded-full shrink-0 ${
|
||||
job.running
|
||||
? 'bg-blue-400 shadow-lg shadow-blue-400/40'
|
||||
: job.enabled
|
||||
? 'bg-emerald-400 shadow-lg shadow-emerald-400/40'
|
||||
: 'bg-gray-500'
|
||||
}`}
|
||||
/>
|
||||
<div>
|
||||
<span className="text-sm font-medium text-gray-200">{job.label}</span>
|
||||
<div className="mt-0.5 flex flex-wrap items-center gap-3">
|
||||
{/* Live state only — a persisted error must not read as the
|
||||
current status forever, so this never consults last_run_*. */}
|
||||
<span
|
||||
className={`text-[11px] font-medium ${
|
||||
job.running
|
||||
? 'text-blue-300'
|
||||
: job.runtime_status === 'rate_limited' || job.runtime_status === 'deferred'
|
||||
? 'text-amber-300'
|
||||
: job.runtime_status === 'error'
|
||||
? 'text-red-300'
|
||||
: job.enabled
|
||||
? 'text-emerald-400'
|
||||
: 'text-gray-500'
|
||||
}`}
|
||||
>
|
||||
{job.running
|
||||
? 'Running'
|
||||
: job.runtime_status === 'rate_limited'
|
||||
? 'Paused (rate-limited)'
|
||||
: job.runtime_status === 'deferred'
|
||||
? 'Deferred (retrying)'
|
||||
: job.runtime_status === 'error'
|
||||
? 'Last run error'
|
||||
: job.enabled
|
||||
? 'Active'
|
||||
: 'Inactive'}
|
||||
</span>
|
||||
{job.enabled && <NextRun job={job} labels={labels} />}
|
||||
{!job.registered && (
|
||||
<span className="text-[11px] text-red-400">Not registered</span>
|
||||
)}
|
||||
</div>
|
||||
<Membership job={job} labels={labels} />
|
||||
{/* Persisted, so this survives a deploy — unlike runtime_* above. */}
|
||||
{!job.running && job.last_run_at && (
|
||||
<div className={`mt-1 text-[11px] ${lastRunColor(job.last_run_status)}`}>
|
||||
Last run {formatAgo(job.last_run_at)}
|
||||
{job.last_run_status ? ` · ${job.last_run_status}` : ''}
|
||||
{job.last_run_message ? ` — ${job.last_run_message}` : ''}
|
||||
</div>
|
||||
)}
|
||||
{!job.running && !job.last_run_at && (
|
||||
<div className="mt-1 text-[11px] text-gray-600">No run recorded yet</div>
|
||||
)}
|
||||
{job.running && (
|
||||
<div className="mt-2 space-y-1.5">
|
||||
<div className="flex items-center justify-between text-[11px] text-gray-400">
|
||||
<span>
|
||||
{job.runtime_processed ?? 0}
|
||||
{typeof job.runtime_total === 'number' ? ` / ${job.runtime_total}` : ''}
|
||||
{' '}processed
|
||||
</span>
|
||||
{typeof job.runtime_progress_pct === 'number' && (
|
||||
<span>{Math.max(0, Math.min(100, job.runtime_progress_pct)).toFixed(0)}%</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="h-1.5 w-56 overflow-hidden rounded-full bg-slate-700/80">
|
||||
<div
|
||||
className="h-full bg-blue-400 transition-all duration-500"
|
||||
style={{
|
||||
width: `${
|
||||
typeof job.runtime_progress_pct === 'number'
|
||||
? Math.max(5, Math.min(100, job.runtime_progress_pct))
|
||||
: 30
|
||||
}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{job.runtime_current_ticker && (
|
||||
<div className="text-[11px] text-gray-500">Current: {job.runtime_current_ticker}</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onToggle(job)}
|
||||
disabled={togglePending}
|
||||
className={`rounded-lg border px-3 py-1.5 text-xs transition-all duration-200 disabled:opacity-50 ${
|
||||
job.enabled
|
||||
? 'border-red-500/20 bg-red-500/10 text-red-400 hover:bg-red-500/20'
|
||||
: 'border-emerald-500/20 bg-emerald-500/10 text-emerald-400 hover:bg-emerald-500/20'
|
||||
}`}
|
||||
>
|
||||
{job.enabled ? 'Disable' : 'Enable'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onTrigger(job)}
|
||||
disabled={triggerPending || !job.enabled || anyJobRunning}
|
||||
className="btn-primary px-3 py-1.5 text-xs disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
<span>
|
||||
{job.running
|
||||
? 'Running…'
|
||||
: triggerPending
|
||||
? 'Triggering…'
|
||||
: anyJobRunning
|
||||
? 'Blocked'
|
||||
: 'Trigger Now'}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{anyJobRunning && !job.running && (
|
||||
<div className="mt-2 text-[11px] text-gray-500">
|
||||
Manual trigger blocked while {runningJobLabel ?? 'another job'} is running.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function JobControls() {
|
||||
const { data: jobs, isLoading } = useJobs();
|
||||
const toggleJob = useToggleJob();
|
||||
const triggerJob = useTriggerJob();
|
||||
const anyJobRunning = (jobs ?? []).some((job) => job.running);
|
||||
const runningJob = jobs?.find((job) => job.running);
|
||||
const pausedJob = jobs?.find((job) => !job.running && job.runtime_status === 'rate_limited');
|
||||
const runningJobLabel = runningJob?.label;
|
||||
const all = jobs ?? [];
|
||||
// Job id -> display label, so a step can name its parent pipeline.
|
||||
const labels = Object.fromEntries(all.map((job) => [job.name, job.label]));
|
||||
const anyJobRunning = all.some((job) => job.running);
|
||||
const runningJob = all.find((job) => job.running);
|
||||
const pausedJob = all.find((job) => !job.running && job.runtime_status === 'rate_limited');
|
||||
|
||||
if (isLoading) return <SkeletonTable rows={4} cols={3} />;
|
||||
|
||||
const known = new Set<string>(SECTIONS.map((s) => s.key));
|
||||
const groups: { key: string; title: string; hint: string; jobs: JobStatus[] }[] = [
|
||||
...SECTIONS.map((section) => ({
|
||||
...section,
|
||||
jobs: all.filter((job) => job.category === section.key),
|
||||
})),
|
||||
{
|
||||
key: 'other',
|
||||
title: 'Other',
|
||||
hint: 'uncategorised',
|
||||
jobs: all.filter((job) => !job.category || !known.has(job.category)),
|
||||
},
|
||||
];
|
||||
|
||||
const cardProps = {
|
||||
labels,
|
||||
anyJobRunning,
|
||||
runningJobLabel: runningJob?.label,
|
||||
onToggle: (job: JobStatus) =>
|
||||
toggleJob.mutate({ jobName: job.name, enabled: !job.enabled }),
|
||||
onTrigger: (job: JobStatus) => triggerJob.mutate(job.name),
|
||||
togglePending: toggleJob.isPending,
|
||||
triggerPending: triggerJob.isPending,
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="space-y-6">
|
||||
{runningJob && (
|
||||
<div className="rounded-xl border border-blue-400/30 bg-blue-500/10 px-4 py-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
@@ -60,7 +309,7 @@ export function JobControls() {
|
||||
: ''}
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-2 h-1.5 w-full rounded-full bg-slate-700/80 overflow-hidden">
|
||||
<div className="mt-2 h-1.5 w-full overflow-hidden rounded-full bg-slate-700/80">
|
||||
<div
|
||||
className="h-full bg-blue-400 transition-all duration-500"
|
||||
style={{
|
||||
@@ -78,9 +327,7 @@ export function JobControls() {
|
||||
</div>
|
||||
)}
|
||||
{runningJob.runtime_message && (
|
||||
<div className="mt-1 text-[11px] text-blue-100/80">
|
||||
{runningJob.runtime_message}
|
||||
</div>
|
||||
<div className="mt-1 text-[11px] text-blue-100/80">{runningJob.runtime_message}</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
@@ -106,136 +353,23 @@ export function JobControls() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{jobs?.map((job) => (
|
||||
<div key={job.name} className="glass p-4 glass-hover">
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
<div className="flex items-center gap-3">
|
||||
{/* Status dot */}
|
||||
<span
|
||||
className={`inline-block h-2.5 w-2.5 rounded-full shrink-0 ${
|
||||
job.running
|
||||
? 'bg-blue-400 shadow-lg shadow-blue-400/40'
|
||||
: job.enabled
|
||||
? 'bg-emerald-400 shadow-lg shadow-emerald-400/40'
|
||||
: 'bg-gray-500'
|
||||
}`}
|
||||
/>
|
||||
<div>
|
||||
<span className="text-sm font-medium text-gray-200">{job.label}</span>
|
||||
<div className="flex items-center gap-3 mt-0.5">
|
||||
<span
|
||||
className={`text-[11px] font-medium ${
|
||||
job.running
|
||||
? 'text-blue-300'
|
||||
: job.runtime_status === 'rate_limited'
|
||||
? 'text-amber-300'
|
||||
: job.runtime_status === 'error'
|
||||
? 'text-red-300'
|
||||
: job.enabled
|
||||
? 'text-emerald-400'
|
||||
: 'text-gray-500'
|
||||
}`}
|
||||
>
|
||||
{job.running
|
||||
? 'Running'
|
||||
: job.runtime_status === 'rate_limited'
|
||||
? 'Paused (rate-limited)'
|
||||
: job.runtime_status === 'error'
|
||||
? 'Last run error'
|
||||
: job.enabled
|
||||
? 'Active'
|
||||
: 'Inactive'}
|
||||
</span>
|
||||
{job.via_pipeline ? (
|
||||
<span className="text-[11px] text-gray-500">runs via pipeline</span>
|
||||
) : (
|
||||
job.enabled && job.next_run_at && (
|
||||
<span className="text-[11px] text-gray-500">
|
||||
Next run {formatNextRun(job.next_run_at)}
|
||||
</span>
|
||||
)
|
||||
)}
|
||||
{!job.registered && (
|
||||
<span className="text-[11px] text-red-400">Not registered</span>
|
||||
)}
|
||||
</div>
|
||||
{!job.running && job.runtime_finished_at && (
|
||||
<div className={`mt-1 text-[11px] ${lastRunColor(job.runtime_status)}`}>
|
||||
Last run {formatAgo(job.runtime_finished_at)}
|
||||
{job.runtime_status ? ` · ${job.runtime_status}` : ''}
|
||||
{job.runtime_message ? ` — ${job.runtime_message}` : ''}
|
||||
</div>
|
||||
)}
|
||||
{job.running && (
|
||||
<div className="mt-2 space-y-1.5">
|
||||
<div className="flex items-center justify-between text-[11px] text-gray-400">
|
||||
<span>
|
||||
{job.runtime_processed ?? 0}
|
||||
{typeof job.runtime_total === 'number' ? ` / ${job.runtime_total}` : ''}
|
||||
{' '}processed
|
||||
</span>
|
||||
{typeof job.runtime_progress_pct === 'number' && (
|
||||
<span>{Math.max(0, Math.min(100, job.runtime_progress_pct)).toFixed(0)}%</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="h-1.5 w-56 rounded-full bg-slate-700/80 overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-blue-400 transition-all duration-500"
|
||||
style={{
|
||||
width: `${
|
||||
typeof job.runtime_progress_pct === 'number'
|
||||
? Math.max(5, Math.min(100, job.runtime_progress_pct))
|
||||
: 30
|
||||
}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{job.runtime_current_ticker && (
|
||||
<div className="text-[11px] text-gray-500">Current: {job.runtime_current_ticker}</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleJob.mutate({ jobName: job.name, enabled: !job.enabled })}
|
||||
disabled={toggleJob.isPending}
|
||||
className={`rounded-lg border px-3 py-1.5 text-xs transition-all duration-200 disabled:opacity-50 ${
|
||||
job.enabled
|
||||
? 'border-red-500/20 bg-red-500/10 text-red-400 hover:bg-red-500/20'
|
||||
: 'border-emerald-500/20 bg-emerald-500/10 text-emerald-400 hover:bg-emerald-500/20'
|
||||
}`}
|
||||
>
|
||||
{job.enabled ? 'Disable' : 'Enable'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => triggerJob.mutate(job.name)}
|
||||
disabled={triggerJob.isPending || !job.enabled || anyJobRunning}
|
||||
className="btn-primary px-3 py-1.5 text-xs disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
<span>
|
||||
{job.running
|
||||
? 'Running…'
|
||||
: triggerJob.isPending
|
||||
? 'Triggering…'
|
||||
: anyJobRunning
|
||||
? 'Blocked'
|
||||
: 'Trigger Now'}
|
||||
{groups.map(
|
||||
(group) =>
|
||||
group.jobs.length > 0 && (
|
||||
<section key={group.key} className="space-y-3">
|
||||
<h3 className="text-xs font-medium uppercase tracking-widest text-gray-500">
|
||||
{group.title}
|
||||
<span className="ml-2 num text-gray-600">{group.jobs.length}</span>
|
||||
<span className="ml-2 normal-case tracking-normal text-gray-600">
|
||||
{group.hint}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{anyJobRunning && !job.running && (
|
||||
<div className="mt-2 text-[11px] text-gray-500">
|
||||
Manual trigger blocked while {runningJobLabel ?? 'another job'} is running.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</h3>
|
||||
{group.jobs.map((job) => (
|
||||
<JobCard key={job.name} job={job} {...cardProps} />
|
||||
))}
|
||||
</section>
|
||||
),
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ const DEFAULTS: ScheduleConfig = {
|
||||
schedule_daily_pipeline_cron: '0 2 * * *',
|
||||
schedule_dolt_earnings_cron: '30 2 * * *',
|
||||
schedule_sec_fundamentals_cron: '0 4 * * *',
|
||||
schedule_fundamentals_parity_cron: '30 5 * * *',
|
||||
schedule_near_close_pipeline_cron: '30 15 * * mon-fri',
|
||||
schedule_after_close_pipeline_cron: '45 16 * * mon-fri',
|
||||
schedule_intraday_pipeline_cron: '0 10-15 * * mon-fri',
|
||||
schedule_fundamentals_cron: '0 1 * * mon',
|
||||
schedule_backtest_cron: '0 3 * * sun',
|
||||
schedule_ticker_universe_cron: '0 1 * * *',
|
||||
};
|
||||
|
||||
const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: boolean }[] = [
|
||||
@@ -24,25 +24,19 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
||||
{
|
||||
key: 'schedule_daily_pipeline_cron',
|
||||
label: 'Morning pipeline',
|
||||
hint: 'OHLCV → benchmark → sentiment → regime → alerts (no R:R scan). Default 02:00 ET so regime-quadrant changes hit Telegram in the morning.',
|
||||
hint: 'OHLCV → benchmark → sentiment → trend/risk → alerts (no R:R scan). Default 02:00 ET so risk-quadrant changes hit Telegram in the morning.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_dolt_earnings_cron',
|
||||
label: 'Dolt earnings (shadow)',
|
||||
hint: 'Pull and import earnings dates/results daily at 02:30 ET. Live scoring remains untouched before A5.',
|
||||
label: 'Dolt earnings',
|
||||
hint: 'Pull and import earnings dates/results daily at 02:30 ET. The fundamentals cache refresh uses these local events.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_sec_fundamentals_cron',
|
||||
label: 'SEC fundamentals (shadow)',
|
||||
hint: 'Import tracked-universe SEC facts daily at 04:00 ET. Unchanged revisions become no-op runs.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_fundamentals_parity_cron',
|
||||
label: 'Fundamentals parity report',
|
||||
hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the shadow imports.',
|
||||
label: 'SEC fundamentals',
|
||||
hint: 'Import tracked-universe SEC facts daily at 04:00 ET, then refresh the fundamentals cache scoring reads. Disabling the job stops the SEC fetch only — the local cache refresh still runs.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
@@ -64,9 +58,15 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_fundamentals_cron',
|
||||
label: 'Legacy fundamentals (weekly)',
|
||||
hint: 'Existing provider chain retained until the A5 parity approval and A6 removal.',
|
||||
key: 'schedule_backtest_cron',
|
||||
label: 'Backtest',
|
||||
hint: 'Replay history and refresh the Track Record report. Default Sunday 03:00 ET. Was a 168h interval, which restarted on every deploy and so could defer indefinitely.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_ticker_universe_cron',
|
||||
label: 'Ticker universe sync',
|
||||
hint: 'Refresh the tracked-symbol universe. Default 01:00 ET daily, before the morning pipeline.',
|
||||
mono: true,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -378,16 +378,15 @@ export function TradeChart({
|
||||
// it wanders left as more post-entry bars arrive.
|
||||
const WINDOW = 21;
|
||||
const MID = 10;
|
||||
let start: number;
|
||||
let entryIdx: number;
|
||||
if (postCount <= MID + 1) {
|
||||
start = Math.max(0, entryAbs - MID);
|
||||
entryIdx = entryAbs - start;
|
||||
} else {
|
||||
const start = postCount <= MID + 1
|
||||
? Math.max(0, entryAbs - MID)
|
||||
// Enough history: keep the latest WINDOW bars; entry falls where it falls.
|
||||
start = Math.max(0, bars.length - WINDOW);
|
||||
entryIdx = entryAbs - start;
|
||||
}
|
||||
: Math.max(0, bars.length - WINDOW);
|
||||
// A trade older than the window entered before the first visible bar. Clamp to
|
||||
// the left edge — a negative index reads past the start of `series`/`stopPath`
|
||||
// and NaNs out the price and trail paths entirely.
|
||||
const entryBeforeWindow = entryAbs < start;
|
||||
const entryIdx = Math.max(0, entryAbs - start);
|
||||
const windowBars = bars.slice(start);
|
||||
const series = windowBars.map((b) => b.close);
|
||||
if (series.length < 2) return null;
|
||||
@@ -601,7 +600,11 @@ export function TradeChart({
|
||||
{entryIdx === lastIdx && (
|
||||
<circle cx={px(entryIdx)} cy={py(series[entryIdx])} r="2" fill="var(--ink-3)" />
|
||||
)}
|
||||
<circle cx={px(entryIdx)} cy={py(entry)} r="3.5" fill="var(--ink-2)" stroke="var(--surface)" strokeWidth="1.5" />
|
||||
{/* Entry marker only when the entry bar is actually in the window — for an
|
||||
older trade the entry level line carries it instead. */}
|
||||
{!entryBeforeWindow && (
|
||||
<circle cx={px(entryIdx)} cy={py(entry)} r="3.5" fill="var(--ink-2)" stroke="var(--surface)" strokeWidth="1.5" />
|
||||
)}
|
||||
<circle cx={px(lastIdx)} cy={py(series[lastIdx])} r="4" fill={nowCol} stroke="var(--surface)" strokeWidth="2" />
|
||||
</svg>
|
||||
);
|
||||
|
||||
@@ -22,6 +22,63 @@ function pnlColor(v: number): string {
|
||||
return 'text-gray-300';
|
||||
}
|
||||
|
||||
function maxHoldText(trade: PaperTrade): string | null {
|
||||
const remaining = trade.sessions_remaining;
|
||||
if (remaining == null) return null;
|
||||
const held = trade.sessions_held ?? 0;
|
||||
if (remaining < 0) return `${held} held · past max hold`;
|
||||
if (remaining === 0) return `${held} held · max hold reached`;
|
||||
return `${held} held · ${remaining} remaining`;
|
||||
}
|
||||
|
||||
function maxHoldColor(trade: PaperTrade): string {
|
||||
const remaining = trade.sessions_remaining;
|
||||
if (remaining == null) return 'text-gray-400';
|
||||
const holdDays = Math.max(1, (trade.sessions_held ?? 0) + remaining);
|
||||
const warningAt = Math.max(1, Math.ceil(holdDays * 0.2));
|
||||
return remaining <= warningAt ? 'text-amber-300' : 'text-gray-400';
|
||||
}
|
||||
|
||||
/** Quiet secondary telemetry below the R bar. Exact timing stays in the
|
||||
* expanded row; this only communicates how far through max hold the trade is. */
|
||||
function HoldProgress({ trade }: { trade: PaperTrade }) {
|
||||
const held = trade.sessions_held;
|
||||
const remaining = trade.sessions_remaining;
|
||||
if (held == null || remaining == null) return null;
|
||||
|
||||
const total = Math.max(1, held + Math.max(0, remaining));
|
||||
const elapsedPct = remaining <= 0
|
||||
? 100
|
||||
: Math.min(100, Math.max(0, (held / total) * 100));
|
||||
const warningAt = Math.max(1, Math.ceil(total * 0.2));
|
||||
const urgent = remaining <= warningAt;
|
||||
const color = urgent ? 'bg-amber-400/75' : 'bg-sky-400/40';
|
||||
|
||||
return (
|
||||
<div
|
||||
className="relative h-[3px] rounded-full bg-white/[0.06]"
|
||||
role="progressbar"
|
||||
aria-label="Holding period"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={total}
|
||||
aria-valuenow={Math.min(held, total)}
|
||||
aria-valuetext={remaining < 0
|
||||
? `${held} sessions held, past maximum hold`
|
||||
: `${held} sessions held, ${remaining} remaining`}
|
||||
title="Holding-period progress — click for the exact session count"
|
||||
>
|
||||
<span
|
||||
className={`absolute inset-y-0 left-0 rounded-full ${color}`}
|
||||
style={{ width: `${elapsedPct}%` }}
|
||||
/>
|
||||
<span
|
||||
className={`absolute top-1/2 h-[5px] w-[2px] -translate-x-1/2 -translate-y-1/2 rounded-full ${color}`}
|
||||
style={{ left: `${elapsedPct}%` }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DirTag({ direction }: { direction: string }) {
|
||||
const isLong = direction === 'long';
|
||||
return (
|
||||
@@ -46,10 +103,22 @@ function Detail({ label, value, valueClass = 'text-gray-100' }: {
|
||||
);
|
||||
}
|
||||
|
||||
function Fact({ label, value, valueClass = 'text-gray-300' }: {
|
||||
label: string;
|
||||
value: ReactNode;
|
||||
valueClass?: string;
|
||||
}) {
|
||||
return (
|
||||
<span className="num inline-flex items-baseline gap-1.5 whitespace-nowrap">
|
||||
<span className="text-[9px] uppercase tracking-[0.14em] text-gray-600">{label}</span>
|
||||
<span className={`text-[11px] ${valueClass}`}>{value}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/** Expanded row: full trade detail + price chart with entry / trail path. */
|
||||
function TradeDetail({ trade, exitLabel, exitMode, atrMultiplier, trailingPct, onClose, closing }: {
|
||||
function TradeDetail({ trade, exitMode, atrMultiplier, trailingPct, onClose, closing }: {
|
||||
trade: PaperTrade;
|
||||
exitLabel: string | null;
|
||||
exitMode: 'time' | 'trailing' | 'atr_trailing' | 'target';
|
||||
atrMultiplier: number;
|
||||
trailingPct: number;
|
||||
@@ -66,30 +135,28 @@ function TradeDetail({ trade, exitLabel, exitMode, atrMultiplier, trailingPct, o
|
||||
staleTime: 5 * 60_000,
|
||||
});
|
||||
const opened = new Date(trade.opened_at).toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
|
||||
const holdText = maxHoldText(trade);
|
||||
const exitRuleText = exitMode === 'atr_trailing'
|
||||
? `${atrMultiplier.toFixed(1)}× ATR trail`
|
||||
: exitMode === 'trailing'
|
||||
? `${Math.round(trailingPct)}% trailing stop`
|
||||
: exitMode === 'target'
|
||||
? 'target / stop'
|
||||
: null;
|
||||
const chartHint = trailMoved || exitMode === 'atr_trailing' || exitMode === 'trailing'
|
||||
? 'entry · now · stop · trail · gate'
|
||||
: 'entry · now · stop · gate';
|
||||
return (
|
||||
<div className="flex flex-col gap-4 px-2 pb-4 pt-1">
|
||||
<dl className="grid grid-cols-2 gap-x-8 gap-y-3 sm:grid-cols-4">
|
||||
<Detail label="opened" value={`${opened} · ${trade.shares} shares`} />
|
||||
<Detail label="entry → now" value={
|
||||
`${formatPrice(trade.entry_price)} → ${trade.current_price != null ? formatPrice(trade.current_price) : '—'}`
|
||||
} />
|
||||
<dl className="grid grid-cols-2 gap-x-8 gap-y-3 md:grid-cols-4 xl:grid-cols-2">
|
||||
<Detail
|
||||
label="P&L"
|
||||
value={p ? `${money(p.pnl)} · ${p.pct >= 0 ? '+' : ''}${p.pct.toFixed(1)}%` : '—'}
|
||||
valueClass={p ? pnlColor(p.pnl) : 'text-gray-500'}
|
||||
/>
|
||||
<Detail
|
||||
label="alpha vs SPY"
|
||||
value={
|
||||
trade.alpha_pct != null
|
||||
? `${trade.alpha_pct >= 0 ? '+' : ''}${trade.alpha_pct.toFixed(1)}%${trade.alpha_usd != null ? ` · ${money(trade.alpha_usd)}` : ''}`
|
||||
: '—'
|
||||
}
|
||||
valueClass={trade.alpha_pct != null ? pnlColor(trade.alpha_pct) : 'text-gray-500'}
|
||||
/>
|
||||
<Detail label="entry → now" value={
|
||||
`${formatPrice(trade.entry_price)} → ${trade.current_price != null ? formatPrice(trade.current_price) : '—'}`
|
||||
} />
|
||||
<Detail
|
||||
label={trailMoved ? 'trail' : 'stop'}
|
||||
value={
|
||||
@@ -105,27 +172,36 @@ function TradeDetail({ trade, exitLabel, exitMode, atrMultiplier, trailingPct, o
|
||||
}
|
||||
/>
|
||||
<Detail
|
||||
label="target"
|
||||
label="alpha vs SPY"
|
||||
value={
|
||||
trade.alpha_pct != null
|
||||
? `${trade.alpha_pct >= 0 ? '+' : ''}${trade.alpha_pct.toFixed(1)}%${trade.alpha_usd != null ? ` · ${money(trade.alpha_usd)}` : ''}`
|
||||
: '—'
|
||||
}
|
||||
valueClass={trade.alpha_pct != null ? pnlColor(trade.alpha_pct) : 'text-gray-500'}
|
||||
/>
|
||||
</dl>
|
||||
<div className="flex flex-wrap items-center gap-x-5 gap-y-2 border-t border-white/[0.06] pt-3">
|
||||
<Fact label="position" value={`${trade.shares} shares`} />
|
||||
<Fact
|
||||
label="holding"
|
||||
value={
|
||||
<>
|
||||
{formatPrice(trade.target)}
|
||||
{exitMode !== 'target' && (
|
||||
<span className="ml-1.5 text-[10px] text-gray-500">screening only</span>
|
||||
)}
|
||||
opened {opened}
|
||||
{holdText && <span className={maxHoldColor(trade)}> · {holdText}</span>}
|
||||
</>
|
||||
}
|
||||
/>
|
||||
<Detail label="exit rule" value={exitLabel ?? 'target/stop'} />
|
||||
<div className="flex items-end">
|
||||
<button
|
||||
onClick={onClose}
|
||||
disabled={closing}
|
||||
className="rounded-md border border-white/[0.1] px-3 py-1.5 text-xs text-gray-300 transition-colors hover:bg-white/[0.06] hover:text-white disabled:opacity-50"
|
||||
>
|
||||
Sell at market
|
||||
</button>
|
||||
</div>
|
||||
</dl>
|
||||
<Fact label="screening target" value={formatPrice(trade.target)} />
|
||||
{exitRuleText && <Fact label="exit" value={exitRuleText} />}
|
||||
<button
|
||||
onClick={onClose}
|
||||
disabled={closing}
|
||||
className="ml-auto rounded-md border border-white/[0.1] px-3 py-1.5 text-[11px] text-gray-300 transition-colors hover:bg-white/[0.06] hover:text-white disabled:opacity-50"
|
||||
>
|
||||
Sell at market
|
||||
</button>
|
||||
</div>
|
||||
{ohlcv.data && (
|
||||
<div>
|
||||
<p className="num text-[9.5px] uppercase tracking-[0.16em] text-gray-500">
|
||||
@@ -173,13 +249,14 @@ export function OpenTradesPanel() {
|
||||
const trailingPct = policy?.trailing_pct ?? 12;
|
||||
const exitLabel = policy
|
||||
? policy.mode === 'atr_trailing'
|
||||
? `${atrMultiplier.toFixed(1)}x ATR trailing stop / ${policy.hold_days}d max`
|
||||
? `${atrMultiplier.toFixed(1)}x ATR trailing stop / ${policy.hold_days} sessions max`
|
||||
: policy.mode === 'trailing'
|
||||
? `trailing ${Math.round(trailingPct)}%`
|
||||
: policy.mode === 'time'
|
||||
? `${policy.hold_days}d hold`
|
||||
? `${policy.hold_days}-session hold`
|
||||
: 'target/stop'
|
||||
: null;
|
||||
const hasMaxHold = exitMode === 'atr_trailing' || exitMode === 'time';
|
||||
|
||||
const rows = trades ?? [];
|
||||
|
||||
@@ -245,7 +322,10 @@ export function OpenTradesPanel() {
|
||||
<span className="num hidden text-xs text-gray-400 sm:block">
|
||||
{formatPrice(t.entry_price)} → {t.current_price != null ? formatPrice(t.current_price) : '—'}
|
||||
</span>
|
||||
<RBar r={p?.r ?? null} max={rMax} />
|
||||
<div className={`min-w-0 ${hasMaxHold ? 'space-y-1.5' : ''}`}>
|
||||
<RBar r={p?.r ?? null} max={rMax} />
|
||||
{hasMaxHold && <HoldProgress trade={t} />}
|
||||
</div>
|
||||
<span className={`num text-right text-[13px] font-semibold ${p?.r != null ? pnlColor(p.r) : 'text-gray-500'}`}>
|
||||
{p?.r != null ? `${p.r >= 0 ? '+' : ''}${p.r.toFixed(2)}R` : '—'}
|
||||
</span>
|
||||
@@ -256,7 +336,6 @@ export function OpenTradesPanel() {
|
||||
{open && (
|
||||
<TradeDetail
|
||||
trade={t}
|
||||
exitLabel={exitLabel}
|
||||
exitMode={exitMode}
|
||||
atrMultiplier={atrMultiplier}
|
||||
trailingPct={trailingPct}
|
||||
|
||||
@@ -7,7 +7,7 @@ const navItems = [
|
||||
{ to: '/', label: 'Overview', end: true },
|
||||
{ to: '/market', label: 'Market', end: false },
|
||||
{ to: '/signals', label: 'Signals', end: false },
|
||||
{ to: '/regime', label: 'Regime', end: false },
|
||||
{ to: '/regime', label: 'Risk', end: false },
|
||||
];
|
||||
|
||||
export default function MobileNav() {
|
||||
|
||||
@@ -13,7 +13,8 @@ const navItems = [
|
||||
{ to: '/', label: 'Overview', end: true },
|
||||
{ to: '/market', label: 'Market', end: false },
|
||||
{ to: '/signals', label: 'Signals', end: false },
|
||||
{ to: '/regime', label: 'Regime', end: false },
|
||||
// Route stays /regime so existing links keep working; only the label changes.
|
||||
{ to: '/regime', label: 'Risk', end: false },
|
||||
];
|
||||
|
||||
const linkClasses = (isActive: boolean) =>
|
||||
@@ -84,7 +85,7 @@ export default function TopBar() {
|
||||
</div>
|
||||
|
||||
<div className="ml-auto flex items-center gap-5">
|
||||
{/* Market regime — ambient status; the full picture lives on /regime */}
|
||||
{/* SPY trend — ambient status; the full picture lives on /regime */}
|
||||
{regime.data && (
|
||||
<NavLink
|
||||
to="/regime"
|
||||
@@ -99,7 +100,7 @@ export default function TopBar() {
|
||||
>
|
||||
<span className={`inline-block h-1.5 w-1.5 rounded-full ${regimeDot(regime.data.label)}`} />
|
||||
<span className="text-[11px] capitalize text-gray-500 transition-colors group-hover:text-gray-300">
|
||||
{regime.data.label} regime
|
||||
{regime.data.label} trend
|
||||
</span>
|
||||
</NavLink>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
CartesianGrid,
|
||||
Line,
|
||||
LineChart,
|
||||
ReferenceArea,
|
||||
ReferenceLine,
|
||||
ResponsiveContainer,
|
||||
Scatter,
|
||||
ScatterChart,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
ZAxis,
|
||||
} from 'recharts';
|
||||
import { getRegimeHistory, getRegimeMonitor } from '../../api/regime';
|
||||
import { Callout } from '../ui/Callout';
|
||||
import { SkeletonCard } from '../ui/Skeleton';
|
||||
import { formatDate } from '../../lib/format';
|
||||
import { FUNDAMENTAL_VISUAL, QUADRANT_WASH, REGIME_VISUAL } from '../../lib/regime';
|
||||
import type { EvidenceQuality, FundamentalState } from '../../lib/types';
|
||||
|
||||
// Lazy-loaded (see RegimePage) so recharts stays in the regime-tab chunk.
|
||||
// Time and Path are two projections of one series, so they share a card and a
|
||||
// query rather than sitting in two panels that look like different data.
|
||||
|
||||
const VIEWS = ['Time', 'Path'] as const;
|
||||
type View = (typeof VIEWS)[number];
|
||||
|
||||
const RANGES = [
|
||||
{ key: '1M', days: 30 },
|
||||
{ key: '3M', days: 90 },
|
||||
{ key: '6M', days: 182 },
|
||||
{ key: 'All', days: Number.POSITIVE_INFINITY },
|
||||
] as const;
|
||||
type RangeKey = (typeof RANGES)[number]['key'];
|
||||
|
||||
/** Sessions drawn in Path view. The full series is unreadable as a path. */
|
||||
const PATH_TRAIL = 60;
|
||||
|
||||
const STATE_COLOR = REGIME_VISUAL.state;
|
||||
const WARNING_COLOR = REGIME_VISUAL.warning;
|
||||
const FUNDAMENTAL_SYMBOL: Record<FundamentalState, string> = {
|
||||
supportive: '▲',
|
||||
neutral: '●',
|
||||
adverse: '◆',
|
||||
unknown: '○',
|
||||
};
|
||||
|
||||
// Fall back to the shipped constants, not v2's shared 60/60, so a missing
|
||||
// quadrant_config cannot draw dividers that disagree with the alert path.
|
||||
const DEFAULT_STATE_DIVIDER = 50;
|
||||
const DEFAULT_WARNING_DIVIDER = 40;
|
||||
|
||||
interface PathPoint {
|
||||
x: number;
|
||||
y: number;
|
||||
date: string;
|
||||
/** The third channel as recorded that day. Colours the dot; never moves it. */
|
||||
fundamental: FundamentalState;
|
||||
evidence: EvidenceQuality;
|
||||
/** Raw dated observations are interactive dots; the smoothed copy is line-only. */
|
||||
raw: boolean;
|
||||
recency: number;
|
||||
}
|
||||
|
||||
/** Centered moving average to de-noise the path; today (last) kept exact. */
|
||||
function smoothTrail(points: PathPoint[], half = 2): PathPoint[] {
|
||||
const n = points.length;
|
||||
return points.map((p, i) => {
|
||||
if (i === n - 1) return { ...p, raw: false };
|
||||
let sx = 0;
|
||||
let sy = 0;
|
||||
let c = 0;
|
||||
for (let j = Math.max(0, i - half); j <= Math.min(n - 1, i + half); j++) {
|
||||
sx += points[j].x;
|
||||
sy += points[j].y;
|
||||
c += 1;
|
||||
}
|
||||
return { ...p, x: sx / c, y: sy / c, raw: false };
|
||||
});
|
||||
}
|
||||
|
||||
function FundamentalGlyph({
|
||||
cx,
|
||||
cy,
|
||||
state,
|
||||
size,
|
||||
opacity = 1,
|
||||
}: {
|
||||
cx: number;
|
||||
cy: number;
|
||||
state: FundamentalState;
|
||||
size: number;
|
||||
opacity?: number;
|
||||
}) {
|
||||
const visual = FUNDAMENTAL_VISUAL[state] ?? FUNDAMENTAL_VISUAL.unknown;
|
||||
const common = { fill: visual.color, opacity, stroke: '#11131c', strokeWidth: 1 };
|
||||
if (visual.glyph === 'up') {
|
||||
return <polygon points={`${cx},${cy - size} ${cx - size},${cy + size} ${cx + size},${cy + size}`} {...common} />;
|
||||
}
|
||||
if (visual.glyph === 'diamond') {
|
||||
return <polygon points={`${cx},${cy - size} ${cx - size},${cy} ${cx},${cy + size} ${cx + size},${cy}`} {...common} />;
|
||||
}
|
||||
if (visual.glyph === 'ring') {
|
||||
return <circle cx={cx} cy={cy} r={size - 0.5} fill="transparent" opacity={opacity} stroke={visual.color} strokeWidth={1.5} />;
|
||||
}
|
||||
return <circle cx={cx} cy={cy} r={size - 0.5} {...common} />;
|
||||
}
|
||||
|
||||
function PathPointShape({ cx = 0, cy = 0, payload }: { cx?: number; cy?: number; payload?: PathPoint }) {
|
||||
if (!payload) return <g />;
|
||||
return (
|
||||
<FundamentalGlyph
|
||||
cx={cx}
|
||||
cy={cy}
|
||||
state={payload.fundamental}
|
||||
size={3.25 + payload.recency * 1.25}
|
||||
opacity={0.58 + payload.recency * 0.42}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function LatestPointShape({ cx = 0, cy = 0, payload }: { cx?: number; cy?: number; payload?: PathPoint }) {
|
||||
if (!payload) return <g />;
|
||||
return (
|
||||
<g>
|
||||
<circle cx={cx} cy={cy} r={7} fill="transparent" stroke="#ffffff" strokeWidth={1.75} />
|
||||
<FundamentalGlyph cx={cx} cy={cy} state={payload.fundamental} size={4.5} />
|
||||
</g>
|
||||
);
|
||||
}
|
||||
|
||||
function SegmentedControl<T extends string>({
|
||||
options,
|
||||
value,
|
||||
onChange,
|
||||
label,
|
||||
}: {
|
||||
options: readonly T[];
|
||||
value: T;
|
||||
onChange: (next: T) => void;
|
||||
label: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex gap-1" role="group" aria-label={label}>
|
||||
{options.map((option) => (
|
||||
<button
|
||||
key={option}
|
||||
type="button"
|
||||
aria-pressed={value === option}
|
||||
onClick={() => onChange(option)}
|
||||
className={`min-h-9 rounded px-3 py-2 text-xs font-medium tabular-nums transition-colors ${
|
||||
value === option ? 'bg-white/10 text-blue-300' : 'text-gray-400 hover:text-gray-200'
|
||||
}`}
|
||||
>
|
||||
{option}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function PathTip({ active, payload }: { active?: boolean; payload?: { payload: PathPoint }[] }) {
|
||||
if (!active || !payload?.length) return null;
|
||||
const p = payload.find((item) => item.payload.raw)?.payload ?? payload[0].payload;
|
||||
const visual = FUNDAMENTAL_VISUAL[p.fundamental] ?? FUNDAMENTAL_VISUAL.unknown;
|
||||
const evidence = p.evidence === 'unavailable' ? 'Unavailable' : `${p.evidence.replace(/_/g, ' ')} evidence`;
|
||||
return (
|
||||
<div className="glass px-3 py-2 text-xs">
|
||||
<div className="text-gray-300">{formatDate(p.date)}</div>
|
||||
<div className="text-gray-400">
|
||||
State <span style={{ color: STATE_COLOR }}>{Math.round(p.x)}</span> · Warning{' '}
|
||||
<span style={{ color: WARNING_COLOR }}>{Math.round(p.y)}</span>
|
||||
</div>
|
||||
<div className="mt-0.5 text-gray-400">
|
||||
Fundamentals <span style={{ color: visual.color }}>{visual.label}</span> · {evidence}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function RegimeChart() {
|
||||
const [view, setView] = useState<View>('Time');
|
||||
const [range, setRange] = useState<RangeKey>('3M');
|
||||
const history = useQuery({ queryKey: ['regime', 'history'], queryFn: () => getRegimeHistory(800) });
|
||||
const monitor = useQuery({ queryKey: ['regime', 'monitor'], queryFn: getRegimeMonitor });
|
||||
|
||||
const xDiv = monitor.data?.quadrant_config?.state_divider ?? DEFAULT_STATE_DIVIDER;
|
||||
const yDiv = monitor.data?.quadrant_config?.warning_divider ?? DEFAULT_WARNING_DIVIDER;
|
||||
const basketAsOf = monitor.data?.basket?.basket_asof;
|
||||
|
||||
const series = useMemo(() => {
|
||||
const data = history.data ?? [];
|
||||
if (view === 'Path') {
|
||||
return data
|
||||
.filter((p) => p.state != null && p.warning != null)
|
||||
.slice(-PATH_TRAIL);
|
||||
}
|
||||
const days = RANGES.find((r) => r.key === range)!.days;
|
||||
if (!Number.isFinite(days)) return data;
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - days);
|
||||
return data.filter((p) => new Date(p.date) >= cutoff);
|
||||
}, [history.data, view, range]);
|
||||
|
||||
const pathPoints = useMemo<PathPoint[]>(
|
||||
() => series.map((p, index, points) => ({
|
||||
x: p.state as number,
|
||||
y: p.warning as number,
|
||||
date: p.date,
|
||||
fundamental: p.fundamental_state ?? 'unknown',
|
||||
evidence: p.evidence_quality ?? 'unavailable',
|
||||
raw: true,
|
||||
recency: points.length <= 1 ? 1 : index / (points.length - 1),
|
||||
})),
|
||||
[series],
|
||||
);
|
||||
const trail = useMemo(() => (view === 'Path' ? smoothTrail(pathPoints) : []), [pathPoints, view]);
|
||||
const latest = view === 'Path' && pathPoints.length ? pathPoints[pathPoints.length - 1] : null;
|
||||
|
||||
// Only warn about pre-freeze history when the drawn window actually reaches
|
||||
// back past the freeze date.
|
||||
const crossesFreeze = Boolean(basketAsOf && series.length && series[0].date < basketAsOf);
|
||||
const enoughData = view === 'Path' ? pathPoints.length > 0 : series.length >= 2;
|
||||
|
||||
return (
|
||||
<div className="glass p-5">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-xs uppercase tracking-wider text-gray-400">
|
||||
{view === 'Time' ? 'State & Warning over time' : `State × Warning path · last ${PATH_TRAIL} sessions`}
|
||||
</span>
|
||||
<SegmentedControl options={VIEWS} value={view} onChange={setView} label="Chart view" />
|
||||
</div>
|
||||
{view === 'Time' ? (
|
||||
<SegmentedControl options={RANGES.map((r) => r.key)} value={range} onChange={setRange} label="Time range" />
|
||||
) : (
|
||||
latest && (
|
||||
<span className="text-xs text-gray-400">
|
||||
now: State <span style={{ color: STATE_COLOR }}>{Math.round(latest.x)}</span> · Warning{' '}
|
||||
<span style={{ color: WARNING_COLOR }}>{Math.round(latest.y)}</span>
|
||||
</span>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
|
||||
{history.isLoading ? (
|
||||
<SkeletonCard className="mt-3 h-72" />
|
||||
) : !enoughData ? (
|
||||
<Callout variant="empty">Not enough coverage-qualified history yet — it accumulates as the daily job runs.</Callout>
|
||||
) : (
|
||||
<>
|
||||
<div
|
||||
className="mt-3 h-72"
|
||||
role="img"
|
||||
aria-label={view === 'Time' ? 'State and Warning scores over time' : 'State by Warning path with fundamental context symbols'}
|
||||
>
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
{view === 'Time' ? (
|
||||
<LineChart data={series} margin={{ top: 6, right: 8, left: 0, bottom: 0 }}>
|
||||
<CartesianGrid stroke="rgba(255,255,255,0.05)" vertical={false} />
|
||||
<XAxis
|
||||
dataKey="date"
|
||||
tick={{ fill: '#9aa0b0', fontSize: 10 }}
|
||||
tickFormatter={(d) => formatDate(String(d))}
|
||||
minTickGap={28}
|
||||
tickLine={false}
|
||||
axisLine={{ stroke: 'rgba(255,255,255,0.08)' }}
|
||||
/>
|
||||
{/* width must clear a 3-digit label: the old chart paired
|
||||
width 28 with margin.left -18 and clipped every tick. */}
|
||||
<YAxis
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 25, 50, 75, 100]}
|
||||
tick={{ fill: '#9aa0b0', fontSize: 10 }}
|
||||
width={34}
|
||||
tickLine={false}
|
||||
axisLine={false}
|
||||
/>
|
||||
{/* The two axes have different thresholds, so each divider is
|
||||
drawn in its series' colour rather than as shared gridlines. */}
|
||||
<ReferenceLine y={xDiv} stroke={STATE_COLOR} strokeOpacity={0.25} strokeDasharray="4 4" />
|
||||
<ReferenceLine y={yDiv} stroke={WARNING_COLOR} strokeOpacity={0.25} strokeDasharray="4 4" />
|
||||
<Tooltip
|
||||
contentStyle={{
|
||||
background: 'rgba(17,24,39,0.95)',
|
||||
border: '1px solid rgba(255,255,255,0.1)',
|
||||
borderRadius: 8,
|
||||
fontSize: 12,
|
||||
}}
|
||||
labelStyle={{ color: '#9ca3af' }}
|
||||
labelFormatter={(l) => formatDate(String(l))}
|
||||
formatter={(value) => (value == null ? '—' : Math.round(Number(value)))}
|
||||
/>
|
||||
<Line type="monotone" dataKey="state" name="State" stroke={STATE_COLOR} dot={false} strokeWidth={1.5} isAnimationActive={false} />
|
||||
<Line type="monotone" dataKey="warning" name="Warning" stroke={WARNING_COLOR} dot={false} strokeWidth={1.5} isAnimationActive={false} />
|
||||
</LineChart>
|
||||
) : (
|
||||
<ScatterChart margin={{ top: 10, right: 16, bottom: 22, left: 0 }}>
|
||||
{/* One neutral at four opacities: denser = more axes elevated.
|
||||
Hue here would collide with the fundamental glyphs drawn
|
||||
on top of it — see QUADRANT_WASH. */}
|
||||
<ReferenceArea x1={0} x2={xDiv} y1={yDiv} y2={100} fill="#ffffff" fillOpacity={QUADRANT_WASH.early_warning} stroke="none" />
|
||||
<ReferenceArea x1={xDiv} x2={100} y1={yDiv} y2={100} fill="#ffffff" fillOpacity={QUADRANT_WASH.active_stress} stroke="none" />
|
||||
<ReferenceArea x1={0} x2={xDiv} y1={0} y2={yDiv} fill="#ffffff" fillOpacity={QUADRANT_WASH.healthy} stroke="none" />
|
||||
<ReferenceArea x1={xDiv} x2={100} y1={0} y2={yDiv} fill="#ffffff" fillOpacity={QUADRANT_WASH.stabilizing} stroke="none" />
|
||||
<CartesianGrid stroke="rgba(255,255,255,0.04)" />
|
||||
<ReferenceLine x={xDiv} stroke="rgba(255,255,255,0.12)" />
|
||||
<ReferenceLine y={yDiv} stroke="rgba(255,255,255,0.12)" />
|
||||
<XAxis
|
||||
type="number"
|
||||
dataKey="x"
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 20, 40, 60, 80, 100]}
|
||||
tick={{ fill: '#9aa0b0', fontSize: 10 }}
|
||||
tickLine={false}
|
||||
axisLine={{ stroke: 'rgba(255,255,255,0.08)' }}
|
||||
label={{ value: 'State →', position: 'insideBottom', offset: -12, fill: '#9aa0b0', fontSize: 10 }}
|
||||
/>
|
||||
<YAxis
|
||||
type="number"
|
||||
dataKey="y"
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 20, 40, 60, 80, 100]}
|
||||
tick={{ fill: '#9aa0b0', fontSize: 10 }}
|
||||
width={30}
|
||||
tickLine={false}
|
||||
axisLine={false}
|
||||
label={{ value: 'Warning', angle: -90, position: 'insideLeft', fill: '#9aa0b0', fontSize: 10 }}
|
||||
/>
|
||||
<ZAxis range={[18, 18]} />
|
||||
<Tooltip cursor={{ strokeDasharray: '3 3', stroke: 'rgba(255,255,255,0.2)' }} content={<PathTip />} />
|
||||
<Scatter
|
||||
data={trail}
|
||||
line={{ stroke: 'rgba(255,255,255,0.18)', strokeWidth: 1.5 }}
|
||||
shape={(props: { cx?: number; cy?: number }) => <circle cx={props.cx} cy={props.cy} r={0} />}
|
||||
tooltipType="none"
|
||||
isAnimationActive={false}
|
||||
/>
|
||||
<Scatter data={pathPoints} shape={<PathPointShape />} isAnimationActive={false} />
|
||||
{latest && (
|
||||
<Scatter
|
||||
data={[latest]}
|
||||
isAnimationActive={false}
|
||||
shape={<LatestPointShape />}
|
||||
/>
|
||||
)}
|
||||
</ScatterChart>
|
||||
)}
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
|
||||
{view === 'Time' ? (
|
||||
<div className="mt-2 flex flex-wrap items-center gap-4 text-xs text-gray-400">
|
||||
<span className="flex items-center gap-1.5">
|
||||
<span className="inline-block h-2 w-3 rounded-sm" style={{ background: STATE_COLOR }} />
|
||||
State
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5">
|
||||
<span className="inline-block h-2 w-3 rounded-sm" style={{ background: WARNING_COLOR }} />
|
||||
Warning
|
||||
</span>
|
||||
<span className="text-gray-400">dashed = each axis's elevated threshold ({xDiv} / {yDiv})</span>
|
||||
</div>
|
||||
) : (
|
||||
<div className="mt-2 grid grid-cols-1 gap-x-4 gap-y-1 text-xs text-gray-400 sm:grid-cols-2">
|
||||
{/* Swatches, not coloured words: the quadrant names used the
|
||||
fundamental channel's colours, so "Stabilizing" was rendered in
|
||||
the adverse hue while meaning damage receding. */}
|
||||
{([
|
||||
['active_stress', 'Active stress', 'damaged and deteriorating'],
|
||||
['early_warning', 'Early warning', 'calm, fragility rising'],
|
||||
['stabilizing', 'Stabilizing', 'damage remains, warning lower'],
|
||||
['healthy', 'Healthy', 'calm, broadly supported'],
|
||||
] as const).map(([key, name, gloss]) => (
|
||||
<span key={key} className="flex items-center gap-1.5">
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className="inline-block h-3 w-3 shrink-0 rounded-sm border border-white/10"
|
||||
style={{ background: `rgba(255,255,255,${QUADRANT_WASH[key] * 4})` }}
|
||||
/>
|
||||
<span className="text-gray-300">{name}</span> — {gloss}
|
||||
</span>
|
||||
))}
|
||||
<span className="text-gray-400 sm:col-span-2">Raw dated points grow toward today; the connecting line is smoothed. White ring = today.</span>
|
||||
<span className="mt-1 flex flex-wrap items-center gap-x-3 gap-y-1 text-gray-400 sm:col-span-2">
|
||||
<span>symbol + colour = fundamentals:</span>
|
||||
{(['supportive', 'neutral', 'adverse', 'unknown'] as const).map((state) => (
|
||||
<span key={state} className="inline-flex items-center gap-1.5">
|
||||
<span aria-hidden="true" style={{ color: FUNDAMENTAL_VISUAL[state].color }}>{FUNDAMENTAL_SYMBOL[state]}</span>
|
||||
{FUNDAMENTAL_VISUAL[state].label}
|
||||
</span>
|
||||
))}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{crossesFreeze && (
|
||||
<p className="mt-2 text-xs text-gray-400">
|
||||
History before {basketAsOf} is reconstructed against today's basket — retrospective, not a live record.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
import { useMemo } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
ScatterChart,
|
||||
Scatter,
|
||||
Cell,
|
||||
XAxis,
|
||||
YAxis,
|
||||
ZAxis,
|
||||
CartesianGrid,
|
||||
Tooltip,
|
||||
ResponsiveContainer,
|
||||
ReferenceLine,
|
||||
ReferenceArea,
|
||||
} from 'recharts';
|
||||
import { getRegimeHistory, getRegimeMonitor } from '../../api/regime';
|
||||
import { Callout } from '../ui/Callout';
|
||||
import { SkeletonCard } from '../ui/Skeleton';
|
||||
|
||||
// Lazy-loaded (see RegimePage) so recharts stays in the regime-tab chunk.
|
||||
|
||||
// Quadrant boundaries come from the backend v2 methodology response.
|
||||
const TRAIL = 60; // sessions shown
|
||||
|
||||
interface QPoint {
|
||||
x: number;
|
||||
y: number;
|
||||
date: string;
|
||||
}
|
||||
|
||||
/** Centered moving average to de-noise the path; today (last) kept exact. */
|
||||
function smoothTrail(points: QPoint[], half = 2): QPoint[] {
|
||||
const n = points.length;
|
||||
return points.map((p, i) => {
|
||||
if (i === n - 1) return { ...p };
|
||||
let sx = 0;
|
||||
let sy = 0;
|
||||
let c = 0;
|
||||
for (let j = Math.max(0, i - half); j <= Math.min(n - 1, i + half); j++) {
|
||||
sx += points[j].x;
|
||||
sy += points[j].y;
|
||||
c += 1;
|
||||
}
|
||||
return { x: sx / c, y: sy / c, date: p.date };
|
||||
});
|
||||
}
|
||||
|
||||
/** Recency gradient: 0 = oldest (muted slate), 1 = newest (bright blue). */
|
||||
function recencyColor(t: number): string {
|
||||
const lerp = (a: number, b: number) => Math.round(a + (b - a) * t);
|
||||
const r = lerp(71, 96);
|
||||
const g = lerp(85, 165);
|
||||
const b = lerp(105, 250);
|
||||
const alpha = (0.3 + 0.7 * t).toFixed(2);
|
||||
return `rgba(${r}, ${g}, ${b}, ${alpha})`;
|
||||
}
|
||||
|
||||
function QuadrantTip({ active, payload }: { active?: boolean; payload?: { payload: QPoint }[] }) {
|
||||
if (!active || !payload?.length) return null;
|
||||
const p = payload[0].payload;
|
||||
return (
|
||||
<div className="glass px-2.5 py-1.5 text-[11px]">
|
||||
<div className="text-gray-300">{p.date}</div>
|
||||
<div className="text-gray-400">
|
||||
State <span className="text-blue-300">{Math.round(p.x)}</span> · Warning{' '}
|
||||
<span className="text-orange-300">{Math.round(p.y)}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function RegimeQuadrant() {
|
||||
const history = useQuery({ queryKey: ['regime', 'history'], queryFn: () => getRegimeHistory(800) });
|
||||
const monitor = useQuery({ queryKey: ['regime', 'monitor'], queryFn: getRegimeMonitor });
|
||||
const xDiv = monitor.data?.quadrant_config?.state_divider ?? 60;
|
||||
const yDiv = monitor.data?.quadrant_config?.warning_divider ?? 60;
|
||||
|
||||
const points = useMemo<QPoint[]>(() => {
|
||||
const data = history.data ?? [];
|
||||
return data
|
||||
.filter((p) => p.state != null && p.warning != null)
|
||||
.slice(-TRAIL)
|
||||
.map((p) => ({ x: p.state as number, y: p.warning as number, date: p.date }));
|
||||
}, [history.data]);
|
||||
|
||||
const trail = useMemo(() => smoothTrail(points), [points]);
|
||||
const latest = points.length ? points[points.length - 1] : null;
|
||||
|
||||
return (
|
||||
<div className="glass p-5">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="text-[11px] uppercase tracking-wider text-gray-500">
|
||||
State × Warning quadrant — last {TRAIL} sessions
|
||||
</div>
|
||||
{latest && (
|
||||
<div className="text-[11px] text-gray-500">
|
||||
now: State <span className="text-blue-300">{Math.round(latest.x)}</span> · Warning{' '}
|
||||
<span className="text-orange-300">{Math.round(latest.y)}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{history.isLoading ? (
|
||||
<SkeletonCard className="mt-3 h-72" />
|
||||
) : !points.length ? (
|
||||
<Callout variant="empty">
|
||||
Not enough coverage-qualified v2 history yet.
|
||||
</Callout>
|
||||
) : (
|
||||
<>
|
||||
<div className="mt-3 h-80">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<ScatterChart margin={{ top: 10, right: 16, bottom: 22, left: 0 }}>
|
||||
{/* Quadrant shading (drawn first, behind everything) */}
|
||||
<ReferenceArea x1={0} x2={xDiv} y1={yDiv} y2={100} fill="#f59e0b" fillOpacity={0.07} stroke="none" />
|
||||
<ReferenceArea x1={xDiv} x2={100} y1={yDiv} y2={100} fill="#f97316" fillOpacity={0.07} stroke="none" />
|
||||
<ReferenceArea x1={0} x2={xDiv} y1={0} y2={yDiv} fill="#10b981" fillOpacity={0.07} stroke="none" />
|
||||
<ReferenceArea x1={xDiv} x2={100} y1={0} y2={yDiv} fill="#ef4444" fillOpacity={0.08} stroke="none" />
|
||||
<CartesianGrid stroke="rgba(255,255,255,0.04)" />
|
||||
<ReferenceLine x={xDiv} stroke="rgba(255,255,255,0.12)" />
|
||||
<ReferenceLine y={yDiv} stroke="rgba(255,255,255,0.12)" />
|
||||
<XAxis
|
||||
type="number"
|
||||
dataKey="x"
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 20, 40, 60, 80, 100]}
|
||||
tick={{ fill: '#6b7280', fontSize: 10 }}
|
||||
tickLine={false}
|
||||
axisLine={{ stroke: 'rgba(255,255,255,0.08)' }}
|
||||
label={{ value: 'State →', position: 'insideBottom', offset: -12, fill: '#6b7280', fontSize: 10 }}
|
||||
/>
|
||||
<YAxis
|
||||
type="number"
|
||||
dataKey="y"
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 20, 40, 60, 80, 100]}
|
||||
tick={{ fill: '#6b7280', fontSize: 10 }}
|
||||
width={30}
|
||||
tickLine={false}
|
||||
axisLine={false}
|
||||
label={{ value: 'Warning', angle: -90, position: 'insideLeft', fill: '#6b7280', fontSize: 10 }}
|
||||
/>
|
||||
<ZAxis range={[13, 13]} />
|
||||
<Tooltip cursor={{ strokeDasharray: '3 3', stroke: 'rgba(255,255,255,0.2)' }} content={<QuadrantTip />} />
|
||||
{/* Smoothed trail with a recency gradient (old → new) */}
|
||||
<Scatter
|
||||
data={trail}
|
||||
line={{ stroke: 'rgba(96,165,250,0.18)', strokeWidth: 1.5 }}
|
||||
isAnimationActive={false}
|
||||
>
|
||||
{trail.map((_, i) => (
|
||||
<Cell key={i} fill={recencyColor(trail.length <= 1 ? 1 : i / (trail.length - 1))} />
|
||||
))}
|
||||
</Scatter>
|
||||
{/* Today */}
|
||||
{latest && (
|
||||
<Scatter
|
||||
data={[latest]}
|
||||
isAnimationActive={false}
|
||||
shape={(props: { cx?: number; cy?: number }) => (
|
||||
<circle cx={props.cx} cy={props.cy} r={6} fill="#ffffff" stroke="#60a5fa" strokeWidth={2} />
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</ScatterChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
|
||||
<div className="mt-2 grid grid-cols-1 gap-x-4 gap-y-1 text-[11px] text-gray-500 sm:grid-cols-2">
|
||||
<span><span className="text-amber-400">Early warning</span> — state calm, fragility rising</span>
|
||||
<span><span className="text-orange-400">Active stress</span> — damaged and deteriorating</span>
|
||||
<span><span className="text-emerald-400">Healthy</span> — calm and broadly supported</span>
|
||||
<span><span className="text-red-400">Stressed / stabilizing</span> — damage remains, warning lower</span>
|
||||
</div>
|
||||
<p className="mt-2 text-[11px] leading-relaxed text-gray-600">
|
||||
White dot = today; the trail fades from muted (older) to bright blue (newer) over the last {TRAIL}{' '}
|
||||
sessions, smoothed. The path matters more than a single point. Risk thermometer — not an entry, exit,
|
||||
or sizing signal.
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
import { useState, useMemo } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
LineChart,
|
||||
Line,
|
||||
XAxis,
|
||||
YAxis,
|
||||
CartesianGrid,
|
||||
Tooltip,
|
||||
ResponsiveContainer,
|
||||
ReferenceLine,
|
||||
} from 'recharts';
|
||||
import { getRegimeHistory } from '../../api/regime';
|
||||
import { Callout } from '../ui/Callout';
|
||||
import { SkeletonCard } from '../ui/Skeleton';
|
||||
import { formatDate } from '../../lib/format';
|
||||
|
||||
// Lazy-loaded (see RegimePage) so recharts only ships in the regime-tab chunk.
|
||||
|
||||
const HISTORY_RANGES = [
|
||||
{ key: '1M', days: 30 },
|
||||
{ key: '3M', days: 90 },
|
||||
{ key: '6M', days: 182 },
|
||||
{ key: 'All', days: 100000 },
|
||||
] as const;
|
||||
type HistoryRange = (typeof HISTORY_RANGES)[number]['key'];
|
||||
|
||||
const HISTORY_SERIES = [
|
||||
{ key: 'state', label: 'State', color: '#60a5fa' },
|
||||
{ key: 'warning', label: 'Warning', color: '#fb923c' },
|
||||
] as const;
|
||||
|
||||
export default function ScoreHistoryChart() {
|
||||
const [range, setRange] = useState<HistoryRange>('3M');
|
||||
const history = useQuery({ queryKey: ['regime', 'history'], queryFn: () => getRegimeHistory(800) });
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
const data = history.data ?? [];
|
||||
const days = HISTORY_RANGES.find((r) => r.key === range)!.days;
|
||||
if (range === 'All') return data;
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - days);
|
||||
return data.filter((p) => new Date(p.date) >= cutoff);
|
||||
}, [history.data, range]);
|
||||
|
||||
return (
|
||||
<div className="glass p-5">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="text-[11px] uppercase tracking-wider text-gray-500">Score history</div>
|
||||
<div className="flex gap-1">
|
||||
{HISTORY_RANGES.map((r) => (
|
||||
<button
|
||||
key={r.key}
|
||||
type="button"
|
||||
onClick={() => setRange(r.key)}
|
||||
className={`rounded px-2 py-1 text-[11px] font-medium tabular-nums transition-colors ${
|
||||
range === r.key ? 'bg-white/10 text-blue-300' : 'text-gray-500 hover:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
{r.key}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{history.isLoading ? (
|
||||
<SkeletonCard className="mt-3 h-56" />
|
||||
) : filtered.length < 2 ? (
|
||||
<Callout variant="empty">Not enough history yet — it accumulates as the daily job runs.</Callout>
|
||||
) : (
|
||||
<>
|
||||
<div className="mt-3 h-60">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<LineChart data={filtered} margin={{ top: 6, right: 8, left: -18, bottom: 0 }}>
|
||||
<CartesianGrid stroke="rgba(255,255,255,0.05)" vertical={false} />
|
||||
<XAxis
|
||||
dataKey="date"
|
||||
tick={{ fill: '#6b7280', fontSize: 10 }}
|
||||
tickFormatter={(d) => formatDate(String(d))}
|
||||
minTickGap={28}
|
||||
tickLine={false}
|
||||
axisLine={{ stroke: 'rgba(255,255,255,0.08)' }}
|
||||
/>
|
||||
<YAxis
|
||||
domain={[0, 100]}
|
||||
ticks={[0, 30, 60, 80, 100]}
|
||||
tick={{ fill: '#6b7280', fontSize: 10 }}
|
||||
width={28}
|
||||
tickLine={false}
|
||||
axisLine={false}
|
||||
/>
|
||||
<ReferenceLine y={30} stroke="rgba(255,255,255,0.06)" />
|
||||
<ReferenceLine y={60} stroke="rgba(255,255,255,0.06)" />
|
||||
<ReferenceLine y={80} stroke="rgba(255,255,255,0.06)" />
|
||||
<Tooltip
|
||||
contentStyle={{
|
||||
background: 'rgba(17,24,39,0.95)',
|
||||
border: '1px solid rgba(255,255,255,0.1)',
|
||||
borderRadius: 8,
|
||||
fontSize: 12,
|
||||
}}
|
||||
labelStyle={{ color: '#9ca3af' }}
|
||||
labelFormatter={(l) => formatDate(String(l))}
|
||||
formatter={(value) => (value == null ? '—' : Math.round(Number(value)))}
|
||||
/>
|
||||
{HISTORY_SERIES.map((s) => (
|
||||
<Line
|
||||
key={s.key}
|
||||
type="monotone"
|
||||
dataKey={s.key}
|
||||
name={s.label}
|
||||
stroke={s.color}
|
||||
dot={false}
|
||||
strokeWidth={1.5}
|
||||
isAnimationActive={false}
|
||||
/>
|
||||
))}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
<div className="mt-2 flex flex-wrap gap-4">
|
||||
{HISTORY_SERIES.map((s) => (
|
||||
<span key={s.key} className="flex items-center gap-1.5 text-[11px] text-gray-400">
|
||||
<span className="inline-block h-2 w-3 rounded-sm" style={{ background: s.color }} />
|
||||
{s.label}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -9,35 +9,8 @@ import { Disclosure } from '../ui/Disclosure';
|
||||
import { Dropdown } from '../ui/Dropdown';
|
||||
import { Section } from '../ui/Section';
|
||||
import { useToast } from '../ui/Toast';
|
||||
import type { BacktestCurvePoint, BacktestPortfolioMonitorRun } from '../../lib/types';
|
||||
|
||||
function fmtR(v: number | null | undefined): string {
|
||||
if (v === null || v === undefined) return '—';
|
||||
return `${v > 0 ? '+' : ''}${v.toFixed(2)}R`;
|
||||
}
|
||||
function fmtPct(v: number | null): string {
|
||||
return v === null ? '—' : `${v.toFixed(1)}%`;
|
||||
}
|
||||
function fmtMoney(v: number | null | undefined): string {
|
||||
if (v === null || v === undefined) return '—';
|
||||
return v.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
||||
}
|
||||
function fmtSignedPct(v: number | null | undefined): string {
|
||||
if (v === null || v === undefined) return '—';
|
||||
return `${v > 0 ? '+' : ''}${v.toFixed(1)}%`;
|
||||
}
|
||||
function fmtDrawdown(v: number | null | undefined): string {
|
||||
return v === null || v === undefined ? '—' : `-${Math.abs(v).toFixed(1)}%`;
|
||||
}
|
||||
function fmtDays(v: number | null | undefined): string {
|
||||
return v === null || v === undefined ? '—' : `${v.toFixed(1)}d`;
|
||||
}
|
||||
function rColor(v: number | null): string {
|
||||
if (v === null) return 'text-gray-400';
|
||||
if (v > 0) return 'text-emerald-400';
|
||||
if (v < 0) return 'text-red-400';
|
||||
return 'text-gray-300';
|
||||
}
|
||||
import { BacktestRecommendationCard } from './BacktestRecommendationCard';
|
||||
import { PortfolioMonitorPanel } from './PortfolioMonitorPanel';
|
||||
|
||||
function timeAgo(iso: string): string {
|
||||
const mins = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000);
|
||||
@@ -48,95 +21,14 @@ function timeAgo(iso: string): string {
|
||||
return `${Math.floor(hrs / 24)}d ago`;
|
||||
}
|
||||
|
||||
function Stat({ label, value, valueClass = 'text-gray-100', sub }: {
|
||||
label: string; value: string; valueClass?: string; sub?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="glass p-4">
|
||||
<p className="section-index">{label}</p>
|
||||
<p className={`num mt-1.5 text-2xl font-semibold ${valueClass}`}>{value}</p>
|
||||
{sub && <p className="mt-1 text-xs text-gray-500">{sub}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function curvePath(
|
||||
points: BacktestCurvePoint[],
|
||||
min: number,
|
||||
max: number,
|
||||
w: number,
|
||||
h: number,
|
||||
pad: number,
|
||||
startMs: number,
|
||||
endMs: number,
|
||||
): string {
|
||||
if (points.length < 2) return '';
|
||||
const span = Math.max(max - min, 1);
|
||||
const timeSpan = Math.max(endMs - startMs, 1);
|
||||
return points
|
||||
.map((p, i) => {
|
||||
const t = new Date(p.date).getTime();
|
||||
const x = pad + ((t - startMs) / timeSpan) * (w - pad * 2);
|
||||
const value = p.return_pct ?? 0;
|
||||
const y = pad + (1 - (value - min) / span) * (h - pad * 2);
|
||||
return `${i === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`;
|
||||
})
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
function EquityCurveChart({ run }: { run: BacktestPortfolioMonitorRun }) {
|
||||
const portfolio = run.equity_curve ?? [];
|
||||
const benchmark = run.benchmark_curve ?? [];
|
||||
const values = [...portfolio, ...benchmark]
|
||||
.map((p) => p.return_pct)
|
||||
.filter((v): v is number => v !== null && v !== undefined);
|
||||
if (portfolio.length < 2 || values.length === 0) {
|
||||
return <Callout variant="empty">No equity curve points for this selection.</Callout>;
|
||||
}
|
||||
|
||||
const min = Math.min(0, ...values);
|
||||
const max = Math.max(0, ...values);
|
||||
const times = [...portfolio, ...benchmark]
|
||||
.map((p) => new Date(p.date).getTime())
|
||||
.filter((v) => Number.isFinite(v));
|
||||
if (times.length === 0) {
|
||||
return <Callout variant="empty">No dated equity curve points for this selection.</Callout>;
|
||||
}
|
||||
const startMs = Math.min(...times);
|
||||
const endMs = Math.max(...times);
|
||||
const w = 720;
|
||||
const h = 240;
|
||||
const pad = 28;
|
||||
const portfolioPath = curvePath(portfolio, min, max, w, h, pad, startMs, endMs);
|
||||
const benchmarkPath = curvePath(benchmark, min, max, w, h, pad, startMs, endMs);
|
||||
const lastPortfolio = portfolio[portfolio.length - 1]?.return_pct ?? null;
|
||||
const lastBenchmark = benchmark[benchmark.length - 1]?.return_pct ?? run.spy_return_pct;
|
||||
|
||||
return (
|
||||
<div className="glass overflow-hidden">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-white/[0.05] px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-gray-100">{run.label}</p>
|
||||
<p className="text-[11px] text-gray-500">{run.start_date} - {run.end_date}</p>
|
||||
</div>
|
||||
<div className="flex gap-4 text-xs">
|
||||
<span className="text-blue-300">Portfolio {fmtSignedPct(lastPortfolio)}</span>
|
||||
<span className="text-gray-400">S&P 500 {fmtSignedPct(lastBenchmark)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<svg viewBox={`0 0 ${w} ${h}`} className="h-64 w-full" role="img" aria-label="Portfolio return compared with S&P 500">
|
||||
<line x1={pad} y1={h - pad} x2={w - pad} y2={h - pad} stroke="rgba(255,255,255,0.12)" />
|
||||
<line x1={pad} y1={pad} x2={pad} y2={h - pad} stroke="rgba(255,255,255,0.12)" />
|
||||
{benchmarkPath && (
|
||||
<path d={benchmarkPath} fill="none" stroke="rgba(156,163,175,0.9)" strokeWidth="2" strokeDasharray="5 5" />
|
||||
)}
|
||||
<path d={portfolioPath} fill="none" stroke="rgb(96,165,250)" strokeWidth="3" />
|
||||
<text x={pad} y={pad - 8} className="fill-gray-500 text-[10px]">{fmtSignedPct(max)}</text>
|
||||
<text x={pad} y={h - 8} className="fill-gray-500 text-[10px]">{fmtSignedPct(min)}</text>
|
||||
</svg>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const TARGET_MODEL_OPTIONS = [
|
||||
{ value: 'production_gtl', label: 'Live GTL — production' },
|
||||
{ value: 'structural_sr', label: 'Structural S/R — comparison' },
|
||||
];
|
||||
const CADENCE_OPTIONS = [
|
||||
{ value: 'weekly', label: 'Weekly — default' },
|
||||
{ value: 'daily', label: 'Daily — research' },
|
||||
];
|
||||
|
||||
export function BacktestPanel() {
|
||||
const { data: report, isLoading } = useBacktestReport();
|
||||
@@ -150,8 +42,19 @@ export function BacktestPanel() {
|
||||
const monitor = report?.portfolio_monitor ?? null;
|
||||
const activeStrategy =
|
||||
selectedStrategy || monitor?.production_strategy || monitor?.strategies[0]?.strategy || '';
|
||||
// Default to the window the recommendation was computed on, so the tiles and
|
||||
// the recommendation never open showing different numbers. They used to: the
|
||||
// backend preferred "all" while this defaulted to "3y". The 3y fallback is
|
||||
// only for reports predating basis_lookback.
|
||||
const basisLookback = report?.recommendation?.basis_lookback ?? null;
|
||||
const activeLookback =
|
||||
selectedLookback || (monitor?.lookbacks.some((l) => l.lookback === '3y') ? '3y' : monitor?.lookbacks[0]?.lookback) || '';
|
||||
selectedLookback ||
|
||||
(basisLookback && monitor?.lookbacks.some((l) => l.lookback === basisLookback)
|
||||
? basisLookback
|
||||
: monitor?.lookbacks.some((l) => l.lookback === '3y')
|
||||
? '3y'
|
||||
: monitor?.lookbacks[0]?.lookback) ||
|
||||
'';
|
||||
const monitorRun = useMemo(
|
||||
() =>
|
||||
monitor?.runs.find((row) => row.strategy === activeStrategy && row.lookback === activeLookback) ??
|
||||
@@ -178,7 +81,61 @@ export function BacktestPanel() {
|
||||
return (
|
||||
<Section title="Is the strategy working?" hint="portfolio simulation of the promoted strategy vs S&P 500">
|
||||
<div className="space-y-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
{/* Run status and the controls that start a new run, on one line. The
|
||||
explainer sits BELOW this row rather than beside it — sharing a flex
|
||||
row meant expanding it shoved every control down the page. */}
|
||||
<div className="flex flex-wrap items-end justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<p className="section-index">Last run</p>
|
||||
{report ? (
|
||||
<p className="mt-1 text-xs text-gray-400">
|
||||
{timeAgo(report.generated_at)} · {report.tickers} tickers ·{' '}
|
||||
{report.candidates} setups ({report.qualified} qualified) ·{' '}
|
||||
{report.params.entry_cadence ?? 'weekly'},{' '}
|
||||
{report.params.horizon_days}d horizon
|
||||
{report.params.cost_per_side_pct != null && (
|
||||
<> · net of {report.params.cost_per_side_pct}%/side</>
|
||||
)}
|
||||
{' · '}
|
||||
<span className={report.params.is_production_target_model === false ? 'text-amber-300' : 'text-blue-300'}>
|
||||
{report.params.target_model_label ?? 'Unknown (legacy report)'}
|
||||
</span>
|
||||
</p>
|
||||
) : (
|
||||
<p className="mt-1 text-xs text-gray-500">Never run</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* flex-wrap is load-bearing: two dropdowns plus the button overflow a
|
||||
narrow viewport otherwise. */}
|
||||
<div className="flex flex-wrap items-end gap-2">
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="backtest-target-model">Target model</label>
|
||||
<Dropdown
|
||||
id="backtest-target-model"
|
||||
className="w-56 normal-case tracking-normal"
|
||||
value={targetModel}
|
||||
onChange={(v) => setTargetModel(v as BacktestTargetModel)}
|
||||
options={TARGET_MODEL_OPTIONS}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="backtest-cadence">Entry cadence</label>
|
||||
<Dropdown
|
||||
id="backtest-cadence"
|
||||
className="w-44 normal-case tracking-normal"
|
||||
value={cadence}
|
||||
onChange={(v) => setCadence(v as BacktestCadence)}
|
||||
options={CADENCE_OPTIONS}
|
||||
/>
|
||||
</div>
|
||||
<Button onClick={() => run.mutate()} loading={run.isPending} className="shrink-0">
|
||||
{run.isPending ? 'Starting…' : report ? 'Re-run' : 'Run backtest'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Disclosure summary="How this is measured">
|
||||
<p className="max-w-2xl text-xs text-gray-400">
|
||||
The backtest replays the current config at the selected cadence — at each point the setup is
|
||||
@@ -187,114 +144,30 @@ export function BacktestPanel() {
|
||||
fundamentals are held neutral (no point-in-time history). ~6 months is roughly one market regime,
|
||||
so read it as directional.
|
||||
</p>
|
||||
<p className="mt-2 max-w-2xl text-xs text-gray-400">
|
||||
<strong className="text-gray-300">Live GTL</strong> is the exact target path the scanner and the
|
||||
scheduled backtest use; <strong className="text-gray-300">Structural S/R</strong> is a comparison
|
||||
arm sourcing targets from chart structure. <strong className="text-gray-300">Weekly</strong> steps
|
||||
five sessions at a time and is what the server runs; <strong className="text-gray-300">Daily</strong>
|
||||
{' '}is roughly 5× the replay work.
|
||||
</p>
|
||||
</Disclosure>
|
||||
<div className="flex w-full flex-col gap-3 sm:w-auto sm:items-end">
|
||||
<fieldset className="grid w-full grid-cols-1 gap-2 sm:w-[34rem] sm:grid-cols-2">
|
||||
<legend className="mb-1 text-[11px] font-medium uppercase tracking-wider text-gray-500">
|
||||
Target model for this run
|
||||
</legend>
|
||||
<label
|
||||
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-blue-400/60 ${
|
||||
targetModel === 'production_gtl'
|
||||
? 'border-blue-400/60 bg-blue-500/10'
|
||||
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
className="sr-only"
|
||||
type="radio"
|
||||
name="backtest-target-model"
|
||||
value="production_gtl"
|
||||
checked={targetModel === 'production_gtl'}
|
||||
onChange={() => setTargetModel('production_gtl')}
|
||||
/>
|
||||
<span className="flex items-center justify-between gap-2 text-sm font-medium text-gray-100">
|
||||
Live GTL
|
||||
<span className="rounded-full border border-blue-400/40 bg-blue-400/10 px-2 py-0.5 text-[9px] font-semibold uppercase tracking-widest text-blue-300">
|
||||
Production
|
||||
</span>
|
||||
</span>
|
||||
<span className="mt-1 block text-[11px] leading-4 text-gray-500">
|
||||
Exact target path used by the live scanner and scheduled backtest.
|
||||
</span>
|
||||
</label>
|
||||
<label
|
||||
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-amber-400/60 ${
|
||||
targetModel === 'structural_sr'
|
||||
? 'border-amber-400/50 bg-amber-500/10'
|
||||
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
className="sr-only"
|
||||
type="radio"
|
||||
name="backtest-target-model"
|
||||
value="structural_sr"
|
||||
checked={targetModel === 'structural_sr'}
|
||||
onChange={() => setTargetModel('structural_sr')}
|
||||
/>
|
||||
<span className="text-sm font-medium text-gray-200">Structural S/R</span>
|
||||
<span className="mt-1 block text-[11px] leading-4 text-gray-500">
|
||||
Comparison only; uses chart structure as the target source.
|
||||
</span>
|
||||
</label>
|
||||
</fieldset>
|
||||
<fieldset className="grid w-full grid-cols-2 gap-2 sm:w-[34rem]">
|
||||
<legend className="mb-1 text-[11px] font-medium uppercase tracking-wider text-gray-500">
|
||||
Entry cadence
|
||||
</legend>
|
||||
<label
|
||||
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-blue-400/60 ${
|
||||
cadence === 'weekly'
|
||||
? 'border-blue-400/60 bg-blue-500/10'
|
||||
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
className="sr-only"
|
||||
type="radio"
|
||||
name="backtest-cadence"
|
||||
value="weekly"
|
||||
checked={cadence === 'weekly'}
|
||||
onChange={() => setCadence('weekly')}
|
||||
/>
|
||||
<span className="flex items-center justify-between gap-2 text-sm font-medium text-gray-100">
|
||||
Weekly
|
||||
<span className="rounded-full border border-blue-400/40 bg-blue-400/10 px-2 py-0.5 text-[9px] font-semibold uppercase tracking-widest text-blue-300">
|
||||
Default
|
||||
</span>
|
||||
</span>
|
||||
<span className="mt-1 block text-[11px] leading-4 text-gray-500">
|
||||
Resource-safe server run at five-session intervals.
|
||||
</span>
|
||||
</label>
|
||||
<label
|
||||
className={`cursor-pointer rounded-lg border px-3 py-2 transition-colors focus-within:ring-2 focus-within:ring-amber-400/60 ${
|
||||
cadence === 'daily'
|
||||
? 'border-amber-400/50 bg-amber-500/10'
|
||||
: 'border-white/10 bg-white/[0.03] hover:border-white/20'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
className="sr-only"
|
||||
type="radio"
|
||||
name="backtest-cadence"
|
||||
value="daily"
|
||||
checked={cadence === 'daily'}
|
||||
onChange={() => setCadence('daily')}
|
||||
/>
|
||||
<span className="text-sm font-medium text-gray-200">Daily</span>
|
||||
<span className="mt-1 block text-[11px] leading-4 text-amber-300/80">
|
||||
Research run: roughly 5× the replay work; prefer the offline snapshot runner.
|
||||
</span>
|
||||
</label>
|
||||
</fieldset>
|
||||
<Button onClick={() => run.mutate()} loading={run.isPending} className="shrink-0">
|
||||
{run.isPending ? 'Starting…' : report ? 'Re-run backtest' : 'Run backtest'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Only surfaced for non-default choices — zero noise on the common path,
|
||||
but a non-production selection still announces itself, which is what
|
||||
the old always-amber cards were really for. */}
|
||||
{(cadence === 'daily' || targetModel === 'structural_sr') && (
|
||||
<div className="space-y-1 text-[11px] text-amber-300/80">
|
||||
{cadence === 'daily' && (
|
||||
<p>Daily replays ~5× the work — prefer the offline snapshot runner.</p>
|
||||
)}
|
||||
{targetModel === 'structural_sr' && (
|
||||
<p>Comparison arm — not the live scanner's target path.</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isLoading && <Callout variant="empty">Loading…</Callout>}
|
||||
|
||||
{!isLoading && !report && (
|
||||
@@ -306,131 +179,21 @@ export function BacktestPanel() {
|
||||
|
||||
{report && (
|
||||
<>
|
||||
<p className="text-[11px] text-gray-500">
|
||||
Ran {timeAgo(report.generated_at)} · {report.tickers} tickers · {report.candidates} setups
|
||||
({report.qualified} qualified) · {report.params.entry_cadence ?? 'weekly'} cadence,
|
||||
{' '}{report.params.horizon_days}-day horizon
|
||||
{report.params.cost_per_side_pct != null && (
|
||||
<> · net of {report.params.cost_per_side_pct}%/side costs</>
|
||||
)}
|
||||
{' '}· target model:{' '}
|
||||
<span className={report.params.is_production_target_model === false ? 'text-amber-300' : 'text-blue-300'}>
|
||||
{report.params.target_model_label ?? 'Unknown (legacy report)'}
|
||||
</span>
|
||||
</p>
|
||||
<PortfolioMonitorPanel
|
||||
monitor={monitor}
|
||||
monitorRun={monitorRun}
|
||||
activeStrategy={activeStrategy}
|
||||
activeLookback={activeLookback}
|
||||
onStrategyChange={setSelectedStrategy}
|
||||
onLookbackChange={setSelectedLookback}
|
||||
basisLookback={basisLookback}
|
||||
basisLookbackLabel={report.recommendation?.basis_lookback_label ?? null}
|
||||
productionStrategy={monitor?.production_strategy ?? null}
|
||||
/>
|
||||
|
||||
{monitor && monitorRun ? (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-end justify-between gap-3">
|
||||
<div>
|
||||
<p className="section-index">Portfolio monitor</p>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Simulated book for the selected strategy and lookback, compared with the S&P 500.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="monitor-strategy">Strategy</label>
|
||||
<Dropdown
|
||||
id="monitor-strategy"
|
||||
className="w-64 normal-case tracking-normal"
|
||||
value={activeStrategy}
|
||||
onChange={setSelectedStrategy}
|
||||
options={monitor.strategies.map((s) => ({
|
||||
value: s.strategy,
|
||||
label: `${s.is_production ? 'Production: ' : ''}${s.label}`,
|
||||
}))}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="monitor-lookback">Lookback</label>
|
||||
<Dropdown
|
||||
id="monitor-lookback"
|
||||
className="w-36 normal-case tracking-normal"
|
||||
value={activeLookback}
|
||||
onChange={setSelectedLookback}
|
||||
options={monitor.lookbacks.map((l) => ({ value: l.lookback, label: l.label }))}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<Stat label="CAGR" value={fmtSignedPct(monitorRun.cagr_pct)} valueClass={rColor(monitorRun.cagr_pct)} />
|
||||
<Stat label="Sharpe" value={monitorRun.sharpe == null ? '—' : monitorRun.sharpe.toFixed(2)} />
|
||||
<Stat label="Max Drawdown" value={fmtDrawdown(monitorRun.max_drawdown_pct)} valueClass="text-amber-400" />
|
||||
<Stat
|
||||
label="Total Return"
|
||||
value={fmtSignedPct(monitorRun.total_return_pct)}
|
||||
valueClass={rColor(monitorRun.total_return_pct)}
|
||||
sub={`vs S&P 500 ${fmtSignedPct(monitorRun.spy_return_pct)}`}
|
||||
/>
|
||||
<Stat label="Trades" value={String(monitorRun.trades)} sub={`${fmtPct(monitorRun.win_rate)} win rate`} />
|
||||
</div>
|
||||
|
||||
<EquityCurveChart run={monitorRun} />
|
||||
|
||||
<p className="text-[11px] text-gray-500">
|
||||
Avg hold {fmtDays(monitorRun.avg_hold_days)} · Best {fmtR(monitorRun.best_trade_r)} / Worst{' '}
|
||||
{fmtR(monitorRun.worst_trade_r)} · Avg P&L per trade {fmtMoney(monitorRun.avg_trade_pnl)}
|
||||
{monitorRun.reentry_policy === 'gate_reset' ? (
|
||||
<> · Re-entry after gate failure and fresh qualification</>
|
||||
) : null}
|
||||
</p>
|
||||
|
||||
{monitorRun.yearly_returns && monitorRun.yearly_returns.length > 0 && (
|
||||
<div className="glass overflow-x-auto p-4">
|
||||
<p className="section-index mb-2">Per-year returns</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{monitorRun.yearly_returns.map((y) => (
|
||||
<div key={y.year} className="rounded border border-white/10 px-3 py-1.5">
|
||||
<span className="num text-xs text-gray-500">{y.year}</span>{' '}
|
||||
<span className={`num text-sm font-semibold ${rColor(y.return_pct)}`}>
|
||||
{fmtSignedPct(y.return_pct)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{monitor.note && <p className="text-[11px] text-gray-600">{monitor.note}</p>}
|
||||
</div>
|
||||
) : (
|
||||
<Callout variant="empty">
|
||||
This report predates the portfolio monitor — re-run the backtest to populate it.
|
||||
</Callout>
|
||||
{report.recommendation && (
|
||||
<BacktestRecommendationCard recommendation={report.recommendation} />
|
||||
)}
|
||||
|
||||
{report.recommendation && report.recommendation.items.length > 0 && (
|
||||
<div className="glass border border-blue-400/20 p-4">
|
||||
<p className="section-index">What this backtest recommends</p>
|
||||
{report.recommendation.headline && (
|
||||
<p className="mt-1.5 text-sm font-semibold text-gray-100">
|
||||
{report.recommendation.headline}
|
||||
</p>
|
||||
)}
|
||||
<ul className="mt-2 space-y-1">
|
||||
{report.recommendation.items.map((item) => (
|
||||
<li
|
||||
key={item.topic + item.text}
|
||||
className={`text-xs ${item.text.includes('WARNING') || item.text.includes('LAGS') ? 'text-amber-400' : 'text-gray-400'}`}
|
||||
>
|
||||
{item.text}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
{report.recommendation.note && (
|
||||
<p className="mt-2 text-[11px] text-gray-600">{report.recommendation.note}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<p className="text-[11px] text-gray-600">
|
||||
Strategy research — gate tuning, exit sweeps, factor rank-IC — now runs locally against a
|
||||
database snapshot (see README). This page keeps only what says whether the promoted strategy
|
||||
is worth trading; your realized results up top show what it is actually delivering.
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { Disclosure } from '../ui/Disclosure';
|
||||
import type { BacktestRecommendation } from '../../lib/types';
|
||||
|
||||
/**
|
||||
* The verdict, ahead of the tuning detail.
|
||||
*
|
||||
* Two problems this solves. All eight findings used to render as equal-weight
|
||||
* bullets, so "does this strategy work" sat in the same register as "which
|
||||
* cutoff scored best". And the headline — which is a *description of the
|
||||
* config*, not a verdict — was the loudest thing on the card while every actual
|
||||
* finding was small grey text.
|
||||
*
|
||||
* So: findings first, each split into a label and its detail; the config
|
||||
* description demoted to a footer where it belongs.
|
||||
*/
|
||||
const PRIMARY_TOPICS = new Set(['production', 'benchmark', 'robustness']);
|
||||
|
||||
/**
|
||||
* Mirrors how the backend phrases a bad result — `_build_recommendation` emits
|
||||
* "Robustness WARNING: …" and "Book vs SPY: LAGS …". There is deliberately no
|
||||
* `severity` field on the payload; if that changes, this is the one place to fix.
|
||||
*/
|
||||
function isWarning(text: string): boolean {
|
||||
return text.includes('WARNING') || text.includes('LAGS');
|
||||
}
|
||||
|
||||
/**
|
||||
* Every backend string self-prefixes ("Gate: keep the R:R floor…"), so the
|
||||
* prefix IS the label — no need for a chip that would just repeat it, and no
|
||||
* need to reword anything server-side. Split on the first colon; if a string
|
||||
* ever stops carrying one, it renders whole as detail.
|
||||
*/
|
||||
function splitLabel(text: string): { label: string | null; detail: string } {
|
||||
const at = text.indexOf(': ');
|
||||
if (at === -1 || at > 48) return { label: null, detail: text };
|
||||
return { label: text.slice(0, at), detail: text.slice(at + 2) };
|
||||
}
|
||||
|
||||
function Finding({ text, primary }: { text: string; primary: boolean }) {
|
||||
const warn = isWarning(text);
|
||||
const { label, detail } = splitLabel(text);
|
||||
return (
|
||||
<li className="flex flex-col gap-0.5 sm:flex-row sm:gap-3">
|
||||
{label && (
|
||||
<span
|
||||
className={`shrink-0 text-[11px] font-semibold uppercase tracking-wider sm:w-44 sm:pt-0.5 ${
|
||||
warn ? 'text-amber-400' : 'text-gray-500'
|
||||
}`}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
)}
|
||||
<span
|
||||
className={`${primary ? 'text-sm' : 'text-xs'} ${
|
||||
warn ? 'text-amber-300' : primary ? 'text-gray-200' : 'text-gray-400'
|
||||
}`}
|
||||
>
|
||||
{detail}
|
||||
</span>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
export function BacktestRecommendationCard({
|
||||
recommendation,
|
||||
}: {
|
||||
recommendation: BacktestRecommendation;
|
||||
}) {
|
||||
const items = recommendation.items;
|
||||
if (items.length === 0) return null;
|
||||
|
||||
// A warning is always visible, whatever its topic — burying "the edge
|
||||
// disappears without the top 5% of winners" behind a disclosure would defeat
|
||||
// the point of surfacing it at all.
|
||||
const primary = items.filter((i) => PRIMARY_TOPICS.has(i.topic) || isWarning(i.text));
|
||||
const secondary = items.filter((i) => !PRIMARY_TOPICS.has(i.topic) && !isWarning(i.text));
|
||||
const warningCount = items.filter((i) => isWarning(i.text)).length;
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="glass border border-blue-400/20 p-4">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<p className="section-index">What this backtest recommends</p>
|
||||
{/* No headline means the backend found no production monitor row, so
|
||||
nothing here describes the production book. Zero keyword warnings
|
||||
is then absence of data, not a clean bill of health — a green chip
|
||||
beside "this report predates the portfolio monitor" would be a
|
||||
success badge for missing data. */}
|
||||
{!recommendation.headline ? (
|
||||
<span className="rounded-full border border-white/15 bg-white/[0.05] px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wider text-gray-400">
|
||||
baseline unavailable
|
||||
</span>
|
||||
) : warningCount > 0 ? (
|
||||
<span className="rounded-full border border-amber-400/40 bg-amber-400/10 px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wider text-amber-300">
|
||||
⚠ {warningCount} warning{warningCount > 1 ? 's' : ''}
|
||||
</span>
|
||||
) : (
|
||||
<span className="rounded-full border border-emerald-400/30 bg-emerald-400/10 px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wider text-emerald-300">
|
||||
no warnings
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{primary.length > 0 && (
|
||||
<ul className="mt-3 space-y-2.5">
|
||||
{primary.map((item) => (
|
||||
<Finding key={item.topic + item.text} text={item.text} primary />
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
|
||||
{/* The config description, demoted: it says what the strategy IS, which
|
||||
is context for the findings above rather than a finding itself. */}
|
||||
{recommendation.headline && (
|
||||
<div className="mt-3 border-t border-white/[0.06] pt-3">
|
||||
<p className="section-index">Configuration under test</p>
|
||||
<p className="mt-1 text-xs leading-relaxed text-gray-500">{recommendation.headline}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{recommendation.note && (
|
||||
<p className="mt-2 text-[11px] text-gray-600">{recommendation.note}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Outside the card body on purpose: Disclosure renders its own glass-sm
|
||||
panel, so nesting it inside the bordered card double-frames it. */}
|
||||
{secondary.length > 0 && (
|
||||
<Disclosure summary={`Gate and cutoff detail (${secondary.length})`}>
|
||||
<ul className="space-y-2">
|
||||
{secondary.map((item) => (
|
||||
<Finding key={item.topic + item.text} text={item.text} primary={false} />
|
||||
))}
|
||||
</ul>
|
||||
</Disclosure>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { Callout } from '../ui/Callout';
|
||||
import { fmtSignedPct } from '../../lib/format';
|
||||
import type { BacktestCurvePoint, BacktestPortfolioMonitorRun } from '../../lib/types';
|
||||
|
||||
/**
|
||||
* Portfolio return vs S&P 500 for one monitor run.
|
||||
*
|
||||
* Hand-rolled SVG on purpose: two polylines and two axis rules do not justify a
|
||||
* charting dependency, and the shape is fixed. Lives in `signals/` rather than
|
||||
* `ui/` because it is typed to the backtest payload — generalising it for a
|
||||
* single caller would be the wrong trade.
|
||||
*/
|
||||
function curvePath(
|
||||
points: BacktestCurvePoint[],
|
||||
min: number,
|
||||
max: number,
|
||||
w: number,
|
||||
h: number,
|
||||
pad: number,
|
||||
startMs: number,
|
||||
endMs: number,
|
||||
): string {
|
||||
if (points.length < 2) return '';
|
||||
const span = Math.max(max - min, 1);
|
||||
const timeSpan = Math.max(endMs - startMs, 1);
|
||||
return points
|
||||
.map((p, i) => {
|
||||
const t = new Date(p.date).getTime();
|
||||
const x = pad + ((t - startMs) / timeSpan) * (w - pad * 2);
|
||||
const value = p.return_pct ?? 0;
|
||||
const y = pad + (1 - (value - min) / span) * (h - pad * 2);
|
||||
return `${i === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`;
|
||||
})
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
export function EquityCurveChart({ run }: { run: BacktestPortfolioMonitorRun }) {
|
||||
const portfolio = run.equity_curve ?? [];
|
||||
const benchmark = run.benchmark_curve ?? [];
|
||||
const values = [...portfolio, ...benchmark]
|
||||
.map((p) => p.return_pct)
|
||||
.filter((v): v is number => v !== null && v !== undefined);
|
||||
if (portfolio.length < 2 || values.length === 0) {
|
||||
return <Callout variant="empty">No equity curve points for this selection.</Callout>;
|
||||
}
|
||||
|
||||
const min = Math.min(0, ...values);
|
||||
const max = Math.max(0, ...values);
|
||||
const times = [...portfolio, ...benchmark]
|
||||
.map((p) => new Date(p.date).getTime())
|
||||
.filter((v) => Number.isFinite(v));
|
||||
if (times.length === 0) {
|
||||
return <Callout variant="empty">No dated equity curve points for this selection.</Callout>;
|
||||
}
|
||||
const startMs = Math.min(...times);
|
||||
const endMs = Math.max(...times);
|
||||
const w = 720;
|
||||
const h = 240;
|
||||
const pad = 28;
|
||||
const portfolioPath = curvePath(portfolio, min, max, w, h, pad, startMs, endMs);
|
||||
const benchmarkPath = curvePath(benchmark, min, max, w, h, pad, startMs, endMs);
|
||||
const lastPortfolio = portfolio[portfolio.length - 1]?.return_pct ?? null;
|
||||
const lastBenchmark = benchmark[benchmark.length - 1]?.return_pct ?? run.spy_return_pct;
|
||||
|
||||
return (
|
||||
<div className="glass overflow-hidden">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-white/[0.05] px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-gray-100">{run.label}</p>
|
||||
<p className="text-[11px] text-gray-500">{run.start_date} - {run.end_date}</p>
|
||||
</div>
|
||||
<div className="flex gap-4 text-xs">
|
||||
<span className="text-blue-300">Portfolio {fmtSignedPct(lastPortfolio)}</span>
|
||||
<span className="text-gray-400">S&P 500 {fmtSignedPct(lastBenchmark)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<svg viewBox={`0 0 ${w} ${h}`} className="h-64 w-full" role="img" aria-label="Portfolio return compared with S&P 500">
|
||||
<line x1={pad} y1={h - pad} x2={w - pad} y2={h - pad} stroke="rgba(255,255,255,0.12)" />
|
||||
<line x1={pad} y1={pad} x2={pad} y2={h - pad} stroke="rgba(255,255,255,0.12)" />
|
||||
{benchmarkPath && (
|
||||
<path d={benchmarkPath} fill="none" stroke="rgba(156,163,175,0.9)" strokeWidth="2" strokeDasharray="5 5" />
|
||||
)}
|
||||
<path d={portfolioPath} fill="none" stroke="rgb(96,165,250)" strokeWidth="3" />
|
||||
<text x={pad} y={pad - 8} className="fill-gray-500 text-[10px]">{fmtSignedPct(max)}</text>
|
||||
<text x={pad} y={h - 8} className="fill-gray-500 text-[10px]">{fmtSignedPct(min)}</text>
|
||||
</svg>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+6
-24
@@ -5,8 +5,7 @@ import { triggerJob, resetTrackRecord } from '../../api/admin';
|
||||
import { Button } from '../ui/Button';
|
||||
import { Disclosure } from '../ui/Disclosure';
|
||||
import { useToast } from '../ui/Toast';
|
||||
import { BacktestPanel } from './BacktestPanel';
|
||||
import { MyTradesPanel } from './MyTradesPanel';
|
||||
import { fmtR, rColor } from '../../lib/format';
|
||||
|
||||
// Need at least this many matured setups before the pipeline check means anything;
|
||||
// below it the live sample is too noisy to compare.
|
||||
@@ -16,18 +15,6 @@ const DRIFT_TOLERANCE_R = 0.2;
|
||||
|
||||
type PipelineStatus = 'building' | 'tracking' | 'drift' | 'no-backtest';
|
||||
|
||||
function fmtR(value: number | null): string {
|
||||
if (value === null) return '—';
|
||||
return `${value > 0 ? '+' : ''}${value.toFixed(2)}R`;
|
||||
}
|
||||
|
||||
function rColor(value: number | null): string {
|
||||
if (value === null) return 'text-gray-400';
|
||||
if (value > 0) return 'text-emerald-400';
|
||||
if (value < 0) return 'text-red-400';
|
||||
return 'text-gray-300';
|
||||
}
|
||||
|
||||
function StatusChip({ status }: { status: PipelineStatus }) {
|
||||
const styles: Record<PipelineStatus, { cls: string; label: string }> = {
|
||||
tracking: { cls: 'border-emerald-500/30 bg-emerald-500/15 text-emerald-300', label: '✓ in sync' },
|
||||
@@ -39,7 +26,7 @@ function StatusChip({ status }: { status: PipelineStatus }) {
|
||||
return <span className={`shrink-0 rounded-full border px-2.5 py-1 text-xs font-medium ${s.cls}`}>{s.label}</span>;
|
||||
}
|
||||
|
||||
export function TrackRecordPanel() {
|
||||
export function EvaluationPanel() {
|
||||
const queryClient = useQueryClient();
|
||||
const toast = useToast();
|
||||
|
||||
@@ -101,19 +88,14 @@ export function TrackRecordPanel() {
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Your real, realized results come first; the strategy simulation follows. */}
|
||||
<MyTradesPanel />
|
||||
<div className="border-t border-white/[0.06]" />
|
||||
<BacktestPanel />
|
||||
|
||||
<Disclosure summary="Track-record maintenance">
|
||||
<Disclosure summary="Setup-grading diagnostic & maintenance">
|
||||
<div className="space-y-4 pt-1">
|
||||
<p className="max-w-2xl text-xs text-gray-500">
|
||||
<span className="text-amber-300/90">Diagnostic only — not production P&L.</span>{' '}
|
||||
Grades gate-level touch vs stop (the rejected take-profit model). Production exits are
|
||||
initial stop / ATR trail / max hold — see paper trades and the portfolio monitor above.
|
||||
Target before stop = win, stop first = loss (same-bar both = loss), neither in 30 trading
|
||||
days = expired at 0R. Only matured windows count. Scores{' '}
|
||||
initial stop / ATR trail / max hold — see the Paper Trades tab and the portfolio monitor
|
||||
above. Target before stop = win, stop first = loss (same-bar both = loss), neither in 30
|
||||
trading days = expired at 0R. Only matured windows count. Scores{' '}
|
||||
<span className="text-gray-300">all</span> setups as a control group; runs nightly.
|
||||
</p>
|
||||
|
||||
@@ -2,22 +2,10 @@ import { useMemo } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { usePaperTrades } from '../../hooks/usePaperTrades';
|
||||
import { tradePnl } from '../../lib/paperTrade';
|
||||
import { formatPrice } from '../../lib/format';
|
||||
import { formatPrice, fmtR, fmtSignedMoney, rColor } from '../../lib/format';
|
||||
import { Section } from '../ui/Section';
|
||||
import { Callout } from '../ui/Callout';
|
||||
|
||||
function money(v: number): string {
|
||||
return `${v >= 0 ? '+' : '−'}$${Math.abs(v).toFixed(2)}`;
|
||||
}
|
||||
function fmtR(v: number | null): string {
|
||||
return v === null ? '—' : `${v > 0 ? '+' : ''}${v.toFixed(2)}R`;
|
||||
}
|
||||
function color(v: number | null): string {
|
||||
if (v === null) return 'text-gray-400';
|
||||
if (v > 0) return 'text-emerald-400';
|
||||
if (v < 0) return 'text-red-400';
|
||||
return 'text-gray-300';
|
||||
}
|
||||
import { StatTile } from '../ui/StatTile';
|
||||
|
||||
// How the trade was closed — useful context on real trades at almost no cost.
|
||||
function reasonMeta(reason: string | null): { label: string; cls: string } {
|
||||
@@ -31,18 +19,6 @@ function reasonMeta(reason: string | null): { label: string; cls: string } {
|
||||
}
|
||||
}
|
||||
|
||||
function Stat({ label, value, valueClass = 'text-gray-100', sub }: {
|
||||
label: string; value: string; valueClass?: string; sub?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="glass p-4">
|
||||
<p className="section-index">{label}</p>
|
||||
<p className={`num mt-1.5 text-2xl font-semibold ${valueClass}`}>{value}</p>
|
||||
{sub && <p className="mt-1 text-xs text-gray-500">{sub}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function MyTradesPanel() {
|
||||
const { data: closed, isLoading } = usePaperTrades('closed');
|
||||
|
||||
@@ -70,7 +46,10 @@ export function MyTradesPanel() {
|
||||
if (isLoading) return null;
|
||||
|
||||
return (
|
||||
<Section title="My Trades" hint="your realized paper-trading results">
|
||||
<Section
|
||||
title="Closed Trades"
|
||||
hint="realized paper-trading results — open positions are on the Dashboard"
|
||||
>
|
||||
{stats.total === 0 ? (
|
||||
<Callout variant="empty">
|
||||
No closed trades yet. Take setups as paper trades and they’ll resolve here when price hits
|
||||
@@ -79,11 +58,11 @@ export function MyTradesPanel() {
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<Stat label="Hit Rate" value={stats.hitRate != null ? `${stats.hitRate.toFixed(1)}%` : '—'} sub={`${stats.wins}W / ${stats.losses}L`} />
|
||||
<Stat label="Expectancy" value={fmtR(stats.avgR)} valueClass={color(stats.avgR)} sub="avg R per closed trade" />
|
||||
<Stat label="Total R" value={fmtR(stats.totalR)} valueClass={color(stats.totalR)} sub={`${stats.total} closed`} />
|
||||
<Stat label="Total P&L" value={money(stats.totalPnl)} valueClass={color(stats.totalPnl)} sub="realized, all closed" />
|
||||
<Stat label="Alpha vs S&P 500" value={stats.totalAlpha != null ? money(stats.totalAlpha) : '—'} valueClass={color(stats.totalAlpha)} sub="realized vs buy-and-hold SPY" />
|
||||
<StatTile label="Hit Rate" value={stats.hitRate != null ? `${stats.hitRate.toFixed(1)}%` : '—'} sub={`${stats.wins}W / ${stats.losses}L`} />
|
||||
<StatTile label="Expectancy" value={fmtR(stats.avgR)} valueClass={rColor(stats.avgR)} sub="avg R per closed trade" />
|
||||
<StatTile label="Total R" value={fmtR(stats.totalR)} valueClass={rColor(stats.totalR)} sub={`${stats.total} closed`} />
|
||||
<StatTile label="Total P&L" value={fmtSignedMoney(stats.totalPnl)} valueClass={rColor(stats.totalPnl)} sub="realized, all closed" />
|
||||
<StatTile label="Alpha vs S&P 500" value={stats.totalAlpha != null ? fmtSignedMoney(stats.totalAlpha) : '—'} valueClass={rColor(stats.totalAlpha)} sub="realized vs buy-and-hold SPY" />
|
||||
</div>
|
||||
|
||||
<div className="glass overflow-x-auto">
|
||||
@@ -112,9 +91,9 @@ export function MyTradesPanel() {
|
||||
</td>
|
||||
<td className="num px-4 py-2.5 text-right text-gray-300">{formatPrice(t.entry_price)}</td>
|
||||
<td className="num px-4 py-2.5 text-right text-gray-300">{t.close_price != null ? formatPrice(t.close_price) : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right font-semibold ${p ? color(p.pnl) : 'text-gray-500'}`}>{p ? money(p.pnl) : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right ${p?.r != null ? color(p.r) : 'text-gray-500'}`}>{p?.r != null ? fmtR(p.r) : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right ${t.alpha_pct != null ? color(t.alpha_pct) : 'text-gray-500'}`} title="Return vs. S&P 500 over the holding period">{t.alpha_pct != null ? `${t.alpha_pct >= 0 ? '+' : ''}${t.alpha_pct.toFixed(1)}%` : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right font-semibold ${p ? rColor(p.pnl) : 'text-gray-500'}`}>{p ? fmtSignedMoney(p.pnl) : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right ${p?.r != null ? rColor(p.r) : 'text-gray-500'}`}>{p?.r != null ? fmtR(p.r) : '—'}</td>
|
||||
<td className={`num px-4 py-2.5 text-right ${t.alpha_pct != null ? rColor(t.alpha_pct) : 'text-gray-500'}`} title="Return vs. S&P 500 over the holding period">{t.alpha_pct != null ? `${t.alpha_pct >= 0 ? '+' : ''}${t.alpha_pct.toFixed(1)}%` : '—'}</td>
|
||||
<td className="px-4 py-2.5">
|
||||
<span className={`num text-[10px] font-semibold uppercase tracking-wider ${reasonMeta(t.close_reason).cls}`} title="How the trade was closed">
|
||||
{reasonMeta(t.close_reason).label}
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { Callout } from '../ui/Callout';
|
||||
import { Dropdown } from '../ui/Dropdown';
|
||||
import { StatTile } from '../ui/StatTile';
|
||||
import { EquityCurveChart } from './EquityCurveChart';
|
||||
import {
|
||||
fmtDays,
|
||||
fmtDrawdown,
|
||||
fmtPct,
|
||||
fmtR,
|
||||
fmtRatio,
|
||||
fmtSignedMoney,
|
||||
fmtSignedPct,
|
||||
rColor,
|
||||
} from '../../lib/format';
|
||||
import type {
|
||||
BacktestPortfolioMonitor,
|
||||
BacktestPortfolioMonitorRun,
|
||||
} from '../../lib/types';
|
||||
|
||||
/**
|
||||
* The simulated book for one strategy/lookback selection, against the S&P 500.
|
||||
*
|
||||
* Selection state deliberately stays in BacktestPanel — it also resolves which
|
||||
* run this panel receives, so splitting it here would mean resolving twice.
|
||||
*/
|
||||
export function PortfolioMonitorPanel({
|
||||
monitor,
|
||||
monitorRun,
|
||||
activeStrategy,
|
||||
activeLookback,
|
||||
onStrategyChange,
|
||||
onLookbackChange,
|
||||
basisLookback = null,
|
||||
basisLookbackLabel = null,
|
||||
productionStrategy = null,
|
||||
}: {
|
||||
monitor: BacktestPortfolioMonitor | null | undefined;
|
||||
monitorRun: BacktestPortfolioMonitorRun | null | undefined;
|
||||
activeStrategy: string;
|
||||
activeLookback: string;
|
||||
onStrategyChange: (v: string) => void;
|
||||
onLookbackChange: (v: string) => void;
|
||||
/** The window the recommendation below was computed on. */
|
||||
basisLookback?: string | null;
|
||||
basisLookbackLabel?: string | null;
|
||||
productionStrategy?: string | null;
|
||||
}) {
|
||||
if (!monitor || !monitorRun) {
|
||||
return (
|
||||
<Callout variant="empty">
|
||||
This report predates the portfolio monitor — re-run the backtest to populate it.
|
||||
</Callout>
|
||||
);
|
||||
}
|
||||
|
||||
// Key ABSENT (not null) means the cached report predates these metrics.
|
||||
// Gated on sortino specifically: calmar and avg_trade_pnl have always been
|
||||
// emitted, so testing those would half-populate the row with dashes.
|
||||
const isLegacyRun = monitorRun.sortino === undefined;
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-end justify-between gap-3">
|
||||
<div>
|
||||
<p className="section-index">Portfolio monitor</p>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Simulated book for the selected strategy and lookback, compared with the S&P 500.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="monitor-strategy">Strategy</label>
|
||||
<Dropdown
|
||||
id="monitor-strategy"
|
||||
className="w-64 normal-case tracking-normal"
|
||||
value={activeStrategy}
|
||||
onChange={onStrategyChange}
|
||||
options={monitor.strategies.map((s) => ({
|
||||
value: s.strategy,
|
||||
// "Production: " prefix dropped — a bullet costs one character
|
||||
// instead of twelve, and the full config is spelled out under
|
||||
// the chart anyway.
|
||||
label: `${s.is_production ? '● ' : ''}${s.label}`,
|
||||
}))}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1 text-[11px] uppercase tracking-wider text-gray-500">
|
||||
<label htmlFor="monitor-lookback">Lookback</label>
|
||||
<Dropdown
|
||||
id="monitor-lookback"
|
||||
className="w-36 normal-case tracking-normal"
|
||||
value={activeLookback}
|
||||
onChange={onLookbackChange}
|
||||
options={monitor.lookbacks.map((l) => ({ value: l.lookback, label: l.label }))}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* The recommendation below is baked into the report and cannot follow a
|
||||
dropdown. On load the two agree by construction; say so plainly the
|
||||
moment a selection moves off that basis. */}
|
||||
{((basisLookback && activeLookback !== basisLookback) ||
|
||||
(productionStrategy && activeStrategy !== productionStrategy)) && (
|
||||
<p className="text-[11px] text-amber-300/80">
|
||||
Showing{' '}
|
||||
{productionStrategy && activeStrategy !== productionStrategy
|
||||
? 'a comparison strategy'
|
||||
: 'a different window'}
|
||||
. The recommendation below is computed on the production strategy over{' '}
|
||||
{basisLookbackLabel ?? basisLookback} — these tiles will not match it.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Tier 1 — what the book returned. */}
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<StatTile
|
||||
label="Total Return"
|
||||
value={fmtSignedPct(monitorRun.total_return_pct)}
|
||||
valueClass={rColor(monitorRun.total_return_pct)}
|
||||
sub={`vs S&P 500 ${fmtSignedPct(monitorRun.spy_return_pct)}`}
|
||||
/>
|
||||
<StatTile label="CAGR" value={fmtSignedPct(monitorRun.cagr_pct)} valueClass={rColor(monitorRun.cagr_pct)} />
|
||||
<StatTile label="Max Drawdown" value={fmtDrawdown(monitorRun.max_drawdown_pct)} valueClass="text-amber-400" />
|
||||
<StatTile
|
||||
label="EV / trade"
|
||||
value={fmtSignedMoney(monitorRun.avg_trade_pnl)}
|
||||
valueClass={rColor(monitorRun.avg_trade_pnl)}
|
||||
title="Average realized P&L per closed trade. Scales with position size, so it carries no quality band."
|
||||
/>
|
||||
<StatTile label="Trades" value={String(monitorRun.trades)} sub={`${fmtPct(monitorRun.win_rate)} win rate`} />
|
||||
</div>
|
||||
|
||||
{/* Tier 2 — how good that return was. Smaller and labelled on purpose:
|
||||
ten equal tiles would read as ten equally important facts. */}
|
||||
{isLegacyRun ? (
|
||||
<p className="text-[11px] text-gray-600">
|
||||
Risk-adjusted quality metrics appear after the next backtest run.
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="flex flex-wrap items-baseline justify-between gap-2">
|
||||
<p className="section-index">Risk-adjusted quality</p>
|
||||
<p className="text-[11px] text-gray-600">
|
||||
Bands are set stricter than textbook ranges — this universe is today's
|
||||
survivors replayed backward, which flatters every ratio.
|
||||
</p>
|
||||
</div>
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<StatTile
|
||||
label="Sharpe"
|
||||
value={fmtRatio(monitorRun.sharpe)}
|
||||
metric="sharpe"
|
||||
raw={monitorRun.sharpe}
|
||||
title="Return per unit of total volatility (annualized). Penalizes upside swings as well as downside."
|
||||
/>
|
||||
<StatTile
|
||||
label="Sortino"
|
||||
value={fmtRatio(monitorRun.sortino)}
|
||||
metric="sortino"
|
||||
raw={monitorRun.sortino}
|
||||
title="Return per unit of downside deviation (annualized). Punishes losing days only, unlike Sharpe."
|
||||
/>
|
||||
<StatTile
|
||||
label="Calmar (MAR)"
|
||||
value={fmtRatio(monitorRun.calmar)}
|
||||
metric="calmar"
|
||||
raw={monitorRun.calmar}
|
||||
title="CAGR divided by maximum drawdown — return earned per unit of worst-case pain."
|
||||
/>
|
||||
<StatTile
|
||||
label="Gain / Pain"
|
||||
value={fmtRatio(monitorRun.gain_to_pain)}
|
||||
metric="gain_to_pain"
|
||||
raw={monitorRun.gain_to_pain}
|
||||
title="Sum of monthly returns divided by the absolute sum of the negative ones (Schwager)."
|
||||
/>
|
||||
<StatTile
|
||||
label="Profit Factor ($)"
|
||||
value={fmtRatio(monitorRun.profit_factor)}
|
||||
metric="profit_factor"
|
||||
raw={monitorRun.profit_factor}
|
||||
title="Gross winning dollars divided by gross losing dollars, across closed trades."
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<EquityCurveChart run={monitorRun} />
|
||||
|
||||
{/* avg_trade_pnl is a tile now (EV / trade) — not repeated here. */}
|
||||
<p className="text-[11px] text-gray-500">
|
||||
Avg hold {fmtDays(monitorRun.avg_hold_days)} · Best {fmtR(monitorRun.best_trade_r)} / Worst{' '}
|
||||
{fmtR(monitorRun.worst_trade_r)}
|
||||
{monitorRun.reentry_policy === 'gate_reset' ? (
|
||||
<> · Re-entry after gate failure and fresh qualification</>
|
||||
) : null}
|
||||
</p>
|
||||
|
||||
{monitorRun.yearly_returns && monitorRun.yearly_returns.length > 0 && (
|
||||
<div className="glass overflow-x-auto p-4">
|
||||
<p className="section-index mb-2">Per-year returns</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{monitorRun.yearly_returns.map((y) => (
|
||||
<div key={y.year} className="rounded border border-white/10 px-3 py-1.5">
|
||||
<span className="num text-xs text-gray-500">{y.year}</span>{' '}
|
||||
<span className={`num text-sm font-semibold ${rColor(y.return_pct)}`}>
|
||||
{fmtSignedPct(y.return_pct)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{monitor.note && <p className="text-[11px] text-gray-600">{monitor.note}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -16,7 +16,7 @@ const sizeClasses: Record<Size, string> = {
|
||||
md: 'px-4 py-2 text-sm',
|
||||
};
|
||||
|
||||
export function Spinner({ className = 'h-4 w-4' }: { className?: string }) {
|
||||
function Spinner({ className = 'h-4 w-4' }: { className?: string }) {
|
||||
return (
|
||||
<svg className={`animate-spin ${className}`} viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4" />
|
||||
|
||||
@@ -9,7 +9,7 @@ interface DisclosureProps {
|
||||
export function Disclosure({ summary, children }: DisclosureProps) {
|
||||
return (
|
||||
<details className="glass-sm group">
|
||||
<summary className="flex cursor-pointer select-none items-center gap-2 px-4 py-2.5 text-xs font-medium text-gray-400 transition-colors hover:text-gray-200 [&::-webkit-details-marker]:hidden">
|
||||
<summary className="flex min-h-11 cursor-pointer select-none items-center gap-2 px-4 py-2.5 text-xs font-medium text-gray-400 transition-colors hover:text-gray-200 [&::-webkit-details-marker]:hidden">
|
||||
<span className="inline-block transition-transform duration-200 group-open:rotate-90">▸</span>
|
||||
{summary}
|
||||
</summary>
|
||||
|
||||
@@ -86,7 +86,12 @@ export function Dropdown({
|
||||
onClick={() => setOpen((v) => !v)}
|
||||
className="input-glass flex w-full items-center justify-between gap-2 px-3 py-1.5 text-left text-sm"
|
||||
>
|
||||
<span className={selected ? 'text-gray-200' : 'text-gray-500'}>
|
||||
{/* truncate, not wrap: a long option name used to push the trigger to
|
||||
three lines and shove the whole control row out of alignment. */}
|
||||
<span
|
||||
className={`truncate ${selected ? 'text-gray-200' : 'text-gray-500'}`}
|
||||
title={selected ? selected.label : undefined}
|
||||
>
|
||||
{selected ? selected.label : placeholder}
|
||||
</span>
|
||||
<svg
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
const pulse = 'animate-pulse rounded-lg bg-white/[0.05]';
|
||||
|
||||
export function SkeletonLine({ className = '' }: { className?: string }) {
|
||||
return <div className={`${pulse} h-4 w-full ${className}`} />;
|
||||
}
|
||||
|
||||
export function SkeletonCard({ className = '' }: { className?: string }) {
|
||||
return <div className={`${pulse} h-32 w-full ${className}`} />;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
BAND_STYLE,
|
||||
bandTicks,
|
||||
classifyMetric,
|
||||
meterFraction,
|
||||
} from '../../lib/metricBands';
|
||||
|
||||
/**
|
||||
* One labelled metric.
|
||||
*
|
||||
* Optionally carries a quality meter: pass `metric` (a key in METRIC_BANDS) and
|
||||
* the numeric `raw` value. The meter is the answer to "2.72 — is that good?" —
|
||||
* a track showing where the value sits, ticks at the band edges, and the band
|
||||
* word. Colour never travels alone; the word is always rendered beside it.
|
||||
*
|
||||
* Every tile is the same size. Hierarchy comes from grouping and section
|
||||
* labels, not from shrinking one row — two sizes read as inconsistent rather
|
||||
* than as a deliberate ranking.
|
||||
*/
|
||||
export function StatTile({
|
||||
label,
|
||||
value,
|
||||
valueClass = 'text-gray-100',
|
||||
sub,
|
||||
title,
|
||||
metric,
|
||||
raw,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
valueClass?: string;
|
||||
sub?: string;
|
||||
/** Native tooltip — how the metric is defined. */
|
||||
title?: string;
|
||||
/** Key into METRIC_BANDS; enables the quality meter. */
|
||||
metric?: string;
|
||||
/** Numeric value the meter reads (the formatted `value` is display-only). */
|
||||
raw?: number | null;
|
||||
}) {
|
||||
const band = metric ? classifyMetric(metric, raw) : null;
|
||||
const style = band ? BAND_STYLE[band] : null;
|
||||
|
||||
return (
|
||||
<div className="glass flex flex-col p-4" title={title}>
|
||||
<p className="section-index">{label}</p>
|
||||
<p className={`num mt-1.5 text-2xl font-semibold ${valueClass}`}>{value}</p>
|
||||
|
||||
{style && metric && (
|
||||
<div className="mt-2.5">
|
||||
<div className="relative h-1.5 overflow-hidden rounded-full bg-white/[0.07]">
|
||||
<div
|
||||
className={`h-full rounded-full ${style.fill}`}
|
||||
style={{ width: `${meterFraction(metric, raw) * 100}%` }}
|
||||
/>
|
||||
{/* Band edges — where "fair" becomes "good", and so on. */}
|
||||
{bandTicks(metric).map((t) => (
|
||||
<span
|
||||
key={t}
|
||||
className="absolute top-0 h-full w-px bg-black/50"
|
||||
style={{ left: `${t * 100}%` }}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<p className={`mt-1.5 text-[11px] font-medium ${style.text}`}>{style.label}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sub && <p className="mt-1 text-xs text-gray-500">{sub}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
/* Dev-only visual harness for FundamentalsPanel. Served at /harness.html by
|
||||
* `vite`. Not imported by the app. Renders the three key states so desktop and
|
||||
* mobile can be eyeballed with representative fixtures. */
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import '../styles/globals.css';
|
||||
import { FundamentalsPanel } from '../components/ticker/FundamentalsPanel';
|
||||
import type { FundamentalResponse, MetricItem } from '../lib/types';
|
||||
|
||||
function h(period: string, value: number | null) {
|
||||
return { period_end: period, value };
|
||||
}
|
||||
const P = ['2025-06-30', '2025-09-30', '2025-12-31', '2026-03-28'];
|
||||
|
||||
function dateFromToday(days: number): string {
|
||||
const date = new Date();
|
||||
date.setHours(12, 0, 0, 0);
|
||||
date.setDate(date.getDate() + days);
|
||||
return [
|
||||
date.getFullYear(),
|
||||
String(date.getMonth() + 1).padStart(2, '0'),
|
||||
String(date.getDate()).padStart(2, '0'),
|
||||
].join('-');
|
||||
}
|
||||
|
||||
function metric(key: string, value: number | null, hist: (number | null)[],
|
||||
industry: MetricItem['industry'] = null,
|
||||
caveat: string | null = null): MetricItem {
|
||||
return {
|
||||
key: key as MetricItem['key'], value,
|
||||
history: hist.map((v, i) => h(P[i], v)),
|
||||
industry, period_end: '2026-03-28', filed_date: '2026-05-01', caveat,
|
||||
source: 'sec',
|
||||
};
|
||||
}
|
||||
const ind = (median: number, favorable_percentile: number) =>
|
||||
({ label: 'SIC 35 peers', median, favorable_percentile, peer_count: 12 });
|
||||
|
||||
const legacy = {
|
||||
pe_ratio: null, revenue_growth: null, earnings_surprise: null, market_cap: null,
|
||||
next_earnings_date: null, fetched_at: null, unavailable_fields: {},
|
||||
};
|
||||
|
||||
const full: FundamentalResponse = {
|
||||
symbol: 'AAPL', ...legacy,
|
||||
earnings: {
|
||||
next: { date: dateFromToday(12), session: 'amc', days_until: 12 },
|
||||
recent: [
|
||||
{ announce_date: '2025-08-01', period_end: '2025-06-30', eps_estimate: 1.4, eps_actual: 1.6, surprise_pct: 14.3 },
|
||||
{ announce_date: '2025-11-01', period_end: '2025-09-30', eps_estimate: 1.7, eps_actual: 1.9, surprise_pct: 11.8 },
|
||||
{ announce_date: '2026-02-01', period_end: '2025-12-31', eps_estimate: 2.6, eps_actual: 2.4, surprise_pct: -7.7 },
|
||||
{ announce_date: '2026-05-01', period_end: '2026-03-28', eps_estimate: 1.5, eps_actual: 1.65, surprise_pct: 10.0 },
|
||||
],
|
||||
},
|
||||
metrics: [
|
||||
metric('revenue_growth_yoy', 18, [8, 11, 15, 18], ind(11, 82)),
|
||||
metric('eps_growth_yoy', 24, [10, 18, 22, 24], ind(15, 70)),
|
||||
metric('operating_margin', 32, [30, 31, 31, 32], ind(22, 88)),
|
||||
metric('fcf_margin', 28, [24, 25, 27, 28], ind(18, 80)),
|
||||
metric('net_debt', 16.2e9, [46e9, 44e9, 24e9, 16.2e9], null),
|
||||
metric('net_debt_to_ebitda', 1.4, [1.9, 1.7, 1.5, 1.4], ind(2.1, 68)),
|
||||
metric('share_count_change_yoy', -1.7, [-2.4, -2.2, -2.3, -1.7], null),
|
||||
],
|
||||
valuation: {
|
||||
pe: 29.2, fcf_yield: 3.8, market_cap_est: 3.2e12,
|
||||
pe_industry: ind(23.5, 30), fcf_yield_industry: ind(3.1, 70), price_date: '2026-05-01',
|
||||
},
|
||||
reads: {
|
||||
header: 'growth accelerating · margins improving · valuation priced above peers',
|
||||
by_key: {
|
||||
revenue_growth_yoy: 'accelerating', eps_growth_yoy: 'accelerating',
|
||||
operating_margin: 'improving', fcf_margin: 'improving',
|
||||
share_count_change_yoy: 'buying back', net_debt_to_ebitda: 'conservative leverage',
|
||||
pe: 'priced above peers', fcf_yield: 'above peers', net_debt: null,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const partial: FundamentalResponse = {
|
||||
symbol: 'NEWCO', ...legacy,
|
||||
earnings: { next: { date: dateFromToday(0), session: 'unknown', days_until: 0 }, recent: [] },
|
||||
metrics: [
|
||||
metric('revenue_growth_yoy', 12, [null, 8, 10, 12], null),
|
||||
metric(
|
||||
'eps_growth_yoy',
|
||||
null,
|
||||
[null, null, null, null],
|
||||
null,
|
||||
'Not comparable: share count changed at least 25%; possible split or corporate action.',
|
||||
),
|
||||
metric('operating_margin', 25, [24, 24, 25, 25], null),
|
||||
metric('fcf_margin', null, [null, null, null, null], null),
|
||||
metric('net_debt', null, [], null),
|
||||
metric('net_debt_to_ebitda', 1.9, [1.7, 1.8, 1.9, 1.9], null),
|
||||
metric(
|
||||
'share_count_change_yoy',
|
||||
null,
|
||||
[1.8, 2.0, 2.0, null],
|
||||
null,
|
||||
'Not comparable: share count changed at least 25%; possible split or corporate action.',
|
||||
),
|
||||
],
|
||||
valuation: {
|
||||
pe: 15.2, fcf_yield: null, market_cap_est: 5.4e8,
|
||||
pe_industry: null, fcf_yield_industry: null, price_date: '2026-05-01',
|
||||
},
|
||||
reads: {
|
||||
header: 'growth steady · margins stable',
|
||||
by_key: {
|
||||
revenue_growth_yoy: 'steady', operating_margin: 'stable',
|
||||
share_count_change_yoy: '2.1% dilution', net_debt_to_ebitda: null,
|
||||
pe: null, fcf_yield: null, eps_growth_yoy: null, fcf_margin: null, net_debt: null,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const empty: FundamentalResponse = {
|
||||
symbol: 'ADR', ...legacy,
|
||||
earnings: { next: null, recent: [] },
|
||||
metrics: [
|
||||
'revenue_growth_yoy', 'eps_growth_yoy', 'operating_margin', 'fcf_margin',
|
||||
'net_debt', 'net_debt_to_ebitda', 'share_count_change_yoy',
|
||||
].map((k) => metric(k, null, [])),
|
||||
valuation: null,
|
||||
reads: { header: null, by_key: {} },
|
||||
};
|
||||
|
||||
function Case({ title, data }: { title: string; data: FundamentalResponse }) {
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-1.5 text-[11px] uppercase tracking-widest text-gray-500">{title}</div>
|
||||
<FundamentalsPanel data={data} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<div className="mx-auto max-w-3xl space-y-8 p-6">
|
||||
<p className="text-[11px] uppercase tracking-widest text-gray-500">
|
||||
Desktop width (~768px, two columns). Resize the browser to ~390px to check mobile (single column).
|
||||
</p>
|
||||
<Case title="Full" data={full} />
|
||||
<Case title="Partial · insufficient peers" data={partial} />
|
||||
<Case title="Empty" data={empty} />
|
||||
</div>,
|
||||
);
|
||||
@@ -316,14 +316,6 @@ export function useJobs() {
|
||||
});
|
||||
}
|
||||
|
||||
export function useFundamentalsParityReport() {
|
||||
return useQuery({
|
||||
queryKey: ['admin', 'fundamentals-parity'],
|
||||
queryFn: () => adminApi.getFundamentalsParityReport(),
|
||||
refetchInterval: 15_000,
|
||||
});
|
||||
}
|
||||
|
||||
export function usePipelineReadiness() {
|
||||
return useQuery({
|
||||
queryKey: ['admin', 'pipeline-readiness'],
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user