Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
23de8c9540 | ||
|
|
c2a3b56aaa | ||
|
|
e1607ddbff | ||
|
|
13f3636b6a | ||
|
|
3e83d63b05 | ||
|
|
f5d4b516ab | ||
|
|
3ff0fd9f1c |
+3
-17
@@ -18,16 +18,9 @@ OPENAI_API_KEY=
|
|||||||
OPENAI_MODEL=gpt-4o-mini
|
OPENAI_MODEL=gpt-4o-mini
|
||||||
OPENAI_SENTIMENT_BATCH_SIZE=5
|
OPENAI_SENTIMENT_BATCH_SIZE=5
|
||||||
|
|
||||||
# Fundamentals Provider — Financial Modeling Prep
|
# Dolt bulk data — local clone of post-no-preference/earnings. Together with the
|
||||||
FMP_API_KEY=
|
# SEC EDGAR block below this is the ONLY fundamentals source; there is no
|
||||||
|
# provider-API fallback.
|
||||||
# Fundamentals Provider — Finnhub (optional fallback)
|
|
||||||
FINNHUB_API_KEY=
|
|
||||||
|
|
||||||
# Fundamentals Provider — Alpha Vantage (optional fallback)
|
|
||||||
ALPHA_VANTAGE_API_KEY=
|
|
||||||
|
|
||||||
# Dolt bulk data — local clone of post-no-preference/earnings (workstream A).
|
|
||||||
# DOLT_BINARY: path to the dolt CLI (set the full path in dev if it's not on PATH,
|
# DOLT_BINARY: path to the dolt CLI (set the full path in dev if it's not on PATH,
|
||||||
# e.g. Windows: C:\Program Files\Dolt\bin\dolt.exe). DOLT_DATA_DIR holds the
|
# e.g. Windows: C:\Program Files\Dolt\bin\dolt.exe). DOLT_DATA_DIR holds the
|
||||||
# clones; in PRODUCTION it MUST be outside the deploy tree (deploy is
|
# clones; in PRODUCTION it MUST be outside the deploy tree (deploy is
|
||||||
@@ -52,10 +45,6 @@ SEC_REQUEST_SPACING_SECONDS=0.2
|
|||||||
SEC_MAX_RETRIES=4
|
SEC_MAX_RETRIES=4
|
||||||
SEC_REQUEST_TIMEOUT_SECONDS=30.0
|
SEC_REQUEST_TIMEOUT_SECONDS=30.0
|
||||||
|
|
||||||
# A5 read-only parity report archive. In production keep this outside the
|
|
||||||
# rsync deployment tree, e.g. /var/lib/signal-platform/reports/fundamentals-parity.
|
|
||||||
FUNDAMENTALS_PARITY_REPORT_DIR=reports/fundamentals-parity
|
|
||||||
|
|
||||||
# Regime Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
# Regime Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||||
# Optional: without it the volatility (V1) and credit (C1) pillars show as n/a.
|
# Optional: without it the volatility (V1) and credit (C1) pillars show as n/a.
|
||||||
FRED_API_KEY=
|
FRED_API_KEY=
|
||||||
@@ -63,10 +52,7 @@ FRED_API_KEY=
|
|||||||
# Scheduled Jobs
|
# Scheduled Jobs
|
||||||
DATA_COLLECTOR_FREQUENCY=daily
|
DATA_COLLECTOR_FREQUENCY=daily
|
||||||
SENTIMENT_POLL_INTERVAL_MINUTES=30
|
SENTIMENT_POLL_INTERVAL_MINUTES=30
|
||||||
FUNDAMENTAL_FETCH_FREQUENCY=daily
|
|
||||||
RR_SCAN_FREQUENCY=daily
|
RR_SCAN_FREQUENCY=daily
|
||||||
FUNDAMENTAL_RATE_LIMIT_RETRIES=3
|
|
||||||
FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS=15
|
|
||||||
|
|
||||||
# Scoring Defaults
|
# Scoring Defaults
|
||||||
DEFAULT_WATCHLIST_AUTO_SIZE=10
|
DEFAULT_WATCHLIST_AUTO_SIZE=10
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ Hourly mid-session (Mon–Fri ~10:00–15:00 ET): only **OHLCV → Outcome Eval*
|
|||||||
|
|
||||||
### Other jobs
|
### Other jobs
|
||||||
|
|
||||||
Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs).
|
Dolt earnings import (daily 02:30 ET) · SEC fundamentals import (daily 04:00 ET, also refreshes the fundamentals cache scoring reads) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs).
|
||||||
|
|
||||||
### From score to "top pick"
|
### From score to "top pick"
|
||||||
|
|
||||||
@@ -255,18 +255,11 @@ A systematic single-variable sweep (offline prod snapshot, production gate/rank/
|
|||||||
| ATR trail multiple {1.5–4.0} | **Keep 3.0** | Return+Sharpe peak; ≤2.0 whipsaws out the momentum right tail; ≥2.5 is a plateau |
|
| ATR trail multiple {1.5–4.0} | **Keep 3.0** | Return+Sharpe peak; ≤2.0 whipsaws out the momentum right tail; ≥2.5 is a plateau |
|
||||||
| SPY 200d-MA regime overlay (block entries / go flat) | **Reject** | Halves return (315%→138%) with zero drawdown benefit — the ATR trail already manages downside, and the filter blocks the recovery-phase entries that make the money |
|
| SPY 200d-MA regime overlay (block entries / go flat) | **Reject** | Halves return (315%→138%) with zero drawdown benefit — the ATR trail already manages downside, and the filter blocks the recovery-phase entries that make the money |
|
||||||
| Momentum lookback: 6-1, 3-1, 12-7 (Novy-Marx), composites | **Keep residual 12-1** | 6-1/3-1 rank-IC ≈ 0; 12-7 IC 0.045 / t 1.58 — weaker than residual 12-1 (0.055 / t 1.98) |
|
| Momentum lookback: 6-1, 3-1, 12-7 (Novy-Marx), composites | **Keep residual 12-1** | 6-1/3-1 rank-IC ≈ 0; 12-7 IC 0.045 / t 1.58 — weaker than residual 12-1 (0.055 / t 1.98) |
|
||||||
| Selection cutoff {70, 75, 85, 90} × book size {10, 15, 20} | **Keep cutoff 80; capacity reopened** | The older weekly replay favored 80 × 10, but its no-cap-pressure conclusion is superseded by 519 book-full rejections versus 472 trades under the current daily gate-reset control |
|
| Selection cutoff {70, 75, 85, 90} × book size {10, 15, 20} | **Keep 80 × 10** | Monotonically worse in both directions from 80; the 10-slot cap never binds (<10 concurrent) |
|
||||||
| Position sizing: equal-weight, inverse-vol, risk-% sweep | **Keep 1% fixed-fractional** | See the inverse-vol warning below |
|
| Position sizing: equal-weight, inverse-vol, risk-% sweep | **Keep 1% fixed-fractional** | See the inverse-vol warning below |
|
||||||
| Post-stop re-entry: immediate, fixed 2–5 sessions, gate resets, confirmation filters | **Keep normal gate reset for the 10-position production book** | Sharpe 1.77 vs 1.67 immediate and 1.47 cooldown 5; rerun before changing portfolio capacity |
|
| Post-stop re-entry: immediate, fixed 2–5 sessions, gate resets, confirmation filters | **Keep normal gate reset for the 10-position production book** | Sharpe 1.77 vs 1.67 immediate and 1.47 cooldown 5; rerun before changing portfolio capacity |
|
||||||
| FIP path-smoothness as an in-book tie-breaker/filter | **Reject** (but see the lead below) | Non-monotonic across FIP quintiles within the qualified set; either half of a median split underperforms the full book — thinning the entry stream costs more compounding than the tilt returns |
|
| FIP path-smoothness as an in-book tie-breaker/filter | **Reject** (but see the lead below) | Non-monotonic across FIP quintiles within the qualified set; either half of a median split underperforms the full book — thinning the entry stream costs more compounding than the tilt returns |
|
||||||
|
|
||||||
> **Capacity correction (2026-08-05):** the table's older weekly conclusion
|
|
||||||
> that the ten-slot cap never binds is superseded. Under the current daily
|
|
||||||
> gate-reset Phase A control, 472 trades were admitted and 519 qualified entries
|
|
||||||
> were rejected because the book was full (52.4% of admitted+blocked
|
|
||||||
> opportunities). Cutoff 80 remains the signal setting; portfolio capacity is
|
|
||||||
> reopened in the focused capacity-bracket study.
|
|
||||||
|
|
||||||
Two findings future sessions must not re-litigate:
|
Two findings future sessions must not re-litigate:
|
||||||
|
|
||||||
- **The "inverse-vol sizing win" (July 2026) was mis-attributed — do not resurrect.** The diagnostic sized `notional = equity × 1% / vol_6m`, and the 20% notional cap bound on 95% of entries, so it actually measured "~5 positions × 20% notional each" — a concentration/risk-appetite bump economically equivalent to raising risk to 1.5%, not vol-managed sizing. Genuine inverse-vol sizing (risk budget × median-vol/vol) cuts max drawdown to −18.2% but costs ~58pp total return at flat Sharpe: a risk-preference trade, not edge.
|
- **The "inverse-vol sizing win" (July 2026) was mis-attributed — do not resurrect.** The diagnostic sized `notional = equity × 1% / vol_6m`, and the 20% notional cap bound on 95% of entries, so it actually measured "~5 positions × 20% notional each" — a concentration/risk-appetite bump economically equivalent to raising risk to 1.5%, not vol-managed sizing. Genuine inverse-vol sizing (risk budget × median-vol/vol) cuts max drawdown to −18.2% but costs ~58pp total return at flat Sharpe: a risk-preference trade, not edge.
|
||||||
@@ -308,13 +301,13 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m
|
|||||||
| Charts | Canvas 2D candlestick chart with S/R overlays |
|
| Charts | Canvas 2D candlestick chart with S/R overlays |
|
||||||
| Routing | React Router v6 (SPA) |
|
| Routing | React Router v6 (SPA) |
|
||||||
| HTTP | Axios with JWT interceptor |
|
| HTTP | Axios with JWT interceptor |
|
||||||
| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); Fundamentals chain: FMP → Finnhub → Alpha Vantage; FRED (regime); Telegram (alerts) |
|
| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); SEC EDGAR Company Facts + DoltHub earnings (fundamentals, bulk import); FRED (regime); Telegram (alerts) |
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
### Backend
|
### Backend
|
||||||
- Ticker registry with full cascade delete
|
- Ticker registry with full cascade delete
|
||||||
- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint
|
- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint — free public sources (Wikipedia / NASDAQ Trader), then the cached snapshot, then a built-in seed list. The seeds are representative, not complete, so a *fresh* install bootstrapped while the public source is unreachable gets a partial universe; a warm instance falls through to its cache.
|
||||||
- OHLCV price storage with upsert and validation
|
- OHLCV price storage with upsert and validation
|
||||||
- Technical indicators: ADX, EMA, RSI, ATR, Volume Profile, Pivot Points, EMA Cross
|
- Technical indicators: ADX, EMA, RSI, ATR, Volume Profile, Pivot Points, EMA Cross
|
||||||
- Structural Support/Resistance detection with rejection/recency strength, ATR-adaptive merging and a hard cap; persisted for charts and alerts
|
- Structural Support/Resistance detection with rejection/recency strength, ATR-adaptive merging and a hard cap; persisted for charts and alerts
|
||||||
@@ -590,18 +583,12 @@ Configure in `.env` (copy from `.env.example`):
|
|||||||
| `OPENAI_API_KEY` | For sentiment (OpenAI path) | — | OpenAI API key |
|
| `OPENAI_API_KEY` | For sentiment (OpenAI path) | — | OpenAI API key |
|
||||||
| `OPENAI_MODEL` | No | `gpt-4o-mini` | OpenAI model name |
|
| `OPENAI_MODEL` | No | `gpt-4o-mini` | OpenAI model name |
|
||||||
| `OPENAI_SENTIMENT_BATCH_SIZE` | No | `5` | Micro-batch size for sentiment collector |
|
| `OPENAI_SENTIMENT_BATCH_SIZE` | No | `5` | Micro-batch size for sentiment collector |
|
||||||
| `FMP_API_KEY` | Optional (fundamentals) | — | Financial Modeling Prep API key (first provider in chain) |
|
|
||||||
| `FINNHUB_API_KEY` | Optional (fundamentals) | — | Finnhub API key (fallback provider) |
|
|
||||||
| `ALPHA_VANTAGE_API_KEY` | Optional (fundamentals) | — | Alpha Vantage API key (fallback provider) |
|
|
||||||
| `FRED_API_KEY` | Optional (regime) | — | FRED key for the regime monitor (VIX, credit spreads) |
|
| `FRED_API_KEY` | Optional (regime) | — | FRED key for the regime monitor (VIX, credit spreads) |
|
||||||
| `TELEGRAM_BOT_TOKEN` | Optional (alerts) | — | Telegram bot token for alerts (can also be set in Admin) |
|
| `TELEGRAM_BOT_TOKEN` | Optional (alerts) | — | Telegram bot token for alerts (can also be set in Admin) |
|
||||||
| `TELEGRAM_CHAT_ID` | Optional (alerts) | — | Telegram chat id for alerts |
|
| `TELEGRAM_CHAT_ID` | Optional (alerts) | — | Telegram chat id for alerts |
|
||||||
| `DATA_COLLECTOR_FREQUENCY` | No | `daily` | OHLCV collection schedule (legacy — see note below) |
|
| `DATA_COLLECTOR_FREQUENCY` | No | `daily` | OHLCV collection schedule (legacy — see note below) |
|
||||||
| `SENTIMENT_POLL_INTERVAL_MINUTES` | No | `30` | Sentiment polling interval |
|
| `SENTIMENT_POLL_INTERVAL_MINUTES` | No | `30` | Sentiment polling interval |
|
||||||
| `FUNDAMENTAL_FETCH_FREQUENCY` | No | `weekly` | Fundamentals fetch cadence |
|
|
||||||
| `RR_SCAN_FREQUENCY` | No | `daily` | R:R scanner schedule |
|
| `RR_SCAN_FREQUENCY` | No | `daily` | R:R scanner schedule |
|
||||||
| `FUNDAMENTAL_RATE_LIMIT_RETRIES` | No | `3` | Retries per ticker on fundamentals rate-limit |
|
|
||||||
| `FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS` | No | `15` | Base backoff seconds for fundamentals retry (exponential) |
|
|
||||||
| `DEFAULT_WATCHLIST_AUTO_SIZE` | No | `10` | Auto-watchlist size |
|
| `DEFAULT_WATCHLIST_AUTO_SIZE` | No | `10` | Auto-watchlist size |
|
||||||
| `DEFAULT_RR_THRESHOLD` | No | `1.5` | Minimum R:R ratio for setups |
|
| `DEFAULT_RR_THRESHOLD` | No | `1.5` | Minimum R:R ratio for setups |
|
||||||
| `DB_POOL_SIZE` | No | `5` | Database connection pool size |
|
| `DB_POOL_SIZE` | No | `5` | Database connection pool size |
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
"""Retire the legacy fundamentals settings (A6)
|
||||||
|
|
||||||
|
Revision ID: 029
|
||||||
|
Revises: 028
|
||||||
|
Create Date: 2026-08-07 00:00:00.000000
|
||||||
|
|
||||||
|
A6 removed the FMP/Finnhub/Alpha Vantage providers, the weekly
|
||||||
|
``fundamental_collector`` job and the A5 parity report. Five SystemSetting rows
|
||||||
|
are left over. They are NOT all deleted, because the deploy runs migrations
|
||||||
|
before restarting the service: for a short window — and for the whole of any
|
||||||
|
rollback — pre-A6 code is still live, and it reads absent rows permissively
|
||||||
|
(cutover absent -> disabled; ``job_<name>_enabled`` absent -> enabled). Deleting
|
||||||
|
both would hand a rolled-back process a re-armed legacy collector writing over
|
||||||
|
the SEC/Dolt cache.
|
||||||
|
|
||||||
|
So the two rows that carry behavior become tombstones pinned to the safe value,
|
||||||
|
and only the inert ones are deleted. The tombstones are dropped in a later
|
||||||
|
release once the rollback window has closed; ``SettingsForm`` hides them
|
||||||
|
meanwhile.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "029"
|
||||||
|
down_revision: Union[str, None] = "028"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
# Behavior-bearing under pre-A6 code -> pin to the safe value, keep the row.
|
||||||
|
_TOMBSTONES: dict[str, str] = {
|
||||||
|
"fundamental_data_sec_dolt_cutover_enabled": "true",
|
||||||
|
"job_fundamental_collector_enabled": "false",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Inert either way: an absent cron falls back to a default for a job that no
|
||||||
|
# longer registers, and the parity report never wrote anything.
|
||||||
|
_OBSOLETE: tuple[str, ...] = (
|
||||||
|
"schedule_fundamentals_cron",
|
||||||
|
"schedule_fundamentals_parity_cron",
|
||||||
|
"job_fundamentals_parity_report_enabled",
|
||||||
|
)
|
||||||
|
|
||||||
|
_settings = sa.table(
|
||||||
|
"system_settings",
|
||||||
|
sa.column("id", sa.Integer),
|
||||||
|
sa.column("key", sa.String),
|
||||||
|
sa.column("value", sa.Text),
|
||||||
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
now = sa.func.now()
|
||||||
|
|
||||||
|
for key, pinned in _TOMBSTONES.items():
|
||||||
|
row = conn.execute(
|
||||||
|
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||||
|
).fetchone()
|
||||||
|
old_value = row[0] if row is not None else None
|
||||||
|
print(f"a6_tombstone {key}: {old_value!r} -> {pinned!r}", flush=True)
|
||||||
|
if row is None:
|
||||||
|
conn.execute(
|
||||||
|
sa.insert(_settings).values(key=key, value=pinned, updated_at=now)
|
||||||
|
)
|
||||||
|
elif old_value != pinned:
|
||||||
|
conn.execute(
|
||||||
|
sa.update(_settings)
|
||||||
|
.where(_settings.c.key == key)
|
||||||
|
.values(value=pinned, updated_at=now)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Print the value before deleting — a bare DELETE cannot be undone from the
|
||||||
|
# migration output.
|
||||||
|
for key in _OBSOLETE:
|
||||||
|
row = conn.execute(
|
||||||
|
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
print(f"a6_delete {key}: absent", flush=True)
|
||||||
|
continue
|
||||||
|
print(f"a6_delete {key}: {row[0]!r}", flush=True)
|
||||||
|
conn.execute(sa.delete(_settings).where(_settings.c.key == key))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""No-op.
|
||||||
|
|
||||||
|
The deleted rows configured jobs this revision's code no longer registers,
|
||||||
|
and the tombstones already hold the values pre-A6 code needs. Recreating
|
||||||
|
them would restore nothing useful; the printed values above cover recovery.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""Drop the A6 rollback tombstones
|
||||||
|
|
||||||
|
Revision ID: 030
|
||||||
|
Revises: 029
|
||||||
|
Create Date: 2026-08-07 00:00:00.000000
|
||||||
|
|
||||||
|
Migration ``029`` kept two SystemSetting rows alive as rollback tombstones,
|
||||||
|
pinned to the values a pre-A6 process needed to behave safely. A6 is deployed
|
||||||
|
and healthy, and the provider keys are gone from the production ``.env`` — which
|
||||||
|
makes the legacy collector inert regardless of any settings row — so the
|
||||||
|
tombstones have no remaining job.
|
||||||
|
|
||||||
|
Nothing in the current codebase reads either key.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "030"
|
||||||
|
down_revision: Union[str, None] = "029"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
# The safe values 029 pinned. Kept here so downgrade restores real protection
|
||||||
|
# rather than leaving a rolled-back process reading absent rows permissively.
|
||||||
|
_TOMBSTONES: dict[str, str] = {
|
||||||
|
"fundamental_data_sec_dolt_cutover_enabled": "true",
|
||||||
|
"job_fundamental_collector_enabled": "false",
|
||||||
|
}
|
||||||
|
|
||||||
|
_settings = sa.table(
|
||||||
|
"system_settings",
|
||||||
|
sa.column("id", sa.Integer),
|
||||||
|
sa.column("key", sa.String),
|
||||||
|
sa.column("value", sa.Text),
|
||||||
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
for key in _TOMBSTONES:
|
||||||
|
row = conn.execute(
|
||||||
|
sa.select(_settings.c.value).where(_settings.c.key == key)
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
print(f"a6_tombstone_drop {key}: absent", flush=True)
|
||||||
|
continue
|
||||||
|
print(f"a6_tombstone_drop {key}: {row[0]!r}", flush=True)
|
||||||
|
conn.execute(sa.delete(_settings).where(_settings.c.key == key))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Restore the tombstones at their safe values.
|
||||||
|
|
||||||
|
Unlike 029's no-op downgrade, this one is meaningful: going back past this
|
||||||
|
revision implies going back toward code that still reads these keys.
|
||||||
|
"""
|
||||||
|
conn = op.get_bind()
|
||||||
|
now = sa.func.now()
|
||||||
|
for key, pinned in _TOMBSTONES.items():
|
||||||
|
exists = conn.execute(
|
||||||
|
sa.select(_settings.c.id).where(_settings.c.key == key)
|
||||||
|
).fetchone()
|
||||||
|
if exists is None:
|
||||||
|
conn.execute(
|
||||||
|
sa.insert(_settings).values(key=key, value=pinned, updated_at=now)
|
||||||
|
)
|
||||||
@@ -28,15 +28,6 @@ class Settings(BaseSettings):
|
|||||||
deepseek_api_key: str = ""
|
deepseek_api_key: str = ""
|
||||||
xai_api_key: str = ""
|
xai_api_key: str = ""
|
||||||
|
|
||||||
# Fundamentals Provider — Financial Modeling Prep
|
|
||||||
fmp_api_key: str = ""
|
|
||||||
|
|
||||||
# Fundamentals Provider — Finnhub (optional fallback)
|
|
||||||
finnhub_api_key: str = ""
|
|
||||||
|
|
||||||
# Fundamentals Provider — Alpha Vantage (optional fallback)
|
|
||||||
alpha_vantage_api_key: str = ""
|
|
||||||
|
|
||||||
# Dolt bulk-data — local clone of post-no-preference/earnings (workstream A).
|
# Dolt bulk-data — local clone of post-no-preference/earnings (workstream A).
|
||||||
# dolt_binary: full path when not on PATH (dev/Windows install). dolt_data_dir
|
# dolt_binary: full path when not on PATH (dev/Windows install). dolt_data_dir
|
||||||
# holds the clones; in production it MUST be outside the deploy tree (deploy is
|
# holds the clones; in production it MUST be outside the deploy tree (deploy is
|
||||||
@@ -61,10 +52,6 @@ class Settings(BaseSettings):
|
|||||||
sec_max_retries: int = 4
|
sec_max_retries: int = 4
|
||||||
sec_request_timeout_seconds: float = 30.0
|
sec_request_timeout_seconds: float = 30.0
|
||||||
|
|
||||||
# A5 read-only comparison artifacts. Production must keep this outside the
|
|
||||||
# rsync deployment tree so the 5-7 day review window survives deploys.
|
|
||||||
fundamentals_parity_report_dir: str = "reports/fundamentals-parity"
|
|
||||||
|
|
||||||
# Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
# Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
||||||
# the volatility (P5) and credit-spread (F2) signals are reported as n/a.
|
# the volatility (P5) and credit-spread (F2) signals are reported as n/a.
|
||||||
fred_api_key: str = ""
|
fred_api_key: str = ""
|
||||||
@@ -86,15 +73,8 @@ class Settings(BaseSettings):
|
|||||||
# the score window is 7 days).
|
# the score window is 7 days).
|
||||||
sentiment_fresh_hours: int = 120
|
sentiment_fresh_hours: int = 120
|
||||||
sentiment_top_composite: int = 30
|
sentiment_top_composite: int = 30
|
||||||
fundamental_fetch_frequency: str = "weekly" # quarterly-ish data; weekly conserves API quota
|
|
||||||
rr_scan_frequency: str = "daily" # legacy label; qualifying scan is cron near-close
|
rr_scan_frequency: str = "daily" # legacy label; qualifying scan is cron near-close
|
||||||
# alerts_frequency removed: alerts fire only via morning + near-close pipelines
|
# alerts_frequency removed: alerts fire only via morning + near-close pipelines
|
||||||
fundamental_rate_limit_retries: int = 3
|
|
||||||
fundamental_rate_limit_backoff_seconds: int = 15
|
|
||||||
# Pause between tickers in the bulk fundamentals job. Free tiers throttle
|
|
||||||
# hard (Finnhub ~60 calls/min, ~3 calls/ticker → ~3s/ticker); without
|
|
||||||
# spacing the job bursts straight into 429s. 0 disables.
|
|
||||||
fundamental_request_spacing_seconds: float = 3.0
|
|
||||||
|
|
||||||
# Scoring Defaults
|
# Scoring Defaults
|
||||||
default_watchlist_auto_size: int = 10
|
default_watchlist_auto_size: int = 10
|
||||||
|
|||||||
@@ -1,174 +0,0 @@
|
|||||||
"""Financial Modeling Prep (FMP) fundamentals provider using httpx.
|
|
||||||
|
|
||||||
Uses the stable API endpoints (https://financialmodelingprep.com/stable/)
|
|
||||||
which replaced the legacy /api/v3/ endpoints deprecated in Aug 2025.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.exceptions import ProviderError, RateLimitError
|
|
||||||
from app.providers.protocol import FundamentalData
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_FMP_STABLE_URL = "https://financialmodelingprep.com/stable"
|
|
||||||
|
|
||||||
# Resolve CA bundle for explicit httpx verify
|
|
||||||
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
|
|
||||||
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
|
|
||||||
_CA_BUNDLE_PATH: str | bool = True # use system default
|
|
||||||
else:
|
|
||||||
_CA_BUNDLE_PATH = _CA_BUNDLE
|
|
||||||
|
|
||||||
|
|
||||||
class FMPFundamentalProvider:
|
|
||||||
"""Fetches fundamental data from Financial Modeling Prep REST API."""
|
|
||||||
|
|
||||||
def __init__(self, api_key: str) -> None:
|
|
||||||
if not api_key:
|
|
||||||
raise ProviderError("FMP API key is required")
|
|
||||||
self._api_key = api_key
|
|
||||||
|
|
||||||
# Mapping from FMP endpoint name to the FundamentalData field it populates
|
|
||||||
_ENDPOINT_FIELD_MAP: dict[str, str] = {
|
|
||||||
"ratios-ttm": "pe_ratio",
|
|
||||||
"financial-growth": "revenue_growth",
|
|
||||||
"earnings": "earnings_surprise",
|
|
||||||
}
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
"""Fetch P/E, revenue growth, earnings surprise, and market cap.
|
|
||||||
|
|
||||||
Fetches from multiple stable endpoints. If a supplementary endpoint
|
|
||||||
(ratios, growth, earnings) returns 402 (paid tier), we gracefully
|
|
||||||
degrade and return partial data rather than failing entirely, and
|
|
||||||
record the affected field in ``unavailable_fields``.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
endpoints_402: set[str] = set()
|
|
||||||
|
|
||||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
|
||||||
params = {"symbol": ticker, "apikey": self._api_key}
|
|
||||||
|
|
||||||
# Profile is the primary source — must succeed
|
|
||||||
profile = await self._fetch_json(client, "profile", params, ticker)
|
|
||||||
|
|
||||||
# Supplementary sources — degrade gracefully on 402
|
|
||||||
ratios, was_402 = await self._fetch_json_optional(client, "ratios-ttm", params, ticker)
|
|
||||||
if was_402:
|
|
||||||
endpoints_402.add("ratios-ttm")
|
|
||||||
|
|
||||||
growth, was_402 = await self._fetch_json_optional(client, "financial-growth", params, ticker)
|
|
||||||
if was_402:
|
|
||||||
endpoints_402.add("financial-growth")
|
|
||||||
|
|
||||||
earnings, was_402 = await self._fetch_json_optional(client, "earnings", params, ticker)
|
|
||||||
if was_402:
|
|
||||||
endpoints_402.add("earnings")
|
|
||||||
|
|
||||||
pe_ratio = self._safe_float(ratios.get("priceToEarningsRatioTTM"))
|
|
||||||
revenue_growth = self._safe_float(growth.get("revenueGrowth"))
|
|
||||||
market_cap = self._safe_float(profile.get("marketCap"))
|
|
||||||
earnings_surprise = self._compute_earnings_surprise(earnings)
|
|
||||||
|
|
||||||
# Build unavailable_fields from 402 endpoints
|
|
||||||
unavailable_fields: dict[str, str] = {
|
|
||||||
self._ENDPOINT_FIELD_MAP[ep]: "requires paid plan"
|
|
||||||
for ep in endpoints_402
|
|
||||||
if ep in self._ENDPOINT_FIELD_MAP
|
|
||||||
}
|
|
||||||
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker,
|
|
||||||
pe_ratio=pe_ratio,
|
|
||||||
revenue_growth=revenue_growth,
|
|
||||||
earnings_surprise=earnings_surprise,
|
|
||||||
market_cap=market_cap,
|
|
||||||
fetched_at=datetime.now(timezone.utc),
|
|
||||||
unavailable_fields=unavailable_fields,
|
|
||||||
)
|
|
||||||
|
|
||||||
except (ProviderError, RateLimitError):
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
logger.error("FMP provider error for %s: %s", ticker, exc)
|
|
||||||
raise ProviderError(f"FMP provider error for {ticker}: {exc}") from exc
|
|
||||||
|
|
||||||
async def _fetch_json(
|
|
||||||
self,
|
|
||||||
client: httpx.AsyncClient,
|
|
||||||
endpoint: str,
|
|
||||||
params: dict,
|
|
||||||
ticker: str,
|
|
||||||
) -> dict:
|
|
||||||
"""Fetch a stable endpoint and return the first item (or empty dict)."""
|
|
||||||
url = f"{_FMP_STABLE_URL}/{endpoint}"
|
|
||||||
resp = await client.get(url, params=params)
|
|
||||||
self._check_response(resp, ticker, endpoint)
|
|
||||||
data = resp.json()
|
|
||||||
if isinstance(data, list):
|
|
||||||
return data[0] if data else {}
|
|
||||||
return data if isinstance(data, dict) else {}
|
|
||||||
|
|
||||||
async def _fetch_json_optional(
|
|
||||||
self,
|
|
||||||
client: httpx.AsyncClient,
|
|
||||||
endpoint: str,
|
|
||||||
params: dict,
|
|
||||||
ticker: str,
|
|
||||||
) -> tuple[dict, bool]:
|
|
||||||
"""Fetch a stable endpoint, returning ``({}, True)`` on 402 (paid tier).
|
|
||||||
|
|
||||||
Returns a tuple of (data_dict, was_402) so callers can track which
|
|
||||||
endpoints required a paid plan.
|
|
||||||
"""
|
|
||||||
url = f"{_FMP_STABLE_URL}/{endpoint}"
|
|
||||||
resp = await client.get(url, params=params)
|
|
||||||
if resp.status_code == 402:
|
|
||||||
logger.warning("FMP %s requires paid plan — skipping for %s", endpoint, ticker)
|
|
||||||
return {}, True
|
|
||||||
self._check_response(resp, ticker, endpoint)
|
|
||||||
data = resp.json()
|
|
||||||
if isinstance(data, list):
|
|
||||||
return (data[0] if data else {}, False)
|
|
||||||
return (data if isinstance(data, dict) else {}, False)
|
|
||||||
|
|
||||||
def _compute_earnings_surprise(self, earnings_data: dict) -> float | None:
|
|
||||||
"""Compute earnings surprise % from the most recent actual vs estimated EPS."""
|
|
||||||
actual = self._safe_float(earnings_data.get("epsActual"))
|
|
||||||
estimated = self._safe_float(earnings_data.get("epsEstimated"))
|
|
||||||
if actual is None or estimated is None or estimated == 0:
|
|
||||||
return None
|
|
||||||
return ((actual - estimated) / abs(estimated)) * 100
|
|
||||||
|
|
||||||
def _check_response(
|
|
||||||
self, resp: httpx.Response, ticker: str, endpoint: str
|
|
||||||
) -> None:
|
|
||||||
"""Raise appropriate errors for non-200 responses."""
|
|
||||||
if resp.status_code == 429:
|
|
||||||
raise RateLimitError(f"FMP rate limit hit for {ticker} ({endpoint})")
|
|
||||||
if resp.status_code == 403:
|
|
||||||
raise ProviderError(
|
|
||||||
f"FMP {endpoint} access denied for {ticker}: HTTP 403 — check API key validity and plan tier"
|
|
||||||
)
|
|
||||||
if resp.status_code != 200:
|
|
||||||
raise ProviderError(
|
|
||||||
f"FMP {endpoint} error for {ticker}: HTTP {resp.status_code}"
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _safe_float(value: object) -> float | None:
|
|
||||||
"""Convert a value to float, returning None on failure."""
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return float(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
@@ -1,354 +0,0 @@
|
|||||||
"""Chained fundamentals provider with fallback adapters.
|
|
||||||
|
|
||||||
Order:
|
|
||||||
1) FMP (if configured)
|
|
||||||
2) Finnhub (if configured)
|
|
||||||
3) Alpha Vantage (if configured)
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
from datetime import date, datetime, timedelta, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.config import settings
|
|
||||||
from app.exceptions import ProviderError, RateLimitError
|
|
||||||
from app.providers.fmp import FMPFundamentalProvider
|
|
||||||
from app.providers.protocol import FundamentalData, FundamentalProvider
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
|
|
||||||
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
|
|
||||||
_CA_BUNDLE_PATH: str | bool = True
|
|
||||||
else:
|
|
||||||
_CA_BUNDLE_PATH = _CA_BUNDLE
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_float(value: object) -> float | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return float(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _to_api_symbol(symbol: str) -> str:
|
|
||||||
"""Convert internal symbol format (BRK-B) to API format (BRK.B).
|
|
||||||
|
|
||||||
Finnhub and Alpha Vantage use dot-separated share class notation.
|
|
||||||
"""
|
|
||||||
return symbol.replace("-", ".")
|
|
||||||
|
|
||||||
|
|
||||||
class FinnhubFundamentalProvider:
|
|
||||||
"""Fundamentals provider backed by Finnhub free endpoints."""
|
|
||||||
|
|
||||||
def __init__(self, api_key: str) -> None:
|
|
||||||
if not api_key:
|
|
||||||
raise ProviderError("Finnhub API key is required")
|
|
||||||
self._api_key = api_key
|
|
||||||
self._base_url = "https://finnhub.io/api/v1"
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
unavailable: dict[str, str] = {}
|
|
||||||
api_symbol = _to_api_symbol(ticker)
|
|
||||||
|
|
||||||
today = date.today()
|
|
||||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
|
||||||
profile_resp = await client.get(
|
|
||||||
f"{self._base_url}/stock/profile2",
|
|
||||||
params={"symbol": api_symbol, "token": self._api_key},
|
|
||||||
)
|
|
||||||
metric_resp = await client.get(
|
|
||||||
f"{self._base_url}/stock/metric",
|
|
||||||
params={"symbol": api_symbol, "metric": "all", "token": self._api_key},
|
|
||||||
)
|
|
||||||
earnings_resp = await client.get(
|
|
||||||
f"{self._base_url}/stock/earnings",
|
|
||||||
params={"symbol": api_symbol, "limit": 1, "token": self._api_key},
|
|
||||||
)
|
|
||||||
calendar_resp = await client.get(
|
|
||||||
f"{self._base_url}/calendar/earnings",
|
|
||||||
params={
|
|
||||||
"symbol": api_symbol,
|
|
||||||
"from": today.isoformat(),
|
|
||||||
"to": (today + timedelta(days=120)).isoformat(),
|
|
||||||
"token": self._api_key,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
for resp, endpoint in (
|
|
||||||
(profile_resp, "profile2"),
|
|
||||||
(metric_resp, "stock/metric"),
|
|
||||||
(earnings_resp, "stock/earnings"),
|
|
||||||
(calendar_resp, "calendar/earnings"),
|
|
||||||
):
|
|
||||||
if resp.status_code == 429:
|
|
||||||
raise RateLimitError(f"Finnhub rate limit hit for {ticker} ({endpoint})")
|
|
||||||
if resp.status_code in (401, 403):
|
|
||||||
raise ProviderError(f"Finnhub access denied for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
|
||||||
if resp.status_code != 200:
|
|
||||||
raise ProviderError(f"Finnhub error for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
|
||||||
|
|
||||||
profile_payload = profile_resp.json() if profile_resp.text else {}
|
|
||||||
metric_payload = metric_resp.json() if metric_resp.text else {}
|
|
||||||
earnings_payload = earnings_resp.json() if earnings_resp.text else []
|
|
||||||
|
|
||||||
metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {}
|
|
||||||
# Finnhub profile2 marketCapitalization is in millions of USD.
|
|
||||||
# Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage.
|
|
||||||
market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization"))
|
|
||||||
market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None
|
|
||||||
pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual"))
|
|
||||||
revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y"))
|
|
||||||
|
|
||||||
earnings_surprise = None
|
|
||||||
if isinstance(earnings_payload, list) and earnings_payload:
|
|
||||||
first = earnings_payload[0] if isinstance(earnings_payload[0], dict) else {}
|
|
||||||
earnings_surprise = _safe_float(first.get("surprisePercent"))
|
|
||||||
|
|
||||||
next_earnings_date = self._next_earnings(calendar_resp)
|
|
||||||
|
|
||||||
if pe_ratio is None:
|
|
||||||
unavailable["pe_ratio"] = "not available from provider payload"
|
|
||||||
if revenue_growth is None:
|
|
||||||
unavailable["revenue_growth"] = "not available from provider payload"
|
|
||||||
if earnings_surprise is None:
|
|
||||||
unavailable["earnings_surprise"] = "not available from provider payload"
|
|
||||||
if market_cap is None:
|
|
||||||
unavailable["market_cap"] = "not available from provider payload"
|
|
||||||
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker,
|
|
||||||
pe_ratio=pe_ratio,
|
|
||||||
revenue_growth=revenue_growth,
|
|
||||||
earnings_surprise=earnings_surprise,
|
|
||||||
market_cap=market_cap,
|
|
||||||
fetched_at=datetime.now(timezone.utc),
|
|
||||||
next_earnings_date=next_earnings_date,
|
|
||||||
unavailable_fields=unavailable,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _next_earnings(resp: httpx.Response) -> date | None:
|
|
||||||
"""Earliest upcoming earnings date from Finnhub's calendar payload."""
|
|
||||||
try:
|
|
||||||
payload = resp.json() if resp.text else {}
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
entries = payload.get("earningsCalendar", []) if isinstance(payload, dict) else []
|
|
||||||
dates: list[date] = []
|
|
||||||
today = date.today()
|
|
||||||
for entry in entries if isinstance(entries, list) else []:
|
|
||||||
raw = entry.get("date") if isinstance(entry, dict) else None
|
|
||||||
if not raw:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
parsed = date.fromisoformat(raw)
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
if parsed >= today:
|
|
||||||
dates.append(parsed)
|
|
||||||
return min(dates) if dates else None
|
|
||||||
|
|
||||||
|
|
||||||
class AlphaVantageFundamentalProvider:
|
|
||||||
"""Fundamentals provider backed by Alpha Vantage free endpoints."""
|
|
||||||
|
|
||||||
def __init__(self, api_key: str) -> None:
|
|
||||||
if not api_key:
|
|
||||||
raise ProviderError("Alpha Vantage API key is required")
|
|
||||||
self._api_key = api_key
|
|
||||||
self._base_url = "https://www.alphavantage.co/query"
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
unavailable: dict[str, str] = {}
|
|
||||||
api_symbol = _to_api_symbol(ticker)
|
|
||||||
|
|
||||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
|
||||||
overview_resp = await client.get(
|
|
||||||
self._base_url,
|
|
||||||
params={"function": "OVERVIEW", "symbol": api_symbol, "apikey": self._api_key},
|
|
||||||
)
|
|
||||||
earnings_resp = await client.get(
|
|
||||||
self._base_url,
|
|
||||||
params={"function": "EARNINGS", "symbol": api_symbol, "apikey": self._api_key},
|
|
||||||
)
|
|
||||||
income_resp = await client.get(
|
|
||||||
self._base_url,
|
|
||||||
params={"function": "INCOME_STATEMENT", "symbol": api_symbol, "apikey": self._api_key},
|
|
||||||
)
|
|
||||||
|
|
||||||
for resp, endpoint in (
|
|
||||||
(overview_resp, "OVERVIEW"),
|
|
||||||
(earnings_resp, "EARNINGS"),
|
|
||||||
(income_resp, "INCOME_STATEMENT"),
|
|
||||||
):
|
|
||||||
if resp.status_code == 429:
|
|
||||||
raise RateLimitError(f"Alpha Vantage rate limit hit for {ticker} ({endpoint})")
|
|
||||||
if resp.status_code != 200:
|
|
||||||
raise ProviderError(f"Alpha Vantage error for {ticker} ({endpoint}): HTTP {resp.status_code}")
|
|
||||||
|
|
||||||
overview = overview_resp.json() if overview_resp.text else {}
|
|
||||||
earnings = earnings_resp.json() if earnings_resp.text else {}
|
|
||||||
income = income_resp.json() if income_resp.text else {}
|
|
||||||
|
|
||||||
if isinstance(overview, dict) and overview.get("Information"):
|
|
||||||
raise ProviderError(f"Alpha Vantage unavailable for {ticker}: {overview.get('Information')}")
|
|
||||||
if isinstance(overview, dict) and overview.get("Note"):
|
|
||||||
raise RateLimitError(f"Alpha Vantage rate limit for {ticker}: {overview.get('Note')}")
|
|
||||||
|
|
||||||
pe_ratio = _safe_float((overview or {}).get("PERatio"))
|
|
||||||
market_cap = _safe_float((overview or {}).get("MarketCapitalization"))
|
|
||||||
|
|
||||||
earnings_surprise = None
|
|
||||||
quarterly = earnings.get("quarterlyEarnings", []) if isinstance(earnings, dict) else []
|
|
||||||
if isinstance(quarterly, list) and quarterly:
|
|
||||||
first = quarterly[0] if isinstance(quarterly[0], dict) else {}
|
|
||||||
earnings_surprise = _safe_float(first.get("surprisePercentage"))
|
|
||||||
|
|
||||||
revenue_growth = None
|
|
||||||
annual = income.get("annualReports", []) if isinstance(income, dict) else []
|
|
||||||
if isinstance(annual, list) and len(annual) >= 2:
|
|
||||||
curr = _safe_float((annual[0] or {}).get("totalRevenue"))
|
|
||||||
prev = _safe_float((annual[1] or {}).get("totalRevenue"))
|
|
||||||
if curr is not None and prev not in (None, 0):
|
|
||||||
revenue_growth = ((curr - prev) / abs(prev)) * 100.0
|
|
||||||
|
|
||||||
if pe_ratio is None:
|
|
||||||
unavailable["pe_ratio"] = "not available from provider payload"
|
|
||||||
if revenue_growth is None:
|
|
||||||
unavailable["revenue_growth"] = "not available from provider payload"
|
|
||||||
if earnings_surprise is None:
|
|
||||||
unavailable["earnings_surprise"] = "not available from provider payload"
|
|
||||||
if market_cap is None:
|
|
||||||
unavailable["market_cap"] = "not available from provider payload"
|
|
||||||
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker,
|
|
||||||
pe_ratio=pe_ratio,
|
|
||||||
revenue_growth=revenue_growth,
|
|
||||||
earnings_surprise=earnings_surprise,
|
|
||||||
market_cap=market_cap,
|
|
||||||
fetched_at=datetime.now(timezone.utc),
|
|
||||||
unavailable_fields=unavailable,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
_FUNDAMENTAL_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise", "market_cap")
|
|
||||||
|
|
||||||
|
|
||||||
class ChainedFundamentalProvider:
|
|
||||||
"""Merge fundamentals across providers, filling gaps from later sources.
|
|
||||||
|
|
||||||
A single provider rarely covers everything on free tiers — FMP's free plan,
|
|
||||||
for example, returns only market cap (the ratios/growth/earnings endpoints
|
|
||||||
402). Rather than stop at the first provider with *any* field, we take each
|
|
||||||
field from the first provider that supplies it, so FMP's market cap is
|
|
||||||
combined with Finnhub's P/E and earnings surprise.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, providers: list[tuple[str, FundamentalProvider]]) -> None:
|
|
||||||
if not providers:
|
|
||||||
raise ProviderError("No fundamental providers configured")
|
|
||||||
self._providers = providers
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str, allow_partial: bool = False) -> FundamentalData:
|
|
||||||
"""Merge fundamentals across providers.
|
|
||||||
|
|
||||||
``allow_partial`` controls behaviour when a fallback provider is *rate
|
|
||||||
limited* and we end up with missing fields. By default we raise
|
|
||||||
RateLimitError so the caller (the bulk collector) can back off and retry
|
|
||||||
the ticker once the window frees — otherwise a transient 429 on Finnhub
|
|
||||||
would be silently stored as market-cap-only. Pass ``allow_partial=True``
|
|
||||||
(manual single fetches, or the collector's final give-up attempt) to
|
|
||||||
accept whatever was gathered instead of raising.
|
|
||||||
"""
|
|
||||||
merged: dict[str, float | None] = {f: None for f in _FUNDAMENTAL_FIELDS}
|
|
||||||
field_source: dict[str, str] = {}
|
|
||||||
errors: list[str] = []
|
|
||||||
rate_limited = False
|
|
||||||
next_earnings_date = None
|
|
||||||
|
|
||||||
for provider_name, provider in self._providers:
|
|
||||||
if all(merged[f] is not None for f in _FUNDAMENTAL_FIELDS) and next_earnings_date:
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
data = await provider.fetch_fundamentals(ticker)
|
|
||||||
except RateLimitError as exc:
|
|
||||||
rate_limited = True
|
|
||||||
errors.append(f"{provider_name}: RateLimitError: {exc}")
|
|
||||||
continue
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"{provider_name}: {type(exc).__name__}: {exc}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if next_earnings_date is None and data.next_earnings_date is not None:
|
|
||||||
next_earnings_date = data.next_earnings_date
|
|
||||||
|
|
||||||
for field in _FUNDAMENTAL_FIELDS:
|
|
||||||
if merged[field] is None:
|
|
||||||
value = getattr(data, field)
|
|
||||||
if value is not None:
|
|
||||||
merged[field] = value
|
|
||||||
field_source[field] = provider_name
|
|
||||||
|
|
||||||
missing = [f for f in _FUNDAMENTAL_FIELDS if merged[f] is None]
|
|
||||||
|
|
||||||
# A rate limit left data incomplete: signal it (unless partial is OK) so
|
|
||||||
# the collector backs off rather than persisting a degraded record.
|
|
||||||
if rate_limited and missing and not allow_partial:
|
|
||||||
attempts = "; ".join(errors[:6])
|
|
||||||
raise RateLimitError(
|
|
||||||
f"Fundamentals incomplete for {ticker} due to provider rate limits "
|
|
||||||
f"(missing {', '.join(missing)}). Attempts: {attempts}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if all(merged[f] is None for f in _FUNDAMENTAL_FIELDS):
|
|
||||||
attempts = "; ".join(errors[:6]) if errors else "no usable metrics from any provider"
|
|
||||||
raise ProviderError(f"All fundamentals providers failed for {ticker}. Attempts: {attempts}")
|
|
||||||
|
|
||||||
unavailable: dict[str, str] = {
|
|
||||||
field: "not available from any configured provider"
|
|
||||||
for field in _FUNDAMENTAL_FIELDS
|
|
||||||
if merged[field] is None
|
|
||||||
}
|
|
||||||
# Record which provider supplied each field for transparency.
|
|
||||||
for field, src in field_source.items():
|
|
||||||
unavailable[f"source_{field}"] = src
|
|
||||||
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker,
|
|
||||||
pe_ratio=merged["pe_ratio"],
|
|
||||||
revenue_growth=merged["revenue_growth"],
|
|
||||||
earnings_surprise=merged["earnings_surprise"],
|
|
||||||
market_cap=merged["market_cap"],
|
|
||||||
fetched_at=datetime.now(timezone.utc),
|
|
||||||
next_earnings_date=next_earnings_date,
|
|
||||||
unavailable_fields=unavailable,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def build_fundamental_provider_chain() -> FundamentalProvider:
|
|
||||||
providers: list[tuple[str, FundamentalProvider]] = []
|
|
||||||
|
|
||||||
if settings.fmp_api_key:
|
|
||||||
providers.append(("fmp", FMPFundamentalProvider(settings.fmp_api_key)))
|
|
||||||
if settings.finnhub_api_key:
|
|
||||||
providers.append(("finnhub", FinnhubFundamentalProvider(settings.finnhub_api_key)))
|
|
||||||
if settings.alpha_vantage_api_key:
|
|
||||||
providers.append(("alpha_vantage", AlphaVantageFundamentalProvider(settings.alpha_vantage_api_key)))
|
|
||||||
|
|
||||||
if not providers:
|
|
||||||
raise ProviderError(
|
|
||||||
"No fundamentals provider configured. Set one of FMP_API_KEY, FINNHUB_API_KEY, ALPHA_VANTAGE_API_KEY"
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info("Fundamentals provider chain configured: %s", [name for name, _ in providers])
|
|
||||||
return ChainedFundamentalProvider(providers)
|
|
||||||
@@ -44,20 +44,6 @@ class SentimentData:
|
|||||||
recommendation: str | None = None # "buy" | "hold" | "avoid" — actionable LLM view
|
recommendation: str | None = None # "buy" | "hold" | "avoid" — actionable LLM view
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class FundamentalData:
|
|
||||||
"""Fundamental metrics returned by fundamental providers."""
|
|
||||||
|
|
||||||
ticker: str
|
|
||||||
pe_ratio: float | None
|
|
||||||
revenue_growth: float | None
|
|
||||||
earnings_surprise: float | None
|
|
||||||
market_cap: float | None
|
|
||||||
fetched_at: datetime
|
|
||||||
next_earnings_date: date | None = None
|
|
||||||
unavailable_fields: dict[str, str] = field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Provider Protocols
|
# Provider Protocols
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -81,9 +67,5 @@ class SentimentProvider(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
class FundamentalProvider(Protocol):
|
# No fundamentals provider protocol: since A6 fundamentals come only from the
|
||||||
"""Protocol for fundamental data providers."""
|
# batch SEC/Dolt imports, never from a request-time provider call.
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
"""Fetch fundamental data for a ticker."""
|
|
||||||
...
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ from app.schemas.admin import (
|
|||||||
AlertConfigUpdate,
|
AlertConfigUpdate,
|
||||||
CreateUserRequest,
|
CreateUserRequest,
|
||||||
DataCleanupRequest,
|
DataCleanupRequest,
|
||||||
FundamentalsCutoverConfigUpdate,
|
|
||||||
JobTriggerRequest,
|
JobTriggerRequest,
|
||||||
JobToggle,
|
JobToggle,
|
||||||
RecommendationConfigUpdate,
|
RecommendationConfigUpdate,
|
||||||
@@ -138,27 +137,6 @@ async def list_settings(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/admin/settings/fundamentals-cutover", response_model=APIEnvelope)
|
|
||||||
async def get_fundamentals_cutover_settings(
|
|
||||||
_admin: User = Depends(require_admin),
|
|
||||||
db: AsyncSession = Depends(get_db),
|
|
||||||
):
|
|
||||||
config = await admin_service.get_fundamentals_cutover_config(db)
|
|
||||||
return APIEnvelope(status="success", data=config)
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/admin/settings/fundamentals-cutover", response_model=APIEnvelope)
|
|
||||||
async def update_fundamentals_cutover_settings(
|
|
||||||
body: FundamentalsCutoverConfigUpdate,
|
|
||||||
_admin: User = Depends(require_admin),
|
|
||||||
db: AsyncSession = Depends(get_db),
|
|
||||||
):
|
|
||||||
config = await admin_service.update_fundamentals_cutover_config(
|
|
||||||
db, body.enabled
|
|
||||||
)
|
|
||||||
return APIEnvelope(status="success", data=config)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/admin/settings/recommendations", response_model=APIEnvelope)
|
@router.get("/admin/settings/recommendations", response_model=APIEnvelope)
|
||||||
async def get_recommendation_settings(
|
async def get_recommendation_settings(
|
||||||
_admin: User = Depends(require_admin),
|
_admin: User = Depends(require_admin),
|
||||||
@@ -475,36 +453,6 @@ async def toggle_job(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/admin/fundamentals-parity", response_model=APIEnvelope)
|
|
||||||
async def get_fundamentals_parity_report(
|
|
||||||
_admin: User = Depends(require_admin),
|
|
||||||
):
|
|
||||||
"""Latest read-only A5 source/score comparison, or null before first run."""
|
|
||||||
return APIEnvelope(
|
|
||||||
status="success", data=admin_service.get_fundamentals_parity_report()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/admin/fundamentals-parity/csv", response_model=APIEnvelope)
|
|
||||||
async def get_fundamentals_parity_csv(
|
|
||||||
_admin: User = Depends(require_admin),
|
|
||||||
):
|
|
||||||
"""Latest flattened A5 report for an authenticated browser download."""
|
|
||||||
artifact = admin_service.get_fundamentals_parity_csv()
|
|
||||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
|
||||||
return APIEnvelope(status="success", data=data)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/admin/fundamentals-parity/json", response_model=APIEnvelope)
|
|
||||||
async def get_fundamentals_parity_json(
|
|
||||||
_admin: User = Depends(require_admin),
|
|
||||||
):
|
|
||||||
"""Canonical A5 JSON artifact for an authenticated browser download."""
|
|
||||||
artifact = admin_service.get_fundamentals_parity_json()
|
|
||||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
|
||||||
return APIEnvelope(status="success", data=data)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# System events (operational warnings / errors)
|
# System events (operational warnings / errors)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ from app.models.sr_level import SRLevel
|
|||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.providers.alpaca import AlpacaOHLCVProvider
|
from app.providers.alpaca import AlpacaOHLCVProvider
|
||||||
from app.providers.fundamentals_chain import build_fundamental_provider_chain
|
|
||||||
from app.services.rr_scanner_service import (
|
from app.services.rr_scanner_service import (
|
||||||
resolve_activation_ranks_for_symbol,
|
resolve_activation_ranks_for_symbol,
|
||||||
scan_ticker,
|
scan_ticker,
|
||||||
@@ -31,7 +30,6 @@ from app.services.rr_scanner_service import (
|
|||||||
from app.services.sentiment_provider_service import build_sentiment_provider
|
from app.services.sentiment_provider_service import build_sentiment_provider
|
||||||
from app.schemas.common import APIEnvelope
|
from app.schemas.common import APIEnvelope
|
||||||
from app.services import (
|
from app.services import (
|
||||||
fundamental_service,
|
|
||||||
ingestion_service,
|
ingestion_service,
|
||||||
scoring_service,
|
scoring_service,
|
||||||
sentiment_service,
|
sentiment_service,
|
||||||
@@ -185,34 +183,14 @@ async def fetch_symbol(
|
|||||||
sources_out["sentiment"] = {"status": "error", "message": str(exc)}
|
sources_out["sentiment"] = {"status": "error", "message": str(exc)}
|
||||||
|
|
||||||
# --- Fundamentals ---
|
# --- Fundamentals ---
|
||||||
|
# No per-ticker fetch exists any more: fundamental_data is rebuilt for the
|
||||||
|
# whole universe by the nightly SEC + Dolt imports, from local PostgreSQL.
|
||||||
|
# The source key is still accepted so older clients get a truthful answer.
|
||||||
if "fundamentals" in requested:
|
if "fundamentals" in requested:
|
||||||
if settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key:
|
sources_out["fundamentals"] = {
|
||||||
try:
|
"status": "skipped",
|
||||||
fundamentals_provider = build_fundamental_provider_chain()
|
"message": "Fundamentals refresh nightly from the SEC + Dolt imports",
|
||||||
# Manual single fetch: take whatever we can get (a lone 429 on a
|
}
|
||||||
# fallback shouldn't fail the whole refresh).
|
|
||||||
fdata = await fundamentals_provider.fetch_fundamentals(
|
|
||||||
symbol_upper, allow_partial=True
|
|
||||||
)
|
|
||||||
await fundamental_service.store_fundamental(
|
|
||||||
db,
|
|
||||||
symbol=symbol_upper,
|
|
||||||
pe_ratio=fdata.pe_ratio,
|
|
||||||
revenue_growth=fdata.revenue_growth,
|
|
||||||
earnings_surprise=fdata.earnings_surprise,
|
|
||||||
market_cap=fdata.market_cap,
|
|
||||||
next_earnings_date=fdata.next_earnings_date,
|
|
||||||
unavailable_fields=fdata.unavailable_fields,
|
|
||||||
)
|
|
||||||
sources_out["fundamentals"] = {"status": "ok", "message": None}
|
|
||||||
except Exception as exc:
|
|
||||||
logger.error("Fundamentals fetch failed for %s: %s", symbol_upper, exc)
|
|
||||||
sources_out["fundamentals"] = {"status": "error", "message": str(exc)}
|
|
||||||
else:
|
|
||||||
sources_out["fundamentals"] = {
|
|
||||||
"status": "skipped",
|
|
||||||
"message": "No fundamentals provider key configured",
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- Derived pipeline: S/R levels (free, always) ---
|
# --- Derived pipeline: S/R levels (free, always) ---
|
||||||
try:
|
try:
|
||||||
|
|||||||
+54
-271
@@ -1,9 +1,9 @@
|
|||||||
"""APScheduler job definitions and FastAPI lifespan integration.
|
"""APScheduler job definitions and FastAPI lifespan integration.
|
||||||
|
|
||||||
Defines four scheduled jobs:
|
Defines the scheduled jobs, among them:
|
||||||
- Data Collector (OHLCV fetch for all tickers)
|
- Data Collector (OHLCV fetch for all tickers)
|
||||||
- Sentiment Collector (sentiment for all tickers)
|
- Sentiment Collector (sentiment for all tickers)
|
||||||
- Fundamental Collector (fundamentals for all tickers)
|
- Dolt Earnings / SEC Fundamentals imports (bulk fundamentals sources)
|
||||||
- R:R Scanner (trade setup scan for all tickers)
|
- R:R Scanner (trade setup scan for all tickers)
|
||||||
|
|
||||||
Each job processes tickers independently, logs errors as structured JSON,
|
Each job processes tickers independently, logs errors as structured JSON,
|
||||||
@@ -25,22 +25,18 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.database import async_session_factory
|
from app.database import async_session_factory
|
||||||
from app.models.fundamental import FundamentalData
|
|
||||||
from app.models.ohlcv import OHLCVRecord
|
from app.models.ohlcv import OHLCVRecord
|
||||||
from app.models.sentiment import SentimentScore
|
from app.models.sentiment import SentimentScore
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.exceptions import ProviderError
|
from app.exceptions import ProviderError
|
||||||
from app.providers.alpaca import AlpacaOHLCVProvider
|
from app.providers.alpaca import AlpacaOHLCVProvider
|
||||||
from app.providers.fundamentals_chain import build_fundamental_provider_chain
|
|
||||||
from app.providers.protocol import SentimentData
|
from app.providers.protocol import SentimentData
|
||||||
from app.services import (
|
from app.services import (
|
||||||
fundamental_service,
|
|
||||||
ingestion_service,
|
ingestion_service,
|
||||||
pipeline_run,
|
pipeline_run,
|
||||||
sentiment_service,
|
sentiment_service,
|
||||||
settings_store,
|
settings_store,
|
||||||
shadow_book_service,
|
shadow_book_service,
|
||||||
fundamentals_parity_service,
|
|
||||||
fundamental_data_refresh_service,
|
fundamental_data_refresh_service,
|
||||||
)
|
)
|
||||||
from app.services.data_import import (
|
from app.services.data_import import (
|
||||||
@@ -93,7 +89,6 @@ _last_successful: dict[str, str | None] = {
|
|||||||
"data_collector": None,
|
"data_collector": None,
|
||||||
"data_backfill": None,
|
"data_backfill": None,
|
||||||
"sentiment_collector": None,
|
"sentiment_collector": None,
|
||||||
"fundamental_collector": None,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Jobs whose per-run progress is surfaced to Admin → Jobs. (outcome_evaluator is
|
# Jobs whose per-run progress is surfaced to Admin → Jobs. (outcome_evaluator is
|
||||||
@@ -102,10 +97,8 @@ _JOB_NAMES = [
|
|||||||
"data_collector",
|
"data_collector",
|
||||||
"data_backfill",
|
"data_backfill",
|
||||||
"sentiment_collector",
|
"sentiment_collector",
|
||||||
"fundamental_collector",
|
|
||||||
"dolt_earnings_import",
|
"dolt_earnings_import",
|
||||||
"sec_fundamentals_import",
|
"sec_fundamentals_import",
|
||||||
"fundamentals_parity_report",
|
|
||||||
"rr_scanner",
|
"rr_scanner",
|
||||||
"ticker_universe_sync",
|
"ticker_universe_sync",
|
||||||
"alerts",
|
"alerts",
|
||||||
@@ -261,7 +254,14 @@ def _runtime_finish(
|
|||||||
processed: int,
|
processed: int,
|
||||||
total: int | None,
|
total: int | None,
|
||||||
message: str | None = None,
|
message: str | None = None,
|
||||||
|
emit_event: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
"""Finalize a job's runtime row, optionally raising a durable event.
|
||||||
|
|
||||||
|
``emit_event=False`` is for a *re-finalize* that only rewords an outcome an
|
||||||
|
earlier call already reported. The dedup key includes the message, so a
|
||||||
|
reworded error would otherwise land in Admin → System Events twice.
|
||||||
|
"""
|
||||||
runtime = _job_runtime.get(job_name, {})
|
runtime = _job_runtime.get(job_name, {})
|
||||||
runtime.update({
|
runtime.update({
|
||||||
"running": False,
|
"running": False,
|
||||||
@@ -275,7 +275,7 @@ def _runtime_finish(
|
|||||||
})
|
})
|
||||||
_job_runtime[job_name] = runtime
|
_job_runtime[job_name] = runtime
|
||||||
# Durable event for error / rate-limit finishes (badge + Admin → Jobs panel).
|
# Durable event for error / rate-limit finishes (badge + Admin → Jobs panel).
|
||||||
if status in ("error", "rate_limited"):
|
if emit_event and status in ("error", "rate_limited"):
|
||||||
severity = "error" if status == "error" else "warning"
|
severity = "error" if status == "error" else "warning"
|
||||||
try:
|
try:
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
@@ -466,23 +466,6 @@ async def _get_sentiment_priority_tickers(db: AsyncSession) -> list[str]:
|
|||||||
return priority_syms + filler_syms
|
return priority_syms + filler_syms
|
||||||
|
|
||||||
|
|
||||||
async def _get_fundamental_priority_tickers(db: AsyncSession) -> list[str]:
|
|
||||||
"""Return symbols prioritized for fundamentals refresh.
|
|
||||||
|
|
||||||
Priority:
|
|
||||||
1) Tickers with no fundamentals snapshot yet
|
|
||||||
2) Tickers with existing fundamentals, oldest fetched_at first
|
|
||||||
3) Alphabetical tiebreaker
|
|
||||||
"""
|
|
||||||
missing_first = case((FundamentalData.fetched_at.is_(None), 0), else_=1)
|
|
||||||
result = await db.execute(
|
|
||||||
select(Ticker.symbol)
|
|
||||||
.outerjoin(FundamentalData, FundamentalData.ticker_id == Ticker.id)
|
|
||||||
.order_by(missing_first.asc(), FundamentalData.fetched_at.asc(), Ticker.symbol.asc())
|
|
||||||
)
|
|
||||||
return list(result.scalars().all())
|
|
||||||
|
|
||||||
|
|
||||||
def _resume_tickers(symbols: list[str], job_name: str) -> list[str]:
|
def _resume_tickers(symbols: list[str], job_name: str) -> list[str]:
|
||||||
"""Reorder tickers to resume after the last successful one (rate-limit resume).
|
"""Reorder tickers to resume after the last successful one (rate-limit resume).
|
||||||
|
|
||||||
@@ -816,149 +799,16 @@ async def collect_sentiment() -> None:
|
|||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Job: Fundamental Collector
|
# Jobs: bulk fundamentals source imports
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
async def collect_fundamentals() -> None:
|
async def _run_source_import(job_name: str, importer: SourceImporter) -> bool:
|
||||||
"""Fetch fundamentals for all tracked tickers via FMP.
|
|
||||||
|
|
||||||
Processes each ticker independently. On rate limit, records last
|
|
||||||
successful ticker for resume.
|
|
||||||
"""
|
|
||||||
job_name = "fundamental_collector"
|
|
||||||
_log_event(logging.INFO, "job_start", job=job_name)
|
|
||||||
_runtime_start(job_name)
|
|
||||||
processed = 0
|
|
||||||
total: int | None = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with async_session_factory() as db:
|
|
||||||
if not await _is_job_enabled(db, job_name):
|
|
||||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled")
|
|
||||||
return
|
|
||||||
if await fundamental_data_refresh_service.is_enabled(db):
|
|
||||||
message = "SEC + Dolt fundamentals cutover is active"
|
|
||||||
_log_event(
|
|
||||||
logging.INFO,
|
|
||||||
"job_skipped",
|
|
||||||
job=job_name,
|
|
||||||
reason="sec_dolt_cutover_active",
|
|
||||||
)
|
|
||||||
_runtime_finish(
|
|
||||||
job_name,
|
|
||||||
"skipped",
|
|
||||||
processed=0,
|
|
||||||
total=0,
|
|
||||||
message=message,
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
symbols = await _get_fundamental_priority_tickers(db)
|
|
||||||
if not symbols:
|
|
||||||
_log_event(logging.INFO, "job_complete", job=job_name, tickers=0)
|
|
||||||
_runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers")
|
|
||||||
return
|
|
||||||
|
|
||||||
total = len(symbols)
|
|
||||||
_runtime_progress(job_name, processed=0, total=total)
|
|
||||||
|
|
||||||
if not (settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key):
|
|
||||||
_log_event(logging.WARNING, "job_skipped", job=job_name, reason="no fundamentals provider keys configured")
|
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=total, message="No fundamentals provider keys configured")
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
provider = build_fundamental_provider_chain()
|
|
||||||
except Exception as exc:
|
|
||||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
|
||||||
_runtime_finish(job_name, "error", processed=0, total=total, message=str(exc))
|
|
||||||
return
|
|
||||||
|
|
||||||
max_retries = max(0, settings.fundamental_rate_limit_retries)
|
|
||||||
base_backoff = max(1, settings.fundamental_rate_limit_backoff_seconds)
|
|
||||||
spacing = max(0.0, settings.fundamental_request_spacing_seconds)
|
|
||||||
|
|
||||||
async def _store(symbol: str, data) -> None:
|
|
||||||
async with async_session_factory() as db:
|
|
||||||
await fundamental_service.store_fundamental(
|
|
||||||
db,
|
|
||||||
symbol=symbol,
|
|
||||||
pe_ratio=data.pe_ratio,
|
|
||||||
revenue_growth=data.revenue_growth,
|
|
||||||
earnings_surprise=data.earnings_surprise,
|
|
||||||
market_cap=data.market_cap,
|
|
||||||
next_earnings_date=data.next_earnings_date,
|
|
||||||
unavailable_fields=data.unavailable_fields,
|
|
||||||
)
|
|
||||||
|
|
||||||
for symbol in symbols:
|
|
||||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
|
||||||
attempt = 0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
data = await provider.fetch_fundamentals(symbol)
|
|
||||||
await _store(symbol, data)
|
|
||||||
_last_successful[job_name] = symbol
|
|
||||||
processed += 1
|
|
||||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
|
||||||
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol)
|
|
||||||
break
|
|
||||||
except Exception as exc:
|
|
||||||
msg = str(exc).lower()
|
|
||||||
if "rate" in msg or "429" in msg:
|
|
||||||
if attempt < max_retries:
|
|
||||||
wait_seconds = base_backoff * (2 ** attempt)
|
|
||||||
attempt += 1
|
|
||||||
_log_event(logging.WARNING, "rate_limited_retry", job=job_name, ticker=symbol, attempt=attempt, max_retries=max_retries, wait_seconds=wait_seconds, processed=processed)
|
|
||||||
_runtime_progress(
|
|
||||||
job_name,
|
|
||||||
processed=processed,
|
|
||||||
total=total,
|
|
||||||
current_ticker=symbol,
|
|
||||||
message=f"Rate-limited at {symbol}; retry {attempt}/{max_retries} in {wait_seconds}s",
|
|
||||||
)
|
|
||||||
await asyncio.sleep(wait_seconds)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Retries exhausted: store whatever partial data we can
|
|
||||||
# still get (e.g. FMP market cap) and move on, rather than
|
|
||||||
# aborting the whole run and leaving every later ticker
|
|
||||||
# untouched.
|
|
||||||
_log_event(logging.WARNING, "rate_limited_partial", job=job_name, ticker=symbol, processed=processed)
|
|
||||||
try:
|
|
||||||
data = await provider.fetch_fundamentals(symbol, allow_partial=True)
|
|
||||||
await _store(symbol, data)
|
|
||||||
processed += 1
|
|
||||||
except Exception as exc2:
|
|
||||||
_log_job_error(job_name, symbol, exc2)
|
|
||||||
break
|
|
||||||
_log_job_error(job_name, symbol, exc)
|
|
||||||
break
|
|
||||||
|
|
||||||
if spacing:
|
|
||||||
await asyncio.sleep(spacing)
|
|
||||||
|
|
||||||
_last_successful[job_name] = None
|
|
||||||
_log_event(logging.INFO, "job_complete", job=job_name, tickers=processed)
|
|
||||||
_runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers")
|
|
||||||
except Exception as exc:
|
|
||||||
_log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc))
|
|
||||||
_runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Jobs: shadow fundamentals sources
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool:
|
|
||||||
"""Run an importer and return whether its scheduled job was enabled.
|
"""Run an importer and return whether its scheduled job was enabled.
|
||||||
|
|
||||||
The SEC wrapper uses the return value to run its activated local cache step
|
The SEC wrapper uses the return value only to word its runtime message: its
|
||||||
after deferred, failed, no-op, promoted, or source-locked attempts while honoring
|
local cache step runs after deferred, failed, no-op, promoted, source-locked
|
||||||
the job-level disable switch.
|
and disabled attempts alike.
|
||||||
"""
|
"""
|
||||||
_log_event(logging.INFO, "job_start", job=job_name)
|
_log_event(logging.INFO, "job_start", job=job_name)
|
||||||
_runtime_start(job_name, total=1)
|
_runtime_start(job_name, total=1)
|
||||||
@@ -968,7 +818,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool:
|
|||||||
if not await _is_job_enabled(db, job_name):
|
if not await _is_job_enabled(db, job_name):
|
||||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
||||||
return
|
return False
|
||||||
|
|
||||||
run = await run_import(importer)
|
run = await run_import(importer)
|
||||||
if run is None:
|
if run is None:
|
||||||
@@ -1015,25 +865,26 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
async def run_dolt_earnings_import() -> None:
|
async def run_dolt_earnings_import() -> None:
|
||||||
"""Pull and import the Dolt earnings calendar/results feed in shadow."""
|
"""Pull and import the Dolt earnings calendar/results feed."""
|
||||||
await _run_shadow_import("dolt_earnings_import", DoltEarningsImporter())
|
await _run_source_import("dolt_earnings_import", DoltEarningsImporter())
|
||||||
|
|
||||||
|
|
||||||
async def run_sec_fundamentals_import() -> None:
|
async def run_sec_fundamentals_import() -> None:
|
||||||
"""Import SEC facts, then run the activated local compat-cache refresh.
|
"""Import SEC facts, then refresh the local compat cache.
|
||||||
|
|
||||||
The refresh is deliberately separate from the network import result. Once
|
The refresh is deliberately independent of the network import: it reads only
|
||||||
activated it therefore still runs from stored snapshots/earnings/prices when
|
stored snapshots, earnings events and closes, so it runs identically when SEC
|
||||||
SEC is unavailable, unchanged, or another SEC import owns the source lock.
|
is unavailable, unchanged, or owned by another import — and also when the
|
||||||
|
job's ingestion is switched off in Admin → Jobs. Disabling the job stops
|
||||||
|
SEC network access, not the cache; prices and earnings move daily even when
|
||||||
|
no filing does, and `fundamental_data` feeds scoring.
|
||||||
"""
|
"""
|
||||||
job_name = "sec_fundamentals_import"
|
job_name = "sec_fundamentals_import"
|
||||||
job_enabled = await _run_shadow_import(job_name, SecFundamentalsImporter())
|
import_ran = await _run_source_import(job_name, SecFundamentalsImporter())
|
||||||
if not job_enabled:
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with async_session_factory() as db:
|
async with async_session_factory() as db:
|
||||||
summary = await fundamental_data_refresh_service.refresh_if_enabled(db)
|
summary = await fundamental_data_refresh_service.refresh(db)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
_runtime_finish(
|
_runtime_finish(
|
||||||
job_name, "error", processed=0, total=1, message="Cancelled"
|
job_name, "error", processed=0, total=1, message="Cancelled"
|
||||||
@@ -1051,79 +902,38 @@ async def run_sec_fundamentals_import() -> None:
|
|||||||
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
||||||
return
|
return
|
||||||
|
|
||||||
if not summary["enabled"]:
|
|
||||||
_log_event(
|
|
||||||
logging.INFO,
|
|
||||||
"fundamental_data_refresh_skipped",
|
|
||||||
job=job_name,
|
|
||||||
reason="cutover_disabled",
|
|
||||||
setting=fundamental_data_refresh_service.ACTIVATION_KEY,
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
_log_event(
|
_log_event(
|
||||||
logging.INFO,
|
logging.INFO,
|
||||||
"fundamental_data_refresh_complete",
|
"fundamental_data_refresh_complete",
|
||||||
job=job_name,
|
job=job_name,
|
||||||
**summary,
|
**summary,
|
||||||
)
|
)
|
||||||
|
cache_message = (
|
||||||
|
f"cache {summary['refreshed']} · "
|
||||||
|
f"{summary['score_inputs_changed']} score inputs changed"
|
||||||
|
)
|
||||||
|
# Every outcome carries the cache summary — including deferred, failed and
|
||||||
|
# source-locked ones. The import status is what varies; the refresh always
|
||||||
|
# happened, and Admin → Jobs is the only place an operator sees that.
|
||||||
|
#
|
||||||
|
# This only rewords what _run_source_import already finalized, so it must not
|
||||||
|
# emit a second durable event: the dedup key includes the message, and a
|
||||||
|
# failure would otherwise show up twice in Admin → System Events.
|
||||||
runtime = get_job_runtime_snapshot(job_name)
|
runtime = get_job_runtime_snapshot(job_name)
|
||||||
if runtime.get("status") == "completed":
|
if import_ran:
|
||||||
|
status = str(runtime.get("status") or "completed")
|
||||||
import_message = runtime.get("message") or "import completed"
|
import_message = runtime.get("message") or "import completed"
|
||||||
cache_message = (
|
processed = 1 if status == "completed" else 0
|
||||||
f"cache {summary['refreshed']} · "
|
else:
|
||||||
f"{summary['score_inputs_changed']} score inputs changed"
|
status, import_message, processed = "completed", "Import disabled", 1
|
||||||
)
|
_runtime_finish(
|
||||||
_runtime_finish(
|
job_name,
|
||||||
job_name,
|
status,
|
||||||
"completed",
|
processed=processed,
|
||||||
processed=1,
|
total=1,
|
||||||
total=1,
|
message=f"{import_message} · {cache_message}",
|
||||||
message=f"{import_message} · {cache_message}",
|
emit_event=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def run_fundamentals_parity_report() -> None:
|
|
||||||
"""Generate the A5 comparison bundle without mutating live fundamentals/scores."""
|
|
||||||
job_name = "fundamentals_parity_report"
|
|
||||||
_log_event(logging.INFO, "job_start", job=job_name)
|
|
||||||
_runtime_start(job_name, total=1)
|
|
||||||
try:
|
|
||||||
async with async_session_factory() as db:
|
|
||||||
if not await _is_job_enabled(db, job_name):
|
|
||||||
_runtime_finish(
|
|
||||||
job_name, "skipped", processed=0, total=1, message="Disabled"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
report, artifacts = await fundamentals_parity_service.generate_and_store(
|
|
||||||
db, settings.fundamentals_parity_report_dir
|
|
||||||
)
|
|
||||||
summary = report["summary"]
|
|
||||||
message = (
|
|
||||||
f"{summary['universe_count']} tickers · "
|
|
||||||
f"{summary['fundamental_score_material_changes']} material score changes"
|
|
||||||
)
|
|
||||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
|
||||||
_log_event(
|
|
||||||
logging.INFO,
|
|
||||||
"job_complete",
|
|
||||||
job=job_name,
|
|
||||||
generated_at=report["generated_at"],
|
|
||||||
json_path=artifacts["json"],
|
|
||||||
csv_path=artifacts["csv"],
|
|
||||||
)
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
|
||||||
_log_event(
|
|
||||||
logging.ERROR,
|
|
||||||
"job_error",
|
|
||||||
job=job_name,
|
|
||||||
error_type=type(exc).__name__,
|
|
||||||
message=str(exc),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1666,19 +1476,16 @@ SCHEDULE_DEFAULTS: dict[str, str] = {
|
|||||||
"schedule_timezone": "America/New_York",
|
"schedule_timezone": "America/New_York",
|
||||||
# Morning data/display refresh (no qualifying R:R scan).
|
# Morning data/display refresh (no qualifying R:R scan).
|
||||||
"schedule_daily_pipeline_cron": "0 2 * * *",
|
"schedule_daily_pipeline_cron": "0 2 * * *",
|
||||||
# Bulk source imports. The SEC job writes the legacy compat cache only after
|
# Bulk source imports. The SEC job also refreshes the fundamental_data compat
|
||||||
# the explicit, default-off A5 cutover setting is enabled.
|
# cache that scoring reads — locally, from stored snapshots/earnings/closes.
|
||||||
"schedule_dolt_earnings_cron": "30 2 * * *",
|
"schedule_dolt_earnings_cron": "30 2 * * *",
|
||||||
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
||||||
"schedule_fundamentals_parity_cron": "30 5 * * *",
|
|
||||||
# Fetch in-progress bars → scan → Telegram (manual MOC window).
|
# Fetch in-progress bars → scan → Telegram (manual MOC window).
|
||||||
"schedule_near_close_pipeline_cron": "30 15 * * mon-fri",
|
"schedule_near_close_pipeline_cron": "30 15 * * mon-fri",
|
||||||
# Fetch final bars → outcome eval (must not run on the partial near-close bar).
|
# Fetch final bars → outcome eval (must not run on the partial near-close bar).
|
||||||
"schedule_after_close_pipeline_cron": "45 16 * * mon-fri",
|
"schedule_after_close_pipeline_cron": "45 16 * * mon-fri",
|
||||||
# Hourly mid-session price + outcome (10:00–15:00 ET Mon–Fri).
|
# Hourly mid-session price + outcome (10:00–15:00 ET Mon–Fri).
|
||||||
"schedule_intraday_pipeline_cron": "0 10-15 * * mon-fri",
|
"schedule_intraday_pipeline_cron": "0 10-15 * * mon-fri",
|
||||||
# Weekly fundamentals early Monday NY.
|
|
||||||
"schedule_fundamentals_cron": "0 1 * * mon",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# job id -> schedule setting key
|
# job id -> schedule setting key
|
||||||
@@ -1686,11 +1493,9 @@ _CRON_JOBS: dict[str, str] = {
|
|||||||
"daily_pipeline": "schedule_daily_pipeline_cron",
|
"daily_pipeline": "schedule_daily_pipeline_cron",
|
||||||
"dolt_earnings_import": "schedule_dolt_earnings_cron",
|
"dolt_earnings_import": "schedule_dolt_earnings_cron",
|
||||||
"sec_fundamentals_import": "schedule_sec_fundamentals_cron",
|
"sec_fundamentals_import": "schedule_sec_fundamentals_cron",
|
||||||
"fundamentals_parity_report": "schedule_fundamentals_parity_cron",
|
|
||||||
"near_close_pipeline": "schedule_near_close_pipeline_cron",
|
"near_close_pipeline": "schedule_near_close_pipeline_cron",
|
||||||
"after_close_pipeline": "schedule_after_close_pipeline_cron",
|
"after_close_pipeline": "schedule_after_close_pipeline_cron",
|
||||||
"intraday_pipeline": "schedule_intraday_pipeline_cron",
|
"intraday_pipeline": "schedule_intraday_pipeline_cron",
|
||||||
"fundamental_collector": "schedule_fundamentals_cron",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1779,7 +1584,7 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
"schedule_dolt_earnings_cron",
|
"schedule_dolt_earnings_cron",
|
||||||
),
|
),
|
||||||
id="dolt_earnings_import",
|
id="dolt_earnings_import",
|
||||||
name="Dolt Earnings Import (shadow)",
|
name="Dolt Earnings Import",
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
@@ -1793,17 +1598,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
name="SEC Fundamentals Import",
|
name="SEC Fundamentals Import",
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
scheduler.add_job(
|
|
||||||
run_fundamentals_parity_report,
|
|
||||||
_cron_trigger(
|
|
||||||
cfg["schedule_fundamentals_parity_cron"],
|
|
||||||
tz,
|
|
||||||
"schedule_fundamentals_parity_cron",
|
|
||||||
),
|
|
||||||
id="fundamentals_parity_report",
|
|
||||||
name="Fundamentals Parity Report (read-only)",
|
|
||||||
replace_existing=True,
|
|
||||||
)
|
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
run_near_close_pipeline,
|
run_near_close_pipeline,
|
||||||
_cron_trigger(
|
_cron_trigger(
|
||||||
@@ -1831,13 +1625,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
_cron_trigger(cfg["schedule_intraday_pipeline_cron"], tz, "schedule_intraday_pipeline_cron"),
|
_cron_trigger(cfg["schedule_intraday_pipeline_cron"], tz, "schedule_intraday_pipeline_cron"),
|
||||||
id="intraday_pipeline", name="Intraday Pipeline", replace_existing=True,
|
id="intraday_pipeline", name="Intraday Pipeline", replace_existing=True,
|
||||||
)
|
)
|
||||||
# Fundamentals — quarterly-ish data; weekly by default (conserves API quota).
|
|
||||||
# Its own early cron so the slow, rate-limited fetch finishes before the day.
|
|
||||||
scheduler.add_job(
|
|
||||||
collect_fundamentals,
|
|
||||||
_cron_trigger(cfg["schedule_fundamentals_cron"], tz, "schedule_fundamentals_cron"),
|
|
||||||
id="fundamental_collector", name="Fundamental Collector", replace_existing=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Independent interval jobs (own cadence, no ordering dependency)
|
# Independent interval jobs (own cadence, no ordering dependency)
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
@@ -1879,9 +1666,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
},
|
},
|
||||||
dolt_earnings_import={"cron": cfg["schedule_dolt_earnings_cron"]},
|
dolt_earnings_import={"cron": cfg["schedule_dolt_earnings_cron"]},
|
||||||
sec_fundamentals_import={"cron": cfg["schedule_sec_fundamentals_cron"]},
|
sec_fundamentals_import={"cron": cfg["schedule_sec_fundamentals_cron"]},
|
||||||
fundamentals_parity_report={
|
|
||||||
"cron": cfg["schedule_fundamentals_parity_cron"]
|
|
||||||
},
|
|
||||||
near_close_pipeline={
|
near_close_pipeline={
|
||||||
"cron": cfg["schedule_near_close_pipeline_cron"],
|
"cron": cfg["schedule_near_close_pipeline_cron"],
|
||||||
"steps": [name for name, _ in _NEAR_CLOSE_PIPELINE_STEPS],
|
"steps": [name for name, _ in _NEAR_CLOSE_PIPELINE_STEPS],
|
||||||
@@ -1894,7 +1678,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
"cron": cfg["schedule_intraday_pipeline_cron"],
|
"cron": cfg["schedule_intraday_pipeline_cron"],
|
||||||
"steps": [name for name, _ in _INTRADAY_PIPELINE_STEPS],
|
"steps": [name for name, _ in _INTRADAY_PIPELINE_STEPS],
|
||||||
},
|
},
|
||||||
fundamental_collector={"cron": cfg["schedule_fundamentals_cron"]},
|
|
||||||
independent=["ticker_universe_sync", "backtest"],
|
independent=["ticker_universe_sync", "backtest"],
|
||||||
manual_only=["alerts", "data_backfill", "event_study"],
|
manual_only=["alerts", "data_backfill", "event_study"],
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -73,11 +73,6 @@ class ActivationConfigUpdate(BaseModel):
|
|||||||
exclude_neutral: bool | None = None
|
exclude_neutral: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
class FundamentalsCutoverConfigUpdate(BaseModel):
|
|
||||||
"""Switch the legacy fundamentals cache from quota APIs to SEC/Dolt."""
|
|
||||||
enabled: bool
|
|
||||||
|
|
||||||
|
|
||||||
class ScheduleConfigUpdate(BaseModel):
|
class ScheduleConfigUpdate(BaseModel):
|
||||||
"""Cron schedule for the pipelines + fundamentals. Crons are 5-field
|
"""Cron schedule for the pipelines + fundamentals. Crons are 5-field
|
||||||
(min hour dom month dow); timezone is an IANA name (e.g. America/New_York)."""
|
(min hour dom month dow); timezone is an IANA name (e.g. America/New_York)."""
|
||||||
@@ -85,11 +80,9 @@ class ScheduleConfigUpdate(BaseModel):
|
|||||||
schedule_daily_pipeline_cron: str | None = Field(default=None, max_length=120)
|
schedule_daily_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_dolt_earnings_cron: str | None = Field(default=None, max_length=120)
|
schedule_dolt_earnings_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_sec_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
schedule_sec_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_fundamentals_parity_cron: str | None = Field(default=None, max_length=120)
|
|
||||||
schedule_near_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
schedule_near_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_after_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
schedule_after_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_intraday_pipeline_cron: str | None = Field(default=None, max_length=120)
|
schedule_intraday_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||||
schedule_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
|
||||||
|
|
||||||
|
|
||||||
class PerformanceConfigUpdate(BaseModel):
|
class PerformanceConfigUpdate(BaseModel):
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from app.models.settings import SystemSetting
|
|||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.models.trade_setup import TradeSetup
|
from app.models.trade_setup import TradeSetup
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.services import fundamental_data_refresh_service, settings_store
|
from app.services import settings_store
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -159,28 +159,6 @@ async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSettin
|
|||||||
return setting
|
return setting
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Fundamentals source cutover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def get_fundamentals_cutover_config(db: AsyncSession) -> dict[str, bool]:
|
|
||||||
"""Return the explicit A5 cache-cutover switch (default off)."""
|
|
||||||
return {"enabled": await fundamental_data_refresh_service.is_enabled(db)}
|
|
||||||
|
|
||||||
|
|
||||||
async def update_fundamentals_cutover_config(
|
|
||||||
db: AsyncSession, enabled: bool
|
|
||||||
) -> dict[str, bool]:
|
|
||||||
"""Activate or pause SEC/Dolt writes to the legacy fundamentals cache."""
|
|
||||||
await settings_store.upsert_setting(
|
|
||||||
db,
|
|
||||||
fundamental_data_refresh_service.ACTIVATION_KEY,
|
|
||||||
"true" if enabled else "false",
|
|
||||||
)
|
|
||||||
await db.commit()
|
|
||||||
return await get_fundamentals_cutover_config(db)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Activation thresholds
|
# Activation thresholds
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -633,10 +611,8 @@ VALID_JOB_NAMES = {
|
|||||||
"data_backfill",
|
"data_backfill",
|
||||||
"benchmark_collector",
|
"benchmark_collector",
|
||||||
"sentiment_collector",
|
"sentiment_collector",
|
||||||
"fundamental_collector",
|
|
||||||
"dolt_earnings_import",
|
"dolt_earnings_import",
|
||||||
"sec_fundamentals_import",
|
"sec_fundamentals_import",
|
||||||
"fundamentals_parity_report",
|
|
||||||
"rr_scanner",
|
"rr_scanner",
|
||||||
"ticker_universe_sync",
|
"ticker_universe_sync",
|
||||||
"outcome_evaluator",
|
"outcome_evaluator",
|
||||||
@@ -657,10 +633,8 @@ JOB_LABELS = {
|
|||||||
"data_backfill": "Data Backfill (deep history)",
|
"data_backfill": "Data Backfill (deep history)",
|
||||||
"benchmark_collector": "Benchmark Collector",
|
"benchmark_collector": "Benchmark Collector",
|
||||||
"sentiment_collector": "Sentiment Collector",
|
"sentiment_collector": "Sentiment Collector",
|
||||||
"fundamental_collector": "Fundamental Collector",
|
"dolt_earnings_import": "Dolt Earnings Import",
|
||||||
"dolt_earnings_import": "Dolt Earnings Import (shadow)",
|
|
||||||
"sec_fundamentals_import": "SEC Fundamentals Import",
|
"sec_fundamentals_import": "SEC Fundamentals Import",
|
||||||
"fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
|
|
||||||
"rr_scanner": "R:R Scanner",
|
"rr_scanner": "R:R Scanner",
|
||||||
"ticker_universe_sync": "Ticker Universe Sync",
|
"ticker_universe_sync": "Ticker Universe Sync",
|
||||||
"outcome_evaluator": "Outcome Evaluator",
|
"outcome_evaluator": "Outcome Evaluator",
|
||||||
@@ -799,30 +773,3 @@ async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSe
|
|||||||
|
|
||||||
key = f"job_{job_name}_enabled"
|
key = f"job_{job_name}_enabled"
|
||||||
return await update_setting(db, key, str(enabled).lower())
|
return await update_setting(db, key, str(enabled).lower())
|
||||||
|
|
||||||
|
|
||||||
def get_fundamentals_parity_report() -> dict | None:
|
|
||||||
"""Return the latest compact A5 summary, if the job has run."""
|
|
||||||
from app.config import settings
|
|
||||||
from app.services.fundamentals_parity_service import load_latest
|
|
||||||
|
|
||||||
report = load_latest(settings.fundamentals_parity_report_dir)
|
|
||||||
if report is not None:
|
|
||||||
report.pop("rows", None) # full per-ticker data is download-only
|
|
||||||
return report
|
|
||||||
|
|
||||||
|
|
||||||
def get_fundamentals_parity_csv() -> tuple[str, str] | None:
|
|
||||||
"""Return the latest A5 CSV filename and content for authenticated download."""
|
|
||||||
from app.config import settings
|
|
||||||
from app.services.fundamentals_parity_service import load_latest_csv
|
|
||||||
|
|
||||||
return load_latest_csv(settings.fundamentals_parity_report_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def get_fundamentals_parity_json() -> tuple[str, str] | None:
|
|
||||||
"""Return the canonical A5 JSON artifact for authenticated download."""
|
|
||||||
from app.config import settings
|
|
||||||
from app.services.fundamentals_parity_service import load_latest_json
|
|
||||||
|
|
||||||
return load_latest_json(settings.fundamentals_parity_report_dir)
|
|
||||||
|
|||||||
@@ -1320,7 +1320,6 @@ def _replay_candidates_for_period(
|
|||||||
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||||
include_short_candidates: bool = False,
|
include_short_candidates: bool = False,
|
||||||
include_universe_rank_observations: bool = False,
|
include_universe_rank_observations: bool = False,
|
||||||
outcome_horizon_sessions: int = HORIZON,
|
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Slim picklable replay used by local event studies.
|
"""Slim picklable replay used by local event studies.
|
||||||
|
|
||||||
@@ -1344,13 +1343,10 @@ def _replay_candidates_for_period(
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
cadence = validate_backtest_cadence(cadence)
|
cadence = validate_backtest_cadence(cadence)
|
||||||
replay_horizon = int(outcome_horizon_sessions)
|
|
||||||
if replay_horizon < 0:
|
|
||||||
raise ValueError('outcome_horizon_sessions must be non-negative')
|
|
||||||
candidates: list[dict] = []
|
candidates: list[dict] = []
|
||||||
for i in range(
|
for i in range(
|
||||||
MIN_LOOKBACK - 1,
|
MIN_LOOKBACK - 1,
|
||||||
len(bars) - replay_horizon,
|
len(bars) - HORIZON,
|
||||||
backtest_step_sessions(cadence),
|
backtest_step_sessions(cadence),
|
||||||
):
|
):
|
||||||
if bars[i].date < start_date:
|
if bars[i].date < start_date:
|
||||||
@@ -1705,7 +1701,14 @@ def _gate_ablation(candidates: list[dict], activation: dict, threshold: float) -
|
|||||||
# the QUALIFIED setups at their detection close, best momentum first while
|
# the QUALIFIED setups at their detection close, best momentum first while
|
||||||
# slots and cash allow.
|
# slots and cash allow.
|
||||||
SIM_STARTING_CAPITAL = 10_000.0
|
SIM_STARTING_CAPITAL = 10_000.0
|
||||||
SIM_MAX_POSITIONS = 10
|
# Headroom, not a target: the count cap should never bind. The capacity study
|
||||||
|
# (reports/portfolio-construction-prod505-capacity-bracket-daily-v1) showed a book
|
||||||
|
# that never hits the count cap earns +1.1pp CAGR over the old 10 (51 cohorts of 175
|
||||||
|
# better, 2 worse) at unchanged drawdown, because the blocked entries were as good as
|
||||||
|
# the taken ones — capacity costs trade COUNT, not trade quality. The real ceiling is
|
||||||
|
# cash plus SIM_NOTIONAL_CAP, which saturates the book near 12 positions, so 15/20/None
|
||||||
|
# are the same experiment. Judge any future change here on CAGR, never on EV per trade.
|
||||||
|
SIM_MAX_POSITIONS = 15
|
||||||
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
|
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
|
||||||
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
|
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
|
||||||
_EULER_MASCHERONI = 0.5772156649015329
|
_EULER_MASCHERONI = 0.5772156649015329
|
||||||
@@ -1946,7 +1949,6 @@ def _make_gate_reset_reentry_fn(
|
|||||||
cadence: str,
|
cadence: str,
|
||||||
qualified_fn: Callable[[dict], bool] | None = None,
|
qualified_fn: Callable[[dict], bool] | None = None,
|
||||||
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
||||||
evaluation_horizon_sessions: int = HORIZON,
|
|
||||||
) -> Callable[[str, int, dict, Any], dict | None]:
|
) -> Callable[[str, int, dict, Any], dict | None]:
|
||||||
"""Build the production post-stop gate-reset callback.
|
"""Build the production post-stop gate-reset callback.
|
||||||
|
|
||||||
@@ -1964,18 +1966,11 @@ def _make_gate_reset_reentry_fn(
|
|||||||
|
|
||||||
evaluation_ords: dict[str, set[int]] = {}
|
evaluation_ords: dict[str, set[int]] = {}
|
||||||
step_sessions = backtest_step_sessions(cadence)
|
step_sessions = backtest_step_sessions(cadence)
|
||||||
evaluation_horizon = int(evaluation_horizon_sessions)
|
|
||||||
if evaluation_horizon < 0:
|
|
||||||
raise ValueError('evaluation_horizon_sessions must be non-negative')
|
|
||||||
for symbol, columns in prices.items():
|
for symbol, columns in prices.items():
|
||||||
ordinals = columns[0]
|
ordinals = columns[0]
|
||||||
evaluation_ords[symbol] = {
|
evaluation_ords[symbol] = {
|
||||||
int(ordinals[index])
|
int(ordinals[index])
|
||||||
for index in range(
|
for index in range(MIN_LOOKBACK - 1, len(ordinals) - HORIZON, step_sessions)
|
||||||
MIN_LOOKBACK - 1,
|
|
||||||
len(ordinals) - evaluation_horizon,
|
|
||||||
step_sessions,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
qualified_by_symbol_date: dict[tuple[str, int], dict] = {}
|
qualified_by_symbol_date: dict[tuple[str, int], dict] = {}
|
||||||
@@ -2022,7 +2017,7 @@ def _simulate_portfolio(
|
|||||||
*,
|
*,
|
||||||
qualified_fn: Callable[[dict], bool] | None = None,
|
qualified_fn: Callable[[dict], bool] | None = None,
|
||||||
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
||||||
max_positions: int | None = SIM_MAX_POSITIONS,
|
max_positions: int = SIM_MAX_POSITIONS,
|
||||||
risk_per_trade: float = SIM_RISK_PER_TRADE,
|
risk_per_trade: float = SIM_RISK_PER_TRADE,
|
||||||
atr_trail_multiplier: float = ATR_TRAIL_MULTIPLIER,
|
atr_trail_multiplier: float = ATR_TRAIL_MULTIPLIER,
|
||||||
cost_per_side: float = COST_PER_SIDE,
|
cost_per_side: float = COST_PER_SIDE,
|
||||||
@@ -2046,12 +2041,6 @@ def _simulate_portfolio(
|
|||||||
corr_lookback: int = 120,
|
corr_lookback: int = 120,
|
||||||
corr_action: str = "skip",
|
corr_action: str = "skip",
|
||||||
corr_min_overlap: int = 60,
|
corr_min_overlap: int = 60,
|
||||||
min_initial_risk_fraction: float | None = None,
|
|
||||||
weekly_top_n_rebalance: bool = False,
|
|
||||||
daily_rank_map: dict[tuple[str, str], dict[str, float | None]] | None = None,
|
|
||||||
measurement_start_date: date | None = None,
|
|
||||||
hard_end_date: date | None = None,
|
|
||||||
include_capacity_diagnostics: bool = False,
|
|
||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
"""Replay the qualified setups as ONE capital-constrained book and report
|
"""Replay the qualified setups as ONE capital-constrained book and report
|
||||||
portfolio economics from the daily equity curve (return, CAGR, drawdown,
|
portfolio economics from the daily equity curve (return, CAGR, drawdown,
|
||||||
@@ -2101,20 +2090,6 @@ def _simulate_portfolio(
|
|||||||
raise ValueError("corr_action must be 'skip' or 'half_size'")
|
raise ValueError("corr_action must be 'skip' or 'half_size'")
|
||||||
if vol_target is not None and vol_target <= 0:
|
if vol_target is not None and vol_target <= 0:
|
||||||
raise ValueError("vol_target must be positive when set")
|
raise ValueError("vol_target must be positive when set")
|
||||||
if max_positions is not None and int(max_positions) <= 0:
|
|
||||||
raise ValueError("max_positions must be positive or None")
|
|
||||||
if min_initial_risk_fraction is not None and not (
|
|
||||||
0.0 < float(min_initial_risk_fraction) < 1.0
|
|
||||||
):
|
|
||||||
raise ValueError("min_initial_risk_fraction must be between 0 and 1")
|
|
||||||
if weekly_top_n_rebalance and (
|
|
||||||
max_positions is None or daily_rank_map is None
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
"weekly_top_n_rebalance requires max_positions and daily_rank_map"
|
|
||||||
)
|
|
||||||
if weekly_top_n_rebalance and fill_mode != FILL_MODE_CLOSE:
|
|
||||||
raise ValueError("weekly_top_n_rebalance requires fill_mode=close")
|
|
||||||
clamp_lo, clamp_hi = float(vol_clamp[0]), float(vol_clamp[1])
|
clamp_lo, clamp_hi = float(vol_clamp[0]), float(vol_clamp[1])
|
||||||
if clamp_lo <= 0 or clamp_hi < clamp_lo:
|
if clamp_lo <= 0 or clamp_hi < clamp_lo:
|
||||||
raise ValueError("vol_clamp must satisfy 0 < lo <= hi")
|
raise ValueError("vol_clamp must satisfy 0 < lo <= hi")
|
||||||
@@ -2126,26 +2101,8 @@ def _simulate_portfolio(
|
|||||||
|
|
||||||
entries_by_ord: dict[int, list[dict]] = defaultdict(list)
|
entries_by_ord: dict[int, list[dict]] = defaultdict(list)
|
||||||
start_ord = start_date.toordinal() if start_date is not None else None
|
start_ord = start_date.toordinal() if start_date is not None else None
|
||||||
measurement_start_ord = (
|
|
||||||
measurement_start_date.toordinal()
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else start_ord
|
|
||||||
)
|
|
||||||
hard_end_ord = hard_end_date.toordinal() if hard_end_date is not None else None
|
|
||||||
# Explicit simulator/holdout end dates are exclusive split boundaries.
|
# Explicit simulator/holdout end dates are exclusive split boundaries.
|
||||||
end_ord = end_date.toordinal() if end_date is not None else None
|
end_ord = end_date.toordinal() if end_date is not None else None
|
||||||
if (
|
|
||||||
start_ord is not None
|
|
||||||
and measurement_start_ord is not None
|
|
||||||
and measurement_start_ord < start_ord
|
|
||||||
):
|
|
||||||
raise ValueError("measurement_start_date cannot precede start_date")
|
|
||||||
if (
|
|
||||||
hard_end_ord is not None
|
|
||||||
and measurement_start_ord is not None
|
|
||||||
and hard_end_ord <= measurement_start_ord
|
|
||||||
):
|
|
||||||
raise ValueError("hard_end_date must follow measurement_start_date")
|
|
||||||
for c in candidates:
|
for c in candidates:
|
||||||
if not qualified_fn(c) or c.get("direction") != "long":
|
if not qualified_fn(c) or c.get("direction") != "long":
|
||||||
continue
|
continue
|
||||||
@@ -2154,8 +2111,6 @@ def _simulate_portfolio(
|
|||||||
continue
|
continue
|
||||||
if end_ord is not None and entry_ord >= end_ord:
|
if end_ord is not None and entry_ord >= end_ord:
|
||||||
continue # holdout/validation: entries strictly before the split
|
continue # holdout/validation: entries strictly before the split
|
||||||
if hard_end_ord is not None and entry_ord >= hard_end_ord:
|
|
||||||
continue
|
|
||||||
if not c.get("entry") or not c.get("stop"):
|
if not c.get("entry") or not c.get("stop"):
|
||||||
continue
|
continue
|
||||||
entries_by_ord[entry_ord].append(c)
|
entries_by_ord[entry_ord].append(c)
|
||||||
@@ -2168,12 +2123,7 @@ def _simulate_portfolio(
|
|||||||
}
|
}
|
||||||
|
|
||||||
first_ord = start_ord if start_ord is not None else min(entries_by_ord)
|
first_ord = start_ord if start_ord is not None else min(entries_by_ord)
|
||||||
full_calendar = sorted({o for cols in prices.values() for o in cols[0]})
|
calendar = sorted({o for cols in prices.values() for o in cols[0] if o >= first_ord})
|
||||||
calendar = [
|
|
||||||
o
|
|
||||||
for o in full_calendar
|
|
||||||
if o >= first_ord and (hard_end_ord is None or o < hard_end_ord)
|
|
||||||
]
|
|
||||||
if not calendar:
|
if not calendar:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -2181,39 +2131,20 @@ def _simulate_portfolio(
|
|||||||
# fill lag). Prevents trailing flat-cash after the last resolvable entry —
|
# fill lag). Prevents trailing flat-cash after the last resolvable entry —
|
||||||
# the clear-air train-window bug — for train, validation, and full-period
|
# the clear-air train-window bug — for train, validation, and full-period
|
||||||
# books alike (including max-hold sweeps out to 90 days).
|
# books alike (including max-hold sweeps out to 90 days).
|
||||||
if hard_end_ord is None:
|
last_signal_ord = max(entries_by_ord)
|
||||||
last_signal_ord = max(entries_by_ord)
|
resolve_pad = hold_days + (1 if fill_mode in DELAYED_FILL_MODES else 0)
|
||||||
resolve_pad = hold_days + (1 if fill_mode in DELAYED_FILL_MODES else 0)
|
cut = bisect.bisect_left(calendar, last_signal_ord) + resolve_pad + 1
|
||||||
cut = bisect.bisect_left(calendar, last_signal_ord) + resolve_pad + 1
|
calendar = calendar[:cut]
|
||||||
calendar = calendar[:cut]
|
|
||||||
if not calendar:
|
if not calendar:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
weekly_rebalance_ords: set[int] = set()
|
|
||||||
for index, session_ord in enumerate(full_calendar):
|
|
||||||
session_date = date.fromordinal(session_ord)
|
|
||||||
iso = session_date.isocalendar()
|
|
||||||
if index + 1 < len(full_calendar):
|
|
||||||
next_iso = date.fromordinal(full_calendar[index + 1]).isocalendar()
|
|
||||||
if (iso.year, iso.week) != (next_iso.year, next_iso.week):
|
|
||||||
weekly_rebalance_ords.add(session_ord)
|
|
||||||
elif session_date.weekday() == 4:
|
|
||||||
weekly_rebalance_ords.add(session_ord)
|
|
||||||
|
|
||||||
cash = SIM_STARTING_CAPITAL
|
cash = SIM_STARTING_CAPITAL
|
||||||
positions: dict[str, dict] = {}
|
positions: dict[str, dict] = {}
|
||||||
curve: list[tuple[int, float]] = []
|
curve: list[tuple[int, float]] = []
|
||||||
trades: list[dict] = []
|
trades: list[dict] = []
|
||||||
skipped_full = 0
|
skipped_full = 0
|
||||||
measurement_skipped_full = 0
|
|
||||||
skipped_cooldown = 0
|
skipped_cooldown = 0
|
||||||
skipped_corr = 0
|
skipped_corr = 0
|
||||||
skipped_min_initial_risk = 0
|
|
||||||
measurement_skipped_min_initial_risk = 0
|
|
||||||
opened_positions = 0
|
|
||||||
measurement_opened_positions = 0
|
|
||||||
weekly_rank_rejected_entries = 0
|
|
||||||
measurement_weekly_rank_rejected_entries = 0
|
|
||||||
skipped_missing_fill = 0
|
skipped_missing_fill = 0
|
||||||
skipped_gap_cap = 0
|
skipped_gap_cap = 0
|
||||||
cooldown_until_index: dict[str, int] = {}
|
cooldown_until_index: dict[str, int] = {}
|
||||||
@@ -2228,12 +2159,6 @@ def _simulate_portfolio(
|
|||||||
vol_scalars: list[float] = []
|
vol_scalars: list[float] = []
|
||||||
overnight_slippage_pct: list[float] = []
|
overnight_slippage_pct: list[float] = []
|
||||||
pending_delayed: list[dict] = []
|
pending_delayed: list[dict] = []
|
||||||
measurement_start_equity: float | None = None
|
|
||||||
measurement_start_position_count: int | None = None
|
|
||||||
capacity_samples: list[dict[str, float | int]] = []
|
|
||||||
weekly_rebalance_events: list[dict] = []
|
|
||||||
rebalance_exit_index: dict[str, tuple[int, int]] = {}
|
|
||||||
rebalance_reentry_events: list[dict] = []
|
|
||||||
|
|
||||||
def _bar(sym: str, o: int):
|
def _bar(sym: str, o: int):
|
||||||
idx = index_of.get(sym, {}).get(o)
|
idx = index_of.get(sym, {}).get(o)
|
||||||
@@ -2303,13 +2228,6 @@ def _simulate_portfolio(
|
|||||||
cost = proceeds * cost_rate
|
cost = proceeds * cost_rate
|
||||||
cash += proceeds - cost
|
cash += proceeds - cost
|
||||||
risk = pos["entry"] - pos["initial_stop"]
|
risk = pos["entry"] - pos["initial_stop"]
|
||||||
initial_risk_dollars = pos["shares"] * risk
|
|
||||||
net_pnl = (
|
|
||||||
proceeds
|
|
||||||
- pos["shares"] * pos["entry"]
|
|
||||||
- cost
|
|
||||||
- pos["entry_cost"]
|
|
||||||
)
|
|
||||||
trades.append({
|
trades.append({
|
||||||
"symbol": sym,
|
"symbol": sym,
|
||||||
"entry_ord": pos["entry_ord"],
|
"entry_ord": pos["entry_ord"],
|
||||||
@@ -2318,13 +2236,8 @@ def _simulate_portfolio(
|
|||||||
"initial_stop": pos["initial_stop"],
|
"initial_stop": pos["initial_stop"],
|
||||||
"active_stop": pos["stop"],
|
"active_stop": pos["stop"],
|
||||||
"fill": fill,
|
"fill": fill,
|
||||||
"shares": pos["shares"],
|
"pnl": proceeds - pos["shares"] * pos["entry"] - cost - pos["entry_cost"],
|
||||||
"initial_risk_dollars": initial_risk_dollars,
|
|
||||||
"pnl": net_pnl,
|
|
||||||
"r": (fill - pos["entry"]) / risk if risk > 0 else 0.0,
|
"r": (fill - pos["entry"]) / risk if risk > 0 else 0.0,
|
||||||
"net_r": net_pnl / initial_risk_dollars
|
|
||||||
if initial_risk_dollars > 0
|
|
||||||
else 0.0,
|
|
||||||
"hold": pos["bars_held"],
|
"hold": pos["bars_held"],
|
||||||
"reason": reason,
|
"reason": reason,
|
||||||
"stop_refreshes": pos["stop_refreshes"],
|
"stop_refreshes": pos["stop_refreshes"],
|
||||||
@@ -2339,13 +2252,6 @@ def _simulate_portfolio(
|
|||||||
|
|
||||||
cooldown_sessions = max(0, int(reentry_cooldown_sessions))
|
cooldown_sessions = max(0, int(reentry_cooldown_sessions))
|
||||||
for calendar_index, o in enumerate(calendar):
|
for calendar_index, o in enumerate(calendar):
|
||||||
in_measurement = (
|
|
||||||
measurement_start_ord is None or o >= measurement_start_ord
|
|
||||||
)
|
|
||||||
if in_measurement and measurement_start_equity is None:
|
|
||||||
measurement_start_equity = _marked_equity()
|
|
||||||
measurement_start_position_count = len(positions)
|
|
||||||
|
|
||||||
# 1) exits on today's bars (stop intraday, target intraday, time at close)
|
# 1) exits on today's bars (stop intraday, target intraday, time at close)
|
||||||
for sym in list(positions):
|
for sym in list(positions):
|
||||||
pos = positions[sym]
|
pos = positions[sym]
|
||||||
@@ -2459,82 +2365,6 @@ def _simulate_portfolio(
|
|||||||
reverse=True,
|
reverse=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
weekly_selected_entries: list[dict] | None = None
|
|
||||||
if weekly_top_n_rebalance and o in weekly_rebalance_ords:
|
|
||||||
assert max_positions is not None
|
|
||||||
assert daily_rank_map is not None
|
|
||||||
asof = date.fromordinal(o).isoformat()
|
|
||||||
protected: set[str] = set()
|
|
||||||
ranked_pool: list[tuple[float, int, str, dict | None]] = []
|
|
||||||
for sym in positions:
|
|
||||||
rank_row = daily_rank_map.get((sym, asof))
|
|
||||||
current_rank = (
|
|
||||||
rank_row.get("strategy_rank") if rank_row is not None else None
|
|
||||||
)
|
|
||||||
if current_rank is None or _bar(sym, o) is None:
|
|
||||||
protected.add(sym)
|
|
||||||
continue
|
|
||||||
ranked_pool.append((float(current_rank), 0, sym, None))
|
|
||||||
|
|
||||||
entrants_by_symbol: dict[str, dict] = {}
|
|
||||||
for candidate in signal_todays:
|
|
||||||
sym = str(candidate["symbol"])
|
|
||||||
if sym in positions or sym in entrants_by_symbol:
|
|
||||||
continue
|
|
||||||
entrants_by_symbol[sym] = candidate
|
|
||||||
eligible_entrants = 0
|
|
||||||
for sym, candidate in entrants_by_symbol.items():
|
|
||||||
rank_row = daily_rank_map.get((sym, asof))
|
|
||||||
current_rank = (
|
|
||||||
rank_row.get("strategy_rank") if rank_row is not None else None
|
|
||||||
)
|
|
||||||
if current_rank is None:
|
|
||||||
continue
|
|
||||||
eligible_entrants += 1
|
|
||||||
ranked_pool.append((float(current_rank), 1, sym, candidate))
|
|
||||||
|
|
||||||
available_slots = max(0, int(max_positions) - len(protected))
|
|
||||||
ranked_pool.sort(key=lambda row: (-row[0], row[1], row[2]))
|
|
||||||
selected = ranked_pool[:available_slots]
|
|
||||||
selected_holding_symbols = {
|
|
||||||
sym for _rank, kind, sym, _candidate in selected if kind == 0
|
|
||||||
}
|
|
||||||
weekly_selected_entries = [
|
|
||||||
candidate
|
|
||||||
for _rank, kind, _sym, candidate in selected
|
|
||||||
if kind == 1 and candidate is not None
|
|
||||||
]
|
|
||||||
selected_entrant_symbols = {
|
|
||||||
str(candidate["symbol"]) for candidate in weekly_selected_entries
|
|
||||||
}
|
|
||||||
rejected_now = max(0, eligible_entrants - len(selected_entrant_symbols))
|
|
||||||
weekly_rank_rejected_entries += rejected_now
|
|
||||||
if in_measurement:
|
|
||||||
measurement_weekly_rank_rejected_entries += rejected_now
|
|
||||||
|
|
||||||
exited_symbols: list[str] = []
|
|
||||||
for sym in list(positions):
|
|
||||||
if sym in protected or sym in selected_holding_symbols:
|
|
||||||
continue
|
|
||||||
bar = _bar(sym, o)
|
|
||||||
if bar is None:
|
|
||||||
continue
|
|
||||||
_close_trade(sym, float(bar.close), "weekly_rebalance")
|
|
||||||
rebalance_exit_index[sym] = (calendar_index, o)
|
|
||||||
exited_symbols.append(sym)
|
|
||||||
|
|
||||||
weekly_rebalance_events.append({
|
|
||||||
"ord": o,
|
|
||||||
"fresh_entrant_pool": len(entrants_by_symbol),
|
|
||||||
"rank_eligible_entrant_pool": eligible_entrants,
|
|
||||||
"selected_entrants": len(selected_entrant_symbols),
|
|
||||||
"replacements": len(exited_symbols),
|
|
||||||
"exited_symbols": sorted(exited_symbols),
|
|
||||||
"selected_entrant_symbols": sorted(selected_entrant_symbols),
|
|
||||||
"measurement": in_measurement,
|
|
||||||
})
|
|
||||||
equity = _marked_equity()
|
|
||||||
|
|
||||||
if fill_mode in DELAYED_FILL_MODES:
|
if fill_mode in DELAYED_FILL_MODES:
|
||||||
fill_candidates = sorted(
|
fill_candidates = sorted(
|
||||||
pending_delayed,
|
pending_delayed,
|
||||||
@@ -2543,11 +2373,7 @@ def _simulate_portfolio(
|
|||||||
)
|
)
|
||||||
pending_delayed = []
|
pending_delayed = []
|
||||||
else:
|
else:
|
||||||
fill_candidates = (
|
fill_candidates = signal_todays
|
||||||
weekly_selected_entries
|
|
||||||
if weekly_selected_entries is not None
|
|
||||||
else signal_todays
|
|
||||||
)
|
|
||||||
|
|
||||||
def _corr_scale_for(sym: str, asof_idx: int) -> float | None:
|
def _corr_scale_for(sym: str, asof_idx: int) -> float | None:
|
||||||
"""1.0 ok, 0.5 half-size, None = skip. Missing history → uncorrelated."""
|
"""1.0 ok, 0.5 half-size, None = skip. Missing history → uncorrelated."""
|
||||||
@@ -2592,21 +2418,15 @@ def _simulate_portfolio(
|
|||||||
corr_scale: float,
|
corr_scale: float,
|
||||||
fill_bar: Any | None,
|
fill_bar: Any | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
nonlocal cash, equity, skipped_full, measurement_skipped_full
|
nonlocal cash, equity, skipped_full, skipped_cooldown, post_stop_events
|
||||||
nonlocal skipped_cooldown, post_stop_events
|
|
||||||
nonlocal skipped_min_initial_risk
|
|
||||||
nonlocal measurement_skipped_min_initial_risk
|
|
||||||
nonlocal opened_positions, measurement_opened_positions
|
|
||||||
sym = c["symbol"]
|
sym = c["symbol"]
|
||||||
if sym in positions:
|
if sym in positions:
|
||||||
return
|
return
|
||||||
if calendar_index < cooldown_until_index.get(sym, -1):
|
if calendar_index < cooldown_until_index.get(sym, -1):
|
||||||
skipped_cooldown += 1
|
skipped_cooldown += 1
|
||||||
return
|
return
|
||||||
if max_positions is not None and len(positions) >= max_positions:
|
if len(positions) >= max_positions:
|
||||||
skipped_full += 1
|
skipped_full += 1
|
||||||
if in_measurement:
|
|
||||||
measurement_skipped_full += 1
|
|
||||||
return
|
return
|
||||||
risk_ps = entry - stop
|
risk_ps = entry - stop
|
||||||
if risk_ps <= 0 or entry <= 0:
|
if risk_ps <= 0 or entry <= 0:
|
||||||
@@ -2623,16 +2443,6 @@ def _simulate_portfolio(
|
|||||||
(equity * SIM_NOTIONAL_CAP) / entry,
|
(equity * SIM_NOTIONAL_CAP) / entry,
|
||||||
max(cash, 0.0) / (entry * (1.0 + cost_rate)),
|
max(cash, 0.0) / (entry * (1.0 + cost_rate)),
|
||||||
)
|
)
|
||||||
initial_risk_dollars = shares * risk_ps
|
|
||||||
if (
|
|
||||||
min_initial_risk_fraction is not None
|
|
||||||
and initial_risk_dollars
|
|
||||||
< equity * float(min_initial_risk_fraction)
|
|
||||||
):
|
|
||||||
skipped_min_initial_risk += 1
|
|
||||||
if in_measurement:
|
|
||||||
measurement_skipped_min_initial_risk += 1
|
|
||||||
return
|
|
||||||
if shares * entry < 1.0:
|
if shares * entry < 1.0:
|
||||||
return
|
return
|
||||||
entry_cost = shares * entry * cost_rate
|
entry_cost = shares * entry * cost_rate
|
||||||
@@ -2672,21 +2482,6 @@ def _simulate_portfolio(
|
|||||||
"vol_scalar": scalar,
|
"vol_scalar": scalar,
|
||||||
"corr_scale": corr_scale,
|
"corr_scale": corr_scale,
|
||||||
}
|
}
|
||||||
opened_positions += 1
|
|
||||||
if in_measurement:
|
|
||||||
measurement_opened_positions += 1
|
|
||||||
prior_rebalance_exit = rebalance_exit_index.pop(sym, None)
|
|
||||||
if prior_rebalance_exit is not None:
|
|
||||||
prior_exit_index, prior_exit_ord = prior_rebalance_exit
|
|
||||||
rebalance_reentry_events.append({
|
|
||||||
"symbol": sym,
|
|
||||||
"exit_ord": prior_exit_ord,
|
|
||||||
"exit_calendar_index": prior_exit_index,
|
|
||||||
"reentry_calendar_index": calendar_index,
|
|
||||||
"wait_sessions": calendar_index - prior_exit_index,
|
|
||||||
"reentry_ord": entry_ord,
|
|
||||||
"measurement": in_measurement,
|
|
||||||
})
|
|
||||||
# next_open only: fill is at the open, so the rest of the bar can stop out.
|
# next_open only: fill is at the open, so the rest of the bar can stop out.
|
||||||
# stale_close fills at the close — same-day stop after entry does not apply.
|
# stale_close fills at the close — same-day stop after entry does not apply.
|
||||||
# bars_held stays 0 on the fill day (matches close-fill cadence).
|
# bars_held stays 0 on the fill day (matches close-fill cadence).
|
||||||
@@ -2788,25 +2583,7 @@ def _simulate_portfolio(
|
|||||||
# Queue today's signals for the next session's fill.
|
# Queue today's signals for the next session's fill.
|
||||||
pending_delayed.extend(signal_todays)
|
pending_delayed.extend(signal_todays)
|
||||||
|
|
||||||
marked_equity = _marked_equity()
|
curve.append((o, _marked_equity()))
|
||||||
if in_measurement and include_capacity_diagnostics:
|
|
||||||
gross_notional = sum(
|
|
||||||
pos["shares"] * pos["last_close"] for pos in positions.values()
|
|
||||||
)
|
|
||||||
capacity_samples.append({
|
|
||||||
"positions": len(positions),
|
|
||||||
"cash_pct": cash / marked_equity * 100.0
|
|
||||||
if marked_equity > 0
|
|
||||||
else 0.0,
|
|
||||||
"gross_exposure_pct": gross_notional / marked_equity * 100.0
|
|
||||||
if marked_equity > 0
|
|
||||||
else 0.0,
|
|
||||||
"at_capacity": int(
|
|
||||||
max_positions is not None
|
|
||||||
and len(positions) >= max_positions
|
|
||||||
),
|
|
||||||
})
|
|
||||||
curve.append((o, marked_equity))
|
|
||||||
|
|
||||||
# Close whatever is still open at its last mark so final equity is realized.
|
# Close whatever is still open at its last mark so final equity is realized.
|
||||||
for sym in list(positions):
|
for sym in list(positions):
|
||||||
@@ -2814,57 +2591,32 @@ def _simulate_portfolio(
|
|||||||
final_equity = cash
|
final_equity = cash
|
||||||
curve[-1] = (calendar[-1], final_equity)
|
curve[-1] = (calendar[-1], final_equity)
|
||||||
|
|
||||||
metric_start_ord = (
|
total_return_pct = (final_equity / SIM_STARTING_CAPITAL - 1.0) * 100.0
|
||||||
measurement_start_ord if measurement_start_ord is not None else calendar[0]
|
years = (calendar[-1] - calendar[0]) / 365.25
|
||||||
)
|
|
||||||
metric_curve = [(day_ord, eq) for day_ord, eq in curve if day_ord >= metric_start_ord]
|
|
||||||
if not metric_curve:
|
|
||||||
return None
|
|
||||||
metric_base_equity = (
|
|
||||||
measurement_start_equity
|
|
||||||
if measurement_start_date is not None and measurement_start_equity is not None
|
|
||||||
else SIM_STARTING_CAPITAL
|
|
||||||
)
|
|
||||||
total_return_pct = (final_equity / metric_base_equity - 1.0) * 100.0
|
|
||||||
years = (calendar[-1] - metric_start_ord) / 365.25
|
|
||||||
cagr_pct = (
|
cagr_pct = (
|
||||||
((final_equity / metric_base_equity) ** (1.0 / years) - 1.0) * 100.0
|
((final_equity / SIM_STARTING_CAPITAL) ** (1.0 / years) - 1.0) * 100.0
|
||||||
if years > 0.25 and final_equity > 0
|
if years > 0.25 and final_equity > 0
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
peak = float("-inf")
|
peak = float("-inf")
|
||||||
max_dd = 0.0
|
max_dd = 0.0
|
||||||
drawdown_equities = (
|
for _, eq in curve:
|
||||||
[metric_base_equity, *(eq for _, eq in metric_curve)]
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else [eq for _, eq in metric_curve]
|
|
||||||
)
|
|
||||||
for eq in drawdown_equities:
|
|
||||||
peak = max(peak, eq)
|
peak = max(peak, eq)
|
||||||
if peak > 0:
|
if peak > 0:
|
||||||
max_dd = max(max_dd, (peak - eq) / peak)
|
max_dd = max(max_dd, (peak - eq) / peak)
|
||||||
|
|
||||||
return_equities = (
|
rets = [b / a - 1.0 for (_, a), (_, b) in zip(curve, curve[1:]) if a > 0]
|
||||||
[metric_base_equity, *(eq for _, eq in metric_curve)]
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else [eq for _, eq in metric_curve]
|
|
||||||
)
|
|
||||||
rets = [
|
|
||||||
b / a - 1.0
|
|
||||||
for a, b in zip(return_equities, return_equities[1:])
|
|
||||||
if a > 0
|
|
||||||
]
|
|
||||||
diag = sharpe_diagnostics(rets)
|
diag = sharpe_diagnostics(rets)
|
||||||
sharpe = diag["sharpe"]
|
sharpe = diag["sharpe"]
|
||||||
|
|
||||||
# Per-calendar-year returns off the equity curve — shows whether every year
|
# Per-calendar-year returns off the equity curve — shows whether every year
|
||||||
# contributed or one exceptional stretch carried the result.
|
# contributed or one exceptional stretch carried the result.
|
||||||
yearly: list[dict] = []
|
yearly: list[dict] = []
|
||||||
year_start_eq = metric_base_equity
|
year_start_eq = curve[0][1]
|
||||||
cur_year = date.fromordinal(metric_start_ord).year
|
cur_year = date.fromordinal(curve[0][0]).year
|
||||||
last_eq = metric_base_equity
|
last_eq = curve[0][1]
|
||||||
for o, eq in metric_curve:
|
for o, eq in curve:
|
||||||
y = date.fromordinal(o).year
|
y = date.fromordinal(o).year
|
||||||
if y != cur_year:
|
if y != cur_year:
|
||||||
yearly.append({
|
yearly.append({
|
||||||
@@ -2883,29 +2635,24 @@ def _simulate_portfolio(
|
|||||||
),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
metric_trades = [
|
pnls = [t["pnl"] for t in trades]
|
||||||
trade for trade in trades if trade["entry_ord"] >= metric_start_ord
|
|
||||||
]
|
|
||||||
pnls = [t["pnl"] for t in metric_trades]
|
|
||||||
wins = sum(1 for p in pnls if p > 0)
|
wins = sum(1 for p in pnls if p > 0)
|
||||||
reason_counts = {
|
reason_counts = {
|
||||||
reason: sum(1 for t in metric_trades if t["reason"] == reason)
|
reason: sum(1 for t in trades if t["reason"] == reason)
|
||||||
for reason in sorted({t["reason"] for t in metric_trades})
|
for reason in sorted({t["reason"] for t in trades})
|
||||||
}
|
}
|
||||||
spy_pct = None
|
spy_pct = None
|
||||||
if spy_closes:
|
if spy_closes:
|
||||||
from app.services.benchmark_service import benchmark_return_pct
|
from app.services.benchmark_service import benchmark_return_pct
|
||||||
|
|
||||||
spy_pct = benchmark_return_pct(
|
spy_pct = benchmark_return_pct(
|
||||||
spy_closes,
|
spy_closes, date.fromordinal(calendar[0]), date.fromordinal(calendar[-1])
|
||||||
date.fromordinal(metric_start_ord),
|
|
||||||
date.fromordinal(calendar[-1]),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
curve_payload: list[dict] | None = None
|
curve_payload: list[dict] | None = None
|
||||||
benchmark_payload: list[dict] | None = None
|
benchmark_payload: list[dict] | None = None
|
||||||
if include_curve:
|
if include_curve:
|
||||||
curve_base = metric_base_equity
|
curve_base = curve[0][1] if curve else SIM_STARTING_CAPITAL
|
||||||
curve_payload = [
|
curve_payload = [
|
||||||
{
|
{
|
||||||
"date": date.fromordinal(o).isoformat(),
|
"date": date.fromordinal(o).isoformat(),
|
||||||
@@ -2914,12 +2661,12 @@ def _simulate_portfolio(
|
|||||||
if curve_base > 0
|
if curve_base > 0
|
||||||
else None,
|
else None,
|
||||||
}
|
}
|
||||||
for o, eq in metric_curve
|
for o, eq in curve
|
||||||
]
|
]
|
||||||
if spy_closes:
|
if spy_closes:
|
||||||
benchmark_payload = []
|
benchmark_payload = []
|
||||||
base_spy = None
|
base_spy = None
|
||||||
for o, _ in metric_curve:
|
for o, _ in curve:
|
||||||
d = date.fromordinal(o)
|
d = date.fromordinal(o)
|
||||||
close = spy_closes.get(d)
|
close = spy_closes.get(d)
|
||||||
if close is None or close <= 0:
|
if close is None or close <= 0:
|
||||||
@@ -2938,8 +2685,6 @@ def _simulate_portfolio(
|
|||||||
calmar = float(cagr_pct) / max_dd_pct
|
calmar = float(cagr_pct) / max_dd_pct
|
||||||
result = {
|
result = {
|
||||||
"starting_capital": SIM_STARTING_CAPITAL,
|
"starting_capital": SIM_STARTING_CAPITAL,
|
||||||
"measurement_start_equity": round(metric_base_equity, 2),
|
|
||||||
"measurement_start_positions": measurement_start_position_count or 0,
|
|
||||||
"cost_per_side_pct": round(cost_rate * 100.0, 3),
|
"cost_per_side_pct": round(cost_rate * 100.0, 3),
|
||||||
"fill_mode": fill_mode,
|
"fill_mode": fill_mode,
|
||||||
"final_equity": round(final_equity, 2),
|
"final_equity": round(final_equity, 2),
|
||||||
@@ -2953,161 +2698,23 @@ def _simulate_portfolio(
|
|||||||
"n_returns": diag["n_returns"],
|
"n_returns": diag["n_returns"],
|
||||||
"return_skew": diag["return_skew"],
|
"return_skew": diag["return_skew"],
|
||||||
"return_kurtosis": diag["return_kurtosis"],
|
"return_kurtosis": diag["return_kurtosis"],
|
||||||
"trades": len(metric_trades),
|
"trades": len(trades),
|
||||||
"win_rate": (
|
"win_rate": round(wins / len(trades) * 100.0, 1) if trades else None,
|
||||||
round(wins / len(metric_trades) * 100.0, 1)
|
|
||||||
if metric_trades
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
"avg_trade_pnl": round(sum(pnls) / len(pnls), 2) if pnls else None,
|
"avg_trade_pnl": round(sum(pnls) / len(pnls), 2) if pnls else None,
|
||||||
"best_trade_r": (
|
"best_trade_r": round(max(t["r"] for t in trades), 2) if trades else None,
|
||||||
round(max(t["r"] for t in metric_trades), 2)
|
"worst_trade_r": round(min(t["r"] for t in trades), 2) if trades else None,
|
||||||
if metric_trades
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
"worst_trade_r": (
|
|
||||||
round(min(t["r"] for t in metric_trades), 2)
|
|
||||||
if metric_trades
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
"best_trade_pnl": round(max(pnls), 2) if pnls else None,
|
"best_trade_pnl": round(max(pnls), 2) if pnls else None,
|
||||||
"worst_trade_pnl": round(min(pnls), 2) if pnls else None,
|
"worst_trade_pnl": round(min(pnls), 2) if pnls else None,
|
||||||
"avg_hold_days": (
|
"avg_hold_days": (
|
||||||
round(
|
round(sum(t["hold"] for t in trades) / len(trades), 1) if trades else None
|
||||||
sum(t["hold"] for t in metric_trades) / len(metric_trades),
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
if metric_trades
|
|
||||||
else None
|
|
||||||
),
|
),
|
||||||
"exit_reasons": reason_counts,
|
"exit_reasons": reason_counts,
|
||||||
"skipped_book_full": skipped_full,
|
"skipped_book_full": skipped_full,
|
||||||
"spy_return_pct": round(spy_pct, 1) if spy_pct is not None else None,
|
"spy_return_pct": round(spy_pct, 1) if spy_pct is not None else None,
|
||||||
"yearly_returns": yearly,
|
"yearly_returns": yearly,
|
||||||
"start_date": date.fromordinal(metric_start_ord).isoformat(),
|
"start_date": date.fromordinal(calendar[0]).isoformat(),
|
||||||
"end_date": date.fromordinal(calendar[-1]).isoformat(),
|
"end_date": date.fromordinal(calendar[-1]).isoformat(),
|
||||||
}
|
}
|
||||||
if measurement_start_date is not None:
|
|
||||||
result["simulation_start_date"] = date.fromordinal(calendar[0]).isoformat()
|
|
||||||
if hard_end_date is not None:
|
|
||||||
result["hard_end_date_exclusive"] = hard_end_date.isoformat()
|
|
||||||
if measurement_start_date is not None:
|
|
||||||
result["measurement_skipped_book_full"] = measurement_skipped_full
|
|
||||||
result["measurement_opened_positions"] = measurement_opened_positions
|
|
||||||
if min_initial_risk_fraction is not None:
|
|
||||||
result["min_initial_risk_fraction"] = float(min_initial_risk_fraction)
|
|
||||||
result["skipped_min_initial_risk"] = skipped_min_initial_risk
|
|
||||||
result["measurement_skipped_min_initial_risk"] = (
|
|
||||||
measurement_skipped_min_initial_risk
|
|
||||||
)
|
|
||||||
if include_capacity_diagnostics:
|
|
||||||
measured_opened = (
|
|
||||||
measurement_opened_positions
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else opened_positions
|
|
||||||
)
|
|
||||||
measured_full = (
|
|
||||||
measurement_skipped_full
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else skipped_full
|
|
||||||
)
|
|
||||||
capacity_opportunities = measured_opened + measured_full
|
|
||||||
result["opened_positions"] = measured_opened
|
|
||||||
result["capacity_opportunities"] = capacity_opportunities
|
|
||||||
result["blocked_fraction"] = (
|
|
||||||
round(measured_full / capacity_opportunities, 6)
|
|
||||||
if capacity_opportunities
|
|
||||||
else 0.0
|
|
||||||
)
|
|
||||||
result["avg_positions"] = (
|
|
||||||
round(
|
|
||||||
sum(float(sample["positions"]) for sample in capacity_samples)
|
|
||||||
/ len(capacity_samples),
|
|
||||||
4,
|
|
||||||
)
|
|
||||||
if capacity_samples
|
|
||||||
else 0.0
|
|
||||||
)
|
|
||||||
result["peak_positions"] = (
|
|
||||||
max(int(sample["positions"]) for sample in capacity_samples)
|
|
||||||
if capacity_samples
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
result["sessions_at_capacity"] = sum(
|
|
||||||
int(sample["at_capacity"]) for sample in capacity_samples
|
|
||||||
)
|
|
||||||
result["sessions_measured"] = len(capacity_samples)
|
|
||||||
result["avg_cash_pct"] = (
|
|
||||||
round(
|
|
||||||
sum(float(sample["cash_pct"]) for sample in capacity_samples)
|
|
||||||
/ len(capacity_samples),
|
|
||||||
4,
|
|
||||||
)
|
|
||||||
if capacity_samples
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
result["avg_gross_exposure_pct"] = (
|
|
||||||
round(
|
|
||||||
sum(
|
|
||||||
float(sample["gross_exposure_pct"])
|
|
||||||
for sample in capacity_samples
|
|
||||||
)
|
|
||||||
/ len(capacity_samples),
|
|
||||||
4,
|
|
||||||
)
|
|
||||||
if capacity_samples
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
if weekly_top_n_rebalance:
|
|
||||||
measured_events = [
|
|
||||||
event for event in weekly_rebalance_events if event["measurement"]
|
|
||||||
]
|
|
||||||
measured_reentries = [
|
|
||||||
event for event in rebalance_reentry_events if event["measurement"]
|
|
||||||
]
|
|
||||||
result["weekly_rank_rejected_entries"] = (
|
|
||||||
measurement_weekly_rank_rejected_entries
|
|
||||||
if measurement_start_date is not None
|
|
||||||
else weekly_rank_rejected_entries
|
|
||||||
)
|
|
||||||
result["weekly_rebalance_events"] = [
|
|
||||||
{
|
|
||||||
**{
|
|
||||||
key: value
|
|
||||||
for key, value in event.items()
|
|
||||||
if key not in {"ord", "measurement"}
|
|
||||||
},
|
|
||||||
"date": date.fromordinal(event["ord"]).isoformat(),
|
|
||||||
}
|
|
||||||
for event in measured_events
|
|
||||||
]
|
|
||||||
result["rebalance_reentry_events"] = [
|
|
||||||
{
|
|
||||||
**{
|
|
||||||
key: value
|
|
||||||
for key, value in event.items()
|
|
||||||
if key
|
|
||||||
not in {
|
|
||||||
"exit_ord",
|
|
||||||
"reentry_ord",
|
|
||||||
"measurement",
|
|
||||||
"exit_calendar_index",
|
|
||||||
"reentry_calendar_index",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"exit_date": date.fromordinal(event["exit_ord"]).isoformat(),
|
|
||||||
"reentry_date": date.fromordinal(
|
|
||||||
event["reentry_ord"]
|
|
||||||
).isoformat(),
|
|
||||||
}
|
|
||||||
for event in measured_reentries
|
|
||||||
]
|
|
||||||
for session_limit in (5, 10, 20):
|
|
||||||
result[f"rebalance_reentries_within_{session_limit}_sessions"] = sum(
|
|
||||||
1
|
|
||||||
for event in measured_reentries
|
|
||||||
if int(event["wait_sessions"]) <= session_limit
|
|
||||||
)
|
|
||||||
if vol_target is not None:
|
if vol_target is not None:
|
||||||
result["vol_target"] = vol_target
|
result["vol_target"] = vol_target
|
||||||
result["vol_lookback"] = int(vol_lookback)
|
result["vol_lookback"] = int(vol_lookback)
|
||||||
@@ -3182,7 +2789,7 @@ def _simulate_portfolio(
|
|||||||
"entry_date": date.fromordinal(trade["entry_ord"]).isoformat(),
|
"entry_date": date.fromordinal(trade["entry_ord"]).isoformat(),
|
||||||
"exit_date": date.fromordinal(trade["exit_ord"]).isoformat(),
|
"exit_date": date.fromordinal(trade["exit_ord"]).isoformat(),
|
||||||
}
|
}
|
||||||
for trade in metric_trades
|
for trade in trades
|
||||||
]
|
]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
"""A5 activation: refresh the legacy fundamentals cache from local bulk data."""
|
"""Refresh the fundamentals compat cache from local SEC/Dolt bulk data.
|
||||||
|
|
||||||
|
``fundamental_data`` is the table scoring reads. This is its only writer.
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -12,38 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from app.database import insert_for_session
|
from app.database import insert_for_session
|
||||||
from app.models.fundamental import FundamentalData
|
from app.models.fundamental import FundamentalData
|
||||||
from app.models.score import CompositeScore, DimensionScore
|
from app.models.score import CompositeScore, DimensionScore
|
||||||
from app.services import fundamentals_candidate_service, settings_store
|
from app.services import fundamentals_candidate_service
|
||||||
|
|
||||||
|
|
||||||
# Absence is deliberately false. Production activation therefore requires one
|
|
||||||
# explicit, durable SystemSetting change after the A5 evidence is approved.
|
|
||||||
ACTIVATION_KEY = "fundamental_data_sec_dolt_cutover_enabled"
|
|
||||||
_SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
_SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
||||||
|
|
||||||
|
|
||||||
async def is_enabled(db: AsyncSession) -> bool:
|
|
||||||
raw = await settings_store.get_value(db, ACTIVATION_KEY, "false")
|
|
||||||
return str(raw).strip().lower() == "true"
|
|
||||||
|
|
||||||
|
|
||||||
async def refresh_if_enabled(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
now: datetime | None = None,
|
|
||||||
today: date | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Refresh atomically when activated; otherwise perform no writes."""
|
|
||||||
if not await is_enabled(db):
|
|
||||||
return {
|
|
||||||
"enabled": False,
|
|
||||||
"refreshed": 0,
|
|
||||||
"score_inputs_changed": 0,
|
|
||||||
"dimension_scores_staled": 0,
|
|
||||||
"composite_scores_staled": 0,
|
|
||||||
}
|
|
||||||
return await refresh(db, now=now, today=today)
|
|
||||||
|
|
||||||
|
|
||||||
async def refresh(
|
async def refresh(
|
||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
*,
|
*,
|
||||||
@@ -117,7 +93,6 @@ async def refresh(
|
|||||||
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
return {
|
return {
|
||||||
"enabled": True,
|
|
||||||
"refreshed": len(candidates),
|
"refreshed": len(candidates),
|
||||||
"score_inputs_changed": len(changed_ids),
|
"score_inputs_changed": len(changed_ids),
|
||||||
"dimension_scores_staled": len(dimension_ids),
|
"dimension_scores_staled": len(dimension_ids),
|
||||||
|
|||||||
@@ -1,22 +1,19 @@
|
|||||||
"""Fundamental data service.
|
"""Fundamental data read access.
|
||||||
|
|
||||||
Stores fundamental data (P/E, revenue growth, earnings surprise, market cap)
|
``fundamental_data`` is the compat cache scoring reads. It is written solely by
|
||||||
and marks the fundamental dimension score as stale on new data.
|
``fundamental_data_refresh_service`` from SEC snapshots, Dolt earnings events and
|
||||||
|
stored closes; nothing fetches it per ticker.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from sqlalchemy import select, update
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.database import insert_for_session
|
|
||||||
from app.exceptions import NotFoundError
|
from app.exceptions import NotFoundError
|
||||||
from app.models.fundamental import FundamentalData
|
from app.models.fundamental import FundamentalData
|
||||||
from app.models.score import DimensionScore
|
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -32,65 +29,6 @@ async def _get_ticker(db: AsyncSession, symbol: str) -> Ticker:
|
|||||||
return ticker
|
return ticker
|
||||||
|
|
||||||
|
|
||||||
async def store_fundamental(
|
|
||||||
db: AsyncSession,
|
|
||||||
symbol: str,
|
|
||||||
pe_ratio: float | None = None,
|
|
||||||
revenue_growth: float | None = None,
|
|
||||||
earnings_surprise: float | None = None,
|
|
||||||
market_cap: float | None = None,
|
|
||||||
next_earnings_date=None,
|
|
||||||
unavailable_fields: dict[str, str] | None = None,
|
|
||||||
) -> FundamentalData:
|
|
||||||
"""Store or update fundamental data for a ticker.
|
|
||||||
|
|
||||||
Keeps a single latest snapshot per ticker. On new data, marks the
|
|
||||||
fundamental dimension score as stale (if one exists).
|
|
||||||
"""
|
|
||||||
ticker = await _get_ticker(db, symbol)
|
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
unavailable_fields_json = json.dumps(unavailable_fields or {})
|
|
||||||
|
|
||||||
stmt = insert_for_session(db, FundamentalData).values(
|
|
||||||
ticker_id=ticker.id,
|
|
||||||
pe_ratio=pe_ratio,
|
|
||||||
revenue_growth=revenue_growth,
|
|
||||||
earnings_surprise=earnings_surprise,
|
|
||||||
market_cap=market_cap,
|
|
||||||
next_earnings_date=next_earnings_date,
|
|
||||||
fetched_at=now,
|
|
||||||
unavailable_fields_json=unavailable_fields_json,
|
|
||||||
)
|
|
||||||
stmt = stmt.on_conflict_do_update(
|
|
||||||
index_elements=["ticker_id"],
|
|
||||||
set_={
|
|
||||||
"pe_ratio": stmt.excluded.pe_ratio,
|
|
||||||
"revenue_growth": stmt.excluded.revenue_growth,
|
|
||||||
"earnings_surprise": stmt.excluded.earnings_surprise,
|
|
||||||
"market_cap": stmt.excluded.market_cap,
|
|
||||||
"next_earnings_date": stmt.excluded.next_earnings_date,
|
|
||||||
"fetched_at": stmt.excluded.fetched_at,
|
|
||||||
"unavailable_fields_json": stmt.excluded.unavailable_fields_json,
|
|
||||||
},
|
|
||||||
).returning(FundamentalData)
|
|
||||||
record = (await db.execute(stmt)).scalar_one()
|
|
||||||
|
|
||||||
# Mark fundamental dimension score as stale if it exists
|
|
||||||
# TODO: Use DimensionScore service when built
|
|
||||||
await db.execute(
|
|
||||||
update(DimensionScore)
|
|
||||||
.where(
|
|
||||||
DimensionScore.ticker_id == ticker.id,
|
|
||||||
DimensionScore.dimension == "fundamental",
|
|
||||||
)
|
|
||||||
.values(is_stale=True)
|
|
||||||
)
|
|
||||||
|
|
||||||
await db.commit()
|
|
||||||
return record
|
|
||||||
|
|
||||||
|
|
||||||
async def get_fundamental(
|
async def get_fundamental(
|
||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
symbol: str,
|
symbol: str,
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
"""Local SEC/Dolt candidate values for the legacy fundamentals cache.
|
"""Local SEC/Dolt candidate values for the fundamentals compat cache.
|
||||||
|
|
||||||
This is the single read path shared by the A5 parity report and the activated
|
This is the read path behind the ``fundamental_data`` refresh. It never contacts
|
||||||
``fundamental_data`` refresh. It never contacts SEC or Dolt: every input comes
|
SEC or Dolt: every input comes from PostgreSQL, so price- and earnings-driven
|
||||||
from PostgreSQL, so price- and earnings-driven values can still refresh when an
|
values can still refresh when an upstream import is unchanged or unavailable.
|
||||||
upstream import is unchanged or unavailable.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|||||||
@@ -1,498 +0,0 @@
|
|||||||
"""Read-only A5 comparison of legacy and SEC/Dolt fundamental inputs.
|
|
||||||
|
|
||||||
The report deliberately does not write ``fundamental_data`` or score tables.
|
|
||||||
It reconstructs the current legacy and candidate fundamental scores, projects
|
|
||||||
their composite-score/rank effect with the active weights, and archives a
|
|
||||||
timestamped JSON + CSV bundle for explicit human approval.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import csv
|
|
||||||
import io
|
|
||||||
import json
|
|
||||||
import math
|
|
||||||
import os
|
|
||||||
import statistics
|
|
||||||
from datetime import date, datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Iterable
|
|
||||||
from zoneinfo import ZoneInfo
|
|
||||||
|
|
||||||
from sqlalchemy import select, text
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from app.models.data_import_run import DataImportRun
|
|
||||||
from app.models.fundamental import FundamentalData
|
|
||||||
from app.services import fundamentals_candidate_service as candidate_service
|
|
||||||
|
|
||||||
REPORT_VERSION = 1
|
|
||||||
APPROVAL_STATUS = "pending_explicit_approval"
|
|
||||||
FIELD_KEYS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
|
||||||
MIN_SCORE_METRICS = 2
|
|
||||||
|
|
||||||
# Materiality is a review aid, never an automatic cutover verdict. Definition
|
|
||||||
# changes remain visible even when a delta falls inside these bands.
|
|
||||||
FIELD_TOLERANCES = {
|
|
||||||
"pe_ratio": {"absolute": 1.0, "relative_pct": 10.0},
|
|
||||||
"revenue_growth": {"absolute": 2.0, "relative_pct": None},
|
|
||||||
"earnings_surprise": {"absolute": 2.0, "relative_pct": None},
|
|
||||||
}
|
|
||||||
DEFINITION_NOTES = {
|
|
||||||
"pe_ratio": (
|
|
||||||
"Legacy provider P/E convention versus latest close divided by "
|
|
||||||
"SEC-derived TTM diluted EPS."
|
|
||||||
),
|
|
||||||
"revenue_growth": (
|
|
||||||
"Legacy provider growth convention versus SEC-derived TTM revenue YoY."
|
|
||||||
),
|
|
||||||
"earnings_surprise": (
|
|
||||||
"Legacy provider latest surprise versus latest completed Dolt earnings "
|
|
||||||
"event with actual and estimate."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def fundamental_score(
|
|
||||||
pe_ratio: float | None,
|
|
||||||
revenue_growth: float | None,
|
|
||||||
earnings_surprise: float | None,
|
|
||||||
) -> float | None:
|
|
||||||
"""Match the production fundamental-dimension formula without persistence."""
|
|
||||||
scores: list[float] = []
|
|
||||||
if _finite(pe_ratio) and pe_ratio > 0:
|
|
||||||
scores.append(max(0.0, min(100.0, 100.0 - (pe_ratio - 15.0) * (100.0 / 30.0))))
|
|
||||||
if _finite(revenue_growth):
|
|
||||||
scores.append(max(0.0, min(100.0, 50.0 + revenue_growth * 2.5)))
|
|
||||||
if _finite(earnings_surprise):
|
|
||||||
scores.append(max(0.0, min(100.0, 50.0 + earnings_surprise * 5.0)))
|
|
||||||
return sum(scores) / len(scores) if len(scores) >= MIN_SCORE_METRICS else None
|
|
||||||
|
|
||||||
|
|
||||||
async def build_report(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
generated_at: datetime | None = None,
|
|
||||||
today: date | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Build a point-in-time parity report from one database session."""
|
|
||||||
generated_at = generated_at or datetime.now(timezone.utc)
|
|
||||||
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
|
||||||
|
|
||||||
# A report must not mix rows from before and after a concurrent import
|
|
||||||
# promotion. The scheduled job provides a fresh session, so establish the
|
|
||||||
# production snapshot before its first query and have Postgres enforce the
|
|
||||||
# no-write contract as well. SQLite tests retain their normal transaction.
|
|
||||||
if db.get_bind().dialect.name == "postgresql":
|
|
||||||
connection = await db.connection(
|
|
||||||
execution_options={"isolation_level": "REPEATABLE READ"}
|
|
||||||
)
|
|
||||||
await connection.execute(text("SET TRANSACTION READ ONLY"))
|
|
||||||
|
|
||||||
candidates = await candidate_service.build_candidates(db, today=today)
|
|
||||||
ticker_ids = [candidate.ticker_id for candidate in candidates]
|
|
||||||
legacy_by_ticker = await _legacy_values(db, ticker_ids)
|
|
||||||
source_runs = await _source_runs(db)
|
|
||||||
|
|
||||||
rows: list[dict[str, Any]] = []
|
|
||||||
for candidate in candidates:
|
|
||||||
legacy = legacy_by_ticker.get(candidate.ticker_id)
|
|
||||||
candidate_values = {
|
|
||||||
"pe_ratio": candidate.pe_ratio,
|
|
||||||
"revenue_growth": candidate.revenue_growth,
|
|
||||||
"earnings_surprise": candidate.earnings_surprise,
|
|
||||||
}
|
|
||||||
legacy_values = {
|
|
||||||
"pe_ratio": legacy.pe_ratio if legacy else None,
|
|
||||||
"revenue_growth": legacy.revenue_growth if legacy else None,
|
|
||||||
"earnings_surprise": legacy.earnings_surprise if legacy else None,
|
|
||||||
}
|
|
||||||
fields = {
|
|
||||||
key: _field_comparison(key, legacy_values[key], candidate_values[key])
|
|
||||||
for key in FIELD_KEYS
|
|
||||||
}
|
|
||||||
legacy_score = fundamental_score(**legacy_values)
|
|
||||||
candidate_score = fundamental_score(**candidate_values)
|
|
||||||
rows.append(
|
|
||||||
{
|
|
||||||
"symbol": candidate.symbol,
|
|
||||||
"cik": candidate.cik,
|
|
||||||
"legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
|
|
||||||
"price_date": _iso(candidate.price_date),
|
|
||||||
"fields": fields,
|
|
||||||
"scores": {
|
|
||||||
"legacy_fundamental": _round(legacy_score),
|
|
||||||
"candidate_fundamental": _round(candidate_score),
|
|
||||||
"fundamental_delta": _delta(legacy_score, candidate_score),
|
|
||||||
"legacy_fundamental_rank": None,
|
|
||||||
"candidate_fundamental_rank": None,
|
|
||||||
"fundamental_rank_change": None,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
_attach_ranks(rows, "legacy_fundamental", "legacy_fundamental_rank")
|
|
||||||
_attach_ranks(rows, "candidate_fundamental", "candidate_fundamental_rank")
|
|
||||||
for row in rows:
|
|
||||||
scores = row["scores"]
|
|
||||||
scores["fundamental_rank_change"] = _rank_change(
|
|
||||||
scores["legacy_fundamental_rank"], scores["candidate_fundamental_rank"]
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"report_version": REPORT_VERSION,
|
|
||||||
"generated_at": generated_at.isoformat(),
|
|
||||||
"as_of_date": today.isoformat(),
|
|
||||||
"approval_status": APPROVAL_STATUS,
|
|
||||||
"read_only": True,
|
|
||||||
"fundamental_score_formula": (
|
|
||||||
"Equal-weighted mean of 2+ available sub-scores: P/E = "
|
|
||||||
"clamp(100-(pe-15)*(100/30)); revenue growth = "
|
|
||||||
"clamp(50+growth*2.5); earnings surprise = "
|
|
||||||
"clamp(50+surprise*5)."
|
|
||||||
),
|
|
||||||
"source_runs": source_runs,
|
|
||||||
"definition_notes": DEFINITION_NOTES,
|
|
||||||
"materiality_notes": {
|
|
||||||
"fields": FIELD_TOLERANCES,
|
|
||||||
"fundamental_score_absolute": 5.0,
|
|
||||||
"automatic_cutover": False,
|
|
||||||
},
|
|
||||||
"summary": _summary(rows),
|
|
||||||
"rows": rows,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def store_report(report: dict[str, Any], report_dir: str | Path) -> dict[str, str]:
|
|
||||||
"""Atomically archive JSON/CSV artifacts and update the latest manifest."""
|
|
||||||
directory = Path(report_dir).expanduser().resolve()
|
|
||||||
directory.mkdir(parents=True, exist_ok=True)
|
|
||||||
stamp = _artifact_stamp(report["generated_at"])
|
|
||||||
json_name = f"fundamentals-parity-{stamp}.json"
|
|
||||||
csv_name = f"fundamentals-parity-{stamp}.csv"
|
|
||||||
json_path = directory / json_name
|
|
||||||
csv_path = directory / csv_name
|
|
||||||
|
|
||||||
_atomic_write(json_path, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
|
||||||
_atomic_write(csv_path, report_csv(report))
|
|
||||||
manifest = {
|
|
||||||
"generated_at": report["generated_at"],
|
|
||||||
"json_file": json_name,
|
|
||||||
"csv_file": csv_name,
|
|
||||||
}
|
|
||||||
_atomic_write(
|
|
||||||
directory / "latest.json",
|
|
||||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"json": str(json_path),
|
|
||||||
"csv": str(csv_path),
|
|
||||||
"manifest": str(directory / "latest.json"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def generate_and_store(
|
|
||||||
db: AsyncSession,
|
|
||||||
report_dir: str | Path,
|
|
||||||
*,
|
|
||||||
generated_at: datetime | None = None,
|
|
||||||
today: date | None = None,
|
|
||||||
) -> tuple[dict[str, Any], dict[str, str]]:
|
|
||||||
report = await build_report(db, generated_at=generated_at, today=today)
|
|
||||||
return report, store_report(report, report_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def load_latest(report_dir: str | Path) -> dict[str, Any] | None:
|
|
||||||
manifest = _load_manifest(report_dir)
|
|
||||||
if manifest is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
path = _manifest_artifact(report_dir, manifest, "json_file")
|
|
||||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
return loaded if isinstance(loaded, dict) else None
|
|
||||||
|
|
||||||
|
|
||||||
def load_latest_csv(report_dir: str | Path) -> tuple[str, str] | None:
|
|
||||||
return _load_latest_text_artifact(report_dir, "csv_file")
|
|
||||||
|
|
||||||
|
|
||||||
def load_latest_json(report_dir: str | Path) -> tuple[str, str] | None:
|
|
||||||
return _load_latest_text_artifact(report_dir, "json_file")
|
|
||||||
|
|
||||||
|
|
||||||
def _load_latest_text_artifact(
|
|
||||||
report_dir: str | Path, manifest_key: str
|
|
||||||
) -> tuple[str, str] | None:
|
|
||||||
manifest = _load_manifest(report_dir)
|
|
||||||
if manifest is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
path = _manifest_artifact(report_dir, manifest, manifest_key)
|
|
||||||
return path.name, path.read_text(encoding="utf-8")
|
|
||||||
except (OSError, TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def report_csv(report: dict[str, Any]) -> str:
|
|
||||||
output = io.StringIO(newline="")
|
|
||||||
columns = [
|
|
||||||
"symbol",
|
|
||||||
"cik",
|
|
||||||
"legacy_fetched_at",
|
|
||||||
"price_date",
|
|
||||||
*(
|
|
||||||
f"{field}_{suffix}"
|
|
||||||
for field in FIELD_KEYS
|
|
||||||
for suffix in ("legacy", "candidate", "absolute_delta", "relative_delta_pct", "material")
|
|
||||||
),
|
|
||||||
"legacy_fundamental",
|
|
||||||
"candidate_fundamental",
|
|
||||||
"fundamental_delta",
|
|
||||||
"legacy_fundamental_rank",
|
|
||||||
"candidate_fundamental_rank",
|
|
||||||
"fundamental_rank_change",
|
|
||||||
]
|
|
||||||
writer = csv.DictWriter(output, fieldnames=columns)
|
|
||||||
writer.writeheader()
|
|
||||||
for row in report.get("rows", []):
|
|
||||||
flat = {
|
|
||||||
"symbol": row["symbol"],
|
|
||||||
"cik": row.get("cik"),
|
|
||||||
"legacy_fetched_at": row.get("legacy_fetched_at"),
|
|
||||||
"price_date": row.get("price_date"),
|
|
||||||
**row["scores"],
|
|
||||||
}
|
|
||||||
for field in FIELD_KEYS:
|
|
||||||
comparison = row["fields"][field]
|
|
||||||
for suffix in (
|
|
||||||
"legacy",
|
|
||||||
"candidate",
|
|
||||||
"absolute_delta",
|
|
||||||
"relative_delta_pct",
|
|
||||||
"material",
|
|
||||||
):
|
|
||||||
flat[f"{field}_{suffix}"] = comparison.get(suffix)
|
|
||||||
writer.writerow(flat)
|
|
||||||
return output.getvalue()
|
|
||||||
|
|
||||||
|
|
||||||
async def _legacy_values(
|
|
||||||
db: AsyncSession, ticker_ids: list[int]
|
|
||||||
) -> dict[int, FundamentalData]:
|
|
||||||
if not ticker_ids:
|
|
||||||
return {}
|
|
||||||
rows = (
|
|
||||||
await db.execute(
|
|
||||||
select(FundamentalData).where(FundamentalData.ticker_id.in_(ticker_ids))
|
|
||||||
)
|
|
||||||
).scalars()
|
|
||||||
return {row.ticker_id: row for row in rows}
|
|
||||||
|
|
||||||
|
|
||||||
async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
|
|
||||||
sources = ("sec_facts", "dolt_earnings")
|
|
||||||
rows = (
|
|
||||||
await db.execute(
|
|
||||||
select(DataImportRun)
|
|
||||||
.where(
|
|
||||||
DataImportRun.source.in_(sources),
|
|
||||||
DataImportRun.status.in_(("promoted", "no_op")),
|
|
||||||
)
|
|
||||||
.order_by(DataImportRun.id.desc())
|
|
||||||
)
|
|
||||||
).scalars()
|
|
||||||
latest: dict[str, dict[str, Any] | None] = {source: None for source in sources}
|
|
||||||
for row in rows:
|
|
||||||
if latest[row.source] is None:
|
|
||||||
latest[row.source] = {
|
|
||||||
"run_id": row.id,
|
|
||||||
"status": row.status,
|
|
||||||
"revision": row.revision,
|
|
||||||
"source_max_date": _iso(row.source_max_date),
|
|
||||||
"completed_at": _iso(row.completed_at),
|
|
||||||
}
|
|
||||||
return latest
|
|
||||||
|
|
||||||
|
|
||||||
def _field_comparison(
|
|
||||||
key: str, legacy: float | None, candidate: float | None
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
legacy = float(legacy) if _finite(legacy) else None
|
|
||||||
candidate = float(candidate) if _finite(candidate) else None
|
|
||||||
absolute = _delta(legacy, candidate)
|
|
||||||
relative = (
|
|
||||||
None
|
|
||||||
if absolute is None or legacy in (None, 0)
|
|
||||||
else round(absolute / abs(legacy) * 100.0, 4)
|
|
||||||
)
|
|
||||||
tolerance = FIELD_TOLERANCES[key]
|
|
||||||
material = False
|
|
||||||
if absolute is not None:
|
|
||||||
material = abs(absolute) > tolerance["absolute"]
|
|
||||||
relative_limit = tolerance["relative_pct"]
|
|
||||||
if relative_limit is not None:
|
|
||||||
material = material and relative is not None and abs(relative) > relative_limit
|
|
||||||
return {
|
|
||||||
"legacy": _round(legacy),
|
|
||||||
"candidate": _round(candidate),
|
|
||||||
"absolute_delta": absolute,
|
|
||||||
"relative_delta_pct": relative,
|
|
||||||
"material": material,
|
|
||||||
"definition_changed": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _attach_ranks(rows: list[dict[str, Any]], value_key: str, rank_key: str) -> None:
|
|
||||||
values = [
|
|
||||||
row["scores"][value_key]
|
|
||||||
for row in rows
|
|
||||||
if _finite(row["scores"][value_key])
|
|
||||||
]
|
|
||||||
for row in rows:
|
|
||||||
value = row["scores"][value_key]
|
|
||||||
row["scores"][rank_key] = (
|
|
||||||
1 + sum(other > value for other in values) if _finite(value) else None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
||||||
field_stats = {}
|
|
||||||
for key in FIELD_KEYS:
|
|
||||||
comparisons = [row["fields"][key] for row in rows]
|
|
||||||
deltas = [
|
|
||||||
abs(item["absolute_delta"])
|
|
||||||
for item in comparisons
|
|
||||||
if item["absolute_delta"] is not None
|
|
||||||
]
|
|
||||||
field_stats[key] = {
|
|
||||||
"legacy_available": sum(item["legacy"] is not None for item in comparisons),
|
|
||||||
"candidate_available": sum(
|
|
||||||
item["candidate"] is not None for item in comparisons
|
|
||||||
),
|
|
||||||
"both_available": len(deltas),
|
|
||||||
"material_differences": sum(item["material"] for item in comparisons),
|
|
||||||
"median_absolute_delta": _round(statistics.median(deltas) if deltas else None),
|
|
||||||
"p95_absolute_delta": _round(_percentile(deltas, 0.95)),
|
|
||||||
"max_absolute_delta": _round(max(deltas) if deltas else None),
|
|
||||||
}
|
|
||||||
|
|
||||||
fundamental_deltas = _score_deltas(rows, "fundamental_delta")
|
|
||||||
changed_rows = sorted(
|
|
||||||
(
|
|
||||||
{
|
|
||||||
"symbol": row["symbol"],
|
|
||||||
"fundamental_delta": row["scores"]["fundamental_delta"],
|
|
||||||
"fundamental_rank_change": row["scores"]["fundamental_rank_change"],
|
|
||||||
}
|
|
||||||
for row in rows
|
|
||||||
if row["scores"]["fundamental_delta"] is not None
|
|
||||||
),
|
|
||||||
key=lambda item: (
|
|
||||||
abs(item["fundamental_delta"] or 0),
|
|
||||||
),
|
|
||||||
reverse=True,
|
|
||||||
)[:20]
|
|
||||||
return {
|
|
||||||
"universe_count": len(rows),
|
|
||||||
"legacy_fundamental_score_available": _count_score(
|
|
||||||
rows, "legacy_fundamental"
|
|
||||||
),
|
|
||||||
"candidate_fundamental_score_available": _count_score(
|
|
||||||
rows, "candidate_fundamental"
|
|
||||||
),
|
|
||||||
"fundamental_scores_compared": len(fundamental_deltas),
|
|
||||||
"fundamental_score_material_changes": sum(
|
|
||||||
abs(delta) > 5.0 for delta in fundamental_deltas
|
|
||||||
),
|
|
||||||
"fundamental_rank_changes": _rank_change_count(
|
|
||||||
rows, "fundamental_rank_change"
|
|
||||||
),
|
|
||||||
"field_stats": field_stats,
|
|
||||||
"largest_changes": changed_rows,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _score_deltas(rows: Iterable[dict[str, Any]], key: str) -> list[float]:
|
|
||||||
return [
|
|
||||||
row["scores"][key]
|
|
||||||
for row in rows
|
|
||||||
if row["scores"][key] is not None
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _count_score(rows: Iterable[dict[str, Any]], key: str) -> int:
|
|
||||||
return sum(row["scores"][key] is not None for row in rows)
|
|
||||||
|
|
||||||
|
|
||||||
def _rank_change_count(rows: Iterable[dict[str, Any]], key: str) -> int:
|
|
||||||
return sum(
|
|
||||||
row["scores"][key] not in (None, 0)
|
|
||||||
for row in rows
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _rank_change(legacy: int | None, candidate: int | None) -> int | None:
|
|
||||||
# Positive means the candidate improved its rank.
|
|
||||||
return legacy - candidate if legacy is not None and candidate is not None else None
|
|
||||||
|
|
||||||
|
|
||||||
def _delta(legacy: float | None, candidate: float | None) -> float | None:
|
|
||||||
if not _finite(legacy) or not _finite(candidate):
|
|
||||||
return None
|
|
||||||
return round(candidate - legacy, 4)
|
|
||||||
|
|
||||||
|
|
||||||
def _round(value: float | None, digits: int = 4) -> float | None:
|
|
||||||
return round(float(value), digits) if _finite(value) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _percentile(values: list[float], quantile: float) -> float | None:
|
|
||||||
if not values:
|
|
||||||
return None
|
|
||||||
ordered = sorted(values)
|
|
||||||
index = max(0, math.ceil(quantile * len(ordered)) - 1)
|
|
||||||
return ordered[index]
|
|
||||||
|
|
||||||
|
|
||||||
def _finite(value: Any) -> bool:
|
|
||||||
return (
|
|
||||||
isinstance(value, (int, float))
|
|
||||||
and not isinstance(value, bool)
|
|
||||||
and math.isfinite(value)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _iso(value: Any) -> str | None:
|
|
||||||
return value.isoformat() if value is not None else None
|
|
||||||
|
|
||||||
|
|
||||||
def _artifact_stamp(raw: str) -> str:
|
|
||||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
||||||
return parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
|
||||||
|
|
||||||
|
|
||||||
def _atomic_write(path: Path, content: str) -> None:
|
|
||||||
temp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
||||||
temp.write_text(content, encoding="utf-8", newline="")
|
|
||||||
os.replace(temp, path)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_manifest(report_dir: str | Path) -> dict[str, Any] | None:
|
|
||||||
path = Path(report_dir).expanduser().resolve() / "latest.json"
|
|
||||||
try:
|
|
||||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
return loaded if isinstance(loaded, dict) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _manifest_artifact(
|
|
||||||
report_dir: str | Path, manifest: dict[str, Any], key: str
|
|
||||||
) -> Path:
|
|
||||||
directory = Path(report_dir).expanduser().resolve()
|
|
||||||
name = Path(str(manifest.get(key, ""))).name
|
|
||||||
if not name:
|
|
||||||
raise ValueError(f"Latest parity manifest has no {key}")
|
|
||||||
return directory / name
|
|
||||||
@@ -12,7 +12,6 @@ from app.models.data_import_run import DataImportRun
|
|||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
from app.models.sec_filing_gap import SecFilingGap
|
from app.models.sec_filing_gap import SecFilingGap
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.services import fundamental_data_refresh_service
|
|
||||||
|
|
||||||
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
||||||
|
|
||||||
@@ -78,8 +77,6 @@ async def blocked_reasons_by_cik(
|
|||||||
ciks: set[str] | None = None,
|
ciks: set[str] | None = None,
|
||||||
) -> dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Current SEC blocker code by CIK; no historical audit scan."""
|
"""Current SEC blocker code by CIK; no historical audit scan."""
|
||||||
if not await fundamental_data_refresh_service.is_enabled(db):
|
|
||||||
return {}
|
|
||||||
if ciks is not None and not ciks:
|
if ciks is not None and not ciks:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|||||||
@@ -497,8 +497,8 @@ async def _compute_fundamental_score(
|
|||||||
"reason": "Earnings surprise data not available",
|
"reason": "Earnings surprise data not available",
|
||||||
})
|
})
|
||||||
|
|
||||||
# Require at least two real metrics — a single available metric (e.g. only
|
# Require at least two real metrics — a single available metric (e.g. an
|
||||||
# market cap is free on FMP) does not make a meaningful fundamental score.
|
# issuer with only a market cap) does not make a meaningful fundamental score.
|
||||||
MIN_METRICS = 2
|
MIN_METRICS = 2
|
||||||
if len(scores) < MIN_METRICS:
|
if len(scores) < MIN_METRICS:
|
||||||
unavailable.append({
|
unavailable.append({
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ logger = logging.getLogger(__name__)
|
|||||||
_WWW = "https://www.sec.gov"
|
_WWW = "https://www.sec.gov"
|
||||||
_DATA = "https://data.sec.gov"
|
_DATA = "https://data.sec.gov"
|
||||||
|
|
||||||
# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py).
|
# Resolve CA bundle for explicit httpx verify (matches app/providers/alpaca.py).
|
||||||
_CA = os.environ.get("SSL_CERT_FILE", "")
|
_CA = os.environ.get("SSL_CERT_FILE", "")
|
||||||
_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
|
_CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True
|
||||||
|
|
||||||
|
|||||||
@@ -40,10 +40,12 @@ KEY_CAPACITY = "shadow_book_capacity"
|
|||||||
KEY_RISK_PCT = "shadow_book_risk_pct"
|
KEY_RISK_PCT = "shadow_book_risk_pct"
|
||||||
KEY_START_EQUITY = "shadow_book_start_equity"
|
KEY_START_EQUITY = "shadow_book_start_equity"
|
||||||
|
|
||||||
# Matches the validated configuration: 10-position book, 1% fixed-fractional
|
# Matches the validated configuration: 1% fixed-fractional risk, and a count cap
|
||||||
# risk. Start equity is only a sizing base — comparisons are drawn in percent
|
# set as headroom rather than a target — see backtest_service.SIM_MAX_POSITIONS,
|
||||||
# and R-multiples, never in raw currency.
|
# which this must track. NOTIONAL_CAP below saturates the book near 12 positions,
|
||||||
DEFAULT_CAPACITY = 10
|
# so the count cap should simply never bind. Start equity is only a sizing base —
|
||||||
|
# comparisons are drawn in percent and R-multiples, never in raw currency.
|
||||||
|
DEFAULT_CAPACITY = 15
|
||||||
DEFAULT_RISK_PCT = 1.0
|
DEFAULT_RISK_PCT = 1.0
|
||||||
DEFAULT_START_EQUITY = 100_000.0
|
DEFAULT_START_EQUITY = 100_000.0
|
||||||
|
|
||||||
|
|||||||
@@ -113,116 +113,6 @@ def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
|
|||||||
return sorted(deduped)
|
return sorted(deduped)
|
||||||
|
|
||||||
|
|
||||||
def _extract_symbols_from_fmp_payload(payload: object) -> list[str]:
|
|
||||||
if not isinstance(payload, list):
|
|
||||||
return []
|
|
||||||
|
|
||||||
symbols: list[str] = []
|
|
||||||
for item in payload:
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
continue
|
|
||||||
candidate = item.get("symbol") or item.get("ticker")
|
|
||||||
if isinstance(candidate, str):
|
|
||||||
symbols.append(candidate)
|
|
||||||
return symbols
|
|
||||||
|
|
||||||
|
|
||||||
async def _try_fmp_urls(
|
|
||||||
client: httpx.AsyncClient,
|
|
||||||
urls: list[str],
|
|
||||||
) -> tuple[list[str], list[str]]:
|
|
||||||
failures: list[str] = []
|
|
||||||
for url in urls:
|
|
||||||
endpoint = url.split("?")[0]
|
|
||||||
try:
|
|
||||||
response = await client.get(url)
|
|
||||||
except httpx.HTTPError as exc:
|
|
||||||
failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if response.status_code != 200:
|
|
||||||
failures.append(f"{endpoint}: HTTP {response.status_code}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
payload = response.json()
|
|
||||||
except ValueError:
|
|
||||||
failures.append(f"{endpoint}: invalid JSON payload")
|
|
||||||
continue
|
|
||||||
|
|
||||||
symbols = _extract_symbols_from_fmp_payload(payload)
|
|
||||||
if symbols:
|
|
||||||
return symbols, failures
|
|
||||||
|
|
||||||
failures.append(f"{endpoint}: empty/unsupported payload")
|
|
||||||
|
|
||||||
return [], failures
|
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
|
|
||||||
if not settings.fmp_api_key:
|
|
||||||
raise ValidationError(
|
|
||||||
"FMP API key is required for universe bootstrap (set FMP_API_KEY)"
|
|
||||||
)
|
|
||||||
|
|
||||||
api_key = settings.fmp_api_key
|
|
||||||
stable_base = "https://financialmodelingprep.com/stable"
|
|
||||||
legacy_base = "https://financialmodelingprep.com/api/v3"
|
|
||||||
|
|
||||||
stable_candidates: dict[str, list[str]] = {
|
|
||||||
"sp500": [
|
|
||||||
f"{stable_base}/sp500-constituent?apikey={api_key}",
|
|
||||||
f"{stable_base}/sp500-constituents?apikey={api_key}",
|
|
||||||
],
|
|
||||||
"nasdaq100": [
|
|
||||||
f"{stable_base}/nasdaq-100-constituent?apikey={api_key}",
|
|
||||||
f"{stable_base}/nasdaq100-constituent?apikey={api_key}",
|
|
||||||
f"{stable_base}/nasdaq-100-constituents?apikey={api_key}",
|
|
||||||
],
|
|
||||||
"nasdaq_all": [
|
|
||||||
f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
|
||||||
f"{stable_base}/available-traded/list?apikey={api_key}",
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
legacy_candidates: dict[str, list[str]] = {
|
|
||||||
"sp500": [
|
|
||||||
f"{legacy_base}/sp500_constituent?apikey={api_key}",
|
|
||||||
f"{legacy_base}/sp500_constituent",
|
|
||||||
],
|
|
||||||
"nasdaq100": [
|
|
||||||
f"{legacy_base}/nasdaq_constituent?apikey={api_key}",
|
|
||||||
f"{legacy_base}/nasdaq_constituent",
|
|
||||||
],
|
|
||||||
"nasdaq_all": [
|
|
||||||
f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
failures: list[str] = []
|
|
||||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
|
||||||
stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe])
|
|
||||||
failures.extend(stable_failures)
|
|
||||||
|
|
||||||
if stable_symbols:
|
|
||||||
return stable_symbols
|
|
||||||
|
|
||||||
legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe])
|
|
||||||
failures.extend(legacy_failures)
|
|
||||||
|
|
||||||
if legacy_symbols:
|
|
||||||
return legacy_symbols
|
|
||||||
|
|
||||||
if failures:
|
|
||||||
reason = "; ".join(failures[:6])
|
|
||||||
logger.warning("FMP universe fetch failed for %s: %s", universe, reason)
|
|
||||||
raise ProviderError(
|
|
||||||
f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}"
|
|
||||||
)
|
|
||||||
|
|
||||||
raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
|
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_wiki_constituent_symbols(
|
async def _fetch_wiki_constituent_symbols(
|
||||||
client: httpx.AsyncClient,
|
client: httpx.AsyncClient,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -351,13 +241,16 @@ async def fetch_universe_symbols(
|
|||||||
|
|
||||||
Fallback order:
|
Fallback order:
|
||||||
1) Free public sources (Wikipedia/NASDAQ trader)
|
1) Free public sources (Wikipedia/NASDAQ trader)
|
||||||
2) FMP endpoints (if available)
|
2) Cached snapshot in SystemSetting
|
||||||
3) Cached snapshot in SystemSetting
|
3) Built-in seed symbols
|
||||||
4) Built-in seed symbols
|
|
||||||
|
|
||||||
Returns ``(symbols, source_label)`` so bootstrap UI can show where the
|
Returns ``(symbols, source_label)`` so bootstrap UI can show where the
|
||||||
list came from (important when Wikipedia/FMP fail and a stale cache still
|
list came from (important when the public source fails and a stale cache
|
||||||
lists BK instead of BNY).
|
still lists BK instead of BNY).
|
||||||
|
|
||||||
|
The seeds are representative, not complete, so a *fresh* install whose
|
||||||
|
public source is down bootstraps a partial universe. A warm instance is
|
||||||
|
unaffected — it falls through to its cached snapshot.
|
||||||
"""
|
"""
|
||||||
normalised_universe = _validate_universe(universe)
|
normalised_universe = _validate_universe(universe)
|
||||||
failures: list[str] = []
|
failures: list[str] = []
|
||||||
@@ -369,15 +262,6 @@ async def fetch_universe_symbols(
|
|||||||
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
|
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
|
||||||
return cleaned_public, public_source or "public"
|
return cleaned_public, public_source or "public"
|
||||||
|
|
||||||
try:
|
|
||||||
fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
|
|
||||||
cleaned_fmp = _normalise_symbols(fmp_symbols)
|
|
||||||
if cleaned_fmp:
|
|
||||||
await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
|
|
||||||
return cleaned_fmp, "fmp"
|
|
||||||
except (ProviderError, ValidationError) as exc:
|
|
||||||
failures.append(str(exc))
|
|
||||||
|
|
||||||
cached_symbols = await _read_cached_symbols(db, normalised_universe)
|
cached_symbols = await _read_cached_symbols(db, normalised_universe)
|
||||||
if cached_symbols:
|
if cached_symbols:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ MIN_FREE_GB="${DOLT_MIN_FREE_DISK_GB:-5}"
|
|||||||
EARNINGS_DIR="${DOLT_DATA_DIR}/${DOLT_EARNINGS_SUBDIR}"
|
EARNINGS_DIR="${DOLT_DATA_DIR}/${DOLT_EARNINGS_SUBDIR}"
|
||||||
DOLT_IDENTITY_NAME="${DOLT_IDENTITY_NAME:-Signal Platform}"
|
DOLT_IDENTITY_NAME="${DOLT_IDENTITY_NAME:-Signal Platform}"
|
||||||
DOLT_IDENTITY_EMAIL="${DOLT_IDENTITY_EMAIL:-signal-platform@localhost}"
|
DOLT_IDENTITY_EMAIL="${DOLT_IDENTITY_EMAIL:-signal-platform@localhost}"
|
||||||
FUNDAMENTALS_PARITY_REPORT_DIR="${FUNDAMENTALS_PARITY_REPORT_DIR:-/var/lib/signal-platform/reports/fundamentals-parity}"
|
|
||||||
|
|
||||||
fail() {
|
fail() {
|
||||||
echo "ERROR: $*" >&2
|
echo "ERROR: $*" >&2
|
||||||
@@ -80,8 +79,6 @@ check_env() {
|
|||||||
|| fail "set DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR in $ENV_FILE"
|
|| fail "set DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR in $ENV_FILE"
|
||||||
grep -Eq '^SEC_USER_AGENT=.*@.*' "$ENV_FILE" \
|
grep -Eq '^SEC_USER_AGENT=.*@.*' "$ENV_FILE" \
|
||||||
|| fail "SEC_USER_AGENT in $ENV_FILE must contain a real contact email"
|
|| fail "SEC_USER_AGENT in $ENV_FILE must contain a real contact email"
|
||||||
grep -Fqx "FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR" "$ENV_FILE" \
|
|
||||||
|| fail "set FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR in $ENV_FILE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
check_all() {
|
check_all() {
|
||||||
@@ -104,15 +101,6 @@ check_all() {
|
|||||||
identity_email="$(repo_config_value user.email 2>/dev/null || true)"
|
identity_email="$(repo_config_value user.email 2>/dev/null || true)"
|
||||||
[[ -n "$identity_name" ]] || fail "missing Dolt user.name for $EARNINGS_DIR"
|
[[ -n "$identity_name" ]] || fail "missing Dolt user.name for $EARNINGS_DIR"
|
||||||
[[ -n "$identity_email" ]] || fail "missing Dolt user.email for $EARNINGS_DIR"
|
[[ -n "$identity_email" ]] || fail "missing Dolt user.email for $EARNINGS_DIR"
|
||||||
[[ -d "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \
|
|
||||||
|| fail "missing parity report directory: $FUNDAMENTALS_PARITY_REPORT_DIR"
|
|
||||||
if [[ "$(id -un)" == "$APP_USER" ]]; then
|
|
||||||
[[ -w "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \
|
|
||||||
|| fail "parity report directory is not writable by $APP_USER"
|
|
||||||
else
|
|
||||||
runuser -u "$APP_USER" -- test -w "$FUNDAMENTALS_PARITY_REPORT_DIR" \
|
|
||||||
|| fail "parity report directory is not writable by $APP_USER"
|
|
||||||
fi
|
|
||||||
check_free_space
|
check_free_space
|
||||||
check_env
|
check_env
|
||||||
echo "OK: Dolt $DOLT_VERSION and earnings clone are provisioned"
|
echo "OK: Dolt $DOLT_VERSION and earnings clone are provisioned"
|
||||||
@@ -139,7 +127,6 @@ fi
|
|||||||
version_ok || fail "Dolt $DOLT_VERSION installation failed"
|
version_ok || fail "Dolt $DOLT_VERSION installation failed"
|
||||||
|
|
||||||
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$DOLT_DATA_DIR"
|
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$DOLT_DATA_DIR"
|
||||||
install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$FUNDAMENTALS_PARITY_REPORT_DIR"
|
|
||||||
check_free_space
|
check_free_space
|
||||||
|
|
||||||
if [[ ! -d "$EARNINGS_DIR/.dolt" ]]; then
|
if [[ ! -d "$EARNINGS_DIR/.dolt" ]]; then
|
||||||
|
|||||||
@@ -1,15 +1,18 @@
|
|||||||
# Dolt bulk-data integration — implementation plan
|
# Dolt bulk-data integration — implementation plan
|
||||||
|
|
||||||
Status: approved 2026-07-21, revised through four review rounds; direction: KISS
|
Status: **workstream A complete and deployed** (A0–A6, last step 2026-08-07);
|
||||||
backend, UI value first. Hand-off document for the implementing agent;
|
**workstream B dropped 2026-08-07** — see § Why B was dropped. Approved 2026-07-21,
|
||||||
self-contained.
|
revised through five review rounds; direction: KISS backend, UI value first.
|
||||||
|
Originally a hand-off document for the implementing agent; now the design record.
|
||||||
|
Current operations live in `docs/fundamentals-deployment.md`.
|
||||||
|
|
||||||
## Objective
|
## Objective
|
||||||
|
|
||||||
Replace the free-tier fundamentals APIs (FMP, Finnhub, Alpha Vantage) with bulk
|
Replace the free-tier fundamentals APIs (FMP, Finnhub, Alpha Vantage) with bulk
|
||||||
data: SEC Company Facts for fundamentals, the DoltHub earnings repo for the
|
data: SEC Company Facts for fundamentals and the DoltHub earnings repo for the
|
||||||
earnings calendar/history, and — later, independently — the DoltHub stocks repo for
|
earnings calendar/history. PostgreSQL stays the production system of record.
|
||||||
historical OHLCV. PostgreSQL stays the production system of record.
|
(A third source — the DoltHub stocks repo for historical OHLCV — was planned as
|
||||||
|
workstream B and dropped; Alpaca remains the price source.)
|
||||||
|
|
||||||
**Delivery order: two independent workstreams.**
|
**Delivery order: two independent workstreams.**
|
||||||
|
|
||||||
@@ -17,9 +20,9 @@ historical OHLCV. PostgreSQL stays the production system of record.
|
|||||||
FundamentalsPanel + decommission FMP/Finnhub/Alpha Vantage. Valuation uses the
|
FundamentalsPanel + decommission FMP/Finnhub/Alpha Vantage. Valuation uses the
|
||||||
existing Alpaca closes already in `ohlcv_records`. This alone achieves the goal
|
existing Alpaca closes already in `ohlcv_records`. This alone achieves the goal
|
||||||
(killing the quota-limited APIs) and delivers all the UI value.
|
(killing the quota-limited APIs) and delivers all the UI value.
|
||||||
- **Workstream B (later, optional until needed):** replace historical OHLCV with
|
- **Workstream B — DROPPED 2026-08-07, see below.** Would have replaced historical
|
||||||
the Dolt stocks repo. The most complex machinery (4.7 GB clone, split
|
OHLCV with the Dolt stocks repo. Its design is retained further down as a record,
|
||||||
adjustment, source-bar table, reconciliation) lives here and blocks nothing in A.
|
not as a backlog item.
|
||||||
|
|
||||||
**Guiding principle: KISS.** Plain daily importers with staging and atomic
|
**Guiding principle: KISS.** Plain daily importers with staging and atomic
|
||||||
promotion — no forensic replay, no permanent archive store, no conflict tables, no
|
promotion — no forensic replay, no permanent archive store, no conflict tables, no
|
||||||
@@ -70,8 +73,9 @@ notes (retain a CC BY-SA 4.0 reference + attribution to `post-no-preference/earn
|
|||||||
and a note of the transformations applied — e.g. in a repo `NOTICE`/attribution file
|
and a note of the transformations applied — e.g. in a repo `NOTICE`/attribution file
|
||||||
and the importer module); **no public API, bulk export, or redistribution** of the
|
and the importer module); **no public API, bulk export, or redistribution** of the
|
||||||
data; re-review licensing before any public or commercial access. The
|
data; re-review licensing before any public or commercial access. The
|
||||||
`post-no-preference/stocks` repo (workstream B) is **not** covered here and will be
|
`post-no-preference/stocks` repo (workstream B) is **not** covered here. B was
|
||||||
reviewed separately if B begins.
|
dropped before any licensing review, so that repo has never been assessed — any
|
||||||
|
future use of it starts that review from scratch.
|
||||||
|
|
||||||
## Schema
|
## Schema
|
||||||
|
|
||||||
@@ -119,7 +123,9 @@ reviewed separately if B begins.
|
|||||||
cache, repopulated by the daily SEC job — but only after the phase-A5 parity
|
cache, repopulated by the daily SEC job — but only after the phase-A5 parity
|
||||||
gate.
|
gate.
|
||||||
|
|
||||||
**Migration 027 (workstream B, written when B starts):**
|
**Migration 027 (workstream B — NEVER WRITTEN; B was dropped, and `027` was
|
||||||
|
subsequently used for `fundamental_snapshots.weighted_avg_diluted_shares`). The
|
||||||
|
design below is a record only:**
|
||||||
|
|
||||||
- `ohlcv_source_bars` — source-truth bar table, required because `ohlcv_records`
|
- `ohlcv_source_bars` — source-truth bar table, required because `ohlcv_records`
|
||||||
allows one row per (ticker_id, date) (`app/models/ohlcv.py:12`) and Alpaca
|
allows one row per (ticker_id, date) (`app/models/ohlcv.py:12`) and Alpaca
|
||||||
@@ -219,7 +225,7 @@ Workstream A:
|
|||||||
**The new API valuation object is not stored anywhere** — it is computed at
|
**The new API valuation object is not stored anywhere** — it is computed at
|
||||||
request time (below). No valuation cache or table exists.
|
request time (below). No valuation cache or table exists.
|
||||||
|
|
||||||
Workstream B:
|
Workstream B (dropped — never built):
|
||||||
|
|
||||||
- Dolt OHLCV+splits pull/import: `0 2 * * tue-sat` ET. If source_max_date is not
|
- Dolt OHLCV+splits pull/import: `0 2 * * tue-sat` ET. If source_max_date is not
|
||||||
fresh, retry hourly until ~06:00, then give up quietly. After a successful
|
fresh, retry hourly until ~06:00, then give up quietly. After a successful
|
||||||
@@ -430,16 +436,55 @@ workstream B — Alpaca remains the price source throughout.
|
|||||||
approval** — see the handoff section below. Step (c) is implemented behind the
|
approval** — see the handoff section below. Step (c) is implemented behind the
|
||||||
default-off `fundamental_data_sec_dolt_cutover_enabled` SystemSetting; the
|
default-off `fundamental_data_sec_dolt_cutover_enabled` SystemSetting; the
|
||||||
remaining production action is flipping that switch on and observing it.
|
remaining production action is flipping that switch on and observing it.
|
||||||
- A6. Remove FMP/Finnhub/Alpha Vantage; keep monitoring + manual fallback.
|
- A6. **DONE 2026-08-07.** FMP/Finnhub/Alpha Vantage removed, along with the
|
||||||
|
weekly `fundamental_collector` job, the A5 cutover toggle (SEC+Dolt is now the
|
||||||
|
unconditional path) and the parity report. Migration `029` tombstoned the two
|
||||||
|
behavior-bearing settings rows for the rollback window and `030` dropped them
|
||||||
|
once the deploy was confirmed healthy; the archived parity bundles stay as the
|
||||||
|
A5 evidence trail.
|
||||||
|
|
||||||
**Workstream B (independent, start when wanted):**
|
**Workstream B — DROPPED 2026-08-07.** The phases below are recorded for anyone
|
||||||
|
who revisits the decision; none of them are scheduled work.
|
||||||
|
|
||||||
- B0. Stocks clone (~4.7 GB) provisioned; migration 027.
|
- ~~B0. Stocks clone (~4.7 GB) provisioned; migration 027.~~
|
||||||
- B1. OHLCV + split adjustment in shadow (writes `ohlcv_source_bars` only; Alpaca
|
- ~~B1. OHLCV + split adjustment in shadow (writes `ohlcv_source_bars` only; Alpaca
|
||||||
keeps owning `ohlcv_records`); historical backfill.
|
keeps owning `ohlcv_records`); historical backfill.~~
|
||||||
- B2. Reconciliation window (≥ 2 weeks) vs Alpaca; review validation summaries.
|
- ~~B2. Reconciliation window (≥ 2 weeks) vs Alpaca; review validation summaries.~~
|
||||||
- B3. Promote Dolt as historical OHLCV source (canonical rebuilt from raw source
|
- ~~B3. Promote Dolt as historical OHLCV source (canonical rebuilt from raw source
|
||||||
bars + splits); morning pipeline → 03:00.
|
bars + splits); morning pipeline → 03:00.~~
|
||||||
|
|
||||||
|
### Why B was dropped
|
||||||
|
|
||||||
|
Reviewed after A6 shipped. Four reasons, in order of weight:
|
||||||
|
|
||||||
|
1. **Its motivation no longer exists.** B was scoped inside a plan whose goal was
|
||||||
|
killing the quota-limited free-tier APIs. Alpaca was never one of them, and the
|
||||||
|
plan always said so (§ Decommissioning: "Alpaca remains the price source
|
||||||
|
throughout"). A6 achieved the goal. What remained was swapping one working
|
||||||
|
price source for another.
|
||||||
|
2. **Its only concrete benefit is reachable far more cheaply.** The prize was
|
||||||
|
`corporate_actions`, the documented fix for the KLAC-class post-filing split
|
||||||
|
(TTM EPS pre-split vs a post-split price → P/E 6.19 instead of ~13, invisible to
|
||||||
|
snapshots). That needs *split events*, not 4.7 GB of bars — and the Alpaca SDK
|
||||||
|
already in the venv exposes them via
|
||||||
|
`alpaca.data.historical.corporate_actions.CorporateActionsClient.get_corporate_actions`
|
||||||
|
with `CorporateActionsRequest` / `CorporateActionsType`. See the follow-up below.
|
||||||
|
3. **The benefit is small.** Fundamentals carry 20% of the composite, P/E is one of
|
||||||
|
three fundamental inputs, and only names that split between their last 10-Q and
|
||||||
|
today are affected — a handful at a time, self-correcting at the next filing.
|
||||||
|
4. **B would add a risk the current setup does not carry.** By design a newly
|
||||||
|
published split rewrites a symbol's entire adjusted history. A backtest↔prod
|
||||||
|
parity guard exists precisely because changed history invalidates comparisons;
|
||||||
|
B makes history mutable as a routine event. It also needs its own license
|
||||||
|
review — the A0 CC BY-SA decision covers only `post-no-preference/earnings`.
|
||||||
|
|
||||||
|
**Optional follow-up, not scheduled:** a small `corporate_actions` table populated
|
||||||
|
from Alpaca, used to null or correct P/E when a split post-dates the newest
|
||||||
|
snapshot. Roughly a day's work; captures essentially all of B's value with no
|
||||||
|
clone, no `ohlcv_source_bars`, no split-adjustment pipeline and no reconciliation
|
||||||
|
window. Worth doing only if the wart starts costing something — it has been visible
|
||||||
|
and harmless since July 2026. Note that migration numbering has moved on: head is
|
||||||
|
`030`, so any such table would be `031+`, not the `027` named below.
|
||||||
|
|
||||||
## Test plan
|
## Test plan
|
||||||
|
|
||||||
@@ -464,8 +509,8 @@ workstream B — Alpaca remains the price source throughout.
|
|||||||
falls back from P/E to FCF yield for the valuation segment when P/E is null.
|
falls back from P/E to FCF yield for the valuation segment when P/E is null.
|
||||||
- Peer comparison disappears below 5 peer issuers; favorable-percentile direction
|
- Peer comparison disappears below 5 peer issuers; favorable-percentile direction
|
||||||
correct for both polarities.
|
correct for both polarities.
|
||||||
- Workstream B: split-adjusted OHLCV matches Alpaca on representative normal /
|
- ~~Workstream B: split-adjusted OHLCV matches Alpaca on representative normal /
|
||||||
split / reverse-split symbols.
|
split / reverse-split symbols.~~ (dropped)
|
||||||
- UI states: positive, adverse, neutral, insufficient history, insufficient
|
- UI states: positive, adverse, neutral, insufficient history, insufficient
|
||||||
peers; mobile layout; non-color accessibility.
|
peers; mobile layout; non-color accessibility.
|
||||||
- Unit, integration, scheduler and frontend suites pass.
|
- Unit, integration, scheduler and frontend suites pass.
|
||||||
@@ -492,30 +537,32 @@ Post-fix: candidate scores 504 of 511 vs legacy's 507 (gap = PSKY/Q new registra
|
|||||||
FITB, all explained); revenue-growth agreement 0.0038 median abs delta where both exist.
|
FITB, all explained); revenue-growth agreement 0.0038 median abs delta where both exist.
|
||||||
Dennis reviewed the evidence 2026-07-24 and directed proceeding to cutover.
|
Dennis reviewed the evidence 2026-07-24 and directed proceeding to cutover.
|
||||||
|
|
||||||
**Task 1 — A5 activation (IMPLEMENTED 2026-07-24; production switch remains).** The
|
**Task 1 — A5 activation: DONE.** Implemented 2026-07-24, switched on and observed
|
||||||
post-activation local refresh of `fundamental_data` derives `pe_ratio` and
|
in production, and made unconditional by A6 (2026-08-07) — there is no longer a
|
||||||
`market_cap` from newest valid snapshots × latest PostgreSQL close, `revenue_growth`
|
switch, an Admin card, or a weekly legacy collector to skip. The local refresh of
|
||||||
from snapshots, `earnings_surprise`/`next_earnings_date` from `earnings_events`; mark
|
`fundamental_data` derives `pe_ratio` and `market_cap` from newest valid snapshots ×
|
||||||
affected cached fundamental scores stale; must run identically when SEC is unreachable.
|
latest PostgreSQL close, `revenue_growth` from snapshots, and
|
||||||
|
`earnings_surprise`/`next_earnings_date` from `earnings_events`; it marks affected
|
||||||
|
cached fundamental scores stale and runs identically when SEC is unreachable.
|
||||||
It consumes `fundamentals_derivation.derive()` outputs, NOT raw snapshot fields —
|
It consumes `fundamentals_derivation.derive()` outputs, NOT raw snapshot fields —
|
||||||
that path carries the split guard (`ttm_diluted_eps`
|
that path carries the split guard (`ttm_diluted_eps` nulls when contaminated, with
|
||||||
nulls when contaminated, with `ttm_diluted_eps_caveat`) and the multi-class share
|
`ttm_diluted_eps_caveat`) and the multi-class share fallback (`shares_outstanding` +
|
||||||
fallback (`shares_outstanding` + `shares_outstanding_estimated`). Parity and activation
|
`shares_outstanding_estimated`). See `docs/fundamentals-deployment.md` for current
|
||||||
share the same candidate builder. Activation is the explicit
|
operations and rollback.
|
||||||
`fundamental_data_sec_dolt_cutover_enabled` SystemSetting and defaults off. It is
|
|
||||||
managed by the **Fundamentals data source** card in Admin → Settings; while active,
|
|
||||||
the weekly legacy collector skips itself so it cannot overwrite the SEC/Dolt cache.
|
|
||||||
See `docs/fundamentals-deployment.md` for the production flip and rollback procedure.
|
|
||||||
|
|
||||||
**Task 2 — A6 decommissioning.** After a short observation window: remove
|
**Task 2 — A6 decommissioning: DONE 2026-08-07.** The cutover ran on and was
|
||||||
FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual
|
observed in production, so the legacy providers, their config/env keys, the weekly
|
||||||
fallback. Gated by the acceptance criteria above — especially forward-calendar
|
collector job and the parity report were all removed. Two consequences to carry:
|
||||||
timeliness from `dolt_earnings` (its `source_max_date` ran ~5 weeks ahead as of
|
(1) `fundamental_data` now has no provider fallback — recovery is restore-from-backup;
|
||||||
2026-07-23, which passes).
|
(2) disabling **SEC Fundamentals Import** stops the SEC fetch only, because the local
|
||||||
|
cache refresh was deliberately moved outside the job-enable check. No follow-ups
|
||||||
|
remain: migration `030` dropped the tombstone rows after the deploy was verified.
|
||||||
|
|
||||||
**Known caveats to carry (documented in the findings report, not bugs to fix):**
|
**Known caveats to carry (documented in the findings report, not bugs to fix):**
|
||||||
- KLAC-class post-filing splits: P/E wrong until the next 10-Q; undetectable from
|
- KLAC-class post-filing splits: P/E wrong until the next 10-Q; undetectable from
|
||||||
snapshots. Workstream B's `corporate_actions` table is the natural future fix.
|
snapshots. Still open and still harmless. The fix, if ever wanted, is a small
|
||||||
|
`corporate_actions` table fed from Alpaca — **not** workstream B, which was
|
||||||
|
dropped; see § Why B was dropped.
|
||||||
- BRK-B: no share count exists anywhere in companyfacts → no market cap, correctly.
|
- BRK-B: no share count exists anywhere in companyfacts → no market cap, correctly.
|
||||||
- FITB: unscored (split guard + no taggable revenue) — the one name that lost its
|
- FITB: unscored (split guard + no taggable revenue) — the one name that lost its
|
||||||
score relative to legacy; composite renormalises.
|
score relative to legacy; composite renormalises.
|
||||||
@@ -528,7 +575,7 @@ timeliness from `dolt_earnings` (its `source_max_date` ran ~5 weeks ahead as of
|
|||||||
|
|
||||||
## Deferred (explicitly, until a concrete need appears)
|
## Deferred (explicitly, until a concrete need appears)
|
||||||
|
|
||||||
- Workstream B itself is deferred relative to A and blocks nothing in A.
|
- Workstream B: **dropped** 2026-08-07, not deferred — see § Why B was dropped.
|
||||||
- Exact byte-level source replay of historical imports; permanent archive store.
|
- Exact byte-level source replay of historical imports; permanent archive store.
|
||||||
- Point-in-time backtest enforcement (`accepted_at` is stored now; derivation and
|
- Point-in-time backtest enforcement (`accepted_at` is stored now; derivation and
|
||||||
backtest visibility rules are built only when fundamentals enter
|
backtest visibility rules are built only when fundamentals enter
|
||||||
|
|||||||
@@ -1,17 +1,21 @@
|
|||||||
# Fundamentals production deployment
|
# Fundamentals production deployment
|
||||||
|
|
||||||
This is the one-time production setup for the Dolt earnings and SEC fundamentals
|
This is the one-time production setup for the Dolt earnings and SEC fundamentals
|
||||||
imports. The A5 scoring cutover was approved on 2026-07-24; the compat-cache write
|
imports. Since A6 (2026-08) these are the *only* fundamentals sources — the
|
||||||
path is still default-off until the explicit production switch below is set. Do
|
FMP/Finnhub/Alpha Vantage providers, the weekly legacy collector and the A5 parity
|
||||||
not add OS cron entries: the application scheduler owns both jobs.
|
report are gone, and the cache write path is unconditional. Do not add OS cron
|
||||||
|
entries: the application scheduler owns both jobs.
|
||||||
|
|
||||||
## What the deployment adds
|
## What the deployment adds
|
||||||
|
|
||||||
- `Dolt Earnings Import (shadow)` runs daily at 02:30 America/New_York.
|
- `Dolt Earnings Import` runs daily at 02:30 America/New_York.
|
||||||
- `SEC Fundamentals Import` runs daily at 04:00 America/New_York. Its local
|
- `SEC Fundamentals Import` runs daily at 04:00 America/New_York, then refreshes
|
||||||
`fundamental_data` refresh runs only when the A5 switch is enabled.
|
`fundamental_data` — the compat cache scoring reads — from stored snapshots,
|
||||||
- `Fundamentals Parity Report (read-only)` runs daily at 05:30 America/New_York.
|
earnings events and closes.
|
||||||
- Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs.
|
- Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs.
|
||||||
|
**Disabling the SEC job stops its SEC network fetch only**; the local cache
|
||||||
|
refresh still runs, because prices and earnings move daily even when no filing
|
||||||
|
does.
|
||||||
- Cron expressions are editable in Admin → Schedule.
|
- Cron expressions are editable in Admin → Schedule.
|
||||||
- Every attempt is recorded in `data_import_runs`; failures also create a system
|
- Every attempt is recorded in `data_import_runs`; failures also create a system
|
||||||
event. A failed validation does not promote partial data.
|
event. A failed validation does not promote partial data.
|
||||||
@@ -36,14 +40,16 @@ DOLT_EARNINGS_SUBDIR=earnings
|
|||||||
DOLT_MIN_FREE_DISK_GB=5.0
|
DOLT_MIN_FREE_DISK_GB=5.0
|
||||||
SEC_USER_AGENT=signal-platform/1.0 (contact: real-address@example.com)
|
SEC_USER_AGENT=signal-platform/1.0 (contact: real-address@example.com)
|
||||||
SEC_REQUEST_SPACING_SECONDS=0.2
|
SEC_REQUEST_SPACING_SECONDS=0.2
|
||||||
FUNDAMENTALS_PARITY_REPORT_DIR=/var/lib/signal-platform/reports/fundamentals-parity
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Use a real monitored contact address. Keep at least 5 GB free at the Dolt data
|
Use a real monitored contact address. Keep at least 5 GB free at the Dolt data
|
||||||
path; 8–10 GB gives comfortable growth headroom. The data directory must stay
|
path; 8–10 GB gives comfortable growth headroom. The data directory must stay
|
||||||
outside `/opt/signalplatform`, because deployments use `rsync --delete` there.
|
outside `/opt/signalplatform`, because deployments use `rsync --delete` there.
|
||||||
The parity-report directory is also persistent and owned by the service user;
|
|
||||||
its small timestamped JSON/CSV bundles form the temporary A5 review trail.
|
`FMP_API_KEY`, `FINNHUB_API_KEY` and `ALPHA_VANTAGE_API_KEY` must be **removed**
|
||||||
|
from this file. Nothing reads them any more, and leaving them installed is the
|
||||||
|
one thing that would let a rolled-back pre-A6 process resume the legacy
|
||||||
|
collector and overwrite the SEC/Dolt cache.
|
||||||
|
|
||||||
## One-time provisioning
|
## One-time provisioning
|
||||||
|
|
||||||
@@ -80,7 +86,7 @@ a reviewed change to `DOLT_VERSION`, followed by the same provision/check flow.
|
|||||||
|
|
||||||
In Admin → Jobs, wait until no other job is running, then:
|
In Admin → Jobs, wait until no other job is running, then:
|
||||||
|
|
||||||
1. Trigger **Dolt Earnings Import (shadow)**. Expect `completed` with import
|
1. Trigger **Dolt Earnings Import**. Expect `completed` with import
|
||||||
status `promoted`; a repeat without an upstream change should report `no_op`.
|
status `promoted`; a repeat without an upstream change should report `no_op`.
|
||||||
2. Trigger **SEC Fundamentals Import**. The first run performs the
|
2. Trigger **SEC Fundamentals Import**. The first run performs the
|
||||||
tracked-universe history backfill and can take materially longer than a daily
|
tracked-universe history backfill and can take materially longer than a daily
|
||||||
@@ -91,27 +97,7 @@ In Admin → Jobs, wait until no other job is running, then:
|
|||||||
data and still handles partial/missing issuers cleanly. A ticker held by the
|
data and still handles partial/missing issuers cleanly. A ticker held by the
|
||||||
quality gate should show **New setups paused** with the specific SEC reason.
|
quality gate should show **New setups paused** with the specific SEC reason.
|
||||||
|
|
||||||
## A5 parity observation window
|
## Verification
|
||||||
|
|
||||||
After both shadow imports are healthy, trigger **Fundamentals Parity Report
|
|
||||||
(read-only)** once in Admin → Jobs. The **A5 Fundamentals Parity** card above
|
|
||||||
the jobs shows the latest coverage/delta summary and provides authenticated JSON
|
|
||||||
and CSV downloads. The canonical server-side bundles are archived at:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/var/lib/signal-platform/reports/fundamentals-parity/
|
|
||||||
```
|
|
||||||
|
|
||||||
The scheduler then generates one report daily at 05:30 New York time, after the
|
|
||||||
02:30 Dolt and 04:00 SEC jobs. Review 5–7 consecutive reports before making the
|
|
||||||
cutover decision. A report never writes `fundamental_data`, dimension/composite
|
|
||||||
scores, rankings, qualification state, or an approval flag. Materiality bands
|
|
||||||
only highlight rows for review; A5 still requires explicit approval.
|
|
||||||
|
|
||||||
Each bundle contains legacy and candidate P/E, revenue growth, and earnings
|
|
||||||
surprise; definition notes; source revisions and price dates; recomputed legacy
|
|
||||||
and candidate fundamental scores; and per-universe fundamental-rank changes.
|
|
||||||
Definition changes remain explicit even when numeric deltas are small.
|
|
||||||
|
|
||||||
Optional database verification:
|
Optional database verification:
|
||||||
|
|
||||||
@@ -162,45 +148,24 @@ Expect `OK: source lock is busy`. This is the remaining live-PostgreSQL
|
|||||||
mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory
|
mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory
|
||||||
locks. A second Admin trigger should independently report the job as busy.
|
locks. A second Admin trigger should independently report the job as busy.
|
||||||
|
|
||||||
## A5 production activation (approved 2026-07-24)
|
## The fundamentals cache
|
||||||
|
|
||||||
The write path is controlled by the SystemSetting
|
`fundamental_data` is the compat cache scoring reads. The SEC Fundamentals
|
||||||
`fundamental_data_sec_dolt_cutover_enabled`. An absent value, `false`, or any
|
Import rebuilds it every run from data already in PostgreSQL: newest valid
|
||||||
value other than `true` leaves `fundamental_data` untouched. Before enabling it,
|
snapshots x latest close for `pe_ratio` and `market_cap`, snapshots alone for
|
||||||
confirm the normal PostgreSQL backup containing `fundamental_data` is current.
|
`revenue_growth`, and `earnings_events` for `earnings_surprise` and
|
||||||
|
`next_earnings_date`. It therefore also runs after an SEC network/validation
|
||||||
|
failure, a `no_op`, a source-lock skip, or with the job disabled — no network
|
||||||
|
access is involved. The job message appends the cache row count and the changed
|
||||||
|
score-input count.
|
||||||
|
|
||||||
In **Admin → Settings → Fundamentals data source**:
|
A refresh marks affected fundamental and composite score caches stale. The
|
||||||
|
normal 15:30 near-close scanner recomputes them before using the rankings; until
|
||||||
|
then, reads truthfully expose the stale state.
|
||||||
|
|
||||||
1. Turn on **Use SEC + Dolt for scoring inputs** and accept the confirmation.
|
Verify the refreshed rows:
|
||||||
2. Click **Run refresh now**. The SEC import may be `promoted` or `no_op`; either
|
|
||||||
result runs the local cache refresh.
|
|
||||||
|
|
||||||
The weekly legacy collector is automatically skipped while the switch is on, so
|
|
||||||
it cannot overwrite the activated cache. The switch remains visible even before
|
|
||||||
its SystemSetting row exists because the safe default is off.
|
|
||||||
|
|
||||||
If the Admin UI is unavailable, enable the cutover directly in PostgreSQL:
|
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
INSERT INTO system_settings (key, value, updated_at)
|
|
||||||
VALUES ('fundamental_data_sec_dolt_cutover_enabled', 'true', now())
|
|
||||||
ON CONFLICT (key) DO UPDATE
|
|
||||||
SET value = EXCLUDED.value, updated_at = now();
|
|
||||||
```
|
|
||||||
|
|
||||||
Then trigger **SEC Fundamentals Import** once in Admin → Jobs. Once enabled, the
|
|
||||||
same refresh also runs after an SEC network/validation failure or a source-lock
|
|
||||||
skip, because it reads only PostgreSQL snapshots, earnings events, and closes.
|
|
||||||
The job message appends the cache row count and changed score-input count when
|
|
||||||
the import itself completed successfully.
|
|
||||||
|
|
||||||
Verify the switch and refreshed rows:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
SELECT key, value, updated_at
|
|
||||||
FROM system_settings
|
|
||||||
WHERE key = 'fundamental_data_sec_dolt_cutover_enabled';
|
|
||||||
|
|
||||||
SELECT count(*) AS rows,
|
SELECT count(*) AS rows,
|
||||||
max(fetched_at) AS refreshed_at,
|
max(fetched_at) AS refreshed_at,
|
||||||
count(pe_ratio) AS pe_available,
|
count(pe_ratio) AS pe_available,
|
||||||
@@ -213,28 +178,26 @@ SELECT dimension, is_stale, count(*)
|
|||||||
FROM dimension_scores
|
FROM dimension_scores
|
||||||
WHERE dimension = 'fundamental'
|
WHERE dimension = 'fundamental'
|
||||||
GROUP BY dimension, is_stale;
|
GROUP BY dimension, is_stale;
|
||||||
|
|
||||||
SELECT is_stale, count(*)
|
|
||||||
FROM composite_scores
|
|
||||||
GROUP BY is_stale;
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The first refresh intentionally marks affected fundamental and composite score
|
|
||||||
caches stale. The normal 15:30 near-close scanner recomputes them before using
|
|
||||||
the rankings; until then, reads truthfully expose the stale state. Observe at
|
|
||||||
least several scheduled cycles before A6 removes the legacy providers.
|
|
||||||
|
|
||||||
## Failure and rollback
|
## Failure and rollback
|
||||||
|
|
||||||
- To stop the A5 cache writes without stopping SEC snapshot ingestion, turn off
|
- **There is no provider fallback any more, and no Admin switch that freezes the
|
||||||
**Use SEC + Dolt for scoring inputs** in Admin → Settings. If the UI is
|
cache.** Disabling **SEC Fundamentals Import** stops SEC network access only;
|
||||||
unavailable, set `fundamental_data_sec_dolt_cutover_enabled` back to `false`
|
the 04:00 job still rebuilds `fundamental_data` from the stored snapshots,
|
||||||
with the SQL above (changing only the value). This prevents the next local
|
earnings events and closes.
|
||||||
refresh but does not restore rows already replaced. Restore `fundamental_data`
|
- Restoring `fundamental_data` from the PostgreSQL backup is therefore a
|
||||||
from the pre-cutover database backup, or—before A6—manually run the legacy
|
*temporary* fix on its own: if the bad values come from the snapshots or from
|
||||||
Fundamental Collector if its provider keys and quota are still available.
|
the derivation code, the next scheduled run reproduces them. Fix the cause —
|
||||||
- Disable a failing source-import job in Admin → Jobs only when ingestion itself
|
restore or repair `fundamental_snapshots` / `earnings_events`, or revert the
|
||||||
must stop. Existing promoted snapshots/events remain available.
|
parser change and re-run `scripts/reparse_fundamentals.py --apply`.
|
||||||
|
- To genuinely freeze the cache while you work, stop the service
|
||||||
|
(`sudo systemctl stop signalplatform.service`) — that stops the scheduler with
|
||||||
|
it. There is no finer-grained control, by design: a silently frozen scoring
|
||||||
|
input is worse than an obvious outage.
|
||||||
|
- Disable a failing source-import job in Admin → Jobs when SEC network access
|
||||||
|
itself must stop. Existing promoted snapshots and events remain available, and
|
||||||
|
the job's runtime message still reports the cache result.
|
||||||
- Inspect the job runtime, latest `data_import_runs.validation_json`, service
|
- Inspect the job runtime, latest `data_import_runs.validation_json`, service
|
||||||
logs, and Admin → System Events before retrying.
|
logs, and Admin → System Events before retrying.
|
||||||
- `unresolved_filing` is emitted once when a filing enters automatic retry. It
|
- `unresolved_filing` is emitted once when a filing enters automatic retry. It
|
||||||
@@ -250,8 +213,3 @@ least several scheduled cycles before A6 removes the legacy providers.
|
|||||||
- The Dolt clone is a reproducible cache and does not need a bespoke backup.
|
- The Dolt clone is a reproducible cache and does not need a bespoke backup.
|
||||||
PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import
|
PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import
|
||||||
audit rows) must remain covered by the normal production database backup.
|
audit rows) must remain covered by the normal production database backup.
|
||||||
- Do not proceed to A6 until the activated cache has completed the observation
|
|
||||||
window and the forward earnings calendar remains timely.
|
|
||||||
- If report generation fails, inspect Admin → System Events and verify
|
|
||||||
`FUNDAMENTALS_PARITY_REPORT_DIR` exists and is writable by `deploy`. Existing
|
|
||||||
reports and all live data remain untouched.
|
|
||||||
|
|||||||
+17
-13
@@ -25,7 +25,7 @@ score, Structural S/R, the Gate Target Ladder, sentiment, fundamentals) is
|
|||||||
| 1.5× ATR initial stop | Real exit | Cuts losers fast |
|
| 1.5× ATR initial stop | Real exit | Cuts losers fast |
|
||||||
| 3× ATR trailing stop, 30-day max hold | Real exit | Best Sharpe of every exit tested |
|
| 3× ATR trailing stop, 30-day max hold | Real exit | Best Sharpe of every exit tested |
|
||||||
| Post-stop normal gate reset | Re-entry policy | Stop always closes; a later gate failure and subsequent fresh qualification define the next signal episode. The selected study arm reached Sharpe 1.77 / CAGR 48.3% at capacity 10; live scan-before-outcome timing is stricter (Sharpe 1.68 / CAGR 44.8% analogue). [Full study](post-stop-reentry.md) |
|
| Post-stop normal gate reset | Re-entry policy | Stop always closes; a later gate failure and subsequent fresh qualification define the next signal episode. The selected study arm reached Sharpe 1.77 / CAGR 48.3% at capacity 10; live scan-before-outcome timing is stricter (Sharpe 1.68 / CAGR 44.8% analogue). [Full study](post-stop-reentry.md) |
|
||||||
| Max 10 concurrent positions, 1% risk per trade | Sizing | The cap binds by signal count, but the focused bracket found negligible opportunity cost: cap 15 admitted every blocked setup and added only 0.0018 R/trade in affected paths. [Findings](portfolio-capacity-bracket-findings.md) |
|
| Max **15** concurrent positions, 1% risk per trade | Sizing | Raised from 10 (2026-08-05) so the count cap never binds: +1.075pp CAGR paired, 51 paths better / 2 worse, drawdown unchanged. Cash plus the 20% notional cap saturates the book near 12. [Findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||||
| Structural S/R | Human-facing product context | Clean, capped zones for charts and alerts; not read by the scanner |
|
| Structural S/R | Human-facing product context | Clean, capped zones for charts and alerts; not read by the scanner |
|
||||||
| Gate Target Ladder | Screening machinery | Volume-free transient proposals preserve the production candidate set exactly; never an exit |
|
| Gate Target Ladder | Screening machinery | Volume-free transient proposals preserve the production candidate set exactly; never an exit |
|
||||||
|
|
||||||
@@ -61,7 +61,7 @@ invites overfitting.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| ATR trail multiple {1.5–4.0} | **Keep 3.0** — ≤2.0 whipsaws out the right tail; ≥2.5 is a plateau |
|
| ATR trail multiple {1.5–4.0} | **Keep 3.0** — ≤2.0 whipsaws out the right tail; ≥2.5 is a plateau |
|
||||||
| Momentum lookback (6-1, 3-1, 12-7 Novy-Marx, composites) | **Keep residual 12-1** — the others have IC ≈ 0 or weaker t-stats |
|
| Momentum lookback (6-1, 3-1, 12-7 Novy-Marx, composites) | **Keep residual 12-1** — the others have IC ≈ 0 or weaker t-stats |
|
||||||
| Selection cutoff {70…90} × book size {10, 15, 20} | **Keep 80 × 10** — the focused daily bracket found no meaningful gain from cap 15, while weekly rank replacement hurt. [Findings](portfolio-capacity-bracket-findings.md) |
|
| Selection cutoff {70…90} × book size {10, 15, 20} | **Keep cutoff 80; book size now 15** — the focused daily bracket found cap 15 worth +1.075pp CAGR (the weekly replay's contrary reading was EV-per-trade). Weekly rank replacement hurt. [Findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||||
| Position sizing (equal-weight, inverse-vol, risk-% sweep) | **Keep 1% fixed-fractional** |
|
| Position sizing (equal-weight, inverse-vol, risk-% sweep) | **Keep 1% fixed-fractional** |
|
||||||
| Primary-target probability floor | **Keep 20%** — pruned lottery targets, 1,428 → 1,089 qualified, lifted Sharpe |
|
| Primary-target probability floor | **Keep 20%** — pruned lottery targets, 1,428 → 1,089 qualified, lifted Sharpe |
|
||||||
| Primary-target R:R selector | **Keep 1.5** — target choice is intentionally independent of the later 2.0 activation floor |
|
| Primary-target R:R selector | **Keep 1.5** — target choice is intentionally independent of the later 2.0 activation floor |
|
||||||
@@ -146,7 +146,7 @@ knobs.
|
|||||||
| **Broader universe** | Composition changes factor signs (fip tug-of-war); vol-tilt on breadth is only a **directional hypothesis** (auth. −0.048 / t −1.36) | Any prod broaden must re-validate 80/20 tilt; offline research only; research.sqlite requires completion manifest |
|
| **Broader universe** | Composition changes factor signs (fip tug-of-war); vol-tilt on breadth is only a **directional hypothesis** (auth. −0.048 / t −1.36) | Any prod broaden must re-validate 80/20 tilt; offline research only; research.sqlite requires completion manifest |
|
||||||
| **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships |
|
| **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships |
|
||||||
| **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR |
|
| **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR |
|
||||||
| **Minimum effective-risk floor** | In cap-never-bound paths, the confounded 0.5% floor arm removed about 8% of fills while EV rose from 0.328 to 0.399 R and PF from 1.60 to 1.75, with exposure nearly unchanged | Run the frozen single-variable cap-10 A/B. [Specification](effective-risk-floor-ab.md) / [capacity findings](portfolio-capacity-bracket-findings.md) |
|
| **Minimum effective-risk floor** | ⛔ CLOSED NEGATIVE, not run. The floor lifts EV/trade (+0.032) and PF (+0.073) *by deleting trades* — 11.4 fewer per path, never one more — and costs **−0.753pp CAGR**, −0.047 Sharpe, −0.051 Calmar | Do not run the A/B; its EV-based pass rule would have shipped it. [Withdrawn specification](effective-risk-floor-ab.md) / [findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -198,15 +198,19 @@ qualification. The [daily re-entry matrix](post-stop-reentry.md) supports this
|
|||||||
for the current 10-position book, but not as a universal rule for other
|
for the current 10-position book, but not as a universal rule for other
|
||||||
portfolio capacities.
|
portfolio capacities.
|
||||||
|
|
||||||
Capacity is now closed as a negative result. The current daily Phase A control
|
Capacity is closed **positive**: the count cap was raised 10 → 15 so it no longer
|
||||||
does reject 519 qualified entries because the ten-slot book is full versus 472
|
binds, worth **+1.075pp CAGR** paired across 175 paths (51 better, 2 worse) at
|
||||||
admitted trades, so the older weekly “cap never binds” claim was stale. But the
|
unchanged drawdown. Fifteen is headroom, not a target — cap15 peaked at 12 with
|
||||||
clean cap-15 arm admitted every opportunity the strategy requested and added
|
zero full-book skips, so cash plus the 20% notional cap is the real ceiling.
|
||||||
only 0.0018 R/trade in paths where cap 10 bound. Weekly current-rank replacement
|
|
||||||
reduced mean EV and created substantial churn. Keep cap 10 and do not build the
|
An earlier reading of this run concluded "keep cap 10, added only 0.0018 R/trade."
|
||||||
replacement policy. See the [frozen specification](portfolio-capacity-bracket.md)
|
That was **EV per trade**, which is the wrong metric for a treatment that changes
|
||||||
and the separate [capacity findings](portfolio-capacity-bracket-findings.md).
|
trade *count*: flat EV/trade means the blocked entries were as good as the taken
|
||||||
The only open follow-up from that run is the
|
ones, so refusing them cost their whole contribution to return. Weekly
|
||||||
[frozen confound-free 0.5% minimum effective-risk-floor A/B](effective-risk-floor-ab.md).
|
current-rank replacement remains rejected (−0.043 EV R, 24% churn). The 0.5%
|
||||||
|
effective-risk-floor A/B is **closed negative** without being run — it costs
|
||||||
|
0.75pp of CAGR while raising EV/trade, and its frozen pass rule would have shipped
|
||||||
|
it. See the [frozen specification](portfolio-capacity-bracket.md) and the
|
||||||
|
[capacity findings](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||||
|
|
||||||
The next real evidence is **forward**, not backward: the live paper-trade record.
|
The next real evidence is **forward**, not backward: the live paper-trade record.
|
||||||
|
|||||||
@@ -1,8 +1,27 @@
|
|||||||
# Effective initial-risk floor A/B - frozen specification
|
# Effective initial-risk floor A/B - frozen specification
|
||||||
|
|
||||||
|
> ## ⛔ CLOSED 2026-08-05 — NEGATIVE. DO NOT RUN.
|
||||||
|
>
|
||||||
|
> This A/B was never executed because the capacity-bracket run already contains
|
||||||
|
> it. `cap15_incumbent` (peak 12, zero blocked, no floor) and `cash_unbounded`
|
||||||
|
> (peak 12, floor) have the same effective capacity and differ essentially only
|
||||||
|
> by `min_initial_risk_fraction`. Paired over 175 paths, the 0.5% floor gives
|
||||||
|
> **EV/trade +0.032 and profit factor +0.073, but CAGR −0.753pp, total return
|
||||||
|
> −0.765pp, Sharpe −0.047, Calmar −0.051**, and it removes 11.4 trades per path
|
||||||
|
> while never adding one (174 worse / 0 better).
|
||||||
|
>
|
||||||
|
> **The pass rule below is unsafe.** It promotes on paired EV, and the floor
|
||||||
|
> raises EV per trade *precisely by deleting trades* that were net positive
|
||||||
|
> contributors — so this specification would have shipped a change costing
|
||||||
|
> 0.75pp of CAGR. Any successor study must decide on CAGR/total return and treat
|
||||||
|
> EV per trade as a diagnostic.
|
||||||
|
>
|
||||||
|
> See [portfolio-capacity-bracket-findings.md](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||||
|
> Retained as a record of what was specified and why it was withdrawn.
|
||||||
|
|
||||||
Date frozen: 2026-08-05
|
Date frozen: 2026-08-05
|
||||||
Branch: research/portfolio-capacity-rebalancing
|
Branch: research/portfolio-capacity-rebalancing (deleted; tag `research/portfolio-capacity-final`)
|
||||||
Runner: scripts/run_portfolio_construction_matrix.py
|
Runner: scripts/run_portfolio_construction_matrix.py (not on main; see tag)
|
||||||
Study ID: risk-floor-ab
|
Study ID: risk-floor-ab
|
||||||
|
|
||||||
## Question
|
## Question
|
||||||
|
|||||||
@@ -32,8 +32,13 @@ Mechanics guards confirmed before reading results: calendar truncation asserted
|
|||||||
skipped_book_full = 519 versus 472 admitted trades, so the ten-slot book
|
skipped_book_full = 519 versus 472 admitted trades, so the ten-slot book
|
||||||
refuses 52.4% of admitted+blocked qualified opportunities. The older weekly
|
refuses 52.4% of admitted+blocked qualified opportunities. The older weekly
|
||||||
claim that the cap never bound is stale and does not apply to this daily
|
claim that the cap never bound is stale and does not apply to this daily
|
||||||
gate-reset configuration. Capacity is now isolated in the
|
gate-reset configuration. Capacity was isolated in the
|
||||||
[focused bracket study](portfolio-capacity-bracket.md).
|
[focused bracket study](portfolio-capacity-bracket.md) and **resolved: the count
|
||||||
|
cap was raised 10 → 15 so it no longer binds (+1.075pp CAGR paired, 51 paths
|
||||||
|
better / 2 worse, drawdown unchanged).** Note that the blocked *count* was a poor
|
||||||
|
guide in both directions — one path had 244 blocked entries and relieving all of
|
||||||
|
them moved CAGR by −0.1pp. See the
|
||||||
|
[findings correction](portfolio-capacity-bracket-findings.md#correction-2026-08-05-ev-per-trade-was-the-wrong-lens).
|
||||||
|
|
||||||
Validation SE ≈ 0.72 — almost no arm clears a 1-SE delta.
|
Validation SE ≈ 0.72 — almost no arm clears a 1-SE delta.
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,16 @@
|
|||||||
|
|
||||||
Date interpreted: 2026-08-05
|
Date interpreted: 2026-08-05
|
||||||
|
|
||||||
Status: **capacity and weekly replacement closed as negative results; the
|
Status: **SUPERSEDED IN PART — see [Correction](#correction-2026-08-05-ev-per-trade-was-the-wrong-lens)
|
||||||
minimum effective-risk floor remains an open single-variable follow-up.**
|
at the foot of this document before acting on anything here.** Weekly replacement
|
||||||
|
is closed as a negative result and that still holds. The capacity decision below
|
||||||
|
("keep cap 10") and the recommendation to run the effective-risk-floor A/B were
|
||||||
|
both reached on EV per trade and are **reversed** by the correction: the count cap
|
||||||
|
was raised so it no longer binds, and the floor A/B is closed as negative.
|
||||||
|
|
||||||
|
> The runner (`scripts/run_portfolio_construction_matrix.py`), the research
|
||||||
|
> simulator hooks, and the study's unit tests were deliberately not merged to
|
||||||
|
> main. They live at tag `research/portfolio-capacity-final`.
|
||||||
|
|
||||||
This document interprets the frozen v2 run without modifying its generated
|
This document interprets the frozen v2 run without modifying its generated
|
||||||
outputs:
|
outputs:
|
||||||
@@ -116,9 +124,96 @@ start-date evidence, but they necessarily mix initialization with market regime.
|
|||||||
|
|
||||||
## Final decisions
|
## Final decisions
|
||||||
|
|
||||||
1. Keep cap 10; its measured opportunity cost is negligible.
|
1. ~~Keep cap 10; its measured opportunity cost is negligible.~~ **REVERSED —
|
||||||
2. Reject weekly rank replacement.
|
see the correction below.**
|
||||||
|
2. Reject weekly rank replacement. *(Stands.)*
|
||||||
3. Do not interpret the `cash_unbounded` improvement as a capacity effect.
|
3. Do not interpret the `cash_unbounded` improvement as a capacity effect.
|
||||||
4. Run only the focused cap-10 effective-risk-floor A/B next.
|
*(Stands — and it is not a floor effect worth having either; see below.)*
|
||||||
|
4. ~~Run only the focused cap-10 effective-risk-floor A/B next.~~ **REVERSED —
|
||||||
|
that A/B is answered and negative; do not run it.**
|
||||||
5. Report means, inert fractions, and absolute dispersion beside medians and
|
5. Report means, inert fractions, and absolute dispersion beside medians and
|
||||||
ratios in future sparse-treatment studies.
|
ratios in future sparse-treatment studies. *(Stands, and see below — the
|
||||||
|
metric itself matters as much as the summary statistic.)*
|
||||||
|
|
||||||
|
## Correction 2026-08-05: EV per trade was the wrong lens
|
||||||
|
|
||||||
|
Everything above judged the arms on **mean paired net EV per trade**. That is the
|
||||||
|
wrong metric for any treatment that changes how many trades the book takes.
|
||||||
|
Capacity does not change trade *quality*; it changes trade *count*. A flat EV/trade
|
||||||
|
delta therefore does not mean "no benefit" — it means the blocked entries were
|
||||||
|
**just as good** as the taken ones, so refusing them cost their entire
|
||||||
|
contribution to return. Re-running the same paired comparison on CAGR inverts two
|
||||||
|
conclusions.
|
||||||
|
|
||||||
|
### Capacity: raise the cap (reverses decision 1)
|
||||||
|
|
||||||
|
`cap15_incumbent` versus `cap10_incumbent`, paired, all 175 paths, 0.10% per fill:
|
||||||
|
|
||||||
|
| Metric | Mean Δ | Worse / better |
|
||||||
|
|---|---:|---:|
|
||||||
|
| Trades | +1.00 | **0 / 76** (never fewer) |
|
||||||
|
| **CAGR pp** | **+1.075** | 2 / 51 |
|
||||||
|
| Total return pp | +1.079 | 1 / 51 |
|
||||||
|
| Max drawdown pp | +0.007 | 1 / 2 |
|
||||||
|
| Calmar | +0.062 | **1 / 51** |
|
||||||
|
| Sharpe | +0.022 | 10 / 28 |
|
||||||
|
| Net EV R/trade | +0.001 | 47 / 29 |
|
||||||
|
|
||||||
|
Restricted to the 105 paths where the cap actually bound: **+1.791pp CAGR**.
|
||||||
|
|
||||||
|
The honest tail: exactly one path was materially hurt — `empty-2023-04`, CAGR
|
||||||
|
87.2 → 81.2 (−6.0pp), drawdown 13.0 → 14.3, from two extra trades. Second-worst
|
||||||
|
was −0.1pp. The best paths (+6.6/+6.7/+6.9pp) came with *identical* drawdown. Best
|
||||||
|
and worst magnitudes are symmetric at roughly ±6pp, but the frequency is 51:1.
|
||||||
|
|
||||||
|
Blocked count is not lost value in either direction: `empty-2021-05` had **244**
|
||||||
|
blocked entries under cap 10, and relieving every one of them moved CAGR by
|
||||||
|
−0.1pp.
|
||||||
|
|
||||||
|
**Shipped:** `SIM_MAX_POSITIONS` and `shadow_book_service.DEFAULT_CAPACITY` raised
|
||||||
|
10 → 15. Fifteen is headroom, not a target — cap15 peaked at 12 with zero
|
||||||
|
full-book skips, so cash plus the 20% notional cap is the real ceiling and
|
||||||
|
15/20/None are the same experiment.
|
||||||
|
|
||||||
|
### Effective-risk floor: closed negative (reverses decision 4)
|
||||||
|
|
||||||
|
The floor A/B does not need running — this study already contains it.
|
||||||
|
`cap15_incumbent` (peak 12, zero blocked, no floor) and `cash_unbounded` (peak 12,
|
||||||
|
floor) have the same effective capacity and differ essentially only by
|
||||||
|
`min_initial_risk_fraction`. Paired, n=175, 0.10% per fill, floor minus no-floor:
|
||||||
|
|
||||||
|
| Metric | Mean Δ | Worse / better |
|
||||||
|
|---|---:|---:|
|
||||||
|
| Net EV R/trade | **+0.032** | 53 / 121 |
|
||||||
|
| Profit factor | **+0.073** | 46 / 128 |
|
||||||
|
| Trades | **−11.4** | **174 / 0** (never adds one) |
|
||||||
|
| **CAGR pp** | **−0.753** | 105 / 68 |
|
||||||
|
| Total return pp | −0.765 | 105 / 68 |
|
||||||
|
| Sharpe | −0.047 | 108 / 65 |
|
||||||
|
| Calmar | −0.051 | 103 / 71 |
|
||||||
|
| Max drawdown pp | +0.333 (worse) | — |
|
||||||
|
|
||||||
|
The same trap, mirrored: the floor raises per-trade quality *precisely by deleting
|
||||||
|
trades*, and the deleted trades were net positive contributors. The frozen
|
||||||
|
specification in [effective-risk-floor-ab.md](effective-risk-floor-ab.md) would
|
||||||
|
have passed it on paired EV and shipped a change costing 0.75pp of CAGR.
|
||||||
|
|
||||||
|
Genuinely open, low priority: 0.005 clearly over-cuts, but the sizing code's real
|
||||||
|
floor is a **$1** minimum, which is no floor at all. Whether something near 0.001
|
||||||
|
strips true dust without cutting real trades is untested, and only worth revisiting
|
||||||
|
if live broker order minimums force it.
|
||||||
|
|
||||||
|
### Start-date sensitivity is real but not a capacity artifact
|
||||||
|
|
||||||
|
Within-year spread of EV across monthly start dates is ~0.672 R and is
|
||||||
|
*identical* for `cap10` (0.672), `cap15` (0.672) and `cash_unbounded` (0.677). It
|
||||||
|
is small-sample noise — roughly 84 trades per 252-session window drawn from a
|
||||||
|
fat-tailed R distribution gives an EV standard error near 0.15–0.25 R — not a
|
||||||
|
queueing artifact. No construction policy reduces it.
|
||||||
|
|
||||||
|
### Rule for future studies
|
||||||
|
|
||||||
|
Choose the metric from the treatment's mechanism before reading any table. If a
|
||||||
|
treatment changes trade count, CAGR and total return are the decision metrics and
|
||||||
|
EV per trade is a diagnostic. The generated report's headline tables lead with
|
||||||
|
ΔEV net R, which is what made this error easy to make twice.
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import type {
|
|||||||
AdminUser,
|
AdminUser,
|
||||||
AlertConfig,
|
AlertConfig,
|
||||||
AlertTestResult,
|
AlertTestResult,
|
||||||
FundamentalsCutoverConfig,
|
|
||||||
PipelineReadiness,
|
PipelineReadiness,
|
||||||
RecommendationConfig,
|
RecommendationConfig,
|
||||||
ScheduleConfig,
|
ScheduleConfig,
|
||||||
@@ -57,18 +56,6 @@ export function updateSetting(key: string, value: string) {
|
|||||||
.then((r) => r.data);
|
.then((r) => r.data);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getFundamentalsCutoverSettings() {
|
|
||||||
return apiClient
|
|
||||||
.get<FundamentalsCutoverConfig>('admin/settings/fundamentals-cutover')
|
|
||||||
.then((r) => r.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function updateFundamentalsCutoverSettings(enabled: boolean) {
|
|
||||||
return apiClient
|
|
||||||
.put<FundamentalsCutoverConfig>('admin/settings/fundamentals-cutover', { enabled })
|
|
||||||
.then((r) => r.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getRecommendationSettings() {
|
export function getRecommendationSettings() {
|
||||||
return apiClient
|
return apiClient
|
||||||
.get<RecommendationConfig>('admin/settings/recommendations')
|
.get<RecommendationConfig>('admin/settings/recommendations')
|
||||||
@@ -246,40 +233,6 @@ export interface TriggerJobResponse {
|
|||||||
cadence?: BacktestCadence;
|
cadence?: BacktestCadence;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ParityFieldStats {
|
|
||||||
legacy_available: number;
|
|
||||||
candidate_available: number;
|
|
||||||
both_available: number;
|
|
||||||
material_differences: number;
|
|
||||||
median_absolute_delta: number | null;
|
|
||||||
p95_absolute_delta: number | null;
|
|
||||||
max_absolute_delta: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface FundamentalsParityReport {
|
|
||||||
report_version: number;
|
|
||||||
generated_at: string;
|
|
||||||
as_of_date: string;
|
|
||||||
approval_status: string;
|
|
||||||
read_only: boolean;
|
|
||||||
summary: {
|
|
||||||
universe_count: number;
|
|
||||||
legacy_fundamental_score_available: number;
|
|
||||||
candidate_fundamental_score_available: number;
|
|
||||||
fundamental_scores_compared: number;
|
|
||||||
fundamental_score_material_changes: number;
|
|
||||||
fundamental_rank_changes: number;
|
|
||||||
field_stats: Record<string, ParityFieldStats>;
|
|
||||||
};
|
|
||||||
source_runs: Record<string, {
|
|
||||||
run_id: number;
|
|
||||||
status: string;
|
|
||||||
revision: string | null;
|
|
||||||
source_max_date: string | null;
|
|
||||||
completed_at: string | null;
|
|
||||||
} | null>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type BacktestTargetModel = 'production_gtl' | 'structural_sr';
|
export type BacktestTargetModel = 'production_gtl' | 'structural_sr';
|
||||||
export type BacktestCadence = 'weekly' | 'daily';
|
export type BacktestCadence = 'weekly' | 'daily';
|
||||||
|
|
||||||
@@ -306,24 +259,6 @@ export function triggerJob(
|
|||||||
.then((r) => r.data);
|
.then((r) => r.data);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getFundamentalsParityReport() {
|
|
||||||
return apiClient
|
|
||||||
.get<FundamentalsParityReport | null>('admin/fundamentals-parity')
|
|
||||||
.then((r) => r.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getFundamentalsParityCsv() {
|
|
||||||
return apiClient
|
|
||||||
.get<{ filename: string; content: string } | null>('admin/fundamentals-parity/csv')
|
|
||||||
.then((r) => r.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getFundamentalsParityJson() {
|
|
||||||
return apiClient
|
|
||||||
.get<{ filename: string; content: string } | null>('admin/fundamentals-parity/json')
|
|
||||||
.then((r) => r.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
// System events (operational warnings / errors)
|
// System events (operational warnings / errors)
|
||||||
export interface SystemEvent {
|
export interface SystemEvent {
|
||||||
id: number;
|
id: number;
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ export interface FetchDataResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Provider sources that cost an API call/quota. */
|
/** Provider sources that cost an API call/quota. */
|
||||||
export type FetchSource = 'ohlcv' | 'sentiment' | 'fundamentals';
|
export type FetchSource = 'ohlcv' | 'sentiment';
|
||||||
/** Source selector: omit → fetch all; array → those providers; 'recompute' → derived only (free). */
|
/** Source selector: omit → fetch all; array → those providers; 'recompute' → derived only (free). */
|
||||||
export type FetchSelector = FetchSource[] | 'recompute';
|
export type FetchSelector = FetchSource[] | 'recompute';
|
||||||
|
|
||||||
|
|||||||
@@ -1,176 +0,0 @@
|
|||||||
import {
|
|
||||||
useFundamentalsCutoverSettings,
|
|
||||||
useJobs,
|
|
||||||
useTriggerJob,
|
|
||||||
useUpdateFundamentalsCutoverSettings,
|
|
||||||
} from '../../hooks/useAdmin';
|
|
||||||
import { SkeletonCard } from '../ui/Skeleton';
|
|
||||||
|
|
||||||
const SEC_JOB = 'sec_fundamentals_import';
|
|
||||||
|
|
||||||
function formatRun(iso: string | null | undefined): string {
|
|
||||||
if (!iso) return 'not run in this process';
|
|
||||||
const minutes = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000);
|
|
||||||
if (minutes < 1) return 'just now';
|
|
||||||
if (minutes < 60) return `${minutes}m ago`;
|
|
||||||
const hours = Math.floor(minutes / 60);
|
|
||||||
return hours < 24 ? `${hours}h ago` : `${Math.floor(hours / 24)}d ago`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function FundamentalsCutoverSettings() {
|
|
||||||
const cutover = useFundamentalsCutoverSettings();
|
|
||||||
const update = useUpdateFundamentalsCutoverSettings();
|
|
||||||
const trigger = useTriggerJob();
|
|
||||||
const { data: jobs } = useJobs();
|
|
||||||
|
|
||||||
if (cutover.isLoading) return <SkeletonCard />;
|
|
||||||
if (cutover.isError || !cutover.data) {
|
|
||||||
return (
|
|
||||||
<p className="text-sm text-red-400">
|
|
||||||
{(cutover.error as Error)?.message || 'Failed to load fundamentals data source'}
|
|
||||||
</p>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const enabled = cutover.data.enabled;
|
|
||||||
const secJob = jobs?.find((job) => job.name === SEC_JOB);
|
|
||||||
const runningJob = jobs?.find((job) => job.running);
|
|
||||||
const refreshBlocked = Boolean(runningJob && runningJob.name !== SEC_JOB);
|
|
||||||
|
|
||||||
const changeSource = () => {
|
|
||||||
const next = !enabled;
|
|
||||||
const confirmed = window.confirm(
|
|
||||||
next
|
|
||||||
? 'Activate SEC + Dolt fundamentals? The next SEC import will replace the legacy cache and mark affected scores stale.'
|
|
||||||
: 'Pause SEC + Dolt cache refreshes? Existing cache values will stay in place; legacy values are not restored automatically.',
|
|
||||||
);
|
|
||||||
if (confirmed) update.mutate(next);
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<section className="glass overflow-hidden" aria-labelledby="fundamentals-source-title">
|
|
||||||
<div className={`h-0.5 ${enabled ? 'bg-gradient-to-r from-sky-500 via-cyan-300 to-emerald-400' : 'bg-white/[0.06]'}`} />
|
|
||||||
<div className="space-y-5 p-5">
|
|
||||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
|
||||||
<div>
|
|
||||||
<div className="flex items-center gap-2">
|
|
||||||
<h3 id="fundamentals-source-title" className="text-sm font-semibold text-gray-200">
|
|
||||||
Fundamentals data source
|
|
||||||
</h3>
|
|
||||||
<span
|
|
||||||
className={`rounded-full border px-2 py-0.5 text-[10px] font-semibold uppercase tracking-[0.14em] ${
|
|
||||||
enabled
|
|
||||||
? 'border-cyan-400/25 bg-cyan-400/10 text-cyan-300'
|
|
||||||
: 'border-white/10 bg-white/[0.04] text-gray-500'
|
|
||||||
}`}
|
|
||||||
>
|
|
||||||
{enabled ? 'SEC + Dolt active' : 'Legacy cache'}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<p className="mt-1 max-w-3xl text-xs leading-relaxed text-gray-500">
|
|
||||||
Controls what repopulates <span className="num text-gray-400">fundamental_data</span>, the
|
|
||||||
compatibility cache used by scoring. SEC filings supply P/E, growth and estimated market
|
|
||||||
cap; Dolt supplies earnings dates and surprises. Everything is derived locally from PostgreSQL.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="grid grid-cols-[minmax(0,1fr)_5rem_minmax(0,1fr)] items-center gap-3 rounded-xl border border-white/[0.06] bg-black/10 px-4 py-3">
|
|
||||||
<div className={enabled ? 'text-gray-600' : 'text-amber-200/90'}>
|
|
||||||
<div className="num text-[10px] uppercase tracking-[0.16em]">Legacy APIs</div>
|
|
||||||
<div className="mt-0.5 text-[11px]">FMP / Finnhub / Alpha Vantage</div>
|
|
||||||
</div>
|
|
||||||
<div className="relative h-px bg-white/10" aria-hidden="true">
|
|
||||||
<span
|
|
||||||
className={`absolute top-1/2 h-2.5 w-2.5 -translate-y-1/2 rounded-full border-2 border-[#0e120f] transition-all duration-300 ${
|
|
||||||
enabled
|
|
||||||
? 'right-0 bg-cyan-300 shadow-[0_0_12px_rgba(103,232,249,0.55)]'
|
|
||||||
: 'left-0 bg-amber-300'
|
|
||||||
}`}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={`text-right ${enabled ? 'text-cyan-200' : 'text-gray-600'}`}>
|
|
||||||
<div className="num text-[10px] uppercase tracking-[0.16em]">SEC + Dolt</div>
|
|
||||||
<div className="mt-0.5 text-[11px]">Bulk imports → PostgreSQL cache</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="grid gap-4 border-t border-white/[0.06] pt-4 md:grid-cols-2">
|
|
||||||
<div className="flex items-start justify-between gap-4 rounded-xl bg-white/[0.025] p-3.5">
|
|
||||||
<div>
|
|
||||||
<div className="num text-[10px] uppercase tracking-[0.14em] text-gray-600">1 · Source</div>
|
|
||||||
<div className="mt-1 text-sm text-gray-200">Use SEC + Dolt for scoring inputs</div>
|
|
||||||
<p className="mt-1 text-[11px] leading-relaxed text-gray-500">
|
|
||||||
While active, the weekly legacy collector is skipped so it cannot overwrite the new cache.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
role="switch"
|
|
||||||
aria-checked={enabled}
|
|
||||||
aria-label="Use SEC and Dolt fundamentals"
|
|
||||||
onClick={changeSource}
|
|
||||||
disabled={update.isPending}
|
|
||||||
className={`relative mt-1 inline-flex h-6 w-11 shrink-0 rounded-full border-2 border-transparent transition-colors focus:outline-none focus:ring-2 focus:ring-cyan-400/70 focus:ring-offset-2 focus:ring-offset-[#0e120f] disabled:cursor-wait disabled:opacity-50 ${
|
|
||||||
enabled ? 'bg-gradient-to-r from-sky-500 to-cyan-400' : 'bg-white/10'
|
|
||||||
}`}
|
|
||||||
>
|
|
||||||
<span
|
|
||||||
className={`pointer-events-none inline-block h-5 w-5 rounded-full bg-white shadow transition-transform ${
|
|
||||||
enabled ? 'translate-x-5' : 'translate-x-0'
|
|
||||||
}`}
|
|
||||||
/>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="rounded-xl bg-white/[0.025] p-3.5">
|
|
||||||
<div className="num text-[10px] uppercase tracking-[0.14em] text-gray-600">2 · Refresh</div>
|
|
||||||
<div className="mt-1 flex flex-wrap items-center justify-between gap-3">
|
|
||||||
<div>
|
|
||||||
<div className="text-sm text-gray-200">Apply the source now</div>
|
|
||||||
<p className="mt-1 text-[11px] text-gray-500">
|
|
||||||
{secJob?.running
|
|
||||||
? 'SEC import and cache refresh are running.'
|
|
||||||
: secJob?.runtime_message || `Last SEC run: ${formatRun(secJob?.runtime_finished_at)}`}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
onClick={() => trigger.mutate(SEC_JOB)}
|
|
||||||
disabled={
|
|
||||||
!enabled ||
|
|
||||||
trigger.isPending ||
|
|
||||||
Boolean(secJob?.running) ||
|
|
||||||
refreshBlocked ||
|
|
||||||
secJob?.enabled === false
|
|
||||||
}
|
|
||||||
className="btn-primary px-3 py-2 text-xs disabled:cursor-not-allowed disabled:opacity-40"
|
|
||||||
>
|
|
||||||
<span>
|
|
||||||
{secJob?.running
|
|
||||||
? 'Refreshing…'
|
|
||||||
: trigger.isPending
|
|
||||||
? 'Starting…'
|
|
||||||
: refreshBlocked
|
|
||||||
? 'Another job is running'
|
|
||||||
: 'Run refresh now'}
|
|
||||||
</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
{!enabled && (
|
|
||||||
<p className="mt-2 text-[11px] text-amber-300/70">Activate the source before running the refresh.</p>
|
|
||||||
)}
|
|
||||||
{enabled && secJob?.enabled === false && (
|
|
||||||
<p className="mt-2 text-[11px] text-amber-300/70">Enable the SEC Fundamentals job on the Jobs tab first.</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<p className="text-[11px] leading-relaxed text-gray-600">
|
|
||||||
Rollback pauses future writes only. To restore pre-cutover values, use the database backup or
|
|
||||||
pause this source and manually run the legacy collector while its provider keys remain installed.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
import { useState } from 'react';
|
|
||||||
import {
|
|
||||||
getFundamentalsParityCsv,
|
|
||||||
getFundamentalsParityJson,
|
|
||||||
} from '../../api/admin';
|
|
||||||
import { useFundamentalsParityReport } from '../../hooks/useAdmin';
|
|
||||||
import { SkeletonTable } from '../ui/Skeleton';
|
|
||||||
|
|
||||||
const FIELD_LABELS: Record<string, string> = {
|
|
||||||
pe_ratio: 'P/E',
|
|
||||||
revenue_growth: 'Revenue growth',
|
|
||||||
earnings_surprise: 'Earnings surprise',
|
|
||||||
};
|
|
||||||
|
|
||||||
function downloadText(filename: string, content: string, type: string) {
|
|
||||||
const blob = new Blob([content], { type });
|
|
||||||
const url = URL.createObjectURL(blob);
|
|
||||||
const anchor = document.createElement('a');
|
|
||||||
anchor.href = url;
|
|
||||||
anchor.download = filename;
|
|
||||||
anchor.click();
|
|
||||||
URL.revokeObjectURL(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function FundamentalsParityPanel() {
|
|
||||||
const { data: report, isLoading, isError, error } = useFundamentalsParityReport();
|
|
||||||
const [downloading, setDownloading] = useState(false);
|
|
||||||
|
|
||||||
if (isLoading) return <SkeletonTable rows={2} cols={4} />;
|
|
||||||
if (isError) {
|
|
||||||
return <p className="text-sm text-red-400">{(error as Error).message}</p>;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!report) {
|
|
||||||
return (
|
|
||||||
<div className="glass p-5">
|
|
||||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
|
||||||
<p className="mt-1 text-xs text-gray-500">
|
|
||||||
No report yet. Trigger “Fundamentals Parity Report (read-only)” below.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const summary = report.summary;
|
|
||||||
const generated = new Date(report.generated_at).toLocaleString();
|
|
||||||
|
|
||||||
async function downloadCsv() {
|
|
||||||
setDownloading(true);
|
|
||||||
try {
|
|
||||||
const artifact = await getFundamentalsParityCsv();
|
|
||||||
if (artifact) downloadText(artifact.filename, artifact.content, 'text/csv;charset=utf-8');
|
|
||||||
} finally {
|
|
||||||
setDownloading(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function downloadJson() {
|
|
||||||
setDownloading(true);
|
|
||||||
try {
|
|
||||||
const artifact = await getFundamentalsParityJson();
|
|
||||||
if (artifact) downloadText(artifact.filename, artifact.content, 'application/json;charset=utf-8');
|
|
||||||
} finally {
|
|
||||||
setDownloading(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="glass p-5 space-y-4">
|
|
||||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
|
||||||
<div>
|
|
||||||
<div className="flex flex-wrap items-center gap-2">
|
|
||||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
|
||||||
<span className="rounded-full border border-amber-400/20 bg-amber-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-amber-300">
|
|
||||||
approval pending
|
|
||||||
</span>
|
|
||||||
<span className="rounded-full border border-cyan-400/20 bg-cyan-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-cyan-300">
|
|
||||||
read-only
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<p className="mt-1 text-xs text-gray-500">
|
|
||||||
Generated {generated} · as of {report.as_of_date} · {summary.universe_count} tracked tickers
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<div className="flex gap-2">
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white"
|
|
||||||
onClick={downloadJson}
|
|
||||||
disabled={downloading}
|
|
||||||
>
|
|
||||||
Download JSON
|
|
||||||
</button>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white disabled:opacity-50"
|
|
||||||
onClick={downloadCsv}
|
|
||||||
disabled={downloading}
|
|
||||||
>
|
|
||||||
{downloading ? 'Preparing…' : 'Download CSV'}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
|
|
||||||
<Summary label="Candidate score coverage" value={`${summary.candidate_fundamental_score_available}/${summary.universe_count}`} />
|
|
||||||
<Summary label="Scores compared" value={summary.fundamental_scores_compared} />
|
|
||||||
<Summary label="Material score moves" value={summary.fundamental_score_material_changes} />
|
|
||||||
<Summary label="Fundamental rank moves" value={summary.fundamental_rank_changes} />
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="overflow-x-auto">
|
|
||||||
<table className="w-full text-left text-xs">
|
|
||||||
<thead className="text-[10px] uppercase tracking-wider text-gray-500">
|
|
||||||
<tr>
|
|
||||||
<th className="pb-2 pr-4 font-medium">Field</th>
|
|
||||||
<th className="pb-2 px-3 font-medium">Legacy</th>
|
|
||||||
<th className="pb-2 px-3 font-medium">Candidate</th>
|
|
||||||
<th className="pb-2 px-3 font-medium">Compared</th>
|
|
||||||
<th className="pb-2 px-3 font-medium">Material</th>
|
|
||||||
<th className="pb-2 pl-3 font-medium">Median |Δ|</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody className="divide-y divide-white/[0.06] text-gray-300">
|
|
||||||
{Object.entries(summary.field_stats).map(([key, stats]) => (
|
|
||||||
<tr key={key}>
|
|
||||||
<td className="py-2.5 pr-4">{FIELD_LABELS[key] ?? key}</td>
|
|
||||||
<td className="py-2.5 px-3 num">{stats.legacy_available}</td>
|
|
||||||
<td className="py-2.5 px-3 num">{stats.candidate_available}</td>
|
|
||||||
<td className="py-2.5 px-3 num">{stats.both_available}</td>
|
|
||||||
<td className="py-2.5 px-3 num">{stats.material_differences}</td>
|
|
||||||
<td className="py-2.5 pl-3 num">
|
|
||||||
{stats.median_absolute_delta == null ? 'n/a' : stats.median_absolute_delta.toFixed(2)}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<p className="text-[11px] leading-relaxed text-gray-500">
|
|
||||||
Materiality bands highlight review candidates only. They do not approve a cutover or write fundamentals,
|
|
||||||
scores, rankings, or qualification state.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function Summary({ label, value }: { label: string; value: string | number }) {
|
|
||||||
return (
|
|
||||||
<div className="rounded-lg border border-white/[0.07] bg-white/[0.025] px-3 py-2.5">
|
|
||||||
<div className="text-[10px] uppercase tracking-wider text-gray-500">{label}</div>
|
|
||||||
<div className="mt-1 num text-lg text-gray-200">{value}</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -8,11 +8,9 @@ const DEFAULTS: ScheduleConfig = {
|
|||||||
schedule_daily_pipeline_cron: '0 2 * * *',
|
schedule_daily_pipeline_cron: '0 2 * * *',
|
||||||
schedule_dolt_earnings_cron: '30 2 * * *',
|
schedule_dolt_earnings_cron: '30 2 * * *',
|
||||||
schedule_sec_fundamentals_cron: '0 4 * * *',
|
schedule_sec_fundamentals_cron: '0 4 * * *',
|
||||||
schedule_fundamentals_parity_cron: '30 5 * * *',
|
|
||||||
schedule_near_close_pipeline_cron: '30 15 * * mon-fri',
|
schedule_near_close_pipeline_cron: '30 15 * * mon-fri',
|
||||||
schedule_after_close_pipeline_cron: '45 16 * * mon-fri',
|
schedule_after_close_pipeline_cron: '45 16 * * mon-fri',
|
||||||
schedule_intraday_pipeline_cron: '0 10-15 * * mon-fri',
|
schedule_intraday_pipeline_cron: '0 10-15 * * mon-fri',
|
||||||
schedule_fundamentals_cron: '0 1 * * mon',
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: boolean }[] = [
|
const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: boolean }[] = [
|
||||||
@@ -30,19 +28,13 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
|||||||
{
|
{
|
||||||
key: 'schedule_dolt_earnings_cron',
|
key: 'schedule_dolt_earnings_cron',
|
||||||
label: 'Dolt earnings',
|
label: 'Dolt earnings',
|
||||||
hint: 'Pull and import earnings dates/results daily at 02:30 ET. The activated cache refresh uses these local events.',
|
hint: 'Pull and import earnings dates/results daily at 02:30 ET. The fundamentals cache refresh uses these local events.',
|
||||||
mono: true,
|
mono: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'schedule_sec_fundamentals_cron',
|
key: 'schedule_sec_fundamentals_cron',
|
||||||
label: 'SEC fundamentals',
|
label: 'SEC fundamentals',
|
||||||
hint: 'Import tracked-universe SEC facts daily at 04:00 ET and refresh the scoring cache when the cutover is active.',
|
hint: 'Import tracked-universe SEC facts daily at 04:00 ET, then refresh the fundamentals cache scoring reads. Disabling the job stops the SEC fetch only — the local cache refresh still runs.',
|
||||||
mono: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: 'schedule_fundamentals_parity_cron',
|
|
||||||
label: 'Fundamentals parity report',
|
|
||||||
hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the bulk imports.',
|
|
||||||
mono: true,
|
mono: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -63,12 +55,6 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
|||||||
hint: 'Refresh prices + resolve outcomes mid-session. Default hourly 10:00–15:00 ET weekdays.',
|
hint: 'Refresh prices + resolve outcomes mid-session. Default hourly 10:00–15:00 ET weekdays.',
|
||||||
mono: true,
|
mono: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
key: 'schedule_fundamentals_cron',
|
|
||||||
label: 'Legacy fundamentals (weekly)',
|
|
||||||
hint: 'Fallback provider chain. Automatically skipped while the SEC + Dolt cutover is active.',
|
|
||||||
mono: true,
|
|
||||||
},
|
|
||||||
];
|
];
|
||||||
|
|
||||||
export function ScheduleSettings() {
|
export function ScheduleSettings() {
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ import { useSettings, useUpdateSetting } from '../../hooks/useAdmin';
|
|||||||
import { SkeletonTable } from '../ui/Skeleton';
|
import { SkeletonTable } from '../ui/Skeleton';
|
||||||
import type { SystemSetting } from '../../lib/types';
|
import type { SystemSetting } from '../../lib/types';
|
||||||
|
|
||||||
const MANAGED_SETTINGS = new Set(['fundamental_data_sec_dolt_cutover_enabled']);
|
|
||||||
|
|
||||||
export function SettingsForm() {
|
export function SettingsForm() {
|
||||||
const { data: settings, isLoading, isError, error } = useSettings();
|
const { data: settings, isLoading, isError, error } = useSettings();
|
||||||
const updateSetting = useUpdateSetting();
|
const updateSetting = useUpdateSetting();
|
||||||
@@ -34,11 +32,10 @@ export function SettingsForm() {
|
|||||||
if (isLoading) return <SkeletonTable rows={4} cols={2} />;
|
if (isLoading) return <SkeletonTable rows={4} cols={2} />;
|
||||||
if (isError) return <p className="text-sm text-red-400">{(error as Error)?.message || 'Failed to load settings'}</p>;
|
if (isError) return <p className="text-sm text-red-400">{(error as Error)?.message || 'Failed to load settings'}</p>;
|
||||||
if (!settings || settings.length === 0) return <p className="text-sm text-gray-500">No settings found.</p>;
|
if (!settings || settings.length === 0) return <p className="text-sm text-gray-500">No settings found.</p>;
|
||||||
const visibleSettings = settings.filter((setting) => !MANAGED_SETTINGS.has(setting.key));
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
{visibleSettings.map((setting) => (
|
{settings.map((setting) => (
|
||||||
<div key={setting.key} className="glass p-4 flex flex-wrap items-center gap-3 glass-hover">
|
<div key={setting.key} className="glass p-4 flex flex-wrap items-center gap-3 glass-hover">
|
||||||
<label className="min-w-[140px] text-sm font-medium text-gray-300">{setting.key}</label>
|
<label className="min-w-[140px] text-sm font-medium text-gray-300">{setting.key}</label>
|
||||||
{setting.key === 'registration' ? (
|
{setting.key === 'registration' ? (
|
||||||
|
|||||||
@@ -90,36 +90,6 @@ export function useUpdateSetting() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useFundamentalsCutoverSettings() {
|
|
||||||
return useQuery({
|
|
||||||
queryKey: ['admin', 'fundamentals-cutover'],
|
|
||||||
queryFn: () => adminApi.getFundamentalsCutoverSettings(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function useUpdateFundamentalsCutoverSettings() {
|
|
||||||
const qc = useQueryClient();
|
|
||||||
const { addToast } = useToast();
|
|
||||||
|
|
||||||
return useMutation({
|
|
||||||
mutationFn: (enabled: boolean) =>
|
|
||||||
adminApi.updateFundamentalsCutoverSettings(enabled),
|
|
||||||
onSuccess: (config) => {
|
|
||||||
qc.setQueryData(['admin', 'fundamentals-cutover'], config);
|
|
||||||
qc.invalidateQueries({ queryKey: ['admin', 'settings'] });
|
|
||||||
addToast(
|
|
||||||
config.enabled ? 'success' : 'info',
|
|
||||||
config.enabled
|
|
||||||
? 'SEC + Dolt fundamentals activated'
|
|
||||||
: 'SEC + Dolt cache refresh paused',
|
|
||||||
);
|
|
||||||
},
|
|
||||||
onError: (error: Error) => {
|
|
||||||
addToast('error', error.message || 'Failed to update fundamentals data source');
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function useRecommendationSettings() {
|
export function useRecommendationSettings() {
|
||||||
return useQuery({
|
return useQuery({
|
||||||
queryKey: ['admin', 'recommendation-settings'],
|
queryKey: ['admin', 'recommendation-settings'],
|
||||||
@@ -346,14 +316,6 @@ export function useJobs() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useFundamentalsParityReport() {
|
|
||||||
return useQuery({
|
|
||||||
queryKey: ['admin', 'fundamentals-parity'],
|
|
||||||
queryFn: () => adminApi.getFundamentalsParityReport(),
|
|
||||||
refetchInterval: 15_000,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function usePipelineReadiness() {
|
export function usePipelineReadiness() {
|
||||||
return useQuery({
|
return useQuery({
|
||||||
queryKey: ['admin', 'pipeline-readiness'],
|
queryKey: ['admin', 'pipeline-readiness'],
|
||||||
|
|||||||
@@ -187,21 +187,15 @@ export interface ActivationConfig {
|
|||||||
exclude_neutral: boolean;
|
exclude_neutral: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FundamentalsCutoverConfig {
|
|
||||||
enabled: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cron schedule for morning / near-close / after-close / intraday + fundamentals
|
// Cron schedule for morning / near-close / after-close / intraday + fundamentals
|
||||||
export interface ScheduleConfig {
|
export interface ScheduleConfig {
|
||||||
schedule_timezone: string;
|
schedule_timezone: string;
|
||||||
schedule_daily_pipeline_cron: string;
|
schedule_daily_pipeline_cron: string;
|
||||||
schedule_dolt_earnings_cron: string;
|
schedule_dolt_earnings_cron: string;
|
||||||
schedule_sec_fundamentals_cron: string;
|
schedule_sec_fundamentals_cron: string;
|
||||||
schedule_fundamentals_parity_cron: string;
|
|
||||||
schedule_near_close_pipeline_cron: string;
|
schedule_near_close_pipeline_cron: string;
|
||||||
schedule_after_close_pipeline_cron: string;
|
schedule_after_close_pipeline_cron: string;
|
||||||
schedule_intraday_pipeline_cron: string;
|
schedule_intraday_pipeline_cron: string;
|
||||||
schedule_fundamentals_cron: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runtime sentiment LLM configuration
|
// Runtime sentiment LLM configuration
|
||||||
@@ -892,7 +886,7 @@ export interface TickerUniverseSetting {
|
|||||||
|
|
||||||
export interface TickerUniverseBootstrapResult {
|
export interface TickerUniverseBootstrapResult {
|
||||||
universe: TickerUniverse;
|
universe: TickerUniverse;
|
||||||
/** Where the member list came from: wikipedia_sp500 | fmp | cache | seed | … */
|
/** Where the member list came from: wikipedia_sp500 | nasdaq_trader | cache | seed | … */
|
||||||
source?: string;
|
source?: string;
|
||||||
total_universe_symbols: number;
|
total_universe_symbols: number;
|
||||||
added: number;
|
added: number;
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ import { AlertSettings } from '../components/admin/AlertSettings';
|
|||||||
import { SentimentProviderSettings } from '../components/admin/SentimentProviderSettings';
|
import { SentimentProviderSettings } from '../components/admin/SentimentProviderSettings';
|
||||||
import { DataCleanup } from '../components/admin/DataCleanup';
|
import { DataCleanup } from '../components/admin/DataCleanup';
|
||||||
import { JobControls } from '../components/admin/JobControls';
|
import { JobControls } from '../components/admin/JobControls';
|
||||||
import { FundamentalsParityPanel } from '../components/admin/FundamentalsParityPanel';
|
|
||||||
import { FundamentalsCutoverSettings } from '../components/admin/FundamentalsCutoverSettings';
|
|
||||||
import { PerformanceSettings } from '../components/admin/PerformanceSettings';
|
import { PerformanceSettings } from '../components/admin/PerformanceSettings';
|
||||||
import { PipelineReadinessPanel } from '../components/admin/PipelineReadinessPanel';
|
import { PipelineReadinessPanel } from '../components/admin/PipelineReadinessPanel';
|
||||||
import { SystemEventsPanel } from '../components/admin/SystemEventsPanel';
|
import { SystemEventsPanel } from '../components/admin/SystemEventsPanel';
|
||||||
@@ -37,7 +35,6 @@ export default function AdminPage() {
|
|||||||
{activeTab === 'Tickers' && <TickerManagement />}
|
{activeTab === 'Tickers' && <TickerManagement />}
|
||||||
{activeTab === 'Settings' && (
|
{activeTab === 'Settings' && (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
<FundamentalsCutoverSettings />
|
|
||||||
<ActivationSettings />
|
<ActivationSettings />
|
||||||
<ExitPolicySettings />
|
<ExitPolicySettings />
|
||||||
<PerformanceSettings />
|
<PerformanceSettings />
|
||||||
@@ -51,7 +48,6 @@ export default function AdminPage() {
|
|||||||
{activeTab === 'Jobs' && (
|
{activeTab === 'Jobs' && (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
<ScheduleSettings />
|
<ScheduleSettings />
|
||||||
<FundamentalsParityPanel />
|
|
||||||
<JobControls />
|
<JobControls />
|
||||||
<PipelineReadinessPanel />
|
<PipelineReadinessPanel />
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ interface DataStatusItem {
|
|||||||
available: boolean;
|
available: boolean;
|
||||||
timestamp?: string | null;
|
timestamp?: string | null;
|
||||||
timestampLabel?: string | null;
|
timestampLabel?: string | null;
|
||||||
selector: FetchSelector; // what a refresh of this row fetches
|
selector?: FetchSelector; // what a refresh fetches; omit for rows with no manual refresh
|
||||||
paid?: boolean; // provider call that may cost money/quota
|
paid?: boolean; // provider call that may cost money/quota
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,14 +138,16 @@ function DataFreshnessBar({
|
|||||||
) : !item.available ? (
|
) : !item.available ? (
|
||||||
<span className="text-[10px] text-gray-600">no data</span>
|
<span className="text-[10px] text-gray-600">no data</span>
|
||||||
) : null}
|
) : null}
|
||||||
<button
|
{item.selector && (
|
||||||
onClick={() => onRefresh(item)}
|
<button
|
||||||
disabled={busy}
|
onClick={() => onRefresh(item)}
|
||||||
title={item.paid ? `Fetch ${item.label} (uses provider quota)` : `Recompute ${item.label}`}
|
disabled={busy}
|
||||||
className="ml-0.5 text-gray-500 hover:text-blue-300 disabled:opacity-40 transition-colors"
|
title={item.paid ? `Fetch ${item.label} (uses provider quota)` : `Recompute ${item.label}`}
|
||||||
>
|
className="ml-0.5 text-gray-500 hover:text-blue-300 disabled:opacity-40 transition-colors"
|
||||||
<RefreshIcon spinning={pendingLabel === item.label} />
|
>
|
||||||
</button>
|
<RefreshIcon spinning={pendingLabel === item.label} />
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
{item.paid && <span className="text-[9px] text-amber-500/70" title="Uses a paid/quota provider call">$</span>}
|
{item.paid && <span className="text-[9px] text-amber-500/70" title="Uses a paid/quota provider call">$</span>}
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
@@ -226,11 +228,11 @@ export default function TickerDetailPage() {
|
|||||||
paid: true,
|
paid: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
// Rebuilt for the whole universe by the nightly SEC + Dolt imports —
|
||||||
|
// there is no per-ticker fetch to offer here.
|
||||||
label: 'Fundamentals',
|
label: 'Fundamentals',
|
||||||
available: !!fundamentals.data && fundamentals.data.fetched_at !== null,
|
available: !!fundamentals.data && fundamentals.data.fetched_at !== null,
|
||||||
timestamp: fundamentals.data?.fetched_at,
|
timestamp: fundamentals.data?.fetched_at,
|
||||||
selector: ['fundamentals'] as FetchSelector,
|
|
||||||
paid: true,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: 'S/R Levels',
|
label: 'S/R Levels',
|
||||||
@@ -247,6 +249,7 @@ export default function TickerDetailPage() {
|
|||||||
], [ohlcv.data, sentiment.data, fundamentals.data, srLevels.data, scores.data]);
|
], [ohlcv.data, sentiment.data, fundamentals.data, srLevels.data, scores.data]);
|
||||||
|
|
||||||
const handleRefresh = (item: DataStatusItem) => {
|
const handleRefresh = (item: DataStatusItem) => {
|
||||||
|
if (!item.selector) return;
|
||||||
setRefreshingLabel(item.label);
|
setRefreshingLabel(item.label);
|
||||||
ingestion.mutate(
|
ingestion.mutate(
|
||||||
{ symbol, sources: item.selector },
|
{ symbol, sources: item.selector },
|
||||||
|
|||||||
@@ -1,515 +0,0 @@
|
|||||||
"""Bulk-only historical earnings backfill for a local SQLite snapshot.
|
|
||||||
|
|
||||||
The job uses FMP's date-range earnings-calendar endpoint. One request covers all
|
|
||||||
symbols in a date window; per-symbol endpoints are intentionally not available
|
|
||||||
in this task runner. Successful windows are committed independently so a later
|
|
||||||
run resumes after a daily quota boundary without repeating completed windows.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
python scripts/backfill_earnings_events.py --snapshot backtest_snapshots/prod.sqlite \
|
|
||||||
--from-date 2012-01-01 --window-days 30 --limit 250
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import math
|
|
||||||
import sys
|
|
||||||
from datetime import date, datetime, timedelta, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from sqlalchemy import create_engine, text
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
if str(ROOT) not in sys.path:
|
|
||||||
sys.path.insert(0, str(ROOT))
|
|
||||||
|
|
||||||
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
|
||||||
|
|
||||||
bootstrap_ssl()
|
|
||||||
|
|
||||||
FMP_STABLE = "https://financialmodelingprep.com/stable"
|
|
||||||
EVENTS_DDL = """
|
|
||||||
CREATE TABLE IF NOT EXISTS earnings_events (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
symbol TEXT NOT NULL,
|
|
||||||
announce_date TEXT NOT NULL,
|
|
||||||
announce_time TEXT,
|
|
||||||
eps_estimate REAL,
|
|
||||||
eps_actual REAL,
|
|
||||||
revenue_estimate REAL,
|
|
||||||
revenue_actual REAL,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
fetched_at TEXT NOT NULL,
|
|
||||||
UNIQUE(symbol, announce_date)
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
META_DDL = """
|
|
||||||
CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
|
|
||||||
symbol TEXT PRIMARY KEY,
|
|
||||||
status TEXT NOT NULL,
|
|
||||||
n_events INTEGER NOT NULL DEFAULT 0,
|
|
||||||
updated_at TEXT NOT NULL,
|
|
||||||
note TEXT
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
WINDOW_DDL = """
|
|
||||||
CREATE TABLE IF NOT EXISTS earnings_backfill_windows (
|
|
||||||
from_date TEXT NOT NULL,
|
|
||||||
to_date TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL,
|
|
||||||
requests INTEGER NOT NULL DEFAULT 0,
|
|
||||||
rows_raw INTEGER NOT NULL DEFAULT 0,
|
|
||||||
rows_universe INTEGER NOT NULL DEFAULT 0,
|
|
||||||
duplicate_rows INTEGER NOT NULL DEFAULT 0,
|
|
||||||
restated_rows INTEGER NOT NULL DEFAULT 0,
|
|
||||||
updated_at TEXT NOT NULL,
|
|
||||||
note TEXT,
|
|
||||||
PRIMARY KEY(from_date, to_date)
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
|
||||||
parser.add_argument("--from-date", default="2012-01-01")
|
|
||||||
parser.add_argument("--to-date", default=None)
|
|
||||||
parser.add_argument("--window-days", type=int, default=30)
|
|
||||||
parser.add_argument("--limit", type=int, default=250)
|
|
||||||
parser.add_argument("--sleep", type=float, default=0.35)
|
|
||||||
parser.add_argument(
|
|
||||||
"--refetch-windows",
|
|
||||||
action="store_true",
|
|
||||||
help="Re-fetch date windows already logged as done.",
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _ensure_tables(engine) -> None:
|
|
||||||
with engine.begin() as conn:
|
|
||||||
conn.execute(text(EVENTS_DDL))
|
|
||||||
conn.execute(text(META_DDL))
|
|
||||||
conn.execute(text(WINDOW_DDL))
|
|
||||||
|
|
||||||
|
|
||||||
def _number(value: Any) -> float | None:
|
|
||||||
if value is None or value == "":
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
result = float(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
return result if math.isfinite(result) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _normalise_session(value: Any) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
cleaned = str(value).strip().lower().replace("_", " ").replace("-", " ")
|
|
||||||
aliases = {
|
|
||||||
"bmo": "bmo",
|
|
||||||
"before market open": "bmo",
|
|
||||||
"before open": "bmo",
|
|
||||||
"amc": "amc",
|
|
||||||
"after market close": "amc",
|
|
||||||
"after close": "amc",
|
|
||||||
"during market hours": "during",
|
|
||||||
"dmh": "during",
|
|
||||||
}
|
|
||||||
return aliases.get(cleaned, cleaned or None)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_bulk_item(item: dict) -> dict | None:
|
|
||||||
symbol = str(item.get("symbol") or "").strip().upper().replace(".", "-")
|
|
||||||
raw_date = item.get("date") or item.get("earningsDate")
|
|
||||||
if not symbol or not raw_date:
|
|
||||||
return None
|
|
||||||
return {
|
|
||||||
"symbol": symbol,
|
|
||||||
"announce_date": str(raw_date)[:10],
|
|
||||||
"announce_time": _normalise_session(
|
|
||||||
item.get("time") or item.get("announceTime")
|
|
||||||
),
|
|
||||||
"eps_estimate": _number(
|
|
||||||
item.get("epsEstimated")
|
|
||||||
if item.get("epsEstimated") is not None
|
|
||||||
else item.get("estimatedEarning")
|
|
||||||
),
|
|
||||||
"eps_actual": _number(
|
|
||||||
item.get("epsActual")
|
|
||||||
if item.get("epsActual") is not None
|
|
||||||
else item.get("eps")
|
|
||||||
),
|
|
||||||
"revenue_estimate": _number(item.get("revenueEstimated")),
|
|
||||||
"revenue_actual": _number(item.get("revenueActual")),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _windows(start: date, end: date, window_days: int) -> list[tuple[date, date]]:
|
|
||||||
if window_days < 1:
|
|
||||||
raise ValueError("window_days must be positive")
|
|
||||||
result: list[tuple[date, date]] = []
|
|
||||||
cursor = start
|
|
||||||
while cursor <= end:
|
|
||||||
window_end = min(end, cursor + timedelta(days=window_days - 1))
|
|
||||||
result.append((cursor, window_end))
|
|
||||||
cursor = window_end + timedelta(days=1)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _dedupe_bulk_rows(rows: list[dict]) -> tuple[list[dict], int, int]:
|
|
||||||
"""Prefer the most complete duplicate; use the later row as the tie-break."""
|
|
||||||
fields = (
|
|
||||||
"announce_time",
|
|
||||||
"eps_estimate",
|
|
||||||
"eps_actual",
|
|
||||||
"revenue_estimate",
|
|
||||||
"revenue_actual",
|
|
||||||
)
|
|
||||||
chosen: dict[tuple[str, str], dict] = {}
|
|
||||||
duplicate_extras = 0
|
|
||||||
restated = 0
|
|
||||||
for row in rows:
|
|
||||||
key = (str(row["symbol"]), str(row["announce_date"]))
|
|
||||||
previous = chosen.get(key)
|
|
||||||
if previous is None:
|
|
||||||
chosen[key] = row
|
|
||||||
continue
|
|
||||||
duplicate_extras += 1
|
|
||||||
if any(
|
|
||||||
previous.get(field) is not None
|
|
||||||
and row.get(field) is not None
|
|
||||||
and previous.get(field) != row.get(field)
|
|
||||||
for field in fields
|
|
||||||
):
|
|
||||||
restated += 1
|
|
||||||
previous_score = sum(previous.get(field) is not None for field in fields)
|
|
||||||
new_score = sum(row.get(field) is not None for field in fields)
|
|
||||||
if new_score >= previous_score:
|
|
||||||
chosen[key] = row
|
|
||||||
return list(chosen.values()), duplicate_extras, restated
|
|
||||||
|
|
||||||
|
|
||||||
def _upsert_events(conn, rows: list[dict]) -> int:
|
|
||||||
if not rows:
|
|
||||||
return 0
|
|
||||||
fetched_at = datetime.now(timezone.utc).isoformat()
|
|
||||||
statement = text(
|
|
||||||
"""
|
|
||||||
INSERT INTO earnings_events (
|
|
||||||
symbol, announce_date, announce_time, eps_estimate, eps_actual,
|
|
||||||
revenue_estimate, revenue_actual, source, fetched_at
|
|
||||||
) VALUES (
|
|
||||||
:symbol, :announce_date, :announce_time, :eps_estimate, :eps_actual,
|
|
||||||
:revenue_estimate, :revenue_actual, 'fmp_earnings_calendar', :fetched_at
|
|
||||||
)
|
|
||||||
ON CONFLICT(symbol, announce_date) DO UPDATE SET
|
|
||||||
announce_time=COALESCE(excluded.announce_time, earnings_events.announce_time),
|
|
||||||
eps_estimate=COALESCE(excluded.eps_estimate, earnings_events.eps_estimate),
|
|
||||||
eps_actual=COALESCE(excluded.eps_actual, earnings_events.eps_actual),
|
|
||||||
revenue_estimate=COALESCE(excluded.revenue_estimate, earnings_events.revenue_estimate),
|
|
||||||
revenue_actual=COALESCE(excluded.revenue_actual, earnings_events.revenue_actual),
|
|
||||||
source=excluded.source,
|
|
||||||
fetched_at=excluded.fetched_at
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
conn.execute(statement, [{**row, "fetched_at": fetched_at} for row in rows])
|
|
||||||
return len(rows)
|
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_bulk_window(
|
|
||||||
client: httpx.AsyncClient, api_key: str, start: date, end: date
|
|
||||||
) -> tuple[list[dict], int, str | None]:
|
|
||||||
response = await client.get(
|
|
||||||
f"{FMP_STABLE}/earnings-calendar",
|
|
||||||
params={"from": start.isoformat(), "to": end.isoformat(), "apikey": api_key},
|
|
||||||
)
|
|
||||||
if response.status_code in (402, 403):
|
|
||||||
return [], response.status_code, "bulk_endpoint_unavailable"
|
|
||||||
if response.status_code == 429:
|
|
||||||
return [], response.status_code, "daily_limit_reached"
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
if not isinstance(payload, list):
|
|
||||||
return [], response.status_code, f"unexpected_payload:{type(payload).__name__}"
|
|
||||||
rows = []
|
|
||||||
for item in payload:
|
|
||||||
if isinstance(item, dict):
|
|
||||||
parsed = _parse_bulk_item(item)
|
|
||||||
if parsed:
|
|
||||||
rows.append(parsed)
|
|
||||||
return rows, response.status_code, None
|
|
||||||
|
|
||||||
|
|
||||||
def _write_window_status(
|
|
||||||
engine,
|
|
||||||
*,
|
|
||||||
start: date,
|
|
||||||
end: date,
|
|
||||||
status: str,
|
|
||||||
raw_n: int = 0,
|
|
||||||
universe_n: int = 0,
|
|
||||||
duplicate_n: int = 0,
|
|
||||||
restated_n: int = 0,
|
|
||||||
note: str | None = None,
|
|
||||||
) -> None:
|
|
||||||
with engine.begin() as conn:
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"""
|
|
||||||
INSERT INTO earnings_backfill_windows(
|
|
||||||
from_date, to_date, status, requests, rows_raw, rows_universe,
|
|
||||||
duplicate_rows, restated_rows, updated_at, note
|
|
||||||
) VALUES (:a, :b, :status, 1, :raw, :uni, :dup, :rest, :now, :note)
|
|
||||||
ON CONFLICT(from_date, to_date) DO UPDATE SET
|
|
||||||
status=excluded.status,
|
|
||||||
requests=earnings_backfill_windows.requests + 1,
|
|
||||||
rows_raw=excluded.rows_raw,
|
|
||||||
rows_universe=excluded.rows_universe,
|
|
||||||
duplicate_rows=excluded.duplicate_rows,
|
|
||||||
restated_rows=excluded.restated_rows,
|
|
||||||
updated_at=excluded.updated_at,
|
|
||||||
note=excluded.note
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{
|
|
||||||
"a": start.isoformat(),
|
|
||||||
"b": end.isoformat(),
|
|
||||||
"status": status,
|
|
||||||
"raw": raw_n,
|
|
||||||
"uni": universe_n,
|
|
||||||
"dup": duplicate_n,
|
|
||||||
"rest": restated_n,
|
|
||||||
"now": datetime.now(timezone.utc).isoformat(),
|
|
||||||
"note": note,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _main() -> None:
|
|
||||||
args = _parse_args()
|
|
||||||
snapshot = Path(args.snapshot)
|
|
||||||
if not snapshot.exists():
|
|
||||||
raise SystemExit(f"Snapshot not found: {snapshot}")
|
|
||||||
|
|
||||||
from app.config import settings
|
|
||||||
|
|
||||||
if not settings.fmp_api_key:
|
|
||||||
raise SystemExit("FMP_API_KEY required")
|
|
||||||
start = date.fromisoformat(args.from_date)
|
|
||||||
end = date.fromisoformat(args.to_date) if args.to_date else date.today()
|
|
||||||
if start > end:
|
|
||||||
raise SystemExit("--from-date must not be after --to-date")
|
|
||||||
|
|
||||||
engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True)
|
|
||||||
_ensure_tables(engine)
|
|
||||||
all_windows = _windows(start, end, int(args.window_days))
|
|
||||||
with engine.connect() as conn:
|
|
||||||
symbols = [
|
|
||||||
str(row[0]).upper().replace(".", "-")
|
|
||||||
for row in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol"))
|
|
||||||
]
|
|
||||||
completed = {
|
|
||||||
(str(row[0]), str(row[1]))
|
|
||||||
for row in conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT from_date, to_date FROM earnings_backfill_windows "
|
|
||||||
"WHERE status='done'"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
pending = [
|
|
||||||
window
|
|
||||||
for window in all_windows
|
|
||||||
if args.refetch_windows
|
|
||||||
or (window[0].isoformat(), window[1].isoformat()) not in completed
|
|
||||||
]
|
|
||||||
universe = set(symbols)
|
|
||||||
print(f"Snapshot: {snapshot}")
|
|
||||||
print(f"Universe: {len(symbols)} symbols")
|
|
||||||
print(f"Window: {start} -> {end}")
|
|
||||||
print(
|
|
||||||
f"Bulk windows: {len(all_windows)} total; "
|
|
||||||
f"{len(all_windows) - len(pending)} done; {len(pending)} pending"
|
|
||||||
)
|
|
||||||
print("Provider: FMP bulk earnings-calendar only")
|
|
||||||
|
|
||||||
requests_this_run = 0
|
|
||||||
rows_upserted = 0
|
|
||||||
duplicate_rows = 0
|
|
||||||
restated_rows = 0
|
|
||||||
stop_note: str | None = None
|
|
||||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
||||||
for index, (window_start, window_end) in enumerate(pending, 1):
|
|
||||||
if requests_this_run >= int(args.limit):
|
|
||||||
stop_note = "request_budget_exhausted"
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
raw_rows, status_code, error = await _fetch_bulk_window(
|
|
||||||
client, settings.fmp_api_key, window_start, window_end
|
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
raw_rows, status_code = [], 0
|
|
||||||
error = f"request_error:{type(exc).__name__}:{exc}"
|
|
||||||
requests_this_run += 1
|
|
||||||
if error:
|
|
||||||
_write_window_status(
|
|
||||||
engine,
|
|
||||||
start=window_start,
|
|
||||||
end=window_end,
|
|
||||||
status="error",
|
|
||||||
note=f"http={status_code} {error}"[:300],
|
|
||||||
)
|
|
||||||
stop_note = error
|
|
||||||
print(
|
|
||||||
f"STOP {window_start}..{window_end}: {error} "
|
|
||||||
f"(http={status_code}, request={requests_this_run})"
|
|
||||||
)
|
|
||||||
break
|
|
||||||
|
|
||||||
in_universe = [row for row in raw_rows if row["symbol"] in universe]
|
|
||||||
deduped, duplicate_n, restated_n = _dedupe_bulk_rows(in_universe)
|
|
||||||
with engine.begin() as conn:
|
|
||||||
rows_upserted += _upsert_events(conn, deduped)
|
|
||||||
_write_window_status(
|
|
||||||
engine,
|
|
||||||
start=window_start,
|
|
||||||
end=window_end,
|
|
||||||
status="done",
|
|
||||||
raw_n=len(raw_rows),
|
|
||||||
universe_n=len(deduped),
|
|
||||||
duplicate_n=duplicate_n,
|
|
||||||
restated_n=restated_n,
|
|
||||||
note="bulk",
|
|
||||||
)
|
|
||||||
duplicate_rows += duplicate_n
|
|
||||||
restated_rows += restated_n
|
|
||||||
if index == 1 or index % 10 == 0 or index == len(pending):
|
|
||||||
print(
|
|
||||||
f"progress windows={index}/{len(pending)} "
|
|
||||||
f"requests={requests_this_run}/{args.limit} "
|
|
||||||
f"last={window_start}..{window_end} rows={len(deduped)}"
|
|
||||||
)
|
|
||||||
if args.sleep > 0:
|
|
||||||
await asyncio.sleep(float(args.sleep))
|
|
||||||
|
|
||||||
with engine.begin() as conn:
|
|
||||||
windows_done = int(
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT COUNT(*) FROM earnings_backfill_windows "
|
|
||||||
"WHERE status='done' AND from_date >= :a AND to_date <= :b"
|
|
||||||
),
|
|
||||||
{"a": start.isoformat(), "b": end.isoformat()},
|
|
||||||
).scalar_one()
|
|
||||||
)
|
|
||||||
complete = windows_done >= len(all_windows)
|
|
||||||
if complete:
|
|
||||||
now = datetime.now(timezone.utc).isoformat()
|
|
||||||
for symbol in symbols:
|
|
||||||
count = int(
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT COUNT(*) FROM earnings_events "
|
|
||||||
"WHERE symbol=:symbol AND announce_date BETWEEN :a AND :b"
|
|
||||||
),
|
|
||||||
{"symbol": symbol, "a": start.isoformat(), "b": end.isoformat()},
|
|
||||||
).scalar_one()
|
|
||||||
)
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"""
|
|
||||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
|
||||||
VALUES (:symbol, 'done', :count, :now, 'bulk_complete')
|
|
||||||
ON CONFLICT(symbol) DO UPDATE SET
|
|
||||||
status='done', n_events=excluded.n_events,
|
|
||||||
updated_at=excluded.updated_at, note=excluded.note
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{"symbol": symbol, "count": count, "now": now},
|
|
||||||
)
|
|
||||||
params = {"a": start.isoformat(), "b": end.isoformat()}
|
|
||||||
total_events = int(
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT COUNT(*) FROM earnings_events "
|
|
||||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
|
||||||
"AND announce_date BETWEEN :a AND :b"
|
|
||||||
),
|
|
||||||
params,
|
|
||||||
).scalar_one()
|
|
||||||
)
|
|
||||||
paired_events = int(
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT COUNT(*) FROM earnings_events "
|
|
||||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
|
||||||
"AND announce_date BETWEEN :a AND :b "
|
|
||||||
"AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL"
|
|
||||||
),
|
|
||||||
params,
|
|
||||||
).scalar_one()
|
|
||||||
)
|
|
||||||
date_range = conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events "
|
|
||||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
|
||||||
"AND announce_date BETWEEN :a AND :b"
|
|
||||||
),
|
|
||||||
params,
|
|
||||||
).fetchone()
|
|
||||||
done_symbols = int(
|
|
||||||
conn.execute(
|
|
||||||
text("SELECT COUNT(*) FROM earnings_backfill_meta WHERE status='done'")
|
|
||||||
).scalar_one()
|
|
||||||
)
|
|
||||||
totals = conn.execute(
|
|
||||||
text(
|
|
||||||
"SELECT COALESCE(SUM(requests),0), COALESCE(SUM(duplicate_rows),0), "
|
|
||||||
"COALESCE(SUM(restated_rows),0) FROM earnings_backfill_windows "
|
|
||||||
"WHERE from_date >= :a AND to_date <= :b"
|
|
||||||
),
|
|
||||||
params,
|
|
||||||
).fetchone()
|
|
||||||
|
|
||||||
summary = {
|
|
||||||
"mode": "fmp_bulk_date_range_only",
|
|
||||||
"window": {"from": start.isoformat(), "to": end.isoformat()},
|
|
||||||
"window_days": int(args.window_days),
|
|
||||||
"bulk_windows_total": len(all_windows),
|
|
||||||
"bulk_windows_done": windows_done,
|
|
||||||
"bulk_requests_this_run": requests_this_run,
|
|
||||||
"bulk_requests_logged_total": int(totals[0]),
|
|
||||||
"rows_upserted_this_run": rows_upserted,
|
|
||||||
"duplicate_rows_this_run": duplicate_rows,
|
|
||||||
"restated_rows_this_run": restated_rows,
|
|
||||||
"duplicate_rows_logged_total": int(totals[1]),
|
|
||||||
"restated_rows_logged_total": int(totals[2]),
|
|
||||||
"dedupe_policy": (
|
|
||||||
"UNIQUE(symbol, announce_date); prefer more non-null fields, then "
|
|
||||||
"the provider's later occurrence; non-null bulk fields replace prior "
|
|
||||||
"values while null bulk fields retain existing values"
|
|
||||||
),
|
|
||||||
"events_in_window": total_events,
|
|
||||||
"events_with_actual_and_estimate": paired_events,
|
|
||||||
"symbols_done": done_symbols,
|
|
||||||
"symbols_universe": len(symbols),
|
|
||||||
"announce_date_range": {"min": date_range[0], "max": date_range[1]},
|
|
||||||
"request_budget": int(args.limit),
|
|
||||||
"stop_note": stop_note,
|
|
||||||
"complete": complete,
|
|
||||||
}
|
|
||||||
output = Path("reports/earnings-backfill-status.json")
|
|
||||||
output.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
||||||
print(json.dumps(summary, indent=2))
|
|
||||||
print(f"Wrote {output}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(_main())
|
|
||||||
@@ -128,11 +128,10 @@ async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
|
|||||||
"""Return sorted unique symbols and source labels.
|
"""Return sorted unique symbols and source labels.
|
||||||
|
|
||||||
Offline-safe: does **not** use production Postgres or SystemSetting cache
|
Offline-safe: does **not** use production Postgres or SystemSetting cache
|
||||||
(those require a schema). Public sources first, then FMP, then seeds.
|
(those require a schema). Public sources first, then seeds.
|
||||||
"""
|
"""
|
||||||
from app.services.ticker_universe_service import (
|
from app.services.ticker_universe_service import (
|
||||||
_SEED_UNIVERSES,
|
_SEED_UNIVERSES,
|
||||||
_fetch_universe_symbols_from_fmp,
|
|
||||||
_fetch_universe_symbols_from_public,
|
_fetch_universe_symbols_from_public,
|
||||||
_normalise_symbols,
|
_normalise_symbols,
|
||||||
)
|
)
|
||||||
@@ -150,19 +149,11 @@ async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
|
|||||||
cleaned = _normalise_symbols(public_symbols)
|
cleaned = _normalise_symbols(public_symbols)
|
||||||
if cleaned:
|
if cleaned:
|
||||||
src = public_source or "public"
|
src = public_source or "public"
|
||||||
else:
|
elif public_failures:
|
||||||
if public_failures:
|
print(
|
||||||
print(
|
f" WARNING: public fetch {universe}: "
|
||||||
f" WARNING: public fetch {universe}: "
|
f"{'; '.join(public_failures[:3])}"
|
||||||
f"{'; '.join(public_failures[:3])}"
|
)
|
||||||
)
|
|
||||||
try:
|
|
||||||
fmp_symbols = await _fetch_universe_symbols_from_fmp(universe)
|
|
||||||
cleaned = _normalise_symbols(fmp_symbols)
|
|
||||||
if cleaned:
|
|
||||||
src = "fmp"
|
|
||||||
except Exception as exc:
|
|
||||||
print(f" WARNING: FMP fetch {universe}: {exc}")
|
|
||||||
|
|
||||||
if not cleaned:
|
if not cleaned:
|
||||||
cleaned = _normalise_symbols(_SEED_UNIVERSES.get(universe, []))
|
cleaned = _normalise_symbols(_SEED_UNIVERSES.get(universe, []))
|
||||||
|
|||||||
@@ -1,764 +0,0 @@
|
|||||||
'''Pure helpers for the focused daily portfolio-capacity research matrix.'''
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import math
|
|
||||||
import random
|
|
||||||
import statistics
|
|
||||||
from collections import defaultdict
|
|
||||||
from datetime import date, timedelta
|
|
||||||
from typing import Any, Iterable
|
|
||||||
|
|
||||||
|
|
||||||
ARMS: tuple[dict[str, Any], ...] = (
|
|
||||||
{
|
|
||||||
'id': 'cap10_incumbent',
|
|
||||||
'label': 'Cap 10, arrival-order incumbents',
|
|
||||||
'max_positions': 10,
|
|
||||||
'min_initial_risk_fraction': None,
|
|
||||||
'weekly_top_n_rebalance': False,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'id': 'cash_unbounded',
|
|
||||||
'label': 'Cash-constrained, no count cap',
|
|
||||||
'max_positions': None,
|
|
||||||
'min_initial_risk_fraction': 0.005,
|
|
||||||
'weekly_top_n_rebalance': False,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'id': 'cap10_weekly_top10',
|
|
||||||
'label': 'Cap 10, weekly current-rank top 10',
|
|
||||||
'max_positions': 10,
|
|
||||||
'min_initial_risk_fraction': None,
|
|
||||||
'weekly_top_n_rebalance': True,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'id': 'cap15_incumbent',
|
|
||||||
'label': 'Cap 15, arrival-order incumbents',
|
|
||||||
'max_positions': 15,
|
|
||||||
'min_initial_risk_fraction': None,
|
|
||||||
'weekly_top_n_rebalance': False,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
ARM_BY_ID = {arm['id']: arm for arm in ARMS}
|
|
||||||
RISK_FLOOR_ARMS: tuple[dict[str, Any], ...] = (
|
|
||||||
ARMS[0],
|
|
||||||
{
|
|
||||||
'id': 'cap10_min_risk_005',
|
|
||||||
'label': 'Cap 10, 0.5% minimum effective initial risk',
|
|
||||||
'max_positions': 10,
|
|
||||||
'min_initial_risk_fraction': 0.005,
|
|
||||||
'weekly_top_n_rebalance': False,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
COSTS_PER_SIDE_PCT = (0.1, 0.2)
|
|
||||||
ANCHOR_YEARS = tuple(range(2019, 2026))
|
|
||||||
SCORING_SESSIONS = 504
|
|
||||||
MEASUREMENT_SESSIONS = 252
|
|
||||||
RESIDUAL_BENCHMARK_SESSIONS = 252
|
|
||||||
WARM_SEED_MIN_OFFSET = 63
|
|
||||||
WARM_SEED_MAX_OFFSET = 126
|
|
||||||
BOOTSTRAP_REPLICATES = 10_000
|
|
||||||
BOOTSTRAP_SEED = 20260805
|
|
||||||
PRIMARY_METRICS = (
|
|
||||||
'ev_net_r',
|
|
||||||
'calmar',
|
|
||||||
'profit_factor',
|
|
||||||
'gain_to_pain',
|
|
||||||
'sortino',
|
|
||||||
)
|
|
||||||
PAIRED_METRICS = (
|
|
||||||
*PRIMARY_METRICS,
|
|
||||||
'cagr_pct',
|
|
||||||
'max_drawdown_pct',
|
|
||||||
'total_return_pct',
|
|
||||||
'sharpe',
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _end_exclusive(
|
|
||||||
sessions: list[date], start_index: int, count: int
|
|
||||||
) -> date:
|
|
||||||
end_index = start_index + count
|
|
||||||
if end_index < len(sessions):
|
|
||||||
return sessions[end_index]
|
|
||||||
return sessions[-1] + timedelta(days=1)
|
|
||||||
|
|
||||||
|
|
||||||
def build_cohort_manifest(session_dates: Iterable[date]) -> dict[str, Any]:
|
|
||||||
sessions = sorted(set(session_dates))
|
|
||||||
minimum = RESIDUAL_BENCHMARK_SESSIONS + SCORING_SESSIONS
|
|
||||||
if len(sessions) <= minimum + MEASUREMENT_SESSIONS:
|
|
||||||
raise ValueError('Snapshot is too short for the frozen cohort design')
|
|
||||||
|
|
||||||
index_of = {session: index for index, session in enumerate(sessions)}
|
|
||||||
first_eligible_index = RESIDUAL_BENCHMARK_SESSIONS - 1 + SCORING_SESSIONS
|
|
||||||
last_eligible_index = len(sessions) - MEASUREMENT_SESSIONS
|
|
||||||
|
|
||||||
first_by_month: dict[tuple[int, int], date] = {}
|
|
||||||
for session in sessions:
|
|
||||||
first_by_month.setdefault((session.year, session.month), session)
|
|
||||||
|
|
||||||
empty: list[dict[str, Any]] = []
|
|
||||||
for (year, month), session in sorted(first_by_month.items()):
|
|
||||||
index = index_of[session]
|
|
||||||
if year not in ANCHOR_YEARS:
|
|
||||||
continue
|
|
||||||
if index < first_eligible_index or index > last_eligible_index:
|
|
||||||
continue
|
|
||||||
empty.append({
|
|
||||||
'protocol': 'empty_book',
|
|
||||||
'path_id': f'empty-{year:04d}-{month:02d}',
|
|
||||||
'cluster': year,
|
|
||||||
'simulation_start': session.isoformat(),
|
|
||||||
'measurement_start': session.isoformat(),
|
|
||||||
'hard_end_exclusive': _end_exclusive(
|
|
||||||
sessions, index, MEASUREMENT_SESSIONS
|
|
||||||
).isoformat(),
|
|
||||||
})
|
|
||||||
|
|
||||||
first_by_year: dict[int, date] = {}
|
|
||||||
for session in sessions:
|
|
||||||
first_by_year.setdefault(session.year, session)
|
|
||||||
|
|
||||||
warm: list[dict[str, Any]] = []
|
|
||||||
warm_seed_counts: dict[str, int] = {}
|
|
||||||
for year in ANCHOR_YEARS:
|
|
||||||
anchor = first_by_year.get(year)
|
|
||||||
if anchor is None:
|
|
||||||
continue
|
|
||||||
anchor_index = index_of[anchor]
|
|
||||||
if (
|
|
||||||
anchor_index < WARM_SEED_MAX_OFFSET
|
|
||||||
or anchor_index > last_eligible_index
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
seed_window = sessions[
|
|
||||||
anchor_index - WARM_SEED_MAX_OFFSET:
|
|
||||||
anchor_index - WARM_SEED_MIN_OFFSET + 1
|
|
||||||
]
|
|
||||||
first_by_iso_week: dict[tuple[int, int], date] = {}
|
|
||||||
for session in seed_window:
|
|
||||||
iso = session.isocalendar()
|
|
||||||
first_by_iso_week.setdefault((iso.year, iso.week), session)
|
|
||||||
seeds = sorted(first_by_iso_week.values())
|
|
||||||
warm_seed_counts[str(year)] = len(seeds)
|
|
||||||
for seed_index, seed in enumerate(seeds, 1):
|
|
||||||
warm.append({
|
|
||||||
'protocol': 'warm_book',
|
|
||||||
'path_id': f'warm-{year}-seed-{seed_index:02d}',
|
|
||||||
'cluster': year,
|
|
||||||
'simulation_start': seed.isoformat(),
|
|
||||||
'measurement_start': anchor.isoformat(),
|
|
||||||
'hard_end_exclusive': _end_exclusive(
|
|
||||||
sessions, anchor_index, MEASUREMENT_SESSIONS
|
|
||||||
).isoformat(),
|
|
||||||
'seed_offset_sessions': anchor_index - index_of[seed],
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
'snapshot_first_session': sessions[0].isoformat(),
|
|
||||||
'snapshot_last_session': sessions[-1].isoformat(),
|
|
||||||
'session_count': len(sessions),
|
|
||||||
'expected_clusters': list(ANCHOR_YEARS),
|
|
||||||
'empty_book': empty,
|
|
||||||
'warm_book': warm,
|
|
||||||
'empty_cluster_counts': dict(
|
|
||||||
sorted(
|
|
||||||
(
|
|
||||||
str(year),
|
|
||||||
sum(1 for row in empty if row['cluster'] == year),
|
|
||||||
)
|
|
||||||
for year in {row['cluster'] for row in empty}
|
|
||||||
)
|
|
||||||
),
|
|
||||||
'warm_seed_counts': warm_seed_counts,
|
|
||||||
'empty_cluster_count': len({row['cluster'] for row in empty}),
|
|
||||||
'warm_cluster_count': len({row['cluster'] for row in warm}),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def validate_cohort_manifest(manifest: dict[str, Any]) -> list[str]:
|
|
||||||
errors: list[str] = []
|
|
||||||
expected = set(ANCHOR_YEARS)
|
|
||||||
empty_clusters = {row['cluster'] for row in manifest['empty_book']}
|
|
||||||
warm_clusters = {row['cluster'] for row in manifest['warm_book']}
|
|
||||||
if empty_clusters != expected:
|
|
||||||
errors.append(
|
|
||||||
f'empty-book clusters {sorted(empty_clusters)} != {sorted(expected)}'
|
|
||||||
)
|
|
||||||
if warm_clusters != expected:
|
|
||||||
errors.append(
|
|
||||||
f'warm-book clusters {sorted(warm_clusters)} != {sorted(expected)}'
|
|
||||||
)
|
|
||||||
for year in ANCHOR_YEARS:
|
|
||||||
seed_count = int(manifest['warm_seed_counts'].get(str(year), 0))
|
|
||||||
if seed_count < 12:
|
|
||||||
errors.append(f'warm anchor {year} has only {seed_count} seeds')
|
|
||||||
return errors
|
|
||||||
|
|
||||||
|
|
||||||
def build_cells(
|
|
||||||
manifest: dict[str, Any],
|
|
||||||
*,
|
|
||||||
arms: tuple[dict[str, Any], ...] = ARMS,
|
|
||||||
protocols: tuple[str, ...] = ('empty_book', 'warm_book'),
|
|
||||||
costs: tuple[float, ...] = COSTS_PER_SIDE_PCT,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
paths = [
|
|
||||||
path
|
|
||||||
for protocol in protocols
|
|
||||||
for path in manifest[protocol]
|
|
||||||
]
|
|
||||||
cells: list[dict[str, Any]] = []
|
|
||||||
for cost in costs:
|
|
||||||
for path in paths:
|
|
||||||
for arm in arms:
|
|
||||||
cell_id = (
|
|
||||||
f'{arm["id"]}|{path["protocol"]}|{path["path_id"]}'
|
|
||||||
f'|cost={cost:.1f}'
|
|
||||||
)
|
|
||||||
cells.append({
|
|
||||||
**path,
|
|
||||||
'cell_id': cell_id,
|
|
||||||
'arm_id': arm['id'],
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
})
|
|
||||||
return cells
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(values: Iterable[float], probability: float) -> float | None:
|
|
||||||
ordered = sorted(float(value) for value in values if value is not None)
|
|
||||||
if not ordered:
|
|
||||||
return None
|
|
||||||
if len(ordered) == 1:
|
|
||||||
return ordered[0]
|
|
||||||
location = (len(ordered) - 1) * probability
|
|
||||||
lower = math.floor(location)
|
|
||||||
upper = math.ceil(location)
|
|
||||||
if lower == upper:
|
|
||||||
return ordered[lower]
|
|
||||||
weight = location - lower
|
|
||||||
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
|
|
||||||
|
|
||||||
|
|
||||||
def iqr(values: Iterable[float]) -> float | None:
|
|
||||||
clean: list[float] = []
|
|
||||||
for value in values:
|
|
||||||
if value is None:
|
|
||||||
continue
|
|
||||||
parsed = float(value)
|
|
||||||
if math.isfinite(parsed):
|
|
||||||
clean.append(parsed)
|
|
||||||
q25 = percentile(clean, 0.25)
|
|
||||||
q75 = percentile(clean, 0.75)
|
|
||||||
if q25 is None or q75 is None:
|
|
||||||
return None
|
|
||||||
return q75 - q25
|
|
||||||
|
|
||||||
|
|
||||||
def median(values: Iterable[float | None]) -> float | None:
|
|
||||||
clean = [float(value) for value in values if value is not None]
|
|
||||||
return statistics.median(clean) if clean else None
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_ratio(numerator: float | None, denominator: float | None) -> float | None:
|
|
||||||
if numerator is None or denominator is None:
|
|
||||||
return None
|
|
||||||
if abs(denominator) <= 1e-12:
|
|
||||||
return 1.0 if abs(numerator) <= 1e-12 else None
|
|
||||||
return numerator / denominator
|
|
||||||
|
|
||||||
|
|
||||||
def _stable_seed(*parts: object) -> int:
|
|
||||||
digest = hashlib.sha256('|'.join(map(str, parts)).encode('utf-8')).digest()
|
|
||||||
return BOOTSTRAP_SEED + int.from_bytes(digest[:4], 'big')
|
|
||||||
|
|
||||||
|
|
||||||
def bootstrap_median_interval(
|
|
||||||
values: Iterable[float | None],
|
|
||||||
*,
|
|
||||||
seed_parts: tuple[object, ...],
|
|
||||||
replicates: int = BOOTSTRAP_REPLICATES,
|
|
||||||
) -> dict[str, float | int | None]:
|
|
||||||
clean = [float(value) for value in values if value is not None]
|
|
||||||
if not clean:
|
|
||||||
return {'n': 0, 'point': None, 'p05': None, 'p95': None}
|
|
||||||
rng = random.Random(_stable_seed(*seed_parts))
|
|
||||||
draws = [
|
|
||||||
statistics.median(rng.choices(clean, k=len(clean)))
|
|
||||||
for _ in range(replicates)
|
|
||||||
]
|
|
||||||
return {
|
|
||||||
'n': len(clean),
|
|
||||||
'replicates': replicates,
|
|
||||||
'point': statistics.median(clean),
|
|
||||||
'p05': percentile(draws, 0.05),
|
|
||||||
'p95': percentile(draws, 0.95),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _monthly_returns(
|
|
||||||
equity_curve: list[dict[str, Any]], base_equity: float
|
|
||||||
) -> list[float]:
|
|
||||||
month_ends: dict[tuple[int, int], float] = {}
|
|
||||||
for point in equity_curve:
|
|
||||||
point_date = date.fromisoformat(str(point['date']))
|
|
||||||
month_ends[(point_date.year, point_date.month)] = float(point['equity'])
|
|
||||||
previous = float(base_equity)
|
|
||||||
returns: list[float] = []
|
|
||||||
for month in sorted(month_ends):
|
|
||||||
equity = month_ends[month]
|
|
||||||
if previous > 0:
|
|
||||||
returns.append(equity / previous - 1.0)
|
|
||||||
previous = equity
|
|
||||||
return returns
|
|
||||||
|
|
||||||
|
|
||||||
def _time_underwater(equities: list[float]) -> tuple[int, float]:
|
|
||||||
peak = float('-inf')
|
|
||||||
current = 0
|
|
||||||
longest = 0
|
|
||||||
underwater = 0
|
|
||||||
for equity in equities:
|
|
||||||
peak = max(peak, equity)
|
|
||||||
if peak > 0 and equity < peak - 1e-9:
|
|
||||||
current += 1
|
|
||||||
underwater += 1
|
|
||||||
longest = max(longest, current)
|
|
||||||
else:
|
|
||||||
current = 0
|
|
||||||
percentage = underwater / len(equities) * 100.0 if equities else 0.0
|
|
||||||
return longest, percentage
|
|
||||||
|
|
||||||
|
|
||||||
def summarize_simulation(sim: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
trades = list(sim.get('trade_details') or [])
|
|
||||||
equity_curve = list(sim.get('equity_curve') or [])
|
|
||||||
net_rs = [float(trade['net_r']) for trade in trades]
|
|
||||||
positive_rs = [value for value in net_rs if value > 0]
|
|
||||||
negative_rs = [value for value in net_rs if value < 0]
|
|
||||||
ev_net_r = statistics.fmean(net_rs) if net_rs else None
|
|
||||||
profit_factor = (
|
|
||||||
sum(positive_rs) / abs(sum(negative_rs))
|
|
||||||
if negative_rs
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
base_equity = float(
|
|
||||||
sim.get('measurement_start_equity') or sim.get('starting_capital') or 0.0
|
|
||||||
)
|
|
||||||
curve_equities = [float(point['equity']) for point in equity_curve]
|
|
||||||
daily_equities = [base_equity, *curve_equities]
|
|
||||||
daily_returns = [
|
|
||||||
current / previous - 1.0
|
|
||||||
for previous, current in zip(daily_equities, daily_equities[1:])
|
|
||||||
if previous > 0
|
|
||||||
]
|
|
||||||
downside_deviation = (
|
|
||||||
math.sqrt(
|
|
||||||
statistics.fmean(min(value, 0.0) ** 2 for value in daily_returns)
|
|
||||||
)
|
|
||||||
if daily_returns
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
sortino = (
|
|
||||||
statistics.fmean(daily_returns) / downside_deviation * math.sqrt(252.0)
|
|
||||||
if downside_deviation is not None and downside_deviation > 0
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
monthly_returns = _monthly_returns(equity_curve, base_equity)
|
|
||||||
negative_monthly = sum(value for value in monthly_returns if value < 0)
|
|
||||||
gain_to_pain = (
|
|
||||||
sum(monthly_returns) / abs(negative_monthly)
|
|
||||||
if negative_monthly < 0
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
longest_underwater, underwater_pct = _time_underwater(daily_equities)
|
|
||||||
|
|
||||||
transaction_cost = sum(
|
|
||||||
float(trade.get('transaction_cost') or 0.0) for trade in trades
|
|
||||||
)
|
|
||||||
traded_notional = sum(
|
|
||||||
float(trade.get('shares') or 0.0)
|
|
||||||
* (float(trade.get('entry') or 0.0) + float(trade.get('fill') or 0.0))
|
|
||||||
for trade in trades
|
|
||||||
)
|
|
||||||
turnover_multiple = (
|
|
||||||
traded_notional / base_equity if base_equity > 0 else None
|
|
||||||
)
|
|
||||||
|
|
||||||
ordered_rs = sorted(net_rs, reverse=True)
|
|
||||||
ev_without_best: dict[str, float | None] = {}
|
|
||||||
for count in (1, 5, 10):
|
|
||||||
remaining = ordered_rs[count:]
|
|
||||||
ev_without_best[str(count)] = (
|
|
||||||
statistics.fmean(remaining) if remaining else None
|
|
||||||
)
|
|
||||||
|
|
||||||
events = list(sim.get('weekly_rebalance_events') or [])
|
|
||||||
entrant_sizes = [int(event['fresh_entrant_pool']) for event in events]
|
|
||||||
eligible_sizes = [
|
|
||||||
int(event['rank_eligible_entrant_pool']) for event in events
|
|
||||||
]
|
|
||||||
replacements = [int(event['replacements']) for event in events]
|
|
||||||
|
|
||||||
capacity_skips = int(
|
|
||||||
sim.get('measurement_skipped_book_full', sim.get('skipped_book_full', 0))
|
|
||||||
)
|
|
||||||
opened = int(sim.get('opened_positions', sim.get('trades', 0)))
|
|
||||||
capacity_opportunities = opened + capacity_skips
|
|
||||||
|
|
||||||
result = {
|
|
||||||
'start_date': sim.get('start_date'),
|
|
||||||
'end_date': sim.get('end_date'),
|
|
||||||
'simulation_start_date': sim.get('simulation_start_date'),
|
|
||||||
'measurement_start_equity': base_equity,
|
|
||||||
'measurement_start_positions': sim.get('measurement_start_positions', 0),
|
|
||||||
'trades': len(trades),
|
|
||||||
'ev_net_r': ev_net_r,
|
|
||||||
'profit_factor': profit_factor,
|
|
||||||
'gain_to_pain': gain_to_pain,
|
|
||||||
'sortino': sortino,
|
|
||||||
'ev_without_best': ev_without_best,
|
|
||||||
'total_return_pct': sim.get('total_return_pct'),
|
|
||||||
'cagr_pct': sim.get('cagr_pct'),
|
|
||||||
'max_drawdown_pct': sim.get('max_drawdown_pct'),
|
|
||||||
'calmar': sim.get('calmar'),
|
|
||||||
'sharpe': sim.get('sharpe'),
|
|
||||||
'win_rate': sim.get('win_rate'),
|
|
||||||
'avg_hold_days': sim.get('avg_hold_days'),
|
|
||||||
'longest_underwater_sessions': longest_underwater,
|
|
||||||
'underwater_pct': underwater_pct,
|
|
||||||
'transaction_cost': transaction_cost,
|
|
||||||
'turnover_multiple': turnover_multiple,
|
|
||||||
'skipped_book_full': capacity_skips,
|
|
||||||
'opened_positions': opened,
|
|
||||||
'capacity_opportunities': capacity_opportunities,
|
|
||||||
'blocked_fraction': (
|
|
||||||
capacity_skips / capacity_opportunities
|
|
||||||
if capacity_opportunities
|
|
||||||
else 0.0
|
|
||||||
),
|
|
||||||
'skipped_min_initial_risk': int(
|
|
||||||
sim.get('measurement_skipped_min_initial_risk', 0)
|
|
||||||
),
|
|
||||||
'avg_positions': sim.get('avg_positions'),
|
|
||||||
'peak_positions': sim.get('peak_positions'),
|
|
||||||
'sessions_at_capacity': sim.get('sessions_at_capacity'),
|
|
||||||
'sessions_measured': sim.get('sessions_measured'),
|
|
||||||
'avg_cash_pct': sim.get('avg_cash_pct'),
|
|
||||||
'avg_gross_exposure_pct': sim.get('avg_gross_exposure_pct'),
|
|
||||||
'exit_reasons': sim.get('exit_reasons'),
|
|
||||||
}
|
|
||||||
if events:
|
|
||||||
result['weekly_rebalance'] = {
|
|
||||||
'events': len(events),
|
|
||||||
'zero_entrant_fraction': (
|
|
||||||
sum(1 for value in entrant_sizes if value == 0) / len(events)
|
|
||||||
),
|
|
||||||
'entrant_pool_mean': statistics.fmean(entrant_sizes),
|
|
||||||
'entrant_pool_median': statistics.median(entrant_sizes),
|
|
||||||
'entrant_pool_p90': percentile(entrant_sizes, 0.9),
|
|
||||||
'eligible_pool_mean': statistics.fmean(eligible_sizes),
|
|
||||||
'replacements': sum(replacements),
|
|
||||||
'weekly_rank_rejected_entries': int(
|
|
||||||
sim.get('weekly_rank_rejected_entries', 0)
|
|
||||||
),
|
|
||||||
'reentries_within_5_sessions': int(
|
|
||||||
sim.get('rebalance_reentries_within_5_sessions', 0)
|
|
||||||
),
|
|
||||||
'reentries_within_10_sessions': int(
|
|
||||||
sim.get('rebalance_reentries_within_10_sessions', 0)
|
|
||||||
),
|
|
||||||
'reentries_within_20_sessions': int(
|
|
||||||
sim.get('rebalance_reentries_within_20_sessions', 0)
|
|
||||||
),
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _cluster_rows(
|
|
||||||
cells: list[dict[str, Any]],
|
|
||||||
*,
|
|
||||||
arm_id: str,
|
|
||||||
protocol: str,
|
|
||||||
cost: float,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
treatment = {
|
|
||||||
row['path_id']: row
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == arm_id
|
|
||||||
and row['protocol'] == protocol
|
|
||||||
and float(row['cost_per_side_pct']) == cost
|
|
||||||
}
|
|
||||||
control = {
|
|
||||||
row['path_id']: row
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == 'cap10_incumbent'
|
|
||||||
and row['protocol'] == protocol
|
|
||||||
and float(row['cost_per_side_pct']) == cost
|
|
||||||
}
|
|
||||||
shared_paths = sorted(set(treatment) & set(control))
|
|
||||||
by_cluster: dict[int, list[tuple[dict, dict]]] = defaultdict(list)
|
|
||||||
for path_id in shared_paths:
|
|
||||||
row = treatment[path_id]
|
|
||||||
by_cluster[int(row['cluster'])].append((row, control[path_id]))
|
|
||||||
|
|
||||||
summaries: list[dict[str, Any]] = []
|
|
||||||
for cluster, pairs in sorted(by_cluster.items()):
|
|
||||||
metrics: dict[str, Any] = {}
|
|
||||||
for metric in PAIRED_METRICS:
|
|
||||||
arm_values = [
|
|
||||||
pair[0]['metrics'].get(metric)
|
|
||||||
for pair in pairs
|
|
||||||
if pair[0]['metrics'].get(metric) is not None
|
|
||||||
and math.isfinite(float(pair[0]['metrics'][metric]))
|
|
||||||
]
|
|
||||||
control_values = [
|
|
||||||
pair[1]['metrics'].get(metric)
|
|
||||||
for pair in pairs
|
|
||||||
if pair[1]['metrics'].get(metric) is not None
|
|
||||||
and math.isfinite(float(pair[1]['metrics'][metric]))
|
|
||||||
]
|
|
||||||
deltas = [
|
|
||||||
float(arm['metrics'][metric])
|
|
||||||
- float(base['metrics'][metric])
|
|
||||||
for arm, base in pairs
|
|
||||||
if arm['metrics'].get(metric) is not None
|
|
||||||
and base['metrics'].get(metric) is not None
|
|
||||||
and math.isfinite(float(arm['metrics'][metric]))
|
|
||||||
and math.isfinite(float(base['metrics'][metric]))
|
|
||||||
]
|
|
||||||
arm_median = median(arm_values)
|
|
||||||
control_median = median(control_values)
|
|
||||||
metrics[metric] = {
|
|
||||||
'arm_median': arm_median,
|
|
||||||
'control_median': control_median,
|
|
||||||
'paired_delta_median': median(deltas),
|
|
||||||
'arm_control_ratio': _safe_ratio(
|
|
||||||
arm_median, control_median
|
|
||||||
),
|
|
||||||
'paired_paths': len(deltas),
|
|
||||||
}
|
|
||||||
summaries.append({
|
|
||||||
'cluster': cluster,
|
|
||||||
'paths': len(pairs),
|
|
||||||
'metrics': metrics,
|
|
||||||
})
|
|
||||||
return summaries
|
|
||||||
|
|
||||||
|
|
||||||
def aggregate_results(
|
|
||||||
cells: list[dict[str, Any]],
|
|
||||||
*,
|
|
||||||
arms: tuple[dict[str, Any], ...] = ARMS,
|
|
||||||
protocols: tuple[str, ...] = ('empty_book', 'warm_book'),
|
|
||||||
costs: tuple[float, ...] = COSTS_PER_SIDE_PCT,
|
|
||||||
include_warm_dispersion: bool = True,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
paired: list[dict[str, Any]] = []
|
|
||||||
path_distributions: list[dict[str, Any]] = []
|
|
||||||
for cost in costs:
|
|
||||||
for protocol in protocols:
|
|
||||||
control_by_path = {
|
|
||||||
row['path_id']: row
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == 'cap10_incumbent'
|
|
||||||
and row['protocol'] == protocol
|
|
||||||
and float(row['cost_per_side_pct']) == float(cost)
|
|
||||||
}
|
|
||||||
for arm in arms:
|
|
||||||
arm_id = str(arm['id'])
|
|
||||||
clusters = _cluster_rows(
|
|
||||||
cells,
|
|
||||||
arm_id=arm_id,
|
|
||||||
protocol=protocol,
|
|
||||||
cost=float(cost),
|
|
||||||
)
|
|
||||||
headline: dict[str, Any] = {}
|
|
||||||
for metric in PAIRED_METRICS:
|
|
||||||
deltas = [
|
|
||||||
cluster['metrics'][metric]['paired_delta_median']
|
|
||||||
for cluster in clusters
|
|
||||||
]
|
|
||||||
arm_levels = [
|
|
||||||
cluster['metrics'][metric]['arm_median']
|
|
||||||
for cluster in clusters
|
|
||||||
]
|
|
||||||
control_levels = [
|
|
||||||
cluster['metrics'][metric]['control_median']
|
|
||||||
for cluster in clusters
|
|
||||||
]
|
|
||||||
arm_level = median(arm_levels)
|
|
||||||
control_level = median(control_levels)
|
|
||||||
metric_summary: dict[str, Any] = {
|
|
||||||
'paired_delta_median': median(deltas),
|
|
||||||
'arm_median': arm_level,
|
|
||||||
'control_median': control_level,
|
|
||||||
'arm_control_ratio': _safe_ratio(
|
|
||||||
arm_level, control_level
|
|
||||||
),
|
|
||||||
}
|
|
||||||
if metric in ('ev_net_r', 'calmar'):
|
|
||||||
metric_summary['bootstrap_90'] = (
|
|
||||||
bootstrap_median_interval(
|
|
||||||
deltas,
|
|
||||||
seed_parts=(
|
|
||||||
arm_id,
|
|
||||||
protocol,
|
|
||||||
cost,
|
|
||||||
metric,
|
|
||||||
'paired-delta',
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
headline[metric] = metric_summary
|
|
||||||
paired.append({
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'protocol': protocol,
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
'clusters': clusters,
|
|
||||||
'headline': headline,
|
|
||||||
})
|
|
||||||
treatment_by_path = {
|
|
||||||
row['path_id']: row
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == arm_id
|
|
||||||
and row['protocol'] == protocol
|
|
||||||
and float(row['cost_per_side_pct']) == float(cost)
|
|
||||||
}
|
|
||||||
shared_paths = sorted(
|
|
||||||
set(treatment_by_path) & set(control_by_path)
|
|
||||||
)
|
|
||||||
path_metrics: dict[str, Any] = {}
|
|
||||||
for metric in PAIRED_METRICS:
|
|
||||||
deltas = [
|
|
||||||
float(treatment_by_path[path_id]['metrics'][metric])
|
|
||||||
- float(control_by_path[path_id]['metrics'][metric])
|
|
||||||
for path_id in shared_paths
|
|
||||||
if treatment_by_path[path_id]['metrics'].get(metric)
|
|
||||||
is not None
|
|
||||||
and control_by_path[path_id]['metrics'].get(metric)
|
|
||||||
is not None
|
|
||||||
and math.isfinite(
|
|
||||||
float(treatment_by_path[path_id]['metrics'][metric])
|
|
||||||
)
|
|
||||||
and math.isfinite(
|
|
||||||
float(control_by_path[path_id]['metrics'][metric])
|
|
||||||
)
|
|
||||||
]
|
|
||||||
path_metrics[metric] = {
|
|
||||||
'paired_paths': len(deltas),
|
|
||||||
'paired_delta_mean': (
|
|
||||||
statistics.fmean(deltas) if deltas else None
|
|
||||||
),
|
|
||||||
'paired_delta_median': median(deltas),
|
|
||||||
'paired_delta_p25': percentile(deltas, 0.25),
|
|
||||||
'paired_delta_p75': percentile(deltas, 0.75),
|
|
||||||
'positive_fraction': (
|
|
||||||
sum(delta > 0.0 for delta in deltas) / len(deltas)
|
|
||||||
if deltas
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
'identical_fraction': (
|
|
||||||
sum(abs(delta) <= 1e-12 for delta in deltas)
|
|
||||||
/ len(deltas)
|
|
||||||
if deltas
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
}
|
|
||||||
path_distributions.append({
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'protocol': protocol,
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
'metrics': path_metrics,
|
|
||||||
})
|
|
||||||
|
|
||||||
warm_rows = [
|
|
||||||
row for row in cells if row['protocol'] == 'warm_book'
|
|
||||||
]
|
|
||||||
warm_dispersion: list[dict[str, Any]] = []
|
|
||||||
for cost in costs:
|
|
||||||
for arm in arms:
|
|
||||||
arm_id = str(arm['id'])
|
|
||||||
anchor_rows: list[dict[str, Any]] = []
|
|
||||||
for cluster in ANCHOR_YEARS:
|
|
||||||
arm_paths = [
|
|
||||||
row
|
|
||||||
for row in warm_rows
|
|
||||||
if row['arm_id'] == arm_id
|
|
||||||
and int(row['cluster']) == cluster
|
|
||||||
and float(row['cost_per_side_pct']) == float(cost)
|
|
||||||
]
|
|
||||||
control_by_path = {
|
|
||||||
row['path_id']: row
|
|
||||||
for row in warm_rows
|
|
||||||
if row['arm_id'] == 'cap10_incumbent'
|
|
||||||
and int(row['cluster']) == cluster
|
|
||||||
and float(row['cost_per_side_pct']) == float(cost)
|
|
||||||
}
|
|
||||||
metric_rows: dict[str, Any] = {}
|
|
||||||
for metric in ('ev_net_r', 'calmar'):
|
|
||||||
arm_spread = iqr(
|
|
||||||
row['metrics'].get(metric) for row in arm_paths
|
|
||||||
)
|
|
||||||
control_spread = iqr(
|
|
||||||
control_by_path[row['path_id']]['metrics'].get(metric)
|
|
||||||
for row in arm_paths
|
|
||||||
if row['path_id'] in control_by_path
|
|
||||||
)
|
|
||||||
metric_rows[metric] = {
|
|
||||||
'arm_iqr': arm_spread,
|
|
||||||
'control_iqr': control_spread,
|
|
||||||
'iqr_ratio': _safe_ratio(
|
|
||||||
arm_spread, control_spread
|
|
||||||
),
|
|
||||||
}
|
|
||||||
anchor_rows.append({
|
|
||||||
'cluster': cluster,
|
|
||||||
'seeds': len(arm_paths),
|
|
||||||
'metrics': metric_rows,
|
|
||||||
})
|
|
||||||
|
|
||||||
headline: dict[str, Any] = {}
|
|
||||||
for metric in ('ev_net_r', 'calmar'):
|
|
||||||
ratios = [
|
|
||||||
row['metrics'][metric]['iqr_ratio']
|
|
||||||
for row in anchor_rows
|
|
||||||
]
|
|
||||||
headline[metric] = {
|
|
||||||
'median_iqr_ratio': median(ratios),
|
|
||||||
'bootstrap_90': bootstrap_median_interval(
|
|
||||||
ratios,
|
|
||||||
seed_parts=(
|
|
||||||
arm_id,
|
|
||||||
cost,
|
|
||||||
metric,
|
|
||||||
'warm-iqr-ratio',
|
|
||||||
),
|
|
||||||
),
|
|
||||||
}
|
|
||||||
warm_dispersion.append({
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
'anchors': anchor_rows,
|
|
||||||
'headline': headline,
|
|
||||||
})
|
|
||||||
|
|
||||||
if not include_warm_dispersion:
|
|
||||||
warm_dispersion = []
|
|
||||||
|
|
||||||
return {
|
|
||||||
'paired_per_year': paired,
|
|
||||||
'paired_path_distributions': path_distributions,
|
|
||||||
'warm_seed_dispersion': warm_dispersion,
|
|
||||||
'bootstrap': {
|
|
||||||
'replicates': BOOTSTRAP_REPLICATES,
|
|
||||||
'seed': BOOTSTRAP_SEED,
|
|
||||||
'interval': 'central 90% percentile, context only',
|
|
||||||
'resampling_unit': 'seven annual paired summaries',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -15,10 +15,10 @@ Cost: a reparse cannot be served from the database -- the facts a fixed parser n
|
|||||||
accepts were never stored -- so it refetches Company Facts for every tracked issuer
|
accepts were never stored -- so it refetches Company Facts for every tracked issuer
|
||||||
under the SEC fair-access throttle. Expect a long run and a lot of network.
|
under the SEC fair-access throttle. Expect a long run and a lot of network.
|
||||||
|
|
||||||
Scope note: this rewrites ``fundamental_snapshots`` only. As of the A5 gate those
|
Scope note: this rewrites ``fundamental_snapshots`` only. Those rows now feed both
|
||||||
rows feed the fundamentals API/UI and the parity report; scoring still reads the
|
the fundamentals API/UI *and* — through the nightly ``fundamental_data`` refresh —
|
||||||
legacy ``fundamental_data`` table, so a reparse does not move composite scores or
|
the fundamental dimension of the composite score, so a reparse does move scores
|
||||||
backtests until the cutover happens.
|
and backtests. Run it deliberately.
|
||||||
|
|
||||||
Examples
|
Examples
|
||||||
--------
|
--------
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,6 @@
|
|||||||
#
|
#
|
||||||
# Kept after Tier-1 cleanup:
|
# Kept after Tier-1 cleanup:
|
||||||
# --ssl-check diagnose corporate CA / proxy
|
# --ssl-check diagnose corporate CA / proxy
|
||||||
# --earnings-only resume FMP earnings backfill + 2a/2b (parked)
|
|
||||||
# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix
|
# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix
|
||||||
#
|
#
|
||||||
# Prerequisites: git checkout research branch, .env, deep research.sqlite for
|
# Prerequisites: git checkout research branch, .env, deep research.sqlite for
|
||||||
@@ -21,8 +20,6 @@ cd "$ROOT"
|
|||||||
RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}"
|
RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}"
|
||||||
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
|
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
|
||||||
WORKERS="${WORKERS:-8}"
|
WORKERS="${WORKERS:-8}"
|
||||||
FMP_LIMIT="${FMP_LIMIT:-250}"
|
|
||||||
FMP_SLEEP="${FMP_SLEEP:-0.35}"
|
|
||||||
PYTHON="${PYTHON:-python3}"
|
PYTHON="${PYTHON:-python3}"
|
||||||
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"
|
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"
|
||||||
PHASE=""
|
PHASE=""
|
||||||
@@ -35,7 +32,6 @@ usage() {
|
|||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--ssl-check) PHASE=ssl; shift ;;
|
--ssl-check) PHASE=ssl; shift ;;
|
||||||
--earnings-only) PHASE=earnings; shift ;;
|
|
||||||
--prod-book-matrix) PHASE=prod_book; shift ;;
|
--prod-book-matrix) PHASE=prod_book; shift ;;
|
||||||
--corp-proxy) USE_CORP_PROXY=1; shift ;;
|
--corp-proxy) USE_CORP_PROXY=1; shift ;;
|
||||||
--workers) WORKERS="$2"; shift 2 ;;
|
--workers) WORKERS="$2"; shift 2 ;;
|
||||||
@@ -46,7 +42,7 @@ while [[ $# -gt 0 ]]; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
if [[ -z "$PHASE" ]]; then
|
if [[ -z "$PHASE" ]]; then
|
||||||
echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2
|
echo "Pick a phase: --ssl-check | --prod-book-matrix" >&2
|
||||||
usage 1
|
usage 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -104,7 +100,6 @@ print(json.dumps(ssl_status(), indent=2))
|
|||||||
print("bootstrap ->", bootstrap_ssl())
|
print("bootstrap ->", bootstrap_ssl())
|
||||||
for url in (
|
for url in (
|
||||||
"https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
|
"https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
|
||||||
"https://financialmodelingprep.com/stable/profile?symbol=AAPL",
|
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"})
|
req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"})
|
||||||
@@ -118,17 +113,6 @@ PY
|
|||||||
setup_ssl
|
setup_ssl
|
||||||
case "$PHASE" in
|
case "$PHASE" in
|
||||||
ssl) ssl_check ;;
|
ssl) ssl_check ;;
|
||||||
earnings)
|
|
||||||
need_file "$PROD_SNAP"
|
|
||||||
need_file "$RESEARCH_SNAP"
|
|
||||||
log "Earnings Task 2 bulk backfill + registered 2a/2b closeout"
|
|
||||||
"$PYTHON" scripts/backfill_earnings_events.py \
|
|
||||||
--snapshot "$PROD_SNAP" --from-date 2016-01-04 --window-days 30 \
|
|
||||||
--limit "$FMP_LIMIT" --sleep "$FMP_SLEEP"
|
|
||||||
"$PYTHON" scripts/run_earnings_research.py \
|
|
||||||
--snapshot "$RESEARCH_SNAP" --universe-snapshot "$PROD_SNAP" \
|
|
||||||
--earnings-snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn
|
|
||||||
;;
|
|
||||||
prod_book)
|
prod_book)
|
||||||
need_file "$RESEARCH_SNAP"
|
need_file "$RESEARCH_SNAP"
|
||||||
log "Production book universe × horizon matrix"
|
log "Production book universe × horizon matrix"
|
||||||
|
|||||||
@@ -8,9 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from app.exceptions import ValidationError
|
from app.exceptions import ValidationError
|
||||||
from app.services.admin_service import (
|
from app.services.admin_service import (
|
||||||
get_activation_config,
|
get_activation_config,
|
||||||
get_fundamentals_cutover_config,
|
|
||||||
update_activation_config,
|
update_activation_config,
|
||||||
update_fundamentals_cutover_config,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -78,18 +76,3 @@ class TestActivationConfig:
|
|||||||
async def test_rejects_out_of_range_confidence(self, session: AsyncSession):
|
async def test_rejects_out_of_range_confidence(self, session: AsyncSession):
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
await update_activation_config(session, {"min_confidence": 120.0})
|
await update_activation_config(session, {"min_confidence": 120.0})
|
||||||
|
|
||||||
|
|
||||||
class TestFundamentalsCutoverConfig:
|
|
||||||
async def test_defaults_off_when_unset(self, session: AsyncSession):
|
|
||||||
assert await get_fundamentals_cutover_config(session) == {"enabled": False}
|
|
||||||
|
|
||||||
async def test_round_trips_explicit_switch(self, session: AsyncSession):
|
|
||||||
assert await update_fundamentals_cutover_config(session, True) == {
|
|
||||||
"enabled": True
|
|
||||||
}
|
|
||||||
assert await get_fundamentals_cutover_config(session) == {"enabled": True}
|
|
||||||
|
|
||||||
assert await update_fundamentals_cutover_config(session, False) == {
|
|
||||||
"enabled": False
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
|
||||||
from scripts.backfill_earnings_events import _dedupe_bulk_rows, _windows
|
|
||||||
from scripts.import_dolthub_earnings import _align_symbol
|
from scripts.import_dolthub_earnings import _align_symbol
|
||||||
from scripts.run_earnings_research import (
|
from scripts.run_earnings_research import (
|
||||||
_analyse_2a_trades,
|
_analyse_2a_trades,
|
||||||
@@ -41,42 +40,6 @@ def test_dolthub_alignment_allows_fiscal_period_label_after_announcement() -> No
|
|||||||
assert matches == [(0, 0), (1, 1)]
|
assert matches == [(0, 0), (1, 1)]
|
||||||
|
|
||||||
|
|
||||||
def test_bulk_windows_cover_range_without_overlap() -> None:
|
|
||||||
result = _windows(date(2020, 1, 1), date(2020, 1, 10), 4)
|
|
||||||
assert result == [
|
|
||||||
(date(2020, 1, 1), date(2020, 1, 4)),
|
|
||||||
(date(2020, 1, 5), date(2020, 1, 8)),
|
|
||||||
(date(2020, 1, 9), date(2020, 1, 10)),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_bulk_dedupe_prefers_more_complete_and_counts_restatement() -> None:
|
|
||||||
rows = [
|
|
||||||
{
|
|
||||||
"symbol": "AAPL",
|
|
||||||
"announce_date": "2024-01-01",
|
|
||||||
"announce_time": None,
|
|
||||||
"eps_estimate": 1.0,
|
|
||||||
"eps_actual": 1.1,
|
|
||||||
"revenue_estimate": None,
|
|
||||||
"revenue_actual": None,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"symbol": "AAPL",
|
|
||||||
"announce_date": "2024-01-01",
|
|
||||||
"announce_time": "amc",
|
|
||||||
"eps_estimate": 1.0,
|
|
||||||
"eps_actual": 1.2,
|
|
||||||
"revenue_estimate": 10.0,
|
|
||||||
"revenue_actual": 11.0,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
deduped, duplicates, restated = _dedupe_bulk_rows(rows)
|
|
||||||
assert duplicates == 1
|
|
||||||
assert restated == 1
|
|
||||||
assert deduped == [rows[1]]
|
|
||||||
|
|
||||||
|
|
||||||
def test_2a_uses_net_r_strict_hold_and_next_session_stop() -> None:
|
def test_2a_uses_net_r_strict_hold_and_next_session_stop() -> None:
|
||||||
calendar = [
|
calendar = [
|
||||||
date(2024, 1, 2),
|
date(2024, 1, 2),
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
"""Unit tests for FinnhubFundamentalProvider unit conversions."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from unittest.mock import AsyncMock, patch
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.providers.fundamentals_chain import FinnhubFundamentalProvider
|
|
||||||
|
|
||||||
|
|
||||||
def _mock_response(status_code: int, json_data: object = None) -> httpx.Response:
|
|
||||||
return httpx.Response(
|
|
||||||
status_code=status_code,
|
|
||||||
json=json_data if json_data is not None else {},
|
|
||||||
request=httpx.Request("GET", "https://example.com"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def provider() -> FinnhubFundamentalProvider:
|
|
||||||
return FinnhubFundamentalProvider(api_key="test-key")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_finnhub_market_cap_converted_from_millions_to_dollars(provider):
|
|
||||||
"""Finnhub marketCapitalization is in millions — store absolute USD.
|
|
||||||
|
|
||||||
SPCX-scale example: ~$1.8T → Finnhub reports 1_800_000 (millions).
|
|
||||||
Without conversion the UI showed 1.8M / micro cap.
|
|
||||||
"""
|
|
||||||
profile = {"marketCapitalization": 1_800_000} # millions → $1.8T
|
|
||||||
metrics = {"metric": {"peTTM": 40.0, "revenueGrowthTTMYoy": 25.0}}
|
|
||||||
earnings = [{"surprisePercent": 2.5}]
|
|
||||||
calendar = {"earningsCalendar": []}
|
|
||||||
|
|
||||||
async def mock_get(url, params=None):
|
|
||||||
if "profile2" in url:
|
|
||||||
return _mock_response(200, profile)
|
|
||||||
if "stock/metric" in url:
|
|
||||||
return _mock_response(200, metrics)
|
|
||||||
if "stock/earnings" in url:
|
|
||||||
return _mock_response(200, earnings)
|
|
||||||
if "calendar/earnings" in url:
|
|
||||||
return _mock_response(200, calendar)
|
|
||||||
return _mock_response(200, {})
|
|
||||||
|
|
||||||
with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
|
|
||||||
instance = AsyncMock()
|
|
||||||
instance.get.side_effect = mock_get
|
|
||||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
|
||||||
instance.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
MockClient.return_value = instance
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("SPCX")
|
|
||||||
|
|
||||||
assert result.market_cap == 1_800_000 * 1_000_000 # $1.8T
|
|
||||||
assert result.pe_ratio == 40.0
|
|
||||||
assert result.revenue_growth == 25.0
|
|
||||||
assert result.earnings_surprise == 2.5
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_finnhub_market_cap_none_when_missing(provider):
|
|
||||||
profile: dict = {}
|
|
||||||
metrics = {"metric": {}}
|
|
||||||
earnings: list = []
|
|
||||||
calendar = {"earningsCalendar": []}
|
|
||||||
|
|
||||||
async def mock_get(url, params=None):
|
|
||||||
if "profile2" in url:
|
|
||||||
return _mock_response(200, profile)
|
|
||||||
if "stock/metric" in url:
|
|
||||||
return _mock_response(200, metrics)
|
|
||||||
if "stock/earnings" in url:
|
|
||||||
return _mock_response(200, earnings)
|
|
||||||
if "calendar/earnings" in url:
|
|
||||||
return _mock_response(200, calendar)
|
|
||||||
return _mock_response(200, {})
|
|
||||||
|
|
||||||
with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
|
|
||||||
instance = AsyncMock()
|
|
||||||
instance.get.side_effect = mock_get
|
|
||||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
|
||||||
instance.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
MockClient.return_value = instance
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("XYZ")
|
|
||||||
|
|
||||||
assert result.market_cap is None
|
|
||||||
assert "market_cap" in result.unavailable_fields
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
"""Unit tests for FMPFundamentalProvider 402 reason recording."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from unittest.mock import AsyncMock, patch
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.providers.fmp import FMPFundamentalProvider
|
|
||||||
|
|
||||||
|
|
||||||
def _mock_response(status_code: int, json_data: object = None) -> httpx.Response:
|
|
||||||
"""Build a fake httpx.Response."""
|
|
||||||
resp = httpx.Response(
|
|
||||||
status_code=status_code,
|
|
||||||
json=json_data if json_data is not None else {},
|
|
||||||
request=httpx.Request("GET", "https://example.com"),
|
|
||||||
)
|
|
||||||
return resp
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def provider() -> FMPFundamentalProvider:
|
|
||||||
return FMPFundamentalProvider(api_key="test-key")
|
|
||||||
|
|
||||||
|
|
||||||
class TestFetchJsonOptional402Tracking:
|
|
||||||
"""_fetch_json_optional returns (data, was_402) tuple."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_returns_empty_dict_and_true_on_402(self, provider):
|
|
||||||
mock_client = AsyncMock()
|
|
||||||
mock_client.get.return_value = _mock_response(402)
|
|
||||||
|
|
||||||
data, was_402 = await provider._fetch_json_optional(
|
|
||||||
mock_client, "ratios-ttm", {}, "AAPL"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert data == {}
|
|
||||||
assert was_402 is True
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_returns_data_and_false_on_200(self, provider):
|
|
||||||
mock_client = AsyncMock()
|
|
||||||
mock_client.get.return_value = _mock_response(
|
|
||||||
200, [{"priceToEarningsRatioTTM": 25.5}]
|
|
||||||
)
|
|
||||||
|
|
||||||
data, was_402 = await provider._fetch_json_optional(
|
|
||||||
mock_client, "ratios-ttm", {}, "AAPL"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert data == {"priceToEarningsRatioTTM": 25.5}
|
|
||||||
assert was_402 is False
|
|
||||||
|
|
||||||
|
|
||||||
class TestFetchFundamentals402Recording:
|
|
||||||
"""fetch_fundamentals records 402 endpoints in unavailable_fields."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_all_402_records_all_fields(self, provider):
|
|
||||||
"""When all supplementary endpoints return 402, all three fields are recorded."""
|
|
||||||
profile_resp = _mock_response(200, [{"marketCap": 1_000_000}])
|
|
||||||
ratios_resp = _mock_response(402)
|
|
||||||
growth_resp = _mock_response(402)
|
|
||||||
earnings_resp = _mock_response(402)
|
|
||||||
|
|
||||||
async def mock_get(url, params=None):
|
|
||||||
if "profile" in url:
|
|
||||||
return profile_resp
|
|
||||||
if "ratios-ttm" in url:
|
|
||||||
return ratios_resp
|
|
||||||
if "financial-growth" in url:
|
|
||||||
return growth_resp
|
|
||||||
if "earnings" in url:
|
|
||||||
return earnings_resp
|
|
||||||
return _mock_response(200, [{}])
|
|
||||||
|
|
||||||
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
|
|
||||||
instance = AsyncMock()
|
|
||||||
instance.get.side_effect = mock_get
|
|
||||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
|
||||||
instance.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
MockClient.return_value = instance
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
assert result.unavailable_fields == {
|
|
||||||
"pe_ratio": "requires paid plan",
|
|
||||||
"revenue_growth": "requires paid plan",
|
|
||||||
"earnings_surprise": "requires paid plan",
|
|
||||||
}
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_mixed_200_402_records_only_402_fields(self, provider):
|
|
||||||
"""When only ratios-ttm returns 402, only pe_ratio is recorded."""
|
|
||||||
profile_resp = _mock_response(200, [{"marketCap": 2_000_000}])
|
|
||||||
ratios_resp = _mock_response(402)
|
|
||||||
growth_resp = _mock_response(200, [{"revenueGrowth": 0.15}])
|
|
||||||
earnings_resp = _mock_response(200, [{"epsActual": 3.0, "epsEstimated": 2.5}])
|
|
||||||
|
|
||||||
async def mock_get(url, params=None):
|
|
||||||
if "profile" in url:
|
|
||||||
return profile_resp
|
|
||||||
if "ratios-ttm" in url:
|
|
||||||
return ratios_resp
|
|
||||||
if "financial-growth" in url:
|
|
||||||
return growth_resp
|
|
||||||
if "earnings" in url:
|
|
||||||
return earnings_resp
|
|
||||||
return _mock_response(200, [{}])
|
|
||||||
|
|
||||||
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
|
|
||||||
instance = AsyncMock()
|
|
||||||
instance.get.side_effect = mock_get
|
|
||||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
|
||||||
instance.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
MockClient.return_value = instance
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
assert result.unavailable_fields == {"pe_ratio": "requires paid plan"}
|
|
||||||
assert result.revenue_growth == 0.15
|
|
||||||
assert result.earnings_surprise is not None
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_no_402_empty_unavailable_fields(self, provider):
|
|
||||||
"""When all endpoints succeed, unavailable_fields is empty."""
|
|
||||||
profile_resp = _mock_response(200, [{"marketCap": 3_000_000}])
|
|
||||||
ratios_resp = _mock_response(200, [{"priceToEarningsRatioTTM": 20.0}])
|
|
||||||
growth_resp = _mock_response(200, [{"revenueGrowth": 0.10}])
|
|
||||||
earnings_resp = _mock_response(200, [{"epsActual": 2.0, "epsEstimated": 1.8}])
|
|
||||||
|
|
||||||
async def mock_get(url, params=None):
|
|
||||||
if "profile" in url:
|
|
||||||
return profile_resp
|
|
||||||
if "ratios-ttm" in url:
|
|
||||||
return ratios_resp
|
|
||||||
if "financial-growth" in url:
|
|
||||||
return growth_resp
|
|
||||||
if "earnings" in url:
|
|
||||||
return earnings_resp
|
|
||||||
return _mock_response(200, [{}])
|
|
||||||
|
|
||||||
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
|
|
||||||
instance = AsyncMock()
|
|
||||||
instance.get.side_effect = mock_get
|
|
||||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
|
||||||
instance.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
MockClient.return_value = instance
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
assert result.unavailable_fields == {}
|
|
||||||
assert result.pe_ratio == 20.0
|
|
||||||
@@ -15,7 +15,6 @@ from app.models.fundamental import FundamentalData
|
|||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
from app.models.ohlcv import OHLCVRecord
|
from app.models.ohlcv import OHLCVRecord
|
||||||
from app.models.score import CompositeScore, DimensionScore
|
from app.models.score import CompositeScore, DimensionScore
|
||||||
from app.models.settings import SystemSetting
|
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.services import fundamentals_candidate_service as candidates
|
from app.services import fundamentals_candidate_service as candidates
|
||||||
from app.services import fundamentals_derivation as deriv
|
from app.services import fundamentals_derivation as deriv
|
||||||
@@ -76,49 +75,9 @@ def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]:
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
async def test_default_off_performs_no_candidate_read_or_write(
|
async def test_refresh_updates_all_fields_and_invalidates_scores(
|
||||||
session: AsyncSession, monkeypatch
|
|
||||||
):
|
|
||||||
ticker = Ticker(symbol="AAA")
|
|
||||||
session.add(ticker)
|
|
||||||
await session.flush()
|
|
||||||
session.add(
|
|
||||||
FundamentalData(
|
|
||||||
ticker_id=ticker.id,
|
|
||||||
pe_ratio=12,
|
|
||||||
revenue_growth=3,
|
|
||||||
earnings_surprise=1,
|
|
||||||
market_cap=100,
|
|
||||||
fetched_at=NOW,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await session.commit()
|
|
||||||
|
|
||||||
async def should_not_read(*args, **kwargs):
|
|
||||||
raise AssertionError("default-off refresh derived candidates")
|
|
||||||
|
|
||||||
monkeypatch.setattr(candidates, "build_candidates", should_not_read)
|
|
||||||
summary = await refresh_service.refresh_if_enabled(session, today=TODAY)
|
|
||||||
|
|
||||||
stored = await session.scalar(
|
|
||||||
select(FundamentalData).where(FundamentalData.ticker_id == ticker.id)
|
|
||||||
)
|
|
||||||
assert summary == {
|
|
||||||
"enabled": False,
|
|
||||||
"refreshed": 0,
|
|
||||||
"score_inputs_changed": 0,
|
|
||||||
"dimension_scores_staled": 0,
|
|
||||||
"composite_scores_staled": 0,
|
|
||||||
}
|
|
||||||
assert stored.pe_ratio == 12
|
|
||||||
|
|
||||||
|
|
||||||
async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
|
|
||||||
session: AsyncSession,
|
session: AsyncSession,
|
||||||
):
|
):
|
||||||
session.add(
|
|
||||||
SystemSetting(key=refresh_service.ACTIVATION_KEY, value="true")
|
|
||||||
)
|
|
||||||
first = Ticker(symbol="AAA", cik="0000000001")
|
first = Ticker(symbol="AAA", cik="0000000001")
|
||||||
second = Ticker(symbol="AAB", cik="0000000001")
|
second = Ticker(symbol="AAB", cik="0000000001")
|
||||||
session.add_all([first, second])
|
session.add_all([first, second])
|
||||||
@@ -191,7 +150,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
|
|||||||
)
|
)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
summary = await refresh_service.refresh_if_enabled(
|
summary = await refresh_service.refresh(
|
||||||
session, now=NOW, today=TODAY
|
session, now=NOW, today=TODAY
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -225,7 +184,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
|
|||||||
for row in (*dimensions, *composites):
|
for row in (*dimensions, *composites):
|
||||||
row.is_stale = False
|
row.is_stale = False
|
||||||
await session.commit()
|
await session.commit()
|
||||||
unchanged = await refresh_service.refresh_if_enabled(
|
unchanged = await refresh_service.refresh(
|
||||||
session, now=NOW + timedelta(hours=1), today=TODAY
|
session, now=NOW + timedelta(hours=1), today=TODAY
|
||||||
)
|
)
|
||||||
assert unchanged["score_inputs_changed"] == 0
|
assert unchanged["score_inputs_changed"] == 0
|
||||||
|
|||||||
@@ -1,13 +1,20 @@
|
|||||||
"""Unit tests for fundamental_service — unavailable_fields persistence."""
|
"""Unit tests for fundamental_service — the surviving read path.
|
||||||
|
|
||||||
|
Writes to ``fundamental_data`` are covered by test_fundamental_data_refresh.py;
|
||||||
|
this file only guards the lookup used by the router and scoring.
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
from app.database import Base
|
from app.database import Base
|
||||||
|
from app.exceptions import NotFoundError
|
||||||
|
from app.models.fundamental import FundamentalData
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.services import fundamental_service
|
from app.services import fundamental_service
|
||||||
|
|
||||||
@@ -43,57 +50,36 @@ async def ticker(session: AsyncSession) -> Ticker:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_store_fundamental_persists_unavailable_fields(
|
async def test_get_fundamental_returns_the_cached_row(
|
||||||
session: AsyncSession, ticker: Ticker
|
session: AsyncSession, ticker: Ticker
|
||||||
):
|
):
|
||||||
"""unavailable_fields dict is serialized to JSON and stored."""
|
fields = {"pe_ratio": "split guard applied"}
|
||||||
fields = {"pe_ratio": "requires paid plan", "revenue_growth": "requires paid plan"}
|
session.add(
|
||||||
|
FundamentalData(
|
||||||
record = await fundamental_service.store_fundamental(
|
ticker_id=ticker.id,
|
||||||
session,
|
pe_ratio=None,
|
||||||
symbol="AAPL",
|
market_cap=1_000_000.0,
|
||||||
pe_ratio=None,
|
fetched_at=datetime.now(timezone.utc),
|
||||||
revenue_growth=None,
|
unavailable_fields_json=json.dumps(fields),
|
||||||
market_cap=1_000_000.0,
|
)
|
||||||
unavailable_fields=fields,
|
|
||||||
)
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
record = await fundamental_service.get_fundamental(session, symbol="aapl")
|
||||||
|
|
||||||
|
assert record is not None
|
||||||
|
assert record.market_cap == 1_000_000.0
|
||||||
assert json.loads(record.unavailable_fields_json) == fields
|
assert json.loads(record.unavailable_fields_json) == fields
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_store_fundamental_defaults_to_empty_dict(
|
async def test_get_fundamental_returns_none_without_a_cached_row(
|
||||||
session: AsyncSession, ticker: Ticker
|
session: AsyncSession, ticker: Ticker
|
||||||
):
|
):
|
||||||
"""When unavailable_fields is not provided, column defaults to '{}'."""
|
assert await fundamental_service.get_fundamental(session, symbol="AAPL") is None
|
||||||
record = await fundamental_service.store_fundamental(
|
|
||||||
session,
|
|
||||||
symbol="AAPL",
|
|
||||||
pe_ratio=25.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert json.loads(record.unavailable_fields_json) == {}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_store_fundamental_updates_unavailable_fields(
|
async def test_get_fundamental_rejects_an_unknown_symbol(session: AsyncSession):
|
||||||
session: AsyncSession, ticker: Ticker
|
with pytest.raises(NotFoundError):
|
||||||
):
|
await fundamental_service.get_fundamental(session, symbol="NOPE")
|
||||||
"""Updating an existing record also updates unavailable_fields_json."""
|
|
||||||
# First store
|
|
||||||
await fundamental_service.store_fundamental(
|
|
||||||
session,
|
|
||||||
symbol="AAPL",
|
|
||||||
pe_ratio=None,
|
|
||||||
unavailable_fields={"pe_ratio": "requires paid plan"},
|
|
||||||
)
|
|
||||||
|
|
||||||
# Second store — fields now available
|
|
||||||
record = await fundamental_service.store_fundamental(
|
|
||||||
session,
|
|
||||||
symbol="AAPL",
|
|
||||||
pe_ratio=25.0,
|
|
||||||
unavailable_fields={},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert json.loads(record.unavailable_fields_json) == {}
|
|
||||||
|
|||||||
@@ -1,181 +0,0 @@
|
|||||||
"""Unit tests for chained fundamentals provider fallback behavior."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.exceptions import ProviderError, RateLimitError
|
|
||||||
from app.providers.fundamentals_chain import ChainedFundamentalProvider
|
|
||||||
from app.providers.protocol import FundamentalData
|
|
||||||
|
|
||||||
|
|
||||||
class _FailProvider:
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
self._message = message
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
raise ProviderError(f"{self._message} ({ticker})")
|
|
||||||
|
|
||||||
|
|
||||||
class _RateLimitedProvider:
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
raise RateLimitError(f"rate limit hit for {ticker}")
|
|
||||||
|
|
||||||
|
|
||||||
class _DataProvider:
|
|
||||||
def __init__(self, data: FundamentalData) -> None:
|
|
||||||
self._data = data
|
|
||||||
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker,
|
|
||||||
pe_ratio=self._data.pe_ratio,
|
|
||||||
revenue_growth=self._data.revenue_growth,
|
|
||||||
earnings_surprise=self._data.earnings_surprise,
|
|
||||||
market_cap=self._data.market_cap,
|
|
||||||
fetched_at=self._data.fetched_at,
|
|
||||||
unavailable_fields=self._data.unavailable_fields,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_chained_provider_uses_fallback_provider_on_primary_failure():
|
|
||||||
fallback_data = FundamentalData(
|
|
||||||
ticker="AAPL",
|
|
||||||
pe_ratio=25.0,
|
|
||||||
revenue_growth=None,
|
|
||||||
earnings_surprise=None,
|
|
||||||
market_cap=1_000_000.0,
|
|
||||||
fetched_at=datetime.now(timezone.utc),
|
|
||||||
unavailable_fields={},
|
|
||||||
)
|
|
||||||
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("primary", _FailProvider("primary down")),
|
|
||||||
("fallback", _DataProvider(fallback_data)),
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
assert result.pe_ratio == 25.0
|
|
||||||
assert result.market_cap == 1_000_000.0
|
|
||||||
assert result.unavailable_fields.get("source_pe_ratio") == "fallback"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_chained_provider_merges_fields_across_providers():
|
|
||||||
"""Primary supplies only market cap; fallback fills P/E and earnings."""
|
|
||||||
primary_data = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
|
||||||
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
|
||||||
)
|
|
||||||
fallback_data = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=18.0, revenue_growth=12.0, earnings_surprise=4.0,
|
|
||||||
market_cap=999.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
|
||||||
)
|
|
||||||
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("fmp", _DataProvider(primary_data)),
|
|
||||||
("finnhub", _DataProvider(fallback_data)),
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
# market cap from primary (first to supply it), the rest from fallback
|
|
||||||
assert result.market_cap == 2_000_000.0
|
|
||||||
assert result.pe_ratio == 18.0
|
|
||||||
assert result.revenue_growth == 12.0
|
|
||||||
assert result.earnings_surprise == 4.0
|
|
||||||
assert result.unavailable_fields.get("source_market_cap") == "fmp"
|
|
||||||
assert result.unavailable_fields.get("source_pe_ratio") == "finnhub"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_chained_provider_raises_when_all_providers_fail():
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("p1", _FailProvider("p1 failed")),
|
|
||||||
("p2", _FailProvider("p2 failed")),
|
|
||||||
])
|
|
||||||
|
|
||||||
with pytest.raises(ProviderError) as exc:
|
|
||||||
await provider.fetch_fundamentals("MSFT")
|
|
||||||
|
|
||||||
assert "All fundamentals providers failed" in str(exc.value)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rate_limited_fallback_raises_when_incomplete():
|
|
||||||
"""FMP gives market cap; the fallback is rate-limited → chain signals it so
|
|
||||||
the collector can back off instead of storing a degraded record."""
|
|
||||||
primary_data = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
|
||||||
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
|
||||||
)
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("fmp", _DataProvider(primary_data)),
|
|
||||||
("finnhub", _RateLimitedProvider()),
|
|
||||||
])
|
|
||||||
|
|
||||||
with pytest.raises(RateLimitError):
|
|
||||||
await provider.fetch_fundamentals("AAPL")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rate_limited_fallback_allows_partial():
|
|
||||||
"""With allow_partial=True the chain returns the market cap it did get."""
|
|
||||||
primary_data = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
|
||||||
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
|
||||||
)
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("fmp", _DataProvider(primary_data)),
|
|
||||||
("finnhub", _RateLimitedProvider()),
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL", allow_partial=True)
|
|
||||||
assert result.market_cap == 2_000_000.0
|
|
||||||
assert result.pe_ratio is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rate_limited_but_complete_does_not_raise():
|
|
||||||
"""If every field is filled, a rate limit on a later (unused) provider is moot."""
|
|
||||||
full = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=20.0, revenue_growth=10.0, earnings_surprise=2.0,
|
|
||||||
market_cap=5.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
|
||||||
)
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("fmp", _DataProvider(full)),
|
|
||||||
("finnhub", _RateLimitedProvider()),
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
assert result.pe_ratio == 20.0
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_chain_merges_next_earnings_date():
|
|
||||||
"""Earnings date is taken from the first provider that supplies it."""
|
|
||||||
from datetime import date as _date
|
|
||||||
|
|
||||||
primary = FundamentalData(
|
|
||||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
|
||||||
market_cap=100.0, fetched_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
class _EarningsProvider:
|
|
||||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
|
||||||
return FundamentalData(
|
|
||||||
ticker=ticker, pe_ratio=10.0, revenue_growth=5.0, earnings_surprise=1.0,
|
|
||||||
market_cap=None, fetched_at=datetime.now(timezone.utc),
|
|
||||||
next_earnings_date=_date(2026, 7, 1),
|
|
||||||
)
|
|
||||||
|
|
||||||
provider = ChainedFundamentalProvider([
|
|
||||||
("fmp", _DataProvider(primary)),
|
|
||||||
("finnhub", _EarningsProvider()),
|
|
||||||
])
|
|
||||||
result = await provider.fetch_fundamentals("AAPL")
|
|
||||||
assert result.next_earnings_date == _date(2026, 7, 1)
|
|
||||||
@@ -1,209 +0,0 @@
|
|||||||
"""A5 fundamentals parity report: read-only comparison + artifact archive."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from datetime import date, datetime, timezone
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
|
|
||||||
from app.models.data_import_run import DataImportRun
|
|
||||||
from app.models.earnings_event import EarningsEvent
|
|
||||||
from app.models.fundamental import FundamentalData
|
|
||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
|
||||||
from app.models.ohlcv import OHLCVRecord
|
|
||||||
from app.models.ticker import Ticker
|
|
||||||
from app.services.fundamentals_parity_service import (
|
|
||||||
build_report,
|
|
||||||
fundamental_score,
|
|
||||||
load_latest,
|
|
||||||
load_latest_csv,
|
|
||||||
load_latest_json,
|
|
||||||
store_report,
|
|
||||||
)
|
|
||||||
|
|
||||||
UTC = timezone.utc
|
|
||||||
GENERATED = datetime(2026, 7, 23, 10, 30, tzinfo=UTC)
|
|
||||||
|
|
||||||
|
|
||||||
def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]:
|
|
||||||
rows = []
|
|
||||||
periods = ("Q1", "Q2", "Q3", "FY")
|
|
||||||
months = (3, 6, 9, 12)
|
|
||||||
for fy, multiplier in ((2025, 1.0), (2026, 1.1)):
|
|
||||||
revenues = [100 * multiplier, 110 * multiplier, 120 * multiplier, 130 * multiplier]
|
|
||||||
eps = [1.0 * multiplier, 1.1 * multiplier, 1.2 * multiplier, 1.3 * multiplier]
|
|
||||||
for index, period in enumerate(periods):
|
|
||||||
period_end = date(fy, months[index], 28)
|
|
||||||
rows.append(
|
|
||||||
FundamentalSnapshot(
|
|
||||||
cik=cik,
|
|
||||||
accession=f"{cik}-{fy}-{period}",
|
|
||||||
form="10-K" if period == "FY" else "10-Q",
|
|
||||||
filed_date=period_end,
|
|
||||||
accepted_at=datetime(fy, months[index], 28, tzinfo=UTC),
|
|
||||||
period_end=period_end,
|
|
||||||
fiscal_year=fy,
|
|
||||||
fiscal_period=period,
|
|
||||||
revenue=sum(revenues[: index + 1]),
|
|
||||||
operating_income=sum(revenues[: index + 1]) * 0.2,
|
|
||||||
diluted_eps=sum(eps[: index + 1]),
|
|
||||||
cfo=sum(revenues[: index + 1]) * 0.25,
|
|
||||||
capex=sum(revenues[: index + 1]) * 0.05,
|
|
||||||
depreciation_amortization=sum(revenues[: index + 1]) * 0.05,
|
|
||||||
cash_and_st_investments=40,
|
|
||||||
total_debt=100,
|
|
||||||
shares_outstanding=1000,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
async def _seed(db_session):
|
|
||||||
first = Ticker(symbol="AAA", cik="0000000001", sic="3571")
|
|
||||||
second = Ticker(symbol="BBB", cik=None, sic=None)
|
|
||||||
db_session.add_all([first, second])
|
|
||||||
await db_session.flush()
|
|
||||||
db_session.add_all(_snapshot_rows(first.cik))
|
|
||||||
db_session.add_all(
|
|
||||||
[
|
|
||||||
FundamentalData(
|
|
||||||
ticker_id=first.id,
|
|
||||||
pe_ratio=25,
|
|
||||||
revenue_growth=5,
|
|
||||||
earnings_surprise=0,
|
|
||||||
fetched_at=GENERATED,
|
|
||||||
),
|
|
||||||
FundamentalData(
|
|
||||||
ticker_id=second.id,
|
|
||||||
pe_ratio=12,
|
|
||||||
revenue_growth=3,
|
|
||||||
earnings_surprise=None,
|
|
||||||
fetched_at=GENERATED,
|
|
||||||
),
|
|
||||||
OHLCVRecord(
|
|
||||||
ticker_id=first.id,
|
|
||||||
date=date(2026, 7, 22),
|
|
||||||
open=100,
|
|
||||||
high=100,
|
|
||||||
low=100,
|
|
||||||
close=100,
|
|
||||||
volume=100,
|
|
||||||
),
|
|
||||||
EarningsEvent(
|
|
||||||
ticker_id=first.id,
|
|
||||||
announce_date=date(2026, 7, 1),
|
|
||||||
session="amc",
|
|
||||||
eps_estimate=2,
|
|
||||||
eps_actual=2.2,
|
|
||||||
source="dolt_earnings",
|
|
||||||
),
|
|
||||||
DataImportRun(
|
|
||||||
source="sec_facts",
|
|
||||||
revision="sec-rev",
|
|
||||||
status="promoted",
|
|
||||||
source_max_date=date(2026, 7, 22),
|
|
||||||
started_at=GENERATED,
|
|
||||||
completed_at=GENERATED,
|
|
||||||
),
|
|
||||||
DataImportRun(
|
|
||||||
source="dolt_earnings",
|
|
||||||
revision="dolt-rev",
|
|
||||||
status="no_op",
|
|
||||||
source_max_date=date(2026, 7, 22),
|
|
||||||
started_at=GENERATED,
|
|
||||||
completed_at=GENERATED,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
await db_session.flush()
|
|
||||||
|
|
||||||
|
|
||||||
def test_score_formula_matches_production_rules():
|
|
||||||
score = fundamental_score(pe_ratio=15, revenue_growth=0, earnings_surprise=0)
|
|
||||||
assert score == pytest.approx((100 + 50 + 50) / 3)
|
|
||||||
assert fundamental_score(pe_ratio=15, revenue_growth=None, earnings_surprise=None) is None
|
|
||||||
|
|
||||||
async def test_report_compares_sources_and_leaves_database_untouched(db_session):
|
|
||||||
await _seed(db_session)
|
|
||||||
before = await db_session.scalar(select(func.count()).select_from(FundamentalData))
|
|
||||||
|
|
||||||
report = await build_report(
|
|
||||||
db_session,
|
|
||||||
generated_at=GENERATED,
|
|
||||||
today=date(2026, 7, 23),
|
|
||||||
)
|
|
||||||
|
|
||||||
after = await db_session.scalar(select(func.count()).select_from(FundamentalData))
|
|
||||||
assert before == after == 2
|
|
||||||
assert not db_session.new and not db_session.dirty and not db_session.deleted
|
|
||||||
assert report["read_only"] is True
|
|
||||||
assert report["approval_status"] == "pending_explicit_approval"
|
|
||||||
assert report["source_runs"]["sec_facts"]["revision"] == "sec-rev"
|
|
||||||
assert report["source_runs"]["dolt_earnings"]["revision"] == "dolt-rev"
|
|
||||||
|
|
||||||
first = next(row for row in report["rows"] if row["symbol"] == "AAA")
|
|
||||||
assert first["fields"]["pe_ratio"]["candidate"] == pytest.approx(
|
|
||||||
100 / 5.06, abs=1e-4
|
|
||||||
)
|
|
||||||
assert first["fields"]["revenue_growth"]["candidate"] == pytest.approx(10)
|
|
||||||
assert first["fields"]["earnings_surprise"]["candidate"] == pytest.approx(10)
|
|
||||||
assert first["scores"]["candidate_fundamental"] is not None
|
|
||||||
assert report["summary"]["universe_count"] == 2
|
|
||||||
assert report["summary"]["field_stats"]["pe_ratio"]["both_available"] == 1
|
|
||||||
|
|
||||||
|
|
||||||
async def test_artifacts_archive_and_latest_manifest(db_session, tmp_path):
|
|
||||||
await _seed(db_session)
|
|
||||||
report = await build_report(
|
|
||||||
db_session,
|
|
||||||
generated_at=GENERATED,
|
|
||||||
today=date(2026, 7, 23),
|
|
||||||
)
|
|
||||||
|
|
||||||
paths = store_report(report, tmp_path)
|
|
||||||
|
|
||||||
assert tmp_path.joinpath("latest.json").exists()
|
|
||||||
assert paths["json"].endswith(".json") and paths["csv"].endswith(".csv")
|
|
||||||
assert load_latest(tmp_path)["generated_at"] == GENERATED.isoformat()
|
|
||||||
csv_artifact = load_latest_csv(tmp_path)
|
|
||||||
assert csv_artifact is not None
|
|
||||||
assert csv_artifact[0].endswith(".csv")
|
|
||||||
assert "legacy_fundamental,candidate_fundamental" in csv_artifact[1]
|
|
||||||
assert "AAA" in csv_artifact[1]
|
|
||||||
json_artifact = load_latest_json(tmp_path)
|
|
||||||
assert json_artifact is not None and '"rows"' in json_artifact[1]
|
|
||||||
|
|
||||||
|
|
||||||
async def test_admin_endpoints_return_compact_summary_and_downloads(
|
|
||||||
client, db_session, tmp_path, monkeypatch
|
|
||||||
):
|
|
||||||
from app.config import settings
|
|
||||||
from app.dependencies import require_admin
|
|
||||||
from app.main import app
|
|
||||||
|
|
||||||
await _seed(db_session)
|
|
||||||
report = await build_report(
|
|
||||||
db_session,
|
|
||||||
generated_at=GENERATED,
|
|
||||||
today=date(2026, 7, 23),
|
|
||||||
)
|
|
||||||
store_report(report, tmp_path)
|
|
||||||
monkeypatch.setattr(settings, "fundamentals_parity_report_dir", str(tmp_path))
|
|
||||||
app.dependency_overrides[require_admin] = lambda: None
|
|
||||||
try:
|
|
||||||
summary_response = await client.get("/api/v1/admin/fundamentals-parity")
|
|
||||||
assert summary_response.status_code == 200
|
|
||||||
summary = summary_response.json()["data"]
|
|
||||||
assert summary["summary"]["universe_count"] == 2
|
|
||||||
assert "rows" not in summary
|
|
||||||
|
|
||||||
csv_response = await client.get("/api/v1/admin/fundamentals-parity/csv")
|
|
||||||
assert csv_response.status_code == 200
|
|
||||||
assert "AAA" in csv_response.json()["data"]["content"]
|
|
||||||
|
|
||||||
json_response = await client.get("/api/v1/admin/fundamentals-parity/json")
|
|
||||||
assert json_response.status_code == 200
|
|
||||||
assert '"rows"' in json_response.json()["data"]["content"]
|
|
||||||
finally:
|
|
||||||
app.dependency_overrides.pop(require_admin, None)
|
|
||||||
@@ -6,7 +6,6 @@ from datetime import date, datetime, timezone
|
|||||||
from app.models.data_import_run import DataImportRun
|
from app.models.data_import_run import DataImportRun
|
||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
from app.models.sec_filing_gap import SecFilingGap
|
from app.models.sec_filing_gap import SecFilingGap
|
||||||
from app.models.settings import SystemSetting
|
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.services import fundamentals_quality_service
|
from app.services import fundamentals_quality_service
|
||||||
|
|
||||||
@@ -19,12 +18,6 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks(
|
|||||||
healthy = Ticker(symbol="HEALTHY", cik="0000000003")
|
healthy = Ticker(symbol="HEALTHY", cik="0000000003")
|
||||||
db_session.add_all([missing, no_history, healthy])
|
db_session.add_all([missing, no_history, healthy])
|
||||||
await db_session.flush()
|
await db_session.flush()
|
||||||
db_session.add(
|
|
||||||
SystemSetting(
|
|
||||||
key="fundamental_data_sec_dolt_cutover_enabled",
|
|
||||||
value="true",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
db_session.add(
|
db_session.add(
|
||||||
DataImportRun(
|
DataImportRun(
|
||||||
source="sec_facts",
|
source="sec_facts",
|
||||||
@@ -44,38 +37,11 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async def test_sec_quality_gate_is_inactive_before_cutover(db_session):
|
|
||||||
ticker = Ticker(symbol="SHADOW", cik="0000000042")
|
|
||||||
db_session.add(ticker)
|
|
||||||
await db_session.flush()
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
db_session.add(
|
|
||||||
SecFilingGap(
|
|
||||||
cik=ticker.cik,
|
|
||||||
accession="SHADOW-Q",
|
|
||||||
form="10-Q",
|
|
||||||
index_date=date.today(),
|
|
||||||
reason="not_in_companyfacts",
|
|
||||||
first_seen_at=now,
|
|
||||||
last_attempted_at=now,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await db_session.flush()
|
|
||||||
|
|
||||||
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def test_active_gap_is_blocked_until_a_later_filing_supersedes_it(db_session):
|
async def test_active_gap_is_blocked_until_a_later_filing_supersedes_it(db_session):
|
||||||
ticker = Ticker(symbol="HIST", cik="0000000043")
|
ticker = Ticker(symbol="HIST", cik="0000000043")
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
db_session.add_all([
|
db_session.add_all([
|
||||||
ticker,
|
ticker,
|
||||||
SystemSetting(
|
|
||||||
key="fundamental_data_sec_dolt_cutover_enabled",
|
|
||||||
value="true",
|
|
||||||
),
|
|
||||||
SecFilingGap(
|
SecFilingGap(
|
||||||
cik=ticker.cik,
|
cik=ticker.cik,
|
||||||
accession="HIST-Q",
|
accession="HIST-Q",
|
||||||
@@ -116,10 +82,6 @@ async def test_gap_without_index_date_uses_first_seen_date_for_supersession(
|
|||||||
first_seen = datetime(2026, 5, 1, 12, tzinfo=timezone.utc)
|
first_seen = datetime(2026, 5, 1, 12, tzinfo=timezone.utc)
|
||||||
db_session.add_all([
|
db_session.add_all([
|
||||||
ticker,
|
ticker,
|
||||||
SystemSetting(
|
|
||||||
key="fundamental_data_sec_dolt_cutover_enabled",
|
|
||||||
value="true",
|
|
||||||
),
|
|
||||||
SecFilingGap(
|
SecFilingGap(
|
||||||
cik=ticker.cik,
|
cik=ticker.cik,
|
||||||
accession="DATELESS-Q",
|
accession="DATELESS-Q",
|
||||||
@@ -157,10 +119,6 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session):
|
|||||||
ticker = Ticker(symbol="NEWREG", cik="0000000044")
|
ticker = Ticker(symbol="NEWREG", cik="0000000044")
|
||||||
db_session.add_all([
|
db_session.add_all([
|
||||||
ticker,
|
ticker,
|
||||||
SystemSetting(
|
|
||||||
key="fundamental_data_sec_dolt_cutover_enabled",
|
|
||||||
value="true",
|
|
||||||
),
|
|
||||||
DataImportRun(
|
DataImportRun(
|
||||||
source="sec_facts",
|
source="sec_facts",
|
||||||
status="promoted",
|
status="promoted",
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""A6: `sources=fundamentals` is accepted but never fetches from a provider.
|
||||||
|
|
||||||
|
`fundamental_data` is rebuilt for the whole universe by the nightly SEC + Dolt
|
||||||
|
imports, so there is no per-ticker fetch left. The source key stays valid so an
|
||||||
|
older client gets a truthful `skipped` instead of a silent omission.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.models.ticker import Ticker
|
||||||
|
|
||||||
|
|
||||||
|
async def test_fundamentals_source_reports_skipped(client, db_session):
|
||||||
|
from app.dependencies import require_access
|
||||||
|
from app.main import app
|
||||||
|
|
||||||
|
app.dependency_overrides[require_access] = lambda: None
|
||||||
|
try:
|
||||||
|
db_session.add(Ticker(symbol="AAPL"))
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/v1/ingestion/fetch/AAPL", params={"sources": "fundamentals"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
source = resp.json()["data"]["sources"]["fundamentals"]
|
||||||
|
assert source["status"] == "skipped"
|
||||||
|
assert "SEC + Dolt" in source["message"]
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.pop(require_access, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fundamentals_remains_a_recognised_source_key():
|
||||||
|
"""Older clients keep getting an entry for it rather than a missing key."""
|
||||||
|
from app.routers.ingestion import _parse_requested_sources
|
||||||
|
|
||||||
|
assert "fundamentals" in _parse_requested_sources("fundamentals")
|
||||||
|
assert "fundamentals" in _parse_requested_sources(None) # None => all sources
|
||||||
@@ -1,905 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import pickle
|
|
||||||
import sqlite3
|
|
||||||
from datetime import date, timedelta
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.services import backtest_service as bt
|
|
||||||
from scripts.portfolio_capacity_research import (
|
|
||||||
ANCHOR_YEARS,
|
|
||||||
RISK_FLOOR_ARMS,
|
|
||||||
aggregate_results,
|
|
||||||
bootstrap_median_interval,
|
|
||||||
build_cells,
|
|
||||||
build_cohort_manifest,
|
|
||||||
iqr,
|
|
||||||
summarize_simulation,
|
|
||||||
validate_cohort_manifest,
|
|
||||||
)
|
|
||||||
from scripts.run_portfolio_construction_matrix import (
|
|
||||||
CACHE_VERSION,
|
|
||||||
STUDIES,
|
|
||||||
_assert_clean_worktree,
|
|
||||||
_build_candidate_cache,
|
|
||||||
_checkpoint_state,
|
|
||||||
_construction_candidate_view,
|
|
||||||
_construction_universe_errors,
|
|
||||||
_json_hash,
|
|
||||||
_load_snapshot,
|
|
||||||
_markdown,
|
|
||||||
_operational_summary,
|
|
||||||
_risk_floor_markdown,
|
|
||||||
_worker_init,
|
|
||||||
_worker_run_cell,
|
|
||||||
_write_cell_checkpoint,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _prices(ords: list[int], close: float = 100.0) -> tuple:
|
|
||||||
closes = [close] * len(ords)
|
|
||||||
return (
|
|
||||||
ords,
|
|
||||||
list(closes),
|
|
||||||
[value + 1.0 for value in closes],
|
|
||||||
[value - 1.0 for value in closes],
|
|
||||||
list(closes),
|
|
||||||
[1_000_000] * len(ords),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _candidate(
|
|
||||||
symbol: str,
|
|
||||||
day: date,
|
|
||||||
*,
|
|
||||||
entry: float = 100.0,
|
|
||||||
stop: float = 80.0,
|
|
||||||
rank: float = 90.0,
|
|
||||||
) -> dict:
|
|
||||||
return {
|
|
||||||
'qualified': True,
|
|
||||||
'direction': 'long',
|
|
||||||
'symbol': symbol,
|
|
||||||
'date': day.isoformat(),
|
|
||||||
'entry': entry,
|
|
||||||
'stop': stop,
|
|
||||||
'target': entry + 100.0,
|
|
||||||
'momentum_percentile': rank,
|
|
||||||
'activation_momentum_percentile': rank,
|
|
||||||
'residual_high_vol_blend_80_20': rank,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _business_days(start: date, end: date) -> list[date]:
|
|
||||||
days: list[date] = []
|
|
||||||
current = start
|
|
||||||
while current <= end:
|
|
||||||
if current.weekday() < 5:
|
|
||||||
days.append(current)
|
|
||||||
current += timedelta(days=1)
|
|
||||||
return days
|
|
||||||
|
|
||||||
|
|
||||||
def test_new_simulator_option_defaults_match_explicit_defaults():
|
|
||||||
start = date(2025, 1, 6)
|
|
||||||
ords = [start.toordinal() + offset for offset in range(8)]
|
|
||||||
prices = {'AAA': _prices(ords)}
|
|
||||||
candidates = [_candidate('AAA', start)]
|
|
||||||
|
|
||||||
legacy = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
3,
|
|
||||||
include_trades=True,
|
|
||||||
)
|
|
||||||
explicit = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
3,
|
|
||||||
max_positions=10,
|
|
||||||
min_initial_risk_fraction=None,
|
|
||||||
weekly_top_n_rebalance=False,
|
|
||||||
measurement_start_date=None,
|
|
||||||
hard_end_date=None,
|
|
||||||
include_capacity_diagnostics=False,
|
|
||||||
include_trades=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert legacy == explicit
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_snapshot_accepts_pre_sec_ticker_schema(tmp_path, monkeypatch):
|
|
||||||
snapshot = tmp_path / 'legacy-research.sqlite'
|
|
||||||
with sqlite3.connect(snapshot) as connection:
|
|
||||||
connection.executescript(
|
|
||||||
'''
|
|
||||||
CREATE TABLE tickers (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
symbol VARCHAR(10) NOT NULL UNIQUE,
|
|
||||||
name VARCHAR(120),
|
|
||||||
created_at DATETIME NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE ohlcv_records (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
ticker_id INTEGER NOT NULL,
|
|
||||||
date DATE NOT NULL,
|
|
||||||
open FLOAT NOT NULL,
|
|
||||||
high FLOAT NOT NULL,
|
|
||||||
low FLOAT NOT NULL,
|
|
||||||
close FLOAT NOT NULL,
|
|
||||||
volume BIGINT NOT NULL,
|
|
||||||
created_at DATETIME NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE research_rank_only (
|
|
||||||
symbol VARCHAR(10) PRIMARY KEY
|
|
||||||
);
|
|
||||||
INSERT INTO tickers VALUES
|
|
||||||
(1, 'LEGACY', 'Legacy Co', '2024-01-01 00:00:00'),
|
|
||||||
(2, 'RANK', 'Rank Only Co', '2024-01-01 00:00:00');
|
|
||||||
INSERT INTO ohlcv_records VALUES
|
|
||||||
(1, 1, '2024-01-02', 100, 102, 99, 101, 1000000,
|
|
||||||
'2024-01-02 00:00:00'),
|
|
||||||
(2, 2, '2024-01-02', 50, 51, 49, 50, 500000,
|
|
||||||
'2024-01-02 00:00:00');
|
|
||||||
INSERT INTO research_rank_only VALUES ('RANK');
|
|
||||||
'''
|
|
||||||
)
|
|
||||||
|
|
||||||
async def recommendation_config(_db):
|
|
||||||
return {}
|
|
||||||
|
|
||||||
async def activation_config(_db):
|
|
||||||
return {'min_momentum_percentile': 80.0}
|
|
||||||
|
|
||||||
async def exit_policy(_db):
|
|
||||||
return {'mode': 'atr_trailing', 'hold_days': 30, 'atr_multiplier': 3.0}
|
|
||||||
|
|
||||||
async def benchmark_closes(_db, *, days, refresh):
|
|
||||||
assert days is None
|
|
||||||
assert refresh is False
|
|
||||||
return {date(2024, 1, 2): 100.0}
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
'app.services.recommendation_service.get_recommendation_config',
|
|
||||||
recommendation_config,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
'app.services.admin_service.get_activation_config',
|
|
||||||
activation_config,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
'app.services.paper_trade_service.get_exit_policy',
|
|
||||||
exit_policy,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
'app.services.backtest_service._load_benchmark_closes_for_backtest',
|
|
||||||
benchmark_closes,
|
|
||||||
)
|
|
||||||
|
|
||||||
loaded = asyncio.run(_load_snapshot(snapshot, quiet=True))
|
|
||||||
|
|
||||||
assert loaded['symbols'] == ['LEGACY', 'RANK']
|
|
||||||
assert loaded['construction_symbols'] == {'LEGACY'}
|
|
||||||
assert loaded['prices']['LEGACY'] == (
|
|
||||||
[date(2024, 1, 2).toordinal()],
|
|
||||||
[100.0],
|
|
||||||
[102.0],
|
|
||||||
[99.0],
|
|
||||||
[101.0],
|
|
||||||
[1_000_000],
|
|
||||||
)
|
|
||||||
assert loaded['prices']['RANK'][4] == [50.0]
|
|
||||||
assert loaded['construction_universe_manifest'][
|
|
||||||
'construction_ticker_rows'
|
|
||||||
] == 1
|
|
||||||
assert loaded['construction_universe_manifest']['rank_only_ticker_rows'] == 1
|
|
||||||
with sqlite3.connect(snapshot) as connection:
|
|
||||||
columns = {
|
|
||||||
row[1] for row in connection.execute('PRAGMA table_info(tickers)')
|
|
||||||
}
|
|
||||||
assert {'cik', 'sic', 'sic_description'}.isdisjoint(columns)
|
|
||||||
|
|
||||||
|
|
||||||
def test_construction_view_filters_rank_only_rows_without_rebuilding_cache():
|
|
||||||
manifest = {
|
|
||||||
'ranking_ticker_rows': 506,
|
|
||||||
'ranking_symbols_with_prices': 506,
|
|
||||||
'construction_ticker_rows': 505,
|
|
||||||
'construction_symbols_with_prices': 505,
|
|
||||||
'rank_only_ticker_rows': 1,
|
|
||||||
'rank_only_symbols_with_prices': 1,
|
|
||||||
'rank_only_unknown_symbols': 0,
|
|
||||||
}
|
|
||||||
cached = {
|
|
||||||
'key': {'version': 'existing-broad-cache'},
|
|
||||||
'qualified_candidates': [
|
|
||||||
{'symbol': 'PROD', 'date': '2025-01-02'},
|
|
||||||
{'symbol': 'RANK', 'date': '2025-01-02'},
|
|
||||||
],
|
|
||||||
'qualified_long_count': 2,
|
|
||||||
'daily_rank_map': {
|
|
||||||
('RANK', '2025-01-02'): {'strategy_rank': 99.0},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
view = _construction_candidate_view(
|
|
||||||
cached,
|
|
||||||
{
|
|
||||||
'construction_symbols': {'PROD'},
|
|
||||||
'construction_universe_manifest': manifest,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert [row['symbol'] for row in view['qualified_candidates']] == ['PROD']
|
|
||||||
assert view['raw_full_universe_qualified_long_count'] == 2
|
|
||||||
assert view['filtered_rank_only_qualified_long_count'] == 1
|
|
||||||
assert view['qualified_long_count'] == 1
|
|
||||||
assert ('RANK', '2025-01-02') in view['daily_rank_map']
|
|
||||||
assert len(cached['qualified_candidates']) == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_existing_broad_candidate_cache_key_remains_reusable(tmp_path, monkeypatch):
|
|
||||||
snapshot = tmp_path / 'research.sqlite'
|
|
||||||
snapshot.write_bytes(b'snapshot-placeholder')
|
|
||||||
cache_path = tmp_path / 'broad-cache.pkl'
|
|
||||||
snapshot_data = {
|
|
||||||
'recommendation_config': {'rr': 3.0},
|
|
||||||
'activation': {'min_momentum_percentile': 80.0},
|
|
||||||
'runtime_config': {'ranking_key': 'test'},
|
|
||||||
'universe_manifest': {
|
|
||||||
'ticker_rows': 4655,
|
|
||||||
'symbols_with_prices': 4654,
|
|
||||||
'symbols_sha256': 'symbols',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
key = {
|
|
||||||
'version': CACHE_VERSION,
|
|
||||||
'snapshot': str(snapshot.resolve()),
|
|
||||||
'snapshot_sha256': 'snapshot-hash',
|
|
||||||
'cadence': 'daily',
|
|
||||||
'outcome_horizon_sessions': 0,
|
|
||||||
'recommendation_config_hash': _json_hash(
|
|
||||||
snapshot_data['recommendation_config']
|
|
||||||
),
|
|
||||||
'activation_hash': _json_hash(snapshot_data['activation']),
|
|
||||||
'runtime_config': snapshot_data['runtime_config'],
|
|
||||||
'universe_manifest': snapshot_data['universe_manifest'],
|
|
||||||
}
|
|
||||||
cached = {'key': key, 'qualified_candidates': [{'symbol': 'PROD'}]}
|
|
||||||
cache_path.write_bytes(pickle.dumps(cached))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
bt,
|
|
||||||
'_replay_candidates_for_period',
|
|
||||||
lambda *_args: pytest.fail('existing cache should avoid replay'),
|
|
||||||
)
|
|
||||||
|
|
||||||
loaded = _build_candidate_cache(
|
|
||||||
snapshot_data,
|
|
||||||
snapshot=snapshot,
|
|
||||||
snapshot_sha256='snapshot-hash',
|
|
||||||
cache_path=cache_path,
|
|
||||||
workers=1,
|
|
||||||
quiet=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert loaded == cached
|
|
||||||
|
|
||||||
|
|
||||||
def test_construction_universe_guard_rejects_leaked_broad_book():
|
|
||||||
valid = {
|
|
||||||
'ranking_ticker_rows': 4655,
|
|
||||||
'construction_ticker_rows': 506,
|
|
||||||
'construction_symbols_with_prices': 506,
|
|
||||||
'rank_only_ticker_rows': 4149,
|
|
||||||
'rank_only_unknown_symbols': 0,
|
|
||||||
}
|
|
||||||
assert _construction_universe_errors(valid) == []
|
|
||||||
|
|
||||||
leaked = {
|
|
||||||
**valid,
|
|
||||||
'construction_ticker_rows': 4655,
|
|
||||||
'construction_symbols_with_prices': 4654,
|
|
||||||
'rank_only_ticker_rows': 0,
|
|
||||||
}
|
|
||||||
errors = _construction_universe_errors(leaked)
|
|
||||||
assert any('450-600' in error for error in errors)
|
|
||||||
|
|
||||||
|
|
||||||
def test_unbounded_count_and_effective_risk_floor():
|
|
||||||
start = date(2025, 1, 6)
|
|
||||||
ords = [start.toordinal() + offset for offset in range(4)]
|
|
||||||
symbols = [f'S{index}' for index in range(25)]
|
|
||||||
prices = {symbol: _prices(ords) for symbol in symbols}
|
|
||||||
candidates = [
|
|
||||||
_candidate(symbol, start, stop=80.0, rank=100.0 - index)
|
|
||||||
for index, symbol in enumerate(symbols)
|
|
||||||
]
|
|
||||||
|
|
||||||
capped = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
max_positions=1,
|
|
||||||
hard_end_date=start + timedelta(days=4),
|
|
||||||
measurement_start_date=start,
|
|
||||||
include_capacity_diagnostics=True,
|
|
||||||
)
|
|
||||||
unbounded = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
max_positions=None,
|
|
||||||
min_initial_risk_fraction=0.005,
|
|
||||||
hard_end_date=start + timedelta(days=4),
|
|
||||||
measurement_start_date=start,
|
|
||||||
include_capacity_diagnostics=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert capped is not None and unbounded is not None
|
|
||||||
assert capped['peak_positions'] == 1
|
|
||||||
assert capped['measurement_skipped_book_full'] == 24
|
|
||||||
assert unbounded['peak_positions'] > 1
|
|
||||||
assert unbounded['measurement_skipped_book_full'] == 0
|
|
||||||
assert unbounded['skipped_min_initial_risk'] > 0
|
|
||||||
assert unbounded['peak_positions'] == unbounded['trades']
|
|
||||||
|
|
||||||
|
|
||||||
def test_measurement_window_carries_state_but_excludes_pre_anchor_trade_ev():
|
|
||||||
start = date(2025, 1, 6)
|
|
||||||
anchor = start + timedelta(days=2)
|
|
||||||
hard_end = start + timedelta(days=7)
|
|
||||||
ords = [
|
|
||||||
start.toordinal() + offset
|
|
||||||
for offset in range((hard_end - start).days)
|
|
||||||
]
|
|
||||||
prices = {
|
|
||||||
'AAA': _prices(ords, 100.0),
|
|
||||||
'BBB': _prices(ords, 100.0),
|
|
||||||
}
|
|
||||||
candidates = [
|
|
||||||
_candidate('AAA', start),
|
|
||||||
_candidate('BBB', anchor + timedelta(days=1)),
|
|
||||||
]
|
|
||||||
|
|
||||||
sim = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
start_date=start,
|
|
||||||
end_date=hard_end,
|
|
||||||
measurement_start_date=anchor,
|
|
||||||
hard_end_date=hard_end,
|
|
||||||
include_curve=True,
|
|
||||||
include_trades=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sim is not None
|
|
||||||
assert sim['simulation_start_date'] == start.isoformat()
|
|
||||||
assert sim['start_date'] == anchor.isoformat()
|
|
||||||
assert sim['measurement_start_positions'] == 1
|
|
||||||
assert sim['trades'] == 1
|
|
||||||
assert [trade['symbol'] for trade in sim['trade_details']] == ['BBB']
|
|
||||||
assert sim['equity_curve'][0]['date'] == anchor.isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def test_weekly_top10_uses_current_rank_for_both_sides_not_entry_rank():
|
|
||||||
monday = date(2025, 1, 6)
|
|
||||||
friday = date(2025, 1, 10)
|
|
||||||
sessions = _business_days(monday, friday)
|
|
||||||
ords = [session.toordinal() for session in sessions]
|
|
||||||
prices = {
|
|
||||||
'AAA': _prices(ords),
|
|
||||||
'BBB': _prices(ords),
|
|
||||||
}
|
|
||||||
candidates = [
|
|
||||||
_candidate('AAA', monday, rank=99.0),
|
|
||||||
_candidate('BBB', friday, rank=10.0),
|
|
||||||
]
|
|
||||||
rank_map = {
|
|
||||||
('AAA', friday.isoformat()): {'strategy_rank': 10.0},
|
|
||||||
('BBB', friday.isoformat()): {'strategy_rank': 90.0},
|
|
||||||
}
|
|
||||||
|
|
||||||
sim = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
max_positions=1,
|
|
||||||
weekly_top_n_rebalance=True,
|
|
||||||
daily_rank_map=rank_map,
|
|
||||||
measurement_start_date=monday,
|
|
||||||
hard_end_date=friday + timedelta(days=1),
|
|
||||||
include_trades=True,
|
|
||||||
include_capacity_diagnostics=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sim is not None
|
|
||||||
assert [trade['symbol'] for trade in sim['trade_details']] == ['AAA', 'BBB']
|
|
||||||
assert sim['trade_details'][0]['reason'] == 'weekly_rebalance'
|
|
||||||
event = sim['weekly_rebalance_events'][0]
|
|
||||||
assert event['exited_symbols'] == ['AAA']
|
|
||||||
assert event['selected_entrant_symbols'] == ['BBB']
|
|
||||||
|
|
||||||
|
|
||||||
def test_weekly_top10_incumbent_wins_exact_current_rank_tie():
|
|
||||||
monday = date(2025, 1, 6)
|
|
||||||
friday = date(2025, 1, 10)
|
|
||||||
sessions = _business_days(monday, friday)
|
|
||||||
ords = [session.toordinal() for session in sessions]
|
|
||||||
prices = {
|
|
||||||
'AAA': _prices(ords),
|
|
||||||
'BBB': _prices(ords),
|
|
||||||
}
|
|
||||||
candidates = [
|
|
||||||
_candidate('AAA', monday, rank=10.0),
|
|
||||||
_candidate('BBB', friday, rank=99.0),
|
|
||||||
]
|
|
||||||
rank_map = {
|
|
||||||
('AAA', friday.isoformat()): {'strategy_rank': 80.0},
|
|
||||||
('BBB', friday.isoformat()): {'strategy_rank': 80.0},
|
|
||||||
}
|
|
||||||
|
|
||||||
sim = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
max_positions=1,
|
|
||||||
weekly_top_n_rebalance=True,
|
|
||||||
daily_rank_map=rank_map,
|
|
||||||
measurement_start_date=monday,
|
|
||||||
hard_end_date=friday + timedelta(days=1),
|
|
||||||
include_trades=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sim is not None
|
|
||||||
assert [trade['symbol'] for trade in sim['trade_details']] == ['AAA']
|
|
||||||
assert sim['trade_details'][0]['reason'] == 'open_at_end'
|
|
||||||
assert sim['weekly_rebalance_events'][0]['replacements'] == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_weekly_rebalance_exit_bypasses_cooldown_and_churn_is_counted():
|
|
||||||
first_monday = date(2025, 1, 6)
|
|
||||||
friday = date(2025, 1, 10)
|
|
||||||
next_monday = date(2025, 1, 13)
|
|
||||||
sessions = _business_days(first_monday, next_monday)
|
|
||||||
ords = [session.toordinal() for session in sessions]
|
|
||||||
prices = {
|
|
||||||
'AAA': _prices(ords),
|
|
||||||
'BBB': (
|
|
||||||
ords,
|
|
||||||
[100.0] * len(ords),
|
|
||||||
[101.0] * len(ords),
|
|
||||||
[99.0] * (len(ords) - 1) + [70.0],
|
|
||||||
[100.0] * len(ords),
|
|
||||||
[1_000_000] * len(ords),
|
|
||||||
),
|
|
||||||
}
|
|
||||||
candidates = [
|
|
||||||
_candidate('AAA', first_monday, rank=99.0),
|
|
||||||
_candidate('BBB', friday, rank=10.0),
|
|
||||||
_candidate('AAA', next_monday, rank=99.0),
|
|
||||||
]
|
|
||||||
rank_map = {
|
|
||||||
('AAA', friday.isoformat()): {'strategy_rank': 10.0},
|
|
||||||
('BBB', friday.isoformat()): {'strategy_rank': 90.0},
|
|
||||||
}
|
|
||||||
|
|
||||||
sim = bt._simulate_portfolio(
|
|
||||||
candidates,
|
|
||||||
prices,
|
|
||||||
None,
|
|
||||||
'hold',
|
|
||||||
30,
|
|
||||||
max_positions=1,
|
|
||||||
reentry_cooldown_sessions=5,
|
|
||||||
weekly_top_n_rebalance=True,
|
|
||||||
daily_rank_map=rank_map,
|
|
||||||
measurement_start_date=first_monday,
|
|
||||||
hard_end_date=next_monday + timedelta(days=1),
|
|
||||||
include_trades=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sim is not None
|
|
||||||
assert [trade['symbol'] for trade in sim['trade_details']] == [
|
|
||||||
'AAA',
|
|
||||||
'BBB',
|
|
||||||
'AAA',
|
|
||||||
]
|
|
||||||
assert sim['trade_details'][0]['reason'] == 'weekly_rebalance'
|
|
||||||
assert sim['rebalance_reentries_within_5_sessions'] == 1
|
|
||||||
assert sim['skipped_cooldown'] == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_cohort_manifest_realizes_seven_frozen_clusters():
|
|
||||||
sessions = _business_days(date(2016, 1, 4), date(2026, 7, 17))
|
|
||||||
manifest = build_cohort_manifest(sessions)
|
|
||||||
|
|
||||||
assert validate_cohort_manifest(manifest) == []
|
|
||||||
assert manifest['empty_cluster_count'] == 7
|
|
||||||
assert manifest['warm_cluster_count'] == 7
|
|
||||||
assert set(map(int, manifest['empty_cluster_counts'])) == set(ANCHOR_YEARS)
|
|
||||||
assert all(
|
|
||||||
int(count) >= 12 for count in manifest['warm_seed_counts'].values()
|
|
||||||
)
|
|
||||||
cells = build_cells(manifest)
|
|
||||||
assert len(cells) == (
|
|
||||||
len(manifest['empty_book']) + len(manifest['warm_book'])
|
|
||||||
) * 4 * 2
|
|
||||||
floor_cells = build_cells(manifest, arms=RISK_FLOOR_ARMS)
|
|
||||||
assert len(floor_cells) == (
|
|
||||||
len(manifest['empty_book']) + len(manifest['warm_book'])
|
|
||||||
) * 2 * 2
|
|
||||||
assert {row['arm_id'] for row in floor_cells} == {
|
|
||||||
'cap10_incumbent',
|
|
||||||
'cap10_min_risk_005',
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_risk_floor_study_changes_only_the_effective_risk_floor():
|
|
||||||
control, treatment = RISK_FLOOR_ARMS
|
|
||||||
|
|
||||||
assert control['max_positions'] == treatment['max_positions'] == 10
|
|
||||||
assert (
|
|
||||||
control['weekly_top_n_rebalance']
|
|
||||||
== treatment['weekly_top_n_rebalance']
|
|
||||||
is False
|
|
||||||
)
|
|
||||||
assert control['min_initial_risk_fraction'] is None
|
|
||||||
assert treatment['min_initial_risk_fraction'] == 0.005
|
|
||||||
assert STUDIES['risk-floor-ab']['arms'] == RISK_FLOOR_ARMS
|
|
||||||
assert STUDIES['capacity-bracket']['arms'] != RISK_FLOOR_ARMS
|
|
||||||
|
|
||||||
|
|
||||||
def test_zero_outcome_horizon_extends_rank_replay_to_last_session(monkeypatch):
|
|
||||||
monkeypatch.setattr(bt, '_window_setups', lambda *_args, **_kwargs: [])
|
|
||||||
count = bt.MIN_LOOKBACK + bt.HORIZON
|
|
||||||
start = date(2025, 1, 1)
|
|
||||||
ords = [start.toordinal() + offset for offset in range(count)]
|
|
||||||
columns = _prices(ords)
|
|
||||||
|
|
||||||
legacy = bt._replay_candidates_for_period(
|
|
||||||
'AAA',
|
|
||||||
columns,
|
|
||||||
{},
|
|
||||||
{},
|
|
||||||
None,
|
|
||||||
date.min,
|
|
||||||
'daily',
|
|
||||||
True,
|
|
||||||
True,
|
|
||||||
)
|
|
||||||
zero_horizon = bt._replay_candidates_for_period(
|
|
||||||
'AAA',
|
|
||||||
columns,
|
|
||||||
{},
|
|
||||||
{},
|
|
||||||
None,
|
|
||||||
date.min,
|
|
||||||
'daily',
|
|
||||||
True,
|
|
||||||
True,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert len(zero_horizon) == len(legacy) + bt.HORIZON
|
|
||||||
assert zero_horizon[-1]['date'] == date.fromordinal(ords[-1]).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def test_gain_to_pain_uses_all_monthly_returns_and_net_r():
|
|
||||||
sim = {
|
|
||||||
'measurement_start_equity': 100.0,
|
|
||||||
'trade_details': [
|
|
||||||
{
|
|
||||||
'net_r': 1.0,
|
|
||||||
'pnl': 10.0,
|
|
||||||
'shares': 1.0,
|
|
||||||
'entry': 100.0,
|
|
||||||
'fill': 110.0,
|
|
||||||
'transaction_cost': 0.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'net_r': -0.5,
|
|
||||||
'pnl': -5.0,
|
|
||||||
'shares': 1.0,
|
|
||||||
'entry': 100.0,
|
|
||||||
'fill': 95.0,
|
|
||||||
'transaction_cost': 0.0,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
'equity_curve': [
|
|
||||||
{'date': '2025-01-31', 'equity': 110.0},
|
|
||||||
{'date': '2025-02-28', 'equity': 99.0},
|
|
||||||
],
|
|
||||||
'trades': 2,
|
|
||||||
'skipped_book_full': 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
summary = summarize_simulation(sim)
|
|
||||||
|
|
||||||
assert summary['ev_net_r'] == pytest.approx(0.25)
|
|
||||||
assert summary['profit_factor'] == pytest.approx(2.0)
|
|
||||||
# Monthly returns are +10% and -10%; all-return numerator is zero.
|
|
||||||
assert summary['gain_to_pain'] == pytest.approx(0.0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_simple_cluster_bootstrap_is_deterministic_and_not_a_gate():
|
|
||||||
first = bootstrap_median_interval(
|
|
||||||
[1, 2, 3, 4, 5, 6, 7],
|
|
||||||
seed_parts=('determinism',),
|
|
||||||
replicates=500,
|
|
||||||
)
|
|
||||||
second = bootstrap_median_interval(
|
|
||||||
[1, 2, 3, 4, 5, 6, 7],
|
|
||||||
seed_parts=('determinism',),
|
|
||||||
replicates=500,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert first == second
|
|
||||||
assert first['point'] == 4
|
|
||||||
assert first['p05'] <= first['point'] <= first['p95']
|
|
||||||
|
|
||||||
|
|
||||||
def test_iqr_materializes_generator_before_both_quantiles():
|
|
||||||
assert iqr(value for value in (0.0, 1.0, 2.0, 3.0)) == pytest.approx(1.5)
|
|
||||||
|
|
||||||
|
|
||||||
def test_aggregate_reports_paired_years_and_separate_warm_iqrs():
|
|
||||||
cells: list[dict] = []
|
|
||||||
for cost in (0.1, 0.2):
|
|
||||||
for cluster in ANCHOR_YEARS:
|
|
||||||
for seed in range(3):
|
|
||||||
path_id = f'warm-{cluster}-{seed}'
|
|
||||||
for arm_id, shift in (
|
|
||||||
('cap10_incumbent', 0.0),
|
|
||||||
('cash_unbounded', 0.2),
|
|
||||||
('cap10_weekly_top10', 0.1),
|
|
||||||
('cap15_incumbent', 0.05),
|
|
||||||
):
|
|
||||||
cells.append({
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'protocol': 'warm_book',
|
|
||||||
'path_id': path_id,
|
|
||||||
'cluster': cluster,
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
'metrics': {
|
|
||||||
'ev_net_r': seed + shift,
|
|
||||||
'calmar': 1.0 + seed * 0.1 + shift,
|
|
||||||
'profit_factor': 1.5 + shift,
|
|
||||||
'gain_to_pain': 2.0 + shift,
|
|
||||||
'sortino': 1.0 + shift,
|
|
||||||
'cagr_pct': 10.0 + shift,
|
|
||||||
'max_drawdown_pct': 5.0,
|
|
||||||
'total_return_pct': 10.0 + shift,
|
|
||||||
'sharpe': 1.0 + shift,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
for arm_id, shift in (
|
|
||||||
('cap10_incumbent', 0.0),
|
|
||||||
('cash_unbounded', 0.2),
|
|
||||||
('cap10_weekly_top10', 0.1),
|
|
||||||
('cap15_incumbent', 0.05),
|
|
||||||
):
|
|
||||||
cells.append({
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'protocol': 'empty_book',
|
|
||||||
'path_id': f'empty-{cluster}',
|
|
||||||
'cluster': cluster,
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
'metrics': {
|
|
||||||
'ev_net_r': 1.0 + shift,
|
|
||||||
'calmar': 2.0 + shift,
|
|
||||||
'profit_factor': 1.5 + shift,
|
|
||||||
'gain_to_pain': 2.0 + shift,
|
|
||||||
'sortino': 1.0 + shift,
|
|
||||||
'cagr_pct': 10.0 + shift,
|
|
||||||
'max_drawdown_pct': 5.0,
|
|
||||||
'total_return_pct': 10.0 + shift,
|
|
||||||
'sharpe': 1.0 + shift,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
report = aggregate_results(cells)
|
|
||||||
|
|
||||||
cash_empty = next(
|
|
||||||
row
|
|
||||||
for row in report['paired_per_year']
|
|
||||||
if row['arm_id'] == 'cash_unbounded'
|
|
||||||
and row['protocol'] == 'empty_book'
|
|
||||||
and row['cost_per_side_pct'] == 0.1
|
|
||||||
)
|
|
||||||
assert cash_empty['headline']['ev_net_r']['paired_delta_median'] == pytest.approx(
|
|
||||||
0.2
|
|
||||||
)
|
|
||||||
cash_paths = next(
|
|
||||||
row
|
|
||||||
for row in report['paired_path_distributions']
|
|
||||||
if row['arm_id'] == 'cash_unbounded'
|
|
||||||
and row['protocol'] == 'empty_book'
|
|
||||||
and row['cost_per_side_pct'] == 0.1
|
|
||||||
)
|
|
||||||
assert cash_paths['metrics']['ev_net_r']['paired_delta_mean'] == pytest.approx(
|
|
||||||
0.2
|
|
||||||
)
|
|
||||||
assert cash_paths['metrics']['ev_net_r']['positive_fraction'] == 1.0
|
|
||||||
assert cash_paths['metrics']['ev_net_r']['identical_fraction'] == 0.0
|
|
||||||
cash_warm = next(
|
|
||||||
row
|
|
||||||
for row in report['warm_seed_dispersion']
|
|
||||||
if row['arm_id'] == 'cash_unbounded'
|
|
||||||
and row['cost_per_side_pct'] == 0.1
|
|
||||||
)
|
|
||||||
assert set(cash_warm['headline']) == {'ev_net_r', 'calmar'}
|
|
||||||
assert 'D' not in cash_warm
|
|
||||||
assert cash_warm['headline']['ev_net_r']['median_iqr_ratio'] == pytest.approx(
|
|
||||||
1.0
|
|
||||||
)
|
|
||||||
assert cash_warm['headline']['calmar']['median_iqr_ratio'] == pytest.approx(
|
|
||||||
1.0
|
|
||||||
)
|
|
||||||
assert cash_warm['headline']['ev_net_r']['bootstrap_90']['n'] == 7
|
|
||||||
markdown = _markdown({
|
|
||||||
'generated_at': '2026-08-05T00:00:00Z',
|
|
||||||
'analysis': report,
|
|
||||||
'operational_summary': _operational_summary(cells),
|
|
||||||
'validation': {
|
|
||||||
'construction_universe_manifest': {
|
|
||||||
'construction_symbols_with_prices': 506,
|
|
||||||
'rank_only_symbols_with_prices': 4148,
|
|
||||||
'ranking_symbols_with_prices': 4654,
|
|
||||||
},
|
|
||||||
'candidate_rank_coverage': {
|
|
||||||
'construction_qualified_longs': 5000,
|
|
||||||
'filtered_rank_only_qualified_longs': 137000,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
assert 'ΔGain-to-Pain' in markdown
|
|
||||||
assert '0.10% per fill' in markdown
|
|
||||||
assert '0.20% per fill' in markdown
|
|
||||||
assert 'Tradable setup symbols with prices: 506.' in markdown
|
|
||||||
assert 'Rank-only qualified rows removed: 137000.' in markdown
|
|
||||||
assert 'formal promotion gate' in markdown
|
|
||||||
|
|
||||||
focused_cells = [
|
|
||||||
row
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == 'cap10_incumbent'
|
|
||||||
] + [
|
|
||||||
{
|
|
||||||
**row,
|
|
||||||
'arm_id': 'cap10_min_risk_005',
|
|
||||||
}
|
|
||||||
for row in cells
|
|
||||||
if row['arm_id'] == 'cash_unbounded'
|
|
||||||
]
|
|
||||||
focused_analysis = aggregate_results(
|
|
||||||
focused_cells,
|
|
||||||
arms=RISK_FLOOR_ARMS,
|
|
||||||
include_warm_dispersion=False,
|
|
||||||
)
|
|
||||||
assert focused_analysis['warm_seed_dispersion'] == []
|
|
||||||
focused_markdown = _risk_floor_markdown({
|
|
||||||
'generated_at': '2026-08-05T00:00:00Z',
|
|
||||||
'arms': list(RISK_FLOOR_ARMS),
|
|
||||||
'protocols': ['empty_book', 'warm_book'],
|
|
||||||
'costs_per_side_pct': [0.1, 0.2],
|
|
||||||
'analysis': focused_analysis,
|
|
||||||
'operational_summary': _operational_summary(
|
|
||||||
focused_cells,
|
|
||||||
arms=RISK_FLOOR_ARMS,
|
|
||||||
),
|
|
||||||
})
|
|
||||||
assert '# Effective initial-risk floor A/B' in focused_markdown
|
|
||||||
assert 'Mean dEV' in focused_markdown
|
|
||||||
assert 'Identical' in focused_markdown
|
|
||||||
assert 'Mean dGtP' in focused_markdown
|
|
||||||
assert 'Mean dCalmar/MAR' in focused_markdown
|
|
||||||
assert 'Floor rejects' in focused_markdown
|
|
||||||
assert 'not independent evidence' in focused_markdown
|
|
||||||
|
|
||||||
|
|
||||||
def test_synthetic_worker_matrix_covers_four_arms_protocols_and_costs(monkeypatch):
|
|
||||||
monkeypatch.setenv('BACKTEST_SNAPSHOT_OFFLINE', '0')
|
|
||||||
monkeypatch.setenv('BACKTEST_ALLOW_SPAWN', '0')
|
|
||||||
start = date(2025, 1, 6)
|
|
||||||
sessions = _business_days(start, date(2025, 1, 17))
|
|
||||||
ords = [session.toordinal() for session in sessions]
|
|
||||||
symbols = [f'S{index}' for index in range(12)]
|
|
||||||
prices = {symbol: _prices(ords) for symbol in symbols}
|
|
||||||
candidates = [
|
|
||||||
_candidate(symbol, start, rank=99.0 - index)
|
|
||||||
for index, symbol in enumerate(symbols[:11])
|
|
||||||
]
|
|
||||||
friday = date(2025, 1, 10)
|
|
||||||
candidates.append(_candidate('S11', friday, rank=99.0))
|
|
||||||
rank_map = {
|
|
||||||
(symbol, friday.isoformat()): {
|
|
||||||
'strategy_rank': 100.0 if symbol == 'S11' else float(index)
|
|
||||||
}
|
|
||||||
for index, symbol in enumerate(symbols)
|
|
||||||
}
|
|
||||||
_worker_init({
|
|
||||||
'qualified_candidates': candidates,
|
|
||||||
'daily_rank_map': rank_map,
|
|
||||||
'prices': prices,
|
|
||||||
'benchmark_closes': None,
|
|
||||||
'ranking_key': 'residual_high_vol_blend_80_20',
|
|
||||||
'exit_policy': 'hold',
|
|
||||||
'hold_days': 30,
|
|
||||||
'risk_per_trade': 0.01,
|
|
||||||
'atr_trail_multiplier': 3.0,
|
|
||||||
})
|
|
||||||
|
|
||||||
rows = []
|
|
||||||
for protocol, measurement_start in (
|
|
||||||
('empty_book', start),
|
|
||||||
('warm_book', date(2025, 1, 8)),
|
|
||||||
):
|
|
||||||
for cost in (0.1, 0.2):
|
|
||||||
for arm_id in (
|
|
||||||
'cap10_incumbent',
|
|
||||||
'cash_unbounded',
|
|
||||||
'cap10_weekly_top10',
|
|
||||||
'cap15_incumbent',
|
|
||||||
):
|
|
||||||
rows.append(_worker_run_cell({
|
|
||||||
'cell_id': f'{arm_id}|{protocol}|{cost}',
|
|
||||||
'arm_id': arm_id,
|
|
||||||
'protocol': protocol,
|
|
||||||
'path_id': f'{protocol}-synthetic',
|
|
||||||
'cluster': 2025,
|
|
||||||
'simulation_start': start.isoformat(),
|
|
||||||
'measurement_start': measurement_start.isoformat(),
|
|
||||||
'hard_end_exclusive': date(2025, 1, 14).isoformat(),
|
|
||||||
'cost_per_side_pct': cost,
|
|
||||||
}))
|
|
||||||
|
|
||||||
assert len(rows) == 16
|
|
||||||
assert {row['arm_id'] for row in rows} == {
|
|
||||||
'cap10_incumbent',
|
|
||||||
'cash_unbounded',
|
|
||||||
'cap10_weekly_top10',
|
|
||||||
'cap15_incumbent',
|
|
||||||
}
|
|
||||||
assert {row['protocol'] for row in rows} == {'empty_book', 'warm_book'}
|
|
||||||
assert {row['cost_per_side_pct'] for row in rows} == {0.1, 0.2}
|
|
||||||
assert all('ev_net_r' in row['metrics'] for row in rows)
|
|
||||||
|
|
||||||
|
|
||||||
def test_checkpoint_resume_rejects_fingerprint_mismatch(tmp_path):
|
|
||||||
checkpoint = tmp_path / 'checkpoint'
|
|
||||||
completed = _checkpoint_state(checkpoint, 'fingerprint-a', resume=False)
|
|
||||||
assert completed == {}
|
|
||||||
_write_cell_checkpoint(
|
|
||||||
checkpoint,
|
|
||||||
{'cell_id': 'one', 'metrics': {'ev_net_r': 1.0}},
|
|
||||||
)
|
|
||||||
resumed = _checkpoint_state(checkpoint, 'fingerprint-a', resume=True)
|
|
||||||
assert set(resumed) == {'one'}
|
|
||||||
with pytest.raises(SystemExit, match='fingerprint mismatch'):
|
|
||||||
_checkpoint_state(checkpoint, 'fingerprint-b', resume=True)
|
|
||||||
|
|
||||||
|
|
||||||
def test_dirty_worktree_guard(monkeypatch):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
'scripts.run_portfolio_construction_matrix._git_output',
|
|
||||||
lambda *_args: ' M changed.py',
|
|
||||||
)
|
|
||||||
with pytest.raises(SystemExit, match='dirty worktree'):
|
|
||||||
_assert_clean_worktree()
|
|
||||||
@@ -24,7 +24,6 @@ from app.models.ohlcv import OHLCVRecord
|
|||||||
from app.models.paper_trade import PaperTrade
|
from app.models.paper_trade import PaperTrade
|
||||||
from app.models.signal_context_snapshot import SignalContextSnapshot
|
from app.models.signal_context_snapshot import SignalContextSnapshot
|
||||||
from app.models.sec_filing_gap import SecFilingGap
|
from app.models.sec_filing_gap import SecFilingGap
|
||||||
from app.models.settings import SystemSetting
|
|
||||||
from app.models.sr_level import SRLevel
|
from app.models.sr_level import SRLevel
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.models.trade_setup import TradeSetup
|
from app.models.trade_setup import TradeSetup
|
||||||
@@ -524,10 +523,6 @@ async def test_get_trade_setups_hides_active_sec_filing_gap(
|
|||||||
db_session.add(ticker)
|
db_session.add(ticker)
|
||||||
await db_session.flush()
|
await db_session.flush()
|
||||||
db_session.add_all([
|
db_session.add_all([
|
||||||
SystemSetting(
|
|
||||||
key="fundamental_data_sec_dolt_cutover_enabled",
|
|
||||||
value="true",
|
|
||||||
),
|
|
||||||
SecFilingGap(
|
SecFilingGap(
|
||||||
cik=ticker.cik,
|
cik=ticker.cik,
|
||||||
accession="0000000042-26-000001",
|
accession="0000000042-26-000001",
|
||||||
|
|||||||
@@ -66,26 +66,11 @@ class TestTradingDayCrons:
|
|||||||
assert "Mon" in weekdays, f"{key} skips Mondays — numeric day-of-week?"
|
assert "Mon" in weekdays, f"{key} skips Mondays — numeric day-of-week?"
|
||||||
assert {"Sat", "Sun"}.isdisjoint(weekdays), f"{key} fires on a weekend"
|
assert {"Sat", "Sun"}.isdisjoint(weekdays), f"{key} fires on a weekend"
|
||||||
|
|
||||||
def test_fundamentals_runs_on_monday(self):
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from apscheduler.triggers.cron import CronTrigger
|
|
||||||
|
|
||||||
trigger = CronTrigger.from_crontab(
|
|
||||||
SCHEDULE_DEFAULTS["schedule_fundamentals_cron"],
|
|
||||||
timezone=SCHEDULE_DEFAULTS["schedule_timezone"],
|
|
||||||
)
|
|
||||||
fire = trigger.get_next_fire_time(
|
|
||||||
None, datetime(2026, 7, 19, tzinfo=trigger.timezone)
|
|
||||||
)
|
|
||||||
assert fire.strftime("%a") == "Mon"
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("key", "hour", "minute"),
|
("key", "hour", "minute"),
|
||||||
(
|
(
|
||||||
("schedule_dolt_earnings_cron", 2, 30),
|
("schedule_dolt_earnings_cron", 2, 30),
|
||||||
("schedule_sec_fundamentals_cron", 4, 0),
|
("schedule_sec_fundamentals_cron", 4, 0),
|
||||||
("schedule_fundamentals_parity_cron", 5, 30),
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_shadow_imports_run_daily_at_expected_et_time(
|
def test_shadow_imports_run_daily_at_expected_et_time(
|
||||||
@@ -122,7 +107,7 @@ class TestScheduleConfig:
|
|||||||
|
|
||||||
async def test_rejects_bad_cron(self, session: AsyncSession):
|
async def test_rejects_bad_cron(self, session: AsyncSession):
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
await update_schedule_config(session, {"schedule_fundamentals_cron": "every monday"})
|
await update_schedule_config(session, {"schedule_daily_pipeline_cron": "every monday"})
|
||||||
|
|
||||||
async def test_rejects_bad_timezone(self, session: AsyncSession):
|
async def test_rejects_bad_timezone(self, session: AsyncSession):
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Unit tests for app.scheduler module."""
|
"""Unit tests for app.scheduler module."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -12,9 +13,7 @@ from app.scheduler import (
|
|||||||
_parse_frequency,
|
_parse_frequency,
|
||||||
_resume_tickers,
|
_resume_tickers,
|
||||||
_last_successful,
|
_last_successful,
|
||||||
_run_shadow_import,
|
_run_source_import,
|
||||||
collect_fundamentals,
|
|
||||||
run_fundamentals_parity_report,
|
|
||||||
run_sec_fundamentals_import,
|
run_sec_fundamentals_import,
|
||||||
configure_scheduler,
|
configure_scheduler,
|
||||||
get_job_runtime_snapshot,
|
get_job_runtime_snapshot,
|
||||||
@@ -123,10 +122,8 @@ class TestConfigureScheduler:
|
|||||||
"data_backfill",
|
"data_backfill",
|
||||||
"benchmark_collector",
|
"benchmark_collector",
|
||||||
"sentiment_collector",
|
"sentiment_collector",
|
||||||
"fundamental_collector",
|
|
||||||
"dolt_earnings_import",
|
"dolt_earnings_import",
|
||||||
"sec_fundamentals_import",
|
"sec_fundamentals_import",
|
||||||
"fundamentals_parity_report",
|
|
||||||
"rr_scanner",
|
"rr_scanner",
|
||||||
"shadow_book",
|
"shadow_book",
|
||||||
"ticker_universe_sync",
|
"ticker_universe_sync",
|
||||||
@@ -157,10 +154,8 @@ class TestConfigureScheduler:
|
|||||||
"intraday_pipeline",
|
"intraday_pipeline",
|
||||||
"data_collector",
|
"data_collector",
|
||||||
"data_backfill",
|
"data_backfill",
|
||||||
"fundamental_collector",
|
|
||||||
"dolt_earnings_import",
|
"dolt_earnings_import",
|
||||||
"sec_fundamentals_import",
|
"sec_fundamentals_import",
|
||||||
"fundamentals_parity_report",
|
|
||||||
"market_regime",
|
"market_regime",
|
||||||
"near_close_pipeline",
|
"near_close_pipeline",
|
||||||
"regime_monitor",
|
"regime_monitor",
|
||||||
@@ -181,42 +176,7 @@ class _SessionContext:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
class TestFundamentalCollector:
|
class TestSourceImportJobs:
|
||||||
@staticmethod
|
|
||||||
def _session_factory():
|
|
||||||
return _SessionContext()
|
|
||||||
|
|
||||||
async def test_skips_legacy_provider_when_cutover_is_active(self, monkeypatch):
|
|
||||||
async def enabled(db, job_name):
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def cutover_enabled(db):
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def unexpected_ticker_lookup(db):
|
|
||||||
raise AssertionError("legacy ticker lookup must not run after cutover")
|
|
||||||
|
|
||||||
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
|
||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"app.scheduler.fundamental_data_refresh_service.is_enabled",
|
|
||||||
cutover_enabled,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"app.scheduler._get_fundamental_priority_tickers",
|
|
||||||
unexpected_ticker_lookup,
|
|
||||||
)
|
|
||||||
|
|
||||||
await collect_fundamentals()
|
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("fundamental_collector")
|
|
||||||
assert runtime["status"] == "skipped"
|
|
||||||
assert runtime["processed"] == 0
|
|
||||||
assert runtime["total"] == 0
|
|
||||||
assert runtime["message"] == "SEC + Dolt fundamentals cutover is active"
|
|
||||||
|
|
||||||
|
|
||||||
class TestShadowImportJobs:
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _session_factory():
|
def _session_factory():
|
||||||
return _SessionContext()
|
return _SessionContext()
|
||||||
@@ -234,7 +194,7 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", imported)
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
|
|
||||||
await _run_shadow_import("dolt_earnings_import", object())
|
await _run_source_import("dolt_earnings_import", object())
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("dolt_earnings_import")
|
runtime = get_job_runtime_snapshot("dolt_earnings_import")
|
||||||
assert runtime["status"] == "completed"
|
assert runtime["status"] == "completed"
|
||||||
@@ -254,7 +214,7 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", imported)
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
|
|
||||||
await _run_shadow_import("sec_fundamentals_import", object())
|
await _run_source_import("sec_fundamentals_import", object())
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
assert runtime["status"] == "error"
|
assert runtime["status"] == "error"
|
||||||
@@ -276,7 +236,7 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", imported)
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
|
|
||||||
await _run_shadow_import("sec_fundamentals_import", object())
|
await _run_source_import("sec_fundamentals_import", object())
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
assert runtime["status"] == STATUS_DEFERRED
|
assert runtime["status"] == STATUS_DEFERRED
|
||||||
@@ -294,7 +254,7 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", imported)
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
|
|
||||||
await _run_shadow_import("dolt_earnings_import", object())
|
await _run_source_import("dolt_earnings_import", object())
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("dolt_earnings_import")
|
runtime = get_job_runtime_snapshot("dolt_earnings_import")
|
||||||
assert runtime["status"] == "skipped"
|
assert runtime["status"] == "skipped"
|
||||||
@@ -311,14 +271,15 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
|
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
|
||||||
|
|
||||||
await _run_shadow_import("sec_fundamentals_import", object())
|
await _run_source_import("sec_fundamentals_import", object())
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
assert runtime["status"] == "skipped"
|
assert runtime["status"] == "skipped"
|
||||||
assert runtime["message"] == "Disabled"
|
assert runtime["message"] == "Disabled"
|
||||||
|
|
||||||
async def test_sec_failure_still_runs_activated_local_refresh(self, monkeypatch):
|
async def test_sec_failure_still_runs_local_cache_refresh(self, monkeypatch):
|
||||||
calls = []
|
calls = []
|
||||||
|
events = []
|
||||||
|
|
||||||
async def enabled(db, job_name):
|
async def enabled(db, job_name):
|
||||||
return True
|
return True
|
||||||
@@ -329,29 +290,40 @@ class TestShadowImportJobs:
|
|||||||
async def refreshed(db):
|
async def refreshed(db):
|
||||||
calls.append(db)
|
calls.append(db)
|
||||||
return {
|
return {
|
||||||
"enabled": True,
|
|
||||||
"refreshed": 511,
|
"refreshed": 511,
|
||||||
"score_inputs_changed": 2,
|
"score_inputs_changed": 2,
|
||||||
"dimension_scores_staled": 2,
|
"dimension_scores_staled": 2,
|
||||||
"composite_scores_staled": 2,
|
"composite_scores_staled": 2,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async def record(**kwargs):
|
||||||
|
events.append(kwargs)
|
||||||
|
|
||||||
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", unavailable)
|
monkeypatch.setattr("app.scheduler.run_import", unavailable)
|
||||||
|
monkeypatch.setattr("app.scheduler._record_system_event", record)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
"app.scheduler.fundamental_data_refresh_service.refresh",
|
||||||
refreshed,
|
refreshed,
|
||||||
)
|
)
|
||||||
|
|
||||||
await run_sec_fundamentals_import()
|
await run_sec_fundamentals_import()
|
||||||
|
await asyncio.sleep(0) # let the fire-and-forget event task run
|
||||||
|
|
||||||
assert len(calls) == 1
|
assert len(calls) == 1
|
||||||
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
assert runtime["status"] == "error"
|
assert runtime["status"] == "error"
|
||||||
assert runtime["message"] == "SEC unavailable"
|
# the failure stays the headline, but the cache result is still visible
|
||||||
|
assert runtime["message"] == (
|
||||||
|
"SEC unavailable · cache 511 · 2 score inputs changed"
|
||||||
|
)
|
||||||
|
# Rewording the outcome must not duplicate the durable event: the dedup
|
||||||
|
# key includes the message, so a second finish would show up twice in
|
||||||
|
# Admin → System Events.
|
||||||
|
assert len(events) == 1, events
|
||||||
|
|
||||||
async def test_sec_success_surfaces_activated_refresh_summary(self, monkeypatch):
|
async def test_sec_success_surfaces_cache_refresh_summary(self, monkeypatch):
|
||||||
async def enabled(db, job_name):
|
async def enabled(db, job_name):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -362,7 +334,6 @@ class TestShadowImportJobs:
|
|||||||
|
|
||||||
async def refreshed(db):
|
async def refreshed(db):
|
||||||
return {
|
return {
|
||||||
"enabled": True,
|
|
||||||
"refreshed": 511,
|
"refreshed": 511,
|
||||||
"score_inputs_changed": 2,
|
"score_inputs_changed": 2,
|
||||||
"dimension_scores_staled": 2,
|
"dimension_scores_staled": 2,
|
||||||
@@ -373,7 +344,7 @@ class TestShadowImportJobs:
|
|||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", imported)
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
"app.scheduler.fundamental_data_refresh_service.refresh",
|
||||||
refreshed,
|
refreshed,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -385,52 +356,79 @@ class TestShadowImportJobs:
|
|||||||
"no_op · abcdef123456 · cache 511 · 2 score inputs changed"
|
"no_op · abcdef123456 · cache 511 · 2 score inputs changed"
|
||||||
)
|
)
|
||||||
|
|
||||||
async def test_disabled_sec_job_does_not_run_local_refresh(self, monkeypatch):
|
async def test_source_locked_sec_run_still_reports_the_cache_refresh(
|
||||||
async def disabled(db, job_name):
|
self, monkeypatch
|
||||||
return False
|
):
|
||||||
|
"""A skipped import keeps its skip status but shows the cache advanced."""
|
||||||
|
|
||||||
async def should_not_run(*args, **kwargs):
|
async def enabled(db, job_name):
|
||||||
raise AssertionError("disabled SEC job ran work")
|
return True
|
||||||
|
|
||||||
|
async def locked(importer):
|
||||||
|
return None # another import owns the source lock
|
||||||
|
|
||||||
|
async def refreshed(db):
|
||||||
|
return {
|
||||||
|
"refreshed": 511,
|
||||||
|
"score_inputs_changed": 0,
|
||||||
|
"dimension_scores_staled": 0,
|
||||||
|
"composite_scores_staled": 0,
|
||||||
|
}
|
||||||
|
|
||||||
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
|
monkeypatch.setattr("app.scheduler.run_import", locked)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
"app.scheduler.fundamental_data_refresh_service.refresh",
|
||||||
should_not_run,
|
refreshed,
|
||||||
)
|
)
|
||||||
|
|
||||||
await run_sec_fundamentals_import()
|
await run_sec_fundamentals_import()
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
assert runtime["status"] == "skipped"
|
assert runtime["status"] == "skipped"
|
||||||
assert runtime["message"] == "Disabled"
|
assert runtime["message"] == (
|
||||||
|
"Another import for this source is already running · "
|
||||||
|
"cache 511 · 0 score inputs changed"
|
||||||
async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch):
|
|
||||||
async def enabled(db, job_name):
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def generated(db, report_dir):
|
|
||||||
return (
|
|
||||||
{
|
|
||||||
"generated_at": "2026-07-23T10:30:00+00:00",
|
|
||||||
"summary": {
|
|
||||||
"universe_count": 511,
|
|
||||||
"fundamental_score_material_changes": 12,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{"json": "report.json", "csv": "report.csv"},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
monkeypatch.setattr("app.scheduler.async_session_factory", TestShadowImportJobs._session_factory)
|
async def test_disabled_sec_job_still_refreshes_local_cache(self, monkeypatch):
|
||||||
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
"""Disabling the job stops the SEC fetch, not the local cache.
|
||||||
monkeypatch.setattr(
|
|
||||||
"app.scheduler.fundamentals_parity_service.generate_and_store", generated
|
|
||||||
)
|
|
||||||
|
|
||||||
await run_fundamentals_parity_report()
|
The cache is derived from stored snapshots, earnings events and closes.
|
||||||
|
Prices and earnings move daily even when no filing does, and there is no
|
||||||
|
provider fallback since A6 — freezing it would silently stale scoring.
|
||||||
|
"""
|
||||||
|
calls = []
|
||||||
|
|
||||||
runtime = get_job_runtime_snapshot("fundamentals_parity_report")
|
async def disabled(db, job_name):
|
||||||
assert runtime["status"] == "completed"
|
return False
|
||||||
assert runtime["message"] == "511 tickers · 12 material score changes"
|
|
||||||
|
async def should_not_run(*args, **kwargs):
|
||||||
|
raise AssertionError("disabled SEC job hit the network")
|
||||||
|
|
||||||
|
async def refreshed(db):
|
||||||
|
calls.append(db)
|
||||||
|
return {
|
||||||
|
"refreshed": 511,
|
||||||
|
"score_inputs_changed": 2,
|
||||||
|
"dimension_scores_staled": 2,
|
||||||
|
"composite_scores_staled": 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
|
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
|
||||||
|
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.scheduler.fundamental_data_refresh_service.refresh",
|
||||||
|
refreshed,
|
||||||
|
)
|
||||||
|
|
||||||
|
await run_sec_fundamentals_import()
|
||||||
|
|
||||||
|
assert len(calls) == 1
|
||||||
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
|
assert runtime["status"] == "completed"
|
||||||
|
assert runtime["message"] == (
|
||||||
|
"Import disabled · cache 511 · 2 score inputs changed"
|
||||||
|
)
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
from app.database import Base
|
from app.database import Base
|
||||||
from app.exceptions import ProviderError
|
|
||||||
from app.models.settings import SystemSetting
|
from app.models.settings import SystemSetting
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
from app.services import ticker_universe_service
|
from app.services import ticker_universe_service
|
||||||
@@ -97,11 +96,7 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai
|
|||||||
async def _fake_public(_universe: str):
|
async def _fake_public(_universe: str):
|
||||||
return [], ["public failed"], None
|
return [], ["public failed"], None
|
||||||
|
|
||||||
async def _fake_fmp(_universe: str):
|
|
||||||
raise ProviderError("fmp failed")
|
|
||||||
|
|
||||||
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
|
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
|
||||||
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
|
|
||||||
|
|
||||||
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
|
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
|
||||||
assert symbols == ["AAPL", "MSFT"]
|
assert symbols == ["AAPL", "MSFT"]
|
||||||
@@ -116,11 +111,7 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail(
|
|||||||
async def _fake_public(_universe: str):
|
async def _fake_public(_universe: str):
|
||||||
return [], ["public failed"], None
|
return [], ["public failed"], None
|
||||||
|
|
||||||
async def _fake_fmp(_universe: str):
|
|
||||||
raise ProviderError("fmp failed")
|
|
||||||
|
|
||||||
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
|
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
|
||||||
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
|
|
||||||
|
|
||||||
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
|
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
|
||||||
assert "AAPL" in symbols
|
assert "AAPL" in symbols
|
||||||
|
|||||||
Reference in New Issue
Block a user