dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: decommission FMP, Finnhub and Alpha Vantage (A6)

Browse Source 
The A5 cutover has been on and observed in production, so SEC Company Facts +
DoltHub earnings are already the live source for `fundamental_data`. This
removes everything the legacy path still occupied.

Gone: the three providers and their config/env keys; the weekly
`fundamental_collector` job; the cutover toggle (SEC + Dolt is now the
unconditional path, so `off` can no longer silently freeze scoring inputs); the
A5 parity report, whose deltas became structurally zero once the candidate
builder started writing the table it compared against; and the FMP tier of
universe bootstrap.

Two behavioral notes:

- Disabling **SEC Fundamentals Import** now stops the SEC network fetch only.
  The local cache refresh moved outside the job-enable check, because candidates
  also derive from daily closes and earnings events — freezing those on an
  ingestion pause would stale scoring with no fallback left to recover from.
- `/ingestion/fetch?sources=fundamentals` still accepts the key and reports
  `skipped`; there is no per-ticker fetch any more.

Migration 029 does not blanket-delete the leftover settings rows. Migrations run
before the service restart, and pre-A6 code reads an absent `job_*_enabled` row
as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe
values (hidden in Admin) and only the inert three are deleted. Removing the
provider keys from the production `.env` is the matching rollout step.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dennisthiessen
2026-08-07 11:19:28 +02:00
co-authored by Claude Opus 5
parent f5d4b516ab
commit 3e83d63b05
51 changed files with 368 additions and 3787 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_activation_settings.py
-17
View File
@@ -8,9 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.exceptions import ValidationError
from app.services.admin_service import (
get_activation_config,
get_fundamentals_cutover_config,
update_activation_config,
update_fundamentals_cutover_config,
)
@@ -78,18 +76,3 @@ class TestActivationConfig:
async def test_rejects_out_of_range_confidence(self, session: AsyncSession):
with pytest.raises(ValidationError):
await update_activation_config(session, {"min_confidence": 120.0})
class TestFundamentalsCutoverConfig:
async def test_defaults_off_when_unset(self, session: AsyncSession):
assert await get_fundamentals_cutover_config(session) == {"enabled": False}
async def test_round_trips_explicit_switch(self, session: AsyncSession):
assert await update_fundamentals_cutover_config(session, True) == {
"enabled": True
}
assert await get_fundamentals_cutover_config(session) == {"enabled": True}
assert await update_fundamentals_cutover_config(session, False) == {
"enabled": False
}
tests/unit/test_earnings_research.py
-37
View File
@@ -1,6 +1,5 @@
from datetime import date, timedelta
from scripts.backfill_earnings_events import _dedupe_bulk_rows, _windows
from scripts.import_dolthub_earnings import _align_symbol
from scripts.run_earnings_research import (
_analyse_2a_trades,
@@ -41,42 +40,6 @@ def test_dolthub_alignment_allows_fiscal_period_label_after_announcement() -> No
assert matches == [(0, 0), (1, 1)]
def test_bulk_windows_cover_range_without_overlap() -> None:
result = _windows(date(2020, 1, 1), date(2020, 1, 10), 4)
assert result == [
(date(2020, 1, 1), date(2020, 1, 4)),
(date(2020, 1, 5), date(2020, 1, 8)),
(date(2020, 1, 9), date(2020, 1, 10)),
]
def test_bulk_dedupe_prefers_more_complete_and_counts_restatement() -> None:
rows = [
{
"symbol": "AAPL",
"announce_date": "2024-01-01",
"announce_time": None,
"eps_estimate": 1.0,
"eps_actual": 1.1,
"revenue_estimate": None,
"revenue_actual": None,
},
{
"symbol": "AAPL",
"announce_date": "2024-01-01",
"announce_time": "amc",
"eps_estimate": 1.0,
"eps_actual": 1.2,
"revenue_estimate": 10.0,
"revenue_actual": 11.0,
},
]
deduped, duplicates, restated = _dedupe_bulk_rows(rows)
assert duplicates == 1
assert restated == 1
assert deduped == [rows[1]]
def test_2a_uses_net_r_strict_hold_and_next_session_stop() -> None:
calendar = [
date(2024, 1, 2),
tests/unit/test_finnhub_provider.py
-92
View File
@@ -1,92 +0,0 @@
"""Unit tests for FinnhubFundamentalProvider unit conversions."""
from __future__ import annotations
from unittest.mock import AsyncMock, patch
import httpx
import pytest
from app.providers.fundamentals_chain import FinnhubFundamentalProvider
def _mock_response(status_code: int, json_data: object = None) -> httpx.Response:
return httpx.Response(
status_code=status_code,
json=json_data if json_data is not None else {},
request=httpx.Request("GET", "https://example.com"),
)
@pytest.fixture
def provider() -> FinnhubFundamentalProvider:
return FinnhubFundamentalProvider(api_key="test-key")
@pytest.mark.asyncio
async def test_finnhub_market_cap_converted_from_millions_to_dollars(provider):
"""Finnhub marketCapitalization is in millions — store absolute USD.
SPCX-scale example: ~$1.8T → Finnhub reports 1_800_000 (millions).
Without conversion the UI showed 1.8M / micro cap.
"""
profile = {"marketCapitalization": 1_800_000} # millions → $1.8T
metrics = {"metric": {"peTTM": 40.0, "revenueGrowthTTMYoy": 25.0}}
earnings = [{"surprisePercent": 2.5}]
calendar = {"earningsCalendar": []}
async def mock_get(url, params=None):
if "profile2" in url:
return _mock_response(200, profile)
if "stock/metric" in url:
return _mock_response(200, metrics)
if "stock/earnings" in url:
return _mock_response(200, earnings)
if "calendar/earnings" in url:
return _mock_response(200, calendar)
return _mock_response(200, {})
with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
instance = AsyncMock()
instance.get.side_effect = mock_get
instance.__aenter__ = AsyncMock(return_value=instance)
instance.__aexit__ = AsyncMock(return_value=False)
MockClient.return_value = instance
result = await provider.fetch_fundamentals("SPCX")
assert result.market_cap == 1_800_000 * 1_000_000 # $1.8T
assert result.pe_ratio == 40.0
assert result.revenue_growth == 25.0
assert result.earnings_surprise == 2.5
@pytest.mark.asyncio
async def test_finnhub_market_cap_none_when_missing(provider):
profile: dict = {}
metrics = {"metric": {}}
earnings: list = []
calendar = {"earningsCalendar": []}
async def mock_get(url, params=None):
if "profile2" in url:
return _mock_response(200, profile)
if "stock/metric" in url:
return _mock_response(200, metrics)
if "stock/earnings" in url:
return _mock_response(200, earnings)
if "calendar/earnings" in url:
return _mock_response(200, calendar)
return _mock_response(200, {})
with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
instance = AsyncMock()
instance.get.side_effect = mock_get
instance.__aenter__ = AsyncMock(return_value=instance)
instance.__aexit__ = AsyncMock(return_value=False)
MockClient.return_value = instance
result = await provider.fetch_fundamentals("XYZ")
assert result.market_cap is None
assert "market_cap" in result.unavailable_fields
tests/unit/test_fmp_provider.py
-156
View File
@@ -1,156 +0,0 @@
"""Unit tests for FMPFundamentalProvider 402 reason recording."""
from __future__ import annotations
from unittest.mock import AsyncMock, patch
import httpx
import pytest
from app.providers.fmp import FMPFundamentalProvider
def _mock_response(status_code: int, json_data: object = None) -> httpx.Response:
"""Build a fake httpx.Response."""
resp = httpx.Response(
status_code=status_code,
json=json_data if json_data is not None else {},
request=httpx.Request("GET", "https://example.com"),
)
return resp
@pytest.fixture
def provider() -> FMPFundamentalProvider:
return FMPFundamentalProvider(api_key="test-key")
class TestFetchJsonOptional402Tracking:
"""_fetch_json_optional returns (data, was_402) tuple."""
@pytest.mark.asyncio
async def test_returns_empty_dict_and_true_on_402(self, provider):
mock_client = AsyncMock()
mock_client.get.return_value = _mock_response(402)
data, was_402 = await provider._fetch_json_optional(
mock_client, "ratios-ttm", {}, "AAPL"
)
assert data == {}
assert was_402 is True
@pytest.mark.asyncio
async def test_returns_data_and_false_on_200(self, provider):
mock_client = AsyncMock()
mock_client.get.return_value = _mock_response(
200, [{"priceToEarningsRatioTTM": 25.5}]
)
data, was_402 = await provider._fetch_json_optional(
mock_client, "ratios-ttm", {}, "AAPL"
)
assert data == {"priceToEarningsRatioTTM": 25.5}
assert was_402 is False
class TestFetchFundamentals402Recording:
"""fetch_fundamentals records 402 endpoints in unavailable_fields."""
@pytest.mark.asyncio
async def test_all_402_records_all_fields(self, provider):
"""When all supplementary endpoints return 402, all three fields are recorded."""
profile_resp = _mock_response(200, [{"marketCap": 1_000_000}])
ratios_resp = _mock_response(402)
growth_resp = _mock_response(402)
earnings_resp = _mock_response(402)
async def mock_get(url, params=None):
if "profile" in url:
return profile_resp
if "ratios-ttm" in url:
return ratios_resp
if "financial-growth" in url:
return growth_resp
if "earnings" in url:
return earnings_resp
return _mock_response(200, [{}])
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
instance = AsyncMock()
instance.get.side_effect = mock_get
instance.__aenter__ = AsyncMock(return_value=instance)
instance.__aexit__ = AsyncMock(return_value=False)
MockClient.return_value = instance
result = await provider.fetch_fundamentals("AAPL")
assert result.unavailable_fields == {
"pe_ratio": "requires paid plan",
"revenue_growth": "requires paid plan",
"earnings_surprise": "requires paid plan",
}
@pytest.mark.asyncio
async def test_mixed_200_402_records_only_402_fields(self, provider):
"""When only ratios-ttm returns 402, only pe_ratio is recorded."""
profile_resp = _mock_response(200, [{"marketCap": 2_000_000}])
ratios_resp = _mock_response(402)
growth_resp = _mock_response(200, [{"revenueGrowth": 0.15}])
earnings_resp = _mock_response(200, [{"epsActual": 3.0, "epsEstimated": 2.5}])
async def mock_get(url, params=None):
if "profile" in url:
return profile_resp
if "ratios-ttm" in url:
return ratios_resp
if "financial-growth" in url:
return growth_resp
if "earnings" in url:
return earnings_resp
return _mock_response(200, [{}])
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
instance = AsyncMock()
instance.get.side_effect = mock_get
instance.__aenter__ = AsyncMock(return_value=instance)
instance.__aexit__ = AsyncMock(return_value=False)
MockClient.return_value = instance
result = await provider.fetch_fundamentals("AAPL")
assert result.unavailable_fields == {"pe_ratio": "requires paid plan"}
assert result.revenue_growth == 0.15
assert result.earnings_surprise is not None
@pytest.mark.asyncio
async def test_no_402_empty_unavailable_fields(self, provider):
"""When all endpoints succeed, unavailable_fields is empty."""
profile_resp = _mock_response(200, [{"marketCap": 3_000_000}])
ratios_resp = _mock_response(200, [{"priceToEarningsRatioTTM": 20.0}])
growth_resp = _mock_response(200, [{"revenueGrowth": 0.10}])
earnings_resp = _mock_response(200, [{"epsActual": 2.0, "epsEstimated": 1.8}])
async def mock_get(url, params=None):
if "profile" in url:
return profile_resp
if "ratios-ttm" in url:
return ratios_resp
if "financial-growth" in url:
return growth_resp
if "earnings" in url:
return earnings_resp
return _mock_response(200, [{}])
with patch("app.providers.fmp.httpx.AsyncClient") as MockClient:
instance = AsyncMock()
instance.get.side_effect = mock_get
instance.__aenter__ = AsyncMock(return_value=instance)
instance.__aexit__ = AsyncMock(return_value=False)
MockClient.return_value = instance
result = await provider.fetch_fundamentals("AAPL")
assert result.unavailable_fields == {}
assert result.pe_ratio == 20.0
tests/unit/test_fundamental_data_refresh.py
+3 -44
View File
@@ -15,7 +15,6 @@ from app.models.fundamental import FundamentalData
from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.ohlcv import OHLCVRecord
from app.models.score import CompositeScore, DimensionScore
from app.models.settings import SystemSetting
from app.models.ticker import Ticker
from app.services import fundamentals_candidate_service as candidates
from app.services import fundamentals_derivation as deriv
@@ -76,49 +75,9 @@ def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]:
return rows
async def test_default_off_performs_no_candidate_read_or_write(
session: AsyncSession, monkeypatch
):
ticker = Ticker(symbol="AAA")
session.add(ticker)
await session.flush()
session.add(
FundamentalData(
ticker_id=ticker.id,
pe_ratio=12,
revenue_growth=3,
earnings_surprise=1,
market_cap=100,
fetched_at=NOW,
)
)
await session.commit()
async def should_not_read(*args, **kwargs):
raise AssertionError("default-off refresh derived candidates")
monkeypatch.setattr(candidates, "build_candidates", should_not_read)
summary = await refresh_service.refresh_if_enabled(session, today=TODAY)
stored = await session.scalar(
select(FundamentalData).where(FundamentalData.ticker_id == ticker.id)
)
assert summary == {
"enabled": False,
"refreshed": 0,
"score_inputs_changed": 0,
"dimension_scores_staled": 0,
"composite_scores_staled": 0,
}
assert stored.pe_ratio == 12
async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
async def test_refresh_updates_all_fields_and_invalidates_scores(
session: AsyncSession,
):
session.add(
SystemSetting(key=refresh_service.ACTIVATION_KEY, value="true")
)
first = Ticker(symbol="AAA", cik="0000000001")
second = Ticker(symbol="AAB", cik="0000000001")
session.add_all([first, second])
@@ -191,7 +150,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
)
await session.commit()
summary = await refresh_service.refresh_if_enabled(
summary = await refresh_service.refresh(
session, now=NOW, today=TODAY
)
@@ -225,7 +184,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
for row in (*dimensions, *composites):
row.is_stale = False
await session.commit()
unchanged = await refresh_service.refresh_if_enabled(
unchanged = await refresh_service.refresh(
session, now=NOW + timedelta(hours=1), today=TODAY
)
assert unchanged["score_inputs_changed"] == 0
tests/unit/test_fundamental_service.py
+28 -42
View File
@@ -1,13 +1,20 @@
"""Unit tests for fundamental_service — unavailable_fields persistence."""
"""Unit tests for fundamental_service — the surviving read path.
Writes to ``fundamental_data`` are covered by test_fundamental_data_refresh.py;
this file only guards the lookup used by the router and scoring.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.database import Base
from app.exceptions import NotFoundError
from app.models.fundamental import FundamentalData
from app.models.ticker import Ticker
from app.services import fundamental_service
@@ -43,57 +50,36 @@ async def ticker(session: AsyncSession) -> Ticker:
@pytest.mark.asyncio
async def test_store_fundamental_persists_unavailable_fields(
async def test_get_fundamental_returns_the_cached_row(
session: AsyncSession, ticker: Ticker
):
"""unavailable_fields dict is serialized to JSON and stored."""
fields = {"pe_ratio": "requires paid plan", "revenue_growth": "requires paid plan"}
record = await fundamental_service.store_fundamental(
session,
symbol="AAPL",
pe_ratio=None,
revenue_growth=None,
market_cap=1_000_000.0,
unavailable_fields=fields,
fields = {"pe_ratio": "split guard applied"}
session.add(
FundamentalData(
ticker_id=ticker.id,
pe_ratio=None,
market_cap=1_000_000.0,
fetched_at=datetime.now(timezone.utc),
unavailable_fields_json=json.dumps(fields),
)
)
await session.commit()
record = await fundamental_service.get_fundamental(session, symbol="aapl")
assert record is not None
assert record.market_cap == 1_000_000.0
assert json.loads(record.unavailable_fields_json) == fields
@pytest.mark.asyncio
async def test_store_fundamental_defaults_to_empty_dict(
async def test_get_fundamental_returns_none_without_a_cached_row(
session: AsyncSession, ticker: Ticker
):
"""When unavailable_fields is not provided, column defaults to '{}'."""
record = await fundamental_service.store_fundamental(
session,
symbol="AAPL",
pe_ratio=25.0,
)
assert json.loads(record.unavailable_fields_json) == {}
assert await fundamental_service.get_fundamental(session, symbol="AAPL") is None
@pytest.mark.asyncio
async def test_store_fundamental_updates_unavailable_fields(
session: AsyncSession, ticker: Ticker
):
"""Updating an existing record also updates unavailable_fields_json."""
# First store
await fundamental_service.store_fundamental(
session,
symbol="AAPL",
pe_ratio=None,
unavailable_fields={"pe_ratio": "requires paid plan"},
)
# Second store — fields now available
record = await fundamental_service.store_fundamental(
session,
symbol="AAPL",
pe_ratio=25.0,
unavailable_fields={},
)
assert json.loads(record.unavailable_fields_json) == {}
async def test_get_fundamental_rejects_an_unknown_symbol(session: AsyncSession):
with pytest.raises(NotFoundError):
await fundamental_service.get_fundamental(session, symbol="NOPE")
tests/unit/test_fundamentals_chain_provider.py
-181
View File
@@ -1,181 +0,0 @@
"""Unit tests for chained fundamentals provider fallback behavior."""
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from app.exceptions import ProviderError, RateLimitError
from app.providers.fundamentals_chain import ChainedFundamentalProvider
from app.providers.protocol import FundamentalData
class _FailProvider:
def __init__(self, message: str) -> None:
self._message = message
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
raise ProviderError(f"{self._message} ({ticker})")
class _RateLimitedProvider:
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
raise RateLimitError(f"rate limit hit for {ticker}")
class _DataProvider:
def __init__(self, data: FundamentalData) -> None:
self._data = data
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
return FundamentalData(
ticker=ticker,
pe_ratio=self._data.pe_ratio,
revenue_growth=self._data.revenue_growth,
earnings_surprise=self._data.earnings_surprise,
market_cap=self._data.market_cap,
fetched_at=self._data.fetched_at,
unavailable_fields=self._data.unavailable_fields,
)
@pytest.mark.asyncio
async def test_chained_provider_uses_fallback_provider_on_primary_failure():
fallback_data = FundamentalData(
ticker="AAPL",
pe_ratio=25.0,
revenue_growth=None,
earnings_surprise=None,
market_cap=1_000_000.0,
fetched_at=datetime.now(timezone.utc),
unavailable_fields={},
)
provider = ChainedFundamentalProvider([
("primary", _FailProvider("primary down")),
("fallback", _DataProvider(fallback_data)),
])
result = await provider.fetch_fundamentals("AAPL")
assert result.pe_ratio == 25.0
assert result.market_cap == 1_000_000.0
assert result.unavailable_fields.get("source_pe_ratio") == "fallback"
@pytest.mark.asyncio
async def test_chained_provider_merges_fields_across_providers():
"""Primary supplies only market cap; fallback fills P/E and earnings."""
primary_data = FundamentalData(
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
)
fallback_data = FundamentalData(
ticker="AAPL", pe_ratio=18.0, revenue_growth=12.0, earnings_surprise=4.0,
market_cap=999.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
)
provider = ChainedFundamentalProvider([
("fmp", _DataProvider(primary_data)),
("finnhub", _DataProvider(fallback_data)),
])
result = await provider.fetch_fundamentals("AAPL")
# market cap from primary (first to supply it), the rest from fallback
assert result.market_cap == 2_000_000.0
assert result.pe_ratio == 18.0
assert result.revenue_growth == 12.0
assert result.earnings_surprise == 4.0
assert result.unavailable_fields.get("source_market_cap") == "fmp"
assert result.unavailable_fields.get("source_pe_ratio") == "finnhub"
@pytest.mark.asyncio
async def test_chained_provider_raises_when_all_providers_fail():
provider = ChainedFundamentalProvider([
("p1", _FailProvider("p1 failed")),
("p2", _FailProvider("p2 failed")),
])
with pytest.raises(ProviderError) as exc:
await provider.fetch_fundamentals("MSFT")
assert "All fundamentals providers failed" in str(exc.value)
@pytest.mark.asyncio
async def test_rate_limited_fallback_raises_when_incomplete():
"""FMP gives market cap; the fallback is rate-limited → chain signals it so
the collector can back off instead of storing a degraded record."""
primary_data = FundamentalData(
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
)
provider = ChainedFundamentalProvider([
("fmp", _DataProvider(primary_data)),
("finnhub", _RateLimitedProvider()),
])
with pytest.raises(RateLimitError):
await provider.fetch_fundamentals("AAPL")
@pytest.mark.asyncio
async def test_rate_limited_fallback_allows_partial():
"""With allow_partial=True the chain returns the market cap it did get."""
primary_data = FundamentalData(
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
)
provider = ChainedFundamentalProvider([
("fmp", _DataProvider(primary_data)),
("finnhub", _RateLimitedProvider()),
])
result = await provider.fetch_fundamentals("AAPL", allow_partial=True)
assert result.market_cap == 2_000_000.0
assert result.pe_ratio is None
@pytest.mark.asyncio
async def test_rate_limited_but_complete_does_not_raise():
"""If every field is filled, a rate limit on a later (unused) provider is moot."""
full = FundamentalData(
ticker="AAPL", pe_ratio=20.0, revenue_growth=10.0, earnings_surprise=2.0,
market_cap=5.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
)
provider = ChainedFundamentalProvider([
("fmp", _DataProvider(full)),
("finnhub", _RateLimitedProvider()),
])
result = await provider.fetch_fundamentals("AAPL")
assert result.pe_ratio == 20.0
@pytest.mark.asyncio
async def test_chain_merges_next_earnings_date():
"""Earnings date is taken from the first provider that supplies it."""
from datetime import date as _date
primary = FundamentalData(
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
market_cap=100.0, fetched_at=datetime.now(timezone.utc),
)
class _EarningsProvider:
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
return FundamentalData(
ticker=ticker, pe_ratio=10.0, revenue_growth=5.0, earnings_surprise=1.0,
market_cap=None, fetched_at=datetime.now(timezone.utc),
next_earnings_date=_date(2026, 7, 1),
)
provider = ChainedFundamentalProvider([
("fmp", _DataProvider(primary)),
("finnhub", _EarningsProvider()),
])
result = await provider.fetch_fundamentals("AAPL")
assert result.next_earnings_date == _date(2026, 7, 1)
tests/unit/test_fundamentals_parity.py
-209
View File
@@ -1,209 +0,0 @@
"""A5 fundamentals parity report: read-only comparison + artifact archive."""
from __future__ import annotations
from datetime import date, datetime, timezone
import pytest
from sqlalchemy import func, select
from app.models.data_import_run import DataImportRun
from app.models.earnings_event import EarningsEvent
from app.models.fundamental import FundamentalData
from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.ohlcv import OHLCVRecord
from app.models.ticker import Ticker
from app.services.fundamentals_parity_service import (
build_report,
fundamental_score,
load_latest,
load_latest_csv,
load_latest_json,
store_report,
)
UTC = timezone.utc
GENERATED = datetime(2026, 7, 23, 10, 30, tzinfo=UTC)
def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]:
rows = []
periods = ("Q1", "Q2", "Q3", "FY")
months = (3, 6, 9, 12)
for fy, multiplier in ((2025, 1.0), (2026, 1.1)):
revenues = [100 * multiplier, 110 * multiplier, 120 * multiplier, 130 * multiplier]
eps = [1.0 * multiplier, 1.1 * multiplier, 1.2 * multiplier, 1.3 * multiplier]
for index, period in enumerate(periods):
period_end = date(fy, months[index], 28)
rows.append(
FundamentalSnapshot(
cik=cik,
accession=f"{cik}-{fy}-{period}",
form="10-K" if period == "FY" else "10-Q",
filed_date=period_end,
accepted_at=datetime(fy, months[index], 28, tzinfo=UTC),
period_end=period_end,
fiscal_year=fy,
fiscal_period=period,
revenue=sum(revenues[: index + 1]),
operating_income=sum(revenues[: index + 1]) * 0.2,
diluted_eps=sum(eps[: index + 1]),
cfo=sum(revenues[: index + 1]) * 0.25,
capex=sum(revenues[: index + 1]) * 0.05,
depreciation_amortization=sum(revenues[: index + 1]) * 0.05,
cash_and_st_investments=40,
total_debt=100,
shares_outstanding=1000,
)
)
return rows
async def _seed(db_session):
first = Ticker(symbol="AAA", cik="0000000001", sic="3571")
second = Ticker(symbol="BBB", cik=None, sic=None)
db_session.add_all([first, second])
await db_session.flush()
db_session.add_all(_snapshot_rows(first.cik))
db_session.add_all(
[
FundamentalData(
ticker_id=first.id,
pe_ratio=25,
revenue_growth=5,
earnings_surprise=0,
fetched_at=GENERATED,
),
FundamentalData(
ticker_id=second.id,
pe_ratio=12,
revenue_growth=3,
earnings_surprise=None,
fetched_at=GENERATED,
),
OHLCVRecord(
ticker_id=first.id,
date=date(2026, 7, 22),
open=100,
high=100,
low=100,
close=100,
volume=100,
),
EarningsEvent(
ticker_id=first.id,
announce_date=date(2026, 7, 1),
session="amc",
eps_estimate=2,
eps_actual=2.2,
source="dolt_earnings",
),
DataImportRun(
source="sec_facts",
revision="sec-rev",
status="promoted",
source_max_date=date(2026, 7, 22),
started_at=GENERATED,
completed_at=GENERATED,
),
DataImportRun(
source="dolt_earnings",
revision="dolt-rev",
status="no_op",
source_max_date=date(2026, 7, 22),
started_at=GENERATED,
completed_at=GENERATED,
),
]
)
await db_session.flush()
def test_score_formula_matches_production_rules():
score = fundamental_score(pe_ratio=15, revenue_growth=0, earnings_surprise=0)
assert score == pytest.approx((100 + 50 + 50) / 3)
assert fundamental_score(pe_ratio=15, revenue_growth=None, earnings_surprise=None) is None
async def test_report_compares_sources_and_leaves_database_untouched(db_session):
await _seed(db_session)
before = await db_session.scalar(select(func.count()).select_from(FundamentalData))
report = await build_report(
db_session,
generated_at=GENERATED,
today=date(2026, 7, 23),
)
after = await db_session.scalar(select(func.count()).select_from(FundamentalData))
assert before == after == 2
assert not db_session.new and not db_session.dirty and not db_session.deleted
assert report["read_only"] is True
assert report["approval_status"] == "pending_explicit_approval"
assert report["source_runs"]["sec_facts"]["revision"] == "sec-rev"
assert report["source_runs"]["dolt_earnings"]["revision"] == "dolt-rev"
first = next(row for row in report["rows"] if row["symbol"] == "AAA")
assert first["fields"]["pe_ratio"]["candidate"] == pytest.approx(
100 / 5.06, abs=1e-4
)
assert first["fields"]["revenue_growth"]["candidate"] == pytest.approx(10)
assert first["fields"]["earnings_surprise"]["candidate"] == pytest.approx(10)
assert first["scores"]["candidate_fundamental"] is not None
assert report["summary"]["universe_count"] == 2
assert report["summary"]["field_stats"]["pe_ratio"]["both_available"] == 1
async def test_artifacts_archive_and_latest_manifest(db_session, tmp_path):
await _seed(db_session)
report = await build_report(
db_session,
generated_at=GENERATED,
today=date(2026, 7, 23),
)
paths = store_report(report, tmp_path)
assert tmp_path.joinpath("latest.json").exists()
assert paths["json"].endswith(".json") and paths["csv"].endswith(".csv")
assert load_latest(tmp_path)["generated_at"] == GENERATED.isoformat()
csv_artifact = load_latest_csv(tmp_path)
assert csv_artifact is not None
assert csv_artifact[0].endswith(".csv")
assert "legacy_fundamental,candidate_fundamental" in csv_artifact[1]
assert "AAA" in csv_artifact[1]
json_artifact = load_latest_json(tmp_path)
assert json_artifact is not None and '"rows"' in json_artifact[1]
async def test_admin_endpoints_return_compact_summary_and_downloads(
client, db_session, tmp_path, monkeypatch
):
from app.config import settings
from app.dependencies import require_admin
from app.main import app
await _seed(db_session)
report = await build_report(
db_session,
generated_at=GENERATED,
today=date(2026, 7, 23),
)
store_report(report, tmp_path)
monkeypatch.setattr(settings, "fundamentals_parity_report_dir", str(tmp_path))
app.dependency_overrides[require_admin] = lambda: None
try:
summary_response = await client.get("/api/v1/admin/fundamentals-parity")
assert summary_response.status_code == 200
summary = summary_response.json()["data"]
assert summary["summary"]["universe_count"] == 2
assert "rows" not in summary
csv_response = await client.get("/api/v1/admin/fundamentals-parity/csv")
assert csv_response.status_code == 200
assert "AAA" in csv_response.json()["data"]["content"]
json_response = await client.get("/api/v1/admin/fundamentals-parity/json")
assert json_response.status_code == 200
assert '"rows"' in json_response.json()["data"]["content"]
finally:
app.dependency_overrides.pop(require_admin, None)
tests/unit/test_fundamentals_quality_service.py
-42
View File
@@ -6,7 +6,6 @@ from datetime import date, datetime, timezone
from app.models.data_import_run import DataImportRun
from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.sec_filing_gap import SecFilingGap
from app.models.settings import SystemSetting
from app.models.ticker import Ticker
from app.services import fundamentals_quality_service
@@ -19,12 +18,6 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks(
healthy = Ticker(symbol="HEALTHY", cik="0000000003")
db_session.add_all([missing, no_history, healthy])
await db_session.flush()
db_session.add(
SystemSetting(
key="fundamental_data_sec_dolt_cutover_enabled",
value="true",
)
)
db_session.add(
DataImportRun(
source="sec_facts",
@@ -44,38 +37,11 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks(
}
async def test_sec_quality_gate_is_inactive_before_cutover(db_session):
ticker = Ticker(symbol="SHADOW", cik="0000000042")
db_session.add(ticker)
await db_session.flush()
now = datetime.now(timezone.utc)
db_session.add(
SecFilingGap(
cik=ticker.cik,
accession="SHADOW-Q",
form="10-Q",
index_date=date.today(),
reason="not_in_companyfacts",
first_seen_at=now,
last_attempted_at=now,
)
)
await db_session.flush()
assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set()
async def test_active_gap_is_blocked_until_a_later_filing_supersedes_it(db_session):
ticker = Ticker(symbol="HIST", cik="0000000043")
now = datetime.now(timezone.utc)
db_session.add_all([
ticker,
SystemSetting(
key="fundamental_data_sec_dolt_cutover_enabled",
value="true",
),
SecFilingGap(
cik=ticker.cik,
accession="HIST-Q",
@@ -116,10 +82,6 @@ async def test_gap_without_index_date_uses_first_seen_date_for_supersession(
first_seen = datetime(2026, 5, 1, 12, tzinfo=timezone.utc)
db_session.add_all([
ticker,
SystemSetting(
key="fundamental_data_sec_dolt_cutover_enabled",
value="true",
),
SecFilingGap(
cik=ticker.cik,
accession="DATELESS-Q",
@@ -157,10 +119,6 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session):
ticker = Ticker(symbol="NEWREG", cik="0000000044")
db_session.add_all([
ticker,
SystemSetting(
key="fundamental_data_sec_dolt_cutover_enabled",
value="true",
),
DataImportRun(
source="sec_facts",
status="promoted",
tests/unit/test_ingestion_fundamentals_source.py
+38
View File
@@ -0,0 +1,38 @@
"""A6: `sources=fundamentals` is accepted but never fetches from a provider.
`fundamental_data` is rebuilt for the whole universe by the nightly SEC + Dolt
imports, so there is no per-ticker fetch left. The source key stays valid so an
older client gets a truthful `skipped` instead of a silent omission.
"""
from __future__ import annotations
from app.models.ticker import Ticker
async def test_fundamentals_source_reports_skipped(client, db_session):
from app.dependencies import require_access
from app.main import app
app.dependency_overrides[require_access] = lambda: None
try:
db_session.add(Ticker(symbol="AAPL"))
await db_session.flush()
resp = await client.post(
"/api/v1/ingestion/fetch/AAPL", params={"sources": "fundamentals"}
)
assert resp.status_code == 200
source = resp.json()["data"]["sources"]["fundamentals"]
assert source["status"] == "skipped"
assert "SEC + Dolt" in source["message"]
finally:
app.dependency_overrides.pop(require_access, None)
def test_fundamentals_remains_a_recognised_source_key():
"""Older clients keep getting an entry for it rather than a missing key."""
from app.routers.ingestion import _parse_requested_sources
assert "fundamentals" in _parse_requested_sources("fundamentals")
assert "fundamentals" in _parse_requested_sources(None) # None => all sources
tests/unit/test_rr_scanner_preservation.py
-5
View File
@@ -24,7 +24,6 @@ from app.models.ohlcv import OHLCVRecord
from app.models.paper_trade import PaperTrade
from app.models.signal_context_snapshot import SignalContextSnapshot
from app.models.sec_filing_gap import SecFilingGap
from app.models.settings import SystemSetting
from app.models.sr_level import SRLevel
from app.models.ticker import Ticker
from app.models.trade_setup import TradeSetup
@@ -524,10 +523,6 @@ async def test_get_trade_setups_hides_active_sec_filing_gap(
db_session.add(ticker)
await db_session.flush()
db_session.add_all([
SystemSetting(
key="fundamental_data_sec_dolt_cutover_enabled",
value="true",
),
SecFilingGap(
cik=ticker.cik,
accession="0000000042-26-000001",
tests/unit/test_schedule_config.py
+1 -16
View File
@@ -66,26 +66,11 @@ class TestTradingDayCrons:
assert "Mon" in weekdays, f"{key} skips Mondays — numeric day-of-week?"
assert {"Sat", "Sun"}.isdisjoint(weekdays), f"{key} fires on a weekend"
def test_fundamentals_runs_on_monday(self):
from datetime import datetime
from apscheduler.triggers.cron import CronTrigger
trigger = CronTrigger.from_crontab(
SCHEDULE_DEFAULTS["schedule_fundamentals_cron"],
timezone=SCHEDULE_DEFAULTS["schedule_timezone"],
)
fire = trigger.get_next_fire_time(
None, datetime(2026, 7, 19, tzinfo=trigger.timezone)
)
assert fire.strftime("%a") == "Mon"
@pytest.mark.parametrize(
("key", "hour", "minute"),
(
("schedule_dolt_earnings_cron", 2, 30),
("schedule_sec_fundamentals_cron", 4, 0),
("schedule_fundamentals_parity_cron", 5, 30),
),
)
def test_shadow_imports_run_daily_at_expected_et_time(
@@ -122,7 +107,7 @@ class TestScheduleConfig:
async def test_rejects_bad_cron(self, session: AsyncSession):
with pytest.raises(ValidationError):
await update_schedule_config(session, {"schedule_fundamentals_cron": "every monday"})
await update_schedule_config(session, {"schedule_daily_pipeline_cron": "every monday"})
async def test_rejects_bad_timezone(self, session: AsyncSession):
with pytest.raises(ValidationError):
tests/unit/test_scheduler.py
+28 -78
View File
@@ -13,8 +13,6 @@ from app.scheduler import (
_resume_tickers,
_last_successful,
_run_shadow_import,
collect_fundamentals,
run_fundamentals_parity_report,
run_sec_fundamentals_import,
configure_scheduler,
get_job_runtime_snapshot,
@@ -123,10 +121,8 @@ class TestConfigureScheduler:
"data_backfill",
"benchmark_collector",
"sentiment_collector",
"fundamental_collector",
"dolt_earnings_import",
"sec_fundamentals_import",
"fundamentals_parity_report",
"rr_scanner",
"shadow_book",
"ticker_universe_sync",
@@ -157,10 +153,8 @@ class TestConfigureScheduler:
"intraday_pipeline",
"data_collector",
"data_backfill",
"fundamental_collector",
"dolt_earnings_import",
"sec_fundamentals_import",
"fundamentals_parity_report",
"market_regime",
"near_close_pipeline",
"regime_monitor",
@@ -181,41 +175,6 @@ class _SessionContext:
return None
class TestFundamentalCollector:
@staticmethod
def _session_factory():
return _SessionContext()
async def test_skips_legacy_provider_when_cutover_is_active(self, monkeypatch):
async def enabled(db, job_name):
return True
async def cutover_enabled(db):
return True
async def unexpected_ticker_lookup(db):
raise AssertionError("legacy ticker lookup must not run after cutover")
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.is_enabled",
cutover_enabled,
)
monkeypatch.setattr(
"app.scheduler._get_fundamental_priority_tickers",
unexpected_ticker_lookup,
)
await collect_fundamentals()
runtime = get_job_runtime_snapshot("fundamental_collector")
assert runtime["status"] == "skipped"
assert runtime["processed"] == 0
assert runtime["total"] == 0
assert runtime["message"] == "SEC + Dolt fundamentals cutover is active"
class TestShadowImportJobs:
@staticmethod
def _session_factory():
@@ -329,7 +288,6 @@ class TestShadowImportJobs:
async def refreshed(db):
calls.append(db)
return {
"enabled": True,
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
@@ -340,7 +298,7 @@ class TestShadowImportJobs:
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr("app.scheduler.run_import", unavailable)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
@@ -351,7 +309,7 @@ class TestShadowImportJobs:
assert runtime["status"] == "error"
assert runtime["message"] == "SEC unavailable"
async def test_sec_success_surfaces_activated_refresh_summary(self, monkeypatch):
async def test_sec_success_surfaces_cache_refresh_summary(self, monkeypatch):
async def enabled(db, job_name):
return True
@@ -362,7 +320,6 @@ class TestShadowImportJobs:
async def refreshed(db):
return {
"enabled": True,
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
@@ -373,7 +330,7 @@ class TestShadowImportJobs:
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr("app.scheduler.run_import", imported)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
@@ -385,52 +342,45 @@ class TestShadowImportJobs:
"no_op · abcdef123456 · cache 511 · 2 score inputs changed"
)
async def test_disabled_sec_job_does_not_run_local_refresh(self, monkeypatch):
async def test_disabled_sec_job_still_refreshes_local_cache(self, monkeypatch):
"""Disabling the job stops the SEC fetch, not the local cache.
The cache is derived from stored snapshots, earnings events and closes.
Prices and earnings move daily even when no filing does, and there is no
provider fallback since A6 — freezing it would silently stale scoring.
"""
calls = []
async def disabled(db, job_name):
return False
async def should_not_run(*args, **kwargs):
raise AssertionError("disabled SEC job ran work")
raise AssertionError("disabled SEC job hit the network")
async def refreshed(db):
calls.append(db)
return {
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
"composite_scores_staled": 2,
}
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
should_not_run,
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
await run_sec_fundamentals_import()
assert len(calls) == 1
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
assert runtime["status"] == "skipped"
assert runtime["message"] == "Disabled"
async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch):
async def enabled(db, job_name):
return True
async def generated(db, report_dir):
return (
{
"generated_at": "2026-07-23T10:30:00+00:00",
"summary": {
"universe_count": 511,
"fundamental_score_material_changes": 12,
},
},
{"json": "report.json", "csv": "report.csv"},
assert runtime["status"] == "completed"
assert runtime["message"] == (
"Import disabled · cache 511 · 2 score inputs changed"
)
monkeypatch.setattr("app.scheduler.async_session_factory", TestShadowImportJobs._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr(
"app.scheduler.fundamentals_parity_service.generate_and_store", generated
)
await run_fundamentals_parity_report()
runtime = get_job_runtime_snapshot("fundamentals_parity_report")
assert runtime["status"] == "completed"
assert runtime["message"] == "511 tickers · 12 material score changes"
tests/unit/test_ticker_universe_service.py
-9
View File
@@ -10,7 +10,6 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.database import Base
from app.exceptions import ProviderError
from app.models.settings import SystemSetting
from app.models.ticker import Ticker
from app.services import ticker_universe_service
@@ -97,11 +96,7 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai
async def _fake_public(_universe: str):
return [], ["public failed"], None
async def _fake_fmp(_universe: str):
raise ProviderError("fmp failed")
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
assert symbols == ["AAPL", "MSFT"]
@@ -116,11 +111,7 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail(
async def _fake_public(_universe: str):
return [], ["public failed"], None
async def _fake_fmp(_universe: str):
raise ProviderError("fmp failed")
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
assert "AAPL" in symbols