dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: decommission FMP, Finnhub and Alpha Vantage (A6)

Browse Source 
The A5 cutover has been on and observed in production, so SEC Company Facts +
DoltHub earnings are already the live source for `fundamental_data`. This
removes everything the legacy path still occupied.

Gone: the three providers and their config/env keys; the weekly
`fundamental_collector` job; the cutover toggle (SEC + Dolt is now the
unconditional path, so `off` can no longer silently freeze scoring inputs); the
A5 parity report, whose deltas became structurally zero once the candidate
builder started writing the table it compared against; and the FMP tier of
universe bootstrap.

Two behavioral notes:

- Disabling **SEC Fundamentals Import** now stops the SEC network fetch only.
  The local cache refresh moved outside the job-enable check, because candidates
  also derive from daily closes and earnings events — freezing those on an
  ingestion pause would stale scoring with no fallback left to recover from.
- `/ingestion/fetch?sources=fundamentals` still accepts the key and reports
  `skipped`; there is no per-ticker fetch any more.

Migration 029 does not blanket-delete the leftover settings rows. Migrations run
before the service restart, and pre-A6 code reads an absent `job_*_enabled` row
as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe
values (hidden in Admin) and only the inert three are deleted. Removing the
provider keys from the production `.env` is the matching rollout step.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dennisthiessen
2026-08-07 11:19:28 +02:00
co-authored by Claude Opus 5
parent f5d4b516ab
commit 3e83d63b05
51 changed files with 368 additions and 3787 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_scheduler.py
+28 -78
View File
@@ -13,8 +13,6 @@ from app.scheduler import (
_resume_tickers,
_last_successful,
_run_shadow_import,
collect_fundamentals,
run_fundamentals_parity_report,
run_sec_fundamentals_import,
configure_scheduler,
get_job_runtime_snapshot,
@@ -123,10 +121,8 @@ class TestConfigureScheduler:
"data_backfill",
"benchmark_collector",
"sentiment_collector",
"fundamental_collector",
"dolt_earnings_import",
"sec_fundamentals_import",
"fundamentals_parity_report",
"rr_scanner",
"shadow_book",
"ticker_universe_sync",
@@ -157,10 +153,8 @@ class TestConfigureScheduler:
"intraday_pipeline",
"data_collector",
"data_backfill",
"fundamental_collector",
"dolt_earnings_import",
"sec_fundamentals_import",
"fundamentals_parity_report",
"market_regime",
"near_close_pipeline",
"regime_monitor",
@@ -181,41 +175,6 @@ class _SessionContext:
return None
class TestFundamentalCollector:
@staticmethod
def _session_factory():
return _SessionContext()
async def test_skips_legacy_provider_when_cutover_is_active(self, monkeypatch):
async def enabled(db, job_name):
return True
async def cutover_enabled(db):
return True
async def unexpected_ticker_lookup(db):
raise AssertionError("legacy ticker lookup must not run after cutover")
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.is_enabled",
cutover_enabled,
)
monkeypatch.setattr(
"app.scheduler._get_fundamental_priority_tickers",
unexpected_ticker_lookup,
)
await collect_fundamentals()
runtime = get_job_runtime_snapshot("fundamental_collector")
assert runtime["status"] == "skipped"
assert runtime["processed"] == 0
assert runtime["total"] == 0
assert runtime["message"] == "SEC + Dolt fundamentals cutover is active"
class TestShadowImportJobs:
@staticmethod
def _session_factory():
@@ -329,7 +288,6 @@ class TestShadowImportJobs:
async def refreshed(db):
calls.append(db)
return {
"enabled": True,
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
@@ -340,7 +298,7 @@ class TestShadowImportJobs:
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr("app.scheduler.run_import", unavailable)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
@@ -351,7 +309,7 @@ class TestShadowImportJobs:
assert runtime["status"] == "error"
assert runtime["message"] == "SEC unavailable"
async def test_sec_success_surfaces_activated_refresh_summary(self, monkeypatch):
async def test_sec_success_surfaces_cache_refresh_summary(self, monkeypatch):
async def enabled(db, job_name):
return True
@@ -362,7 +320,6 @@ class TestShadowImportJobs:
async def refreshed(db):
return {
"enabled": True,
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
@@ -373,7 +330,7 @@ class TestShadowImportJobs:
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr("app.scheduler.run_import", imported)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
@@ -385,52 +342,45 @@ class TestShadowImportJobs:
"no_op · abcdef123456 · cache 511 · 2 score inputs changed"
)
async def test_disabled_sec_job_does_not_run_local_refresh(self, monkeypatch):
async def test_disabled_sec_job_still_refreshes_local_cache(self, monkeypatch):
"""Disabling the job stops the SEC fetch, not the local cache.
The cache is derived from stored snapshots, earnings events and closes.
Prices and earnings move daily even when no filing does, and there is no
provider fallback since A6 — freezing it would silently stale scoring.
"""
calls = []
async def disabled(db, job_name):
return False
async def should_not_run(*args, **kwargs):
raise AssertionError("disabled SEC job ran work")
raise AssertionError("disabled SEC job hit the network")
async def refreshed(db):
calls.append(db)
return {
"refreshed": 511,
"score_inputs_changed": 2,
"dimension_scores_staled": 2,
"composite_scores_staled": 2,
}
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
monkeypatch.setattr(
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
should_not_run,
"app.scheduler.fundamental_data_refresh_service.refresh",
refreshed,
)
await run_sec_fundamentals_import()
assert len(calls) == 1
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
assert runtime["status"] == "skipped"
assert runtime["message"] == "Disabled"
async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch):
async def enabled(db, job_name):
return True
async def generated(db, report_dir):
return (
{
"generated_at": "2026-07-23T10:30:00+00:00",
"summary": {
"universe_count": 511,
"fundamental_score_material_changes": 12,
},
},
{"json": "report.json", "csv": "report.csv"},
assert runtime["status"] == "completed"
assert runtime["message"] == (
"Import disabled · cache 511 · 2 score inputs changed"
)
monkeypatch.setattr("app.scheduler.async_session_factory", TestShadowImportJobs._session_factory)
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
monkeypatch.setattr(
"app.scheduler.fundamentals_parity_service.generate_and_store", generated
)
await run_fundamentals_parity_report()
runtime = get_job_runtime_snapshot("fundamentals_parity_report")
assert runtime["status"] == "completed"
assert runtime["message"] == "511 tickers · 12 material score changes"