diff --git a/app/routers/admin.py b/app/routers/admin.py index 8ff9ad7..c555f5e 100644 --- a/app/routers/admin.py +++ b/app/routers/admin.py @@ -13,6 +13,7 @@ from app.schemas.admin import ( AlertConfigUpdate, CreateUserRequest, DataCleanupRequest, + FundamentalsCutoverConfigUpdate, JobTriggerRequest, JobToggle, RecommendationConfigUpdate, @@ -137,6 +138,27 @@ async def list_settings( ) +@router.get("/admin/settings/fundamentals-cutover", response_model=APIEnvelope) +async def get_fundamentals_cutover_settings( + _admin: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + config = await admin_service.get_fundamentals_cutover_config(db) + return APIEnvelope(status="success", data=config) + + +@router.put("/admin/settings/fundamentals-cutover", response_model=APIEnvelope) +async def update_fundamentals_cutover_settings( + body: FundamentalsCutoverConfigUpdate, + _admin: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + config = await admin_service.update_fundamentals_cutover_config( + db, body.enabled + ) + return APIEnvelope(status="success", data=config) + + @router.get("/admin/settings/recommendations", response_model=APIEnvelope) async def get_recommendation_settings( _admin: User = Depends(require_admin), diff --git a/app/scheduler.py b/app/scheduler.py index e35ad07..42a8d52 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -826,6 +826,22 @@ async def collect_fundamentals() -> None: _log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled") _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled") return + if await fundamental_data_refresh_service.is_enabled(db): + message = "SEC + Dolt fundamentals cutover is active" + _log_event( + logging.INFO, + "job_skipped", + job=job_name, + reason="sec_dolt_cutover_active", + ) + _runtime_finish( + job_name, + "skipped", + processed=0, + total=0, + message=message, + ) + return symbols = await _get_fundamental_priority_tickers(db) if not symbols: diff --git a/app/schemas/admin.py b/app/schemas/admin.py index 3127750..e86f941 100644 --- a/app/schemas/admin.py +++ b/app/schemas/admin.py @@ -73,6 +73,11 @@ class ActivationConfigUpdate(BaseModel): exclude_neutral: bool | None = None +class FundamentalsCutoverConfigUpdate(BaseModel): + """Switch the legacy fundamentals cache from quota APIs to SEC/Dolt.""" + enabled: bool + + class ScheduleConfigUpdate(BaseModel): """Cron schedule for the pipelines + fundamentals. Crons are 5-field (min hour dom month dow); timezone is an IANA name (e.g. America/New_York).""" diff --git a/app/services/admin_service.py b/app/services/admin_service.py index f7f850b..d614c5d 100644 --- a/app/services/admin_service.py +++ b/app/services/admin_service.py @@ -17,7 +17,7 @@ from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.models.trade_setup import TradeSetup from app.models.user import User -from app.services import settings_store +from app.services import fundamental_data_refresh_service, settings_store logger = logging.getLogger(__name__) @@ -159,6 +159,28 @@ async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSettin return setting +# --------------------------------------------------------------------------- +# Fundamentals source cutover +# --------------------------------------------------------------------------- + +async def get_fundamentals_cutover_config(db: AsyncSession) -> dict[str, bool]: + """Return the explicit A5 cache-cutover switch (default off).""" + return {"enabled": await fundamental_data_refresh_service.is_enabled(db)} + + +async def update_fundamentals_cutover_config( + db: AsyncSession, enabled: bool +) -> dict[str, bool]: + """Activate or pause SEC/Dolt writes to the legacy fundamentals cache.""" + await settings_store.upsert_setting( + db, + fundamental_data_refresh_service.ACTIVATION_KEY, + "true" if enabled else "false", + ) + await db.commit() + return await get_fundamentals_cutover_config(db) + + # --------------------------------------------------------------------------- # Activation thresholds # --------------------------------------------------------------------------- diff --git a/docs/dolt-integration-plan.md b/docs/dolt-integration-plan.md index 8758e67..6c58781 100644 --- a/docs/dolt-integration-plan.md +++ b/docs/dolt-integration-plan.md @@ -502,8 +502,10 @@ that path carries the split guard (`ttm_diluted_eps` nulls when contaminated, with `ttm_diluted_eps_caveat`) and the multi-class share fallback (`shares_outstanding` + `shares_outstanding_estimated`). Parity and activation share the same candidate builder. Activation is the explicit -`fundamental_data_sec_dolt_cutover_enabled` SystemSetting and defaults off; see -`docs/fundamentals-deployment.md` for the production flip and rollback procedure. +`fundamental_data_sec_dolt_cutover_enabled` SystemSetting and defaults off. It is +managed by the **Fundamentals data source** card in Admin → Settings; while active, +the weekly legacy collector skips itself so it cannot overwrite the SEC/Dolt cache. +See `docs/fundamentals-deployment.md` for the production flip and rollback procedure. **Task 2 — A6 decommissioning.** After a short observation window: remove FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual diff --git a/docs/fundamentals-deployment.md b/docs/fundamentals-deployment.md index f6bbb39..1b0ea7f 100644 --- a/docs/fundamentals-deployment.md +++ b/docs/fundamentals-deployment.md @@ -163,7 +163,17 @@ The write path is controlled by the SystemSetting value other than `true` leaves `fundamental_data` untouched. Before enabling it, confirm the normal PostgreSQL backup containing `fundamental_data` is current. -Enable the cutover in PostgreSQL: +In **Admin → Settings → Fundamentals data source**: + +1. Turn on **Use SEC + Dolt for scoring inputs** and accept the confirmation. +2. Click **Run refresh now**. The SEC import may be `promoted` or `no_op`; either + result runs the local cache refresh. + +The weekly legacy collector is automatically skipped while the switch is on, so +it cannot overwrite the activated cache. The switch remains visible even before +its SystemSetting row exists because the safe default is off. + +If the Admin UI is unavailable, enable the cutover directly in PostgreSQL: ```sql INSERT INTO system_settings (key, value, updated_at) @@ -172,8 +182,7 @@ ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now(); ``` -Then trigger **SEC Fundamentals Import** once in Admin → Jobs. The import may be -`promoted` or `no_op`; either result runs the local refresh. Once enabled, the +Then trigger **SEC Fundamentals Import** once in Admin → Jobs. Once enabled, the same refresh also runs after an SEC network/validation failure or a source-lock skip, because it reads only PostgreSQL snapshots, earnings events, and closes. The job message appends the cache row count and changed score-input count when @@ -211,12 +220,13 @@ least several scheduled cycles before A6 removes the legacy providers. ## Failure and rollback -- To stop the A5 cache writes without stopping SEC snapshot ingestion, set - `fundamental_data_sec_dolt_cutover_enabled` back to `false` with the SQL above - (changing only the value). This prevents the next local refresh but does not - restore rows already replaced. Restore `fundamental_data` from the pre-cutover - database backup, or—before A6—manually run the legacy Fundamental Collector if - its provider keys and quota are still available. +- To stop the A5 cache writes without stopping SEC snapshot ingestion, turn off + **Use SEC + Dolt for scoring inputs** in Admin → Settings. If the UI is + unavailable, set `fundamental_data_sec_dolt_cutover_enabled` back to `false` + with the SQL above (changing only the value). This prevents the next local + refresh but does not restore rows already replaced. Restore `fundamental_data` + from the pre-cutover database backup, or—before A6—manually run the legacy + Fundamental Collector if its provider keys and quota are still available. - Disable a failing source-import job in Admin → Jobs only when ingestion itself must stop. Existing promoted snapshots/events remain available. - Inspect the job runtime, latest `data_import_runs.validation_json`, service diff --git a/frontend/src/api/admin.ts b/frontend/src/api/admin.ts index bf2fb28..d24ff9e 100644 --- a/frontend/src/api/admin.ts +++ b/frontend/src/api/admin.ts @@ -4,6 +4,7 @@ import type { AdminUser, AlertConfig, AlertTestResult, + FundamentalsCutoverConfig, PipelineReadiness, RecommendationConfig, ScheduleConfig, @@ -56,6 +57,18 @@ export function updateSetting(key: string, value: string) { .then((r) => r.data); } +export function getFundamentalsCutoverSettings() { + return apiClient + .get('admin/settings/fundamentals-cutover') + .then((r) => r.data); +} + +export function updateFundamentalsCutoverSettings(enabled: boolean) { + return apiClient + .put('admin/settings/fundamentals-cutover', { enabled }) + .then((r) => r.data); +} + export function getRecommendationSettings() { return apiClient .get('admin/settings/recommendations') diff --git a/frontend/src/components/admin/FundamentalsCutoverSettings.tsx b/frontend/src/components/admin/FundamentalsCutoverSettings.tsx new file mode 100644 index 0000000..4c58c41 --- /dev/null +++ b/frontend/src/components/admin/FundamentalsCutoverSettings.tsx @@ -0,0 +1,176 @@ +import { + useFundamentalsCutoverSettings, + useJobs, + useTriggerJob, + useUpdateFundamentalsCutoverSettings, +} from '../../hooks/useAdmin'; +import { SkeletonCard } from '../ui/Skeleton'; + +const SEC_JOB = 'sec_fundamentals_import'; + +function formatRun(iso: string | null | undefined): string { + if (!iso) return 'not run in this process'; + const minutes = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000); + if (minutes < 1) return 'just now'; + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + return hours < 24 ? `${hours}h ago` : `${Math.floor(hours / 24)}d ago`; +} + +export function FundamentalsCutoverSettings() { + const cutover = useFundamentalsCutoverSettings(); + const update = useUpdateFundamentalsCutoverSettings(); + const trigger = useTriggerJob(); + const { data: jobs } = useJobs(); + + if (cutover.isLoading) return ; + if (cutover.isError || !cutover.data) { + return ( +

+ {(cutover.error as Error)?.message || 'Failed to load fundamentals data source'} +

+ ); + } + + const enabled = cutover.data.enabled; + const secJob = jobs?.find((job) => job.name === SEC_JOB); + const runningJob = jobs?.find((job) => job.running); + const refreshBlocked = Boolean(runningJob && runningJob.name !== SEC_JOB); + + const changeSource = () => { + const next = !enabled; + const confirmed = window.confirm( + next + ? 'Activate SEC + Dolt fundamentals? The next SEC import will replace the legacy cache and mark affected scores stale.' + : 'Pause SEC + Dolt cache refreshes? Existing cache values will stay in place; legacy values are not restored automatically.', + ); + if (confirmed) update.mutate(next); + }; + + return ( +
+
+
+
+
+
+

+ Fundamentals data source +

+ + {enabled ? 'SEC + Dolt active' : 'Legacy cache'} + +
+

+ Controls what repopulates fundamental_data, the + compatibility cache used by scoring. SEC filings supply P/E, growth and estimated market + cap; Dolt supplies earnings dates and surprises. Everything is derived locally from PostgreSQL. +

+
+
+ +
+
+
Legacy APIs
+
FMP / Finnhub / Alpha Vantage
+
+ +
+
SEC + Dolt
+
Bulk imports → PostgreSQL cache
+
+
+ +
+
+
+
1 · Source
+
Use SEC + Dolt for scoring inputs
+

+ While active, the weekly legacy collector is skipped so it cannot overwrite the new cache. +

+
+ +
+ +
+
2 · Refresh
+
+
+
Apply the source now
+

+ {secJob?.running + ? 'SEC import and cache refresh are running.' + : secJob?.runtime_message || `Last SEC run: ${formatRun(secJob?.runtime_finished_at)}`} +

+
+ +
+ {!enabled && ( +

Activate the source before running the refresh.

+ )} + {enabled && secJob?.enabled === false && ( +

Enable the SEC Fundamentals job on the Jobs tab first.

+ )} +
+
+ +

+ Rollback pauses future writes only. To restore pre-cutover values, use the database backup or + pause this source and manually run the legacy collector while its provider keys remain installed. +

+
+
+ ); +} diff --git a/frontend/src/components/admin/ScheduleSettings.tsx b/frontend/src/components/admin/ScheduleSettings.tsx index ccb44b5..e084fc6 100644 --- a/frontend/src/components/admin/ScheduleSettings.tsx +++ b/frontend/src/components/admin/ScheduleSettings.tsx @@ -29,20 +29,20 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b }, { key: 'schedule_dolt_earnings_cron', - label: 'Dolt earnings (shadow)', - hint: 'Pull and import earnings dates/results daily at 02:30 ET. Live scoring remains untouched before A5.', + label: 'Dolt earnings', + hint: 'Pull and import earnings dates/results daily at 02:30 ET. The activated cache refresh uses these local events.', mono: true, }, { key: 'schedule_sec_fundamentals_cron', - label: 'SEC fundamentals (shadow)', - hint: 'Import tracked-universe SEC facts daily at 04:00 ET. Unchanged revisions become no-op runs.', + label: 'SEC fundamentals', + hint: 'Import tracked-universe SEC facts daily at 04:00 ET and refresh the scoring cache when the cutover is active.', mono: true, }, { key: 'schedule_fundamentals_parity_cron', label: 'Fundamentals parity report', - hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the shadow imports.', + hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the bulk imports.', mono: true, }, { @@ -66,7 +66,7 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b { key: 'schedule_fundamentals_cron', label: 'Legacy fundamentals (weekly)', - hint: 'Existing provider chain retained until the A5 parity approval and A6 removal.', + hint: 'Fallback provider chain. Automatically skipped while the SEC + Dolt cutover is active.', mono: true, }, ]; diff --git a/frontend/src/components/admin/SettingsForm.tsx b/frontend/src/components/admin/SettingsForm.tsx index 5e4f49b..cdd2993 100644 --- a/frontend/src/components/admin/SettingsForm.tsx +++ b/frontend/src/components/admin/SettingsForm.tsx @@ -3,6 +3,8 @@ import { useSettings, useUpdateSetting } from '../../hooks/useAdmin'; import { SkeletonTable } from '../ui/Skeleton'; import type { SystemSetting } from '../../lib/types'; +const MANAGED_SETTINGS = new Set(['fundamental_data_sec_dolt_cutover_enabled']); + export function SettingsForm() { const { data: settings, isLoading, isError, error } = useSettings(); const updateSetting = useUpdateSetting(); @@ -32,10 +34,11 @@ export function SettingsForm() { if (isLoading) return ; if (isError) return

{(error as Error)?.message || 'Failed to load settings'}

; if (!settings || settings.length === 0) return

No settings found.

; + const visibleSettings = settings.filter((setting) => !MANAGED_SETTINGS.has(setting.key)); return (
- {settings.map((setting) => ( + {visibleSettings.map((setting) => (
{setting.key === 'registration' ? ( diff --git a/frontend/src/hooks/useAdmin.ts b/frontend/src/hooks/useAdmin.ts index 3b5d31a..f0da952 100644 --- a/frontend/src/hooks/useAdmin.ts +++ b/frontend/src/hooks/useAdmin.ts @@ -90,6 +90,36 @@ export function useUpdateSetting() { }); } +export function useFundamentalsCutoverSettings() { + return useQuery({ + queryKey: ['admin', 'fundamentals-cutover'], + queryFn: () => adminApi.getFundamentalsCutoverSettings(), + }); +} + +export function useUpdateFundamentalsCutoverSettings() { + const qc = useQueryClient(); + const { addToast } = useToast(); + + return useMutation({ + mutationFn: (enabled: boolean) => + adminApi.updateFundamentalsCutoverSettings(enabled), + onSuccess: (config) => { + qc.setQueryData(['admin', 'fundamentals-cutover'], config); + qc.invalidateQueries({ queryKey: ['admin', 'settings'] }); + addToast( + config.enabled ? 'success' : 'info', + config.enabled + ? 'SEC + Dolt fundamentals activated' + : 'SEC + Dolt cache refresh paused', + ); + }, + onError: (error: Error) => { + addToast('error', error.message || 'Failed to update fundamentals data source'); + }, + }); +} + export function useRecommendationSettings() { return useQuery({ queryKey: ['admin', 'recommendation-settings'], diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index aef0736..ea4d858 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -187,6 +187,10 @@ export interface ActivationConfig { exclude_neutral: boolean; } +export interface FundamentalsCutoverConfig { + enabled: boolean; +} + // Cron schedule for morning / near-close / after-close / intraday + fundamentals export interface ScheduleConfig { schedule_timezone: string; diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 520faee..6ec3d47 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -6,6 +6,7 @@ import { SentimentProviderSettings } from '../components/admin/SentimentProvider import { DataCleanup } from '../components/admin/DataCleanup'; import { JobControls } from '../components/admin/JobControls'; import { FundamentalsParityPanel } from '../components/admin/FundamentalsParityPanel'; +import { FundamentalsCutoverSettings } from '../components/admin/FundamentalsCutoverSettings'; import { PerformanceSettings } from '../components/admin/PerformanceSettings'; import { PipelineReadinessPanel } from '../components/admin/PipelineReadinessPanel'; import { SystemEventsPanel } from '../components/admin/SystemEventsPanel'; @@ -36,6 +37,7 @@ export default function AdminPage() { {activeTab === 'Tickers' && } {activeTab === 'Settings' && (
+ diff --git a/tests/unit/test_activation_settings.py b/tests/unit/test_activation_settings.py index 5a2f36f..1c8cc98 100644 --- a/tests/unit/test_activation_settings.py +++ b/tests/unit/test_activation_settings.py @@ -8,7 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.exceptions import ValidationError from app.services.admin_service import ( get_activation_config, + get_fundamentals_cutover_config, update_activation_config, + update_fundamentals_cutover_config, ) @@ -76,3 +78,18 @@ class TestActivationConfig: async def test_rejects_out_of_range_confidence(self, session: AsyncSession): with pytest.raises(ValidationError): await update_activation_config(session, {"min_confidence": 120.0}) + + +class TestFundamentalsCutoverConfig: + async def test_defaults_off_when_unset(self, session: AsyncSession): + assert await get_fundamentals_cutover_config(session) == {"enabled": False} + + async def test_round_trips_explicit_switch(self, session: AsyncSession): + assert await update_fundamentals_cutover_config(session, True) == { + "enabled": True + } + assert await get_fundamentals_cutover_config(session) == {"enabled": True} + + assert await update_fundamentals_cutover_config(session, False) == { + "enabled": False + } diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 8a9a23b..d6acb68 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -11,6 +11,7 @@ from app.scheduler import ( _resume_tickers, _last_successful, _run_shadow_import, + collect_fundamentals, run_fundamentals_parity_report, run_sec_fundamentals_import, configure_scheduler, @@ -169,6 +170,41 @@ class _SessionContext: return None +class TestFundamentalCollector: + @staticmethod + def _session_factory(): + return _SessionContext() + + async def test_skips_legacy_provider_when_cutover_is_active(self, monkeypatch): + async def enabled(db, job_name): + return True + + async def cutover_enabled(db): + return True + + async def unexpected_ticker_lookup(db): + raise AssertionError("legacy ticker lookup must not run after cutover") + + monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory) + monkeypatch.setattr("app.scheduler._is_job_enabled", enabled) + monkeypatch.setattr( + "app.scheduler.fundamental_data_refresh_service.is_enabled", + cutover_enabled, + ) + monkeypatch.setattr( + "app.scheduler._get_fundamental_priority_tickers", + unexpected_ticker_lookup, + ) + + await collect_fundamentals() + + runtime = get_job_runtime_snapshot("fundamental_collector") + assert runtime["status"] == "skipped" + assert runtime["processed"] == 0 + assert runtime["total"] == 0 + assert runtime["message"] == "SEC + Dolt fundamentals cutover is active" + + class TestShadowImportJobs: @staticmethod def _session_factory():