chore: decommission FMP, Finnhub and Alpha Vantage (A6)
The A5 cutover has been on and observed in production, so SEC Company Facts + DoltHub earnings are already the live source for `fundamental_data`. This removes everything the legacy path still occupied. Gone: the three providers and their config/env keys; the weekly `fundamental_collector` job; the cutover toggle (SEC + Dolt is now the unconditional path, so `off` can no longer silently freeze scoring inputs); the A5 parity report, whose deltas became structurally zero once the candidate builder started writing the table it compared against; and the FMP tier of universe bootstrap. Two behavioral notes: - Disabling **SEC Fundamentals Import** now stops the SEC network fetch only. The local cache refresh moved outside the job-enable check, because candidates also derive from daily closes and earnings events — freezing those on an ingestion pause would stale scoring with no fallback left to recover from. - `/ingestion/fetch?sources=fundamentals` still accepts the key and reports `skipped`; there is no per-ticker fetch any more. Migration 029 does not blanket-delete the leftover settings rows. Migrations run before the service restart, and pre-A6 code reads an absent `job_*_enabled` row as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe values (hidden in Admin) and only the inert three are deleted. Removing the provider keys from the production `.env` is the matching rollout step. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -1,176 +0,0 @@
|
||||
import {
|
||||
useFundamentalsCutoverSettings,
|
||||
useJobs,
|
||||
useTriggerJob,
|
||||
useUpdateFundamentalsCutoverSettings,
|
||||
} from '../../hooks/useAdmin';
|
||||
import { SkeletonCard } from '../ui/Skeleton';
|
||||
|
||||
const SEC_JOB = 'sec_fundamentals_import';
|
||||
|
||||
function formatRun(iso: string | null | undefined): string {
|
||||
if (!iso) return 'not run in this process';
|
||||
const minutes = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000);
|
||||
if (minutes < 1) return 'just now';
|
||||
if (minutes < 60) return `${minutes}m ago`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
return hours < 24 ? `${hours}h ago` : `${Math.floor(hours / 24)}d ago`;
|
||||
}
|
||||
|
||||
export function FundamentalsCutoverSettings() {
|
||||
const cutover = useFundamentalsCutoverSettings();
|
||||
const update = useUpdateFundamentalsCutoverSettings();
|
||||
const trigger = useTriggerJob();
|
||||
const { data: jobs } = useJobs();
|
||||
|
||||
if (cutover.isLoading) return <SkeletonCard />;
|
||||
if (cutover.isError || !cutover.data) {
|
||||
return (
|
||||
<p className="text-sm text-red-400">
|
||||
{(cutover.error as Error)?.message || 'Failed to load fundamentals data source'}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
const enabled = cutover.data.enabled;
|
||||
const secJob = jobs?.find((job) => job.name === SEC_JOB);
|
||||
const runningJob = jobs?.find((job) => job.running);
|
||||
const refreshBlocked = Boolean(runningJob && runningJob.name !== SEC_JOB);
|
||||
|
||||
const changeSource = () => {
|
||||
const next = !enabled;
|
||||
const confirmed = window.confirm(
|
||||
next
|
||||
? 'Activate SEC + Dolt fundamentals? The next SEC import will replace the legacy cache and mark affected scores stale.'
|
||||
: 'Pause SEC + Dolt cache refreshes? Existing cache values will stay in place; legacy values are not restored automatically.',
|
||||
);
|
||||
if (confirmed) update.mutate(next);
|
||||
};
|
||||
|
||||
return (
|
||||
<section className="glass overflow-hidden" aria-labelledby="fundamentals-source-title">
|
||||
<div className={`h-0.5 ${enabled ? 'bg-gradient-to-r from-sky-500 via-cyan-300 to-emerald-400' : 'bg-white/[0.06]'}`} />
|
||||
<div className="space-y-5 p-5">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="flex items-center gap-2">
|
||||
<h3 id="fundamentals-source-title" className="text-sm font-semibold text-gray-200">
|
||||
Fundamentals data source
|
||||
</h3>
|
||||
<span
|
||||
className={`rounded-full border px-2 py-0.5 text-[10px] font-semibold uppercase tracking-[0.14em] ${
|
||||
enabled
|
||||
? 'border-cyan-400/25 bg-cyan-400/10 text-cyan-300'
|
||||
: 'border-white/10 bg-white/[0.04] text-gray-500'
|
||||
}`}
|
||||
>
|
||||
{enabled ? 'SEC + Dolt active' : 'Legacy cache'}
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 max-w-3xl text-xs leading-relaxed text-gray-500">
|
||||
Controls what repopulates <span className="num text-gray-400">fundamental_data</span>, the
|
||||
compatibility cache used by scoring. SEC filings supply P/E, growth and estimated market
|
||||
cap; Dolt supplies earnings dates and surprises. Everything is derived locally from PostgreSQL.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_5rem_minmax(0,1fr)] items-center gap-3 rounded-xl border border-white/[0.06] bg-black/10 px-4 py-3">
|
||||
<div className={enabled ? 'text-gray-600' : 'text-amber-200/90'}>
|
||||
<div className="num text-[10px] uppercase tracking-[0.16em]">Legacy APIs</div>
|
||||
<div className="mt-0.5 text-[11px]">FMP / Finnhub / Alpha Vantage</div>
|
||||
</div>
|
||||
<div className="relative h-px bg-white/10" aria-hidden="true">
|
||||
<span
|
||||
className={`absolute top-1/2 h-2.5 w-2.5 -translate-y-1/2 rounded-full border-2 border-[#0e120f] transition-all duration-300 ${
|
||||
enabled
|
||||
? 'right-0 bg-cyan-300 shadow-[0_0_12px_rgba(103,232,249,0.55)]'
|
||||
: 'left-0 bg-amber-300'
|
||||
}`}
|
||||
/>
|
||||
</div>
|
||||
<div className={`text-right ${enabled ? 'text-cyan-200' : 'text-gray-600'}`}>
|
||||
<div className="num text-[10px] uppercase tracking-[0.16em]">SEC + Dolt</div>
|
||||
<div className="mt-0.5 text-[11px]">Bulk imports → PostgreSQL cache</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-4 border-t border-white/[0.06] pt-4 md:grid-cols-2">
|
||||
<div className="flex items-start justify-between gap-4 rounded-xl bg-white/[0.025] p-3.5">
|
||||
<div>
|
||||
<div className="num text-[10px] uppercase tracking-[0.14em] text-gray-600">1 · Source</div>
|
||||
<div className="mt-1 text-sm text-gray-200">Use SEC + Dolt for scoring inputs</div>
|
||||
<p className="mt-1 text-[11px] leading-relaxed text-gray-500">
|
||||
While active, the weekly legacy collector is skipped so it cannot overwrite the new cache.
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={enabled}
|
||||
aria-label="Use SEC and Dolt fundamentals"
|
||||
onClick={changeSource}
|
||||
disabled={update.isPending}
|
||||
className={`relative mt-1 inline-flex h-6 w-11 shrink-0 rounded-full border-2 border-transparent transition-colors focus:outline-none focus:ring-2 focus:ring-cyan-400/70 focus:ring-offset-2 focus:ring-offset-[#0e120f] disabled:cursor-wait disabled:opacity-50 ${
|
||||
enabled ? 'bg-gradient-to-r from-sky-500 to-cyan-400' : 'bg-white/10'
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`pointer-events-none inline-block h-5 w-5 rounded-full bg-white shadow transition-transform ${
|
||||
enabled ? 'translate-x-5' : 'translate-x-0'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="rounded-xl bg-white/[0.025] p-3.5">
|
||||
<div className="num text-[10px] uppercase tracking-[0.14em] text-gray-600">2 · Refresh</div>
|
||||
<div className="mt-1 flex flex-wrap items-center justify-between gap-3">
|
||||
<div>
|
||||
<div className="text-sm text-gray-200">Apply the source now</div>
|
||||
<p className="mt-1 text-[11px] text-gray-500">
|
||||
{secJob?.running
|
||||
? 'SEC import and cache refresh are running.'
|
||||
: secJob?.runtime_message || `Last SEC run: ${formatRun(secJob?.runtime_finished_at)}`}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => trigger.mutate(SEC_JOB)}
|
||||
disabled={
|
||||
!enabled ||
|
||||
trigger.isPending ||
|
||||
Boolean(secJob?.running) ||
|
||||
refreshBlocked ||
|
||||
secJob?.enabled === false
|
||||
}
|
||||
className="btn-primary px-3 py-2 text-xs disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
<span>
|
||||
{secJob?.running
|
||||
? 'Refreshing…'
|
||||
: trigger.isPending
|
||||
? 'Starting…'
|
||||
: refreshBlocked
|
||||
? 'Another job is running'
|
||||
: 'Run refresh now'}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
{!enabled && (
|
||||
<p className="mt-2 text-[11px] text-amber-300/70">Activate the source before running the refresh.</p>
|
||||
)}
|
||||
{enabled && secJob?.enabled === false && (
|
||||
<p className="mt-2 text-[11px] text-amber-300/70">Enable the SEC Fundamentals job on the Jobs tab first.</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="text-[11px] leading-relaxed text-gray-600">
|
||||
Rollback pauses future writes only. To restore pre-cutover values, use the database backup or
|
||||
pause this source and manually run the legacy collector while its provider keys remain installed.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
getFundamentalsParityCsv,
|
||||
getFundamentalsParityJson,
|
||||
} from '../../api/admin';
|
||||
import { useFundamentalsParityReport } from '../../hooks/useAdmin';
|
||||
import { SkeletonTable } from '../ui/Skeleton';
|
||||
|
||||
const FIELD_LABELS: Record<string, string> = {
|
||||
pe_ratio: 'P/E',
|
||||
revenue_growth: 'Revenue growth',
|
||||
earnings_surprise: 'Earnings surprise',
|
||||
};
|
||||
|
||||
function downloadText(filename: string, content: string, type: string) {
|
||||
const blob = new Blob([content], { type });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const anchor = document.createElement('a');
|
||||
anchor.href = url;
|
||||
anchor.download = filename;
|
||||
anchor.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
export function FundamentalsParityPanel() {
|
||||
const { data: report, isLoading, isError, error } = useFundamentalsParityReport();
|
||||
const [downloading, setDownloading] = useState(false);
|
||||
|
||||
if (isLoading) return <SkeletonTable rows={2} cols={4} />;
|
||||
if (isError) {
|
||||
return <p className="text-sm text-red-400">{(error as Error).message}</p>;
|
||||
}
|
||||
|
||||
if (!report) {
|
||||
return (
|
||||
<div className="glass p-5">
|
||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
No report yet. Trigger “Fundamentals Parity Report (read-only)” below.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const summary = report.summary;
|
||||
const generated = new Date(report.generated_at).toLocaleString();
|
||||
|
||||
async function downloadCsv() {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const artifact = await getFundamentalsParityCsv();
|
||||
if (artifact) downloadText(artifact.filename, artifact.content, 'text/csv;charset=utf-8');
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadJson() {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const artifact = await getFundamentalsParityJson();
|
||||
if (artifact) downloadText(artifact.filename, artifact.content, 'application/json;charset=utf-8');
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="glass p-5 space-y-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h3 className="text-sm font-semibold text-gray-200">A5 Fundamentals Parity</h3>
|
||||
<span className="rounded-full border border-amber-400/20 bg-amber-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-amber-300">
|
||||
approval pending
|
||||
</span>
|
||||
<span className="rounded-full border border-cyan-400/20 bg-cyan-400/10 px-2 py-0.5 text-[10px] uppercase tracking-wide text-cyan-300">
|
||||
read-only
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Generated {generated} · as of {report.as_of_date} · {summary.universe_count} tracked tickers
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white"
|
||||
onClick={downloadJson}
|
||||
disabled={downloading}
|
||||
>
|
||||
Download JSON
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-white/10 px-3 py-1.5 text-xs text-gray-300 hover:text-white disabled:opacity-50"
|
||||
onClick={downloadCsv}
|
||||
disabled={downloading}
|
||||
>
|
||||
{downloading ? 'Preparing…' : 'Download CSV'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<Summary label="Candidate score coverage" value={`${summary.candidate_fundamental_score_available}/${summary.universe_count}`} />
|
||||
<Summary label="Scores compared" value={summary.fundamental_scores_compared} />
|
||||
<Summary label="Material score moves" value={summary.fundamental_score_material_changes} />
|
||||
<Summary label="Fundamental rank moves" value={summary.fundamental_rank_changes} />
|
||||
</div>
|
||||
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-left text-xs">
|
||||
<thead className="text-[10px] uppercase tracking-wider text-gray-500">
|
||||
<tr>
|
||||
<th className="pb-2 pr-4 font-medium">Field</th>
|
||||
<th className="pb-2 px-3 font-medium">Legacy</th>
|
||||
<th className="pb-2 px-3 font-medium">Candidate</th>
|
||||
<th className="pb-2 px-3 font-medium">Compared</th>
|
||||
<th className="pb-2 px-3 font-medium">Material</th>
|
||||
<th className="pb-2 pl-3 font-medium">Median |Δ|</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-white/[0.06] text-gray-300">
|
||||
{Object.entries(summary.field_stats).map(([key, stats]) => (
|
||||
<tr key={key}>
|
||||
<td className="py-2.5 pr-4">{FIELD_LABELS[key] ?? key}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.legacy_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.candidate_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.both_available}</td>
|
||||
<td className="py-2.5 px-3 num">{stats.material_differences}</td>
|
||||
<td className="py-2.5 pl-3 num">
|
||||
{stats.median_absolute_delta == null ? 'n/a' : stats.median_absolute_delta.toFixed(2)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<p className="text-[11px] leading-relaxed text-gray-500">
|
||||
Materiality bands highlight review candidates only. They do not approve a cutover or write fundamentals,
|
||||
scores, rankings, or qualification state.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Summary({ label, value }: { label: string; value: string | number }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-white/[0.07] bg-white/[0.025] px-3 py-2.5">
|
||||
<div className="text-[10px] uppercase tracking-wider text-gray-500">{label}</div>
|
||||
<div className="mt-1 num text-lg text-gray-200">{value}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -8,11 +8,9 @@ const DEFAULTS: ScheduleConfig = {
|
||||
schedule_daily_pipeline_cron: '0 2 * * *',
|
||||
schedule_dolt_earnings_cron: '30 2 * * *',
|
||||
schedule_sec_fundamentals_cron: '0 4 * * *',
|
||||
schedule_fundamentals_parity_cron: '30 5 * * *',
|
||||
schedule_near_close_pipeline_cron: '30 15 * * mon-fri',
|
||||
schedule_after_close_pipeline_cron: '45 16 * * mon-fri',
|
||||
schedule_intraday_pipeline_cron: '0 10-15 * * mon-fri',
|
||||
schedule_fundamentals_cron: '0 1 * * mon',
|
||||
};
|
||||
|
||||
const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: boolean }[] = [
|
||||
@@ -30,19 +28,13 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
||||
{
|
||||
key: 'schedule_dolt_earnings_cron',
|
||||
label: 'Dolt earnings',
|
||||
hint: 'Pull and import earnings dates/results daily at 02:30 ET. The activated cache refresh uses these local events.',
|
||||
hint: 'Pull and import earnings dates/results daily at 02:30 ET. The fundamentals cache refresh uses these local events.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_sec_fundamentals_cron',
|
||||
label: 'SEC fundamentals',
|
||||
hint: 'Import tracked-universe SEC facts daily at 04:00 ET and refresh the scoring cache when the cutover is active.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_fundamentals_parity_cron',
|
||||
label: 'Fundamentals parity report',
|
||||
hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the bulk imports.',
|
||||
hint: 'Import tracked-universe SEC facts daily at 04:00 ET, then refresh the fundamentals cache scoring reads. Disabling the job stops the SEC fetch only — the local cache refresh still runs.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
@@ -63,12 +55,6 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b
|
||||
hint: 'Refresh prices + resolve outcomes mid-session. Default hourly 10:00–15:00 ET weekdays.',
|
||||
mono: true,
|
||||
},
|
||||
{
|
||||
key: 'schedule_fundamentals_cron',
|
||||
label: 'Legacy fundamentals (weekly)',
|
||||
hint: 'Fallback provider chain. Automatically skipped while the SEC + Dolt cutover is active.',
|
||||
mono: true,
|
||||
},
|
||||
];
|
||||
|
||||
export function ScheduleSettings() {
|
||||
|
||||
@@ -3,7 +3,13 @@ import { useSettings, useUpdateSetting } from '../../hooks/useAdmin';
|
||||
import { SkeletonTable } from '../ui/Skeleton';
|
||||
import type { SystemSetting } from '../../lib/types';
|
||||
|
||||
const MANAGED_SETTINGS = new Set(['fundamental_data_sec_dolt_cutover_enabled']);
|
||||
// Retired keys kept in the database as rollback tombstones (migration 029).
|
||||
// They no longer control anything; hide them so nobody edits a dead switch.
|
||||
// Delete both the rows and this filter once the A6 rollback window has closed.
|
||||
const MANAGED_SETTINGS = new Set([
|
||||
'fundamental_data_sec_dolt_cutover_enabled',
|
||||
'job_fundamental_collector_enabled',
|
||||
]);
|
||||
|
||||
export function SettingsForm() {
|
||||
const { data: settings, isLoading, isError, error } = useSettings();
|
||||
|
||||
Reference in New Issue
Block a user