fix: refresh same-day OHLCV bars

This commit is contained in:
2026-08-03 23:13:17 +02:00
parent 7bcdf77ef9
commit 7d703ea524
4 changed files with 102 additions and 9 deletions
+3 -3
View File
@@ -518,7 +518,7 @@ async def collect_ohlcv(
Uses AlpacaOHLCVProvider. Processes each ticker independently.
On rate limit, records last successful ticker for resume.
Start date is resolved by ingestion progress:
- existing ticker: resume from last_ingested_date + 1
- existing ticker: overlap last_ingested_date so partial bars refresh
- new ticker: backfill the configured history window
``full_backfill`` forces every ticker to re-fetch the full
@@ -1496,8 +1496,8 @@ _DAILY_PIPELINE_STEPS = [
("alerts", "dispatch_alerts_job"),
]
# Near-close (~15:30 ET MonFri): refresh in-progress day-t bars (already how
# the intraday pipeline keeps the dashboard live), then the only daily
# Near-close (~15:30 ET MonFri): refresh in-progress day-t bars (incremental
# ingestion overlaps the latest stored session), then the only daily
# qualifying R:R scan, then Telegram immediately so manual fills can still hit
# MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan
# may see ~15:15 prices — immaterial for a 12-1 momentum signal.
+6 -1
View File
@@ -129,7 +129,12 @@ async def fetch_and_ingest(
if bar_count < minimum_backfill_bars:
start_date = backfill_start
elif progress is not None:
start_date = progress.last_ingested_date + timedelta(days=1)
# Re-fetch the latest stored session so an in-progress daily bar can
# be overwritten as the market moves. Starting one day later makes
# every subsequent intraday, near-close, and manual refresh skip
# today's bar once the first partial snapshot has been stored.
# The price-store upsert keeps this one-session overlap idempotent.
start_date = progress.last_ingested_date
else:
start_date = backfill_start
+42 -3
View File
@@ -64,10 +64,43 @@ function timeAgo(iso: string): string {
return `${days}d ago`;
}
function marketDate(date = new Date()): string {
const parts = new Intl.DateTimeFormat('en-US', {
timeZone: 'America/New_York',
year: 'numeric',
month: '2-digit',
day: '2-digit',
}).formatToParts(date);
const value = (type: Intl.DateTimeFormatPartTypes) =>
parts.find((part) => part.type === type)?.value ?? '';
return value('year') + '-' + value('month') + '-' + value('day');
}
function formatSessionDate(isoDate: string): string {
if (isoDate === marketDate()) return 'Today';
// Parse date-only market sessions explicitly. Parsing YYYY-MM-DD directly as
// a Date means midnight UTC and makes today's bar look many hours old.
const [year, month, day] = isoDate.split('-').map(Number);
if (!year || !month || !day) return isoDate;
return new Intl.DateTimeFormat(undefined, {
month: 'short',
day: 'numeric',
year: year === new Date().getFullYear() ? undefined : 'numeric',
timeZone: 'UTC',
}).format(new Date(Date.UTC(year, month - 1, day)));
}
function formatOHLCVFreshness(sessionDate: string, updatedAt?: string | null): string {
const session = formatSessionDate(sessionDate);
return updatedAt ? session + ' · updated ' + timeAgo(updatedAt) : session;
}
interface DataStatusItem {
label: string;
available: boolean;
timestamp?: string | null;
timestampLabel?: string | null;
selector: FetchSelector; // what a refresh of this row fetches
paid?: boolean; // provider call that may cost money/quota
}
@@ -100,7 +133,7 @@ function DataFreshnessBar({
}`} />
<span className="text-xs text-gray-400">{item.label}</span>
{item.available && item.timestamp ? (
<span className="text-[10px] text-gray-500">{timeAgo(item.timestamp)}</span>
<span className="text-[10px] text-gray-500">{item.timestampLabel ?? timeAgo(item.timestamp)}</span>
) : !item.available ? (
<span className="text-[10px] text-gray-600">no data</span>
) : null}
@@ -171,10 +204,16 @@ export default function TickerDetailPage() {
const dataStatus: DataStatusItem[] = useMemo(() => [
{
label: 'OHLCV',
// Market age of the latest bar (session date), not DB insert time —
// created_at stays frozen when the provider returns no new sessions.
// Keep the market session date distinct from the last successful bar
// write; treating YYYY-MM-DD as an instant makes today's session look old.
available: !!ohlcv.data && ohlcv.data.length > 0,
timestamp: ohlcv.data?.[ohlcv.data.length - 1]?.date,
timestampLabel: ohlcv.data?.length
? formatOHLCVFreshness(
ohlcv.data[ohlcv.data.length - 1].date,
ohlcv.data[ohlcv.data.length - 1].created_at,
)
: null,
selector: ['ohlcv'] as FetchSelector,
paid: true,
},
+51 -2
View File
@@ -6,6 +6,8 @@ from datetime import date, timedelta
import pytest
from app.models.ohlcv import OHLCVRecord
from app.models.settings import IngestionProgress
from app.models.ticker import Ticker
from app.providers.protocol import OHLCVData
from app.services import ingestion_service as svc
@@ -18,9 +20,12 @@ async def session():
yield s
async def _add_ticker(session, symbol: str) -> None:
session.add(Ticker(symbol=symbol))
async def _add_ticker(session, symbol: str) -> Ticker:
ticker = Ticker(symbol=symbol)
session.add(ticker)
await session.commit()
await session.refresh(ticker)
return ticker
def _bars(symbol: str, n: int) -> list[OHLCVData]:
@@ -51,6 +56,50 @@ async def test_happy_path_ingests_bars(session):
assert result.records_ingested == 3
async def test_incremental_fetch_overlaps_latest_session_and_updates_partial_bar(session):
"""Once today exists, a live refresh must fetch and overwrite it again."""
ticker = await _add_ticker(session, "LIVE")
today = date.today()
session.add_all([
OHLCVRecord(
ticker_id=ticker.id,
date=today - timedelta(days=i),
open=100.0,
high=101.0,
low=99.0,
close=100.0,
volume=1000,
)
for i in range(200)
])
session.add(IngestionProgress(ticker_id=ticker.id, last_ingested_date=today))
await session.commit()
provider = MockMarketDataProvider(ohlcv_data=[
OHLCVData(
ticker="LIVE",
date=today,
open=100.0,
high=124.0,
low=99.0,
close=123.0,
volume=2000,
)
])
result = await svc.fetch_and_ingest(session, provider, "LIVE")
assert provider.calls == [{
"ticker": "LIVE",
"start_date": today,
"end_date": today,
}]
assert result.status == "complete"
assert result.records_ingested == 1
records = await svc.price_service.query_ohlcv(session, "LIVE", today, today)
assert records[0].close == 123.0
assert records[0].volume == 2000
async def test_empty_fetch_with_existing_history_is_up_to_date(session):
# Covered ticker, just no new bars in the window → complete, not no_data.
await _add_ticker(session, "BBB")