fix: refresh same-day OHLCV bars

This commit is contained in:
2026-08-03 23:13:17 +02:00
parent 7bcdf77ef9
commit 7d703ea524
4 changed files with 102 additions and 9 deletions
+3 -3
View File
@@ -518,7 +518,7 @@ async def collect_ohlcv(
Uses AlpacaOHLCVProvider. Processes each ticker independently. Uses AlpacaOHLCVProvider. Processes each ticker independently.
On rate limit, records last successful ticker for resume. On rate limit, records last successful ticker for resume.
Start date is resolved by ingestion progress: Start date is resolved by ingestion progress:
- existing ticker: resume from last_ingested_date + 1 - existing ticker: overlap last_ingested_date so partial bars refresh
- new ticker: backfill the configured history window - new ticker: backfill the configured history window
``full_backfill`` forces every ticker to re-fetch the full ``full_backfill`` forces every ticker to re-fetch the full
@@ -1496,8 +1496,8 @@ _DAILY_PIPELINE_STEPS = [
("alerts", "dispatch_alerts_job"), ("alerts", "dispatch_alerts_job"),
] ]
# Near-close (~15:30 ET MonFri): refresh in-progress day-t bars (already how # Near-close (~15:30 ET MonFri): refresh in-progress day-t bars (incremental
# the intraday pipeline keeps the dashboard live), then the only daily # ingestion overlaps the latest stored session), then the only daily
# qualifying R:R scan, then Telegram immediately so manual fills can still hit # qualifying R:R scan, then Telegram immediately so manual fills can still hit
# MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan # MOC cutoffs (~15:50/15:55). Under a 15-minute delayed SIP feed a 15:30 scan
# may see ~15:15 prices — immaterial for a 12-1 momentum signal. # may see ~15:15 prices — immaterial for a 12-1 momentum signal.
+6 -1
View File
@@ -129,7 +129,12 @@ async def fetch_and_ingest(
if bar_count < minimum_backfill_bars: if bar_count < minimum_backfill_bars:
start_date = backfill_start start_date = backfill_start
elif progress is not None: elif progress is not None:
start_date = progress.last_ingested_date + timedelta(days=1) # Re-fetch the latest stored session so an in-progress daily bar can
# be overwritten as the market moves. Starting one day later makes
# every subsequent intraday, near-close, and manual refresh skip
# today's bar once the first partial snapshot has been stored.
# The price-store upsert keeps this one-session overlap idempotent.
start_date = progress.last_ingested_date
else: else:
start_date = backfill_start start_date = backfill_start
+42 -3
View File
@@ -64,10 +64,43 @@ function timeAgo(iso: string): string {
return `${days}d ago`; return `${days}d ago`;
} }
function marketDate(date = new Date()): string {
const parts = new Intl.DateTimeFormat('en-US', {
timeZone: 'America/New_York',
year: 'numeric',
month: '2-digit',
day: '2-digit',
}).formatToParts(date);
const value = (type: Intl.DateTimeFormatPartTypes) =>
parts.find((part) => part.type === type)?.value ?? '';
return value('year') + '-' + value('month') + '-' + value('day');
}
function formatSessionDate(isoDate: string): string {
if (isoDate === marketDate()) return 'Today';
// Parse date-only market sessions explicitly. Parsing YYYY-MM-DD directly as
// a Date means midnight UTC and makes today's bar look many hours old.
const [year, month, day] = isoDate.split('-').map(Number);
if (!year || !month || !day) return isoDate;
return new Intl.DateTimeFormat(undefined, {
month: 'short',
day: 'numeric',
year: year === new Date().getFullYear() ? undefined : 'numeric',
timeZone: 'UTC',
}).format(new Date(Date.UTC(year, month - 1, day)));
}
function formatOHLCVFreshness(sessionDate: string, updatedAt?: string | null): string {
const session = formatSessionDate(sessionDate);
return updatedAt ? session + ' · updated ' + timeAgo(updatedAt) : session;
}
interface DataStatusItem { interface DataStatusItem {
label: string; label: string;
available: boolean; available: boolean;
timestamp?: string | null; timestamp?: string | null;
timestampLabel?: string | null;
selector: FetchSelector; // what a refresh of this row fetches selector: FetchSelector; // what a refresh of this row fetches
paid?: boolean; // provider call that may cost money/quota paid?: boolean; // provider call that may cost money/quota
} }
@@ -100,7 +133,7 @@ function DataFreshnessBar({
}`} /> }`} />
<span className="text-xs text-gray-400">{item.label}</span> <span className="text-xs text-gray-400">{item.label}</span>
{item.available && item.timestamp ? ( {item.available && item.timestamp ? (
<span className="text-[10px] text-gray-500">{timeAgo(item.timestamp)}</span> <span className="text-[10px] text-gray-500">{item.timestampLabel ?? timeAgo(item.timestamp)}</span>
) : !item.available ? ( ) : !item.available ? (
<span className="text-[10px] text-gray-600">no data</span> <span className="text-[10px] text-gray-600">no data</span>
) : null} ) : null}
@@ -171,10 +204,16 @@ export default function TickerDetailPage() {
const dataStatus: DataStatusItem[] = useMemo(() => [ const dataStatus: DataStatusItem[] = useMemo(() => [
{ {
label: 'OHLCV', label: 'OHLCV',
// Market age of the latest bar (session date), not DB insert time — // Keep the market session date distinct from the last successful bar
// created_at stays frozen when the provider returns no new sessions. // write; treating YYYY-MM-DD as an instant makes today's session look old.
available: !!ohlcv.data && ohlcv.data.length > 0, available: !!ohlcv.data && ohlcv.data.length > 0,
timestamp: ohlcv.data?.[ohlcv.data.length - 1]?.date, timestamp: ohlcv.data?.[ohlcv.data.length - 1]?.date,
timestampLabel: ohlcv.data?.length
? formatOHLCVFreshness(
ohlcv.data[ohlcv.data.length - 1].date,
ohlcv.data[ohlcv.data.length - 1].created_at,
)
: null,
selector: ['ohlcv'] as FetchSelector, selector: ['ohlcv'] as FetchSelector,
paid: true, paid: true,
}, },
+51 -2
View File
@@ -6,6 +6,8 @@ from datetime import date, timedelta
import pytest import pytest
from app.models.ohlcv import OHLCVRecord
from app.models.settings import IngestionProgress
from app.models.ticker import Ticker from app.models.ticker import Ticker
from app.providers.protocol import OHLCVData from app.providers.protocol import OHLCVData
from app.services import ingestion_service as svc from app.services import ingestion_service as svc
@@ -18,9 +20,12 @@ async def session():
yield s yield s
async def _add_ticker(session, symbol: str) -> None: async def _add_ticker(session, symbol: str) -> Ticker:
session.add(Ticker(symbol=symbol)) ticker = Ticker(symbol=symbol)
session.add(ticker)
await session.commit() await session.commit()
await session.refresh(ticker)
return ticker
def _bars(symbol: str, n: int) -> list[OHLCVData]: def _bars(symbol: str, n: int) -> list[OHLCVData]:
@@ -51,6 +56,50 @@ async def test_happy_path_ingests_bars(session):
assert result.records_ingested == 3 assert result.records_ingested == 3
async def test_incremental_fetch_overlaps_latest_session_and_updates_partial_bar(session):
"""Once today exists, a live refresh must fetch and overwrite it again."""
ticker = await _add_ticker(session, "LIVE")
today = date.today()
session.add_all([
OHLCVRecord(
ticker_id=ticker.id,
date=today - timedelta(days=i),
open=100.0,
high=101.0,
low=99.0,
close=100.0,
volume=1000,
)
for i in range(200)
])
session.add(IngestionProgress(ticker_id=ticker.id, last_ingested_date=today))
await session.commit()
provider = MockMarketDataProvider(ohlcv_data=[
OHLCVData(
ticker="LIVE",
date=today,
open=100.0,
high=124.0,
low=99.0,
close=123.0,
volume=2000,
)
])
result = await svc.fetch_and_ingest(session, provider, "LIVE")
assert provider.calls == [{
"ticker": "LIVE",
"start_date": today,
"end_date": today,
}]
assert result.status == "complete"
assert result.records_ingested == 1
records = await svc.price_service.query_ohlcv(session, "LIVE", today, today)
assert records[0].close == 123.0
assert records[0].volume == 2000
async def test_empty_fetch_with_existing_history_is_up_to_date(session): async def test_empty_fetch_with_existing_history_is_up_to_date(session):
# Covered ticker, just no new bars in the window → complete, not no_data. # Covered ticker, just no new bars in the window → complete, not no_data.
await _add_ticker(session, "BBB") await _add_ticker(session, "BBB")