fix: preserve OHLCV stale detection
This commit is contained in:
+8
-1
@@ -512,6 +512,7 @@ async def collect_ohlcv(
|
||||
job_name: str = "data_collector",
|
||||
*,
|
||||
refetch_days: int = 0,
|
||||
refresh_sr: bool = True,
|
||||
) -> None:
|
||||
"""Fetch latest daily OHLCV for all tracked tickers.
|
||||
|
||||
@@ -580,6 +581,7 @@ async def collect_ohlcv(
|
||||
try:
|
||||
result = await ingestion_service.fetch_and_ingest(
|
||||
db, provider, symbol, start_date=backfill_start, end_date=end_date,
|
||||
refresh_sr=refresh_sr,
|
||||
)
|
||||
_last_successful[job_name] = symbol
|
||||
processed += 1
|
||||
@@ -619,6 +621,11 @@ async def collect_ohlcv(
|
||||
_runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc))
|
||||
|
||||
|
||||
async def collect_ohlcv_for_scan() -> None:
|
||||
"""Near-close fetch; the scanner immediately rebuilds S/R per ticker."""
|
||||
await collect_ohlcv(refresh_sr=False)
|
||||
|
||||
|
||||
async def backfill_ohlcv() -> None:
|
||||
"""Deep historical backfill: re-fetch the full ``settings.ohlcv_history_days``
|
||||
window for every ticker, ignoring incremental resume.
|
||||
@@ -1484,7 +1491,7 @@ async def sync_ticker_universe() -> None:
|
||||
_FINAL_REFETCH_DAYS = 5
|
||||
|
||||
_DAILY_PIPELINE_STEPS = [
|
||||
("data_collector", "collect_ohlcv"),
|
||||
("data_collector", "collect_ohlcv_for_scan"),
|
||||
("benchmark_collector", "collect_benchmark"),
|
||||
("sentiment_collector", "collect_sentiment"),
|
||||
("market_regime", "compute_market_regime"),
|
||||
|
||||
@@ -100,6 +100,8 @@ async def fetch_and_ingest(
|
||||
symbol: str,
|
||||
start_date: date | None = None,
|
||||
end_date: date | None = None,
|
||||
*,
|
||||
refresh_sr: bool = True,
|
||||
) -> IngestionResult:
|
||||
"""Fetch OHLCV data from provider and upsert into Price Store.
|
||||
|
||||
@@ -244,7 +246,7 @@ async def fetch_and_ingest(
|
||||
ticker.symbol,
|
||||
ingested_count,
|
||||
)
|
||||
if ingested_count > 0:
|
||||
if ingested_count > 0 and refresh_sr:
|
||||
await _refresh_structural_sr(db, ticker.symbol)
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
@@ -254,9 +256,28 @@ async def fetch_and_ingest(
|
||||
message=f"Rate limited. Ingested {ingested_count} records. Resume available.",
|
||||
)
|
||||
|
||||
if ingested_count > 0:
|
||||
if ingested_count > 0 and refresh_sr:
|
||||
await _refresh_structural_sr(db, ticker.symbol)
|
||||
|
||||
# Incremental fetches deliberately overlap the latest stored session so an
|
||||
# in-progress bar can be updated. A halted/delisted symbol can therefore
|
||||
# return one old bar forever; non-empty no longer means fresh. Judge stale
|
||||
# state from the newest stored session after the upserts instead.
|
||||
latest = await _get_latest_ohlcv_date(db, ticker.id)
|
||||
gap_days = (end_date - latest).days if latest is not None else None
|
||||
if gap_days is not None and gap_days > _STALE_OHLCV_GAP_DAYS:
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
records_ingested=ingested_count,
|
||||
last_date=latest,
|
||||
status="stale",
|
||||
message=(
|
||||
f"No new bars since {latest.isoformat()} ({gap_days}d gap). "
|
||||
"The symbol may be halted, delisted, or renamed under a new ticker — "
|
||||
"check the listing and add/fetch the current symbol if it changed."
|
||||
),
|
||||
)
|
||||
|
||||
return IngestionResult(
|
||||
symbol=ticker.symbol,
|
||||
records_ingested=ingested_count,
|
||||
|
||||
Reference in New Issue
Block a user