Add system alerts log with nav badge and Admin Alerts tab.
Persist job and ingestion warnings/errors for 7 days, surface a dismissible top-nav badge, treat stale OHLCV as a warning (e.g. ticker renames), and show market bar age on the ticker freshness chip.
This commit is contained in:
@@ -152,6 +152,28 @@ def _log_job_error(job_name: str, ticker: str, error: Exception) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _record_system_event(
|
||||
*,
|
||||
severity: str,
|
||||
source: str,
|
||||
code: str,
|
||||
message: str,
|
||||
symbol: str | None = None,
|
||||
dedup_key: str | None = None,
|
||||
) -> None:
|
||||
"""Best-effort durable event for Admin → Jobs and the top-nav badge."""
|
||||
from app.services.system_event_service import log_event_standalone
|
||||
|
||||
await log_event_standalone(
|
||||
severity=severity,
|
||||
source=source,
|
||||
code=code,
|
||||
message=message,
|
||||
symbol=symbol,
|
||||
dedup_key=dedup_key,
|
||||
)
|
||||
|
||||
|
||||
def _runtime_start(job_name: str, total: int | None = None, message: str | None = None) -> None:
|
||||
_job_runtime[job_name] = {
|
||||
**_idle_runtime(),
|
||||
@@ -206,6 +228,22 @@ def _runtime_finish(
|
||||
"message": message,
|
||||
})
|
||||
_job_runtime[job_name] = runtime
|
||||
# Durable event for error / rate-limit finishes (badge + Admin → Jobs panel).
|
||||
if status in ("error", "rate_limited"):
|
||||
severity = "error" if status == "error" else "warning"
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(
|
||||
_record_system_event(
|
||||
severity=severity,
|
||||
source=job_name,
|
||||
code=f"job_{status}",
|
||||
message=message or f"Job {job_name} finished with status {status}",
|
||||
dedup_key=f"job:{job_name}:{status}:{(message or '')[:80]}"[:200],
|
||||
)
|
||||
)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def get_job_runtime_snapshot(job_name: str | None = None) -> dict[str, dict[str, object]] | dict[str, object]:
|
||||
@@ -489,6 +527,15 @@ async def collect_ohlcv(full_backfill: bool = False, job_name: str = "data_colle
|
||||
processed += 1
|
||||
_runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol)
|
||||
_log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol, status=result.status, records=result.records_ingested)
|
||||
if result.status == "stale":
|
||||
await _record_system_event(
|
||||
severity="warning",
|
||||
source=job_name,
|
||||
code="ohlcv_stale",
|
||||
message=result.message or f"No new OHLCV bars for {symbol}",
|
||||
symbol=symbol,
|
||||
dedup_key=f"ohlcv_stale:{symbol}",
|
||||
)
|
||||
if result.status == "partial":
|
||||
# Rate limited — stop and resume next run
|
||||
_log_event(logging.WARNING, "rate_limited", job=job_name, ticker=symbol, processed=processed)
|
||||
@@ -496,6 +543,14 @@ async def collect_ohlcv(full_backfill: bool = False, job_name: str = "data_colle
|
||||
return
|
||||
except Exception as exc:
|
||||
_log_job_error(job_name, symbol, exc)
|
||||
await _record_system_event(
|
||||
severity="error",
|
||||
source=job_name,
|
||||
code="job_ticker_error",
|
||||
message=f"{type(exc).__name__}: {exc}",
|
||||
symbol=symbol,
|
||||
dedup_key=f"job_ticker_error:{job_name}:{symbol}:{type(exc).__name__}",
|
||||
)
|
||||
|
||||
# Reset resume pointer on full completion
|
||||
_last_successful[job_name] = None
|
||||
|
||||
Reference in New Issue
Block a user