"""Single source for JobRunState reads/writes. Mirrors ``settings_store``: ``record_finish`` never commits — the caller owns the transaction — and reads are batched so the admin listing stays one query. """ from __future__ import annotations import logging from collections.abc import Iterable from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncSession from app.models.job_run_state import JobRunState logger = logging.getLogger(__name__) def _as_datetime(value: object) -> datetime | None: """Runtime snapshots carry ISO strings; the column wants a datetime.""" if isinstance(value, datetime): return value if isinstance(value, str) and value: try: return datetime.fromisoformat(value) except ValueError: return None return None async def get_map(db: AsyncSession, job_names: Iterable[str]) -> dict[str, JobRunState]: """Return {job_name: row} for the given jobs that have ever finished. ``populate_existing`` because rows are written by core upserts, which leave any previously-loaded ORM instance in the identity map stale. """ result = await db.execute( select(JobRunState) .where(JobRunState.job_name.in_(list(job_names))) .execution_options(populate_existing=True) ) return {row.job_name: row for row in result.scalars().all()} def _insert_for(db: AsyncSession): """ON CONFLICT is dialect-specific; prod is Postgres, tests are SQLite.""" dialect = db.get_bind().dialect.name return pg_insert if dialect == "postgresql" else sqlite_insert async def record_finish(db: AsyncSession, job_name: str, runtime: dict) -> None: """Upsert the last-run row from a scheduler runtime snapshot. Atomic, and newer-wins. Select-then-insert loses races that really happen here: pipelines are separate scheduler jobs that can overlap, and they share step ids -- data_collector belongs to all four. Two of them finishing that step together would both see no row and both insert, and the loser's IntegrityError is swallowed by the caller, so the run silently vanishes. The ``where`` guard is the other half: without it a slower pipeline finishing an *older* run last would rewind finished_at and the status with it, so the panel would report a stale outcome as the latest one. """ finished_at = _as_datetime(runtime.get("finished_at")) or datetime.now(timezone.utc) message = runtime.get("message") now = datetime.now(timezone.utc) values = { "job_name": job_name, "status": str(runtime.get("status") or "completed"), "started_at": _as_datetime(runtime.get("started_at")), "finished_at": finished_at, "processed": runtime.get("processed"), "total": runtime.get("total"), "message": str(message)[:4000] if message else None, # Set explicitly: the model's onupdate hook does not fire for a core # INSERT ... ON CONFLICT DO UPDATE. "updated_at": now, } statement = _insert_for(db)(JobRunState).values(**values) await db.execute( statement.on_conflict_do_update( index_elements=[JobRunState.job_name], set_={key: statement.excluded[key] for key in values if key != "job_name"}, where=JobRunState.finished_at < statement.excluded.finished_at, ) )