dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dolt): A2 review — subprocess timeouts, stronger initial gate, HASHOF

Browse Source 
Addresses the A2 review:

1. Every dolt subprocess is now bounded by a hard timeout
   (dolt_command_timeout_seconds, default 600s); on expiry the process is killed
   and DoltError raised — a hung pull/sql can no longer pin the import
   connection and advisory lock indefinitely. Tested (timeout + non-zero exit).
2. Initial-load validate is stronger: besides zero-future, an initial load now
   requires a real forward horizon (>= 21d, under the ~35d observed on the
   clone) AND universe coverage >= 50% (a broken symbol join can't seed a hollow
   calendar). Subsequent runs keep the 50% collapse gate.
3. Revision uses DOLT_HASHOF('HEAD') — formally HEAD, not dolt_log-by-timestamp.
4. Free-disk floor raised 2 GB -> 5 GB (safe headroom over the ~1.7 GB clone).

Full suite 702 passed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
dennisthiessen
2026-07-22 11:50:50 +02:00
co-authored by Claude Opus 4.8
parent a4e33d7a39
commit 5d275c8df7
6 changed files with 161 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_dolt_client.py
+47
View File
@@ -0,0 +1,47 @@
"""Tests for the async dolt subprocess wrapper's failure handling.
Uses the Python interpreter as a stand-in subprocess (cross-platform, no dolt
needed) to prove a non-zero exit and a hung command both raise DoltError — the
latter is what stops a hung pull from pinning the import connection + advisory
lock forever.
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import pytest
from app.services import dolt_client
from app.services.dolt_client import DoltError
async def test_run_raises_on_nonzero_exit():
with pytest.raises(DoltError) as exc:
await dolt_client._run(
sys.executable, ["-c", "import sys; sys.exit(3)"], cwd=Path.cwd(), timeout=30
)
assert "3" in str(exc.value)
async def test_run_times_out_and_kills():
start = time.monotonic()
with pytest.raises(DoltError) as exc:
await dolt_client._run(
sys.executable,
["-c", "import time; time.sleep(30)"],
cwd=Path.cwd(),
timeout=0.5,
)
elapsed = time.monotonic() - start
assert "timed out" in str(exc.value)
assert elapsed < 10 # killed promptly, not waited out
async def test_run_returns_stdout_on_success():
out = await dolt_client._run(
sys.executable, ["-c", "print('hello')"], cwd=Path.cwd(), timeout=30
)
assert out.strip() == "hello"
tests/unit/test_dolt_earnings_importer.py
+21 -19
View File
@@ -57,13 +57,13 @@ class FakeDolt:
self.commit = commit
self.pulled = False
async def pull(self, repo_dir, *, binary):
async def pull(self, repo_dir, *, binary, timeout=None):
self.pulled = True
async def current_commit(self, repo_dir, *, binary):
async def current_commit(self, repo_dir, *, binary, timeout=None):
return self.commit
async def query_csv(self, repo_dir, sql, *, binary):
async def query_csv(self, repo_dir, sql, *, binary, timeout=None):
if "earnings_calendar" in sql:
return self.calendar
if "eps_history" in sql:
@@ -111,18 +111,18 @@ async def test_stage_and_promote_basic(engine):
factory = _factory(engine)
await _seed_tickers(factory, ["AAPL"])
fake = FakeDolt(
calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-01")],
calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-20")],
history=[_hist("AAPL", "2026-03-31", 1.5, 1.4)], # only the reported quarter
)
run = await run_import(_importer(fake), engine=engine)
assert run.status == STATUS_PROMOTED
assert run.source_max_date == date(2026, 8, 1)
assert run.source_max_date == date(2026, 8, 20)
events = await _events(factory)
assert len(events) == 2
past = next(e for e in events if e.announce_date == date(2026, 5, 1))
future = next(e for e in events if e.announce_date == date(2026, 8, 1))
future = next(e for e in events if e.announce_date == date(2026, 8, 20))
# past announcement paired to the reported quarter
assert past.eps_actual == 1.5 and past.eps_estimate == 1.4
assert past.period_end == date(2026, 3, 31) and past.session == "amc"
@@ -136,7 +136,7 @@ async def test_symbol_normalisation_join(engine):
factory = _factory(engine)
ids = await _seed_tickers(factory, ["AAPL", "BRK.B"])
fake = FakeDolt(
calendar=[_cal("AAPL", "2026-08-01"), _cal("BRK.B", "2026-08-05")], # dotted source symbol
calendar=[_cal("AAPL", "2026-08-20"), _cal("BRK.B", "2026-08-25")], # dotted source symbol
history=[],
)
run = await run_import(_importer(fake), engine=engine)
@@ -151,15 +151,15 @@ async def test_reschedule_moves_future_row(engine):
factory = _factory(engine)
await _seed_tickers(factory, ["AAPL"])
fake1 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c1")
fake1 = FakeDolt(calendar=[_cal("AAPL", "2026-08-20")], history=[], commit="c1")
await run_import(_importer(fake1), engine=engine)
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-08")], history=[], commit="c2") # moved
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-27")], history=[], commit="c2") # moved
run2 = await run_import(_importer(fake2), engine=engine)
assert run2.status == STATUS_PROMOTED
dates = {e.announce_date for e in await _events(factory)}
assert dates == {date(2026, 8, 8)} # old future date gone, new one present
assert dates == {date(2026, 8, 27)} # old future date gone, new one present
async def test_cancellation_removes_future_row(engine):
@@ -183,17 +183,17 @@ async def test_past_row_never_deleted(engine):
await _seed_tickers(factory, ["AAPL"])
fake1 = FakeDolt(
calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-01")], history=[], commit="c1"
calendar=[_cal("AAPL", "2026-05-01"), _cal("AAPL", "2026-08-20")], history=[], commit="c1"
)
await run_import(_importer(fake1), engine=engine)
# Second import's calendar omits the past date but keeps a future one.
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c2")
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-20")], history=[], commit="c2")
await run_import(_importer(fake2), engine=engine)
dates = {e.announce_date for e in await _events(factory)}
assert date(2026, 5, 1) in dates # past result survived
assert date(2026, 8, 1) in dates
assert date(2026, 8, 20) in dates
async def test_validate_fails_when_no_future(engine):
@@ -214,10 +214,10 @@ async def test_validate_fails_on_forward_collapse(engine):
fake1 = FakeDolt(
calendar=[
_cal("AAPL", "2026-08-01"),
_cal("MSFT", "2026-08-02"),
_cal("NVDA", "2026-08-03"),
_cal("AMZN", "2026-08-04"),
_cal("AAPL", "2026-08-20"),
_cal("MSFT", "2026-08-21"),
_cal("NVDA", "2026-08-22"),
_cal("AMZN", "2026-08-23"),
],
history=[],
commit="c1",
@@ -226,7 +226,7 @@ async def test_validate_fails_on_forward_collapse(engine):
assert len([e for e in await _events(factory) if e.announce_date > TODAY]) == 4
# Only one future row now → 1 < 50% of 4 → fail-closed, no destructive wipe.
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-01")], history=[], commit="c2")
fake2 = FakeDolt(calendar=[_cal("AAPL", "2026-08-20")], history=[], commit="c2")
run2 = await run_import(_importer(fake2), engine=engine)
assert run2.status == STATUS_FAILED
@@ -249,7 +249,9 @@ async def test_real_clone_smoke(engine):
from app.services import dolt_client
factory = _factory(engine)
await _seed_tickers(factory, ["AAPL", "MSFT", "JPM"])
# A few tickers spanning near + further-out reporters so the initial-load
# forward-horizon gate (>= 21d) is satisfied on the fixed clone.
await _seed_tickers(factory, ["AAPL", "MSFT", "NVDA", "JPM", "BRK.B"])
imp = DoltEarningsImporter(
repo_dir=_CLONE_DIR, binary=_DOLT_BIN, today=date.today(), do_pull=False, dolt=dolt_client
)