research: Task 2 closed — SUE dead, earnings gap informational
Earnings backfill sourced from the public DoltHub earnings repo at a pinned commit rather than the FMP API: reproducible for anyone re-running the study, and it burns no request quota. 12,414 events, 98.6% of symbols with >=8 announcements, 99.2% paired actual/estimate, no keyed duplicates. 2a earnings-gap diagnostic: INFORMATIONAL, no filter shipped. The pre-earnings cohort's right tail was better, so the registered avoid-earnings condition failed. Note the raw 23/266 vs 115/574 incidence gap is largely a duration confound -- severe losses stop out fast and have less time to span an announcement -- so it is not evidence that holding through earnings is safe. 2b SUE: FAIL against the pre-registered +0.03 bar (unconditional IC +0.0151 over 56 reliable windows, momentum-conditional +0.0213). Signs stable across eras, so this is a clean null rather than an ambiguous one, consistent with post-earnings drift having decayed in large caps. Closes the Tier-1 arc: Task 1 dead on deep evidence, Task 2 dead here, Task 3 complete as diagnostic. No in-sample research thread remains open. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,657 @@
|
||||
"""Import the public post-no-preference/earnings DoltHub database.
|
||||
|
||||
The earnings calendar and EPS history are separate tables in the source. This
|
||||
importer aligns them monotonically per symbol, keeps every calendar event for
|
||||
the defensive gap study, and stores the longer EPS history separately for SUE
|
||||
scaling. EPS history without an announcement date is never exposed as a live
|
||||
signal event.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EVENTS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
symbol TEXT NOT NULL,
|
||||
announce_date TEXT NOT NULL,
|
||||
announce_time TEXT,
|
||||
eps_estimate REAL,
|
||||
eps_actual REAL,
|
||||
revenue_estimate REAL,
|
||||
revenue_actual REAL,
|
||||
source TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
period_end_date TEXT,
|
||||
UNIQUE(symbol, announce_date)
|
||||
)
|
||||
"""
|
||||
META_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
|
||||
symbol TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
n_events INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL,
|
||||
note TEXT
|
||||
)
|
||||
"""
|
||||
SURPRISE_HISTORY_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_surprise_history (
|
||||
symbol TEXT NOT NULL,
|
||||
period_end_date TEXT NOT NULL,
|
||||
eps_estimate REAL,
|
||||
eps_actual REAL,
|
||||
source TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY(symbol, period_end_date)
|
||||
)
|
||||
"""
|
||||
|
||||
SKIP_EVENT_COST = 45.0
|
||||
SKIP_PERIOD_COST = 45.0
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
||||
parser.add_argument("--calendar-csv", required=True)
|
||||
parser.add_argument("--history-csv", required=True)
|
||||
parser.add_argument("--from-date", default="2020-01-22")
|
||||
parser.add_argument("--to-date", required=True)
|
||||
parser.add_argument("--source-commit", required=True)
|
||||
parser.add_argument(
|
||||
"--source-url",
|
||||
default="https://www.dolthub.com/repositories/post-no-preference/earnings",
|
||||
)
|
||||
parser.add_argument("--max-period-lag-days", type=int, default=90)
|
||||
parser.add_argument("--max-period-lead-days", type=int, default=14)
|
||||
parser.add_argument(
|
||||
"--status-output", default="reports/earnings-backfill-status.json"
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _normalise_symbol(value: Any) -> str:
|
||||
return str(value or "").strip().upper().replace(".", "-")
|
||||
|
||||
|
||||
def _normalise_session(value: Any) -> str | None:
|
||||
cleaned = str(value or "").strip().lower().replace("_", " ").replace("-", " ")
|
||||
aliases = {
|
||||
"before market open": "bmo",
|
||||
"before open": "bmo",
|
||||
"bmo": "bmo",
|
||||
"after market close": "amc",
|
||||
"after close": "amc",
|
||||
"amc": "amc",
|
||||
"during market hours": "during",
|
||||
"dmh": "during",
|
||||
}
|
||||
return aliases.get(cleaned, cleaned or None)
|
||||
|
||||
|
||||
def _number(value: Any) -> float | None:
|
||||
if value is None or str(value).strip() == "":
|
||||
return None
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return result if math.isfinite(result) else None
|
||||
|
||||
|
||||
def _read_calendar(
|
||||
path: Path,
|
||||
universe: set[str],
|
||||
start: date,
|
||||
end: date,
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
raw_rows = 0
|
||||
universe_rows = 0
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||||
for raw in csv.DictReader(handle):
|
||||
raw_rows += 1
|
||||
symbol = _normalise_symbol(raw.get("act_symbol"))
|
||||
raw_date = str(raw.get("date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
event_date = date.fromisoformat(raw_date)
|
||||
if not start <= event_date <= end:
|
||||
continue
|
||||
universe_rows += 1
|
||||
row = {
|
||||
"symbol": symbol,
|
||||
"announce_date": event_date,
|
||||
"announce_time": _normalise_session(raw.get("when")),
|
||||
}
|
||||
key = (symbol, event_date)
|
||||
previous = by_key.get(key)
|
||||
if previous is not None:
|
||||
duplicate_rows += 1
|
||||
if (
|
||||
previous.get("announce_time") is not None
|
||||
and row.get("announce_time") is not None
|
||||
and previous["announce_time"] != row["announce_time"]
|
||||
):
|
||||
restated_rows += 1
|
||||
if row.get("announce_time") is not None:
|
||||
by_key[key] = row
|
||||
else:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda item: item["announce_date"])
|
||||
return grouped, {
|
||||
"raw_rows": raw_rows,
|
||||
"universe_rows_in_window": universe_rows,
|
||||
"deduped_rows_in_window": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def _read_history(
|
||||
path: Path, universe: set[str]
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
raw_rows = 0
|
||||
universe_rows = 0
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
fields = ("eps_actual", "eps_estimate")
|
||||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||||
for raw in csv.DictReader(handle):
|
||||
raw_rows += 1
|
||||
symbol = _normalise_symbol(raw.get("act_symbol"))
|
||||
raw_date = str(raw.get("period_end_date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
universe_rows += 1
|
||||
period_end = date.fromisoformat(raw_date)
|
||||
row = {
|
||||
"symbol": symbol,
|
||||
"period_end_date": period_end,
|
||||
"eps_actual": _number(raw.get("reported")),
|
||||
"eps_estimate": _number(raw.get("estimate")),
|
||||
}
|
||||
key = (symbol, period_end)
|
||||
previous = by_key.get(key)
|
||||
if previous is not None:
|
||||
duplicate_rows += 1
|
||||
if any(
|
||||
previous.get(field) is not None
|
||||
and row.get(field) is not None
|
||||
and previous[field] != row[field]
|
||||
for field in fields
|
||||
):
|
||||
restated_rows += 1
|
||||
previous_score = sum(previous.get(field) is not None for field in fields)
|
||||
row_score = sum(row.get(field) is not None for field in fields)
|
||||
if row_score >= previous_score:
|
||||
by_key[key] = row
|
||||
else:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda item: item["period_end_date"])
|
||||
return grouped, {
|
||||
"raw_rows": raw_rows,
|
||||
"universe_rows": universe_rows,
|
||||
"deduped_rows": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def _match_cost(event: dict[str, Any], period: dict[str, Any]) -> float:
|
||||
delta = (event["announce_date"] - period["period_end_date"]).days
|
||||
missing_session_penalty = 3.0 if event.get("announce_time") is None else 0.0
|
||||
return float(abs(delta - 30)) + missing_session_penalty
|
||||
|
||||
|
||||
def _align_symbol(
|
||||
events: list[dict[str, Any]],
|
||||
periods: list[dict[str, Any]],
|
||||
*,
|
||||
max_lag_days: int,
|
||||
max_lead_days: int,
|
||||
) -> tuple[list[tuple[int, int]], list[int], list[int]]:
|
||||
"""Return a minimum-cost monotonic calendar-to-period alignment."""
|
||||
n_events = len(events)
|
||||
n_periods = len(periods)
|
||||
scores = [[0.0] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
choices = [[""] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
for event_index in range(n_events - 1, -1, -1):
|
||||
scores[event_index][n_periods] = (
|
||||
scores[event_index + 1][n_periods] + SKIP_EVENT_COST
|
||||
)
|
||||
choices[event_index][n_periods] = "event"
|
||||
for period_index in range(n_periods - 1, -1, -1):
|
||||
scores[n_events][period_index] = (
|
||||
scores[n_events][period_index + 1] + SKIP_PERIOD_COST
|
||||
)
|
||||
choices[n_events][period_index] = "period"
|
||||
|
||||
for event_index in range(n_events - 1, -1, -1):
|
||||
for period_index in range(n_periods - 1, -1, -1):
|
||||
options = [
|
||||
(
|
||||
scores[event_index + 1][period_index] + SKIP_EVENT_COST,
|
||||
2,
|
||||
"event",
|
||||
),
|
||||
(
|
||||
scores[event_index][period_index + 1] + SKIP_PERIOD_COST,
|
||||
1,
|
||||
"period",
|
||||
),
|
||||
]
|
||||
delta = (
|
||||
events[event_index]["announce_date"]
|
||||
- periods[period_index]["period_end_date"]
|
||||
).days
|
||||
if -max_lead_days <= delta <= max_lag_days:
|
||||
options.append(
|
||||
(
|
||||
scores[event_index + 1][period_index + 1]
|
||||
+ _match_cost(events[event_index], periods[period_index]),
|
||||
0,
|
||||
"match",
|
||||
)
|
||||
)
|
||||
score, _, choice = min(options)
|
||||
scores[event_index][period_index] = score
|
||||
choices[event_index][period_index] = choice
|
||||
|
||||
matches: list[tuple[int, int]] = []
|
||||
unmatched_events: list[int] = []
|
||||
unmatched_periods: list[int] = []
|
||||
event_index = 0
|
||||
period_index = 0
|
||||
while event_index < n_events or period_index < n_periods:
|
||||
if event_index >= n_events:
|
||||
unmatched_periods.extend(range(period_index, n_periods))
|
||||
break
|
||||
if period_index >= n_periods:
|
||||
unmatched_events.extend(range(event_index, n_events))
|
||||
break
|
||||
choice = choices[event_index][period_index]
|
||||
if choice == "match":
|
||||
matches.append((event_index, period_index))
|
||||
event_index += 1
|
||||
period_index += 1
|
||||
elif choice == "period":
|
||||
unmatched_periods.append(period_index)
|
||||
period_index += 1
|
||||
else:
|
||||
unmatched_events.append(event_index)
|
||||
event_index += 1
|
||||
return matches, unmatched_events, unmatched_periods
|
||||
|
||||
|
||||
def _ensure_schema(connection: sqlite3.Connection) -> None:
|
||||
connection.execute(EVENTS_DDL)
|
||||
columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(earnings_events)")
|
||||
}
|
||||
if "period_end_date" not in columns:
|
||||
connection.execute("ALTER TABLE earnings_events ADD COLUMN period_end_date TEXT")
|
||||
connection.execute(META_DDL)
|
||||
connection.execute(SURPRISE_HISTORY_DDL)
|
||||
|
||||
|
||||
def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
calendar_csv = Path(args.calendar_csv)
|
||||
history_csv = Path(args.history_csv)
|
||||
for path in (snapshot, calendar_csv, history_csv):
|
||||
if not path.exists():
|
||||
raise SystemExit(f"Missing input: {path}")
|
||||
start = date.fromisoformat(args.from_date)
|
||||
end = date.fromisoformat(args.to_date)
|
||||
if start > end:
|
||||
raise SystemExit("--from-date must not be after --to-date")
|
||||
|
||||
connection = sqlite3.connect(snapshot)
|
||||
try:
|
||||
universe = {
|
||||
_normalise_symbol(row[0])
|
||||
for row in connection.execute("SELECT symbol FROM tickers")
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
calendar, calendar_stats = _read_calendar(calendar_csv, universe, start, end)
|
||||
history, history_stats = _read_history(history_csv, universe)
|
||||
|
||||
aligned_events: list[dict[str, Any]] = []
|
||||
pairing_deltas: list[int] = []
|
||||
unmatched_calendar = 0
|
||||
unmatched_periods_in_pairing_window = 0
|
||||
matched = 0
|
||||
for symbol in sorted(universe):
|
||||
events = calendar.get(symbol, [])
|
||||
lower = start - timedelta(days=int(args.max_period_lag_days))
|
||||
upper = end + timedelta(days=int(args.max_period_lead_days))
|
||||
periods = [
|
||||
row
|
||||
for row in history.get(symbol, [])
|
||||
if lower <= row["period_end_date"] <= upper
|
||||
]
|
||||
matches, unmatched_events, unmatched_periods = _align_symbol(
|
||||
events,
|
||||
periods,
|
||||
max_lag_days=int(args.max_period_lag_days),
|
||||
max_lead_days=int(args.max_period_lead_days),
|
||||
)
|
||||
matched_by_event = {event_index: period_index for event_index, period_index in matches}
|
||||
matched += len(matches)
|
||||
unmatched_calendar += len(unmatched_events)
|
||||
unmatched_periods_in_pairing_window += len(unmatched_periods)
|
||||
for event_index, event in enumerate(events):
|
||||
row = dict(event)
|
||||
period_index = matched_by_event.get(event_index)
|
||||
if period_index is None:
|
||||
row.update(
|
||||
{
|
||||
"period_end_date": None,
|
||||
"eps_actual": None,
|
||||
"eps_estimate": None,
|
||||
}
|
||||
)
|
||||
else:
|
||||
period = periods[period_index]
|
||||
row.update(
|
||||
{
|
||||
"period_end_date": period["period_end_date"],
|
||||
"eps_actual": period["eps_actual"],
|
||||
"eps_estimate": period["eps_estimate"],
|
||||
}
|
||||
)
|
||||
pairing_deltas.append(
|
||||
(event["announce_date"] - period["period_end_date"]).days
|
||||
)
|
||||
aligned_events.append(row)
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
source = f"dolthub_post_no_preference@{args.source_commit}"
|
||||
conflicting_existing_rows = 0
|
||||
conflicting_existing_fields = 0
|
||||
preserved_existing_fields = 0
|
||||
incoming_keys = {
|
||||
(row["symbol"], row["announce_date"].isoformat()) for row in aligned_events
|
||||
}
|
||||
connection = sqlite3.connect(snapshot)
|
||||
try:
|
||||
_ensure_schema(connection)
|
||||
existing = {
|
||||
(str(row[0]), str(row[1])): row
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT symbol, announce_date, announce_time, eps_estimate,
|
||||
eps_actual, period_end_date, source
|
||||
FROM earnings_events
|
||||
WHERE announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
(start.isoformat(), end.isoformat()),
|
||||
)
|
||||
}
|
||||
upsert = """
|
||||
INSERT INTO earnings_events(
|
||||
symbol, announce_date, announce_time, eps_estimate, eps_actual,
|
||||
revenue_estimate, revenue_actual, source, fetched_at, period_end_date
|
||||
) VALUES (?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?)
|
||||
ON CONFLICT(symbol, announce_date) DO UPDATE SET
|
||||
announce_time=COALESCE(earnings_events.announce_time, excluded.announce_time),
|
||||
eps_estimate=COALESCE(earnings_events.eps_estimate, excluded.eps_estimate),
|
||||
eps_actual=COALESCE(earnings_events.eps_actual, excluded.eps_actual),
|
||||
period_end_date=COALESCE(excluded.period_end_date, earnings_events.period_end_date),
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
for row in aligned_events:
|
||||
key = (row["symbol"], row["announce_date"].isoformat())
|
||||
old = existing.get(key)
|
||||
retained = 0
|
||||
conflicts = 0
|
||||
if old is not None:
|
||||
old_values = {
|
||||
"announce_time": old[2],
|
||||
"eps_estimate": old[3],
|
||||
"eps_actual": old[4],
|
||||
"period_end_date": old[5],
|
||||
}
|
||||
new_values = {
|
||||
"announce_time": row.get("announce_time"),
|
||||
"eps_estimate": row.get("eps_estimate"),
|
||||
"eps_actual": row.get("eps_actual"),
|
||||
"period_end_date": (
|
||||
row["period_end_date"].isoformat()
|
||||
if row.get("period_end_date")
|
||||
else None
|
||||
),
|
||||
}
|
||||
for field, new_value in new_values.items():
|
||||
old_value = old_values[field]
|
||||
if field != "period_end_date" and old_value is not None:
|
||||
retained += 1
|
||||
if new_value is not None and old_value is not None:
|
||||
if field in {"eps_estimate", "eps_actual"}:
|
||||
differs = not math.isclose(
|
||||
float(new_value), float(old_value), rel_tol=0.0, abs_tol=1e-9
|
||||
)
|
||||
else:
|
||||
differs = str(new_value) != str(old_value)
|
||||
conflicts += int(differs)
|
||||
conflicting_existing_rows += int(conflicts > 0)
|
||||
conflicting_existing_fields += conflicts
|
||||
preserved_existing_fields += retained
|
||||
row_source = source
|
||||
if retained and old is not None:
|
||||
row_source = f"{old[6]}+calendar:{source}"
|
||||
connection.execute(
|
||||
upsert,
|
||||
(
|
||||
row["symbol"],
|
||||
row["announce_date"].isoformat(),
|
||||
row.get("announce_time"),
|
||||
row.get("eps_estimate"),
|
||||
row.get("eps_actual"),
|
||||
row_source,
|
||||
now,
|
||||
(
|
||||
row["period_end_date"].isoformat()
|
||||
if row.get("period_end_date")
|
||||
else None
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
history_upsert = """
|
||||
INSERT INTO earnings_surprise_history(
|
||||
symbol, period_end_date, eps_estimate, eps_actual, source, fetched_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(symbol, period_end_date) DO UPDATE SET
|
||||
eps_estimate=COALESCE(excluded.eps_estimate, earnings_surprise_history.eps_estimate),
|
||||
eps_actual=COALESCE(excluded.eps_actual, earnings_surprise_history.eps_actual),
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
for symbol, rows in history.items():
|
||||
connection.executemany(
|
||||
history_upsert,
|
||||
[
|
||||
(
|
||||
symbol,
|
||||
row["period_end_date"].isoformat(),
|
||||
row.get("eps_estimate"),
|
||||
row.get("eps_actual"),
|
||||
source,
|
||||
now,
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
for symbol in sorted(universe):
|
||||
count = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol=? AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
(symbol, start.isoformat(), end.isoformat()),
|
||||
).fetchone()[0]
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
||||
VALUES (?, 'done', ?, ?, 'dolthub_bulk_complete')
|
||||
ON CONFLICT(symbol) DO UPDATE SET
|
||||
status='done', n_events=excluded.n_events,
|
||||
updated_at=excluded.updated_at, note=excluded.note
|
||||
""",
|
||||
(symbol, count, now),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
params = (start.isoformat(), end.isoformat())
|
||||
total_events = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
paired_events = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
date_range = connection.execute(
|
||||
"""
|
||||
SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()
|
||||
source_symbols = set(calendar)
|
||||
history_complete = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_surprise_history
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL
|
||||
"""
|
||||
).fetchone()[0]
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
deltas = sorted(pairing_deltas)
|
||||
summary = {
|
||||
"mode": "dolthub_public_bulk_clone",
|
||||
"window": {"from": start.isoformat(), "to": end.isoformat()},
|
||||
"coverage_amendment": {
|
||||
"approved_by_user": True,
|
||||
"reason": "FMP free tier blocks historical bulk earnings",
|
||||
"original_start": "2016-01-04",
|
||||
"amended_announcement_start": start.isoformat(),
|
||||
},
|
||||
"source": {
|
||||
"repository": args.source_url,
|
||||
"commit": args.source_commit,
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"upstream_provider_documented": False,
|
||||
},
|
||||
"bulk_windows_total": 1,
|
||||
"bulk_windows_done": 1,
|
||||
"bulk_requests_logged_total": 1,
|
||||
"bulk_exports": 2,
|
||||
"calendar": calendar_stats,
|
||||
"eps_history": {**history_stats, "complete_actual_and_estimate": history_complete},
|
||||
"pairing": {
|
||||
"method": "minimum-cost monotonic alignment per symbol",
|
||||
"allowed_announce_minus_period_end_days": [
|
||||
-int(args.max_period_lead_days),
|
||||
int(args.max_period_lag_days),
|
||||
],
|
||||
"matched_calendar_events": matched,
|
||||
"unmatched_calendar_events": unmatched_calendar,
|
||||
"unmatched_periods_in_pairing_window": unmatched_periods_in_pairing_window,
|
||||
"announce_minus_period_end_days": {
|
||||
"min": min(deltas) if deltas else None,
|
||||
"median": deltas[len(deltas) // 2] if deltas else None,
|
||||
"max": max(deltas) if deltas else None,
|
||||
},
|
||||
"pre_2020_eps_history_use": (
|
||||
"trailing_surprise_stdev_only; never treated as an announcement "
|
||||
"or live signal event"
|
||||
),
|
||||
},
|
||||
"duplicate_rows_logged_total": (
|
||||
calendar_stats["duplicate_rows"] + history_stats["duplicate_rows"]
|
||||
),
|
||||
"restated_rows_logged_total": (
|
||||
calendar_stats["restated_rows"]
|
||||
+ history_stats["restated_rows"]
|
||||
+ conflicting_existing_rows
|
||||
),
|
||||
"conflicting_existing_rows": conflicting_existing_rows,
|
||||
"conflicting_existing_fields": conflicting_existing_fields,
|
||||
"preserved_existing_fields": preserved_existing_fields,
|
||||
"existing_enrichment_events_not_in_dolthub_calendar": max(
|
||||
0, total_events - len(incoming_keys)
|
||||
),
|
||||
"dedupe_policy": (
|
||||
"UNIQUE(symbol, announce_date); normalise dot/dash symbols; retain one "
|
||||
"calendar row per key; preserve existing non-null session/EPS values from "
|
||||
"the prior FMP/Alpha Vantage partial backfill, then fill nulls and all "
|
||||
"remaining symbols from DoltHub; attach DoltHub period-end alignment"
|
||||
),
|
||||
"events_in_window": total_events,
|
||||
"events_with_actual_and_estimate": paired_events,
|
||||
"symbols_done": len(universe),
|
||||
"symbols_universe": len(universe),
|
||||
"symbols_with_dolthub_calendar": len(source_symbols),
|
||||
"symbols_without_dolthub_calendar": sorted(universe - source_symbols),
|
||||
"announce_date_range": {"min": date_range[0], "max": date_range[1]},
|
||||
"complete": True,
|
||||
}
|
||||
output = Path(args.status_output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, indent=2))
|
||||
print(f"Wrote {output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
Reference in New Issue
Block a user