research: Task 2 closed — SUE dead, earnings gap informational
Earnings backfill sourced from the public DoltHub earnings repo at a pinned commit rather than the FMP API: reproducible for anyone re-running the study, and it burns no request quota. 12,414 events, 98.6% of symbols with >=8 announcements, 99.2% paired actual/estimate, no keyed duplicates. 2a earnings-gap diagnostic: INFORMATIONAL, no filter shipped. The pre-earnings cohort's right tail was better, so the registered avoid-earnings condition failed. Note the raw 23/266 vs 115/574 incidence gap is largely a duration confound -- severe losses stop out fast and have less time to span an announcement -- so it is not evidence that holding through earnings is safe. 2b SUE: FAIL against the pre-registered +0.03 bar (unconditional IC +0.0151 over 56 reliable windows, momentum-conditional +0.0213). Signs stable across eras, so this is a clean null rather than an ambiguous one, consistent with post-earnings drift having decayed in large caps. Closes the Tier-1 arc: Task 1 dead on deep evidence, Task 2 dead here, Task 3 complete as diagnostic. No in-sample research thread remains open. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
+388
-405
@@ -1,15 +1,13 @@
|
||||
"""Backfill historical earnings into a snapshot ``earnings_events`` table.
|
||||
"""Bulk-only historical earnings backfill for a local SQLite snapshot.
|
||||
|
||||
Prefers FMP bulk date-range ``earnings-calendar`` (one request per window).
|
||||
On free-tier 402/403, falls back to per-symbol ``/stable/earnings`` with
|
||||
resume support and request counting (≈250 req/day free tier).
|
||||
The job uses FMP's date-range earnings-calendar endpoint. One request covers all
|
||||
symbols in a date window; per-symbol endpoints are intentionally not available
|
||||
in this task runner. Successful windows are committed independently so a later
|
||||
run resumes after a daily quota boundary without repeating completed windows.
|
||||
|
||||
Research only — writes to the local snapshot SQLite, never production Postgres.
|
||||
|
||||
Example
|
||||
-------
|
||||
python scripts/backfill_earnings_events.py \\
|
||||
--snapshot backtest_snapshots/prod.sqlite --limit 250
|
||||
Example:
|
||||
python scripts/backfill_earnings_events.py --snapshot backtest_snapshots/prod.sqlite \
|
||||
--from-date 2012-01-01 --window-days 30 --limit 250
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,10 +15,11 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import create_engine, text
|
||||
@@ -34,7 +33,7 @@ from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
||||
bootstrap_ssl()
|
||||
|
||||
FMP_STABLE = "https://financialmodelingprep.com/stable"
|
||||
DDL = """
|
||||
EVENTS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
symbol TEXT NOT NULL,
|
||||
@@ -49,7 +48,6 @@ CREATE TABLE IF NOT EXISTS earnings_events (
|
||||
UNIQUE(symbol, announce_date)
|
||||
)
|
||||
"""
|
||||
# Side table tracks which symbols have been fully pulled (resume).
|
||||
META_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
|
||||
symbol TEXT PRIMARY KEY,
|
||||
@@ -59,239 +57,238 @@ CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
|
||||
note TEXT
|
||||
)
|
||||
"""
|
||||
WINDOW_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_backfill_windows (
|
||||
from_date TEXT NOT NULL,
|
||||
to_date TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
requests INTEGER NOT NULL DEFAULT 0,
|
||||
rows_raw INTEGER NOT NULL DEFAULT 0,
|
||||
rows_universe INTEGER NOT NULL DEFAULT 0,
|
||||
duplicate_rows INTEGER NOT NULL DEFAULT 0,
|
||||
restated_rows INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL,
|
||||
note TEXT,
|
||||
PRIMARY KEY(from_date, to_date)
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
||||
p.add_argument(
|
||||
"--from-date",
|
||||
default="2020-01-01",
|
||||
help="Bulk calendar window start (also filters per-symbol rows).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--to-date",
|
||||
default=None,
|
||||
help="Bulk calendar window end (default: today).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=250,
|
||||
help="Max FMP requests this run (free-tier cushion).",
|
||||
)
|
||||
p.add_argument("--sleep", type=float, default=0.35)
|
||||
p.add_argument(
|
||||
"--force-symbol",
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
||||
parser.add_argument("--from-date", default="2012-01-01")
|
||||
parser.add_argument("--to-date", default=None)
|
||||
parser.add_argument("--window-days", type=int, default=30)
|
||||
parser.add_argument("--limit", type=int, default=250)
|
||||
parser.add_argument("--sleep", type=float, default=0.35)
|
||||
parser.add_argument(
|
||||
"--refetch-windows",
|
||||
action="store_true",
|
||||
help="Skip bulk attempt; go straight to per-symbol.",
|
||||
help="Re-fetch date windows already logged as done.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--refetch-done",
|
||||
action="store_true",
|
||||
help="Re-fetch symbols already marked done.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--provider",
|
||||
choices=("fmp", "alpha_vantage", "auto"),
|
||||
default="auto",
|
||||
help="Earnings provider. auto tries FMP bulk then FMP/AV per-symbol.",
|
||||
)
|
||||
return p.parse_args()
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _ensure_tables(engine) -> None:
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text(DDL))
|
||||
conn.execute(text(EVENTS_DDL))
|
||||
conn.execute(text(META_DDL))
|
||||
conn.execute(text(WINDOW_DDL))
|
||||
|
||||
|
||||
def _upsert_events(conn, rows: list[dict], source: str) -> int:
|
||||
if not rows:
|
||||
return 0
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
written = 0
|
||||
for r in rows:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO earnings_events (
|
||||
symbol, announce_date, announce_time,
|
||||
eps_estimate, eps_actual, revenue_estimate, revenue_actual,
|
||||
source, fetched_at
|
||||
) VALUES (
|
||||
:symbol, :announce_date, :announce_time,
|
||||
:eps_estimate, :eps_actual, :revenue_estimate, :revenue_actual,
|
||||
:source, :fetched_at
|
||||
)
|
||||
ON CONFLICT(symbol, announce_date) DO UPDATE SET
|
||||
announce_time=excluded.announce_time,
|
||||
eps_estimate=excluded.eps_estimate,
|
||||
eps_actual=excluded.eps_actual,
|
||||
revenue_estimate=excluded.revenue_estimate,
|
||||
revenue_actual=excluded.revenue_actual,
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
),
|
||||
{
|
||||
"symbol": r["symbol"],
|
||||
"announce_date": r["announce_date"],
|
||||
"announce_time": r.get("announce_time"),
|
||||
"eps_estimate": r.get("eps_estimate"),
|
||||
"eps_actual": r.get("eps_actual"),
|
||||
"revenue_estimate": r.get("revenue_estimate"),
|
||||
"revenue_actual": r.get("revenue_actual"),
|
||||
"source": source,
|
||||
"fetched_at": now,
|
||||
},
|
||||
)
|
||||
written += 1
|
||||
return written
|
||||
def _number(value: Any) -> float | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return result if math.isfinite(result) else None
|
||||
|
||||
|
||||
def _normalise_session(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = str(value).strip().lower().replace("_", " ").replace("-", " ")
|
||||
aliases = {
|
||||
"bmo": "bmo",
|
||||
"before market open": "bmo",
|
||||
"before open": "bmo",
|
||||
"amc": "amc",
|
||||
"after market close": "amc",
|
||||
"after close": "amc",
|
||||
"during market hours": "during",
|
||||
"dmh": "during",
|
||||
}
|
||||
return aliases.get(cleaned, cleaned or None)
|
||||
|
||||
|
||||
def _parse_bulk_item(item: dict) -> dict | None:
|
||||
sym = (item.get("symbol") or "").strip().upper()
|
||||
d = item.get("date") or item.get("earningsDate")
|
||||
if not sym or not d:
|
||||
symbol = str(item.get("symbol") or "").strip().upper().replace(".", "-")
|
||||
raw_date = item.get("date") or item.get("earningsDate")
|
||||
if not symbol or not raw_date:
|
||||
return None
|
||||
return {
|
||||
"symbol": sym.replace(".", "-"),
|
||||
"announce_date": str(d)[:10],
|
||||
"announce_time": item.get("time") or item.get("announceTime"),
|
||||
"eps_estimate": _f(item.get("epsEstimated") or item.get("estimatedEarning")),
|
||||
"eps_actual": _f(item.get("epsActual") or item.get("eps")),
|
||||
"revenue_estimate": _f(item.get("revenueEstimated")),
|
||||
"revenue_actual": _f(item.get("revenueActual")),
|
||||
"symbol": symbol,
|
||||
"announce_date": str(raw_date)[:10],
|
||||
"announce_time": _normalise_session(
|
||||
item.get("time") or item.get("announceTime")
|
||||
),
|
||||
"eps_estimate": _number(
|
||||
item.get("epsEstimated")
|
||||
if item.get("epsEstimated") is not None
|
||||
else item.get("estimatedEarning")
|
||||
),
|
||||
"eps_actual": _number(
|
||||
item.get("epsActual")
|
||||
if item.get("epsActual") is not None
|
||||
else item.get("eps")
|
||||
),
|
||||
"revenue_estimate": _number(item.get("revenueEstimated")),
|
||||
"revenue_actual": _number(item.get("revenueActual")),
|
||||
}
|
||||
|
||||
|
||||
def _parse_symbol_item(item: dict, symbol: str) -> dict | None:
|
||||
d = item.get("date")
|
||||
if not d:
|
||||
return None
|
||||
return {
|
||||
"symbol": symbol.replace(".", "-").upper(),
|
||||
"announce_date": str(d)[:10],
|
||||
"announce_time": item.get("time"),
|
||||
"eps_estimate": _f(item.get("epsEstimated")),
|
||||
"eps_actual": _f(item.get("epsActual")),
|
||||
"revenue_estimate": _f(item.get("revenueEstimated")),
|
||||
"revenue_actual": _f(item.get("revenueActual")),
|
||||
}
|
||||
def _windows(start: date, end: date, window_days: int) -> list[tuple[date, date]]:
|
||||
if window_days < 1:
|
||||
raise ValueError("window_days must be positive")
|
||||
result: list[tuple[date, date]] = []
|
||||
cursor = start
|
||||
while cursor <= end:
|
||||
window_end = min(end, cursor + timedelta(days=window_days - 1))
|
||||
result.append((cursor, window_end))
|
||||
cursor = window_end + timedelta(days=1)
|
||||
return result
|
||||
|
||||
|
||||
def _f(v) -> float | None:
|
||||
if v is None or v == "":
|
||||
return None
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
def _dedupe_bulk_rows(rows: list[dict]) -> tuple[list[dict], int, int]:
|
||||
"""Prefer the most complete duplicate; use the later row as the tie-break."""
|
||||
fields = (
|
||||
"announce_time",
|
||||
"eps_estimate",
|
||||
"eps_actual",
|
||||
"revenue_estimate",
|
||||
"revenue_actual",
|
||||
)
|
||||
chosen: dict[tuple[str, str], dict] = {}
|
||||
duplicate_extras = 0
|
||||
restated = 0
|
||||
for row in rows:
|
||||
key = (str(row["symbol"]), str(row["announce_date"]))
|
||||
previous = chosen.get(key)
|
||||
if previous is None:
|
||||
chosen[key] = row
|
||||
continue
|
||||
duplicate_extras += 1
|
||||
if any(
|
||||
previous.get(field) is not None
|
||||
and row.get(field) is not None
|
||||
and previous.get(field) != row.get(field)
|
||||
for field in fields
|
||||
):
|
||||
restated += 1
|
||||
previous_score = sum(previous.get(field) is not None for field in fields)
|
||||
new_score = sum(row.get(field) is not None for field in fields)
|
||||
if new_score >= previous_score:
|
||||
chosen[key] = row
|
||||
return list(chosen.values()), duplicate_extras, restated
|
||||
|
||||
|
||||
async def _try_bulk(
|
||||
client: httpx.AsyncClient,
|
||||
api_key: str,
|
||||
def _upsert_events(conn, rows: list[dict]) -> int:
|
||||
if not rows:
|
||||
return 0
|
||||
fetched_at = datetime.now(timezone.utc).isoformat()
|
||||
statement = text(
|
||||
"""
|
||||
INSERT INTO earnings_events (
|
||||
symbol, announce_date, announce_time, eps_estimate, eps_actual,
|
||||
revenue_estimate, revenue_actual, source, fetched_at
|
||||
) VALUES (
|
||||
:symbol, :announce_date, :announce_time, :eps_estimate, :eps_actual,
|
||||
:revenue_estimate, :revenue_actual, 'fmp_earnings_calendar', :fetched_at
|
||||
)
|
||||
ON CONFLICT(symbol, announce_date) DO UPDATE SET
|
||||
announce_time=COALESCE(excluded.announce_time, earnings_events.announce_time),
|
||||
eps_estimate=COALESCE(excluded.eps_estimate, earnings_events.eps_estimate),
|
||||
eps_actual=COALESCE(excluded.eps_actual, earnings_events.eps_actual),
|
||||
revenue_estimate=COALESCE(excluded.revenue_estimate, earnings_events.revenue_estimate),
|
||||
revenue_actual=COALESCE(excluded.revenue_actual, earnings_events.revenue_actual),
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
)
|
||||
conn.execute(statement, [{**row, "fetched_at": fetched_at} for row in rows])
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _fetch_bulk_window(
|
||||
client: httpx.AsyncClient, api_key: str, start: date, end: date
|
||||
) -> tuple[list[dict], int, str | None]:
|
||||
response = await client.get(
|
||||
f"{FMP_STABLE}/earnings-calendar",
|
||||
params={"from": start.isoformat(), "to": end.isoformat(), "apikey": api_key},
|
||||
)
|
||||
if response.status_code in (402, 403):
|
||||
return [], response.status_code, "bulk_endpoint_unavailable"
|
||||
if response.status_code == 429:
|
||||
return [], response.status_code, "daily_limit_reached"
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, list):
|
||||
return [], response.status_code, f"unexpected_payload:{type(payload).__name__}"
|
||||
rows = []
|
||||
for item in payload:
|
||||
if isinstance(item, dict):
|
||||
parsed = _parse_bulk_item(item)
|
||||
if parsed:
|
||||
rows.append(parsed)
|
||||
return rows, response.status_code, None
|
||||
|
||||
|
||||
def _write_window_status(
|
||||
engine,
|
||||
*,
|
||||
start: date,
|
||||
end: date,
|
||||
*,
|
||||
window_days: int = 30,
|
||||
) -> tuple[list[dict], int, str | None]:
|
||||
"""Return (rows, requests_used, error_note)."""
|
||||
rows: list[dict] = []
|
||||
reqs = 0
|
||||
cur = start
|
||||
while cur <= end:
|
||||
win_end = min(end, cur + timedelta(days=window_days - 1))
|
||||
resp = await client.get(
|
||||
f"{FMP_STABLE}/earnings-calendar",
|
||||
params={
|
||||
"from": cur.isoformat(),
|
||||
"to": win_end.isoformat(),
|
||||
"apikey": api_key,
|
||||
status: str,
|
||||
raw_n: int = 0,
|
||||
universe_n: int = 0,
|
||||
duplicate_n: int = 0,
|
||||
restated_n: int = 0,
|
||||
note: str | None = None,
|
||||
) -> None:
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_windows(
|
||||
from_date, to_date, status, requests, rows_raw, rows_universe,
|
||||
duplicate_rows, restated_rows, updated_at, note
|
||||
) VALUES (:a, :b, :status, 1, :raw, :uni, :dup, :rest, :now, :note)
|
||||
ON CONFLICT(from_date, to_date) DO UPDATE SET
|
||||
status=excluded.status,
|
||||
requests=earnings_backfill_windows.requests + 1,
|
||||
rows_raw=excluded.rows_raw,
|
||||
rows_universe=excluded.rows_universe,
|
||||
duplicate_rows=excluded.duplicate_rows,
|
||||
restated_rows=excluded.restated_rows,
|
||||
updated_at=excluded.updated_at,
|
||||
note=excluded.note
|
||||
"""
|
||||
),
|
||||
{
|
||||
"a": start.isoformat(),
|
||||
"b": end.isoformat(),
|
||||
"status": status,
|
||||
"raw": raw_n,
|
||||
"uni": universe_n,
|
||||
"dup": duplicate_n,
|
||||
"rest": restated_n,
|
||||
"now": datetime.now(timezone.utc).isoformat(),
|
||||
"note": note,
|
||||
},
|
||||
)
|
||||
reqs += 1
|
||||
if resp.status_code in (402, 403):
|
||||
return [], reqs, f"bulk_unavailable status={resp.status_code}"
|
||||
if resp.status_code == 429:
|
||||
return rows, reqs, "rate_limited"
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return [], reqs, f"unexpected bulk payload type={type(data)}"
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
parsed = _parse_bulk_item(item)
|
||||
if parsed:
|
||||
rows.append(parsed)
|
||||
cur = win_end + timedelta(days=1)
|
||||
return rows, reqs, None
|
||||
|
||||
|
||||
async def _fetch_symbol(
|
||||
client: httpx.AsyncClient, api_key: str, symbol: str
|
||||
) -> list[dict]:
|
||||
resp = await client.get(
|
||||
f"{FMP_STABLE}/earnings",
|
||||
params={"symbol": symbol, "apikey": api_key},
|
||||
)
|
||||
if resp.status_code == 429:
|
||||
raise RuntimeError("rate_limited")
|
||||
if resp.status_code == 402:
|
||||
return []
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
parsed = _parse_symbol_item(item, symbol)
|
||||
if parsed:
|
||||
out.append(parsed)
|
||||
return out
|
||||
|
||||
|
||||
async def _fetch_symbol_alpha_vantage(
|
||||
client: httpx.AsyncClient, api_key: str, symbol: str
|
||||
) -> list[dict]:
|
||||
"""Alpha Vantage EARNINGS — includes reportedDate (announce) + estimate/actual."""
|
||||
resp = await client.get(
|
||||
"https://www.alphavantage.co/query",
|
||||
params={"function": "EARNINGS", "symbol": symbol, "apikey": api_key},
|
||||
)
|
||||
if resp.status_code == 429:
|
||||
raise RuntimeError("rate_limited")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
note = str(data.get("Note") or data.get("Information") or "")
|
||||
if "rate limit" in note.lower() or "Thank you for using Alpha Vantage" in note:
|
||||
raise RuntimeError("rate_limited")
|
||||
if data.get("Error Message"):
|
||||
return []
|
||||
quarterly = data.get("quarterlyEarnings") or []
|
||||
out: list[dict] = []
|
||||
for item in quarterly:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
# Prefer announce (reportedDate); fall back to fiscal end (worse PIT).
|
||||
ad = item.get("reportedDate") or item.get("fiscalDateEnding")
|
||||
if not ad:
|
||||
continue
|
||||
out.append({
|
||||
"symbol": symbol.replace(".", "-").upper(),
|
||||
"announce_date": str(ad)[:10],
|
||||
"announce_time": item.get("reportTime"),
|
||||
"eps_estimate": _f(item.get("estimatedEPS")),
|
||||
"eps_actual": _f(item.get("reportedEPS")),
|
||||
"revenue_estimate": None,
|
||||
"revenue_actual": None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
@@ -304,228 +301,214 @@ async def _main() -> None:
|
||||
|
||||
if not settings.fmp_api_key:
|
||||
raise SystemExit("FMP_API_KEY required")
|
||||
|
||||
start = date.fromisoformat(args.from_date)
|
||||
end = date.fromisoformat(args.to_date) if args.to_date else date.today()
|
||||
engine = create_engine(
|
||||
f"sqlite:///{snapshot.resolve().as_posix()}",
|
||||
future=True,
|
||||
)
|
||||
_ensure_tables(engine)
|
||||
if start > end:
|
||||
raise SystemExit("--from-date must not be after --to-date")
|
||||
|
||||
engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True)
|
||||
_ensure_tables(engine)
|
||||
all_windows = _windows(start, end, int(args.window_days))
|
||||
with engine.connect() as conn:
|
||||
symbols = [
|
||||
str(r[0]).upper().replace(".", "-")
|
||||
for r in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol"))
|
||||
str(row[0]).upper().replace(".", "-")
|
||||
for row in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol"))
|
||||
]
|
||||
done = set()
|
||||
if not args.refetch_done:
|
||||
done = {
|
||||
str(r[0])
|
||||
for r in conn.execute(
|
||||
text(
|
||||
"SELECT symbol FROM earnings_backfill_meta "
|
||||
"WHERE status='done' AND n_events > 0"
|
||||
)
|
||||
completed = {
|
||||
(str(row[0]), str(row[1]))
|
||||
for row in conn.execute(
|
||||
text(
|
||||
"SELECT from_date, to_date FROM earnings_backfill_windows "
|
||||
"WHERE status='done'"
|
||||
)
|
||||
}
|
||||
|
||||
pending = [s for s in symbols if s not in done]
|
||||
print(f"Snapshot: {snapshot}")
|
||||
print(f"Universe: {len(symbols)}; pending: {len(pending)}; done: {len(done)}")
|
||||
print(f"Window filter: {start} → {end}")
|
||||
print(f"Provider: {args.provider}")
|
||||
|
||||
req_budget = int(args.limit)
|
||||
reqs_used = 0
|
||||
events_written = 0
|
||||
mode = "per_symbol"
|
||||
use_av = args.provider in ("alpha_vantage", "auto") and bool(
|
||||
getattr(settings, "alpha_vantage_api_key", "")
|
||||
)
|
||||
use_fmp = args.provider in ("fmp", "auto") and bool(settings.fmp_api_key)
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
if (
|
||||
not args.force_symbol
|
||||
and req_budget > 0
|
||||
and use_fmp
|
||||
and args.provider != "alpha_vantage"
|
||||
):
|
||||
print("Attempting bulk earnings-calendar…")
|
||||
bulk_rows, bulk_reqs, err = await _try_bulk(
|
||||
client, settings.fmp_api_key, start, end
|
||||
)
|
||||
reqs_used += bulk_reqs
|
||||
if err:
|
||||
print(f" Bulk unavailable: {err} (requests={bulk_reqs})")
|
||||
else:
|
||||
# Filter to universe.
|
||||
uni = set(symbols)
|
||||
bulk_rows = [r for r in bulk_rows if r["symbol"] in uni]
|
||||
with engine.begin() as conn:
|
||||
events_written += _upsert_events(conn, bulk_rows, "fmp_earnings_calendar")
|
||||
for sym in symbols:
|
||||
n = conn.execute(
|
||||
text(
|
||||
"SELECT COUNT(*) FROM earnings_events WHERE symbol=:s"
|
||||
),
|
||||
{"s": sym},
|
||||
).scalar_one()
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
||||
VALUES (:s, 'done', :n, :t, 'bulk')
|
||||
ON CONFLICT(symbol) DO UPDATE SET
|
||||
status='done', n_events=excluded.n_events,
|
||||
updated_at=excluded.updated_at, note=excluded.note
|
||||
"""
|
||||
),
|
||||
{
|
||||
"s": sym,
|
||||
"n": int(n),
|
||||
"t": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
mode = "bulk"
|
||||
print(f" Bulk wrote {events_written} events; requests={bulk_reqs}")
|
||||
pending = []
|
||||
}
|
||||
pending = [
|
||||
window
|
||||
for window in all_windows
|
||||
if args.refetch_windows
|
||||
or (window[0].isoformat(), window[1].isoformat()) not in completed
|
||||
]
|
||||
universe = set(symbols)
|
||||
print(f"Snapshot: {snapshot}")
|
||||
print(f"Universe: {len(symbols)} symbols")
|
||||
print(f"Window: {start} -> {end}")
|
||||
print(
|
||||
f"Bulk windows: {len(all_windows)} total; "
|
||||
f"{len(all_windows) - len(pending)} done; {len(pending)} pending"
|
||||
)
|
||||
print("Provider: FMP bulk earnings-calendar only")
|
||||
|
||||
# Per-symbol fallback / completion.
|
||||
fmp_limited = False
|
||||
for sym in pending:
|
||||
if reqs_used >= req_budget:
|
||||
print(f"Request budget exhausted ({req_budget}). Resume later.")
|
||||
requests_this_run = 0
|
||||
rows_upserted = 0
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
stop_note: str | None = None
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
for index, (window_start, window_end) in enumerate(pending, 1):
|
||||
if requests_this_run >= int(args.limit):
|
||||
stop_note = "request_budget_exhausted"
|
||||
break
|
||||
items: list[dict] = []
|
||||
source = "fmp_earnings"
|
||||
note = "per_symbol"
|
||||
try:
|
||||
if use_fmp and not fmp_limited and args.provider != "alpha_vantage":
|
||||
items = await _fetch_symbol(client, settings.fmp_api_key, sym)
|
||||
source = "fmp_earnings"
|
||||
note = "fmp_per_symbol"
|
||||
# Empty list may mean soft-limit or no data — try AV if available.
|
||||
if not items and use_av:
|
||||
items = await _fetch_symbol_alpha_vantage(
|
||||
client, settings.alpha_vantage_api_key, sym
|
||||
)
|
||||
source = "alpha_vantage_earnings"
|
||||
note = "av_after_fmp_empty"
|
||||
reqs_used += 1 # count AV call separately below too
|
||||
elif use_av:
|
||||
items = await _fetch_symbol_alpha_vantage(
|
||||
client, settings.alpha_vantage_api_key, sym
|
||||
)
|
||||
source = "alpha_vantage_earnings"
|
||||
note = "av_per_symbol"
|
||||
else:
|
||||
raise RuntimeError("no provider available")
|
||||
raw_rows, status_code, error = await _fetch_bulk_window(
|
||||
client, settings.fmp_api_key, window_start, window_end
|
||||
)
|
||||
except Exception as exc:
|
||||
msg = str(exc)
|
||||
print(f" FAIL {sym}: {msg}")
|
||||
reqs_used += 1
|
||||
if "rate_limited" in msg and note.startswith("fmp"):
|
||||
fmp_limited = True
|
||||
with engine.begin() as conn:
|
||||
raw_rows, status_code = [], 0
|
||||
error = f"request_error:{type(exc).__name__}:{exc}"
|
||||
requests_this_run += 1
|
||||
if error:
|
||||
_write_window_status(
|
||||
engine,
|
||||
start=window_start,
|
||||
end=window_end,
|
||||
status="error",
|
||||
note=f"http={status_code} {error}"[:300],
|
||||
)
|
||||
stop_note = error
|
||||
print(
|
||||
f"STOP {window_start}..{window_end}: {error} "
|
||||
f"(http={status_code}, request={requests_this_run})"
|
||||
)
|
||||
break
|
||||
|
||||
in_universe = [row for row in raw_rows if row["symbol"] in universe]
|
||||
deduped, duplicate_n, restated_n = _dedupe_bulk_rows(in_universe)
|
||||
with engine.begin() as conn:
|
||||
rows_upserted += _upsert_events(conn, deduped)
|
||||
_write_window_status(
|
||||
engine,
|
||||
start=window_start,
|
||||
end=window_end,
|
||||
status="done",
|
||||
raw_n=len(raw_rows),
|
||||
universe_n=len(deduped),
|
||||
duplicate_n=duplicate_n,
|
||||
restated_n=restated_n,
|
||||
note="bulk",
|
||||
)
|
||||
duplicate_rows += duplicate_n
|
||||
restated_rows += restated_n
|
||||
if index == 1 or index % 10 == 0 or index == len(pending):
|
||||
print(
|
||||
f"progress windows={index}/{len(pending)} "
|
||||
f"requests={requests_this_run}/{args.limit} "
|
||||
f"last={window_start}..{window_end} rows={len(deduped)}"
|
||||
)
|
||||
if args.sleep > 0:
|
||||
await asyncio.sleep(float(args.sleep))
|
||||
|
||||
with engine.begin() as conn:
|
||||
windows_done = int(
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT COUNT(*) FROM earnings_backfill_windows "
|
||||
"WHERE status='done' AND from_date >= :a AND to_date <= :b"
|
||||
),
|
||||
{"a": start.isoformat(), "b": end.isoformat()},
|
||||
).scalar_one()
|
||||
)
|
||||
complete = windows_done >= len(all_windows)
|
||||
if complete:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
for symbol in symbols:
|
||||
count = int(
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
||||
VALUES (:s, 'error', 0, :t, :n)
|
||||
ON CONFLICT(symbol) DO UPDATE SET
|
||||
status='error', updated_at=excluded.updated_at, note=excluded.note
|
||||
"""
|
||||
"SELECT COUNT(*) FROM earnings_events "
|
||||
"WHERE symbol=:symbol AND announce_date BETWEEN :a AND :b"
|
||||
),
|
||||
{
|
||||
"s": sym,
|
||||
"t": datetime.now(timezone.utc).isoformat(),
|
||||
"n": msg[:200],
|
||||
},
|
||||
)
|
||||
if args.sleep > 0:
|
||||
await asyncio.sleep(args.sleep)
|
||||
continue
|
||||
|
||||
reqs_used += 1
|
||||
# Keep all rows with dates on/before end — SUE needs trailing history.
|
||||
filtered = [
|
||||
r for r in items if r["announce_date"] <= end.isoformat()
|
||||
]
|
||||
# Do NOT mark empty as done — leave pending for another provider/day.
|
||||
status = "done" if filtered else "empty"
|
||||
with engine.begin() as conn:
|
||||
n_w = _upsert_events(conn, filtered, source) if filtered else 0
|
||||
events_written += n_w
|
||||
{"symbol": symbol, "a": start.isoformat(), "b": end.isoformat()},
|
||||
).scalar_one()
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
||||
VALUES (:s, :st, :n, :t, :note)
|
||||
VALUES (:symbol, 'done', :count, :now, 'bulk_complete')
|
||||
ON CONFLICT(symbol) DO UPDATE SET
|
||||
status=excluded.status, n_events=excluded.n_events,
|
||||
updated_at=excluded.updated_at, note=excluded.note
|
||||
status='done', n_events=excluded.n_events,
|
||||
updated_at=excluded.updated_at, note=excluded.note
|
||||
"""
|
||||
),
|
||||
{
|
||||
"s": sym,
|
||||
"st": status,
|
||||
"n": len(filtered),
|
||||
"t": datetime.now(timezone.utc).isoformat(),
|
||||
"note": note,
|
||||
},
|
||||
{"symbol": symbol, "count": count, "now": now},
|
||||
)
|
||||
if reqs_used % 10 == 0 or reqs_used == 1:
|
||||
print(
|
||||
f" progress reqs={reqs_used}/{req_budget} last={sym} "
|
||||
f"events_batch={len(filtered)} src={source}"
|
||||
)
|
||||
# AV free tier is ~5/min or 25/day — be polite when using it.
|
||||
sleep_s = float(args.sleep)
|
||||
if source.startswith("alpha_vantage"):
|
||||
sleep_s = max(sleep_s, 12.0)
|
||||
if sleep_s > 0:
|
||||
await asyncio.sleep(sleep_s)
|
||||
|
||||
with engine.connect() as conn:
|
||||
params = {"a": start.isoformat(), "b": end.isoformat()}
|
||||
total_events = int(
|
||||
conn.execute(text("SELECT COUNT(*) FROM earnings_events")).scalar_one()
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT COUNT(*) FROM earnings_events "
|
||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
||||
"AND announce_date BETWEEN :a AND :b"
|
||||
),
|
||||
params,
|
||||
).scalar_one()
|
||||
)
|
||||
done_n = int(
|
||||
paired_events = int(
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT COUNT(*) FROM earnings_events "
|
||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
||||
"AND announce_date BETWEEN :a AND :b "
|
||||
"AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL"
|
||||
),
|
||||
params,
|
||||
).scalar_one()
|
||||
)
|
||||
date_range = conn.execute(
|
||||
text(
|
||||
"SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events "
|
||||
"WHERE symbol IN (SELECT symbol FROM tickers) "
|
||||
"AND announce_date BETWEEN :a AND :b"
|
||||
),
|
||||
params,
|
||||
).fetchone()
|
||||
done_symbols = int(
|
||||
conn.execute(
|
||||
text("SELECT COUNT(*) FROM earnings_backfill_meta WHERE status='done'")
|
||||
).scalar_one()
|
||||
)
|
||||
d_range = conn.execute(
|
||||
text("SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events")
|
||||
totals = conn.execute(
|
||||
text(
|
||||
"SELECT COALESCE(SUM(requests),0), COALESCE(SUM(duplicate_rows),0), "
|
||||
"COALESCE(SUM(restated_rows),0) FROM earnings_backfill_windows "
|
||||
"WHERE from_date >= :a AND to_date <= :b"
|
||||
),
|
||||
params,
|
||||
).fetchone()
|
||||
with_actual = int(
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT COUNT(*) FROM earnings_events "
|
||||
"WHERE eps_actual IS NOT NULL AND eps_estimate IS NOT NULL"
|
||||
)
|
||||
).scalar_one()
|
||||
)
|
||||
|
||||
summary = {
|
||||
"mode": mode,
|
||||
"fmp_requests": reqs_used,
|
||||
"events_written_this_run": events_written,
|
||||
"total_events": total_events,
|
||||
"symbols_done": done_n,
|
||||
"mode": "fmp_bulk_date_range_only",
|
||||
"window": {"from": start.isoformat(), "to": end.isoformat()},
|
||||
"window_days": int(args.window_days),
|
||||
"bulk_windows_total": len(all_windows),
|
||||
"bulk_windows_done": windows_done,
|
||||
"bulk_requests_this_run": requests_this_run,
|
||||
"bulk_requests_logged_total": int(totals[0]),
|
||||
"rows_upserted_this_run": rows_upserted,
|
||||
"duplicate_rows_this_run": duplicate_rows,
|
||||
"restated_rows_this_run": restated_rows,
|
||||
"duplicate_rows_logged_total": int(totals[1]),
|
||||
"restated_rows_logged_total": int(totals[2]),
|
||||
"dedupe_policy": (
|
||||
"UNIQUE(symbol, announce_date); prefer more non-null fields, then "
|
||||
"the provider's later occurrence; non-null bulk fields replace prior "
|
||||
"values while null bulk fields retain existing values"
|
||||
),
|
||||
"events_in_window": total_events,
|
||||
"events_with_actual_and_estimate": paired_events,
|
||||
"symbols_done": done_symbols,
|
||||
"symbols_universe": len(symbols),
|
||||
"announce_date_range": {"min": d_range[0], "max": d_range[1]},
|
||||
"events_with_actual_and_estimate": with_actual,
|
||||
"budget": req_budget,
|
||||
"complete": done_n >= len(symbols),
|
||||
"announce_date_range": {"min": date_range[0], "max": date_range[1]},
|
||||
"request_budget": int(args.limit),
|
||||
"stop_note": stop_note,
|
||||
"complete": complete,
|
||||
}
|
||||
output = Path("reports/earnings-backfill-status.json")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, indent=2))
|
||||
out = Path("reports") / "earnings-backfill-status.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Wrote {out}")
|
||||
print(f"Wrote {output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -97,6 +97,14 @@ def _parse_args() -> argparse.Namespace:
|
||||
default=5,
|
||||
help="Retries per symbol on RateLimitError.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--source-symbols-only",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Refresh only symbols present in --source. Useful for repairing "
|
||||
"per-symbol depth without re-fetching the broad rank-only pool."
|
||||
),
|
||||
)
|
||||
p.add_argument("--quiet", action="store_true")
|
||||
return p.parse_args()
|
||||
|
||||
@@ -218,6 +226,15 @@ async def _main() -> None:
|
||||
if not source.exists():
|
||||
raise SystemExit(f"Source snapshot not found: {source}")
|
||||
|
||||
source_engine = create_engine(
|
||||
f"sqlite:///{source.resolve().as_posix()}", future=True
|
||||
)
|
||||
with source_engine.connect() as conn:
|
||||
source_symbols = {
|
||||
str(row[0]) for row in conn.execute(text("SELECT symbol FROM tickers"))
|
||||
}
|
||||
source_engine.dispose()
|
||||
|
||||
# Any rebuild/update invalidates prior completion until we finish cleanly.
|
||||
clear_manifest(output)
|
||||
|
||||
@@ -241,7 +258,12 @@ async def _main() -> None:
|
||||
start = end - timedelta(days=int(args.history_days))
|
||||
|
||||
print("Resolving universe pool (nasdaq_all ∪ sp500)…")
|
||||
pool, sources = await _resolve_pool()
|
||||
if args.source_symbols_only:
|
||||
pool = sorted(source_symbols)
|
||||
sources = {"pool": "source_snapshot"}
|
||||
print(" source snapshot: symbol pool selected")
|
||||
else:
|
||||
pool, sources = await _resolve_pool()
|
||||
print(f"Pool size: {len(pool)} (sources={sources})")
|
||||
|
||||
# Sync sqlite via raw SQL — one short transaction per symbol so a failed
|
||||
@@ -257,7 +279,7 @@ async def _main() -> None:
|
||||
text("SELECT id, symbol FROM tickers")
|
||||
).fetchall()
|
||||
existing_ids = {str(sym): int(tid) for tid, sym in existing_rows}
|
||||
prod_symbols = set(existing_ids)
|
||||
prod_symbols = set(source_symbols)
|
||||
|
||||
bar_counts: dict[str, int] = {}
|
||||
for sym, tid in existing_ids.items():
|
||||
@@ -363,7 +385,7 @@ async def _main() -> None:
|
||||
for b in bars
|
||||
],
|
||||
)
|
||||
if is_new:
|
||||
if is_new and sym not in prod_symbols:
|
||||
write.execute(
|
||||
text(
|
||||
"INSERT OR REPLACE INTO research_rank_only "
|
||||
@@ -385,6 +407,39 @@ async def _main() -> None:
|
||||
f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}"
|
||||
)
|
||||
|
||||
benchmark_rows = 0
|
||||
try:
|
||||
benchmark_bars = await _fetch_symbol_bars(
|
||||
provider,
|
||||
"SPY",
|
||||
start,
|
||||
end,
|
||||
max_retries=args.max_retries,
|
||||
sleep_s=args.sleep,
|
||||
)
|
||||
with engine.begin() as write:
|
||||
write.execute(
|
||||
text(
|
||||
"DELETE FROM benchmark_prices WHERE symbol='SPY' "
|
||||
"AND date >= :start AND date <= :end"
|
||||
),
|
||||
{"start": start.isoformat(), "end": end.isoformat()},
|
||||
)
|
||||
if benchmark_bars:
|
||||
write.execute(
|
||||
text(
|
||||
"INSERT INTO benchmark_prices(symbol, date, close) "
|
||||
"VALUES ('SPY', :date, :close)"
|
||||
),
|
||||
[
|
||||
{"date": bar.date.isoformat(), "close": float(bar.close)}
|
||||
for bar in benchmark_bars
|
||||
],
|
||||
)
|
||||
benchmark_rows = len(benchmark_bars)
|
||||
except Exception as exc:
|
||||
print(f" benchmark SPY refresh FAIL {exc}")
|
||||
|
||||
rank_only_n = conn.execute(
|
||||
text("SELECT COUNT(*) FROM research_rank_only")
|
||||
).scalar_one()
|
||||
@@ -409,6 +464,8 @@ async def _main() -> None:
|
||||
"prod_symbols_at_start": len(prod_symbols),
|
||||
"pool_size": len(pool),
|
||||
"to_fetch": len(to_fetch),
|
||||
"source_symbols_only": bool(args.source_symbols_only),
|
||||
"benchmark_spy_rows": benchmark_rows,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,657 @@
|
||||
"""Import the public post-no-preference/earnings DoltHub database.
|
||||
|
||||
The earnings calendar and EPS history are separate tables in the source. This
|
||||
importer aligns them monotonically per symbol, keeps every calendar event for
|
||||
the defensive gap study, and stores the longer EPS history separately for SUE
|
||||
scaling. EPS history without an announcement date is never exposed as a live
|
||||
signal event.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EVENTS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
symbol TEXT NOT NULL,
|
||||
announce_date TEXT NOT NULL,
|
||||
announce_time TEXT,
|
||||
eps_estimate REAL,
|
||||
eps_actual REAL,
|
||||
revenue_estimate REAL,
|
||||
revenue_actual REAL,
|
||||
source TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
period_end_date TEXT,
|
||||
UNIQUE(symbol, announce_date)
|
||||
)
|
||||
"""
|
||||
META_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
|
||||
symbol TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
n_events INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL,
|
||||
note TEXT
|
||||
)
|
||||
"""
|
||||
SURPRISE_HISTORY_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS earnings_surprise_history (
|
||||
symbol TEXT NOT NULL,
|
||||
period_end_date TEXT NOT NULL,
|
||||
eps_estimate REAL,
|
||||
eps_actual REAL,
|
||||
source TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY(symbol, period_end_date)
|
||||
)
|
||||
"""
|
||||
|
||||
SKIP_EVENT_COST = 45.0
|
||||
SKIP_PERIOD_COST = 45.0
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
||||
parser.add_argument("--calendar-csv", required=True)
|
||||
parser.add_argument("--history-csv", required=True)
|
||||
parser.add_argument("--from-date", default="2020-01-22")
|
||||
parser.add_argument("--to-date", required=True)
|
||||
parser.add_argument("--source-commit", required=True)
|
||||
parser.add_argument(
|
||||
"--source-url",
|
||||
default="https://www.dolthub.com/repositories/post-no-preference/earnings",
|
||||
)
|
||||
parser.add_argument("--max-period-lag-days", type=int, default=90)
|
||||
parser.add_argument("--max-period-lead-days", type=int, default=14)
|
||||
parser.add_argument(
|
||||
"--status-output", default="reports/earnings-backfill-status.json"
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _normalise_symbol(value: Any) -> str:
|
||||
return str(value or "").strip().upper().replace(".", "-")
|
||||
|
||||
|
||||
def _normalise_session(value: Any) -> str | None:
|
||||
cleaned = str(value or "").strip().lower().replace("_", " ").replace("-", " ")
|
||||
aliases = {
|
||||
"before market open": "bmo",
|
||||
"before open": "bmo",
|
||||
"bmo": "bmo",
|
||||
"after market close": "amc",
|
||||
"after close": "amc",
|
||||
"amc": "amc",
|
||||
"during market hours": "during",
|
||||
"dmh": "during",
|
||||
}
|
||||
return aliases.get(cleaned, cleaned or None)
|
||||
|
||||
|
||||
def _number(value: Any) -> float | None:
|
||||
if value is None or str(value).strip() == "":
|
||||
return None
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return result if math.isfinite(result) else None
|
||||
|
||||
|
||||
def _read_calendar(
|
||||
path: Path,
|
||||
universe: set[str],
|
||||
start: date,
|
||||
end: date,
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
raw_rows = 0
|
||||
universe_rows = 0
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||||
for raw in csv.DictReader(handle):
|
||||
raw_rows += 1
|
||||
symbol = _normalise_symbol(raw.get("act_symbol"))
|
||||
raw_date = str(raw.get("date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
event_date = date.fromisoformat(raw_date)
|
||||
if not start <= event_date <= end:
|
||||
continue
|
||||
universe_rows += 1
|
||||
row = {
|
||||
"symbol": symbol,
|
||||
"announce_date": event_date,
|
||||
"announce_time": _normalise_session(raw.get("when")),
|
||||
}
|
||||
key = (symbol, event_date)
|
||||
previous = by_key.get(key)
|
||||
if previous is not None:
|
||||
duplicate_rows += 1
|
||||
if (
|
||||
previous.get("announce_time") is not None
|
||||
and row.get("announce_time") is not None
|
||||
and previous["announce_time"] != row["announce_time"]
|
||||
):
|
||||
restated_rows += 1
|
||||
if row.get("announce_time") is not None:
|
||||
by_key[key] = row
|
||||
else:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda item: item["announce_date"])
|
||||
return grouped, {
|
||||
"raw_rows": raw_rows,
|
||||
"universe_rows_in_window": universe_rows,
|
||||
"deduped_rows_in_window": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def _read_history(
|
||||
path: Path, universe: set[str]
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, int]]:
|
||||
by_key: dict[tuple[str, date], dict[str, Any]] = {}
|
||||
raw_rows = 0
|
||||
universe_rows = 0
|
||||
duplicate_rows = 0
|
||||
restated_rows = 0
|
||||
fields = ("eps_actual", "eps_estimate")
|
||||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||||
for raw in csv.DictReader(handle):
|
||||
raw_rows += 1
|
||||
symbol = _normalise_symbol(raw.get("act_symbol"))
|
||||
raw_date = str(raw.get("period_end_date") or "")[:10]
|
||||
if symbol not in universe or not raw_date:
|
||||
continue
|
||||
universe_rows += 1
|
||||
period_end = date.fromisoformat(raw_date)
|
||||
row = {
|
||||
"symbol": symbol,
|
||||
"period_end_date": period_end,
|
||||
"eps_actual": _number(raw.get("reported")),
|
||||
"eps_estimate": _number(raw.get("estimate")),
|
||||
}
|
||||
key = (symbol, period_end)
|
||||
previous = by_key.get(key)
|
||||
if previous is not None:
|
||||
duplicate_rows += 1
|
||||
if any(
|
||||
previous.get(field) is not None
|
||||
and row.get(field) is not None
|
||||
and previous[field] != row[field]
|
||||
for field in fields
|
||||
):
|
||||
restated_rows += 1
|
||||
previous_score = sum(previous.get(field) is not None for field in fields)
|
||||
row_score = sum(row.get(field) is not None for field in fields)
|
||||
if row_score >= previous_score:
|
||||
by_key[key] = row
|
||||
else:
|
||||
by_key[key] = row
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in by_key.values():
|
||||
grouped[row["symbol"]].append(row)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda item: item["period_end_date"])
|
||||
return grouped, {
|
||||
"raw_rows": raw_rows,
|
||||
"universe_rows": universe_rows,
|
||||
"deduped_rows": len(by_key),
|
||||
"duplicate_rows": duplicate_rows,
|
||||
"restated_rows": restated_rows,
|
||||
}
|
||||
|
||||
|
||||
def _match_cost(event: dict[str, Any], period: dict[str, Any]) -> float:
|
||||
delta = (event["announce_date"] - period["period_end_date"]).days
|
||||
missing_session_penalty = 3.0 if event.get("announce_time") is None else 0.0
|
||||
return float(abs(delta - 30)) + missing_session_penalty
|
||||
|
||||
|
||||
def _align_symbol(
|
||||
events: list[dict[str, Any]],
|
||||
periods: list[dict[str, Any]],
|
||||
*,
|
||||
max_lag_days: int,
|
||||
max_lead_days: int,
|
||||
) -> tuple[list[tuple[int, int]], list[int], list[int]]:
|
||||
"""Return a minimum-cost monotonic calendar-to-period alignment."""
|
||||
n_events = len(events)
|
||||
n_periods = len(periods)
|
||||
scores = [[0.0] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
choices = [[""] * (n_periods + 1) for _ in range(n_events + 1)]
|
||||
for event_index in range(n_events - 1, -1, -1):
|
||||
scores[event_index][n_periods] = (
|
||||
scores[event_index + 1][n_periods] + SKIP_EVENT_COST
|
||||
)
|
||||
choices[event_index][n_periods] = "event"
|
||||
for period_index in range(n_periods - 1, -1, -1):
|
||||
scores[n_events][period_index] = (
|
||||
scores[n_events][period_index + 1] + SKIP_PERIOD_COST
|
||||
)
|
||||
choices[n_events][period_index] = "period"
|
||||
|
||||
for event_index in range(n_events - 1, -1, -1):
|
||||
for period_index in range(n_periods - 1, -1, -1):
|
||||
options = [
|
||||
(
|
||||
scores[event_index + 1][period_index] + SKIP_EVENT_COST,
|
||||
2,
|
||||
"event",
|
||||
),
|
||||
(
|
||||
scores[event_index][period_index + 1] + SKIP_PERIOD_COST,
|
||||
1,
|
||||
"period",
|
||||
),
|
||||
]
|
||||
delta = (
|
||||
events[event_index]["announce_date"]
|
||||
- periods[period_index]["period_end_date"]
|
||||
).days
|
||||
if -max_lead_days <= delta <= max_lag_days:
|
||||
options.append(
|
||||
(
|
||||
scores[event_index + 1][period_index + 1]
|
||||
+ _match_cost(events[event_index], periods[period_index]),
|
||||
0,
|
||||
"match",
|
||||
)
|
||||
)
|
||||
score, _, choice = min(options)
|
||||
scores[event_index][period_index] = score
|
||||
choices[event_index][period_index] = choice
|
||||
|
||||
matches: list[tuple[int, int]] = []
|
||||
unmatched_events: list[int] = []
|
||||
unmatched_periods: list[int] = []
|
||||
event_index = 0
|
||||
period_index = 0
|
||||
while event_index < n_events or period_index < n_periods:
|
||||
if event_index >= n_events:
|
||||
unmatched_periods.extend(range(period_index, n_periods))
|
||||
break
|
||||
if period_index >= n_periods:
|
||||
unmatched_events.extend(range(event_index, n_events))
|
||||
break
|
||||
choice = choices[event_index][period_index]
|
||||
if choice == "match":
|
||||
matches.append((event_index, period_index))
|
||||
event_index += 1
|
||||
period_index += 1
|
||||
elif choice == "period":
|
||||
unmatched_periods.append(period_index)
|
||||
period_index += 1
|
||||
else:
|
||||
unmatched_events.append(event_index)
|
||||
event_index += 1
|
||||
return matches, unmatched_events, unmatched_periods
|
||||
|
||||
|
||||
def _ensure_schema(connection: sqlite3.Connection) -> None:
|
||||
connection.execute(EVENTS_DDL)
|
||||
columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(earnings_events)")
|
||||
}
|
||||
if "period_end_date" not in columns:
|
||||
connection.execute("ALTER TABLE earnings_events ADD COLUMN period_end_date TEXT")
|
||||
connection.execute(META_DDL)
|
||||
connection.execute(SURPRISE_HISTORY_DDL)
|
||||
|
||||
|
||||
def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
calendar_csv = Path(args.calendar_csv)
|
||||
history_csv = Path(args.history_csv)
|
||||
for path in (snapshot, calendar_csv, history_csv):
|
||||
if not path.exists():
|
||||
raise SystemExit(f"Missing input: {path}")
|
||||
start = date.fromisoformat(args.from_date)
|
||||
end = date.fromisoformat(args.to_date)
|
||||
if start > end:
|
||||
raise SystemExit("--from-date must not be after --to-date")
|
||||
|
||||
connection = sqlite3.connect(snapshot)
|
||||
try:
|
||||
universe = {
|
||||
_normalise_symbol(row[0])
|
||||
for row in connection.execute("SELECT symbol FROM tickers")
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
calendar, calendar_stats = _read_calendar(calendar_csv, universe, start, end)
|
||||
history, history_stats = _read_history(history_csv, universe)
|
||||
|
||||
aligned_events: list[dict[str, Any]] = []
|
||||
pairing_deltas: list[int] = []
|
||||
unmatched_calendar = 0
|
||||
unmatched_periods_in_pairing_window = 0
|
||||
matched = 0
|
||||
for symbol in sorted(universe):
|
||||
events = calendar.get(symbol, [])
|
||||
lower = start - timedelta(days=int(args.max_period_lag_days))
|
||||
upper = end + timedelta(days=int(args.max_period_lead_days))
|
||||
periods = [
|
||||
row
|
||||
for row in history.get(symbol, [])
|
||||
if lower <= row["period_end_date"] <= upper
|
||||
]
|
||||
matches, unmatched_events, unmatched_periods = _align_symbol(
|
||||
events,
|
||||
periods,
|
||||
max_lag_days=int(args.max_period_lag_days),
|
||||
max_lead_days=int(args.max_period_lead_days),
|
||||
)
|
||||
matched_by_event = {event_index: period_index for event_index, period_index in matches}
|
||||
matched += len(matches)
|
||||
unmatched_calendar += len(unmatched_events)
|
||||
unmatched_periods_in_pairing_window += len(unmatched_periods)
|
||||
for event_index, event in enumerate(events):
|
||||
row = dict(event)
|
||||
period_index = matched_by_event.get(event_index)
|
||||
if period_index is None:
|
||||
row.update(
|
||||
{
|
||||
"period_end_date": None,
|
||||
"eps_actual": None,
|
||||
"eps_estimate": None,
|
||||
}
|
||||
)
|
||||
else:
|
||||
period = periods[period_index]
|
||||
row.update(
|
||||
{
|
||||
"period_end_date": period["period_end_date"],
|
||||
"eps_actual": period["eps_actual"],
|
||||
"eps_estimate": period["eps_estimate"],
|
||||
}
|
||||
)
|
||||
pairing_deltas.append(
|
||||
(event["announce_date"] - period["period_end_date"]).days
|
||||
)
|
||||
aligned_events.append(row)
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
source = f"dolthub_post_no_preference@{args.source_commit}"
|
||||
conflicting_existing_rows = 0
|
||||
conflicting_existing_fields = 0
|
||||
preserved_existing_fields = 0
|
||||
incoming_keys = {
|
||||
(row["symbol"], row["announce_date"].isoformat()) for row in aligned_events
|
||||
}
|
||||
connection = sqlite3.connect(snapshot)
|
||||
try:
|
||||
_ensure_schema(connection)
|
||||
existing = {
|
||||
(str(row[0]), str(row[1])): row
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT symbol, announce_date, announce_time, eps_estimate,
|
||||
eps_actual, period_end_date, source
|
||||
FROM earnings_events
|
||||
WHERE announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
(start.isoformat(), end.isoformat()),
|
||||
)
|
||||
}
|
||||
upsert = """
|
||||
INSERT INTO earnings_events(
|
||||
symbol, announce_date, announce_time, eps_estimate, eps_actual,
|
||||
revenue_estimate, revenue_actual, source, fetched_at, period_end_date
|
||||
) VALUES (?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?)
|
||||
ON CONFLICT(symbol, announce_date) DO UPDATE SET
|
||||
announce_time=COALESCE(earnings_events.announce_time, excluded.announce_time),
|
||||
eps_estimate=COALESCE(earnings_events.eps_estimate, excluded.eps_estimate),
|
||||
eps_actual=COALESCE(earnings_events.eps_actual, excluded.eps_actual),
|
||||
period_end_date=COALESCE(excluded.period_end_date, earnings_events.period_end_date),
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
for row in aligned_events:
|
||||
key = (row["symbol"], row["announce_date"].isoformat())
|
||||
old = existing.get(key)
|
||||
retained = 0
|
||||
conflicts = 0
|
||||
if old is not None:
|
||||
old_values = {
|
||||
"announce_time": old[2],
|
||||
"eps_estimate": old[3],
|
||||
"eps_actual": old[4],
|
||||
"period_end_date": old[5],
|
||||
}
|
||||
new_values = {
|
||||
"announce_time": row.get("announce_time"),
|
||||
"eps_estimate": row.get("eps_estimate"),
|
||||
"eps_actual": row.get("eps_actual"),
|
||||
"period_end_date": (
|
||||
row["period_end_date"].isoformat()
|
||||
if row.get("period_end_date")
|
||||
else None
|
||||
),
|
||||
}
|
||||
for field, new_value in new_values.items():
|
||||
old_value = old_values[field]
|
||||
if field != "period_end_date" and old_value is not None:
|
||||
retained += 1
|
||||
if new_value is not None and old_value is not None:
|
||||
if field in {"eps_estimate", "eps_actual"}:
|
||||
differs = not math.isclose(
|
||||
float(new_value), float(old_value), rel_tol=0.0, abs_tol=1e-9
|
||||
)
|
||||
else:
|
||||
differs = str(new_value) != str(old_value)
|
||||
conflicts += int(differs)
|
||||
conflicting_existing_rows += int(conflicts > 0)
|
||||
conflicting_existing_fields += conflicts
|
||||
preserved_existing_fields += retained
|
||||
row_source = source
|
||||
if retained and old is not None:
|
||||
row_source = f"{old[6]}+calendar:{source}"
|
||||
connection.execute(
|
||||
upsert,
|
||||
(
|
||||
row["symbol"],
|
||||
row["announce_date"].isoformat(),
|
||||
row.get("announce_time"),
|
||||
row.get("eps_estimate"),
|
||||
row.get("eps_actual"),
|
||||
row_source,
|
||||
now,
|
||||
(
|
||||
row["period_end_date"].isoformat()
|
||||
if row.get("period_end_date")
|
||||
else None
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
history_upsert = """
|
||||
INSERT INTO earnings_surprise_history(
|
||||
symbol, period_end_date, eps_estimate, eps_actual, source, fetched_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(symbol, period_end_date) DO UPDATE SET
|
||||
eps_estimate=COALESCE(excluded.eps_estimate, earnings_surprise_history.eps_estimate),
|
||||
eps_actual=COALESCE(excluded.eps_actual, earnings_surprise_history.eps_actual),
|
||||
source=excluded.source,
|
||||
fetched_at=excluded.fetched_at
|
||||
"""
|
||||
for symbol, rows in history.items():
|
||||
connection.executemany(
|
||||
history_upsert,
|
||||
[
|
||||
(
|
||||
symbol,
|
||||
row["period_end_date"].isoformat(),
|
||||
row.get("eps_estimate"),
|
||||
row.get("eps_actual"),
|
||||
source,
|
||||
now,
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
for symbol in sorted(universe):
|
||||
count = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol=? AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
(symbol, start.isoformat(), end.isoformat()),
|
||||
).fetchone()[0]
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
|
||||
VALUES (?, 'done', ?, ?, 'dolthub_bulk_complete')
|
||||
ON CONFLICT(symbol) DO UPDATE SET
|
||||
status='done', n_events=excluded.n_events,
|
||||
updated_at=excluded.updated_at, note=excluded.note
|
||||
""",
|
||||
(symbol, count, now),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
params = (start.isoformat(), end.isoformat())
|
||||
total_events = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
paired_events = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
date_range = connection.execute(
|
||||
"""
|
||||
SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND announce_date BETWEEN ? AND ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()
|
||||
source_symbols = set(calendar)
|
||||
history_complete = int(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM earnings_surprise_history
|
||||
WHERE symbol IN (SELECT symbol FROM tickers)
|
||||
AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL
|
||||
"""
|
||||
).fetchone()[0]
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
deltas = sorted(pairing_deltas)
|
||||
summary = {
|
||||
"mode": "dolthub_public_bulk_clone",
|
||||
"window": {"from": start.isoformat(), "to": end.isoformat()},
|
||||
"coverage_amendment": {
|
||||
"approved_by_user": True,
|
||||
"reason": "FMP free tier blocks historical bulk earnings",
|
||||
"original_start": "2016-01-04",
|
||||
"amended_announcement_start": start.isoformat(),
|
||||
},
|
||||
"source": {
|
||||
"repository": args.source_url,
|
||||
"commit": args.source_commit,
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"upstream_provider_documented": False,
|
||||
},
|
||||
"bulk_windows_total": 1,
|
||||
"bulk_windows_done": 1,
|
||||
"bulk_requests_logged_total": 1,
|
||||
"bulk_exports": 2,
|
||||
"calendar": calendar_stats,
|
||||
"eps_history": {**history_stats, "complete_actual_and_estimate": history_complete},
|
||||
"pairing": {
|
||||
"method": "minimum-cost monotonic alignment per symbol",
|
||||
"allowed_announce_minus_period_end_days": [
|
||||
-int(args.max_period_lead_days),
|
||||
int(args.max_period_lag_days),
|
||||
],
|
||||
"matched_calendar_events": matched,
|
||||
"unmatched_calendar_events": unmatched_calendar,
|
||||
"unmatched_periods_in_pairing_window": unmatched_periods_in_pairing_window,
|
||||
"announce_minus_period_end_days": {
|
||||
"min": min(deltas) if deltas else None,
|
||||
"median": deltas[len(deltas) // 2] if deltas else None,
|
||||
"max": max(deltas) if deltas else None,
|
||||
},
|
||||
"pre_2020_eps_history_use": (
|
||||
"trailing_surprise_stdev_only; never treated as an announcement "
|
||||
"or live signal event"
|
||||
),
|
||||
},
|
||||
"duplicate_rows_logged_total": (
|
||||
calendar_stats["duplicate_rows"] + history_stats["duplicate_rows"]
|
||||
),
|
||||
"restated_rows_logged_total": (
|
||||
calendar_stats["restated_rows"]
|
||||
+ history_stats["restated_rows"]
|
||||
+ conflicting_existing_rows
|
||||
),
|
||||
"conflicting_existing_rows": conflicting_existing_rows,
|
||||
"conflicting_existing_fields": conflicting_existing_fields,
|
||||
"preserved_existing_fields": preserved_existing_fields,
|
||||
"existing_enrichment_events_not_in_dolthub_calendar": max(
|
||||
0, total_events - len(incoming_keys)
|
||||
),
|
||||
"dedupe_policy": (
|
||||
"UNIQUE(symbol, announce_date); normalise dot/dash symbols; retain one "
|
||||
"calendar row per key; preserve existing non-null session/EPS values from "
|
||||
"the prior FMP/Alpha Vantage partial backfill, then fill nulls and all "
|
||||
"remaining symbols from DoltHub; attach DoltHub period-end alignment"
|
||||
),
|
||||
"events_in_window": total_events,
|
||||
"events_with_actual_and_estimate": paired_events,
|
||||
"symbols_done": len(universe),
|
||||
"symbols_universe": len(universe),
|
||||
"symbols_with_dolthub_calendar": len(source_symbols),
|
||||
"symbols_without_dolthub_calendar": sorted(universe - source_symbols),
|
||||
"announce_date_range": {"min": date_range[0], "max": date_range[1]},
|
||||
"complete": True,
|
||||
}
|
||||
output = Path(args.status_output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, indent=2))
|
||||
print(f"Wrote {output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
+1370
-726
File diff suppressed because it is too large
Load Diff
@@ -120,12 +120,14 @@ case "$PHASE" in
|
||||
ssl) ssl_check ;;
|
||||
earnings)
|
||||
need_file "$PROD_SNAP"
|
||||
log "Earnings backfill + research (parked experiment)"
|
||||
need_file "$RESEARCH_SNAP"
|
||||
log "Earnings Task 2 bulk backfill + registered 2a/2b closeout"
|
||||
"$PYTHON" scripts/backfill_earnings_events.py \
|
||||
--snapshot "$PROD_SNAP" --provider fmp --force-symbol \
|
||||
--snapshot "$PROD_SNAP" --from-date 2016-01-04 --window-days 30 \
|
||||
--limit "$FMP_LIMIT" --sleep "$FMP_SLEEP"
|
||||
"$PYTHON" scripts/run_earnings_research.py \
|
||||
--snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn
|
||||
--snapshot "$RESEARCH_SNAP" --universe-snapshot "$PROD_SNAP" \
|
||||
--earnings-snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn
|
||||
;;
|
||||
prod_book)
|
||||
need_file "$RESEARCH_SNAP"
|
||||
|
||||
Reference in New Issue
Block a user