feat(dolt): migration 026 + models — fundamentals/earnings schema (A1 schema)
First reviewable slice of workstream A: schema only, no importers, no data. - data_import_runs: lean batch-import audit (source/revision/status, row_counts_json + validation_json as Text-holding-JSON per repo convention). - fundamental_snapshots: CIK-keyed, one immutable row per accession; stores per-period raw facts (duration = cumulative YTD/FY, balance-sheet = period-end) plus period_start/period_end/fiscal_year/fiscal_period so discrete quarters, Q4, TTM and YoY are derived at read time. - earnings_events: Dolt-sourced calendar + surprise history, unique (ticker_id, announce_date). - tickers: nullable cik/sic/sic_description — the ticker<->issuer join point. fundamental_data is left untouched (cutover gated separately at A5). Models registered in app/models/__init__.py; Ticker gains an earnings_events relationship. Verified: create_all builds the tables, mappers configure, and migration 026 renders valid Postgres DDL up and down. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,9 @@ from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.user import User
|
||||
from app.models.sentiment import SentimentScore
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.score import DimensionScore, CompositeScore
|
||||
from app.models.sr_level import SRLevel
|
||||
from app.models.trade_setup import TradeSetup
|
||||
@@ -21,6 +24,9 @@ __all__ = [
|
||||
"User",
|
||||
"SentimentScore",
|
||||
"FundamentalData",
|
||||
"FundamentalSnapshot",
|
||||
"EarningsEvent",
|
||||
"DataImportRun",
|
||||
"DimensionScore",
|
||||
"CompositeScore",
|
||||
"SRLevel",
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import Date, DateTime, Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class DataImportRun(Base):
|
||||
"""One row per bulk-import attempt (SEC facts / Dolt earnings / Dolt stocks).
|
||||
|
||||
Lean audit record for the batch import framework: every attempt is logged,
|
||||
whether it promoted, was a ``no_op`` (unchanged revision), or ``failed``.
|
||||
``row_counts`` and ``validation`` hold JSON strings (repo convention — see
|
||||
``fundamental_data.unavailable_fields_json``), not JSONB; the validation
|
||||
blob carries reconciliation/discrepancy summaries so no separate conflicts
|
||||
table is needed. One run per source at a time is enforced at write time by a
|
||||
Postgres advisory lock keyed by ``source``.
|
||||
"""
|
||||
|
||||
__tablename__ = "data_import_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_data_import_runs_source_started", "source", "started_at"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
# sec_facts | dolt_earnings | dolt_stocks
|
||||
source: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# Dolt commit hash, or SEC archive SHA-256. Null until known.
|
||||
revision: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# running | validated | promoted | no_op | failed
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
source_max_date: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
row_counts_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
validation_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
error_details: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -0,0 +1,42 @@
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import Date, DateTime, Float, ForeignKey, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class EarningsEvent(Base):
|
||||
"""Earnings calendar + surprise history, sourced from the DoltHub earnings repo.
|
||||
|
||||
Forward rows (``announce_date`` > today) are the calendar; past rows are
|
||||
results. Rescheduling is handled in the importer's promotion transaction:
|
||||
this source's future-dated rows are deleted and re-inserted from the new
|
||||
snapshot so moved/cancelled dates never linger; past rows are never deleted.
|
||||
"""
|
||||
|
||||
__tablename__ = "earnings_events"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("ticker_id", "announce_date", name="uq_earnings_ticker_announce"),
|
||||
Index("ix_earnings_events_announce_date", "announce_date"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
ticker_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tickers.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
announce_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
# bmo | amc | unknown (source coverage is partial)
|
||||
session: Mapped[str] = mapped_column(String(10), nullable=False, default="unknown")
|
||||
period_end: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
eps_estimate: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
eps_actual: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
source: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
import_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("data_import_runs.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
ticker = relationship("Ticker", back_populates="earnings_events")
|
||||
@@ -0,0 +1,73 @@
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import Date, DateTime, Float, ForeignKey, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class FundamentalSnapshot(Base):
|
||||
"""CIK-keyed, one immutable row per SEC accession.
|
||||
|
||||
Keyed by issuer (CIK), not ticker — multi-class issuers (GOOG/GOOGL) share
|
||||
one CIK and one set of fundamentals; the ``tickers.cik`` column is the only
|
||||
join point. Amendments are retained: every accession is a distinct immutable
|
||||
row, and readers pick the newest valid ``accepted_at`` per
|
||||
(cik, fiscal_year, fiscal_period) at read time — no flags, no mutation.
|
||||
|
||||
**Facts are stored as the filing reports them, never as derived quarters.**
|
||||
Duration facts (revenue, net_income, operating_income, diluted_eps, cfo,
|
||||
capex, depreciation_amortization) hold the filing's normalized **cumulative
|
||||
YTD/FY** value over (period_start -> period_end). Balance-sheet facts
|
||||
(cash_and_st_investments, total_debt, shares_outstanding) are **period-end**
|
||||
values. ``shares_outstanding`` is a point-in-time count
|
||||
(``dei:EntityCommonStockSharesOutstanding``, summed across share classes for
|
||||
a multi-class issuer) — deliberately not the weighted-average diluted share
|
||||
count, since both consumers (estimated market cap, YoY dilution read) want a
|
||||
point-in-time value. Discrete quarters (10-Q YTD deltas, Q4 = FY - Q1..Q3), TTM, YoY and
|
||||
the quarter tape are all derived at read time — so non-calendar fiscal years
|
||||
resolve correctly and a later amendment never leaves a stale frozen quarter.
|
||||
"""
|
||||
|
||||
__tablename__ = "fundamental_snapshots"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("accession", name="uq_fundamental_snapshots_accession"),
|
||||
Index("ix_fundamental_snapshots_cik_period", "cik", "fiscal_year", "fiscal_period"),
|
||||
Index("ix_fundamental_snapshots_cik_period_end", "cik", "period_end"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cik: Mapped[str] = mapped_column(String(10), nullable=False)
|
||||
accession: Mapped[str] = mapped_column(String(25), nullable=False)
|
||||
form: Mapped[str] = mapped_column(String(12), nullable=False) # 10-Q, 10-K, 10-K/A ...
|
||||
filed_date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
# Kept although PIT enforcement is deferred (one timestamp now vs painful retrofit).
|
||||
accepted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
# Period identity — required to align non-calendar fiscal years and to derive
|
||||
# discrete quarters from cumulative facts.
|
||||
period_start: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
period_end: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
fiscal_year: Mapped[int] = mapped_column(nullable=False)
|
||||
fiscal_period: Mapped[str] = mapped_column(String(4), nullable=False) # Q1|Q2|Q3|Q4|FY
|
||||
|
||||
# Duration facts — cumulative YTD/FY over (period_start -> period_end).
|
||||
revenue: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
net_income: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
operating_income: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
diluted_eps: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
cfo: Mapped[float | None] = mapped_column(Float, nullable=True) # cash flow from operations
|
||||
capex: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
depreciation_amortization: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
|
||||
# Balance-sheet facts — period-end values.
|
||||
cash_and_st_investments: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
total_debt: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
shares_outstanding: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
|
||||
import_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("data_import_runs.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
@@ -14,6 +14,13 @@ class Ticker(Base):
|
||||
# Company name (e.g. "Biogen Inc."); backfilled from Alpaca, nullable for
|
||||
# symbols Alpaca doesn't know.
|
||||
name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
# SEC issuer identity, refreshed by the SEC fundamentals import from
|
||||
# company_tickers.json / submissions. The only ticker<->issuer join point;
|
||||
# multi-class tickers (GOOG/GOOGL) share these values. Nullable: not every
|
||||
# symbol resolves to a CIK (e.g. ADRs, foreign issuers not in SEC data).
|
||||
cik: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
sic: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
sic_description: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow, nullable=False
|
||||
)
|
||||
@@ -28,3 +35,4 @@ class Ticker(Base):
|
||||
trade_setups = relationship("TradeSetup", back_populates="ticker", cascade="all, delete-orphan")
|
||||
watchlist_entries = relationship("WatchlistEntry", back_populates="ticker", cascade="all, delete-orphan")
|
||||
ingestion_progress = relationship("IngestionProgress", back_populates="ticker", cascade="all, delete-orphan", uselist=False)
|
||||
earnings_events = relationship("EarningsEvent", back_populates="ticker", cascade="all, delete-orphan")
|
||||
|
||||
Reference in New Issue
Block a user