signal-platform/scripts/reparse_fundamentals.py

"""Re-derive every stored SEC snapshot with the current parser.

Snapshots are immutable per accession, so a parser fix does not reach rows that
are already stored: a normal import skips them and only logs a
``snapshot_discrepancy``. This script is the deliberate, manual exception --
it restages every accession from SEC Company Facts and rewrites the rows whose
reconstruction changed.

**Dry run by default.** Nothing is written unless ``--apply`` is passed. The dry
run stages and validates exactly as the real run does (both are read-only) and
reports the full blast radius: how many rows would change, which fields, and
per-symbol before/after samples.

Cost: a reparse cannot be served from the database -- the facts a fixed parser now
accepts were never stored -- so it refetches Company Facts for every tracked issuer
under the SEC fair-access throttle. Expect a long run and a lot of network.

Scope note: this rewrites ``fundamental_snapshots`` only. Those rows now feed both
the fundamentals API/UI *and* — through the nightly ``fundamental_data`` refresh —
the fundamental dimension of the composite score, so a reparse does move scores
and backtests. Run it deliberately.

Examples
--------
  # dry run: report what would change, write nothing
  python scripts/reparse_fundamentals.py

  # dry run, showing more per-field detail
  python scripts/reparse_fundamentals.py --samples 40

  # actually rewrite the changed rows
  python scripts/reparse_fundamentals.py --apply
"""

from __future__ import annotations

import argparse
import asyncio
import sys
from collections import Counter
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.database import async_session_factory  # noqa: E402
from app.services.data_import import run_import  # noqa: E402
from app.services.sec_fundamentals_importer import SecFundamentalsImporter  # noqa: E402


def _parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--apply", action="store_true",
                    help="rewrite changed rows (default: dry run, writes nothing)")
    ap.add_argument("--samples", type=int, default=20,
                    help="how many changed accessions to show in detail (default 20)")
    return ap.parse_args()


async def _dry_run(samples: int) -> int:
    importer = SecFundamentalsImporter(reparse=True)
    async with async_session_factory() as db:
        print("staging every tracked issuer from SEC Company Facts (this is the slow part)...")
        revision = await importer.detect_revision(db)
        staged = await importer.stage(db)
        result = await importer.validate(db, staged)

    print(f"\nrevision            : {revision}")
    print(f"issuers fetched     : {staged.issuers_fetched}")
    print(f"rows reconstructed  : {len(staged.rows)}")
    print(f"already stored      : {len(staged.existing_accessions)}")
    print(f"WOULD BE REWRITTEN  : {len(staged.discrepancies)}")
    print(f"new inserts         : {len(staged.rows) - len(staged.existing_accessions)}")
    print(f"validation ok       : {result.ok}")
    if not result.ok:
        print(f"validation messages : {result.messages}")

    if staged.discrepancies:
        field_counts = Counter(f for d in staged.discrepancies for f in d["fields"])
        print("\nchanged fields (accession count per field):")
        for name, count in field_counts.most_common():
            print(f"   {name:28s} {count}")

        by_accession = {r.accession: r for r in staged.rows}
        print(f"\nfirst {min(samples, len(staged.discrepancies))} changed accessions:")
        for d in staged.discrepancies[:samples]:
            row = by_accession.get(d["accession"])
            where = f"{row.cik} {row.fiscal_year} {row.fiscal_period}" if row else "?"
            print(f"   {d['accession']}  {where:28s} {', '.join(d['fields'])}")

    print(
        "\nDRY RUN -- nothing was written."
        "\nCheck that the changes are the *kinds* you expect (recovered nulls,"
        "\ncorrected values) and sample issuers you did not anticipate before"
        "\nre-running with --apply."
    )
    return 0 if result.ok else 1


async def _apply() -> int:
    # force=True: the revision tracks SEC, which has not changed — the staleness
    # is on our side, so the normal no-op gate would skip this.
    run = await run_import(SecFundamentalsImporter(reparse=True), force=True)
    if run is None:
        print("another sec_facts import holds the lock; nothing done")
        return 1
    print(f"run {run.id}: status={run.status}")
    print(f"  revision   : {run.revision}")
    print(f"  row_counts : {run.row_counts_json}")
    if run.error_details:
        print(f"  error      : {run.error_details}")
    return 0 if run.status == "promoted" else 1


def main() -> int:
    args = _parse_args()
    return asyncio.run(_apply() if args.apply else _dry_run(args.samples))


if __name__ == "__main__":
    raise SystemExit(main())