Track backtest reports in git; add a report comparison tool

The reports are the evidence behind the production baseline, so they belong
next to the README that quotes them rather than living only on one machine.
Un-ignores reports/*.json (~2.6 MB compressed for all 11); the snapshot DBs
they run against stay ignored.

Renames the reports to a single dated scheme so they sort chronologically and
say what they measured. Each name is derived from the report's own contents
(the atr_trail_sweep / regime_overlay / blue_sky_projected / sizing_test
sections, and the qualified counts that identify the A/B arms), not from the
ad-hoc slugs they carried before. The run the README quotes is now
backtest-20260711-prod-baseline.json.

reports/compare_reports.py loads every report into one sortable table
(portfolio monitor, entry variants, exit policies, portfolio sim), filters by
report and lookback, and highlights the best row for a chosen metric — max
drawdown correctly ranking lowest-as-best. Stdlib tkinter, no dependencies.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

reports/compare_reports.py

@@ -0,0 +1,281 @@
+"""Compare backtest report JSONs side by side.
+
+Reads every backtest-*.json in this folder and shows one table of runs across
+all of them, so a strategy can be compared across reports (or reports against
+each other) without hand-diffing JSON. Highlights the best row for the chosen
+metric. Stdlib only (tkinter) — run it with any Python 3:
+
+    python reports/compare_reports.py
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import tkinter as tk
+from tkinter import ttk
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+# Each report section exposes the same portfolio metrics; only the key naming
+# the run (and whether it has lookbacks) differs.
+SECTIONS = {
+    "Portfolio monitor (prod strategies)": {
+        "path": ("portfolio_monitor", "runs"),
+        "name_key": "strategy",
+        "lookbacks": True,
+    },
+    "Entry variants": {
+        "path": ("strategy_variants", "variants"),
+        "name_key": "variant",
+        "lookbacks": False,
+    },
+    "Exit policies": {
+        "path": ("exit_policy_variants", "variants"),
+        "name_key": "exit_policy",
+        "lookbacks": False,
+    },
+    "Portfolio sim (policies)": {
+        "path": ("portfolio_sim", "policies"),
+        "name_key": "policy",
+        "lookbacks": False,
+    },
+}
+
+# label, json key, format, higher_is_better (None = not rankable)
+COLUMNS = [
+    ("Report", "_report", "{}", None),
+    ("Run", "_name", "{}", None),
+    ("Lookback", "_lookback", "{}", None),
+    ("CAGR %", "cagr_pct", "{:+.1f}", True),
+    ("Max DD %", "max_drawdown_pct", "-{:.1f}", False),
+    ("Sharpe", "sharpe", "{:.2f}", True),
+    ("Total ret %", "total_return_pct", "{:+.1f}", True),
+    ("SPY %", "spy_return_pct", "{:+.1f}", None),
+    ("Trades", "trades", "{:.0f}", None),
+    ("Win %", "win_rate", "{:.1f}", True),
+    ("Hold d", "avg_hold_days", "{:.1f}", None),
+]
+RANKABLE = [c[0] for c in COLUMNS if c[3] is not None]
+
+
+def load_reports() -> list[dict]:
+    reports = []
+    for path in sorted(glob.glob(os.path.join(HERE, "backtest-*.json"))):
+        try:
+            with open(path, encoding="utf-8") as fh:
+                data = json.load(fh)
+        except (OSError, json.JSONDecodeError) as exc:
+            print(f"skipping {os.path.basename(path)}: {exc}")
+            continue
+        name = os.path.basename(path)[len("backtest-") : -len(".json")]
+        reports.append({
+            "name": name,
+            "data": data,
+            "generated": (data.get("generated_at") or "")[:16].replace("T", " "),
+            "qualified": data.get("qualified"),
+        })
+    return reports
+
+
+def rows_for(report: dict, section: str) -> list[dict]:
+    """Flatten one report's section into rows, tagging report/run/lookback."""
+    cfg = SECTIONS[section]
+    top, inner = cfg["path"]
+    block = report["data"].get(top) or {}
+    raw = block.get(inner) or []
+    rows = []
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        row = dict(item)
+        row["_report"] = report["name"]
+        row["_name"] = item.get(cfg["name_key"]) or "?"
+        row["_lookback"] = item.get("lookback") or "-"
+        row["_production"] = bool(item.get("is_production"))
+        rows.append(row)
+    return rows
+
+
+class App:
+    def __init__(self, root: tk.Tk, reports: list[dict]) -> None:
+        self.root = root
+        self.reports = reports
+        self.rows: list[dict] = []
+        root.title("Backtest report comparison")
+        root.geometry("1250x680")
+
+        controls = ttk.Frame(root, padding=8)
+        controls.pack(fill="x")
+
+        ttk.Label(controls, text="Section").pack(side="left")
+        self.section = ttk.Combobox(
+            controls, values=list(SECTIONS), state="readonly", width=32
+        )
+        self.section.current(0)
+        self.section.pack(side="left", padx=(4, 14))
+        self.section.bind("<<ComboboxSelected>>", lambda _e: self.on_section())
+
+        ttk.Label(controls, text="Lookback").pack(side="left")
+        self.lookback = ttk.Combobox(controls, state="readonly", width=10)
+        self.lookback.pack(side="left", padx=(4, 14))
+        self.lookback.bind("<<ComboboxSelected>>", lambda _e: self.refresh())
+
+        ttk.Label(controls, text="Best by").pack(side="left")
+        self.metric = ttk.Combobox(
+            controls, values=RANKABLE, state="readonly", width=12
+        )
+        self.metric.set("Sharpe")
+        self.metric.pack(side="left", padx=(4, 14))
+        self.metric.bind("<<ComboboxSelected>>", lambda _e: self.refresh())
+
+        body = ttk.Frame(root, padding=(8, 0))
+        body.pack(fill="both", expand=True)
+
+        left = ttk.Frame(body)
+        left.pack(side="left", fill="y", padx=(0, 8))
+        ttk.Label(left, text="Reports (select to filter)").pack(anchor="w")
+        self.report_list = tk.Listbox(
+            left, selectmode="extended", width=34, height=24, exportselection=False
+        )
+        for rep in self.reports:
+            label = f"{rep['name']}  ({rep['generated'][5:]}"
+            label += f", q={rep['qualified']})" if rep["qualified"] else ")"
+            self.report_list.insert("end", label)
+        self.report_list.select_set(0, "end")
+        self.report_list.pack(fill="y", expand=True)
+        self.report_list.bind("<<ListboxSelect>>", lambda _e: self.refresh())
+
+        headers = [c[0] for c in COLUMNS]
+        self.tree = ttk.Treeview(body, columns=headers, show="headings")
+        for label, _key, _fmt, _hib in COLUMNS:
+            self.tree.heading(
+                label, text=label, command=lambda c=label: self.sort_by(c)
+            )
+            width = 190 if label == "Run" else (150 if label == "Report" else 82)
+            self.tree.column(label, width=width, anchor="w" if width > 100 else "e")
+        scroll = ttk.Scrollbar(body, orient="vertical", command=self.tree.yview)
+        self.tree.configure(yscrollcommand=scroll.set)
+        self.tree.pack(side="left", fill="both", expand=True)
+        scroll.pack(side="left", fill="y")
+
+        self.tree.tag_configure("best", background="#c8e6c9")
+        self.tree.tag_configure("prod", font=("TkDefaultFont", 9, "bold"))
+
+        self.status = ttk.Label(root, padding=8, anchor="w")
+        self.status.pack(fill="x")
+
+        self.sort_col: str | None = None
+        self.sort_desc = True
+        self.on_section()
+
+    def selected_reports(self) -> list[dict]:
+        picked = self.report_list.curselection()
+        return [self.reports[i] for i in picked] if picked else self.reports
+
+    def on_section(self) -> None:
+        cfg = SECTIONS[self.section.get()]
+        if cfg["lookbacks"]:
+            seen: list[str] = []
+            for rep in self.reports:
+                for row in rows_for(rep, self.section.get()):
+                    if row["_lookback"] not in seen:
+                        seen.append(row["_lookback"])
+            self.lookback.configure(values=["(all rows)"] + seen, state="readonly")
+            self.lookback.set("all" if "all" in seen else "(all rows)")
+        else:
+            self.lookback.configure(values=["(n/a)"], state="disabled")
+            self.lookback.set("(n/a)")
+        self.sort_col = None
+        self.refresh()
+
+    def refresh(self) -> None:
+        section = self.section.get()
+        rows: list[dict] = []
+        missing: list[str] = []
+        for rep in self.selected_reports():
+            found = rows_for(rep, section)
+            if not found:
+                missing.append(rep["name"])
+            rows.extend(found)
+
+        chosen = self.lookback.get()
+        if SECTIONS[section]["lookbacks"] and chosen not in ("(all rows)", "(n/a)"):
+            rows = [r for r in rows if r["_lookback"] == chosen]
+
+        metric_label = self.metric.get()
+        key, higher = next(
+            (c[1], c[3]) for c in COLUMNS if c[0] == metric_label
+        )
+        ranked = [r for r in rows if isinstance(r.get(key), (int, float))]
+        best = None
+        if ranked:
+            best = (max if higher else min)(ranked, key=lambda r: r[key])
+
+        sort_col = self.sort_col or metric_label
+        sort_key, sort_hib = next(
+            (c[1], c[3]) for c in COLUMNS if c[0] == sort_col
+        )
+        if self.sort_col is None:
+            # Default order: best value first for the chosen metric.
+            reverse = bool(higher)
+        else:
+            reverse = self.sort_desc
+
+        def sort_value(row: dict):
+            val = row.get(sort_key)
+            if isinstance(val, (int, float)):
+                return (1, val, "")
+            return (0, 0.0, str(val or ""))
+
+        rows.sort(key=sort_value, reverse=reverse)
+
+        self.tree.delete(*self.tree.get_children())
+        for row in rows:
+            values = []
+            for label, k, fmt, _hib in COLUMNS:
+                val = row.get(k)
+                if isinstance(val, (int, float)):
+                    values.append(fmt.format(val))
+                else:
+                    values.append("-" if val in (None, "") else str(val))
+            tags = []
+            if best is not None and row is best:
+                tags.append("best")
+            if row.get("_production"):
+                tags.append("prod")
+            self.tree.insert("", "end", values=values, tags=tags)
+
+        self.rows = rows
+        parts = [f"{len(rows)} runs from {len(self.selected_reports())} report(s)"]
+        if best is not None:
+            parts.append(
+                f"best {metric_label}: {best['_name']} "
+                f"({best['_report']}) = {best[key]:.2f}"
+            )
+        if missing:
+            parts.append(f"no '{section}' data in: {', '.join(missing)}")
+        parts.append("bold = production row")
+        self.status.configure(text="   |   ".join(parts))
+
+    def sort_by(self, column: str) -> None:
+        if self.sort_col == column:
+            self.sort_desc = not self.sort_desc
+        else:
+            self.sort_col = column
+            self.sort_desc = True
+        self.refresh()
+
+
+def main() -> None:
+    reports = load_reports()
+    if not reports:
+        raise SystemExit(f"No backtest-*.json found in {HERE}")
+    root = tk.Tk()
+    App(root, reports)
+    root.mainloop()
+
+
+if __name__ == "__main__":
+    main()