"""Compare backtest report JSONs side by side. Reads every backtest-*.json in this folder and shows one table of runs across all of them, so a strategy can be compared across reports (or reports against each other) without hand-diffing JSON. Highlights the best row for the chosen metric. Stdlib only (tkinter) — run it with any Python 3: python reports/compare_reports.py """ from __future__ import annotations import glob import json import os import tkinter as tk from tkinter import ttk HERE = os.path.dirname(os.path.abspath(__file__)) # Each report section exposes the same portfolio metrics; only the key naming # the run (and whether it has lookbacks) differs. SECTIONS = { "Portfolio monitor (prod strategies)": { "path": ("portfolio_monitor", "runs"), "name_key": "strategy", "lookbacks": True, }, "Entry variants": { "path": ("strategy_variants", "variants"), "name_key": "variant", "lookbacks": False, }, "Exit policies": { "path": ("exit_policy_variants", "variants"), "name_key": "exit_policy", "lookbacks": False, }, "Portfolio sim (policies)": { "path": ("portfolio_sim", "policies"), "name_key": "policy", "lookbacks": False, }, } # label, json key, format, higher_is_better (None = not rankable) COLUMNS = [ ("Report", "_report", "{}", None), ("Run", "_name", "{}", None), ("Lookback", "_lookback", "{}", None), ("CAGR %", "cagr_pct", "{:+.1f}", True), ("Max DD %", "max_drawdown_pct", "-{:.1f}", False), ("Sharpe", "sharpe", "{:.2f}", True), ("Total ret %", "total_return_pct", "{:+.1f}", True), ("SPY %", "spy_return_pct", "{:+.1f}", None), ("Trades", "trades", "{:.0f}", None), ("Win %", "win_rate", "{:.1f}", True), ("Hold d", "avg_hold_days", "{:.1f}", None), ] RANKABLE = [c[0] for c in COLUMNS if c[3] is not None] def load_reports() -> list[dict]: reports = [] for path in sorted(glob.glob(os.path.join(HERE, "backtest-*.json"))): try: with open(path, encoding="utf-8") as fh: data = json.load(fh) except (OSError, json.JSONDecodeError) as exc: print(f"skipping {os.path.basename(path)}: {exc}") continue name = os.path.basename(path)[len("backtest-") : -len(".json")] reports.append({ "name": name, "data": data, "generated": (data.get("generated_at") or "")[:16].replace("T", " "), "qualified": data.get("qualified"), }) return reports def rows_for(report: dict, section: str) -> list[dict]: """Flatten one report's section into rows, tagging report/run/lookback.""" cfg = SECTIONS[section] top, inner = cfg["path"] block = report["data"].get(top) or {} raw = block.get(inner) or [] rows = [] for item in raw: if not isinstance(item, dict): continue row = dict(item) row["_report"] = report["name"] row["_name"] = item.get(cfg["name_key"]) or "?" row["_lookback"] = item.get("lookback") or "-" row["_production"] = bool(item.get("is_production")) rows.append(row) return rows class App: def __init__(self, root: tk.Tk, reports: list[dict]) -> None: self.root = root self.reports = reports self.rows: list[dict] = [] root.title("Backtest report comparison") root.geometry("1250x680") controls = ttk.Frame(root, padding=8) controls.pack(fill="x") ttk.Label(controls, text="Section").pack(side="left") self.section = ttk.Combobox( controls, values=list(SECTIONS), state="readonly", width=32 ) self.section.current(0) self.section.pack(side="left", padx=(4, 14)) self.section.bind("<>", lambda _e: self.on_section()) ttk.Label(controls, text="Lookback").pack(side="left") self.lookback = ttk.Combobox(controls, state="readonly", width=10) self.lookback.pack(side="left", padx=(4, 14)) self.lookback.bind("<>", lambda _e: self.refresh()) ttk.Label(controls, text="Best by").pack(side="left") self.metric = ttk.Combobox( controls, values=RANKABLE, state="readonly", width=12 ) self.metric.set("Sharpe") self.metric.pack(side="left", padx=(4, 14)) self.metric.bind("<>", lambda _e: self.refresh()) body = ttk.Frame(root, padding=(8, 0)) body.pack(fill="both", expand=True) left = ttk.Frame(body) left.pack(side="left", fill="y", padx=(0, 8)) ttk.Label(left, text="Reports (select to filter)").pack(anchor="w") self.report_list = tk.Listbox( left, selectmode="extended", width=34, height=24, exportselection=False ) for rep in self.reports: label = f"{rep['name']} ({rep['generated'][5:]}" label += f", q={rep['qualified']})" if rep["qualified"] else ")" self.report_list.insert("end", label) self.report_list.select_set(0, "end") self.report_list.pack(fill="y", expand=True) self.report_list.bind("<>", lambda _e: self.refresh()) headers = [c[0] for c in COLUMNS] self.tree = ttk.Treeview(body, columns=headers, show="headings") for label, _key, _fmt, _hib in COLUMNS: self.tree.heading( label, text=label, command=lambda c=label: self.sort_by(c) ) width = 190 if label == "Run" else (150 if label == "Report" else 82) self.tree.column(label, width=width, anchor="w" if width > 100 else "e") scroll = ttk.Scrollbar(body, orient="vertical", command=self.tree.yview) self.tree.configure(yscrollcommand=scroll.set) self.tree.pack(side="left", fill="both", expand=True) scroll.pack(side="left", fill="y") self.tree.tag_configure("best", background="#c8e6c9") self.tree.tag_configure("prod", font=("TkDefaultFont", 9, "bold")) self.status = ttk.Label(root, padding=8, anchor="w") self.status.pack(fill="x") self.sort_col: str | None = None self.sort_desc = True self.on_section() def selected_reports(self) -> list[dict]: picked = self.report_list.curselection() return [self.reports[i] for i in picked] if picked else self.reports def on_section(self) -> None: cfg = SECTIONS[self.section.get()] if cfg["lookbacks"]: seen: list[str] = [] for rep in self.reports: for row in rows_for(rep, self.section.get()): if row["_lookback"] not in seen: seen.append(row["_lookback"]) self.lookback.configure(values=["(all rows)"] + seen, state="readonly") self.lookback.set("all" if "all" in seen else "(all rows)") else: self.lookback.configure(values=["(n/a)"], state="disabled") self.lookback.set("(n/a)") self.sort_col = None self.refresh() def refresh(self) -> None: section = self.section.get() rows: list[dict] = [] missing: list[str] = [] for rep in self.selected_reports(): found = rows_for(rep, section) if not found: missing.append(rep["name"]) rows.extend(found) chosen = self.lookback.get() if SECTIONS[section]["lookbacks"] and chosen not in ("(all rows)", "(n/a)"): rows = [r for r in rows if r["_lookback"] == chosen] metric_label = self.metric.get() key, higher = next( (c[1], c[3]) for c in COLUMNS if c[0] == metric_label ) ranked = [r for r in rows if isinstance(r.get(key), (int, float))] best = None if ranked: best = (max if higher else min)(ranked, key=lambda r: r[key]) sort_col = self.sort_col or metric_label sort_key, sort_hib = next( (c[1], c[3]) for c in COLUMNS if c[0] == sort_col ) if self.sort_col is None: # Default order: best value first for the chosen metric. reverse = bool(higher) else: reverse = self.sort_desc def sort_value(row: dict): val = row.get(sort_key) if isinstance(val, (int, float)): return (1, val, "") return (0, 0.0, str(val or "")) rows.sort(key=sort_value, reverse=reverse) self.tree.delete(*self.tree.get_children()) for row in rows: values = [] for label, k, fmt, _hib in COLUMNS: val = row.get(k) if isinstance(val, (int, float)): values.append(fmt.format(val)) else: values.append("-" if val in (None, "") else str(val)) tags = [] if best is not None and row is best: tags.append("best") if row.get("_production"): tags.append("prod") self.tree.insert("", "end", values=values, tags=tags) self.rows = rows parts = [f"{len(rows)} runs from {len(self.selected_reports())} report(s)"] if best is not None: parts.append( f"best {metric_label}: {best['_name']} " f"({best['_report']}) = {best[key]:.2f}" ) if missing: parts.append(f"no '{section}' data in: {', '.join(missing)}") parts.append("bold = production row") self.status.configure(text=" | ".join(parts)) def sort_by(self, column: str) -> None: if self.sort_col == column: self.sort_desc = not self.sort_desc else: self.sort_col = column self.sort_desc = True self.refresh() def main() -> None: reports = load_reports() if not reports: raise SystemExit(f"No backtest-*.json found in {HERE}") root = tk.Tk() App(root, reports) root.mainloop() if __name__ == "__main__": main()