signal-platform/reports/compare_reports.py

"""Compare backtest report JSONs side by side.

Reads every backtest-*.json in this folder and shows one table of runs across
all of them, so a strategy can be compared across reports (or reports against
each other) without hand-diffing JSON. Highlights the best row for the chosen
metric. Stdlib only (tkinter) — run it with any Python 3:

    python reports/compare_reports.py
"""

from __future__ import annotations

import glob
import json
import os
import tkinter as tk
from tkinter import ttk

HERE = os.path.dirname(os.path.abspath(__file__))

# Each report section exposes the same portfolio metrics; only the key naming
# the run (and whether it has lookbacks) differs.
SECTIONS = {
    "Portfolio monitor (prod strategies)": {
        "path": ("portfolio_monitor", "runs"),
        "name_key": "strategy",
        "lookbacks": True,
    },
    "Entry variants": {
        "path": ("strategy_variants", "variants"),
        "name_key": "variant",
        "lookbacks": False,
    },
    "Exit policies": {
        "path": ("exit_policy_variants", "variants"),
        "name_key": "exit_policy",
        "lookbacks": False,
    },
    "Portfolio sim (policies)": {
        "path": ("portfolio_sim", "policies"),
        "name_key": "policy",
        "lookbacks": False,
    },
}

# label, json key, format, higher_is_better (None = not rankable)
COLUMNS = [
    ("Report", "_report", "{}", None),
    ("Run", "_name", "{}", None),
    ("Lookback", "_lookback", "{}", None),
    ("CAGR %", "cagr_pct", "{:+.1f}", True),
    ("Max DD %", "max_drawdown_pct", "-{:.1f}", False),
    ("Sharpe", "sharpe", "{:.2f}", True),
    ("Total ret %", "total_return_pct", "{:+.1f}", True),
    ("SPY %", "spy_return_pct", "{:+.1f}", None),
    ("Trades", "trades", "{:.0f}", None),
    ("Win %", "win_rate", "{:.1f}", True),
    ("Hold d", "avg_hold_days", "{:.1f}", None),
]
RANKABLE = [c[0] for c in COLUMNS if c[3] is not None]


def load_reports() -> list[dict]:
    reports = []
    for path in sorted(glob.glob(os.path.join(HERE, "backtest-*.json"))):
        try:
            with open(path, encoding="utf-8") as fh:
                data = json.load(fh)
        except (OSError, json.JSONDecodeError) as exc:
            print(f"skipping {os.path.basename(path)}: {exc}")
            continue
        name = os.path.basename(path)[len("backtest-") : -len(".json")]
        reports.append({
            "name": name,
            "data": data,
            "generated": (data.get("generated_at") or "")[:16].replace("T", " "),
            "qualified": data.get("qualified"),
        })
    return reports


def rows_for(report: dict, section: str) -> list[dict]:
    """Flatten one report's section into rows, tagging report/run/lookback."""
    cfg = SECTIONS[section]
    top, inner = cfg["path"]
    block = report["data"].get(top) or {}
    raw = block.get(inner) or []
    rows = []
    for item in raw:
        if not isinstance(item, dict):
            continue
        row = dict(item)
        row["_report"] = report["name"]
        row["_name"] = item.get(cfg["name_key"]) or "?"
        row["_lookback"] = item.get("lookback") or "-"
        row["_production"] = bool(item.get("is_production"))
        rows.append(row)
    return rows


class App:
    def __init__(self, root: tk.Tk, reports: list[dict]) -> None:
        self.root = root
        self.reports = reports
        self.rows: list[dict] = []
        root.title("Backtest report comparison")
        root.geometry("1250x680")

        controls = ttk.Frame(root, padding=8)
        controls.pack(fill="x")

        ttk.Label(controls, text="Section").pack(side="left")
        self.section = ttk.Combobox(
            controls, values=list(SECTIONS), state="readonly", width=32
        )
        self.section.current(0)
        self.section.pack(side="left", padx=(4, 14))
        self.section.bind("<<ComboboxSelected>>", lambda _e: self.on_section())

        ttk.Label(controls, text="Lookback").pack(side="left")
        self.lookback = ttk.Combobox(controls, state="readonly", width=10)
        self.lookback.pack(side="left", padx=(4, 14))
        self.lookback.bind("<<ComboboxSelected>>", lambda _e: self.refresh())

        ttk.Label(controls, text="Best by").pack(side="left")
        self.metric = ttk.Combobox(
            controls, values=RANKABLE, state="readonly", width=12
        )
        self.metric.set("Sharpe")
        self.metric.pack(side="left", padx=(4, 14))
        self.metric.bind("<<ComboboxSelected>>", lambda _e: self.refresh())

        body = ttk.Frame(root, padding=(8, 0))
        body.pack(fill="both", expand=True)

        left = ttk.Frame(body)
        left.pack(side="left", fill="y", padx=(0, 8))
        ttk.Label(left, text="Reports (select to filter)").pack(anchor="w")
        self.report_list = tk.Listbox(
            left, selectmode="extended", width=34, height=24, exportselection=False
        )
        for rep in self.reports:
            label = f"{rep['name']}  ({rep['generated'][5:]}"
            label += f", q={rep['qualified']})" if rep["qualified"] else ")"
            self.report_list.insert("end", label)
        self.report_list.select_set(0, "end")
        self.report_list.pack(fill="y", expand=True)
        self.report_list.bind("<<ListboxSelect>>", lambda _e: self.refresh())

        headers = [c[0] for c in COLUMNS]
        self.tree = ttk.Treeview(body, columns=headers, show="headings")
        for label, _key, _fmt, _hib in COLUMNS:
            self.tree.heading(
                label, text=label, command=lambda c=label: self.sort_by(c)
            )
            width = 190 if label == "Run" else (150 if label == "Report" else 82)
            self.tree.column(label, width=width, anchor="w" if width > 100 else "e")
        scroll = ttk.Scrollbar(body, orient="vertical", command=self.tree.yview)
        self.tree.configure(yscrollcommand=scroll.set)
        self.tree.pack(side="left", fill="both", expand=True)
        scroll.pack(side="left", fill="y")

        self.tree.tag_configure("best", background="#c8e6c9")
        self.tree.tag_configure("prod", font=("TkDefaultFont", 9, "bold"))

        self.status = ttk.Label(root, padding=8, anchor="w")
        self.status.pack(fill="x")

        self.sort_col: str | None = None
        self.sort_desc = True
        self.on_section()

    def selected_reports(self) -> list[dict]:
        picked = self.report_list.curselection()
        return [self.reports[i] for i in picked] if picked else self.reports

    def on_section(self) -> None:
        cfg = SECTIONS[self.section.get()]
        if cfg["lookbacks"]:
            seen: list[str] = []
            for rep in self.reports:
                for row in rows_for(rep, self.section.get()):
                    if row["_lookback"] not in seen:
                        seen.append(row["_lookback"])
            self.lookback.configure(values=["(all rows)"] + seen, state="readonly")
            self.lookback.set("all" if "all" in seen else "(all rows)")
        else:
            self.lookback.configure(values=["(n/a)"], state="disabled")
            self.lookback.set("(n/a)")
        self.sort_col = None
        self.refresh()

    def refresh(self) -> None:
        section = self.section.get()
        rows: list[dict] = []
        missing: list[str] = []
        for rep in self.selected_reports():
            found = rows_for(rep, section)
            if not found:
                missing.append(rep["name"])
            rows.extend(found)

        chosen = self.lookback.get()
        if SECTIONS[section]["lookbacks"] and chosen not in ("(all rows)", "(n/a)"):
            rows = [r for r in rows if r["_lookback"] == chosen]

        metric_label = self.metric.get()
        key, higher = next(
            (c[1], c[3]) for c in COLUMNS if c[0] == metric_label
        )
        ranked = [r for r in rows if isinstance(r.get(key), (int, float))]
        best = None
        if ranked:
            best = (max if higher else min)(ranked, key=lambda r: r[key])

        sort_col = self.sort_col or metric_label
        sort_key, sort_hib = next(
            (c[1], c[3]) for c in COLUMNS if c[0] == sort_col
        )
        if self.sort_col is None:
            # Default order: best value first for the chosen metric.
            reverse = bool(higher)
        else:
            reverse = self.sort_desc

        def sort_value(row: dict):
            val = row.get(sort_key)
            if isinstance(val, (int, float)):
                return (1, val, "")
            return (0, 0.0, str(val or ""))

        rows.sort(key=sort_value, reverse=reverse)

        self.tree.delete(*self.tree.get_children())
        for row in rows:
            values = []
            for label, k, fmt, _hib in COLUMNS:
                val = row.get(k)
                if isinstance(val, (int, float)):
                    values.append(fmt.format(val))
                else:
                    values.append("-" if val in (None, "") else str(val))
            tags = []
            if best is not None and row is best:
                tags.append("best")
            if row.get("_production"):
                tags.append("prod")
            self.tree.insert("", "end", values=values, tags=tags)

        self.rows = rows
        parts = [f"{len(rows)} runs from {len(self.selected_reports())} report(s)"]
        if best is not None:
            parts.append(
                f"best {metric_label}: {best['_name']} "
                f"({best['_report']}) = {best[key]:.2f}"
            )
        if missing:
            parts.append(f"no '{section}' data in: {', '.join(missing)}")
        parts.append("bold = production row")
        self.status.configure(text="   |   ".join(parts))

    def sort_by(self, column: str) -> None:
        if self.sort_col == column:
            self.sort_desc = not self.sort_desc
        else:
            self.sort_col = column
            self.sort_desc = True
        self.refresh()


def main() -> None:
    reports = load_reports()
    if not reports:
        raise SystemExit(f"No backtest-*.json found in {HERE}")
    root = tk.Tk()
    App(root, reports)
    root.mainloop()


if __name__ == "__main__":
    main()