From a94fece53b2606d2904e6c8c3b896eca0e39ddc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:47:08 +0000 Subject: [PATCH 01/15] Bump astral-sh/setup-uv from 5 to 7 Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 5 to 7. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v5...v7) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ee9a8..2354827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Install locked dependencies @@ -59,7 +59,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Build wheel and sdist @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Export locked third-party dependencies From da07700c3be7e6beb444fbf60facdd802ada7c6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:47:11 +0000 Subject: [PATCH 02/15] Bump actions/dependency-review-action from 4 to 5 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4 to 5. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ee9a8..cc95cc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/dependency-review-action@v4 + - uses: actions/dependency-review-action@v5 codeql: name: CodeQL From 8d46db1ee0c76a0daba366d892982f20bd62ba79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:47:13 +0000 Subject: [PATCH 03/15] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ee9a8..e28c054 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - uses: astral-sh/setup-uv@v5 @@ -75,7 +75,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - uses: astral-sh/setup-uv@v5 From 3ecd6414acc895da78c43ab1386bdbc54fa7a786 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:47:16 +0000 Subject: [PATCH 04/15] Bump actions/checkout from 4 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ee9a8..faa3db8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: matrix: python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-python@v5 @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-python@v5 @@ -72,7 +72,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-python@v5 @@ -99,7 +99,7 @@ jobs: contents: read pull-requests: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/dependency-review-action@v4 @@ -113,7 +113,7 @@ jobs: contents: read security-events: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: github/codeql-action/init@v3 From 2efa11132aa142113052bc34647691e31c56e3db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:47:20 +0000 Subject: [PATCH 05/15] Bump github/codeql-action from 3 to 4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ee9a8..9d8ab02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,9 +116,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: github/codeql-action/init@v3 + - uses: github/codeql-action/init@v4 with: languages: python - - uses: github/codeql-action/analyze@v3 + - uses: github/codeql-action/analyze@v4 with: category: "/language:python" From 3fb300b592a4c4cd8571bec1eae90ded81d97046 Mon Sep 17 00:00:00 2001 From: Dominique Broeglin Date: Sun, 5 Jul 2026 16:16:57 +0200 Subject: [PATCH 06/15] feat: warn on skipped malformed JSON lines in session logs --- src/copilot_experiments/sessionlog.py | 13 +++++++++++++ tests/test_sessionlog.py | 26 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/copilot_experiments/sessionlog.py b/src/copilot_experiments/sessionlog.py index d31ecc2..60b3711 100644 --- a/src/copilot_experiments/sessionlog.py +++ b/src/copilot_experiments/sessionlog.py @@ -8,9 +8,13 @@ from pathlib import Path from typing import Any +from rich.console import Console + from . import pricing from .models import Metrics, ModelMetric, TokenEconomics +err = Console(stderr=True) + def session_state_root(base: Path | None = None) -> Path: """Directory where the Copilot CLI stores per-session state.""" @@ -27,6 +31,7 @@ def load_events(path: Path) -> list[dict[str, Any]]: if not path.exists(): return [] events: list[dict[str, Any]] = [] + skipped_lines = 0 for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: @@ -34,7 +39,15 @@ def load_events(path: Path) -> list[dict[str, Any]]: try: events.append(json.loads(line)) except json.JSONDecodeError: + skipped_lines += 1 continue + + if skipped_lines > 0: + err.print( + f"[yellow]Warning: Skipped {skipped_lines} malformed JSON line(s) " + f"in {path.name}[/yellow]" + ) + return events diff --git a/tests/test_sessionlog.py b/tests/test_sessionlog.py index ea49b1b..a44d7ee 100644 --- a/tests/test_sessionlog.py +++ b/tests/test_sessionlog.py @@ -2,7 +2,9 @@ from __future__ import annotations -from copilot_experiments.sessionlog import parse_metrics +from pathlib import Path + +from copilot_experiments.sessionlog import load_events, parse_metrics def _events(): @@ -61,7 +63,7 @@ def test_parse_metrics_empty(): # --------------------------------------------------------------------------- # -def test_load_events_skips_blank_and_invalid_lines(tmp_path): +def test_load_events_skips_blank_and_invalid_lines(tmp_path, capsys): from copilot_experiments.sessionlog import load_events path = tmp_path / "events.jsonl" @@ -72,6 +74,9 @@ def test_load_events_skips_blank_and_invalid_lines(tmp_path): events = load_events(path) assert [e["type"] for e in events] == ["session.start", "user.message"] + out, err = capsys.readouterr() + assert "Skipped 1 malformed JSON line(s)" in err + def test_load_events_missing_file_returns_empty(tmp_path): from copilot_experiments.sessionlog import load_events @@ -121,3 +126,20 @@ def test_compaction_start_tracks_peak_without_shutdown(): econ = extract_economics(events) assert econ.total_tokens is None # no shutdown -> no authoritative totals assert econ.peak_context_tokens == 9500 # 1000 + 8000 + 500 + + +def test_load_events_with_malformed_lines(tmp_path: Path, capsys): + events_file = tmp_path / "events.jsonl" + events_file.write_text( + '{"type": "session.start"}\n' + '{"type": "malformed"\n' # bad json + '{"type": "session.end"}\n', + encoding="utf-8", + ) + events = load_events(events_file) + assert len(events) == 2 + assert events[0]["type"] == "session.start" + assert events[1]["type"] == "session.end" + + out, err = capsys.readouterr() + assert "Skipped 1 malformed JSON line(s)" in err From c73d23a134f18402cd049d940e1b535e55c6adf1 Mon Sep 17 00:00:00 2001 From: Dominique Broeglin Date: Mon, 6 Jul 2026 08:46:15 +0200 Subject: [PATCH 07/15] Add interactive Plotly HTML result dashboards Render each Pier run into a self-contained summary.html dashboard with KPI cards, a ranked leaderboard, and four interactive charts, inspired by the DeepSWE leaderboard and the Copilot agentic-harness report. Good clean, shareable rendering was the goal, so the file opens straight from disk with no server. - New charts.py module (build_dashboard_html / write_dashboard / plotly_available) behind an optional [charts] extra; degrades gracefully with an install hint when Plotly is absent. - New `chart` CLI command (--last/--out/--cdn/--open); `run` now emits summary.html automatically when Plotly is installed. - Populate the previously stubbed variance/coverage aggregates in pier_results.py (std/cv for AIU and tokens, mean_resolved_rate, resolved_at_k_rate, per-task resolved_rate) and fix a CV key mismatch in report.py. - Offline tests (test_charts.py plus aggregate and CLI tests), docs (visualizing-results.md, results-format.md, ADR 0021), and scaffold template updates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 5 +- docs/adr/0021-interactive-html-dashboards.md | 62 ++ docs/results-format.md | 17 + docs/visualizing-results.md | 82 ++ pyproject.toml | 6 + src/copilot_experiments/charts.py | 712 ++++++++++++++++++ src/copilot_experiments/cli.py | 44 ++ src/copilot_experiments/pier_results.py | 50 +- src/copilot_experiments/report.py | 4 +- .../.apm/skills/analyzing-results/SKILL.md | 2 + .../templates/experiment_repo/AGENTS.md.tmpl | 1 + .../templates/experiment_repo/README.md.tmpl | 2 + .../experiment_repo/pyproject.toml.tmpl | 4 +- tests/test_charts.py | 201 +++++ tests/test_pier_results.py | 86 +++ uv.lock | 31 + 16 files changed, 1292 insertions(+), 17 deletions(-) create mode 100644 docs/adr/0021-interactive-html-dashboards.md create mode 100644 docs/visualizing-results.md create mode 100644 src/copilot_experiments/charts.py create mode 100644 tests/test_charts.py diff --git a/README.md b/README.md index bc2949d..e8078f5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ flowchart LR A --> S["native Copilot\ncopilot-session/**/events.jsonl"] P --> O["jobs///\nPier result.json + trials"] S --> C["Copilot-native analysis\nAIU, tokens, tools, turns"] - O --> R["summary.json / summary.md\nshow / inspect / analyze"] + O --> R["summary.json / summary.md / summary.html\nshow / chart / inspect / analyze"] ``` - **Tasks** are Harbor/Pier task directories: `task.toml`, `instruction.md`, `environment/`, @@ -47,6 +47,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open uv run copilot-experiments analyze --last ``` @@ -106,6 +107,7 @@ uv run copilot-experiments analyze --root examples/tracer_bullet --last | `run --resume` | Resume an existing Pier job directory and skip already-completed matching trials. | | `list` | List Pier job configs and copyable run selectors. | | `show ` / `show --last` | Print a summary for a Pier run (`job` or `job/run`). | +| `chart ` / `chart --last` | Write an interactive `summary.html` dashboard for a Pier run (`--open`, `--out`, `--cdn`). | | `inspect ` | Drill into stored trials by `--agent`, `--task`, and `--trial`. | | `analyze ` / `analyze --last` / `analyze --file ` | Render a rich overview of a selected Copilot session log. | @@ -116,6 +118,7 @@ uv run copilot-experiments analyze --root examples/tracer_bullet --last - [`docs/deepswe.md`](docs/deepswe.md) - importing and running DeepSWE tasks through Pier. - [`docs/collecting-run-data.md`](docs/collecting-run-data.md) - everything to collect around a Copilot CLI run, including native `events.jsonl`, Pier artifacts, ATIF, and OTel. - [`docs/results-format.md`](docs/results-format.md) - Pier `jobs/` layout and derived summaries. +- [`docs/visualizing-results.md`](docs/visualizing-results.md) - interactive `summary.html` dashboards. - [`docs/analysis.md`](docs/analysis.md) - native Copilot session analysis. - [`docs/byok-and-local-models.md`](docs/byok-and-local-models.md) - provider env for Copilot CLI. - [`docs/adr/`](docs/adr) - architecture decision records. diff --git a/docs/adr/0021-interactive-html-dashboards.md b/docs/adr/0021-interactive-html-dashboards.md new file mode 100644 index 0000000..a33e699 --- /dev/null +++ b/docs/adr/0021-interactive-html-dashboards.md @@ -0,0 +1,62 @@ +# 0021. Interactive self-contained HTML result dashboards + +- **Status:** Accepted +- **Date:** 2026-06-20 +- **Deciders:** copilot-experiments maintainers + +## Context + +The tool could render results only as Rich terminal tables (`show`, `analyze`, `inspect`) and a +plain `summary.md`. Users asked for high-quality, clean visual outputs of experiment results, +citing the [DeepSWE leaderboard](https://deepswe.datacurve.ai/) and the +[GitHub Copilot agentic-harness benchmarking report](https://github.blog/ai-and-ml/github-copilot/evaluating-performance-and-efficiency-of-the-github-copilot-agentic-harness-across-models-and-tasks/) +as the target look and feel. Those references communicate three things well: task resolution per +model, token/cost efficiency, and a resolution-vs-cost trade-off with run-to-run variance. + +Constraints from our existing ADRs shaped the options: + +- The filesystem is the source of truth ([0002](0002-filesystem-is-source-of-truth.md)); a + dashboard should be a derived artifact next to `summary.json`, not new authoritative state. +- Analysis data is separate from rendering ([0006](0006-separate-analysis-data-from-rendering.md)); + charts must consume the existing summary, not re-derive metrics. +- We deliberately favored a CLI over a web app for analysis + ([0007](0007-cli-rich-analysis-before-web-app.md)); we did not want to (re)introduce a server. +- Tests stay offline; any dependency must be exercisable without network or Docker. + +Candidate approaches: (a) static PNG/SVG via Matplotlib, (b) a served web app, (c) self-contained +interactive HTML. Static images are easy to embed but not explorable and render poorly across +screens. A served app reverses ADR 0007. Self-contained HTML keeps the "just files" model while +giving crisp, interactive, shareable output. + +## Decision + +We will render each Pier run into a single self-contained `summary.html` dashboard built with +[Plotly](https://plotly.com/python/), added as a new `charts.py` module. + +- Plotly is an **optional** dependency behind a `charts` extra (and the dev group). The dashboard + degrades gracefully: `run` skips it when Plotly is absent, and `chart` exits with an install + hint via a `ChartError`. +- `charts.build_dashboard_html(summary)` consumes the existing `summary.json` shape and returns a + complete HTML string; `charts.write_dashboard(job_dir)` writes `summary.html` beside the other + derived summaries. No new source-of-truth state is introduced. +- A new `chart` CLI command produces the dashboard on demand (`--last`, `--out`, `--cdn`, + `--open`), and `run` emits it automatically when Plotly is available. +- By default plotly.js is embedded for zero-dependency offline viewing; `--cdn` produces a small + file that loads plotly.js from the CDN. +- To feed the charts we populated the previously stubbed variance/coverage aggregates in + `pier_results.py` (`std_*`/`cv_*` for AIU and tokens, `mean_resolved_rate`, + `resolved_at_k_rate`, per-task `resolved_rate`). + +## Consequences + +- Runs now yield a crisp, interactive, shareable dashboard with no server, preserving the + filesystem-first, CLI-first model. It sits alongside `summary.json` / `summary.md` and is + regenerable at any time. +- `summary.json` gained documented spread and suite-coverage fields, which also benefit `show` + and any future consumer; `report.py` key names were corrected to match. +- We accept a large optional dependency. The embedded offline build is ~4.5 MB per file; `--cdn` + is available when size matters. Because the extra is optional, the core tool and its offline + tests never require Plotly (chart tests use the `cdn=True` path or monkeypatch the import to + assert graceful degradation). +- Future chart tweaks live entirely in `charts.py` and read only the summary dict, so rendering + stays decoupled from metric derivation per ADR 0006. diff --git a/docs/results-format.md b/docs/results-format.md index 10b6a45..ae1b91a 100644 --- a/docs/results-format.md +++ b/docs/results-format.md @@ -18,6 +18,7 @@ jobs/ copilot-experiments-run.json summary.json # written by copilot-experiments summary.md # written by copilot-experiments + summary.html # written by copilot-experiments (chart / run) / config.json result.json @@ -52,6 +53,7 @@ download. `copilot-experiments` adds `copilot-experiments-run.json` to preserve | `agent/copilot-otel.jsonl` | Copilot OTel file-exporter output, captured by default for Copilot agent runs unless custom OTel destination settings override it. Useful for per-LLM-call spans with input/output/cache-write/nano-AIU details. | | `verifier/reward.txt` / `.json` | Pier verifier reward. Positive reward means solved. | | `summary.json` / `summary.md` | Derived agent/task aggregate summary. | +| `summary.html` | Interactive Plotly dashboard derived from `summary.json` (see [Visualizing results](visualizing-results.md)). | Pier jobs do not persist per-trial `metrics.json` or `analysis.json` files. Those views are derived from `agent/copilot-session/**/events.jsonl` (or `agent/trajectory.json` as a fallback) @@ -69,6 +71,21 @@ when `show`, `analyze`, or `inspect` runs. - Copilot-native token/AIU/tool metrics when native events are available; - nullable fallback metrics for non-Copilot agents. +### Variance and suite-coverage aggregates + +Each agent entry also carries spread and multi-trial coverage fields, used by the +`summary.html` dashboard: + +| Field | Scope | Meaning | +| --- | --- | --- | +| `std_aiu` / `cv_aiu` | agent, task | Population standard deviation and coefficient of variation (`std / mean`) of AIU cost across trials. `null` when no AIU is available; `cv` is `null` when the mean is zero. | +| `std_total_tokens` / `cv_total_tokens` | agent, task | Same spread measures for total token counts. | +| `mean_resolved_rate` | agent | Macro-average of per-task `success_rate` (each task weighted equally, regardless of trial count). | +| `resolved_at_k_rate` | agent | Fraction of tasks solved at least once across their `k` trials (each task's `resolved_rate` averaged). | +| `resolved` / `resolved_rate` | task | Whether a task was solved at least once (`resolved_rate` is `1.0`/`0.0` per task) and, aggregated per agent, the resolved@k coverage. | + +Single-trial tasks report `std_* = 0.0` and `cv_* = 0.0`; empty inputs report `null`. + ## Analyzing a trial ```bash diff --git a/docs/visualizing-results.md b/docs/visualizing-results.md new file mode 100644 index 0000000..cdbd313 --- /dev/null +++ b/docs/visualizing-results.md @@ -0,0 +1,82 @@ +# Visualizing results + +`copilot-experiments` renders each Pier run into a single, self-contained +`summary.html` dashboard inspired by the [DeepSWE leaderboard](https://deepswe.datacurve.ai/) +and the [GitHub Copilot agentic-harness benchmarking report](https://github.blog/ai-and-ml/github-copilot/evaluating-performance-and-efficiency-of-the-github-copilot-agentic-harness-across-models-and-tasks/). +The dashboard is built with [Plotly](https://plotly.com/python/): the charts are interactive +(hover, zoom, legend toggles) yet the file opens straight from disk with no server. + +## Producing a dashboard + +```bash +# Chart the most recent run and open it in a browser. +uv run copilot-experiments chart --last --open + +# Chart a specific run by selector (from `list`). +uv run copilot-experiments chart +uv run copilot-experiments chart / + +# Write it somewhere else, or use the CDN build (see below). +uv run copilot-experiments chart --last --out report.html +uv run copilot-experiments chart --last --cdn +``` + +`run` also emits `summary.html` automatically after each Pier job when Plotly is installed, so a +freshly finished experiment already has a shareable dashboard next to its `summary.json`. + +Plotly is an **optional** dependency. Install it with the `charts` extra: + +```bash +uv sync --extra charts # or: pip install copilot-experiments[charts] +``` + +When Plotly is missing, `run` silently skips the dashboard (everything else still works) and +`chart` exits with an install hint. + +## What the dashboard shows + +The page is derived entirely from `summary.json` (see +[Results format](results-format.md)), so it stays in sync with `show` and `inspect`: + +- **Header + KPI cards** - job identity, status, and headline numbers (agents, tasks, trials, + overall success rate, total AIU). +- **Leaderboard** - agents ranked by success rate, then by cost, with per-agent color that is + reused across every chart. +- **Task resolution** - success rate per agent; grouped by task when the run has more than one + task. Higher is better. +- **Resolution vs. cost** - success rate (y) against average cost per task (x) with ±1σ cost + error bars. "Up and to the left" (high success, low cost) is best. +- **Cost efficiency** - average cost per task per agent with ±1σ spread. Lower is better. +- **Per-task success** - an agent × task heatmap, shown only when the run has more than one task. + +Cost prefers AIU (`avg_aiu` / `std_aiu`) and falls back to token counts +(`avg_total_tokens` / `std_total_tokens`) when no AIU is available, matching the aggregate fields +documented in [Results format](results-format.md#variance-and-suite-coverage-aggregates). + +## Offline vs. CDN builds + +By default the dashboard **embeds** the full plotly.js bundle (~4.5 MB) so it renders with zero +network access - ideal for archiving a run or emailing it to a colleague. Pass `--cdn` to instead +reference plotly.js from `https://cdn.plot.ly/`, producing a small file (tens of KB) that needs an +internet connection to render. + +| Build | Flag | File size | Needs network to view | +| --- | --- | --- | --- | +| Offline (default) | *(none)* | ~4.5 MB | No | +| CDN | `--cdn` | ~40 KB | Yes | + +## Programmatic use + +The dashboard builder is a small public API: + +```python +from copilot_experiments.charts import build_dashboard_html, write_dashboard, plotly_available +from copilot_experiments.pier_results import build_pier_summary + +summary = build_pier_summary(job_dir) +html = build_dashboard_html(summary, cdn=True) # returns a complete HTML string +write_dashboard(job_dir, summary=summary) # writes /summary.html +``` + +`build_dashboard_html` and `write_dashboard` raise `charts.ChartError` (with an install hint) when +Plotly is unavailable; guard optional call sites with `plotly_available()`. diff --git a/pyproject.toml b/pyproject.toml index 8561107..c892d69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,10 @@ dependencies = [ "rich>=13.7", ] +[project.optional-dependencies] +# Interactive, self-contained HTML result dashboards. +charts = ["plotly>=5.20"] + [project.scripts] copilot-experiments = "copilot_experiments.cli:app" @@ -35,6 +39,8 @@ dev = [ "pytest>=8.0", "pytest-cov>=6.0", "ruff>=0.5", + # Installed so the offline test suite exercises the charts module. + "plotly>=5.20", ] [tool.ruff] diff --git a/src/copilot_experiments/charts.py b/src/copilot_experiments/charts.py new file mode 100644 index 0000000..8f45999 --- /dev/null +++ b/src/copilot_experiments/charts.py @@ -0,0 +1,712 @@ +"""Self-contained interactive HTML dashboards for Pier run summaries. + +Renders a Pier ``summary`` (the dict produced by +:func:`copilot_experiments.pier_results.build_pier_summary`) into a single, +shareable ``summary.html`` file. The look and feel is inspired by the DeepSWE +leaderboard and the GitHub Copilot agentic-harness benchmarking report: a clean +header, KPI cards, a ranked leaderboard, and a small set of interactive +[Plotly](https://plotly.com/python/) charts. + +Plotly is an *optional* dependency (``pip install copilot-experiments[charts]``). +The public helpers raise :class:`ChartError` with an install hint when it is +missing so the rest of the tool keeps working without it. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ._util import write_text + +try: # optional dependency + import plotly.graph_objects as go + import plotly.io as pio + from plotly.offline import get_plotlyjs, get_plotlyjs_version + + _PLOTLY_IMPORT_ERROR: ModuleNotFoundError | None = None +except ModuleNotFoundError as exc: # pragma: no cover - exercised via monkeypatch + go = None # type: ignore[assignment] + pio = None # type: ignore[assignment] + get_plotlyjs = None # type: ignore[assignment] + get_plotlyjs_version = None # type: ignore[assignment] + _PLOTLY_IMPORT_ERROR = exc + + +class ChartError(RuntimeError): + """Raised when a dashboard cannot be produced (e.g. Plotly not installed).""" + + +_INSTALL_HINT = ( + "Plotly is required to render HTML dashboards. Install the charts extra with " + "`pip install copilot-experiments[charts]` (or `uv sync --extra charts`)." +) + +# --------------------------------------------------------------------------- +# Theme +# --------------------------------------------------------------------------- + +_FONT_FAMILY = ( + '-apple-system, BlinkMacSystemFont, "Segoe UI", "Noto Sans", Helvetica, Arial, ' + 'sans-serif, "Apple Color Emoji", "Segoe UI Emoji"' +) +_INK = "#1f2328" +_MUTED = "#656d76" +_GRID = "#eaeef2" +_ZERO = "#d0d7de" + +# Stable, readable qualitative palette (assigned by leaderboard rank). +_PALETTE = [ + "#0969da", # blue + "#1a7f37", # green + "#8250df", # purple + "#bc4c00", # orange + "#bf3989", # pink + "#1b7c83", # teal + "#9a6700", # amber + "#cf222e", # red + "#57606a", # gray + "#6639ba", # indigo +] + +_PLOT_CONFIG = { + "displaylogo": False, + "responsive": True, + "modeBarButtonsToRemove": ["lasso2d", "select2d", "autoScale2d"], + "toImageButtonOptions": {"format": "png", "scale": 2}, +} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def build_dashboard_html( + summary: dict[str, Any], *, title: str | None = None, cdn: bool = False +) -> str: + """Render ``summary`` into a complete, self-contained HTML document. + + Parameters + ---------- + summary: + A Pier run summary (see :func:`pier_results.build_pier_summary`). + title: + Optional page/document title. Defaults to the job name. + cdn: + When ``True`` load plotly.js from the public CDN instead of embedding + the ~4.5 MB bundle, producing a tiny file that needs network access to + render. + """ + + _ensure_plotly() + agents = list(summary.get("agents") or []) + ranked = _rank_agents(agents) + colors = {agent["name"]: _PALETTE[i % len(_PALETTE)] for i, agent in enumerate(ranked)} + n_tasks = int(summary.get("n_tasks") or 0) + + charts_html: list[str] = [] + + resolution = _chart_task_resolution(ranked, colors, n_tasks) + if resolution is not None: + charts_html.append( + _chart_card( + "Task resolution", + "Success rate per agent" + + (" across tasks" if n_tasks > 1 else "") + + ". Higher is better.", + resolution, + ) + ) + + scatter = _chart_resolution_vs_cost(ranked, colors) + if scatter is not None: + charts_html.append( + _chart_card( + "Resolution vs. cost", + "Success rate against average cost per task, with \u00b11\u03c3 cost " + "spread across trials. Up and to the left is better.", + scatter, + ) + ) + + cost = _chart_cost_efficiency(ranked, colors) + if cost is not None: + charts_html.append( + _chart_card( + "Cost efficiency", + "Average cost per task per agent, with \u00b11\u03c3 spread across " + "trials. Lower is better.", + cost, + ) + ) + + if n_tasks > 1: + heatmap = _chart_task_heatmap(ranked) + if heatmap is not None: + charts_html.append( + _chart_card( + "Per-task success", + "Success rate for every agent \u00d7 task cell.", + heatmap, + ) + ) + + body = "\n".join( + [ + _header_html(summary), + _kpi_cards_html(summary), + _leaderboard_html(ranked, colors), + '
' + "\n".join(charts_html) + "
" + if charts_html + else _empty_charts_html(), + _footer_html(), + ] + ) + + head_js = _plotly_head(cdn) + doc_title = title or f"{summary.get('job') or 'Experiment'} \u2014 results" + return _PAGE_SHELL.format(title=_esc(doc_title), css=_CSS, head_js=head_js, body=body) + + +def write_dashboard( + job_dir: Path, + *, + summary: dict[str, Any] | None = None, + out_path: Path | None = None, + title: str | None = None, + cdn: bool = False, +) -> Path: + """Build and write a ``summary.html`` dashboard for a Pier job directory. + + Pass ``summary`` to reuse an already-built summary; otherwise it is rebuilt + from ``job_dir``. Returns the path written. Raises :class:`ChartError` when + Plotly is missing. + """ + + _ensure_plotly() + job_dir = Path(job_dir) + if summary is None: + from .pier_results import build_pier_summary + + summary = build_pier_summary(job_dir) + html = build_dashboard_html(summary, title=title, cdn=cdn) + out = Path(out_path) if out_path is not None else job_dir / "summary.html" + write_text(out, html) + return out + + +def plotly_available() -> bool: + """Return ``True`` when the optional Plotly dependency can be imported.""" + + return _PLOTLY_IMPORT_ERROR is None + + +# --------------------------------------------------------------------------- +# Charts +# --------------------------------------------------------------------------- + + +def _chart_task_resolution( + agents: list[dict[str, Any]], + colors: dict[str, str], + n_tasks: int, +) -> str | None: + if not agents: + return None + fig = go.Figure() + if n_tasks > 1: + # One colored series per agent; grouped by task along the x-axis. + task_order = _task_order(agents) + for agent in agents: + by_task = {task["task"]: task for task in agent.get("tasks") or []} + ys = [_pct_value(by_task.get(slug, {}).get("success_rate")) for slug in task_order] + fig.add_bar( + name=_esc(agent["name"]), + x=[label for _, label in task_order.items()], + y=ys, + marker_color=colors[agent["name"]], + hovertemplate="%{fullData.name}
%{x}: %{y:.0f}%", + ) + fig.update_layout(barmode="group", legend_title_text="Agent") + else: + names = [agent["name"] for agent in agents] + fig.add_bar( + x=names, + y=[_pct_value(agent.get("success_rate")) for agent in agents], + marker_color=[colors[name] for name in names], + hovertemplate="%{x}
%{y:.0f}%", + showlegend=False, + ) + fig.update_yaxes(title_text="Success rate", ticksuffix="%", range=[0, 100]) + _apply_theme(fig) + return _fragment(fig, "chart-resolution", height=430) + + +def _chart_resolution_vs_cost(agents: list[dict[str, Any]], colors: dict[str, str]) -> str | None: + metric, points = _cost_points(agents) + if not points: + return None + fig = go.Figure() + for agent in points: + cost = agent["_cost"] + std = agent.get("_cost_std") + fig.add_trace( + go.Scatter( + x=[cost], + y=[_pct_value(agent.get("success_rate"))], + mode="markers+text", + name=_esc(agent["name"]), + text=[_esc(agent["name"])], + textposition="top center", + textfont={"size": 11, "color": _MUTED}, + marker={ + "size": 15, + "color": colors[agent["name"]], + "line": {"width": 1.5, "color": "#ffffff"}, + "opacity": 0.95, + }, + error_x=( + { + "type": "data", + "array": [std], + "thickness": 1.4, + "width": 6, + "color": colors[agent["name"]], + } + if std + else None + ), + hovertemplate=( + f"%{{text}}
Success: %{{y:.0f}}%
{metric['label']}: " + f"%{{x:{metric['fmt']}}}{metric['suffix']}" + ), + showlegend=False, + ) + ) + fig.update_xaxes(title_text=metric["axis"]) + fig.update_yaxes(title_text="Success rate", ticksuffix="%", range=[-5, 105]) + fig.add_annotation( + xref="paper", + yref="paper", + x=0.01, + y=0.99, + text="\u2196 better", + showarrow=False, + font={"size": 12, "color": _MUTED}, + align="left", + ) + _apply_theme(fig) + return _fragment(fig, "chart-scatter", height=460) + + +def _chart_cost_efficiency(agents: list[dict[str, Any]], colors: dict[str, str]) -> str | None: + metric, points = _cost_points(agents) + if not points: + return None + names = [agent["name"] for agent in points] + fig = go.Figure( + go.Bar( + x=names, + y=[agent["_cost"] for agent in points], + marker_color=[colors[name] for name in names], + error_y={ + "type": "data", + "array": [agent.get("_cost_std") or 0 for agent in points], + "thickness": 1.4, + "width": 6, + "color": _MUTED, + }, + hovertemplate=f"%{{x}}
%{{y:{metric['fmt']}}}{metric['suffix']}", + showlegend=False, + ) + ) + fig.update_yaxes(title_text=metric["axis"], rangemode="tozero") + _apply_theme(fig) + return _fragment(fig, "chart-cost", height=430) + + +def _chart_task_heatmap(agents: list[dict[str, Any]]) -> str | None: + task_order = _task_order(agents) + if not task_order: + return None + x_labels = list(task_order.values()) + y_labels = [agent["name"] for agent in agents] + z: list[list[float | None]] = [] + text: list[list[str]] = [] + for agent in agents: + by_task = {task["task"]: task for task in agent.get("tasks") or []} + row_z: list[float | None] = [] + row_t: list[str] = [] + for slug in task_order: + rate = by_task.get(slug, {}).get("success_rate") + row_z.append(None if rate is None else round(rate * 100, 1)) + row_t.append("\u2014" if rate is None else f"{rate * 100:.0f}%") + z.append(row_z) + text.append(row_t) + fig = go.Figure( + go.Heatmap( + z=z, + x=x_labels, + y=y_labels, + text=text, + texttemplate="%{text}", + textfont={"size": 12}, + zmin=0, + zmax=100, + colorscale=[[0, "#fbeaec"], [0.5, "#ffe8b3"], [1.0, "#c3e6cd"]], + colorbar={"title": {"text": "%"}, "ticksuffix": "%", "thickness": 12}, + hovertemplate="%{y}
%{x}: %{z:.0f}%", + xgap=3, + ygap=3, + ) + ) + fig.update_yaxes(autorange="reversed") + _apply_theme(fig) + height = max(320, 120 + 46 * len(y_labels)) + return _fragment(fig, "chart-heatmap", height=height) + + +# --------------------------------------------------------------------------- +# HTML pieces +# --------------------------------------------------------------------------- + + +def _header_html(summary: dict[str, Any]) -> str: + chips = [] + run_id = summary.get("run_id") + if run_id: + chips.append(f'{_esc(str(run_id))}') + status = summary.get("status") + if status: + chips.append(f'{_esc(str(status))}') + started = (summary.get("started_at") or "")[:19].replace("T", " ") + if started: + chips.append(f'started {_esc(started)}') + finished = (summary.get("finished_at") or "")[:19].replace("T", " ") + if finished: + chips.append(f'finished {_esc(finished)}') + selector = summary.get("pier_job_id") + subtitle = f'
{_esc(str(selector))}
' if selector else "" + return ( + '
' + f'
Experiment results
' + f"

{_esc(str(summary.get('job') or 'Experiment'))}

" + f"{subtitle}" + f'
{"".join(chips)}
' + "
" + ) + + +def _kpi_cards_html(summary: dict[str, Any]) -> str: + cards = [ + ("Overall success", _pct(summary.get("overall_success_rate")), "resolved trials"), + ("Total cost", _aiu(summary.get("total_aiu")), "AIU"), + ("Agents", _int(summary.get("n_agents")), "compared"), + ("Tasks", _int(summary.get("n_tasks")), "in suite"), + ("Trials", _int(summary.get("n_trials")), "executed"), + ] + failed = int(summary.get("n_failed_trials") or 0) + if failed: + cards.append(("Harness failures", str(failed), "did not run cleanly")) + items = "".join( + f'
' + f'
{value}
' + f'
{_esc(label)}
' + f'
{_esc(note)}
' + "
" + for label, value, note in cards + ) + return f'
{items}
' + + +def _leaderboard_html(agents: list[dict[str, Any]], colors: dict[str, str]) -> str: + if not agents: + return "" + rows = [] + for rank, agent in enumerate(agents, start=1): + name = agent["name"] + rows.append( + "" + f'{rank}' + f'' + f'{_esc(name)}' + f'{_esc(str(agent.get("model") or "\u2014"))}' + f'{_esc(str(agent.get("reasoning_effort") or "\u2014"))}' + f'{_pct(agent.get("success_rate"))}' + f'{_pct(agent.get("resolved_at_k_rate"))}' + f'{_aiu(agent.get("avg_aiu"))}' + f'{_aiu(agent.get("aiu_per_solve"))}' + f'{_int(agent.get("avg_total_tokens"))}' + f'{_int(agent.get("n_trials"))}' + "" + ) + return ( + '
' + '

Leaderboard

' + '
Ranked by success rate, then cost.
' + '
' + "" + '' + '' + '' + '' + "" + f"{''.join(rows)}" + "
#AgentModelEffortSuccessResolved@kAvg AIUAIU / solveAvg tokensTrials
" + ) + + +def _chart_card(title: str, subtitle: str, fragment: str) -> str: + return ( + '
' + f'

{_esc(title)}

' + f'
{subtitle}
' + f'
{fragment}
' + "
" + ) + + +def _empty_charts_html() -> str: + return ( + '

Charts

' + '
No agent metrics were available to plot.
' + "
" + ) + + +def _footer_html() -> str: + return ( + '
Generated by ' + 'copilot-experiments. Charts are interactive \u2014 ' + "hover for details, drag to zoom, double-click to reset.
" + ) + + +# --------------------------------------------------------------------------- +# Data helpers +# --------------------------------------------------------------------------- + + +def _rank_agents(agents: list[dict[str, Any]]) -> list[dict[str, Any]]: + def key(agent: dict[str, Any]) -> tuple[Any, ...]: + success = agent.get("success_rate") + cost = agent.get("avg_aiu") + if cost is None: + cost = agent.get("avg_total_tokens") + return ( + 0 if success is not None else 1, + -(success or 0.0), + 0 if cost is not None else 1, + cost if cost is not None else 0.0, + str(agent.get("name") or ""), + ) + + return sorted(agents, key=key) + + +def _cost_points(agents: list[dict[str, Any]]) -> tuple[dict[str, str], list[dict[str, Any]]]: + """Attach a cost value/std to each agent, preferring AIU then tokens.""" + + use_aiu = any(agent.get("avg_aiu") is not None for agent in agents) + metric = ( + {"label": "Avg AIU", "axis": "Average cost per task (AIU)", "fmt": ".3f", "suffix": ""} + if use_aiu + else {"label": "Avg tokens", "axis": "Average tokens per task", "fmt": ",.0f", "suffix": ""} + ) + points = [] + for agent in agents: + cost = agent.get("avg_aiu") if use_aiu else agent.get("avg_total_tokens") + if cost is None: + continue + std = agent.get("std_aiu") if use_aiu else agent.get("std_total_tokens") + points.append({**agent, "_cost": cost, "_cost_std": std}) + return metric, points + + +def _task_order(agents: list[dict[str, Any]]) -> dict[str, str]: + """Ordered mapping of task slug -> display label across all agents.""" + + order: dict[str, str] = {} + for agent in agents: + for task in agent.get("tasks") or []: + slug = task.get("task") + if slug and slug not in order: + order[slug] = str(task.get("name") or slug) + return order + + +def _pct_value(rate: float | None) -> float | None: + return None if rate is None else round(rate * 100, 1) + + +# --------------------------------------------------------------------------- +# Rendering helpers +# --------------------------------------------------------------------------- + + +def _ensure_plotly() -> None: + if _PLOTLY_IMPORT_ERROR is not None: + raise ChartError(_INSTALL_HINT) from _PLOTLY_IMPORT_ERROR + + +def _apply_theme(fig: Any) -> None: + fig.update_layout( + template="plotly_white", + font={"family": _FONT_FAMILY, "size": 13, "color": _INK}, + margin={"l": 64, "r": 24, "t": 16, "b": 56}, + paper_bgcolor="rgba(0,0,0,0)", + plot_bgcolor="rgba(0,0,0,0)", + colorway=_PALETTE, + hoverlabel={"font": {"family": _FONT_FAMILY, "size": 12}}, + legend={ + "orientation": "h", + "yanchor": "bottom", + "y": 1.02, + "xanchor": "left", + "x": 0, + "font": {"size": 12}, + }, + bargap=0.28, + bargroupgap=0.12, + ) + fig.update_xaxes( + showgrid=False, + zeroline=False, + linecolor=_ZERO, + ticks="outside", + tickcolor=_ZERO, + tickfont={"color": _MUTED}, + title_font={"size": 12, "color": _MUTED}, + automargin=True, + ) + fig.update_yaxes( + gridcolor=_GRID, + zeroline=True, + zerolinecolor=_ZERO, + linecolor="rgba(0,0,0,0)", + tickfont={"color": _MUTED}, + title_font={"size": 12, "color": _MUTED}, + automargin=True, + ) + + +def _plotly_head(cdn: bool) -> str: + """Return the ```` snippet that makes plotly.js available.""" + + if cdn: + src = f"https://cdn.plot.ly/plotly-{get_plotlyjs_version()}.min.js" + return f'' + return f"" + + +def _fragment(fig: Any, div_id: str, *, height: int) -> str: + return pio.to_html( + fig, + full_html=False, + include_plotlyjs=False, + div_id=div_id, + default_width="100%", + default_height=f"{height}px", + config=_PLOT_CONFIG, + ) + + +def _esc(value: str) -> str: + return ( + str(value) + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def _pct(value: float | None) -> str: + return "\u2014" if value is None else f"{value * 100:.0f}%" + + +def _aiu(value: float | None) -> str: + if value is None: + return "\u2014" + value = float(value) + return f"{value:.3f}" if value < 1 else f"{value:,.2f}" + + +def _int(value: float | None) -> str: + if value is None: + return "\u2014" + return f"{float(value):,.0f}" + + +# --------------------------------------------------------------------------- +# Page shell +# --------------------------------------------------------------------------- + +_CSS = """ +*{box-sizing:border-box} +html{-webkit-text-size-adjust:100%} +body{margin:0;background:#f6f8fa;color:#1f2328;line-height:1.5; + font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif; + -webkit-font-smoothing:antialiased;text-rendering:optimizeLegibility} +.mono{font-family:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,"Liberation Mono",monospace} +.muted{color:#656d76} +.wrap{max-width:1120px;margin:0 auto;padding:36px 24px 72px} +.header{margin-bottom:24px} +.eyebrow{font-size:12px;font-weight:600;letter-spacing:.06em;text-transform:uppercase;color:#8250df} +h1{margin:6px 0 4px;font-size:30px;line-height:1.2;letter-spacing:-.01em} +.subtitle{color:#656d76;font-size:13px} +.chips{display:flex;flex-wrap:wrap;gap:8px;margin-top:14px} +.chip{display:inline-flex;align-items:center;gap:6px;padding:3px 10px;border-radius:999px; + background:#fff;border:1px solid #d0d7de;font-size:12px;color:#424a53} +.chip.status-completed{background:#dafbe1;border-color:#aceebb;color:#1a7f37} +.chip.status-failed,.chip.status-error{background:#ffebe9;border-color:#ffcecb;color:#cf222e} +.chip.status-running,.chip.status-partial{background:#fff8c5;border-color:#eac54f;color:#7d4e00} +.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:14px;margin-bottom:22px} +.kpi{background:#fff;border:1px solid #d0d7de;border-radius:12px;padding:16px 18px; + box-shadow:0 1px 2px rgba(31,35,40,.06)} +.kpi-value{font-size:26px;font-weight:650;letter-spacing:-.01em;line-height:1.1} +.kpi-label{margin-top:4px;font-size:13px;font-weight:600;color:#424a53} +.kpi-note{font-size:12px;color:#8b949e} +.kpi-warn{border-color:#eac54f;background:#fffbdd} +.kpi-warn .kpi-value{color:#7d4e00} +.panel{background:#fff;border:1px solid #d0d7de;border-radius:14px;margin-bottom:22px; + box-shadow:0 1px 2px rgba(31,35,40,.06);overflow:hidden} +.panel-head{padding:18px 20px 0} +.panel-head h2{margin:0;font-size:16px;letter-spacing:-.005em} +.panel-sub{margin-top:3px;color:#656d76;font-size:13px} +.chart-body{padding:8px 12px 14px} +.table-wrap{overflow-x:auto;padding:12px 8px 8px} +table.board{width:100%;border-collapse:collapse;font-size:13px} +table.board th,table.board td{padding:10px 12px;text-align:left;white-space:nowrap} +table.board thead th{font-size:11px;font-weight:600;letter-spacing:.04em;text-transform:uppercase; + color:#656d76;border-bottom:1px solid #d0d7de} +table.board tbody tr{border-bottom:1px solid #eaeef2} +table.board tbody tr:last-child{border-bottom:0} +table.board tbody tr:hover{background:#f6f8fa} +table.board td.num,table.board th.num{text-align:right;font-variant-numeric:tabular-nums} +table.board td.rank,table.board th.rank{text-align:center;color:#8b949e;width:34px} +table.board td.strong{font-weight:650} +.agent-name{font-weight:600} +.dot{display:inline-block;width:9px;height:9px;border-radius:50%;margin-right:8px;vertical-align:middle} +.charts{display:block} +.footer{color:#8b949e;font-size:12px;text-align:center;margin-top:8px} +@media (max-width:640px){h1{font-size:24px}.wrap{padding:24px 16px 56px}} +""" + +_PAGE_SHELL = """ + + + + +{title} + +{head_js} + + +
+{body} +
+ + +""" diff --git a/src/copilot_experiments/cli.py b/src/copilot_experiments/cli.py index 5640c5f..2e37a7d 100644 --- a/src/copilot_experiments/cli.py +++ b/src/copilot_experiments/cli.py @@ -13,6 +13,7 @@ from ._util import read_json from .analysis import analyze_events, analyze_trajectory from .auth import AuthError, preflight_github_token +from .charts import ChartError, plotly_available, write_dashboard from .deepswe import DeepSweImportError, write_deepswe_job_config from .pier_backend import ( PierBackendPreflightError, @@ -273,6 +274,12 @@ def run( summary = write_pier_summary(run_result.job_dir) _print_run_summary(summary) _warn_failed_pier_trials(run_result.job_dir) + if plotly_available(): + try: + html_path = write_dashboard(run_result.job_dir, summary=summary) + console.print(f"[dim]chart:[/dim] {html_path}") + except ChartError: + pass if summary.get("status") != "completed": any_failures = True console.print(f"[dim]results:[/dim] {run_result.job_dir}\n") @@ -342,6 +349,43 @@ def show( console.print(f"\n[dim]{resolved.path / 'summary.md'}[/dim]") +@app.command() +def chart( + selector: str | None = typer.Argument( + None, + help="Pier run selector from `list`: job, run id/prefix, or job/run.", + ), + last: bool = typer.Option(False, "--last", help="Chart the most recent stored Pier run."), + out: Path | None = typer.Option( + None, "--out", help="Write the dashboard here (default: /summary.html)." + ), + cdn: bool = typer.Option( + False, + "--cdn", + help="Load plotly.js from its CDN (tiny file, but needs network to view).", + ), + open_browser: bool = typer.Option( + False, "--open", help="Open the dashboard in your web browser when done." + ), + root: Path | None = typer.Option(None, "--root", help="Experiment repository root."), +) -> None: + """Render an interactive HTML dashboard (summary.html) for a Pier run.""" + + resolved = _resolve_or_exit(root, selector, last=last) + summary = write_pier_summary(resolved.path) + try: + out_path = write_dashboard(resolved.path, summary=summary, out_path=out, cdn=cdn) + except ChartError as exc: + err.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + console.print(f"[green]Wrote[/green] {out_path}") + if open_browser: + import webbrowser + + webbrowser.open(out_path.resolve().as_uri()) + + @app.command() def inspect( selector: str | None = typer.Argument( diff --git a/src/copilot_experiments/pier_results.py b/src/copilot_experiments/pier_results.py index 3bbd5da..9382ec9 100644 --- a/src/copilot_experiments/pier_results.py +++ b/src/copilot_experiments/pier_results.py @@ -2,6 +2,7 @@ from __future__ import annotations +import statistics from collections import defaultdict from datetime import datetime from pathlib import Path @@ -70,7 +71,7 @@ def build_pier_summary(job_dir: Path) -> dict[str, Any]: task_summaries.append(_aggregate_task(task_slug, trials)) cell["tasks"] = task_summaries cell["n_tasks"] = len(task_summaries) - cell.update(_aggregate_agent(all_trials)) + cell.update(_aggregate_agent(all_trials, task_summaries)) agents.append(cell) all_trials = [trial for agent in agents for task in agent["tasks"] for trial in task["_trials"]] @@ -313,49 +314,55 @@ def _job_status(job_result: dict[str, Any]) -> str: def _aggregate_task(task_slug: str, trials: list[dict[str, Any]]) -> dict[str, Any]: graded = [trial["success"] for trial in trials if trial.get("success") is not None] aiu_values = [(trial.get("metrics") or {}).get("aiu") for trial in trials] + token_values = _metric_values(trials, "total_tokens") total_aiu = sum(value for value in aiu_values if value is not None) solved = sum(1 for value in graded if value) + resolved = None if not graded else int(any(graded)) return { "task": task_slug, "name": trials[0].get("task_name") or task_slug, "n_trials": len(trials), "success_rate": (solved / len(graded)) if graded else None, - "resolved": None if not graded else int(any(graded)), + "resolved": resolved, + "resolved_rate": None if resolved is None else float(resolved), "avg_duration_s": _avg([trial.get("duration_s") for trial in trials]), "avg_turns": _avg(_metric_values(trials, "n_turns")), - "avg_total_tokens": _avg(_metric_values(trials, "total_tokens")), - "cv_total_tokens": None, + "avg_total_tokens": _avg(token_values), + "cv_total_tokens": _cv(token_values), "avg_aiu": _avg(aiu_values), - "cv_aiu": None, + "cv_aiu": _cv(aiu_values), "total_aiu": round(total_aiu, 3) if total_aiu else None, "aiu_per_solve": round(total_aiu / solved, 3) if total_aiu and solved else None, "_trials": trials, } -def _aggregate_agent(trials: list[dict[str, Any]]) -> dict[str, Any]: +def _aggregate_agent( + trials: list[dict[str, Any]], task_summaries: list[dict[str, Any]] +) -> dict[str, Any]: graded = [trial["success"] for trial in trials if trial.get("success") is not None] solved = sum(1 for value in graded if value) aiu_values = [(trial.get("metrics") or {}).get("aiu") for trial in trials] + token_values = _metric_values(trials, "total_tokens") total_aiu = sum(value for value in aiu_values if value is not None) return { "success_rate": (solved / len(graded)) if graded else None, - "mean_resolved_rate": None, - "resolved_at_k_rate": None, + "mean_resolved_rate": _avg([task.get("success_rate") for task in task_summaries]), + "resolved_at_k_rate": _avg([task.get("resolved_rate") for task in task_summaries]), "avg_duration_s": _avg([trial.get("duration_s") for trial in trials]), "avg_turns": _avg(_metric_values(trials, "n_turns")), "avg_tool_calls": _avg(_metric_values(trials, "n_tool_calls")), "avg_tool_failures": _avg(_metric_values(trials, "n_tool_failures")), - "avg_total_tokens": _avg(_metric_values(trials, "total_tokens")), - "std_total_tokens": None, - "cv_total_tokens": None, + "avg_total_tokens": _avg(token_values), + "std_total_tokens": _std(token_values), + "cv_total_tokens": _cv(token_values), "avg_input_tokens": _avg(_metric_values(trials, "input_tokens")), "avg_output_tokens": _avg(_metric_values(trials, "output_tokens")), "avg_cache_read_tokens": _avg(_metric_values(trials, "cache_read_tokens")), "avg_reasoning_tokens": _avg(_metric_values(trials, "reasoning_tokens")), "avg_aiu": _avg(aiu_values), - "std_aiu": None, - "cv_aiu": None, + "std_aiu": _std(aiu_values), + "cv_aiu": _cv(aiu_values), "total_aiu": round(total_aiu, 3) if total_aiu else None, "aiu_per_solve": round(total_aiu / solved, 3) if total_aiu and solved else None, "avg_lines_added": _avg(_metric_values(trials, "lines_added")), @@ -384,6 +391,23 @@ def _avg(values: list[Any]) -> float | None: return round(sum(nums) / len(nums), 3) if nums else None +def _std(values: list[Any]) -> float | None: + """Population standard deviation across a cell's trials (0.0 for a single point).""" + nums = [float(value) for value in values if value is not None] + return round(statistics.pstdev(nums), 3) if nums else None + + +def _cv(values: list[Any]) -> float | None: + """Coefficient of variation (std / mean); ``None`` when undefined.""" + nums = [float(value) for value in values if value is not None] + if not nums: + return None + mean = sum(nums) / len(nums) + if mean == 0: + return None + return round(statistics.pstdev(nums) / mean, 3) + + def _metric_values(trials: list[dict[str, Any]], name: str) -> list[Any]: return [(trial.get("metrics") or {}).get(name) for trial in trials] diff --git a/src/copilot_experiments/report.py b/src/copilot_experiments/report.py index 445fa6c..578455f 100644 --- a/src/copilot_experiments/report.py +++ b/src/copilot_experiments/report.py @@ -84,10 +84,10 @@ def summary_markdown(summary: dict, description: str = "") -> str: "{api} |".format( name=agent["name"], avg_aiu=_aiu(agent.get("avg_aiu")), - aiu_cv=_fmt(agent.get("aiu_cv")), + aiu_cv=_fmt(agent.get("cv_aiu")), aiu_solve=_aiu(agent.get("aiu_per_solve")), tokens=_fmt(agent.get("avg_total_tokens")), - token_cv=_fmt(agent.get("total_tokens_cv")), + token_cv=_fmt(agent.get("cv_total_tokens")), api=_fmt(agent.get("avg_api_duration_s")), ) ) diff --git a/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md b/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md index 3f5a3c3..ad351f9 100644 --- a/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md +++ b/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md @@ -16,6 +16,7 @@ jobs/// copilot-experiments-run.json summary.json # derived copilot-experiments summary summary.md # human-readable report + summary.html # interactive Plotly dashboard (chart / run) / result.json # Pier trial result agent/ @@ -32,6 +33,7 @@ tokens, and AIU economics. ```bash copilot-experiments list # runs + success rates copilot-experiments show --last # per-agent comparison table +copilot-experiments chart --last --open # interactive summary.html dashboard copilot-experiments inspect # latest run for that Pier job copilot-experiments inspect / # exact run selector from list copilot-experiments analyze / --agent --trial diff --git a/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl b/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl index d3a27fb..5e53b89 100644 --- a/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl +++ b/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl @@ -37,6 +37,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open uv run copilot-experiments inspect / --agent --trial uv run copilot-experiments analyze / --agent --trial ``` diff --git a/src/copilot_experiments/templates/experiment_repo/README.md.tmpl b/src/copilot_experiments/templates/experiment_repo/README.md.tmpl index 45b18ec..61c12d1 100644 --- a/src/copilot_experiments/templates/experiment_repo/README.md.tmpl +++ b/src/copilot_experiments/templates/experiment_repo/README.md.tmpl @@ -28,6 +28,7 @@ uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments validate uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments run uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments list uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments show --last +uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments chart --last --open # explore results uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments list @@ -57,6 +58,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open # explore results uv run copilot-experiments list diff --git a/src/copilot_experiments/templates/experiment_repo/pyproject.toml.tmpl b/src/copilot_experiments/templates/experiment_repo/pyproject.toml.tmpl index 2fa2bfd..17ed1b3 100644 --- a/src/copilot_experiments/templates/experiment_repo/pyproject.toml.tmpl +++ b/src/copilot_experiments/templates/experiment_repo/pyproject.toml.tmpl @@ -10,7 +10,9 @@ dependencies = [ # To use an unpublished local checkout without syncing this repo, run: # uvx --from /path/to/github-copilot-lab copilot-experiments ... # Add --no-cache before --from when iterating on the harness working tree. - "copilot-experiments @ git+https://github.com/dbroeglin/github-copilot-lab.git", + # The [charts] extra pulls in Plotly for the interactive `chart` dashboards; + # drop it if you only need the terminal reports. + "copilot-experiments[charts] @ git+https://github.com/dbroeglin/github-copilot-lab.git", ] [tool.copilot-experiments] diff --git a/tests/test_charts.py b/tests/test_charts.py new file mode 100644 index 0000000..13cad35 --- /dev/null +++ b/tests/test_charts.py @@ -0,0 +1,201 @@ +"""Tests for the interactive HTML result dashboards.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from copilot_experiments import charts +from copilot_experiments.charts import ( + ChartError, + build_dashboard_html, + plotly_available, + write_dashboard, +) + +_MULTI_TASK_SUMMARY = { + "run_id": "20260612T103300Z_a1b2c3", + "job": "strtools-vs-csvtools", + "pier_job_id": "strtools/20260612T103300Z_a1b2c3", + "status": "completed", + "started_at": "2026-06-12T10:33:00Z", + "finished_at": "2026-06-12T10:52:11Z", + "n_agents": 2, + "n_tasks": 2, + "n_trials": 12, + "n_failed_trials": 0, + "overall_success_rate": 0.75, + "total_aiu": 3.21, + "agents": [ + { + "name": "sonnet-4.6", + "model": "claude-sonnet-4.6", + "reasoning_effort": "medium", + "n_tasks": 2, + "n_trials": 6, + "success_rate": 0.83, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.21, + "std_aiu": 0.04, + "aiu_per_solve": 0.25, + "avg_total_tokens": 120000, + "std_total_tokens": 15000, + "tasks": [ + {"task": "strtools", "name": "strtools", "success_rate": 1.0, "resolved_rate": 1.0}, + { + "task": "csvtools", + "name": "csvtools", + "success_rate": 0.66, + "resolved_rate": 1.0, + }, + ], + }, + { + "name": "haiku-4.5", + "model": "claude-haiku-4.5", + "reasoning_effort": None, + "n_tasks": 2, + "n_trials": 6, + "success_rate": 0.5, + "resolved_at_k_rate": 0.5, + "avg_aiu": 0.08, + "std_aiu": 0.02, + "aiu_per_solve": 0.16, + "avg_total_tokens": 90000, + "std_total_tokens": 8000, + "tasks": [ + {"task": "strtools", "name": "strtools", "success_rate": 1.0, "resolved_rate": 1.0}, + {"task": "csvtools", "name": "csvtools", "success_rate": 0.0, "resolved_rate": 0.0}, + ], + }, + ], +} + +_SINGLE_TASK_SUMMARY = { + "run_id": "run-1", + "job": "textstats", + "pier_job_id": "textstats/run-1", + "status": "completed", + "n_agents": 2, + "n_tasks": 1, + "n_trials": 4, + "overall_success_rate": 0.75, + "total_aiu": 0.9, + "agents": [ + { + "name": "agent-a", + "model": "model-a", + "reasoning_effort": "high", + "n_tasks": 1, + "n_trials": 2, + "success_rate": 1.0, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.3, + "std_aiu": 0.05, + "aiu_per_solve": 0.3, + "avg_total_tokens": 150000, + "tasks": [ + { + "task": "textstats", + "name": "textstats", + "success_rate": 1.0, + "resolved_rate": 1.0, + } + ], + }, + { + "name": "agent-b", + "model": "model-b", + "reasoning_effort": "low", + "n_tasks": 1, + "n_trials": 2, + "success_rate": 0.5, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.15, + "std_aiu": 0.02, + "aiu_per_solve": 0.3, + "avg_total_tokens": 80000, + "tasks": [ + { + "task": "textstats", + "name": "textstats", + "success_rate": 0.5, + "resolved_rate": 1.0, + } + ], + }, + ], +} + + +def test_build_dashboard_html_has_all_sections(): + html = build_dashboard_html(_MULTI_TASK_SUMMARY, cdn=True) + + assert html.startswith("") + assert "Plotly.newPlot" in html + for div_id in ("chart-resolution", "chart-scatter", "chart-cost", "chart-heatmap"): + assert div_id in html + assert "Leaderboard" in html + assert "strtools-vs-csvtools" in html # job title + assert "claude-sonnet-4.6" in html # leaderboard model column + + +def test_build_dashboard_cdn_is_small_and_references_cdn(): + html = build_dashboard_html(_MULTI_TASK_SUMMARY, cdn=True) + + assert "https://cdn.plot.ly/plotly-" in html + assert "