diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e213e44..8e8f5df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,13 +28,13 @@ jobs: matrix: python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Install locked dependencies @@ -53,13 +53,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Build wheel and sdist @@ -72,13 +72,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - name: Export locked third-party dependencies @@ -99,10 +99,10 @@ jobs: contents: read pull-requests: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: actions/dependency-review-action@v4 + - uses: actions/dependency-review-action@v5 codeql: name: CodeQL @@ -113,12 +113,12 @@ jobs: contents: read security-events: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: github/codeql-action/init@v3 + - uses: github/codeql-action/init@v4 with: languages: python - - uses: github/codeql-action/analyze@v3 + - uses: github/codeql-action/analyze@v4 with: category: "/language:python" diff --git a/README.md b/README.md index bc2949d..e8078f5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ flowchart LR A --> S["native Copilot\ncopilot-session/**/events.jsonl"] P --> O["jobs///\nPier result.json + trials"] S --> C["Copilot-native analysis\nAIU, tokens, tools, turns"] - O --> R["summary.json / summary.md\nshow / inspect / analyze"] + O --> R["summary.json / summary.md / summary.html\nshow / chart / inspect / analyze"] ``` - **Tasks** are Harbor/Pier task directories: `task.toml`, `instruction.md`, `environment/`, @@ -47,6 +47,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open uv run copilot-experiments analyze --last ``` @@ -106,6 +107,7 @@ uv run copilot-experiments analyze --root examples/tracer_bullet --last | `run --resume` | Resume an existing Pier job directory and skip already-completed matching trials. | | `list` | List Pier job configs and copyable run selectors. | | `show ` / `show --last` | Print a summary for a Pier run (`job` or `job/run`). | +| `chart ` / `chart --last` | Write an interactive `summary.html` dashboard for a Pier run (`--open`, `--out`, `--cdn`). | | `inspect ` | Drill into stored trials by `--agent`, `--task`, and `--trial`. | | `analyze ` / `analyze --last` / `analyze --file ` | Render a rich overview of a selected Copilot session log. | @@ -116,6 +118,7 @@ uv run copilot-experiments analyze --root examples/tracer_bullet --last - [`docs/deepswe.md`](docs/deepswe.md) - importing and running DeepSWE tasks through Pier. - [`docs/collecting-run-data.md`](docs/collecting-run-data.md) - everything to collect around a Copilot CLI run, including native `events.jsonl`, Pier artifacts, ATIF, and OTel. - [`docs/results-format.md`](docs/results-format.md) - Pier `jobs/` layout and derived summaries. +- [`docs/visualizing-results.md`](docs/visualizing-results.md) - interactive `summary.html` dashboards. - [`docs/analysis.md`](docs/analysis.md) - native Copilot session analysis. - [`docs/byok-and-local-models.md`](docs/byok-and-local-models.md) - provider env for Copilot CLI. - [`docs/adr/`](docs/adr) - architecture decision records. diff --git a/docs/adr/0021-interactive-html-dashboards.md b/docs/adr/0021-interactive-html-dashboards.md new file mode 100644 index 0000000..68f4c45 --- /dev/null +++ b/docs/adr/0021-interactive-html-dashboards.md @@ -0,0 +1,62 @@ +# 0021. Interactive self-contained HTML result dashboards + +- **Status:** Accepted +- **Date:** 2026-06-20 +- **Deciders:** copilot-experiments maintainers + +## Context + +The tool could render results only as Rich terminal tables (`show`, `analyze`, `inspect`) and a +plain `summary.md`. Users asked for high-quality, clean visual outputs of experiment results, +citing the [DeepSWE leaderboard](https://deepswe.datacurve.ai/) and the +[GitHub Copilot agentic-harness benchmarking report](https://github.blog/ai-and-ml/github-copilot/evaluating-performance-and-efficiency-of-the-github-copilot-agentic-harness-across-models-and-tasks/) +as the target look and feel. Those references communicate three things well: task resolution per +model, token/cost efficiency, and a resolution-vs-cost trade-off with run-to-run variance. + +Constraints from our existing ADRs shaped the options: + +- The filesystem is the source of truth ([0002](0002-filesystem-is-source-of-truth.md)); a + dashboard should be a derived artifact next to `summary.json`, not new authoritative state. +- Analysis data is separate from rendering ([0006](0006-separate-analysis-data-from-rendering.md)); + charts must consume the existing summary, not re-derive metrics. +- We deliberately favored a CLI over a web app for analysis + ([0007](0007-cli-rich-analysis-before-web-app.md)); we did not want to (re)introduce a server. +- Tests stay offline; any dependency must be exercisable without network or Docker. + +Candidate approaches: (a) static PNG/SVG via Matplotlib, (b) a served web app, (c) self-contained +interactive HTML. Static images are easy to embed but not explorable and render poorly across +screens. A served app reverses ADR 0007. Self-contained HTML keeps the "just files" model while +giving crisp, interactive, shareable output. + +## Decision + +We will render each Pier run into a single self-contained `summary.html` dashboard built with +[Plotly](https://plotly.com/python/), added as a new `charts.py` module. + +- Plotly is a **core** runtime dependency, so the `chart` command and the automatic `run` + dashboard work with a normal install (`uvx`, `uv tool install`, `pip install`) and no extra to + enable. The import is still guarded defensively: if Plotly is somehow unavailable, `run` skips + the dashboard and `chart` exits with a clear `ChartError` instead of a bare `ImportError`. +- `charts.build_dashboard_html(summary)` consumes the existing `summary.json` shape and returns a + complete HTML string; `charts.write_dashboard(job_dir)` writes `summary.html` beside the other + derived summaries. No new source-of-truth state is introduced. +- A new `chart` CLI command produces the dashboard on demand (`--last`, `--out`, `--cdn`, + `--open`), and `run` emits it automatically after each job. +- By default plotly.js is embedded for zero-dependency offline viewing; `--cdn` produces a small + file that loads plotly.js from the CDN. +- To feed the charts we populated the previously stubbed variance/coverage aggregates in + `pier_results.py` (`std_*`/`cv_*` for AIU and tokens, `mean_resolved_rate`, + `resolved_at_k_rate`, per-task `resolved_rate`). + +## Consequences + +- Runs now yield a crisp, interactive, shareable dashboard with no server, preserving the + filesystem-first, CLI-first model. It sits alongside `summary.json` / `summary.md` and is + regenerable at any time. +- `summary.json` gained documented spread and suite-coverage fields, which also benefit `show` + and any future consumer; `report.py` key names were corrected to match. +- We accept a large runtime dependency (Plotly bundles plotly.js). The embedded offline build is + ~4.5 MB per file; `--cdn` is available when size matters. Offline tests stay fast by using the + `cdn=True` path or monkeypatching the import to assert the defensive `ChartError`. +- Future chart tweaks live entirely in `charts.py` and read only the summary dict, so rendering + stays decoupled from metric derivation per ADR 0006. diff --git a/docs/adr/0022-copilot-cli-egress-allowlist-uses-wildcard-domains.md b/docs/adr/0022-copilot-cli-egress-allowlist-uses-wildcard-domains.md new file mode 100644 index 0000000..7bb9951 --- /dev/null +++ b/docs/adr/0022-copilot-cli-egress-allowlist-uses-wildcard-domains.md @@ -0,0 +1,53 @@ +# 0022. The copilot-cli egress allowlist uses dotted-wildcard domains only + +- **Status:** Accepted +- **Date:** 2026-07-06 +- **Deciders:** project owner, Copilot + +## Context + +The local `copilot-cli` Pier installed agent (`src/copilot_experiments/pier_agents/copilot_cli.py`) +declares a `NetworkAllowlist` so air-gapped DeepSWE tasks (`allow_internet = false`) can still reach +the small set of GitHub hosts the Copilot CLI needs to install and run. + +Pier enforces that allowlist with an **egress proxy that is a Squid instance**. Squid renders the +allowlist into a `dstdomain` ACL, one entry per domain. Squid's `dstdomain` treats a bare domain and +its leading-dot wildcard as an **overlapping, fatal** pair: if both `github.com` and `.github.com` +appear, Squid aborts at startup with + +``` +ERROR: '.github.com' is a subdomain of 'github.com' +FATAL: Bungled ... acl allowed_domains dstdomain +``` + +The proxy container exits (1). The agent (`main`) container depends on the proxy via +`condition: service_healthy`, so every trial fails to start its environment. Pier then surfaces a +confusing downstream symptom — `Could not find the file /logs/artifacts/model.patch` — that hides the +real cause. The original allowlist listed three domains both bare and dotted +(`github.com`/`.github.com`, `githubcopilot.com`/`.githubcopilot.com`, +`githubusercontent.com`/`.githubusercontent.com`), so this crashed 100% of DeepSWE trials. + +## Decision + +We will list **only** the leading-dot wildcard form for any domain whose subdomains we need. In +Squid a `.example.com` entry already matches the apex `example.com` *and* every subdomain, so the +bare entries are both redundant and fatal. The allowlist is: + +```python +[".github.com", ".githubcopilot.com", ".githubusercontent.com", "gh.io"] +``` + +`gh.io` stays bare: it is a single redirector host with no subdomains we call, and its +`/copilot-install` redirect targets `*.githubusercontent.com`, already covered by the wildcard. + +A regression test asserts the wildcard domains are present and that no domain appears both bare and +as a `.domain` wildcard, so this exact Squid conflict cannot be reintroduced silently. + +## Consequences + +- DeepSWE (and any other air-gapped) tasks run through the egress proxy again; the proxy container + starts healthy instead of exiting (1). +- Anyone extending the allowlist must add the **dotted** form (e.g. `.example.com`), not the bare + domain, whenever subdomains are needed — the test enforces this and a code comment explains why. +- Bare domains remain acceptable only for hosts with no subdomain we contact (like `gh.io`); if such + a host ever needs subdomains, switch it to the dotted form rather than listing both. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8791c59..6be3d64 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,3 +37,5 @@ We follow the lightweight format popularized by | [0018](0018-adopt-pytest-cov-for-local-coverage-analysis.md) | Adopt pytest-cov for local coverage analysis | Accepted | | [0019](0019-use-nested-pier-run-directories.md) | Use nested Pier run directories | Accepted | | [0020](0020-remove-legacy-native-harness.md) | Remove the legacy native harness | Accepted | +| [0021](0021-interactive-html-dashboards.md) | Interactive self-contained HTML result dashboards | Accepted | +| [0022](0022-copilot-cli-egress-allowlist-uses-wildcard-domains.md) | The copilot-cli egress allowlist uses dotted-wildcard domains only | Accepted | diff --git a/docs/collecting-run-data.md b/docs/collecting-run-data.md index df31fe1..b56b56c 100644 --- a/docs/collecting-run-data.md +++ b/docs/collecting-run-data.md @@ -102,6 +102,13 @@ jobs/ `events.jsonl`; if no native events exist, `analyze` and summaries can fall back to `agent/trajectory.json`. +> **Root-owned `jobs/` directories.** The Pier Docker backend runs task containers as `root` +> and bind-mounts `jobs/`, so a completed run can leave `jobs/` (and its contents) owned by +> `root`. A subsequent run started as your normal user then cannot create a new run directory +> under it and Pier fails with a bare `PermissionError` on `jobs/`. `validate` and +> `run` preflight this and report the `jobs dir` check as not writable with the fix; to clear it +> manually, run `sudo chown -R "$(id -u):$(id -g)" jobs` (or `sudo rm -rf jobs`) and retry. + ## Standalone capture recipe For ad hoc runs outside Pier, generate a session id and copy the session state after Copilot exits: diff --git a/docs/results-format.md b/docs/results-format.md index 10b6a45..ae1b91a 100644 --- a/docs/results-format.md +++ b/docs/results-format.md @@ -18,6 +18,7 @@ jobs/ copilot-experiments-run.json summary.json # written by copilot-experiments summary.md # written by copilot-experiments + summary.html # written by copilot-experiments (chart / run) / config.json result.json @@ -52,6 +53,7 @@ download. `copilot-experiments` adds `copilot-experiments-run.json` to preserve | `agent/copilot-otel.jsonl` | Copilot OTel file-exporter output, captured by default for Copilot agent runs unless custom OTel destination settings override it. Useful for per-LLM-call spans with input/output/cache-write/nano-AIU details. | | `verifier/reward.txt` / `.json` | Pier verifier reward. Positive reward means solved. | | `summary.json` / `summary.md` | Derived agent/task aggregate summary. | +| `summary.html` | Interactive Plotly dashboard derived from `summary.json` (see [Visualizing results](visualizing-results.md)). | Pier jobs do not persist per-trial `metrics.json` or `analysis.json` files. Those views are derived from `agent/copilot-session/**/events.jsonl` (or `agent/trajectory.json` as a fallback) @@ -69,6 +71,21 @@ when `show`, `analyze`, or `inspect` runs. - Copilot-native token/AIU/tool metrics when native events are available; - nullable fallback metrics for non-Copilot agents. +### Variance and suite-coverage aggregates + +Each agent entry also carries spread and multi-trial coverage fields, used by the +`summary.html` dashboard: + +| Field | Scope | Meaning | +| --- | --- | --- | +| `std_aiu` / `cv_aiu` | agent, task | Population standard deviation and coefficient of variation (`std / mean`) of AIU cost across trials. `null` when no AIU is available; `cv` is `null` when the mean is zero. | +| `std_total_tokens` / `cv_total_tokens` | agent, task | Same spread measures for total token counts. | +| `mean_resolved_rate` | agent | Macro-average of per-task `success_rate` (each task weighted equally, regardless of trial count). | +| `resolved_at_k_rate` | agent | Fraction of tasks solved at least once across their `k` trials (each task's `resolved_rate` averaged). | +| `resolved` / `resolved_rate` | task | Whether a task was solved at least once (`resolved_rate` is `1.0`/`0.0` per task) and, aggregated per agent, the resolved@k coverage. | + +Single-trial tasks report `std_* = 0.0` and `cv_* = 0.0`; empty inputs report `null`. + ## Analyzing a trial ```bash diff --git a/docs/visualizing-results.md b/docs/visualizing-results.md new file mode 100644 index 0000000..57636b4 --- /dev/null +++ b/docs/visualizing-results.md @@ -0,0 +1,77 @@ +# Visualizing results + +`copilot-experiments` renders each Pier run into a single, self-contained +`summary.html` dashboard inspired by the [DeepSWE leaderboard](https://deepswe.datacurve.ai/) +and the [GitHub Copilot agentic-harness benchmarking report](https://github.blog/ai-and-ml/github-copilot/evaluating-performance-and-efficiency-of-the-github-copilot-agentic-harness-across-models-and-tasks/). +The dashboard is built with [Plotly](https://plotly.com/python/): the charts are interactive +(hover, zoom, legend toggles) yet the file opens straight from disk with no server. + +## Producing a dashboard + +```bash +# Chart the most recent run and open it in a browser. +uv run copilot-experiments chart --last --open + +# Chart a specific run by selector (from `list`). +uv run copilot-experiments chart +uv run copilot-experiments chart / + +# Write it somewhere else, or use the CDN build (see below). +uv run copilot-experiments chart --last --out report.html +uv run copilot-experiments chart --last --cdn +``` + +`run` also emits `summary.html` automatically after each Pier job, so a freshly finished +experiment already has a shareable dashboard next to its `summary.json`. + +Plotly ships as a core dependency, so `chart` and the automatic `run` dashboard work out of the +box with a normal install (`uv sync`, `uvx copilot-experiments`, `uv tool install`, or `pip +install copilot-experiments`) -- there is no extra to enable. + +## What the dashboard shows + +The page is derived entirely from `summary.json` (see +[Results format](results-format.md)), so it stays in sync with `show` and `inspect`: + +- **Header + KPI cards** - job identity, status, and headline numbers (agents, tasks, trials, + overall success rate, total AIU). +- **Leaderboard** - agents ranked by success rate, then by cost, with per-agent color that is + reused across every chart. +- **Task resolution** - success rate per agent; grouped by task when the run has more than one + task. Higher is better. +- **Resolution vs. cost** - success rate (y) against average cost per task (x) with ±1σ cost + error bars. "Up and to the left" (high success, low cost) is best. +- **Cost efficiency** - average cost per task per agent with ±1σ spread. Lower is better. +- **Per-task success** - an agent × task heatmap, shown only when the run has more than one task. + +Cost prefers AIU (`avg_aiu` / `std_aiu`) and falls back to token counts +(`avg_total_tokens` / `std_total_tokens`) when no AIU is available, matching the aggregate fields +documented in [Results format](results-format.md#variance-and-suite-coverage-aggregates). + +## Offline vs. CDN builds + +By default the dashboard **embeds** the full plotly.js bundle (~4.5 MB) so it renders with zero +network access - ideal for archiving a run or emailing it to a colleague. Pass `--cdn` to instead +reference plotly.js from `https://cdn.plot.ly/`, producing a small file (tens of KB) that needs an +internet connection to render. + +| Build | Flag | File size | Needs network to view | +| --- | --- | --- | --- | +| Offline (default) | *(none)* | ~4.5 MB | No | +| CDN | `--cdn` | ~40 KB | Yes | + +## Programmatic use + +The dashboard builder is a small public API: + +```python +from copilot_experiments.charts import build_dashboard_html, write_dashboard, plotly_available +from copilot_experiments.pier_results import build_pier_summary + +summary = build_pier_summary(job_dir) +html = build_dashboard_html(summary, cdn=True) # returns a complete HTML string +write_dashboard(job_dir, summary=summary) # writes /summary.html +``` + +`build_dashboard_html` and `write_dashboard` raise `charts.ChartError` (with an install hint) when +Plotly is unavailable; guard optional call sites with `plotly_available()`. diff --git a/pyproject.toml b/pyproject.toml index 8561107..19811ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "pyyaml>=6.0", "typer>=0.12", "rich>=13.7", + # Renders the interactive, self-contained HTML result dashboards (`chart` / `run`). + "plotly>=5.20", ] [project.scripts] diff --git a/src/copilot_experiments/charts.py b/src/copilot_experiments/charts.py new file mode 100644 index 0000000..3af2453 --- /dev/null +++ b/src/copilot_experiments/charts.py @@ -0,0 +1,718 @@ +"""Self-contained interactive HTML dashboards for Pier run summaries. + +Renders a Pier ``summary`` (the dict produced by +:func:`copilot_experiments.pier_results.build_pier_summary`) into a single, +shareable ``summary.html`` file. The look and feel is inspired by the DeepSWE +leaderboard and the GitHub Copilot agentic-harness benchmarking report: a clean +header, KPI cards, a ranked leaderboard, and a small set of interactive +[Plotly](https://plotly.com/python/) charts. + +Plotly is a required runtime dependency. The import is still guarded, so if it +is somehow unavailable (e.g. a broken or partial install) the public helpers +raise :class:`ChartError` with a clear message instead of a bare ``ImportError``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ._util import write_text + +try: # required dependency, guarded defensively against a broken install + import plotly.graph_objects as go + import plotly.io as pio + from plotly.offline import get_plotlyjs, get_plotlyjs_version + + _PLOTLY_IMPORT_ERROR: ModuleNotFoundError | None = None +except ModuleNotFoundError as exc: # pragma: no cover - exercised via monkeypatch + go = None # type: ignore[assignment] + pio = None # type: ignore[assignment] + get_plotlyjs = None # type: ignore[assignment] + get_plotlyjs_version = None # type: ignore[assignment] + _PLOTLY_IMPORT_ERROR = exc + + +class ChartError(RuntimeError): + """Raised when a dashboard cannot be produced (e.g. Plotly not installed).""" + + +_INSTALL_HINT = ( + "Plotly is required to render HTML dashboards but could not be imported. " + "Reinstall copilot-experiments to restore it (for example `uv sync` or " + "`pip install --force-reinstall copilot-experiments`)." +) + +# --------------------------------------------------------------------------- +# Theme +# --------------------------------------------------------------------------- + +_FONT_FAMILY = ( + '-apple-system, BlinkMacSystemFont, "Segoe UI", "Noto Sans", Helvetica, Arial, ' + 'sans-serif, "Apple Color Emoji", "Segoe UI Emoji"' +) +_INK = "#1f2328" +_MUTED = "#656d76" +_GRID = "#eaeef2" +_ZERO = "#d0d7de" + +# Stable, readable qualitative palette (assigned by leaderboard rank). +_PALETTE = [ + "#0969da", # blue + "#1a7f37", # green + "#8250df", # purple + "#bc4c00", # orange + "#bf3989", # pink + "#1b7c83", # teal + "#9a6700", # amber + "#cf222e", # red + "#57606a", # gray + "#6639ba", # indigo +] + +_PLOT_CONFIG = { + "displaylogo": False, + "responsive": True, + "modeBarButtonsToRemove": ["lasso2d", "select2d", "autoScale2d"], + "toImageButtonOptions": {"format": "png", "scale": 2}, +} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def build_dashboard_html( + summary: dict[str, Any], *, title: str | None = None, cdn: bool = False +) -> str: + """Render ``summary`` into a complete, self-contained HTML document. + + Parameters + ---------- + summary: + A Pier run summary (see :func:`pier_results.build_pier_summary`). + title: + Optional page/document title. Defaults to the job name. + cdn: + When ``True`` load plotly.js from the public CDN instead of embedding + the ~4.5 MB bundle, producing a tiny file that needs network access to + render. + """ + + _ensure_plotly() + agents = list(summary.get("agents") or []) + ranked = _rank_agents(agents) + colors = {agent["name"]: _PALETTE[i % len(_PALETTE)] for i, agent in enumerate(ranked)} + n_tasks = int(summary.get("n_tasks") or 0) + + charts_html: list[str] = [] + + resolution = _chart_task_resolution(ranked, colors, n_tasks) + if resolution is not None: + charts_html.append( + _chart_card( + "Task resolution", + "Success rate per agent" + + (" across tasks" if n_tasks > 1 else "") + + ". Higher is better.", + resolution, + ) + ) + + scatter = _chart_resolution_vs_cost(ranked, colors) + if scatter is not None: + charts_html.append( + _chart_card( + "Resolution vs. cost", + "Success rate against average cost per task, with \u00b11\u03c3 cost " + "spread across trials. Up and to the left is better.", + scatter, + ) + ) + + cost = _chart_cost_efficiency(ranked, colors) + if cost is not None: + charts_html.append( + _chart_card( + "Cost efficiency", + "Average cost per task per agent, with \u00b11\u03c3 spread across " + "trials. Lower is better.", + cost, + ) + ) + + if n_tasks > 1: + heatmap = _chart_task_heatmap(ranked) + if heatmap is not None: + charts_html.append( + _chart_card( + "Per-task success", + "Success rate for every agent \u00d7 task cell.", + heatmap, + ) + ) + + body = "\n".join( + [ + _header_html(summary), + _kpi_cards_html(summary), + _leaderboard_html(ranked, colors), + '
' + "\n".join(charts_html) + "
" + if charts_html + else _empty_charts_html(), + _footer_html(), + ] + ) + + head_js = _plotly_head(cdn) + doc_title = title or f"{summary.get('job') or 'Experiment'} \u2014 results" + return _PAGE_SHELL.format(title=_esc(doc_title), css=_CSS, head_js=head_js, body=body) + + +def write_dashboard( + job_dir: Path, + *, + summary: dict[str, Any] | None = None, + out_path: Path | None = None, + title: str | None = None, + cdn: bool = False, +) -> Path: + """Build and write a ``summary.html`` dashboard for a Pier job directory. + + Pass ``summary`` to reuse an already-built summary; otherwise it is rebuilt + from ``job_dir``. Returns the path written. Raises :class:`ChartError` when + Plotly is missing. + """ + + _ensure_plotly() + job_dir = Path(job_dir) + if summary is None: + from .pier_results import build_pier_summary + + summary = build_pier_summary(job_dir) + html = build_dashboard_html(summary, title=title, cdn=cdn) + out = Path(out_path) if out_path is not None else job_dir / "summary.html" + write_text(out, html) + return out + + +def plotly_available() -> bool: + """Return ``True`` when the Plotly dependency can be imported.""" + + return _PLOTLY_IMPORT_ERROR is None + + +# --------------------------------------------------------------------------- +# Charts +# --------------------------------------------------------------------------- + + +def _chart_task_resolution( + agents: list[dict[str, Any]], + colors: dict[str, str], + n_tasks: int, +) -> str | None: + if not agents: + return None + fig = go.Figure() + if n_tasks > 1: + # One colored series per agent; grouped by task along the x-axis. + task_order = _task_order(agents) + for agent in agents: + by_task = {task["task"]: task for task in agent.get("tasks") or []} + ys = [_pct_value(by_task.get(slug, {}).get("success_rate")) for slug in task_order] + fig.add_bar( + name=_esc(agent["name"]), + x=[label for _, label in task_order.items()], + y=ys, + marker_color=colors[agent["name"]], + hovertemplate="%{fullData.name}
%{x}: %{y:.0f}%", + ) + fig.update_layout(barmode="group", legend_title_text="Agent") + else: + names = [agent["name"] for agent in agents] + fig.add_bar( + x=names, + y=[_pct_value(agent.get("success_rate")) for agent in agents], + marker_color=[colors[name] for name in names], + hovertemplate="%{x}
%{y:.0f}%", + showlegend=False, + ) + fig.update_yaxes(title_text="Success rate", ticksuffix="%", range=[0, 100]) + _apply_theme(fig) + return _fragment(fig, "chart-resolution", height=430) + + +def _chart_resolution_vs_cost(agents: list[dict[str, Any]], colors: dict[str, str]) -> str | None: + metric, points = _cost_points(agents) + if not points: + return None + fig = go.Figure() + for agent in points: + label = _esc(agent["name"]) + cost = agent["_cost"] + std = agent.get("_cost_std") + color = colors[agent["name"]] + # Agent names are identified through the legend rather than printed on + # each point: Plotly has no collision avoidance for scatter text, so long + # names over closely spaced points would overlap. The legend wraps cleanly. + fig.add_trace( + go.Scatter( + x=[cost], + y=[_pct_value(agent.get("success_rate"))], + mode="markers", + name=label, + marker={ + "size": 15, + "color": color, + "line": {"width": 1.5, "color": "#ffffff"}, + "opacity": 0.95, + }, + error_x=( + { + "type": "data", + "array": [std], + "thickness": 1.4, + "width": 6, + "color": color, + } + if std + else None + ), + hovertemplate=( + f"{label}
Success: %{{y:.0f}}%
{metric['label']}: " + f"%{{x:{metric['fmt']}}}{metric['suffix']}" + ), + showlegend=True, + ) + ) + fig.update_xaxes(title_text=metric["axis"], range=list(_cost_axis_bounds(points))) + fig.update_yaxes(title_text="Success rate", ticksuffix="%", range=[-5, 106]) + _apply_theme(fig) + return _fragment(fig, "chart-scatter", height=460) + + +def _chart_cost_efficiency(agents: list[dict[str, Any]], colors: dict[str, str]) -> str | None: + metric, points = _cost_points(agents) + if not points: + return None + names = [agent["name"] for agent in points] + fig = go.Figure( + go.Bar( + x=names, + y=[agent["_cost"] for agent in points], + marker_color=[colors[name] for name in names], + error_y={ + "type": "data", + "array": [agent.get("_cost_std") or 0 for agent in points], + "thickness": 1.4, + "width": 6, + "color": _MUTED, + }, + hovertemplate=f"%{{x}}
%{{y:{metric['fmt']}}}{metric['suffix']}", + showlegend=False, + ) + ) + fig.update_yaxes(title_text=metric["axis"], rangemode="tozero") + _apply_theme(fig) + return _fragment(fig, "chart-cost", height=430) + + +def _chart_task_heatmap(agents: list[dict[str, Any]]) -> str | None: + task_order = _task_order(agents) + if not task_order: + return None + x_labels = list(task_order.values()) + y_labels = [agent["name"] for agent in agents] + z: list[list[float | None]] = [] + text: list[list[str]] = [] + for agent in agents: + by_task = {task["task"]: task for task in agent.get("tasks") or []} + row_z: list[float | None] = [] + row_t: list[str] = [] + for slug in task_order: + rate = by_task.get(slug, {}).get("success_rate") + row_z.append(None if rate is None else round(rate * 100, 1)) + row_t.append("\u2014" if rate is None else f"{rate * 100:.0f}%") + z.append(row_z) + text.append(row_t) + fig = go.Figure( + go.Heatmap( + z=z, + x=x_labels, + y=y_labels, + text=text, + texttemplate="%{text}", + textfont={"size": 12}, + zmin=0, + zmax=100, + colorscale=[[0, "#fbeaec"], [0.5, "#ffe8b3"], [1.0, "#c3e6cd"]], + colorbar={"title": {"text": "%"}, "ticksuffix": "%", "thickness": 12}, + hovertemplate="%{y}
%{x}: %{z:.0f}%", + xgap=3, + ygap=3, + ) + ) + fig.update_yaxes(autorange="reversed") + _apply_theme(fig) + height = max(320, 120 + 46 * len(y_labels)) + return _fragment(fig, "chart-heatmap", height=height) + + +# --------------------------------------------------------------------------- +# HTML pieces +# --------------------------------------------------------------------------- + + +def _header_html(summary: dict[str, Any]) -> str: + chips = [] + run_id = summary.get("run_id") + if run_id: + chips.append(f'{_esc(str(run_id))}') + status = summary.get("status") + if status: + chips.append(f'{_esc(str(status))}') + started = (summary.get("started_at") or "")[:19].replace("T", " ") + if started: + chips.append(f'started {_esc(started)}') + finished = (summary.get("finished_at") or "")[:19].replace("T", " ") + if finished: + chips.append(f'finished {_esc(finished)}') + selector = summary.get("pier_job_id") + subtitle = f'
{_esc(str(selector))}
' if selector else "" + return ( + '
' + f'
Experiment results
' + f"

{_esc(str(summary.get('job') or 'Experiment'))}

" + f"{subtitle}" + f'
{"".join(chips)}
' + "
" + ) + + +def _kpi_cards_html(summary: dict[str, Any]) -> str: + cards = [ + ("Overall success", _pct(summary.get("overall_success_rate")), "resolved trials"), + ("Total cost", _aiu(summary.get("total_aiu")), "AIU"), + ("Agents", _int(summary.get("n_agents")), "compared"), + ("Tasks", _int(summary.get("n_tasks")), "in suite"), + ("Trials", _int(summary.get("n_trials")), "executed"), + ] + failed = int(summary.get("n_failed_trials") or 0) + if failed: + cards.append(("Harness failures", str(failed), "did not run cleanly")) + items = "".join( + f'
' + f'
{value}
' + f'
{_esc(label)}
' + f'
{_esc(note)}
' + "
" + for label, value, note in cards + ) + return f'
{items}
' + + +def _leaderboard_html(agents: list[dict[str, Any]], colors: dict[str, str]) -> str: + if not agents: + return "" + rows = [] + for rank, agent in enumerate(agents, start=1): + name = agent["name"] + rows.append( + "" + f'{rank}' + f'' + f'{_esc(name)}' + f'{_esc(str(agent.get("model") or "\u2014"))}' + f'{_esc(str(agent.get("reasoning_effort") or "\u2014"))}' + f'{_pct(agent.get("success_rate"))}' + f'{_pct(agent.get("resolved_at_k_rate"))}' + f'{_aiu(agent.get("avg_aiu"))}' + f'{_aiu(agent.get("aiu_per_solve"))}' + f'{_int(agent.get("avg_total_tokens"))}' + f'{_int(agent.get("n_trials"))}' + "" + ) + return ( + '
' + '

Leaderboard

' + '
Ranked by success rate, then cost.
' + '
' + "" + '' + '' + '' + '' + "" + f"{''.join(rows)}" + "
#AgentModelEffortSuccessResolved@kAvg AIUAIU / solveAvg tokensTrials
" + ) + + +def _chart_card(title: str, subtitle: str, fragment: str) -> str: + return ( + '
' + f'

{_esc(title)}

' + f'
{subtitle}
' + f'
{fragment}
' + "
" + ) + + +def _empty_charts_html() -> str: + return ( + '

Charts

' + '
No agent metrics were available to plot.
' + "
" + ) + + +def _footer_html() -> str: + return ( + '
Generated by ' + 'copilot-experiments. Charts are interactive \u2014 ' + "hover for details, drag to zoom, double-click to reset.
" + ) + + +# --------------------------------------------------------------------------- +# Data helpers +# --------------------------------------------------------------------------- + + +def _rank_agents(agents: list[dict[str, Any]]) -> list[dict[str, Any]]: + def key(agent: dict[str, Any]) -> tuple[Any, ...]: + success = agent.get("success_rate") + cost = agent.get("avg_aiu") + if cost is None: + cost = agent.get("avg_total_tokens") + return ( + 0 if success is not None else 1, + -(success or 0.0), + 0 if cost is not None else 1, + cost if cost is not None else 0.0, + str(agent.get("name") or ""), + ) + + return sorted(agents, key=key) + + +def _cost_points(agents: list[dict[str, Any]]) -> tuple[dict[str, str], list[dict[str, Any]]]: + """Attach a cost value/std to each agent, preferring AIU then tokens.""" + + use_aiu = any(agent.get("avg_aiu") is not None for agent in agents) + metric = ( + {"label": "Avg AIU", "axis": "Average cost per task (AIU)", "fmt": ".3f", "suffix": ""} + if use_aiu + else {"label": "Avg tokens", "axis": "Average tokens per task", "fmt": ",.0f", "suffix": ""} + ) + points = [] + for agent in agents: + cost = agent.get("avg_aiu") if use_aiu else agent.get("avg_total_tokens") + if cost is None: + continue + std = agent.get("std_aiu") if use_aiu else agent.get("std_total_tokens") + points.append({**agent, "_cost": cost, "_cost_std": std}) + return metric, points + + +def _cost_axis_bounds(points: list[dict[str, Any]]) -> tuple[float, float]: + """Padded x-range for the cost scatter so markers and error bars clear the edges.""" + + lo = min(p["_cost"] - (p.get("_cost_std") or 0) for p in points) + hi = max(p["_cost"] + (p.get("_cost_std") or 0) for p in points) + span = hi - lo + pad = span * 0.1 if span > 0 else (abs(hi) or 1.0) * 0.4 + low = lo - pad + if lo >= 0: # keep a cost axis from dipping below zero + low = max(0.0, low) + return low, hi + pad + + +def _task_order(agents: list[dict[str, Any]]) -> dict[str, str]: + """Ordered mapping of task slug -> display label across all agents.""" + + order: dict[str, str] = {} + for agent in agents: + for task in agent.get("tasks") or []: + slug = task.get("task") + if slug and slug not in order: + order[slug] = str(task.get("name") or slug) + return order + + +def _pct_value(rate: float | None) -> float | None: + return None if rate is None else round(rate * 100, 1) + + +# --------------------------------------------------------------------------- +# Rendering helpers +# --------------------------------------------------------------------------- + + +def _ensure_plotly() -> None: + if _PLOTLY_IMPORT_ERROR is not None: + raise ChartError(_INSTALL_HINT) from _PLOTLY_IMPORT_ERROR + + +def _apply_theme(fig: Any) -> None: + fig.update_layout( + template="plotly_white", + font={"family": _FONT_FAMILY, "size": 13, "color": _INK}, + margin={"l": 64, "r": 24, "t": 16, "b": 56}, + paper_bgcolor="rgba(0,0,0,0)", + plot_bgcolor="rgba(0,0,0,0)", + colorway=_PALETTE, + hoverlabel={"font": {"family": _FONT_FAMILY, "size": 12}}, + legend={ + "orientation": "h", + "yanchor": "bottom", + "y": 1.02, + "xanchor": "left", + "x": 0, + "font": {"size": 12}, + }, + bargap=0.28, + bargroupgap=0.12, + ) + fig.update_xaxes( + showgrid=False, + zeroline=False, + linecolor=_ZERO, + ticks="outside", + tickcolor=_ZERO, + tickfont={"color": _MUTED}, + title_font={"size": 12, "color": _MUTED}, + automargin=True, + ) + fig.update_yaxes( + gridcolor=_GRID, + zeroline=True, + zerolinecolor=_ZERO, + linecolor="rgba(0,0,0,0)", + tickfont={"color": _MUTED}, + title_font={"size": 12, "color": _MUTED}, + automargin=True, + ) + + +def _plotly_head(cdn: bool) -> str: + """Return the ```` snippet that makes plotly.js available.""" + + if cdn: + src = f"https://cdn.plot.ly/plotly-{get_plotlyjs_version()}.min.js" + return f'' + return f"" + + +def _fragment(fig: Any, div_id: str, *, height: int) -> str: + return pio.to_html( + fig, + full_html=False, + include_plotlyjs=False, + div_id=div_id, + default_width="100%", + default_height=f"{height}px", + config=_PLOT_CONFIG, + ) + + +def _esc(value: str) -> str: + return ( + str(value) + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def _pct(value: float | None) -> str: + return "\u2014" if value is None else f"{value * 100:.0f}%" + + +def _aiu(value: float | None) -> str: + if value is None: + return "\u2014" + value = float(value) + return f"{value:.3f}" if value < 1 else f"{value:,.2f}" + + +def _int(value: float | None) -> str: + if value is None: + return "\u2014" + return f"{float(value):,.0f}" + + +# --------------------------------------------------------------------------- +# Page shell +# --------------------------------------------------------------------------- + +_CSS = """ +*{box-sizing:border-box} +html{-webkit-text-size-adjust:100%} +body{margin:0;background:#f6f8fa;color:#1f2328;line-height:1.5; + font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif; + -webkit-font-smoothing:antialiased;text-rendering:optimizeLegibility} +.mono{font-family:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,"Liberation Mono",monospace} +.muted{color:#656d76} +.wrap{max-width:1120px;margin:0 auto;padding:36px 24px 72px} +.header{margin-bottom:24px} +.eyebrow{font-size:12px;font-weight:600;letter-spacing:.06em;text-transform:uppercase;color:#8250df} +h1{margin:6px 0 4px;font-size:30px;line-height:1.2;letter-spacing:-.01em} +.subtitle{color:#656d76;font-size:13px} +.chips{display:flex;flex-wrap:wrap;gap:8px;margin-top:14px} +.chip{display:inline-flex;align-items:center;gap:6px;padding:3px 10px;border-radius:999px; + background:#fff;border:1px solid #d0d7de;font-size:12px;color:#424a53} +.chip.status-completed{background:#dafbe1;border-color:#aceebb;color:#1a7f37} +.chip.status-failed,.chip.status-error{background:#ffebe9;border-color:#ffcecb;color:#cf222e} +.chip.status-running,.chip.status-partial{background:#fff8c5;border-color:#eac54f;color:#7d4e00} +.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:14px;margin-bottom:22px} +.kpi{background:#fff;border:1px solid #d0d7de;border-radius:12px;padding:16px 18px; + box-shadow:0 1px 2px rgba(31,35,40,.06)} +.kpi-value{font-size:26px;font-weight:650;letter-spacing:-.01em;line-height:1.1} +.kpi-label{margin-top:4px;font-size:13px;font-weight:600;color:#424a53} +.kpi-note{font-size:12px;color:#8b949e} +.kpi-warn{border-color:#eac54f;background:#fffbdd} +.kpi-warn .kpi-value{color:#7d4e00} +.panel{background:#fff;border:1px solid #d0d7de;border-radius:14px;margin-bottom:22px; + box-shadow:0 1px 2px rgba(31,35,40,.06);overflow:hidden} +.panel-head{padding:18px 20px 0} +.panel-head h2{margin:0;font-size:16px;letter-spacing:-.005em} +.panel-sub{margin-top:3px;color:#656d76;font-size:13px} +.chart-body{padding:8px 12px 14px} +.table-wrap{overflow-x:auto;padding:12px 8px 8px} +table.board{width:100%;border-collapse:collapse;font-size:13px} +table.board th,table.board td{padding:10px 12px;text-align:left;white-space:nowrap} +table.board thead th{font-size:11px;font-weight:600;letter-spacing:.04em;text-transform:uppercase; + color:#656d76;border-bottom:1px solid #d0d7de} +table.board tbody tr{border-bottom:1px solid #eaeef2} +table.board tbody tr:last-child{border-bottom:0} +table.board tbody tr:hover{background:#f6f8fa} +table.board td.num,table.board th.num{text-align:right;font-variant-numeric:tabular-nums} +table.board td.rank,table.board th.rank{text-align:center;color:#8b949e;width:34px} +table.board td.strong{font-weight:650} +.agent-name{font-weight:600} +.dot{display:inline-block;width:9px;height:9px;border-radius:50%;margin-right:8px;vertical-align:middle} +.charts{display:block} +.footer{color:#8b949e;font-size:12px;text-align:center;margin-top:8px} +@media (max-width:640px){h1{font-size:24px}.wrap{padding:24px 16px 56px}} +""" + +_PAGE_SHELL = """ + + + + +{title} + +{head_js} + + +
+{body} +
+ + +""" diff --git a/src/copilot_experiments/cli.py b/src/copilot_experiments/cli.py index 5640c5f..b1a94c7 100644 --- a/src/copilot_experiments/cli.py +++ b/src/copilot_experiments/cli.py @@ -13,15 +13,18 @@ from ._util import read_json from .analysis import analyze_events, analyze_trajectory from .auth import AuthError, preflight_github_token +from .charts import ChartError, plotly_available, write_dashboard from .deepswe import DeepSweImportError, write_deepswe_job_config from .pier_backend import ( PierBackendPreflightError, PierJobSpec, + check_jobs_dir_writable, discover_pier_job_configs, inject_copilot_token, preflight_pier_backend, prepare_pier_job_for_run, run_pier_job, + sync_saved_run_config, ) from .pier_results import ( describe_missing_pier_analysis_source, @@ -251,6 +254,8 @@ def run( if verbose: prepared.config.debug = True inject_copilot_token(prepared.config, auth.token) + if prepared.resumed: + sync_saved_run_config(prepared.config) console.print(f"[bold]Running Pier job[/bold] {prepared.label}") if prepared.resumed: console.print(f"[dim]resume:[/dim] reusing existing Pier run {prepared.label}") @@ -273,6 +278,12 @@ def run( summary = write_pier_summary(run_result.job_dir) _print_run_summary(summary) _warn_failed_pier_trials(run_result.job_dir) + if plotly_available(): + try: + html_path = write_dashboard(run_result.job_dir, summary=summary) + console.print(f"[dim]chart:[/dim] {html_path}") + except ChartError: + pass if summary.get("status") != "completed": any_failures = True console.print(f"[dim]results:[/dim] {run_result.job_dir}\n") @@ -342,6 +353,43 @@ def show( console.print(f"\n[dim]{resolved.path / 'summary.md'}[/dim]") +@app.command() +def chart( + selector: str | None = typer.Argument( + None, + help="Pier run selector from `list`: job, run id/prefix, or job/run.", + ), + last: bool = typer.Option(False, "--last", help="Chart the most recent stored Pier run."), + out: Path | None = typer.Option( + None, "--out", help="Write the dashboard here (default: /summary.html)." + ), + cdn: bool = typer.Option( + False, + "--cdn", + help="Load plotly.js from its CDN (tiny file, but needs network to view).", + ), + open_browser: bool = typer.Option( + False, "--open", help="Open the dashboard in your web browser when done." + ), + root: Path | None = typer.Option(None, "--root", help="Experiment repository root."), +) -> None: + """Render an interactive HTML dashboard (summary.html) for a Pier run.""" + + resolved = _resolve_or_exit(root, selector, last=last) + summary = write_pier_summary(resolved.path) + try: + out_path = write_dashboard(resolved.path, summary=summary, out_path=out, cdn=cdn) + except ChartError as exc: + err.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + console.print(f"[green]Wrote[/green] {out_path}") + if open_browser: + import webbrowser + + webbrowser.open(out_path.resolve().as_uri()) + + @app.command() def inspect( selector: str | None = typer.Argument( @@ -499,6 +547,12 @@ def _validate_pier_specs(specs: list[PierJobSpec]) -> list[ValidationCheck]: checks.append(ValidationCheck(f"{prefix}: backend", False, str(exc))) else: checks.append(ValidationCheck(f"{prefix}: backend", True, "preflight OK")) + try: + check_jobs_dir_writable(spec.config) + except PierBackendPreflightError as exc: + checks.append(ValidationCheck(f"{prefix}: jobs dir", False, str(exc))) + else: + checks.append(ValidationCheck(f"{prefix}: jobs dir", True, "writable")) if all(check.ok for check in checks): try: diff --git a/src/copilot_experiments/pier_agents/copilot_cli.py b/src/copilot_experiments/pier_agents/copilot_cli.py index 02ced35..a8bb33c 100644 --- a/src/copilot_experiments/pier_agents/copilot_cli.py +++ b/src/copilot_experiments/pier_agents/copilot_cli.py @@ -98,14 +98,16 @@ def parse_version(self, stdout: str) -> str: return match.group(1) if match else text def network_allowlist(self) -> NetworkAllowlist: + # Pier's egress proxy is Squid, whose dstdomain ACL treats a bare domain + # and its ".domain" wildcard as a fatal conflict ("'.github.com' is a + # subdomain of 'github.com'"). Listing both crashes the proxy container + # (exit 1) and fails every trial. A leading-dot entry already matches the + # apex domain *and* all subdomains, so we list only the wildcard forms + # (plus the bare redirector host gh.io, which has no subdomains). return NetworkAllowlist( domains=[ - "api.github.com", - "github.com", ".github.com", - "githubcopilot.com", ".githubcopilot.com", - "githubusercontent.com", ".githubusercontent.com", "gh.io", ] diff --git a/src/copilot_experiments/pier_backend.py b/src/copilot_experiments/pier_backend.py index 0081974..31d36a3 100644 --- a/src/copilot_experiments/pier_backend.py +++ b/src/copilot_experiments/pier_backend.py @@ -4,8 +4,10 @@ import asyncio import json +import os import shutil import subprocess +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -140,6 +142,47 @@ def preflight_pier_backend(config: Any) -> None: _preflight_docker_backend() +def check_jobs_dir_writable( + config: Any, + *, + writable: Callable[[Path], bool] | None = None, +) -> None: + """Fail fast when the harness cannot create the job's run directory. + + Pier creates ``jobs_dir / job_name`` with ``Path.mkdir(parents=True)``. When an + earlier run left that tree owned by another user -- most commonly ``root``, because + Pier's Docker backend runs containers as root and bind-mounts the jobs directory -- + the current user can no longer create entries under it and Pier raises a bare + ``PermissionError``. Detect that here and surface an actionable remediation instead. + """ + + is_writable = writable or (lambda directory: os.access(directory, os.W_OK | os.X_OK)) + target = _job_dir(config) + anchor = _nearest_existing_ancestor(target) + if is_writable(anchor): + return + raise PierBackendPreflightError( + f"Cannot write to {anchor}, which is needed to create the run directory " + f"{target}. An earlier run probably created it as another user (for example " + f"root, via the Pier Docker backend), so the current user can no longer create " + f"entries under it. Reclaim ownership with " + f'`sudo chown -R "$(id -u):$(id -g)" {anchor}` ' + f"or remove the stale output directory, then retry." + ) + + +def _nearest_existing_ancestor(path: Path) -> Path: + """Return ``path`` or its closest existing ancestor directory.""" + + for candidate in (path, *path.parents): + if candidate.exists(): + return candidate + return Path(path.anchor) if path.anchor else Path(".") + + +_TOKEN_ENV_KEYS: frozenset[str] = frozenset({"COPILOT_GITHUB_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"}) + + def inject_copilot_token(config: Any, token: str) -> None: """Inject a GitHub token into local Copilot agents without persisting it to config.""" @@ -155,6 +198,38 @@ def inject_copilot_token(config: Any, token: str) -> None: agent.env.setdefault("GH_TOKEN", token) +def sync_saved_run_config(config: Any) -> None: + """Overwrite the Copilot token env vars in ``config.json`` of an existing run dir. + + Call this after ``inject_copilot_token`` and before ``run_pier_job`` when resuming. + Pier reads ``config.json`` on ``Job.create`` and raises ``FileExistsError`` when the + stored config differs from the one passed to it — most commonly because the auth token + has rotated between runs. Patching only the token fields in the raw JSON preserves + all other config fields exactly as Pier wrote them, so the comparison always passes. + """ + run_dir = Path(config.jobs_dir) / str(config.job_name) + config_path = run_dir / "config.json" + if not config_path.exists(): + return + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + # Update token env vars in-place in the raw dict so no secret-masking occurs. + for saved_agent, live_agent in zip(data.get("agents", []), config.agents, strict=False): + env: dict[str, str] = saved_agent.setdefault("env", {}) + for key in _TOKEN_ENV_KEYS: + live_val = live_agent.env.get(key) + if live_val is not None: + env[key] = live_val + else: + env.pop(key, None) + config_path.write_text( + json.dumps(data, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + except Exception: + pass + + def prepare_pier_job_for_run( config: Any, *, @@ -175,9 +250,8 @@ def prepare_pier_job_for_run( if resume: existing = _latest_existing_run_dir(prepared) if existing is not None: - prepared.jobs_dir = existing.parent - prepared.job_name = existing.name - return PreparedPierJob(prepared, requested_name, existing.name, resumed=True) + resumed_config = _config_for_resume(prepared, existing) + return PreparedPierJob(resumed_config, requested_name, existing.name, resumed=True) base_run_name = (now or datetime.now()).strftime("%Y%m%d-%H%M%S") run_name = base_run_name @@ -214,6 +288,47 @@ def _job_dir(config: Any) -> Path: return Path(config.jobs_dir) / str(config.job_name) +def _config_for_resume(current_config: Any, run_dir: Path) -> Any: + """Reconstruct the Pier config from a saved run directory for resuming. + + Pier's ``Job.create`` raises ``FileExistsError`` when the config passed to it + differs from the one stored in ``config.json``. This helper rebuilds the config + from the saved file (so all structural fields match), then clears the token env + vars so that ``inject_copilot_token`` can inject fresh values. Falls back to + the current in-memory config (pointing at the run dir) when the file cannot be + loaded or parsed. + """ + config_path = run_dir / "config.json" + if config_path.exists(): + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + loaded = type(current_config).model_validate(data) + _clear_copilot_token_env(loaded) + # Always anchor jobs_dir/job_name to the actual run directory — the saved + # values may be relative or point to a different machine layout. + loaded.jobs_dir = run_dir.parent + loaded.job_name = run_dir.name + return loaded + except Exception: + pass + current_config.jobs_dir = run_dir.parent + current_config.job_name = run_dir.name + return current_config + + +def _clear_copilot_token_env(config: Any) -> None: + """Remove Copilot token env vars from agents so fresh values can be injected.""" + for agent in config.agents: + is_copilot = ( + agent.import_path == COPILOT_CLI_AGENT_IMPORT_PATH + or agent.name == COPILOT_CLI_AGENT_NAME + ) + if not is_copilot: + continue + for key in _TOKEN_ENV_KEYS: + agent.env.pop(key, None) + + def _latest_existing_run_dir(config: Any) -> Path | None: """Return the latest resumable run directory for a stable job config.""" diff --git a/src/copilot_experiments/pier_results.py b/src/copilot_experiments/pier_results.py index 3bbd5da..9382ec9 100644 --- a/src/copilot_experiments/pier_results.py +++ b/src/copilot_experiments/pier_results.py @@ -2,6 +2,7 @@ from __future__ import annotations +import statistics from collections import defaultdict from datetime import datetime from pathlib import Path @@ -70,7 +71,7 @@ def build_pier_summary(job_dir: Path) -> dict[str, Any]: task_summaries.append(_aggregate_task(task_slug, trials)) cell["tasks"] = task_summaries cell["n_tasks"] = len(task_summaries) - cell.update(_aggregate_agent(all_trials)) + cell.update(_aggregate_agent(all_trials, task_summaries)) agents.append(cell) all_trials = [trial for agent in agents for task in agent["tasks"] for trial in task["_trials"]] @@ -313,49 +314,55 @@ def _job_status(job_result: dict[str, Any]) -> str: def _aggregate_task(task_slug: str, trials: list[dict[str, Any]]) -> dict[str, Any]: graded = [trial["success"] for trial in trials if trial.get("success") is not None] aiu_values = [(trial.get("metrics") or {}).get("aiu") for trial in trials] + token_values = _metric_values(trials, "total_tokens") total_aiu = sum(value for value in aiu_values if value is not None) solved = sum(1 for value in graded if value) + resolved = None if not graded else int(any(graded)) return { "task": task_slug, "name": trials[0].get("task_name") or task_slug, "n_trials": len(trials), "success_rate": (solved / len(graded)) if graded else None, - "resolved": None if not graded else int(any(graded)), + "resolved": resolved, + "resolved_rate": None if resolved is None else float(resolved), "avg_duration_s": _avg([trial.get("duration_s") for trial in trials]), "avg_turns": _avg(_metric_values(trials, "n_turns")), - "avg_total_tokens": _avg(_metric_values(trials, "total_tokens")), - "cv_total_tokens": None, + "avg_total_tokens": _avg(token_values), + "cv_total_tokens": _cv(token_values), "avg_aiu": _avg(aiu_values), - "cv_aiu": None, + "cv_aiu": _cv(aiu_values), "total_aiu": round(total_aiu, 3) if total_aiu else None, "aiu_per_solve": round(total_aiu / solved, 3) if total_aiu and solved else None, "_trials": trials, } -def _aggregate_agent(trials: list[dict[str, Any]]) -> dict[str, Any]: +def _aggregate_agent( + trials: list[dict[str, Any]], task_summaries: list[dict[str, Any]] +) -> dict[str, Any]: graded = [trial["success"] for trial in trials if trial.get("success") is not None] solved = sum(1 for value in graded if value) aiu_values = [(trial.get("metrics") or {}).get("aiu") for trial in trials] + token_values = _metric_values(trials, "total_tokens") total_aiu = sum(value for value in aiu_values if value is not None) return { "success_rate": (solved / len(graded)) if graded else None, - "mean_resolved_rate": None, - "resolved_at_k_rate": None, + "mean_resolved_rate": _avg([task.get("success_rate") for task in task_summaries]), + "resolved_at_k_rate": _avg([task.get("resolved_rate") for task in task_summaries]), "avg_duration_s": _avg([trial.get("duration_s") for trial in trials]), "avg_turns": _avg(_metric_values(trials, "n_turns")), "avg_tool_calls": _avg(_metric_values(trials, "n_tool_calls")), "avg_tool_failures": _avg(_metric_values(trials, "n_tool_failures")), - "avg_total_tokens": _avg(_metric_values(trials, "total_tokens")), - "std_total_tokens": None, - "cv_total_tokens": None, + "avg_total_tokens": _avg(token_values), + "std_total_tokens": _std(token_values), + "cv_total_tokens": _cv(token_values), "avg_input_tokens": _avg(_metric_values(trials, "input_tokens")), "avg_output_tokens": _avg(_metric_values(trials, "output_tokens")), "avg_cache_read_tokens": _avg(_metric_values(trials, "cache_read_tokens")), "avg_reasoning_tokens": _avg(_metric_values(trials, "reasoning_tokens")), "avg_aiu": _avg(aiu_values), - "std_aiu": None, - "cv_aiu": None, + "std_aiu": _std(aiu_values), + "cv_aiu": _cv(aiu_values), "total_aiu": round(total_aiu, 3) if total_aiu else None, "aiu_per_solve": round(total_aiu / solved, 3) if total_aiu and solved else None, "avg_lines_added": _avg(_metric_values(trials, "lines_added")), @@ -384,6 +391,23 @@ def _avg(values: list[Any]) -> float | None: return round(sum(nums) / len(nums), 3) if nums else None +def _std(values: list[Any]) -> float | None: + """Population standard deviation across a cell's trials (0.0 for a single point).""" + nums = [float(value) for value in values if value is not None] + return round(statistics.pstdev(nums), 3) if nums else None + + +def _cv(values: list[Any]) -> float | None: + """Coefficient of variation (std / mean); ``None`` when undefined.""" + nums = [float(value) for value in values if value is not None] + if not nums: + return None + mean = sum(nums) / len(nums) + if mean == 0: + return None + return round(statistics.pstdev(nums) / mean, 3) + + def _metric_values(trials: list[dict[str, Any]], name: str) -> list[Any]: return [(trial.get("metrics") or {}).get(name) for trial in trials] diff --git a/src/copilot_experiments/report.py b/src/copilot_experiments/report.py index 445fa6c..6f7c254 100644 --- a/src/copilot_experiments/report.py +++ b/src/copilot_experiments/report.py @@ -84,10 +84,10 @@ def summary_markdown(summary: dict, description: str = "") -> str: "{api} |".format( name=agent["name"], avg_aiu=_aiu(agent.get("avg_aiu")), - aiu_cv=_fmt(agent.get("aiu_cv")), + aiu_cv=_fmt(agent.get("cv_aiu")), aiu_solve=_aiu(agent.get("aiu_per_solve")), tokens=_fmt(agent.get("avg_total_tokens")), - token_cv=_fmt(agent.get("total_tokens_cv")), + token_cv=_fmt(agent.get("cv_total_tokens")), api=_fmt(agent.get("avg_api_duration_s")), ) ) @@ -115,7 +115,7 @@ def summary_markdown(summary: dict, description: str = "") -> str: for agent in summary["agents"]: for task in agent.get("tasks", []): lines.append( - f"| {agent['name']} | {task['task_slug']} | {task['n_trials']} | " + f"| {agent['name']} | {task['task']} | {task['n_trials']} | " f"{_pct(task.get('success_rate'))} | {_pct(task.get('resolved_rate'))} | " f"{_aiu(task.get('avg_aiu'))} |" ) diff --git a/src/copilot_experiments/sessionlog.py b/src/copilot_experiments/sessionlog.py index d31ecc2..60b3711 100644 --- a/src/copilot_experiments/sessionlog.py +++ b/src/copilot_experiments/sessionlog.py @@ -8,9 +8,13 @@ from pathlib import Path from typing import Any +from rich.console import Console + from . import pricing from .models import Metrics, ModelMetric, TokenEconomics +err = Console(stderr=True) + def session_state_root(base: Path | None = None) -> Path: """Directory where the Copilot CLI stores per-session state.""" @@ -27,6 +31,7 @@ def load_events(path: Path) -> list[dict[str, Any]]: if not path.exists(): return [] events: list[dict[str, Any]] = [] + skipped_lines = 0 for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: @@ -34,7 +39,15 @@ def load_events(path: Path) -> list[dict[str, Any]]: try: events.append(json.loads(line)) except json.JSONDecodeError: + skipped_lines += 1 continue + + if skipped_lines > 0: + err.print( + f"[yellow]Warning: Skipped {skipped_lines} malformed JSON line(s) " + f"in {path.name}[/yellow]" + ) + return events diff --git a/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md b/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md index 3f5a3c3..ad351f9 100644 --- a/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md +++ b/src/copilot_experiments/templates/experiment_repo/.apm/skills/analyzing-results/SKILL.md @@ -16,6 +16,7 @@ jobs/// copilot-experiments-run.json summary.json # derived copilot-experiments summary summary.md # human-readable report + summary.html # interactive Plotly dashboard (chart / run) / result.json # Pier trial result agent/ @@ -32,6 +33,7 @@ tokens, and AIU economics. ```bash copilot-experiments list # runs + success rates copilot-experiments show --last # per-agent comparison table +copilot-experiments chart --last --open # interactive summary.html dashboard copilot-experiments inspect # latest run for that Pier job copilot-experiments inspect / # exact run selector from list copilot-experiments analyze / --agent --trial diff --git a/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl b/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl index d3a27fb..5e53b89 100644 --- a/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl +++ b/src/copilot_experiments/templates/experiment_repo/AGENTS.md.tmpl @@ -37,6 +37,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open uv run copilot-experiments inspect / --agent --trial uv run copilot-experiments analyze / --agent --trial ``` diff --git a/src/copilot_experiments/templates/experiment_repo/README.md.tmpl b/src/copilot_experiments/templates/experiment_repo/README.md.tmpl index 45b18ec..5da967a 100644 --- a/src/copilot_experiments/templates/experiment_repo/README.md.tmpl +++ b/src/copilot_experiments/templates/experiment_repo/README.md.tmpl @@ -28,6 +28,7 @@ uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments validate uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments run uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments list uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments show --last +uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments chart --last --open # explore results uvx --from "$COPILOT_EXPERIMENTS_REPO" copilot-experiments list @@ -57,6 +58,7 @@ uv run copilot-experiments validate uv run copilot-experiments run uv run copilot-experiments list uv run copilot-experiments show --last +uv run copilot-experiments chart --last --open # explore results uv run copilot-experiments list @@ -64,6 +66,23 @@ uv run copilot-experiments inspect / --agent copilot-cli --tri uv run copilot-experiments analyze / --agent copilot-cli --trial 1 ``` +## Troubleshooting + +### `PermissionError` creating a run directory under `jobs/` + +The Pier Docker backend runs task containers as `root` and bind-mounts the `jobs/` +directory, so a completed run can leave `jobs/` (and its contents) owned by `root`. A later +run started as your normal user then cannot create a new run directory under it, and Pier +fails with a bare `PermissionError: [Errno 13] Permission denied: '.../jobs/'`. + +`validate` and `run` now preflight this and report `jobs dir` as not writable with the fix, +but if you hit it directly, reclaim ownership (or remove the stale output) and retry: + +```bash +sudo chown -R "$(id -u):$(id -g)" jobs # reclaim ownership, or +sudo rm -rf jobs # discard stale outputs +``` + ## Writing experiments See `tasks/example-fix-bug/` and `experiments/example.yaml`. diff --git a/tests/test_charts.py b/tests/test_charts.py new file mode 100644 index 0000000..9519430 --- /dev/null +++ b/tests/test_charts.py @@ -0,0 +1,201 @@ +"""Tests for the interactive HTML result dashboards.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from copilot_experiments import charts +from copilot_experiments.charts import ( + ChartError, + build_dashboard_html, + plotly_available, + write_dashboard, +) + +_MULTI_TASK_SUMMARY = { + "run_id": "20260612T103300Z_a1b2c3", + "job": "strtools-vs-csvtools", + "pier_job_id": "strtools/20260612T103300Z_a1b2c3", + "status": "completed", + "started_at": "2026-06-12T10:33:00Z", + "finished_at": "2026-06-12T10:52:11Z", + "n_agents": 2, + "n_tasks": 2, + "n_trials": 12, + "n_failed_trials": 0, + "overall_success_rate": 0.75, + "total_aiu": 3.21, + "agents": [ + { + "name": "sonnet-4.6", + "model": "claude-sonnet-4.6", + "reasoning_effort": "medium", + "n_tasks": 2, + "n_trials": 6, + "success_rate": 0.83, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.21, + "std_aiu": 0.04, + "aiu_per_solve": 0.25, + "avg_total_tokens": 120000, + "std_total_tokens": 15000, + "tasks": [ + {"task": "strtools", "name": "strtools", "success_rate": 1.0, "resolved_rate": 1.0}, + { + "task": "csvtools", + "name": "csvtools", + "success_rate": 0.66, + "resolved_rate": 1.0, + }, + ], + }, + { + "name": "haiku-4.5", + "model": "claude-haiku-4.5", + "reasoning_effort": None, + "n_tasks": 2, + "n_trials": 6, + "success_rate": 0.5, + "resolved_at_k_rate": 0.5, + "avg_aiu": 0.08, + "std_aiu": 0.02, + "aiu_per_solve": 0.16, + "avg_total_tokens": 90000, + "std_total_tokens": 8000, + "tasks": [ + {"task": "strtools", "name": "strtools", "success_rate": 1.0, "resolved_rate": 1.0}, + {"task": "csvtools", "name": "csvtools", "success_rate": 0.0, "resolved_rate": 0.0}, + ], + }, + ], +} + +_SINGLE_TASK_SUMMARY = { + "run_id": "run-1", + "job": "textstats", + "pier_job_id": "textstats/run-1", + "status": "completed", + "n_agents": 2, + "n_tasks": 1, + "n_trials": 4, + "overall_success_rate": 0.75, + "total_aiu": 0.9, + "agents": [ + { + "name": "agent-a", + "model": "model-a", + "reasoning_effort": "high", + "n_tasks": 1, + "n_trials": 2, + "success_rate": 1.0, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.3, + "std_aiu": 0.05, + "aiu_per_solve": 0.3, + "avg_total_tokens": 150000, + "tasks": [ + { + "task": "textstats", + "name": "textstats", + "success_rate": 1.0, + "resolved_rate": 1.0, + } + ], + }, + { + "name": "agent-b", + "model": "model-b", + "reasoning_effort": "low", + "n_tasks": 1, + "n_trials": 2, + "success_rate": 0.5, + "resolved_at_k_rate": 1.0, + "avg_aiu": 0.15, + "std_aiu": 0.02, + "aiu_per_solve": 0.3, + "avg_total_tokens": 80000, + "tasks": [ + { + "task": "textstats", + "name": "textstats", + "success_rate": 0.5, + "resolved_rate": 1.0, + } + ], + }, + ], +} + + +def test_build_dashboard_html_has_all_sections(): + html = build_dashboard_html(_MULTI_TASK_SUMMARY, cdn=True) + + assert html.startswith("") + assert "Plotly.newPlot" in html + for div_id in ("chart-resolution", "chart-scatter", "chart-cost", "chart-heatmap"): + assert div_id in html + assert "Leaderboard" in html + assert "strtools-vs-csvtools" in html # job title + assert "claude-sonnet-4.6" in html # leaderboard model column + + +def test_build_dashboard_cdn_is_small_and_references_cdn(): + html = build_dashboard_html(_MULTI_TASK_SUMMARY, cdn=True) + + assert "https://cdn.plot.ly/plotly-" in html + assert "