diff --git a/.github/lsp.json b/.github/lsp.json index 7535212849..e58456ac43 100644 --- a/.github/lsp.json +++ b/.github/lsp.json @@ -21,24 +21,6 @@ ".go": "go" }, "rootUri": "go" - }, - "rust-analyzer": { - "command": "rust-analyzer", - "fileExtensions": { - ".rs": "rust" - }, - "initializationOptions": { - "cargo": { - "buildScripts": { - "enable": true - }, - "allFeatures": true - }, - "checkOnSave": true, - "check": { - "command": "clippy" - } - } } } } diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e42b1e6adc..cd96dd0fa9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -171,13 +171,17 @@ jobs: - name: Set version run: sed -i -E 's/^version = ".*"$/version = "${{ needs.version.outputs.version }}"/' Cargo.toml - name: Snapshot CLI version + hashes for build.rs - run: bash scripts/snapshot-bundled-cli-version.sh - - name: Verify cli-version.txt exists run: | - if [[ ! -f cli-version.txt ]]; then - echo "::error::cli-version.txt was not generated. The Snapshot step must run before packaging." - exit 1 - fi + bash scripts/snapshot-bundled-cli-version.sh + bash scripts/snapshot-bundled-in-process-version.sh + - name: Verify CLI version snapshots exist + run: | + for snapshot in cli-version.txt cli-version-in-process.txt; do + if [[ ! -f "${snapshot}" ]]; then + echo "::error::${snapshot} was not generated. The Snapshot step must run before packaging." + exit 1 + fi + done - name: Package (dry run) run: cargo publish --dry-run --allow-dirty - name: Upload artifact diff --git a/.github/workflows/rust-sdk-tests.yml b/.github/workflows/rust-sdk-tests.yml index f75a5d6a29..8e13e16b23 100644 --- a/.github/workflows/rust-sdk-tests.yml +++ b/.github/workflows/rust-sdk-tests.yml @@ -29,7 +29,7 @@ permissions: jobs: test: - name: "Rust SDK Tests" + name: "Rust SDK Tests (${{ matrix.os }}, default)" if: github.event.repository.fork == false env: POWERSHELL_UPDATECHECK: Off @@ -84,7 +84,7 @@ jobs: # Share the bundled-CLI archive cache with the `bundle` job: build.rs # now downloads in both modes (embed for `bundle`, extract-to-cache # for this `test` job's `--no-default-features` build). - - name: Cache bundled CLI tarball + - name: Cache bundled CLI archives uses: actions/cache@v4 with: path: ./rust/.bundled-cli-cache @@ -98,7 +98,7 @@ jobs: if: runner.os == 'Linux' env: BUNDLED_CLI_CACHE_DIR: ${{ github.workspace }}/rust/.bundled-cli-cache - run: cargo clippy --all-targets --features test-support -- --no-deps -D warnings -D clippy::unwrap_used -D clippy::disallowed_macros -D clippy::await_holding_invalid_type + run: cargo clippy --all-targets --features test-support,bundled-in-process -- --no-deps -D warnings -D clippy::unwrap_used -D clippy::disallowed_macros -D clippy::await_holding_invalid_type - name: cargo doc if: runner.os == 'Linux' @@ -129,6 +129,84 @@ jobs: # The dedicated `bundle` job below exercises the embed pipeline. run: cargo test --no-default-features --features test-support -- --test-threads=4 --nocapture + # Exercises the in-process FFI transport (`Transport::InProcess`, the Rust + # analogue of the .NET `RuntimeConnection.ForInProcess()`), mirroring the + # `inprocess` transport cell in dotnet-sdk-tests.yml. Sets + # COPILOT_SDK_DEFAULT_CONNECTION=inprocess so the client hosts the runtime + # cdylib in-process instead of spawning a stdio child, then runs the whole + # E2E suite over the in-process transport. The suite runs serially in-process + # (the harness forces concurrency to 1) because it mirrors each test's + # environment onto the shared process environment the in-process worker inherits. + # Runs the whole E2E suite over the in-process transport on supported hosts. + test-inprocess: + name: "Rust SDK Tests (${{ matrix.os }}, inprocess)" + if: github.event.repository.fork == false + env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + strategy: + fail-fast: false + matrix: + # TODO: Re-enable Windows after fixing the napi-oop peer shutdown crash. + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: ./rust + steps: + - uses: actions/checkout@v6.0.2 + + - uses: ./.github/actions/setup-copilot + id: setup-copilot + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + with: + toolchain: "1.94.0" + + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 + with: + workspaces: "rust" + prefix-key: v1-rust-no-bin + cache-bin: false + + - name: Read pinned @github/copilot CLI version + id: cli-version + working-directory: ./nodejs + run: | + version=$(node -p "require('./package-lock.json').packages['node_modules/@github/copilot'].version") + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "Pinned CLI version: $version" + + - name: Cache bundled CLI archives + uses: actions/cache@v4 + with: + path: ./rust/.bundled-cli-cache + key: bundled-cli-${{ matrix.os }}-${{ steps.cli-version.outputs.version }} + + - name: Install test harness dependencies + working-directory: ./test/harness + run: npm ci --ignore-scripts + + - name: Warm up PowerShell + if: runner.os == 'Windows' + run: pwsh.exe -Command "Write-Host 'PowerShell ready'" + + - name: Select in-process transport + run: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" + + - name: cargo test (in-process transport, full E2E suite) + timeout-minutes: 60 + env: + COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} + COPILOT_CLI_PATH: ${{ steps.setup-copilot.outputs.cli-path }} + BUNDLED_CLI_CACHE_DIR: ${{ github.workspace }}/rust/.bundled-cli-cache + # The harness forces serial execution in-process (both the async semaphore and + # libtest via --test-threads=1) because it mirrors each test's environment onto + # the shared process environment, so RUST_E2E_CONCURRENCY is not set here. + run: cargo test --no-default-features --features test-support,bundled-in-process --test e2e -- --test-threads=1 --nocapture + # Validates the bundled-CLI build path on all three supported # platforms. While the regular `cargo test` job above also exercises # build.rs (bundling is on by default now), this matrix job is the @@ -136,7 +214,7 @@ jobs: # extract / embed pipeline. Catches regressions before they ship to # crates.io and before bundling consumers hit them downstream. bundle: - name: "Rust SDK Bundled CLI Build" + name: "Rust SDK Bundled CLI Build (${{ matrix.os }})" if: github.event.repository.fork == false env: CARGO_TERM_COLOR: always @@ -180,13 +258,15 @@ jobs: # ~130 MB on every CI invocation. Keyed by OS + CLI version so old # archives drop out when the pinned version bumps, keeping the # cache bounded. - - name: Cache bundled CLI tarball + - name: Cache bundled CLI archives uses: actions/cache@v4 with: path: ./rust/.bundled-cli-cache key: bundled-cli-${{ matrix.os }}-${{ steps.cli-version.outputs.version }} - - name: cargo build (bundled-cli is the default feature) + - name: Test bundled CLI build paths env: BUNDLED_CLI_CACHE_DIR: ${{ github.workspace }}/rust/.bundled-cli-cache - run: cargo build + run: | + cargo build + cargo test --features bundled-in-process --lib embedded_archive_contains_only_expected_files diff --git a/rust/.gitignore b/rust/.gitignore index c4095ffc0f..c149fa3946 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,3 +1,4 @@ /target Cargo.lock.bak cli-version.txt +cli-version-in-process.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index aa9fe67ab9..23a179cd1e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -434,6 +434,7 @@ dependencies = [ "getrandom 0.2.17", "http", "indexmap", + "libloading", "parking_lot", "regex", "reqwest", @@ -774,6 +775,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.16" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 6529e013f5..4810d83f44 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -13,6 +13,7 @@ readme = "README.md" license = "MIT" include = [ "src/**/*", + "build/**/*", "examples/**/*", "tests/**/*", "build.rs", @@ -20,6 +21,7 @@ include = [ "README.md", "LICENSE", "cli-version.txt", + "cli-version-in-process.txt", ] [lib] @@ -28,6 +30,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] +bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -49,10 +52,13 @@ tokio-stream = { version = "0.1", features = ["sync"] } tokio-util = { version = "0.7", default-features = false } tracing = "0.1" dirs = "5" +libloading = { version = "0.8", optional = true } parking_lot = "0.12" regex = "1" getrandom = "0.2" uuid = { version = "1", default-features = false, features = ["v4"] } +flate2 = { version = "1", optional = true } +tar = { version = "0.4", optional = true } # LLM inference callback transport: idiomatic HTTP/WebSocket forwarding for the # `CopilotRequestHandler`, plus base64/byte/stream plumbing for the chunk protocol. base64 = "0.22" @@ -65,10 +71,6 @@ tokio-tungstenite = { version = "0.24", default-features = false, features = ["c [target.'cfg(windows)'.dependencies] zip = { version = "2", default-features = false, features = ["deflate"], optional = true } -[target.'cfg(not(windows))'.dependencies] -flate2 = { version = "1", optional = true } -tar = { version = "0.4", optional = true } - [dev-dependencies] rusqlite = { version = "0.35", features = ["bundled"] } schemars = "1" @@ -89,8 +91,10 @@ name = "protocol_version_test" required-features = ["test-support"] [build-dependencies] +base64 = "0.22" dirs = "5" flate2 = "1" +serde_json = "1" sha2 = "0.10" tar = "0.4" ureq = { version = "2", default-features = false, features = ["tls"] } diff --git a/rust/README.md b/rust/README.md index 6d92224088..7dda184838 100644 --- a/rust/README.md +++ b/rust/README.md @@ -72,15 +72,15 @@ client.stop().await?; **`ClientOptions`:** -| Field | Type | Description | -| ------------- | --------------------------- | --------------------------------------------------------------- | -| `program` | `CliProgram` | `Resolve` (default: auto-detect) or `Path(PathBuf)` (explicit) | -| `prefix_args` | `Vec` | Args before `--server` (e.g. script path for node) | -| `cwd` | `PathBuf` | Working directory for CLI process | -| `env` | `Vec<(OsString, OsString)>` | Environment variables for CLI process | -| `env_remove` | `Vec` | Environment variables to remove | -| `extra_args` | `Vec` | Extra CLI flags | -| `transport` | `Transport` | `Stdio` (default), `Tcp { port }`, or `External { host, port }` | +| Field | Type | Description | +| ------------------- | --------------------------- | ----------------------------------------------------------------- | +| `program` | `CliProgram` | `Resolve` (default: auto-detect) or `Path(PathBuf)` (explicit) | +| `prefix_args` | `Vec` | Args before `--server` (e.g. script path for node) | +| `working_directory` | `PathBuf` | Working directory for CLI process (empty = host process's cwd) | +| `env` | `Vec<(OsString, OsString)>` | Environment variables for CLI process | +| `env_remove` | `Vec` | Environment variables to remove | +| `extra_args` | `Vec` | Extra CLI flags | +| `transport` | `Transport` | `Default`, `Stdio`, `InProcess`, `Tcp`, or `External` | With the default `CliProgram::Resolve`, `Client::start()` resolves the CLI in this order: an explicit `CliProgram::Path(path)`, the `COPILOT_CLI_PATH` env var, then the bundled CLI that was embedded at build time. There is no PATH scanning — if you've opted out of bundling (`default-features = false`) you must supply either `CliProgram::Path` or `COPILOT_CLI_PATH`. @@ -749,7 +749,7 @@ none of them are scheduled for removal. caller-supplied `AsyncRead` / `AsyncWrite`. Useful for testing, in-process embedding, or custom transports. Other SDKs are spawn-only or fixed-stdio. -- **`enum Transport { Stdio, Tcp, External }`** — explicit, exhaustive +- **`enum Transport { Default, Stdio, InProcess, Tcp, External }`** — explicit transport selector on `ClientOptions::transport`. Node/Python/Go rely on conditional config field combinations instead. - **Split `prefix_args` / `extra_args`** on `ClientOptions` — separate @@ -776,7 +776,14 @@ none of them are scheduled for removal. ## Embedded CLI -The SDK provisions the Copilot CLI binary at build time. By default the `bundled-cli` feature embeds the verified binary directly in your compiled crate, so end-user binaries are self-contained — no env var setup, no separate install, just `cargo build`. +The SDK provisions the Copilot CLI binary at build time. By default the +`bundled-cli` feature embeds only the verified CLI executable in your compiled +crate. Enable `bundled-in-process` to additionally embed the native +runtime library and use `Transport::InProcess`: + +```toml +github-copilot-sdk = { version = "0.1", features = ["bundled-in-process"] } +``` For builds that prefer a smaller artifact, disable the `bundled-cli` feature: @@ -795,7 +802,7 @@ github-copilot-sdk = { version = "0.1", default-features = false } > together. > > **Convenience on the build machine only.** As a special case, -> `build.rs` downloads and SHA-verifies the compatible CLI version and +> `build.rs` downloads and integrity-verifies the compatible CLI version and > drops it into the build machine's per-user cache; the runtime > resolver on that same machine will pick it up automatically. This > makes local development and CI ergonomic, but it does **not** carry @@ -812,8 +819,11 @@ github-copilot-sdk = { version = "0.1", default-features = false } The resolved version is baked into the crate via `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` regardless of mode. The runtime resolver consumes it to recompute the on-disk path by convention, so no absolute paths leak into the rlib. -2. **Build time:** `build.rs` downloads the platform-appropriate archive from the [`github/copilot-cli` GitHub Releases](https://github.com/github/copilot-cli/releases) (`copilot-{platform}.tar.gz` on macOS/Linux, `.zip` on Windows), live-fetches the matching `SHA256SUMS.txt`, and verifies the archive hash. Then: - - **`bundled-cli` on (default, release):** embeds the raw archive bytes via `include_bytes!()`. Runtime extracts on first `Client::start()`. +2. **Build time:** `build.rs` downloads the platform-specific npm package and + verifies its `sha512` integrity against the lockfile or publish snapshot. + Then: + - **`bundled-cli` on (default):** creates and embeds a minimal archive containing only the CLI executable. + - **`bundled-in-process` on:** the minimal archive additionally contains the platform-native runtime library (`.dll`, `.so`, or `.dylib`); no other npm package files are embedded. - **`bundled-cli` off:** extracts the binary directly into the platform cache (staging file + atomic rename), idempotent across rebuilds. If the extracted binary is already present at the expected path, the download is skipped entirely — the extracted binary *is* the cache. 3. **Runtime:** in both modes the binary lives at: @@ -899,10 +909,11 @@ Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `win32-x64` ## Features -| Feature | Default | Description | -| -------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bundled-cli` | ✓ | Build-time CLI embedding. Pulls in `tar`+`flate2` (Linux/macOS) or `zip` (Windows). Disable via `default-features = false` to opt out (e.g. when shipping a smaller binary or when always supplying the CLI via `CliProgram::Path` / `COPILOT_CLI_PATH`). | -| `derive` | — | `schema_for::()` for generating JSON Schema from Rust types (adds `schemars`). Enable when defining [tool parameters](#tool-registration). | +| Feature | Default | Description | +| ------- | ------- | ----------- | +| `bundled-cli` | ✓ | Embeds only the CLI executable. Disable via `default-features = false` when supplying the CLI via `CliProgram::Path` or `COPILOT_CLI_PATH`. | +| `bundled-in-process` | — | Enables `Transport::InProcess`, implies `bundled-cli`, and additionally embeds only the platform-native runtime library. | +| `derive` | — | `schema_for::()` for generating JSON Schema from Rust types (adds `schemars`). | ```toml # These examples use registry syntax for illustration; until the crate is @@ -911,7 +922,10 @@ Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `win32-x64` # Default — bundles the Copilot CLI in your binary. github-copilot-sdk = "0.1" -# Opt out of bundling — resolve CLI from COPILOT_CLI_PATH or system PATH instead. +# Enable the in-process transport and bundle its native runtime library. +github-copilot-sdk = { version = "0.1", features = ["bundled-in-process"] } + +# Opt out of bundling — supply the CLI explicitly at runtime. github-copilot-sdk = { version = "0.1", default-features = false } # Derive JSON Schema for tool parameters (adds to default bundled-cli). diff --git a/rust/build.rs b/rust/build.rs index 66d1de7bc8..d04cf2870b 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,709 +1,11 @@ -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; -use std::time::Duration; +#[cfg(feature = "bundled-in-process")] +#[path = "build/in_process.rs"] +mod implementation; -use sha2::Digest; +#[cfg(not(feature = "bundled-in-process"))] +#[path = "build/out_of_process.rs"] +mod implementation; fn main() { - println!("cargo:rerun-if-env-changed=DOCS_RS"); - println!("cargo:rerun-if-env-changed=COPILOT_SKIP_CLI_DOWNLOAD"); - println!("cargo:rerun-if-env-changed=COPILOT_CLI_EXTRACT_DIR"); - println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); - println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); - println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); - println!("cargo:rerun-if-changed=cli-version.txt"); - - // Only declare the lockfile rerun when the lockfile actually exists. - // Cargo treats `rerun-if-changed` for a missing path as "always rerun" - // — so unconditionally declaring this on consumers without a sibling - // `nodejs/` (vendored slots, published crates) would force build.rs - // to re-run on every `cargo build` even when nothing has changed. - // The lockfile path is only the source-of-truth in this repo's - // contributor builds; everywhere else `cli-version.txt` is canonical. - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); - let lockfile = Path::new(&manifest_dir) - .join("..") - .join("nodejs") - .join("package-lock.json"); - if lockfile.is_file() { - println!("cargo:rerun-if-changed={}", lockfile.display()); - } - - // Hard opt-out: disable the entire download / bundle / cache mechanism - // in one step. For consumers who always supply the CLI via - // `CliProgram::Path` or `COPILOT_CLI_PATH` and don't want build.rs to - // touch the network (offline builds, locked-down CI, etc.). Works - // regardless of the `bundled-cli` cargo feature state — with neither - // `has_bundled_cli` nor `has_extracted_cli` emitted, runtime resolution - // falls straight through to `Error::BinaryNotFound` unless an explicit - // path source resolves first. - if std::env::var_os("COPILOT_SKIP_CLI_DOWNLOAD").is_some() { - println!( - "cargo:warning=COPILOT_SKIP_CLI_DOWNLOAD is set — skipping CLI download/bundle/cache" - ); - return; - } - - // docs.rs builds in a sandboxed environment without network access. - // Skip the CLI download so documentation can be generated successfully. - if std::env::var_os("DOCS_RS").is_some() { - println!("cargo:warning=DOCS_RS is set — skipping CLI download/bundle/cache"); - return; - } - - let Some(platform) = target_platform() else { - println!("cargo:warning=Unsupported target platform for Copilot CLI bundling — skipping"); - return; - }; - - let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR is always set by cargo"); - let out = Path::new(&out_dir); - - // Resolve version + per-asset SHA-256 from one of two sources, in order: - // 1. `cli-version.txt` snapshot at the crate root (published-crate - // consumer; generated by the publish workflow). Combined format: - // `version=X` line + per-asset hash lines. Committing the hashes - // makes the publish workflow the trust boundary — an attacker who - // later re-points the release tag can't silently poison consumer - // builds. - // 2. Sibling `../nodejs/package-lock.json` (contributor build inside - // the github/copilot-sdk repo; live SHA256SUMS.txt fetch). Matches - // the .NET `_GetCopilotCliVersion` MSBuild target and the Go - // `cmd/bundler` tool. - let (version, expected_hash) = resolve_version_and_hash(platform.asset_name); - - // Bake the version into the crate regardless of mode. This is the - // single source of truth for "what CLI version did build.rs target", - // consumed by both the embed-mode path computation in embeddedcli.rs - // and the runtime path computation in resolve.rs (when `bundled-cli` - // is off). It's a small, machine-independent datum: no absolute - // paths, no username/home leakage, so sccache / cross-machine - // `target/` reuse stays cache-coherent. - println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); - - let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); - let cache_dir = std::env::var("BUNDLED_CLI_CACHE_DIR") - .ok() - .map(std::path::PathBuf::from); - - // Versioned cache key since copilot asset names don't include the version. - let cache_key = format!("v{version}-{}", platform.asset_name); - - if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { - // Embed mode: we need the archive bytes to bake into the rlib, so - // always run the download (cache hit short-circuits inside - // `cached_download`). - let archive = cached_download( - &format!("{base_url}/{}", platform.asset_name), - &cache_key, - &expected_hash, - &cache_dir, - ); - verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); - emit_embedded(out, &archive); - println!("cargo:rustc-cfg=has_bundled_cli"); - } else { - // With `bundled-cli` off the extracted binary *is* the cache. - // Skip the upstream download entirely when it already exists at - // the expected path. No two separate caches. - // - // Runtime resolution (see `src/resolve.rs::extracted_cli_path`) - // recomputes this same path from `COPILOT_SDK_CLI_VERSION` + the - // OS-derived binary name + optional `COPILOT_CLI_EXTRACT_DIR`, - // so we don't bake an absolute path into the crate. - let install_dir = extracted_install_dir(&version); - let final_path = install_dir.join(platform.binary_name); - - // Invalidate build.rs whenever the cached binary disappears (cache GC, - // manual rm, OS reset, switching extract dir). Without this, cargo - // replays the saved `has_extracted_cli` cfg from its build-script - // output cache even when the file is gone, and runtime resolution - // fails with BinaryNotFound. - println!("cargo:rerun-if-changed={}", final_path.display()); - - if !final_path.is_file() { - let archive = cached_download( - &format!("{base_url}/{}", platform.asset_name), - &cache_key, - &expected_hash, - &cache_dir, - ); - verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); - extract_to_cache(&archive, &install_dir, platform); - } - - // Re-check after potential download+extract above; not an `else` - // because we need to verify the extraction actually produced the file. - if final_path.is_file() { - println!("cargo:rustc-cfg=has_extracted_cli"); - } - } -} - -/// Install directory used when `bundled-cli` is off. Mirrors the runtime -/// convention in `src/resolve.rs::extracted_cli_path`: both sides MUST -/// compute the same path from the same inputs, otherwise the runtime -/// resolver won't find what build.rs extracted. -/// -/// If `COPILOT_CLI_EXTRACT_DIR` is set the binary lives directly under -/// that directory (no per-version subdir) — useful for vendored slots and -/// for `.cargo/config.toml [env]`-style pinning that's symmetric between -/// build-time write and runtime read. Otherwise the binary lives under -/// `/github-copilot-sdk/cli//`. -fn extracted_install_dir(version: &str) -> PathBuf { - if let Some(custom) = std::env::var_os("COPILOT_CLI_EXTRACT_DIR") { - PathBuf::from(custom) - } else { - let cache = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); - cache - .join("github-copilot-sdk") - .join("cli") - .join(sanitize_version(version)) - } -} - -/// Emit the `bundled_cli.rs` glue + `copilot_cli.archive` blob into `OUT_DIR` -/// for embed mode (`bundled-cli` cargo feature on). The version is exposed -/// crate-wide via the unconditional `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` -/// emit; the binary name is OS-derived at runtime — so all we need to -/// generate here is the archive blob include. -fn emit_embedded(out: &Path, archive: &[u8]) { - std::fs::write(out.join("copilot_cli.archive"), archive) - .expect("failed to write copilot_cli.archive"); - - let generated = r#"// Auto-generated by github-copilot-sdk build.rs. Do not edit. -pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); -"#; - - std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); -} - -/// Resolve the CLI version and the expected SHA-256 hash for the current -/// target's archive. Picks one of two sources in order. Panics with a clear -/// error if neither is available. -fn resolve_version_and_hash(asset_name: &str) -> (String, String) { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); - - // 1. Snapshot file at the crate root (published-crate consumer, - // vendored-slot consumer). Combined version + per-asset hashes. - let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); - if snapshot.is_file() { - let contents = std::fs::read_to_string(&snapshot) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); - return parse_snapshot(&contents, asset_name) - .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); - } - - // 2. Lockfile fallback (contributor build inside github/copilot-sdk) — - // read version, fetch live SHA256SUMS. - let lockfile = Path::new(&manifest_dir) - .join("..") - .join("nodejs") - .join("package-lock.json"); - if lockfile.is_file() { - let version = read_version_from_package_lock(&lockfile); - let hash = fetch_live_sha256(&version, asset_name); - return (version, hash); - } - - panic!( - "Could not resolve the Copilot CLI version.\n\ - Tried:\n\ - - {} (missing)\n\ - - {} (missing)\n\ - In a published crate or vendored slot, `cli-version.txt` should be present.\n\ - Inside the github/copilot-sdk repo, `../nodejs/package-lock.json` is the source.", - snapshot.display(), - lockfile.display(), - ); -} - -/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per -/// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map -/// asset filename to hex SHA-256. Blank lines and lines starting with `#` -/// are skipped. -fn parse_snapshot(contents: &str, asset_name: &str) -> Result<(String, String), String> { - let mut version: Option = None; - let mut hash: Option = None; - for (line_no, raw) in contents.lines().enumerate() { - let line = raw.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let (key, value) = line - .split_once('=') - .ok_or_else(|| format!("line {}: expected `key=value`, got `{raw}`", line_no + 1))?; - match key.trim() { - "version" => version = Some(value.trim().to_string()), - k if k == asset_name => hash = Some(value.trim().to_string()), - _ => {} - } - } - let version = version.ok_or("missing `version=` line")?; - let hash = hash.ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; - Ok((version, hash)) -} - -/// Read the `@github/copilot` version from `nodejs/package-lock.json`. -fn read_version_from_package_lock(path: &Path) -> String { - let contents = std::fs::read_to_string(path) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); - // Minimal JSON walk: find `"node_modules/@github/copilot"` object and - // its `"version"` field. Full JSON parsing keeps build.rs dep-light by - // using a regex; the file is generated by npm and we're matching an - // exact key path. - let key = "\"node_modules/@github/copilot\""; - let key_pos = contents - .find(key) - .unwrap_or_else(|| panic!("{} does not contain {key}", path.display())); - let after_key = &contents[key_pos + key.len()..]; - let version_key = "\"version\""; - let v_pos = after_key - .find(version_key) - .unwrap_or_else(|| panic!("no `version` field found near {key} in {}", path.display())); - let after_v = &after_key[v_pos + version_key.len()..]; - let q1 = after_v.find('"').expect("malformed version"); - let after_q1 = &after_v[q1 + 1..]; - let q2 = after_q1.find('"').expect("malformed version"); - after_q1[..q2].to_string() -} - -/// Fetch the live `SHA256SUMS.txt` for the given version from GitHub Releases -/// and pluck out the entry for `asset_name`. -fn fetch_live_sha256(version: &str, asset_name: &str) -> String { - let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); - let checksums_url = format!("{base_url}/SHA256SUMS.txt"); - let checksums = download_with_retry(&checksums_url); - let checksums_text = - std::str::from_utf8(&checksums).expect("checksums file is not valid UTF-8"); - find_sha256_for_asset(checksums_text, asset_name) -} - -#[derive(Clone, Copy)] -struct Platform { - asset_name: &'static str, - binary_name: &'static str, -} - -fn target_platform() -> Option { - let os = std::env::var("CARGO_CFG_TARGET_OS").ok()?; - let arch = std::env::var("CARGO_CFG_TARGET_ARCH").ok()?; - - match (os.as_str(), arch.as_str()) { - ("macos", "aarch64") => Some(Platform { - asset_name: "copilot-darwin-arm64.tar.gz", - binary_name: "copilot", - }), - ("macos", "x86_64") => Some(Platform { - asset_name: "copilot-darwin-x64.tar.gz", - binary_name: "copilot", - }), - ("linux", "x86_64") => Some(Platform { - asset_name: "copilot-linux-x64.tar.gz", - binary_name: "copilot", - }), - ("linux", "aarch64") => Some(Platform { - asset_name: "copilot-linux-arm64.tar.gz", - binary_name: "copilot", - }), - ("windows", "x86_64") => Some(Platform { - asset_name: "copilot-win32-x64.zip", - binary_name: "copilot.exe", - }), - ("windows", "aarch64") => Some(Platform { - asset_name: "copilot-win32-arm64.zip", - binary_name: "copilot.exe", - }), - _ => None, - } -} - -/// Write the single binary entry from `archive` to -/// `/` and return the resulting path. -/// Idempotent — returns the existing path if a previous build already -/// populated the target. -/// -/// Uses file-level staging + atomic rename so a concurrent reader during -/// a parallel `cargo build` race never observes a partially-written -/// binary. `fs::rename` for files is atomic on both Unix and Windows -/// (Windows uses `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`); for -/// directories it is not, which is why we stage at file granularity. -fn extract_to_cache(archive: &[u8], install_dir: &Path, platform: Platform) -> PathBuf { - let final_path = install_dir.join(platform.binary_name); - - // Caller already gated on `final_path.is_file()`; this is a safety - // net for any future caller that forgets. - if final_path.is_file() { - return final_path; - } - - std::fs::create_dir_all(install_dir).unwrap_or_else(|e| { - panic!( - "failed to create install dir {}: {e}", - install_dir.display() - ) - }); - - let bytes = extract_binary_bytes(archive, platform); - - // Staging file is a sibling of the final binary so the rename stays - // on the same filesystem (cross-fs rename is not atomic). PID + nanos - // disambiguate concurrent builds racing on the same cache. - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let staging_path = install_dir.join(format!( - ".{}.staging-{}-{nanos}", - platform.binary_name, - std::process::id(), - )); - - { - let mut f = std::fs::File::create(&staging_path).unwrap_or_else(|e| { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to create staging file {}: {e}", - staging_path.display() - ); - }); - - if let Err(e) = f.write_all(&bytes) { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to write staging file {}: {e}", - staging_path.display() - ); - } - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - if let Err(e) = f.set_permissions(std::fs::Permissions::from_mode(0o755)) { - let _ = std::fs::remove_file(&staging_path); - panic!("failed to chmod {}: {e}", staging_path.display()); - } - } - - // Backdate the staged binary to the Unix epoch before it lands. We emit - // `cargo:rerun-if-changed` on `final_path` (see caller) so a *deleted* - // cache binary forces a re-extract — but cargo stamps the build-script - // `output` reference when the script is spawned, seconds before this - // freshly-downloaded binary is written. A current mtime would therefore - // be *newer* than that reference, so the next identical `cargo` - // invocation would see the watched file as "changed" and pointlessly - // rerun build.rs + recompile the crate + relink every downstream crate. - // Pinning to the epoch keeps the file unambiguously older than any real - // build reference; `rename` preserves mtime (same inode), so it lands - // already-backdated and a no-change rebuild stays a true no-op. The - // deleted-file recovery contract is untouched: a missing file can't be - // stat'd, so cargo still treats it as stale and reruns regardless. - // - // Best-effort: a filesystem that refuses the epoch (e.g. FAT's 1980 floor - // clamps it — still older than any real reference) or rejects the call - // just reverts to the pre-fix redundant-rebuild behaviour, never a broken - // build. - if let Err(e) = f.set_modified(std::time::SystemTime::UNIX_EPOCH) { - println!( - "cargo:warning=Could not backdate {} (a redundant rebuild may occur): {e}", - staging_path.display() - ); - } - } - - // Atomic file-replace on both Unix and Windows. If a concurrent build - // already produced the same file the rename overwrites it; the bytes - // are SHA-verified-identical so replacement is safe. - if let Err(e) = std::fs::rename(&staging_path, &final_path) { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to rename {} -> {}: {e}", - staging_path.display(), - final_path.display() - ); - } - - // Surface where the binary landed so contributors can find it. Quiet - // on the hot path: the caller's `is_file()` short-circuit (and the - // safety net at the top of this function) means this only fires on a - // true cache miss. - println!( - "cargo:warning=Extracted Copilot CLI to {}", - final_path.display() - ); - - final_path -} - -/// Replace characters outside `[a-zA-Z0-9._-]` with `_` so the version -/// string is always safe to use as a path component. Kept in sync with -/// `embeddedcli::sanitize_version` and `resolve::sanitize_version` so all -/// three resolve to the same cache directory for any given version. -fn sanitize_version(version: &str) -> String { - version - .chars() - .map(|c| match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => c, - _ => '_', - }) - .collect() -} - -/// Extract the single `binary_name` entry from the release archive. Reused -/// between embed mode's `verify_binary_present_in_archive` and the -/// `extract_to_cache` path used when `bundled-cli` is off. Panics if the -/// entry isn't found — callers have already invoked -/// `verify_binary_present_in_archive`. -fn extract_binary_bytes(archive: &[u8], platform: Platform) -> Vec { - if platform.asset_name.ends_with(".zip") { - let cursor = std::io::Cursor::new(archive); - let mut zip = zip::ZipArchive::new(cursor) - .unwrap_or_else(|e| panic!("failed to open zip archive: {e}")); - for i in 0..zip.len() { - let mut entry = zip - .by_index(i) - .unwrap_or_else(|e| panic!("failed to read zip entry {i}: {e}")); - let name = entry.name().to_string(); - if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) - { - let mut bytes = Vec::with_capacity(entry.size() as usize); - std::io::copy(&mut entry, &mut bytes) - .unwrap_or_else(|e| panic!("failed to read zip entry bytes: {e}")); - return bytes; - } - } - } else { - let gz = flate2::read::GzDecoder::new(archive); - let mut tar = tar::Archive::new(gz); - for entry in tar - .entries() - .unwrap_or_else(|e| panic!("failed to read tar entries: {e}")) - { - let mut entry = entry.unwrap_or_else(|e| panic!("failed to read tar entry: {e}")); - let path = entry - .path() - .unwrap_or_else(|e| panic!("failed to read tar entry path: {e}")); - let name = path.to_string_lossy().into_owned(); - if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) - { - let mut bytes = Vec::with_capacity(entry.size() as usize); - entry - .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read tar entry bytes: {e}")); - return bytes; - } - } - } - panic!( - "binary `{}` not found in archive `{}`", - platform.binary_name, platform.asset_name - ); -} - -/// Read a file from the download cache, or download it (with retries) and save -/// to cache. Verifies SHA-256 on every path. Evicts stale/corrupt cache entries -/// automatically. Cache I/O failures are treated as cache misses — they never -/// break the build. -fn cached_download( - url: &str, - cache_key: &str, - expected_hash: &str, - cache_dir: &Option, -) -> Vec { - if let Some(dir) = cache_dir { - let cached_path = dir.join(cache_key); - if cached_path.is_file() { - match std::fs::read(&cached_path) { - Ok(data) if hex_sha256(&data) == expected_hash => { - // Silent cache hit — nothing to surface. - return data; - } - Ok(_) => { - println!("cargo:warning=Cached archive hash mismatch, re-downloading"); - let _ = std::fs::remove_file(&cached_path); - } - Err(e) => { - println!( - "cargo:warning=Failed to read cache {}, re-downloading: {e}", - cached_path.display() - ); - } - } - } - } - - println!("cargo:warning=Downloading {url}"); - let data = download_with_retry(url); - let actual_hash = hex_sha256(&data); - if actual_hash != expected_hash { - panic!( - "Archive integrity check failed for {url}!\n expected: {expected_hash}\n actual: {actual_hash}\n \ - This could indicate a corrupted download or a supply-chain attack." - ); - } - - if let Some(dir) = cache_dir { - if let Err(e) = std::fs::create_dir_all(dir) { - println!( - "cargo:warning=Failed to create cache directory {}: {e}", - dir.display() - ); - } else { - let cached_path = dir.join(cache_key); - println!("cargo:warning=Caching archive at {}", cached_path.display()); - if let Err(e) = std::fs::write(&cached_path, &data) { - println!( - "cargo:warning=Failed to write cache file {}: {e}", - cached_path.display() - ); - } - } - } - - data -} - -/// Maximum number of HTTP attempts (one initial + this many retries on transient errors). -const MAX_RETRIES: u32 = 3; - -/// Download `url` with bounded retries on transient network errors. Backoff is -/// exponential starting at 1s. 4xx responses fail fast; 5xx and connect/read -/// errors are retried. -fn download_with_retry(url: &str) -> Vec { - let mut attempt = 0u32; - loop { - attempt += 1; - match try_download(url) { - Ok(bytes) => return bytes, - Err(err) if err.transient && attempt <= MAX_RETRIES => { - let backoff = Duration::from_secs(1u64 << (attempt - 1)); - println!( - "cargo:warning=Transient download failure for {url} (attempt {attempt}/{}): {} — retrying in {}s", - MAX_RETRIES + 1, - err.message, - backoff.as_secs(), - ); - std::thread::sleep(backoff); - } - Err(err) => panic!("Failed to download {url}: {}", err.message), - } - } -} - -struct DownloadError { - message: String, - transient: bool, -} - -fn try_download(url: &str) -> Result, DownloadError> { - let agent = ureq::AgentBuilder::new() - .timeout_connect(Duration::from_secs(30)) - .timeout_read(Duration::from_secs(120)) - .build(); - - match agent.get(url).call() { - Ok(response) => { - let mut bytes = Vec::new(); - response - .into_reader() - .read_to_end(&mut bytes) - .map_err(|e| DownloadError { - message: format!("read error: {e}"), - transient: true, - })?; - Ok(bytes) - } - // 5xx — server-side, treat as transient. - Err(ureq::Error::Status(code, response)) if (500..600).contains(&code) => { - Err(DownloadError { - message: format!("HTTP {code} {}", response.status_text()), - transient: true, - }) - } - // 4xx — client-side, fail fast. - Err(ureq::Error::Status(code, response)) => Err(DownloadError { - message: format!("HTTP {code} {}", response.status_text()), - transient: false, - }), - // Transport-layer (DNS, connect, TLS, read timeout) — treat as transient. - Err(ureq::Error::Transport(t)) => Err(DownloadError { - message: format!("transport error: {t}"), - transient: true, - }), - } -} - -fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { - for line in sums.lines() { - // Format: " " (two spaces) - if let Some((hash, name)) = line.split_once(" ") - && name.trim() == asset_name - { - return hash.trim().to_string(); - } - } - panic!("SHA256SUMS.txt does not contain an entry for {asset_name}"); -} - -fn sha256(data: &[u8]) -> [u8; 32] { - let mut hasher = sha2::Sha256::new(); - hasher.update(data); - hasher.finalize().into() -} - -/// Walks the downloaded archive at build time to confirm an entry matching -/// `binary_name` exists. Panics with a clear message if not — defends against -/// silent breakage if the upstream archive layout ever changes. -fn verify_binary_present_in_archive(archive: &[u8], binary_name: &str, asset_name: &str) { - let found = if asset_name.ends_with(".zip") { - archive_contains_zip_entry(archive, binary_name) - } else { - archive_contains_tar_entry(archive, binary_name) - }; - if !found { - panic!( - "Copilot CLI archive `{asset_name}` does not contain an entry named `{binary_name}`. \ - The upstream archive layout may have changed; runtime extraction would fail. \ - Update `verify_binary_present_in_archive` in build.rs and the matching `extract_binary` in src/embeddedcli.rs." - ); - } -} - -fn archive_contains_tar_entry(targz: &[u8], binary_name: &str) -> bool { - let gz = flate2::read::GzDecoder::new(targz); - let mut archive = tar::Archive::new(gz); - let Ok(entries) = archive.entries() else { - return false; - }; - for entry in entries.flatten() { - let Ok(path) = entry.path() else { - continue; - }; - let name = path.to_string_lossy(); - if name == binary_name || name.ends_with(&format!("/{binary_name}")) { - return true; - } - } - false -} - -fn archive_contains_zip_entry(zip_bytes: &[u8], binary_name: &str) -> bool { - let cursor = std::io::Cursor::new(zip_bytes); - let Ok(mut archive) = zip::ZipArchive::new(cursor) else { - return false; - }; - for i in 0..archive.len() { - let Ok(entry) = archive.by_index(i) else { - continue; - }; - let name = entry.name(); - if name == binary_name || name.ends_with(&format!("/{binary_name}")) { - return true; - } - } - false -} - -fn hex_sha256(data: &[u8]) -> String { - sha256(data).iter().map(|b| format!("{b:02x}")).collect() + implementation::main(); } diff --git a/rust/build/in_process.rs b/rust/build/in_process.rs new file mode 100644 index 0000000000..5e7a773266 --- /dev/null +++ b/rust/build/in_process.rs @@ -0,0 +1,712 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use base64::Engine; +use sha2::Digest; + +pub(crate) fn main() { + println!("cargo:rerun-if-env-changed=DOCS_RS"); + println!("cargo:rerun-if-env-changed=COPILOT_SKIP_CLI_DOWNLOAD"); + println!("cargo:rerun-if-env-changed=COPILOT_CLI_EXTRACT_DIR"); + println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); + println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); + println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); + println!("cargo:rerun-if-changed=cli-version-in-process.txt"); + + // Only declare the lockfile rerun when the lockfile actually exists. + // Cargo treats `rerun-if-changed` for a missing path as "always rerun" + // — so unconditionally declaring this on consumers without a sibling + // `nodejs/` (vendored slots, published crates) would force build.rs + // to re-run on every `cargo build` even when nothing has changed. + // The lockfile path is only the source-of-truth in this repo's + // contributor builds; everywhere else `cli-version-in-process.txt` is canonical. + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + let lockfile = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package-lock.json"); + if lockfile.is_file() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + } + + // Hard opt-out: disable the entire download / bundle / cache mechanism + // in one step. For consumers who always supply the CLI via + // `CliProgram::Path` or `COPILOT_CLI_PATH` and don't want build.rs to + // touch the network (offline builds, locked-down CI, etc.). Works + // regardless of the `bundled-cli` cargo feature state — with neither + // `has_bundled_cli` nor `has_extracted_cli` emitted, runtime resolution + // falls straight through to `Error::BinaryNotFound` unless an explicit + // path source resolves first. + if std::env::var_os("COPILOT_SKIP_CLI_DOWNLOAD").is_some() { + println!( + "cargo:warning=COPILOT_SKIP_CLI_DOWNLOAD is set — skipping CLI download/bundle/cache" + ); + return; + } + + // docs.rs builds in a sandboxed environment without network access. + // Skip the CLI download so documentation can be generated successfully. + if std::env::var_os("DOCS_RS").is_some() { + println!("cargo:warning=DOCS_RS is set — skipping CLI download/bundle/cache"); + return; + } + + let Some(platform) = target_platform() else { + println!("cargo:warning=Unsupported target platform for Copilot CLI bundling — skipping"); + return; + }; + + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR is always set by cargo"); + let out = Path::new(&out_dir); + + // Resolve version + npm integrity from one of two sources, in order: + // 1. `cli-version-in-process.txt` snapshot at the crate root (published-crate + // consumer; generated by the publish workflow). Combined format: + // `version=X` line + per-package integrity lines. Committing these + // makes the publish workflow the trust boundary — an attacker who + // later re-points the release tag can't silently poison consumer + // builds. + // 2. Sibling `../nodejs/package-lock.json` (contributor build inside + // the github/copilot-sdk repo), whose platform-package integrity is + // the same trust source npm uses. + let (version, expected_integrity) = resolve_version_and_integrity(platform.package_name); + + // Bake the version into the crate regardless of mode. This is the + // single source of truth for "what CLI version did build.rs target", + // consumed by both the embed-mode path computation in embeddedcli.rs + // and the runtime path computation in resolve.rs (when `bundled-cli` + // is off). It's a small, machine-independent datum: no absolute + // paths, no username/home leakage, so sccache / cross-machine + // `target/` reuse stays cache-coherent. + println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); + + let archive_name = format!("{}-{version}.tgz", platform.package_name); + let download_url = format!( + "https://registry.npmjs.org/@github/{}/-/{}", + platform.package_name, archive_name + ); + let cache_dir = std::env::var("BUNDLED_CLI_CACHE_DIR") + .ok() + .map(std::path::PathBuf::from); + + let cache_key = format!("v{version}-{archive_name}"); + let include_runtime = std::env::var_os("CARGO_FEATURE_BUNDLED_IN_PROCESS").is_some(); + + if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { + let archive = cached_download(&download_url, &cache_key, &expected_integrity, &cache_dir); + verify_binary_present_in_archive(&archive, platform.binary_name, &archive_name); + emit_embedded(out, &archive, platform, include_runtime); + println!("cargo:rustc-cfg=has_bundled_cli"); + } else { + // With `bundled-cli` off the extracted binary *is* the cache. + // Skip the upstream download entirely when it already exists at + // the expected path. No two separate caches. + // + // Runtime resolution (see `src/resolve.rs::extracted_cli_path`) + // recomputes this same path from `COPILOT_SDK_CLI_VERSION` + the + // OS-derived binary name + optional `COPILOT_CLI_EXTRACT_DIR`, + // so we don't bake an absolute path into the crate. + let install_dir = extracted_install_dir(&version); + let final_path = install_dir.join(platform.binary_name); + + // Invalidate build.rs whenever the cached binary disappears (cache GC, + // manual rm, OS reset, switching extract dir). Without this, cargo + // replays the saved `has_extracted_cli` cfg from its build-script + // output cache even when the file is gone, and runtime resolution + // fails with BinaryNotFound. + println!("cargo:rerun-if-changed={}", final_path.display()); + + if !final_path.is_file() { + let archive = + cached_download(&download_url, &cache_key, &expected_integrity, &cache_dir); + verify_binary_present_in_archive(&archive, platform.binary_name, &archive_name); + extract_to_cache(&archive, &install_dir, platform); + } + + // Re-check after potential download+extract above; not an `else` + // because we need to verify the extraction actually produced the file. + if final_path.is_file() { + println!("cargo:rustc-cfg=has_extracted_cli"); + } + } +} + +/// Install directory used when `bundled-cli` is off. Mirrors the runtime +/// convention in `src/resolve.rs::extracted_cli_path`: both sides MUST +/// compute the same path from the same inputs, otherwise the runtime +/// resolver won't find what build.rs extracted. +/// +/// If `COPILOT_CLI_EXTRACT_DIR` is set the binary lives directly under +/// that directory (no per-version subdir) — useful for vendored slots and +/// for `.cargo/config.toml [env]`-style pinning that's symmetric between +/// build-time write and runtime read. Otherwise the binary lives under +/// `/github-copilot-sdk/cli//`. +fn extracted_install_dir(version: &str) -> PathBuf { + if let Some(custom) = std::env::var_os("COPILOT_CLI_EXTRACT_DIR") { + PathBuf::from(custom) + } else { + let cache = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); + cache + .join("github-copilot-sdk") + .join("cli") + .join(sanitize_version(version)) + } +} + +/// Emit the `bundled_cli.rs` glue + `copilot_cli.archive` blob into `OUT_DIR` +/// for embed mode (`bundled-cli` cargo feature on). The version is exposed +/// crate-wide via the unconditional `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` +/// emit; the binary name is OS-derived at runtime — so all we need to +/// generate here is the archive blob include. +fn emit_embedded(out: &Path, package: &[u8], platform: Platform, include_runtime: bool) { + let archive = build_embedded_archive(package, platform, include_runtime); + std::fs::write(out.join("copilot_cli.archive"), archive) + .expect("failed to write copilot_cli.archive"); + + let generated = r#"// Auto-generated by github-copilot-sdk build.rs. Do not edit. +pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); +"#; + + std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); +} + +fn build_embedded_archive(package: &[u8], platform: Platform, include_runtime: bool) -> Vec { + let encoder = flate2::GzBuilder::new() + .mtime(0) + .write(Vec::new(), flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + append_archive_file( + &mut archive, + platform.binary_name, + &extract_binary_bytes(package, platform), + 0o755, + ); + if include_runtime { + let runtime = extract_runtime_library_bytes(package).unwrap_or_else(|| { + panic!( + "package `{}` does not contain the native runtime library required by the `bundled-in-process` feature", + platform.package_name + ) + }); + append_archive_file( + &mut archive, + platform.runtime_library_name(), + &runtime, + 0o644, + ); + } + let encoder = archive + .into_inner() + .expect("failed to finish minimal embedded CLI archive"); + encoder + .finish() + .expect("failed to compress minimal embedded CLI archive") +} + +fn append_archive_file( + archive: &mut tar::Builder, + path: &str, + bytes: &[u8], + mode: u32, +) { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes) + .unwrap_or_else(|e| panic!("failed to add `{path}` to embedded CLI archive: {e}")); +} + +/// Resolve the CLI version and npm integrity for the current target's +/// platform package. Picks one of two sources in order. Panics with a clear +/// error if neither is available. +fn resolve_version_and_integrity(package_name: &str) -> (String, String) { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + + // 1. Snapshot file at the crate root (published-crate consumer, + // vendored-slot consumer). Combined version + per-asset hashes. + let snapshot = Path::new(&manifest_dir).join("cli-version-in-process.txt"); + if snapshot.is_file() { + let contents = std::fs::read_to_string(&snapshot) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); + return parse_snapshot(&contents, package_name) + .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); + } + + // 2. Lockfile fallback (contributor build inside github/copilot-sdk). + let lockfile = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package-lock.json"); + if lockfile.is_file() { + return read_version_and_integrity_from_package_lock(&lockfile, package_name); + } + + panic!( + "Could not resolve the Copilot CLI version.\n\ + Tried:\n\ + - {} (missing)\n\ + - {} (missing)\n\ + In a published crate or vendored slot, `cli-version-in-process.txt` should be present.\n\ + Inside the github/copilot-sdk repo, `../nodejs/package-lock.json` is the source.", + snapshot.display(), + lockfile.display(), + ); +} + +/// Parse the `cli-version-in-process.txt` snapshot file. Format is one `key=value` per +/// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map +/// platform package name to npm integrity. Blank lines and lines starting with `#` +/// are skipped. +fn parse_snapshot(contents: &str, package_name: &str) -> Result<(String, String), String> { + let mut version: Option = None; + let mut integrity: Option = None; + for (line_no, raw) in contents.lines().enumerate() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + return Err(format!( + "line {}: expected `key=value`, got `{raw}`", + line_no + 1 + )); + }; + match key.trim() { + "version" => version = Some(value.trim().to_string()), + k if k == package_name => integrity = Some(value.trim().to_string()), + _ => {} + } + } + let version = version.ok_or("missing `version=` line")?; + let integrity = + integrity.ok_or_else(|| format!("missing integrity for package `{package_name}`"))?; + Ok((version, integrity)) +} + +fn read_version_and_integrity_from_package_lock( + path: &Path, + package_name: &str, +) -> (String, String) { + let contents = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + let lock: serde_json::Value = serde_json::from_str(&contents) + .unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display())); + let cli_key = "node_modules/@github/copilot"; + let version = lock["packages"][cli_key]["version"] + .as_str() + .unwrap_or_else(|| panic!("{cli_key} has no version in {}", path.display())); + let platform_key = format!("node_modules/@github/{package_name}"); + let integrity = lock["packages"][&platform_key]["integrity"] + .as_str() + .unwrap_or_else(|| panic!("{platform_key} has no integrity in {}", path.display())); + (version.to_string(), integrity.to_string()) +} + +#[derive(Clone, Copy)] +struct Platform { + package_name: &'static str, + binary_name: &'static str, +} + +impl Platform { + fn runtime_library_name(&self) -> &'static str { + if self.package_name.contains("win32") { + "copilot_runtime.dll" + } else if self.package_name.contains("darwin") { + "libcopilot_runtime.dylib" + } else { + "libcopilot_runtime.so" + } + } +} + +fn target_platform() -> Option { + let os = std::env::var("CARGO_CFG_TARGET_OS").ok()?; + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").ok()?; + + match (os.as_str(), arch.as_str()) { + ("macos", "aarch64") => Some(Platform { + package_name: "copilot-darwin-arm64", + binary_name: "copilot", + }), + ("macos", "x86_64") => Some(Platform { + package_name: "copilot-darwin-x64", + binary_name: "copilot", + }), + ("linux", "x86_64") => Some(Platform { + package_name: "copilot-linux-x64", + binary_name: "copilot", + }), + ("linux", "aarch64") => Some(Platform { + package_name: "copilot-linux-arm64", + binary_name: "copilot", + }), + ("windows", "x86_64") => Some(Platform { + package_name: "copilot-win32-x64", + binary_name: "copilot.exe", + }), + ("windows", "aarch64") => Some(Platform { + package_name: "copilot-win32-arm64", + binary_name: "copilot.exe", + }), + _ => None, + } +} + +/// Write the single binary entry from `archive` to +/// `/` and return the resulting path. +/// Idempotent — returns the existing path if a previous build already +/// populated the target. +/// +/// Uses file-level staging + atomic rename so a concurrent reader during +/// a parallel `cargo build` race never observes a partially-written +/// binary. `fs::rename` for files is atomic on both Unix and Windows +/// (Windows uses `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`); for +/// directories it is not, which is why we stage at file granularity. +fn extract_to_cache(archive: &[u8], install_dir: &Path, platform: Platform) -> PathBuf { + let final_path = install_dir.join(platform.binary_name); + + // Caller already gated on `final_path.is_file()`; this is a safety + // net for any future caller that forgets. + if final_path.is_file() { + return final_path; + } + + std::fs::create_dir_all(install_dir).unwrap_or_else(|e| { + panic!( + "failed to create install dir {}: {e}", + install_dir.display() + ) + }); + + let bytes = extract_binary_bytes(archive, platform); + + // Staging file is a sibling of the final binary so the rename stays + // on the same filesystem (cross-fs rename is not atomic). PID + nanos + // disambiguate concurrent builds racing on the same cache. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let staging_path = install_dir.join(format!( + ".{}.staging-{}-{nanos}", + platform.binary_name, + std::process::id(), + )); + + { + let mut f = std::fs::File::create(&staging_path).unwrap_or_else(|e| { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to create staging file {}: {e}", + staging_path.display() + ); + }); + + if let Err(e) = f.write_all(&bytes) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to write staging file {}: {e}", + staging_path.display() + ); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Err(e) = f.set_permissions(std::fs::Permissions::from_mode(0o755)) { + let _ = std::fs::remove_file(&staging_path); + panic!("failed to chmod {}: {e}", staging_path.display()); + } + } + + // Backdate the staged binary to the Unix epoch before it lands. We emit + // `cargo:rerun-if-changed` on `final_path` (see caller) so a *deleted* + // cache binary forces a re-extract — but cargo stamps the build-script + // `output` reference when the script is spawned, seconds before this + // freshly-downloaded binary is written. A current mtime would therefore + // be *newer* than that reference, so the next identical `cargo` + // invocation would see the watched file as "changed" and pointlessly + // rerun build.rs + recompile the crate + relink every downstream crate. + // Pinning to the epoch keeps the file unambiguously older than any real + // build reference; `rename` preserves mtime (same inode), so it lands + // already-backdated and a no-change rebuild stays a true no-op. The + // deleted-file recovery contract is untouched: a missing file can't be + // stat'd, so cargo still treats it as stale and reruns regardless. + // + // Best-effort: a filesystem that refuses the epoch (e.g. FAT's 1980 floor + // clamps it — still older than any real reference) or rejects the call + // just reverts to the pre-fix redundant-rebuild behaviour, never a broken + // build. + if let Err(e) = f.set_modified(std::time::SystemTime::UNIX_EPOCH) { + println!( + "cargo:warning=Could not backdate {} (a redundant rebuild may occur): {e}", + staging_path.display() + ); + } + } + + // Atomic file-replace on both Unix and Windows. If a concurrent build + // already produced the same file the rename overwrites it; the bytes + // are integrity-verified-identical so replacement is safe. + if let Err(e) = std::fs::rename(&staging_path, &final_path) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to rename {} -> {}: {e}", + staging_path.display(), + final_path.display() + ); + } + + // Surface where the binary landed so contributors can find it. Quiet + // on the hot path: the caller's `is_file()` short-circuit (and the + // safety net at the top of this function) means this only fires on a + // true cache miss. + println!( + "cargo:warning=Extracted Copilot CLI to {}", + final_path.display() + ); + + final_path +} + +fn extract_runtime_library_bytes(archive: &[u8]) -> Option> { + let gz = flate2::read::GzDecoder::new(archive); + let mut tar = tar::Archive::new(gz); + for entry in tar.entries().ok()? { + let mut entry = entry.ok()?; + let name = entry.path().ok()?.to_string_lossy().into_owned(); + if name == "runtime.node" || name.ends_with("/runtime.node") { + let mut bytes = Vec::with_capacity(entry.size() as usize); + entry.read_to_end(&mut bytes).ok()?; + return Some(bytes); + } + } + None +} + +/// Replace characters outside `[a-zA-Z0-9._-]` with `_` so the version +/// string is always safe to use as a path component. Kept in sync with +/// `embeddedcli::sanitize_version` and `resolve::sanitize_version` so all +/// three resolve to the same cache directory for any given version. +fn sanitize_version(version: &str) -> String { + version + .chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => c, + _ => '_', + }) + .collect() +} + +/// Extract the single `binary_name` entry from the npm package archive. Reused +/// between embed mode's `verify_binary_present_in_archive` and the +/// `extract_to_cache` path used when `bundled-cli` is off. Panics if the +/// entry isn't found — callers have already invoked +/// `verify_binary_present_in_archive`. +fn extract_binary_bytes(archive: &[u8], platform: Platform) -> Vec { + let gz = flate2::read::GzDecoder::new(archive); + let mut tar = tar::Archive::new(gz); + for entry in tar + .entries() + .unwrap_or_else(|e| panic!("failed to read tar entries: {e}")) + { + let mut entry = entry.unwrap_or_else(|e| panic!("failed to read tar entry: {e}")); + let path = entry + .path() + .unwrap_or_else(|e| panic!("failed to read tar entry path: {e}")); + let name = path.to_string_lossy().into_owned(); + if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) { + let mut bytes = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut bytes) + .unwrap_or_else(|e| panic!("failed to read tar entry bytes: {e}")); + return bytes; + } + } + panic!( + "binary `{}` not found in package `{}`", + platform.binary_name, platform.package_name + ); +} + +/// Read a file from the download cache, or download it (with retries) and save +/// to cache. Verifies npm integrity on every path. Evicts stale/corrupt cache entries +/// automatically. Cache I/O failures are treated as cache misses — they never +/// break the build. +fn cached_download( + url: &str, + cache_key: &str, + expected_integrity: &str, + cache_dir: &Option, +) -> Vec { + if let Some(dir) = cache_dir { + let cached_path = dir.join(cache_key); + if cached_path.is_file() { + match std::fs::read(&cached_path) { + Ok(data) if verify_integrity(&data, expected_integrity) => { + // Silent cache hit — nothing to surface. + return data; + } + Ok(_) => { + println!("cargo:warning=Cached archive hash mismatch, re-downloading"); + let _ = std::fs::remove_file(&cached_path); + } + Err(e) => { + println!( + "cargo:warning=Failed to read cache {}, re-downloading: {e}", + cached_path.display() + ); + } + } + } + } + + println!("cargo:warning=Downloading {url}"); + let data = download_with_retry(url); + if !verify_integrity(&data, expected_integrity) { + panic!( + "Archive integrity check failed for {url}!\n expected: {expected_integrity}\n \ + This could indicate a corrupted download or a supply-chain attack." + ); + } + + if let Some(dir) = cache_dir { + if let Err(e) = std::fs::create_dir_all(dir) { + println!( + "cargo:warning=Failed to create cache directory {}: {e}", + dir.display() + ); + } else { + let cached_path = dir.join(cache_key); + println!("cargo:warning=Caching archive at {}", cached_path.display()); + if let Err(e) = std::fs::write(&cached_path, &data) { + println!( + "cargo:warning=Failed to write cache file {}: {e}", + cached_path.display() + ); + } + } + } + + data +} + +/// Maximum number of HTTP attempts (one initial + this many retries on transient errors). +const MAX_RETRIES: u32 = 3; + +/// Download `url` with bounded retries on transient network errors. Backoff is +/// exponential starting at 1s. 4xx responses fail fast; 5xx and connect/read +/// errors are retried. +fn download_with_retry(url: &str) -> Vec { + let mut attempt = 0u32; + loop { + attempt += 1; + match try_download(url) { + Ok(bytes) => return bytes, + Err(err) if err.transient && attempt <= MAX_RETRIES => { + let backoff = Duration::from_secs(1u64 << (attempt - 1)); + println!( + "cargo:warning=Transient download failure for {url} (attempt {attempt}/{}): {} — retrying in {}s", + MAX_RETRIES + 1, + err.message, + backoff.as_secs(), + ); + std::thread::sleep(backoff); + } + Err(err) => panic!("Failed to download {url}: {}", err.message), + } + } +} + +struct DownloadError { + message: String, + transient: bool, +} + +fn try_download(url: &str) -> Result, DownloadError> { + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_secs(30)) + .timeout_read(Duration::from_secs(120)) + .build(); + + match agent.get(url).call() { + Ok(response) => { + let mut bytes = Vec::new(); + response + .into_reader() + .read_to_end(&mut bytes) + .map_err(|e| DownloadError { + message: format!("read error: {e}"), + transient: true, + })?; + Ok(bytes) + } + // 5xx — server-side, treat as transient. + Err(ureq::Error::Status(code, response)) if (500..600).contains(&code) => { + Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: true, + }) + } + // 4xx — client-side, fail fast. + Err(ureq::Error::Status(code, response)) => Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: false, + }), + // Transport-layer (DNS, connect, TLS, read timeout) — treat as transient. + Err(ureq::Error::Transport(t)) => Err(DownloadError { + message: format!("transport error: {t}"), + transient: true, + }), + } +} + +/// Walks the downloaded archive at build time to confirm an entry matching +/// `binary_name` exists. Panics with a clear message if not. +fn verify_binary_present_in_archive(archive: &[u8], binary_name: &str, package_name: &str) { + let found = archive_contains_tar_entry(archive, binary_name); + if !found { + panic!( + "Copilot CLI package `{package_name}` does not contain an entry named `{binary_name}`. \ + The package layout may have changed; runtime extraction would fail. \ + Update `verify_binary_present_in_archive` in build.rs and the matching `extract_binary` in src/embeddedcli.rs." + ); + } +} + +fn archive_contains_tar_entry(targz: &[u8], binary_name: &str) -> bool { + let gz = flate2::read::GzDecoder::new(targz); + let mut archive = tar::Archive::new(gz); + let Ok(entries) = archive.entries() else { + return false; + }; + for entry in entries.flatten() { + let Ok(path) = entry.path() else { + continue; + }; + let name = path.to_string_lossy(); + if name == binary_name || name.ends_with(&format!("/{binary_name}")) { + return true; + } + } + false +} + +fn verify_integrity(data: &[u8], integrity: &str) -> bool { + let Some(encoded) = integrity.strip_prefix("sha512-") else { + return false; + }; + let Ok(expected) = base64::engine::general_purpose::STANDARD.decode(encoded) else { + return false; + }; + let mut hasher = sha2::Sha512::new(); + hasher.update(data); + hasher.finalize().as_slice() == expected +} diff --git a/rust/build/out_of_process.rs b/rust/build/out_of_process.rs new file mode 100644 index 0000000000..bb6732a036 --- /dev/null +++ b/rust/build/out_of_process.rs @@ -0,0 +1,712 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use sha2::Digest; + +pub(crate) fn main() { + println!("cargo:rerun-if-env-changed=DOCS_RS"); + println!("cargo:rerun-if-env-changed=COPILOT_SKIP_CLI_DOWNLOAD"); + println!("cargo:rerun-if-env-changed=COPILOT_CLI_EXTRACT_DIR"); + println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); + println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); + println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); + println!("cargo:rerun-if-changed=cli-version.txt"); + + // Only declare the lockfile rerun when the lockfile actually exists. + // Cargo treats `rerun-if-changed` for a missing path as "always rerun" + // — so unconditionally declaring this on consumers without a sibling + // `nodejs/` (vendored slots, published crates) would force build.rs + // to re-run on every `cargo build` even when nothing has changed. + // The lockfile path is only the source-of-truth in this repo's + // contributor builds; everywhere else `cli-version.txt` is canonical. + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + let lockfile = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package-lock.json"); + if lockfile.is_file() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + } + + // Hard opt-out: disable the entire download / bundle / cache mechanism + // in one step. For consumers who always supply the CLI via + // `CliProgram::Path` or `COPILOT_CLI_PATH` and don't want build.rs to + // touch the network (offline builds, locked-down CI, etc.). Works + // regardless of the `bundled-cli` cargo feature state — with neither + // `has_bundled_cli` nor `has_extracted_cli` emitted, runtime resolution + // falls straight through to `Error::BinaryNotFound` unless an explicit + // path source resolves first. + if std::env::var_os("COPILOT_SKIP_CLI_DOWNLOAD").is_some() { + println!( + "cargo:warning=COPILOT_SKIP_CLI_DOWNLOAD is set — skipping CLI download/bundle/cache" + ); + return; + } + + // docs.rs builds in a sandboxed environment without network access. + // Skip the CLI download so documentation can be generated successfully. + if std::env::var_os("DOCS_RS").is_some() { + println!("cargo:warning=DOCS_RS is set — skipping CLI download/bundle/cache"); + return; + } + + let Some(platform) = target_platform() else { + println!("cargo:warning=Unsupported target platform for Copilot CLI bundling — skipping"); + return; + }; + + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR is always set by cargo"); + let out = Path::new(&out_dir); + + // Resolve version + per-asset SHA-256 from one of two sources, in order: + // 1. `cli-version.txt` snapshot at the crate root (published-crate + // consumer; generated by the publish workflow). Combined format: + // `version=X` line + per-asset hash lines. Committing the hashes + // makes the publish workflow the trust boundary — an attacker who + // later re-points the release tag can't silently poison consumer + // builds. + // 2. Sibling `../nodejs/package-lock.json` (contributor build inside + // the github/copilot-sdk repo; live SHA256SUMS.txt fetch). Matches + // the .NET `_GetCopilotCliVersion` MSBuild target and the Go + // `cmd/bundler` tool. + let (version, expected_hash) = resolve_version_and_hash(platform.asset_name); + + // Bake the version into the crate regardless of mode. This is the + // single source of truth for "what CLI version did build.rs target", + // consumed by both the embed-mode path computation in embeddedcli.rs + // and the runtime path computation in resolve.rs (when `bundled-cli` + // is off). It's a small, machine-independent datum: no absolute + // paths, no username/home leakage, so sccache / cross-machine + // `target/` reuse stays cache-coherent. + println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); + + let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); + let cache_dir = std::env::var("BUNDLED_CLI_CACHE_DIR") + .ok() + .map(std::path::PathBuf::from); + + // Versioned cache key since copilot asset names don't include the version. + let cache_key = format!("v{version}-{}", platform.asset_name); + + if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { + // Embed mode: we need the archive bytes to bake into the rlib, so + // always run the download (cache hit short-circuits inside + // `cached_download`). + let archive = cached_download( + &format!("{base_url}/{}", platform.asset_name), + &cache_key, + &expected_hash, + &cache_dir, + ); + verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); + emit_embedded(out, &archive); + println!("cargo:rustc-cfg=has_bundled_cli"); + } else { + // With `bundled-cli` off the extracted binary *is* the cache. + // Skip the upstream download entirely when it already exists at + // the expected path. No two separate caches. + // + // Runtime resolution (see `src/resolve.rs::extracted_cli_path`) + // recomputes this same path from `COPILOT_SDK_CLI_VERSION` + the + // OS-derived binary name + optional `COPILOT_CLI_EXTRACT_DIR`, + // so we don't bake an absolute path into the crate. + let install_dir = extracted_install_dir(&version); + let final_path = install_dir.join(platform.binary_name); + + // Invalidate build.rs whenever the cached binary disappears (cache GC, + // manual rm, OS reset, switching extract dir). Without this, cargo + // replays the saved `has_extracted_cli` cfg from its build-script + // output cache even when the file is gone, and runtime resolution + // fails with BinaryNotFound. + println!("cargo:rerun-if-changed={}", final_path.display()); + + if !final_path.is_file() { + let archive = cached_download( + &format!("{base_url}/{}", platform.asset_name), + &cache_key, + &expected_hash, + &cache_dir, + ); + verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); + extract_to_cache(&archive, &install_dir, platform); + } + + // Re-check after potential download+extract above; not an `else` + // because we need to verify the extraction actually produced the file. + if final_path.is_file() { + println!("cargo:rustc-cfg=has_extracted_cli"); + } + } +} + +/// Install directory used when `bundled-cli` is off. Mirrors the runtime +/// convention in `src/resolve.rs::extracted_cli_path`: both sides MUST +/// compute the same path from the same inputs, otherwise the runtime +/// resolver won't find what build.rs extracted. +/// +/// If `COPILOT_CLI_EXTRACT_DIR` is set the binary lives directly under +/// that directory (no per-version subdir) — useful for vendored slots and +/// for `.cargo/config.toml [env]`-style pinning that's symmetric between +/// build-time write and runtime read. Otherwise the binary lives under +/// `/github-copilot-sdk/cli//`. +fn extracted_install_dir(version: &str) -> PathBuf { + if let Some(custom) = std::env::var_os("COPILOT_CLI_EXTRACT_DIR") { + PathBuf::from(custom) + } else { + let cache = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); + cache + .join("github-copilot-sdk") + .join("cli") + .join(sanitize_version(version)) + } +} + +/// Emit the `bundled_cli.rs` glue + `copilot_cli.archive` blob into `OUT_DIR` +/// for embed mode (`bundled-cli` cargo feature on). The version is exposed +/// crate-wide via the unconditional `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` +/// emit; the binary name is OS-derived at runtime — so all we need to +/// generate here is the archive blob include. +fn emit_embedded(out: &Path, archive: &[u8]) { + std::fs::write(out.join("copilot_cli.archive"), archive) + .expect("failed to write copilot_cli.archive"); + + let generated = r#"// Auto-generated by github-copilot-sdk build.rs. Do not edit. +pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); +"#; + + std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); +} + +/// Resolve the CLI version and the expected SHA-256 hash for the current +/// target's archive. Picks one of two sources in order. Panics with a clear +/// error if neither is available. +fn resolve_version_and_hash(asset_name: &str) -> (String, String) { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + + // 1. Snapshot file at the crate root (published-crate consumer, + // vendored-slot consumer). Combined version + per-asset hashes. + let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); + if snapshot.is_file() { + let contents = std::fs::read_to_string(&snapshot) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); + return parse_snapshot(&contents, asset_name) + .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); + } + + // 2. Lockfile fallback (contributor build inside github/copilot-sdk) — + // read version, fetch live SHA256SUMS. + let lockfile = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package-lock.json"); + if lockfile.is_file() { + let version = read_version_from_package_lock(&lockfile); + let hash = fetch_live_sha256(&version, asset_name); + return (version, hash); + } + + panic!( + "Could not resolve the Copilot CLI version.\n\ + Tried:\n\ + - {} (missing)\n\ + - {} (missing)\n\ + In a published crate or vendored slot, `cli-version.txt` should be present.\n\ + Inside the github/copilot-sdk repo, `../nodejs/package-lock.json` is the source.", + snapshot.display(), + lockfile.display(), + ); +} + +/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per +/// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map +/// asset filename to hex SHA-256. Blank lines and lines starting with `#` +/// are skipped. +fn parse_snapshot(contents: &str, asset_name: &str) -> Result<(String, String), String> { + let mut version: Option = None; + let mut hash: Option = None; + for (line_no, raw) in contents.lines().enumerate() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + return Err(format!( + "line {}: expected `key=value`, got `{raw}`", + line_no + 1 + )); + }; + match key.trim() { + "version" => version = Some(value.trim().to_string()), + k if k == asset_name => hash = Some(value.trim().to_string()), + _ => {} + } + } + let version = version.ok_or("missing `version=` line")?; + let hash = hash.ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; + Ok((version, hash)) +} + +/// Read the `@github/copilot` version from `nodejs/package-lock.json`. +fn read_version_from_package_lock(path: &Path) -> String { + let contents = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + // Minimal JSON walk: find `"node_modules/@github/copilot"` object and + // its `"version"` field. Full JSON parsing keeps build.rs dep-light by + // using a regex; the file is generated by npm and we're matching an + // exact key path. + let key = "\"node_modules/@github/copilot\""; + let key_pos = contents + .find(key) + .unwrap_or_else(|| panic!("{} does not contain {key}", path.display())); + let after_key = &contents[key_pos + key.len()..]; + let version_key = "\"version\""; + let v_pos = after_key + .find(version_key) + .unwrap_or_else(|| panic!("no `version` field found near {key} in {}", path.display())); + let after_v = &after_key[v_pos + version_key.len()..]; + let q1 = after_v.find('"').expect("malformed version"); + let after_q1 = &after_v[q1 + 1..]; + let q2 = after_q1.find('"').expect("malformed version"); + after_q1[..q2].to_string() +} + +/// Fetch the live `SHA256SUMS.txt` for the given version from GitHub Releases +/// and pluck out the entry for `asset_name`. +fn fetch_live_sha256(version: &str, asset_name: &str) -> String { + let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); + let checksums_url = format!("{base_url}/SHA256SUMS.txt"); + let checksums = download_with_retry(&checksums_url); + let checksums_text = + std::str::from_utf8(&checksums).expect("checksums file is not valid UTF-8"); + find_sha256_for_asset(checksums_text, asset_name) +} + +#[derive(Clone, Copy)] +struct Platform { + asset_name: &'static str, + binary_name: &'static str, +} + +fn target_platform() -> Option { + let os = std::env::var("CARGO_CFG_TARGET_OS").ok()?; + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").ok()?; + + match (os.as_str(), arch.as_str()) { + ("macos", "aarch64") => Some(Platform { + asset_name: "copilot-darwin-arm64.tar.gz", + binary_name: "copilot", + }), + ("macos", "x86_64") => Some(Platform { + asset_name: "copilot-darwin-x64.tar.gz", + binary_name: "copilot", + }), + ("linux", "x86_64") => Some(Platform { + asset_name: "copilot-linux-x64.tar.gz", + binary_name: "copilot", + }), + ("linux", "aarch64") => Some(Platform { + asset_name: "copilot-linux-arm64.tar.gz", + binary_name: "copilot", + }), + ("windows", "x86_64") => Some(Platform { + asset_name: "copilot-win32-x64.zip", + binary_name: "copilot.exe", + }), + ("windows", "aarch64") => Some(Platform { + asset_name: "copilot-win32-arm64.zip", + binary_name: "copilot.exe", + }), + _ => None, + } +} + +/// Write the single binary entry from `archive` to +/// `/` and return the resulting path. +/// Idempotent — returns the existing path if a previous build already +/// populated the target. +/// +/// Uses file-level staging + atomic rename so a concurrent reader during +/// a parallel `cargo build` race never observes a partially-written +/// binary. `fs::rename` for files is atomic on both Unix and Windows +/// (Windows uses `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`); for +/// directories it is not, which is why we stage at file granularity. +fn extract_to_cache(archive: &[u8], install_dir: &Path, platform: Platform) -> PathBuf { + let final_path = install_dir.join(platform.binary_name); + + // Caller already gated on `final_path.is_file()`; this is a safety + // net for any future caller that forgets. + if final_path.is_file() { + return final_path; + } + + std::fs::create_dir_all(install_dir).unwrap_or_else(|e| { + panic!( + "failed to create install dir {}: {e}", + install_dir.display() + ) + }); + + let bytes = extract_binary_bytes(archive, platform); + + // Staging file is a sibling of the final binary so the rename stays + // on the same filesystem (cross-fs rename is not atomic). PID + nanos + // disambiguate concurrent builds racing on the same cache. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let staging_path = install_dir.join(format!( + ".{}.staging-{}-{nanos}", + platform.binary_name, + std::process::id(), + )); + + { + let mut f = std::fs::File::create(&staging_path).unwrap_or_else(|e| { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to create staging file {}: {e}", + staging_path.display() + ); + }); + + if let Err(e) = f.write_all(&bytes) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to write staging file {}: {e}", + staging_path.display() + ); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Err(e) = f.set_permissions(std::fs::Permissions::from_mode(0o755)) { + let _ = std::fs::remove_file(&staging_path); + panic!("failed to chmod {}: {e}", staging_path.display()); + } + } + + // Backdate the staged binary to the Unix epoch before it lands. We emit + // `cargo:rerun-if-changed` on `final_path` (see caller) so a *deleted* + // cache binary forces a re-extract — but cargo stamps the build-script + // `output` reference when the script is spawned, seconds before this + // freshly-downloaded binary is written. A current mtime would therefore + // be *newer* than that reference, so the next identical `cargo` + // invocation would see the watched file as "changed" and pointlessly + // rerun build.rs + recompile the crate + relink every downstream crate. + // Pinning to the epoch keeps the file unambiguously older than any real + // build reference; `rename` preserves mtime (same inode), so it lands + // already-backdated and a no-change rebuild stays a true no-op. The + // deleted-file recovery contract is untouched: a missing file can't be + // stat'd, so cargo still treats it as stale and reruns regardless. + // + // Best-effort: a filesystem that refuses the epoch (e.g. FAT's 1980 floor + // clamps it — still older than any real reference) or rejects the call + // just reverts to the pre-fix redundant-rebuild behaviour, never a broken + // build. + if let Err(e) = f.set_modified(std::time::SystemTime::UNIX_EPOCH) { + println!( + "cargo:warning=Could not backdate {} (a redundant rebuild may occur): {e}", + staging_path.display() + ); + } + } + + // Atomic file-replace on both Unix and Windows. If a concurrent build + // already produced the same file the rename overwrites it; the bytes + // are SHA-verified-identical so replacement is safe. + if let Err(e) = std::fs::rename(&staging_path, &final_path) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to rename {} -> {}: {e}", + staging_path.display(), + final_path.display() + ); + } + + // Surface where the binary landed so contributors can find it. Quiet + // on the hot path: the caller's `is_file()` short-circuit (and the + // safety net at the top of this function) means this only fires on a + // true cache miss. + println!( + "cargo:warning=Extracted Copilot CLI to {}", + final_path.display() + ); + + final_path +} + +/// Replace characters outside `[a-zA-Z0-9._-]` with `_` so the version +/// string is always safe to use as a path component. Kept in sync with +/// `embeddedcli::sanitize_version` and `resolve::sanitize_version` so all +/// three resolve to the same cache directory for any given version. +fn sanitize_version(version: &str) -> String { + version + .chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => c, + _ => '_', + }) + .collect() +} + +/// Extract the single `binary_name` entry from the release archive. Reused +/// between embed mode's `verify_binary_present_in_archive` and the +/// `extract_to_cache` path used when `bundled-cli` is off. Panics if the +/// entry isn't found — callers have already invoked +/// `verify_binary_present_in_archive`. +fn extract_binary_bytes(archive: &[u8], platform: Platform) -> Vec { + if platform.asset_name.ends_with(".zip") { + let cursor = std::io::Cursor::new(archive); + let mut zip = zip::ZipArchive::new(cursor) + .unwrap_or_else(|e| panic!("failed to open zip archive: {e}")); + for i in 0..zip.len() { + let mut entry = zip + .by_index(i) + .unwrap_or_else(|e| panic!("failed to read zip entry {i}: {e}")); + let name = entry.name().to_string(); + if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) + { + let mut bytes = Vec::with_capacity(entry.size() as usize); + std::io::copy(&mut entry, &mut bytes) + .unwrap_or_else(|e| panic!("failed to read zip entry bytes: {e}")); + return bytes; + } + } + } else { + let gz = flate2::read::GzDecoder::new(archive); + let mut tar = tar::Archive::new(gz); + for entry in tar + .entries() + .unwrap_or_else(|e| panic!("failed to read tar entries: {e}")) + { + let mut entry = entry.unwrap_or_else(|e| panic!("failed to read tar entry: {e}")); + let path = entry + .path() + .unwrap_or_else(|e| panic!("failed to read tar entry path: {e}")); + let name = path.to_string_lossy().into_owned(); + if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) + { + let mut bytes = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut bytes) + .unwrap_or_else(|e| panic!("failed to read tar entry bytes: {e}")); + return bytes; + } + } + } + panic!( + "binary `{}` not found in archive `{}`", + platform.binary_name, platform.asset_name + ); +} + +/// Read a file from the download cache, or download it (with retries) and save +/// to cache. Verifies SHA-256 on every path. Evicts stale/corrupt cache entries +/// automatically. Cache I/O failures are treated as cache misses — they never +/// break the build. +fn cached_download( + url: &str, + cache_key: &str, + expected_hash: &str, + cache_dir: &Option, +) -> Vec { + if let Some(dir) = cache_dir { + let cached_path = dir.join(cache_key); + if cached_path.is_file() { + match std::fs::read(&cached_path) { + Ok(data) if hex_sha256(&data) == expected_hash => { + // Silent cache hit — nothing to surface. + return data; + } + Ok(_) => { + println!("cargo:warning=Cached archive hash mismatch, re-downloading"); + let _ = std::fs::remove_file(&cached_path); + } + Err(e) => { + println!( + "cargo:warning=Failed to read cache {}, re-downloading: {e}", + cached_path.display() + ); + } + } + } + } + + println!("cargo:warning=Downloading {url}"); + let data = download_with_retry(url); + let actual_hash = hex_sha256(&data); + if actual_hash != expected_hash { + panic!( + "Archive integrity check failed for {url}!\n expected: {expected_hash}\n actual: {actual_hash}\n \ + This could indicate a corrupted download or a supply-chain attack." + ); + } + + if let Some(dir) = cache_dir { + if let Err(e) = std::fs::create_dir_all(dir) { + println!( + "cargo:warning=Failed to create cache directory {}: {e}", + dir.display() + ); + } else { + let cached_path = dir.join(cache_key); + println!("cargo:warning=Caching archive at {}", cached_path.display()); + if let Err(e) = std::fs::write(&cached_path, &data) { + println!( + "cargo:warning=Failed to write cache file {}: {e}", + cached_path.display() + ); + } + } + } + + data +} + +/// Maximum number of HTTP attempts (one initial + this many retries on transient errors). +const MAX_RETRIES: u32 = 3; + +/// Download `url` with bounded retries on transient network errors. Backoff is +/// exponential starting at 1s. 4xx responses fail fast; 5xx and connect/read +/// errors are retried. +fn download_with_retry(url: &str) -> Vec { + let mut attempt = 0u32; + loop { + attempt += 1; + match try_download(url) { + Ok(bytes) => return bytes, + Err(err) if err.transient && attempt <= MAX_RETRIES => { + let backoff = Duration::from_secs(1u64 << (attempt - 1)); + println!( + "cargo:warning=Transient download failure for {url} (attempt {attempt}/{}): {} — retrying in {}s", + MAX_RETRIES + 1, + err.message, + backoff.as_secs(), + ); + std::thread::sleep(backoff); + } + Err(err) => panic!("Failed to download {url}: {}", err.message), + } + } +} + +struct DownloadError { + message: String, + transient: bool, +} + +fn try_download(url: &str) -> Result, DownloadError> { + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_secs(30)) + .timeout_read(Duration::from_secs(120)) + .build(); + + match agent.get(url).call() { + Ok(response) => { + let mut bytes = Vec::new(); + response + .into_reader() + .read_to_end(&mut bytes) + .map_err(|e| DownloadError { + message: format!("read error: {e}"), + transient: true, + })?; + Ok(bytes) + } + // 5xx — server-side, treat as transient. + Err(ureq::Error::Status(code, response)) if (500..600).contains(&code) => { + Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: true, + }) + } + // 4xx — client-side, fail fast. + Err(ureq::Error::Status(code, response)) => Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: false, + }), + // Transport-layer (DNS, connect, TLS, read timeout) — treat as transient. + Err(ureq::Error::Transport(t)) => Err(DownloadError { + message: format!("transport error: {t}"), + transient: true, + }), + } +} + +fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { + for line in sums.lines() { + // Format: " " (two spaces) + if let Some((hash, name)) = line.split_once(" ") + && name.trim() == asset_name + { + return hash.trim().to_string(); + } + } + panic!("SHA256SUMS.txt does not contain an entry for {asset_name}"); +} + +fn sha256(data: &[u8]) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + hasher.finalize().into() +} + +/// Walks the downloaded archive at build time to confirm an entry matching +/// `binary_name` exists. Panics with a clear message if not — defends against +/// silent breakage if the upstream archive layout ever changes. +fn verify_binary_present_in_archive(archive: &[u8], binary_name: &str, asset_name: &str) { + let found = if asset_name.ends_with(".zip") { + archive_contains_zip_entry(archive, binary_name) + } else { + archive_contains_tar_entry(archive, binary_name) + }; + if !found { + panic!( + "Copilot CLI archive `{asset_name}` does not contain an entry named `{binary_name}`. \ + The upstream archive layout may have changed; runtime extraction would fail. \ + Update `verify_binary_present_in_archive` in build.rs and the matching `extract_binary` in src/embeddedcli.rs." + ); + } +} + +fn archive_contains_tar_entry(targz: &[u8], binary_name: &str) -> bool { + let gz = flate2::read::GzDecoder::new(targz); + let mut archive = tar::Archive::new(gz); + let Ok(entries) = archive.entries() else { + return false; + }; + for entry in entries.flatten() { + let Ok(path) = entry.path() else { + continue; + }; + let name = path.to_string_lossy(); + if name == binary_name || name.ends_with(&format!("/{binary_name}")) { + return true; + } + } + false +} + +fn archive_contains_zip_entry(zip_bytes: &[u8], binary_name: &str) -> bool { + let cursor = std::io::Cursor::new(zip_bytes); + let Ok(mut archive) = zip::ZipArchive::new(cursor) else { + return false; + }; + for i in 0..archive.len() { + let Ok(entry) = archive.by_index(i) else { + continue; + }; + let name = entry.name(); + if name == binary_name || name.ends_with(&format!("/{binary_name}")) { + return true; + } + } + false +} + +fn hex_sha256(data: &[u8]) -> String { + sha256(data).iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/rust/scripts/snapshot-bundled-in-process-version.sh b/rust/scripts/snapshot-bundled-in-process-version.sh new file mode 100755 index 0000000000..5a4cde73fa --- /dev/null +++ b/rust/scripts/snapshot-bundled-in-process-version.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Snapshot the Copilot CLI version + per-platform npm integrity values for the +# rust crate's bundled-in-process build path. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUST_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${RUST_DIR}/.." && pwd)" +LOCKFILE="${REPO_ROOT}/nodejs/package-lock.json" +OUTPUT="${RUST_DIR}/cli-version-in-process.txt" + +if [[ ! -f "${LOCKFILE}" ]]; then + echo "error: ${LOCKFILE} not found" >&2 + exit 1 +fi + +VERSION="$(node -e "console.log(require('${LOCKFILE}').packages['node_modules/@github/copilot'].version)")" +if [[ -z "${VERSION}" ]]; then + echo "error: could not read @github/copilot version from ${LOCKFILE}" >&2 + exit 1 +fi + +PACKAGES=( + "copilot-darwin-arm64" + "copilot-darwin-x64" + "copilot-linux-arm64" + "copilot-linux-x64" + "copilot-win32-arm64" + "copilot-win32-x64" +) + +declare -A INTEGRITIES +for package in "${PACKAGES[@]}"; do + integrity="$(node -e "console.log(require('${LOCKFILE}').packages['node_modules/@github/${package}'].integrity)")" + if [[ -z "${integrity}" ]]; then + echo "error: package-lock.json missing integrity for @github/${package}" >&2 + exit 1 + fi + INTEGRITIES[$package]="${integrity}" +done + +{ + echo "# Auto-generated by rust/scripts/snapshot-bundled-in-process-version.sh" + echo "# Do not edit. Regenerated by the publish workflow on every release." + echo "version=${VERSION}" + for package in "${PACKAGES[@]}"; do + echo "${package}=${INTEGRITIES[$package]}" + done +} > "${OUTPUT}" + +echo "Wrote ${OUTPUT} (version=${VERSION}, ${#PACKAGES[@]} integrity values)" diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 56f97e0c0e..40900a4d22 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -2,14 +2,11 @@ //! crate (gated on the `bundled-cli` cargo feature, which is in the default //! feature set). //! -//! build.rs downloads the platform's `copilot-{platform}.{tar.gz,zip}` -//! archive from GitHub Releases, SHA-256 verifies it against the version -//! pinned in `cli-version.txt` (or `../nodejs/package-lock.json` when -//! building inside the github/copilot-sdk repo itself), and embeds the -//! **raw archive bytes** -//! into the consumer's compiled artifact via `include_bytes!()`. Extraction -//! to a real on-disk path is deferred until the first call to -//! [`path`] / [`install_at`]. +//! Normal builds embed the platform release archive from GitHub Releases. +//! Builds with `bundled-in-process` instead embed a minimal archive from the +//! platform npm package containing the CLI executable and native runtime +//! library. Extraction to a real on-disk path is deferred until the first call +//! to [`path`] / [`install_at`]. //! //! The embedded bytes are part of the consumer's signed binary and therefore //! trusted *as the source of truth* — but the bytes that land on disk are not. @@ -31,7 +28,7 @@ // off but still needs to exercise them. #[cfg(any(has_bundled_cli, test))] use std::fs; -#[cfg(all(has_bundled_cli, not(windows)))] +#[cfg(all(has_bundled_cli, any(feature = "bundled-in-process", not(windows))))] use std::io::Read; #[cfg(any(has_bundled_cli, test))] use std::io::Write; @@ -44,8 +41,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tracing::{info, warn}; // When the `bundled-cli` cargo feature is enabled and the target platform is -// supported, build.rs generates `bundled_cli.rs` exposing the raw archive -// bytes. The CLI version is exposed crate-wide via the +// supported, build.rs generates `bundled_cli.rs` exposing the selected archive. +// The CLI version is exposed crate-wide via the // `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` emit (see `build.rs`), and the // binary name is OS-derived — so no other generated constants are needed. #[cfg(has_bundled_cli)] @@ -157,8 +154,53 @@ fn default_install_dir(version: &str) -> PathBuf { #[cfg(has_bundled_cli)] const MAX_PUBLISH_ATTEMPTS: u32 = 3; +// Natural platform shared-library name for the in-process FFI runtime. +#[cfg(all(has_bundled_cli, feature = "bundled-in-process", windows))] +const RUNTIME_LIBRARY_NAME: &str = "copilot_runtime.dll"; +#[cfg(all(has_bundled_cli, feature = "bundled-in-process", target_os = "macos"))] +const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.dylib"; +#[cfg(all( + has_bundled_cli, + feature = "bundled-in-process", + not(windows), + not(target_os = "macos") +))] +const RUNTIME_LIBRARY_NAME: &str = "libcopilot_runtime.so"; + #[cfg(has_bundled_cli)] fn install(install_dir: &Path, archive: &[u8]) -> Result { + let final_path = install_cli(install_dir, archive)?; + #[cfg(feature = "bundled-in-process")] + { + install_runtime_library(install_dir, archive)?; + } + Ok(final_path) +} + +#[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] +fn install_runtime_library(install_dir: &Path, archive: &[u8]) -> Result<(), EmbeddedCliError> { + let target = install_dir.join(RUNTIME_LIBRARY_NAME); + if fs::metadata(&target).map(|m| m.len() > 0).unwrap_or(false) { + return Ok(()); + } + let bytes = extract_binary(archive, RUNTIME_LIBRARY_NAME)?; + if bytes.is_empty() { + return Err(EmbeddedCliError::with_message( + EmbeddedCliErrorKind::Verification, + "embedded runtime library is empty", + )); + } + let tmp = write_temp_file(install_dir, &bytes)?; + if let Err(e) = publish(&tmp, &target) { + let _ = fs::remove_file(&tmp); + return Err(e); + } + tracing::debug!(path = %target.display(), "in-process FFI runtime library installed"); + Ok(()) +} + +#[cfg(has_bundled_cli)] +fn install_cli(install_dir: &Path, archive: &[u8]) -> Result { let verbose = std::env::var("COPILOT_CLI_INSTALL_VERBOSE").ok().as_deref() == Some("1"); fs::create_dir_all(install_dir) @@ -438,7 +480,7 @@ fn read_marker_len(marker_path: &Path) -> Option { .ok() } -#[cfg(all(has_bundled_cli, not(windows)))] +#[cfg(all(has_bundled_cli, any(feature = "bundled-in-process", not(windows))))] fn extract_binary(archive: &[u8], binary_name: &str) -> Result, EmbeddedCliError> { let gz = flate2::read::GzDecoder::new(archive); let mut tar = tar::Archive::new(gz); @@ -463,7 +505,7 @@ fn extract_binary(archive: &[u8], binary_name: &str) -> Result, Embedded Err(EmbeddedCliErrorKind::BinaryNotFoundInArchive.into()) } -#[cfg(all(has_bundled_cli, windows))] +#[cfg(all(has_bundled_cli, not(feature = "bundled-in-process"), windows))] fn extract_binary(archive: &[u8], binary_name: &str) -> Result, EmbeddedCliError> { let cursor = std::io::Cursor::new(archive); let mut zip = zip::ZipArchive::new(cursor) @@ -499,9 +541,9 @@ fn sanitize_version(version: &str) -> String { #[allow(dead_code)] enum EmbeddedCliErrorKind { CreateDir, - #[cfg(not(windows))] + #[cfg(any(feature = "bundled-in-process", not(windows)))] Archive, - #[cfg(windows)] + #[cfg(all(not(feature = "bundled-in-process"), windows))] Zip, BinaryNotFoundInArchive, Io, @@ -519,9 +561,9 @@ impl std::fmt::Display for EmbeddedCliErrorKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { EmbeddedCliErrorKind::CreateDir => f.write_str("failed to create install directory"), - #[cfg(not(windows))] + #[cfg(any(feature = "bundled-in-process", not(windows)))] EmbeddedCliErrorKind::Archive => f.write_str("failed to read archive entry"), - #[cfg(windows)] + #[cfg(all(not(feature = "bundled-in-process"), windows))] EmbeddedCliErrorKind::Zip => f.write_str("failed to read zip archive"), EmbeddedCliErrorKind::BinaryNotFoundInArchive => { f.write_str("CLI binary not found in embedded archive") @@ -626,6 +668,33 @@ impl std::error::Error for EmbeddedCliError { mod tests { use super::*; + #[cfg(all(has_bundled_cli, feature = "bundled-in-process"))] + #[test] + fn embedded_archive_contains_only_expected_files() { + let gz = flate2::read::GzDecoder::new(build_time::CLI_ARCHIVE); + let mut archive = tar::Archive::new(gz); + let mut names: Vec = archive + .entries() + .expect("archive entries") + .map(|entry| { + entry + .expect("archive entry") + .path() + .expect("archive path") + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + + let mut expected = vec![ + CLI_BINARY_NAME.to_string(), + RUNTIME_LIBRARY_NAME.to_string(), + ]; + expected.sort(); + assert_eq!(names, expected); + } + /// Bytes whose header looks like a valid executable image on the host /// platform, so `looks_like_valid_image` accepts them. `extra` padding /// bytes follow the magic so size checks have something to disagree about. diff --git a/rust/src/ffi.rs b/rust/src/ffi.rs new file mode 100644 index 0000000000..f784b1a6d1 --- /dev/null +++ b/rust/src/ffi.rs @@ -0,0 +1,633 @@ +//! In-process FFI transport: hosts the Copilot runtime by loading its native +//! library and speaking JSON-RPC over its C ABI, +//! instead of spawning a CLI child process and communicating over stdio/TCP. +//! +//! The runtime's `host_start` export spawns the residual TypeScript worker +//! itself — the packaged single-file CLI (`copilot --embedded-host`) or, for +//! dev, `node dist-cli/index.js --embedded-host`. JSON-RPC frames are pumped +//! across the ABI: writes go to `connection_write`; inbound frames arrive on a +//! native callback that feeds an async reader. The framing is unchanged — the +//! same LSP `Content-Length:` frames the stdio transport uses. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::task::{Context, Poll}; + +use libloading::Library; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::mpsc; +use tracing::debug; + +use crate::{Error, ErrorKind}; + +type OutboundCallback = unsafe extern "C" fn(*mut c_void, *const u8, usize); +type HostStartFn = unsafe extern "C" fn(*const u8, usize, *const u8, usize) -> u32; +type HostShutdownFn = unsafe extern "C" fn(u32) -> bool; +#[allow(clippy::type_complexity)] +type ConnectionOpenFn = unsafe extern "C" fn( + u32, + OutboundCallback, + *mut c_void, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, +) -> u32; +type ConnectionWriteFn = unsafe extern "C" fn(u32, *const u8, usize) -> bool; +type ConnectionCloseFn = unsafe extern "C" fn(u32) -> bool; + +/// State handed to the native side as `user_data` so the outbound callback can +/// route inbound frames back to the reader. +struct CallbackState { + tx: mpsc::UnboundedSender>, + active_callbacks: AtomicUsize, + closing: AtomicBool, +} + +extern "C" fn on_outbound(user_data: *mut c_void, bytes: *const u8, len: usize) { + if user_data.is_null() || bytes.is_null() || len == 0 { + return; + } + let state = unsafe { &*(user_data as *const CallbackState) }; + state.active_callbacks.fetch_add(1, Ordering::SeqCst); + if state.closing.load(Ordering::SeqCst) { + state.active_callbacks.fetch_sub(1, Ordering::SeqCst); + return; + } + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + let _ = state.tx.send(slice.to_vec()); + state.active_callbacks.fetch_sub(1, Ordering::SeqCst); +} + +/// Bound exports and connection lifecycle state, shared between the +/// [`FfiWriter`] and the owning [`Client`]. The cdylib itself is loaded +/// process-globally and never unloaded (see [`load_library`]), so this holds +/// only the bound fn pointers and connection state. +pub(crate) struct FfiShared { + host_shutdown: HostShutdownFn, + connection_write: ConnectionWriteFn, + connection_close: ConnectionCloseFn, + server_id: AtomicU32, + connection_id: AtomicU32, + callback_state: AtomicPtr, + closed: AtomicBool, + operation_lock: parking_lot::Mutex<()>, + library_path: PathBuf, +} + +// The raw fn pointers and the boxed callback state are safe to move across +// threads: the native side copies buffers synchronously and the callback only +// forwards to a thread-safe channel sender. +unsafe impl Send for FfiShared {} +unsafe impl Sync for FfiShared {} + +impl FfiShared { + /// Close the connection, shut the host down, and free the callback state. + /// Idempotent; called from [`Client::stop`], drop, and on startup failure. + pub(crate) fn close(&self) { + let _operation = self.operation_lock.lock(); + if self.closed.swap(true, Ordering::SeqCst) { + return; + } + let state = self.callback_state.load(Ordering::SeqCst); + if !state.is_null() { + unsafe { &*state }.closing.store(true, Ordering::SeqCst); + } + let conn = self.connection_id.swap(0, Ordering::SeqCst); + if conn != 0 { + unsafe { (self.connection_close)(conn) }; + } + let server = self.server_id.swap(0, Ordering::SeqCst); + if server != 0 { + unsafe { (self.host_shutdown)(server) }; + } + // Free the callback state only after the connection is closed and the + // host is shut down, so native can no longer invoke the callback. + let state = self + .callback_state + .swap(std::ptr::null_mut(), Ordering::SeqCst); + if !state.is_null() { + while unsafe { &*state }.active_callbacks.load(Ordering::SeqCst) != 0 { + std::thread::yield_now(); + } + drop(unsafe { Box::from_raw(state) }); + } + debug!(library = %self.library_path.display(), "FFI runtime connection closed"); + } + + fn write_frame(&self, frame: &[u8]) -> bool { + let _operation = self.operation_lock.lock(); + if self.closed.load(Ordering::SeqCst) { + return false; + } + let conn = self.connection_id.load(Ordering::SeqCst); + if conn == 0 { + return false; + } + unsafe { (self.connection_write)(conn, frame.as_ptr(), frame.len()) } + } +} + +impl Drop for FfiShared { + fn drop(&mut self) { + self.close(); + } +} + +/// Read side of the FFI transport, fed by the native outbound callback via an +/// unbounded channel. Implements [`AsyncRead`] for the JSON-RPC read loop. +pub(crate) struct FfiReader { + rx: mpsc::UnboundedReceiver>, + leftover: Vec, + pos: usize, +} + +impl AsyncRead for FfiReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.pos >= self.leftover.len() { + match self.rx.poll_recv(cx) { + Poll::Ready(Some(chunk)) => { + self.leftover = chunk; + self.pos = 0; + } + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => return Poll::Pending, + } + } + let available = self.leftover.len() - self.pos; + let n = available.min(buf.remaining()); + let start = self.pos; + buf.put_slice(&self.leftover[start..start + n]); + self.pos += n; + Poll::Ready(Ok(())) + } +} + +/// Write side of the FFI transport. Each frame is forwarded synchronously to +/// the native `connection_write` export (native copies before returning). +pub(crate) struct FfiWriter { + shared: Arc, +} + +impl AsyncWrite for FfiWriter { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + if self.shared.write_frame(buf) { + Poll::Ready(Ok(buf.len())) + } else { + Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "failed to write a frame to the in-process runtime connection", + ))) + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +/// Prepared FFI host: the bound cdylib exports plus the spawn arguments needed +/// to start the runtime worker. The cdylib is loaded process-globally and never +/// unloaded (see [`load_library`]). +pub(crate) struct FfiHost { + library_path: PathBuf, + entrypoint: PathBuf, + environment: Vec<(String, String)>, + args: Vec, + host_start: HostStartFn, + host_shutdown: HostShutdownFn, + connection_open: ConnectionOpenFn, + connection_write: ConnectionWriteFn, + connection_close: ConnectionCloseFn, +} + +// SAFETY: as for `FfiShared` — the bound exports are plain fn pointers, safe to +// move to the blocking thread that starts the host. +unsafe impl Send for FfiHost {} + +impl FfiHost { + /// Load the cdylib next to `entrypoint` and bind its exports. + /// + /// `entrypoint` is the packaged single-file CLI binary or, for dev, a + /// `.js` file launched via `node`. The native library is resolved relative + /// to the entrypoint directory, supporting both packaged and development + /// layouts. + pub(crate) fn create( + entrypoint: &Path, + environment: Vec<(String, String)>, + args: Vec, + ) -> Result { + let entrypoint = std::fs::canonicalize(entrypoint) + .map(path_for_child_process) + .map_err(|e| { + Error::with_message( + ErrorKind::InvalidConfig, + format!( + "failed to resolve in-process CLI entrypoint '{}': {e}", + entrypoint.display() + ), + ) + })?; + let library_path = + std::fs::canonicalize(resolve_library_path(&entrypoint)?).map_err(|e| { + Error::with_message( + ErrorKind::InvalidConfig, + format!("failed to resolve in-process runtime library: {e}"), + ) + })?; + let lib = load_library(&library_path)?; + + let host_start = *bind::(lib, b"copilot_runtime_host_start\0", &library_path)?; + let host_shutdown = + *bind::(lib, b"copilot_runtime_host_shutdown\0", &library_path)?; + let connection_open = + *bind::(lib, b"copilot_runtime_connection_open\0", &library_path)?; + let connection_write = + *bind::(lib, b"copilot_runtime_connection_write\0", &library_path)?; + let connection_close = + *bind::(lib, b"copilot_runtime_connection_close\0", &library_path)?; + + Ok(Self { + library_path, + entrypoint, + environment, + args, + host_start, + host_shutdown, + connection_open, + connection_write, + connection_close, + }) + } + + /// Start the runtime worker and open the FFI JSON-RPC connection. + /// + /// `host_start` blocks until the worker connects back and signals + /// readiness (up to ~30s), and must not run on an async executor thread, so + /// the blocking handshake is offloaded to [`tokio::task::spawn_blocking`]. + pub(crate) async fn start(self) -> Result<(FfiReader, FfiWriter, Arc), Error> { + tokio::task::spawn_blocking(move || self.start_blocking()) + .await + .map_err(|e| { + Error::with_message( + ErrorKind::InvalidConfig, + format!("in-process runtime startup task failed: {e}"), + ) + })? + } + + fn start_blocking(self) -> Result<(FfiReader, FfiWriter, Arc), Error> { + let argv = build_argv_json(&self.entrypoint, &self.args); + let env = build_env_json(&self.environment); + + let (env_ptr, env_len) = match &env { + Some(bytes) => (bytes.as_ptr(), bytes.len()), + None => (std::ptr::null(), 0), + }; + + let server_id = unsafe { (self.host_start)(argv.as_ptr(), argv.len(), env_ptr, env_len) }; + + if server_id == 0 { + return Err(Error::with_message( + ErrorKind::InvalidConfig, + format!( + "copilot_runtime_host_start failed (library '{}', entrypoint '{}')", + self.library_path.display(), + self.entrypoint.display() + ), + )); + } + + let (tx, rx) = mpsc::unbounded_channel::>(); + let state_ptr = Box::into_raw(Box::new(CallbackState { + tx, + active_callbacks: AtomicUsize::new(0), + closing: AtomicBool::new(false), + })); + let connection_id = unsafe { + (self.connection_open)( + server_id, + on_outbound, + state_ptr as *mut c_void, + std::ptr::null(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 0, + ) + }; + if connection_id == 0 { + drop(unsafe { Box::from_raw(state_ptr) }); + unsafe { (self.host_shutdown)(server_id) }; + return Err(Error::with_message( + ErrorKind::InvalidConfig, + "copilot_runtime_connection_open failed", + )); + } + + let shared = Arc::new(FfiShared { + host_shutdown: self.host_shutdown, + connection_write: self.connection_write, + connection_close: self.connection_close, + server_id: AtomicU32::new(server_id), + connection_id: AtomicU32::new(connection_id), + callback_state: AtomicPtr::new(state_ptr), + closed: AtomicBool::new(false), + operation_lock: parking_lot::Mutex::new(()), + library_path: self.library_path.clone(), + }); + + debug!( + library = %self.library_path.display(), + server_id, connection_id, "FFI runtime host started" + ); + + let reader = FfiReader { + rx, + leftover: Vec::new(), + pos: 0, + }; + let writer = FfiWriter { + shared: Arc::clone(&shared), + }; + Ok((reader, writer, shared)) + } +} + +fn bind<'lib, T>( + lib: &'lib Library, + symbol: &[u8], + library_path: &Path, +) -> Result, Error> { + match unsafe { lib.get::(symbol) } { + Ok(export) => Ok(export), + Err(e) => Err(Error::with_message( + ErrorKind::InvalidConfig, + format!( + "in-process runtime library '{}' is missing an expected export ({}): {e}", + library_path.display(), + String::from_utf8_lossy(symbol.strip_suffix(b"\0").unwrap_or(symbol)) + ), + )), + } +} + +/// Loads the runtime cdylib once per process and never unloads it, returning a +/// `'static` reference. Subsequent loads of the same path reuse the first +/// handle. +/// +/// The library stays mapped because native worker threads can outlive an +/// individual connection teardown. +fn load_library(library_path: &Path) -> Result<&'static Library, Error> { + static LIBRARIES: OnceLock>> = + OnceLock::new(); + let cache = LIBRARIES.get_or_init(|| parking_lot::Mutex::new(HashMap::new())); + + let mut guard = cache.lock(); + if let Some(lib) = guard.get(library_path) { + return Ok(*lib); + } + + let lib = unsafe { Library::new(library_path) }.map_err(|e| { + Error::with_message( + ErrorKind::InvalidConfig, + format!( + "failed to load in-process runtime library '{}': {e}", + library_path.display() + ), + ) + })?; + // Leak the library so it is never unloaded for the process lifetime. + let leaked: &'static Library = Box::leak(Box::new(lib)); + guard.insert(library_path.to_path_buf(), leaked); + Ok(leaked) +} + +/// The natural platform shared-library file name for the runtime cdylib — the +/// `.node` file renamed to what the Rust cdylib would be called on this OS. +fn natural_library_name() -> &'static str { + if cfg!(windows) { + "copilot_runtime.dll" + } else if cfg!(target_os = "macos") { + "libcopilot_runtime.dylib" + } else { + "libcopilot_runtime.so" + } +} + +/// The package prebuild folder name for the current host. +pub(crate) fn prebuilds_folder() -> Option { + let platform = if cfg!(target_os = "windows") { + "win32" + } else if cfg!(target_os = "macos") { + "darwin" + } else if cfg!(target_os = "linux") { + "linux" + } else { + return None; + }; + let arch = if cfg!(target_arch = "x86_64") { + "x64" + } else if cfg!(target_arch = "aarch64") { + "arm64" + } else { + return None; + }; + Some(format!("{platform}-{arch}")) +} + +fn resolve_library_path(entrypoint: &Path) -> Result { + let dir = entrypoint.parent().ok_or_else(|| { + Error::with_message( + ErrorKind::InvalidConfig, + format!( + "could not determine directory for CLI entrypoint '{}'", + entrypoint.display() + ), + ) + })?; + + // Bundled/flat layout: natural shared-library name next to the CLI. + let flat = dir.join(natural_library_name()); + if flat.is_file() { + return Ok(flat); + } + + // Development package layout. + let prebuilds = + prebuilds_folder().map(|folder| dir.join("prebuilds").join(folder).join("runtime.node")); + if let Some(prebuilds_path) = &prebuilds + && prebuilds_path.is_file() + { + return Ok(prebuilds_path.clone()); + } + + Err(Error::with_message( + ErrorKind::BinaryNotFound { + name: natural_library_name().into(), + hint: Some(format!( + "native runtime library not found next to '{}'. Enable the \ + `bundled-in-process` feature or set COPILOT_CLI_PATH to a compatible CLI package.", + entrypoint.display() + )), + }, + "native runtime library not found", + )) +} + +#[cfg(windows)] +fn path_for_child_process(path: PathBuf) -> PathBuf { + use std::ffi::OsString; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + + const VERBATIM_PREFIX: &[u16] = &[b'\\' as u16, b'\\' as u16, b'?' as u16, b'\\' as u16]; + const UNC_PREFIX: &[u16] = &[b'U' as u16, b'N' as u16, b'C' as u16, b'\\' as u16]; + + let encoded: Vec = path.as_os_str().encode_wide().collect(); + let Some(stripped) = encoded.strip_prefix(VERBATIM_PREFIX) else { + return path; + }; + let normalized = if let Some(unc_path) = stripped.strip_prefix(UNC_PREFIX) { + let mut result = vec![b'\\' as u16, b'\\' as u16]; + result.extend_from_slice(unc_path); + result + } else { + stripped.to_vec() + }; + PathBuf::from(OsString::from_wide(&normalized)) +} + +#[cfg(not(windows))] +fn path_for_child_process(path: PathBuf) -> PathBuf { + path +} + +fn build_argv_json(entrypoint: &Path, extra_args: &[String]) -> Vec { + // A `.js` entrypoint (dev / dist-cli) is launched via node; the packaged + // single-file CLI binary embeds its own Node and is invoked directly. + let entrypoint_str = entrypoint.to_string_lossy().into_owned(); + let is_js = entrypoint + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("js")); + let mut argv: Vec = if is_js { + vec![ + "node".to_string(), + entrypoint_str, + "--embedded-host".to_string(), + "--no-auto-update".to_string(), + ] + } else { + vec![ + entrypoint_str, + "--embedded-host".to_string(), + "--no-auto-update".to_string(), + ] + }; + argv.extend_from_slice(extra_args); + serde_json::to_vec(&argv).expect("argv serializes") +} + +fn build_env_json(environment: &[(String, String)]) -> Option> { + if environment.is_empty() { + return None; + } + let map: serde_json::Map = environment + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + Some(serde_json::to_vec(&map).expect("env serializes")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn argv_pins_worker_and_appends_client_options() { + let argv: Vec = serde_json::from_slice(&build_argv_json( + Path::new("copilot"), + &["--log-level".into(), "debug".into()], + )) + .unwrap(); + + assert_eq!( + argv, + [ + "copilot", + "--embedded-host", + "--no-auto-update", + "--log-level", + "debug" + ] + ); + } + + #[test] + fn javascript_entrypoint_uses_node() { + let argv: Vec = + serde_json::from_slice(&build_argv_json(Path::new("index.js"), &[])).unwrap(); + + assert_eq!( + argv, + ["node", "index.js", "--embedded-host", "--no-auto-update"] + ); + } + + #[cfg(windows)] + #[test] + fn child_process_path_removes_windows_verbatim_prefix() { + assert_eq!( + path_for_child_process(PathBuf::from(r"\\?\D:\a\copilot-sdk\index.js")), + PathBuf::from(r"D:\a\copilot-sdk\index.js") + ); + assert_eq!( + path_for_child_process(PathBuf::from(r"\\?\UNC\server\share\copilot-sdk\index.js")), + PathBuf::from(r"\\server\share\copilot-sdk\index.js") + ); + } + + #[test] + fn environment_is_omitted_when_empty() { + assert_eq!(build_env_json(&[]), None); + } + + #[test] + fn environment_serializes_worker_overrides() { + let env: serde_json::Value = serde_json::from_slice( + &build_env_json(&[ + ("COPILOT_HOME".into(), "state".into()), + ("COPILOT_DISABLE_KEYTAR".into(), "1".into()), + ]) + .unwrap(), + ) + .unwrap(); + + assert_eq!( + env, + serde_json::json!({ + "COPILOT_HOME": "state", + "COPILOT_DISABLE_KEYTAR": "1", + }) + ); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4006a8f44a..8e5685ea2b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -10,6 +10,9 @@ mod canvas_dispatch; #[cfg(feature = "bundled-cli")] pub(crate) mod embeddedcli; mod errors; +/// In-process FFI transport hosting the runtime cdylib (`Transport::InProcess`). +#[cfg(feature = "bundled-in-process")] +pub(crate) mod ffi; pub use errors::*; /// Connection-level Copilot request handler — intercept and replace the /// model-layer HTTP and WebSocket traffic the runtime issues for both CAPI and @@ -113,9 +116,26 @@ const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); #[derive(Debug, Default)] #[non_exhaustive] pub enum Transport { - /// Communicate over stdin/stdout pipes (default). + /// Resolve the transport from `COPILOT_SDK_DEFAULT_CONNECTION`, falling + /// back to [`Transport::Stdio`] when the variable is unset. #[default] + Default, + /// Communicate over stdin/stdout pipes (default). Stdio, + /// Host the runtime in-process over FFI (no child process). + /// + /// Loads the native runtime library next to the resolved CLI entrypoint + /// and speaks JSON-RPC over its C ABI. The runtime spawns its + /// own worker; the SDK never launches a CLI child process. This is + /// **experimental**. Per-client [`ClientOptions::working_directory`], + /// [`ClientOptions::env`]/[`ClientOptions::env_remove`], + /// [`ClientOptions::telemetry`], and [`ClientOptions::github_token`] are + /// not supported because native runtime code shares the host process. + /// [`ClientOptions::base_directory`] remains supported because it is + /// passed to the spawned worker as `COPILOT_HOME`. + /// + /// Requires the `bundled-in-process` Cargo feature. + InProcess, /// Spawn the CLI with `--port` and connect via TCP. Tcp { /// Port to listen on (0 for OS-assigned). @@ -212,6 +232,8 @@ pub struct ClientOptions { /// Arguments prepended before `--server` (e.g. the script path for node). pub prefix_args: Vec, /// Working directory for the CLI process. + /// + /// Setting this option is not supported with [`Transport::InProcess`]. pub working_directory: PathBuf, /// Environment variables set on the child process. pub env: Vec<(OsString, OsString)>, @@ -593,7 +615,7 @@ impl Default for ClientOptions { Self { program: CliProgram::Resolve, prefix_args: Vec::new(), - working_directory: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), + working_directory: PathBuf::new(), env: Vec::new(), env_remove: Vec::new(), extra_args: Vec::new(), @@ -854,6 +876,72 @@ fn generate_connection_token() -> String { hex } +/// Environment variable that overrides the transport used when the caller +/// leaves [`ClientOptions::transport`] at [`Transport::Default`]. +/// Accepts `"inprocess"` or `"stdio"` (case-insensitive); unset preserves +/// stdio. Any other value is an error. +const DEFAULT_CONNECTION_ENV_VAR: &str = "COPILOT_SDK_DEFAULT_CONNECTION"; + +/// Resolve a transport override from [`DEFAULT_CONNECTION_ENV_VAR`]. +fn resolve_default_transport(options: &ClientOptions) -> Result { + let configured = options + .env + .iter() + .find(|(key, _)| { + key.to_string_lossy() + .eq_ignore_ascii_case(DEFAULT_CONNECTION_ENV_VAR) + }) + .map(|(_, value)| value.to_string_lossy().into_owned()); + let process = std::env::var(DEFAULT_CONNECTION_ENV_VAR).ok(); + resolve_default_transport_value(configured.as_deref().or(process.as_deref())) +} + +fn resolve_default_transport_value(value: Option<&str>) -> Result { + match value { + None => Ok(Transport::Stdio), + Some(v) if v.is_empty() || v.eq_ignore_ascii_case("stdio") => Ok(Transport::Stdio), + Some(v) if v.eq_ignore_ascii_case("inprocess") => Ok(Transport::InProcess), + Some(v) => Err(Error::with_message( + ErrorKind::InvalidConfig, + format!( + "invalid {DEFAULT_CONNECTION_ENV_VAR} value '{v}'. \ + Expected 'inprocess', 'stdio', or unset." + ), + )), + } +} + +#[cfg(any(feature = "bundled-in-process", test))] +fn validate_inprocess_options(options: &ClientOptions) -> Result<()> { + let unsupported = if !options.working_directory.as_os_str().is_empty() { + Some("working_directory") + } else if !options.env.is_empty() { + Some("env") + } else if !options.env_remove.is_empty() { + Some("env_remove") + } else if options.telemetry.is_some() { + Some("telemetry") + } else if options.github_token.is_some() { + Some("github_token") + } else if !options.prefix_args.is_empty() { + Some("prefix_args") + } else { + None + }; + + if let Some(option) = unsupported { + return Err(Error::with_message( + ErrorKind::InvalidConfig, + format!( + "ClientOptions::{option} is not supported with Transport::InProcess; \ + configure process-global settings on the host process instead" + ), + )); + } + + Ok(()) +} + /// Connection to a GitHub Copilot CLI server (stdio, TCP, or external). /// /// Cheaply cloneable — cloning shares the underlying connection. @@ -874,6 +962,10 @@ impl std::fmt::Debug for Client { struct ClientInner { child: parking_lot::Mutex>, + #[cfg(feature = "bundled-in-process")] + /// In-process FFI runtime host, set only for [`Transport::InProcess`]. + /// Closing it tears down the FFI connection and worker. + ffi_host: parking_lot::Mutex>>, rpc: JsonRpcClient, cwd: PathBuf, request_rx: parking_lot::Mutex>>, @@ -920,6 +1012,21 @@ impl Client { /// backend. pub async fn start(options: ClientOptions) -> Result { let start_time = Instant::now(); + let mut options = options; + if matches!(options.transport, Transport::Default) { + options.transport = resolve_default_transport(&options)?; + } + if matches!(options.transport, Transport::InProcess) { + #[cfg(not(feature = "bundled-in-process"))] + { + return Err(Error::with_message( + ErrorKind::InvalidConfig, + "Transport::InProcess requires the `bundled-in-process` Cargo feature", + )); + } + #[cfg(feature = "bundled-in-process")] + validate_inprocess_options(&options)?; + } if options.mode == ClientMode::Empty && options.base_directory.is_none() && options.session_fs.is_none() @@ -974,9 +1081,9 @@ impl Client { // to the server. For Tcp, the SDK auto-generates one when the // caller leaves it unset so the loopback listener is safe by // default. - let mut options = options; let effective_connection_token: Option = match &mut options.transport { - Transport::Stdio => None, + Transport::Default => unreachable!("default transport resolved above"), + Transport::Stdio | Transport::InProcess => None, Transport::Tcp { connection_token, .. } => Some( @@ -1020,8 +1127,17 @@ impl Client { resolved } }; + let working_directory = { + let cwd = options.working_directory.clone(); + if cwd.as_os_str().is_empty() { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) + } else { + cwd + } + }; let client = match options.transport { + Transport::Default => unreachable!("default transport resolved above"), Transport::External { ref host, port, @@ -1041,7 +1157,7 @@ impl Client { reader, writer, None, - options.working_directory, + working_directory, options.on_list_models, session_fs_config.is_some(), session_fs_sqlite_declared, @@ -1055,7 +1171,8 @@ impl Client { port, connection_token: _, } => { - let (mut child, actual_port) = Self::spawn_tcp(&program, &options, port).await?; + let (mut child, actual_port) = + Self::spawn_tcp(&program, &options, &working_directory, port).await?; let connect_start = Instant::now(); let stream = TcpStream::connect(("127.0.0.1", actual_port)).await?; debug!( @@ -1069,7 +1186,7 @@ impl Client { reader, writer, Some(child), - options.working_directory, + working_directory, options.on_list_models, session_fs_config.is_some(), session_fs_sqlite_declared, @@ -1080,7 +1197,7 @@ impl Client { )? } Transport::Stdio => { - let mut child = Self::spawn_stdio(&program, &options)?; + let mut child = Self::spawn_stdio(&program, &options, &working_directory)?; let stdin = child.stdin.take().expect("stdin is piped"); let stdout = child.stdout.take().expect("stdout is piped"); Self::drain_stderr(&mut child); @@ -1088,7 +1205,7 @@ impl Client { stdout, stdin, Some(child), - options.working_directory, + working_directory, options.on_list_models, session_fs_config.is_some(), session_fs_sqlite_declared, @@ -1098,8 +1215,57 @@ impl Client { options.mode, )? } + Transport::InProcess => { + #[cfg(feature = "bundled-in-process")] + { + info!(entrypoint = %program.display(), "hosting copilot runtime in-process (FFI)"); + let mut environment = Vec::new(); + if let Some(base_directory) = &options.base_directory { + let value = base_directory.to_str().ok_or_else(|| { + Error::with_message( + ErrorKind::InvalidConfig, + "base_directory must be valid UTF-8 for Transport::InProcess", + ) + })?; + environment.push(("COPILOT_HOME".to_string(), value.to_string())); + } + if options.mode == ClientMode::Empty { + environment.push(("COPILOT_DISABLE_KEYTAR".to_string(), "1".to_string())); + } + let mut args = Vec::new(); + args.extend( + Self::log_level_args(&options) + .into_iter() + .map(str::to_string), + ); + args.extend(Self::session_idle_timeout_args(&options)); + args.extend(Self::remote_args(&options)); + if options.use_logged_in_user == Some(false) { + args.push("--no-auto-login".to_string()); + } + args.extend(options.extra_args.clone()); + let host = crate::ffi::FfiHost::create(&program, environment, args)?; + let (reader, writer, shared) = host.start().await?; + let client = Self::from_transport( + reader, + writer, + None, + working_directory, + options.on_list_models, + session_fs_config.is_some(), + session_fs_sqlite_declared, + options.on_get_trace_context, + options.on_github_telemetry, + effective_connection_token.clone(), + options.mode, + )?; + *client.inner.ffi_host.lock() = Some(shared); + client + } + #[cfg(not(feature = "bundled-in-process"))] + unreachable!("in-process feature validation returned above") + } }; - debug!( elapsed_ms = start_time.elapsed().as_millis(), "Client::start transport setup complete" @@ -1298,6 +1464,8 @@ impl Client { let client = Self { inner: Arc::new(ClientInner { child: parking_lot::Mutex::new(child), + #[cfg(feature = "bundled-in-process")] + ffi_host: parking_lot::Mutex::new(None), rpc, cwd, request_rx: parking_lot::Mutex::new(Some(request_rx)), @@ -1366,7 +1534,7 @@ impl Client { }); } - fn build_command(program: &Path, options: &ClientOptions) -> Command { + fn build_command(program: &Path, options: &ClientOptions, working_directory: &Path) -> Command { let mut command = Command::new(program); for arg in &options.prefix_args { command.arg(arg); @@ -1424,7 +1592,7 @@ impl Client { command.env_remove(key); } command - .current_dir(&options.working_directory) + .current_dir(working_directory) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1487,9 +1655,13 @@ impl Client { } } - fn spawn_stdio(program: &Path, options: &ClientOptions) -> Result { - info!(cwd = ?options.working_directory, program = %program.display(), "spawning copilot CLI (stdio)"); - let mut command = Self::build_command(program, options); + fn spawn_stdio( + program: &Path, + options: &ClientOptions, + working_directory: &Path, + ) -> Result { + info!(cwd = ?working_directory, program = %program.display(), "spawning copilot CLI (stdio)"); + let mut command = Self::build_command(program, options, working_directory); command .args(["--server", "--stdio", "--no-auto-update"]) .args(Self::log_level_args(options)) @@ -1507,9 +1679,14 @@ impl Client { Ok(child) } - async fn spawn_tcp(program: &Path, options: &ClientOptions, port: u16) -> Result<(Child, u16)> { - info!(cwd = ?options.working_directory, program = %program.display(), port = %port, "spawning copilot CLI (tcp)"); - let mut command = Self::build_command(program, options); + async fn spawn_tcp( + program: &Path, + options: &ClientOptions, + working_directory: &Path, + port: u16, + ) -> Result<(Child, u16)> { + info!(cwd = ?working_directory, program = %program.display(), port = %port, "spawning copilot CLI (tcp)"); + let mut command = Self::build_command(program, options, working_directory); command .args(["--server", "--port", &port.to_string(), "--no-auto-update"]) .args(Self::log_level_args(options)) @@ -2068,6 +2245,9 @@ impl Client { } let should_shutdown_runtime = self.inner.child.lock().is_some(); + #[cfg(feature = "bundled-in-process")] + let should_shutdown_runtime = + should_shutdown_runtime || self.inner.ffi_host.lock().is_some(); if should_shutdown_runtime { let runtime_shutdown_start = Instant::now(); match tokio::time::timeout(RUNTIME_SHUTDOWN_TIMEOUT, self.rpc().runtime().shutdown()) @@ -2124,6 +2304,16 @@ impl Client { } } + // The runtime.shutdown RPC above already asked the worker to clean up; + // closing here tears down the transport. + #[cfg(feature = "bundled-in-process")] + { + if let Some(host) = self.inner.ffi_host.lock().take() { + self.inner.rpc.force_close(); + host.close(); + } + } + info!(pid = ?pid, errors = errors.len(), "CLI process stopped"); if errors.is_empty() { Ok(()) @@ -2170,6 +2360,12 @@ impl Client { error!(pid = ?pid, error = %e, "failed to send kill signal"); } self.inner.rpc.force_close(); + #[cfg(feature = "bundled-in-process")] + { + if let Some(host) = self.inner.ffi_host.lock().take() { + host.close(); + } + } // Drop all session channels so any awaiters see a closed channel // instead of waiting for responses that will never arrive. self.inner.router.clear(); @@ -2226,6 +2422,13 @@ impl Drop for ClientInner { info!(pid = ?pid, "kill signal sent for CLI process on drop"); } } + #[cfg(feature = "bundled-in-process")] + { + if let Some(host) = self.ffi_host.lock().take() { + self.rpc.force_close(); + host.close(); + } + } } } @@ -2290,6 +2493,66 @@ mod tests { assert!(opts.enable_remote_sessions); } + #[test] + fn default_transport_values_resolve_without_process_state() { + assert!(matches!( + resolve_default_transport_value(None).unwrap(), + Transport::Stdio + )); + assert!(matches!( + resolve_default_transport_value(Some("stdio")).unwrap(), + Transport::Stdio + )); + assert!(matches!( + resolve_default_transport_value(Some("INPROCESS")).unwrap(), + Transport::InProcess + )); + assert!(resolve_default_transport_value(Some("tcp")).is_err()); + } + + #[test] + fn inprocess_rejects_process_scoped_options() { + let invalid = [ + ClientOptions::new().with_cwd("."), + ClientOptions::new().with_env([("KEY", "value")]), + ClientOptions::new().with_env_remove(["KEY"]), + ClientOptions::new().with_telemetry(TelemetryConfig::default()), + ClientOptions::new().with_github_token("token"), + ClientOptions::new().with_prefix_args(["index.js"]), + ]; + + for options in invalid { + assert!(validate_inprocess_options(&options).is_err()); + } + } + + #[test] + fn inprocess_allows_worker_and_rpc_options() { + let options = ClientOptions::new() + .with_base_directory("state") + .with_log_level(LogLevel::Debug) + .with_session_idle_timeout_seconds(10) + .with_use_logged_in_user(false) + .with_enable_remote_sessions(true) + .with_extra_args(["--verbose"]); + + assert!(validate_inprocess_options(&options).is_ok()); + } + + #[cfg(not(feature = "bundled-in-process"))] + #[tokio::test] + async fn inprocess_requires_cargo_feature() { + let error = Client::start( + ClientOptions::new() + .with_program(CliProgram::Path("copilot".into())) + .with_transport(Transport::InProcess), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("bundled-in-process")); + } + #[test] fn is_transport_failure_rejects_other_protocol_errors() { let err = Error::from(ErrorKind::Protocol(ProtocolErrorKind::CliStartupTimeout)); @@ -2303,7 +2566,7 @@ mod tests { env_remove: vec![std::ffi::OsString::from("COPILOT_SDK_AUTH_TOKEN")], ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); // get_envs() iter yields the latest action per key — None means removed. let action = cmd .as_std() @@ -2327,7 +2590,7 @@ mod tests { )], ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); let value = cmd .as_std() .get_envs() @@ -2342,7 +2605,7 @@ mod tests { github_token: Some("just-the-token".to_string()), ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); let value = cmd .as_std() .get_envs() @@ -2410,7 +2673,7 @@ mod tests { }), ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert_eq!( env_value(&cmd, "COPILOT_OTEL_ENABLED"), Some(std::ffi::OsStr::new("true")), @@ -2444,7 +2707,7 @@ mod tests { #[test] fn build_command_omits_otel_env_when_telemetry_none() { let opts = ClientOptions::default(); - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); for key in [ "COPILOT_OTEL_ENABLED", "OTEL_EXPORTER_OTLP_ENDPOINT", @@ -2470,7 +2733,7 @@ mod tests { }), ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); // The one set field plus the implicit enabled flag should propagate. assert_eq!( env_value(&cmd, "COPILOT_OTEL_ENABLED"), @@ -2505,7 +2768,7 @@ mod tests { )], ..Default::default() }; - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert_eq!( env_value(&cmd, "OTEL_EXPORTER_OTLP_ENDPOINT"), Some(std::ffi::OsStr::new("http://from-user-env:4318")), @@ -2516,14 +2779,14 @@ mod tests { #[test] fn build_command_sets_copilot_home_env_when_configured() { let opts = ClientOptions::new().with_base_directory(PathBuf::from("/custom/copilot")); - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert_eq!( env_value(&cmd, "COPILOT_HOME"), Some(std::ffi::OsStr::new("/custom/copilot")), ); let opts = ClientOptions::default(); - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert!(env_value(&cmd, "COPILOT_HOME").is_none()); } @@ -2533,14 +2796,14 @@ mod tests { port: 0, connection_token: Some("secret-token".to_string()), }); - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert_eq!( env_value(&cmd, "COPILOT_CONNECTION_TOKEN"), Some(std::ffi::OsStr::new("secret-token")), ); let opts = ClientOptions::default(); - let cmd = Client::build_command(Path::new("/bin/echo"), &opts); + let cmd = Client::build_command(Path::new("/bin/echo"), &opts, Path::new("/tmp")); assert!(env_value(&cmd, "COPILOT_CONNECTION_TOKEN").is_none()); } @@ -2591,8 +2854,9 @@ mod tests { }), ..Default::default() }; - let cmd_true = Client::build_command(Path::new("/bin/echo"), &opts_true); - let cmd_false = Client::build_command(Path::new("/bin/echo"), &opts_false); + let cmd_true = Client::build_command(Path::new("/bin/echo"), &opts_true, Path::new("/tmp")); + let cmd_false = + Client::build_command(Path::new("/bin/echo"), &opts_false, Path::new("/tmp")); assert_eq!( env_value( &cmd_true, @@ -2802,6 +3066,8 @@ mod tests { Client { inner: Arc::new(ClientInner { child: parking_lot::Mutex::new(None), + #[cfg(feature = "bundled-in-process")] + ffi_host: parking_lot::Mutex::new(None), rpc: { let (req_tx, _req_rx) = mpsc::unbounded_channel(); let (notif_tx, _notif_rx) = broadcast::channel(16); diff --git a/rust/tests/cli_resolution_test.rs b/rust/tests/cli_resolution_test.rs index c3044a9e75..9e4927e676 100644 --- a/rust/tests/cli_resolution_test.rs +++ b/rust/tests/cli_resolution_test.rs @@ -196,21 +196,27 @@ async fn extract_dir_runtime_override_is_honored() { let _ = fake; } -/// Build-time version pin: `cli-version.txt` (when present) must be a -/// combined snapshot — a `version=X.Y.Z` line plus per-asset hash lines. +/// Build-time version pins, when present, must match the selected bundling +/// implementation's checksum format. /// When absent, build.rs falls through to `../nodejs/package-lock.json` — /// both are accepted, this test only checks the pin file's format if it's /// there. #[test] fn pin_file_when_present_is_well_formed() { let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let pin = PathBuf::from(manifest_dir).join("cli-version.txt"); + let (filename, value_prefix) = if cfg!(feature = "bundled-in-process") { + ("cli-version-in-process.txt", Some("sha512-")) + } else { + ("cli-version.txt", None) + }; + let pin = PathBuf::from(manifest_dir).join(filename); if !pin.is_file() { // Contributor build path — no assertion needed. return; } - let contents = std::fs::read_to_string(&pin).expect("read cli-version.txt"); + let contents = std::fs::read_to_string(&pin).expect("read CLI version snapshot"); let mut saw_version = false; + let mut package_count = 0; for raw in contents.lines() { let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -222,9 +228,28 @@ fn pin_file_when_present_is_well_formed() { assert!(!value.trim().is_empty(), "empty value for key {key:?}"); if key.trim() == "version" { saw_version = true; + } else { + if let Some(prefix) = value_prefix { + assert!( + value.trim().starts_with(prefix), + "invalid npm integrity for key {key:?}" + ); + } else { + assert_eq!( + value.trim().len(), + 64, + "invalid SHA-256 hash for key {key:?}" + ); + assert!( + value.trim().bytes().all(|byte| byte.is_ascii_hexdigit()), + "invalid SHA-256 hash for key {key:?}" + ); + } + package_count += 1; } } - assert!(saw_version, "cli-version.txt missing `version=` line"); + assert!(saw_version, "{filename} missing `version=` line"); + assert_eq!(package_count, 6); } /// With `bundled-cli` on AND a supported target, `install_bundled_cli` @@ -246,6 +271,26 @@ fn install_bundled_cli_returns_extracted_path() { first, second, "install_bundled_cli must be idempotent across calls" ); + + #[cfg(feature = "bundled-in-process")] + { + let runtime_name = if cfg!(windows) { + "copilot_runtime.dll" + } else if cfg!(target_os = "macos") { + "libcopilot_runtime.dylib" + } else { + "libcopilot_runtime.so" + }; + let runtime = first + .parent() + .expect("install directory") + .join(runtime_name); + assert!( + runtime.is_file(), + "bundled runtime library was not installed: {}", + runtime.display() + ); + } } /// `install_bundled_cli` returns the same path the runtime resolver diff --git a/rust/tests/e2e.rs b/rust/tests/e2e.rs index 62412963b8..3a698abd18 100644 --- a/rust/tests/e2e.rs +++ b/rust/tests/e2e.rs @@ -37,6 +37,9 @@ mod github_telemetry; mod hooks; #[path = "e2e/hooks_extended.rs"] mod hooks_extended; +#[cfg(feature = "bundled-in-process")] +#[path = "e2e/inprocess.rs"] +mod inprocess; #[path = "e2e/mcp_and_agents.rs"] mod mcp_and_agents; #[path = "e2e/mcp_oauth.rs"] diff --git a/rust/tests/e2e/byok_bearer_token_provider.rs b/rust/tests/e2e/byok_bearer_token_provider.rs index fc3ef89d94..a7989d157f 100644 --- a/rust/tests/e2e/byok_bearer_token_provider.rs +++ b/rust/tests/e2e/byok_bearer_token_provider.rs @@ -142,6 +142,21 @@ async fn run_turn( #[tokio::test] async fn callback_token_is_applied_as_authorization_header() { + // The runtime's LLM inference provider slot is process-global and is never released + // when the registering connection disconnects (runtime `shared_api/llm_inference.rs`). + // Over the in-process transport all clients share this process's runtime, so once a + // BYOK provider is registered here and the client stops, the dangling registration + // routes every later model-inference request (list-models, tool-using turns, hooks, + // …) to the dead connection and hangs them. Registering a BYOK provider in-process + // therefore poisons the shared runtime for the rest of the suite. The BYOK bearer-token + // wiring is covered over stdio (a separate child process per test); the SDK-side + // request/response plumbing is transport-agnostic. + if super::support::skip_inprocess( + "registering a BYOK LLM inference provider is process-global in-process and is never \ + released on disconnect, poisoning later model-inference tests", + ) { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); @@ -189,6 +204,18 @@ async fn callback_token_is_applied_as_authorization_header() { #[tokio::test] async fn reacquires_a_fresh_token_for_each_request() { + // The runtime registers the LLM inference provider per connection and, by design, + // never releases the slot on disconnect (runtime `shared_api/llm_inference.rs`). Over + // the in-process transport every client shares this process's runtime, so a second + // provider-registering client is refused ("Another client is already the LLM + // inference provider"). The BYOK bearer-token behavior over the in-process transport + // is covered by `callback_token_is_applied_as_authorization_header`; this scenario's + // provider-dispatch logic is transport-agnostic and is covered over stdio. + if super::support::skip_inprocess( + "llmInference.setProvider is process-global in-process; a second provider client is refused", + ) { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); @@ -252,6 +279,16 @@ async fn reacquires_a_fresh_token_for_each_request() { #[tokio::test] async fn dispatches_token_acquisition_per_provider() { + // See `reacquires_a_fresh_token_for_each_request`: in-process, the process-global LLM + // inference provider registration is not released on disconnect, so this additional + // provider-registering client is refused. The BYOK transport path is covered in-process + // by `callback_token_is_applied_as_authorization_header`; the per-provider dispatch + // logic exercised here is transport-agnostic and covered over stdio. + if super::support::skip_inprocess( + "llmInference.setProvider is process-global in-process; a second provider client is refused", + ) { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); diff --git a/rust/tests/e2e/client.rs b/rust/tests/e2e/client.rs index 114e828ac9..6dd0f27acf 100644 --- a/rust/tests/e2e/client.rs +++ b/rust/tests/e2e/client.rs @@ -64,8 +64,7 @@ async fn should_get_authenticated_status() { Box::pin(async move { ctx.set_default_copilot_user(); let client = Client::start( - ctx.client_options() - .with_github_token(super::support::DEFAULT_TEST_TOKEN), + ctx.client_options_with_github_token(super::support::DEFAULT_TEST_TOKEN), ) .await .expect("start client"); @@ -85,8 +84,7 @@ async fn should_list_models_when_authenticated() { Box::pin(async move { ctx.set_default_copilot_user(); let client = Client::start( - ctx.client_options() - .with_github_token(super::support::DEFAULT_TEST_TOKEN), + ctx.client_options_with_github_token(super::support::DEFAULT_TEST_TOKEN), ) .await .expect("start client"); diff --git a/rust/tests/e2e/client_options.rs b/rust/tests/e2e/client_options.rs index dbf3c5c83d..c279557a66 100644 --- a/rust/tests/e2e/client_options.rs +++ b/rust/tests/e2e/client_options.rs @@ -6,7 +6,7 @@ use github_copilot_sdk::rpc::{OpenCanvasInstance, RemoteSessionMode}; use github_copilot_sdk::session_events::{ReasoningSummary, SessionLimitsConfig}; use github_copilot_sdk::{ CliProgram, Client, ClientOptions, ExtensionInfo, ProviderConfig, ResumeSessionConfig, - SessionConfig, SessionId, + SessionConfig, SessionId, Transport, }; use serde::Deserialize; use serde_json::{Value, json}; @@ -347,6 +347,7 @@ impl FakeCli { ]) .with_github_token(token) .with_use_logged_in_user(false) + .with_transport(Transport::Stdio) } fn path(&self, name: &str) -> PathBuf { diff --git a/rust/tests/e2e/copilot_request_handler.rs b/rust/tests/e2e/copilot_request_handler.rs index 2dd1411734..46b4e510cd 100644 --- a/rust/tests/e2e/copilot_request_handler.rs +++ b/rust/tests/e2e/copilot_request_handler.rs @@ -502,6 +502,9 @@ async fn start_ws_upstream(counters: HandlerCounters) -> String { #[tokio::test] async fn services_http_and_websocket_via_handler() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); @@ -624,6 +627,9 @@ impl CopilotRequestHandler for RecordingHandler { #[tokio::test] async fn threads_session_id_into_inference() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); @@ -757,6 +763,9 @@ impl CopilotRequestHandler for ThrowingHandler { #[tokio::test] async fn surfaces_handler_errors() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); @@ -823,6 +832,9 @@ impl CopilotRequestHandler for CancellingHandler { #[tokio::test] async fn observes_runtime_driven_cancel() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); diff --git a/rust/tests/e2e/inprocess.rs b/rust/tests/e2e/inprocess.rs new file mode 100644 index 0000000000..0c183a27df --- /dev/null +++ b/rust/tests/e2e/inprocess.rs @@ -0,0 +1,25 @@ +use super::support::with_e2e_context; + +/// Starts an in-process client, performs a round-trip, and stops cleanly. +/// Fails hard if the in-process runtime library cannot be loaded. +#[tokio::test] +async fn should_start_ping_and_stop_inprocess_client() { + with_e2e_context("client", "should_start_ping_and_stop_stdio_client", |ctx| { + Box::pin(async move { + let client = ctx.start_inprocess_client().await; + + let response = client + .ping(Some("hello from rust in-process")) + .await + .expect("ping over in-process FFI transport"); + assert_eq!(response.message, "pong: hello from rust in-process"); + assert!(!response.timestamp.is_empty()); + + let status = client.get_status().await.expect("get status"); + assert!(status.protocol_version > 0); + + client.stop().await.expect("stop in-process client"); + }) + }) + .await; +} diff --git a/rust/tests/e2e/per_session_auth.rs b/rust/tests/e2e/per_session_auth.rs index b2fd11e4d4..efb005b590 100644 --- a/rust/tests/e2e/per_session_auth.rs +++ b/rust/tests/e2e/per_session_auth.rs @@ -7,6 +7,9 @@ use super::support::with_e2e_context; #[tokio::test] async fn session_uses_client_token_when_no_session_token_is_supplied() { + if super::support::skip_inprocess("client-level GitHub tokens are not supported in-process") { + return; + } with_e2e_context( "per-session-auth", "session_uses_client_token_when_no_session_token_is_supplied", @@ -47,6 +50,9 @@ async fn session_uses_client_token_when_no_session_token_is_supplied() { #[tokio::test] async fn session_token_overrides_client_token() { + if super::support::skip_inprocess("client-level GitHub tokens are not supported in-process") { + return; + } with_e2e_context( "per-session-auth", "session_token_overrides_client_token", @@ -93,7 +99,11 @@ async fn session_auth_status_is_unauthenticated_without_token() { "session_auth_status_is_unauthenticated_without_token", |ctx| { Box::pin(async move { - let client = ctx.start_client().await; + let client = github_copilot_sdk::Client::start( + ctx.client_options().with_use_logged_in_user(false), + ) + .await + .expect("start client"); let session = client .create_session( SessionConfig::default() diff --git a/rust/tests/e2e/provider_endpoint.rs b/rust/tests/e2e/provider_endpoint.rs index df6d5941dd..3953aad669 100644 --- a/rust/tests/e2e/provider_endpoint.rs +++ b/rust/tests/e2e/provider_endpoint.rs @@ -15,6 +15,7 @@ fn opt_in_env() -> (OsString, OsString) { } #[tokio::test] +#[allow(deprecated)] async fn byok_provider_endpoint_returns_configured_endpoint() { with_e2e_context( "provider-endpoint", @@ -22,7 +23,9 @@ async fn byok_provider_endpoint_returns_configured_endpoint() { |ctx| { Box::pin(async move { let mut options = ctx.client_options(); - options.env.push(opt_in_env()); + if !super::support::is_inprocess_default() { + options.env.push(opt_in_env()); + } let client = github_copilot_sdk::Client::start(options) .await .expect("start client"); @@ -86,6 +89,7 @@ async fn byok_provider_endpoint_returns_configured_endpoint() { } #[tokio::test] +#[allow(deprecated)] async fn capi_provider_endpoint_returns_resolved_credentials() { with_e2e_context( "provider-endpoint", @@ -93,8 +97,10 @@ async fn capi_provider_endpoint_returns_resolved_credentials() { |ctx| { Box::pin(async move { ctx.set_default_copilot_user(); - let mut options = ctx.client_options().with_github_token(DEFAULT_TEST_TOKEN); - options.env.push(opt_in_env()); + let mut options = ctx.client_options_with_github_token(DEFAULT_TEST_TOKEN); + if !super::support::is_inprocess_default() { + options.env.push(opt_in_env()); + } let client = github_copilot_sdk::Client::start(options) .await .expect("start client"); diff --git a/rust/tests/e2e/rpc_server.rs b/rust/tests/e2e/rpc_server.rs index 665041f49d..d0beab2452 100644 --- a/rust/tests/e2e/rpc_server.rs +++ b/rust/tests/e2e/rpc_server.rs @@ -54,7 +54,7 @@ async fn should_call_rpc_models_list_with_typed_result() { Box::pin(async move { let token = "rpc-models-token"; ctx.set_copilot_user_by_token_with_login(token, "rpc-user"); - let client = Client::start(ctx.client_options().with_github_token(token)) + let client = Client::start(ctx.client_options_with_github_token(token)) .await .expect("start client"); @@ -95,7 +95,7 @@ async fn should_call_rpc_account_get_quota_when_authenticated() { } })), ); - let client = Client::start(ctx.client_options().with_github_token(token)) + let client = Client::start(ctx.client_options_with_github_token(token)) .await .expect("start client"); diff --git a/rust/tests/e2e/rpc_session_state_extras.rs b/rust/tests/e2e/rpc_session_state_extras.rs index 148a4151c1..3bd6c99bee 100644 --- a/rust/tests/e2e/rpc_session_state_extras.rs +++ b/rust/tests/e2e/rpc_session_state_extras.rs @@ -22,7 +22,7 @@ async fn should_list_models_for_session() { Box::pin(async move { let token = "rpc-session-model-list-token"; ctx.set_copilot_user_by_token_with_login(token, "rpc-session-extras-user"); - let client = Client::start(ctx.client_options().with_github_token(token)) + let client = Client::start(ctx.client_options_with_github_token(token)) .await .expect("start authenticated client"); let session = client diff --git a/rust/tests/e2e/rpc_workspace_checkpoints.rs b/rust/tests/e2e/rpc_workspace_checkpoints.rs index 2c185a535a..0a8bf56152 100644 --- a/rust/tests/e2e/rpc_workspace_checkpoints.rs +++ b/rust/tests/e2e/rpc_workspace_checkpoints.rs @@ -40,6 +40,12 @@ async fn should_list_no_checkpoints_for_fresh_session() { #[tokio::test] async fn should_return_null_or_empty_content_for_unknown_checkpoint() { + // In-process, session.workspaces.readCheckpoint is answered by the native runtime, + // which decodes the checkpoint number as a u32 and rejects the i64::MAX sentinel this + // test uses. Covered by the default (stdio) transport. See issue #1934. + if super::support::skip_inprocess("readCheckpoint decodes the id as u32 in-process") { + return; + } with_e2e_context( "rpc_workspace_checkpoints", "should_return_null_or_empty_content_for_unknown_checkpoint", diff --git a/rust/tests/e2e/session_config.rs b/rust/tests/e2e/session_config.rs index 0a7a5eb11b..dd498e3761 100644 --- a/rust/tests/e2e/session_config.rs +++ b/rust/tests/e2e/session_config.rs @@ -515,6 +515,9 @@ fn assert_anthropic_document_citations_enabled(request_body: &[u8]) { #[tokio::test] async fn should_enable_citations_for_anthropic_file_attachments_on_create() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context_no_snapshot(|ctx| { Box::pin(async move { ctx.set_default_copilot_user(); diff --git a/rust/tests/e2e/subagent_hooks.rs b/rust/tests/e2e/subagent_hooks.rs index 8a21169c46..fe94c36779 100644 --- a/rust/tests/e2e/subagent_hooks.rs +++ b/rust/tests/e2e/subagent_hooks.rs @@ -15,6 +15,9 @@ use super::support::with_e2e_context; #[tokio::test] async fn should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls() { + if super::support::skip_inprocess("LLM inference providers are process-global in-process") { + return; + } with_e2e_context( "subagent_hooks", "should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls", diff --git a/rust/tests/e2e/support.rs b/rust/tests/e2e/support.rs index 7e47d8fbae..7479baeeba 100644 --- a/rust/tests/e2e/support.rs +++ b/rust/tests/e2e/support.rs @@ -36,6 +36,13 @@ where .await .unwrap_or_else(|err| panic!("create E2E context: {err}")); + // In-process hosting: the runtime loads into this test process and its worker + // inherits the ambient environment (per-client env is not honored in-process, see + // https://github.com/github/copilot-sdk/issues/1934), so mirror this context's env + // onto the process for the duration of the test and restore on drop. Safe because + // E2E_CONCURRENCY is 1 in-process, serializing the whole critical section. + let _env_guard = InProcessEnvGuard::activate(&ctx); + let timed_out = tokio::time::timeout(default_test_timeout(), test(&mut ctx)) .await .is_err(); @@ -65,6 +72,10 @@ where .await .unwrap_or_else(|err| panic!("create E2E context: {err}")); + // See `with_e2e_context` for why the in-process transport mirrors env onto the + // process (restored on drop). + let _env_guard = InProcessEnvGuard::activate(&ctx); + let timed_out = tokio::time::timeout(default_test_timeout(), test(&mut ctx)) .await .is_err(); @@ -104,6 +115,7 @@ impl E2eContext { proxy: Some(proxy), }; ctx.configure(category, snapshot_name)?; + ctx.set_default_copilot_user(); Ok(ctx) } @@ -133,6 +145,7 @@ impl E2eContext { .map_err(|err| { std::io::Error::other(format!("configure proxy without snapshot failed: {err}")) })?; + ctx.set_default_copilot_user(); Ok(ctx) } @@ -164,12 +177,43 @@ impl E2eContext { self.client_options().with_transport(transport) } + pub fn client_options_with_github_token(&self, token: &str) -> ClientOptions { + let options = self.client_options(); + if is_inprocess_default() { + // SAFETY: the in-process E2E suite is serialized for the full + // lifetime of InProcessEnvGuard. + unsafe { + std::env::set_var("GH_TOKEN", token); + std::env::set_var("GITHUB_TOKEN", token); + } + options + } else { + options.with_github_token(token) + } + } + pub async fn start_client(&self) -> Client { Client::start(self.client_options()) .await .expect("start E2E client") } + /// Start a client that hosts the runtime in-process over FFI + /// ([`Transport::InProcess`]). Unlike the stdio harness, the CLI + /// entrypoint is passed as the program directly (the FFI host builds the + /// `node --embedded-host` argv itself and loads the sibling + /// runtime cdylib), so a `.js` entrypoint is not split into node + + /// prefix_args here. + pub async fn start_inprocess_client(&self) -> Client { + let options = ClientOptions::new() + .with_use_logged_in_user(false) + .with_program(CliProgram::Path(self.cli_path.clone())) + .with_transport(Transport::InProcess); + Client::start(options) + .await + .expect("start in-process FFI E2E client") + } + /// Start a client wired to a Copilot request handler, appending `extra_env` /// to the spawned runtime's environment (used to flip the WebSocket ExP /// flag for the WS transport tests). @@ -302,11 +346,13 @@ impl E2eContext { ), ("COPILOT_MCP_APPS".into(), "true".into()), ("MCP_APPS".into(), "true".into()), + ("GH_TOKEN".into(), DEFAULT_TEST_TOKEN.into()), + ("GITHUB_TOKEN".into(), DEFAULT_TEST_TOKEN.into()), + ("GH_ENTERPRISE_TOKEN".into(), "".into()), + ("GITHUB_ENTERPRISE_TOKEN".into(), "".into()), + ("COPILOT_HMAC_KEY".into(), "".into()), + ("CAPI_HMAC_KEY".into(), "".into()), ]); - if std::env::var("GITHUB_ACTIONS").as_deref() == Ok("true") { - env.push(("GH_TOKEN".into(), "fake-token-for-e2e-tests".into())); - env.push(("GITHUB_TOKEN".into(), "fake-token-for-e2e-tests".into())); - } env } @@ -528,6 +574,12 @@ fn default_test_timeout() -> Duration { } fn e2e_concurrency() -> usize { + // The in-process transport mirrors per-test environment onto the shared process + // environment (see `InProcessEnvGuard`), which is only coherent when one test runs + // at a time. Force serial execution in-process; otherwise honor RUST_E2E_CONCURRENCY. + if is_inprocess_default() { + return 1; + } std::env::var("RUST_E2E_CONCURRENCY") .ok() .and_then(|value| value.parse::().ok()) @@ -535,6 +587,90 @@ fn e2e_concurrency() -> usize { .unwrap_or(4) } +/// True when the E2E suite runs over the in-process (FFI) transport, i.e. the SDK +/// resolves `COPILOT_SDK_DEFAULT_CONNECTION=inprocess` to [`Transport::InProcess`]. +pub fn is_inprocess_default() -> bool { + std::env::var("COPILOT_SDK_DEFAULT_CONNECTION") + .map(|value| value.eq_ignore_ascii_case("inprocess")) + .unwrap_or(false) +} + +/// Skip guard for E2E tests exercising features the in-process (FFI) transport does not +/// support (the runtime loads into the shared host process). Returns `true` — and logs — +/// when running in-process so the caller can `return` early; such tests remain covered +/// by the default (stdio) transport. See . +pub fn skip_inprocess(reason: &str) -> bool { + if is_inprocess_default() { + eprintln!("skipping test over the in-process (FFI) transport: {reason}"); + true + } else { + false + } +} + +/// Mirrors an [`E2eContext`]'s environment onto the real process environment for the +/// in-process transport, whose worker inherits this process's ambient environment +/// rather than a per-client env block. Restores the previous values on drop. Only the +/// in-process transport needs this; for stdio/tcp the environment is handed to the +/// spawned child directly. Auth flows via GH_TOKEN/GITHUB_TOKEN and HMAC is disabled so +/// host-side auth resolution picks the token the replay snapshots expect. +struct InProcessEnvGuard { + saved: Vec<(OsString, Option)>, + previous_cwd: PathBuf, +} + +impl InProcessEnvGuard { + /// Returns `Some` guard (having applied the env) when in-process, else `None`. + fn activate(ctx: &E2eContext) -> Option { + if !is_inprocess_default() { + return None; + } + let mut pairs: Vec<(OsString, OsString)> = ctx.environment(); + pairs.push(("COPILOT_SDK_AUTH_TOKEN".into(), "".into())); + // Some tests opt into gated runtime APIs via per-client `options.env`, which the + // in-process transport does not pass to the shared worker (see issue #1934). + // These are process-global runtime gates (not per-client behavior), so applying + // them to the host process for the serial in-process suite is equivalent and + // inert for tests that don't exercise the gated API. + pairs.push(("COPILOT_ALLOW_GET_PROVIDER_ENDPOINT".into(), "true".into())); + pairs.push(( + "COPILOT_EXP_COPILOT_CLI_WEBSOCKET_RESPONSES".into(), + "true".into(), + )); + pairs.push(( + "COPILOT_EXP_COPILOT_CLI_SESSION_BASED_SUBAGENTS".into(), + "true".into(), + )); + + let mut saved: Vec<(OsString, Option)> = Vec::new(); + for (key, value) in &pairs { + saved.push((key.clone(), std::env::var_os(key))); + // SAFETY: the E2E suite runs serially in-process (concurrency 1), so no + // other thread races these process-wide env mutations. + unsafe { std::env::set_var(key, value) }; + } + let previous_cwd = std::env::current_dir().expect("read in-process test cwd"); + std::env::set_current_dir(ctx.work_dir()).expect("set in-process test cwd"); + Some(Self { + saved, + previous_cwd, + }) + } +} + +impl Drop for InProcessEnvGuard { + fn drop(&mut self) { + std::env::set_current_dir(&self.previous_cwd).expect("restore in-process test cwd"); + for (key, previous) in self.saved.iter().rev() { + // SAFETY: as in `activate` — serial execution in-process. + match previous { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + } +} + pub fn get_system_message(exchange: &serde_json::Value) -> String { exchange .get("request") @@ -617,11 +753,24 @@ fn cli_path(repo_root: &Path) -> std::io::Result { )) } +#[allow(deprecated)] fn client_options_for_cli( cli_path: &Path, cwd: &Path, env: Vec<(OsString, OsString)>, ) -> ClientOptions { + // When the in-process FFI transport is the default (matrix cell that sets + // COPILOT_SDK_DEFAULT_CONNECTION=inprocess), pass the CLI entrypoint + // directly: the FFI host builds the `node --embedded-host` + // argv itself and loads the sibling runtime cdylib. Splitting a `.js` + // entrypoint into node + prefix_args (the stdio layout) would point the + // library resolver at node's directory instead. + let inprocess_default = std::env::var("COPILOT_SDK_DEFAULT_CONNECTION") + .map(|value| value.eq_ignore_ascii_case("inprocess")) + .unwrap_or(false); + if inprocess_default { + return ClientOptions::new().with_program(CliProgram::Path(cli_path.to_path_buf())); + } let options = ClientOptions::new() .with_cwd(cwd) .with_env(env) diff --git a/rust/tests/e2e/telemetry.rs b/rust/tests/e2e/telemetry.rs index 38bf4a404a..f6905427ae 100644 --- a/rust/tests/e2e/telemetry.rs +++ b/rust/tests/e2e/telemetry.rs @@ -13,6 +13,11 @@ use super::support::{assistant_message_content, with_e2e_context}; #[tokio::test] async fn should_export_file_telemetry_for_sdk_interactions() { + // Telemetry lowers to environment variables the in-process worker cannot receive + // per-client; covered by the default (stdio) transport. See issue #1934. + if super::support::skip_inprocess("telemetry configuration is not honored in-process") { + return; + } with_e2e_context( "telemetry", "should_export_file_telemetry_for_sdk_interactions",