skills.e2e.test.ts

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *--------------------------------------------------------------------------------------------*/

import * as fs from "fs";
import * as path from "path";
import { beforeEach, describe, expect, it } from "vitest";
import type { CustomAgentConfig } from "../../src/index.js";
import { approveAll } from "../../src/index.js";
import { createSdkTestContext } from "./harness/sdkTestContext.js";

describe("Skills Configuration", async () => {
    const { copilotClient: client, workDir } = await createSdkTestContext({ logLevel: "debug" });
    const SKILL_MARKER = "PINEAPPLE_COCONUT_42";
    const skillsDir = path.join(workDir, ".test_skills");

    beforeEach(() => {
        // Ensure we start fresh each time
        if (fs.existsSync(skillsDir)) {
            fs.rmSync(skillsDir, { recursive: true, force: true });
        }
    });

    function createSkillDir(): string {
        fs.mkdirSync(skillsDir, { recursive: true });

        // Create a skill subdirectory with SKILL.md
        const skillSubdir = path.join(skillsDir, "test-skill");
        fs.mkdirSync(skillSubdir, { recursive: true });

        // Create a skill that instructs the model to include a specific marker in responses
        const skillContent = `---
name: test-skill
description: A test skill that adds a marker to responses
---

# Test Skill Instructions

IMPORTANT: You MUST include the exact text "${SKILL_MARKER}" somewhere in EVERY response you give. This is a mandatory requirement. Include it naturally in your response.
`;
        fs.writeFileSync(path.join(skillSubdir, "SKILL.md"), skillContent);
        return skillsDir;
    }

    describe("Skill Behavior", () => {
        it("should load and apply skill from skillDirectories", async () => {
            const skillsDir = createSkillDir();
            const session = await client.createSession({
                onPermissionRequest: approveAll,
                skillDirectories: [skillsDir],
            });

            expect(session.sessionId).toBeDefined();

            // The skill instructs the model to include a marker - verify it appears
            const message = await session.sendAndWait({
                prompt: "Say hello briefly using the test skill.",
            });

            expect(message?.data.content).toContain(SKILL_MARKER);

            await session.disconnect();
        });

        it("should not apply skill when disabled via disabledSkills", async () => {
            const skillsDir = createSkillDir();
            const session = await client.createSession({
                onPermissionRequest: approveAll,
                skillDirectories: [skillsDir],
                disabledSkills: ["test-skill"],
            });

            expect(session.sessionId).toBeDefined();

            // The skill is disabled, so the marker should NOT appear
            const message = await session.sendAndWait({
                prompt: "Say hello briefly using the test skill.",
            });

            expect(message?.data.content).not.toContain(SKILL_MARKER);

            await session.disconnect();
        });

        // Skipped because the underlying feature doesn't work correctly yet.
        // - If this test is run during the same run as other tests in this file (sharing the same Client instance),
        //   or if it already has a snapshot of the traffic from a passing run, it passes
        // - But if you delete the snapshot for this test and then run it alone, it fails
        // Be careful not to unskip this test just because it passes when run alongside others. It needs to pass when
        // run alone and without any prior snapshot.
        // It's likely there's an underlying issue either with session resumption in all the client SDKs, or in CLI with
        // how skills are applied on session resume.
        // Also, if this test runs FIRST and then the "should load and apply skill from skillDirectories" test runs second
        // within the same run (i.e., sharing the same Client instance), then the second test fails too. There's definitely
        // some state being shared or cached incorrectly.
        it("should allow agent with skills to invoke skill", async () => {
            const skillsDir = createSkillDir();
            const customAgents: CustomAgentConfig[] = [
                {
                    name: "skill-agent",
                    description: "An agent with access to test-skill",
                    prompt: "You are a helpful test agent.",
                    skills: ["test-skill"],
                },
            ];

            const session = await client.createSession({
                onPermissionRequest: approveAll,
                skillDirectories: [skillsDir],
                customAgents,
                agent: "skill-agent",
            });

            expect(session.sessionId).toBeDefined();

            // The agent has skills: ["test-skill"], so the skill content is preloaded into its context
            const message = await session.sendAndWait({
                prompt: "Say hello briefly using the test skill.",
            });

            expect(message?.data.content).toContain(SKILL_MARKER);

            await session.disconnect();
        });

        it("should not provide skills to agent without skills field", async () => {
            const skillsDir = createSkillDir();
            const customAgents: CustomAgentConfig[] = [
                {
                    name: "no-skill-agent",
                    description: "An agent without skills access",
                    prompt: "You are a helpful test agent.",
                },
            ];

            const session = await client.createSession({
                onPermissionRequest: approveAll,
                skillDirectories: [skillsDir],
                customAgents,
                agent: "no-skill-agent",
            });

            expect(session.sessionId).toBeDefined();

            // The agent has no skills field, so no skill content is injected
            const message = await session.sendAndWait({
                prompt: "Say hello briefly using the test skill.",
            });

            expect(message?.data.content).not.toContain(SKILL_MARKER);

            await session.disconnect();
        });

        it.skip("should apply skill on session resume with skillDirectories", async () => {
            const skillsDir = createSkillDir();

            // Create a session without skills first
            const session1 = await client.createSession({ onPermissionRequest: approveAll });
            const sessionId = session1.sessionId;

            // First message without skill - marker should not appear
            const message1 = await session1.sendAndWait({ prompt: "Say hi." });
            expect(message1?.data.content).not.toContain(SKILL_MARKER);

            // Resume with skillDirectories - skill should now be active
            const session2 = await client.resumeSession(sessionId, {
                onPermissionRequest: approveAll,
                skillDirectories: [skillsDir],
            });

            expect(session2.sessionId).toBe(sessionId);

            // Now the skill should be applied
            const message2 = await session2.sendAndWait({
                prompt: "Say hello again using the test skill.",
            });

            expect(message2?.data.content).toContain(SKILL_MARKER);

            await session2.disconnect();
        });
    });
});