SkillsTest.java

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *--------------------------------------------------------------------------------------------*/

package com.github.copilot.sdk;

import static org.junit.jupiter.api.Assertions.*;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import com.github.copilot.sdk.generated.AssistantMessageEvent;
import com.github.copilot.sdk.json.CustomAgentConfig;
import com.github.copilot.sdk.json.MessageOptions;
import com.github.copilot.sdk.json.PermissionHandler;
import com.github.copilot.sdk.json.SessionConfig;

/**
 * Tests for skills configuration functionality.
 *
 * <p>
 * These tests verify that skills can be loaded from skill directories and
 * disabled via configuration. Snapshots are stored in test/snapshots/skills/.
 * </p>
 */
public class SkillsTest {

    private static E2ETestContext ctx;
    private static final String SKILL_MARKER = "PINEAPPLE_COCONUT_42";
    private Path skillsDir;

    @BeforeAll
    static void setup() throws Exception {
        ctx = E2ETestContext.create();
    }

    @AfterAll
    static void teardown() throws Exception {
        if (ctx != null) {
            ctx.close();
        }
    }

    @BeforeEach
    void setupEach() throws IOException {
        // Ensure we start fresh each time
        skillsDir = ctx.getWorkDir().resolve(".test_skills");
        if (Files.exists(skillsDir)) {
            deleteDirectory(skillsDir);
        }
    }

    private void deleteDirectory(Path dir) throws IOException {
        if (Files.exists(dir)) {
            Files.walk(dir).sorted((a, b) -> b.compareTo(a)) // reverse order for deletion
                    .forEach(path -> {
                        try {
                            Files.delete(path);
                        } catch (IOException e) {
                            // Ignore
                        }
                    });
        }
    }

    private Path createSkillDir() throws IOException {
        Files.createDirectories(skillsDir);

        // Create a skill subdirectory with SKILL.md
        Path skillSubdir = skillsDir.resolve("test-skill");
        Files.createDirectories(skillSubdir);

        // Create a skill that instructs the model to include a specific marker in
        // responses
        String skillContent = """
                ---
                name: test-skill
                description: A test skill that adds a marker to responses
                ---

                # Test Skill Instructions

                IMPORTANT: You MUST include the exact text "%s" somewhere in EVERY response you give. This is a mandatory requirement. Include it naturally in your response.
                """
                .formatted(SKILL_MARKER);

        Files.writeString(skillSubdir.resolve("SKILL.md"), skillContent);
        return skillsDir;
    }

    /**
     * Verifies that skills are loaded and applied from skill directories.
     *
     * @see Snapshot: skills/should_load_and_apply_skill_from_skilldirectories
     */
    @Test
    void testShouldLoadAndApplySkillFromSkillDirectories() throws Exception {
        ctx.configureForTest("skills", "should_load_and_apply_skill_from_skilldirectories");

        Path skillsDir = createSkillDir();

        SessionConfig config = new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL)
                .setSkillDirectories(List.of(skillsDir.toString()));

        try (CopilotClient client = ctx.createClient()) {
            CopilotSession session = client.createSession(config).get();

            assertNotNull(session.getSessionId());

            // The skill instructs the model to include a marker - verify it appears
            AssistantMessageEvent response = session
                    .sendAndWait(new MessageOptions().setPrompt("Say hello briefly using the test skill."))
                    .get(60, TimeUnit.SECONDS);

            assertNotNull(response);
            assertTrue(response.getData().content().contains(SKILL_MARKER),
                    "Response should contain skill marker '" + SKILL_MARKER + "': " + response.getData().content());

            session.close();
        }
    }

    /**
     * Verifies that skills are not applied when disabled via disabledSkills.
     *
     * @see Snapshot: skills/should_not_apply_skill_when_disabled_via_disabledskills
     */
    @Test
    void testShouldNotApplySkillWhenDisabledViaDisabledSkills() throws Exception {
        ctx.configureForTest("skills", "should_not_apply_skill_when_disabled_via_disabledskills");

        Path skillsDir = createSkillDir();

        SessionConfig config = new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL)
                .setSkillDirectories(List.of(skillsDir.toString())).setDisabledSkills(List.of("test-skill"));

        try (CopilotClient client = ctx.createClient()) {
            CopilotSession session = client.createSession(config).get();

            assertNotNull(session.getSessionId());

            // The skill is disabled, so the marker should NOT appear
            AssistantMessageEvent response = session
                    .sendAndWait(new MessageOptions().setPrompt("Say hello briefly using the test skill."))
                    .get(60, TimeUnit.SECONDS);

            assertNotNull(response);
            assertFalse(response.getData().content().contains(SKILL_MARKER),
                    "Response should NOT contain skill marker when skill is disabled: " + response.getData().content());

            session.close();
        }
    }

    /**
     * Verifies that an agent with a Skills field can preload and invoke the skill.
     *
     * @see Snapshot: skills/should_allow_agent_with_skills_to_invoke_skill
     */
    @Test
    void testShouldAllowAgentWithSkillsToInvokeSkill() throws Exception {
        ctx.configureForTest("skills", "should_allow_agent_with_skills_to_invoke_skill");

        Path skillsDirPath = createSkillDir();

        var agent = new CustomAgentConfig().setName("skill-agent").setDescription("An agent with access to test-skill")
                .setPrompt("You are a helpful test agent.").setSkills(List.of("test-skill"));

        SessionConfig config = new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL)
                .setSkillDirectories(List.of(skillsDirPath.toString())).setCustomAgents(List.of(agent))
                .setAgent("skill-agent");

        try (CopilotClient client = ctx.createClient()) {
            CopilotSession session = client.createSession(config).get();

            assertNotNull(session.getSessionId());

            // The agent has Skills = ["test-skill"], so the skill content is preloaded
            AssistantMessageEvent response = session
                    .sendAndWait(new MessageOptions().setPrompt("Say hello briefly using the test skill."))
                    .get(60, TimeUnit.SECONDS);

            assertNotNull(response);
            assertTrue(response.getData().content().contains(SKILL_MARKER),
                    "Response should contain skill marker '" + SKILL_MARKER + "': " + response.getData().content());

            session.close();
        }
    }

    /**
     * Verifies that an agent without a Skills field does not get skill content
     * injected.
     *
     * @see Snapshot: skills/should_not_provide_skills_to_agent_without_skills_field
     */
    @Test
    void testShouldNotProvideSkillsToAgentWithoutSkillsField() throws Exception {
        ctx.configureForTest("skills", "should_not_provide_skills_to_agent_without_skills_field");

        Path skillsDirPath = createSkillDir();

        var agent = new CustomAgentConfig().setName("no-skill-agent").setDescription("An agent without skills access")
                .setPrompt("You are a helpful test agent.");

        SessionConfig config = new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL)
                .setSkillDirectories(List.of(skillsDirPath.toString())).setCustomAgents(List.of(agent))
                .setAgent("no-skill-agent");

        try (CopilotClient client = ctx.createClient()) {
            CopilotSession session = client.createSession(config).get();

            assertNotNull(session.getSessionId());

            // The agent has no Skills field, so no skill content is injected
            AssistantMessageEvent response = session
                    .sendAndWait(new MessageOptions().setPrompt("Say hello briefly using the test skill."))
                    .get(60, TimeUnit.SECONDS);

            assertNotNull(response);
            assertFalse(response.getData().content().contains(SKILL_MARKER),
                    "Response should NOT contain skill marker when agent has no Skills field: "
                            + response.getData().content());

            session.close();
        }
    }
}