/** * Extracts code blocks from markdown documentation files. * Outputs individual files for validation by language-specific tools. */ import * as fs from "fs"; import * as path from "path"; import { glob } from "glob"; const DOCS_DIR = path.resolve(import.meta.dirname, "../../docs"); const OUTPUT_DIR = path.resolve(import.meta.dirname, "../../docs/.validation"); // Map markdown language tags to our canonical names const LANGUAGE_MAP: Record = { typescript: "typescript", ts: "typescript", javascript: "typescript", // Treat JS as TS for validation js: "typescript", python: "python", py: "python", go: "go", golang: "go", csharp: "csharp", "c#": "csharp", cs: "csharp", java: "java", }; interface CodeBlock { language: string; code: string; file: string; line: number; skip: boolean; hidden: boolean; wrapAsync: boolean; } interface ExtractionManifest { extractedAt: string; blocks: { id: string; sourceFile: string; sourceLine: number; language: string; outputFile: string; }[]; } function parseMarkdownCodeBlocks( content: string, filePath: string ): CodeBlock[] { const blocks: CodeBlock[] = []; const lines = content.split("\n"); let inCodeBlock = false; let currentLang = ""; let currentCode: string[] = []; let blockStartLine = 0; let skipNext = false; let wrapAsync = false; let inHiddenBlock = false; for (let i = 0; i < lines.length; i++) { const line = lines[i]; // Check for validation directives if (line.includes("")) { skipNext = true; continue; } if (line.includes("")) { wrapAsync = true; continue; } if (line.includes("")) { inHiddenBlock = true; continue; } if (line.includes("")) { inHiddenBlock = false; // Skip the next visible code block since the hidden one replaces it skipNext = true; continue; } // Start of code block if (!inCodeBlock && line.startsWith("```")) { const lang = line.slice(3).trim().toLowerCase(); if (lang && LANGUAGE_MAP[lang]) { inCodeBlock = true; currentLang = LANGUAGE_MAP[lang]; currentCode = []; blockStartLine = i + 1; // 1-indexed line number } continue; } // End of code block if (inCodeBlock && line.startsWith("```")) { blocks.push({ language: currentLang, code: currentCode.join("\n"), file: filePath, line: blockStartLine, skip: skipNext, hidden: inHiddenBlock, wrapAsync: wrapAsync, }); inCodeBlock = false; currentLang = ""; currentCode = []; // Only reset skipNext when NOT in a hidden block — hidden blocks // can contain multiple code fences that all get validated. if (!inHiddenBlock) { skipNext = false; } wrapAsync = false; continue; } // Inside code block if (inCodeBlock) { currentCode.push(line); } } return blocks; } function generateFileName( block: CodeBlock, index: number, langCounts: Map ): string { const count = langCounts.get(block.language) || 0; langCounts.set(block.language, count + 1); const sourceBasename = path.basename(block.file, ".md"); const ext = getExtension(block.language); return `${sourceBasename}_${count}${ext}`; } function getExtension(language: string): string { switch (language) { case "typescript": return ".ts"; case "python": return ".py"; case "go": return ".go"; case "csharp": return ".cs"; case "java": return ".java"; default: return ".txt"; } } /** * Detect code fragments that can't be validated as standalone files. * These are typically partial snippets showing configuration options * or code that's meant to be part of a larger context. */ function shouldSkipFragment(block: CodeBlock): boolean { const code = block.code.trim(); // TypeScript/JavaScript: Skip bare object literals (config snippets) if (block.language === "typescript") { // Starts with property: value pattern (e.g., "provider: {") if (/^[a-zA-Z_]+\s*:\s*[\{\[]/.test(code)) { return true; } // Starts with just an object/array that's not assigned if (/^\{[\s\S]*\}$/.test(code) && !code.includes("import ") && !code.includes("export ")) { return true; } } // Go: Skip fragments that are just type definitions without package if (block.language === "go") { // Function signatures without bodies (interface definitions shown in docs) if (/^func\s+\w+\([^)]*\)\s*\([^)]*\)\s*$/.test(code)) { return true; } } // Java: Skip interface definitions, annotations-only, or method signatures without bodies if (block.language === "java") { // Just an annotation if (/^@\w+/.test(code) && !code.includes("{")) { return true; } // Method signature without body if (/^(public|private|protected)?\s*(static\s+)?[\w<>\[\]]+\s+\w+\([^)]*\)\s*(throws\s+[\w,\s]+)?;\s*$/.test(code)) { return true; } } return false; } function wrapCodeForValidation(block: CodeBlock): string { let code = block.code; // Python: auto-detect async code and wrap if needed if (block.language === "python") { const hasAwait = /\bawait\b/.test(code); const hasAsyncDef = /\basync\s+def\b/.test(code); // Check if await is used outside of any async def // Simple heuristic: if await appears at column 0 or after assignment at column 0 const lines = code.split("\n"); let awaitOutsideFunction = false; let inAsyncFunction = false; let indentLevel = 0; for (const line of lines) { const trimmed = line.trimStart(); const leadingSpaces = line.length - trimmed.length; // Track if we're in an async function if (trimmed.startsWith("async def ")) { inAsyncFunction = true; indentLevel = leadingSpaces; } else if (inAsyncFunction && leadingSpaces <= indentLevel && trimmed && !trimmed.startsWith("#")) { // Dedented back, we're out of the function inAsyncFunction = false; } // Check for await outside function if (trimmed.includes("await ") && !inAsyncFunction) { awaitOutsideFunction = true; break; } } const needsWrap = block.wrapAsync || awaitOutsideFunction || (hasAwait && !hasAsyncDef); if (needsWrap) { const indented = code .split("\n") .map((l) => " " + l) .join("\n"); code = `import asyncio\n\nasync def main():\n${indented}\n\nasyncio.run(main())`; } } // Go: ensure package declaration if (block.language === "go" && !code.includes("package ")) { code = `package main\n\n${code}`; } // Go: add main function if missing and has statements outside functions if (block.language === "go" && !code.includes("func main()")) { // Check if code has statements that need to be in main const hasStatements = /^[a-z]/.test(code.trim().split("\n").pop() || ""); if (hasStatements) { // This is a snippet, wrap it const lines = code.split("\n"); const packageLine = lines.find((l) => l.startsWith("package ")) || ""; const imports = lines.filter( (l) => l.startsWith("import ") || l.startsWith('import (') ); const rest = lines.filter( (l) => !l.startsWith("package ") && !l.startsWith("import ") && !l.startsWith("import (") && !l.startsWith(")") && !l.startsWith("\t") // import block lines ); // Only wrap if there are loose statements (not type/func definitions) const hasLooseStatements = rest.some( (l) => l.trim() && !l.startsWith("type ") && !l.startsWith("func ") && !l.startsWith("//") && !l.startsWith("var ") && !l.startsWith("const ") ); if (!hasLooseStatements) { // Code has proper structure, just ensure it has a main code = code + "\n\nfunc main() {}"; } } } // C#: wrap in a class to avoid top-level statements conflicts // (C# only allows one file with top-level statements per project) if (block.language === "csharp") { // Check if it's a complete file (has namespace or class) const hasStructure = code.includes("namespace ") || code.includes("class ") || code.includes("record ") || code.includes("public delegate "); if (!hasStructure) { // Extract any existing using statements const lines = code.split("\n"); const usings: string[] = []; const rest: string[] = []; for (const line of lines) { if (line.trim().startsWith("using ") && line.trim().endsWith(";")) { usings.push(line); } else { rest.push(line); } } // Always ensure SDK usings are present. If the snippet already // declares any GitHub.Copilot using, assume the author curated // them and don't add others (avoids name ambiguities like // ModelCapabilities living in both namespaces). const hasAnyCopilotUsing = usings.some(u => u.includes("GitHub.Copilot;") || u.includes("GitHub.Copilot."), ); if (!hasAnyCopilotUsing) { usings.push("using GitHub.Copilot;"); usings.push("using GitHub.Copilot.Rpc;"); } // Generate a unique class name based on block location const className = `ValidationClass_${block.file.replace(/[^a-zA-Z0-9]/g, "_")}_${block.line}`; // Wrap in async method to support await const hasAwait = code.includes("await "); const indentedCode = rest.map(l => " " + l).join("\n"); if (hasAwait) { code = `${usings.join("\n")} public static class ${className} { public static async Task Main() { ${indentedCode} } }`; } else { code = `${usings.join("\n")} public static class ${className} { public static void Main() { ${indentedCode} } }`; } } else { // Has structure. Only add SDK usings if neither namespace is present; // if the snippet declares its own using GitHub.Copilot statement, // assume the author curated imports (avoids ambiguities like // ModelCapabilities living in both namespaces). if (!code.includes("using GitHub.Copilot")) { code = "using GitHub.Copilot;\nusing GitHub.Copilot.Rpc;\n" + code; } } } // Java: wrap in a class for compilation if (block.language === "java") { const hasClass = code.includes("class ") || code.includes("interface ") || code.includes("enum "); if (!hasClass) { // Extract any existing import statements const lines = code.split("\n"); const imports: string[] = []; const rest: string[] = []; for (const line of lines) { if (line.trim().startsWith("import ")) { imports.push(line); } else { rest.push(line); } } // Add default imports if no SDK imports are present const hasAnyCopilotImport = imports.some(i => i.includes("com.github.copilot"), ); if (!hasAnyCopilotImport) { imports.push("import com.github.copilot.*;"); imports.push("import com.github.copilot.rpc.*;"); imports.push("import java.util.*;"); imports.push("import java.util.concurrent.*;"); } // Generate a unique class name from block.file and block.line let className = `${block.file.replace(/[^a-zA-Z0-9]/g, "_")}_${block.line}`; if (/^\d/.test(className)) { className = "Snippet_" + className; } const indentedCode = rest.map(l => " " + l).join("\n"); code = `${imports.join("\n")} public class ${className} { public static void main(String[] args) throws Exception { ${indentedCode} } }`; } else { // Has class structure. Only add SDK imports if not already present. if (!code.includes("import com.github.copilot")) { code = "import com.github.copilot.*;\nimport com.github.copilot.rpc.*;\nimport java.util.*;\nimport java.util.concurrent.*;\n" + code; } } } return code; } async function main() { console.log("📖 Extracting code blocks from documentation...\n"); // Clean output directory if (fs.existsSync(OUTPUT_DIR)) { fs.rmSync(OUTPUT_DIR, { recursive: true }); } fs.mkdirSync(OUTPUT_DIR, { recursive: true }); // Create language subdirectories for (const lang of ["typescript", "python", "go", "csharp", "java"]) { fs.mkdirSync(path.join(OUTPUT_DIR, lang), { recursive: true }); } // Find all markdown files const mdFiles = await glob("**/*.md", { cwd: DOCS_DIR, ignore: [".validation/**", "node_modules/**", "IMPROVEMENT_PLAN.md"], }); console.log(`Found ${mdFiles.length} markdown files\n`); const manifest: ExtractionManifest = { extractedAt: new Date().toISOString(), blocks: [], }; const langCounts = new Map(); let totalBlocks = 0; let skippedBlocks = 0; let hiddenBlocks = 0; for (const mdFile of mdFiles) { const fullPath = path.join(DOCS_DIR, mdFile); const content = fs.readFileSync(fullPath, "utf-8"); const blocks = parseMarkdownCodeBlocks(content, mdFile); for (const block of blocks) { if (block.skip) { skippedBlocks++; continue; } if (block.hidden) { hiddenBlocks++; } // Skip empty or trivial blocks if (block.code.trim().length < 10) { continue; } // Skip incomplete code fragments that can't be validated standalone if (shouldSkipFragment(block)) { skippedBlocks++; continue; } const fileName = generateFileName(block, totalBlocks, langCounts); const wrappedCode = wrapCodeForValidation(block); // For Java, filename must match the public class name let actualFileName = fileName; if (block.language === "java") { const classMatch = wrappedCode.match(/public class (\w+)/); if (classMatch) { actualFileName = classMatch[1] + ".java"; } } const outputPath = path.join(OUTPUT_DIR, block.language, actualFileName); // Add source location comment const sourceComment = getSourceComment( block.language, block.file, block.line ); const finalCode = sourceComment + "\n" + wrappedCode; fs.writeFileSync(outputPath, finalCode); manifest.blocks.push({ id: `${block.language}/${actualFileName}`, sourceFile: block.file, sourceLine: block.line, language: block.language, outputFile: `${block.language}/${actualFileName}`, }); totalBlocks++; } } // Write manifest fs.writeFileSync( path.join(OUTPUT_DIR, "manifest.json"), JSON.stringify(manifest, null, 2) ); // Summary console.log("Extraction complete!\n"); console.log(" Language Count"); console.log(" ─────────────────────"); for (const [lang, count] of langCounts) { console.log(` ${lang.padEnd(14)} ${count}`); } console.log(" ─────────────────────"); console.log(` Total ${totalBlocks}`); if (skippedBlocks > 0) { console.log(` Skipped ${skippedBlocks}`); } if (hiddenBlocks > 0) { console.log(` Hidden ${hiddenBlocks}`); } console.log(`\nOutput: ${OUTPUT_DIR}`); } function getSourceComment( language: string, file: string, line: number ): string { // Normalize path separators to forward slashes to avoid issues // (e.g., Java interprets \u as a unicode escape sequence) const normalizedFile = file.replace(/\\/g, "/"); const location = `Source: ${normalizedFile}:${line}`; switch (language) { case "typescript": case "go": case "csharp": return `// ${location}`; case "python": return `# ${location}`; default: return `// ${location}`; } } main().catch((err) => { console.error("Extraction failed:", err); process.exit(1); });