/** * Validates extracted documentation code blocks. * Runs language-specific type/compile checks. */ import { execFileSync, execSync } from "child_process"; import * as fs from "fs"; import { glob } from "glob"; import * as path from "path"; const ROOT_DIR = path.resolve(import.meta.dirname, "../.."); const VALIDATION_DIR = path.join(ROOT_DIR, "docs/.validation"); interface ValidationResult { file: string; sourceFile: string; sourceLine: number; success: boolean; errors: string[]; } interface Manifest { blocks: { id: string; sourceFile: string; sourceLine: number; language: string; outputFile: string; }[]; } function loadManifest(): Manifest { const manifestPath = path.join(VALIDATION_DIR, "manifest.json"); if (!fs.existsSync(manifestPath)) { console.error( "❌ No manifest found. Run extraction first: npm run extract", ); process.exit(1); } return JSON.parse(fs.readFileSync(manifestPath, "utf-8")); } async function validateTypeScript(): Promise { const results: ValidationResult[] = []; const tsDir = path.join(VALIDATION_DIR, "typescript"); const manifest = loadManifest(); if (!fs.existsSync(tsDir)) { console.log(" No TypeScript files to validate"); return results; } // Create a temporary tsconfig for validation const tsconfig = { compilerOptions: { target: "ES2022", module: "NodeNext", moduleResolution: "NodeNext", strict: true, skipLibCheck: true, noEmit: true, esModuleInterop: true, allowSyntheticDefaultImports: true, resolveJsonModule: true, types: ["node"], paths: { "@github/copilot-sdk": [path.join(ROOT_DIR, "nodejs/src/index.ts")], }, }, include: ["./**/*.ts"], }; const tsconfigPath = path.join(tsDir, "tsconfig.json"); fs.writeFileSync(tsconfigPath, JSON.stringify(tsconfig, null, 2)); try { // Run tsc const tscPath = path.join(ROOT_DIR, "nodejs/node_modules/.bin/tsc"); execFileSync(tscPath, ["--project", tsconfigPath], { encoding: "utf-8", cwd: tsDir, }); // All files passed const files = await glob("*.ts", { cwd: tsDir }); for (const file of files) { if (file === "tsconfig.json") continue; const block = manifest.blocks.find( (b) => b.outputFile === `typescript/${file}`, ); results.push({ file: `typescript/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: true, errors: [], }); } } catch (err: any) { // Parse tsc output for errors const output = err.stdout || err.stderr || err.message || ""; const errorLines = output.split("\n"); const fileErrors = new Map(); let currentFile = ""; for (const line of errorLines) { const match = line.match(/^(.+\.ts)\((\d+),(\d+)\): error/); if (match) { currentFile = match[1]; if (!fileErrors.has(currentFile)) { fileErrors.set(currentFile, []); } fileErrors.get(currentFile)!.push(line); } else if (currentFile && line.trim()) { fileErrors.get(currentFile)?.push(line); } } // Create results const files = await glob("*.ts", { cwd: tsDir }); for (const file of files) { if (file === "tsconfig.json") continue; const fullPath = path.join(tsDir, file); const block = manifest.blocks.find( (b) => b.outputFile === `typescript/${file}`, ); const errors = fileErrors.get(fullPath) || fileErrors.get(file) || []; results.push({ file: `typescript/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: errors.length === 0, errors, }); } } return results; } async function validatePython(): Promise { const results: ValidationResult[] = []; const pyDir = path.join(VALIDATION_DIR, "python"); const manifest = loadManifest(); if (!fs.existsSync(pyDir)) { console.log(" No Python files to validate"); return results; } const files = await glob("*.py", { cwd: pyDir }); for (const file of files) { const fullPath = path.join(pyDir, file); const block = manifest.blocks.find( (b) => b.outputFile === `python/${file}`, ); const errors: string[] = []; // Syntax check with py_compile try { execFileSync("python3", ["-m", "py_compile", fullPath], { encoding: "utf-8", }); } catch (err: any) { errors.push(err.stdout || err.stderr || err.message || "Syntax error"); } // Type check with mypy (if available) if (errors.length === 0) { try { execFileSync( "python3", [ "-m", "mypy", fullPath, "--ignore-missing-imports", "--no-error-summary", ], { encoding: "utf-8" }, ); } catch (err: any) { const output = err.stdout || err.stderr || err.message || ""; // Filter out "Success" messages and notes const typeErrors = output .split("\n") .filter( (l: string) => l.includes(": error:") && !l.includes("Cannot find implementation"), ); if (typeErrors.length > 0) { errors.push(...typeErrors); } } } results.push({ file: `python/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: errors.length === 0, errors, }); } return results; } async function validateGo(): Promise { const results: ValidationResult[] = []; const goDir = path.join(VALIDATION_DIR, "go"); const manifest = loadManifest(); if (!fs.existsSync(goDir)) { console.log(" No Go files to validate"); return results; } // Create a go.mod for the validation directory const goMod = `module docs-validation go 1.21 require github.com/github/copilot-sdk/go v0.0.0 replace github.com/github/copilot-sdk/go => ${path.join(ROOT_DIR, "go")} `; fs.writeFileSync(path.join(goDir, "go.mod"), goMod); // Run go mod tidy to fetch dependencies try { execFileSync("go", ["mod", "tidy"], { encoding: "utf-8", cwd: goDir, env: { ...process.env, GO111MODULE: "on" }, }); } catch (err: any) { // go mod tidy might fail if there are syntax errors, continue anyway } const files = await glob("*.go", { cwd: goDir }); // Try to compile each file individually for (const file of files) { const fullPath = path.join(goDir, file); const block = manifest.blocks.find((b) => b.outputFile === `go/${file}`); const errors: string[] = []; try { // Use go vet for syntax and basic checks execFileSync("go", ["build", "-o", "/dev/null", fullPath], { encoding: "utf-8", cwd: goDir, env: { ...process.env, GO111MODULE: "on" }, }); } catch (err: any) { const output = err.stdout || err.stderr || err.message || ""; errors.push( ...output .split("\n") .filter((l: string) => l.trim() && !l.startsWith("#")), ); } results.push({ file: `go/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: errors.length === 0, errors, }); } return results; } async function validateCSharp(): Promise { const results: ValidationResult[] = []; const csDir = path.join(VALIDATION_DIR, "csharp"); const manifest = loadManifest(); if (!fs.existsSync(csDir)) { console.log(" No C# files to validate"); return results; } // Create a minimal csproj for validation const csproj = ` Library net8.0 enable enable CS8019;CS0168;CS0219;GHCP001 `; fs.writeFileSync(path.join(csDir, "DocsValidation.csproj"), csproj); const files = await glob("*.cs", { cwd: csDir }); // Compile all files together try { execFileSync( "dotnet", ["build", path.join(csDir, "DocsValidation.csproj")], { encoding: "utf-8", cwd: csDir, }, ); // All files passed for (const file of files) { const block = manifest.blocks.find( (b) => b.outputFile === `csharp/${file}`, ); results.push({ file: `csharp/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: true, errors: [], }); } } catch (err: any) { const output = err.stdout || err.stderr || err.message || ""; // Parse errors by file const fileErrors = new Map(); for (const line of output.split("\n")) { const match = line.match(/([^/\\]+\.cs)\((\d+),(\d+)\): error/); if (match) { const fileName = match[1]; if (!fileErrors.has(fileName)) { fileErrors.set(fileName, []); } fileErrors.get(fileName)!.push(line); } } for (const file of files) { const block = manifest.blocks.find( (b) => b.outputFile === `csharp/${file}`, ); const errors = fileErrors.get(file) || []; results.push({ file: `csharp/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: errors.length === 0, errors, }); } } return results; } async function validateJava(): Promise { const results: ValidationResult[] = []; const javaDir = path.join(VALIDATION_DIR, "java"); const manifest = loadManifest(); if (!fs.existsSync(javaDir)) { console.log(" No Java files to validate"); return results; } // Create a minimal Maven project structure const srcDir = path.join(javaDir, "src", "main", "java"); fs.mkdirSync(srcDir, { recursive: true }); // Copy all .java files into src/main/java/ (copy, not move, for idempotency) const files = await glob("*.java", { cwd: javaDir }); for (const file of files) { fs.copyFileSync(path.join(javaDir, file), path.join(srcDir, file)); } // Read the SDK version from java/pom.xml const sdkPomPath = path.join(ROOT_DIR, "java", "pom.xml"); const sdkPomContent = fs.readFileSync(sdkPomPath, "utf-8"); const versionMatch = sdkPomContent.match( /copilot-sdk-java<\/artifactId>\s*([^<]+)<\/version>/, ); const sdkVersion = versionMatch ? versionMatch[1] : "1.0.0-SNAPSHOT"; // Create pom.xml that references the local SDK const pomXml = ` 4.0.0 docs docs-validation-java 1.0.0 17 17 UTF-8 com.github copilot-sdk-java ${sdkVersion} `; fs.writeFileSync(path.join(javaDir, "pom.xml"), pomXml); // First, install the local SDK into the local Maven repo const pomPath = path.join(ROOT_DIR, "java", "pom.xml"); try { execSync(`mvn install -f "${pomPath}" -DskipTests -q`, { encoding: "utf-8", cwd: path.join(ROOT_DIR, "java"), }); } catch (err: any) { // If SDK install fails, all Java snippets fail const errorMsg = `SDK install failed: ${(err.stderr || err.message || "").slice(0, 200)}`; for (const file of files) { const block = manifest.blocks.find( (b) => b.outputFile === `java/${file}`, ); results.push({ file: `java/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: false, errors: [errorMsg], }); } return results; } // Compile the validation project try { const validationPom = path.join(javaDir, "pom.xml"); execSync(`mvn compile -f "${validationPom}" -q`, { encoding: "utf-8", cwd: javaDir, }); // All files passed for (const file of files) { const block = manifest.blocks.find( (b) => b.outputFile === `java/${file}`, ); results.push({ file: `java/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: true, errors: [], }); } } catch (err: any) { const output = err.stdout || err.stderr || err.message || ""; // Parse javac errors from Maven output // Format: [ERROR] /path/to/File.java:[line,col] error: message const fileErrors = new Map(); for (const line of output.split("\n")) { const match = line.match( /\[ERROR\]\s+.*[/\\]([^/\\]+\.java):\[(\d+),(\d+)\]\s*(.*)/, ); if (match) { const fileName = match[1]; if (!fileErrors.has(fileName)) { fileErrors.set(fileName, []); } fileErrors.get(fileName)!.push(`${fileName}:${match[2]}: ${match[4]}`); } } for (const file of files) { const block = manifest.blocks.find( (b) => b.outputFile === `java/${file}`, ); const errors = fileErrors.get(file) || []; results.push({ file: `java/${file}`, sourceFile: block?.sourceFile || "unknown", sourceLine: block?.sourceLine || 0, success: errors.length === 0, errors, }); } } return results; } function printResults( results: ValidationResult[], language: string, ): { failed: number; passed: number; failures: ValidationResult[] } { const failed = results.filter((r) => !r.success); const passed = results.filter((r) => r.success); if (failed.length === 0) { console.log(` ✅ ${passed.length} files passed`); return { failed: 0, passed: passed.length, failures: [] }; } console.log(` ❌ ${failed.length} failed, ${passed.length} passed\n`); for (const result of failed) { console.log(` ┌─ ${result.sourceFile}:${result.sourceLine}`); console.log(` │ Extracted to: ${result.file}`); for (const error of result.errors.slice(0, 5)) { console.log(` │ ${error}`); } if (result.errors.length > 5) { console.log(` │ ... and ${result.errors.length - 5} more errors`); } console.log(` └─`); } return { failed: failed.length, passed: passed.length, failures: failed }; } function writeGitHubSummary( summaryData: { language: string; passed: number; failed: number; failures: ValidationResult[]; }[], ) { const summaryFile = process.env.GITHUB_STEP_SUMMARY; if (!summaryFile) return; const totalPassed = summaryData.reduce((sum, d) => sum + d.passed, 0); const totalFailed = summaryData.reduce((sum, d) => sum + d.failed, 0); const allPassed = totalFailed === 0; let summary = `## 📖 Documentation Validation Results\n\n`; if (allPassed) { summary += `✅ **All ${totalPassed} code blocks passed validation**\n\n`; } else { summary += `❌ **${totalFailed} failures** out of ${totalPassed + totalFailed} code blocks\n\n`; } summary += `| Language | Status | Passed | Failed |\n`; summary += `|----------|--------|--------|--------|\n`; for (const { language, passed, failed } of summaryData) { const status = failed === 0 ? "✅" : "❌"; summary += `| ${language} | ${status} | ${passed} | ${failed} |\n`; } if (totalFailed > 0) { summary += `\n### Failures\n\n`; for (const { language, failures } of summaryData) { if (failures.length === 0) continue; summary += `#### ${language}\n\n`; for (const f of failures) { summary += `- **${f.sourceFile}:${f.sourceLine}**\n`; summary += ` \`\`\`\n ${f.errors.slice(0, 3).join("\n ")}\n \`\`\`\n`; } } } fs.appendFileSync(summaryFile, summary); } async function main() { const args = process.argv.slice(2); const langArg = args.find((a) => a.startsWith("--lang=")); const targetLang = langArg?.split("=")[1]; console.log("🔍 Validating documentation code blocks...\n"); if (!fs.existsSync(VALIDATION_DIR)) { console.error("❌ No extracted code found. Run extraction first:"); console.error(" npm run extract"); process.exit(1); } let totalFailed = 0; const summaryData: { language: string; passed: number; failed: number; failures: ValidationResult[]; }[] = []; const validators: [string, () => Promise][] = [ ["TypeScript", validateTypeScript], ["Python", validatePython], ["Go", validateGo], ["C#", validateCSharp], ["Java", validateJava], ]; for (const [name, validator] of validators) { const langKey = name.toLowerCase().replace("#", "sharp"); if (targetLang && langKey !== targetLang) continue; console.log(`\n${name}:`); const results = await validator(); const { failed, passed, failures } = printResults(results, name); totalFailed += failed; summaryData.push({ language: name, passed, failed, failures }); } // Write GitHub Actions summary writeGitHubSummary(summaryData); console.log("\n" + "─".repeat(40)); if (totalFailed > 0) { console.log(`\n❌ Validation failed: ${totalFailed} file(s) have errors`); console.log("\nTo fix:"); console.log(" 1. Check the error messages above"); console.log(" 2. Update the code blocks in the markdown files"); console.log(" 3. Re-run: npm run validate"); console.log("\nTo skip a code block, add before it:"); console.log(" "); console.log("\nTo validate a complete version while showing a snippet:"); console.log(" "); console.log(" ```lang"); console.log(" // full compilable code"); console.log(" ```"); console.log(" "); console.log(" ```lang"); console.log(" // visible snippet (auto-skipped)"); console.log(" ```"); process.exit(1); } console.log("\n✅ All documentation code blocks are valid!"); } main().catch((err) => { console.error("Validation failed:", err); process.exit(1); });