/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ /** * Python code generator for session-events and RPC types. */ import fs from "fs/promises"; import type { JSONSchema7 } from "json-schema"; import { FetchingJSONSchemaStore, InputData, JSONSchemaInput, quicktype } from "quicktype-core"; import { cloneSchemaForCodegen, getApiSchemaPath, getRpcSchemaTypeName, getSessionEventsSchemaPath, hoistTitledSchemas, isObjectSchema, isVoidSchema, isRpcMethod, postProcessSchema, writeGeneratedFile, isNodeFullyExperimental, type ApiSchema, type RpcMethod, } from "./utils.js"; // ── Utilities ─────────────────────────────────────────────────────────────── /** * Modernize quicktype's Python 3.7 output to Python 3.11+ syntax: * - Optional[T] → T | None * - List[T] → list[T] * - Dict[K, V] → dict[K, V] * - Type[T] → type[T] * - Callable from collections.abc instead of typing * - Clean up unused typing imports */ function replaceBalancedBrackets(code: string, prefix: string, replacer: (inner: string) => string): string { let result = ""; let i = 0; while (i < code.length) { const idx = code.indexOf(prefix + "[", i); if (idx === -1) { result += code.slice(i); break; } result += code.slice(i, idx); const start = idx + prefix.length + 1; // after '[' let depth = 1; let j = start; while (j < code.length && depth > 0) { if (code[j] === "[") depth++; else if (code[j] === "]") depth--; j++; } const inner = code.slice(start, j - 1); result += replacer(inner); i = j; } return result; } /** Split a string by commas, but only at the top bracket depth (ignores commas inside [...]) */ function splitTopLevelCommas(s: string): string[] { const parts: string[] = []; let depth = 0; let start = 0; for (let i = 0; i < s.length; i++) { if (s[i] === "[") depth++; else if (s[i] === "]") depth--; else if (s[i] === "," && depth === 0) { parts.push(s.slice(start, i)); start = i + 1; } } parts.push(s.slice(start)); return parts; } function modernizePython(code: string): string { // Replace Optional[X] with X | None (handles arbitrarily nested brackets) code = replaceBalancedBrackets(code, "Optional", (inner) => `${inner} | None`); // Replace Union[X, Y] with X | Y (split only at top-level commas, not inside brackets) // Run iteratively to handle nested Union inside Dict/List let prev = ""; while (prev !== code) { prev = code; code = replaceBalancedBrackets(code, "Union", (inner) => { return splitTopLevelCommas(inner).map((s: string) => s.trim()).join(" | "); }); } // Replace List[X] with list[X] code = code.replace(/\bList\[/g, "list["); // Replace Dict[K, V] with dict[K, V] code = code.replace(/\bDict\[/g, "dict["); // Replace Type[T] with type[T] code = code.replace(/\bType\[/g, "type["); // Move Callable from typing to collections.abc code = code.replace( /from typing import (.*), Callable$/m, "from typing import $1\nfrom collections.abc import Callable" ); code = code.replace( /from typing import Callable, (.*)$/m, "from typing import $1\nfrom collections.abc import Callable" ); // Remove now-unused imports from typing (Optional, List, Dict, Type) code = code.replace(/from typing import (.+)$/m, (_match, imports: string) => { const items = imports.split(",").map((s: string) => s.trim()); const remove = new Set(["Optional", "List", "Dict", "Type", "Union"]); const kept = items.filter((i: string) => !remove.has(i)); return `from typing import ${kept.join(", ")}`; }); return code; } function collapsePlaceholderPythonDataclasses(code: string): string { const classBlockRe = /(@dataclass\r?\nclass\s+(\w+):[\s\S]*?)(?=^@dataclass|^class\s+\w+|^def\s+\w+|\Z)/gm; const matches = [...code.matchAll(classBlockRe)].map((match) => ({ fullBlock: match[1], name: match[2], normalizedBody: normalizePythonDataclassBlock(match[1], match[2]), })); const groups = new Map(); for (const match of matches) { const group = groups.get(match.normalizedBody) ?? []; group.push(match); groups.set(match.normalizedBody, group); } for (const group of groups.values()) { if (group.length < 2) continue; const canonical = chooseCanonicalPlaceholderDuplicate(group.map(({ name }) => name)); if (!canonical) continue; for (const duplicate of group) { if (duplicate.name === canonical) continue; if (!isPlaceholderTypeName(duplicate.name)) continue; code = code.replace(duplicate.fullBlock, ""); code = code.replace(new RegExp(`\\b${duplicate.name}\\b`, "g"), canonical); } } return code.replace(/\n{3,}/g, "\n\n"); } function normalizePythonDataclassBlock(block: string, name: string): string { return block .replace(/^@dataclass\r?\nclass\s+\w+:/, "@dataclass\nclass:") .replace(new RegExp(`\\b${name}\\b`, "g"), "SelfType") .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line.length > 0) .join("\n"); } function chooseCanonicalPlaceholderDuplicate(names: string[]): string | undefined { const specificNames = names.filter((name) => !isPlaceholderTypeName(name)); if (specificNames.length === 0) return undefined; return specificNames.sort((left, right) => right.length - left.length || left.localeCompare(right))[0]; } function isPlaceholderTypeName(name: string): boolean { return name.endsWith("Class"); } function toSnakeCase(s: string): string { return s .replace(/([a-z])([A-Z])/g, "$1_$2") .replace(/[._]/g, "_") .toLowerCase(); } function toPascalCase(s: string): string { return s .split(/[._]/) .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) .join(""); } function collectRpcMethods(node: Record): RpcMethod[] { const results: RpcMethod[] = []; for (const value of Object.values(node)) { if (isRpcMethod(value)) { results.push(value); } else if (typeof value === "object" && value !== null) { results.push(...collectRpcMethods(value as Record)); } } return results; } function pythonResultTypeName(method: RpcMethod): string { return getRpcSchemaTypeName(method.result, toPascalCase(method.rpcMethod) + "Result"); } function pythonParamsTypeName(method: RpcMethod): string { return getRpcSchemaTypeName(method.params, toPascalCase(method.rpcMethod) + "Request"); } // ── Session Events ────────────────────────────────────────────────────────── async function generateSessionEvents(schemaPath?: string): Promise { console.log("Python: generating session-events..."); const resolvedPath = schemaPath ?? (await getSessionEventsSchemaPath()); const schema = cloneSchemaForCodegen(JSON.parse(await fs.readFile(resolvedPath, "utf-8")) as JSONSchema7); const resolvedSchema = (schema.definitions?.SessionEvent as JSONSchema7) || schema; const processed = postProcessSchema(resolvedSchema); // Hoist titled inline schemas (enums etc.) to definitions so quicktype // uses the schema-defined names instead of its own structural heuristics. const { rootDefinitions: hoistedRoots, sharedDefinitions } = hoistTitledSchemas({ SessionEvent: processed }); const hoisted = hoistedRoots.SessionEvent; if (Object.keys(sharedDefinitions).length > 0) { hoisted.definitions = { ...hoisted.definitions, ...sharedDefinitions }; } const schemaInput = new JSONSchemaInput(new FetchingJSONSchemaStore()); await schemaInput.addSource({ name: "SessionEvent", schema: JSON.stringify(hoisted) }); const inputData = new InputData(); inputData.addInput(schemaInput); const result = await quicktype({ inputData, lang: "python", rendererOptions: { "python-version": "3.7" }, }); let code = result.lines.join("\n"); // Fix dataclass field ordering (Any fields need defaults) code = code.replace(/: Any$/gm, ": Any = None"); // Fix bare except: to use Exception (required by ruff/pylint) code = code.replace(/except:/g, "except Exception:"); // Modernize to Python 3.11+ syntax code = modernizePython(code); // Add UNKNOWN enum value for forward compatibility code = code.replace( /^(class SessionEventType\(Enum\):.*?)(^\s*\n@dataclass)/ms, `$1 # UNKNOWN is used for forward compatibility UNKNOWN = "unknown" @classmethod def _missing_(cls, value: object) -> "SessionEventType": """Handle unknown event types gracefully for forward compatibility.""" return cls.UNKNOWN $2` ); const banner = `""" AUTO-GENERATED FILE - DO NOT EDIT Generated from: session-events.schema.json """ `; const outPath = await writeGeneratedFile("python/copilot/generated/session_events.py", banner + code); console.log(` ✓ ${outPath}`); } // ── RPC Types ─────────────────────────────────────────────────────────────── async function generateRpc(schemaPath?: string): Promise { console.log("Python: generating RPC types..."); const resolvedPath = schemaPath ?? (await getApiSchemaPath()); const schema = cloneSchemaForCodegen(JSON.parse(await fs.readFile(resolvedPath, "utf-8")) as ApiSchema); const allMethods = [ ...collectRpcMethods(schema.server || {}), ...collectRpcMethods(schema.session || {}), ...collectRpcMethods(schema.clientSession || {}), ]; // Build a combined schema for quicktype const combinedSchema: JSONSchema7 = { $schema: "http://json-schema.org/draft-07/schema#", definitions: {}, }; for (const method of allMethods) { if (!isVoidSchema(method.result)) { combinedSchema.definitions![pythonResultTypeName(method)] = method.result; } if (method.params?.properties && Object.keys(method.params.properties).length > 0) { if (method.rpcMethod.startsWith("session.")) { const filtered: JSONSchema7 = { ...method.params, properties: Object.fromEntries( Object.entries(method.params.properties).filter(([k]) => k !== "sessionId") ), required: method.params.required?.filter((r) => r !== "sessionId"), }; if (Object.keys(filtered.properties!).length > 0) { combinedSchema.definitions![pythonParamsTypeName(method)] = filtered; } } else { combinedSchema.definitions![pythonParamsTypeName(method)] = method.params; } } } const { rootDefinitions, sharedDefinitions } = hoistTitledSchemas(combinedSchema.definitions! as Record); // Generate types via quicktype const schemaInput = new JSONSchemaInput(new FetchingJSONSchemaStore()); for (const [name, def] of Object.entries(rootDefinitions)) { await schemaInput.addSource({ name, schema: JSON.stringify({ ...def, definitions: sharedDefinitions, }), }); } const inputData = new InputData(); inputData.addInput(schemaInput); const qtResult = await quicktype({ inputData, lang: "python", rendererOptions: { "python-version": "3.7" }, }); let typesCode = qtResult.lines.join("\n"); // Fix dataclass field ordering typesCode = typesCode.replace(/: Any$/gm, ": Any = None"); // Fix bare except: to use Exception (required by ruff/pylint) typesCode = typesCode.replace(/except:/g, "except Exception:"); // Remove unnecessary pass when class has methods (quicktype generates pass for empty schemas) typesCode = typesCode.replace(/^(\s*)pass\n\n(\s*@staticmethod)/gm, "$2"); // Modernize to Python 3.11+ syntax typesCode = modernizePython(typesCode); typesCode = collapsePlaceholderPythonDataclasses(typesCode); // Annotate experimental data types const experimentalTypeNames = new Set(); for (const method of allMethods) { if (method.stability !== "experimental") continue; experimentalTypeNames.add(pythonResultTypeName(method)); const paramsTypeName = pythonParamsTypeName(method); if (rootDefinitions[paramsTypeName]) { experimentalTypeNames.add(paramsTypeName); } } for (const typeName of experimentalTypeNames) { typesCode = typesCode.replace( new RegExp(`^(@dataclass\\n)?class ${typeName}[:(]`, "m"), (match) => `# Experimental: this type is part of an experimental API and may change or be removed.\n${match}` ); } // Extract actual class names generated by quicktype (may differ from toPascalCase, // e.g. quicktype produces "SessionMCPList" not "SessionMcpList") const actualTypeNames = new Map(); const classRe = /^class\s+(\w+)\b/gm; let cm; while ((cm = classRe.exec(typesCode)) !== null) { actualTypeNames.set(cm[1].toLowerCase(), cm[1]); } const resolveType = (name: string): string => actualTypeNames.get(name.toLowerCase()) ?? name; const lines: string[] = []; lines.push(`""" AUTO-GENERATED FILE - DO NOT EDIT Generated from: api.schema.json """ from typing import TYPE_CHECKING if TYPE_CHECKING: from .._jsonrpc import JsonRpcClient from collections.abc import Callable from dataclasses import dataclass from typing import Protocol `); lines.push(typesCode); lines.push(` def _timeout_kwargs(timeout: float | None) -> dict: """Build keyword arguments for optional timeout forwarding.""" if timeout is not None: return {"timeout": timeout} return {} def _patch_model_capabilities(data: dict) -> dict: """Ensure model capabilities have required fields. TODO: Remove once the runtime schema correctly marks these fields as optional. Some models (e.g. embedding models) may omit 'limits' or 'supports' in their capabilities, or omit 'max_context_window_tokens' within limits. The generated deserializer requires these fields, so we supply defaults here. """ for model in data.get("models", []): caps = model.get("capabilities") if caps is None: model["capabilities"] = {"supports": {}, "limits": {"max_context_window_tokens": 0}} continue if "supports" not in caps: caps["supports"] = {} if "limits" not in caps: caps["limits"] = {"max_context_window_tokens": 0} elif "max_context_window_tokens" not in caps["limits"]: caps["limits"]["max_context_window_tokens"] = 0 return data `); // Emit RPC wrapper classes if (schema.server) { emitRpcWrapper(lines, schema.server, false, resolveType); } if (schema.session) { emitRpcWrapper(lines, schema.session, true, resolveType); } if (schema.clientSession) { emitClientSessionApiRegistration(lines, schema.clientSession, resolveType); } // Patch models.list to normalize capabilities before deserialization let finalCode = lines.join("\n"); finalCode = finalCode.replace( `ModelList.from_dict(await self._client.request("models.list"`, `ModelList.from_dict(_patch_model_capabilities(await self._client.request("models.list"`, ); // Close the extra paren opened by _patch_model_capabilities( finalCode = finalCode.replace( /(_patch_model_capabilities\(await self\._client\.request\("models\.list",\s*\{[^)]*\)[^)]*\))/, "$1)", ); const outPath = await writeGeneratedFile("python/copilot/generated/rpc.py", finalCode); console.log(` ✓ ${outPath}`); } function emitRpcWrapper(lines: string[], node: Record, isSession: boolean, resolveType: (name: string) => string): void { const groups = Object.entries(node).filter(([, v]) => typeof v === "object" && v !== null && !isRpcMethod(v)); const topLevelMethods = Object.entries(node).filter(([, v]) => isRpcMethod(v)); const wrapperName = isSession ? "SessionRpc" : "ServerRpc"; // Emit API classes for groups for (const [groupName, groupNode] of groups) { const prefix = isSession ? "" : "Server"; const apiName = prefix + toPascalCase(groupName) + "Api"; const groupExperimental = isNodeFullyExperimental(groupNode as Record); if (isSession) { if (groupExperimental) { lines.push(`# Experimental: this API group is experimental and may change or be removed.`); } lines.push(`class ${apiName}:`); lines.push(` def __init__(self, client: "JsonRpcClient", session_id: str):`); lines.push(` self._client = client`); lines.push(` self._session_id = session_id`); } else { if (groupExperimental) { lines.push(`# Experimental: this API group is experimental and may change or be removed.`); } lines.push(`class ${apiName}:`); lines.push(` def __init__(self, client: "JsonRpcClient"):`); lines.push(` self._client = client`); } lines.push(``); for (const [key, value] of Object.entries(groupNode as Record)) { if (!isRpcMethod(value)) continue; emitMethod(lines, key, value, isSession, resolveType, groupExperimental); } lines.push(``); } // Emit wrapper class if (isSession) { lines.push(`class ${wrapperName}:`); lines.push(` """Typed session-scoped RPC methods."""`); lines.push(` def __init__(self, client: "JsonRpcClient", session_id: str):`); lines.push(` self._client = client`); lines.push(` self._session_id = session_id`); for (const [groupName] of groups) { lines.push(` self.${toSnakeCase(groupName)} = ${toPascalCase(groupName)}Api(client, session_id)`); } } else { lines.push(`class ${wrapperName}:`); lines.push(` """Typed server-scoped RPC methods."""`); lines.push(` def __init__(self, client: "JsonRpcClient"):`); lines.push(` self._client = client`); for (const [groupName] of groups) { lines.push(` self.${toSnakeCase(groupName)} = Server${toPascalCase(groupName)}Api(client)`); } } lines.push(``); // Top-level methods for (const [key, value] of topLevelMethods) { if (!isRpcMethod(value)) continue; emitMethod(lines, key, value, isSession, resolveType, false); } lines.push(``); } function emitMethod(lines: string[], name: string, method: RpcMethod, isSession: boolean, resolveType: (name: string) => string, groupExperimental = false): void { const methodName = toSnakeCase(name); const hasResult = !isVoidSchema(method.result); const resultType = hasResult ? resolveType(pythonResultTypeName(method)) : "None"; const resultIsObject = isObjectSchema(method.result); const paramProps = method.params?.properties || {}; const nonSessionParams = Object.keys(paramProps).filter((k) => k !== "sessionId"); const hasParams = isSession ? nonSessionParams.length > 0 : Object.keys(paramProps).length > 0; const paramsType = resolveType(pythonParamsTypeName(method)); // Build signature with typed params + optional timeout const sig = hasParams ? ` async def ${methodName}(self, params: ${paramsType}, *, timeout: float | None = None) -> ${resultType}:` : ` async def ${methodName}(self, *, timeout: float | None = None) -> ${resultType}:`; lines.push(sig); if (method.stability === "experimental" && !groupExperimental) { lines.push(` """.. warning:: This API is experimental and may change or be removed in future versions."""`); } // For object results use .from_dict(); for enums/primitives use direct construction const deserialize = (expr: string) => resultIsObject ? `${resultType}.from_dict(${expr})` : `${resultType}(${expr})`; // Build request body with proper serialization/deserialization if (isSession) { if (hasParams) { lines.push(` params_dict = {k: v for k, v in params.to_dict().items() if v is not None}`); lines.push(` params_dict["sessionId"] = self._session_id`); if (hasResult) { lines.push(` return ${deserialize(`await self._client.request("${method.rpcMethod}", params_dict, **_timeout_kwargs(timeout))`)}`); } else { lines.push(` await self._client.request("${method.rpcMethod}", params_dict, **_timeout_kwargs(timeout))`); } } else { if (hasResult) { lines.push(` return ${deserialize(`await self._client.request("${method.rpcMethod}", {"sessionId": self._session_id}, **_timeout_kwargs(timeout))`)}`); } else { lines.push(` await self._client.request("${method.rpcMethod}", {"sessionId": self._session_id}, **_timeout_kwargs(timeout))`); } } } else { if (hasParams) { lines.push(` params_dict = {k: v for k, v in params.to_dict().items() if v is not None}`); if (hasResult) { lines.push(` return ${deserialize(`await self._client.request("${method.rpcMethod}", params_dict, **_timeout_kwargs(timeout))`)}`); } else { lines.push(` await self._client.request("${method.rpcMethod}", params_dict, **_timeout_kwargs(timeout))`); } } else { if (hasResult) { lines.push(` return ${deserialize(`await self._client.request("${method.rpcMethod}", {}, **_timeout_kwargs(timeout))`)}`); } else { lines.push(` await self._client.request("${method.rpcMethod}", {}, **_timeout_kwargs(timeout))`); } } } lines.push(``); } function emitClientSessionApiRegistration( lines: string[], node: Record, resolveType: (name: string) => string ): void { const groups = Object.entries(node).filter(([, value]) => typeof value === "object" && value !== null && !isRpcMethod(value)); for (const [groupName, groupNode] of groups) { const handlerName = `${toPascalCase(groupName)}Handler`; const groupExperimental = isNodeFullyExperimental(groupNode as Record); if (groupExperimental) { lines.push(`# Experimental: this API group is experimental and may change or be removed.`); } lines.push(`class ${handlerName}(Protocol):`); for (const [methodName, value] of Object.entries(groupNode as Record)) { if (!isRpcMethod(value)) continue; emitClientSessionHandlerMethod(lines, methodName, value, resolveType, groupExperimental); } lines.push(``); } lines.push(`@dataclass`); lines.push(`class ClientSessionApiHandlers:`); if (groups.length === 0) { lines.push(` pass`); } else { for (const [groupName] of groups) { lines.push(` ${toSnakeCase(groupName)}: ${toPascalCase(groupName)}Handler | None = None`); } } lines.push(``); lines.push(`def register_client_session_api_handlers(`); lines.push(` client: "JsonRpcClient",`); lines.push(` get_handlers: Callable[[str], ClientSessionApiHandlers],`); lines.push(`) -> None:`); lines.push(` """Register client-session request handlers on a JSON-RPC connection."""`); if (groups.length === 0) { lines.push(` return`); } else { for (const [groupName, groupNode] of groups) { for (const [methodName, value] of Object.entries(groupNode as Record)) { if (!isRpcMethod(value)) continue; emitClientSessionRegistrationMethod( lines, groupName, methodName, value, resolveType ); } } } lines.push(``); } function emitClientSessionHandlerMethod( lines: string[], name: string, method: RpcMethod, resolveType: (name: string) => string, groupExperimental = false ): void { const paramsType = resolveType(pythonParamsTypeName(method)); const resultType = !isVoidSchema(method.result) ? resolveType(pythonResultTypeName(method)) : "None"; lines.push(` async def ${toSnakeCase(name)}(self, params: ${paramsType}) -> ${resultType}:`); if (method.stability === "experimental" && !groupExperimental) { lines.push(` """.. warning:: This API is experimental and may change or be removed in future versions."""`); } lines.push(` pass`); } function emitClientSessionRegistrationMethod( lines: string[], groupName: string, methodName: string, method: RpcMethod, resolveType: (name: string) => string ): void { const handlerVariableName = `handle_${toSnakeCase(groupName)}_${toSnakeCase(methodName)}`; const paramsType = resolveType(pythonParamsTypeName(method)); const resultType = !isVoidSchema(method.result) ? resolveType(pythonResultTypeName(method)) : null; const handlerField = toSnakeCase(groupName); const handlerMethod = toSnakeCase(methodName); lines.push(` async def ${handlerVariableName}(params: dict) -> dict | None:`); lines.push(` request = ${paramsType}.from_dict(params)`); lines.push(` handler = get_handlers(request.session_id).${handlerField}`); lines.push( ` if handler is None: raise RuntimeError(f"No ${handlerField} handler registered for session: {request.session_id}")` ); if (resultType) { lines.push(` result = await handler.${handlerMethod}(request)`); if (isObjectSchema(method.result)) { lines.push(` return result.to_dict()`); } else { lines.push(` return result.value if hasattr(result, 'value') else result`); } } else { lines.push(` await handler.${handlerMethod}(request)`); lines.push(` return None`); } lines.push(` client.set_request_handler("${method.rpcMethod}", ${handlerVariableName})`); } // ── Main ──────────────────────────────────────────────────────────────────── async function generate(sessionSchemaPath?: string, apiSchemaPath?: string): Promise { await generateSessionEvents(sessionSchemaPath); try { await generateRpc(apiSchemaPath); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT" && !apiSchemaPath) { console.log("Python: skipping RPC (api.schema.json not found)"); } else { throw err; } } } const sessionArg = process.argv[2] || undefined; const apiArg = process.argv[3] || undefined; generate(sessionArg, apiArg).catch((err) => { console.error("Python generation failed:", err); process.exit(1); });