From 4648ca1533c0096120fb3e22837353c7fe421725 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 25 Jun 2026 10:27:02 -0700 Subject: [PATCH] Make indexing and per-source chunk optional so minimal RAG configs validate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the three v1.13.3+ Zod-validator complaints in issue #88 that block a docs-compliant minimal RAG config (local embeddings) from booting: - IndexingConfigSchema fields (auto_reindex, reindex_hour_utc, stale_threshold_hours) now have sensible defaults (false / 3 / 24), and the top-level `indexing` block defaults to `{}`. A manual-only / minimal config no longer hard-fails for omitting a cron schedule. The now-dead `hasRag && !cfg.indexing` superRefine check is removed; the embedding presence check stays. - BaseSourceFields.chunk is now optional (defaults to `{}`). Its inner fields were already all-optional, so a source can omit `chunk` and fall back to the chunker's per-type defaults. - Local embeddings were never a schema bug — LocalEmbeddingConfigSchema is already in the embedding union. The real cause is the prod Docker image (`npm ci --omit=dev`) omitting the optional @xenova/transformers peer dep. Clarified the Dockerfile guidance and added a deploy-docs callout; kept the dep optional (not a hard dependency). Existing valid configs are preserved (defaults only fill omitted fields; explicit values pass through unchanged). Closes #88 --- Dockerfile | 9 +- docs/deploy/index.html | 4 + src/__tests__/minimal-rag-config.test.ts | 123 +++++++++++++++++++++++ src/__tests__/tool-config.test.ts | 16 ++- src/types.ts | 44 +++++--- 5 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 src/__tests__/minimal-rag-config.test.ts diff --git a/Dockerfile b/Dockerfile index 428711f..98028bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,9 +21,14 @@ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev -# Optional: install these if your config uses document sources or local embeddings +# Optional peer dependencies. `npm ci --omit=dev` above intentionally +# SKIPS these (they are declared as peerDependencies, not dependencies), +# so the default prod image is lean. If your pathfinder.yaml uses any of +# the features below, UNCOMMENT the matching line and rebuild — otherwise +# the server will throw a clear "Install " error at runtime when that +# feature is first exercised (e.g. embedding.provider: local). # RUN npm install pdf-parse mammoth # For type: document (PDF/DOCX) -# RUN npm install @xenova/transformers # For embedding.provider: local +# RUN npm install @xenova/transformers # For embedding.provider: local (transformers.js / in-process CPU embeddings) COPY --from=build /app/dist/ ./dist/ COPY docs/analytics.html ./docs/analytics.html COPY deploy/copilotkit-docs.yaml ./copilotkit-docs.yaml diff --git a/docs/deploy/index.html b/docs/deploy/index.html index 5ec1c3d..655a764 100644 --- a/docs/deploy/index.html +++ b/docs/deploy/index.html @@ -444,6 +444,10 @@

Optional dependencies

Run pathfinder validate to detect missing optional dependencies and get install instructions.

+
+

Docker images: optional peer deps are not bundled

+

The official ghcr.io/copilotkit/pathfinder image is built with npm ci --omit=dev, which intentionally omits these optional peer dependencies to keep the image lean. If your config uses embedding.provider: local (transformers.js, the zero-API-key in-process provider) or type: document sources, the server boots and validates fine, then throws a clear Install <pkg> error the first time that feature runs. To bake the dependency into the image, uncomment the matching RUN npm install … line in the Dockerfile (e.g. RUN npm install @xenova/transformers for local embeddings) and rebuild.

+

Health endpoint

Pathfinder exposes GET /health for monitoring. It returns JSON with uptime, indexing status, chunk counts per source, and index state (last indexed time, commit SHA, errors). Use it for load balancer health checks and deployment verification.

diff --git a/src/__tests__/minimal-rag-config.test.ts b/src/__tests__/minimal-rag-config.test.ts new file mode 100644 index 0000000..218fd88 --- /dev/null +++ b/src/__tests__/minimal-rag-config.test.ts @@ -0,0 +1,123 @@ +import { describe, it, expect } from "vitest"; +import { ServerConfigSchema, IndexingConfigSchema } from "../types.js"; + +// Regression coverage for issue #88: a docs-compliant minimal RAG config +// using local embeddings should validate WITHOUT forcing cron-scheduling +// (`indexing`) fields or an explicit per-source `chunk` object. +// +// This exercises the REAL parse surface — ServerConfigSchema, the same +// schema config.ts/validate.ts run YAML through — not a hand-rolled fake. + +describe("minimal RAG config (issue #88)", () => { + // The docs-compliant config from the issue, focused on the three + // reported schema blockers: a `local` embedding provider, one source + // with NO `chunk`, a search tool, and NO top-level `indexing` block. + // (The issue's crash log lists exactly five errors — sources.0.chunk, + // embedding.provider, and the three indexing fields — and none about + // search-tool limits, so this config supplies the search-tool limit / + // format fields to keep the test pinned to the three claims under fix.) + const minimalIssue88Config = { + server: { name: "tech-ecosystem", version: "1.13.3" }, + embedding: { + provider: "local", + model: "Xenova/all-MiniLM-L6-v2", + dimensions: 384, + }, + sources: [ + { + name: "docs", + type: "code", + path: "/app/docs", + file_patterns: ["**/*"], + }, + ], + tools: [ + { + name: "search-docs", + type: "search", + description: "Search the docs", + source: "docs", + default_limit: 5, + max_limit: 20, + result_format: "docs", + }, + ], + }; + + it("accepts a local-embedding config with no chunk and no indexing block", () => { + const result = ServerConfigSchema.safeParse(minimalIssue88Config); + if (!result.success) { + // Surface the exact failing issues so red/green output is legible. + console.error(JSON.stringify(result.error.issues, null, 2)); + } + expect(result.success).toBe(true); + }); + + it("defaults indexing fields when the indexing block is omitted", () => { + const result = ServerConfigSchema.parse(minimalIssue88Config); + expect(result.indexing).toEqual({ + auto_reindex: false, + reindex_hour_utc: 3, + stale_threshold_hours: 24, + }); + }); + + it("IndexingConfigSchema parses an empty object via defaults", () => { + const result = IndexingConfigSchema.parse({}); + expect(result).toEqual({ + auto_reindex: false, + reindex_hour_utc: 3, + stale_threshold_hours: 24, + }); + }); + + it("REGRESSION: a full explicit config still validates unchanged", () => { + const fullConfig = { + server: { name: "full", version: "1.0.0" }, + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + indexing: { + auto_reindex: true, + reindex_hour_utc: 5, + stale_threshold_hours: 48, + }, + sources: [ + { + name: "docs", + type: "markdown", + path: "./docs", + file_patterns: ["**/*.md"], + chunk: { target_tokens: 512, overlap_tokens: 64 }, + }, + ], + tools: [ + { + name: "search-docs", + type: "search", + description: "Search the docs", + source: "docs", + default_limit: 5, + max_limit: 20, + result_format: "docs", + }, + ], + }; + const result = ServerConfigSchema.parse(fullConfig); + expect(result.indexing).toEqual({ + auto_reindex: true, + reindex_hour_utc: 5, + stale_threshold_hours: 48, + }); + expect(result.embedding).toEqual({ + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }); + // Explicit per-source chunk must be preserved untouched. + const src = result.sources[0]; + expect(src.chunk).toEqual({ target_tokens: 512, overlap_tokens: 64 }); + }); +}); diff --git a/src/__tests__/tool-config.test.ts b/src/__tests__/tool-config.test.ts index b96824f..defc53d 100644 --- a/src/__tests__/tool-config.test.ts +++ b/src/__tests__/tool-config.test.ts @@ -408,7 +408,12 @@ describe("ServerConfigSchema", () => { expect(result.success).toBe(false); }); - it("rejects search config without indexing", () => { + // issue #88: a search config may omit the `indexing` block entirely. + // The indexing block is no longer mandatory for RAG configs — it + // defaults to IndexingConfigSchema's per-field defaults (manual-only, + // i.e. auto_reindex off) so minimal/manual deployments validate without + // a cron schedule. + it("accepts search config without indexing and applies defaults", () => { const { indexing, ...configWithoutIndexing } = minimalConfig; const config = { ...configWithoutIndexing, @@ -425,7 +430,14 @@ describe("ServerConfigSchema", () => { ], }; const result = ServerConfigSchema.safeParse(config); - expect(result.success).toBe(false); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.indexing).toEqual({ + auto_reindex: false, + reindex_hour_utc: 3, + stale_threshold_hours: 24, + }); + } }); describe("server.allowlist", () => { diff --git a/src/types.ts b/src/types.ts index 91826a4..818e1ac 100644 --- a/src/types.ts +++ b/src/types.ts @@ -91,10 +91,18 @@ export const ChunkConfigSchema = z.object({ overlap_lines: z.number().int().nonnegative().optional(), }); -// Base fields shared by all source types +// Base fields shared by all source types. +// +// `chunk` is optional: every field inside ChunkConfigSchema is already +// optional, so a source that omits `chunk` (or supplies an empty object) +// falls back to the chunker's built-in per-source-type defaults. It +// defaults to `{}` so downstream code can read `source.chunk` without a +// null check. (issue #88 — a docs-compliant source like +// `{ name, type, path, file_patterns }` should validate without a +// redundant `chunk: {}`.) const BaseSourceFields = { name: z.string().min(1), - chunk: ChunkConfigSchema, + chunk: ChunkConfigSchema.default({}), version: z.string().optional(), category: z.enum(["faq"]).optional(), }; @@ -296,10 +304,17 @@ export const EmbeddingConfigSchema = z.discriminatedUnion("provider", [ // ── Indexing configuration schemas ──────────────────────────────────────────── +// Indexing cron schedule. All fields have sensible defaults so a minimal +// or manual-only deployment can omit the `indexing` block entirely (issue +// #88). Defaults: auto_reindex off (manual indexing), 03:00 UTC if it is +// ever enabled, and a 24h staleness window. The block itself is given a +// `.default({})` at the ServerConfigSchema level so a `hasRag` config that +// omits `indexing` still resolves to these defaults rather than hard- +// failing purely for lack of a cron schedule. export const IndexingConfigSchema = z.object({ - auto_reindex: z.boolean(), - reindex_hour_utc: z.number().int().min(0).max(23), - stale_threshold_hours: z.number().int().positive(), + auto_reindex: z.boolean().default(false), + reindex_hour_utc: z.number().int().min(0).max(23).default(3), + stale_threshold_hours: z.number().int().positive().default(24), }); // ── Webhook configuration schemas ───────────────────────────────────────────── @@ -371,7 +386,11 @@ export const ServerConfigSchema = z sources: z.array(SourceConfigSchema).min(1), tools: z.array(AnyToolConfigSchema).min(1), embedding: EmbeddingConfigSchema.optional(), - indexing: IndexingConfigSchema.optional(), + // Defaults to `{}` so the IndexingConfigSchema field defaults always + // populate a complete indexing block. This means a `hasRag` config no + // longer hard-fails purely for omitting a cron schedule (issue #88); + // an explicit `indexing` block still overrides the defaults. + indexing: IndexingConfigSchema.default({}), webhook: WebhookConfigSchema.optional(), analytics: AnalyticsConfigSchema.optional(), }) @@ -387,14 +406,11 @@ export const ServerConfigSchema = z path: ["embedding"], }); } - if (hasRag && !cfg.indexing) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - "indexing config is required when search tools are configured.", - path: ["indexing"], - }); - } + // Note: `indexing` no longer needs a hasRag presence check — the + // schema field defaults to `{}` (resolved to IndexingConfigSchema's + // per-field defaults), so a complete indexing block is always present. + // A manual-only / minimal RAG config validates without a cron schedule + // (issue #88). const sourceNames = new Set(cfg.sources.map((s) => s.name)); for (const tool of cfg.tools) { if (tool.type === "search" && tool.default_limit > tool.max_limit) {