Make indexing and per-source chunk optional so minimal RAG configs validate (#88)

Dockerfile

@@ -21,9 +21,14 @@ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
 COPY package.json package-lock.json ./
 RUN npm ci --omit=dev
-# Optional: install these if your config uses document sources or local embeddings
+# Optional peer dependencies. `npm ci --omit=dev` above intentionally
+# SKIPS these (they are declared as peerDependencies, not dependencies),
+# so the default prod image is lean. If your pathfinder.yaml uses any of
+# the features below, UNCOMMENT the matching line and rebuild — otherwise
+# the server will throw a clear "Install <pkg>" error at runtime when that
+# feature is first exercised (e.g. embedding.provider: local).
 # RUN npm install pdf-parse mammoth        # For type: document (PDF/DOCX)
-# RUN npm install @xenova/transformers     # For embedding.provider: local
+# RUN npm install @xenova/transformers     # For embedding.provider: local (transformers.js / in-process CPU embeddings)
 COPY --from=build /app/dist/ ./dist/
 COPY docs/analytics.html ./docs/analytics.html
 COPY deploy/copilotkit-docs.yaml ./copilotkit-docs.yaml

docs/deploy/index.html

@@ -444,6 +444,10 @@ <h3 id="optional-dependencies">Optional dependencies</h3>
         </tbody>
     </table>
     <p>Run <code>pathfinder validate</code> to detect missing optional dependencies and get install instructions.</p>
+    <div class="gap-card">
+        <h4>Docker images: optional peer deps are not bundled</h4>
+        <p>The official <code>ghcr.io/copilotkit/pathfinder</code> image is built with <code>npm ci --omit=dev</code>, which intentionally omits these optional <em>peer</em> dependencies to keep the image lean. If your config uses <code>embedding.provider: local</code> (transformers.js, the zero-API-key in-process provider) or <code>type: document</code> sources, the server boots and validates fine, then throws a clear <code>Install &lt;pkg&gt;</code> error the first time that feature runs. To bake the dependency into the image, uncomment the matching <code>RUN npm install …</code> line in the <a href="https://github.com/CopilotKit/pathfinder/blob/main/Dockerfile">Dockerfile</a> (e.g. <code>RUN npm install @xenova/transformers</code> for local embeddings) and rebuild.</p>
+    </div>
 
     <h3 id="health-endpoint">Health endpoint</h3>
     <p>Pathfinder exposes <code>GET /health</code> for monitoring. It returns JSON with uptime, indexing status, chunk counts per source, and index state (last indexed time, commit SHA, errors). Use it for load balancer health checks and deployment verification.</p>

src/__tests__/minimal-rag-config.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, expect } from "vitest";
+import { ServerConfigSchema, IndexingConfigSchema } from "../types.js";
+
+// Regression coverage for issue #88: a docs-compliant minimal RAG config
+// using local embeddings should validate WITHOUT forcing cron-scheduling
+// (`indexing`) fields or an explicit per-source `chunk` object.
+//
+// This exercises the REAL parse surface — ServerConfigSchema, the same
+// schema config.ts/validate.ts run YAML through — not a hand-rolled fake.
+
+describe("minimal RAG config (issue #88)", () => {
+  // The docs-compliant config from the issue, focused on the three
+  // reported schema blockers: a `local` embedding provider, one source
+  // with NO `chunk`, a search tool, and NO top-level `indexing` block.
+  // (The issue's crash log lists exactly five errors — sources.0.chunk,
+  // embedding.provider, and the three indexing fields — and none about
+  // search-tool limits, so this config supplies the search-tool limit /
+  // format fields to keep the test pinned to the three claims under fix.)
+  const minimalIssue88Config = {
+    server: { name: "tech-ecosystem", version: "1.13.3" },
+    embedding: {
+      provider: "local",
+      model: "Xenova/all-MiniLM-L6-v2",
+      dimensions: 384,
+    },
+    sources: [
+      {
+        name: "docs",
+        type: "code",
+        path: "/app/docs",
+        file_patterns: ["**/*"],
+      },
+    ],
+    tools: [
+      {
+        name: "search-docs",
+        type: "search",
+        description: "Search the docs",
+        source: "docs",
+        default_limit: 5,
+        max_limit: 20,
+        result_format: "docs",
+      },
+    ],
+  };
+
+  it("accepts a local-embedding config with no chunk and no indexing block", () => {
+    const result = ServerConfigSchema.safeParse(minimalIssue88Config);
+    if (!result.success) {
+      // Surface the exact failing issues so red/green output is legible.
+      console.error(JSON.stringify(result.error.issues, null, 2));
+    }
+    expect(result.success).toBe(true);
+  });
+
+  it("defaults indexing fields when the indexing block is omitted", () => {
+    const result = ServerConfigSchema.parse(minimalIssue88Config);
+    expect(result.indexing).toEqual({
+      auto_reindex: false,
+      reindex_hour_utc: 3,
+      stale_threshold_hours: 24,
+    });
+  });
+
+  it("IndexingConfigSchema parses an empty object via defaults", () => {
+    const result = IndexingConfigSchema.parse({});
+    expect(result).toEqual({
+      auto_reindex: false,
+      reindex_hour_utc: 3,
+      stale_threshold_hours: 24,
+    });
+  });
+
+  it("REGRESSION: a full explicit config still validates unchanged", () => {
+    const fullConfig = {
+      server: { name: "full", version: "1.0.0" },
+      embedding: {
+        provider: "openai",
+        model: "text-embedding-3-small",
+        dimensions: 1536,
+      },
+      indexing: {
+        auto_reindex: true,
+        reindex_hour_utc: 5,
+        stale_threshold_hours: 48,
+      },
+      sources: [
+        {
+          name: "docs",
+          type: "markdown",
+          path: "./docs",
+          file_patterns: ["**/*.md"],
+          chunk: { target_tokens: 512, overlap_tokens: 64 },
+        },
+      ],
+      tools: [
+        {
+          name: "search-docs",
+          type: "search",
+          description: "Search the docs",
+          source: "docs",
+          default_limit: 5,
+          max_limit: 20,
+          result_format: "docs",
+        },
+      ],
+    };
+    const result = ServerConfigSchema.parse(fullConfig);
+    expect(result.indexing).toEqual({
+      auto_reindex: true,
+      reindex_hour_utc: 5,
+      stale_threshold_hours: 48,
+    });
+    expect(result.embedding).toEqual({
+      provider: "openai",
+      model: "text-embedding-3-small",
+      dimensions: 1536,
+    });
+    // Explicit per-source chunk must be preserved untouched.
+    const src = result.sources[0];
+    expect(src.chunk).toEqual({ target_tokens: 512, overlap_tokens: 64 });
+  });
+});

src/__tests__/tool-config.test.ts

@@ -408,7 +408,12 @@ describe("ServerConfigSchema", () => {
     expect(result.success).toBe(false);
   });
 
-  it("rejects search config without indexing", () => {
+  // issue #88: a search config may omit the `indexing` block entirely.
+  // The indexing block is no longer mandatory for RAG configs — it
+  // defaults to IndexingConfigSchema's per-field defaults (manual-only,
+  // i.e. auto_reindex off) so minimal/manual deployments validate without
+  // a cron schedule.
+  it("accepts search config without indexing and applies defaults", () => {
     const { indexing, ...configWithoutIndexing } = minimalConfig;
     const config = {
       ...configWithoutIndexing,
@@ -425,7 +430,14 @@ describe("ServerConfigSchema", () => {
       ],
     };
     const result = ServerConfigSchema.safeParse(config);
-    expect(result.success).toBe(false);
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.indexing).toEqual({
+        auto_reindex: false,
+        reindex_hour_utc: 3,
+        stale_threshold_hours: 24,
+      });
+    }
   });
 
   describe("server.allowlist", () => {

src/types.ts

@@ -91,10 +91,18 @@ export const ChunkConfigSchema = z.object({
   overlap_lines: z.number().int().nonnegative().optional(),
 });
 
-// Base fields shared by all source types
+// Base fields shared by all source types.
+//
+// `chunk` is optional: every field inside ChunkConfigSchema is already
+// optional, so a source that omits `chunk` (or supplies an empty object)
+// falls back to the chunker's built-in per-source-type defaults. It
+// defaults to `{}` so downstream code can read `source.chunk` without a
+// null check. (issue #88 — a docs-compliant source like
+// `{ name, type, path, file_patterns }` should validate without a
+// redundant `chunk: {}`.)
 const BaseSourceFields = {
   name: z.string().min(1),
-  chunk: ChunkConfigSchema,
+  chunk: ChunkConfigSchema.default({}),
   version: z.string().optional(),
   category: z.enum(["faq"]).optional(),
 };
@@ -296,10 +304,17 @@ export const EmbeddingConfigSchema = z.discriminatedUnion("provider", [
 
 // ── Indexing configuration schemas ────────────────────────────────────────────
 
+// Indexing cron schedule. All fields have sensible defaults so a minimal
+// or manual-only deployment can omit the `indexing` block entirely (issue
+// #88). Defaults: auto_reindex off (manual indexing), 03:00 UTC if it is
+// ever enabled, and a 24h staleness window. The block itself is given a
+// `.default({})` at the ServerConfigSchema level so a `hasRag` config that
+// omits `indexing` still resolves to these defaults rather than hard-
+// failing purely for lack of a cron schedule.
 export const IndexingConfigSchema = z.object({
-  auto_reindex: z.boolean(),
-  reindex_hour_utc: z.number().int().min(0).max(23),
-  stale_threshold_hours: z.number().int().positive(),
+  auto_reindex: z.boolean().default(false),
+  reindex_hour_utc: z.number().int().min(0).max(23).default(3),
+  stale_threshold_hours: z.number().int().positive().default(24),
 });
 
 // ── Webhook configuration schemas ─────────────────────────────────────────────
@@ -371,7 +386,11 @@ export const ServerConfigSchema = z
     sources: z.array(SourceConfigSchema).min(1),
     tools: z.array(AnyToolConfigSchema).min(1),
     embedding: EmbeddingConfigSchema.optional(),
-    indexing: IndexingConfigSchema.optional(),
+    // Defaults to `{}` so the IndexingConfigSchema field defaults always
+    // populate a complete indexing block. This means a `hasRag` config no
+    // longer hard-fails purely for omitting a cron schedule (issue #88);
+    // an explicit `indexing` block still overrides the defaults.
+    indexing: IndexingConfigSchema.default({}),
     webhook: WebhookConfigSchema.optional(),
     analytics: AnalyticsConfigSchema.optional(),
   })
@@ -387,14 +406,11 @@ export const ServerConfigSchema = z
         path: ["embedding"],
       });
     }
-    if (hasRag && !cfg.indexing) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message:
-          "indexing config is required when search tools are configured.",
-        path: ["indexing"],
-      });
-    }
+    // Note: `indexing` no longer needs a hasRag presence check — the
+    // schema field defaults to `{}` (resolved to IndexingConfigSchema's
+    // per-field defaults), so a complete indexing block is always present.
+    // A manual-only / minimal RAG config validates without a cron schedule
+    // (issue #88).
     const sourceNames = new Set(cfg.sources.map((s) => s.name));
     for (const tool of cfg.tools) {
       if (tool.type === "search" && tool.default_limit > tool.max_limit) {