forked from CopilotKit/CopilotKit
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathvalidate-parity.ts
More file actions
1304 lines (1232 loc) · 49.9 KB
/
Copy pathvalidate-parity.ts
File metadata and controls
1304 lines (1232 loc) · 49.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Parity Validator
*
* Enforces demo <-> spec <-> QA-markdown parity across all packages under
* showcase/integrations/. For each package:
* 1. Reads manifest.yaml to extract declared demo IDs.
* 2. Lists tests/e2e/*.spec.ts files.
* 3. Lists qa/*.md files.
* 4. Lists src/app/demos/<id>/ directories.
*
* MUST checks (fail -> exit 1):
* - manifest.yaml exists and is parseable.
* - Every declared demo has a matching src/app/demos/<id>/ directory.
*
* SHOULD checks (warn on stderr, do not fail):
* - Every declared demo has a matching tests/e2e/<id>.spec.ts.
* - Every declared demo has a matching qa/<id>.md.
* - Package demo count matches the baseline (default: BASELINE_DEMO_COUNT).
* - spec count >= demo count (spec count exceeding demo count is
* legitimate — e.g. when a cross-demo spec covers renderer selection
* for multiple demos — so only UNDER-coverage is flagged).
* - qa count >= demo count.
*
* Usage (from showcase/ or showcase/scripts/):
* npx tsx scripts/validate-parity.ts
* npx tsx scripts/validate-parity.ts --baseline=9
* VALIDATE_PARITY_REPO_ROOT=/tmp/fixture npx tsx scripts/validate-parity.ts
*
* Exit codes (aligned with audit.ts on codes 0/1/3/4; intentionally
* diverges from validate-pins.ts on code 2 — validate-pins.ts uses
* 2=internal error, whereas this tool uses 2=invalid CLI input):
* 0 — no MUST failures
* 1 — one or more MUST failures
* 2 — invalid CLI input (bad --baseline / VALIDATE_PARITY_BASELINE)
* 3 — unreadable (packages dir missing or readdir threw)
* 4 — internal error (uncaught exception)
*
* The script resolves packages relative to its own file location by
* default, so the invocation cwd does not matter.
*/
/* eslint-disable @typescript-eslint/no-use-before-define */
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import {
parseManifest,
type Manifest,
type ManifestDemo,
} from "./lib/manifest.js";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// ROOT = showcase/ (NOT the repo root). validate-parity.ts lives at
// showcase/scripts/validate-parity.ts, so path.resolve(__dirname, "..")
// resolves to showcase/. DEFAULT_PACKAGES_DIR is showcase/integrations/.
const ROOT = path.resolve(__dirname, "..");
const DEFAULT_PACKAGES_DIR = path.join(ROOT, "integrations");
/**
* Baseline expected demo count per package. Packages that deviate from
* this are flagged as warnings (e.g. ones being built out).
*
* RECIPROCAL: the CI workflow .github/workflows/showcase_validate.yml
* reads `baselineDemoCount` from showcase/scripts/fail-baseline.json
* when enforcing the per-package e2e-spec-count floor — there is no
* hardcoded MIN in the workflow anymore. Keep this default in sync
* with `baselineDemoCount` in fail-baseline.json; if one moves,
* move both. The sync is enforced by
* __tests__/baseline-sync.test.ts so drift is caught in CI rather than
* relying on the comment above.
*/
export const BASELINE_DEMO_COUNT = 9;
// Exit code taxonomy aligned with audit.ts on codes 0/1/3/4 so CI callers
// can disambiguate "no anomalies" / "anomalies" / "unreadable" / "internal".
// Intentionally diverges from validate-pins.ts on code 2 (invalid-input here
// vs internal-error in validate-pins.ts) — the tools are NOT aligned on 2.
// `as const` narrows each to its literal type so the union below
// (`ValidateParityExitCode`) is a literal union — guarding callers that
// switch on the value from accidentally matching arbitrary numbers.
const EXIT_OK = 0 as const;
const EXIT_MUST_FAILURE = 1 as const;
const EXIT_INVALID_INPUT = 2 as const;
const EXIT_UNREADABLE = 3 as const;
const EXIT_INTERNAL = 4 as const;
/**
* Strip the leading "/demos/" prefix from a demo route and return the
* on-disk directory name (the segment under src/app/demos/). Returns
* `undefined` when the route is `undefined` OR when the resulting
* segment is empty. parseManifest rejects routes of exactly "/demos/"
* at validation time (demos[i].route must have a non-empty tail), so
* the empty-segment fallback here is defence-in-depth for callers that
* bypass parseManifest — not a primary validation.
*
* Keep in sync with bundle-demo-content.ts, which applies the same
* route → dir transformation when copying per-demo READMEs into the
* bundled content payload.
*/
export function routeToDirName(route: string | undefined): string | undefined {
if (route === undefined) return undefined;
const stripped = route.replace(/^\/demos\//, "");
return stripped.length > 0 ? stripped : undefined;
}
/**
* Literal union of every exit code `runParity` can return. Exposed so
* in-process callers (tests, composed CLIs) can pattern-match against
* the taxonomy without re-declaring magic numbers.
*/
export type ValidateParityExitCode =
| typeof EXIT_OK
| typeof EXIT_MUST_FAILURE
| typeof EXIT_INVALID_INPUT
| typeof EXIT_UNREADABLE
| typeof EXIT_INTERNAL;
/**
* Render an error (and any nested `.cause` chain) for stderr. Walks the
* cause chain depth-first and indents each successive cause so operators
* see both the outer wrapping context (e.g. "audit of <slug> crashed")
* AND the root-cause message/stack without rebuilding the chain by hand.
* Rendering only `err.stack || err.message` drops the chain entirely
* because `Error#stack` does NOT include causes — this helper is the
* missing piece.
*
* Guards against pathological or malicious cyclic cause graphs by
* tracking visited references and capping depth — a self-referential
* `.cause` (or a long synthetic chain) can't hang the validator.
*/
function formatErrorChain(err: unknown): string {
const MAX_DEPTH = 16;
const seen = new WeakSet<object>();
const lines: string[] = [];
const render = (e: unknown): string =>
e instanceof Error ? e.stack || e.message : String(e);
let current: unknown = err;
let depth = 0;
let truncated = false;
while (current !== undefined && current !== null && depth < MAX_DEPTH) {
const prefix = depth === 0 ? "" : `${" ".repeat(depth)}caused by: `;
lines.push(
prefix + render(current).replace(/\n/g, `\n${" ".repeat(depth)}`),
);
if (typeof current !== "object") break;
if (seen.has(current as object)) {
lines.push(`${" ".repeat(depth + 1)}[cyclic cause — stopping]`);
break;
}
seen.add(current as object);
const next = (current as { cause?: unknown }).cause;
if (next === undefined) break;
// If we're at the last allowed depth and there IS still a cause to
// follow, the chain is being truncated. Mark it so the emitter
// below fires — but don't emit when the chain ended naturally at
// MAX_DEPTH (cur.cause undefined), which the `break` above handles.
if (depth + 1 >= MAX_DEPTH) {
truncated = true;
break;
}
current = next;
depth++;
}
if (truncated) {
lines.push(
`${" ".repeat(depth + 1)}[cause chain truncated at depth ${MAX_DEPTH}]`,
);
}
return lines.join("\n");
}
// Re-exported for test callers importing these types from
// "../validate-parity.js"; the canonical definitions live in
// ./lib/manifest.ts.
export type { Manifest, ManifestDemo };
/**
* Typed error raised when `manifest.yaml` exists and could be read but
* its contents do not form a valid Manifest (YAML syntax error, wrong
* top-level shape, non-array demos, missing id, etc.). Callers catch
* with `instanceof ManifestMalformedError` — no string-prefix sniffing.
*/
export class ManifestMalformedError extends Error {
constructor(message: string) {
super(message);
this.name = "ManifestMalformedError";
}
}
/**
* Thrown by `auditPackage` when called with an invalid baseline value
* (non-positive, non-integer, NaN, Infinity). Defence-in-depth: the
* CLI wrapper validates via coerceBaseline, but direct programmatic
* callers of auditPackage should also fail fast rather than silently
* comparing against NaN / 0.
*/
export class InvalidBaselineError extends Error {
constructor(message: string) {
super(message);
this.name = "InvalidBaselineError";
}
}
/**
* Typed error raised when `manifest.yaml` exists on disk but
* readFileSync threw (EACCES, I/O race, etc.). Distinct from
* ManifestMalformedError because the file's contents are not actually
* known to be invalid. Callers catch with `instanceof
* ManifestUnreadableError`.
*/
export class ManifestUnreadableError extends Error {
constructor(message: string) {
super(message);
this.name = "ManifestUnreadableError";
}
}
/**
* Tagged union of per-package entries in `mustErrors` / `warnings`. Each
* variant carries only the structured fields the category needs.
* Categories that want a user-facing string render via `deriveMessage`
* at display time so the struct has a single source of truth — no
* pre-formatted `message` field duplicating the structured data.
*/
export type PackageIssue =
| { category: "missing-manifest" }
| { category: "unreadable-manifest"; error: string }
| { category: "malformed-manifest"; error: string }
| { category: "missing-demo-dir"; demoId: string; expectedDir: string }
| { category: "unreadable-demos-dir"; path: string; error: string }
| { category: "unreadable-specs-dir"; path: string; error: string }
| { category: "unreadable-qa-dir"; path: string; error: string }
| { category: "missing-spec"; demoId: string }
| { category: "missing-qa"; demoId: string }
| { category: "baseline-deviation"; demoCount: number; baseline: number }
| {
category: "spec-under-coverage";
specCount: number;
demoCount: number;
}
| { category: "qa-under-coverage"; qaCount: number; demoCount: number }
| { category: "listing-failed"; path: string; error: string }
| { category: "crashed"; error: string };
/**
* Render a PackageIssue as a user-facing string. Kept as the single
* source of truth for issue rendering so the stderr emitter, JSON
* summary, and any future consumer produce identical text.
*
* Using a discriminated union + exhaustive switch keeps this
* future-proof — adding a new PackageIssue variant without a matching
* case here is a TypeScript error.
*/
export function deriveMessage(issue: PackageIssue): string {
switch (issue.category) {
case "missing-manifest":
return "missing manifest.yaml";
case "unreadable-manifest":
return `unreadable manifest.yaml: ${issue.error}`;
case "malformed-manifest":
return `unparseable manifest.yaml: ${issue.error}`;
case "missing-demo-dir":
// Include both the catalog id AND the expected on-disk directory
// because they can differ: `id` is the catalog key (matched
// against qa/spec filenames) while the on-disk directory comes
// from `demo.route` ("/demos/<dir>" → `src/app/demos/<dir>/`).
// Rendering only the id here hid route/id mismatches — operators
// saw the error but not the path the validator actually probed.
// Mention both the new per-column layout (<pkg>/demos/<dir>/) and
// the legacy shared-tree layout (<pkg>/src/app/demos/<dir>/) since
// auditPackage unions both locations when checking existence.
return issue.demoId === issue.expectedDir
? `demo '${issue.demoId}' declared in manifest but no demos/${issue.expectedDir}/ (or legacy src/app/demos/${issue.expectedDir}/) directory`
: `demo '${issue.demoId}' declared in manifest but no demos/${issue.expectedDir}/ (or legacy src/app/demos/${issue.expectedDir}/) directory (resolved from route)`;
case "unreadable-demos-dir":
return `unreadable demos directory: failed to read directory ${issue.path}: ${issue.error}`;
case "unreadable-specs-dir":
return `unreadable specs directory: failed to read directory ${issue.path}: ${issue.error}`;
case "unreadable-qa-dir":
return `unreadable qa directory: failed to read directory ${issue.path}: ${issue.error}`;
case "missing-spec":
return `demo '${issue.demoId}' has no tests/e2e/${issue.demoId}.spec.ts`;
case "missing-qa":
return `demo '${issue.demoId}' has no qa/${issue.demoId}.md`;
case "baseline-deviation":
return `demo count ${issue.demoCount} deviates from baseline ${issue.baseline}`;
case "spec-under-coverage":
return `spec count ${issue.specCount} < demo count ${issue.demoCount}`;
case "qa-under-coverage":
return `qa count ${issue.qaCount} < demo count ${issue.demoCount}`;
case "listing-failed":
return `failed to read directory ${issue.path}: ${issue.error}`;
case "crashed":
return `audit crashed: ${issue.error}`;
}
}
export interface PackageReport {
readonly slug: string;
// All arrays are `readonly` uniformly: surface any accidental post-
// return push/splice as a compile error. Callers that need a mutable
// copy should clone. Marking mustErrors/warnings readonly while
// leaving demoIds/specFiles/qaFiles/demoDirs mutable would be an
// asymmetric contract and invites drift.
readonly demoIds: readonly string[];
readonly specFiles: readonly string[];
readonly qaFiles: readonly string[];
readonly demoDirs: readonly string[];
readonly mustErrors: readonly PackageIssue[];
readonly warnings: readonly PackageIssue[];
}
/**
* Return shape for `listDirs` / `listFiles`. The tuple keeps entries and
* per-listing warnings correlated without a side-effect parameter —
* callers merge `warnings` into their PackageReport explicitly.
*/
export interface ListResult {
entries: string[];
warnings: PackageIssue[];
}
/**
* Best-effort "does this path exist and is it readable" probe used by
* listDirs/listFiles to distinguish genuinely-missing paths (ENOENT
* → silent, return empty) from permission/I/O failures (EACCES et al
* → surface as listing-failed so the caller can escalate).
*
* `fs.existsSync` CONFLATES these: it returns false for ENOENT AND for
* EACCES (and every other statSync failure), so a package whose
* tests/e2e/ is chmod 0 silently registers as "no tests/e2e/" and the
* whole per-slug cascade gets suppressed. That's the bug this probe
* exists to close — do NOT replace with existsSync.
*
* ENOTDIR handling: ENOTDIR from statSync means "a component of the
* path is a regular file, not a directory" (e.g. stray file committed
* at integrations/foo/tests so walking to integrations/foo/tests/e2e fails).
* That is a misconfiguration signal, NOT a legitimately-absent
* directory. Classifying it as `missing` silently drops the whole
* subtree from parity checks with zero diagnostic. Instead, surface
* it as `unreadable` so the caller emits a listing-failed warning
* and callers upstream can escalate (unreadable-demos-dir / etc.).
*
* Return value:
* { kind: "missing" } — ENOENT only
* { kind: "ok" } — stat succeeded (target may be dir or file; caller decides)
* { kind: "unreadable"; error: string } — ENOTDIR or any other statSync failure
*/
type ProbeResult =
| { kind: "missing" }
| { kind: "ok" }
| { kind: "unreadable"; error: string };
function probeDir(p: string): ProbeResult {
try {
// Deliberately do NOT check isDirectory() here — a non-directory
// at the path (regular file / socket / etc.) should flow through
// to the caller's readdirSync, which will throw ENOTDIR and land
// in the existing listing-failed / "could not read" path. Doing
// the isDirectory check here would reclassify "file at
// integrations/" as "missing" and produce a confusing "not found"
// diagnostic when the real problem is ENOTDIR.
fs.statSync(p);
return { kind: "ok" };
} catch (err) {
const code = (err as NodeJS.ErrnoException)?.code;
if (code === "ENOENT") return { kind: "missing" };
// ENOTDIR (a path component is a regular file) is a
// misconfiguration, not a legitimately-absent path — surface it
// as unreadable so it becomes a listing-failed warning rather
// than a silent empty result.
const msg = err instanceof Error ? err.message : String(err);
return { kind: "unreadable", error: msg };
}
}
/**
* List subdirectories of `p`. Non-existent paths (ENOENT) return
* { entries: [], warnings: [] }. Read/permission errors (EACCES, I/O,
* etc.) return empty entries AND a `listing-failed` warning so the
* caller can include it in the PackageReport's `warnings` array. The
* caller is responsible for emitting the stderr `[WARN]` line via
* deriveMessage — helpers do NOT log directly, otherwise operators
* see a duplicated warning line (once from the helper, once from the
* caller's iteration over `warnings[]`).
*
* NB: uses `statSync` (not `existsSync`) to distinguish ENOENT from
* EACCES — see probeDir docstring for why that matters.
*/
export function listDirs(p: string): ListResult {
const probe = probeDir(p);
if (probe.kind === "missing") return { entries: [], warnings: [] };
if (probe.kind === "unreadable") {
const issue: PackageIssue = {
category: "listing-failed",
path: p,
error: probe.error,
};
return { entries: [], warnings: [issue] };
}
try {
const entries = fs
.readdirSync(p, { withFileTypes: true })
.filter((d) => d.isDirectory())
.map((d) => d.name)
.sort();
return { entries, warnings: [] };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
const issue: PackageIssue = {
category: "listing-failed",
path: p,
error: msg,
};
return { entries: [], warnings: [issue] };
}
}
/**
* List files in `p` with the given suffix. Same error-handling contract
* as listDirs: ENOENT → empty ListResult; EACCES / other stat failure
* → empty entries + listing-failed warning. Caller emits the stderr
* `[WARN]` line.
*
* Bare-suffix filenames (e.g. a file literally named `.spec.ts` or
* `.md`) are silently skipped: after stripping the suffix they would
* map to an empty demo-id and could accidentally match a declared demo
* on the empty-string side of the Set comparison. Such files aren't a
* legitimate package-layout artefact, so dropping them is quieter and
* safer than warning about them.
*
* NB: uses `statSync` (not `existsSync`) to distinguish ENOENT from
* EACCES — see probeDir docstring.
*/
export function listFiles(p: string, suffix: string): ListResult {
const probe = probeDir(p);
if (probe.kind === "missing") return { entries: [], warnings: [] };
if (probe.kind === "unreadable") {
const issue: PackageIssue = {
category: "listing-failed",
path: p,
error: probe.error,
};
return { entries: [], warnings: [issue] };
}
try {
const entries = fs
.readdirSync(p, { withFileTypes: true })
.filter((d) => {
if (!d.isFile()) return false;
if (!d.name.endsWith(suffix)) return false;
// Reject files whose entire name IS the suffix (stem length 0).
return d.name.length > suffix.length;
})
.map((d) => d.name)
.sort();
return { entries, warnings: [] };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
const issue: PackageIssue = {
category: "listing-failed",
path: p,
error: msg,
};
return { entries: [], warnings: [issue] };
}
}
/**
* Load and parse a package's manifest.yaml.
*
* Returns:
* - null if the file does not exist ("missing manifest" — caller flags).
* - the parsed Manifest on success.
*
* Throws:
* - `ManifestMalformedError` — file exists but YAML shape is invalid;
* - `ManifestUnreadableError` — readFileSync failed (permissions, I/O
* race).
* Callers discriminate with `instanceof` (see auditPackage).
*
* Delegates to lib/manifest.ts :: parseManifest for shape validation so
* audit.ts / validate-parity.ts / capture-previews.ts apply identical rules.
*/
export function loadManifest(
slug: string,
packagesDir: string = DEFAULT_PACKAGES_DIR,
): Manifest | null {
const manifestPath = path.join(packagesDir, slug, "manifest.yaml");
// Pass the directory slug to parseManifest so its slug-mismatch
// guard fires: if the manifest's `slug:` field disagrees with the
// directory on disk, we get a shape-malformed result instead of
// silently validating a copy-paste / rename mistake.
const parsed = parseManifest(manifestPath, slug);
switch (parsed.kind) {
case "missing":
return null;
case "ok":
return parsed.manifest;
case "malformed":
throw new ManifestMalformedError(parsed.error);
case "unreadable":
throw new ManifestUnreadableError(parsed.error);
}
}
export function auditPackage(
slug: string,
packagesDir: string = DEFAULT_PACKAGES_DIR,
baselineDemoCount: number = BASELINE_DEMO_COUNT,
): PackageReport {
// Defence-in-depth: runParity validates baseline via coerceBaseline,
// but auditPackage is exported for direct use by tests / future
// callers. A NaN / 0 / negative / non-integer baseline silently
// produces nonsense `demoIds.length !== baseline` warnings otherwise.
if (
typeof baselineDemoCount !== "number" ||
!Number.isFinite(baselineDemoCount) ||
!Number.isInteger(baselineDemoCount) ||
baselineDemoCount <= 0
) {
throw new InvalidBaselineError(
`baselineDemoCount must be a positive integer, got ${String(baselineDemoCount)}`,
);
}
const pkgDir = path.join(packagesDir, slug);
const mustErrors: PackageIssue[] = [];
const warnings: PackageIssue[] = [];
// Pre-compute spec/qa/demo-dir listings up-front so the reporter row
// still shows accurate counts even if manifest parsing fails. MUST
// errors still gate the exit code — this only affects the table.
//
// Demos can live at either <pkg>/demos/<cell>/ (per-column container
// layout with one folder per cell containing frontend/ + backend/) OR
// <pkg>/src/app/demos/<cell>/ (legacy shared-tree layout). Union the
// two listings so the check is permissive during the transition — a
// cell declared in manifest passes as long as either directory exists.
const specResult = listFiles(path.join(pkgDir, "tests", "e2e"), ".spec.ts");
const qaResult = listFiles(path.join(pkgDir, "qa"), ".md");
const topLevelDemosDir = path.join(pkgDir, "demos");
const legacyDemosDir = path.join(pkgDir, "src", "app", "demos");
const topLevelDemoDirResult = listDirs(topLevelDemosDir);
const legacyDemoDirResult = listDirs(legacyDemosDir);
const demoDirResult: ListResult = {
entries: Array.from(
new Set([
...topLevelDemoDirResult.entries,
...legacyDemoDirResult.entries,
]),
),
// Forward ALL listing-failed warnings from both probes regardless of
// whether the other probe returned entries. An EACCES on one path
// must not be silently swallowed just because the other path
// happened to be readable — callers rely on these warnings to
// elevate to unreadable-demos-dir MUSTs. A genuinely-missing path
// (ENOENT) produces no warning from listDirs, so this does not add
// noise for columns using only one of the two layouts.
warnings: [
...topLevelDemoDirResult.warnings,
...legacyDemoDirResult.warnings,
],
};
const specFiles = specResult.entries;
const qaFiles = qaResult.entries;
const demoDirs = demoDirResult.entries;
// When src/app/demos/, tests/e2e/, or qa/ is unreadable, elevate to a
// MUST error under category "unreadable-{demos,specs,qa}-dir" and
// SUPPRESS the downstream missing-{demo-dir,spec,qa} cascade so the
// EACCES root cause isn't buried. We detect this by checking whether
// the respective list call returned a listing-failed warning for the
// target path specifically (not an unrelated path).
// For the unreadable-demos-dir elevation, accept a listing-failed
// warning at EITHER candidate path (new top-level demos/ or legacy
// src/app/demos/). Report the failing path verbatim so the operator
// sees which location couldn't be read.
const specsDirPath = path.join(pkgDir, "tests", "e2e");
const qaDirPath = path.join(pkgDir, "qa");
const findListingFailed = (
result: ListResult,
target: string,
): Extract<PackageIssue, { category: "listing-failed" }> | undefined =>
result.warnings.find(
(w): w is Extract<PackageIssue, { category: "listing-failed" }> =>
w.category === "listing-failed" && w.path === target,
);
const topLevelDemosUnreadable = findListingFailed(
demoDirResult,
topLevelDemosDir,
);
const legacyDemosUnreadable = findListingFailed(
demoDirResult,
legacyDemosDir,
);
// Collect all demos-dir listing failures so BOTH paths are surfaced on
// the MUST when both fail (previously only the first match was
// reported, hiding the second EACCES root cause).
const demosDirUnreadableAll: Extract<
PackageIssue,
{ category: "listing-failed" }
>[] = [];
if (topLevelDemosUnreadable)
demosDirUnreadableAll.push(topLevelDemosUnreadable);
if (legacyDemosUnreadable) demosDirUnreadableAll.push(legacyDemosUnreadable);
const demosDirUnreadable = demosDirUnreadableAll[0];
const specsDirUnreadable = findListingFailed(specResult, specsDirPath);
const qaDirUnreadable = findListingFailed(qaResult, qaDirPath);
// Merge listing warnings only when the underlying path was NOT elevated
// to a MUST error. For the elevated paths, push the typed MUST variant
// so consumers see a single authoritative signal instead of a warning
// plus N cascaded per-demo errors.
if (!specsDirUnreadable) {
warnings.push(...specResult.warnings);
} else {
mustErrors.push({
category: "unreadable-specs-dir",
path: specsDirPath,
error: specsDirUnreadable.error,
});
}
if (!qaDirUnreadable) {
warnings.push(...qaResult.warnings);
} else {
mustErrors.push({
category: "unreadable-qa-dir",
path: qaDirPath,
error: qaDirUnreadable.error,
});
}
if (demosDirUnreadableAll.length === 0) {
warnings.push(...demoDirResult.warnings);
} else {
// Emit one unreadable-demos-dir MUST per failing path — if BOTH the
// top-level and legacy demos/ directories EACCES'd, report both so
// operators see every root cause rather than just the first.
for (const failing of demosDirUnreadableAll) {
mustErrors.push({
category: "unreadable-demos-dir",
path: failing.path,
error: failing.error,
});
}
}
let manifest: Manifest | null;
try {
manifest = loadManifest(slug, packagesDir);
} catch (err) {
// Distinguish unreadable from malformed via typed error classes —
// parseManifest is the single source of truth for the classification,
// loadManifest wraps that in ManifestMalformedError /
// ManifestUnreadableError, and we match on the class here.
if (err instanceof ManifestUnreadableError) {
mustErrors.push({
category: "unreadable-manifest",
error: err.message,
});
} else if (err instanceof ManifestMalformedError) {
mustErrors.push({
category: "malformed-manifest",
error: err.message,
});
} else {
// Unknown error class (TypeError, OOM, bug surfacing from a
// future loadManifest refactor, etc.) — re-throw so the CLI's
// top-level catch surfaces [INTERNAL ERROR] with EXIT_INTERNAL.
// Silently bucketing these as malformed-manifest would hide real
// defects behind a legitimate-looking taxonomy entry. Wrap with
// the package slug so operators see which package triggered the
// crash; the original error rides along via `cause` so stacks /
// errno / etc. remain inspectable.
// Outer message is context-only — formatErrorChain unfurls `cause`
// so including the inner message here would render it twice.
throw new Error(`audit of ${slug} crashed`, { cause: err });
}
// Don't early-return: we still return the report with spec/qa/demo
// dir counts populated so the table row is accurate. MUST error
// already gates the exit code.
return {
slug,
demoIds: [],
specFiles,
qaFiles,
demoDirs,
mustErrors,
warnings,
};
}
if (!manifest) {
mustErrors.push({ category: "missing-manifest" });
return {
slug,
demoIds: [],
specFiles,
qaFiles,
demoDirs,
mustErrors,
warnings,
};
}
// Shape validation (top-level mapping, demos array-of-objects-with-id,
// etc.) is performed by parseManifest in ./lib/manifest.ts. By the
// time we reach this point, `manifest.demos` is guaranteed to be
// `readonly ManifestDemo[]` (parseManifest normalizes missing /
// null-valued `demos:` to a shared EMPTY_DEMOS sentinel), so no
// nullish fallback is needed here — dropping the `?? []` keeps the
// Manifest contract single-sourced at parseManifest.
const demos = manifest.demos;
// Informational-only demos (e.g. cli-start with a `command:` field)
// live in the registry but have no on-disk folder to audit. They're
// identified by the `command` field; exclude them from parity checks.
const auditableDemos = demos.filter(
(d) => !(d as { command?: string }).command,
);
const demoIds = auditableDemos.map((d) => d.id);
const demoDirSet = new Set(demoDirs);
const specIdSet = new Set(specFiles.map((f) => f.replace(/\.spec\.ts$/, "")));
const qaIdSet = new Set(qaFiles.map((f) => f.replace(/\.md$/, "")));
// MUST: every declared demo has a demos/<dir>/ directory. The
// expected directory is resolved from `demo.route` ("/demos/<dir>")
// when present, falling back to `demo.id` when absent. `id` is the
// CATALOG identifier (matched against qa/spec filenames); `route`
// is the URL + filesystem path. They are DELIBERATELY separate — a
// manifest with `id: hitl-in-chat` and `route: /demos/hitl` resolves
// to `src/app/demos/hitl/`. bundle-demo-content.ts uses the same
// idiom. Suppressed entirely when the demos/ dir itself is unreadable
// — a single unreadable-demos-dir MUST is clearer than N cascaded
// missing-demo-dir errors that all trace to the same EACCES root cause.
if (!demosDirUnreadable) {
for (const demo of auditableDemos) {
const expectedDir = routeToDirName(demo.route) ?? demo.id;
if (!demoDirSet.has(expectedDir)) {
mustErrors.push({
category: "missing-demo-dir",
demoId: demo.id,
expectedDir,
});
}
}
}
// SHOULD: every declared demo has a spec file. Suppressed when the
// tests/e2e/ dir itself is unreadable — a single unreadable-specs-dir
// MUST is clearer than N cascaded missing-spec warnings that all trace
// to the same EACCES root cause.
if (!specsDirUnreadable) {
for (const id of demoIds) {
if (!specIdSet.has(id)) {
warnings.push({ category: "missing-spec", demoId: id });
}
}
}
// SHOULD: every declared demo has a QA doc. Suppressed when qa/ is
// unreadable — see unreadable-qa-dir rationale above.
if (!qaDirUnreadable) {
for (const id of demoIds) {
if (!qaIdSet.has(id)) {
warnings.push({ category: "missing-qa", demoId: id });
}
}
}
// SHOULD: demo count matches baseline
if (demoIds.length !== baselineDemoCount) {
warnings.push({
category: "baseline-deviation",
demoCount: demoIds.length,
baseline: baselineDemoCount,
});
}
// SHOULD: spec count >= demo count. Spec count EXCEEDING demo count is
// legitimate (e.g. a cross-demo spec covers renderer selection for
// multiple demos and is intentionally not tied to a single declared
// demo), so we only warn on UNDER-coverage. Suppressed when the
// tests/e2e/ dir itself is unreadable — the elevated
// `unreadable-specs-dir` MUST is the authoritative signal; a spurious
// "0 < N" coverage warning on top of it would bury the EACCES root
// cause, same rationale as the per-demo missing-spec suppression above.
if (!specsDirUnreadable && specFiles.length < demoIds.length) {
warnings.push({
category: "spec-under-coverage",
specCount: specFiles.length,
demoCount: demoIds.length,
});
}
// SHOULD: qa count >= demo count. Suppressed when qa/ is unreadable —
// see spec-under-coverage rationale above.
if (!qaDirUnreadable && qaFiles.length < demoIds.length) {
warnings.push({
category: "qa-under-coverage",
qaCount: qaFiles.length,
demoCount: demoIds.length,
});
}
return {
slug,
demoIds,
specFiles,
qaFiles,
demoDirs,
mustErrors,
warnings,
};
}
interface MainOptions {
/**
* Override for the expected demo count per package. Must be a positive
* integer (> 0). NaN / non-integer / non-positive values are rejected
* by `main()` / `runParity()` before they reach `auditPackage`.
*/
baseline?: number;
}
/**
* Discriminated union of rejection reasons emitted by `coerceBaseline`.
* Consumers map the reason into a specific diagnostic message so bad
* input is actionable (e.g. "1.5" → float, "0x10" → hex).
*/
export type CoerceBaselineReason =
| "empty"
| "whitespace"
| "zero"
| "negative"
| "float"
| "hex"
| "non-numeric";
export type CoerceBaselineResult =
| { ok: true; value: number }
| { ok: false; reason: CoerceBaselineReason };
/**
* Validate a candidate baseline value. Returns a discriminated union so
* callers can surface a specific reason in user-facing error messages
* (distinguishing e.g. "1.5" from "0x10" from "abc" matters when the
* CI operator has to guess what they typed wrong).
*
* Used to guard both `--baseline=N` CLI parsing and
* `VALIDATE_PARITY_BASELINE` env-var parsing.
*/
export function coerceBaseline(raw: unknown): CoerceBaselineResult {
if (typeof raw === "number") {
if (!Number.isFinite(raw)) return { ok: false, reason: "non-numeric" };
if (!Number.isInteger(raw)) return { ok: false, reason: "float" };
if (raw === 0) return { ok: false, reason: "zero" };
if (raw < 0) return { ok: false, reason: "negative" };
return { ok: true, value: raw };
}
if (typeof raw !== "string") return { ok: false, reason: "non-numeric" };
if (raw.length === 0) return { ok: false, reason: "empty" };
const trimmed = raw.trim();
if (trimmed.length === 0) return { ok: false, reason: "whitespace" };
// Distinguish specific bad shapes so the error message can be clearer.
if (/^0x[0-9a-fA-F]+$/.test(trimmed)) return { ok: false, reason: "hex" };
if (/^-\d+$/.test(trimmed)) return { ok: false, reason: "negative" };
if (/^-?\d+\.\d+$/.test(trimmed)) return { ok: false, reason: "float" };
if (trimmed === "0" || /^0+$/.test(trimmed))
return { ok: false, reason: "zero" };
// Strict digits-only: rejects leading +, exponent (1e2), leading 0,
// and anything else Number() would otherwise coerce.
if (!/^[1-9]\d*$/.test(trimmed)) return { ok: false, reason: "non-numeric" };
const n = Number(trimmed);
if (!Number.isFinite(n) || !Number.isInteger(n)) {
return { ok: false, reason: "non-numeric" };
}
if (n === 0) return { ok: false, reason: "zero" };
if (n < 0) return { ok: false, reason: "negative" };
return { ok: true, value: n };
}
function parseMainArgs(argv: string[]): MainOptions {
const opts: MainOptions = {};
// Collect all parse errors, mirroring audit.ts parseArgs. Unrecognised
// arguments (typos like `--basline=10`, space-separated `--baseline 9`,
// stray positionals) are flagged loudly instead of being silently
// ignored — otherwise the user thinks they set baseline but the
// validator uses the default.
//
// Track `sawBaseline` and reject duplicate --baseline=. CI shell
// concatenation is a common source of accidental duplicates and
// "last wins" silently hides the user's first intent — mirror
// audit.ts parseArgs which rejects duplicate --json / --slug /
// --strict / --columns for the same reason.
const errors: string[] = [];
let sawBaseline = false;
for (const a of argv) {
// Match anything after --baseline= (including non-digits) so we can
// emit a clear error, rather than silently ignoring e.g.
// `--baseline=abc`.
const m = /^--baseline=(.*)$/.exec(a);
if (m) {
if (sawBaseline) {
errors.push(
`--baseline specified more than once (duplicate value "${m[1]}")`,
);
continue;
}
sawBaseline = true;
const coerced = coerceBaseline(m[1]);
if (!coerced.ok) {
errors.push(
`invalid --baseline value "${m[1]}" (${coerced.reason}; expected a positive integer)`,
);
continue;
}
opts.baseline = coerced.value;
} else {
errors.push(`unrecognised argument: ${a}`);
}
}
if (errors.length > 0) {
// Join all errors so the user sees every problem at once, rather
// than fixing them one at a time across reruns. Matches audit.ts's
// error-collection pattern.
throw new Error(errors.join("; "));
}
return opts;
}
/**
* Column spec for the pass/summary table. Both header and data rows
* are rendered from this single array so widths can never drift — add
* or remove a column here and `buildHeader` / `formatRow` both update
* in lockstep.
*
* `width` is the minimum display width for the column. The first
* column ("package") uses a runtime-derived slug width instead.
*
* `align`: "left" → padEnd, "right" → padStart. Numeric columns align
* right so counts line up visually; string labels align left.
*
* `render(report)` projects a PackageReport into its cell string. The
* same cells are emitted for the header (labels padded to `width`) so
* both sides are character-for-character identical width per column.
*/
export interface HeaderColumn {
readonly label: string;
readonly width: number;
readonly align: "left" | "right";
readonly render: (r: PackageReport) => string;
}
export const HEADER_COLUMNS: readonly HeaderColumn[] = [
{
label: "package",
width: 0 /* derived from slugs at runtime */,
align: "left",
render: (r) => r.slug,
},
{
label: "status",
width: 6,
align: "left",
render: (r) => (r.mustErrors.length > 0 ? "[FAIL]" : "[PASS]"),
},
{
label: "demos",
width: 5,
align: "right",
render: (r) => String(r.demoIds.length),
},
{
label: "specs",
width: 5,
align: "right",
render: (r) => String(r.specFiles.length),
},
{
label: "qa",
width: 3,
align: "right",
render: (r) => String(r.qaFiles.length),
},
{
label: "notes",
width: 10,
align: "left",
render: (r) => (r.warnings.length > 0 ? `${r.warnings.length} warn` : ""),
},
] as const;