Skip to content

Commit 6b299e0

Browse files
TayDa64TayDa64
committed
feat(phase4): event-driven UI watcher with .NET UIA events
Layer 1 – .NET host (Program.cs): - Thread-safe Reply() with lock(_writeLock) - subscribeEvents/unsubscribeEvents commands - FocusChanged, StructureChanged, PropertyChanged event handlers - Debounce timers (50ms property, 100ms structure, adaptive 200ms backoff) - BuildLightElement matching PowerShell watcher format exactly - WalkFocusedWindowElements for initial snapshot + structure refresh - AttachToWindow/DetachFromWindow for per-window event subscription - FindTopLevelWindow walks up from focused element Layer 2 – UIAHost (uia-host.js): - Event routing: check json.type==='event' before _resolvePending - Emit 'uia-event' for unsolicited event messages - subscribeEvents()/unsubscribeEvents() convenience methods Layer 3 – UIWatcher (ui-watcher.js): - MODE state machine: POLLING → STARTING_EVENTS → EVENT_MODE → FALLBACK - startEventMode()/stopEventMode() lifecycle methods - _onUiaEvent handles focusChanged/structureChanged/propertyChanged - Property changes merged into cache via Map-based patching - Health check: 10s no-event timeout → fallback to polling - Auto-retry: 30s in fallback → attempt event mode again - emits 'poll-complete' from events (source: event-*) for downstream compat Layer 4 – index.js: - Inspect mode toggle calls startEventMode/stopEventMode 222 smoke assertions (55 new for Phase 4), 0 failures
1 parent 5dc5b1e commit 6b299e0

6 files changed

Lines changed: 936 additions & 2 deletions

File tree

‎.github/hooks/logs/tool-audit.jsonl‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -600,3 +600,91 @@
600600
{"timestamp":"2026-02-27T12:03:22.179Z","tool":null,"result":null}
601601
{"timestamp":"2026-02-27T12:03:32.090Z","tool":null,"result":null}
602602
{"timestamp":"2026-02-27T12:03:50.291Z","tool":null,"result":null}
603+
{"timestamp":"2026-02-27T12:03:58.770Z","tool":null,"result":null}
604+
{"timestamp":"2026-02-27T12:04:03.366Z","tool":null,"result":null}
605+
{"timestamp":"2026-02-27T12:04:07.834Z","tool":null,"result":null}
606+
{"timestamp":"2026-02-27T12:04:33.394Z","tool":null,"result":null}
607+
{"timestamp":"2026-02-27T12:04:33.443Z","tool":null,"result":null}
608+
{"timestamp":"2026-02-27T12:04:33.463Z","tool":null,"result":null}
609+
{"timestamp":"2026-02-27T12:04:33.479Z","tool":null,"result":null}
610+
{"timestamp":"2026-02-27T12:04:41.431Z","tool":null,"result":null}
611+
{"timestamp":"2026-02-27T12:04:41.548Z","tool":null,"result":null}
612+
{"timestamp":"2026-02-27T12:04:41.557Z","tool":null,"result":null}
613+
{"timestamp":"2026-02-27T12:04:41.585Z","tool":null,"result":null}
614+
{"timestamp":"2026-02-27T12:04:41.631Z","tool":null,"result":null}
615+
{"timestamp":"2026-02-27T12:04:49.847Z","tool":null,"result":null}
616+
{"timestamp":"2026-02-27T12:04:49.865Z","tool":null,"result":null}
617+
{"timestamp":"2026-02-27T12:04:49.883Z","tool":null,"result":null}
618+
{"timestamp":"2026-02-27T12:04:49.956Z","tool":null,"result":null}
619+
{"timestamp":"2026-02-27T12:04:49.995Z","tool":null,"result":null}
620+
{"timestamp":"2026-02-27T12:04:57.627Z","tool":null,"result":null}
621+
{"timestamp":"2026-02-27T12:04:57.659Z","tool":null,"result":null}
622+
{"timestamp":"2026-02-27T12:04:57.730Z","tool":null,"result":null}
623+
{"timestamp":"2026-02-27T12:05:08.430Z","tool":null,"result":null}
624+
{"timestamp":"2026-02-27T12:05:08.452Z","tool":null,"result":null}
625+
{"timestamp":"2026-02-27T12:05:13.895Z","tool":null,"result":null}
626+
{"timestamp":"2026-02-27T12:05:13.911Z","tool":null,"result":null}
627+
{"timestamp":"2026-02-27T12:05:19.516Z","tool":null,"result":null}
628+
{"timestamp":"2026-02-27T12:05:19.548Z","tool":null,"result":null}
629+
{"timestamp":"2026-02-27T12:05:25.078Z","tool":null,"result":null}
630+
{"timestamp":"2026-02-27T12:05:25.155Z","tool":null,"result":null}
631+
{"timestamp":"2026-02-27T12:05:30.218Z","tool":null,"result":null}
632+
{"timestamp":"2026-02-27T12:05:33.990Z","tool":null,"result":null}
633+
{"timestamp":"2026-02-27T12:05:49.257Z","tool":null,"result":null}
634+
{"timestamp":"2026-02-27T12:07:30.827Z","tool":null,"result":null}
635+
{"timestamp":"2026-02-27T12:07:36.522Z","tool":null,"result":null}
636+
{"timestamp":"2026-02-27T12:07:41.186Z","tool":null,"result":null}
637+
{"timestamp":"2026-02-27T12:07:45.826Z","tool":null,"result":null}
638+
{"timestamp":"2026-02-27T12:07:51.023Z","tool":null,"result":null}
639+
{"timestamp":"2026-02-27T12:13:21.340Z","tool":null,"result":null}
640+
{"timestamp":"2026-02-27T12:13:25.334Z","tool":null,"result":null}
641+
{"timestamp":"2026-02-27T12:13:29.208Z","tool":null,"result":null}
642+
{"timestamp":"2026-02-27T12:13:33.009Z","tool":null,"result":null}
643+
{"timestamp":"2026-02-27T12:13:38.688Z","tool":null,"result":null}
644+
{"timestamp":"2026-02-27T12:13:42.506Z","tool":null,"result":null}
645+
{"timestamp":"2026-02-27T12:13:47.357Z","tool":null,"result":null}
646+
{"timestamp":"2026-02-27T12:13:51.349Z","tool":null,"result":null}
647+
{"timestamp":"2026-02-27T12:13:56.247Z","tool":null,"result":null}
648+
{"timestamp":"2026-02-27T12:14:00.653Z","tool":null,"result":null}
649+
{"timestamp":"2026-02-27T12:14:05.941Z","tool":null,"result":null}
650+
{"timestamp":"2026-02-27T12:14:10.181Z","tool":null,"result":null}
651+
{"timestamp":"2026-02-27T12:14:14.048Z","tool":null,"result":null}
652+
{"timestamp":"2026-02-27T12:16:28.180Z","tool":null,"result":null}
653+
{"timestamp":"2026-02-27T12:26:34.744Z","tool":null,"result":null}
654+
{"timestamp":"2026-02-27T12:26:39.241Z","tool":null,"result":null}
655+
{"timestamp":"2026-02-27T12:26:42.834Z","tool":null,"result":null}
656+
{"timestamp":"2026-02-27T12:26:56.144Z","tool":null,"result":null}
657+
{"timestamp":"2026-02-27T12:27:04.997Z","tool":null,"result":null}
658+
{"timestamp":"2026-02-27T12:27:58.950Z","tool":null,"result":null}
659+
{"timestamp":"2026-02-27T12:28:05.186Z","tool":null,"result":null}
660+
{"timestamp":"2026-02-27T12:28:22.353Z","tool":null,"result":null}
661+
{"timestamp":"2026-02-27T12:28:25.560Z","tool":null,"result":null}
662+
{"timestamp":"2026-02-27T12:28:36.740Z","tool":null,"result":null}
663+
{"timestamp":"2026-02-27T12:28:41.774Z","tool":null,"result":null}
664+
{"timestamp":"2026-02-27T12:29:04.126Z","tool":null,"result":null}
665+
{"timestamp":"2026-02-27T12:29:11.425Z","tool":null,"result":null}
666+
{"timestamp":"2026-02-27T12:29:15.287Z","tool":null,"result":null}
667+
{"timestamp":"2026-02-27T12:29:28.754Z","tool":null,"result":null}
668+
{"timestamp":"2026-02-27T12:29:32.709Z","tool":null,"result":null}
669+
{"timestamp":"2026-02-27T12:29:37.633Z","tool":null,"result":null}
670+
{"timestamp":"2026-02-27T12:29:41.168Z","tool":null,"result":null}
671+
{"timestamp":"2026-02-27T12:29:44.423Z","tool":null,"result":null}
672+
{"timestamp":"2026-02-27T12:29:47.775Z","tool":null,"result":null}
673+
{"timestamp":"2026-02-27T12:30:04.864Z","tool":null,"result":null}
674+
{"timestamp":"2026-02-27T12:30:38.066Z","tool":null,"result":null}
675+
{"timestamp":"2026-02-27T12:30:43.608Z","tool":null,"result":null}
676+
{"timestamp":"2026-02-27T12:30:48.325Z","tool":null,"result":null}
677+
{"timestamp":"2026-02-27T12:30:51.749Z","tool":null,"result":null}
678+
{"timestamp":"2026-02-27T12:30:55.788Z","tool":null,"result":null}
679+
{"timestamp":"2026-02-27T12:31:05.538Z","tool":null,"result":null}
680+
{"timestamp":"2026-02-27T12:31:09.833Z","tool":null,"result":null}
681+
{"timestamp":"2026-02-27T12:31:15.577Z","tool":null,"result":null}
682+
{"timestamp":"2026-02-27T12:31:18.724Z","tool":null,"result":null}
683+
{"timestamp":"2026-02-27T12:31:23.224Z","tool":null,"result":null}
684+
{"timestamp":"2026-02-27T12:31:26.685Z","tool":null,"result":null}
685+
{"timestamp":"2026-02-27T12:31:30.242Z","tool":null,"result":null}
686+
{"timestamp":"2026-02-27T12:32:00.842Z","tool":null,"result":null}
687+
{"timestamp":"2026-02-27T12:32:13.214Z","tool":null,"result":null}
688+
{"timestamp":"2026-02-27T12:32:17.775Z","tool":null,"result":null}
689+
{"timestamp":"2026-02-27T12:32:23.309Z","tool":null,"result":null}
690+
{"timestamp":"2026-02-27T12:32:28.874Z","tool":null,"result":null}

‎scripts/smoke-command-system.js‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,99 @@ console.log('\n\x1b[1m[15] Phase 2: element-from-point + stable identity\x1b[0m'
474474
assert('scrollElement falls back to mouseWheel', pa.includes("method: 'mouseWheel'"));
475475
}
476476

477+
// ── [17] Phase 4: Event-driven UI watcher ────────────────────────────────
478+
{
479+
console.log('\n\x1b[1m[17] Phase 4 \u2013 Event-driven UI watcher\x1b[0m');
480+
481+
// ── Layer 1: .NET host event streaming ──
482+
const dotnetPath = path.join(ROOT, 'src', 'native', 'windows-uia-dotnet', 'Program.cs');
483+
const dotnet = fs.readFileSync(dotnetPath, 'utf-8');
484+
485+
// Thread-safe Reply
486+
assert('.NET Reply uses lock(_writeLock)', dotnet.includes('lock (_writeLock)'));
487+
assert('.NET _writeLock is static readonly', dotnet.includes('static readonly object _writeLock'));
488+
489+
// subscribeEvents / unsubscribeEvents commands
490+
assert('.NET host handles subscribeEvents', dotnet.includes('case "subscribeEvents"'));
491+
assert('.NET host handles unsubscribeEvents', dotnet.includes('case "unsubscribeEvents"'));
492+
assert('.NET HandleSubscribeEvents method', dotnet.includes('HandleSubscribeEvents'));
493+
assert('.NET HandleUnsubscribeEvents method', dotnet.includes('HandleUnsubscribeEvents'));
494+
495+
// Event handlers
496+
assert('.NET OnFocusChanged handler', dotnet.includes('OnFocusChanged'));
497+
assert('.NET OnStructureChanged handler', dotnet.includes('OnStructureChanged'));
498+
assert('.NET OnPropertyChanged handler', dotnet.includes('OnPropertyChanged'));
499+
assert('.NET AddAutomationFocusChangedEventHandler', dotnet.includes('AddAutomationFocusChangedEventHandler'));
500+
assert('.NET AddStructureChangedEventHandler', dotnet.includes('AddStructureChangedEventHandler'));
501+
assert('.NET AddAutomationPropertyChangedEventHandler', dotnet.includes('AddAutomationPropertyChangedEventHandler'));
502+
503+
// Event payloads
504+
assert('.NET emits type="event" for focus', dotnet.includes('"focusChanged"'));
505+
assert('.NET emits type="event" for structure', dotnet.includes('"structureChanged"'));
506+
assert('.NET emits type="event" for property', dotnet.includes('"propertyChanged"'));
507+
508+
// BuildLightElement (format-compatible with PS watcher)
509+
assert('.NET BuildLightElement method', dotnet.includes('BuildLightElement'));
510+
assert('.NET WalkFocusedWindowElements method', dotnet.includes('WalkFocusedWindowElements'));
511+
assert('.NET BuildWindowInfo method', dotnet.includes('BuildWindowInfo'));
512+
513+
// Debounce & adaptive backoff
514+
assert('.NET structure debounce timer', dotnet.includes('_structureDebounce'));
515+
assert('.NET property debounce timer', dotnet.includes('_propertyDebounce'));
516+
assert('.NET adaptive backoff (burst detection)', dotnet.includes('_structureEventBurst'));
517+
assert('.NET debounce 200ms backoff', dotnet.includes('_structureDebounceMs = 200'));
518+
519+
// Window tracking & cleanup
520+
assert('.NET AttachToWindow method', dotnet.includes('AttachToWindow'));
521+
assert('.NET DetachFromWindow method', dotnet.includes('DetachFromWindow'));
522+
assert('.NET FindTopLevelWindow method', dotnet.includes('FindTopLevelWindow'));
523+
assert('.NET RemoveFocusChangedEventHandler on unsubscribe', dotnet.includes('RemoveAutomationFocusChangedEventHandler'));
524+
assert('.NET RemoveStructureChangedEventHandler on unsubscribe', dotnet.includes('RemoveStructureChangedEventHandler'));
525+
assert('.NET RemovePropertyChangedEventHandler on unsubscribe', dotnet.includes('RemoveAutomationPropertyChangedEventHandler'));
526+
527+
// ── Layer 2: UIAHost event routing ──
528+
const hostPath = path.join(ROOT, 'src', 'main', 'ui-automation', 'core', 'uia-host.js');
529+
const host = fs.readFileSync(hostPath, 'utf-8');
530+
531+
assert('UIAHost routes events before _resolvePending', host.includes("json.type === 'event'"));
532+
assert('UIAHost emits uia-event', host.includes("this.emit('uia-event', json)"));
533+
assert('UIAHost.subscribeEvents method', host.includes('async subscribeEvents'));
534+
assert('UIAHost.unsubscribeEvents method', host.includes('async unsubscribeEvents'));
535+
assert('UIAHost event routing uses continue to skip pending', host.includes('continue;'));
536+
537+
// ── Layer 3: UIWatcher event mode ──
538+
const watcherPath = path.join(ROOT, 'src', 'main', 'ui-watcher.js');
539+
const watcher = fs.readFileSync(watcherPath, 'utf-8');
540+
541+
assert('UIWatcher imports getSharedUIAHost', watcher.includes("require('./ui-automation/core/uia-host')"));
542+
assert('UIWatcher MODE state enum', watcher.includes("POLLING: 'POLLING'"));
543+
assert('UIWatcher MODE.EVENT_MODE', watcher.includes("EVENT_MODE: 'EVENT_MODE'"));
544+
assert('UIWatcher MODE.FALLBACK', watcher.includes("FALLBACK: 'FALLBACK'"));
545+
assert('UIWatcher MODE.STARTING_EVENTS', watcher.includes("STARTING_EVENTS: 'STARTING_EVENTS'"));
546+
assert('UIWatcher startEventMode method', watcher.includes('async startEventMode'));
547+
assert('UIWatcher stopEventMode method', watcher.includes('async stopEventMode'));
548+
assert('UIWatcher _onUiaEvent handler', watcher.includes('_onUiaEvent'));
549+
assert('UIWatcher handles focusChanged event', watcher.includes("case 'focusChanged'"));
550+
assert('UIWatcher handles structureChanged event', watcher.includes("case 'structureChanged'"));
551+
assert('UIWatcher handles propertyChanged event', watcher.includes("case 'propertyChanged'"));
552+
assert('UIWatcher health check timer (10s)', watcher.includes('10000'));
553+
assert('UIWatcher fallback auto-retry (30s)', watcher.includes('30000'));
554+
assert('UIWatcher emits mode-changed event', watcher.includes("emit('mode-changed'"));
555+
assert('UIWatcher emits poll-complete from events', watcher.includes("source: 'event-structure'"));
556+
assert('UIWatcher emits poll-complete for property patches', watcher.includes("source: 'event-property'"));
557+
assert('UIWatcher propertyChanged merges into cache', watcher.includes('Object.assign(map.get(patch.id), patch)'));
558+
assert('UIWatcher _fallbackToPolling method', watcher.includes('_fallbackToPolling'));
559+
assert('UIWatcher _restartPolling method', watcher.includes('_restartPolling'));
560+
assert('UIWatcher destroy calls stopEventMode', watcher.includes('this.stopEventMode'));
561+
562+
// ── Layer 4: index.js integration ──
563+
const mainJsPath = path.join(ROOT, 'src', 'main', 'index.js');
564+
const mainJs = fs.readFileSync(mainJsPath, 'utf-8');
565+
566+
assert('index.js calls startEventMode on inspect enable', mainJs.includes('startEventMode'));
567+
assert('index.js calls stopEventMode on inspect disable', mainJs.includes('stopEventMode'));
568+
}
569+
477570
// ── Cleanup & Summary ────────────────────────────────────────────────────
478571
cleanup();
479572
// Also remove any screenshot artifacts from root

‎src/main/index.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2069,6 +2069,19 @@ function setupIPC() {
20692069
// Adaptive polling: fast during inspect
20702070
setUIPollingSpeed(newState || overlayMode === 'selection');
20712071

2072+
// Phase 4: switch watcher to event-driven mode during inspect
2073+
if (uiWatcher) {
2074+
if (newState) {
2075+
uiWatcher.startEventMode().catch(err => {
2076+
console.error('[INSPECT] Event mode start failed, polling continues:', err.message);
2077+
});
2078+
} else {
2079+
uiWatcher.stopEventMode().catch(err => {
2080+
console.error('[INSPECT] Event mode stop failed:', err.message);
2081+
});
2082+
}
2083+
}
2084+
20722085
// Notify overlay
20732086
if (overlayWindow && !overlayWindow.isDestroyed()) {
20742087
overlayWindow.webContents.send('inspect-mode-changed', newState);

‎src/main/ui-automation/core/uia-host.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,20 @@ class UIAHost extends EventEmitter {
126126
return resp;
127127
}
128128

129+
/** Subscribe to UIA events (focus, structure, property). Returns initial snapshot. */
130+
async subscribeEvents() {
131+
const resp = await this.send({ cmd: 'subscribeEvents' });
132+
if (!resp.ok) throw new Error(resp.error || 'subscribeEvents failed');
133+
return resp;
134+
}
135+
136+
/** Unsubscribe from all UIA events. */
137+
async unsubscribeEvents() {
138+
const resp = await this.send({ cmd: 'unsubscribeEvents' });
139+
if (!resp.ok) throw new Error(resp.error || 'unsubscribeEvents failed');
140+
return resp;
141+
}
142+
129143
/** Gracefully shut down the host process. */
130144
async stop() {
131145
if (!this._alive || !this._proc) return;
@@ -154,6 +168,11 @@ class UIAHost extends EventEmitter {
154168
if (!line) continue;
155169
try {
156170
const json = JSON.parse(line);
171+
// Phase 4: route unsolicited event messages before pending resolution
172+
if (json.type === 'event') {
173+
this.emit('uia-event', json);
174+
continue;
175+
}
157176
this._resolvePending(json);
158177
} catch (e) {
159178
this.emit('parseError', line, e);

0 commit comments

Comments
 (0)