feat(phase3): pattern-first interaction primitives

- .NET UIA host: 4 new JSONL commands (setValue, scroll, expandCollapse, getText)
  with ResolveElement + GetPatternNames helpers
- Node bridge: 4 new UIAHost convenience methods
- pattern-actions.js: setElementValue, scrollElement, expandElement,
  collapseElement, toggleExpandCollapse, getElementText
- high-level.js: fillField tries ValuePattern first, selectDropdownItem
  tries ExpandCollapsePattern first, both fall back gracefully
- Pattern name normalization (handles both ProgrammaticName + short formats)
- Barrel exports wired through interactions/index.js + ui-automation/index.js
- 152 smoke assertions (41 new), 0 failures

Author: TayDa64

.github/hooks/logs/tool-audit.jsonl

@@ -515,3 +515,62 @@
 {"timestamp":"2026-02-27T11:38:08.110Z","tool":null,"result":null}
 {"timestamp":"2026-02-27T11:38:11.530Z","tool":null,"result":null}
 {"timestamp":"2026-02-27T11:38:17.798Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:39:24.374Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:39:30.052Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:39:46.608Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:39:52.686Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:42:33.440Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:42:37.356Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:14.525Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.270Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.281Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.374Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.375Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.427Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.485Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:42.487Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:51.332Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:51.354Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:51.370Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:57.011Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:47:57.047Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:48:06.604Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:48:06.618Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:48:06.668Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:48:06.671Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:27.613Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:32.032Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:39.222Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:44.242Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:48.749Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:52.864Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:49:57.063Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:50:00.719Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:50:22.446Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:50:57.663Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:51:42.648Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:51:49.352Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:52:34.093Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:52:45.587Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:53:50.953Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:53:58.498Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:54:03.722Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:54:12.692Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:54:17.546Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:54:50.053Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:55:21.749Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:55:28.768Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:55:35.472Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:55:42.258Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:55:46.379Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:56:09.266Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:38.260Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:42.633Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:46.835Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:50.540Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:55.254Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:57:58.845Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:58:02.916Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:58:07.819Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:58:11.826Z","tool":null,"result":null}
+{"timestamp":"2026-02-27T11:58:21.660Z","tool":null,"result":null}

scripts/smoke-command-system.js

@@ -379,6 +379,81 @@ console.log('\n\x1b[1m[15] Phase 2: element-from-point + stable identity\x1b[0m'
   assert('detectRegions maps clickPoint from .NET or PS', isContent.includes('e.clickPoint'));
 }
 
+// ── [16] Phase 3: Pattern-first interaction primitives ───────────────────
+{
+  console.log('\n\x1b[1m[16] Phase 3 \u2013 Pattern-first interaction primitives\x1b[0m');
+
+  // .NET host has all 4 new handlers
+  const dotnetPath = path.join(ROOT, 'src', 'native', 'windows-uia-dotnet', 'Program.cs');
+  const dotnet = fs.readFileSync(dotnetPath, 'utf-8');
+  assert('.NET host handles setValue command', dotnet.includes('case "setValue"'));
+  assert('.NET host handles scroll command', dotnet.includes('case "scroll"'));
+  assert('.NET host handles expandCollapse command', dotnet.includes('case "expandCollapse"'));
+  assert('.NET host handles getText command', dotnet.includes('case "getText"'));
+  assert('.NET HandleSetValue method', dotnet.includes('HandleSetValue'));
+  assert('.NET HandleScroll method', dotnet.includes('HandleScroll'));
+  assert('.NET HandleExpandCollapse method', dotnet.includes('HandleExpandCollapse'));
+  assert('.NET HandleGetText method', dotnet.includes('HandleGetText'));
+  assert('.NET ResolveElement helper', dotnet.includes('ResolveElement'));
+  assert('.NET GetPatternNames helper', dotnet.includes('GetPatternNames'));
+
+  // Node bridge convenience methods
+  const hostPath = path.join(ROOT, 'src', 'main', 'ui-automation', 'core', 'uia-host.js');
+  const host = fs.readFileSync(hostPath, 'utf-8');
+  assert('UIAHost.setValue bridge method', host.includes('async setValue'));
+  assert('UIAHost.scroll bridge method', host.includes('async scroll'));
+  assert('UIAHost.expandCollapse bridge method', host.includes('async expandCollapse'));
+  assert('UIAHost.getText bridge method', host.includes('async getText'));
+
+  // pattern-actions.js exists with all functions
+  const paPath = path.join(ROOT, 'src', 'main', 'ui-automation', 'interactions', 'pattern-actions.js');
+  assert('pattern-actions.js exists', fs.existsSync(paPath));
+  const pa = fs.readFileSync(paPath, 'utf-8');
+  assert('normalizePatternName helper', pa.includes('function normalizePatternName'));
+  assert('hasPattern helper', pa.includes('function hasPattern'));
+  assert('setElementValue function', pa.includes('async function setElementValue'));
+  assert('scrollElement function', pa.includes('async function scrollElement'));
+  assert('expandElement function', pa.includes('async function expandElement'));
+  assert('collapseElement function', pa.includes('async function collapseElement'));
+  assert('toggleExpandCollapse function', pa.includes('async function toggleExpandCollapse'));
+  assert('getElementText function', pa.includes('async function getElementText'));
+  assert('pattern-actions exports all public functions', 
+    pa.includes('setElementValue') && pa.includes('scrollElement') && 
+    pa.includes('expandElement') && pa.includes('collapseElement') && 
+    pa.includes('getElementText') && pa.includes('normalizePatternName'));
+  assert('pattern-actions returns patternUnsupported flag', pa.includes('patternUnsupported'));
+
+  // high-level.js upgraded with pattern-first strategies
+  const hlPath = path.join(ROOT, 'src', 'main', 'ui-automation', 'interactions', 'high-level.js');
+  const hl = fs.readFileSync(hlPath, 'utf-8');
+  assert('fillField imports setElementValue from pattern-actions', hl.includes("require('./pattern-actions')"));
+  assert('fillField tries ValuePattern first', hl.includes('setElementValue') && hl.includes('preferPattern'));
+  assert('selectDropdownItem tries ExpandCollapsePattern first', hl.includes('expandElement') && hl.includes('ExpandCollapsePattern'));
+
+  // Barrel re-exports from interactions/index.js
+  const intIdx = fs.readFileSync(path.join(ROOT, 'src', 'main', 'ui-automation', 'interactions', 'index.js'), 'utf-8');
+  assert('interactions/index re-exports setElementValue', intIdx.includes('setElementValue'));
+  assert('interactions/index re-exports scrollElement', intIdx.includes('scrollElement'));
+  assert('interactions/index re-exports expandElement', intIdx.includes('expandElement'));
+  assert('interactions/index re-exports collapseElement', intIdx.includes('collapseElement'));
+  assert('interactions/index re-exports toggleExpandCollapse', intIdx.includes('toggleExpandCollapse'));
+  assert('interactions/index re-exports getElementText', intIdx.includes('getElementText'));
+
+  // Main barrel exports
+  const mainIdx = fs.readFileSync(path.join(ROOT, 'src', 'main', 'ui-automation', 'index.js'), 'utf-8');
+  assert('main barrel exports setElementValue', mainIdx.includes('setElementValue'));
+  assert('main barrel exports scrollElement', mainIdx.includes('scrollElement'));
+  assert('main barrel exports expandElement', mainIdx.includes('expandElement'));
+  assert('main barrel exports getElementText', mainIdx.includes('getElementText'));
+  assert('main barrel exports normalizePatternName', mainIdx.includes('normalizePatternName'));
+  assert('main barrel exports hasPattern', mainIdx.includes('hasPattern'));
+
+  // element-click.js handles both pattern name formats
+  const ecPath = path.join(ROOT, 'src', 'main', 'ui-automation', 'interactions', 'element-click.js');
+  const ec = fs.readFileSync(ecPath, 'utf-8');
+  assert('clickElement handles short pattern name format', ec.includes("'Invoke'"));
+}
+
 // ── Cleanup & Summary ────────────────────────────────────────────────────
 cleanup();
 // Also remove any screenshot artifacts from root

src/main/ui-automation/core/uia-host.js

@@ -98,6 +98,34 @@ class UIAHost extends EventEmitter {
     return resp.tree;
   }
 
+  /** Set value on element at (x,y) using ValuePattern. */
+  async setValue(x, y, value) {
+    const resp = await this.send({ cmd: 'setValue', x, y, value });
+    if (!resp.ok) throw new Error(resp.error || 'setValue failed');
+    return resp;
+  }
+
+  /** Scroll element at (x,y) using ScrollPattern. direction: up|down|left|right. amount: percent (0-100) or -1 for small increment. */
+  async scroll(x, y, direction = 'down', amount = -1) {
+    const resp = await this.send({ cmd: 'scroll', x, y, direction, amount });
+    if (!resp.ok) throw new Error(resp.error || 'scroll failed');
+    return resp;
+  }
+
+  /** Expand/collapse element at (x,y). action: expand|collapse|toggle. */
+  async expandCollapse(x, y, action = 'toggle') {
+    const resp = await this.send({ cmd: 'expandCollapse', x, y, action });
+    if (!resp.ok) throw new Error(resp.error || 'expandCollapse failed');
+    return resp;
+  }
+
+  /** Get text from element at (x,y) using TextPattern → ValuePattern → Name fallback. */
+  async getText(x, y) {
+    const resp = await this.send({ cmd: 'getText', x, y });
+    if (!resp.ok) throw new Error(resp.error || 'getText failed');
+    return resp;
+  }
+
   /** Gracefully shut down the host process. */
   async stop() {
     if (!this._alive || !this._proc) return;

src/main/ui-automation/index.js

@@ -92,6 +92,15 @@ const {
   waitAndClick,
   clickAndWaitFor,
   selectFromDropdown,
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
 } = require('./interactions');
 
 // Screenshot
@@ -168,6 +177,16 @@ module.exports = {
   clickAndWaitFor,
   selectFromDropdown,
 
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
+  
   // Screenshot
   screenshot,
   screenshotActiveWindow,

src/main/ui-automation/interactions/element-click.js

@@ -136,7 +136,7 @@ async function clickElement(element, options = {}) {
   const centerY = bounds.y + bounds.height / 2;
 
   // Strategy 1: Try Invoke pattern for buttons
-  if (useInvoke && element.patterns?.includes('InvokePatternIdentifiers.Pattern')) {
+  if (useInvoke && (element.patterns?.includes('InvokePatternIdentifiers.Pattern') || element.patterns?.includes('Invoke'))) {
     log(`Attempting Invoke pattern for "${element.name}"`);
     const invokeResult = await invokeElement(element);
     if (invokeResult.success) {

src/main/ui-automation/interactions/high-level.js

@@ -7,6 +7,7 @@
 
 const { findElement, findElements, waitForElement } = require('../elements');
 const { click, clickByText } = require('./element-click');
+const { setElementValue, expandElement } = require('./pattern-actions');
 const { typeText, sendKeys } = require('../keyboard');
 const { focusWindow, findWindows } = require('../window');
 const { log, sleep } = require('../core/helpers');
@@ -21,9 +22,18 @@ const { log, sleep } = require('../core/helpers');
  * @returns {Promise<{success: boolean}>}
  */
 async function fillField(criteria, text, options = {}) {
-  const { clear = true } = options;
+  const { clear = true, preferPattern = true } = options;
+  
+  // Strategy 1: Try ValuePattern (fast, no focus/click needed)
+  if (preferPattern) {
+    const patternResult = await setElementValue(criteria, text);
+    if (patternResult.success) {
+      log(`fillField: ValuePattern succeeded for "${text.slice(0, 30)}"`);
+      return { success: true, method: 'ValuePattern' };
+    }
+  }
 
-  // Click the field
+  // Strategy 2: Click + type (fallback)
   const clickResult = await click(criteria);
   if (!clickResult.success) {
     return { success: false };
@@ -39,7 +49,7 @@ async function fillField(criteria, text, options = {}) {
 
   // Type text
   const typeResult = await typeText(text);
-  return { success: typeResult.success };
+  return { success: typeResult.success, method: 'sendKeys' };
 }
 
 /**
@@ -52,9 +62,21 @@ async function fillField(criteria, text, options = {}) {
  * @returns {Promise<{success: boolean}>}
  */
 async function selectDropdownItem(dropdownCriteria, itemCriteria, options = {}) {
-  const { itemWait = 1000 } = options;
+  const { itemWait = 1000, preferPattern = true } = options;
+  
+  // Strategy 1: Try ExpandCollapsePattern to open
+  if (preferPattern) {
+    const expandResult = await expandElement(dropdownCriteria);
+    if (expandResult.success) {
+      log(`selectDropdownItem: ExpandCollapsePattern expanded (${expandResult.stateBefore} → ${expandResult.stateAfter})`);
+      await sleep(itemWait);
+      const itemQuery = typeof itemCriteria === 'string' ? { text: itemCriteria } : itemCriteria;
+      const itemResult = await click(itemQuery);
+      return { success: itemResult.success, method: 'ExpandCollapsePattern' };
+    }
+  }
 
-  // Click dropdown to open
+  // Strategy 2: Click to open (fallback)
   const openResult = await click(dropdownCriteria);
   if (!openResult.success) {
     log('selectDropdownItem: Failed to open dropdown', 'warn');
@@ -69,7 +91,7 @@ async function selectDropdownItem(dropdownCriteria, itemCriteria, options = {})
     : itemCriteria;
 
   const itemResult = await click(itemQuery);
-  return { success: itemResult.success };
+  return { success: itemResult.success, method: 'click' };
 }
 
 /**

src/main/ui-automation/interactions/index.js

@@ -25,6 +25,17 @@ const {
   selectFromDropdown,
 } = require('./high-level');
 
+const {
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
+} = require('./pattern-actions');
+
 module.exports = {
   // Element clicks
   click,
@@ -44,4 +55,14 @@ module.exports = {
   waitAndClick,
   clickAndWaitFor,
   selectFromDropdown,
+  
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
 };