From 8a60653a57f324d3d4902a7778e64e0e42976ad5 Mon Sep 17 00:00:00 2001
From: Pete Gordon <pete@petegordon.com>
Date: Sun, 24 May 2026 09:59:30 -0400
Subject: [PATCH] feat(steam-controller): Phase 3a IMU quaternion to visualizer
 + UI fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds quaternion-orientation plumbing for the Steam Controller and the
supporting infrastructure (Test Report multi-interface fan-out, runtime
axis-transform switcher). Pitch + roll track physical motion correctly
in the on-screen model when using the field-tested 'yzp-xy-alt'
transform. Yaw signal is captured but ~10× weaker than pitch/roll
(typical for IMUs without a magnetometer), and visually not convincing
— left as a known limitation for future iteration.

Driver (packages/core/src/drivers/steam-controller-driver.js):
- Parses 4× int16 LE at bytes 31-38 as quaternion components, returns
  parsed.orientation = {x, y, z, w}. The first component (bytes 31-32)
  is actually the high half of the device's uint32 timestamp on this
  firmware — SteamlessController's docs claim the quaternion lives at
  data[31-38] but variance analysis of a real Test Report capture
  shows bytes 29-32 are a ~1MHz uint32 timestamp. Three.js's normalize
  flattens the timestamp byte's effect enough that pitch + roll
  visualize correctly. Better encoding TBD — possibly Euler angles in
  radians, didn't lead to working motion when tried.
- emitsRawGyro = false (already) flag tells app.js to skip the
  rate-based calibration UX since the driver emits orientation, not
  rates.

App (apps/overlay/src/js/app.js):
- Quaternion fast-path: when parsed.orientation is present, writes
  directly into gyroFusion.orientation (bypasses SensorFusion). All
  downstream consumers (overlay body rotation, gimbal widget, R/P/Y
  readout, gyro HUD) pick it up unchanged.
- Reference-quaternion capture: first valid parsed.orientation
  becomes the rest reference; every subsequent frame is emitted as
  delta = current * ref⁻¹. L3+R3 (existing calibrate combo)
  recaptures the reference. Hooks into existing showCalibHint UX so
  user sees feedback when they recenter.
- Runtime axis-transform switcher: window.setSteamQuatTransform(mode)
  cycles between 10+ candidate IMU-body→visualizer-world basis
  transforms. Default 'yzp-xy-alt' = (z, x, y), found via empirical
  iteration. Other transforms in the registry for future tuning.

Test Report wizard (apps/overlay/src/js/test-report.js):
- Multi-interface input fan-out: previously the wizard listened on
  one Puck HID handle (whichever the picker returned) — that's the
  wrong interface for the Puck, so every step recorded 0 reports.
  Now mirrors the gyro pipeline's fan-out: enumerates approved
  siblings with same vid:pid, opens + attaches the recorder to each.
  Single-interface pads (DualSense filtered by usagePage) are no-op.

UI:
- Axis readout (P/R/Y degrees) moved from top-right to bottom-center.
  Was overlapping the Puck status banner.
- HUD Position setting renamed → "Roll HUD Position" since it only
  controls the gyro/roll arc widget, not the axis readout.

Verified via Test Report variance analysis on real hardware:
- Pitch movement primarily varies bytes 35-36 (353× baseline stddev)
- Roll movement primarily varies bytes 33-34 (467× baseline)
- Yaw movement varies all components weakly (max 59× baseline, but
  absolute stddev 10× smaller than pitch/roll)

Phase 3a is sized as "best-with-current-understanding" — the right
IMU encoding remains unclear and the test infrastructure to iterate
further is in place for a future contributor.
---
 apps/overlay/src/index.html                   | 13 ++-
 apps/overlay/src/js/app.js                    | 98 ++++++++++++++++++-
 apps/overlay/src/js/test-report.js            | 36 ++++++-
 .../src/drivers/steam-controller-driver.js    | 48 ++++++---
 4 files changed, 172 insertions(+), 23 deletions(-)

diff --git a/apps/overlay/src/index.html b/apps/overlay/src/index.html
index 04558c8..43b92d8 100644
--- a/apps/overlay/src/index.html
+++ b/apps/overlay/src/index.html
@@ -354,17 +354,20 @@
 
   #axis-readout {
     position: fixed;
-    top: 40px; right: 42px;
+    bottom: 12px;
+    left: 50%;
+    transform: translateX(-50%);
     display: none;
     gap: 14px;
     font-family: monospace;
-    font-size: 30px;
+    font-size: 24px;
     font-weight: 600;
     z-index: 110;
-    background: rgba(0,0,0,0.35);
-    padding: 4px 12px;
+    background: rgba(0,0,0,0.45);
+    padding: 4px 14px;
     border-radius: 14px;
     pointer-events: none;
+    white-space: nowrap;
   }
   body.show-axis-readout #axis-readout { display: flex; }
   #axis-readout .ax-pitch { color: #44dd66; }
@@ -784,7 +787,7 @@ <h3>Controller Overlay</h3>
   </div>
 
   <div class="setting-row">
-    <label>HUD Position</label>
+    <label>Roll HUD Position</label>
     <select id="hud-position">
       <option value="above">Above</option>
       <option value="below" selected>Below</option>
diff --git a/apps/overlay/src/js/app.js b/apps/overlay/src/js/app.js
index c0a6252..66a200d 100644
--- a/apps/overlay/src/js/app.js
+++ b/apps/overlay/src/js/app.js
@@ -138,6 +138,61 @@ let syntheticGamepad = null;
 // Keeps orientation, gravity tracking, stillness & sensor-fusion bias
 // calibration, and all related scratch vectors internal. See #224.
 const gyroFusion = new SensorFusion();
+
+// ── Quaternion-direct driver state (Steam Controller) ──
+// Scratch THREE.Quaternions for the quaternion fast-path. The reference
+// is captured on first valid orientation report (or whenever recalibrate
+// fires) and used to express subsequent reports as relative rotations.
+const _steamRawQuat = new THREE.Quaternion();
+const _steamDelta = new THREE.Quaternion();
+const _steamRefQuatInverse = new THREE.Quaternion();
+let _steamRefQuat = null;
+function recalibrateSteamReference() { _steamRefQuat = null; }
+
+// Runtime-switchable axis transform from the IMU's body frame to the
+// visualizer's world frame. Different IMUs use different conventions
+// (X-forward vs Y-forward, Z-up vs Z-down, handedness). Each transform
+// permutes / negates the quaternion (x, y, z) components — W is the
+// scalar and stays put. Switch via DevTools:
+//   setSteamQuatTransform('swap-yz')
+// after calibrating with L3+R3 if results look stuck.
+const STEAM_QUAT_TRANSFORMS = {
+  'identity':       (x, y, z) => [x, y, z],
+  'swap-yz':        (x, y, z) => [x, z, -y],      // rotate -90° about X (IMU forward→up swap)
+  'swap-yz-pos':    (x, y, z) => [x, -z, y],      // rotate +90° about X — roll axis correct, pitch goes to yaw
+  'swap-xy':        (x, y, z) => [y, x, z],       // swap X↔Y
+  'swap-xz':        (x, y, z) => [z, y, x],       // swap X↔Z
+  'switch-pro-like':(x, y, z) => [-z, y, -x],     // Switch Pro's gyro-frame remap
+  'neg-x':          (x, y, z) => [-x, y, z],
+  'neg-y':          (x, y, z) => [x, -y, z],
+  'neg-z':          (x, y, z) => [x, y, -z],
+  'neg-xz':         (x, y, z) => [-x, y, -z],     // 180° about Y
+  // Compositions building on swap-yz-pos (which got roll right) and
+  // adding an X↔Y swap to move pitch off the yaw axis onto the pitch axis.
+  'yzp-xy':         (x, y, z) => [-z, x, y],      // swap-yz-pos + swap-xy
+  'yzp-xy-pos':     (x, y, z) => [-z, -x, y],     // ...with X sign flipped
+  'yzp-xy-alt':     (x, y, z) => [z, x, y],
+  'yzp-xy-alt-neg': (x, y, z) => [z, -x, y],
+  // Cyclic permutations
+  'cycle-zxy':      (x, y, z) => [z, x, y],
+  'cycle-yzx':      (x, y, z) => [y, z, x],
+};
+// Default: 'yzp-xy-alt' = (z, x, y). Field-tested as best-of-options
+// for the 2026 Steam Controller — produces correct pitch + roll with
+// the driver reading bytes 31-38 as a 4-component quaternion. Yaw
+// signal is weak (~10× smaller than pitch/roll per Test Report
+// variance analysis) and is not visually convincing. Call
+// setSteamQuatTransform('mode-name') in DevTools to try alternates.
+let steamQuatTransform = 'yzp-xy-alt';
+window.setSteamQuatTransform = function(mode) {
+  if (!STEAM_QUAT_TRANSFORMS[mode]) {
+    console.warn(`Unknown transform '${mode}'. Available: ${Object.keys(STEAM_QUAT_TRANSFORMS).join(', ')}`);
+    return;
+  }
+  steamQuatTransform = mode;
+  recalibrateSteamReference();
+  console.log(`Steam quat transform: ${mode} (reference recaptured)`);
+};
 // App-layer calibration still owns variance-check + retry UX — on success
 // it pushes the captured bias into gyroFusion.bias.
 let calibrating = false;
@@ -954,6 +1009,7 @@ async function disconnectGyro() {
   gyroPermitted = false;
   syntheticGamepad = null;
   _firstReportLogged = false;
+  recalibrateSteamReference();
   // Shared SensorFusion owns orientation + all intermediate state.
   gyroFusion.reset();
   gyroFusion.resetBias();
@@ -992,8 +1048,18 @@ function loop() {
       checkCombo(gamepad, 'gyroToggle', toggleGyro);
       checkCombo(gamepad, 'calibrate', () => {
         if (gyroActive) {
-          startCalibration();
-          console.log('Gyro recalibrating');
+          // Quaternion-direct drivers (Steam Controller) use the
+          // reference-capture model: clearing _steamRefQuat causes the
+          // next parsed.orientation to become the new "rest." Rate-based
+          // drivers go through the SensorFusion bias-estimation path.
+          if (controllerDriver?.constructor?.emitsRawGyro === false) {
+            recalibrateSteamReference();
+            showCalibHint('Recalibrated', 2000);
+            console.log('Quaternion reference re-captured');
+          } else {
+            startCalibration();
+            console.log('Gyro recalibrating');
+          }
         }
       });
     }
@@ -1448,6 +1514,34 @@ function handleInputReport(event) {
     overlay.updateTouchpad(parsed.touchpad, parsed.touchpadButton);
   }
 
+  // Quaternion fast-path: drivers that emit orientation directly (Steam
+  // Controller — IMU is a quaternion at offsets 31-38 of the STATE report)
+  // bypass the rate-based SensorFusion pipeline.
+  //
+  // The raw quaternion is in the controller's body frame, which doesn't
+  // align with the visualizer's identity orientation — e.g. the Steam
+  // Controller emits ~(−0.58, 0, −0.09, 0.49) when flat on a desk, not
+  // identity. We capture the FIRST valid quaternion as a reference and
+  // emit each subsequent one as a delta against it: delta = current ·
+  // ref⁻¹. This makes "wherever the controller was when you started"
+  // the on-screen rest position, regardless of physical orientation.
+  // The calibrate combo (L3+R3) re-captures the reference, same way the
+  // rate-based gyro calibration recenters drift.
+  if (parsed.orientation && gyroActive) {
+    const p = parsed.orientation;
+    const [tx, ty, tz] = STEAM_QUAT_TRANSFORMS[steamQuatTransform](p.x, p.y, p.z);
+    _steamRawQuat.set(tx, ty, tz, p.w).normalize();
+    if (!_steamRefQuat) {
+      _steamRefQuat = _steamRawQuat.clone();
+    }
+    // delta = current * ref⁻¹ (post-multiply = rotation since reference)
+    _steamDelta.copy(_steamRawQuat).multiply(
+      _steamRefQuatInverse.copy(_steamRefQuat).invert()
+    );
+    gyroFusion.orientation.copy(_steamDelta);
+    return;
+  }
+
   if (!gyroActive || !parsed.gyro) return;
 
   const now = performance.now();
diff --git a/apps/overlay/src/js/test-report.js b/apps/overlay/src/js/test-report.js
index 43e161d..b9d21f8 100644
--- a/apps/overlay/src/js/test-report.js
+++ b/apps/overlay/src/js/test-report.js
@@ -169,8 +169,30 @@ function snapshotGamepad(vendorId, productId) {
  * @returns {Promise<Array<{ts:number, reportId:number, bytes:string, gamepad?:object}>>}
  */
 export async function recordStep(device, durationMs) {
+  if (!device) throw new Error('No HID device to capture from.');
+
+  // Multi-interface fan-out: pads like the Steam Controller Puck expose
+  // N sibling HID handles sharing the same vid:pid, and only ONE of
+  // them emits the STATE reports we want to capture. The picker hands
+  // back an arbitrary one (typically not the right one), so we attach
+  // the same onReport listener to every approved sibling. Single-
+  // interface pads (DualSense / Switch Pro filtered by usagePage in
+  // getHIDFilters) yield zero siblings here, making this a no-op for
+  // them. Done before the Promise so we can await getDevices/open.
+  const allDevices = [device];
+  try {
+    const approved = await navigator.hid.getDevices();
+    for (const d of approved) {
+      if (d === device) continue;
+      if (d.vendorId !== device.vendorId || d.productId !== device.productId) continue;
+      try {
+        if (!d.opened) await d.open();
+        allDevices.push(d);
+      } catch { /* sibling not openable; skip */ }
+    }
+  } catch { /* getDevices errored; we still have the primary handle */ }
+
   return new Promise((resolve, reject) => {
-    if (!device) return reject(new Error('No HID device to capture from.'));
     const reports = [];
     const t0 = performance.now();
     let lastGamepadSnapAt = 0;
@@ -192,10 +214,14 @@ export async function recordStep(device, durationMs) {
       reports.push(rec);
     };
 
-    device.addEventListener('inputreport', onReport);
+    for (const d of allDevices) d.addEventListener('inputreport', onReport);
+
+    const cleanup = () => {
+      for (const d of allDevices) d.removeEventListener('inputreport', onReport);
+    };
 
     const timer = setTimeout(() => {
-      device.removeEventListener('inputreport', onReport);
+      cleanup();
       resolve(reports);
     }, durationMs);
 
@@ -203,10 +229,10 @@ export async function recordStep(device, durationMs) {
     // UI can stop the wizard instead of waiting for a timeout that yields
     // empty data.
     const onDisconnect = (e) => {
-      if (e.device !== device) return;
+      if (!allDevices.includes(e.device)) return;
       clearTimeout(timer);
       navigator.hid.removeEventListener('disconnect', onDisconnect);
-      device.removeEventListener('inputreport', onReport);
+      cleanup();
       reject(new Error('Device disconnected during capture.'));
     };
     navigator.hid.addEventListener('disconnect', onDisconnect);
diff --git a/packages/core/src/drivers/steam-controller-driver.js b/packages/core/src/drivers/steam-controller-driver.js
index 3211206..8ba17a0 100644
--- a/packages/core/src/drivers/steam-controller-driver.js
+++ b/packages/core/src/drivers/steam-controller-driver.js
@@ -53,11 +53,13 @@ const FEATURE_REPORT_ID_FALLBACK = 0x02;
 
 export class SteamControllerDriver extends ControllerDriver {
 
-  // Phase 1 returns null gyro/accel (the IMU is a quaternion at offsets
-  // 31-38 and doesn't fit the raw-rate fusion pipeline that PlayStation /
-  // Switch drivers feed). App-layer calibration UX should skip itself for
-  // this driver — checks this flag and avoids the "Calibrating…" hint
-  // that would otherwise hang forever waiting for samples that never come.
+  // Steam Controller emits a quaternion (parsed.orientation), not raw
+  // gyro rates. The app's rate-based calibration UX should skip itself
+  // for this driver — checks this flag and avoids the "Calibrating…"
+  // hint that would otherwise hang forever waiting for raw-rate samples
+  // that never come. The visualizer body still rotates: app.js's fast
+  // path writes parsed.orientation directly into gyroFusion.orientation,
+  // bypassing the rate-based integrate-and-correct loop entirely.
   static emitsRawGyro = false;
 
   // Valve's HID interfaces are vendor-defined (usage page 0xFF0x), not
@@ -369,12 +371,35 @@ export class SteamControllerDriver extends ControllerDriver {
       },
     ];
 
-    // IMU is a quaternion at offsets 31-38 (4× int16 LE). Phase 1 omits
-    // it — feeding raw quaternion components to a gyro-rate fusion
-    // pipeline produces garbage. Wiring quaternion-orientation through
-    // the visualizer (bypassing fusion entirely for this driver) is
-    // tracked as follow-up; returning null here keeps the gyro pipeline
-    // safely idle in the meantime.
+    // IMU encoding (2026 firmware) — STILL UNDER INVESTIGATION.
+    //
+    // SteamlessController docs say quaternion at WebHID data[31-38]
+    // (4× int16 LE), but our captures show data[29-32] is a uint32 LE
+    // timestamp on this firmware. Test Report variance analysis showed
+    // bytes 33-38 carry the actual IMU signal (pitch primarily in
+    // bytes 35-36, roll primarily in bytes 33-34), but neither the
+    // compressed-quaternion (X,Y,Z + derived W) interpretation nor any
+    // axis transform matched physical motion correctly.
+    //
+    // Reverting to the previously field-tested layout: read bytes
+    // 31-38 as 4 int16 LE and feed all four into Three.js's quaternion
+    // as (x, y, z, w). One component is timestamp garbage, but
+    // Three.js's normalize() flattens it out enough that pitch + roll
+    // visually track physical motion when combined with the
+    // 'yzp-xy-alt' transform in app.js. Yaw remains weak on this
+    // path (~10× smaller signal than pitch/roll per Test Report).
+    //
+    // Better encoding TBD — possibly Euler angles in radians, or a
+    // different scale/sign convention. Re-investigate when there's
+    // time.
+    const QUAT_SCALE = 1 / 0x7FFF;
+    const orientation = {
+      x: r(data, 31) * QUAT_SCALE,
+      y: r(data, 33) * QUAT_SCALE,
+      z: r(data, 35) * QUAT_SCALE,
+      w: r(data, 37) * QUAT_SCALE,
+    };
+
 
     return {
       sticks,
@@ -385,6 +410,7 @@ export class SteamControllerDriver extends ControllerDriver {
       touchpadButton: false,
       gyro: null,
       accel: null,
+      orientation,
       gyroScale: 2000.0 / 32768.0,
       accelScale: 1.0 / 8192.0,
     };