diff --git a/org.mixedrealitytoolkit.input/CHANGELOG.md b/org.mixedrealitytoolkit.input/CHANGELOG.md index 23b92079a..3d72111c1 100644 --- a/org.mixedrealitytoolkit.input/CHANGELOG.md +++ b/org.mixedrealitytoolkit.input/CHANGELOG.md @@ -4,6 +4,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Unreleased +### Added + +* Added `SpeechHoverMode` and `HoverMode` configuration to `SpeechInteractor` to specify which hover state satisfies the focus requirement for voice commands. [PR #1147](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1147) + ### Changed * Reserialized MRTK XR Rig prefab to remove stale serialized fields. [PR #1110](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1110) diff --git a/org.mixedrealitytoolkit.input/Interactors/Speech/SpeechInteractor.cs b/org.mixedrealitytoolkit.input/Interactors/Speech/SpeechInteractor.cs index c19fe9a99..504bef2c1 100644 --- a/org.mixedrealitytoolkit.input/Interactors/Speech/SpeechInteractor.cs +++ b/org.mixedrealitytoolkit.input/Interactors/Speech/SpeechInteractor.cs @@ -8,6 +8,27 @@ namespace MixedReality.Toolkit.Input { + /// + /// Specifies which hover state satisfies the focus requirement for voice commands on a . + /// + public enum SpeechHoverMode + { + /// + /// Any hover state satisfies the focus requirement. + /// + Any = 0, + + /// + /// Only passive gaze hover (e.g. eye gaze, head gaze) satisfies the focus requirement. + /// + Gaze = 1, + + /// + /// Only active hover (e.g. ray, poke, grab) satisfies the focus requirement. + /// + Active = 2, + } + /// /// A that is driven by a . /// @@ -18,13 +39,26 @@ namespace MixedReality.Toolkit.Input ///
/// At the time was created Unity's XRI did not support selecting /// more than one interactable at a time. Because of this limitation, the - /// drops part of the selection lifecycle management provided by + /// drops part of the selection lifecycle management provided by /// Unity's XRI and manually informs the interaction manager to enter and exit selection states. /// /// [AddComponentMenu("MRTK/Input/Speech Interactor")] public class SpeechInteractor : XRBaseInteractor, ISpeechInteractor { + [SerializeField] + [Tooltip("Which hover state is required on an interactable (when VoiceRequiresFocus is true) for a voice command to select it?")] + private SpeechHoverMode hoverMode = SpeechHoverMode.Any; + + /// + /// Which hover state is required on an interactable (when VoiceRequiresFocus is true) for a voice command to select it? + /// + public SpeechHoverMode HoverMode + { + get => hoverMode; + set => hoverMode = value; + } + [SerializeField] [Tooltip("How long does the interactor remain selecting the interactable after recognizing a voice command?")] private float voiceCommandTriggerTime = 0.3f; @@ -34,7 +68,7 @@ public class SpeechInteractor : XRBaseInteractor, ISpeechInteractor /// public float VoiceCommandTriggerTime => voiceCommandTriggerTime; - private Dictionary> keywordDictionary = new Dictionary>(); + private readonly Dictionary> keywordDictionary = new Dictionary>(); private List<(StatefulInteractable, float)> selectedInteractables = new List<(StatefulInteractable, float)>(); /// @@ -47,7 +81,7 @@ public class SpeechInteractor : XRBaseInteractor, ISpeechInteractor /// Register a new and an associated keyword with this . /// /// - /// When this recognizes the provided , + /// When this recognizes the provided , /// is called, passing along the provided /// . /// @@ -58,13 +92,13 @@ public void RegisterInteractable(StatefulInteractable interactable, string keywo if (!string.IsNullOrWhiteSpace(keyword)) { keyword = keyword.ToLower(); - if (keywordDictionary.TryGetValue(keyword, out List interactableList)) + if (keywordDictionary.TryGetValue(keyword, out HashSet interactables)) { - interactableList.Add(interactable); + interactables.Add(interactable); } else { - keywordDictionary.Add(keyword, new List { interactable }); + keywordDictionary.Add(keyword, new HashSet { interactable }); var subsystem = XRSubsystemHelpers.KeywordRecognitionSubsystem; if (subsystem != null) { @@ -89,7 +123,7 @@ public void UnregisterInteractable(StatefulInteractable interactable, string key if (!string.IsNullOrWhiteSpace(keyword)) { keyword = keyword.ToLower(); - if (keywordDictionary.TryGetValue(keyword, out List interactableList) && interactableList.Remove(interactable)) + if (keywordDictionary.TryGetValue(keyword, out HashSet interactables) && interactables.Remove(interactable)) { return; } @@ -103,21 +137,25 @@ public void UnregisterInteractable(StatefulInteractable interactable, string key private static readonly ProfilerMarker OnKeywordRecognizedPerfMarker = new ProfilerMarker("[MRTK] SpeechInteractor.OnKeywordRecognized"); - private void OnKeywordRecognized(string keyword) + /// + /// Invoked when a keyword registered with this interactor is recognized. + /// + /// The recognized keyword. + protected internal virtual void OnKeywordRecognized(string keyword) { using (OnKeywordRecognizedPerfMarker.Auto()) { - if (keywordDictionary.TryGetValue(keyword, out List interactableList)) + if (keywordDictionary.TryGetValue(keyword, out HashSet interactables)) { - if (interactableList.Count > 0 && interactionManager == null) + if (interactables.Count > 0 && interactionManager == null) { Debug.LogError("The speech interactor does not have an Interaction Manager."); return; } - foreach (var interactable in interactableList) + foreach (var interactable in interactables) { - if (!interactable.VoiceRequiresFocus || interactable.isHovered) + if (!interactable.VoiceRequiresFocus || IsHoverRequirementMet(interactable)) { selectedInteractables.Insert(0, (interactable, VoiceCommandTriggerTime)); interactionManager.SelectEnter(this, interactable as IXRSelectInteractable); @@ -127,6 +165,30 @@ private void OnKeywordRecognized(string keyword) } } + /// + /// Determines whether the hover condition is met for the given interactable based on . + /// + /// The interactable to evaluate. + /// if the hover requirement is met, otherwise . + protected virtual bool IsHoverRequirementMet(StatefulInteractable interactable) + { + if (interactable == null) + { + return false; + } + + switch (hoverMode) + { + case SpeechHoverMode.Gaze: + return interactable.IsGazeHovered?.Active == true; + case SpeechHoverMode.Active: + return interactable.IsActiveHovered?.Active == true; + case SpeechHoverMode.Any: + default: + return interactable.isHovered; + } + } + private static readonly ProfilerMarker ProcessInteractorPerfMarker = new ProfilerMarker("[MRTK] SpeechInteractor.ProcessInteractor"); diff --git a/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs b/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs new file mode 100644 index 000000000..359679c68 --- /dev/null +++ b/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs @@ -0,0 +1,319 @@ +// Copyright (c) Mixed Reality Toolkit Contributors +// Licensed under the BSD 3-Clause + +using MixedReality.Toolkit.Core.Tests; +using NUnit.Framework; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.TestTools; +using UnityEngine.XR.Interaction.Toolkit; + +namespace MixedReality.Toolkit.Input.Tests +{ + /// + /// Tests for verifying the behavior of and its hover requirements. + /// + internal class SpeechInteractorTests : BaseRuntimeTests + { + private GameObject managerGo; + private XRInteractionManager interactionManager; + + private GameObject speechGo; + private SpeechInteractor speechInteractor; + + private GameObject gazeInteractorGo; + private TestGazeInteractor gazeInteractor; + + private GameObject rayInteractorGo; + private TestRayInteractor rayInteractor; + + private List spawnedObjects = new List(); + + private const string TestKeyword = "select"; + + /// + /// Mock interactor implementing to simulate gaze hover. + /// + private class TestGazeInteractor : XRBaseInteractor, IGazeInteractor + { + public List validTargets = new List(); + + public override void GetValidTargets(List targets) + { + targets.Clear(); + targets.AddRange(validTargets); + } + + public override bool CanHover(IXRHoverInteractable interactable) => true; + public override bool isHoverActive => true; + } + + /// + /// Mock interactor implementing to simulate active ray hover. + /// + private class TestRayInteractor : XRBaseInteractor, IRayInteractor + { + public List validTargets = new List(); + + public override void GetValidTargets(List targets) + { + targets.Clear(); + targets.AddRange(validTargets); + } + + public override bool CanHover(IXRHoverInteractable interactable) => true; + public override bool isHoverActive => true; + } + + [UnitySetUp] + public override IEnumerator Setup() + { + yield return base.Setup(); + + managerGo = new GameObject("InteractionManager"); + interactionManager = managerGo.AddComponent(); + + speechGo = new GameObject("SpeechInteractor"); + speechInteractor = speechGo.AddComponent(); + speechInteractor.interactionManager = interactionManager; + + gazeInteractorGo = new GameObject("GazeInteractor"); + gazeInteractor = gazeInteractorGo.AddComponent(); + gazeInteractor.interactionManager = interactionManager; + + rayInteractorGo = new GameObject("RayInteractor"); + rayInteractor = rayInteractorGo.AddComponent(); + rayInteractor.interactionManager = interactionManager; + + yield return null; + } + + [UnityTearDown] + public override IEnumerator TearDown() + { + foreach (var go in spawnedObjects) + { + if (go != null) + { + Object.Destroy(go); + } + } + spawnedObjects.Clear(); + + Object.Destroy(speechGo); + Object.Destroy(gazeInteractorGo); + Object.Destroy(rayInteractorGo); + Object.Destroy(managerGo); + + yield return base.TearDown(); + } + + private StatefulInteractable CreateInteractable(bool voiceRequiresFocus = true, string keyword = TestKeyword) + { + var go = new GameObject("TestInteractable"); + go.AddComponent(); + var stateful = go.AddComponent(); + stateful.VoiceRequiresFocus = voiceRequiresFocus; + stateful.SpeechRecognitionKeyword = keyword; + + spawnedObjects.Add(go); + + LogAssert.Expect(LogType.Warning, "Failed to retrieve a running KeywordRecognitionSubsystem while registering an interactable. " + + "Please make sure the subsystem is correctly set up for this platform or disable this speech interactor if it's unused."); + stateful.interactionManager = interactionManager; + + return stateful; + } + + private void HoverWithGaze(StatefulInteractable target) + { + gazeInteractor.validTargets.Add(target); + interactionManager.HoverEnter(gazeInteractor, target); + } + + private void UnhoverWithGaze(StatefulInteractable target) + { + gazeInteractor.validTargets.Remove(target); + interactionManager.HoverExit(gazeInteractor, target); + } + + private void HoverWithRay(StatefulInteractable target) + { + rayInteractor.validTargets.Add(target); + interactionManager.HoverEnter(rayInteractor, target); + } + + private void UnhoverWithRay(StatefulInteractable target) + { + rayInteractor.validTargets.Remove(target); + interactionManager.HoverExit(rayInteractor, target); + } + + /// + /// Ensures default HoverMode is Any, which allows both gaze and active hover to satisfy focus. + /// + [UnityTest] + public IEnumerator TestDefaultHoverModeIsAny() + { + Assert.AreEqual(SpeechHoverMode.Any, speechInteractor.HoverMode, "Default HoverMode should be Any."); + + var target = CreateInteractable(voiceRequiresFocus: true); + yield return null; + + // 1. Without hover: keyword should not select + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsFalse(target.isSelected, "Should not select without hover when VoiceRequiresFocus is true."); + + // 2. With gaze hover: keyword should select + HoverWithGaze(target); + Assert.IsTrue(target.isHovered, "Interactable should be hovered."); + Assert.IsTrue(target.IsGazeHovered.Active, "Interactable IsGazeHovered should be active."); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Gaze hover should satisfy focus in Any mode."); + UnhoverWithGaze(target); + + // Wait for speech interactor trigger time to expire + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + Assert.IsFalse(target.isSelected); + + // 3. With active (ray) hover: keyword should select + HoverWithRay(target); + Assert.IsTrue(target.isHovered, "Interactable should be hovered."); + Assert.IsTrue(target.IsActiveHovered.Active, "Interactable IsActiveHovered should be active."); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Active hover should satisfy focus in Any mode."); + UnhoverWithRay(target); + + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + } + + /// + /// Ensures that when HoverMode is Gaze, only gaze hover enables voice selection. + /// + [UnityTest] + public IEnumerator TestHoverModeGaze() + { + speechInteractor.HoverMode = SpeechHoverMode.Gaze; + Assert.AreEqual(SpeechHoverMode.Gaze, speechInteractor.HoverMode); + + var target = CreateInteractable(voiceRequiresFocus: true); + yield return null; + + // 1. Without hover: should not select + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsFalse(target.isSelected); + + // 2. With active (ray) hover: should NOT select in Gaze mode + HoverWithRay(target); + Assert.IsTrue(target.isHovered); + Assert.IsTrue(target.IsActiveHovered.Active); + Assert.IsFalse(target.IsGazeHovered.Active); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsFalse(target.isSelected, "Active hover should not satisfy Gaze hover requirement."); + UnhoverWithRay(target); + yield return null; + + // 3. With gaze hover: should select! + HoverWithGaze(target); + Assert.IsTrue(target.IsGazeHovered.Active); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Gaze hover should satisfy Gaze hover requirement."); + UnhoverWithGaze(target); + + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + } + + /// + /// Ensures that when HoverMode is Active, only active hover (ray, poke, grab) enables voice selection. + /// + [UnityTest] + public IEnumerator TestHoverModeActive() + { + speechInteractor.HoverMode = SpeechHoverMode.Active; + Assert.AreEqual(SpeechHoverMode.Active, speechInteractor.HoverMode); + + var target = CreateInteractable(voiceRequiresFocus: true); + yield return null; + + // 1. Without hover: should not select + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsFalse(target.isSelected); + + // 2. With gaze hover: should NOT select in Active mode + HoverWithGaze(target); + Assert.IsTrue(target.isHovered); + Assert.IsTrue(target.IsGazeHovered.Active); + Assert.IsFalse(target.IsActiveHovered.Active); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsFalse(target.isSelected, "Gaze hover should not satisfy Active hover requirement."); + UnhoverWithGaze(target); + yield return null; + + // 3. With active (ray) hover: should select! + HoverWithRay(target); + Assert.IsTrue(target.IsActiveHovered.Active); + + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Active hover should satisfy Active hover requirement."); + UnhoverWithRay(target); + + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + } + + /// + /// Ensures that when VoiceRequiresFocus is false, voice selection occurs regardless of hover state. + /// + [UnityTest] + public IEnumerator TestVoiceRequiresFocusFalse() + { + var target = CreateInteractable(voiceRequiresFocus: false); + yield return null; + + // In Any mode without hover: should select + speechInteractor.HoverMode = SpeechHoverMode.Any; + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Should select in Any mode without hover when VoiceRequiresFocus is false."); + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + + // In Gaze mode without hover: should select + speechInteractor.HoverMode = SpeechHoverMode.Gaze; + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Should select in Gaze mode without hover when VoiceRequiresFocus is false."); + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + + // In Active mode without hover: should select + speechInteractor.HoverMode = SpeechHoverMode.Active; + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Should select in Active mode without hover when VoiceRequiresFocus is false."); + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + } + + /// + /// Ensures that registering an interactable multiple times with the same keyword + /// does not result in duplicate selection or exit events. + /// + [UnityTest] + public IEnumerator TestDuplicateRegistrationDoesNotCauseDuplicateSelectOrExit() + { + var target = CreateInteractable(voiceRequiresFocus: false); + yield return null; + + // Explicitly register the same interactable again with the same keyword + speechInteractor.RegisterInteractable(target, TestKeyword); + + // Trigger voice command + speechInteractor.OnKeywordRecognized(TestKeyword); + Assert.IsTrue(target.isSelected, "Target should be selected."); + + // Wait for voice command duration to expire + yield return new WaitForSeconds(speechInteractor.VoiceCommandTriggerTime + 0.1f); + Assert.IsFalse(target.isSelected, "Target should no longer be selected after trigger time expires."); + } + } +} diff --git a/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs.meta b/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs.meta new file mode 100644 index 000000000..6ad7712e3 --- /dev/null +++ b/org.mixedrealitytoolkit.input/Tests/Runtime/SpeechInteractorTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 438021db11691a746bf0b4490f21f9cc +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: