/************************************************************************************ Copyright : Copyright (c) Facebook Technologies, LLC and its affiliates. All rights reserved. Your use of this SDK or tool is subject to the Oculus SDK License Agreement, available at https://developer.oculus.com/licenses/oculussdk/ Unless required by applicable law or agreed to in writing, the Utilities SDK distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ************************************************************************************/ using UnityEngine; using UnityEngine.Assertions; public class OVRTrackedKeyboardHands : MonoBehaviour { public GameObject LeftHandPresence; public GameObject RightHandPresence; private bool handPresenceInitialized_ = false; private Transform leftHandRoot_; private Transform rightHandRoot_; public OVRTrackedKeyboard KeyboardTracker; private OVRCameraRig cameraRig_; private OVRHand leftHand_; private OVRSkeleton leftHandSkeleton_; private OVRSkeletonRenderer leftHandSkeletonRenderer_; private GameObject leftHandSkeletonRendererGO_; private SkinnedMeshRenderer leftHandSkinnedMeshRenderer_; private OVRMeshRenderer leftHandMeshRenderer_; private OVRHand rightHand_; private OVRSkeleton rightHandSkeleton_; private OVRSkeletonRenderer rightHandSkeletonRenderer_; private GameObject rightHandSkeletonRendererGO_; private OVRMeshRenderer rightHandMeshRenderer_; private SkinnedMeshRenderer rightHandSkinnedMeshRenderer_; public bool RightHandOverKeyboard { get; private set; } = false; public bool LeftHandOverKeyboard { get; private set; } = false; private static readonly float handInnerAlphaThreshold_ = 0.08f; private static readonly float handOuterAlphaThreshold_ = 0.20f; private static readonly float maximumPassthroughHandsDistance_ = 0.18f; private static readonly float minimumModelHandsDistance_ = 0.11f; private TrackedKeyboardHandsVisibilityChangedEvent? lastVisibilityEvent_ = null; private struct HandBoneMapping { public Transform LeftHandTransform; public Transform LeftPresenceTransform; public Transform RightHandTransform; public Transform RightPresenceTransform; public OVRSkeleton.BoneId BoneName; public string HandPresenceLeftBoneName; public string HandPresenceRightBoneName; }; private readonly HandBoneMapping[] boneMappings_ = new HandBoneMapping[] { new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_WristRoot, HandPresenceLeftBoneName = "b_l_wrist", HandPresenceRightBoneName = "b_r_wrist" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Thumb0, HandPresenceLeftBoneName = "b_l_thumb0", HandPresenceRightBoneName = "b_r_thumb0" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Thumb1, HandPresenceLeftBoneName = "b_l_thumb1", HandPresenceRightBoneName = "b_r_thumb1" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Thumb2, HandPresenceLeftBoneName = "b_l_thumb2", HandPresenceRightBoneName = "b_r_thumb2" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Thumb3, HandPresenceLeftBoneName = "b_l_thumb3", HandPresenceRightBoneName = "b_r_thumb3" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Index1, HandPresenceLeftBoneName = "b_l_index1", HandPresenceRightBoneName = "b_r_index1" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Index2, HandPresenceLeftBoneName = "b_l_index2", HandPresenceRightBoneName = "b_r_index2" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Index3, HandPresenceLeftBoneName = "b_l_index3", HandPresenceRightBoneName = "b_r_index3" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Middle1, HandPresenceLeftBoneName = "b_l_middle1", HandPresenceRightBoneName = "b_r_middle1" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Middle2, HandPresenceLeftBoneName = "b_l_middle2", HandPresenceRightBoneName = "b_r_middle2" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Middle3, HandPresenceLeftBoneName = "b_l_middle3", HandPresenceRightBoneName = "b_r_middle3" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Ring1, HandPresenceLeftBoneName = "b_l_ring1", HandPresenceRightBoneName = "b_r_ring1" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Ring2, HandPresenceLeftBoneName = "b_l_ring2", HandPresenceRightBoneName = "b_r_ring2" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Ring3, HandPresenceLeftBoneName = "b_l_ring3", HandPresenceRightBoneName = "b_r_ring3" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Pinky0, HandPresenceLeftBoneName = "b_l_pinky0", HandPresenceRightBoneName = "b_r_pinky0" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Pinky1, HandPresenceLeftBoneName = "b_l_pinky1", HandPresenceRightBoneName = "b_r_pinky1" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Pinky2, HandPresenceLeftBoneName = "b_l_pinky2", HandPresenceRightBoneName = "b_r_pinky2" }, new HandBoneMapping { BoneName = OVRSkeleton.BoneId.Hand_Pinky3, HandPresenceLeftBoneName = "b_l_pinky3", HandPresenceRightBoneName = "b_r_pinky3" } }; public Material HandsMaterial; #region MATERIAL PROPERTIES private const float XSCALE = 0.73f; private const float YSCALE = 0.8f; private const float FORWARD_OFFSET = -0.02f; private int keyboardPositionID_; private int keyboardRotationID_; private int keyboardScaleID_; #endregion private void Awake() { KeyboardTracker.TrackedKeyboardActiveChanged += TrackedKeyboardActiveUpdated; KeyboardTracker.TrackedKeyboardVisibilityChanged += TrackedKeyboardVisibilityChanged; keyboardPositionID_ = Shader.PropertyToID("_KeyboardPosition"); keyboardRotationID_ = Shader.PropertyToID("_KeyboardRotation"); keyboardScaleID_ = Shader.PropertyToID("_KeyboardScale"); } private void Start() { enabled = false; cameraRig_ = FindObjectOfType(); leftHand_ = cameraRig_.leftHandAnchor.GetComponentInChildren(); rightHand_ = cameraRig_.rightHandAnchor.GetComponentInChildren(); leftHandSkeleton_ = leftHand_.GetComponent(); rightHandSkeleton_ = rightHand_.GetComponent(); leftHandMeshRenderer_ = leftHand_.GetComponent(); rightHandMeshRenderer_ = rightHand_.GetComponent(); leftHandSkeletonRenderer_ = leftHand_.GetComponent(); rightHandSkeletonRenderer_ = rightHand_.GetComponent(); if (!leftHandSkeletonRenderer_.enabled) { // App is not using skeleton renderer leftHandSkeletonRenderer_ = null; rightHandSkeletonRenderer_ = null; } leftHandSkinnedMeshRenderer_ = leftHand_.GetComponent(); rightHandSkinnedMeshRenderer_ = rightHand_.GetComponent(); var leftHand = GameObject.Instantiate(LeftHandPresence); var rightHand = GameObject.Instantiate(RightHandPresence); leftHandRoot_ = leftHand.transform; rightHandRoot_ = rightHand.transform; leftHand.SetActive(false); rightHand.SetActive(false); #if !UNITY_EDITOR // GameObject trees for hands only available on-device RetargetHandTrackingToHandPresence(); #endif } private bool AreControllersActive => !(leftHand_.IsTracked || rightHand_.IsTracked); private void LateUpdate() { if (AreControllersActive) { DisableHandObjects(); return; } foreach (var boneEntry in boneMappings_) { boneEntry.LeftPresenceTransform.localRotation = boneEntry.LeftHandTransform.localRotation; boneEntry.RightPresenceTransform.localRotation = boneEntry.RightHandTransform.localRotation; if (boneEntry.BoneName == OVRSkeleton.BoneId.Hand_WristRoot) { boneEntry.LeftPresenceTransform.rotation = boneEntry.LeftHandTransform.rotation; boneEntry.RightPresenceTransform.rotation = boneEntry.RightHandTransform.rotation; var leftScale = leftHand_.HandScale; var rightScale = rightHand_.HandScale; boneEntry.RightPresenceTransform.localScale = new Vector3(rightScale, rightScale, rightScale); boneEntry.LeftPresenceTransform.localScale = new Vector3(leftScale, leftScale, leftScale); } } rightHandRoot_.position = rightHand_.transform.position; rightHandRoot_.rotation = rightHand_.transform.rotation; leftHandRoot_.position = leftHand_.transform.position; leftHandRoot_.rotation = leftHand_.transform.rotation; var leftHandDistance = GetHandDistanceToKeyboard(leftHandSkeleton_); var rightHandDistance = GetHandDistanceToKeyboard(rightHandSkeleton_); LeftHandOverKeyboard = ShouldEnablePassthrough(leftHandDistance); RightHandOverKeyboard = ShouldEnablePassthrough(rightHandDistance); KeyboardTracker.HandsOverKeyboard = RightHandOverKeyboard || LeftHandOverKeyboard; var enableLeftModel = ShouldEnableModel(leftHandDistance); var enableRightModel = ShouldEnableModel(rightHandDistance); SetHandModelsEnabled(enableLeftModel, enableRightModel); if (KeyboardTracker.Presentation == OVRTrackedKeyboard.KeyboardPresentation.PreferOpaque) { // Used mixed reality service hands leftHandRoot_.gameObject.SetActive(false); rightHandRoot_.gameObject.SetActive(false); } else { leftHandRoot_.gameObject.SetActive(LeftHandOverKeyboard); rightHandRoot_.gameObject.SetActive(RightHandOverKeyboard); } var position = KeyboardTracker.ActiveKeyboardTransform?.position; var rotation = KeyboardTracker.ActiveKeyboardTransform?.rotation; var offset = KeyboardTracker.ActiveKeyboardTransform == null ? Vector3.zero : KeyboardTracker.ActiveKeyboardTransform.forward * FORWARD_OFFSET; HandsMaterial.SetVector(keyboardPositionID_, position.HasValue ? position.Value + offset : Vector3.zero); HandsMaterial.SetVector(keyboardRotationID_, rotation.HasValue ? rotation.Value.eulerAngles : Vector3.zero); HandsMaterial.SetVector( keyboardScaleID_, new Vector4( KeyboardTracker.ActiveKeyboardInfo.Dimensions.x * XSCALE, 0.1f, KeyboardTracker.ActiveKeyboardInfo.Dimensions.z * YSCALE, 1f ) ); if (lastVisibilityEvent_ == null || LeftHandOverKeyboard != lastVisibilityEvent_.Value.leftVisible || RightHandOverKeyboard != lastVisibilityEvent_.Value.rightVisible) { lastVisibilityEvent_ = new TrackedKeyboardHandsVisibilityChangedEvent { leftVisible = LeftHandOverKeyboard, rightVisible = RightHandOverKeyboard }; KeyboardTracker.UpdateKeyboardVisibility(); } if (LeftHandOverKeyboard || RightHandOverKeyboard) { var handsIntensity = new OVRPlugin.InsightPassthroughKeyboardHandsIntensity { LeftHandIntensity = ComputeOpacity(leftHandDistance, handInnerAlphaThreshold_, handOuterAlphaThreshold_), RightHandIntensity = ComputeOpacity(rightHandDistance, handInnerAlphaThreshold_, handOuterAlphaThreshold_) }; OVRPlugin.SetInsightPassthroughKeyboardHandsIntensity(KeyboardTracker.PassthroughOverlay.layerId, handsIntensity); } } private bool ShouldEnablePassthrough(float distance) { return distance <= maximumPassthroughHandsDistance_; } private bool ShouldEnableModel(float distance) { return distance >= minimumModelHandsDistance_; } private float GetHandDistanceToKeyboard(OVRSkeleton handSkeleton) { // TODO: Switch back to PointerPose once it's working in OpenXR var pinchPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Index3].Transform.position; var handPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Middle1].Transform.position; var pinkyPosition = handSkeleton.Bones[(int) OVRSkeleton.BoneId.Hand_Pinky3].Transform.position; return Mathf.Min(KeyboardTracker.GetDistanceToKeyboard(pinchPosition), KeyboardTracker.GetDistanceToKeyboard(handPosition), KeyboardTracker.GetDistanceToKeyboard(pinkyPosition)); } private float ComputeOpacity(float distance, float innerThreshold, float outerThreshold) { return Mathf.Clamp((outerThreshold - distance) / (outerThreshold - innerThreshold), 0.0f, 1.0f); } private void SetHandModelsEnabled(bool enableLeftModel, bool enableRightModel) { leftHandMeshRenderer_.enabled = enableLeftModel; rightHandMeshRenderer_.enabled = enableRightModel; leftHandSkinnedMeshRenderer_.enabled = enableLeftModel; rightHandSkinnedMeshRenderer_.enabled = enableRightModel; if (leftHandSkeletonRenderer_ != null) { if (leftHandSkeletonRendererGO_ == null) { leftHandSkeletonRendererGO_ = leftHandSkeletonRenderer_.gameObject.transform.Find("SkeletonRenderer")?.gameObject; rightHandSkeletonRendererGO_ = rightHandSkeletonRenderer_.gameObject.transform.Find("SkeletonRenderer")?.gameObject; } if (leftHandSkeletonRendererGO_ != null) { leftHandSkeletonRendererGO_.SetActive(enableLeftModel); } if (rightHandSkeletonRendererGO_ != null) { rightHandSkeletonRendererGO_.SetActive(enableRightModel); } } } private void RetargetHandTrackingToHandPresence() { Assert.IsTrue(LeftHandPresence != null && RightHandPresence != null); for (int index = 0; index < boneMappings_.Length; index++) { var entry = boneMappings_[index]; var ovrBoneStringLeft = OVRSkeleton.BoneLabelFromBoneId(OVRSkeleton.SkeletonType.HandLeft, entry.BoneName); var ovrBoneStringRight = OVRSkeleton.BoneLabelFromBoneId(OVRSkeleton.SkeletonType.HandRight, entry.BoneName); boneMappings_[index].LeftHandTransform = leftHand_.transform.FindChildRecursive(ovrBoneStringLeft); boneMappings_[index].LeftPresenceTransform = leftHandRoot_.FindChildRecursive(entry.HandPresenceLeftBoneName); boneMappings_[index].RightHandTransform = rightHand_.transform.FindChildRecursive(ovrBoneStringRight); boneMappings_[index].RightPresenceTransform = rightHandRoot_.FindChildRecursive(entry.HandPresenceRightBoneName); Assert.IsTrue( boneMappings_[index].LeftPresenceTransform != null && boneMappings_[index].RightPresenceTransform != null && boneMappings_[index].RightHandTransform != null && boneMappings_[index].LeftHandTransform != null, string.Format( "[tracked_keyboard] - entry.lp {0} && entry.rp {1} && entry.rt {2} && entry.lt {3}, {4}, {5}", boneMappings_[index].LeftPresenceTransform, boneMappings_[index].RightPresenceTransform, boneMappings_[index].RightHandTransform, boneMappings_[index].LeftHandTransform, ovrBoneStringRight, ovrBoneStringLeft ) ); } handPresenceInitialized_ = true; } private void StopHandPresence() { enabled = false; // Re-enable hand models if they are disabled, let OVRHand handle controller/hands switching SetHandModelsEnabled(true, true); DisableHandObjects(); } private void DisableHandObjects() { KeyboardTracker.HandsOverKeyboard = false; RightHandOverKeyboard = false; LeftHandOverKeyboard = false; if (leftHandRoot_ != null) { leftHandRoot_.gameObject.SetActive(false); } if (rightHandRoot_ != null) { rightHandRoot_.gameObject.SetActive(false); } } public void TrackedKeyboardActiveUpdated(OVRTrackedKeyboard.TrackedKeyboardSetActiveEvent e) { if (!e.IsEnabled) { StopHandPresence(); } } public void TrackedKeyboardVisibilityChanged(OVRTrackedKeyboard.TrackedKeyboardVisibilityChangedEvent e) { switch (e.State) { case OVRTrackedKeyboard.TrackedKeyboardState.Offline: case OVRTrackedKeyboard.TrackedKeyboardState.NoTrackableKeyboard: case OVRTrackedKeyboard.TrackedKeyboardState.StartedNotTracked: StopHandPresence(); break; case OVRTrackedKeyboard.TrackedKeyboardState.Valid: enabled = handPresenceInitialized_; break; case OVRTrackedKeyboard.TrackedKeyboardState.Stale: if (e.TrackingTimeout) { StopHandPresence(); } break; case OVRTrackedKeyboard.TrackedKeyboardState.Uninitialized: case OVRTrackedKeyboard.TrackedKeyboardState.Error: case OVRTrackedKeyboard.TrackedKeyboardState.ErrorExtensionFailed: StopHandPresence(); Debug.LogWarning("Invalid state passed into TrackedKeyboardVisibilityChanged " + e.State.ToString()); break; default: throw new System.Exception( $"[tracked_keyboard] - unhandled state: TrackedKeyboardVisibilityChanged {e.State}" ); } } public struct TrackedKeyboardHandsVisibilityChangedEvent { public bool leftVisible; public bool rightVisible; } }