Android build settings + metaxr

This commit is contained in:
2025-05-14 14:00:02 +03:00
parent 6a2bb7475e
commit d5aa21f55c
594 changed files with 200530 additions and 2 deletions

View File

@@ -0,0 +1,50 @@
// @lint-ignore-every LICENSELINT
// Copyright Epic Games, Inc. All Rights Reserved.
namespace UnrealBuildTool.Rules
{
public class OculusXRMovement : ModuleRules
{
public OculusXRMovement(ReadOnlyTargetRules Target) : base(Target)
{
bUseUnity = false;
PublicDependencyModuleNames.AddRange(
new string[] {
"LiveLinkInterface",
"LiveLinkAnimationCore",
});
PrivateDependencyModuleNames.AddRange(
new string[]
{
"Core",
"CoreUObject",
"ApplicationCore",
"Engine",
"InputCore",
"LiveLink",
"HeadMountedDisplay",
"OVRPluginXR",
"OculusXRHMD",
"XRBase",
"OpenXR",
"OpenXRHMD",
});
PrivateIncludePaths.AddRange(
new string[] {
"OculusXRHMD/Private",
});
PublicDependencyModuleNames.AddRange(
new string[]
{
"KhronosOpenXRHeaders",
});
PrivateIncludePathModuleNames.Add("OpenXRHMD");
AddEngineThirdPartyPrivateStaticDependencies(Target, "OpenXR");
}
}
}

View File

@@ -0,0 +1,40 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "LiveLinkOculusXRMovementSourceFactory.h"
#include "IOculusXRMovementModule.h"
#include "Features/IModularFeatures.h"
#include "ILiveLinkClient.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
FText ULiveLinkOculusXRMovementSourceFactory::GetSourceDisplayName() const
{
return LOCTEXT("OculusXRMovementLiveLinkSourceName", "Meta MovementSDK Live Link");
}
FText ULiveLinkOculusXRMovementSourceFactory::GetSourceTooltip() const
{
return LOCTEXT("OculusXRMovementLiveLinkSourceTooltip", "Meta MovementSDK Live Link Source");
}
ULiveLinkOculusXRMovementSourceFactory::EMenuType ULiveLinkOculusXRMovementSourceFactory::GetMenuType() const
{
if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName))
{
const ILiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature<ILiveLinkClient>(ILiveLinkClient::ModularFeatureName);
if (!IOculusXRMovementModule::Get().IsLiveLinkSourceValid() || !LiveLinkClient.HasSourceBeenAdded(IOculusXRMovementModule::Get().GetLiveLinkSource()))
{
return EMenuType::MenuEntry;
}
}
return EMenuType::Disabled;
}
TSharedPtr<ILiveLinkSource> ULiveLinkOculusXRMovementSourceFactory::CreateSource(const FString& ConnectionString) const
{
return IOculusXRMovementModule::Get().GetLiveLinkSource();
}
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,21 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "LiveLinkSourceFactory.h"
#include "LiveLinkOculusXRMovementSourceFactory.generated.h"
UCLASS()
class ULiveLinkOculusXRMovementSourceFactory : public ULiveLinkSourceFactory
{
public:
GENERATED_BODY()
virtual FText GetSourceDisplayName() const override;
virtual FText GetSourceTooltip() const override;
virtual EMenuType GetMenuType() const override;
virtual TSharedPtr<ILiveLinkSource> CreateSource(const FString& ConnectionString) const override;
TSharedPtr<class SLiveLinkOculusXRMovementSourceEditor> ActiveSourceEditor;
};

View File

@@ -0,0 +1,263 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRBodyTrackingComponent.h"
#include "Engine/SkeletalMesh.h"
#include "DrawDebugHelpers.h"
#include "OculusXRHMD.h"
#include "OculusXRPluginWrapper.h"
#include "OculusXRMovementFunctionLibrary.h"
#include "OculusXRMovementLog.h"
#include "OculusXRTelemetryMovementEvents.h"
#if !(UE_BUILD_SHIPPING || UE_BUILD_TEST)
static TAutoConsoleVariable<int32> CVarOVRBodyDebugDraw(
TEXT("ovr.BodyDebugDraw"),
0,
TEXT("Enables or disables debug drawing for body tracking.\n")
TEXT("<=0: disabled (no drawing)\n")
TEXT(" 1: enabled (debug drawing)\n"));
#endif
int UOculusXRBodyTrackingComponent::TrackingInstanceCount = 0;
UOculusXRBodyTrackingComponent::UOculusXRBodyTrackingComponent()
: BodyTrackingMode(EOculusXRBodyTrackingMode::PositionAndRotation)
, ConfidenceThreshold(0.f)
, WorldToMeters(100.f)
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.bStartWithTickEnabled = true;
// Setup defaults
BoneNames.Add(EOculusXRBoneID::BodyRoot, "Root");
BoneNames.Add(EOculusXRBoneID::BodyHips, "Hips");
BoneNames.Add(EOculusXRBoneID::BodySpineLower, "SpineLower");
BoneNames.Add(EOculusXRBoneID::BodySpineMiddle, "SpineMiddle");
BoneNames.Add(EOculusXRBoneID::BodySpineUpper, "SpineUpper");
BoneNames.Add(EOculusXRBoneID::BodyChest, "Chest");
BoneNames.Add(EOculusXRBoneID::BodyNeck, "Neck");
BoneNames.Add(EOculusXRBoneID::BodyHead, "Head");
BoneNames.Add(EOculusXRBoneID::BodyLeftShoulder, "LeftShoulder");
BoneNames.Add(EOculusXRBoneID::BodyLeftScapula, "LeftScapula");
BoneNames.Add(EOculusXRBoneID::BodyLeftArmUpper, "LeftArmUpper");
BoneNames.Add(EOculusXRBoneID::BodyLeftArmLower, "LeftArmLower");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandWristTwist, "LeftHandWristTwist");
BoneNames.Add(EOculusXRBoneID::BodyRightShoulder, "RightShoulder");
BoneNames.Add(EOculusXRBoneID::BodyRightScapula, "RightScapula");
BoneNames.Add(EOculusXRBoneID::BodyRightArmUpper, "RightArmUpper");
BoneNames.Add(EOculusXRBoneID::BodyRightArmLower, "RightArmLower");
BoneNames.Add(EOculusXRBoneID::BodyRightHandWristTwist, "RightHandWristTwist");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandPalm, "LeftHandPalm");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandWrist, "LeftHandWrist");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandThumbMetacarpal, "LeftHandThumbMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandThumbProximal, "LeftHandThumbProximal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandThumbDistal, "LeftHandThumbDistal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandThumbTip, "LeftHandThumbTip");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandIndexMetacarpal, "LeftHandIndexMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandIndexProximal, "LeftHandIndexProximal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandIndexIntermediate, "LeftHandIndexIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandIndexDistal, "LeftHandIndexDistal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandIndexTip, "LeftHandIndexTip");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandMiddleMetacarpal, "LeftHandMiddleMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandMiddleProximal, "LeftHandMiddleProximal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandMiddleIntermediate, "LeftHandMiddleIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandMiddleDistal, "LeftHandMiddleDistal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandMiddleTip, "LeftHandMiddleTip");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandRingMetacarpal, "LeftHandRingMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandRingProximal, "LeftHandRingProximal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandRingIntermediate, "LeftHandRingIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandRingDistal, "LeftHandRingDistal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandRingTip, "LeftHandRingTip");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandLittleMetacarpal, "LeftHandLittleMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandLittleProximal, "LeftHandLittleProximal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandLittleIntermediate, "LeftHandLittleIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandLittleDistal, "LeftHandLittleDistal");
BoneNames.Add(EOculusXRBoneID::BodyLeftHandLittleTip, "LeftHandLittleTip");
BoneNames.Add(EOculusXRBoneID::BodyRightHandPalm, "RightHandPalm");
BoneNames.Add(EOculusXRBoneID::BodyRightHandWrist, "RightHandWrist");
BoneNames.Add(EOculusXRBoneID::BodyRightHandThumbMetacarpal, "RightHandThumbMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandThumbProximal, "RightHandThumbProximal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandThumbDistal, "RightHandThumbDistal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandThumbTip, "RightHandThumbTip");
BoneNames.Add(EOculusXRBoneID::BodyRightHandIndexMetacarpal, "RightHandIndexMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandIndexProximal, "RightHandIndexProximal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandIndexIntermediate, "RightHandIndexIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyRightHandIndexDistal, "RightHandIndexDistal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandIndexTip, "RightHandIndexTip");
BoneNames.Add(EOculusXRBoneID::BodyRightHandMiddleMetacarpal, "RightHandMiddleMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandMiddleProximal, "RightHandMiddleProximal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandMiddleIntermediate, "RightHandMiddleIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyRightHandMiddleDistal, "RightHandMiddleDistal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandMiddleTip, "RightHandMiddleTip");
BoneNames.Add(EOculusXRBoneID::BodyRightHandRingMetacarpal, "RightHandRingMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandRingProximal, "RightHandRingProximal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandRingIntermediate, "RightHandRingIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyRightHandRingDistal, "RightHandRingDistal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandRingTip, "RightHandRingTip");
BoneNames.Add(EOculusXRBoneID::BodyRightHandLittleMetacarpal, "RightHandLittleMetacarpal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandLittleProximal, "RightHandLittleProximal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandLittleIntermediate, "RightHandLittleIntermediate");
BoneNames.Add(EOculusXRBoneID::BodyRightHandLittleDistal, "RightHandLittleDistal");
BoneNames.Add(EOculusXRBoneID::BodyRightHandLittleTip, "RightHandLittleTip");
BoneNames.Add(EOculusXRBoneID::BodyLeftUpperLeg, "LeftUpperLeg");
BoneNames.Add(EOculusXRBoneID::BodyLeftLowerLeg, "LeftLowerLeg");
BoneNames.Add(EOculusXRBoneID::BodyLeftFootAnkleTwist, "LeftFootAnkleTwist");
BoneNames.Add(EOculusXRBoneID::BodyLeftFootAnkle, "LeftFootAnkle");
BoneNames.Add(EOculusXRBoneID::BodyLeftFootSubtalar, "LeftFootSubtalar");
BoneNames.Add(EOculusXRBoneID::BodyLeftFootTransverse, "LeftFootTransverse");
BoneNames.Add(EOculusXRBoneID::BodyLeftFootBall, "LeftFootBall");
BoneNames.Add(EOculusXRBoneID::BodyRightUpperLeg, "RightUpperLeg");
BoneNames.Add(EOculusXRBoneID::BodyRightLowerLeg, "RightLowerLeg");
BoneNames.Add(EOculusXRBoneID::BodyRightFootAnkleTwist, "RightFootAnkleTwist");
BoneNames.Add(EOculusXRBoneID::BodyRightFootAnkle, "RightFootAnkle");
BoneNames.Add(EOculusXRBoneID::BodyRightFootSubtalar, "RightFootSubtalar");
BoneNames.Add(EOculusXRBoneID::BodyRightFootTransverse, "RightFootTransverse");
BoneNames.Add(EOculusXRBoneID::BodyRightFootBall, "RightFootBall");
OculusXRTelemetry::TScopedMarker<OculusXRTelemetry::Events::FMovementSDKBodyStart>(static_cast<int>(GetTypeHash(this)));
}
void UOculusXRBodyTrackingComponent::BeginPlay()
{
Super::BeginPlay();
if (!UOculusXRMovementFunctionLibrary::IsBodyTrackingSupported())
{
// Early exit if body tracking isn't supported
UE_LOG(LogOculusXRMovement, Warning, TEXT("Body tracking is not supported. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
if (!OculusXRHMD::GetUnitScaleFactorFromSettings(GetWorld(), WorldToMeters))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot get world settings. (%s:%s)"), *GetOwner()->GetName(), *GetName());
}
if (!InitializeBodyBones())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to initialize body data. (%s: %s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
if (!UOculusXRMovementFunctionLibrary::StartBodyTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to start body tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
++TrackingInstanceCount;
}
void UOculusXRBodyTrackingComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
if (IsComponentTickEnabled())
{
if (--TrackingInstanceCount == 0)
{
if (!UOculusXRMovementFunctionLibrary::StopBodyTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to stop body tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
}
}
}
Super::EndPlay(EndPlayReason);
}
void UOculusXRBodyTrackingComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
if (UOculusXRMovementFunctionLibrary::TryGetBodyState(BodyState, WorldToMeters))
{
if (BodyState.IsActive && BodyState.Confidence > ConfidenceThreshold)
{
for (int i = 0; i < BodyState.Joints.Num(); ++i)
{
const FOculusXRBodyJoint& Joint = BodyState.Joints[i];
if (!Joint.bIsValid)
{
continue;
}
const FVector& Position = Joint.Position;
const FRotator& Orientation = Joint.Orientation;
#if !(UE_BUILD_SHIPPING || UE_BUILD_TEST)
if (CVarOVRBodyDebugDraw.GetValueOnGameThread() > 0)
{
const FTransform& ParentTransform = GetOwner()->GetActorTransform();
FVector DebugPosition = ParentTransform.TransformPosition(Position);
FRotator DebugOrientation = ParentTransform.TransformRotation(Orientation.Quaternion()).Rotator();
DrawDebugLine(GetWorld(), DebugPosition, DebugPosition + DebugOrientation.Quaternion().GetUpVector(), FColor::Blue);
DrawDebugLine(GetWorld(), DebugPosition, DebugPosition + DebugOrientation.Quaternion().GetForwardVector(), FColor::Red);
DrawDebugLine(GetWorld(), DebugPosition, DebugPosition + DebugOrientation.Quaternion().GetRightVector(), FColor::Green);
}
#endif
int32* BoneIndex = MappedBoneIndices.Find(static_cast<EOculusXRBoneID>(i));
if (BoneIndex != nullptr)
{
switch (BodyTrackingMode)
{
case EOculusXRBodyTrackingMode::PositionAndRotation:
SetBoneTransformByName(BoneNames[static_cast<EOculusXRBoneID>(i)], FTransform(Orientation, Position), EBoneSpaces::ComponentSpace);
break;
case EOculusXRBodyTrackingMode::RotationOnly:
SetBoneRotationByName(BoneNames[static_cast<EOculusXRBoneID>(i)], Orientation, EBoneSpaces::ComponentSpace);
break;
case EOculusXRBodyTrackingMode::NoTracking:
break;
}
}
}
}
}
else
{
UE_LOG(LogOculusXRMovement, Verbose, TEXT("Failed to get body state (%s:%s)."), *GetOwner()->GetName(), *GetName());
}
}
void UOculusXRBodyTrackingComponent::ResetAllBoneTransforms()
{
for (int i = 0; i < BodyState.Joints.Num(); ++i)
{
int32* BoneIndex = MappedBoneIndices.Find(static_cast<EOculusXRBoneID>(i));
if (BoneIndex != nullptr)
{
ResetBoneTransformByName(BoneNames[static_cast<EOculusXRBoneID>(i)]);
}
}
}
bool UOculusXRBodyTrackingComponent::InitializeBodyBones()
{
USkeletalMesh* BodyMesh = Cast<USkeletalMesh>(GetSkinnedAsset());
if (BodyMesh == nullptr)
{
UE_LOG(LogOculusXRMovement, Display, TEXT("No SkeletalMesh in this component."));
return false;
}
for (const auto& it : BoneNames)
{
int32 BoneIndex = GetBoneIndex(it.Value);
if (BoneIndex == INDEX_NONE)
{
UE_LOG(LogOculusXRMovement, Display, TEXT("Could not find bone %s in skeletal mesh %s"), *StaticEnum<EOculusXRBoneID>()->GetValueAsString(it.Key), *BodyMesh->GetName());
}
else
{
MappedBoneIndices.Add(it.Key, BoneIndex);
}
}
return true;
}

View File

@@ -0,0 +1,198 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXREyeTrackingComponent.h"
#include "GameFramework/WorldSettings.h"
#include "GameFramework/PlayerController.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRPluginWrapper.h"
#include "OculusXRMovementFunctionLibrary.h"
#include "OculusXRMovementHelpers.h"
#include "OculusXRMovementLog.h"
#include "OculusXRTelemetryMovementEvents.h"
int UOculusXREyeTrackingComponent::TrackingInstanceCount = 0;
UOculusXREyeTrackingComponent::UOculusXREyeTrackingComponent()
: TargetMeshComponentName(NAME_None)
, bUpdatePosition(true)
, bUpdateRotation(true)
, ConfidenceThreshold(0.f)
, bAcceptInvalid(false)
, WorldToMeters(100.f)
, TargetPoseableMeshComponent(nullptr)
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.bStartWithTickEnabled = true;
EyeToBone.Add(EOculusXREye::Left, "LeftEye");
EyeToBone.Add(EOculusXREye::Right, "RightEye");
OculusXRTelemetry::TScopedMarker<OculusXRTelemetry::Events::FMovementSDKEyeStart>(static_cast<int>(GetTypeHash(this)));
}
void UOculusXREyeTrackingComponent::BeginPlay()
{
Super::BeginPlay();
if (!UOculusXRMovementFunctionLibrary::IsEyeTrackingSupported())
{
// Early exit if eye tracking isn't supported
UE_LOG(LogOculusXRMovement, Warning, TEXT("Eye tracking is not supported. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
// Try & check initializing the eye data
if (!InitializeEyes())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to initialize eye tracking data. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
}
if (!UOculusXRMovementFunctionLibrary::StartEyeTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to start eye tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
++TrackingInstanceCount;
}
void UOculusXREyeTrackingComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
if (IsComponentTickEnabled())
{
if (--TrackingInstanceCount == 0)
{
if (!UOculusXRMovementFunctionLibrary::StopEyeTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to stop eye tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
}
}
}
Super::EndPlay(EndPlayReason);
}
void UOculusXREyeTrackingComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
if (!IsValid(TargetPoseableMeshComponent))
{
UE_LOG(LogOculusXRMovement, VeryVerbose, TEXT("No target mesh specified. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
FOculusXREyeGazesState EyeGazesState;
if (UOculusXRMovementFunctionLibrary::TryGetEyeGazesState(EyeGazesState, WorldToMeters))
{
for (uint8 i = 0u; i < static_cast<uint8>(EOculusXREye::COUNT); ++i)
{
if (PerEyeData[i].EyeIsMapped)
{
const auto& Bone = PerEyeData[i].MappedBoneName;
const auto& EyeGaze = EyeGazesState.EyeGazes[i];
if ((bAcceptInvalid || EyeGaze.bIsValid) && (EyeGaze.Confidence >= ConfidenceThreshold))
{
int32 BoneIndex = TargetPoseableMeshComponent->GetBoneIndex(Bone);
FTransform CurrentTransform = TargetPoseableMeshComponent->GetBoneTransformByName(Bone, EBoneSpaces::ComponentSpace);
if (bUpdatePosition)
{
CurrentTransform.SetLocation(EyeGaze.Position);
}
if (bUpdateRotation)
{
CurrentTransform.SetRotation(EyeGaze.Orientation.Quaternion() * PerEyeData[i].InitialRotation);
}
TargetPoseableMeshComponent->SetBoneTransformByName(Bone, CurrentTransform, EBoneSpaces::ComponentSpace);
}
}
}
}
else
{
UE_LOG(LogOculusXRMovement, VeryVerbose, TEXT("Failed to get Eye state from EyeTrackingComponent. (%s:%s)"), *GetOwner()->GetName(), *GetName());
}
}
void UOculusXREyeTrackingComponent::ClearRotationValues()
{
if (!IsValid(TargetPoseableMeshComponent))
{
UE_LOG(LogOculusXRMovement, VeryVerbose, TEXT("No target mesh specified. (%s:%s)"), *GetOwner()->GetName(), *GetName());
return;
}
for (uint8 i = 0u; i < static_cast<uint8>(EOculusXREye::COUNT); ++i)
{
if (PerEyeData[i].EyeIsMapped)
{
const auto& Bone = PerEyeData[i].MappedBoneName;
int32 BoneIndex = TargetPoseableMeshComponent->GetBoneIndex(Bone);
FTransform CurrentTransform = TargetPoseableMeshComponent->GetBoneTransformByName(Bone, EBoneSpaces::ComponentSpace);
CurrentTransform.SetRotation(PerEyeData[i].InitialRotation);
TargetPoseableMeshComponent->SetBoneTransformByName(Bone, CurrentTransform, EBoneSpaces::ComponentSpace);
}
}
}
bool UOculusXREyeTrackingComponent::InitializeEyes()
{
bool bIsAnythingMapped = false;
TargetPoseableMeshComponent = OculusXRUtility::FindComponentByName<UPoseableMeshComponent>(GetOwner(), TargetMeshComponentName);
if (!IsValid(TargetPoseableMeshComponent))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Could not find mesh with name (%s) for component. (%s:%s)"), *TargetMeshComponentName.ToString(), *GetOwner()->GetName(), *GetName());
return false;
}
for (uint8 i = 0u; i < static_cast<uint8>(EOculusXREye::COUNT); ++i)
{
const EOculusXREye Eye = static_cast<EOculusXREye>(i);
const FName* BoneNameForThisEye = EyeToBone.Find(Eye);
PerEyeData[i].EyeIsMapped = (nullptr != BoneNameForThisEye);
if (PerEyeData[i].EyeIsMapped)
{
int32 BoneIndex = TargetPoseableMeshComponent->GetBoneIndex(*BoneNameForThisEye);
if (BoneIndex == INDEX_NONE)
{
PerEyeData[i].EyeIsMapped = false; // Eye is explicitly mapped to a bone. But the bone name doesn't exist.
UE_LOG(LogOculusXRMovement, Warning, TEXT("Could not find bone by name (%s) in mesh %s. (%s:%s)"), *BoneNameForThisEye->ToString(), *TargetPoseableMeshComponent->GetName(), *GetOwner()->GetName(), *GetName());
}
else
{
PerEyeData[i].MappedBoneName = *BoneNameForThisEye;
PerEyeData[i].InitialRotation = TargetPoseableMeshComponent->GetBoneTransformByName(*BoneNameForThisEye, EBoneSpaces::ComponentSpace).GetRotation();
bIsAnythingMapped = true;
}
}
else
{
UE_LOG(LogOculusXRMovement, Display, TEXT("Eye (%s) is not mapped to any bone on mesh (%s)"), *StaticEnum<EOculusXREye>()->GetValueAsString(Eye), *TargetPoseableMeshComponent->GetName());
}
}
if (!bIsAnythingMapped)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Component name -- %s:%s, doesn't have a valid configuration."), *GetOwner()->GetName(), *GetName());
}
if (!OculusXRHMD::GetUnitScaleFactorFromSettings(GetWorld(), WorldToMeters))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot get world settings. (%s:%s)"), *GetOwner()->GetName(), *GetName());
}
return bIsAnythingMapped;
}

View File

@@ -0,0 +1,315 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRFaceTrackingComponent.h"
#include "OculusXRHMD.h"
#include "OculusXRPluginWrapper.h"
#include "OculusXRMovementFunctionLibrary.h"
#include "OculusXRMovementHelpers.h"
#include "OculusXRMovementLog.h"
#include "OculusXRTelemetryMovementEvents.h"
#include "Engine/SkeletalMesh.h"
#include "Components/SkeletalMeshComponent.h"
#include "Math/UnrealMathUtility.h"
int UOculusXRFaceTrackingComponent::TrackingInstanceCount = 0;
UOculusXRFaceTrackingComponent::UOculusXRFaceTrackingComponent()
: TargetMeshComponentName(NAME_None)
, InvalidFaceDataResetTime(2.0f)
, bUpdateFace(true)
, TargetMeshComponent(nullptr)
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.bStartWithTickEnabled = true;
// Some defaults
ExpressionNames.Add(EOculusXRFaceExpression::BrowLowererL, "browLowerer_L");
ExpressionNames.Add(EOculusXRFaceExpression::BrowLowererR, "browLowerer_R");
ExpressionNames.Add(EOculusXRFaceExpression::CheekPuffL, "cheekPuff_L");
ExpressionNames.Add(EOculusXRFaceExpression::CheekPuffR, "cheekPuff_R");
ExpressionNames.Add(EOculusXRFaceExpression::CheekRaiserL, "cheekRaiser_L");
ExpressionNames.Add(EOculusXRFaceExpression::CheekRaiserR, "cheekRaiser_R");
ExpressionNames.Add(EOculusXRFaceExpression::CheekSuckL, "cheekSuck_L");
ExpressionNames.Add(EOculusXRFaceExpression::CheekSuckR, "cheekSuck_R");
ExpressionNames.Add(EOculusXRFaceExpression::ChinRaiserB, "chinRaiser_B");
ExpressionNames.Add(EOculusXRFaceExpression::ChinRaiserT, "chinRaiser_T");
ExpressionNames.Add(EOculusXRFaceExpression::DimplerL, "dimpler_L");
ExpressionNames.Add(EOculusXRFaceExpression::DimplerR, "dimpler_R");
ExpressionNames.Add(EOculusXRFaceExpression::EyesClosedL, "eyesClosed_L");
ExpressionNames.Add(EOculusXRFaceExpression::EyesClosedR, "eyesClosed_R");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookDownL, "eyesLookDown_L");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookDownR, "eyesLookDown_R");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookLeftL, "eyesLookLeft_L");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookLeftR, "eyesLookLeft_R");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookRightL, "eyesLookRight_L");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookRightR, "eyesLookRight_R");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookUpL, "eyesLookUp_L");
ExpressionNames.Add(EOculusXRFaceExpression::EyesLookUpR, "eyesLookUp_R");
ExpressionNames.Add(EOculusXRFaceExpression::InnerBrowRaiserL, "innerBrowRaiser_L");
ExpressionNames.Add(EOculusXRFaceExpression::InnerBrowRaiserR, "innerBrowRaiser_R");
ExpressionNames.Add(EOculusXRFaceExpression::JawDrop, "jawDrop");
ExpressionNames.Add(EOculusXRFaceExpression::JawSidewaysLeft, "jawSidewaysLeft");
ExpressionNames.Add(EOculusXRFaceExpression::JawSidewaysRight, "jawSidewaysRight");
ExpressionNames.Add(EOculusXRFaceExpression::JawThrust, "jawThrust");
ExpressionNames.Add(EOculusXRFaceExpression::LidTightenerL, "lidTightener_L");
ExpressionNames.Add(EOculusXRFaceExpression::LidTightenerR, "lidTightener_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipCornerDepressorL, "lipCornerDepressor_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipCornerDepressorR, "lipCornerDepressor_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipCornerPullerL, "lipCornerPuller_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipCornerPullerR, "lipCornerPuller_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipFunnelerLB, "lipFunneler_LB");
ExpressionNames.Add(EOculusXRFaceExpression::LipFunnelerLT, "lipFunneler_LT");
ExpressionNames.Add(EOculusXRFaceExpression::LipFunnelerRB, "lipFunneler_RB");
ExpressionNames.Add(EOculusXRFaceExpression::LipFunnelerRT, "lipFunneler_RT");
ExpressionNames.Add(EOculusXRFaceExpression::LipPressorL, "lipPressor_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipPressorR, "lipPressor_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipPuckerL, "lipPucker_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipPuckerR, "lipPucker_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipStretcherL, "lipStretcher_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipStretcherR, "lipStretcher_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipSuckLB, "lipSuck_LB");
ExpressionNames.Add(EOculusXRFaceExpression::LipSuckLT, "lipSuck_LT");
ExpressionNames.Add(EOculusXRFaceExpression::LipSuckRB, "lipSuck_RB");
ExpressionNames.Add(EOculusXRFaceExpression::LipSuckRT, "lipSuck_RT");
ExpressionNames.Add(EOculusXRFaceExpression::LipTightenerL, "lipTightener_L");
ExpressionNames.Add(EOculusXRFaceExpression::LipTightenerR, "lipTightener_R");
ExpressionNames.Add(EOculusXRFaceExpression::LipsToward, "lipsToward");
ExpressionNames.Add(EOculusXRFaceExpression::LowerLipDepressorL, "lowerLipDepressor_L");
ExpressionNames.Add(EOculusXRFaceExpression::LowerLipDepressorR, "lowerLipDepressor_R");
ExpressionNames.Add(EOculusXRFaceExpression::MouthLeft, "mouthLeft");
ExpressionNames.Add(EOculusXRFaceExpression::MouthRight, "mouthRight");
ExpressionNames.Add(EOculusXRFaceExpression::NoseWrinklerL, "noseWrinkler_L");
ExpressionNames.Add(EOculusXRFaceExpression::NoseWrinklerR, "noseWrinkler_R");
ExpressionNames.Add(EOculusXRFaceExpression::OuterBrowRaiserL, "outerBrowRaiser_L");
ExpressionNames.Add(EOculusXRFaceExpression::OuterBrowRaiserR, "outerBrowRaiser_R");
ExpressionNames.Add(EOculusXRFaceExpression::UpperLidRaiserL, "upperLidRaiser_L");
ExpressionNames.Add(EOculusXRFaceExpression::UpperLidRaiserR, "upperLidRaiser_R");
ExpressionNames.Add(EOculusXRFaceExpression::UpperLipRaiserL, "upperLipRaiser_L");
ExpressionNames.Add(EOculusXRFaceExpression::UpperLipRaiserR, "upperLipRaiser_R");
ExpressionNames.Add(EOculusXRFaceExpression::TongueTipInterdental, "tongueTipInterdental");
ExpressionNames.Add(EOculusXRFaceExpression::TongueTipAlveolar, "tongueTipAlveolar");
ExpressionNames.Add(EOculusXRFaceExpression::TongueFrontDorsalPalate, "tongueFrontDorsalPalate");
ExpressionNames.Add(EOculusXRFaceExpression::TongueMidDorsalPalate, "tongueMidDorsalPalate");
ExpressionNames.Add(EOculusXRFaceExpression::TongueBackDorsalVelar, "tongueBackDorsalVelar");
ExpressionNames.Add(EOculusXRFaceExpression::TongueOut, "tongueOut");
const int defaultFaceExpressionModifierLength = 33;
ExpressionModifiers.SetNum(defaultFaceExpressionModifierLength);
ExpressionModifiers[0].FaceExpressions = { EOculusXRFaceExpression::EyesClosedL, EOculusXRFaceExpression::EyesClosedR };
ExpressionModifiers[1].FaceExpressions = { EOculusXRFaceExpression::EyesLookDownL, EOculusXRFaceExpression::EyesLookDownR };
ExpressionModifiers[2].FaceExpressions = { EOculusXRFaceExpression::EyesLookLeftL, EOculusXRFaceExpression::EyesLookLeftR };
ExpressionModifiers[3].FaceExpressions = { EOculusXRFaceExpression::EyesLookRightL, EOculusXRFaceExpression::EyesLookRightR };
ExpressionModifiers[4].FaceExpressions = { EOculusXRFaceExpression::EyesLookUpL, EOculusXRFaceExpression::EyesLookUpR };
ExpressionModifiers[5].FaceExpressions = { EOculusXRFaceExpression::LidTightenerL, EOculusXRFaceExpression::LidTightenerR };
ExpressionModifiers[6].FaceExpressions = { EOculusXRFaceExpression::UpperLidRaiserL, EOculusXRFaceExpression::UpperLidRaiserR };
ExpressionModifiers[7].FaceExpressions = { EOculusXRFaceExpression::JawDrop };
ExpressionModifiers[8].FaceExpressions = { EOculusXRFaceExpression::JawSidewaysLeft, EOculusXRFaceExpression::JawSidewaysRight };
ExpressionModifiers[9].FaceExpressions = { EOculusXRFaceExpression::JawThrust };
ExpressionModifiers[10].FaceExpressions = { EOculusXRFaceExpression::LipFunnelerLB, EOculusXRFaceExpression::LipFunnelerLT };
ExpressionModifiers[11].FaceExpressions = { EOculusXRFaceExpression::LipFunnelerRB, EOculusXRFaceExpression::LipFunnelerRT };
ExpressionModifiers[12].FaceExpressions = { EOculusXRFaceExpression::LipPuckerL, EOculusXRFaceExpression::LipPuckerR };
ExpressionModifiers[13].FaceExpressions = { EOculusXRFaceExpression::LipSuckLB, EOculusXRFaceExpression::LipSuckLT };
ExpressionModifiers[14].FaceExpressions = { EOculusXRFaceExpression::LipSuckRB, EOculusXRFaceExpression::LipSuckRT };
ExpressionModifiers[15].FaceExpressions = { EOculusXRFaceExpression::LipsToward };
ExpressionModifiers[16].FaceExpressions = { EOculusXRFaceExpression::LowerLipDepressorL, EOculusXRFaceExpression::LowerLipDepressorR };
ExpressionModifiers[17].FaceExpressions = { EOculusXRFaceExpression::ChinRaiserB, EOculusXRFaceExpression::ChinRaiserT };
ExpressionModifiers[18].FaceExpressions = { EOculusXRFaceExpression::LipCornerDepressorL, EOculusXRFaceExpression::LipCornerDepressorR };
ExpressionModifiers[19].FaceExpressions = { EOculusXRFaceExpression::LipCornerPullerL, EOculusXRFaceExpression::LipCornerPullerR };
ExpressionModifiers[20].FaceExpressions = { EOculusXRFaceExpression::LipStretcherL, EOculusXRFaceExpression::LipStretcherR };
ExpressionModifiers[21].FaceExpressions = { EOculusXRFaceExpression::MouthLeft, EOculusXRFaceExpression::MouthRight };
ExpressionModifiers[22].FaceExpressions = { EOculusXRFaceExpression::LipPressorL, EOculusXRFaceExpression::LipPressorR };
ExpressionModifiers[23].FaceExpressions = { EOculusXRFaceExpression::LipTightenerL, EOculusXRFaceExpression::LipTightenerR };
ExpressionModifiers[24].FaceExpressions = { EOculusXRFaceExpression::UpperLipRaiserL, EOculusXRFaceExpression::UpperLipRaiserR };
ExpressionModifiers[25].FaceExpressions = { EOculusXRFaceExpression::CheekPuffL, EOculusXRFaceExpression::CheekPuffR };
ExpressionModifiers[26].FaceExpressions = { EOculusXRFaceExpression::CheekRaiserL, EOculusXRFaceExpression::CheekRaiserR };
ExpressionModifiers[27].FaceExpressions = { EOculusXRFaceExpression::CheekSuckL, EOculusXRFaceExpression::CheekSuckR };
ExpressionModifiers[28].FaceExpressions = { EOculusXRFaceExpression::DimplerL, EOculusXRFaceExpression::DimplerR };
ExpressionModifiers[29].FaceExpressions = { EOculusXRFaceExpression::NoseWrinklerL, EOculusXRFaceExpression::NoseWrinklerR };
ExpressionModifiers[30].FaceExpressions = { EOculusXRFaceExpression::BrowLowererL, EOculusXRFaceExpression::BrowLowererR };
ExpressionModifiers[31].FaceExpressions = { EOculusXRFaceExpression::InnerBrowRaiserL, EOculusXRFaceExpression::InnerBrowRaiserR };
ExpressionModifiers[32].FaceExpressions = { EOculusXRFaceExpression::OuterBrowRaiserL, EOculusXRFaceExpression::OuterBrowRaiserR };
OculusXRTelemetry::TScopedMarker<OculusXRTelemetry::Events::FMovementSDKFaceStart>(static_cast<int>(GetTypeHash(this)));
}
void UOculusXRFaceTrackingComponent::BeginPlay()
{
Super::BeginPlay();
if (!UOculusXRMovementFunctionLibrary::IsFaceTrackingSupported())
{
// Early exit if face tracking isn't supported
UE_LOG(LogOculusXRMovement, Warning, TEXT("Face tracking is not supported. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
if (TargetMeshComponentName == NAME_None)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Invalid mesh component name. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
if (!InitializeFaceTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to initialize face tracking. (%s:%s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
if (!UOculusXRMovementFunctionLibrary::StartFaceTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to start face tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
SetComponentTickEnabled(false);
return;
}
++TrackingInstanceCount;
}
void UOculusXRFaceTrackingComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
if (IsComponentTickEnabled())
{
if (--TrackingInstanceCount == 0)
{
if (!UOculusXRMovementFunctionLibrary::StopFaceTracking())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Failed to stop face tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName());
}
}
}
Super::EndPlay(EndPlayReason);
}
void UOculusXRFaceTrackingComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
if (!IsValid(TargetMeshComponent))
{
UE_LOG(LogOculusXRMovement, VeryVerbose, TEXT("No target mesh specified. (%s:%s)"), *GetOwner()->GetName(), *GetName());
return;
}
if (UOculusXRMovementFunctionLibrary::TryGetFaceState(FaceState) && bUpdateFace)
{
InvalidFaceStateTimer = 0.0f;
MorphTargets.ResetMorphTargetCurves(TargetMeshComponent);
for (int32 FaceExpressionIndex = 0; FaceExpressionIndex < static_cast<int32>(EOculusXRFaceExpression::COUNT); ++FaceExpressionIndex)
{
if (ExpressionValid[FaceExpressionIndex])
{
FName ExpressionName = ExpressionNames[static_cast<EOculusXRFaceExpression>(FaceExpressionIndex)];
MorphTargets.SetMorphTarget(ExpressionName, FaceState.ExpressionWeights[FaceExpressionIndex]);
}
}
if (bUseModifiers)
{
for (int32 FaceExpressionModifierIndex = 0; FaceExpressionModifierIndex < ExpressionModifiers.Num(); ++FaceExpressionModifierIndex)
{
for (int32 FaceExpressionIndex = 0; FaceExpressionIndex < ExpressionModifiers[FaceExpressionModifierIndex].FaceExpressions.Num(); ++FaceExpressionIndex)
{
auto Expression = ExpressionModifiers[FaceExpressionModifierIndex].FaceExpressions[FaceExpressionIndex];
if (ExpressionValid[static_cast<int32>(Expression)])
{
FName ExpressionName = ExpressionNames[Expression];
float currentValue = MorphTargets.GetMorphTarget(ExpressionName);
currentValue = FMath::Clamp(
currentValue * ExpressionModifiers[FaceExpressionModifierIndex].Multiplier,
ExpressionModifiers[FaceExpressionModifierIndex].MinValue,
ExpressionModifiers[FaceExpressionModifierIndex].MaxValue);
MorphTargets.SetMorphTarget(ExpressionName, currentValue);
}
}
}
}
}
else
{
InvalidFaceStateTimer += DeltaTime;
if (InvalidFaceStateTimer >= InvalidFaceDataResetTime)
{
MorphTargets.ResetMorphTargetCurves(TargetMeshComponent);
}
}
MorphTargets.ApplyMorphTargets(TargetMeshComponent);
}
void UOculusXRFaceTrackingComponent::SetExpressionValue(EOculusXRFaceExpression Expression, float Value)
{
if (Expression >= EOculusXRFaceExpression::COUNT)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot set expression value with invalid expression index."));
return;
}
if (!ExpressionValid[static_cast<int32>(Expression)])
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot set expression value for an expression with an invalid associated morph target name. Expression name: %s"), *StaticEnum<EOculusXRFaceExpression>()->GetValueAsString(Expression));
return;
}
FName ExpressionName = ExpressionNames[Expression];
MorphTargets.SetMorphTarget(ExpressionName, Value);
}
float UOculusXRFaceTrackingComponent::GetExpressionValue(EOculusXRFaceExpression Expression) const
{
if (Expression >= EOculusXRFaceExpression::COUNT)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot request expression value using an invalid expression index."));
return 0.0f;
}
FName ExpressionName = ExpressionNames[Expression];
if (ExpressionName == NAME_None)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot request expression value for an expression with an invalid associated morph target name. Expression name: %s"), *StaticEnum<EOculusXRFaceExpression>()->GetValueAsString(Expression));
return 0.0f;
}
return MorphTargets.GetMorphTarget(ExpressionName);
}
void UOculusXRFaceTrackingComponent::ClearExpressionValues()
{
MorphTargets.ClearMorphTargets();
}
bool UOculusXRFaceTrackingComponent::InitializeFaceTracking()
{
TargetMeshComponent = OculusXRUtility::FindComponentByName<USkinnedMeshComponent>(GetOwner(), TargetMeshComponentName);
if (!IsValid(TargetMeshComponent))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Could not find skeletal mesh component with name: (%s). (%s:%s)"), *TargetMeshComponentName.ToString(), *GetOwner()->GetName(), *GetName());
return false;
}
if (TargetMeshComponent != nullptr)
{
USkeletalMesh* TargetMesh = Cast<USkeletalMesh>(TargetMeshComponent->GetSkinnedAsset());
if (TargetMesh != nullptr)
{
const TMap<FName, int32>& MorphTargetIndexMap = TargetMesh->GetMorphTargetIndexMap();
for (const auto& it : ExpressionNames)
{
ExpressionValid[static_cast<int32>(it.Key)] = MorphTargetIndexMap.Contains(it.Value);
}
return true;
}
}
return false;
}

View File

@@ -0,0 +1,157 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRLiveLinkRetargetBodyAsset.h"
#include "LiveLinkTypes.h"
#include "Algo/Accumulate.h"
#include "Algo/ForEach.h"
#include "Roles/LiveLinkAnimationTypes.h"
#include "BonePose.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRMovementLog.h"
#include "OculusXRMovement.h"
namespace
{
// EOculusXRAxis to orientation of that direction
FTransform DirectionTransform(EOculusXRAxis Direction)
{
FVector Dir = FVector::ZeroVector;
const uint8 IndexOfDir = static_cast<uint8>(Direction);
const double Sign = IndexOfDir < static_cast<uint8>(EOculusXRAxis::NegativeX) ? 1 : -1;
Dir[IndexOfDir % 3] = Sign * 1.0;
return FTransform(Dir.ToOrientationQuat());
}
} // namespace
UOculusXRLiveLinkRetargetBodyAsset::UOculusXRLiveLinkRetargetBodyAsset(const FObjectInitializer& ObjectInitializer)
: Super(ObjectInitializer), RetargetingMode(EOculusXRRetargetingMode::Full), ForwardMesh(EOculusXRAxis::X), Scale(100.f), TrackingSpaceToMeshSpace(FTransform::Identity), BoneNames(InPlace, NAME_None), LastBoneContainerSerialNumber(0)
{
}
void UOculusXRLiveLinkRetargetBodyAsset::Initialize()
{
TrackingSpaceToMeshSpace = DirectionTransform(ForwardTracking).Inverse() * DirectionTransform(ForwardMesh);
GlobalBoneCorrection = FTransform(GlobalCorrection.RotationOffset, GlobalCorrection.PositionOffset);
for (uint8 BoneId = 0; BoneId < static_cast<uint8>(EOculusXRBoneID::COUNT); ++BoneId)
{
const FTransform LocalCorrectionCombined = Algo::Accumulate(LocalCorrections, FTransform::Identity, [BoneId](FTransform Correction, const FOculusXRBoneCorrectionSet& BoneCorrectionSet) {
if (BoneCorrectionSet.Bones.Contains(static_cast<EOculusXRBoneID>(BoneId)))
{
Correction *= FTransform(BoneCorrectionSet.BoneCorrection.RotationOffset, BoneCorrectionSet.BoneCorrection.PositionOffset);
}
return Correction;
});
LocalBoneCorrections[BoneId] = LocalCorrectionCombined;
const EOculusXRBoneID OculusBoneID = static_cast<EOculusXRBoneID>(BoneId);
if (const FName* NameMapping = BoneRemapping.Find(OculusBoneID))
{
BoneNames[BoneId] = *NameMapping;
}
else
{
BoneNames[BoneId] = NAME_None;
UE_LOG(LogOculusXRMovement, Warning, TEXT("Bone: %s isn't mapped."), *StaticEnum<EOculusXRBoneID>()->GetValueAsString(OculusBoneID));
}
}
if (!OculusXRHMD::GetUnitScaleFactorFromSettings(UObject::GetWorld(), Scale))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Cannot get world settings for body retargetting asset."));
}
LastBoneContainerSerialNumber = 0;
Algo::ForEach(LastSkeletonBoneRemapping, [](FCompactPoseBoneIndex& BoneIndex) { BoneIndex = FCompactPoseBoneIndex(INDEX_NONE); });
}
void UOculusXRLiveLinkRetargetBodyAsset::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose)
{
check(InFrameData);
if (static_cast<int32>(EOculusXRBoneID::COUNT) != InFrameData->Transforms.Num())
{
UE_LOG(LogOculusXRMovement, Error, TEXT("Received wrong data of live link frame. This retargeting asset must be used with Meta MovementSDK Live Link source and Body subject. (received %d bone transforms, expected %d)"), InFrameData->Transforms.Num(), static_cast<int32>(EOculusXRBoneID::COUNT));
return;
}
if ((LastBoneContainerSerialNumber != OutPose.GetBoneContainer().GetSerialNumber()) || (LastBoneContainerSerialNumber == 0))
{
OnBoneContainerChanged(OutPose.GetBoneContainer());
}
FCSPose<FCompactPose> MeshPoses;
MeshPoses.InitPose(OutPose);
for (uint8 BoneId = 0; BoneId < static_cast<uint8>(EOculusXRBoneID::COUNT); ++BoneId)
{
if (const FCompactPoseBoneIndex& BoneIndex = LastSkeletonBoneRemapping[BoneId]; BoneIndex != INDEX_NONE)
{
FTransform BoneTransform = InFrameData->Transforms[BoneId];
BoneTransform.ScaleTranslation(Scale);
BoneTransform *= TrackingSpaceToMeshSpace;
BoneTransform = GlobalBoneCorrection * BoneTransform;
BoneTransform = LocalBoneCorrections[BoneId] * BoneTransform;
check(!BoneTransform.ContainsNaN());
switch (RetargetingMode)
{
case EOculusXRRetargetingMode::Rotations:
BoneTransform.SetLocation(MeshPoses.GetComponentSpaceTransform(BoneIndex).GetLocation());
MeshPoses.SetComponentSpaceTransform(BoneIndex, BoneTransform);
break;
case EOculusXRRetargetingMode::RotationsPlusRoot:
if (BoneId != static_cast<uint8>(EOculusXRBoneID::BodyRoot))
{
BoneTransform.SetLocation(MeshPoses.GetComponentSpaceTransform(BoneIndex).GetLocation());
}
MeshPoses.SetComponentSpaceTransform(BoneIndex, BoneTransform);
break;
case EOculusXRRetargetingMode::RotationsPlusHips:
if (BoneId != static_cast<uint8>(EOculusXRBoneID::BodyHips))
{
BoneTransform.SetLocation(MeshPoses.GetComponentSpaceTransform(BoneIndex).GetLocation());
}
MeshPoses.SetComponentSpaceTransform(BoneIndex, BoneTransform);
break;
case EOculusXRRetargetingMode::Full:
MeshPoses.SetComponentSpaceTransform(BoneIndex, BoneTransform);
break;
case EOculusXRRetargetingMode::None:
default:
break;
}
}
}
FCSPose<FCompactPose>::ConvertComponentPosesToLocalPosesSafe(MeshPoses, OutPose);
}
void UOculusXRLiveLinkRetargetBodyAsset::OnBoneContainerChanged(const FBoneContainer& BoneContainer)
{
LastBoneContainerSerialNumber = 0;
for (uint8 BoneId = 0; BoneId < static_cast<uint8>(EOculusXRBoneID::COUNT); ++BoneId)
{
const auto& BoneName = BoneNames[BoneId];
if (BoneName.IsNone())
{
LastSkeletonBoneRemapping[BoneId] = FCompactPoseBoneIndex(INDEX_NONE);
continue;
}
if (const int32 MeshIndex = BoneContainer.GetPoseBoneIndexForBoneName(BoneName); MeshIndex != INDEX_NONE)
{
LastSkeletonBoneRemapping[BoneId] = BoneContainer.MakeCompactPoseIndex(FMeshPoseBoneIndex(MeshIndex));
if (LastSkeletonBoneRemapping[BoneId] == INDEX_NONE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("Bone %s was intentionally mapped to %s. But this target doesn't exist in skeleton."), *StaticEnum<EOculusXRBoneID>()->GetValueAsString(static_cast<EOculusXRBoneID>(BoneId)), *BoneName.ToString());
}
}
}
LastBoneContainerSerialNumber = BoneContainer.GetSerialNumber();
}

View File

@@ -0,0 +1,82 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRLiveLinkRetargetFaceAsset.h"
#include "LiveLinkTypes.h"
#include "Algo/ForEach.h"
#include "Animation/AnimCurveTypes.h"
#include "BonePose.h"
#include "OculusXRMovement.h"
#include "OculusXRMovementLog.h"
UOculusXRLiveLinkRetargetFaceAsset::UOculusXRLiveLinkRetargetFaceAsset(const FObjectInitializer& ObjectInitializer)
: Super(ObjectInitializer)
{
}
void UOculusXRLiveLinkRetargetFaceAsset::Initialize()
{
LastSkeletonGuid.Invalidate();
#if UE_VERSION_OLDER_THAN(5, 3, 0)
Algo::ForEach(RemappingForLastSkeleton, [](TArray<SmartName::UID_Type>& Arr) { Arr.Reset(); });
#else
Algo::ForEach(RemappingForLastSkeleton, [](TArray<FName>& Arr) { Arr.Reset(); });
#endif
}
void UOculusXRLiveLinkRetargetFaceAsset::BuildPoseAndCurveFromBaseData(float DeltaTime, const FLiveLinkBaseStaticData* InBaseStaticData, const FLiveLinkBaseFrameData* InBaseFrameData, FCompactPose& OutPose, FBlendedCurve& OutCurve)
{
check(InBaseFrameData);
if (static_cast<int32>(EOculusXRFaceExpression::COUNT) != InBaseFrameData->PropertyValues.Num())
{
UE_LOG(LogOculusXRMovement, Error, TEXT("Received wrong data of live link frame. This retargeting asset must be used with Meta MovementSDK Live Link source and Face subject. (received %d face expressions, expected %d)"), InBaseFrameData->PropertyValues.Num(), static_cast<int32>(EOculusXRFaceExpression::COUNT));
return;
}
const USkeleton* Skeleton = OutPose.GetBoneContainer().GetSkeletonAsset();
if (!IsValid(Skeleton))
{
UE_LOG(LogOculusXRMovement, Error, TEXT("No skeleton asset for this retargeting."));
return;
}
if (LastSkeletonGuid != Skeleton->GetGuid())
{
OnSkeletonChanged(Skeleton);
}
for (uint8 ExpressionId = 0; ExpressionId < static_cast<uint8>(EOculusXRFaceExpression::COUNT); ++ExpressionId)
{
#if UE_VERSION_OLDER_THAN(5, 3, 0)
for (const SmartName::UID_Type UID : RemappingForLastSkeleton[ExpressionId])
{
OutCurve.Set(UID, InBaseFrameData->PropertyValues[ExpressionId]);
}
#else
for (const FName Name : RemappingForLastSkeleton[ExpressionId])
{
OutCurve.Set(Name, InBaseFrameData->PropertyValues[ExpressionId]);
}
#endif
}
}
void UOculusXRLiveLinkRetargetFaceAsset::OnSkeletonChanged(const USkeleton* Skeleton)
{
Initialize();
for (const auto& [ExpressionId, CurveMapping] : CurveRemapping)
{
for (const auto& CurveName : CurveMapping.CurveNames)
{
#if UE_VERSION_OLDER_THAN(5, 3, 0)
if (const SmartName::UID_Type UID = Skeleton->GetUIDByName(USkeleton::AnimCurveMappingName, CurveName); UID != SmartName::MaxUID)
{
RemappingForLastSkeleton[static_cast<uint8>(ExpressionId)].Emplace(UID);
}
#else
RemappingForLastSkeleton[static_cast<uint8>(ExpressionId)].Emplace(CurveName);
#endif
}
}
LastSkeletonGuid = Skeleton->GetGuid();
}

View File

@@ -0,0 +1,82 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRLiveLinkRetargetFaceVisemesAsset.h"
#include "LiveLinkTypes.h"
#include "Algo/ForEach.h"
#include "Animation/AnimCurveTypes.h"
#include "BonePose.h"
#include "OculusXRMovement.h"
#include "OculusXRMovementLog.h"
UOculusXRLiveLinkRetargetFaceVisemesAsset::UOculusXRLiveLinkRetargetFaceVisemesAsset(const FObjectInitializer& ObjectInitializer)
: Super(ObjectInitializer)
{
}
void UOculusXRLiveLinkRetargetFaceVisemesAsset::Initialize()
{
LastSkeletonGuid.Invalidate();
#if UE_VERSION_OLDER_THAN(5, 3, 0)
Algo::ForEach(RemappingForLastSkeleton, [](TArray<SmartName::UID_Type>& Arr) { Arr.Reset(); });
#else
Algo::ForEach(RemappingForLastSkeleton, [](TArray<FName>& Arr) { Arr.Reset(); });
#endif
}
void UOculusXRLiveLinkRetargetFaceVisemesAsset::BuildPoseAndCurveFromBaseData(float DeltaTime, const FLiveLinkBaseStaticData* InBaseStaticData, const FLiveLinkBaseFrameData* InBaseFrameData, FCompactPose& OutPose, FBlendedCurve& OutCurve)
{
check(InBaseFrameData);
if (static_cast<int32>(EOculusXRFaceVisemesExpression::COUNT) != InBaseFrameData->PropertyValues.Num())
{
UE_LOG(LogOculusXRMovement, Error, TEXT("Received wrong data of live link frame. This retargeting asset must be used with Meta MovementSDK Live Link source and Face subject. (received %d face expressions, expected %d)"), InBaseFrameData->PropertyValues.Num(), static_cast<int32>(EOculusXRFaceVisemesExpression::COUNT));
return;
}
const USkeleton* Skeleton = OutPose.GetBoneContainer().GetSkeletonAsset();
if (!IsValid(Skeleton))
{
UE_LOG(LogOculusXRMovement, Error, TEXT("No skeleton asset for this retargeting."));
return;
}
if (LastSkeletonGuid != Skeleton->GetGuid())
{
OnSkeletonChanged(Skeleton);
}
for (uint8 ExpressionId = 0; ExpressionId < static_cast<uint8>(EOculusXRFaceVisemesExpression::COUNT); ++ExpressionId)
{
#if UE_VERSION_OLDER_THAN(5, 3, 0)
for (const SmartName::UID_Type UID : RemappingForLastSkeleton[ExpressionId])
{
OutCurve.Set(UID, InBaseFrameData->PropertyValues[ExpressionId]);
}
#else
for (const FName Name : RemappingForLastSkeleton[ExpressionId])
{
OutCurve.Set(Name, InBaseFrameData->PropertyValues[ExpressionId]);
}
#endif
}
}
void UOculusXRLiveLinkRetargetFaceVisemesAsset::OnSkeletonChanged(const USkeleton* Skeleton)
{
Initialize();
for (const auto& [ExpressionId, CurveMapping] : CurveRemapping)
{
for (const auto& CurveName : CurveMapping.CurveNames)
{
#if UE_VERSION_OLDER_THAN(5, 3, 0)
if (const SmartName::UID_Type UID = Skeleton->GetUIDByName(USkeleton::AnimCurveMappingName, CurveName); UID != SmartName::MaxUID)
{
RemappingForLastSkeleton[static_cast<uint8>(ExpressionId)].Emplace(UID);
}
#else
RemappingForLastSkeleton[static_cast<uint8>(ExpressionId)].Emplace(CurveName);
#endif
}
}
LastSkeletonGuid = Skeleton->GetGuid();
}

View File

@@ -0,0 +1,90 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMorphTargetsController.h"
#include "Components/SkeletalMeshComponent.h"
#include "Engine/SkeletalMesh.h"
#include "AnimationRuntime.h"
void FOculusXRMorphTargetsController::ResetMorphTargetCurves(USkinnedMeshComponent* TargetMeshComponent)
{
if (TargetMeshComponent)
{
TargetMeshComponent->ActiveMorphTargets.Reset();
USkeletalMesh* TargetMesh = Cast<USkeletalMesh>(TargetMeshComponent->GetSkinnedAsset());
if (TargetMesh != nullptr)
{
TargetMeshComponent->MorphTargetWeights.SetNum(TargetMesh->GetMorphTargets().Num());
// we need this code to ensure the buffer gets cleared whether or not you have morphtarget curve set
// the case, where you had morphtargets weight on, and when you clear the weight, you want to make sure
// the buffer gets cleared and resized
if (TargetMeshComponent->MorphTargetWeights.Num() > 0)
{
FMemory::Memzero(TargetMeshComponent->MorphTargetWeights.GetData(), TargetMeshComponent->MorphTargetWeights.GetAllocatedSize());
}
}
else
{
TargetMeshComponent->MorphTargetWeights.Reset();
}
}
}
void FOculusXRMorphTargetsController::ApplyMorphTargets(USkinnedMeshComponent* TargetMeshComponent)
{
if (TargetMeshComponent != nullptr)
{
const USkeletalMesh* TargetMesh = Cast<USkeletalMesh>(TargetMeshComponent->GetSkinnedAsset());
if (TargetMesh != nullptr && MorphTargetCurves.Num() > 0)
{
FAnimationRuntime::AppendActiveMorphTargets(TargetMesh, MorphTargetCurves, TargetMeshComponent->ActiveMorphTargets, TargetMeshComponent->MorphTargetWeights);
}
}
}
void FOculusXRMorphTargetsController::SetMorphTarget(FName MorphTargetName, float Value)
{
float* CurveValPtr = MorphTargetCurves.Find(MorphTargetName);
bool bShouldAddToList = FPlatformMath::Abs(Value) > ZERO_ANIMWEIGHT_THRESH;
if (bShouldAddToList)
{
if (CurveValPtr)
{
// sum up, in the future we might normalize, but for now this just sums up
// this won't work well if all of them have full weight - i.e. additive
*CurveValPtr = Value;
}
else
{
MorphTargetCurves.Add(MorphTargetName, Value);
}
}
// if less than ZERO_ANIMWEIGHT_THRESH
// no reason to keep them on the list
else
{
// remove if found
MorphTargetCurves.Remove(MorphTargetName);
}
}
float FOculusXRMorphTargetsController::GetMorphTarget(FName MorphTargetName) const
{
const float* CurveValPtr = MorphTargetCurves.Find(MorphTargetName);
if (CurveValPtr)
{
return *CurveValPtr;
}
else
{
return 0.0f;
}
}
void FOculusXRMorphTargetsController::ClearMorphTargets()
{
MorphTargetCurves.Empty();
}

View File

@@ -0,0 +1,193 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovement.h"
#include "OculusXRMovementLog.h"
#include "OculusXRMovementModule.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRHMD.h"
#include "OculusXRPluginWrapper.h"
#include "OculusXRMovementFunctionsOVR.h"
#include "OculusXRMovementFunctionsOpenXR.h"
#include "Logging/MessageLog.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
bool OculusXRMovement::GetBodyState(FOculusXRBodyState& outOculusXRBodyState, float WorldToMeters)
{
return GetOculusXRMovementFunctionsImpl()->GetBodyState(outOculusXRBodyState, WorldToMeters);
}
bool OculusXRMovement::GetBodySkeleton(FOculusXRBodySkeleton& outOculusXRBodyState, float WorldToMeters)
{
return GetOculusXRMovementFunctionsImpl()->GetBodySkeleton(outOculusXRBodyState, WorldToMeters);
}
bool OculusXRMovement::IsBodyTrackingEnabled()
{
return GetOculusXRMovementFunctionsImpl()->IsBodyTrackingEnabled();
}
bool OculusXRMovement::IsBodyTrackingSupported()
{
return GetOculusXRMovementFunctionsImpl()->IsBodyTrackingSupported();
}
bool OculusXRMovement::RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity)
{
return GetOculusXRMovementFunctionsImpl()->RequestBodyTrackingFidelity(fidelity);
}
bool OculusXRMovement::ResetBodyTrackingCalibration()
{
return GetOculusXRMovementFunctionsImpl()->ResetBodyTrackingCalibration();
}
bool OculusXRMovement::SuggestBodyTrackingCalibrationOverride(float height)
{
return GetOculusXRMovementFunctionsImpl()->SuggestBodyTrackingCalibrationOverride(height);
}
bool OculusXRMovement::StartBodyTracking()
{
return GetOculusXRMovementFunctionsImpl()->StartBodyTracking();
}
bool OculusXRMovement::StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet)
{
return GetOculusXRMovementFunctionsImpl()->StartBodyTrackingByJointSet(jointSet);
}
bool OculusXRMovement::StopBodyTracking()
{
return GetOculusXRMovementFunctionsImpl()->StopBodyTracking();
}
bool OculusXRMovement::GetFaceState(FOculusXRFaceState& outOculusXRFaceState)
{
return GetOculusXRMovementFunctionsImpl()->GetFaceState(outOculusXRFaceState);
}
bool OculusXRMovement::IsFaceTrackingEnabled()
{
return GetOculusXRMovementFunctionsImpl()->IsFaceTrackingEnabled();
}
bool OculusXRMovement::IsFaceTrackingSupported()
{
return GetOculusXRMovementFunctionsImpl()->IsFaceTrackingSupported();
}
bool OculusXRMovement::IsFaceTrackingVisemesEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTrackingVisemesEnabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
bool OculusXRMovement::IsFaceTrackingVisemesSupported()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsSupported = ovrpBool_False;
ovrpResult TrackingSupportedResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTrackingVisemesSupported(&IsSupported);
if (OVRP_SUCCESS(TrackingSupportedResult))
{
bResult = (IsSupported == ovrpBool_True);
}
return bResult;
}
bool OculusXRMovement::SetFaceTrackingVisemesEnabled(bool enabled)
{
return GetOculusXRMovementFunctionsImpl()->SetFaceTrackingVisemesEnabled(enabled);
}
bool OculusXRMovement::GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState)
{
return GetOculusXRMovementFunctionsImpl()->GetFaceVisemesState(outOculusXRFaceVisemesState);
}
bool OculusXRMovement::StartFaceTracking()
{
return GetOculusXRMovementFunctionsImpl()->StartFaceTracking();
}
bool OculusXRMovement::StopFaceTracking()
{
return GetOculusXRMovementFunctionsImpl()->StopFaceTracking();
}
bool OculusXRMovement::GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters)
{
return GetOculusXRMovementFunctionsImpl()->GetEyeGazesState(outOculusXREyeGazesState, WorldToMeters);
}
bool OculusXRMovement::IsEyeTrackingEnabled()
{
return GetOculusXRMovementFunctionsImpl()->IsEyeTrackingEnabled();
}
bool OculusXRMovement::IsEyeTrackingSupported()
{
return GetOculusXRMovementFunctionsImpl()->IsEyeTrackingSupported();
}
bool OculusXRMovement::StartEyeTracking()
{
return GetOculusXRMovementFunctionsImpl()->StartEyeTracking();
}
bool OculusXRMovement::StopEyeTracking()
{
return GetOculusXRMovementFunctionsImpl()->StopEyeTracking();
}
bool OculusXRMovement::IsFullBodyTrackingEnabled()
{
return GetOculusXRMovementFunctionsImpl()->IsFullBodyTrackingEnabled();
}
TSharedPtr<IOculusXRMovementFunctions> OculusXRMovement::MovementFunctionsImpl = nullptr;
TSharedPtr<IOculusXRMovementFunctions> OculusXRMovement::GetOculusXRMovementFunctionsImpl()
{
if (MovementFunctionsImpl == nullptr)
{
const FName SystemName(TEXT("OpenXR"));
const bool IsOpenXR = GEngine->XRSystem.IsValid() && (GEngine->XRSystem->GetSystemName() == SystemName);
if (OculusXRHMD::FOculusXRHMD::GetOculusXRHMD() != nullptr)
{
MovementFunctionsImpl = MakeShared<FOculusXRMovementFunctionsOVR>();
}
else if (IsOpenXR)
{
MovementFunctionsImpl = MakeShared<FOculusXRMovementFunctionsOpenXR>();
}
}
check(MovementFunctionsImpl);
return MovementFunctionsImpl;
}
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,125 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementFunctionLibrary.h"
#include "IOculusXRMovementModule.h"
#include "LiveLinkOculusXRMovementSourceFactory.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRMovement.h"
#include "OculusXRHMD.h"
#include "OculusXRMovementLiveLink.h"
bool UOculusXRMovementFunctionLibrary::TryGetBodyState(FOculusXRBodyState& outBodyState, float WorldToMeters)
{
return OculusXRMovement::GetBodyState(outBodyState, WorldToMeters);
}
bool UOculusXRMovementFunctionLibrary::IsBodyTrackingEnabled()
{
return OculusXRMovement::IsBodyTrackingEnabled();
}
bool UOculusXRMovementFunctionLibrary::IsBodyTrackingSupported()
{
return OculusXRMovement::IsBodyTrackingSupported();
}
bool UOculusXRMovementFunctionLibrary::RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity)
{
return OculusXRMovement::RequestBodyTrackingFidelity(fidelity);
}
bool UOculusXRMovementFunctionLibrary::ResetBodyTrackingCalibration()
{
return OculusXRMovement::ResetBodyTrackingCalibration();
}
bool UOculusXRMovementFunctionLibrary::SuggestBodyTrackingCalibrationOverride(float height)
{
return OculusXRMovement::SuggestBodyTrackingCalibrationOverride(height);
}
bool UOculusXRMovementFunctionLibrary::StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet)
{
return OculusXRMovement::StartBodyTrackingByJointSet(jointSet);
}
bool UOculusXRMovementFunctionLibrary::StartBodyTracking()
{
return OculusXRMovement::StartBodyTracking();
}
bool UOculusXRMovementFunctionLibrary::StopBodyTracking()
{
return OculusXRMovement::StopBodyTracking();
}
bool UOculusXRMovementFunctionLibrary::TryGetFaceState(FOculusXRFaceState& outFaceState)
{
return OculusXRMovement::GetFaceState(outFaceState);
}
bool UOculusXRMovementFunctionLibrary::IsFaceTrackingVisemesSupported()
{
return OculusXRMovement::IsFaceTrackingVisemesSupported();
}
bool UOculusXRMovementFunctionLibrary::IsFaceTrackingVisemesEnabled()
{
return OculusXRMovement::IsFaceTrackingVisemesEnabled();
}
bool UOculusXRMovementFunctionLibrary::SetFaceTrackingVisemesEnabled(bool faceTrackingVisemesEnabled)
{
return OculusXRMovement::SetFaceTrackingVisemesEnabled(faceTrackingVisemesEnabled);
}
bool UOculusXRMovementFunctionLibrary::TryGetFaceVisemesState(FOculusXRFaceVisemesState& outFaceVisemesState)
{
return OculusXRMovement::GetFaceVisemesState(outFaceVisemesState);
}
bool UOculusXRMovementFunctionLibrary::IsFaceTrackingEnabled()
{
return OculusXRMovement::IsFaceTrackingEnabled();
}
bool UOculusXRMovementFunctionLibrary::IsFaceTrackingSupported()
{
return OculusXRMovement::IsFaceTrackingSupported();
}
bool UOculusXRMovementFunctionLibrary::StartFaceTracking()
{
return OculusXRMovement::StartFaceTracking();
}
bool UOculusXRMovementFunctionLibrary::StopFaceTracking()
{
return OculusXRMovement::StopFaceTracking();
}
bool UOculusXRMovementFunctionLibrary::TryGetEyeGazesState(FOculusXREyeGazesState& outEyeGazesState, float WorldToMeters)
{
return OculusXRMovement::GetEyeGazesState(outEyeGazesState, WorldToMeters);
}
bool UOculusXRMovementFunctionLibrary::IsEyeTrackingEnabled()
{
return OculusXRMovement::IsEyeTrackingEnabled();
}
bool UOculusXRMovementFunctionLibrary::IsEyeTrackingSupported()
{
return OculusXRMovement::IsEyeTrackingSupported();
}
bool UOculusXRMovementFunctionLibrary::StartEyeTracking()
{
return OculusXRMovement::StartEyeTracking();
}
bool UOculusXRMovementFunctionLibrary::StopEyeTracking()
{
return OculusXRMovement::StopEyeTracking();
}

View File

@@ -0,0 +1,598 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementFunctionsOVR.h"
#include "OculusXRMovementLog.h"
#include "OculusXRMovementModule.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRHMD.h"
#include "OculusXRPluginWrapper.h"
#include "Logging/MessageLog.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
bool FOculusXRMovementFunctionsOVR::GetBodyState(FOculusXRBodyState& outOculusXRBodyState, float WorldToMeters)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
static_assert(ovrpBoneId_FullBody_End == static_cast<int>(EOculusXRBoneID::COUNT), "The size of the OVRPlugin Bone ID enum should be the same as the EOculusXRBoneID enum.");
const auto AvailableJoints = IsFullBodyTrackingEnabled() ? ovrpBoneId_FullBody_End : ovrpBoneId_Body_End;
checkf(outOculusXRBodyState.Joints.Num() >= AvailableJoints, TEXT("Not enough joints in FOculusXRBodyState::Joints array. You must have at least %d joints"), AvailableJoints);
ovrpBodyState4 OVRBodyState;
ovrpResult OVRBodyStateResult = FOculusXRHMDModule::GetPluginWrapper().GetBodyState4(ovrpStep_Render, OVRP_CURRENT_FRAMEINDEX, &OVRBodyState);
ensureMsgf(OVRBodyStateResult != ovrpFailure_NotYetImplemented, TEXT("Body tracking is not implemented on this platform."));
if (OVRP_SUCCESS(OVRBodyStateResult))
{
outOculusXRBodyState.IsActive = (OVRBodyState.IsActive == ovrpBool_True);
outOculusXRBodyState.Confidence = OVRBodyState.Confidence;
outOculusXRBodyState.SkeletonChangedCount = OVRBodyState.SkeletonChangedCount;
outOculusXRBodyState.Time = static_cast<float>(OVRBodyState.Time);
for (int i = 0; i < AvailableJoints; ++i)
{
ovrpBodyJointLocation OVRJointLocation = OVRBodyState.JointLocations[i];
ovrpPosef OVRJointPose = OVRJointLocation.Pose;
FOculusXRBodyJoint& OculusXRBodyJoint = outOculusXRBodyState.Joints[i];
OculusXRBodyJoint.LocationFlags = OVRJointLocation.LocationFlags;
OculusXRBodyJoint.bIsValid = OVRJointLocation.LocationFlags & (XRSpaceFlags::XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XRSpaceFlags::XR_SPACE_LOCATION_POSITION_VALID_BIT);
OculusXRBodyJoint.Orientation = FRotator(OculusXRHMD::ToFQuat(OVRJointPose.Orientation));
OculusXRBodyJoint.Position = OculusXRHMD::ToFVector(OVRJointPose.Position) * WorldToMeters;
}
if (AvailableJoints < outOculusXRBodyState.Joints.Num())
{
for (int i = AvailableJoints; i < outOculusXRBodyState.Joints.Num(); ++i)
{
outOculusXRBodyState.Joints[i].bIsValid = false;
}
}
return true;
}
return false;
}
bool FOculusXRMovementFunctionsOVR::GetBodySkeleton(FOculusXRBodySkeleton& outOculusXRBodyState, float WorldToMeters)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
ovrpSkeleton3 OVRBodySkeleton = {};
ovrpResult OVRBodyStateResult =
FOculusXRHMDModule::GetPluginWrapper().GetSkeleton3(
(IsFullBodyTrackingEnabled() ? ovrpSkeletonType_FullBody : ovrpSkeletonType_Body),
&OVRBodySkeleton);
if (OVRP_SUCCESS(OVRBodyStateResult))
{
checkf(outOculusXRBodyState.Bones.Num() >= static_cast<int32>(OVRBodySkeleton.NumBones),
TEXT("Not enough bones in OVRBosySkeleton::Bones array. You must have at least %d joints"),
OVRBodySkeleton.NumBones);
outOculusXRBodyState.NumBones = OVRBodySkeleton.NumBones;
for (uint32 i = 0; i < OVRBodySkeleton.NumBones; ++i)
{
ovrpBone OVRBone = OVRBodySkeleton.Bones[i];
ovrpPosef OVRBonePose = OVRBone.Pose;
FOculusXRBodySkeletonBone& OculusXRBone = outOculusXRBodyState.Bones[i];
OculusXRBone.Orientation = FRotator(OculusXRHMD::ToFQuat(OVRBonePose.Orientation));
OculusXRBone.Position = OculusXRHMD::ToFVector(OVRBonePose.Position) * WorldToMeters;
if (OVRBone.ParentBoneIndex == ovrpBoneId_Invalid)
{
OculusXRBone.ParentBoneIndex = EOculusXRBoneID::None;
}
else
{
OculusXRBone.ParentBoneIndex = static_cast<EOculusXRBoneID>(OVRBone.ParentBoneIndex);
}
OculusXRBone.BoneId = static_cast<EOculusXRBoneID>(OVRBone.BoneId);
}
return true;
}
return false;
}
bool FOculusXRMovementFunctionsOVR::IsBodyTrackingEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetBodyTrackingEnabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::IsBodyTrackingSupported()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsSupported = ovrpBool_False;
ovrpResult TrackingSupportedResult = FOculusXRHMDModule::GetPluginWrapper().GetBodyTrackingSupported(&IsSupported);
if (OVRP_SUCCESS(TrackingSupportedResult))
{
bResult = (IsSupported == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
static_assert(static_cast<int>(EOculusXRBodyTrackingFidelity::Low) == static_cast<int>(ovrpBodyTrackingFidelity2::ovrpBodyTrackingFidelity2_Low), "EOculusXRBodyTrackingFidelity and ovrpBodyTrackingFidelity2 should be sync");
static_assert(static_cast<int>(EOculusXRBodyTrackingFidelity::High) == static_cast<int>(ovrpBodyTrackingFidelity2::ovrpBodyTrackingFidelity2_High), "EOculusXRBodyTrackingFidelity and ovrpBodyTrackingFidelity2 should be sync");
auto* OculusXRHMD = OculusXRHMD::FOculusXRHMD::GetOculusXRHMD();
if (OculusXRHMD)
{
OculusXRHMD->GetSettings()->BodyTrackingFidelity = static_cast<EOculusXRHMDBodyTrackingFidelity>(fidelity);
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().RequestBodyTrackingFidelity(static_cast<ovrpBodyTrackingFidelity2>(fidelity)));
}
bool FOculusXRMovementFunctionsOVR::ResetBodyTrackingCalibration()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().ResetBodyTrackingCalibration());
}
bool FOculusXRMovementFunctionsOVR::SuggestBodyTrackingCalibrationOverride(float height)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
ovrpBodyTrackingCalibrationInfo calibrationInfo{ height };
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().SuggestBodyTrackingCalibrationOverride(calibrationInfo));
}
bool FOculusXRMovementFunctionsOVR::StartBodyTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
static_assert(static_cast<int>(EOculusXRBodyJointSet::UpperBody) == static_cast<int>(ovrpBodyJointSet::ovrpBodyJointSet_UpperBody), "EOculusXRBodyJointSet and ovrpBodyJointSet should be sync");
static_assert(static_cast<int>(EOculusXRBodyJointSet::FullBody) == static_cast<int>(ovrpBodyJointSet::ovrpBodyJointSet_FullBody), "EOculusXRBodyJointSet and ovrpBodyJointSet should be sync");
bool result = false;
const auto* OculusXRHMD = OculusXRHMD::FOculusXRHMD::GetOculusXRHMD();
if (OculusXRHMD)
{
const auto JointSet = OculusXRHMD->GetSettings()->BodyTrackingJointSet;
if (!OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StartBodyTracking2(static_cast<ovrpBodyJointSet>(JointSet))))
{
return false;
}
const auto Fidelity = OculusXRHMD->GetSettings()->BodyTrackingFidelity;
FOculusXRHMDModule::GetPluginWrapper().RequestBodyTrackingFidelity(static_cast<ovrpBodyTrackingFidelity2>(Fidelity));
return true;
}
else
{
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StartBodyTracking2(ovrpBodyJointSet::ovrpBodyJointSet_UpperBody));
}
}
bool FOculusXRMovementFunctionsOVR::StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool result = false;
auto* OculusXRHMD = OculusXRHMD::FOculusXRHMD::GetOculusXRHMD();
if (OculusXRHMD)
{
OculusXRHMD->GetSettings()->BodyTrackingJointSet = static_cast<EOculusXRHMDBodyJointSet>(jointSet);
result = StartBodyTracking();
}
else
{
result = OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StartBodyTracking2(static_cast<ovrpBodyJointSet>(jointSet)));
}
return result;
}
bool FOculusXRMovementFunctionsOVR::StopBodyTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
FOculusXRHMDModule::GetPluginWrapper().RequestBodyTrackingFidelity(ovrpBodyTrackingFidelity2::ovrpBodyTrackingFidelity2_Low);
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StopBodyTracking());
}
bool FOculusXRMovementFunctionsOVR::GetFaceState(FOculusXRFaceState& outOculusXRFaceState)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
const auto blendShapeCount = ovrpFaceExpression2_Max;
static_assert(blendShapeCount == static_cast<int>(EOculusXRFaceExpression::COUNT), "The size of the OVRPlugin Face Expression enum should be the same as the EOculusXRFaceExpression enum.");
checkf(outOculusXRFaceState.ExpressionWeightConfidences.Num() >= ovrpFaceConfidence_Max, TEXT("Not enough expression weight confidences in FOculusXRFaceState::ExpressionWeightConfidences. Requires %d available elements in the array."), ovrpFaceConfidence_Max);
checkf(outOculusXRFaceState.ExpressionWeights.Num() >= blendShapeCount, TEXT("Not enough expression weights in FOculusXRFaceState::ExpressionWeights. Requires %d available elements in the array."), blendShapeCount);
ovrpFaceState2 OVRFaceState;
ovrpResult OVRFaceStateResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceState2(ovrpStep_Render, OVRP_CURRENT_FRAMEINDEX, &OVRFaceState);
ensureMsgf(OVRFaceStateResult != ovrpFailure_NotYetImplemented, TEXT("Face tracking is not implemented on this platform."));
if (OVRP_SUCCESS(OVRFaceStateResult))
{
outOculusXRFaceState.bIsValid = (OVRFaceState.Status.IsValid == ovrpBool_True);
outOculusXRFaceState.bIsEyeFollowingBlendshapesValid = (OVRFaceState.Status.IsEyeFollowingBlendshapesValid == ovrpBool_True);
outOculusXRFaceState.Time = static_cast<float>(OVRFaceState.Time);
for (int i = 0; i < blendShapeCount; ++i)
{
outOculusXRFaceState.ExpressionWeights[i] = OVRFaceState.ExpressionWeights[i];
}
for (int i = 0; i < ovrpFaceConfidence_Max; ++i)
{
outOculusXRFaceState.ExpressionWeightConfidences[i] = OVRFaceState.ExpressionWeightConfidences[i];
}
outOculusXRFaceState.DataSource = static_cast<EFaceTrackingDataSource>(OVRFaceState.DataSource);
return true;
}
return false;
}
bool FOculusXRMovementFunctionsOVR::IsFaceTrackingEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTracking2Enabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::IsFaceTrackingSupported()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsSupported = ovrpBool_False;
ovrpResult TrackingSupportedResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTracking2Supported(&IsSupported);
if (OVRP_SUCCESS(TrackingSupportedResult))
{
bResult = (IsSupported == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::StartFaceTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
const auto* OculusXRHMD = OculusXRHMD::FOculusXRHMD::GetOculusXRHMD();
if (OculusXRHMD)
{
ovrpFaceTrackingDataSource2 dataSources[ovrpFaceConstants_FaceTrackingDataSourcesCount];
int count = 0;
for (auto Iterator = OculusXRHMD->GetSettings()->FaceTrackingDataSource.CreateConstIterator(); Iterator; ++Iterator)
{
dataSources[count++] = static_cast<ovrpFaceTrackingDataSource2>(*Iterator);
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StartFaceTracking2(dataSources, count));
}
return false;
}
bool FOculusXRMovementFunctionsOVR::StopFaceTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StopFaceTracking2());
}
bool FOculusXRMovementFunctionsOVR::SetFaceTrackingVisemesEnabled(bool enabled)
{
auto* OculusXRHMD = OculusXRHMD::FOculusXRHMD::GetOculusXRHMD();
if (!OculusXRHMD)
{
return false;
}
OculusXRHMD->GetSettings()->bFaceTrackingVisemesEnabled = enabled;
if (FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTracking2Enabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult) && IsEnabled == ovrpBool_True)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("SetFaceTrackingVisemesEnabled should be called before StartFaceTracking, otherwise it takes no effect."));
return false;
}
}
return true;
}
bool FOculusXRMovementFunctionsOVR::GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
const auto visemeCount = ovrpFaceViseme_Max;
static_assert(visemeCount == static_cast<int>(EOculusXRFaceVisemesExpression::COUNT), "The size of the OVRPlugin Face Visemes Expression enum should be the same as the EOculusXRFaceVisemesExpression enum.");
checkf(outOculusXRFaceVisemesState.ExpressionVisemeWeights.Num() >= visemeCount, TEXT("Not enough expression weights in FOculusXRFaceVisemesState::ExpressionVisemeWeights. Requires %d available elements in the array."), visemeCount);
ovrpFaceVisemesState OVRFaceVisemesState;
ovrpResult OVRFaceVisemesStateResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceVisemesState(ovrpStep_Render, OVRP_CURRENT_FRAMEINDEX, &OVRFaceVisemesState);
if (!OVRP_SUCCESS(OVRFaceVisemesStateResult))
{
return false;
}
outOculusXRFaceVisemesState.bIsValid = (OVRFaceVisemesState.IsValid == ovrpBool_True);
outOculusXRFaceVisemesState.Time = static_cast<float>(OVRFaceVisemesState.Time);
for (int i = 0; i < visemeCount; ++i)
{
outOculusXRFaceVisemesState.ExpressionVisemeWeights[i] = OVRFaceVisemesState.Visemes[i];
}
return true;
}
bool FOculusXRMovementFunctionsOVR::IsFaceTrackingVisemesEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTrackingVisemesEnabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::IsFaceTrackingVisemesSupported()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsSupported = ovrpBool_False;
ovrpResult TrackingSupportedResult = FOculusXRHMDModule::GetPluginWrapper().GetFaceTrackingVisemesSupported(&IsSupported);
if (OVRP_SUCCESS(TrackingSupportedResult))
{
bResult = (IsSupported == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters)
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
static_assert(ovrpEye_Count == (int)EOculusXREye::COUNT, "The size of the OVRPlugin Eye enum should be the same as the EOculusXREye enum.");
checkf(outOculusXREyeGazesState.EyeGazes.Num() >= ovrpEye_Count, TEXT("Not enough eye gaze states in FOculusXREyeGazesState::EyeGazes. Requires %d available elements in the array."), ovrpEye_Count);
ovrpEyeGazesState OVREyeGazesState;
ovrpResult OVREyeGazesStateResult = FOculusXRHMDModule::GetPluginWrapper().GetEyeGazesState(ovrpStep_Render, OVRP_CURRENT_FRAMEINDEX, &OVREyeGazesState);
ensureMsgf(OVREyeGazesStateResult != ovrpFailure_NotYetImplemented, TEXT("Eye tracking is not implemented on this platform."));
if (OVRP_SUCCESS(OVREyeGazesStateResult))
{
outOculusXREyeGazesState.Time = static_cast<float>(OVREyeGazesState.Time);
for (int i = 0; i < ovrpEye_Count; ++i)
{
const auto& EyeGazePose = OVREyeGazesState.EyeGazes[i].Pose;
outOculusXREyeGazesState.EyeGazes[i].Orientation = FRotator(OculusXRHMD::ToFQuat(EyeGazePose.Orientation));
outOculusXREyeGazesState.EyeGazes[i].Position = OculusXRHMD::ToFVector(EyeGazePose.Position) * WorldToMeters;
outOculusXREyeGazesState.EyeGazes[i].bIsValid = (OVREyeGazesState.EyeGazes[i].IsValid == ovrpBool_True);
outOculusXREyeGazesState.EyeGazes[i].Confidence = OVREyeGazesState.EyeGazes[i].Confidence;
}
return true;
}
return false;
}
bool FOculusXRMovementFunctionsOVR::IsEyeTrackingEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetEyeTrackingEnabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::IsEyeTrackingSupported()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsSupported = ovrpBool_False;
ovrpResult TrackingSupportedResult = FOculusXRHMDModule::GetPluginWrapper().GetEyeTrackingSupported(&IsSupported);
if (OVRP_SUCCESS(TrackingSupportedResult))
{
bResult = (IsSupported == ovrpBool_True);
}
return bResult;
}
bool FOculusXRMovementFunctionsOVR::StartEyeTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StartEyeTracking());
}
bool FOculusXRMovementFunctionsOVR::StopEyeTracking()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
return OVRP_SUCCESS(FOculusXRHMDModule::GetPluginWrapper().StopEyeTracking());
}
bool FOculusXRMovementFunctionsOVR::IsFullBodyTrackingEnabled()
{
// Prevent calling plugin functions if the plugin is not available
if (!FOculusXRHMDModule::Get().IsOVRPluginAvailable())
{
return false;
}
bool bResult = false;
ovrpBool IsEnabled = ovrpBool_False;
ovrpResult TrackingEnabledResult = FOculusXRHMDModule::GetPluginWrapper().GetFullBodyTrackingEnabled(&IsEnabled);
if (OVRP_SUCCESS(TrackingEnabledResult))
{
bResult = (IsEnabled == ovrpBool_True);
}
return bResult;
}
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,39 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementFunctions.h"
struct FOculusXRMovementFunctionsOVR : public IOculusXRMovementFunctions
{
public:
virtual bool GetBodyState(FOculusXRBodyState& outOculusXRBodyState, float WorldToMeters) override;
virtual bool IsBodyTrackingEnabled() override;
virtual bool IsBodyTrackingSupported() override;
virtual bool StartBodyTracking() override;
virtual bool StopBodyTracking() override;
virtual bool StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet) override;
virtual bool RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity) override;
virtual bool ResetBodyTrackingCalibration() override;
virtual bool SuggestBodyTrackingCalibrationOverride(float height) override;
virtual bool GetBodySkeleton(FOculusXRBodySkeleton& outOculusXRBodyState, float WorldToMeters) override;
virtual bool GetFaceState(FOculusXRFaceState& outOculusXRFaceState) override;
virtual bool IsFaceTrackingEnabled() override;
virtual bool IsFaceTrackingSupported() override;
virtual bool StartFaceTracking() override;
virtual bool StopFaceTracking() override;
virtual bool SetFaceTrackingVisemesEnabled(bool enabled) override;
virtual bool GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState) override;
virtual bool IsFaceTrackingVisemesEnabled() override;
virtual bool IsFaceTrackingVisemesSupported() override;
virtual bool GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters) override;
virtual bool IsEyeTrackingEnabled() override;
virtual bool IsEyeTrackingSupported() override;
virtual bool StartEyeTracking() override;
virtual bool StopEyeTracking() override;
virtual bool IsFullBodyTrackingEnabled() override;
};

View File

@@ -0,0 +1,156 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementFunctionsOpenXR.h"
#include "OculusXRMovementLog.h"
#include "OculusXRMovementModule.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRHMD.h"
#include "OpenXRHMD.h"
#include "Logging/MessageLog.h"
#include "OpenXR/OculusXROpenXRUtilities.h"
#include "openxr/OculusXRBodyTrackingXR.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
bool FOculusXRMovementFunctionsOpenXR::GetBodyState(FOculusXRBodyState& outState, float WorldToMeters)
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->GetCachedBodyState(outState);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::GetBodySkeleton(FOculusXRBodySkeleton& outSkeleton, float WorldToMeters)
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->GetBodySkeleton(outSkeleton);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::IsBodyTrackingEnabled()
{
return FOculusXRMovementModule::Get().GetXrBodyTracker()->IsBodyTrackingEnabled();
}
bool FOculusXRMovementFunctionsOpenXR::IsBodyTrackingSupported()
{
return FOculusXRMovementModule::Get().GetXrBodyTracker()->IsBodyTrackingSupported();
}
bool FOculusXRMovementFunctionsOpenXR::IsFullBodyTrackingEnabled()
{
return FOculusXRMovementModule::Get().GetXrBodyTracker()->IsFullBodyTrackingEnabled();
}
bool FOculusXRMovementFunctionsOpenXR::StartBodyTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->StartBodyTracking();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet)
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->StartBodyTrackingByJointSet(jointSet);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::StopBodyTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->StopBodyTracking();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity)
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->RequestBodyTrackingFidelity(fidelity);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::ResetBodyTrackingCalibration()
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->ResetBodyTrackingFidelity();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::SuggestBodyTrackingCalibrationOverride(float height)
{
auto result = FOculusXRMovementModule::Get().GetXrBodyTracker()->SuggestBodyTrackingCalibrationOverride(height);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::GetFaceState(FOculusXRFaceState& outOculusXRFaceState)
{
auto result = FOculusXRMovementModule::Get().GetXrFaceTracker()->GetCachedFaceState(outOculusXRFaceState);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::IsFaceTrackingEnabled()
{
return FOculusXRMovementModule::Get().GetXrFaceTracker()->IsFaceTrackingEnabled();
}
bool FOculusXRMovementFunctionsOpenXR::IsFaceTrackingSupported()
{
return FOculusXRMovementModule::Get().GetXrFaceTracker()->IsFaceTrackingSupported();
}
bool FOculusXRMovementFunctionsOpenXR::StartFaceTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrFaceTracker()->StartFaceTracking();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::StopFaceTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrFaceTracker()->StopFaceTracking();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::SetFaceTrackingVisemesEnabled(bool enabled)
{
auto result = FOculusXRMovementModule::Get().GetXrFaceTracker()->SetVisemesEnabled(enabled);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState)
{
auto result = FOculusXRMovementModule::Get().GetXrFaceTracker()->GetCachedVisemeState(outOculusXRFaceVisemesState);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::IsFaceTrackingVisemesEnabled()
{
return FOculusXRMovementModule::Get().GetXrFaceTracker()->IsFaceTrackingVisemesEnabled();
}
bool FOculusXRMovementFunctionsOpenXR::IsFaceTrackingVisemesSupported()
{
return FOculusXRMovementModule::Get().GetXrFaceTracker()->IsFaceTrackingVisemesSupported();
}
bool FOculusXRMovementFunctionsOpenXR::GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters)
{
auto result = FOculusXRMovementModule::Get().GetXrEyeTracker()->GetCachedEyeState(outOculusXREyeGazesState);
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::IsEyeTrackingEnabled()
{
return FOculusXRMovementModule::Get().GetXrEyeTracker()->IsEyeTrackingEnabled();
}
bool FOculusXRMovementFunctionsOpenXR::IsEyeTrackingSupported()
{
return FOculusXRMovementModule::Get().GetXrEyeTracker()->IsEyeTrackingSupported();
}
bool FOculusXRMovementFunctionsOpenXR::StartEyeTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrEyeTracker()->StartEyeTracking();
return XR_SUCCEEDED(result);
}
bool FOculusXRMovementFunctionsOpenXR::StopEyeTracking()
{
auto result = FOculusXRMovementModule::Get().GetXrEyeTracker()->StopEyeTracking();
return XR_SUCCEEDED(result);
}
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,42 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementFunctions.h"
struct FOculusXRMovementFunctionsOpenXR : public IOculusXRMovementFunctions
{
public:
virtual bool GetBodyState(FOculusXRBodyState& outState, float WorldToMeters) override;
virtual bool GetBodySkeleton(FOculusXRBodySkeleton& outSkeleton, float WorldToMeters) override;
virtual bool IsBodyTrackingSupported() override;
virtual bool IsBodyTrackingEnabled() override;
// virtual bool IsFullBodyTrackingSupported() override;
virtual bool IsFullBodyTrackingEnabled() override;
virtual bool StartBodyTracking() override;
virtual bool StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet) override;
virtual bool StopBodyTracking() override;
virtual bool RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity) override;
virtual bool ResetBodyTrackingCalibration() override;
virtual bool SuggestBodyTrackingCalibrationOverride(float height) override;
virtual bool GetFaceState(FOculusXRFaceState& outOculusXRFaceState) override;
virtual bool IsFaceTrackingEnabled() override;
virtual bool IsFaceTrackingSupported() override;
virtual bool StartFaceTracking() override;
virtual bool StopFaceTracking() override;
virtual bool SetFaceTrackingVisemesEnabled(bool enabled) override;
virtual bool GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState) override;
virtual bool IsFaceTrackingVisemesEnabled() override;
virtual bool IsFaceTrackingVisemesSupported() override;
virtual bool GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters) override;
virtual bool IsEyeTrackingEnabled() override;
virtual bool IsEyeTrackingSupported() override;
virtual bool StartEyeTracking() override;
virtual bool StopEyeTracking() override;
};

View File

@@ -0,0 +1,422 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementLiveLink.h"
#include "IHeadMountedDisplayModule.h"
#include "OculusXRHMDModule.h"
#include "OculusXRMovementLog.h"
#include "OculusXRMovement.h"
#include "OculusXRMovementTypes.h"
#include "OculusXRTelemetryMovementEvents.h"
#include "Roles/LiveLinkAnimationTypes.h"
#include "ILiveLinkClient.h"
#define LOCTEXT_NAMESPACE "MetaOculusXRMovement"
namespace
{
constexpr int32 NoParent = -1;
}
namespace MetaXRMovement
{
template <>
void FEyeSubject::InitializeRoleStaticData(FLiveLinkSkeletonStaticData& StaticData) const
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXREye::COUNT);
StaticData.BoneNames.Reserve(FieldsCount);
for (uint8 XRBone = 0; XRBone < FieldsCount; ++XRBone)
{
StaticData.BoneNames.Add(UEnum::GetValueAsName(static_cast<EOculusXREye>(XRBone)));
StaticData.BoneParents.Add(NoParent);
}
}
template <>
void FBodySubject::InitializeRoleStaticData(FLiveLinkSkeletonStaticData& StaticData) const
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRBoneID::COUNT);
StaticData.BoneNames.Reserve(FieldsCount);
for (uint8 XRBone = 0; XRBone < FieldsCount; ++XRBone)
{
StaticData.BoneNames.Add(UEnum::GetValueAsName(static_cast<EOculusXRBoneID>(XRBone)));
StaticData.BoneParents.Add(NoParent);
}
}
template <>
void FFaceSubject::InitializeRoleStaticData(FLiveLinkBaseStaticData& StaticData) const
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRFaceExpression::COUNT);
StaticData.PropertyNames.Reserve(FieldsCount);
for (uint8 XRProperty = 0; XRProperty < FieldsCount; ++XRProperty)
{
StaticData.PropertyNames.Add(UEnum::GetValueAsName(static_cast<EOculusXRFaceExpression>(XRProperty)));
}
}
template <>
void FFaceVisemesSubject::InitializeRoleStaticData(FLiveLinkBaseStaticData& StaticData) const
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRFaceVisemesExpression::COUNT);
StaticData.PropertyNames.Reserve(FieldsCount);
for (uint8 XRProperty = 0; XRProperty < FieldsCount; ++XRProperty)
{
StaticData.PropertyNames.Add(UEnum::GetValueAsName(static_cast<EOculusXRFaceVisemesExpression>(XRProperty)));
}
}
template <typename MetaXRState, typename RoleTypeStaticData, typename RoleTypeFrameData, typename Role>
FLiveLinkStaticDataStruct TSubject<MetaXRState, RoleTypeStaticData, RoleTypeFrameData, Role>::StaticData() const
{
FLiveLinkStaticDataStruct StaticDataStruct(RoleTypeStaticData::StaticStruct());
RoleTypeStaticData& RoleStaticData(*StaticDataStruct.Cast<RoleTypeStaticData>());
InitializeRoleStaticData(RoleStaticData);
return StaticDataStruct;
}
template <typename MetaXRState, typename RoleTypeStaticData, typename RoleTypeFrameData, typename Role>
FLiveLinkFrameDataStruct TSubject<MetaXRState, RoleTypeStaticData, RoleTypeFrameData, Role>::FrameData()
{
FLiveLinkFrameDataStruct FrameDataStruct(RoleTypeFrameData::StaticStruct());
RoleTypeFrameData& FrameData(*FrameDataStruct.Cast<RoleTypeFrameData>());
UpdateFrame(FrameData);
return FrameDataStruct;
}
template <>
FEyeSubject::TSubject()
: Name(TEXT("Eye"))
, bLastFrameIsValid(false)
, bStarted(false)
{
}
template <>
FFaceSubject::TSubject()
: Name(TEXT("Face"))
, bLastFrameIsValid(false)
, bStarted(false)
{
}
template <>
FFaceVisemesSubject::TSubject()
: Name(TEXT("FaceVisemes"))
, bLastFrameIsValid(false)
, bStarted(false)
{
}
template <>
FBodySubject::TSubject()
: Name(TEXT("Body"))
, bLastFrameIsValid(false)
, bStarted(false)
{
}
template <>
bool FEyeSubject::Start()
{
if (!bStarted)
{
bStarted = OculusXRMovement::StartEyeTracking();
}
return bStarted;
}
template <>
bool FEyeSubject::Stop()
{
if (bStarted)
{
bStarted = !OculusXRMovement::StopEyeTracking();
}
return !bStarted;
}
template <>
bool FFaceSubject::Start()
{
if (!bStarted)
{
bStarted = OculusXRMovement::StartFaceTracking();
}
return bStarted;
}
template <>
bool FFaceSubject::Stop()
{
if (bStarted)
{
bStarted = !OculusXRMovement::StopFaceTracking();
}
return !bStarted;
}
template <>
bool FFaceVisemesSubject::Start()
{
if (!bStarted)
{
// TODO calculate consumers
bStarted = OculusXRMovement::StartFaceTracking();
}
return bStarted;
}
template <>
bool FFaceVisemesSubject::Stop()
{
if (bStarted)
{
bStarted = !OculusXRMovement::StopFaceTracking();
}
return !bStarted;
}
template <>
bool FBodySubject::Start()
{
if (!bStarted)
{
bStarted = OculusXRMovement::StartBodyTracking();
}
return bStarted;
}
template <>
bool FBodySubject::Stop()
{
if (bStarted)
{
bStarted = !OculusXRMovement::StopBodyTracking();
}
return !bStarted;
}
template <>
bool FEyeSubject::IsSupported()
{
return OculusXRMovement::IsEyeTrackingSupported();
}
template <>
bool FFaceSubject::IsSupported()
{
return OculusXRMovement::IsFaceTrackingSupported();
}
template <>
bool FFaceVisemesSubject::IsSupported()
{
return OculusXRMovement::IsFaceTrackingSupported();
}
template <>
bool FBodySubject::IsSupported()
{
return OculusXRMovement::IsBodyTrackingSupported();
}
template <>
void FEyeSubject::UpdateFrame(FLiveLinkAnimationFrameData& FrameData)
{
bLastFrameIsValid = OculusXRMovement::GetEyeGazesState(LastState, 1.f)
&& (LastState.EyeGazes[0].bIsValid || LastState.EyeGazes[1].bIsValid);
if (bLastFrameIsValid)
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXREye::COUNT);
FrameData.Transforms.Reserve(FieldsCount);
for (uint8 i = 0u; i < FieldsCount; ++i)
{
const auto& EyeGaze = LastState.EyeGazes[i];
FrameData.Transforms.Emplace(EyeGaze.Orientation, EyeGaze.Position);
}
FrameData.WorldTime = FPlatformTime::Seconds();
}
}
template <>
void FFaceSubject::UpdateFrame(FLiveLinkBaseFrameData& FrameData)
{
bLastFrameIsValid = OculusXRMovement::GetFaceState(LastState) && (LastState.bIsValid);
if (bLastFrameIsValid)
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRFaceExpression::COUNT);
FrameData.PropertyValues.Reserve(FieldsCount);
for (uint8 i = 0u; i < FieldsCount; ++i)
{
FrameData.PropertyValues.Emplace(LastState.ExpressionWeights[i]);
}
FrameData.WorldTime = FPlatformTime::Seconds();
}
}
template <>
void FFaceVisemesSubject::UpdateFrame(FLiveLinkBaseFrameData& FrameData)
{
bLastFrameIsValid = OculusXRMovement::GetFaceVisemesState(LastState) && (LastState.bIsValid);
if (bLastFrameIsValid)
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRFaceVisemesExpression::COUNT);
FrameData.PropertyValues.Reserve(FieldsCount);
for (uint8 i = 0u; i < FieldsCount; ++i)
{
FrameData.PropertyValues.Emplace(LastState.ExpressionVisemeWeights[i]);
}
FrameData.WorldTime = FPlatformTime::Seconds();
}
}
template <>
void FBodySubject::UpdateFrame(FLiveLinkAnimationFrameData& FrameData)
{
bLastFrameIsValid = OculusXRMovement::GetBodyState(LastState, 1.f) && (LastState.IsActive) && (LastState.SkeletonChangedCount > 0);
if (bLastFrameIsValid)
{
constexpr auto FieldsCount = static_cast<uint8>(EOculusXRBoneID::COUNT);
FrameData.Transforms.Reserve(FieldsCount);
for (uint8 i = 0u; i < FieldsCount; ++i)
{
const auto& Joint = LastState.Joints[i];
FrameData.Transforms.Emplace(Joint.Orientation, Joint.Position);
}
FrameData.WorldTime = FPlatformTime::Seconds();
}
}
LiveLinkSource::LiveLinkSource()
: bAnySupported(FEyeSubject::IsSupported() || FFaceSubject::IsSupported() || FFaceVisemesSubject::IsSupported() || FBodySubject::IsSupported())
{
OculusXRTelemetry::TScopedMarker<OculusXRTelemetry::Events::FMovementSDKLiveLinkCreated>();
}
void LiveLinkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid)
{
Client = InClient;
SourceGuid = InSourceGuid;
InitializeMovementSubjects();
UpdateMovementSubjects();
}
bool LiveLinkSource::IsSourceStillValid() const
{
return Client != nullptr;
}
bool LiveLinkSource::RequestSourceShutdown()
{
Client = nullptr;
SourceGuid.Invalidate();
if (!(Body.Stop() && Face.Stop()
&& FaceVisemes.Stop()
&& Eye.Stop()))
{
UE_LOG(LogOculusXRMovement, Error, TEXT("At least one of the trackers cannot stop."));
}
return true;
}
FText LiveLinkSource::GetSourceType() const
{
return LOCTEXT("MetaOculusXRMovementLiveLinkSourceType", "MetaXR MovementSDK");
}
FText LiveLinkSource::GetSourceMachineName() const
{
if (IHeadMountedDisplayModule::IsAvailable())
{
const FString DeviceName = IHeadMountedDisplayModule::Get().GetDeviceSystemName();
return FText::FromString(DeviceName);
}
return LOCTEXT("MetaOculusXRMovementLiveLinkMachineName", "MetaXR Device");
}
FText LiveLinkSource::GetSourceStatus() const
{
if (bAnySupported)
{
return LOCTEXT("MetaOculusXRMovementLiveLinkStatusSupported", "Active");
}
return LOCTEXT("MetaOculusXRMovementLiveLinkStatusNotSupported", "Not Supported");
}
void LiveLinkSource::Tick(float DeltaTime)
{
UpdateMovementSubjects();
}
template <typename SubjectT>
LiveLinkSource::ESubjectInitializationResult LiveLinkSource::InitializeMovementSubject(TOptional<FLiveLinkSubjectKey>& Key, SubjectT& Subject)
{
ESubjectInitializationResult FinalState;
if (Key)
{
if (Key->Source.IsValid())
{ // If the key was already in use. Remove it.
Client->RemoveSubject_AnyThread(*Key);
}
Key.Reset();
}
if (Subject.IsSupported())
{
Key = FLiveLinkSubjectKey(SourceGuid, Subject.Name);
FinalState = ESubjectInitializationResult::Started;
if (!Subject.Start())
{
UE_LOG(LogOculusXRMovement, Error, TEXT("Tracker for LiveLink subject %s cannot start."), *Subject.Name.ToString());
FinalState = ESubjectInitializationResult::StartFailed;
}
using Role = typename std::remove_reference_t<decltype(Subject)>::Role;
Client->PushSubjectStaticData_AnyThread(*Key, Role::StaticClass(), Subject.StaticData());
}
else
{
UE_LOG(LogOculusXRMovement, Log, TEXT("LiveLink subject %s is not supported."), *Subject.Name.ToString());
FinalState = ESubjectInitializationResult::NotSupported;
}
return FinalState;
}
void LiveLinkSource::InitializeMovementSubjects()
{
check(IsInGameThread());
const OculusXRTelemetry::TScopedMarker<OculusXRTelemetry::Events::FMovementSDKLiveLinkInit> LiveLinkInit;
const auto EyeInit = InitializeMovementSubject(KeyEye, Eye);
const auto FaceInit = InitializeMovementSubject(KeyFace, Face);
const auto FaceVisemesInit = InitializeMovementSubject(KeyFaceVisemes, FaceVisemes);
const auto BodyInit = InitializeMovementSubject(KeyBody, Body);
LiveLinkInit.AddAnnotation(StringCast<ANSICHAR>(*Eye.Name.ToString()).Get(), ResultToText[static_cast<int>(EyeInit)])
.AddAnnotation(StringCast<ANSICHAR>(*Face.Name.ToString()).Get(), ResultToText[static_cast<int>(FaceInit)])
.AddAnnotation(StringCast<ANSICHAR>(*FaceVisemes.Name.ToString()).Get(), ResultToText[static_cast<int>(FaceVisemesInit)])
.AddAnnotation(StringCast<ANSICHAR>(*Body.Name.ToString()).Get(), ResultToText[static_cast<int>(BodyInit)]);
}
template <typename SubjectT>
void LiveLinkSource::UpdateMovementSubject(const TOptional<FLiveLinkSubjectKey>& Key, SubjectT& Subject)
{
if (Key)
{
const bool bPreviousFrameValid = Subject.IsLastFrameValid();
auto FrameData = Subject.FrameData();
const bool bFrameValid = Subject.IsLastFrameValid();
if (bPreviousFrameValid != bFrameValid)
{
UE_LOG(LogOculusXRMovement, Log, TEXT("LiveLink subject %s became %s."), *Subject.Name.ToString(), bFrameValid ? TEXT("valid") : TEXT("invalid"));
}
if (bFrameValid)
{
Client->PushSubjectFrameData_AnyThread(*Key, MoveTemp(FrameData));
}
}
}
void LiveLinkSource::UpdateMovementSubjects()
{
check(IsInGameThread());
if (IsSourceStillValid())
{
UpdateMovementSubject(KeyEye, Eye);
UpdateMovementSubject(KeyFace, Face);
UpdateMovementSubject(KeyFaceVisemes, FaceVisemes);
UpdateMovementSubject(KeyBody, Body);
}
}
} // namespace MetaXRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,115 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "ILiveLinkSource.h"
#include "LiveLinkTypes.h"
#include "Roles/LiveLinkAnimationRole.h"
#include "Roles/LiveLinkAnimationTypes.h"
#include "Roles/LiveLinkBasicRole.h"
#include "Tickable.h"
#include "OculusXRMovementTypes.h"
#define LOCTEXT_NAMESPACE "MetaOculusXRMovement"
namespace MetaXRMovement
{
template <typename MetaXRState, typename RoleTypeStaticData, typename RoleTypeFrameData, typename RoleT>
class TSubject
{
public:
explicit TSubject();
using Role = RoleT;
const FLiveLinkSubjectName Name;
FLiveLinkStaticDataStruct StaticData() const;
FLiveLinkFrameDataStruct FrameData();
bool IsLastFrameValid() const { return bLastFrameIsValid; };
bool Start();
bool Stop();
static bool IsSupported();
private:
bool bLastFrameIsValid;
bool bStarted;
MetaXRState LastState;
void InitializeRoleStaticData(RoleTypeStaticData& StaticData) const;
void UpdateFrame(RoleTypeFrameData& FrameData);
};
using FEyeSubject = TSubject<FOculusXREyeGazesState, FLiveLinkSkeletonStaticData, FLiveLinkAnimationFrameData, ULiveLinkAnimationRole>;
using FFaceSubject = TSubject<FOculusXRFaceState, FLiveLinkBaseStaticData, FLiveLinkBaseFrameData, ULiveLinkBasicRole>;
using FBodySubject = TSubject<FOculusXRBodyState, FLiveLinkSkeletonStaticData, FLiveLinkAnimationFrameData, ULiveLinkAnimationRole>;
using FFaceVisemesSubject = TSubject<FOculusXRFaceVisemesState, FLiveLinkBaseStaticData, FLiveLinkBaseFrameData, ULiveLinkBasicRole>;
class LiveLinkSource : public ILiveLinkSource, public FTickableGameObject
{
public:
LiveLinkSource();
virtual ~LiveLinkSource() override = default;
// ILiveLinkSource implementation
virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override;
virtual bool IsSourceStillValid() const override;
virtual bool RequestSourceShutdown() override;
virtual FText GetSourceType() const override;
virtual FText GetSourceMachineName() const override;
virtual FText GetSourceStatus() const override;
// FTickableGameObject implementation
virtual void Tick(float DeltaTime) override;
virtual bool IsTickable() const override { return bAnySupported && Client; };
virtual TStatId GetStatId() const override
{
RETURN_QUICK_DECLARE_CYCLE_STAT(FOculusXRMovementLiveLink, STATGROUP_Tickables);
}
virtual bool IsTickableInEditor() const override { return true; }
virtual bool IsTickableWhenPaused() const override { return true; }
private:
enum class ESubjectInitializationResult
{
Started = 0,
StartFailed = 1,
NotSupported = 2
};
static constexpr const char* ResultToText[]{ "started", "start_failed", "not_supported" };
template <typename SubjectT>
ESubjectInitializationResult InitializeMovementSubject(TOptional<FLiveLinkSubjectKey>& Key, SubjectT& Subject);
void InitializeMovementSubjects();
template <typename SubjectT>
void UpdateMovementSubject(const TOptional<FLiveLinkSubjectKey>& Key, SubjectT& Subject);
void UpdateMovementSubjects();
// LiveLink Data
// The local client to push data updates to
ILiveLinkClient* Client{ nullptr };
// Our identifier in LiveLink
FGuid SourceGuid;
// Whenever any of the trackers is supported.
const bool bAnySupported;
// This subject's keys. Initialized only if a tracker is supported.
TOptional<FLiveLinkSubjectKey> KeyEye;
TOptional<FLiveLinkSubjectKey> KeyFace;
TOptional<FLiveLinkSubjectKey> KeyFaceVisemes;
TOptional<FLiveLinkSubjectKey> KeyBody;
// Subjects
FEyeSubject Eye;
FFaceSubject Face;
FBodySubject Body;
FFaceVisemesSubject FaceVisemes;
};
} // namespace MetaXRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,7 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
DECLARE_LOG_CATEGORY_EXTERN(LogOculusXRMovement, Log, All);

View File

@@ -0,0 +1,64 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementModule.h"
#include "OculusXRHMDModule.h"
#include "OculusXRMovementLog.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
DEFINE_LOG_CATEGORY(LogOculusXRMovement);
//-------------------------------------------------------------------------------------------------
// FOculusXRMovementModule
//-------------------------------------------------------------------------------------------------
FOculusXRMovementModule::FOculusXRMovementModule()
{
}
void FOculusXRMovementModule::StartupModule()
{
BodyTrackingXR = MakeShareable(new XRMovement::FBodyTrackingXR());
BodyTrackingXR->RegisterAsOpenXRExtension();
EyeTrackingXR = MakeShareable(new XRMovement::FEyeTrackingXR());
EyeTrackingXR->RegisterAsOpenXRExtension();
FaceTrackingXR = MakeShareable(new XRMovement::FFaceTrackingXR());
FaceTrackingXR->RegisterAsOpenXRExtension();
}
void FOculusXRMovementModule::ShutdownModule()
{
}
TSharedPtr<ILiveLinkSource> FOculusXRMovementModule::GetLiveLinkSource()
{
if (!MovementSource.IsValid())
{
AddLiveLinkSource();
}
return MovementSource;
}
bool FOculusXRMovementModule::IsLiveLinkSourceValid() const
{
return MovementSource.IsValid();
}
void FOculusXRMovementModule::AddLiveLinkSource()
{
if (!MovementSource.IsValid())
{
MovementSource = MakeShared<MetaXRMovement::LiveLinkSource>();
}
}
void FOculusXRMovementModule::RemoveLiveLinkSource()
{
MovementSource.Reset();
}
IMPLEMENT_MODULE(FOculusXRMovementModule, OculusXRMovement)
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,54 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "openxr/OculusXRBodyTrackingXR.h"
#include "openxr/OculusXREyeTrackingXR.h"
#include "openxr/OculusXRFaceTrackingXR.h"
#include "OculusXRMovement.h"
#include "OculusXRMovementLiveLink.h"
#include "IOculusXRMovementModule.h"
#include "ILiveLinkSource.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
//-------------------------------------------------------------------------------------------------
// FOculusXRMovementModule
//-------------------------------------------------------------------------------------------------
typedef TSharedPtr<XRMovement::FBodyTrackingXR, ESPMode::ThreadSafe> FBodyTrackingXRPtr;
typedef TSharedPtr<XRMovement::FEyeTrackingXR, ESPMode::ThreadSafe> FEyeTrackingXRPtr;
typedef TSharedPtr<XRMovement::FFaceTrackingXR, ESPMode::ThreadSafe> FFaceTrackingXRPtr;
class FOculusXRMovementModule : public IOculusXRMovementModule
{
public:
FOculusXRMovementModule();
static inline FOculusXRMovementModule& Get()
{
return FModuleManager::LoadModuleChecked<FOculusXRMovementModule>("OculusXRMovement");
}
virtual void StartupModule() override;
virtual void ShutdownModule() override;
/* Live link */
virtual TSharedPtr<ILiveLinkSource> GetLiveLinkSource() override;
virtual bool IsLiveLinkSourceValid() const override;
virtual void AddLiveLinkSource() override;
virtual void RemoveLiveLinkSource() override;
FBodyTrackingXRPtr GetXrBodyTracker() { return BodyTrackingXR; }
FEyeTrackingXRPtr GetXrEyeTracker() { return EyeTrackingXR; }
FFaceTrackingXRPtr GetXrFaceTracker() { return FaceTrackingXR; }
private:
FBodyTrackingXRPtr BodyTrackingXR;
FEyeTrackingXRPtr EyeTrackingXR;
FFaceTrackingXRPtr FaceTrackingXR;
TSharedPtr<MetaXRMovement::LiveLinkSource> MovementSource{ nullptr };
};
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,80 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRMovementTypes.h"
#include "OculusXRHMDPrivate.h"
#include "OculusXRHMD.h"
FOculusXRBodyJoint::FOculusXRBodyJoint()
: LocationFlags(0)
, bIsValid(false)
, Orientation(FRotator::ZeroRotator)
, Position(FVector::ZeroVector)
{
}
FOculusXRBodyState::FOculusXRBodyState()
: IsActive(false)
, Confidence(0)
, SkeletonChangedCount(0)
, Time(0.f)
{
Joints.SetNum(static_cast<int32>(EOculusXRBoneID::COUNT));
}
FOculusXRBodySkeletonBone::FOculusXRBodySkeletonBone()
: Orientation(FRotator::ZeroRotator)
, Position(FVector::ZeroVector)
, BoneId(EOculusXRBoneID::None)
, ParentBoneIndex(EOculusXRBoneID::None)
{
}
FOculusXRBodySkeleton::FOculusXRBodySkeleton()
: NumBones(0)
{
Bones.SetNum(static_cast<int32>(EOculusXRBoneID::COUNT));
}
FOculusXRBodySkeletonState::FOculusXRBodySkeletonState()
: BodyState()
, SkeletonState()
{
}
FOculusXRFaceState::FOculusXRFaceState()
: bIsValid(false)
, bIsEyeFollowingBlendshapesValid(false)
, Time(0.f)
, DataSource()
{
ExpressionWeights.SetNum(static_cast<int32>(EOculusXRFaceExpression::COUNT));
ExpressionWeightConfidences.SetNum(static_cast<int32>(EOculusXRFaceConfidence::COUNT));
}
FOculusXRFaceVisemesState::FOculusXRFaceVisemesState()
: bIsValid(false)
, Time(0.f)
{
ExpressionVisemeWeights.SetNum(static_cast<int32>(EOculusXRFaceVisemesExpression::COUNT));
}
FOculusXRFaceExpressionModifier::FOculusXRFaceExpressionModifier()
: MinValue(0.f)
, MaxValue(1.f)
, Multiplier(1.f)
{
}
FOculusXREyeGazeState::FOculusXREyeGazeState()
: Orientation(FRotator::ZeroRotator)
, Position(FVector::ZeroVector)
, Confidence(0.f)
, bIsValid(false)
{
}
FOculusXREyeGazesState::FOculusXREyeGazesState()
: Time(0.f)
{
EyeGazes.SetNum(static_cast<int32>(EOculusXREye::COUNT));
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRTelemetry.h"
namespace OculusXRTelemetry::Events
{
using FMovementSDKLiveLinkCreated = TMarker<191961034>;
using FMovementSDKLiveLinkInit = TMarker<191970472>;
using FMovementSDKBodyStart = TMarker<191958900>;
using FMovementSDKFaceStart = TMarker<191966310>;
using FMovementSDKEyeStart = TMarker<191969182>;
} // namespace OculusXRTelemetry::Events

View File

@@ -0,0 +1,474 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRBodytrackingXR.h"
#include "OpenXRCore.h"
#include "IOpenXRHMDModule.h"
#include "OpenXRHMD.h"
#include "OculusXRMovementLog.h"
#include "OpenXR/OculusXROpenXRUtilities.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
namespace XRMovement
{
PFN_xrCreateBodyTrackerFB xrCreateBodyTrackerFB = nullptr;
PFN_xrDestroyBodyTrackerFB xrDestroyBodyTrackerFB = nullptr;
PFN_xrLocateBodyJointsFB xrLocateBodyJointsFB = nullptr;
PFN_xrGetBodySkeletonFB xrGetBodySkeletonFB = nullptr;
PFN_xrRequestBodyTrackingFidelityMETA xrRequestBodyTrackingFidelityMETA = nullptr;
PFN_xrSuggestBodyTrackingCalibrationOverrideMETA xrSuggestBodyTrackingCalibrationOverrideMETA = nullptr;
PFN_xrResetBodyTrackingCalibrationMETA xrResetBodyTrackingCalibrationMETA = nullptr;
FBodyTrackingXR::FBodyTrackingXR()
: bExtBodyTrackingEnabled(false)
, bExtBodyTrackingFullBodyEnabled(false)
, bExtBodyTrackingFidelityEnabled(false)
, bExtBodyTrackingCalibrationEnabled(false)
, OpenXRHMD(nullptr)
, BodyTracker(nullptr)
, FullBodyTracking(false)
{
}
FBodyTrackingXR::~FBodyTrackingXR()
{
}
void FBodyTrackingXR::RegisterAsOpenXRExtension()
{
#if defined(WITH_OCULUS_BRANCH)
// Feature not enabled on Marketplace build. Currently only for the meta fork
RegisterOpenXRExtensionModularFeature();
#endif
}
bool FBodyTrackingXR::GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
OutExtensions.Add(XR_FB_BODY_TRACKING_EXTENSION_NAME);
return true;
}
bool FBodyTrackingXR::GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
OutExtensions.Add(XR_META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME);
OutExtensions.Add(XR_META_BODY_TRACKING_FIDELITY_EXTENSION_NAME);
OutExtensions.Add(XR_META_BODY_TRACKING_CALIBRATION_EXTENSION_NAME);
return true;
}
const void* FBodyTrackingXR::OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext)
{
if (InModule != nullptr)
{
bExtBodyTrackingEnabled = InModule->IsExtensionEnabled(XR_FB_BODY_TRACKING_EXTENSION_NAME);
bExtBodyTrackingFullBodyEnabled = InModule->IsExtensionEnabled(XR_META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME);
bExtBodyTrackingFidelityEnabled = InModule->IsExtensionEnabled(XR_META_BODY_TRACKING_FIDELITY_EXTENSION_NAME);
bExtBodyTrackingCalibrationEnabled = InModule->IsExtensionEnabled(XR_META_BODY_TRACKING_CALIBRATION_EXTENSION_NAME);
UE_LOG(LogOculusXRMovement, Log, TEXT("[Body Tracking] Extensions available: Tracking: %hs -- Full Body: %hs -- Fidelity: %hs -- Calibration: %hs"),
bExtBodyTrackingEnabled ? "ENABLED" : "DISABLED",
bExtBodyTrackingFullBodyEnabled ? "ENABLED" : "DISABLED",
bExtBodyTrackingFidelityEnabled ? "ENABLED" : "DISABLED",
bExtBodyTrackingCalibrationEnabled ? "ENABLED" : "DISABLED");
}
return InNext;
}
const void* FBodyTrackingXR::OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext)
{
InitOpenXRFunctions(InInstance);
OpenXRHMD = (FOpenXRHMD*)GEngine->XRSystem.Get();
return InNext;
}
void FBodyTrackingXR::OnDestroySession(XrSession InSession)
{
OpenXRHMD = nullptr;
}
void* FBodyTrackingXR::OnWaitFrame(XrSession InSession, void* InNext)
{
Update_GameThread(InSession);
return InNext;
}
XrResult FBodyTrackingXR::StartBodyTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTracking] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsBodyTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTracking] Body tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (BodyTracker != XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Log, TEXT("[StartBodyTracking] Body tracking is already started."));
return XR_SUCCESS;
}
XrBodyTrackerCreateInfoFB createInfo = { XR_TYPE_BODY_TRACKER_CREATE_INFO_FB };
createInfo.next = nullptr;
createInfo.bodyJointSet = XR_BODY_JOINT_SET_DEFAULT_FB;
auto result = XRMovement::xrCreateBodyTrackerFB(OpenXRHMD->GetSession(), &createInfo, &BodyTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTracking] Body tracking failed to start. Result: %d"), result);
return result;
}
return XR_SUCCESS;
}
XrResult FBodyTrackingXR::StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet)
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTrackingByJointSet] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsBodyTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTrackingByJointSet] Body tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (BodyTracker != XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Log, TEXT("[StartBodyTrackingByJointSet] Body tracking is already started."));
return XR_SUCCESS;
}
XrBodyTrackerCreateInfoFB createInfo = { XR_TYPE_BODY_TRACKER_CREATE_INFO_FB };
createInfo.next = nullptr;
switch (jointSet)
{
case EOculusXRBodyJointSet::UpperBody:
createInfo.bodyJointSet = XR_BODY_JOINT_SET_DEFAULT_FB;
break;
case EOculusXRBodyJointSet::FullBody:
createInfo.bodyJointSet = XR_BODY_JOINT_SET_FULL_BODY_META;
if (!IsFullBodySupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTrackingByJointSet] Full body tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
break;
default:
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTrackingByJointSet] Unknown body tracking joint set."));
return XR_ERROR_VALIDATION_FAILURE;
}
auto result = XRMovement::xrCreateBodyTrackerFB(OpenXRHMD->GetSession(), &createInfo, &BodyTracker);
if XR_FAILED (result)
{
BodyTracker = XR_NULL_HANDLE;
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartBodyTrackingByJointSet] Body tracking failed to start. Result: %d"), result);
return result;
}
else
{
FullBodyTracking = (jointSet == EOculusXRBodyJointSet::FullBody);
}
return XR_SUCCESS;
}
XrResult FBodyTrackingXR::StopBodyTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopBodyTracking] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsBodyTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopBodyTracking] Body tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
XrResult result = XR_SUCCESS;
if (IsBodyTrackingEnabled())
{
result = XRMovement::xrDestroyBodyTrackerFB(BodyTracker);
if XR_FAILED (result)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopBodyTracking] Body tracking failed to stop. Result: %d"), result);
}
}
BodyTracker = XR_NULL_HANDLE;
FullBodyTracking = false;
return result;
}
XrResult FBodyTrackingXR::GetCachedBodyState(FOculusXRBodyState& OutState)
{
if (!IsBodyTrackingEnabled())
{
return XR_ERROR_VALIDATION_FAILURE;
}
OutState = CachedBodyState;
return XR_SUCCESS;
}
XrResult FBodyTrackingXR::GetBodySkeleton(FOculusXRBodySkeleton& OutSkeleton)
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[GetBodySkeleton] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
int jointCount = IsFullBodyTrackingEnabled() ? (int)XR_FULL_BODY_JOINT_COUNT_META : (int)XR_BODY_JOINT_COUNT_FB;
// Allocate enough memory for the larger joint set
static_assert((int)XR_FULL_BODY_JOINT_COUNT_META >= (int)XR_BODY_JOINT_COUNT_FB);
XrBodySkeletonJointFB joints[XR_FULL_BODY_JOINT_COUNT_META];
XrBodySkeletonFB bodySkeleton = { XR_TYPE_BODY_SKELETON_FB };
bodySkeleton.jointCount = jointCount;
bodySkeleton.joints = joints;
auto result = XRMovement::xrGetBodySkeletonFB(BodyTracker, &bodySkeleton);
if (XR_FAILED(result))
{
return result;
}
OutSkeleton.NumBones = bodySkeleton.jointCount;
for (uint32 i = 0; i < bodySkeleton.jointCount; ++i)
{
XrBodySkeletonJointFB bone = bodySkeleton.joints[i];
XrPosef bonePose = bone.pose;
FOculusXRBodySkeletonBone& OculusXRBone = OutSkeleton.Bones[i];
OculusXRBone.Orientation = FRotator(ToFQuat(bonePose.orientation));
OculusXRBone.Position = ToFVector(bonePose.position) * OpenXRHMD->GetWorldToMetersScale();
if (bone.parentJoint == XR_BODY_JOINT_NONE_FB)
{
OculusXRBone.ParentBoneIndex = EOculusXRBoneID::None;
}
else
{
OculusXRBone.ParentBoneIndex = static_cast<EOculusXRBoneID>(bone.parentJoint);
}
OculusXRBone.BoneId = static_cast<EOculusXRBoneID>(bone.joint);
}
return XR_SUCCESS;
}
XrResult FBodyTrackingXR::RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity Fidelity)
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[RequestBodyTrackingFidelity] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsFidelitySupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[RequestBodyTrackingFidelity] Fidelity is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (BodyTracker == XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[RequestBodyTrackingFidelity] Body tracking is not started."));
return XR_SUCCESS;
}
XrBodyTrackingFidelityMETA fidelity;
switch (Fidelity)
{
case EOculusXRBodyTrackingFidelity::High:
fidelity = XR_BODY_TRACKING_FIDELITY_HIGH_META;
break;
case EOculusXRBodyTrackingFidelity::Low:
fidelity = XR_BODY_TRACKING_FIDELITY_LOW_META;
break;
default:
UE_LOG(LogOculusXRMovement, Warning, TEXT("[RequestBodyTrackingFidelity] Invalid fidelity level."));
return XR_ERROR_VALIDATION_FAILURE;
}
XrResult result = xrRequestBodyTrackingFidelityMETA(BodyTracker, fidelity);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[RequestBodyTrackingFidelity] Failed to request fidelity level. Result: %d"), result);
}
return result;
}
XrResult FBodyTrackingXR::ResetBodyTrackingFidelity()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[ResetBodyTrackingFidelity] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsFidelitySupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[ResetBodyTrackingFidelity] Fidelity is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (BodyTracker == XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[ResetBodyTrackingFidelity] Body tracking is not started."));
return XR_SUCCESS;
}
XrResult result = xrResetBodyTrackingCalibrationMETA(BodyTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[ResetBodyTrackingFidelity] Failed to request fidelity level. Result: %d"), result);
}
return result;
}
XrResult FBodyTrackingXR::SuggestBodyTrackingCalibrationOverride(float height)
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SuggestBodyTrackingCalibrationOverride] XR state is invalid."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsCalibrationSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SuggestBodyTrackingCalibrationOverride] Calibration is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (BodyTracker == XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SuggestBodyTrackingCalibrationOverride] Body tracking is not started."));
return XR_SUCCESS;
}
XrBodyTrackingCalibrationInfoMETA xrCalibrationInfo = { XR_TYPE_BODY_TRACKING_CALIBRATION_INFO_META };
xrCalibrationInfo.bodyHeight = height;
XrResult result = xrSuggestBodyTrackingCalibrationOverrideMETA(BodyTracker, &xrCalibrationInfo);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SuggestBodyTrackingCalibrationOverride] failed to suggest calibration override! Result: %d"), result);
}
return result;
}
void FBodyTrackingXR::InitOpenXRFunctions(XrInstance InInstance)
{
// XR_FB_Body_Tracking
OculusXR::XRGetInstanceProcAddr(InInstance, "xrCreateBodyTrackerFB", &xrCreateBodyTrackerFB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrDestroyBodyTrackerFB", &xrDestroyBodyTrackerFB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrLocateBodyJointsFB", &xrLocateBodyJointsFB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrGetBodySkeletonFB", &xrGetBodySkeletonFB);
// XR_META_body_tracking_fidelity
OculusXR::XRGetInstanceProcAddr(InInstance, "xrRequestBodyTrackingFidelityMETA", &xrRequestBodyTrackingFidelityMETA);
// XR_META_body_tracking_calibration
OculusXR::XRGetInstanceProcAddr(InInstance, "xrSuggestBodyTrackingCalibrationOverrideMETA", &xrSuggestBodyTrackingCalibrationOverrideMETA);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrResetBodyTrackingCalibrationMETA", &xrResetBodyTrackingCalibrationMETA);
}
void FBodyTrackingXR::Update_GameThread(XrSession InSession)
{
check(IsInGameThread());
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession() || !IsBodyTrackingSupported() || !IsBodyTrackingEnabled())
{
return;
}
static_assert(XR_FULL_BODY_JOINT_COUNT_META == static_cast<int>(EOculusXRBoneID::COUNT), "The size of the XR Bone ID enum should be the same as the EOculusXRBoneID count.");
int jointCount = IsFullBodyTrackingEnabled() ? (int)XR_FULL_BODY_JOINT_COUNT_META : (int)XR_BODY_JOINT_COUNT_FB;
CachedBodyState.Joints.SetNum(static_cast<int>(XR_FULL_BODY_JOINT_COUNT_META));
XrBodyJointsLocateInfoFB info = { XR_TYPE_BODY_JOINTS_LOCATE_INFO_FB };
info.baseSpace = OpenXRHMD->GetTrackingSpace();
info.time = OpenXRHMD->GetDisplayTime();
XrBodyJointLocationsFB locations = { XR_TYPE_BODY_JOINT_LOCATIONS_FB };
XrBodyJointLocationFB jointLocations[XR_FULL_BODY_JOINT_COUNT_META];
locations.jointCount = jointCount;
locations.jointLocations = jointLocations;
XrBodyTrackingCalibrationStatusMETA calibrationStatus = { XR_TYPE_BODY_TRACKING_CALIBRATION_STATUS_META };
calibrationStatus.next = XR_NULL_HANDLE;
if (IsCalibrationSupported())
{
OculusXR::XRAppendToChain(
reinterpret_cast<XrBaseOutStructure*>(&calibrationStatus), reinterpret_cast<XrBaseOutStructure*>(&locations));
}
XrBodyTrackingFidelityStatusMETA fidelityStatus = { XR_TYPE_BODY_TRACKING_FIDELITY_STATUS_META };
fidelityStatus.next = XR_NULL_HANDLE;
if (IsFidelitySupported())
{
OculusXR::XRAppendToChain(
reinterpret_cast<XrBaseOutStructure*>(&fidelityStatus), reinterpret_cast<XrBaseOutStructure*>(&locations));
}
auto result = XRMovement::xrLocateBodyJointsFB(BodyTracker, &info, &locations);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[LocateBodyJoints] Failed to locate joints! Result: %d"), result);
return;
}
CachedBodyState.IsActive = (bool)locations.isActive;
CachedBodyState.Confidence = locations.confidence;
CachedBodyState.SkeletonChangedCount = locations.skeletonChangedCount;
CachedBodyState.Time = locations.time * 1e-9; // FromXrTime
for (int i = 0; i < jointCount; ++i)
{
XrBodyJointLocationFB jointLocation = locations.jointLocations[i];
XrPosef jointPose = jointLocation.pose;
FOculusXRBodyJoint& OculusXRBodyJoint = CachedBodyState.Joints[i];
OculusXRBodyJoint.LocationFlags = jointLocation.locationFlags;
OculusXRBodyJoint.bIsValid = jointLocation.locationFlags & (XRSpaceFlags::XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XRSpaceFlags::XR_SPACE_LOCATION_POSITION_VALID_BIT);
OculusXRBodyJoint.Orientation = FRotator(ToFQuat(jointPose.orientation));
OculusXRBodyJoint.Position = ToFVector(jointPose.position) * OpenXRHMD->GetWorldToMetersScale();
}
// If using less joints than the max count we can just set the remaining joints to null
if (jointCount < CachedBodyState.Joints.Num())
{
for (int i = jointCount; i < CachedBodyState.Joints.Num(); ++i)
{
CachedBodyState.Joints[i].bIsValid = false;
}
}
}
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,74 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementXRIncludes.h"
#include "IOpenXRExtensionPlugin.h"
#include "OculusXRMovementTypes.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
class FOpenXRHMD;
namespace XRMovement
{
extern PFN_xrCreateBodyTrackerFB xrCreateBodyTrackerFB;
extern PFN_xrDestroyBodyTrackerFB xrDestroyBodyTrackerFB;
extern PFN_xrLocateBodyJointsFB xrLocateBodyJointsFB;
extern PFN_xrGetBodySkeletonFB xrGetBodySkeletonFB;
extern PFN_xrRequestBodyTrackingFidelityMETA xrRequestBodyTrackingFidelityMETA;
extern PFN_xrSuggestBodyTrackingCalibrationOverrideMETA xrSuggestBodyTrackingCalibrationOverrideMETA;
extern PFN_xrResetBodyTrackingCalibrationMETA xrResetBodyTrackingCalibrationMETA;
class FBodyTrackingXR : public IOpenXRExtensionPlugin
{
public:
// IOculusXROpenXRHMDPlugin
virtual bool GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual bool GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual const void* OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext) override;
virtual const void* OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext) override;
virtual void OnDestroySession(XrSession InSession) override;
virtual void* OnWaitFrame(XrSession InSession, void* InNext) override;
public:
FBodyTrackingXR();
virtual ~FBodyTrackingXR();
void RegisterAsOpenXRExtension();
bool IsBodyTrackingSupported() const { return bExtBodyTrackingEnabled; }
bool IsFullBodySupported() const { return bExtBodyTrackingFullBodyEnabled; }
bool IsFidelitySupported() const { return bExtBodyTrackingFidelityEnabled; }
bool IsCalibrationSupported() const { return bExtBodyTrackingCalibrationEnabled; }
bool IsBodyTrackingEnabled() const { return BodyTracker != XR_NULL_HANDLE; }
bool IsFullBodyTrackingEnabled() const { return FullBodyTracking; }
XrResult StartBodyTracking();
XrResult StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet);
XrResult StopBodyTracking();
XrResult GetCachedBodyState(FOculusXRBodyState& OutState);
XrResult GetBodySkeleton(FOculusXRBodySkeleton& OutSkeleton);
XrResult RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity Fidelity);
XrResult ResetBodyTrackingFidelity();
XrResult SuggestBodyTrackingCalibrationOverride(float height);
private:
void InitOpenXRFunctions(XrInstance InInstance);
void Update_GameThread(XrSession InSession);
bool bExtBodyTrackingEnabled;
bool bExtBodyTrackingFullBodyEnabled;
bool bExtBodyTrackingFidelityEnabled;
bool bExtBodyTrackingCalibrationEnabled;
FOpenXRHMD* OpenXRHMD;
FOculusXRBodyState CachedBodyState;
XrBodyTrackerFB BodyTracker = XR_NULL_HANDLE;
bool FullBodyTracking{ false };
};
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,192 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXREyeTrackingXR.h"
#include "OpenXRCore.h"
#include "IOpenXRHMDModule.h"
#include "OpenXRHMD.h"
#include "OculusXRMovementLog.h"
#include "OpenXR/OculusXROpenXRUtilities.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
namespace XRMovement
{
PFN_xrCreateEyeTrackerFB xrCreateEyeTrackerFB = nullptr;
PFN_xrDestroyEyeTrackerFB xrDestroyEyeTrackerFB = nullptr;
PFN_xrGetEyeGazesFB xrGetEyeGazesFB = nullptr;
FEyeTrackingXR::FEyeTrackingXR()
: bExtEyeTrackingEnabled(false)
, OpenXRHMD(nullptr)
, EyeTracker(nullptr)
{
CachedEyeState.EyeGazes.SetNum(2);
}
FEyeTrackingXR::~FEyeTrackingXR()
{
}
void FEyeTrackingXR::RegisterAsOpenXRExtension()
{
#if defined(WITH_OCULUS_BRANCH)
// Feature not enabled on Marketplace build. Currently only for the meta fork
RegisterOpenXRExtensionModularFeature();
#endif
}
bool FEyeTrackingXR::GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
OutExtensions.Add(XR_FB_EYE_TRACKING_SOCIAL_EXTENSION_NAME);
return true;
}
bool FEyeTrackingXR::GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
return true;
}
const void* FEyeTrackingXR::OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext)
{
if (InModule != nullptr)
{
bExtEyeTrackingEnabled = InModule->IsExtensionEnabled(XR_FB_EYE_TRACKING_SOCIAL_EXTENSION_NAME);
}
return InNext;
}
const void* FEyeTrackingXR::OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext)
{
InitOpenXRFunctions(InInstance);
OpenXRHMD = (FOpenXRHMD*)GEngine->XRSystem.Get();
return InNext;
}
void FEyeTrackingXR::OnDestroySession(XrSession InSession)
{
OpenXRHMD = nullptr;
}
void* FEyeTrackingXR::OnWaitFrame(XrSession InSession, void* InNext)
{
Update_GameThread(InSession);
return InNext;
}
XrResult FEyeTrackingXR::StartEyeTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartEyeTracking] Cannot start eye tracking, the instance or session is null."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsEyeTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartEyeTracking] Cannot start eye tracking, body tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (EyeTracker != XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartEyeTracking] Cannot start eye tracking, body tracking is already started."));
return XR_SUCCESS;
}
XrEyeTrackerCreateInfoFB createInfo = { XR_TYPE_EYE_TRACKER_CREATE_INFO_FB, nullptr };
auto result = XRMovement::xrCreateEyeTrackerFB(OpenXRHMD->GetSession(), &createInfo, &EyeTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartEyeTracking] Failed to start eye tracking. Result(%d)"), result);
}
return result;
}
XrResult FEyeTrackingXR::StopEyeTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopEyeTracking] Cannot stop eye tracking, the instance or session is null."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsEyeTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopEyeTracking] Cannot stop eye tracking, eye tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
XrResult result = XR_SUCCESS;
if (IsEyeTrackingEnabled())
{
result = XRMovement::xrDestroyEyeTrackerFB(EyeTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopEyeTracking] Failed to stop eye tracking. Result(%d)"), result);
}
}
EyeTracker = XR_NULL_HANDLE;
return result;
}
XrResult FEyeTrackingXR::GetCachedEyeState(FOculusXREyeGazesState& OutState)
{
if (!IsEyeTrackingEnabled())
{
return XR_ERROR_VALIDATION_FAILURE;
}
OutState = CachedEyeState;
return XR_SUCCESS;
}
void FEyeTrackingXR::InitOpenXRFunctions(XrInstance InInstance)
{
// XR_FB_Eye_Tracking_Social
OculusXR::XRGetInstanceProcAddr(InInstance, "xrCreateEyeTrackerFB", &xrCreateEyeTrackerFB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrDestroyEyeTrackerFB", &xrDestroyEyeTrackerFB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrGetEyeGazesFB", &xrGetEyeGazesFB);
}
void FEyeTrackingXR::Update_GameThread(XrSession InSession)
{
check(IsInGameThread());
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession() || !IsEyeTrackingSupported() || !IsEyeTrackingEnabled())
{
return;
}
XrEyeGazesInfoFB info{ XR_TYPE_EYE_GAZES_INFO_FB, nullptr };
info.baseSpace = OpenXRHMD->GetTrackingSpace();
info.time = OpenXRHMD->GetDisplayTime();
XrEyeGazesFB gazes{ XR_TYPE_EYE_GAZES_FB, nullptr };
auto result = XRMovement::xrGetEyeGazesFB(EyeTracker, &info, &gazes);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[EyeGazeStateUpdate] Failed to get gazes state. Result(%d)"), result);
return;
}
auto ApplyGazeToUEType = [this](const XrEyeGazeFB& xrGaze, FOculusXREyeGazeState& outUEGaze) {
outUEGaze.bIsValid = static_cast<bool>(xrGaze.isValid);
outUEGaze.Confidence = xrGaze.gazeConfidence;
outUEGaze.Orientation = FRotator(ToFQuat(xrGaze.gazePose.orientation));
outUEGaze.Position = ToFVector(xrGaze.gazePose.position) * OpenXRHMD->GetWorldToMetersScale();
};
ApplyGazeToUEType(gazes.gaze[0], CachedEyeState.EyeGazes[0]);
ApplyGazeToUEType(gazes.gaze[1], CachedEyeState.EyeGazes[1]);
}
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,55 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementXRIncludes.h"
#include "IOpenXRExtensionPlugin.h"
#include "OculusXRMovementTypes.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
class FOpenXRHMD;
namespace XRMovement
{
extern PFN_xrCreateEyeTrackerFB xrCreateEyeTrackerFB;
extern PFN_xrDestroyEyeTrackerFB xrDestroyEyeTrackerFB;
extern PFN_xrGetEyeGazesFB xrGetEyeGazesFB;
class FEyeTrackingXR : public IOpenXRExtensionPlugin
{
public:
// IOculusXROpenXRHMDPlugin
virtual bool GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual bool GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual const void* OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext) override;
virtual const void* OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext) override;
virtual void OnDestroySession(XrSession InSession) override;
virtual void* OnWaitFrame(XrSession InSession, void* InNext) override;
public:
FEyeTrackingXR();
virtual ~FEyeTrackingXR();
void RegisterAsOpenXRExtension();
bool IsEyeTrackingSupported() const { return bExtEyeTrackingEnabled; }
bool IsEyeTrackingEnabled() const { return EyeTracker != XR_NULL_HANDLE; }
XrResult StartEyeTracking();
XrResult StopEyeTracking();
XrResult GetCachedEyeState(FOculusXREyeGazesState& OutState);
private:
void InitOpenXRFunctions(XrInstance InInstance);
void Update_GameThread(XrSession InSession);
bool bExtEyeTrackingEnabled;
FOpenXRHMD* OpenXRHMD;
FOculusXREyeGazesState CachedEyeState;
XrEyeTrackerFB EyeTracker = XR_NULL_HANDLE;
};
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,259 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#include "OculusXRFaceTrackingXR.h"
#include "OpenXRCore.h"
#include "IOpenXRHMDModule.h"
#include "OpenXRHMD.h"
#include "OculusXRMovementLog.h"
#include "OpenXR/OculusXROpenXRUtilities.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
namespace XRMovement
{
PFN_xrCreateFaceTracker2FB xrCreateFaceTracker2FB = nullptr;
PFN_xrDestroyFaceTracker2FB xrDestroyFaceTracker2FB = nullptr;
PFN_xrGetFaceExpressionWeights2FB xrGetFaceExpressionWeights2FB = nullptr;
FFaceTrackingXR::FFaceTrackingXR()
: bExtFaceTrackingSupported(false)
, bExtFaceTrackingVisemesSupported(false)
, bVisemesEnabled(false)
, OpenXRHMD(nullptr)
, FaceTracker(nullptr)
{
CachedFaceState.ExpressionWeights.SetNum(XR_FACE_EXPRESSION2_COUNT_FB);
CachedFaceState.ExpressionWeightConfidences.SetNum(XR_FACE_CONFIDENCE2_COUNT_FB);
CachedVisemeState.ExpressionVisemeWeights.SetNum(XR_FACE_TRACKING_VISEME_COUNT_METAX1);
}
FFaceTrackingXR::~FFaceTrackingXR()
{
}
void FFaceTrackingXR::RegisterAsOpenXRExtension()
{
#if defined(WITH_OCULUS_BRANCH)
// Feature not enabled on Marketplace build. Currently only for the meta fork
RegisterOpenXRExtensionModularFeature();
#endif
}
bool FFaceTrackingXR::GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
OutExtensions.Add(XR_FB_FACE_TRACKING2_EXTENSION_NAME);
return true;
}
bool FFaceTrackingXR::GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions)
{
OutExtensions.Add(XR_METAX1_FACE_TRACKING_VISEMES_EXTENSION_NAME);
return true;
}
const void* FFaceTrackingXR::OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext)
{
if (InModule != nullptr)
{
bExtFaceTrackingSupported = InModule->IsExtensionEnabled(XR_FB_FACE_TRACKING2_EXTENSION_NAME);
bExtFaceTrackingVisemesSupported = InModule->IsExtensionEnabled(XR_METAX1_FACE_TRACKING_VISEMES_EXTENSION_NAME);
}
return InNext;
}
const void* FFaceTrackingXR::OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext)
{
InitOpenXRFunctions(InInstance);
OpenXRHMD = (FOpenXRHMD*)GEngine->XRSystem.Get();
return InNext;
}
void FFaceTrackingXR::OnDestroySession(XrSession InSession)
{
OpenXRHMD = nullptr;
}
void* FFaceTrackingXR::OnWaitFrame(XrSession InSession, void* InNext)
{
Update_GameThread(InSession);
return InNext;
}
XrResult FFaceTrackingXR::StartFaceTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartFaceTracking] Cannot start face tracking, the instance or session is null."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsFaceTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartFaceTracking] Cannot start face tracking, face tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (FaceTracker != XR_NULL_HANDLE)
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartFaceTracking] Cannot start face tracking, face tracking is already started."));
return XR_SUCCESS;
}
XrFaceTrackerCreateInfo2FB createInfo = { XR_TYPE_FACE_TRACKER_CREATE_INFO2_FB, nullptr };
auto result = XRMovement::xrCreateFaceTracker2FB(OpenXRHMD->GetSession(), &createInfo, &FaceTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StartFaceTracking] Failed to start face tracking. Result(%d)"), result);
}
return result;
}
XrResult FFaceTrackingXR::StopFaceTracking()
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopFaceTracking] Cannot stop face tracking, the instance or session is null."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsFaceTrackingSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopFaceTracking] Cannot stop face tracking, face tracking is unsupported."));
return XR_ERROR_VALIDATION_FAILURE;
}
XrResult result = XR_SUCCESS;
if (IsFaceTrackingEnabled())
{
result = XRMovement::xrDestroyFaceTracker2FB(FaceTracker);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[StopFaceTracking] Failed to stop face tracking. Result(%d)"), result);
}
}
FaceTracker = XR_NULL_HANDLE;
return result;
}
XrResult FFaceTrackingXR::GetCachedFaceState(FOculusXRFaceState& OutState)
{
if (!IsFaceTrackingEnabled())
{
return XR_ERROR_VALIDATION_FAILURE;
}
OutState = CachedFaceState;
return XR_SUCCESS;
}
XrResult FFaceTrackingXR::SetVisemesEnabled(bool enabled)
{
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SetVisemesEnabled] Cannot change viseme state, the instance of session is null."));
return XR_ERROR_VALIDATION_FAILURE;
}
if (!IsFaceTrackingVisemesSupported())
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[SetVisemesEnabled] Cannot change viseme state, visemes are not supported."));
return XR_ERROR_VALIDATION_FAILURE;
}
UE_LOG(LogOculusXRMovement, Log, TEXT("[SetVisemesEnabled] Changing visemes enabled state: %hs"), enabled ? "TRUE" : "FALSE");
bVisemesEnabled = enabled;
return XR_SUCCESS;
}
XrResult FFaceTrackingXR::GetCachedVisemeState(FOculusXRFaceVisemesState& OutState)
{
if (!IsFaceTrackingEnabled() || !IsFaceTrackingVisemesEnabled())
{
return XR_ERROR_VALIDATION_FAILURE;
}
OutState = CachedVisemeState;
return XR_SUCCESS;
}
void FFaceTrackingXR::InitOpenXRFunctions(XrInstance InInstance)
{
// XR_FB_Eye_Tracking_Social
OculusXR::XRGetInstanceProcAddr(InInstance, "xrCreateFaceTracker2FB", &xrCreateFaceTracker2FB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrDestroyFaceTracker2FB", &xrDestroyFaceTracker2FB);
OculusXR::XRGetInstanceProcAddr(InInstance, "xrGetFaceExpressionWeights2FB", &xrGetFaceExpressionWeights2FB);
}
void FFaceTrackingXR::Update_GameThread(XrSession InSession)
{
check(IsInGameThread());
if (!OpenXRHMD || !OpenXRHMD->GetInstance() || !OpenXRHMD->GetSession() || !IsFaceTrackingSupported() || !IsFaceTrackingEnabled())
{
return;
}
XrFaceExpressionInfo2FB info{ XR_TYPE_FACE_EXPRESSION_INFO2_FB, nullptr };
info.time = OpenXRHMD->GetDisplayTime();
float weightsArray[XR_FACE_EXPRESSION2_COUNT_FB];
float confidencesArray[XR_FACE_CONFIDENCE2_COUNT_FB];
XrFaceExpressionWeights2FB weights{ XR_TYPE_FACE_EXPRESSION_WEIGHTS2_FB, nullptr };
weights.weights = weightsArray;
weights.weightCount = XR_FACE_EXPRESSION2_COUNT_FB;
weights.confidences = confidencesArray;
weights.confidenceCount = XR_FACE_CONFIDENCE2_COUNT_FB;
bool useVisemes = IsFaceTrackingVisemesSupported() && IsFaceTrackingVisemesEnabled();
XrFaceTrackingVisemesMETAX1 faceTrackingVisemes{ XR_TYPE_FACE_TRACKING_VISEMES_METAX1 };
if (useVisemes)
{
weights.next = &faceTrackingVisemes;
}
auto result = XRMovement::xrGetFaceExpressionWeights2FB(FaceTracker, &info, &weights);
if (XR_FAILED(result))
{
UE_LOG(LogOculusXRMovement, Warning, TEXT("[FaceExpressionStateUpdate] Failed to get face tracking state. Result(%d)"), result);
return;
}
CachedFaceState.bIsValid = (weights.isValid == XR_TRUE);
CachedFaceState.bIsEyeFollowingBlendshapesValid = (weights.isEyeFollowingBlendshapesValid == XR_TRUE);
CachedFaceState.Time = OculusXR::FromXrTime(weights.time);
switch (weights.dataSource)
{
case XR_FACE_TRACKING_DATA_SOURCE2_AUDIO_FB:
CachedFaceState.DataSource = EFaceTrackingDataSource::Audio;
break;
case XR_FACE_TRACKING_DATA_SOURCE2_VISUAL_FB:
CachedFaceState.DataSource = EFaceTrackingDataSource::Visual;
break;
case XR_FACE_TRACKING_DATA_SOURCE_2FB_MAX_ENUM_FB:
CachedFaceState.DataSource = EFaceTrackingDataSource::MAX;
break;
}
FMemory::Memcpy(CachedFaceState.ExpressionWeights.GetData(), weights.weights, XR_FACE_EXPRESSION2_COUNT_FB);
FMemory::Memcpy(CachedFaceState.ExpressionWeightConfidences.GetData(), weights.confidences, XR_FACE_CONFIDENCE2_COUNT_FB);
if (useVisemes)
{
CachedVisemeState.bIsValid = (faceTrackingVisemes.isValid == XR_TRUE);
CachedVisemeState.Time = CachedFaceState.Time;
FMemory::Memcpy(CachedVisemeState.ExpressionVisemeWeights.GetData(), faceTrackingVisemes.visemes, XR_FACE_TRACKING_VISEME_COUNT_METAX1);
}
}
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,63 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementXRIncludes.h"
#include "IOpenXRExtensionPlugin.h"
#include "OculusXRMovementTypes.h"
#define LOCTEXT_NAMESPACE "OculusXRMovement"
class FOpenXRHMD;
namespace XRMovement
{
extern PFN_xrCreateFaceTracker2FB xrCreateEyeTracker2FB;
extern PFN_xrDestroyFaceTracker2FB xrDestroyEyeTracker2FB;
extern PFN_xrGetFaceExpressionWeights2FB xrGetFaceExpressionWeights2FB;
class FFaceTrackingXR : public IOpenXRExtensionPlugin
{
public:
// IOculusXROpenXRHMDPlugin
virtual bool GetRequiredExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual bool GetOptionalExtensions(TArray<const ANSICHAR*>& OutExtensions) override;
virtual const void* OnCreateInstance(class IOpenXRHMDModule* InModule, const void* InNext) override;
virtual const void* OnCreateSession(XrInstance InInstance, XrSystemId InSystem, const void* InNext) override;
virtual void OnDestroySession(XrSession InSession) override;
virtual void* OnWaitFrame(XrSession InSession, void* InNext) override;
public:
FFaceTrackingXR();
virtual ~FFaceTrackingXR();
void RegisterAsOpenXRExtension();
bool IsFaceTrackingSupported() const { return bExtFaceTrackingSupported; }
bool IsFaceTrackingEnabled() const { return FaceTracker != XR_NULL_HANDLE; }
bool IsFaceTrackingVisemesSupported() const { return bExtFaceTrackingVisemesSupported; }
bool IsFaceTrackingVisemesEnabled() const { return bVisemesEnabled; }
XrResult StartFaceTracking();
XrResult StopFaceTracking();
XrResult GetCachedFaceState(FOculusXRFaceState& OutState);
XrResult SetVisemesEnabled(bool enabled);
XrResult GetCachedVisemeState(FOculusXRFaceVisemesState& OutState);
private:
void InitOpenXRFunctions(XrInstance InInstance);
void Update_GameThread(XrSession InSession);
bool bExtFaceTrackingSupported;
bool bExtFaceTrackingVisemesSupported;
bool bVisemesEnabled;
FOpenXRHMD* OpenXRHMD;
FOculusXRFaceState CachedFaceState;
FOculusXRFaceVisemesState CachedVisemeState;
XrFaceTracker2FB FaceTracker = XR_NULL_HANDLE;
};
} // namespace XRMovement
#undef LOCTEXT_NAMESPACE

View File

@@ -0,0 +1,9 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include <khronos/openxr/openxr.h>
#include <khronos/openxr/meta_openxr_preview/meta_body_tracking_calibration.h>
#include <khronos/openxr/meta_openxr_preview/meta_body_tracking_fidelity.h>
#include <khronos/openxr/meta_openxr_preview/meta_body_tracking_full_body.h>
#include <khronos/openxr/meta_openxr_preview/metax1_face_tracking_visemes.h>

View File

@@ -0,0 +1,59 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "Modules/ModuleManager.h"
/**
* The public interface to this module. In most cases, this interface is only public to sibling modules
* within this plugin.
*/
class IOculusXRMovementModule : public IModuleInterface
{
public:
/**
* Singleton-like access to this module's interface. This is just for convenience!
* Beware of calling this during the shutdown phase, though. Your module might have been unloaded already.
*
* @return Returns singleton instance, loading the module on demand if needed
*/
static inline IOculusXRMovementModule& Get()
{
return FModuleManager::GetModuleChecked<IOculusXRMovementModule>("OculusXRMovement");
}
/**
* Checks to see if this module is loaded and ready. It is only valid to call Get() if IsAvailable() returns true.
*
* @return True if the module is loaded and ready to use
*/
static inline bool IsAvailable()
{
return FModuleManager::Get().IsModuleLoaded("OculusXRMovement");
}
/**
* Returns the LiveLinkSource associated with this IOculusXRMovementModule.
*
* @return Shared pointer to the Meta MovementSDK source.
*/
virtual TSharedPtr<class ILiveLinkSource> GetLiveLinkSource() = 0;
/**
* Checks if the LiveLinkSource has been created.
*
* @return True if the LiveLinkSource has been created with GetLiveLinkSource or AddLiveLinkSource.
*/
virtual bool IsLiveLinkSourceValid() const = 0;
/**
* Make sure Meta MovementSDK Live Link source exist.
*/
virtual void AddLiveLinkSource() = 0;
/**
* Destroy Meta MovementSDK Live Link source.
*/
virtual void RemoveLiveLinkSource() = 0;
};

View File

@@ -0,0 +1,31 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Animation/AnimTypes.h"
#include "LiveLinkRetargetAsset.h"
#include "Containers/StaticArray.h"
#include "OculusXRMovementTypes.h"
#include "Misc/EngineVersionComparison.h"
#include "OculusXRAnimCurveMapping.generated.h"
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRAnimCurveMapping
{
GENERATED_BODY()
FOculusXRAnimCurveMapping(){};
FOculusXRAnimCurveMapping(const std::initializer_list<FName> CurveNamesList)
: CurveNames(CurveNamesList)
{
}
/**
* Skeleton's animation curve names
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
TArray<FName> CurveNames;
};

View File

@@ -0,0 +1,69 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Components/PoseableMeshComponent.h"
#include "OculusXRMovementTypes.h"
#include "OculusXRBodyTrackingComponent.generated.h"
UENUM(BlueprintType)
enum class EOculusXRBodyTrackingMode : uint8
{
PositionAndRotation,
RotationOnly,
NoTracking
};
UCLASS(Blueprintable, meta = (BlueprintSpawnableComponent, DisplayName = "OculusXR Body Tracking Component"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXRBodyTrackingComponent : public UPoseableMeshComponent
{
GENERATED_BODY()
public:
UOculusXRBodyTrackingComponent();
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
/**
* Restore all bones to their initial transforms
*/
UFUNCTION(BlueprintCallable, Category = "OculusXR|Movement")
void ResetAllBoneTransforms();
/**
* How are the results of body tracking applied to the mesh.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
EOculusXRBodyTrackingMode BodyTrackingMode;
/**
* The bone name associated with each bone ID.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
TMap<EOculusXRBoneID, FName> BoneNames;
/**
* Do not apply body state to bones if confidence is lower than this value. Confidence is in range [0,1].
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement", meta = (ClampMin = "0", ClampMax = "1", UIMin = "0", UIMax = "1"))
float ConfidenceThreshold;
private:
bool InitializeBodyBones();
// One meter in unreal world units.
float WorldToMeters;
// The index of each mapped bone after the discovery and association of bone names.
TMap<EOculusXRBoneID, int32> MappedBoneIndices;
// Saved body state.
FOculusXRBodyState BodyState;
// Stop the tracker just once.
static int TrackingInstanceCount;
};

View File

@@ -0,0 +1,99 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Components/ActorComponent.h"
#include "Components/PoseableMeshComponent.h"
#include "OculusXRMovementTypes.h"
#include "OculusXREyeTrackingComponent.generated.h"
struct FOculusXREyeTrackingData
{
public:
FOculusXREyeTrackingData()
: EyeIsMapped(false)
, MappedBoneName(NAME_None)
{
}
bool EyeIsMapped;
FName MappedBoneName;
FQuat InitialRotation;
};
UCLASS(Blueprintable, meta = (BlueprintSpawnableComponent, DisplayName = "OculusXR Eye Tracking Component"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXREyeTrackingComponent : public UActorComponent
{
GENERATED_BODY()
public:
UOculusXREyeTrackingComponent();
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
/**
* Reset the rotation values of the eyes to their initial rotation
*/
UFUNCTION(BlueprintCallable, Category = "Oculus|Movement")
void ClearRotationValues();
/**
* The name of the poseable mesh component that this component targets for eyes glazes movement.
* This must be the name of a component on this actor.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
FName TargetMeshComponentName;
/**
* The map of eye to mesh bone that this component supports.
* Names are validated on (@see BeginPlay) so only valid bone names will be targeted.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
TMap<EOculusXREye, FName> EyeToBone;
/**
* Update the target mesh position when eye state changes
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
bool bUpdatePosition;
/**
* Update the target mesh rotation when eye state changes
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
bool bUpdateRotation;
/**
* Do not accept an eye gaze state if confidence is lower than this value. Confidence is in range [0,1].
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
float ConfidenceThreshold;
/**
* Bypass eye gaze state validity.
*
* @Note: It doesn't check the confidence (@see ConfidenceThreshold). The eye gaze state can be marked as invalid. This flag bypass that state flag.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
bool bAcceptInvalid;
private:
bool InitializeEyes();
// One meter in unreal world units.
float WorldToMeters;
// Per eye, eye tracking data
TStaticArray<FOculusXREyeTrackingData, static_cast<uint32>(EOculusXREye::COUNT)> PerEyeData;
// The mesh component targeted for eyes
UPROPERTY()
UPoseableMeshComponent* TargetPoseableMeshComponent;
// Stop the tracker just once.
static int TrackingInstanceCount;
};

View File

@@ -0,0 +1,104 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Components/SkeletalMeshComponent.h"
#include "OculusXRMorphTargetsController.h"
#include "OculusXRMovementTypes.h"
#include "OculusXRFaceTrackingComponent.generated.h"
UCLASS(Blueprintable, meta = (BlueprintSpawnableComponent, DisplayName = "OculusXR Face Tracking Component"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXRFaceTrackingComponent : public UActorComponent
{
GENERATED_BODY()
public:
UOculusXRFaceTrackingComponent();
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
/**
* Set face expression value with expression key and value(0-1).
*
* @param Expression : The expression key that will be modified.
* @param Value : The new value to assign to the expression, 0 will remove all changes.
*/
UFUNCTION(BlueprintCallable, Category = "Components|OculusXRFaceTracking", meta = (UnsafeDuringActorConstruction = "true"))
void SetExpressionValue(EOculusXRFaceExpression Expression, float Value);
/**
* Get a face expression value given an expression key.
*
* @param Expression : The expression key that will be queried.
*/
UFUNCTION(BlueprintCallable, Category = "Components|OculusXRFaceTracking")
float GetExpressionValue(EOculusXRFaceExpression Expression) const;
/**
* Clears all face expression values.
*/
UFUNCTION(BlueprintCallable, Category = "Components|OculusXRFaceTracking")
void ClearExpressionValues();
/**
* The name of the skinned mesh component that this component targets for facial expression.
* This must be the name of a component on this actor.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
FName TargetMeshComponentName;
/**
* If the face data is invalid for at least this or longer than this time then all face blendshapes/morph targets are reset to zero.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
float InvalidFaceDataResetTime;
/**
* The list of expressions that this component supports.
* Names are validated on startup so only valid morph targets on the skeletal mesh will be targeted.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
TMap<EOculusXRFaceExpression, FName> ExpressionNames;
/**
* An array of optional expression modifiers that can be applied.
*/
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<FOculusXRFaceExpressionModifier> ExpressionModifiers;
/**
* This flag determines if the face should be updated or not during the components tick.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
bool bUpdateFace;
/**
* This flag determines if the face should be modified with Expression Modifiers or not during the components tick.
*/
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "OculusXR|Movement")
bool bUseModifiers;
private:
bool InitializeFaceTracking();
// The mesh component targeted for expressions
UPROPERTY()
USkinnedMeshComponent* TargetMeshComponent;
// Which mapped expressions are valid
TStaticArray<bool, static_cast<uint32>(EOculusXRFaceExpression::COUNT)> ExpressionValid;
// Morph targets controller
FOculusXRMorphTargetsController MorphTargets;
FOculusXRFaceState FaceState;
// Timer that counts up until we reset morph curves if we've failed to get face state
float InvalidFaceStateTimer;
// Stop the tracker just once.
static int TrackingInstanceCount;
};

View File

@@ -0,0 +1,143 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "LiveLinkRetargetAsset.h"
#include "OculusXRMovementTypes.h"
#include "Containers/StaticArray.h"
#include "BonePose.h"
#include "OculusXRLiveLinkRetargetBodyAsset.generated.h"
UENUM(BlueprintType, meta = (DisplayName = "Axis"))
enum class EOculusXRAxis : uint8
{
X = 0 UMETA(DisplayName = "X"),
Y = 1 UMETA(DisplayName = "Y"),
Z = 2 UMETA(DisplayName = "Z"),
NegativeX = 3 UMETA(DisplayName = "-X"),
NegativeY = 4 UMETA(DisplayName = "-Y"),
NegativeZ = 5 UMETA(DisplayName = "-Z"),
};
UENUM(BlueprintType, meta = (DisplayName = "Retargeting mode"))
enum class EOculusXRRetargetingMode : uint8
{
Full UMETA(DisplayName = "Rotations and positions"),
Rotations UMETA(DisplayName = "Only rotations"),
RotationsPlusRoot UMETA(DisplayName = "Rotations and root position"),
RotationsPlusHips UMETA(DisplayName = "Rotations and hips position"),
None UMETA(DisplayName = "Disabled"),
};
USTRUCT(BlueprintType, meta = (DisplayName = "Bone local correction"))
struct OCULUSXRMOVEMENT_API FOculusXRBoneCorrection
{
GENERATED_BODY()
FOculusXRBoneCorrection()
: PositionOffset(FVector::ZeroVector), RotationOffset(FRotator::ZeroRotator){};
/**
* Position offset in local space.
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
FVector PositionOffset;
/**
* Rotation offset in local space.
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
FRotator RotationOffset;
};
USTRUCT(BlueprintType, meta = (DisplayName = "Correction applied to set of bones"))
struct OCULUSXRMOVEMENT_API FOculusXRBoneCorrectionSet
{
GENERATED_BODY()
FOculusXRBoneCorrectionSet(){};
/**
* Set of bones to which the correction will be applied.
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
TSet<EOculusXRBoneID> Bones;
/**
* The correction for this set.
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
FOculusXRBoneCorrection BoneCorrection;
};
UCLASS(Blueprintable, meta = (DisplayName = "MetaXR MovementSDK LiveLink retarget body asset"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXRLiveLinkRetargetBodyAsset : public ULiveLinkRetargetAsset
{
GENERATED_UCLASS_BODY()
virtual void Initialize() override;
virtual void BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) override;
/**
* Remapping from bone ID to target skeleton's bone name.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
TMap<EOculusXRBoneID, FName> BoneRemapping;
/**
* Correction applied to all bones.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
FOculusXRBoneCorrection GlobalCorrection;
/**
* Groups of local bone corrections.
*
* Order matters. A bone can be corrected multiple times.
* Corrections will be applied with the same order as in this array.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
TArray<FOculusXRBoneCorrectionSet> LocalCorrections;
/**
* Switch between retargeting modes.
*/
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
EOculusXRRetargetingMode RetargetingMode;
/**
* Forward vector axis is the direction towards which the target mesh is oriented.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
EOculusXRAxis ForwardMesh;
private:
// Scale the source tracking positions. This will be initialized with WorldToMeters value.
float Scale;
// Movement tracking is oriented towards X axis.
const EOculusXRAxis ForwardTracking{ EOculusXRAxis::X };
// Transform from tracking to mesh space.
FTransform TrackingSpaceToMeshSpace;
// Correction applied to all bones
FTransform GlobalBoneCorrection;
// Correction applied per bone
TStaticArray<FTransform, static_cast<uint8>(EOculusXRBoneID::COUNT)> LocalBoneCorrections;
// Target skeleton's bone name per bone id
TStaticArray<FName, static_cast<uint8>(EOculusXRBoneID::COUNT)> BoneNames;
// Latest bone container serial number
uint16 LastBoneContainerSerialNumber;
// Compact pose indices per bone id
TStaticArray<FCompactPoseBoneIndex, static_cast<uint8>(EOculusXRBoneID::COUNT)> LastSkeletonBoneRemapping{ InPlace, FCompactPoseBoneIndex(INDEX_NONE) };
// Recalculate skeleton dependent mappings
void OnBoneContainerChanged(const FBoneContainer& BoneContainer);
};

View File

@@ -0,0 +1,42 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Animation/AnimTypes.h"
#include "LiveLinkRetargetAsset.h"
#include "Containers/StaticArray.h"
#include "OculusXRMovementTypes.h"
#include "Misc/EngineVersionComparison.h"
#include "OculusXRAnimCurveMapping.h"
#include "OculusXRLiveLinkRetargetFaceAsset.generated.h"
UCLASS(Blueprintable, meta = (DisplayName = "MetaXR MovementSDK LiveLink retarget face asset"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXRLiveLinkRetargetFaceAsset : public ULiveLinkRetargetAsset
{
GENERATED_UCLASS_BODY()
virtual void Initialize() override;
virtual void BuildPoseAndCurveFromBaseData(float DeltaTime, const FLiveLinkBaseStaticData* InBaseStaticData, const FLiveLinkBaseFrameData* InBaseFrameData, FCompactPose& OutPose, FBlendedCurve& OutCurve) override;
/**
* Map face expression to Skeleton's animation curve mapping names.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
TMap<EOculusXRFaceExpression, FOculusXRAnimCurveMapping> CurveRemapping;
private:
// Latest skeleton used to build pose
FGuid LastSkeletonGuid;
// Remapping used for latest used skeleton
#if UE_VERSION_OLDER_THAN(5, 3, 0)
TStaticArray<TArray<SmartName::UID_Type>, static_cast<uint8>(EOculusXRFaceExpression::COUNT)> RemappingForLastSkeleton;
#else
TStaticArray<TArray<FName>, static_cast<uint8>(EOculusXRFaceExpression::COUNT)> RemappingForLastSkeleton;
#endif
// Recalculate skeleton dependent mappings
void OnSkeletonChanged(const USkeleton* Skeleton);
};

View File

@@ -0,0 +1,42 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "Animation/AnimTypes.h"
#include "LiveLinkRetargetAsset.h"
#include "Containers/StaticArray.h"
#include "OculusXRMovementTypes.h"
#include "Misc/EngineVersionComparison.h"
#include "OculusXRAnimCurveMapping.h"
#include "OculusXRLiveLinkRetargetFaceVisemesAsset.generated.h"
UCLASS(Blueprintable, meta = (DisplayName = "MetaXR MovementSDK LiveLink retarget face visemes asset"), ClassGroup = OculusXRHMD)
class OCULUSXRMOVEMENT_API UOculusXRLiveLinkRetargetFaceVisemesAsset : public ULiveLinkRetargetAsset
{
GENERATED_UCLASS_BODY()
virtual void Initialize() override;
virtual void BuildPoseAndCurveFromBaseData(float DeltaTime, const FLiveLinkBaseStaticData* InBaseStaticData, const FLiveLinkBaseFrameData* InBaseFrameData, FCompactPose& OutPose, FBlendedCurve& OutCurve) override;
/**
* Map face expression to Skeleton's animation curve mapping names.
*/
UPROPERTY(EditDefaultsOnly, Category = "OculusXR|Movement")
TMap<EOculusXRFaceVisemesExpression, FOculusXRAnimCurveMapping> CurveRemapping;
private:
// Latest skeleton used to build pose
FGuid LastSkeletonGuid;
// Remapping used for latest used skeleton
#if UE_VERSION_OLDER_THAN(5, 3, 0)
TStaticArray<TArray<SmartName::UID_Type>, static_cast<uint8>(EOculusXRFaceVisemesExpression::COUNT)> RemappingForLastSkeleton;
#else
TStaticArray<TArray<FName>, static_cast<uint8>(EOculusXRFaceVisemesExpression::COUNT)> RemappingForLastSkeleton;
#endif
// Recalculate skeleton dependent mappings
void OnSkeletonChanged(const USkeleton* Skeleton);
};

View File

@@ -0,0 +1,36 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "Components/SkinnedMeshComponent.h"
/*
* Struct that allows applying morph targets data to an arbitrary skinned mesh component
* instead of relying on the skeletal mesh component.
*
* Usage - In a tick method of your choosing:
* 1) ResetMorphTargetCurves(Component) at the start of the update.
* 2) SetMorphTarget(...) as many times as needed based on your data set.
* 3) ApplyMorphTargets(Component) at the end of the update to apply the morph targets to the anim runtime.
*/
struct OCULUSXRMOVEMENT_API FOculusXRMorphTargetsController
{
public:
// Clears active morph targets
void ResetMorphTargetCurves(USkinnedMeshComponent* TargetMeshComponent);
// Will apply morph target data to the underlying runtime skeletal mesh
void ApplyMorphTargets(USkinnedMeshComponent* TargetMeshComponent);
// Sets a specific morph target value
void SetMorphTarget(FName MorphTargetName, float Value);
// Gets a specific morph target value
float GetMorphTarget(FName MorphTargetName) const;
// Clears all morph target curves data
void ClearMorphTargets();
// List of morph targets on this controller
TMap<FName, float> MorphTargetCurves;
};

View File

@@ -0,0 +1,44 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementTypes.h"
#include "OculusXRMovementFunctions.h"
struct OCULUSXRMOVEMENT_API OculusXRMovement
{
public:
static bool GetBodyState(FOculusXRBodyState& outOculusXRBodyState, float WorldToMeters = 100.0f);
static bool IsBodyTrackingEnabled();
static bool IsBodyTrackingSupported();
static bool StartBodyTracking();
static bool StopBodyTracking();
static bool StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet);
static bool RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity);
static bool ResetBodyTrackingCalibration();
static bool SuggestBodyTrackingCalibrationOverride(float height);
static bool GetBodySkeleton(FOculusXRBodySkeleton& outOculusXRBodyState, float WorldToMeters = 100.0f);
static bool GetFaceState(FOculusXRFaceState& outOculusXRFaceState);
static bool IsFaceTrackingEnabled();
static bool IsFaceTrackingSupported();
static bool StartFaceTracking();
static bool StopFaceTracking();
static bool IsFaceTrackingVisemesEnabled();
static bool IsFaceTrackingVisemesSupported();
static bool SetFaceTrackingVisemesEnabled(bool enabled);
static bool GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState);
static bool GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters = 100.0f);
static bool IsEyeTrackingEnabled();
static bool IsEyeTrackingSupported();
static bool StartEyeTracking();
static bool StopEyeTracking();
static bool IsFullBodyTrackingEnabled();
private:
static TSharedPtr<IOculusXRMovementFunctions> GetOculusXRMovementFunctionsImpl();
static TSharedPtr<IOculusXRMovementFunctions> MovementFunctionsImpl;
};

View File

@@ -0,0 +1,83 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementTypes.h"
#include "Kismet/BlueprintFunctionLibrary.h"
#include "OculusXRMovementFunctionLibrary.generated.h"
UCLASS()
class OCULUSXRMOVEMENT_API UOculusXRMovementFunctionLibrary : public UBlueprintFunctionLibrary
{
GENERATED_BODY()
public:
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool TryGetBodyState(FOculusXRBodyState& outBodyState, float WorldToMeters = 100.0f);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool IsBodyTrackingEnabled();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool IsBodyTrackingSupported();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool ResetBodyTrackingCalibration();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool SuggestBodyTrackingCalibrationOverride(float height);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet);
UFUNCTION(BlueprintCallable, meta = (DeprecatedFunction, DeprecationMessage = "StartBodyTracking is deprecated, use StartBodyTrackingByJointSet."), Category = "OculusXR|Body")
static bool StartBodyTracking();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Body")
static bool StopBodyTracking();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool TryGetFaceState(FOculusXRFaceState& outFaceState);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool IsFaceTrackingVisemesSupported();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool IsFaceTrackingVisemesEnabled();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool SetFaceTrackingVisemesEnabled(bool faceTrackingVisemesEnabled);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool TryGetFaceVisemesState(FOculusXRFaceVisemesState& outFaceVisemesState);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool IsFaceTrackingEnabled();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool IsFaceTrackingSupported();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool StartFaceTracking();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Face")
static bool StopFaceTracking();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Eyes")
static bool TryGetEyeGazesState(FOculusXREyeGazesState& outEyeGazesState, float WorldToMeters = 100.0f);
UFUNCTION(BlueprintCallable, Category = "OculusXR|Eyes")
static bool IsEyeTrackingEnabled();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Eyes")
static bool IsEyeTrackingSupported();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Eyes")
static bool StartEyeTracking();
UFUNCTION(BlueprintCallable, Category = "OculusXR|Eyes")
static bool StopEyeTracking();
};

View File

@@ -0,0 +1,39 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "OculusXRMovementTypes.h"
class OCULUSXRMOVEMENT_API IOculusXRMovementFunctions
{
public:
virtual bool GetBodyState(FOculusXRBodyState& outOculusXRBodyState, float WorldToMeters) = 0;
virtual bool IsBodyTrackingEnabled() = 0;
virtual bool IsBodyTrackingSupported() = 0;
virtual bool StartBodyTracking() = 0;
virtual bool StopBodyTracking() = 0;
virtual bool StartBodyTrackingByJointSet(EOculusXRBodyJointSet jointSet) = 0;
virtual bool RequestBodyTrackingFidelity(EOculusXRBodyTrackingFidelity fidelity) = 0;
virtual bool ResetBodyTrackingCalibration() = 0;
virtual bool SuggestBodyTrackingCalibrationOverride(float height) = 0;
virtual bool GetBodySkeleton(FOculusXRBodySkeleton& outOculusXRBodyState, float WorldToMeters) = 0;
virtual bool GetFaceState(FOculusXRFaceState& outOculusXRFaceState) = 0;
virtual bool IsFaceTrackingEnabled() = 0;
virtual bool IsFaceTrackingSupported() = 0;
virtual bool StartFaceTracking() = 0;
virtual bool StopFaceTracking() = 0;
virtual bool SetFaceTrackingVisemesEnabled(bool enabled) = 0;
virtual bool GetFaceVisemesState(FOculusXRFaceVisemesState& outOculusXRFaceVisemesState) = 0;
virtual bool IsFaceTrackingVisemesEnabled() = 0;
virtual bool IsFaceTrackingVisemesSupported() = 0;
virtual bool GetEyeGazesState(FOculusXREyeGazesState& outOculusXREyeGazesState, float WorldToMeters) = 0;
virtual bool IsEyeTrackingEnabled() = 0;
virtual bool IsEyeTrackingSupported() = 0;
virtual bool StartEyeTracking() = 0;
virtual bool StopEyeTracking() = 0;
virtual bool IsFullBodyTrackingEnabled() = 0;
};

View File

@@ -0,0 +1,24 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
namespace OculusXRUtility
{
template <typename T>
T* FindComponentByName(AActor* Actor, const FName& ComponentName)
{
if (IsValid(Actor) && (ComponentName != NAME_None))
{
TArray<T*> ComponentsOfType;
Actor->GetComponents<T>(ComponentsOfType);
T** FoundComponent = ComponentsOfType.FindByPredicate([Name = ComponentName.ToString()](T* Component) { return Component->GetName().Equals(Name); });
if (FoundComponent != nullptr)
{
return *FoundComponent;
}
}
return nullptr;
}
} // namespace OculusXRUtility

View File

@@ -0,0 +1,429 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
#pragma once
#include "CoreMinimal.h"
#include "OculusXRMovementTypes.generated.h"
namespace XRSpaceFlags
{
static const uint64 XR_SPACE_LOCATION_ORIENTATION_VALID_BIT = 0x00000001;
static const uint64 XR_SPACE_LOCATION_POSITION_VALID_BIT = 0x00000002;
static const uint64 XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT = 0x00000004;
static const uint64 XR_SPACE_LOCATION_POSITION_TRACKED_BIT = 0x00000008;
} // namespace XRSpaceFlags
UENUM(BlueprintType)
enum class EOculusXRBodyJointSet : uint8
{
UpperBody = 0,
FullBody = 1
};
UENUM(BlueprintType)
enum class EOculusXRBodyTrackingFidelity : uint8
{
Unset = 0 UMETA(Hidden),
Low = 1,
High = 2,
};
UENUM(BlueprintType)
enum class EOculusXRBoneID : uint8
{
BodyRoot = 0,
BodyHips = 1,
BodySpineLower = 2,
BodySpineMiddle = 3,
BodySpineUpper = 4,
BodyChest = 5,
BodyNeck = 6,
BodyHead = 7,
BodyLeftShoulder = 8,
BodyLeftScapula = 9,
BodyLeftArmUpper = 10,
BodyLeftArmLower = 11,
BodyLeftHandWristTwist = 12,
BodyRightShoulder = 13,
BodyRightScapula = 14,
BodyRightArmUpper = 15,
BodyRightArmLower = 16,
BodyRightHandWristTwist = 17,
BodyLeftHandPalm = 18,
BodyLeftHandWrist = 19,
BodyLeftHandThumbMetacarpal = 20,
BodyLeftHandThumbProximal = 21,
BodyLeftHandThumbDistal = 22,
BodyLeftHandThumbTip = 23,
BodyLeftHandIndexMetacarpal = 24,
BodyLeftHandIndexProximal = 25,
BodyLeftHandIndexIntermediate = 26,
BodyLeftHandIndexDistal = 27,
BodyLeftHandIndexTip = 28,
BodyLeftHandMiddleMetacarpal = 29,
BodyLeftHandMiddleProximal = 30,
BodyLeftHandMiddleIntermediate = 31,
BodyLeftHandMiddleDistal = 32,
BodyLeftHandMiddleTip = 33,
BodyLeftHandRingMetacarpal = 34,
BodyLeftHandRingProximal = 35,
BodyLeftHandRingIntermediate = 36,
BodyLeftHandRingDistal = 37,
BodyLeftHandRingTip = 38,
BodyLeftHandLittleMetacarpal = 39,
BodyLeftHandLittleProximal = 40,
BodyLeftHandLittleIntermediate = 41,
BodyLeftHandLittleDistal = 42,
BodyLeftHandLittleTip = 43,
BodyRightHandPalm = 44,
BodyRightHandWrist = 45,
BodyRightHandThumbMetacarpal = 46,
BodyRightHandThumbProximal = 47,
BodyRightHandThumbDistal = 48,
BodyRightHandThumbTip = 49,
BodyRightHandIndexMetacarpal = 50,
BodyRightHandIndexProximal = 51,
BodyRightHandIndexIntermediate = 52,
BodyRightHandIndexDistal = 53,
BodyRightHandIndexTip = 54,
BodyRightHandMiddleMetacarpal = 55,
BodyRightHandMiddleProximal = 56,
BodyRightHandMiddleIntermediate = 57,
BodyRightHandMiddleDistal = 58,
BodyRightHandMiddleTip = 59,
BodyRightHandRingMetacarpal = 60,
BodyRightHandRingProximal = 61,
BodyRightHandRingIntermediate = 62,
BodyRightHandRingDistal = 63,
BodyRightHandRingTip = 64,
BodyRightHandLittleMetacarpal = 65,
BodyRightHandLittleProximal = 66,
BodyRightHandLittleIntermediate = 67,
BodyRightHandLittleDistal = 68,
BodyRightHandLittleTip = 69,
BodyLeftUpperLeg = 70,
BodyLeftLowerLeg = 71,
BodyLeftFootAnkleTwist = 72,
BodyLeftFootAnkle = 73,
BodyLeftFootSubtalar = 74,
BodyLeftFootTransverse = 75,
BodyLeftFootBall = 76,
BodyRightUpperLeg = 77,
BodyRightLowerLeg = 78,
BodyRightFootAnkleTwist = 79,
BodyRightFootAnkle = 80,
BodyRightFootSubtalar = 81,
BodyRightFootTransverse = 82,
BodyRightFootBall = 83,
COUNT = 84 UMETA(Hidden),
None = 255 UMETA(Hidden),
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRBodyJoint
{
GENERATED_BODY()
public:
FOculusXRBodyJoint();
uint64 LocationFlags;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool bIsValid;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FRotator Orientation;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FVector Position;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRBodyState
{
GENERATED_BODY()
public:
FOculusXRBodyState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool IsActive;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Confidence;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
int SkeletonChangedCount;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Time;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<FOculusXRBodyJoint> Joints;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRBodySkeletonBone
{
GENERATED_BODY()
public:
FOculusXRBodySkeletonBone();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FRotator Orientation;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FVector Position;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
EOculusXRBoneID BoneId;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
EOculusXRBoneID ParentBoneIndex;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRBodySkeleton
{
GENERATED_BODY()
public:
FOculusXRBodySkeleton();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
int NumBones;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<FOculusXRBodySkeletonBone> Bones;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRBodySkeletonState
{
GENERATED_BODY()
public:
FOculusXRBodySkeletonState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FOculusXRBodyState BodyState;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FOculusXRBodySkeleton SkeletonState;
};
UENUM(BlueprintType)
enum class EFaceTrackingDataSource : uint8
{
Visual = 0 UMETA(DisplayName = "Visual"),
Audio = 1 UMETA(DisplayName = "Audio"),
MAX = 2 UMETA(Hidden),
};
UENUM(BlueprintType)
enum class EOculusXRFaceExpression : uint8
{
// Removed invalid to make this supported as a uint8 enum class
BrowLowererL = 0,
BrowLowererR = 1,
CheekPuffL = 2,
CheekPuffR = 3,
CheekRaiserL = 4,
CheekRaiserR = 5,
CheekSuckL = 6,
CheekSuckR = 7,
ChinRaiserB = 8,
ChinRaiserT = 9,
DimplerL = 10,
DimplerR = 11,
EyesClosedL = 12,
EyesClosedR = 13,
EyesLookDownL = 14,
EyesLookDownR = 15,
EyesLookLeftL = 16,
EyesLookLeftR = 17,
EyesLookRightL = 18,
EyesLookRightR = 19,
EyesLookUpL = 20,
EyesLookUpR = 21,
InnerBrowRaiserL = 22,
InnerBrowRaiserR = 23,
JawDrop = 24,
JawSidewaysLeft = 25,
JawSidewaysRight = 26,
JawThrust = 27,
LidTightenerL = 28,
LidTightenerR = 29,
LipCornerDepressorL = 30,
LipCornerDepressorR = 31,
LipCornerPullerL = 32,
LipCornerPullerR = 33,
LipFunnelerLB = 34,
LipFunnelerLT = 35,
LipFunnelerRB = 36,
LipFunnelerRT = 37,
LipPressorL = 38,
LipPressorR = 39,
LipPuckerL = 40,
LipPuckerR = 41,
LipStretcherL = 42,
LipStretcherR = 43,
LipSuckLB = 44,
LipSuckLT = 45,
LipSuckRB = 46,
LipSuckRT = 47,
LipTightenerL = 48,
LipTightenerR = 49,
LipsToward = 50,
LowerLipDepressorL = 51,
LowerLipDepressorR = 52,
MouthLeft = 53,
MouthRight = 54,
NoseWrinklerL = 55,
NoseWrinklerR = 56,
OuterBrowRaiserL = 57,
OuterBrowRaiserR = 58,
UpperLidRaiserL = 59,
UpperLidRaiserR = 60,
UpperLipRaiserL = 61,
UpperLipRaiserR = 62,
TongueTipInterdental = 63,
TongueTipAlveolar = 64,
TongueFrontDorsalPalate = 65,
TongueMidDorsalPalate = 66,
TongueBackDorsalVelar = 67,
TongueOut = 68,
TongueRetreat = 69,
COUNT = 70 UMETA(Hidden),
};
UENUM(BlueprintType)
enum class EOculusXRFaceConfidence : uint8
{
Lower = 0,
Upper = 1,
COUNT = 2,
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRFaceState
{
GENERATED_BODY()
public:
FOculusXRFaceState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<float> ExpressionWeights;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<float> ExpressionWeightConfidences;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool bIsValid;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool bIsEyeFollowingBlendshapesValid;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Time;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
EFaceTrackingDataSource DataSource;
};
UENUM(BlueprintType)
enum class EOculusXRFaceVisemesExpression : uint8
{
SIL = 0,
PP = 1,
FF = 2,
TH = 3,
DD = 4,
KK = 5,
CH = 6,
SS = 7,
NN = 8,
RR = 9,
AA = 10,
E = 11,
IH = 12,
OH = 13,
OU = 14,
COUNT = 15 UMETA(Hidden),
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRFaceVisemesState
{
GENERATED_BODY()
public:
FOculusXRFaceVisemesState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool bIsValid;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Time;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<float> ExpressionVisemeWeights;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXRFaceExpressionModifier
{
GENERATED_BODY()
public:
FOculusXRFaceExpressionModifier();
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
TArray<EOculusXRFaceExpression> FaceExpressions;
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
float MinValue;
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
float MaxValue;
UPROPERTY(EditAnywhere, Category = "OculusXR|Movement")
float Multiplier;
};
UENUM(BlueprintType)
enum class EOculusXREye : uint8
{
Left = 0,
Right = 1,
COUNT = 2,
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXREyeGazeState
{
GENERATED_BODY()
public:
FOculusXREyeGazeState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FRotator Orientation;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
FVector Position;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Confidence;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
bool bIsValid;
};
USTRUCT(BlueprintType)
struct OCULUSXRMOVEMENT_API FOculusXREyeGazesState
{
GENERATED_BODY()
public:
FOculusXREyeGazesState();
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
TArray<FOculusXREyeGazeState> EyeGazes;
UPROPERTY(BlueprintReadOnly, Category = "OculusXR|Movement")
float Time;
};