henrisel/DeltaVR3DModelGeneration
1
0
Fork 0
forked from cgvr/DeltaVR
Code Pull Requests Activity

WIP animate mouth scale based on precalculated voiceline amplitude timelines

Browse Source
This commit is contained in:
henrisel
2026-02-02 19:15:31 +02:00
parent a0d1ee35cd
commit 5a3f566541
43 changed files with 1475 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Assets/_PROJECT/Scripts/Audio/AudioManager.cs
View File

@@ -3,7 +3,6 @@ using FMODUnity;
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using UnityEditor;
using UnityEngine;
using UnityEngine.SceneManagement;
@@ -49,6 +48,7 @@ public class AudioManager : MonoBehaviour
private Bus musicBus;
private Bus sfxBus;
private Bus uiBus;
private Bus voiceoverBus;
const string sid = "00000000-0000-0000-0000-000000000000";
static readonly Guid nullGuid = new Guid(sid);
@@ -112,6 +112,7 @@ public class AudioManager : MonoBehaviour
_instance.ambientBus = RuntimeManager.GetBus("bus:/Ambiences");
_instance.sfxBus = RuntimeManager.GetBus("bus:/SFX");
_instance.uiBus = RuntimeManager.GetBus("bus:/UI");
_instance.voiceoverBus = RuntimeManager.GetBus("bus:/Voiceovers");
_instance.masterVCA = RuntimeManager.GetVCA("vca:/Master");
_instance.musicVCA = RuntimeManager.GetVCA("vca:/Music");
@@ -294,7 +295,7 @@ public class AudioManager : MonoBehaviour
//=====//
//=====//
public void PlayDialogue(string audioTableKey, GameObject emitter = null, float radioAmount = 0f)
public EventInstance PlayDialogue(string audioTableKey, GameObject emitter = null, float radioAmount = 0f)
{
var dialogueEvent = FMODEvents.Instance.VoiceoverAll;
@@ -302,7 +303,7 @@ public class AudioManager : MonoBehaviour
if (dialogueEvent.IsNull)
{
Debug.LogWarning("Dialogue EventReference is not assigned!");
return;
return default;
}
EventInstance instance = RuntimeManager.CreateInstance(dialogueEvent);
@@ -310,7 +311,7 @@ public class AudioManager : MonoBehaviour
if (emitter != null)
{
RuntimeManager.AttachInstanceToGameObject(instance, emitter);
instance.set3DAttributes(FMODUnity.RuntimeUtils.To3DAttributes(emitter.gameObject));
instance.set3DAttributes(RuntimeUtils.To3DAttributes(emitter.gameObject));
}
// Assign the FMOD parameter value (in this case: Continous type)
@@ -334,7 +335,8 @@ public class AudioManager : MonoBehaviour
}
instance.start();
instance.release();
// instance.release();
return instance;
}

2
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/AlienNPC.cs → Assets/_PROJECT/Scripts/ModeGeneration/NPCs/ArcheryRangeNPC.cs
View File

@@ -1,6 +1,6 @@
using UnityEngine;
public class AlienNPC : NPCController
public class ArcheryRangeNPC : NPCController
{
protected override void OnPlayerApproach()
{

0
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/AlienNPC.cs.meta → Assets/_PROJECT/Scripts/ModeGeneration/NPCs/ArcheryRangeNPC.cs.meta
View File

2
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/CafeWaiterNPC.cs
View File

@@ -47,7 +47,7 @@ public class CafeWaiterNPC : NPCController
{
if (state == 0)
{
AudioManager.Instance.PlayDialogue(voiceLineKeys[0], gameObject);
SpeakVoiceLine(0);
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdate;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;

150
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/NPCController.cs
View File

@@ -1,26 +1,42 @@
using DG.Tweening;
using FMOD.Studio;
using System.IO;
using System.Linq;
using UnityEngine;
public abstract class NPCController : MonoBehaviour
{
private Vector3 mouthClosedScale;
private Vector3 mouthOpenScale;
private bool isTalking;
protected Transform playerTransform;
public Transform mouthTransform;
public float mouthScalingMultiplier = 2.5f;
public float mouthMovementDuration = 0.25f;
public string[] voiceLineKeys;
[Header("Mouth Transform")]
public Transform mouth; // assign your billboard mouth object
[Header("Mouth Animation Settings")]
public float minScaleY = 0.3f;
public float maxScaleY = 1.0f;
public float gain = 30f; // multiply RMS to make mouth open larger
public float attack = 0.6f; // faster opening
public float release = 0.2f; // slower closing
[Header("Timeline Folder (StreamingAssets recommended)")]
public string timelineFolder = "CharacterVoicelines";
private float[] rmsCurve;
private EventInstance currentVoicelineEvent;
private bool isSpeaking;
private float smoothed;
private int sampleRate = 50; // RMS samples per second (20ms windows)
// If you change RMS window in Python, update this
// Start is called before the first frame update
void Awake()
{
mouthClosedScale = mouthTransform.localScale;
mouthOpenScale = new Vector3(mouthClosedScale.x, mouthClosedScale.y * mouthScalingMultiplier, mouthClosedScale.z);
isTalking = false;
}
void Start()
@@ -37,6 +53,39 @@ public abstract class NPCController : MonoBehaviour
Vector3 lookTargetPos = new Vector3(playerTransform.position.x, transform.position.y, playerTransform.position.z);
transform.LookAt(lookTargetPos);
}
if (isSpeaking && rmsCurve != null && currentVoicelineEvent.isValid())
{
AnimateMouth();
}
}
private void AnimateMouth()
{
// get FMOD timeline position
currentVoicelineEvent.getTimelinePosition(out int ms);
float time = ms / 1000f;
// find sample index
int index = Mathf.FloorToInt(time * sampleRate);
index = Mathf.Clamp(index, 0, rmsCurve.Length - 1);
float amp = rmsCurve[index] * gain;
// attack/release smoothing
float targetY = Mathf.Clamp(minScaleY + amp, minScaleY, maxScaleY);
if (targetY > smoothed)
smoothed = Mathf.Lerp(smoothed, targetY, attack);
else
smoothed = Mathf.Lerp(smoothed, targetY, release);
// apply mouth scale
Vector3 s = mouth.localScale;
s.y = smoothed;
Debug.Log("mouth scale: " + smoothed);
mouth.localScale = s;
}
private void OnTriggerEnter(Collider other)
@@ -61,39 +110,66 @@ public abstract class NPCController : MonoBehaviour
protected virtual void OnPlayerLeave() {}
public void SpeakVoiceLine(int voiceLineId)
{
AudioManager.Instance.PlayDialogue(voiceLineKeys[voiceLineId], gameObject);
}
public void StartTalking()
{
isTalking = true;
MoveMouth();
}
public void Stoptalking()
{
isTalking = false;
}
private void MoveMouth()
{
if (!isTalking)
if (voiceLineId < 0 || voiceLineId >= voiceLineKeys.Length)
{
Debug.LogError("Invalid voiceLineId!");
return;
}
if (mouthTransform.localScale == mouthClosedScale)
string key = voiceLineKeys[voiceLineId];
LoadCurve(key); // load RMS data
Debug.Log("loaded timeline curve");
currentVoicelineEvent = AudioManager.Instance.PlayDialogue(key, gameObject);
if (!currentVoicelineEvent.isValid())
{
mouthTransform.DOScale(mouthOpenScale, mouthMovementDuration);
}
else
{
mouthTransform.DOScale(mouthClosedScale, mouthMovementDuration);
Debug.LogError("Failed to start dialogue event.");
return;
}
Invoke("MoveMouth", mouthMovementDuration + 0.01f);
//isSpeaking = true;
// Stop mouth on end
currentVoicelineEvent.setCallback((type, inst, param) =>
{
if (type == EVENT_CALLBACK_TYPE.STOPPED)
{
isSpeaking = false;
smoothed = minScaleY;
if (mouth != null)
{
Vector3 s = mouth.localScale;
s.y = minScaleY;
mouth.localScale = s;
}
}
return FMOD.RESULT.OK;
});
}
// ---------------------------
// Load RMS Timeline (.txt)
// ---------------------------
private void LoadCurve(string key)
{
string folderPath = Path.Combine(Application.streamingAssetsPath, timelineFolder);
string filePath = Path.Combine(folderPath, key + ".txt");
if (!File.Exists(filePath))
{
Debug.LogError("Missing RMS timeline file: " + filePath);
rmsCurve = null;
return;
}
var lines = File.ReadAllLines(filePath);
Debug.Log("read lines: " + lines.Length);
rmsCurve = lines.Select(l => float.Parse(l, System.Globalization.CultureInfo.InvariantCulture)).ToArray();
}
}