WIP animate mouth scale based on precalculated voiceline amplitude timelines

Assets/_PROJECT/Scripts/ModeGeneration/NPCs/ArcheryRangeNPC.cs

@@ -1,6 +1,6 @@
 using UnityEngine;
 
-public class AlienNPC : NPCController
+public class ArcheryRangeNPC : NPCController
 {
     protected override void OnPlayerApproach()
     {

Assets/_PROJECT/Scripts/ModeGeneration/NPCs/CafeWaiterNPC.cs

@@ -47,7 +47,7 @@ public class CafeWaiterNPC : NPCController
     {
         if (state == 0)
         {
-            AudioManager.Instance.PlayDialogue(voiceLineKeys[0], gameObject);
+            SpeakVoiceLine(0);
 
             fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdate;
             fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;

Assets/_PROJECT/Scripts/ModeGeneration/NPCs/NPCController.cs

@@ -1,26 +1,42 @@
-using DG.Tweening;
+using FMOD.Studio;
+using System.IO;
+using System.Linq;
 using UnityEngine;
 
 public abstract class NPCController : MonoBehaviour
 {
-    private Vector3 mouthClosedScale;
-    private Vector3 mouthOpenScale;
-    private bool isTalking;
     protected Transform playerTransform;
 
-    public Transform mouthTransform;
-    public float mouthScalingMultiplier = 2.5f;
-    public float mouthMovementDuration = 0.25f;
-
     public string[] voiceLineKeys;
 
+
+    [Header("Mouth Transform")]
+    public Transform mouth;  // assign your billboard mouth object
+
+    [Header("Mouth Animation Settings")]
+    public float minScaleY = 0.3f;
+    public float maxScaleY = 1.0f;
+    public float gain = 30f;      // multiply RMS to make mouth open larger
+    public float attack = 0.6f;   // faster opening
+    public float release = 0.2f;  // slower closing
+
+    [Header("Timeline Folder (StreamingAssets recommended)")]
+    public string timelineFolder = "CharacterVoicelines";
+
+    private float[] rmsCurve;
+    private EventInstance currentVoicelineEvent;
+    private bool isSpeaking;
+    private float smoothed;
+
+    private int sampleRate = 50;   // RMS samples per second (20ms windows)
+    // If you change RMS window in Python, update this
+
+
+
     // Start is called before the first frame update
     void Awake()
     {
-        mouthClosedScale = mouthTransform.localScale;
-        mouthOpenScale = new Vector3(mouthClosedScale.x, mouthClosedScale.y * mouthScalingMultiplier, mouthClosedScale.z);
-
-        isTalking = false;
+        
     }
 
     void Start()
@@ -37,6 +53,39 @@ public abstract class NPCController : MonoBehaviour
             Vector3 lookTargetPos = new Vector3(playerTransform.position.x, transform.position.y, playerTransform.position.z);
             transform.LookAt(lookTargetPos);
         }
+
+        if (isSpeaking && rmsCurve != null && currentVoicelineEvent.isValid())
+        {
+            AnimateMouth();
+        }
+    }
+
+    private void AnimateMouth()
+    {
+
+        // get FMOD timeline position
+        currentVoicelineEvent.getTimelinePosition(out int ms);
+        float time = ms / 1000f;
+
+        // find sample index
+        int index = Mathf.FloorToInt(time * sampleRate);
+        index = Mathf.Clamp(index, 0, rmsCurve.Length - 1);
+
+        float amp = rmsCurve[index] * gain;
+
+        // attack/release smoothing
+        float targetY = Mathf.Clamp(minScaleY + amp, minScaleY, maxScaleY);
+        if (targetY > smoothed)
+            smoothed = Mathf.Lerp(smoothed, targetY, attack);
+        else
+            smoothed = Mathf.Lerp(smoothed, targetY, release);
+
+        // apply mouth scale
+        Vector3 s = mouth.localScale;
+        s.y = smoothed;
+        Debug.Log("mouth scale: " + smoothed);
+        mouth.localScale = s;
+
     }
 
     private void OnTriggerEnter(Collider other)
@@ -61,39 +110,66 @@ public abstract class NPCController : MonoBehaviour
 
     protected virtual void OnPlayerLeave() {}
 
+
     public void SpeakVoiceLine(int voiceLineId)
     {
-        AudioManager.Instance.PlayDialogue(voiceLineKeys[voiceLineId], gameObject);
-    }
-
-    public void StartTalking()
-    {
-        isTalking = true;
-        MoveMouth();
-    }
-
-    public void Stoptalking()
-    {
-        isTalking = false;
-    }
-
-
-    private void MoveMouth()
-    {
-        if (!isTalking)
+        if (voiceLineId < 0 || voiceLineId >= voiceLineKeys.Length)
         {
+            Debug.LogError("Invalid voiceLineId!");
             return;
         }
 
-        if (mouthTransform.localScale == mouthClosedScale)
+        string key = voiceLineKeys[voiceLineId];
+
+        LoadCurve(key);  // load RMS data
+        Debug.Log("loaded timeline curve");
+
+        currentVoicelineEvent = AudioManager.Instance.PlayDialogue(key, gameObject);
+
+        if (!currentVoicelineEvent.isValid())
         {
-            mouthTransform.DOScale(mouthOpenScale, mouthMovementDuration);
-        }
-        else
-        {
-            mouthTransform.DOScale(mouthClosedScale, mouthMovementDuration);
+            Debug.LogError("Failed to start dialogue event.");
+            return;
         }
 
-        Invoke("MoveMouth", mouthMovementDuration + 0.01f);
+        //isSpeaking = true;
+
+        // Stop mouth on end
+        currentVoicelineEvent.setCallback((type, inst, param) =>
+        {
+            if (type == EVENT_CALLBACK_TYPE.STOPPED)
+            {
+                isSpeaking = false;
+                smoothed = minScaleY;
+
+                if (mouth != null)
+                {
+                    Vector3 s = mouth.localScale;
+                    s.y = minScaleY;
+                    mouth.localScale = s;
+                }
+            }
+            return FMOD.RESULT.OK;
+        });
+    }
+
+// ---------------------------
+//   Load RMS Timeline (.txt)
+// ---------------------------
+    private void LoadCurve(string key)
+    {
+        string folderPath = Path.Combine(Application.streamingAssetsPath, timelineFolder);
+        string filePath = Path.Combine(folderPath, key + ".txt");
+
+        if (!File.Exists(filePath))
+        {
+            Debug.LogError("Missing RMS timeline file: " + filePath);
+            rmsCurve = null;
+            return;
+        }
+
+        var lines = File.ReadAllLines(filePath);
+        Debug.Log("read lines: " + lines.Length);
+        rmsCurve = lines.Select(l => float.Parse(l, System.Globalization.CultureInfo.InvariantCulture)).ToArray();
     }
 }