henrisel/DeltaVR3DModelGeneration
1
0
Fork 0
forked from cgvr/DeltaVR
Code Pull Requests Activity

moved archery range model generation logic to npc script, added corresponding voicelines

Browse Source
This commit is contained in:
henrisel
2026-02-08 12:28:59 +02:00
parent 53295a60ed
commit dc55e8d884
12 changed files with 323 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ArcheryRangeModelGenerationController.cs
View File

@@ -1,99 +0,0 @@
using System;
using UnityEngine;
using UnityEngine.UI;
using FMOD.Studio;
public class ArcheryRangeModelGenerationController : MonoBehaviour
{
public MicrophoneStand microphoneStand;
public PushableButton imageGenerationButton;
public PushableButton modelGenerationButton;
public Texture2D GeneratedTexture { get; private set; }
public Image imageDisplay;
public GameObject GeneratedModel { get; private set; }
public float generatedObjectRotationSpeed = 10f;
public Transform modelDisplay;
public Material modelDisplayActiveMaterial;
public Transform wire;
public Material wireActiveMaterial;
public NPCController npcController;
private bool modelGenerationInProgress;
private EventInstance printingSound;
// Start is called before the first frame update
void Start()
{
imageGenerationButton.OnButtonPressed += InvokeImageGeneration;
modelGenerationButton.OnButtonPressed += InvokeModelGeneration;
modelGenerationInProgress = false;
printingSound = AudioManager.Instance.CreateInstance(FMODEvents.Instance.Printing);
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
}
// Update is called once per frame
void Update()
{
if (GeneratedModel != null)
{
GeneratedModel.transform.Rotate(Vector3.up, generatedObjectRotationSpeed * Time.deltaTime);
}
}
private async void InvokeImageGeneration()
{
string inputPrompt = microphoneStand.GetTextOutput();
byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(inputPrompt);
GeneratedTexture = ModelGenerationUtils.CreateTexture(imageBytes);
Sprite sprite = ModelGenerationUtils.CreateSprite(GeneratedTexture);
imageDisplay.sprite = sprite;
imageGenerationButton.Deactivate();
if (!modelGenerationInProgress)
{
modelGenerationButton.Deactivate();
}
npcController.SpeakVoiceLine(3);
}
private async void InvokeModelGeneration()
{
modelGenerationInProgress = true;
printingSound.start();
printingSound.setParameterByName("3DPrinterPrintingJob", 0);
string encodedTexture = Convert.ToBase64String(GeneratedTexture.EncodeToJPG());
byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
GameObject spawnedObject = await ModelGenerationUtils.Instance.SpawnModel(encodedModel);
// Destroy previous generated object
Destroy(GeneratedModel);
spawnedObject.transform.parent = modelDisplay;
spawnedObject.transform.position = modelDisplay.position;
GeneratedModel = spawnedObject;
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
npcController.SpeakVoiceLine(4);
OnModelReady();
modelGenerationButton.Deactivate();
modelGenerationInProgress = false;
}
private void OnModelReady()
{
foreach (MeshRenderer meshRenderer in wire.GetComponentsInChildren<MeshRenderer>())
{
meshRenderer.material = wireActiveMaterial;
}
modelDisplay.GetComponent<MeshRenderer>().material = modelDisplayActiveMaterial;
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.ShapeScannerSuccess, gameObject);
}
}

32
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/MicrophoneStand.cs
View File

@@ -8,9 +8,11 @@ public class MicrophoneStand : MonoBehaviour
public GameObject microphoneOffStatus;
public GameObject microphoneOnStatus;
public NPCController npcController;
public FMODWhisperBridge fmodWhisperBridge;
public delegate void OnPlayerFinishedSpeakingDelegate();
public event OnPlayerFinishedSpeakingDelegate OnPlayerFinishedSpeaking;
// Start is called before the first frame update
void Start()
{
@@ -31,16 +33,12 @@ public class MicrophoneStand : MonoBehaviour
if (controller != null || other.gameObject.tag == "Player Head")
{
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;
microphoneOffStatus.SetActive(false);
microphoneOnStatus.SetActive(true);
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.RadioButton, gameObject);
if (npcController != null)
{
npcController.SpeakVoiceLine(1);
}
fmodWhisperBridge.ActivateRecording();
}
}
@@ -50,31 +48,29 @@ public class MicrophoneStand : MonoBehaviour
KbmController controller = other.GetComponent<KbmController>();
if (controller != null | other.gameObject.tag == "Player Head")
{
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeechFinished;
microphoneOffStatus.SetActive(true);
microphoneOnStatus.SetActive(false);
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.RadioButton, gameObject);
if (npcController != null)
{
npcController.SpeakVoiceLine(2);
}
fmodWhisperBridge.DeactivateRecording();
}
}
private void OnPlayerSpeech(string text)
private void OnPlayerSpeechUpdated(string text)
{
if (string.IsNullOrEmpty(text) || text.Contains("BLANK_AUDIO"))
{
return;
}
outputText.text = text;
}
private void OnPlayerSpeechFinished(string playerText)
{
OnPlayerFinishedSpeaking?.Invoke();
}
public string GetTextOutput()
{
return outputText.text;
}
}

45
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ModelDisplay.cs Normal file
View File

@@ -0,0 +1,45 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class ModelDisplay : MonoBehaviour
{
public Material modelDisplayActiveMaterial;
public Transform wire;
public Material wireActiveMaterial;
public GameObject Model { get; private set; }
public float generatedObjectRotationSpeed = 10f;
// Start is called before the first frame update
void Start()
{
}
// Update is called once per frame
void Update()
{
if (Model != null)
{
Model.transform.Rotate(Vector3.up, generatedObjectRotationSpeed * Time.deltaTime);
}
}
public void DisplayModel(GameObject modelObject)
{
foreach (MeshRenderer meshRenderer in wire.GetComponentsInChildren<MeshRenderer>())
{
meshRenderer.material = wireActiveMaterial;
}
gameObject.GetComponent<MeshRenderer>().material = modelDisplayActiveMaterial;
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.ShapeScannerSuccess, gameObject);
// Destroy previous generated object
Destroy(Model);
modelObject.transform.parent = transform;
modelObject.transform.position = transform.position;
Model = modelObject;
}
}

2
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ArcheryRangeModelGenerationController.cs.meta → Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ModelDisplay.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 99e1fdda206b5d04082b6a45593a1e84
guid: 4034b87c80896fe4d9e18ef54faa1ef6
MonoImporter:
externalObjects: {}
serializedVersion: 2

15
Assets/_PROJECT/Scripts/ModeGeneration/FMODWhisperBridge.cs
View File

@@ -115,11 +115,17 @@ public class FMODWhisperBridge : MonoBehaviour
};
_stream.OnSegmentUpdated += (seg) =>
{
OnWhisperSegmentUpdated?.Invoke(seg.Result);
if (IsSpeechMeaningful(seg.Result))
{
OnWhisperSegmentUpdated?.Invoke(seg.Result);
}
};
_stream.OnSegmentFinished += (seg) =>
{
OnWhisperSegmentFinished?.Invoke(seg.Result);
if (IsSpeechMeaningful(seg.Result))
{
OnWhisperSegmentFinished?.Invoke(seg.Result);
}
};
// If you want Whisper to respect VAD, enable in manager or set useVad (manager controls stream params).
@@ -182,6 +188,11 @@ public class FMODWhisperBridge : MonoBehaviour
isRecordingActivated = false;
}
private bool IsSpeechMeaningful(string userText)
{
return !string.IsNullOrEmpty(userText) && !userText.Contains("BLANK_AUDIO") && !userText.Trim().Equals("[ Silence ]");
}
private void CopyPcm16ToFloatAndFeed(IntPtr src, uint byteLen)
{
int samples = (int)(byteLen / 2); // 2 bytes per sample

109
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/ArcheryRangeNPC.cs
View File

@@ -1,15 +1,118 @@
using System;
using UnityEngine;
using UnityEngine.UI;
using FMOD.Studio;
using System.Threading.Tasks;
public class ArcheryRangeNPC : NPCController
{
protected override void OnPlayerApproach()
[Header("Archery Range Config")]
public MicrophoneStand microphoneStand;
public PushableButton imageGenerationButton;
public PushableButton modelGenerationButton;
public Image imageDisplay;
public ModelDisplay modelDisplay;
private Texture2D GeneratedTexture;
private EventInstance printingSound;
// states:
// 0 - idle
// 1 - player approached
// 2 - player used microphone
// 3 - player pressed the image generation button
// 4 - player pressed the model generation button
// 5 - model spawned into the game
private int state;
private void Awake()
{
Debug.Log("Alien NPC: player approached");
SpeakVoiceLine(0);
state = 0;
}
private void Start()
{
microphoneStand.OnPlayerFinishedSpeaking += OnPlayerUsedMicrophone;
imageGenerationButton.OnButtonPressed += OnImageGenerationButtonPressed;
modelGenerationButton.OnButtonPressed += OnModelGenerationButtonPressed;
printingSound = AudioManager.Instance.CreateInstance(FMODEvents.Instance.Printing);
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
}
protected async override void OnPlayerApproach()
{
if (state == 0)
{
SpeakVoiceLine(0);
await Task.Delay(2000);
SpeakVoiceLine(1);
await Task.Delay(6500);
SpeakVoiceLine(2);
await Task.Delay(3000);
state = 1;
}
}
protected override void OnPlayerLeave()
{
Debug.Log("Alien NPC: player left");
}
private void OnPlayerUsedMicrophone()
{
if (state == 1)
{
SpeakVoiceLine(3);
state = 2;
}
}
private async void OnImageGenerationButtonPressed()
{
if (state == 2)
{
state = 3;
}
string inputPrompt = microphoneStand.GetTextOutput();
byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(inputPrompt);
GeneratedTexture = ModelGenerationUtils.CreateTexture(imageBytes);
Sprite sprite = ModelGenerationUtils.CreateSprite(GeneratedTexture);
imageDisplay.sprite = sprite;
imageGenerationButton.Deactivate();
if (state == 3)
{
modelGenerationButton.Deactivate();
SpeakVoiceLine(4);
}
}
private async void OnModelGenerationButtonPressed()
{
if (state == 3)
{
state = 4;
}
printingSound.start();
printingSound.setParameterByName("3DPrinterPrintingJob", 0);
string encodedTexture = Convert.ToBase64String(GeneratedTexture.EncodeToJPG());
byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
GameObject spawnedObject = await ModelGenerationUtils.Instance.SpawnModel(encodedModel);
modelDisplay.DisplayModel(spawnedObject);
if (state == 4)
{
SpeakVoiceLine(5);
state = 5;
}
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
modelGenerationButton.Deactivate();
}
}

19
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/CafeWaiterNPC.cs
View File

@@ -50,7 +50,7 @@ public class CafeWaiterNPC : NPCController
{
SpeakVoiceLine(0);
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdate;
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;
fmodWhisperBridge.ActivateRecording();
@@ -61,7 +61,7 @@ public class CafeWaiterNPC : NPCController
protected override void OnPlayerLeave()
{
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdate;
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeechFinished;
fmodWhisperBridge.DeactivateRecording();
@@ -80,10 +80,6 @@ public class CafeWaiterNPC : NPCController
{
return;
}
if (IsPlayerSpeechMeaningless(playerText))
{
return;
}
lastPlayerVoiceUpdateTime = Time.time;
if (state == 1)
@@ -111,12 +107,8 @@ public class CafeWaiterNPC : NPCController
}
}
private void OnPlayerSpeechUpdate(string playerText)
private void OnPlayerSpeechUpdated(string playerText)
{
if (IsPlayerSpeechMeaningless(playerText))
{
return;
}
// Only update notepad text when currently listening to player order
if (state == 1)
{
@@ -124,10 +116,7 @@ public class CafeWaiterNPC : NPCController
}
}
private bool IsPlayerSpeechMeaningless(string playerText)
{
return string.IsNullOrEmpty(playerText) || playerText.Contains("BLANK_AUDIO") || playerText.Trim().Equals("[ Silence ]");
}
private async void BringFood()
{