henrisel/DeltaVR3DModelGeneration
1
0
Fork 0
forked from cgvr/DeltaVR
Code Pull Requests Activity

moved archery range model generation logic to npc script, added corresponding voicelines

Browse Source
This commit is contained in:
henrisel
2026-02-08 12:28:59 +02:00
parent 53295a60ed
commit dc55e8d884
12 changed files with 323 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs
View File

@@ -28,7 +28,7 @@ public class ArcheryRange : NetworkBehaviour
public float roundLength = 60f;
public float targetSpawnTime = 3f;
public ArcheryRangeModelGenerationController modelGenerationController;
public ModelDisplay modelDisplay;
public KeyboardManager keyboardManager;
private List<ArcheryTarget> _targets;
@@ -133,14 +133,14 @@ public class ArcheryRange : NetworkBehaviour
private ArcheryTarget SpawnTarget(Vector3 randomPos)
{
GameObject targetObject;
if (modelGenerationController.GeneratedModel == null)
if (modelDisplay.Model == null)
{
// spawn default UFO
targetObject = Instantiate(targetPrefab, randomPos, Quaternion.identity, null);
} else
{
// spawn generated model
targetObject = Instantiate(modelGenerationController.GeneratedModel, randomPos, Quaternion.identity, null);
targetObject = Instantiate(modelDisplay.Model, randomPos, Quaternion.identity, null);
InitializeArcheryTargetObject(targetObject);
}

127
Assets/_PROJECT/Prefabs/ModelGeneration/ModelDisplay.prefab Normal file
View File

@@ -0,0 +1,127 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1 &4841664669366217751
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 8059049557656133817}
- component: {fileID: 4918617990459080428}
- component: {fileID: 163498188678022614}
- component: {fileID: 8145483597018112470}
- component: {fileID: 125370255468251911}
m_Layer: 0
m_Name: ModelDisplay
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &8059049557656133817
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4841664669366217751}
m_LocalRotation: {x: -0, y: -0.57596153, z: -0, w: 0.8174768}
m_LocalPosition: {x: -3.6680002, y: -1.2389998, z: 0.41600037}
m_LocalScale: {x: 1.2, y: 1, z: 1.2}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: -1
m_LocalEulerAnglesHint: {x: 0, y: -70.334, z: 0}
--- !u!33 &4918617990459080428
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4841664669366217751}
m_Mesh: {fileID: 10206, guid: 0000000000000000e000000000000000, type: 0}
--- !u!23 &163498188678022614
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4841664669366217751}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1
m_DynamicOccludee: 1
m_StaticShadowCaster: 0
m_MotionVectors: 1
m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RayTracingMode: 2
m_RayTraceProcedural: 0
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 1892de3c798cec947bcabf6cfc693b8c, type: 2}
m_StaticBatchInfo:
firstSubMesh: 0
subMeshCount: 0
m_StaticBatchRoot: {fileID: 0}
m_ProbeAnchor: {fileID: 0}
m_LightProbeVolumeOverride: {fileID: 0}
m_ScaleInLightmap: 1
m_ReceiveGI: 1
m_PreserveUVs: 0
m_IgnoreNormalsForChartDetection: 0
m_ImportantGI: 0
m_StitchLightmapSeams: 1
m_SelectedEditorRenderState: 3
m_MinimumChartSize: 4
m_AutoUVMaxDistance: 0.5
m_AutoUVMaxAngle: 89
m_LightmapParameters: {fileID: 0}
m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
m_AdditionalVertexStreams: {fileID: 0}
--- !u!136 &8145483597018112470
CapsuleCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4841664669366217751}
m_Material: {fileID: 0}
m_IncludeLayers:
serializedVersion: 2
m_Bits: 0
m_ExcludeLayers:
serializedVersion: 2
m_Bits: 0
m_LayerOverridePriority: 0
m_IsTrigger: 0
m_ProvidesContacts: 0
m_Enabled: 1
serializedVersion: 2
m_Radius: 0.5000001
m_Height: 2
m_Direction: 1
m_Center: {x: 0.000000059604645, y: 0, z: -0.00000008940697}
--- !u!114 &125370255468251911
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4841664669366217751}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 4034b87c80896fe4d9e18ef54faa1ef6, type: 3}
m_Name:
m_EditorClassIdentifier:
modelDisplayActiveMaterial: {fileID: 2100000, guid: e2ac8c5522bae5a4794c3cbdb7bee948,
type: 2}
wire: {fileID: 0}
wireActiveMaterial: {fileID: 2100000, guid: 41f12eac51f3a45459e319b9f41ecb1a, type: 2}
generatedObjectRotationSpeed: 10

7
Assets/_PROJECT/Prefabs/ModelGeneration/ModelDisplay.prefab.meta Normal file
View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: 3b4af50e5f21164468207fe4c7d85d0b
PrefabImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

BIN
Assets/_PROJECT/Scenes/DeltaBuilding_base.unity LFS
View File

Binary file not shown.

99
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ArcheryRangeModelGenerationController.cs
View File

@@ -1,99 +0,0 @@
using System;
using UnityEngine;
using UnityEngine.UI;
using FMOD.Studio;
public class ArcheryRangeModelGenerationController : MonoBehaviour
{
public MicrophoneStand microphoneStand;
public PushableButton imageGenerationButton;
public PushableButton modelGenerationButton;
public Texture2D GeneratedTexture { get; private set; }
public Image imageDisplay;
public GameObject GeneratedModel { get; private set; }
public float generatedObjectRotationSpeed = 10f;
public Transform modelDisplay;
public Material modelDisplayActiveMaterial;
public Transform wire;
public Material wireActiveMaterial;
public NPCController npcController;
private bool modelGenerationInProgress;
private EventInstance printingSound;
// Start is called before the first frame update
void Start()
{
imageGenerationButton.OnButtonPressed += InvokeImageGeneration;
modelGenerationButton.OnButtonPressed += InvokeModelGeneration;
modelGenerationInProgress = false;
printingSound = AudioManager.Instance.CreateInstance(FMODEvents.Instance.Printing);
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
}
// Update is called once per frame
void Update()
{
if (GeneratedModel != null)
{
GeneratedModel.transform.Rotate(Vector3.up, generatedObjectRotationSpeed * Time.deltaTime);
}
}
private async void InvokeImageGeneration()
{
string inputPrompt = microphoneStand.GetTextOutput();
byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(inputPrompt);
GeneratedTexture = ModelGenerationUtils.CreateTexture(imageBytes);
Sprite sprite = ModelGenerationUtils.CreateSprite(GeneratedTexture);
imageDisplay.sprite = sprite;
imageGenerationButton.Deactivate();
if (!modelGenerationInProgress)
{
modelGenerationButton.Deactivate();
}
npcController.SpeakVoiceLine(3);
}
private async void InvokeModelGeneration()
{
modelGenerationInProgress = true;
printingSound.start();
printingSound.setParameterByName("3DPrinterPrintingJob", 0);
string encodedTexture = Convert.ToBase64String(GeneratedTexture.EncodeToJPG());
byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
GameObject spawnedObject = await ModelGenerationUtils.Instance.SpawnModel(encodedModel);
// Destroy previous generated object
Destroy(GeneratedModel);
spawnedObject.transform.parent = modelDisplay;
spawnedObject.transform.position = modelDisplay.position;
GeneratedModel = spawnedObject;
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
npcController.SpeakVoiceLine(4);
OnModelReady();
modelGenerationButton.Deactivate();
modelGenerationInProgress = false;
}
private void OnModelReady()
{
foreach (MeshRenderer meshRenderer in wire.GetComponentsInChildren<MeshRenderer>())
{
meshRenderer.material = wireActiveMaterial;
}
modelDisplay.GetComponent<MeshRenderer>().material = modelDisplayActiveMaterial;
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.ShapeScannerSuccess, gameObject);
}
}

32
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/MicrophoneStand.cs
View File

@@ -8,9 +8,11 @@ public class MicrophoneStand : MonoBehaviour
public GameObject microphoneOffStatus;
public GameObject microphoneOnStatus;
public NPCController npcController;
public FMODWhisperBridge fmodWhisperBridge;
public delegate void OnPlayerFinishedSpeakingDelegate();
public event OnPlayerFinishedSpeakingDelegate OnPlayerFinishedSpeaking;
// Start is called before the first frame update
void Start()
{
@@ -31,16 +33,12 @@ public class MicrophoneStand : MonoBehaviour
if (controller != null || other.gameObject.tag == "Player Head")
{
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;
microphoneOffStatus.SetActive(false);
microphoneOnStatus.SetActive(true);
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.RadioButton, gameObject);
if (npcController != null)
{
npcController.SpeakVoiceLine(1);
}
fmodWhisperBridge.ActivateRecording();
}
}
@@ -50,31 +48,29 @@ public class MicrophoneStand : MonoBehaviour
KbmController controller = other.GetComponent<KbmController>();
if (controller != null | other.gameObject.tag == "Player Head")
{
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeech;
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeechFinished;
microphoneOffStatus.SetActive(true);
microphoneOnStatus.SetActive(false);
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.RadioButton, gameObject);
if (npcController != null)
{
npcController.SpeakVoiceLine(2);
}
fmodWhisperBridge.DeactivateRecording();
}
}
private void OnPlayerSpeech(string text)
private void OnPlayerSpeechUpdated(string text)
{
if (string.IsNullOrEmpty(text) || text.Contains("BLANK_AUDIO"))
{
return;
}
outputText.text = text;
}
private void OnPlayerSpeechFinished(string playerText)
{
OnPlayerFinishedSpeaking?.Invoke();
}
public string GetTextOutput()
{
return outputText.text;
}
}

45
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ModelDisplay.cs Normal file
View File

@@ -0,0 +1,45 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class ModelDisplay : MonoBehaviour
{
public Material modelDisplayActiveMaterial;
public Transform wire;
public Material wireActiveMaterial;
public GameObject Model { get; private set; }
public float generatedObjectRotationSpeed = 10f;
// Start is called before the first frame update
void Start()
{
}
// Update is called once per frame
void Update()
{
if (Model != null)
{
Model.transform.Rotate(Vector3.up, generatedObjectRotationSpeed * Time.deltaTime);
}
}
public void DisplayModel(GameObject modelObject)
{
foreach (MeshRenderer meshRenderer in wire.GetComponentsInChildren<MeshRenderer>())
{
meshRenderer.material = wireActiveMaterial;
}
gameObject.GetComponent<MeshRenderer>().material = modelDisplayActiveMaterial;
AudioManager.Instance.PlayAttachedInstance(FMODEvents.Instance.ShapeScannerSuccess, gameObject);
// Destroy previous generated object
Destroy(Model);
modelObject.transform.parent = transform;
modelObject.transform.position = transform.position;
Model = modelObject;
}
}

2
Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ArcheryRangeModelGenerationController.cs.meta → Assets/_PROJECT/Scripts/ModeGeneration/ArcheryRange/ModelDisplay.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 99e1fdda206b5d04082b6a45593a1e84
guid: 4034b87c80896fe4d9e18ef54faa1ef6
MonoImporter:
externalObjects: {}
serializedVersion: 2

15
Assets/_PROJECT/Scripts/ModeGeneration/FMODWhisperBridge.cs
View File

@@ -115,11 +115,17 @@ public class FMODWhisperBridge : MonoBehaviour
};
_stream.OnSegmentUpdated += (seg) =>
{
OnWhisperSegmentUpdated?.Invoke(seg.Result);
if (IsSpeechMeaningful(seg.Result))
{
OnWhisperSegmentUpdated?.Invoke(seg.Result);
}
};
_stream.OnSegmentFinished += (seg) =>
{
OnWhisperSegmentFinished?.Invoke(seg.Result);
if (IsSpeechMeaningful(seg.Result))
{
OnWhisperSegmentFinished?.Invoke(seg.Result);
}
};
// If you want Whisper to respect VAD, enable in manager or set useVad (manager controls stream params).
@@ -182,6 +188,11 @@ public class FMODWhisperBridge : MonoBehaviour
isRecordingActivated = false;
}
private bool IsSpeechMeaningful(string userText)
{
return !string.IsNullOrEmpty(userText) && !userText.Contains("BLANK_AUDIO") && !userText.Trim().Equals("[ Silence ]");
}
private void CopyPcm16ToFloatAndFeed(IntPtr src, uint byteLen)
{
int samples = (int)(byteLen / 2); // 2 bytes per sample

109
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/ArcheryRangeNPC.cs
View File

@@ -1,15 +1,118 @@
using System;
using UnityEngine;
using UnityEngine.UI;
using FMOD.Studio;
using System.Threading.Tasks;
public class ArcheryRangeNPC : NPCController
{
protected override void OnPlayerApproach()
[Header("Archery Range Config")]
public MicrophoneStand microphoneStand;
public PushableButton imageGenerationButton;
public PushableButton modelGenerationButton;
public Image imageDisplay;
public ModelDisplay modelDisplay;
private Texture2D GeneratedTexture;
private EventInstance printingSound;
// states:
// 0 - idle
// 1 - player approached
// 2 - player used microphone
// 3 - player pressed the image generation button
// 4 - player pressed the model generation button
// 5 - model spawned into the game
private int state;
private void Awake()
{
Debug.Log("Alien NPC: player approached");
SpeakVoiceLine(0);
state = 0;
}
private void Start()
{
microphoneStand.OnPlayerFinishedSpeaking += OnPlayerUsedMicrophone;
imageGenerationButton.OnButtonPressed += OnImageGenerationButtonPressed;
modelGenerationButton.OnButtonPressed += OnModelGenerationButtonPressed;
printingSound = AudioManager.Instance.CreateInstance(FMODEvents.Instance.Printing);
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
}
protected async override void OnPlayerApproach()
{
if (state == 0)
{
SpeakVoiceLine(0);
await Task.Delay(2000);
SpeakVoiceLine(1);
await Task.Delay(6500);
SpeakVoiceLine(2);
await Task.Delay(3000);
state = 1;
}
}
protected override void OnPlayerLeave()
{
Debug.Log("Alien NPC: player left");
}
private void OnPlayerUsedMicrophone()
{
if (state == 1)
{
SpeakVoiceLine(3);
state = 2;
}
}
private async void OnImageGenerationButtonPressed()
{
if (state == 2)
{
state = 3;
}
string inputPrompt = microphoneStand.GetTextOutput();
byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(inputPrompt);
GeneratedTexture = ModelGenerationUtils.CreateTexture(imageBytes);
Sprite sprite = ModelGenerationUtils.CreateSprite(GeneratedTexture);
imageDisplay.sprite = sprite;
imageGenerationButton.Deactivate();
if (state == 3)
{
modelGenerationButton.Deactivate();
SpeakVoiceLine(4);
}
}
private async void OnModelGenerationButtonPressed()
{
if (state == 3)
{
state = 4;
}
printingSound.start();
printingSound.setParameterByName("3DPrinterPrintingJob", 0);
string encodedTexture = Convert.ToBase64String(GeneratedTexture.EncodeToJPG());
byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
GameObject spawnedObject = await ModelGenerationUtils.Instance.SpawnModel(encodedModel);
modelDisplay.DisplayModel(spawnedObject);
if (state == 4)
{
SpeakVoiceLine(5);
state = 5;
}
printingSound.setParameterByName("3DPrinterPrintingJob", 1);
modelGenerationButton.Deactivate();
}
}

19
Assets/_PROJECT/Scripts/ModeGeneration/NPCs/CafeWaiterNPC.cs
View File

@@ -50,7 +50,7 @@ public class CafeWaiterNPC : NPCController
{
SpeakVoiceLine(0);
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdate;
fmodWhisperBridge.OnWhisperSegmentUpdated += OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished += OnPlayerSpeechFinished;
fmodWhisperBridge.ActivateRecording();
@@ -61,7 +61,7 @@ public class CafeWaiterNPC : NPCController
protected override void OnPlayerLeave()
{
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdate;
fmodWhisperBridge.OnWhisperSegmentUpdated -= OnPlayerSpeechUpdated;
fmodWhisperBridge.OnWhisperSegmentFinished -= OnPlayerSpeechFinished;
fmodWhisperBridge.DeactivateRecording();
@@ -80,10 +80,6 @@ public class CafeWaiterNPC : NPCController
{
return;
}
if (IsPlayerSpeechMeaningless(playerText))
{
return;
}
lastPlayerVoiceUpdateTime = Time.time;
if (state == 1)
@@ -111,12 +107,8 @@ public class CafeWaiterNPC : NPCController
}
}
private void OnPlayerSpeechUpdate(string playerText)
private void OnPlayerSpeechUpdated(string playerText)
{
if (IsPlayerSpeechMeaningless(playerText))
{
return;
}
// Only update notepad text when currently listening to player order
if (state == 1)
{
@@ -124,10 +116,7 @@ public class CafeWaiterNPC : NPCController
}
}
private bool IsPlayerSpeechMeaningless(string playerText)
{
return string.IsNullOrEmpty(playerText) || playerText.Contains("BLANK_AUDIO") || playerText.Trim().Equals("[ Silence ]");
}
private async void BringFood()
{