use voice transcription box and model gen box in archery range

Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs

@@ -0,0 +1,59 @@
+using Unity.XR.CoreUtils;
+using UnityEngine;
+
+public class ModelGenerationBox : MonoBehaviour
+{
+    public Material inactiveMaterial;
+    public Material loadingMaterial;
+
+    public Transform modelSpawnPoint;
+    public VoiceTranscriptionBox voiceTranscriptionTestBox;
+
+    private MeshRenderer meshRenderer;
+    private bool isLoading;
+
+    private string lastModelPath;
+    public string LastModelPath
+    {
+        get {
+            return lastModelPath;
+        }
+    }
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        meshRenderer = GetComponent<MeshRenderer>();
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+    async void OnTriggerEnter(Collider other)
+    {
+        if (isLoading) return;
+
+        KbmController controller = other.GetComponent<KbmController>();
+        XROrigin playerOrigin = other.GetComponent<XROrigin>();
+        if (controller != null || playerOrigin != null)
+        {
+            string inputPrompt = voiceTranscriptionTestBox.LastTextOutput;
+
+            isLoading = true;
+            meshRenderer.material = loadingMaterial;
+
+            string modelPath = await PipelineManager.Instance.GenerateModelAsync(inputPrompt);
+            lastModelPath = modelPath;
+
+            GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(modelPath);
+            spawnedObject.transform.parent = modelSpawnPoint;
+            spawnedObject.transform.position = modelSpawnPoint.position;
+
+            isLoading = false;
+            meshRenderer.material = inactiveMaterial;
+        }
+    }
+}

Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 46e67223dce9b7a4783ed36b8ed65f19
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs

@@ -0,0 +1,104 @@
+using GLTFast;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using UnityEngine;
+
+public class PipelineManager : MonoBehaviour
+{
+    public static PipelineManager Instance { get; private set; }
+
+    private void Awake()
+    {
+        Instance = this;
+    }
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+    public async Task<string> GenerateModelAsync(string inputPrompt)
+    {
+        return await Task.Run(() =>
+        {
+            // Path to your virtual environment's python.exe
+            string pythonExe = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\.venv\Scripts\python.exe";
+
+            // Path to your Python script
+            string scriptPath = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\start_pipeline.py";
+
+            // Arguments to pass to the script
+            string arguments = $"{scriptPath} --prompt \"{inputPrompt}\"";
+
+            ProcessStartInfo psi = new ProcessStartInfo
+            {
+                FileName = pythonExe,
+                Arguments = arguments,
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true
+            };
+
+            using (Process process = new Process())
+            {
+                process.StartInfo = psi;
+                process.OutputDataReceived += (sender, e) => UnityEngine.Debug.Log(e.Data);
+                process.ErrorDataReceived += (sender, e) => UnityEngine.Debug.LogError(e.Data);
+
+                process.Start();
+
+                string output = process.StandardOutput.ReadToEnd();
+                string error = process.StandardError.ReadToEnd();
+
+                process.WaitForExit();
+
+
+                // Extract model path from output
+                foreach (string line in output.Split('\n'))
+                {
+                    if (line.StartsWith("Generated 3D model file: "))
+                    {
+                        return line.Replace("Generated 3D model file: ", "").Trim();
+                    }
+                }
+
+                throw new System.Exception("Failed to generate 3D model!");
+            }
+        });
+    }
+
+    public async Task<GameObject> SpawnModel(string modelPath)
+    {
+        var gltf = new GltfImport();
+        bool loadSuccess = await gltf.Load(modelPath);
+        if (loadSuccess)
+        {
+            GameObject spawnedObject = new GameObject("spawned model");
+
+            bool spawnSuccess = await gltf.InstantiateMainSceneAsync(spawnedObject.transform);
+            if (spawnSuccess)
+            {
+                Transform spawnedObjectMainTransform = spawnedObject.transform.GetChild(0).transform;
+                GameObject spawnedObjectBody = spawnedObjectMainTransform.GetChild(0).transform.gameObject;
+                MeshCollider collider = spawnedObjectBody.AddComponent<MeshCollider>();
+                collider.convex = true;
+                MeshRenderer renderer = spawnedObjectBody.GetComponent<MeshRenderer>();
+                renderer.material.SetFloat("metallicFactor", 0);
+
+                spawnedObjectMainTransform.gameObject.AddComponent<Rigidbody>();
+
+                return spawnedObject;
+            }
+        }
+
+        throw new System.Exception("Failed to spawn GameObject from model" + modelPath);
+    }
+}

Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 19e82e42c38cf2d4b912baa8d60c5407
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/_PROJECT/Scripts/ModeGeneration/VoiceTranscriptionBox.cs

@@ -0,0 +1,118 @@
+using System.Diagnostics;
+using TMPro;
+using Unity.XR.CoreUtils;
+using UnityEngine;
+using Whisper;
+using Whisper.Utils;
+
+public class VoiceTranscriptionBox : MonoBehaviour
+{
+    public Material activeMaterial;
+    public Material inactiveMaterial;
+    public Material loadingMaterial;
+
+    private MeshRenderer meshRenderer;
+    private bool isLoading;
+
+
+    public WhisperManager whisper;
+    public MicrophoneRecord microphoneRecord;
+    public TextMeshProUGUI outputText;
+
+    private string _buffer;
+
+    private string lastTextOutput;
+    public string LastTextOutput
+    {
+        get
+        {
+            return lastTextOutput;
+        }
+    }
+
+    private void Awake()
+    {
+        isLoading = false;
+
+        whisper.OnNewSegment += OnNewSegment;
+        whisper.OnProgress += OnProgressHandler;
+
+        microphoneRecord.OnRecordStop += OnRecordStop;
+    }
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        meshRenderer = GetComponent<MeshRenderer>();
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+    void OnTriggerEnter(Collider other)
+    {
+        if (isLoading)
+        {
+            return;
+        }
+
+        KbmController controller = other.GetComponent<KbmController>();
+        XROrigin playerOrigin = other.GetComponent<XROrigin>();
+        if (controller != null || playerOrigin != null)
+        {
+            meshRenderer.material = activeMaterial;
+            microphoneRecord.StartRecord();
+        }
+    }
+
+    private void OnTriggerExit(Collider other)
+    {
+        KbmController controller = other.GetComponent<KbmController>();
+        XROrigin playerOrigin = other.GetComponent<XROrigin>();
+        if (controller != null | playerOrigin != null)
+        {
+            microphoneRecord.StopRecord();
+            meshRenderer.material = loadingMaterial;
+            isLoading = true;
+        }
+    }
+
+
+    private async void OnRecordStop(AudioChunk recordedAudio)
+    {
+        _buffer = "";
+
+        var sw = new Stopwatch();
+        sw.Start();
+
+        var res = await whisper.GetTextAsync(recordedAudio.Data, recordedAudio.Frequency, recordedAudio.Channels);
+        if (res == null)
+            return;
+
+        var time = sw.ElapsedMilliseconds;
+        var rate = recordedAudio.Length / (time * 0.001f);
+        UnityEngine.Debug.Log($"Time: {time} ms\nRate: {rate:F1}x");
+
+        var text = res.Result;
+
+        lastTextOutput = text;
+        outputText.text = text;
+
+        meshRenderer.material = inactiveMaterial;
+        isLoading = false;
+    }
+
+    private void OnProgressHandler(int progress)
+    {
+        UnityEngine.Debug.Log($"Progress: {progress}%");
+    }
+
+    private void OnNewSegment(WhisperSegment segment)
+    {
+        _buffer += segment.Text;
+        UnityEngine.Debug.Log(_buffer + "...");
+    }
+}

Assets/_PROJECT/Scripts/ModeGeneration/VoiceTranscriptionBox.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d28857190597d9a46a8ddf3cf902cc81
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: