diff --git a/Assets/StreamingAssets/Whisper.meta b/Assets/StreamingAssets/Whisper.meta new file mode 100644 index 00000000..50ef6d9f --- /dev/null +++ b/Assets/StreamingAssets/Whisper.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 23fe3883e9cc804429bc54fb860d18f1 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/StreamingAssets/Whisper/ggml-base.bin b/Assets/StreamingAssets/Whisper/ggml-base.bin new file mode 100644 index 00000000..d55d1230 Binary files /dev/null and b/Assets/StreamingAssets/Whisper/ggml-base.bin differ diff --git a/Assets/StreamingAssets/Whisper/ggml-base.bin.meta b/Assets/StreamingAssets/Whisper/ggml-base.bin.meta new file mode 100644 index 00000000..d5994168 --- /dev/null +++ b/Assets/StreamingAssets/Whisper/ggml-base.bin.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f6c028f06eda5904eae3f7a7418b8416 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity b/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity index b172972f..f5d39fa4 100644 --- a/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity +++ b/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:caba633011a1d039f4cdb8d1868cff29011f44e34ea6d28bf0775ffbc395d26e -size 63221788 +oid sha256:e891e80f732ea71162c2ab835931d9c9bb63bf1b69c181356739421445a60690 +size 63234941 diff --git a/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs b/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs new file mode 100644 index 00000000..9eaf1545 --- /dev/null +++ b/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using TMPro; +using UnityEngine; +using Whisper; +using Whisper.Utils; + +public class VoiceTranscriptionTestBox : MonoBehaviour +{ + public Material activeMaterial; + public Material inactiveMaterial; + + private MeshRenderer meshRenderer; + + + public WhisperManager whisper; + public MicrophoneRecord microphoneRecord; + public TextMeshProUGUI outputText; + + private string _buffer; + + private void Awake() + { + whisper.OnNewSegment += OnNewSegment; + whisper.OnProgress += OnProgressHandler; + + microphoneRecord.OnRecordStop += OnRecordStop; + } + + // Start is called before the first frame update + void Start() + { + meshRenderer = GetComponent(); + } + + // Update is called once per frame + void Update() + { + + } + + void OnTriggerEnter(Collider other) + { + KbmController controller = other.GetComponent(); + if (controller != null) + { + meshRenderer.material = activeMaterial; + microphoneRecord.StartRecord(); + } + } + + private void OnTriggerExit(Collider other) + { + KbmController controller = other.GetComponent(); + if (controller != null) + { + meshRenderer.material = inactiveMaterial; + microphoneRecord.StopRecord(); + } + } + + + private async void OnRecordStop(AudioChunk recordedAudio) + { + _buffer = ""; + + var sw = new Stopwatch(); + sw.Start(); + + var res = await whisper.GetTextAsync(recordedAudio.Data, recordedAudio.Frequency, recordedAudio.Channels); + if (res == null) + return; + + var time = sw.ElapsedMilliseconds; + var rate = recordedAudio.Length / (time * 0.001f); + UnityEngine.Debug.Log($"Time: {time} ms\nRate: {rate:F1}x"); + + var text = res.Result; + + outputText.text = text; + } + + private void OnProgressHandler(int progress) + { + UnityEngine.Debug.Log($"Progress: {progress}%"); + } + + private void OnNewSegment(WhisperSegment segment) + { + _buffer += segment.Text; + UnityEngine.Debug.Log(_buffer + "..."); + } +} diff --git a/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs.meta b/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs.meta new file mode 100644 index 00000000..809ce588 --- /dev/null +++ b/Assets/_PROJECT/Scripts/3DModeGeneration/VoiceTranscriptionTestBox.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d28857190597d9a46a8ddf3cf902cc81 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs b/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs new file mode 100644 index 00000000..155008df --- /dev/null +++ b/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs @@ -0,0 +1,57 @@ +using System.Diagnostics; +using UnityEngine; +using Whisper; +using Whisper.Utils; + +public class VoiceTranscription : MonoBehaviour +{ + public WhisperManager whisper; + public MicrophoneRecord microphoneRecord; + + private string _buffer; + + private void Awake() + { + whisper.OnNewSegment += OnNewSegment; + + microphoneRecord.OnRecordStop += OnRecordStop; + } + + // Start is called before the first frame update + void Start() + { + + } + + // Update is called once per frame + void Update() + { + + } + + private async void OnRecordStop(AudioChunk recordedAudio) + { + _buffer = ""; + + var sw = new Stopwatch(); + sw.Start(); + + var res = await whisper.GetTextAsync(recordedAudio.Data, recordedAudio.Frequency, recordedAudio.Channels); + if (res == null) + return; + + var time = sw.ElapsedMilliseconds; + var rate = recordedAudio.Length / (time * 0.001f); + UnityEngine.Debug.Log($"Time: {time} ms\nRate: {rate:F1}x"); + + var text = res.Result; + + UnityEngine.Debug.Log(text); + } + + private void OnNewSegment(WhisperSegment segment) + { + _buffer += segment.Text; + UnityEngine.Debug.Log(_buffer + "..."); + } +} diff --git a/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs.meta b/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs.meta new file mode 100644 index 00000000..1b99e32b --- /dev/null +++ b/Assets/_PROJECT/Scripts/Audio/VoiceTranscription.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: af83274dbfe8bab4599dda694e2545c2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: