forked from cgvr/DeltaVR
Compare commits
6 Commits
ee629e2593
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e55b7e9ab | |||
| dea42068f7 | |||
| b021fd0110 | |||
| 7b45d623d1 | |||
| f42356dd7a | |||
| 50002adb61 |
@@ -3,10 +3,9 @@
|
|||||||
* FMOD ChannelControl errorid
|
* FMOD ChannelControl errorid
|
||||||
* keyboard numbrid ei tööta pärast minigame'i completemist?
|
* keyboard numbrid ei tööta pärast minigame'i completemist?
|
||||||
* speech-to-text:
|
* speech-to-text:
|
||||||
* vahepeal lakkab töötamast lih??? tundub et siis kui pikalt tühjust salvestab
|
* vahepeal lakkab töötamast lih??? tundub et siis kui pikalt tühjust salvestab? ei, vahepeal läheb kohe alguses ka
|
||||||
* sööta talle nulle kui on disabled, sest ikka kuidagi tuvastab eelnevat rääkimist
|
* sööta talle nulle kui on disabled, sest ikka kuidagi tuvastab eelnevat rääkimist
|
||||||
* uurida miks buildis Whisper halvemini töötab
|
* uurida miks buildis Whisper halvemini töötab
|
||||||
* proovida suuremat Whisperi mudelit, äkki töötab mürases keskkonnas paremini
|
|
||||||
* npc character:
|
* npc character:
|
||||||
* klaas on näha temast eespool
|
* klaas on näha temast eespool
|
||||||
* shape detection:
|
* shape detection:
|
||||||
|
|||||||
BIN
Assets/StreamingAssets/Whisper/ggml-base.en.bin
Normal file
BIN
Assets/StreamingAssets/Whisper/ggml-base.en.bin
Normal file
Binary file not shown.
7
Assets/StreamingAssets/Whisper/ggml-base.en.bin.meta
Normal file
7
Assets/StreamingAssets/Whisper/ggml-base.en.bin.meta
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
fileFormatVersion: 2
|
||||||
|
guid: 4fef61b9ad4f7eb488ea2657f1cc700e
|
||||||
|
DefaultImporter:
|
||||||
|
externalObjects: {}
|
||||||
|
userData:
|
||||||
|
assetBundleName:
|
||||||
|
assetBundleVariant:
|
||||||
BIN
Assets/StreamingAssets/Whisper/ggml-tiny.bin
Normal file
BIN
Assets/StreamingAssets/Whisper/ggml-tiny.bin
Normal file
Binary file not shown.
7
Assets/StreamingAssets/Whisper/ggml-tiny.bin.meta
Normal file
7
Assets/StreamingAssets/Whisper/ggml-tiny.bin.meta
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
fileFormatVersion: 2
|
||||||
|
guid: 6f5a0c98d09a6dc4d9d570e23dc96ab9
|
||||||
|
DefaultImporter:
|
||||||
|
externalObjects: {}
|
||||||
|
userData:
|
||||||
|
assetBundleName:
|
||||||
|
assetBundleVariant:
|
||||||
Binary file not shown.
Binary file not shown.
@@ -29,7 +29,7 @@ Transform:
|
|||||||
m_GameObject: {fileID: 1659788510314838016}
|
m_GameObject: {fileID: 1659788510314838016}
|
||||||
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
|
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
|
||||||
m_LocalPosition: {x: 0, y: 0, z: 0}
|
m_LocalPosition: {x: 0, y: 0, z: 0}
|
||||||
m_LocalScale: {x: 0.4, y: 0.35, z: 0.4}
|
m_LocalScale: {x: 0.4, y: 0.38386, z: 0.4}
|
||||||
m_ConstrainProportionsScale: 0
|
m_ConstrainProportionsScale: 0
|
||||||
m_Children:
|
m_Children:
|
||||||
- {fileID: 4211436824399037528}
|
- {fileID: 4211436824399037528}
|
||||||
@@ -149,7 +149,7 @@ Transform:
|
|||||||
m_PrefabAsset: {fileID: 0}
|
m_PrefabAsset: {fileID: 0}
|
||||||
m_GameObject: {fileID: 4039766663692926199}
|
m_GameObject: {fileID: 4039766663692926199}
|
||||||
m_LocalRotation: {x: -0, y: 0.000000014901158, z: -0, w: 1}
|
m_LocalRotation: {x: -0, y: 0.000000014901158, z: -0, w: 1}
|
||||||
m_LocalPosition: {x: 0, y: 0, z: 0}
|
m_LocalPosition: {x: 0, y: -0.36, z: 0}
|
||||||
m_LocalScale: {x: 1, y: 1, z: 1}
|
m_LocalScale: {x: 1, y: 1, z: 1}
|
||||||
m_ConstrainProportionsScale: 0
|
m_ConstrainProportionsScale: 0
|
||||||
m_Children: []
|
m_Children: []
|
||||||
|
|||||||
Binary file not shown.
@@ -4,9 +4,9 @@ using System.Runtime.InteropServices;
|
|||||||
using UnityEngine;
|
using UnityEngine;
|
||||||
using FMOD;
|
using FMOD;
|
||||||
using FMODUnity;
|
using FMODUnity;
|
||||||
using Whisper; // WhisperManager, WhisperStream, WhisperResult
|
using Whisper;
|
||||||
using Whisper.Utils;
|
using Whisper.Utils;
|
||||||
using Debug = UnityEngine.Debug; // AudioChunk
|
using Debug = UnityEngine.Debug;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// FMOD mic is initialized once (Start) and runs continuously in a ring buffer.
|
/// FMOD mic is initialized once (Start) and runs continuously in a ring buffer.
|
||||||
@@ -28,12 +28,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
public int channels = 1;
|
public int channels = 1;
|
||||||
[Range(1, 10)] public int bufferLengthSec = 5;
|
[Range(1, 10)] public int bufferLengthSec = 5;
|
||||||
|
|
||||||
[Header("Loopback (monitor your voice)")]
|
|
||||||
public bool playLoopback = true;
|
|
||||||
[Tooltip("If true, loopback plays only while active; otherwise it’s always on.")]
|
|
||||||
public bool loopbackOnlyWhenActive = true;
|
|
||||||
[Range(0f, 2f)] public float loopbackVolume = 1.0f;
|
|
||||||
|
|
||||||
public delegate void OnWhisperSegmentUpdatedDelegate(string result);
|
public delegate void OnWhisperSegmentUpdatedDelegate(string result);
|
||||||
public event OnWhisperSegmentUpdatedDelegate OnWhisperSegmentUpdated;
|
public event OnWhisperSegmentUpdatedDelegate OnWhisperSegmentUpdated;
|
||||||
|
|
||||||
@@ -44,7 +38,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
private FMOD.System _core;
|
private FMOD.System _core;
|
||||||
private Sound _recSound;
|
private Sound _recSound;
|
||||||
private Channel _playChannel;
|
private Channel _playChannel;
|
||||||
private ChannelGroup _masterGroup;
|
|
||||||
private uint _soundPcmLength; // in samples
|
private uint _soundPcmLength; // in samples
|
||||||
private int _nativeRate;
|
private int _nativeRate;
|
||||||
private int _nativeChannels;
|
private int _nativeChannels;
|
||||||
@@ -54,7 +47,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
|
|
||||||
// Whisper
|
// Whisper
|
||||||
private WhisperStream _stream;
|
private WhisperStream _stream;
|
||||||
private bool _streamStarted;
|
|
||||||
|
|
||||||
// temp conversion buffer
|
// temp conversion buffer
|
||||||
private float[] _floatTmp = new float[0];
|
private float[] _floatTmp = new float[0];
|
||||||
@@ -138,24 +130,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
||||||
Debug.Log("[FMOD→Whisper] Recording started.");
|
Debug.Log("[FMOD→Whisper] Recording started.");
|
||||||
|
|
||||||
// Loopback channel (optional). Start once; pause when inactive if desired.
|
|
||||||
_core.getMasterChannelGroup(out _masterGroup);
|
|
||||||
if (playLoopback)
|
|
||||||
{
|
|
||||||
res = _core.playSound(_recSound, _masterGroup, false, out _playChannel);
|
|
||||||
if (res == RESULT.OK && _playChannel.hasHandle())
|
|
||||||
{
|
|
||||||
_playChannel.setMode(MODE._2D);
|
|
||||||
_playChannel.setVolume(loopbackVolume);
|
|
||||||
if (loopbackOnlyWhenActive) _playChannel.setPaused(true); // keep muted until Activate
|
|
||||||
Debug.Log("[FMOD→Whisper] Loopback playback ready.");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Debug.LogWarning($"[FMOD→Whisper] playSound failed or channel invalid: {res}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// No Whisper stream here. It will be created on ActivateRecording().
|
// No Whisper stream here. It will be created on ActivateRecording().
|
||||||
await System.Threading.Tasks.Task.Yield();
|
await System.Threading.Tasks.Task.Yield();
|
||||||
}
|
}
|
||||||
@@ -181,8 +155,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
|
|
||||||
if (_stream != null)
|
if (_stream != null)
|
||||||
{
|
{
|
||||||
try { _stream.StopStream(); } catch { }
|
_stream.StopStream();
|
||||||
_streamStarted = false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -193,7 +166,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
{
|
{
|
||||||
Debug.LogError($"[FMOD→Whisper] CreateStream exception: {e}");
|
Debug.LogError($"[FMOD→Whisper] CreateStream exception: {e}");
|
||||||
_stream = null;
|
_stream = null;
|
||||||
_streamStarted = false;
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,16 +186,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
whisper.useVad = useVadInStream;
|
whisper.useVad = useVadInStream;
|
||||||
|
|
||||||
_stream.StartStream();
|
_stream.StartStream();
|
||||||
_streamStarted = true;
|
|
||||||
|
|
||||||
|
|
||||||
// --- NEW: Clear the ring buffer and reset read pointer ---
|
|
||||||
// Pause loopback while we clear (optional, but avoids clicks)
|
|
||||||
if (playLoopback && _playChannel.hasHandle())
|
|
||||||
_playChannel.setPaused(true);
|
|
||||||
|
|
||||||
// Clear buffer bytes
|
|
||||||
ClearRecordRingBuffer();
|
|
||||||
|
|
||||||
// Reset our read pointer to the current write head
|
// Reset our read pointer to the current write head
|
||||||
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
||||||
@@ -231,13 +193,8 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
// We’ll skip feeding for one frame to guarantee a clean start
|
// We’ll skip feeding for one frame to guarantee a clean start
|
||||||
_skipOneFeedFrame = true;
|
_skipOneFeedFrame = true;
|
||||||
|
|
||||||
// Unpause loopback if we want it active during recording
|
|
||||||
if (playLoopback && _playChannel.hasHandle() && (!loopbackOnlyWhenActive || isRecordingActivated))
|
|
||||||
_playChannel.setPaused(loopbackOnlyWhenActive ? false : _playChannel.getPaused(out var paused) == FMOD.RESULT.OK && paused ? false : false);
|
|
||||||
|
|
||||||
isRecordingActivated = true;
|
isRecordingActivated = true;
|
||||||
Debug.Log("[FMOD→Whisper] Stream activated (buffer cleared; reading from current head).");
|
Debug.Log("[FMOD→Whisper] Stream activated (buffer cleared; reading from current head).");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -245,14 +202,10 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public void DeactivateRecording()
|
public void DeactivateRecording()
|
||||||
{
|
{
|
||||||
if (!isRecordingActivated && !_streamStarted)
|
if (!isRecordingActivated)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
isRecordingActivated = false;
|
isRecordingActivated = false;
|
||||||
|
|
||||||
// Pause loopback if it should only be active during recording
|
|
||||||
if (playLoopback && loopbackOnlyWhenActive && _playChannel.hasHandle())
|
|
||||||
_playChannel.setPaused(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -277,7 +230,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
return Mathf.Clamp01(Mathf.InverseLerp(-60f, -15f, db));
|
return Mathf.Clamp01(Mathf.InverseLerp(-60f, -15f, db));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void Update()
|
private void Update()
|
||||||
{
|
{
|
||||||
// Always tick FMOD
|
// Always tick FMOD
|
||||||
@@ -285,8 +237,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
if (!_recSound.hasHandle()) return;
|
if (!_recSound.hasHandle()) return;
|
||||||
|
|
||||||
// Compute how many samples recorded since last frame.
|
// Compute how many samples recorded since last frame.
|
||||||
uint recPos;
|
_core.getRecordPosition(recordDriverId, out uint recPos);
|
||||||
_core.getRecordPosition(recordDriverId, out recPos);
|
|
||||||
|
|
||||||
uint deltaSamples = (recPos >= _lastRecordPos)
|
uint deltaSamples = (recPos >= _lastRecordPos)
|
||||||
? (recPos - _lastRecordPos)
|
? (recPos - _lastRecordPos)
|
||||||
@@ -337,7 +288,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2) Feed audio to Whisper
|
// 2) Feed audio to Whisper
|
||||||
if (_streamStarted && _stream != null)
|
if (_stream != null)
|
||||||
{
|
{
|
||||||
if (isRecordingActivated && !_skipOneFeedFrame)
|
if (isRecordingActivated && !_skipOneFeedFrame)
|
||||||
{
|
{
|
||||||
@@ -353,7 +304,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// If skipping, we just discard this frame to ensure no stale data leaks.
|
// If skipping, we just discard this frame to ensure no stale data leaks.
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
@@ -364,7 +314,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
if (_skipOneFeedFrame) _skipOneFeedFrame = false;
|
if (_skipOneFeedFrame) _skipOneFeedFrame = false;
|
||||||
|
|
||||||
_lastRecordPos = recPos;
|
_lastRecordPos = recPos;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private string PostProcessInput(string input)
|
private string PostProcessInput(string input)
|
||||||
@@ -450,56 +399,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void ClearRecordRingBuffer()
|
|
||||||
{
|
|
||||||
if (!_recSound.hasHandle() || _soundPcmLength == 0) return;
|
|
||||||
|
|
||||||
uint totalBytes = _soundPcmLength * (uint)_nativeChannels * 2; // PCM16
|
|
||||||
IntPtr p1, p2;
|
|
||||||
uint len1, len2;
|
|
||||||
|
|
||||||
// Lock the whole buffer (start=0, length=totalBytes)
|
|
||||||
var r = _recSound.@lock(0, totalBytes, out p1, out p2, out len1, out len2);
|
|
||||||
if (r != FMOD.RESULT.OK)
|
|
||||||
{
|
|
||||||
Debug.LogWarning($"[FMOD→Whisper] Could not lock ring buffer to clear: {r}");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (len1 > 0)
|
|
||||||
{
|
|
||||||
// zero p1
|
|
||||||
// We’ll reuse a static zero array to avoid allocating huge buffers repeatedly
|
|
||||||
ZeroMem(p1, (int)len1);
|
|
||||||
}
|
|
||||||
if (len2 > 0)
|
|
||||||
{
|
|
||||||
ZeroMem(p2, (int)len2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
_recSound.unlock(p1, p2, len1, len2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// cheap zeroing helper (avoids allocating len-sized arrays each time)
|
|
||||||
private static readonly byte[] _zeroChunk = new byte[16 * 1024]; // 16 KB
|
|
||||||
private static void ZeroMem(IntPtr dst, int byteLen)
|
|
||||||
{
|
|
||||||
int offset = 0;
|
|
||||||
while (byteLen > 0)
|
|
||||||
{
|
|
||||||
int n = Math.Min(_zeroChunk.Length, byteLen);
|
|
||||||
Marshal.Copy(_zeroChunk, 0, dst + offset, n);
|
|
||||||
offset += n;
|
|
||||||
byteLen -= n;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes RMS (root mean square) from a PCM16 block using only safe code.
|
/// Computes RMS (root mean square) from a PCM16 block using only safe code.
|
||||||
/// Uses the shared _shortOverlay buffer (no allocations).
|
/// Uses the shared _shortOverlay buffer (no allocations).
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"isContinuousLocomotion": true,
|
"isContinuousLocomotion": true,
|
||||||
"continuousLocomotionSpeed": 3.0,
|
"continuousLocomotionSpeed": 3.0,
|
||||||
"volumeMaster": 0.6466318964958191,
|
"volumeMaster": 0.5969846844673157,
|
||||||
"volumeAmbient": 0.1889490932226181,
|
"volumeAmbient": 0.1889490932226181,
|
||||||
"volumeMusic": 0.5,
|
"volumeMusic": 0.5,
|
||||||
"volumeSFX": 0.5,
|
"volumeSFX": 0.5,
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"HighScore":1.0}
|
{"HighScore":212.0}
|
||||||
@@ -4,6 +4,10 @@
|
|||||||
"name": "karlkolm",
|
"name": "karlkolm",
|
||||||
"score": 241.0
|
"score": 241.0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "jass",
|
||||||
|
"score": 212.0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "andreas",
|
"name": "andreas",
|
||||||
"score": 207.0
|
"score": 207.0
|
||||||
@@ -16,10 +20,18 @@
|
|||||||
"name": "rikkss",
|
"name": "rikkss",
|
||||||
"score": 195.0
|
"score": 195.0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "noob",
|
||||||
|
"score": 195.0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "jjkujkkg",
|
"name": "jjkujkkg",
|
||||||
"score": 194.0
|
"score": 194.0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "kuues",
|
||||||
|
"score": 186.0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "gert",
|
"name": "gert",
|
||||||
"score": 184.0
|
"score": 184.0
|
||||||
@@ -32,6 +44,10 @@
|
|||||||
"name": "f",
|
"name": "f",
|
||||||
"score": 181.0
|
"score": 181.0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "kosta",
|
||||||
|
"score": 180.0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "gg",
|
"name": "gg",
|
||||||
"score": 179.0
|
"score": 179.0
|
||||||
@@ -43,18 +59,6 @@
|
|||||||
{
|
{
|
||||||
"name": "rikkss",
|
"name": "rikkss",
|
||||||
"score": 163.0
|
"score": 163.0
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "hendrik",
|
|
||||||
"score": 163.0
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "andeas",
|
|
||||||
"score": 161.0
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "fark..........",
|
|
||||||
"score": 142.0
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user