forked from cgvr/DeltaVR
Compare commits
6 Commits
ee629e2593
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e55b7e9ab | |||
| dea42068f7 | |||
| b021fd0110 | |||
| 7b45d623d1 | |||
| f42356dd7a | |||
| 50002adb61 |
@@ -3,10 +3,9 @@
|
||||
* FMOD ChannelControl errorid
|
||||
* keyboard numbrid ei tööta pärast minigame'i completemist?
|
||||
* speech-to-text:
|
||||
* vahepeal lakkab töötamast lih??? tundub et siis kui pikalt tühjust salvestab
|
||||
* vahepeal lakkab töötamast lih??? tundub et siis kui pikalt tühjust salvestab? ei, vahepeal läheb kohe alguses ka
|
||||
* sööta talle nulle kui on disabled, sest ikka kuidagi tuvastab eelnevat rääkimist
|
||||
* uurida miks buildis Whisper halvemini töötab
|
||||
* proovida suuremat Whisperi mudelit, äkki töötab mürases keskkonnas paremini
|
||||
* npc character:
|
||||
* klaas on näha temast eespool
|
||||
* shape detection:
|
||||
|
||||
BIN
Assets/StreamingAssets/Whisper/ggml-base.en.bin
Normal file
BIN
Assets/StreamingAssets/Whisper/ggml-base.en.bin
Normal file
Binary file not shown.
7
Assets/StreamingAssets/Whisper/ggml-base.en.bin.meta
Normal file
7
Assets/StreamingAssets/Whisper/ggml-base.en.bin.meta
Normal file
@@ -0,0 +1,7 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 4fef61b9ad4f7eb488ea2657f1cc700e
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
BIN
Assets/StreamingAssets/Whisper/ggml-tiny.bin
Normal file
BIN
Assets/StreamingAssets/Whisper/ggml-tiny.bin
Normal file
Binary file not shown.
7
Assets/StreamingAssets/Whisper/ggml-tiny.bin.meta
Normal file
7
Assets/StreamingAssets/Whisper/ggml-tiny.bin.meta
Normal file
@@ -0,0 +1,7 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 6f5a0c98d09a6dc4d9d570e23dc96ab9
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
Binary file not shown.
Binary file not shown.
@@ -29,7 +29,7 @@ Transform:
|
||||
m_GameObject: {fileID: 1659788510314838016}
|
||||
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
|
||||
m_LocalPosition: {x: 0, y: 0, z: 0}
|
||||
m_LocalScale: {x: 0.4, y: 0.35, z: 0.4}
|
||||
m_LocalScale: {x: 0.4, y: 0.38386, z: 0.4}
|
||||
m_ConstrainProportionsScale: 0
|
||||
m_Children:
|
||||
- {fileID: 4211436824399037528}
|
||||
@@ -149,7 +149,7 @@ Transform:
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 4039766663692926199}
|
||||
m_LocalRotation: {x: -0, y: 0.000000014901158, z: -0, w: 1}
|
||||
m_LocalPosition: {x: 0, y: 0, z: 0}
|
||||
m_LocalPosition: {x: 0, y: -0.36, z: 0}
|
||||
m_LocalScale: {x: 1, y: 1, z: 1}
|
||||
m_ConstrainProportionsScale: 0
|
||||
m_Children: []
|
||||
|
||||
Binary file not shown.
@@ -4,9 +4,9 @@ using System.Runtime.InteropServices;
|
||||
using UnityEngine;
|
||||
using FMOD;
|
||||
using FMODUnity;
|
||||
using Whisper; // WhisperManager, WhisperStream, WhisperResult
|
||||
using Whisper;
|
||||
using Whisper.Utils;
|
||||
using Debug = UnityEngine.Debug; // AudioChunk
|
||||
using Debug = UnityEngine.Debug;
|
||||
|
||||
/// <summary>
|
||||
/// FMOD mic is initialized once (Start) and runs continuously in a ring buffer.
|
||||
@@ -28,12 +28,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
public int channels = 1;
|
||||
[Range(1, 10)] public int bufferLengthSec = 5;
|
||||
|
||||
[Header("Loopback (monitor your voice)")]
|
||||
public bool playLoopback = true;
|
||||
[Tooltip("If true, loopback plays only while active; otherwise it’s always on.")]
|
||||
public bool loopbackOnlyWhenActive = true;
|
||||
[Range(0f, 2f)] public float loopbackVolume = 1.0f;
|
||||
|
||||
public delegate void OnWhisperSegmentUpdatedDelegate(string result);
|
||||
public event OnWhisperSegmentUpdatedDelegate OnWhisperSegmentUpdated;
|
||||
|
||||
@@ -44,7 +38,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
private FMOD.System _core;
|
||||
private Sound _recSound;
|
||||
private Channel _playChannel;
|
||||
private ChannelGroup _masterGroup;
|
||||
private uint _soundPcmLength; // in samples
|
||||
private int _nativeRate;
|
||||
private int _nativeChannels;
|
||||
@@ -54,7 +47,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
|
||||
// Whisper
|
||||
private WhisperStream _stream;
|
||||
private bool _streamStarted;
|
||||
|
||||
// temp conversion buffer
|
||||
private float[] _floatTmp = new float[0];
|
||||
@@ -138,24 +130,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
||||
Debug.Log("[FMOD→Whisper] Recording started.");
|
||||
|
||||
// Loopback channel (optional). Start once; pause when inactive if desired.
|
||||
_core.getMasterChannelGroup(out _masterGroup);
|
||||
if (playLoopback)
|
||||
{
|
||||
res = _core.playSound(_recSound, _masterGroup, false, out _playChannel);
|
||||
if (res == RESULT.OK && _playChannel.hasHandle())
|
||||
{
|
||||
_playChannel.setMode(MODE._2D);
|
||||
_playChannel.setVolume(loopbackVolume);
|
||||
if (loopbackOnlyWhenActive) _playChannel.setPaused(true); // keep muted until Activate
|
||||
Debug.Log("[FMOD→Whisper] Loopback playback ready.");
|
||||
}
|
||||
else
|
||||
{
|
||||
Debug.LogWarning($"[FMOD→Whisper] playSound failed or channel invalid: {res}");
|
||||
}
|
||||
}
|
||||
|
||||
// No Whisper stream here. It will be created on ActivateRecording().
|
||||
await System.Threading.Tasks.Task.Yield();
|
||||
}
|
||||
@@ -181,8 +155,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
|
||||
if (_stream != null)
|
||||
{
|
||||
try { _stream.StopStream(); } catch { }
|
||||
_streamStarted = false;
|
||||
_stream.StopStream();
|
||||
}
|
||||
|
||||
try
|
||||
@@ -193,7 +166,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
{
|
||||
Debug.LogError($"[FMOD→Whisper] CreateStream exception: {e}");
|
||||
_stream = null;
|
||||
_streamStarted = false;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -214,16 +186,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
whisper.useVad = useVadInStream;
|
||||
|
||||
_stream.StartStream();
|
||||
_streamStarted = true;
|
||||
|
||||
|
||||
// --- NEW: Clear the ring buffer and reset read pointer ---
|
||||
// Pause loopback while we clear (optional, but avoids clicks)
|
||||
if (playLoopback && _playChannel.hasHandle())
|
||||
_playChannel.setPaused(true);
|
||||
|
||||
// Clear buffer bytes
|
||||
ClearRecordRingBuffer();
|
||||
|
||||
// Reset our read pointer to the current write head
|
||||
_core.getRecordPosition(recordDriverId, out _lastRecordPos);
|
||||
@@ -231,13 +193,8 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
// We’ll skip feeding for one frame to guarantee a clean start
|
||||
_skipOneFeedFrame = true;
|
||||
|
||||
// Unpause loopback if we want it active during recording
|
||||
if (playLoopback && _playChannel.hasHandle() && (!loopbackOnlyWhenActive || isRecordingActivated))
|
||||
_playChannel.setPaused(loopbackOnlyWhenActive ? false : _playChannel.getPaused(out var paused) == FMOD.RESULT.OK && paused ? false : false);
|
||||
|
||||
isRecordingActivated = true;
|
||||
Debug.Log("[FMOD→Whisper] Stream activated (buffer cleared; reading from current head).");
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -245,14 +202,10 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
/// </summary>
|
||||
public void DeactivateRecording()
|
||||
{
|
||||
if (!isRecordingActivated && !_streamStarted)
|
||||
if (!isRecordingActivated)
|
||||
return;
|
||||
|
||||
isRecordingActivated = false;
|
||||
|
||||
// Pause loopback if it should only be active during recording
|
||||
if (playLoopback && loopbackOnlyWhenActive && _playChannel.hasHandle())
|
||||
_playChannel.setPaused(true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -277,7 +230,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
return Mathf.Clamp01(Mathf.InverseLerp(-60f, -15f, db));
|
||||
}
|
||||
|
||||
|
||||
private void Update()
|
||||
{
|
||||
// Always tick FMOD
|
||||
@@ -285,8 +237,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
if (!_recSound.hasHandle()) return;
|
||||
|
||||
// Compute how many samples recorded since last frame.
|
||||
uint recPos;
|
||||
_core.getRecordPosition(recordDriverId, out recPos);
|
||||
_core.getRecordPosition(recordDriverId, out uint recPos);
|
||||
|
||||
uint deltaSamples = (recPos >= _lastRecordPos)
|
||||
? (recPos - _lastRecordPos)
|
||||
@@ -337,7 +288,7 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
}
|
||||
|
||||
// 2) Feed audio to Whisper
|
||||
if (_streamStarted && _stream != null)
|
||||
if (_stream != null)
|
||||
{
|
||||
if (isRecordingActivated && !_skipOneFeedFrame)
|
||||
{
|
||||
@@ -353,7 +304,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// If skipping, we just discard this frame to ensure no stale data leaks.
|
||||
}
|
||||
finally
|
||||
@@ -364,7 +314,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
if (_skipOneFeedFrame) _skipOneFeedFrame = false;
|
||||
|
||||
_lastRecordPos = recPos;
|
||||
|
||||
}
|
||||
|
||||
private string PostProcessInput(string input)
|
||||
@@ -450,56 +399,6 @@ public class FMODWhisperBridge : MonoBehaviour
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void ClearRecordRingBuffer()
|
||||
{
|
||||
if (!_recSound.hasHandle() || _soundPcmLength == 0) return;
|
||||
|
||||
uint totalBytes = _soundPcmLength * (uint)_nativeChannels * 2; // PCM16
|
||||
IntPtr p1, p2;
|
||||
uint len1, len2;
|
||||
|
||||
// Lock the whole buffer (start=0, length=totalBytes)
|
||||
var r = _recSound.@lock(0, totalBytes, out p1, out p2, out len1, out len2);
|
||||
if (r != FMOD.RESULT.OK)
|
||||
{
|
||||
Debug.LogWarning($"[FMOD→Whisper] Could not lock ring buffer to clear: {r}");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (len1 > 0)
|
||||
{
|
||||
// zero p1
|
||||
// We’ll reuse a static zero array to avoid allocating huge buffers repeatedly
|
||||
ZeroMem(p1, (int)len1);
|
||||
}
|
||||
if (len2 > 0)
|
||||
{
|
||||
ZeroMem(p2, (int)len2);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_recSound.unlock(p1, p2, len1, len2);
|
||||
}
|
||||
}
|
||||
|
||||
// cheap zeroing helper (avoids allocating len-sized arrays each time)
|
||||
private static readonly byte[] _zeroChunk = new byte[16 * 1024]; // 16 KB
|
||||
private static void ZeroMem(IntPtr dst, int byteLen)
|
||||
{
|
||||
int offset = 0;
|
||||
while (byteLen > 0)
|
||||
{
|
||||
int n = Math.Min(_zeroChunk.Length, byteLen);
|
||||
Marshal.Copy(_zeroChunk, 0, dst + offset, n);
|
||||
offset += n;
|
||||
byteLen -= n;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes RMS (root mean square) from a PCM16 block using only safe code.
|
||||
/// Uses the shared _shortOverlay buffer (no allocations).
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"isContinuousLocomotion": true,
|
||||
"continuousLocomotionSpeed": 3.0,
|
||||
"volumeMaster": 0.6466318964958191,
|
||||
"volumeMaster": 0.5969846844673157,
|
||||
"volumeAmbient": 0.1889490932226181,
|
||||
"volumeMusic": 0.5,
|
||||
"volumeSFX": 0.5,
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"HighScore":1.0}
|
||||
{"HighScore":212.0}
|
||||
@@ -4,6 +4,10 @@
|
||||
"name": "karlkolm",
|
||||
"score": 241.0
|
||||
},
|
||||
{
|
||||
"name": "jass",
|
||||
"score": 212.0
|
||||
},
|
||||
{
|
||||
"name": "andreas",
|
||||
"score": 207.0
|
||||
@@ -16,10 +20,18 @@
|
||||
"name": "rikkss",
|
||||
"score": 195.0
|
||||
},
|
||||
{
|
||||
"name": "noob",
|
||||
"score": 195.0
|
||||
},
|
||||
{
|
||||
"name": "jjkujkkg",
|
||||
"score": 194.0
|
||||
},
|
||||
{
|
||||
"name": "kuues",
|
||||
"score": 186.0
|
||||
},
|
||||
{
|
||||
"name": "gert",
|
||||
"score": 184.0
|
||||
@@ -32,6 +44,10 @@
|
||||
"name": "f",
|
||||
"score": 181.0
|
||||
},
|
||||
{
|
||||
"name": "kosta",
|
||||
"score": 180.0
|
||||
},
|
||||
{
|
||||
"name": "gg",
|
||||
"score": 179.0
|
||||
@@ -43,18 +59,6 @@
|
||||
{
|
||||
"name": "rikkss",
|
||||
"score": 163.0
|
||||
},
|
||||
{
|
||||
"name": "hendrik",
|
||||
"score": 163.0
|
||||
},
|
||||
{
|
||||
"name": "andeas",
|
||||
"score": 161.0
|
||||
},
|
||||
{
|
||||
"name": "fark..........",
|
||||
"score": 142.0
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user