1
0
forked from cgvr/DeltaVR

10 Commits

34 changed files with 1097 additions and 230 deletions

View File

@@ -1,6 +1,28 @@
### TODO ### TODO
* Artikkel text-to-3d prompt engineeringu kohta: "Sel3DCraft: Interactive Visual Prompts for User-Friendly Text-to-3D Generation" * glTF loading: vahetada ära shader Universal render pipelin Lit, mitte panna buildi kaasa glTf oma
* TRELLIS: postprocessing_utils: texture baking mode: 'opt' vs 'fast' - hardcoded 'opt', kui võimaldada 'fast' siis tuleb error * shape scanner initialisation correct number of confs
* user flow: grab item? mida krabada
* user prefs: settinguid meelde jätta
* võtta spawnitud mudeli mõõtmed: mehscollideri max x, max y, etc? bounding box?
* mängija collide'ib spawnitud mudeliga - ei tohiks
* shape scanner:
* mitte-liigutatavaks, aga samal ajal kõrgus dünaamiliselt õige. või lihtsalt piisavalt madalale asetada või väljaulatuv kang millest krabada
* peenikesemad kiired
* mitte lihtsalt ontriggerenter ja -exit, sest kui mitu objekti lähevad samal ajal sisse
* mustad kiired on halvasti nähtavad pruuni materjali taustal
* kui üks config completed, siis mängijale aru saada: sound effect, "loading"
* mikri vana tekst ei kao ära
* archery range:
* kui midagi laeb (wire aktiivne), siis particle'id voolavad mööda toru
* highscore json tühjaks
* quest marker järjest järgmise tegevuse kohal: mikrofon, siis nupud
* character billboard:
* peaks vaatama pea poole, mitte xr origin
* klaas on näha temast eespool
* pööramine kaamera poole - sujuvalt (slerp)
* küsida Danielilt asukoha kohta
### Notes ### Notes
* TRELLIS: added functionality to specify texture baking optimisation total steps as an argument (`texture_opt_total_steps`), to replace the hardcoded 2500. But this is not tracked in Git (because modified this https://github.com/IgorAherne/trellis-stable-projectorz/releases/tag/latest) * TRELLIS: added functionality to specify texture baking optimisation total steps as an argument (`texture_opt_total_steps`), to replace the hardcoded 2500. But this is not tracked in Git (because modified this https://github.com/IgorAherne/trellis-stable-projectorz/releases/tag/latest)
* Custom Shader Variant Collection to include glTF-pbrMetallicRoughness shader in build

Binary file not shown.

View File

@@ -0,0 +1,144 @@
#if UNITY_EDITOR
using UnityEditor;
using UnityEngine;
[CustomPropertyDrawer(typeof(ShapeScannerConfiguration))]
public class BoolMatrixDrawer : PropertyDrawer
{
private const float ToggleSize = 18f;
private const float RowLabelWidth = 24f;
private const float Padding = 2f;
public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
{
var rowsProp = property.FindPropertyRelative("rows");
var columnsProp = property.FindPropertyRelative("columns");
int rows = rowsProp != null ? rowsProp.arraySize : 0;
int cols = Mathf.Max(0, columnsProp != null ? columnsProp.intValue : 0);
// Header (one line), columns field (one line), buttons (one line), then one line per row
int totalLines = 1 + 1 + 1 + Mathf.Max(1, rows);
float lineHeight = EditorGUIUtility.singleLineHeight + Padding;
return totalLines * lineHeight + Padding;
}
public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
{
EditorGUI.BeginProperty(position, label, property);
var rowsProp = property.FindPropertyRelative("rows");
var columnsProp = property.FindPropertyRelative("columns");
if (rowsProp == null || columnsProp == null)
{
EditorGUI.HelpBox(position, "BoolMatrix is missing 'rows' or 'columns' properties.", MessageType.Error);
EditorGUI.EndProperty();
return;
}
// Layout helper
float lineHeight = EditorGUIUtility.singleLineHeight;
Rect line = new Rect(position.x, position.y, position.width, lineHeight);
// Header
EditorGUI.LabelField(line, label, EditorStyles.boldLabel);
line.y += lineHeight + Padding;
// Draw requiredCorrectPercentage
var requiredCorrectProp = property.FindPropertyRelative("requiredCorrectPercentage");
EditorGUI.PropertyField(line, requiredCorrectProp, new GUIContent("Required Correct Percentage"));
line.y += lineHeight + Padding;
// Columns field
EditorGUI.BeginChangeCheck();
int cols = Mathf.Max(0, EditorGUI.IntField(line, "Columns", columnsProp.intValue));
if (EditorGUI.EndChangeCheck())
{
columnsProp.intValue = cols;
// Resize each row to match new column count
for (int r = 0; r < rowsProp.arraySize; r++)
{
var rowProp = rowsProp.GetArrayElementAtIndex(r);
var cellsProp = rowProp.FindPropertyRelative("cells");
ResizeBoolArray(cellsProp, cols);
}
}
line.y += lineHeight + Padding;
// Row controls (Add/Remove)
using (new EditorGUI.IndentLevelScope())
{
Rect left = new Rect(line.x, line.y, 120, lineHeight);
if (GUI.Button(left, "Add Row"))
{
int newIndex = rowsProp.arraySize;
rowsProp.InsertArrayElementAtIndex(newIndex);
var rowProp = rowsProp.GetArrayElementAtIndex(newIndex);
var cellsProp = rowProp.FindPropertyRelative("cells");
ResizeBoolArray(cellsProp, columnsProp.intValue);
}
Rect right = new Rect(line.x + 130, line.y, 140, lineHeight);
if (GUI.Button(right, "Remove Last Row") && rowsProp.arraySize > 0)
{
rowsProp.DeleteArrayElementAtIndex(rowsProp.arraySize - 1);
}
}
line.y += lineHeight + Padding;
// Draw grid
int rowCount = rowsProp.arraySize;
int colCount = Mathf.Max(0, columnsProp.intValue);
for (int r = 0; r < rowCount; r++)
{
var rowProp = rowsProp.GetArrayElementAtIndex(r);
var cellsProp = rowProp.FindPropertyRelative("cells");
// Ensure row width
if (cellsProp.arraySize != colCount)
ResizeBoolArray(cellsProp, colCount);
// Row label
Rect rowLabel = new Rect(line.x, line.y, RowLabelWidth, lineHeight);
EditorGUI.LabelField(rowLabel, $"R{r}");
// Toggle strip
float startX = rowLabel.x + RowLabelWidth + Padding;
for (int c = 0; c < colCount; c++)
{
Rect toggleRect = new Rect(startX + c * (ToggleSize + 2), line.y, ToggleSize, lineHeight);
var cellProp = cellsProp.GetArrayElementAtIndex(c);
bool newVal = EditorGUI.Toggle(toggleRect, GUIContent.none, cellProp.boolValue);
if (newVal != cellProp.boolValue) cellProp.boolValue = newVal;
}
line.y += lineHeight + Padding;
}
EditorGUI.EndProperty();
}
private void ResizeBoolArray(SerializedProperty listProp, int newSize)
{
if (listProp == null) return;
newSize = Mathf.Max(0, newSize);
// Grow
while (listProp.arraySize < newSize)
{
int i = listProp.arraySize;
listProp.InsertArrayElementAtIndex(i);
var elem = listProp.GetArrayElementAtIndex(i);
elem.boolValue = false;
}
// Shrink
while (listProp.arraySize > newSize)
{
listProp.DeleteArrayElementAtIndex(listProp.arraySize - 1);
}
}
}
#endif

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: e3da5d98c11c1b54fb05eda4624de9e3
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -13,7 +13,6 @@ GameObject:
- component: {fileID: 7203143526414218131} - component: {fileID: 7203143526414218131}
- component: {fileID: 2568780590385977406} - component: {fileID: 2568780590385977406}
- component: {fileID: 1523408157143897080} - component: {fileID: 1523408157143897080}
- component: {fileID: 2918895218826581840}
m_Layer: 0 m_Layer: 0
m_Name: MicrophoneStand 1 m_Name: MicrophoneStand 1
m_TagString: Untagged m_TagString: Untagged
@@ -35,7 +34,7 @@ Transform:
m_Children: m_Children:
- {fileID: 1850758373829337931} - {fileID: 1850758373829337931}
m_Father: {fileID: 0} m_Father: {fileID: 0}
m_RootOrder: 0 m_RootOrder: -1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &4763701867718457502 --- !u!33 &4763701867718457502
MeshFilter: MeshFilter:
@@ -120,41 +119,10 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 357101b2cface4943b04dfd25d4944e3, type: 3} m_Script: {fileID: 11500000, guid: 357101b2cface4943b04dfd25d4944e3, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
whisper: {fileID: 0}
microphoneRecord: {fileID: 2918895218826581840}
microphoneDevice:
outputText: {fileID: 0} outputText: {fileID: 0}
microphoneOffStatus: {fileID: 8977839985090371394} microphoneOffStatus: {fileID: 8977839985090371394}
microphoneOnStatus: {fileID: 6537061652288108950} microphoneOnStatus: {fileID: 6537061652288108950}
--- !u!114 &2918895218826581840 fmodWhisperBridge: {fileID: 0}
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4291579148315658230}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 3bc03a4c19604ea394e364f8fc632928, type: 3}
m_Name:
m_EditorClassIdentifier:
maxLengthSec: 60
loop: 0
frequency: 16000
chunksLengthSec: 0.5
echo: 1
useVad: 1
vadUpdateRateSec: 0.1
vadContextSec: 30
vadLastSec: 1.25
vadThd: 1
vadFreqThd: 100
vadIndicatorImage: {fileID: 0}
vadStop: 0
dropVadPart: 1
vadStopTime: 3
microphoneDropdown: {fileID: 0}
microphoneDefaultLabel: Default microphone
--- !u!1 &5819798980962142350 --- !u!1 &5819798980962142350
GameObject: GameObject:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0

View File

@@ -13,8 +13,7 @@ GameObject:
- component: {fileID: 3363453355800186393} - component: {fileID: 3363453355800186393}
- component: {fileID: 2166102850012183631} - component: {fileID: 2166102850012183631}
- component: {fileID: 8356602476881383464} - component: {fileID: 8356602476881383464}
- component: {fileID: 4280129837482332964} - component: {fileID: 5682436338090300270}
- component: {fileID: 8105025160921831064}
m_Layer: 0 m_Layer: 0
m_Name: ModelGenerationManager m_Name: ModelGenerationManager
m_TagString: Untagged m_TagString: Untagged
@@ -64,6 +63,8 @@ MonoBehaviour:
INVOKEAI_BASE_URL: http://ltat-cgvr9.domenis.ut.ee:9090 INVOKEAI_BASE_URL: http://ltat-cgvr9.domenis.ut.ee:9090
DEFAULT_QUEUE_ID: default DEFAULT_QUEUE_ID: default
MODEL_KEY: 81d45960-08a0-4b8c-a48b-e7d73b21bfe2 MODEL_KEY: 81d45960-08a0-4b8c-a48b-e7d73b21bfe2
promptSuffix: ', single object, front and side fully visible, realistic style,
plain neutral background, clear details, soft studio lighting, true-to-scale'
--- !u!114 &2166102850012183631 --- !u!114 &2166102850012183631
MonoBehaviour: MonoBehaviour:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
@@ -110,7 +111,7 @@ MonoBehaviour:
useVad: 1 useVad: 1
tokensTimestamps: 0 tokensTimestamps: 0
audioCtx: 0 audioCtx: 0
--- !u!114 &4280129837482332964 --- !u!114 &5682436338090300270
MonoBehaviour: MonoBehaviour:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0} m_CorrespondingSourceObject: {fileID: 0}
@@ -119,65 +120,14 @@ MonoBehaviour:
m_GameObject: {fileID: 154411548685861447} m_GameObject: {fileID: 154411548685861447}
m_Enabled: 1 m_Enabled: 1
m_EditorHideFlags: 0 m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: a2836e36774ca1c4bbbee976e17b649c, type: 3} m_Script: {fileID: 11500000, guid: a264cb5321d9f6741a2625b8c63c34a7, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
_componentIndexCache: 0 whisper: {fileID: 8356602476881383464}
_addedNetworkObject: {fileID: 8105025160921831064} useVadInStream: 0
_networkObjectCache: {fileID: 8105025160921831064} recordDriverId: 0
_synchronizeParent: 0 desiredSampleRate: 48000
_packing: channels: 1
Position: 1 bufferLengthSec: 5
Rotation: 1 playLoopback: 0
Scale: 0 loopbackVolume: 1
_interpolation: 2
_extrapolation: 2
_enableTeleport: 0
_teleportThreshold: 1
_clientAuthoritative: 1
_sendToOwner: 1
_synchronizePosition: 1
_positionSnapping:
X: 0
Y: 0
Z: 0
_synchronizeRotation: 1
_rotationSnapping:
X: 0
Y: 0
Z: 0
_synchronizeScale: 1
_scaleSnapping:
X: 0
Y: 0
Z: 0
--- !u!114 &8105025160921831064
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 154411548685861447}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 26b716c41e9b56b4baafaf13a523ba2e, type: 3}
m_Name:
m_EditorClassIdentifier:
<IsNested>k__BackingField: 0
<ComponentIndex>k__BackingField: 0
<PredictedSpawn>k__BackingField: {fileID: 0}
_networkBehaviours:
- {fileID: 4280129837482332964}
<ParentNetworkObject>k__BackingField: {fileID: 0}
<ChildNetworkObjects>k__BackingField: []
_isNetworked: 1
_isGlobal: 0
_initializeOrder: 0
_defaultDespawnType: 0
NetworkObserver: {fileID: 0}
<PrefabId>k__BackingField: 0
<SpawnableCollectionId>k__BackingField: 0
_scenePathHash: 0
<SceneId>k__BackingField: 0
<AssetPathHash>k__BackingField: 17646158845367820466
_sceneNetworkObjects: []

View File

@@ -27,8 +27,8 @@ Transform:
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1013838101896458198} m_GameObject: {fileID: 1013838101896458198}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: -0.21589, z: 0} m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0.64645, y: 0.029855646, z: 0.64645} m_LocalScale: {x: 0.6, y: 0.03, z: 0.6}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: [] m_Children: []
m_Father: {fileID: 8294461198356057691} m_Father: {fileID: 8294461198356057691}
@@ -105,6 +105,146 @@ BoxCollider:
serializedVersion: 3 serializedVersion: 3
m_Size: {x: 1, y: 1, z: 1} m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0} m_Center: {x: 0, y: 0, z: 0}
--- !u!1 &1231701550042884109
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 8594417542550664647}
- component: {fileID: 7869926060146003813}
- component: {fileID: 5476518908687164580}
m_Layer: 0
m_Name: CurrentConfigurationDisplay
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &8594417542550664647
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1231701550042884109}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 6932860525531626823}
m_RootOrder: -1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 0}
m_AnchorMax: {x: 0.5, y: 0}
m_AnchoredPosition: {x: 0, y: 22.45}
m_SizeDelta: {x: 50, y: 14}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &7869926060146003813
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1231701550042884109}
m_CullTransparentMesh: 1
--- !u!114 &5476518908687164580
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1231701550042884109}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_text: '0 / 2
'
m_isRightToLeft: 0
m_fontAsset: {fileID: 11400000, guid: 254d33525bc3919439f569ea33703c5b, type: 2}
m_sharedMaterial: {fileID: 4369893532151414794, guid: 254d33525bc3919439f569ea33703c5b,
type: 2}
m_fontSharedMaterials: []
m_fontMaterial: {fileID: 0}
m_fontMaterials: []
m_fontColor32:
serializedVersion: 2
rgba: 4294967295
m_fontColor: {r: 1, g: 1, b: 1, a: 1}
m_enableVertexGradient: 0
m_colorMode: 3
m_fontColorGradient:
topLeft: {r: 1, g: 1, b: 1, a: 1}
topRight: {r: 1, g: 1, b: 1, a: 1}
bottomLeft: {r: 1, g: 1, b: 1, a: 1}
bottomRight: {r: 1, g: 1, b: 1, a: 1}
m_fontColorGradientPreset: {fileID: 0}
m_spriteAsset: {fileID: 0}
m_tintAllSprites: 0
m_StyleSheet: {fileID: 0}
m_TextStyleHashCode: -1183493901
m_overrideHtmlColors: 0
m_faceColor:
serializedVersion: 2
rgba: 4294967295
m_fontSize: 12
m_fontSizeBase: 12
m_fontWeight: 400
m_enableAutoSizing: 0
m_fontSizeMin: 18
m_fontSizeMax: 72
m_fontStyle: 0
m_HorizontalAlignment: 2
m_VerticalAlignment: 256
m_textAlignment: 65535
m_characterSpacing: 0
m_wordSpacing: 0
m_lineSpacing: 0
m_lineSpacingMax: 0
m_paragraphSpacing: 0
m_charWidthMaxAdj: 0
m_TextWrappingMode: 1
m_wordWrappingRatios: 0.4
m_overflowMode: 0
m_linkedTextComponent: {fileID: 0}
parentLinkedComponent: {fileID: 0}
m_enableKerning: 0
m_ActiveFontFeatures: 6e72656b
m_enableExtraPadding: 0
checkPaddingRequired: 0
m_isRichText: 1
m_EmojiFallbackSupport: 1
m_parseCtrlCharacters: 1
m_isOrthographic: 1
m_isCullingEnabled: 0
m_horizontalMapping: 0
m_verticalMapping: 0
m_uvLineOffset: 0
m_geometrySortingOrder: 0
m_IsTextObjectScaleStatic: 0
m_VertexBufferAutoSizeReduction: 0
m_useMaxVisibleDescender: 1
m_pageToDisplay: 1
m_margin: {x: 0, y: 0, z: 0, w: 0}
m_isUsingLegacyAnimationComponent: 0
m_isVolumetricText: 0
m_hasFontAssetChanged: 0
m_baseMaterial: {fileID: 0}
m_maskOffset: {x: 0, y: 0, z: 0, w: 0}
--- !u!1 &1374309978863673311 --- !u!1 &1374309978863673311
GameObject: GameObject:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
@@ -129,7 +269,7 @@ Transform:
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1374309978863673311} m_GameObject: {fileID: 1374309978863673311}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0.3, y: 0, z: 0.3} m_LocalPosition: {x: 0.275, y: 0, z: 0.275}
m_LocalScale: {x: 1, y: 1, z: 1} m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: [] m_Children: []
@@ -160,7 +300,7 @@ Transform:
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1598434951760464124} m_GameObject: {fileID: 1598434951760464124}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: -0.416, z: 0} m_LocalPosition: {x: 0, y: -0.225, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1} m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: m_Children:
@@ -199,6 +339,7 @@ RectTransform:
m_LocalScale: {x: 1, y: 1, z: 1} m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: m_Children:
- {fileID: 8594417542550664647}
- {fileID: 6492405423818451632} - {fileID: 6492405423818451632}
m_Father: {fileID: 5335160954898573969} m_Father: {fileID: 5335160954898573969}
m_RootOrder: -1 m_RootOrder: -1
@@ -273,8 +414,8 @@ Transform:
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 3769602036287398038} m_GameObject: {fileID: 3769602036287398038}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: -0.61411, z: 0} m_LocalPosition: {x: 0, y: -0.5, z: 0}
m_LocalScale: {x: 0.64645, y: 0.029855646, z: 0.64645} m_LocalScale: {x: 0.6, y: 0.03, z: 0.6}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: [] m_Children: []
m_Father: {fileID: 8294461198356057691} m_Father: {fileID: 8294461198356057691}
@@ -375,7 +516,7 @@ Transform:
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5358414661713154358} m_GameObject: {fileID: 5358414661713154358}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.3, y: 0, z: -0.3} m_LocalPosition: {x: -0.275, y: 0, z: -0.275}
m_LocalScale: {x: 1, y: 1, z: 1} m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: [] m_Children: []
@@ -394,7 +535,7 @@ GameObject:
- component: {fileID: 489185111762359262} - component: {fileID: 489185111762359262}
- component: {fileID: 7443612957244256760} - component: {fileID: 7443612957244256760}
m_Layer: 0 m_Layer: 0
m_Name: Text m_Name: CorrectPercentageDisplay
m_TagString: Untagged m_TagString: Untagged
m_Icon: {fileID: 0} m_Icon: {fileID: 0}
m_NavMeshLayer: 0 m_NavMeshLayer: 0
@@ -415,9 +556,9 @@ RectTransform:
m_Father: {fileID: 6932860525531626823} m_Father: {fileID: 6932860525531626823}
m_RootOrder: -1 m_RootOrder: -1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 0.5} m_AnchorMin: {x: 0.5, y: 0}
m_AnchorMax: {x: 0.5, y: 0.5} m_AnchorMax: {x: 0.5, y: 0}
m_AnchoredPosition: {x: 0, y: 0} m_AnchoredPosition: {x: 0, y: 7}
m_SizeDelta: {x: 50, y: 14} m_SizeDelta: {x: 50, y: 14}
m_Pivot: {x: 0.5, y: 0.5} m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &489185111762359262 --- !u!222 &489185111762359262
@@ -570,25 +711,38 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: b2226de466bbe814d8bfe0cb0fce83f6, type: 3} m_Script: {fileID: 11500000, guid: b2226de466bbe814d8bfe0cb0fce83f6, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
configuration: configurations:
- cells: 010001010000 - columns: 6
- cells: 000101010000 requiredCorrectPercentage: 80
rows:
- cells: 000000000000
- cells: 000001010000
- cells: 000001010000
- cells: 000101010100
- cells: 010101010101 - cells: 010101010101
- cells: 000000000000 - cells: 000000000000
- cells: 000000000000 - columns: 6
- cells: 000000000000 requiredCorrectPercentage: 80
rows:
- cells: 000001000000
- cells: 000101010100
- cells: 000101010101
- cells: 010101010100
- cells: 000101010100
- cells: 000000010000
rayPrefab: {fileID: 1303694035827653214, guid: dfe698ad960ffbd47a7d0ff8f6b5baa3, rayPrefab: {fileID: 1303694035827653214, guid: dfe698ad960ffbd47a7d0ff8f6b5baa3,
type: 3} type: 3}
raySpawnCorner1: {fileID: 4055409281532582849} raySpawnCorner1: {fileID: 4055409281532582849}
raySpawnCorner2: {fileID: 8025195334996356355} raySpawnCorner2: {fileID: 8025195334996356355}
rayParent: {fileID: 4362133469817120903} rayParent: {fileID: 4362133469817120903}
currentConfigurationDisplay: {fileID: 5476518908687164580}
correctPercentageDisplay: {fileID: 7443612957244256760}
requiredAndActive: {fileID: 2100000, guid: dccb608b252977047bd5848d6a497bfb, type: 2} requiredAndActive: {fileID: 2100000, guid: dccb608b252977047bd5848d6a497bfb, type: 2}
requiredAndPassive: {fileID: 2100000, guid: c0707507abffb6149b19e60299403e82, type: 2} requiredAndPassive: {fileID: 2100000, guid: c0707507abffb6149b19e60299403e82, type: 2}
notRequiredAndActive: {fileID: 2100000, guid: c7aad7ea05b9332478242744c9de52b9, notRequiredAndActive: {fileID: 2100000, guid: c7aad7ea05b9332478242744c9de52b9,
type: 2} type: 2}
notRequiredAndPassive: {fileID: 2100000, guid: 991fa2870324ba04aba7bec9e1168afa, notRequiredAndPassive: {fileID: 2100000, guid: 991fa2870324ba04aba7bec9e1168afa,
type: 2} type: 2}
displayText: {fileID: 7443612957244256760}
--- !u!54 &6146655625910388013 --- !u!54 &6146655625910388013
Rigidbody: Rigidbody:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
@@ -707,7 +861,7 @@ MonoBehaviour:
m_Calls: [] m_Calls: []
m_AttachTransform: {fileID: 0} m_AttachTransform: {fileID: 0}
m_SecondaryAttachTransform: {fileID: 0} m_SecondaryAttachTransform: {fileID: 0}
m_UseDynamicAttach: 0 m_UseDynamicAttach: 1
m_MatchAttachPosition: 1 m_MatchAttachPosition: 1
m_MatchAttachRotation: 1 m_MatchAttachRotation: 1
m_SnapToColliderVolume: 1 m_SnapToColliderVolume: 1
@@ -788,8 +942,8 @@ RectTransform:
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0} m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0} m_AnchoredPosition: {x: 0, y: 0.16}
m_SizeDelta: {x: 50, y: 20} m_SizeDelta: {x: 50, y: 30}
m_Pivot: {x: 0.5, y: 0.5} m_Pivot: {x: 0.5, y: 0.5}
--- !u!223 &2825347825077556369 --- !u!223 &2825347825077556369
Canvas: Canvas:

View File

@@ -29,11 +29,11 @@ Transform:
m_GameObject: {fileID: 8688612914795219519} m_GameObject: {fileID: 8688612914795219519}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0.259, y: 0.012, z: -0.26886} m_LocalPosition: {x: 0.259, y: 0.012, z: -0.26886}
m_LocalScale: {x: 0.035900656, y: 0.45784503, z: 0.035900656} m_LocalScale: {x: 0.035, y: 0.5, z: 0.035}
m_ConstrainProportionsScale: 0 m_ConstrainProportionsScale: 0
m_Children: [] m_Children: []
m_Father: {fileID: 0} m_Father: {fileID: 0}
m_RootOrder: -1 m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &8730629787356023745 --- !u!33 &8730629787356023745
MeshFilter: MeshFilter:
@@ -118,5 +118,6 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: db942dc95bda5a149b7e9358de43a923, type: 3} m_Script: {fileID: 11500000, guid: db942dc95bda5a149b7e9358de43a923, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
activeMaterial: {fileID: 0} _activeMaterial: {fileID: 0}
passiveMaterial: {fileID: 0} _passiveMaterial: {fileID: 0}
scannableTag: ShapeScannable

Binary file not shown.

View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2 fileFormatVersion: 2
guid: 8a90a45cf0824d14fb5a2b9f59951013 guid: 9e16c783900f92a4c8942f7d34faa968
DefaultImporter: DefaultImporter:
externalObjects: {} externalObjects: {}
userData: userData:

View File

@@ -1,35 +1,19 @@
using TMPro; using TMPro;
using UnityEngine; using UnityEngine;
using Whisper;
using Whisper.Utils;
public class MicrophoneStand : MonoBehaviour public class MicrophoneStand : MonoBehaviour
{ {
public WhisperManager whisper;
public MicrophoneRecord microphoneRecord;
public string microphoneDevice;
public TextMeshProUGUI outputText; public TextMeshProUGUI outputText;
private WhisperStream stream;
private string textOutput;
public GameObject microphoneOffStatus; public GameObject microphoneOffStatus;
public GameObject microphoneOnStatus; public GameObject microphoneOnStatus;
public FMODWhisperBridge fmodWhisperBridge;
// Start is called before the first frame update // Start is called before the first frame update
async void Start() void Start()
{ {
Debug.Log("Mic devices: " + string.Join(", ", Microphone.devices)); fmodWhisperBridge.OnWhisperResultProcessed += OnWhisperResult;
Debug.Log("Using mic device: " + microphoneDevice);
microphoneRecord.SelectedMicDevice = microphoneDevice;
// This causes about 1 sec long freeze, has to be done once at the start of the game
microphoneRecord.StartRecord();
stream = await whisper.CreateStream(microphoneRecord);
stream.OnResultUpdated += OnWhisperResult;
//stream.StartStream();
microphoneOffStatus.SetActive(true); microphoneOffStatus.SetActive(true);
microphoneOnStatus.SetActive(false); microphoneOnStatus.SetActive(false);
@@ -50,10 +34,7 @@ public class MicrophoneStand : MonoBehaviour
{ {
microphoneOffStatus.SetActive(false); microphoneOffStatus.SetActive(false);
microphoneOnStatus.SetActive(true); microphoneOnStatus.SetActive(true);
fmodWhisperBridge.ActivateRecording();
stream.StartStream();
//microphoneRecord.StartRecord();
Debug.Log("Whisper stream started.");
} }
} }
@@ -64,27 +45,17 @@ public class MicrophoneStand : MonoBehaviour
{ {
microphoneOffStatus.SetActive(true); microphoneOffStatus.SetActive(true);
microphoneOnStatus.SetActive(false); microphoneOnStatus.SetActive(false);
fmodWhisperBridge.DeactivateRecording();
stream.StopStream();
//microphoneRecord.StopRecord();
textOutput = outputText.text;
} }
} }
private void OnWhisperResult(string result) private void OnWhisperResult(string result)
{ {
Debug.Log("Whisper result processed: " + result);
outputText.text = result; outputText.text = result;
} }
private void OnDestroy()
{
microphoneRecord.StopRecord();
Destroy(gameObject);
}
public string GetTextOutput() public string GetTextOutput()
{ {
return textOutput; return outputText.text;
} }
} }

View File

@@ -0,0 +1,39 @@
using System.IO;
using UnityEngine;
public class ConfigManager : MonoBehaviour
{
public GameConfig Config { get; private set; }
public static ConfigManager Instance { get; private set; }
private static string configPath => Path.Combine(Application.persistentDataPath, "config.json");
private void Awake()
{
Instance = this;
LoadConfig();
}
public void LoadConfig()
{
if (File.Exists(configPath))
{
string json = File.ReadAllText(configPath);
Config = JsonUtility.FromJson<GameConfig>(json);
}
else
{
// Create config with default values
Config = new GameConfig();
SaveConfig();
}
Debug.Log("Loaded config from: " + configPath);
}
public void SaveConfig()
{
string json = JsonUtility.ToJson(Config, true);
File.WriteAllText(configPath, json);
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 6a9ed26713a3c4c4f8f6be3bbd4af5c7
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,122 @@
using FMOD;
using FMODUnity;
using System.Runtime.InteropServices;
using UnityEngine;
public class FMODMicLoopback : MonoBehaviour
{
private uint LATENCY_MS = 50;
private uint DRIFT_MS = 1;
private uint samplesRecorded, samplesPlayed = 0;
private int nativeRate, nativeChannels = 0;
private uint recSoundLength = 0;
uint lastPlayPos = 0;
uint lastRecordPos = 0;
private uint driftThreshold = 0;
private uint desiredLatency = 0;
private uint adjustLatency = 0;
private int actualLatency = 0;
uint minRecordDelta = 0xFFFFFFFF;
private FMOD.CREATESOUNDEXINFO exInfo = new FMOD.CREATESOUNDEXINFO();
private FMOD.Sound recSound;
private FMOD.Channel channel;
// Start is called before the first frame update
void Start()
{
/*
Determine latency in samples.
*/
FMODUnity.RuntimeManager.CoreSystem.getRecordDriverInfo(0, out _, 0, out _, out nativeRate, out _, out nativeChannels, out _);
driftThreshold = (uint)(nativeRate * DRIFT_MS) / 1000;
desiredLatency = (uint)(nativeRate * LATENCY_MS) / 1000;
adjustLatency = desiredLatency;
actualLatency = (int)desiredLatency;
/*
Create user sound to record into, then start recording.
*/
exInfo.cbsize = Marshal.SizeOf(typeof(FMOD.CREATESOUNDEXINFO));
exInfo.numchannels = nativeChannels;
exInfo.format = FMOD.SOUND_FORMAT.PCM16;
exInfo.defaultfrequency = nativeRate;
exInfo.length = (uint)(nativeRate * sizeof(short) * nativeChannels);
FMODUnity.RuntimeManager.CoreSystem.createSound("", FMOD.MODE.LOOP_NORMAL | FMOD.MODE.OPENUSER, ref exInfo, out recSound);
FMODUnity.RuntimeManager.CoreSystem.recordStart(0, recSound, true);
recSound.getLength(out recSoundLength, FMOD.TIMEUNIT.PCM);
}
// Update is called once per frame
void Update()
{
/*
Determine how much has been recorded since we last checked
*/
uint recordPos = 0;
FMODUnity.RuntimeManager.CoreSystem.getRecordPosition(0, out recordPos);
uint recordDelta = (recordPos >= lastRecordPos) ? (recordPos - lastRecordPos) : (recordPos + recSoundLength - lastRecordPos);
lastRecordPos = recordPos;
samplesRecorded += recordDelta;
if (recordDelta != 0 && (recordDelta < minRecordDelta))
{
minRecordDelta = recordDelta; // Smallest driver granularity seen so far
adjustLatency = (recordDelta <= desiredLatency) ? desiredLatency : recordDelta; // Adjust our latency if driver granularity is high
}
/*
Delay playback until our desired latency is reached.
*/
if (!channel.hasHandle() && samplesRecorded >= adjustLatency)
{
FMODUnity.RuntimeManager.CoreSystem.getMasterChannelGroup(out FMOD.ChannelGroup mCG);
FMODUnity.RuntimeManager.CoreSystem.playSound(recSound, mCG, false, out channel);
}
/*
Determine how much has been played since we last checked.
*/
if (channel.hasHandle())
{
uint playPos = 0;
channel.getPosition(out playPos, FMOD.TIMEUNIT.PCM);
uint playDelta = (playPos >= lastPlayPos) ? (playPos - lastPlayPos) : (playPos + recSoundLength - lastPlayPos);
lastPlayPos = playPos;
samplesPlayed += playDelta;
// Compensate for any drift.
int latency = (int)(samplesRecorded - samplesPlayed);
actualLatency = (int)((0.97f * actualLatency) + (0.03f * latency));
int playbackRate = nativeRate;
if (actualLatency < (int)(adjustLatency - driftThreshold))
{
// Playback position is catching up to the record position, slow playback down by 2%
playbackRate = nativeRate - (nativeRate / 50);
}
else if (actualLatency > (int)(adjustLatency + driftThreshold))
{
// Playback is falling behind the record position, speed playback up by 2%
playbackRate = nativeRate + (nativeRate / 50);
}
channel.setFrequency((float)playbackRate);
}
}
private void OnDestroy()
{
recSound.release();
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 12623af3ff8b7df4abbaacd753e43555
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,238 @@

using System;
using System.Runtime.InteropServices;
using UnityEngine;
using FMOD;
using FMODUnity;
using Whisper; // WhisperManager, WhisperStream, WhisperResult
using Whisper.Utils; // AudioChunk
/// <summary>
/// Capture microphone with FMOD and feed chunks to WhisperStream (no Unity Microphone).
/// Also (optionally) plays the recorded sound back via FMOD loopback.
/// </summary>
public class FMODWhisperBridge : MonoBehaviour
{
[Header("Whisper")]
[SerializeField] private WhisperManager whisper; // assign in Inspector
[SerializeField] private bool useVadInStream = false; // let WhisperStream do VAD or not
[Header("FMOD capture")]
[Tooltip("Recording device index (0 = default)")]
public int recordDriverId = 0;
[Tooltip("Set 48000 on Quest; falls back to device rate automatically")]
public int desiredSampleRate = 48000;
[Tooltip("Mono recommended for Whisper")]
public int channels = 1;
[Range(1, 10)] public int bufferLengthSec = 5;
[Header("Loopback (monitor your voice)")]
public bool playLoopback = true;
[Range(0f, 2f)] public float loopbackVolume = 1.0f;
public delegate void OnWhisperResultProcessedDelegate(string result);
public event OnWhisperResultProcessedDelegate OnWhisperResultProcessed;
// FMOD
private FMOD.System _core;
private Sound _recSound;
private Channel _playChannel;
private ChannelGroup _masterGroup;
private uint _soundPcmLength; // in samples
private int _nativeRate;
private int _nativeChannels;
// ring-buffer tracking
private uint _lastRecordPos = 0;
// Whisper
private WhisperStream _stream;
private bool _streamStarted;
// temp conversion buffer
private float[] _floatTmp = new float[0];
private bool isRecordingActivated = false;
private void Awake()
{
if (!whisper) whisper = FindObjectOfType<WhisperManager>();
_core = RuntimeManager.CoreSystem; // FMOD core system
}
private async void Start()
{
// Query device info to get native rate/channels.
// (FMOD: getRecordDriverInfo gives you system rate & speaker mode)
string name;
Guid guid;
SPEAKERMODE sm;
int smChannels;
DRIVER_STATE driverState;
// signature: getRecordDriverInfo(id, out name, nameLen, out guid, out systemrate, out speakermode, out speakermodechannels, out driverState)
_core.getRecordDriverInfo(recordDriverId, out name, 256, out guid, out _nativeRate, out sm, out smChannels, out driverState);
_nativeChannels = channels > 0 ? channels : smChannels;
UnityEngine.Debug.Log($"[FMOD→Whisper] Using input device #{recordDriverId}: \"{name}\" rate={_nativeRate} ch={_nativeChannels}");
// Build a user sound buffer that FMOD will fill (OPENUSER | LOOP_NORMAL).
CREATESOUNDEXINFO ex = new CREATESOUNDEXINFO
{
cbsize = Marshal.SizeOf(typeof(CREATESOUNDEXINFO)),
numchannels = _nativeChannels,
defaultfrequency = (_nativeRate > 0) ? _nativeRate : desiredSampleRate,
format = SOUND_FORMAT.PCM16,
length = (uint)(((_nativeRate > 0 ? _nativeRate : desiredSampleRate) * _nativeChannels) * sizeof(short)) // seconds=1 (we loop)
};
_core.createSound("", MODE.OPENUSER | MODE.LOOP_NORMAL | MODE.CREATESAMPLE, ref ex, out _recSound);
_recSound.getLength(out _soundPcmLength, TIMEUNIT.PCM);
// Start FMOD recording into that sound (looping ring buffer).
_core.recordStart(recordDriverId, _recSound, true);
UnityEngine.Debug.Log("[FMOD→Whisper] Recording started.");
// Optional loopback playback using FMOD (plays same sound ring buffer).
_core.getMasterChannelGroup(out _masterGroup);
if (playLoopback)
{
_core.playSound(_recSound, _masterGroup, false, out _playChannel);
_playChannel.setMode(MODE._2D);
_playChannel.setVolume(loopbackVolume);
UnityEngine.Debug.Log("[FMOD→Whisper] Loopback playback started.");
}
// Create Whisper stream WITHOUT MicrophoneRecord, just from (freq, channels).
// We'll push AudioChunk manually.
// NOTE: WhisperStreams sliding window is governed by managers stepSec/keepSec/lengthSec.
_stream = await whisper.CreateStream(ex.defaultfrequency, _nativeChannels);
_stream.OnResultUpdated += (txt) =>
{
//OnWhisperResultProcessed?.Invoke(txt);
//UnityEngine.Debug.Log($"[Whisper] result updated: {txt}");
};
_stream.OnSegmentUpdated += (seg) =>
{
OnWhisperResultProcessed?.Invoke(seg.Result);
//UnityEngine.Debug.Log($"[Whisper] Seg finished: {seg.Result}");
};
// If you want Whisper to respect VAD, enable in manager or set useVad (manager controls stream params).
whisper.useVad = useVadInStream;
_stream.StartStream();
_streamStarted = true;
// prepare temp arrays roughly 100ms of audio
EnsureTmpCapacity((ex.defaultfrequency / 10) * _nativeChannels);
}
private void Update()
{
if (!isRecordingActivated) return;
if (_core.handle != IntPtr.Zero) _core.update();
if (!_streamStarted || !_recSound.hasHandle()) return;
// How many samples recorded since last frame?
uint recPos;
_core.getRecordPosition(recordDriverId, out recPos);
uint deltaSamples = (recPos >= _lastRecordPos)
? (recPos - _lastRecordPos)
: (recPos + _soundPcmLength - _lastRecordPos);
if (deltaSamples == 0) return;
// Well read that region (16-bit) and convert to float[] [-1..1].
// Calculate byte range to lock in sound buffer
uint bytesToRead = deltaSamples * (uint)_nativeChannels * 2; // 16-bit = 2 bytes
uint startBytes = _lastRecordPos * (uint)_nativeChannels * 2;
IntPtr p1, p2;
uint len1, len2;
// Lock can wrap — FMOD splits into p1/p2.
_recSound.@lock(startBytes, bytesToRead, out p1, out p2, out len1, out len2);
try
{
// Convert both parts to float and push to Whisper
if (len1 > 0) CopyPcm16ToFloatAndFeed(p1, len1);
if (len2 > 0) CopyPcm16ToFloatAndFeed(p2, len2);
}
finally
{
_recSound.unlock(p1, p2, len1, len2);
}
_lastRecordPos = recPos;
}
public void ActivateRecording()
{
isRecordingActivated = true;
}
public void DeactivateRecording()
{
isRecordingActivated = false;
}
private void CopyPcm16ToFloatAndFeed(IntPtr src, uint byteLen)
{
int samples = (int)(byteLen / 2); // 2 bytes per sample
EnsureTmpCapacity(samples);
// Marshal the 16-bit PCM into managed space
// We pin a short[] overlay to avoid copying twice
int shorts = samples;
int byteCount = (int)byteLen;
// Use Marshal.Copy into a short[] then convert to float[-1..1]
// (You can also unsafe copy for speed if needed.)
EnsureShortOverlay(shorts, out short[] sBuf);
Marshal.Copy(src, sBuf, 0, shorts);
for (int i = 0; i < shorts; i++)
{
// 32768f avoids clipping at -32768
_floatTmp[i] = Mathf.Clamp(sBuf[i] / 32768f, -1f, 1f);
}
// Build a chunk for WhisperStream; with VAD off, IsVoiceDetected=true is fine.
var chunk = new AudioChunk
{
Data = _floatTmp.AsSpan(0, shorts).ToArray(),
Frequency = (_nativeRate > 0) ? _nativeRate : desiredSampleRate,
Channels = _nativeChannels,
IsVoiceDetected = true
};
_stream.AddToStream(chunk);
}
private short[] _shortOverlay;
private void EnsureShortOverlay(int samples, out short[] buf)
{
if (_shortOverlay == null || _shortOverlay.Length < samples)
_shortOverlay = new short[Mathf.NextPowerOfTwo(samples)];
buf = _shortOverlay;
}
private void EnsureTmpCapacity(int samples)
{
if (_floatTmp == null || _floatTmp.Length < samples)
_floatTmp = new float[Mathf.NextPowerOfTwo(samples)];
}
private void OnDisable()
{
if (_streamStarted)
{
_stream.StopStream();
_streamStarted = false;
}
if (_playChannel.hasHandle()) { _playChannel.stop(); _playChannel.clearHandle(); }
if (_recSound.hasHandle()) { _core.recordStop(recordDriverId); _recSound.release(); _recSound.clearHandle(); }
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: a264cb5321d9f6741a2625b8c63c34a7
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,8 @@
[System.Serializable]
public class GameConfig
{
public string invokeAIUrl = "http://192.168.0.53:9090";
public string invokeAIModelKey = "81d45960-08a0-4b8c-a48b-e7d73b21bfe2";
public string TrellisUrl = "http://192.168.0.53:7960";
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 5412d5465648e6c40bfe932c9098194b
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -14,9 +14,7 @@ public class InvokeAiClient : MonoBehaviour
{ {
public static InvokeAiClient Instance { get; private set; } public static InvokeAiClient Instance { get; private set; }
public string INVOKEAI_BASE_URL;
public string DEFAULT_QUEUE_ID = "default"; public string DEFAULT_QUEUE_ID = "default";
public string MODEL_KEY;
public string promptSuffix = ", single object, front and side fully visible, realistic style, plain neutral background, clear details, soft studio lighting, true-to-scale"; public string promptSuffix = ", single object, front and side fully visible, realistic style, plain neutral background, clear details, soft studio lighting, true-to-scale";
private HttpClient httpClient; private HttpClient httpClient;
@@ -27,7 +25,7 @@ public class InvokeAiClient : MonoBehaviour
{ {
Timeout = TimeSpan.FromSeconds(120) Timeout = TimeSpan.FromSeconds(120)
}; };
httpClient.BaseAddress = new Uri(INVOKEAI_BASE_URL); httpClient.BaseAddress = new Uri(ConfigManager.Instance.Config.invokeAIUrl);
Instance = this; Instance = this;
} }
@@ -520,13 +518,14 @@ public class InvokeAiClient : MonoBehaviour
public async Task<byte[]> GenerateImage(string prompt) public async Task<byte[]> GenerateImage(string prompt)
{ {
string modelKey = ConfigManager.Instance.Config.invokeAIModelKey;
string refinedPrompt = prompt + promptSuffix; string refinedPrompt = prompt + promptSuffix;
JObject args = new JObject() JObject args = new JObject()
{ {
["prompt"] = refinedPrompt, ["prompt"] = refinedPrompt,
["width"] = 512, ["width"] = 512,
["height"] = 512, ["height"] = 512,
["model_key"] = MODEL_KEY, ["model_key"] = modelKey,
}; };
UnityEngine.Debug.Log("Starting image generation..."); UnityEngine.Debug.Log("Starting image generation...");

View File

@@ -2,7 +2,7 @@
using UnityEngine; using UnityEngine;
[RequireComponent(typeof(AudioSource))] [RequireComponent(typeof(AudioSource))]
public class MicrophoneTesting : MonoBehaviour public class MicLoopback : MonoBehaviour
{ {
[Header("Mic settings")] [Header("Mic settings")]
[Tooltip("Leave empty for default device")] [Tooltip("Leave empty for default device")]

View File

@@ -67,17 +67,21 @@ public class ShapeDetectionMinigameController : MonoBehaviour
private void InitializeSpawnedObject(GameObject spawnedObject) private void InitializeSpawnedObject(GameObject spawnedObject)
{ {
Rigidbody rigidbody = spawnedObject.AddComponent<Rigidbody>(); GameObject spawnedObjectParent = new GameObject("SpawnedModelParent");
spawnedObjectParent.transform.parent = modelSpawnPoint;
spawnedObjectParent.transform.position = modelSpawnPoint.transform.position;
Rigidbody rigidbody = spawnedObjectParent.AddComponent<Rigidbody>();
rigidbody.isKinematic = true; rigidbody.isKinematic = true;
//spawnedObject.AddComponent<NetworkObject>(); //spawnedObject.AddComponent<NetworkObject>();
//spawnedObject.AddComponent<NetworkTransform>(); //spawnedObject.AddComponent<NetworkTransform>();
spawnedObject.transform.parent = modelSpawnPoint;
spawnedObject.transform.position = modelSpawnPoint.position; MeshCollider spawnedObjectCollider = spawnedObject.GetComponent<MeshCollider>();
spawnedObjectCollider.convex = false;
spawnedObject.transform.parent = spawnedObjectParent.transform;
spawnedObject.transform.position = spawnedObjectParent.transform.position;
spawnedObject.tag = shapeScannerTag; spawnedObject.tag = shapeScannerTag;
spawnedObject.SetActive(true); spawnedObjectParent.AddComponent<TwoHandScaleGrabInteractable>();
spawnedObject.AddComponent<XRGrabInteractable>();
} }
} }

View File

@@ -1,68 +1,57 @@
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using TMPro; using TMPro;
using UnityEngine; using UnityEngine;
[System.Serializable] [Serializable]
public class BoolRow public class BoolRow
{ {
public bool[] cells; public List<bool> cells = new List<bool>();
}
[Serializable]
public class ShapeScannerConfiguration
{
public int columns = 6;
public float requiredCorrectPercentage = 80.0f;
public List<BoolRow> rows = new List<BoolRow>();
} }
public class ShapeScanner : MonoBehaviour public class ShapeScanner : MonoBehaviour
{ {
public List<BoolRow> configuration; public List<ShapeScannerConfiguration> configurations = new List<ShapeScannerConfiguration>();
public ShapeScannerRay rayPrefab; public ShapeScannerRay rayPrefab;
public Transform raySpawnCorner1; public Transform raySpawnCorner1;
public Transform raySpawnCorner2; public Transform raySpawnCorner2;
public Transform rayParent; public Transform rayParent;
public TextMeshProUGUI currentConfigurationDisplay;
public TextMeshProUGUI correctPercentageDisplay;
public Material requiredAndActive; public Material requiredAndActive;
public Material requiredAndPassive; public Material requiredAndPassive;
public Material notRequiredAndActive; public Material notRequiredAndActive;
public Material notRequiredAndPassive; public Material notRequiredAndPassive;
public TextMeshProUGUI displayText; private List<GameObject> existingRays;
private float raySpawnDistanceX;
private float raySpawnDistanceZ;
private int currentConfiguration;
private int rayCount; private int rayCount;
private int correctRayStates; private int correctRayStates;
private void Awake()
{
correctRayStates = 0;
}
// Start is called before the first frame update // Start is called before the first frame update
void Start() void Start()
{ {
float raySpawnDistanceX = raySpawnCorner2.localPosition.x - raySpawnCorner1.localPosition.x; existingRays = new List<GameObject>();
float raySpawnDistanceZ = raySpawnCorner2.localPosition.z - raySpawnCorner1.localPosition.z; raySpawnDistanceX = raySpawnCorner2.localPosition.x - raySpawnCorner1.localPosition.x;
raySpawnDistanceZ = raySpawnCorner2.localPosition.z - raySpawnCorner1.localPosition.z;
int rayRowCount = configuration.Count; currentConfiguration = 0;
for (int i = 0; i < rayRowCount; i++) InitializeConfiguration();
{
float rayPosX = raySpawnCorner1.localPosition.x + i * raySpawnDistanceX / (rayRowCount - 1);
for (int j = 0; j < rayRowCount; j++)
{
rayCount++;
// Local position
float rayPosZ = raySpawnCorner1.localPosition.z + j * raySpawnDistanceZ / (rayRowCount - 1);
Vector3 rayPos = new Vector3(rayPosX, 0, rayPosZ);
ShapeScannerRay ray = Instantiate(rayPrefab, rayParent);
ray.transform.localPosition = rayPos;
bool rayCollisionRequired = configuration[i].cells[j];
if (rayCollisionRequired)
{
ray.Initialize(this, rayCollisionRequired, requiredAndActive, requiredAndPassive);
} else
{
ray.Initialize(this, rayCollisionRequired, notRequiredAndActive, notRequiredAndPassive);
IncrementCorrectRayCount();
}
}
}
} }
// Update is called once per frame // Update is called once per frame
@@ -71,21 +60,82 @@ public class ShapeScanner : MonoBehaviour
} }
private void InitializeConfiguration()
{
// Delete all existing rays first
foreach (GameObject ray in existingRays)
{
Destroy(ray);
}
ShapeScannerConfiguration configuration = configurations[currentConfiguration];
int rayRowCount = configuration.rows.Count;
for (int i = 0; i < rayRowCount; i++)
{
float rayPosX = raySpawnCorner1.localPosition.x + i * raySpawnDistanceX / (rayRowCount - 1);
for (int j = 0; j < rayRowCount; j++)
{
// Local position
float rayPosZ = raySpawnCorner1.localPosition.z + j * raySpawnDistanceZ / (rayRowCount - 1);
Vector3 rayPos = new Vector3(rayPosX, 0, rayPosZ);
ShapeScannerRay ray = Instantiate(rayPrefab, rayParent);
ray.transform.localPosition = rayPos;
existingRays.Add(ray.gameObject);
bool rayCollisionRequired = configuration.rows[i].cells[j];
if (rayCollisionRequired)
{
ray.Initialize(this, rayCollisionRequired, requiredAndActive, requiredAndPassive);
}
else
{
ray.Initialize(this, rayCollisionRequired, notRequiredAndActive, notRequiredAndPassive);
}
}
}
// Count total rays and required collision rays
rayCount = configuration.rows.SelectMany(row => row.cells).Count();
correctRayStates = configuration.rows.SelectMany(row => row.cells).Count(cell => !cell);
UpdateDisplay(calculateCorrectPercentage());
}
private float calculateCorrectPercentage()
{
return Mathf.RoundToInt((float)correctRayStates / rayCount * 100);
}
public void IncrementCorrectRayCount() public void IncrementCorrectRayCount()
{ {
correctRayStates++; correctRayStates++;
UpdateDisplay(); float correctPercentage = calculateCorrectPercentage();
if (correctPercentage >= configurations[currentConfiguration].requiredCorrectPercentage)
{
UpdateCurrentConfigurationDisplay(currentConfiguration + 1);
if (currentConfiguration + 1 < configurations.Count)
{
currentConfiguration++;
InitializeConfiguration();
} else
{
Debug.Log("Shape checker completed");
}
}
UpdateDisplay(correctPercentage);
} }
public void DecrementCorrectRayCount() public void DecrementCorrectRayCount()
{ {
correctRayStates--; correctRayStates--;
UpdateDisplay(); UpdateDisplay(calculateCorrectPercentage());
} }
private void UpdateDisplay() private void UpdateDisplay(float percentage)
{ {
int percentage = Mathf.RoundToInt((float) correctRayStates / rayCount * 100); correctPercentageDisplay.text = Mathf.Round(percentage).ToString() + " %";
displayText.text = percentage.ToString() + " %"; }
private void UpdateCurrentConfigurationDisplay(int confNumber)
{
currentConfigurationDisplay.text = confNumber.ToString() + " / " + configurations.Count;
} }
} }

View File

@@ -0,0 +1,88 @@
using UnityEditor;
using UnityEngine;
using UnityEngine.XR.Interaction.Toolkit;
public class TwoHandScaleGrabInteractable : XRGrabInteractable
{
[Header("Two-Hand Scaling")]
public float minScale = 0.1f;
public float maxScale = 5f;
public float scaleLerpSpeed = 0f;
private Transform scalingTransform;
private float initialHandsDistance;
private Vector3 initialScale;
private bool twoHandsGrabbing;
void Start()
{
selectMode = InteractableSelectMode.Multiple;
useDynamicAttach = true;
reinitializeDynamicAttachEverySingleGrab = false;
scalingTransform = transform.GetChild(0);
}
protected override void OnSelectEntered(SelectEnterEventArgs args)
{
base.OnSelectEntered(args);
UpdateTwoHandState();
}
protected override void OnSelectExited(SelectExitEventArgs args)
{
Debug.Log("current local scale: " + transform.localScale);
base.OnSelectExited(args);
UpdateTwoHandState();
}
void UpdateTwoHandState()
{
if (interactorsSelecting.Count >= 2)
{
var a = interactorsSelecting[0];
var b = interactorsSelecting[1];
var pA = a.GetAttachTransform(this).position;
var pB = b.GetAttachTransform(this).position;
initialHandsDistance = Vector3.Distance(pA, pB);
initialScale = scalingTransform.localScale;
twoHandsGrabbing = initialHandsDistance > 0.0001f;
}
else
{
twoHandsGrabbing = false;
}
}
public override void ProcessInteractable(XRInteractionUpdateOrder.UpdatePhase updatePhase)
{
base.ProcessInteractable(updatePhase);
if (!twoHandsGrabbing || interactorsSelecting.Count < 2 || initialHandsDistance <= 0.0001f)
return;
var a = interactorsSelecting[0];
var b = interactorsSelecting[1];
var pA = a.GetAttachTransform(this).position;
var pB = b.GetAttachTransform(this).position;
float current = Vector3.Distance(pA, pB);
if (current <= 0.0001f)
return;
float ratio = current / initialHandsDistance;
Debug.Log("distance ratio: " + ratio);
Vector3 desired = initialScale * ratio;
float uniform = Mathf.Clamp(desired.x, minScale, maxScale);
desired = new Vector3(uniform, uniform, uniform);
if (scaleLerpSpeed > 0f)
desired = Vector3.Lerp(transform.localScale, desired, Time.deltaTime * scaleLerpSpeed);
scalingTransform.localScale = desired;
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 22c2ea15df0386746b1b1bc7b58bced7
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -2,7 +2,6 @@ using Newtonsoft.Json.Linq;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Net.Http; using System.Net.Http;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using UnityEngine; using UnityEngine;
@@ -10,16 +9,16 @@ public class TrellisClient : MonoBehaviour
{ {
public static TrellisClient Instance { get; private set; } public static TrellisClient Instance { get; private set; }
public string TRELLIS_BASE_URL;
private HttpClient httpClient; private HttpClient httpClient;
private void Awake() private void Awake()
{ {
httpClient = new HttpClient(); httpClient = new HttpClient();
httpClient.BaseAddress = new Uri(TRELLIS_BASE_URL); httpClient.BaseAddress = new Uri(ConfigManager.Instance.Config.TrellisUrl);
Instance = this; Instance = this;
TestConnection();
} }
// Start is called before the first frame update // Start is called before the first frame update
@@ -34,12 +33,19 @@ public class TrellisClient : MonoBehaviour
} }
private async void TestConnection()
{
var statusResp = await httpClient.GetAsync("status");
Debug.Log("Trellis status: " + statusResp.StatusCode);
}
public async Task<byte[]> GenerateModel( public async Task<byte[]> GenerateModel(
string imageBase64, string imageBase64,
int seed = 42,
int pollIntervalMs = 1000) int pollIntervalMs = 1000)
{ {
int seed = UnityEngine.Random.Range(0, 999999);
// --- Set generation parameters (form-encoded, like Python requests.post(data=params)) --- // --- Set generation parameters (form-encoded, like Python requests.post(data=params)) ---
var form = new Dictionary<string, string> var form = new Dictionary<string, string>
{ {

View File

@@ -0,0 +1,14 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!200 &20000000
ShaderVariantCollection:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_Name: glTFPbrMetallicRoughness
m_Shaders:
- first: {fileID: -6465566751694194690, guid: b9d29dfa1474148e792ac720cbd45122,
type: 3}
second:
variants: []

View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: b852f2aeb9fb71a4aaab9e2d500dc3dc
NativeFormatImporter:
externalObjects: {}
mainObjectFileID: 20000000
userData:
assetBundleName:
assetBundleVariant:

Binary file not shown.

Binary file not shown.

View File

@@ -169,7 +169,7 @@
{ {
"type": "System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", "type": "System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"key": "ShapeBuilder.ActiveShapeIndex", "key": "ShapeBuilder.ActiveShapeIndex",
"value": "{\"m_Value\":2}" "value": "{\"m_Value\":11}"
}, },
{ {
"type": "System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", "type": "System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
@@ -189,7 +189,7 @@
{ {
"type": "UnityEngine.Vector3, UnityEngine.CoreModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", "type": "UnityEngine.Vector3, UnityEngine.CoreModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null",
"key": "ShapeBuilder.LastSize", "key": "ShapeBuilder.LastSize",
"value": "{\"m_Value\":{\"x\":49.08480453491211,\"y\":0.10000000149011612,\"z\":-30.85292625427246}}" "value": "{\"m_Value\":{\"x\":1.3506088256835938,\"y\":0.41489124298095705,\"z\":1.3857231140136719}}"
}, },
{ {
"type": "UnityEngine.Vector3, UnityEngine.CoreModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", "type": "UnityEngine.Vector3, UnityEngine.CoreModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null",
@@ -246,6 +246,21 @@
"key": "ShapeBuilder.Plane", "key": "ShapeBuilder.Plane",
"value": "{}" "value": "{}"
}, },
{
"type": "UnityEngine.ProBuilder.Shapes.Shape, Unity.ProBuilder, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null",
"key": "ShapeBuilder.Prism",
"value": "{}"
},
{
"type": "UnityEngine.ProBuilder.Shapes.Shape, Unity.ProBuilder, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null",
"key": "ShapeBuilder.Torus",
"value": "{}"
},
{
"type": "UnityEngine.ProBuilder.Shapes.Shape, Unity.ProBuilder, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null",
"key": "ShapeBuilder.Pipe",
"value": "{}"
},
{ {
"type": "System.Single, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", "type": "System.Single, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"key": "uv.uvEditorGridSnapIncrement", "key": "uv.uvEditorGridSnapIncrement",