henrisel/DeltaVR3DModelGeneration
1
0
Fork 0
forked from cgvr/DeltaVR
Code Pull Requests Activity

add TrellisClient, use it for model generation

Browse Source
This commit is contained in:
henrisel
2025-12-22 15:07:23 +02:00
parent 1b3b3db1bf
commit 7bc58a48d0
8 changed files with 191 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs
View File

@@ -5,7 +5,6 @@ using FishNet.Object.Synchronizing;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using TMPro;
using Unity.XR.CoreUtils;
using UnityEngine;
@@ -123,7 +122,7 @@ public class ArcheryRange : NetworkBehaviour
}
}
async private void SpawnTarget()
private void SpawnTarget()
{
if (!IsServer) return;
@@ -133,23 +132,21 @@ public class ArcheryRange : NetworkBehaviour
Random.Range(minRandomOffset.z, maxRandomOffset.z)
);
var target = await SpawnTarget(randomPos);
var target = SpawnTarget(randomPos);
_targets.Add(target);
}
async private Task<ArcheryTarget> SpawnTarget(Vector3 randomPos)
private ArcheryTarget SpawnTarget(Vector3 randomPos)
{
GameObject targetObject;
if (modelGenerationBox.LastModelPath == null)
if (modelGenerationBox.GeneratedModel == null)
{
// spawn default UFO
targetObject = Instantiate(targetPrefab, randomPos, Quaternion.identity, null);
} else
{
// spawn generated model
targetObject = await PipelineManager.Instance.SpawnModel(modelGenerationBox.LastModelPath);
targetObject.transform.position = randomPos;
targetObject.transform.rotation = Quaternion.identity;
targetObject = Instantiate(modelGenerationBox.GeneratedModel, randomPos, Quaternion.identity, null);
InitializeArcherytargetObject(targetObject);
}
@@ -165,7 +162,7 @@ public class ArcheryRange : NetworkBehaviour
ArcheryTarget archeryTarget = targetObject.AddComponent<ArcheryTarget>();
archeryTarget.pointsText = archeryTargetPointsText;
Rigidbody rigidbody = targetObject.AddComponent<Rigidbody>();
Rigidbody rigidbody = targetObject.GetComponent<Rigidbody>();
rigidbody.useGravity = false;
rigidbody.isKinematic = true;

4
Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab
View File

@@ -300,4 +300,6 @@ MonoBehaviour:
inactiveMaterial: {fileID: 2100000, guid: 707a698b0ec80454a8c68700bca72941, type: 2}
loadingMaterial: {fileID: 2100000, guid: 33390c6f2eb32df47809c60975868a0c, type: 2}
voiceTranscriptionTestBox: {fileID: 0}
UIImage: {fileID: 6899506685386251279}
imageDisplay: {fileID: 6899506685386251279}
promptSuffix: ', single object, front and side fully visible, realistic style,
plain neutral background, clear details, soft studio lighting, true-to-scale'

BIN
Assets/_PROJECT/Scenes/DeltaBuilding_base.unity LFS
View File

Binary file not shown.

22
Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs
View File

@@ -9,14 +9,16 @@ public class ImageGenerationBox : MonoBehaviour
public Material loadingMaterial;
public VoiceTranscriptionBox voiceTranscriptionTestBox;
public Image UIImage;
public Image imageDisplay;
public Texture2D LastTexture { get; private set; }
public string promptSuffix = ", single object, front and side fully visible, realistic style, plain neutral background, clear details, soft studio lighting, true-to-scale";
private MeshRenderer meshRenderer;
private bool isLoading;
// Start is called before the first frame update
void Start()
// Start is called before the first frame update
void Start()
{
meshRenderer = GetComponent<MeshRenderer>();
}
@@ -42,15 +44,16 @@ public class ImageGenerationBox : MonoBehaviour
meshRenderer.material = loadingMaterial;
byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(refinedPrompt);
Sprite sprite = CreateSprite(imageBytes);
UIImage.sprite = sprite;
LastTexture = CreateTexture(imageBytes);
Sprite sprite = CreateSprite(LastTexture);
imageDisplay.sprite = sprite;
isLoading = false;
meshRenderer.material = inactiveMaterial;
}
}
private Sprite CreateSprite(byte[] imageBytes)
private Texture2D CreateTexture(byte[] imageBytes)
{
var tex = new Texture2D(2, 2, TextureFormat.RGBA32, false);
// ImageConversion.LoadImage returns bool (true = success)
@@ -59,10 +62,15 @@ public class ImageGenerationBox : MonoBehaviour
Destroy(tex);
throw new InvalidOperationException("Failed to decode image bytes into Texture2D.");
}
tex.filterMode = FilterMode.Bilinear;
tex.wrapMode = TextureWrapMode.Clamp;
return tex;
}
private Sprite CreateSprite(Texture2D tex)
{
var sprite = Sprite.Create(
tex,
new Rect(0, 0, tex.width, tex.height),

23
Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs
View File

@@ -1,3 +1,4 @@
using System;
using Unity.XR.CoreUtils;
using UnityEngine;
@@ -7,18 +8,12 @@ public class ModelGenerationBox : MonoBehaviour
public Material loadingMaterial;
public Transform modelSpawnPoint;
public VoiceTranscriptionBox voiceTranscriptionTestBox;
public ImageGenerationBox imageGenerationBox;
private MeshRenderer meshRenderer;
private bool isLoading;
private string lastModelPath;
public string LastModelPath
{
get {
return lastModelPath;
}
}
public GameObject GeneratedModel { get; private set; }
// Start is called before the first frame update
void Start()
@@ -40,18 +35,18 @@ public class ModelGenerationBox : MonoBehaviour
XROrigin playerOrigin = other.GetComponent<XROrigin>();
if (controller != null || playerOrigin != null)
{
string inputPrompt = voiceTranscriptionTestBox.LastTextOutput;
isLoading = true;
meshRenderer.material = loadingMaterial;
string modelPath = await PipelineManager.Instance.GenerateModelAsync(inputPrompt);
lastModelPath = modelPath;
GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(modelPath);
Texture2D inputTexture = imageGenerationBox.LastTexture;
string encodedTexture = Convert.ToBase64String(inputTexture.EncodeToJPG());
byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(encodedModel);
spawnedObject.AddComponent<Rigidbody>();
spawnedObject.transform.parent = modelSpawnPoint;
spawnedObject.transform.position = modelSpawnPoint.position;
GeneratedModel = spawnedObject;
isLoading = false;
meshRenderer.material = inactiveMaterial;

60
Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs
View File

@@ -25,6 +25,10 @@ public class PipelineManager : MonoBehaviour
}
/**
* Generate model by starting a new Python subprocess
* NOT USED ANYMORE
**/
public async Task<string> GenerateModelAsync(string inputPrompt)
{
return await Task.Run(() =>
@@ -76,6 +80,11 @@ public class PipelineManager : MonoBehaviour
});
}
/**
* Spawn model stored on disk
* NOT USED ANYMORE
*
**/
public async Task<GameObject> SpawnModel(string modelPath)
{
var gltf = new GltfImport();
@@ -83,23 +92,42 @@ public class PipelineManager : MonoBehaviour
if (loadSuccess)
{
string objectName = Path.GetFileName(modelPath);
GameObject spawningParent = new GameObject("Parent-" + objectName);
bool spawnSuccess = await gltf.InstantiateMainSceneAsync(spawningParent.transform);
if (spawnSuccess)
{
Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform;
GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject;
MeshCollider collider = spawnedObjectBody.AddComponent<MeshCollider>();
collider.convex = true;
MeshRenderer renderer = spawnedObjectBody.GetComponent<MeshRenderer>();
renderer.material.SetFloat("metallicFactor", 0);
spawnedObjectBody.name = objectName;
return spawnedObjectBody;
}
return await SpawnModel(gltf, objectName);
}
throw new System.Exception("Failed to spawn GameObject from model" + modelPath);
throw new System.Exception("Failed to load GameObject from model" + modelPath);
}
public async Task<GameObject> SpawnModel(byte[] modelBinary)
{
var gltf = new GltfImport();
bool loadSuccess = await gltf.Load(modelBinary);
if (loadSuccess)
{
return await SpawnModel(gltf, "GeneratedModel");
}
throw new System.Exception("Failed to load GameObject from binary!");
}
public async Task<GameObject> SpawnModel(GltfImport gltfImport, string objectName)
{
GameObject spawningParent = new GameObject("Parent-" + objectName);
bool spawnSuccess = await gltfImport.InstantiateMainSceneAsync(spawningParent.transform);
if (spawnSuccess)
{
Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform;
GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject;
MeshCollider collider = spawnedObjectBody.AddComponent<MeshCollider>();
collider.convex = true;
MeshRenderer renderer = spawnedObjectBody.GetComponent<MeshRenderer>();
renderer.material.SetFloat("metallicFactor", 0);
spawnedObjectBody.name = objectName;
return spawnedObjectBody;
}
throw new System.Exception("Failed to spawn GameObject " + objectName);
}
}

101
Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs Normal file
View File

@@ -0,0 +1,101 @@
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
public class TrellisClient : MonoBehaviour
{
public static TrellisClient Instance { get; private set; }
public string TRELLIS_BASE_URL;
private HttpClient httpClient;
private void Awake()
{
httpClient = new HttpClient();
httpClient.BaseAddress = new Uri(TRELLIS_BASE_URL);
Instance = this;
}
// Start is called before the first frame update
void Start()
{
}
// Update is called once per frame
void Update()
{
}
public async Task<byte[]> GenerateModel(
string imageBase64,
int seed = 42,
int pollIntervalMs = 1000)
{
// --- Set generation parameters (form-encoded, like Python requests.post(data=params)) ---
var form = new Dictionary<string, string>
{
["image_base64"] = imageBase64,
["seed"] = seed.ToString(),
["ss_guidance_strength"] = "7.5",
["ss_sampling_steps"] = "10",
["slat_guidance_strength"] = "7.5",
["slat_sampling_steps"] = "10",
["mesh_simplify_ratio"] = "0.99",
["texture_size"] = "1024",
["texture_opt_total_steps"] = "1000",
["output_format"] = "glb"
};
Debug.Log("Starting Trellis generation...");
using (var content = new FormUrlEncodedContent(form))
using (var startResp = await httpClient.PostAsync("generate_no_preview", content))
{
startResp.EnsureSuccessStatusCode();
}
// --- Poll /status until COMPLETE or FAILED ---
while (true)
{
using (var statusResp = await httpClient.GetAsync("status"))
{
statusResp.EnsureSuccessStatusCode();
var json = await statusResp.Content.ReadAsStringAsync();
var status = JObject.Parse(json);
var progress = status.Value<int?>("progress") ?? 0;
var state = status.Value<string>("status") ?? "UNKNOWN";
Debug.Log($"TRELLIS progress: {progress}% (status: {state})");
if (string.Equals(state, "COMPLETE", StringComparison.OrdinalIgnoreCase))
break;
if (string.Equals(state, "FAILED", StringComparison.OrdinalIgnoreCase))
{
var msg = status.Value<string>("message") ?? "Generation failed.";
throw new InvalidOperationException($"Trellis generation failed: {msg}");
}
}
await Task.Delay(pollIntervalMs);
}
// --- Download the model binary ---
Debug.Log("Downloading model...");
using (var downloadResponse = await httpClient.GetAsync("download/model"))
{
downloadResponse.EnsureSuccessStatusCode();
var bytes = await downloadResponse.Content.ReadAsByteArrayAsync();
return bytes;
}
}
}

11
Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 905247c726697644cbf0f39558db46bb
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: