add TrellisClient, use it for model generation

Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs

@@ -5,7 +5,6 @@ using FishNet.Object.Synchronizing;
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Threading.Tasks;
 using TMPro;
 using Unity.XR.CoreUtils;
 using UnityEngine;
@@ -123,7 +122,7 @@ public class ArcheryRange : NetworkBehaviour
         }
     }
 
-    async private void SpawnTarget()
+    private void SpawnTarget()
     {
         if (!IsServer) return;
         
@@ -133,23 +132,21 @@ public class ArcheryRange : NetworkBehaviour
             Random.Range(minRandomOffset.z, maxRandomOffset.z)
         );
 
-        var target = await SpawnTarget(randomPos);
+        var target = SpawnTarget(randomPos);
         _targets.Add(target);
     }
 
-    async private Task<ArcheryTarget> SpawnTarget(Vector3 randomPos)
+    private ArcheryTarget SpawnTarget(Vector3 randomPos)
     {
         GameObject targetObject;
-        if (modelGenerationBox.LastModelPath == null)
+        if (modelGenerationBox.GeneratedModel == null)
         {
             // spawn default UFO
             targetObject = Instantiate(targetPrefab, randomPos, Quaternion.identity, null);
         } else
         {
             // spawn generated model
-            targetObject = await PipelineManager.Instance.SpawnModel(modelGenerationBox.LastModelPath);
+            targetObject = Instantiate(modelGenerationBox.GeneratedModel, randomPos, Quaternion.identity, null);
-            targetObject.transform.position = randomPos;
-            targetObject.transform.rotation = Quaternion.identity;
             InitializeArcherytargetObject(targetObject);
         }
 
@@ -165,7 +162,7 @@ public class ArcheryRange : NetworkBehaviour
         ArcheryTarget archeryTarget = targetObject.AddComponent<ArcheryTarget>();
         archeryTarget.pointsText = archeryTargetPointsText;
 
-        Rigidbody rigidbody = targetObject.AddComponent<Rigidbody>();
+        Rigidbody rigidbody = targetObject.GetComponent<Rigidbody>();
         rigidbody.useGravity = false;
         rigidbody.isKinematic = true;
 

Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab

@@ -300,4 +300,6 @@ MonoBehaviour:
   inactiveMaterial: {fileID: 2100000, guid: 707a698b0ec80454a8c68700bca72941, type: 2}
   loadingMaterial: {fileID: 2100000, guid: 33390c6f2eb32df47809c60975868a0c, type: 2}
   voiceTranscriptionTestBox: {fileID: 0}
-  UIImage: {fileID: 6899506685386251279}
+  imageDisplay: {fileID: 6899506685386251279}
+  promptSuffix: ', single object, front and side fully visible, realistic style,
+    plain neutral background, clear details, soft studio lighting, true-to-scale'

Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs

@@ -9,14 +9,16 @@ public class ImageGenerationBox : MonoBehaviour
     public Material loadingMaterial;
 
     public VoiceTranscriptionBox voiceTranscriptionTestBox;
-    public Image UIImage;
+    public Image imageDisplay;
+    public Texture2D LastTexture { get; private set; }
     public string promptSuffix = ", single object, front and side fully visible, realistic style, plain neutral background, clear details, soft studio lighting, true-to-scale";
 
     private MeshRenderer meshRenderer;
     private bool isLoading;
 
-    // Start is called before the first frame update
+
-    void Start()
+// Start is called before the first frame update
+void Start()
     {
         meshRenderer = GetComponent<MeshRenderer>();
     }
@@ -42,15 +44,16 @@ public class ImageGenerationBox : MonoBehaviour
             meshRenderer.material = loadingMaterial;
 
             byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(refinedPrompt);
-            Sprite sprite = CreateSprite(imageBytes);
+            LastTexture = CreateTexture(imageBytes);
-            UIImage.sprite = sprite;
+            Sprite sprite = CreateSprite(LastTexture);
+            imageDisplay.sprite = sprite;
 
             isLoading = false;
             meshRenderer.material = inactiveMaterial;
         }
     }
 
-    private Sprite CreateSprite(byte[] imageBytes)
+    private Texture2D CreateTexture(byte[] imageBytes)
     {
         var tex = new Texture2D(2, 2, TextureFormat.RGBA32, false);
         // ImageConversion.LoadImage returns bool (true = success)
@@ -59,10 +62,15 @@ public class ImageGenerationBox : MonoBehaviour
             Destroy(tex);
             throw new InvalidOperationException("Failed to decode image bytes into Texture2D.");
         }
-        
+
         tex.filterMode = FilterMode.Bilinear;
         tex.wrapMode = TextureWrapMode.Clamp;
 
+        return tex;
+    }
+
+    private Sprite CreateSprite(Texture2D tex)
+    {
         var sprite = Sprite.Create(
             tex,
             new Rect(0, 0, tex.width, tex.height),

Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs

@@ -1,3 +1,4 @@
+using System;
 using Unity.XR.CoreUtils;
 using UnityEngine;
 
@@ -7,18 +8,12 @@ public class ModelGenerationBox : MonoBehaviour
     public Material loadingMaterial;
 
     public Transform modelSpawnPoint;
-    public VoiceTranscriptionBox voiceTranscriptionTestBox;
+    public ImageGenerationBox imageGenerationBox;
 
     private MeshRenderer meshRenderer;
     private bool isLoading;
 
-    private string lastModelPath;
+    public GameObject GeneratedModel {  get; private set; }
-    public string LastModelPath
-    {
-        get {
-            return lastModelPath;
-        }
-    }
 
     // Start is called before the first frame update
     void Start()
@@ -40,18 +35,18 @@ public class ModelGenerationBox : MonoBehaviour
         XROrigin playerOrigin = other.GetComponent<XROrigin>();
         if (controller != null || playerOrigin != null)
         {
-            string inputPrompt = voiceTranscriptionTestBox.LastTextOutput;
-
             isLoading = true;
             meshRenderer.material = loadingMaterial;
 
-            string modelPath = await PipelineManager.Instance.GenerateModelAsync(inputPrompt);
+            Texture2D inputTexture = imageGenerationBox.LastTexture;
-            lastModelPath = modelPath;
+            string encodedTexture = Convert.ToBase64String(inputTexture.EncodeToJPG());
-
+            byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture);
-            GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(modelPath);
+         
+            GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(encodedModel);
             spawnedObject.AddComponent<Rigidbody>();
             spawnedObject.transform.parent = modelSpawnPoint;
             spawnedObject.transform.position = modelSpawnPoint.position;
+            GeneratedModel = spawnedObject;
 
             isLoading = false;
             meshRenderer.material = inactiveMaterial;

Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs

@@ -25,6 +25,10 @@ public class PipelineManager : MonoBehaviour
         
     }
 
+    /**
+     * Generate model by starting a new Python subprocess
+     * NOT USED ANYMORE
+    **/
     public async Task<string> GenerateModelAsync(string inputPrompt)
     {
         return await Task.Run(() =>
@@ -76,6 +80,11 @@ public class PipelineManager : MonoBehaviour
         });
     }
 
+    /**
+     * Spawn model stored on disk
+     * NOT USED ANYMORE
+     * 
+    **/
     public async Task<GameObject> SpawnModel(string modelPath)
     {
         var gltf = new GltfImport();
@@ -83,23 +92,42 @@ public class PipelineManager : MonoBehaviour
         if (loadSuccess)
         {
             string objectName = Path.GetFileName(modelPath);
-            GameObject spawningParent = new GameObject("Parent-" + objectName);
+            return await SpawnModel(gltf, objectName);
-
-            bool spawnSuccess = await gltf.InstantiateMainSceneAsync(spawningParent.transform);
-            if (spawnSuccess)
-            {
-                Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform;
-                GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject;
-                MeshCollider collider = spawnedObjectBody.AddComponent<MeshCollider>();
-                collider.convex = true;
-                MeshRenderer renderer = spawnedObjectBody.GetComponent<MeshRenderer>();
-                renderer.material.SetFloat("metallicFactor", 0);
-
-                spawnedObjectBody.name = objectName;
-                return spawnedObjectBody;
-            }
         }
 
-        throw new System.Exception("Failed to spawn GameObject from model" + modelPath);
+        throw new System.Exception("Failed to load GameObject from model" + modelPath);
+    }
+
+    public async Task<GameObject> SpawnModel(byte[] modelBinary)
+    {
+        var gltf = new GltfImport();
+        bool loadSuccess = await gltf.Load(modelBinary);
+        if (loadSuccess)
+        {
+            return await SpawnModel(gltf, "GeneratedModel");
+        }
+
+        throw new System.Exception("Failed to load GameObject from binary!");
+    }
+
+    public async Task<GameObject> SpawnModel(GltfImport gltfImport, string objectName)
+    {
+        GameObject spawningParent = new GameObject("Parent-" + objectName);
+
+        bool spawnSuccess = await gltfImport.InstantiateMainSceneAsync(spawningParent.transform);
+        if (spawnSuccess)
+        {
+            Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform;
+            GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject;
+            MeshCollider collider = spawnedObjectBody.AddComponent<MeshCollider>();
+            collider.convex = true;
+            MeshRenderer renderer = spawnedObjectBody.GetComponent<MeshRenderer>();
+            renderer.material.SetFloat("metallicFactor", 0);
+
+            spawnedObjectBody.name = objectName;
+            return spawnedObjectBody;
+        }
+
+        throw new System.Exception("Failed to spawn GameObject " + objectName);
     }
 }

Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs

@@ -0,0 +1,101 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using UnityEngine;
+
+public class TrellisClient : MonoBehaviour
+{
+    public static TrellisClient Instance { get; private set; }
+
+    public string TRELLIS_BASE_URL;
+
+    private HttpClient httpClient;
+
+    private void Awake()
+    {
+        httpClient = new HttpClient();
+        httpClient.BaseAddress = new Uri(TRELLIS_BASE_URL);
+
+        Instance = this;
+    }
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+
+    public async Task<byte[]> GenerateModel(
+            string imageBase64,
+            int seed = 42,
+            int pollIntervalMs = 1000)
+    {
+        // --- Set generation parameters (form-encoded, like Python requests.post(data=params)) ---
+        var form = new Dictionary<string, string>
+        {
+            ["image_base64"] = imageBase64,
+            ["seed"] = seed.ToString(),
+            ["ss_guidance_strength"] = "7.5",
+            ["ss_sampling_steps"] = "10",
+            ["slat_guidance_strength"] = "7.5",
+            ["slat_sampling_steps"] = "10",
+            ["mesh_simplify_ratio"] = "0.99",
+            ["texture_size"] = "1024",
+            ["texture_opt_total_steps"] = "1000",
+            ["output_format"] = "glb"
+        };
+
+        Debug.Log("Starting Trellis generation...");
+        using (var content = new FormUrlEncodedContent(form))
+        using (var startResp = await httpClient.PostAsync("generate_no_preview", content))
+        {
+            startResp.EnsureSuccessStatusCode();
+        }
+
+        // --- Poll /status until COMPLETE or FAILED ---
+        while (true)
+        {
+            using (var statusResp = await httpClient.GetAsync("status"))
+            {
+                statusResp.EnsureSuccessStatusCode();
+                var json = await statusResp.Content.ReadAsStringAsync();
+                var status = JObject.Parse(json);
+
+                var progress = status.Value<int?>("progress") ?? 0;
+                var state = status.Value<string>("status") ?? "UNKNOWN";
+                Debug.Log($"TRELLIS progress: {progress}% (status: {state})");
+
+                if (string.Equals(state, "COMPLETE", StringComparison.OrdinalIgnoreCase))
+                    break;
+
+                if (string.Equals(state, "FAILED", StringComparison.OrdinalIgnoreCase))
+                {
+                    var msg = status.Value<string>("message") ?? "Generation failed.";
+                    throw new InvalidOperationException($"Trellis generation failed: {msg}");
+                }
+            }
+
+            await Task.Delay(pollIntervalMs);
+        }
+
+        // --- Download the model binary ---
+        Debug.Log("Downloading model...");
+        using (var downloadResponse = await httpClient.GetAsync("download/model"))
+        {
+            downloadResponse.EnsureSuccessStatusCode();
+            var bytes = await downloadResponse.Content.ReadAsByteArrayAsync();
+            return bytes;
+        }
+    }
+
+}

Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 905247c726697644cbf0f39558db46bb
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: