diff --git a/Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs b/Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs index 3048cc69..aafc3f07 100644 --- a/Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs +++ b/Assets/_PROJECT/Components/Bow/Scripts/ArcheryRange.cs @@ -5,7 +5,6 @@ using FishNet.Object.Synchronizing; using System; using System.Collections.Generic; using System.Linq; -using System.Threading.Tasks; using TMPro; using Unity.XR.CoreUtils; using UnityEngine; @@ -123,7 +122,7 @@ public class ArcheryRange : NetworkBehaviour } } - async private void SpawnTarget() + private void SpawnTarget() { if (!IsServer) return; @@ -133,23 +132,21 @@ public class ArcheryRange : NetworkBehaviour Random.Range(minRandomOffset.z, maxRandomOffset.z) ); - var target = await SpawnTarget(randomPos); + var target = SpawnTarget(randomPos); _targets.Add(target); } - async private Task SpawnTarget(Vector3 randomPos) + private ArcheryTarget SpawnTarget(Vector3 randomPos) { GameObject targetObject; - if (modelGenerationBox.LastModelPath == null) + if (modelGenerationBox.GeneratedModel == null) { // spawn default UFO targetObject = Instantiate(targetPrefab, randomPos, Quaternion.identity, null); } else { // spawn generated model - targetObject = await PipelineManager.Instance.SpawnModel(modelGenerationBox.LastModelPath); - targetObject.transform.position = randomPos; - targetObject.transform.rotation = Quaternion.identity; + targetObject = Instantiate(modelGenerationBox.GeneratedModel, randomPos, Quaternion.identity, null); InitializeArcherytargetObject(targetObject); } @@ -165,7 +162,7 @@ public class ArcheryRange : NetworkBehaviour ArcheryTarget archeryTarget = targetObject.AddComponent(); archeryTarget.pointsText = archeryTargetPointsText; - Rigidbody rigidbody = targetObject.AddComponent(); + Rigidbody rigidbody = targetObject.GetComponent(); rigidbody.useGravity = false; rigidbody.isKinematic = true; diff --git a/Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab b/Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab index 18dce247..620f34da 100644 --- a/Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab +++ b/Assets/_PROJECT/Prefabs/ModelGeneration/ImageGenerationBox.prefab @@ -300,4 +300,6 @@ MonoBehaviour: inactiveMaterial: {fileID: 2100000, guid: 707a698b0ec80454a8c68700bca72941, type: 2} loadingMaterial: {fileID: 2100000, guid: 33390c6f2eb32df47809c60975868a0c, type: 2} voiceTranscriptionTestBox: {fileID: 0} - UIImage: {fileID: 6899506685386251279} + imageDisplay: {fileID: 6899506685386251279} + promptSuffix: ', single object, front and side fully visible, realistic style, + plain neutral background, clear details, soft studio lighting, true-to-scale' diff --git a/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity b/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity index 6d96c48a..13b300c1 100644 --- a/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity +++ b/Assets/_PROJECT/Scenes/DeltaBuilding_base.unity @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9350888e1de24412c17b98e0860e69b6891be95789d7a56b558f01fa431d1de4 -size 63207713 +oid sha256:329e8815aa533260a7abe80bf6c35c9428f5f81061753352dabcb9d6be828a15 +size 63208000 diff --git a/Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs b/Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs index 36901b86..60829a09 100644 --- a/Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs +++ b/Assets/_PROJECT/Scripts/ModeGeneration/ImageGenerationBox.cs @@ -9,14 +9,16 @@ public class ImageGenerationBox : MonoBehaviour public Material loadingMaterial; public VoiceTranscriptionBox voiceTranscriptionTestBox; - public Image UIImage; + public Image imageDisplay; + public Texture2D LastTexture { get; private set; } public string promptSuffix = ", single object, front and side fully visible, realistic style, plain neutral background, clear details, soft studio lighting, true-to-scale"; private MeshRenderer meshRenderer; private bool isLoading; - // Start is called before the first frame update - void Start() + +// Start is called before the first frame update +void Start() { meshRenderer = GetComponent(); } @@ -42,15 +44,16 @@ public class ImageGenerationBox : MonoBehaviour meshRenderer.material = loadingMaterial; byte[] imageBytes = await InvokeAiClient.Instance.GenerateImage(refinedPrompt); - Sprite sprite = CreateSprite(imageBytes); - UIImage.sprite = sprite; + LastTexture = CreateTexture(imageBytes); + Sprite sprite = CreateSprite(LastTexture); + imageDisplay.sprite = sprite; isLoading = false; meshRenderer.material = inactiveMaterial; } } - private Sprite CreateSprite(byte[] imageBytes) + private Texture2D CreateTexture(byte[] imageBytes) { var tex = new Texture2D(2, 2, TextureFormat.RGBA32, false); // ImageConversion.LoadImage returns bool (true = success) @@ -59,10 +62,15 @@ public class ImageGenerationBox : MonoBehaviour Destroy(tex); throw new InvalidOperationException("Failed to decode image bytes into Texture2D."); } - + tex.filterMode = FilterMode.Bilinear; tex.wrapMode = TextureWrapMode.Clamp; + return tex; + } + + private Sprite CreateSprite(Texture2D tex) + { var sprite = Sprite.Create( tex, new Rect(0, 0, tex.width, tex.height), diff --git a/Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs b/Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs index a42f7d87..838eae47 100644 --- a/Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs +++ b/Assets/_PROJECT/Scripts/ModeGeneration/ModelGenerationBox.cs @@ -1,3 +1,4 @@ +using System; using Unity.XR.CoreUtils; using UnityEngine; @@ -7,18 +8,12 @@ public class ModelGenerationBox : MonoBehaviour public Material loadingMaterial; public Transform modelSpawnPoint; - public VoiceTranscriptionBox voiceTranscriptionTestBox; + public ImageGenerationBox imageGenerationBox; private MeshRenderer meshRenderer; private bool isLoading; - private string lastModelPath; - public string LastModelPath - { - get { - return lastModelPath; - } - } + public GameObject GeneratedModel { get; private set; } // Start is called before the first frame update void Start() @@ -40,18 +35,18 @@ public class ModelGenerationBox : MonoBehaviour XROrigin playerOrigin = other.GetComponent(); if (controller != null || playerOrigin != null) { - string inputPrompt = voiceTranscriptionTestBox.LastTextOutput; - isLoading = true; meshRenderer.material = loadingMaterial; - string modelPath = await PipelineManager.Instance.GenerateModelAsync(inputPrompt); - lastModelPath = modelPath; - - GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(modelPath); + Texture2D inputTexture = imageGenerationBox.LastTexture; + string encodedTexture = Convert.ToBase64String(inputTexture.EncodeToJPG()); + byte[] encodedModel = await TrellisClient.Instance.GenerateModel(encodedTexture); + + GameObject spawnedObject = await PipelineManager.Instance.SpawnModel(encodedModel); spawnedObject.AddComponent(); spawnedObject.transform.parent = modelSpawnPoint; spawnedObject.transform.position = modelSpawnPoint.position; + GeneratedModel = spawnedObject; isLoading = false; meshRenderer.material = inactiveMaterial; diff --git a/Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs b/Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs index 9980ac2e..868c5eb6 100644 --- a/Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs +++ b/Assets/_PROJECT/Scripts/ModeGeneration/PipelineManager.cs @@ -25,6 +25,10 @@ public class PipelineManager : MonoBehaviour } + /** + * Generate model by starting a new Python subprocess + * NOT USED ANYMORE + **/ public async Task GenerateModelAsync(string inputPrompt) { return await Task.Run(() => @@ -76,6 +80,11 @@ public class PipelineManager : MonoBehaviour }); } + /** + * Spawn model stored on disk + * NOT USED ANYMORE + * + **/ public async Task SpawnModel(string modelPath) { var gltf = new GltfImport(); @@ -83,23 +92,42 @@ public class PipelineManager : MonoBehaviour if (loadSuccess) { string objectName = Path.GetFileName(modelPath); - GameObject spawningParent = new GameObject("Parent-" + objectName); - - bool spawnSuccess = await gltf.InstantiateMainSceneAsync(spawningParent.transform); - if (spawnSuccess) - { - Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform; - GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject; - MeshCollider collider = spawnedObjectBody.AddComponent(); - collider.convex = true; - MeshRenderer renderer = spawnedObjectBody.GetComponent(); - renderer.material.SetFloat("metallicFactor", 0); - - spawnedObjectBody.name = objectName; - return spawnedObjectBody; - } + return await SpawnModel(gltf, objectName); } - throw new System.Exception("Failed to spawn GameObject from model" + modelPath); + throw new System.Exception("Failed to load GameObject from model" + modelPath); + } + + public async Task SpawnModel(byte[] modelBinary) + { + var gltf = new GltfImport(); + bool loadSuccess = await gltf.Load(modelBinary); + if (loadSuccess) + { + return await SpawnModel(gltf, "GeneratedModel"); + } + + throw new System.Exception("Failed to load GameObject from binary!"); + } + + public async Task SpawnModel(GltfImport gltfImport, string objectName) + { + GameObject spawningParent = new GameObject("Parent-" + objectName); + + bool spawnSuccess = await gltfImport.InstantiateMainSceneAsync(spawningParent.transform); + if (spawnSuccess) + { + Transform spawnedObjectWorldTransform = spawningParent.transform.GetChild(0).transform; + GameObject spawnedObjectBody = spawnedObjectWorldTransform.GetChild(0).transform.gameObject; + MeshCollider collider = spawnedObjectBody.AddComponent(); + collider.convex = true; + MeshRenderer renderer = spawnedObjectBody.GetComponent(); + renderer.material.SetFloat("metallicFactor", 0); + + spawnedObjectBody.name = objectName; + return spawnedObjectBody; + } + + throw new System.Exception("Failed to spawn GameObject " + objectName); } } diff --git a/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs b/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs new file mode 100644 index 00000000..7c9cd37a --- /dev/null +++ b/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs @@ -0,0 +1,101 @@ +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; + +public class TrellisClient : MonoBehaviour +{ + public static TrellisClient Instance { get; private set; } + + public string TRELLIS_BASE_URL; + + private HttpClient httpClient; + + private void Awake() + { + httpClient = new HttpClient(); + httpClient.BaseAddress = new Uri(TRELLIS_BASE_URL); + + Instance = this; + } + + // Start is called before the first frame update + void Start() + { + + } + + // Update is called once per frame + void Update() + { + + } + + + public async Task GenerateModel( + string imageBase64, + int seed = 42, + int pollIntervalMs = 1000) + { + // --- Set generation parameters (form-encoded, like Python requests.post(data=params)) --- + var form = new Dictionary + { + ["image_base64"] = imageBase64, + ["seed"] = seed.ToString(), + ["ss_guidance_strength"] = "7.5", + ["ss_sampling_steps"] = "10", + ["slat_guidance_strength"] = "7.5", + ["slat_sampling_steps"] = "10", + ["mesh_simplify_ratio"] = "0.99", + ["texture_size"] = "1024", + ["texture_opt_total_steps"] = "1000", + ["output_format"] = "glb" + }; + + Debug.Log("Starting Trellis generation..."); + using (var content = new FormUrlEncodedContent(form)) + using (var startResp = await httpClient.PostAsync("generate_no_preview", content)) + { + startResp.EnsureSuccessStatusCode(); + } + + // --- Poll /status until COMPLETE or FAILED --- + while (true) + { + using (var statusResp = await httpClient.GetAsync("status")) + { + statusResp.EnsureSuccessStatusCode(); + var json = await statusResp.Content.ReadAsStringAsync(); + var status = JObject.Parse(json); + + var progress = status.Value("progress") ?? 0; + var state = status.Value("status") ?? "UNKNOWN"; + Debug.Log($"TRELLIS progress: {progress}% (status: {state})"); + + if (string.Equals(state, "COMPLETE", StringComparison.OrdinalIgnoreCase)) + break; + + if (string.Equals(state, "FAILED", StringComparison.OrdinalIgnoreCase)) + { + var msg = status.Value("message") ?? "Generation failed."; + throw new InvalidOperationException($"Trellis generation failed: {msg}"); + } + } + + await Task.Delay(pollIntervalMs); + } + + // --- Download the model binary --- + Debug.Log("Downloading model..."); + using (var downloadResponse = await httpClient.GetAsync("download/model")) + { + downloadResponse.EnsureSuccessStatusCode(); + var bytes = await downloadResponse.Content.ReadAsByteArrayAsync(); + return bytes; + } + } + +} diff --git a/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs.meta b/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs.meta new file mode 100644 index 00000000..ace291e9 --- /dev/null +++ b/Assets/_PROJECT/Scripts/ModeGeneration/TrellisClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 905247c726697644cbf0f39558db46bb +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: