update env example

3d-generation-pipeline/.env.example

@@ -0,0 +1,4 @@
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_API_TOKEN=
+PIPELINE_FOLDER=
+MODEL_FOLDER=

3d-generation-pipeline/.gitignore

@@ -0,0 +1,4 @@
+.venv
+.env
+images/
+models/

3d-generation-pipeline/README.md

@@ -0,0 +1,2 @@
+### TODO
+* Azure OpenAI API: https://wiki.ut.ee/spaces/AA/pages/218073589/Azure+OpenAI+API+teenus

3d-generation-pipeline/generate_image_local.py

@@ -0,0 +1,28 @@
+import torch
+from diffusers import StableDiffusionPipeline, StableDiffusion3Pipeline
+import time
+
+start_timestamp = time.time()
+#model = "stabilityai/stable-diffusion-3.5-medium" # generation time: 13 min
+model = "stabilityai/stable-diffusion-3-medium-diffusers" # generation time: 10 min
+#model = "stabilityai/stable-diffusion-2" # generation time: 4 sec
+
+pipe = StableDiffusion3Pipeline.from_pretrained(model, torch_dtype=torch.float16)
+#pipe = StableDiffusionPipeline.from_pretrained(model, torch_dtype=torch.float16)
+pipe = pipe.to("cuda")
+
+model_loaded_timestamp = time.time()
+model_load_time = model_loaded_timestamp - start_timestamp
+print(f"model load time: {round(model_load_time)} seconds")
+
+prompt = "A majestic broadsword with a golden pommel, no background"
+image = pipe(
+    prompt,
+    guidance_scale=3.0,
+).images[0]
+
+image_name = "image7.png"
+image.save(f"images/{image_name}")
+
+generation_time = time.time() - model_loaded_timestamp
+print(f"image generation time: {round(generation_time)} seconds")

3d-generation-pipeline/notebooks/cloudflare_API_test.ipynb

@@ -0,0 +1,152 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1dc6faae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import base64\n",
+    "import requests\n",
+    "from dotenv import load_dotenv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b3107275",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "load_dotenv()\n",
+    "\n",
+    "ACCOUNT_ID = os.environ[\"CLOUDFLARE_ACCOUNT_ID\"]\n",
+    "API_TOKEN = os.environ[\"CLOUDFLARE_API_TOKEN\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "999adf95",
+   "metadata": {},
+   "source": [
+    "## Text to image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40b35163",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved: output.jpg  (263282 bytes)\n",
+      "Saved: image9.jpg  (263282 bytes)\n"
+     ]
+    }
+   ],
+   "source": [
+    "MODEL = \"@cf/black-forest-labs/flux-1-schnell\"\n",
+    "URL = f\"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}\"\n",
+    "\n",
+    "payload = {\n",
+    "    \"prompt\": \"a slightly curved broadsword with a fancy golden crossguard\",\n",
+    "}\n",
+    "\n",
+    "headers = {\n",
+    "    \"Authorization\": f\"Bearer {API_TOKEN}\",\n",
+    "    \"Content-Type\": \"application/json\",\n",
+    "}\n",
+    "\n",
+    "resp = requests.post(URL, json=payload, headers=headers, timeout=60)\n",
+    "resp.raise_for_status()\n",
+    "\n",
+    "data = resp.json()\n",
+    "b64 = data[\"result\"][\"image\"]\n",
+    "if not b64:\n",
+    "    raise RuntimeError(f\"Unexpected response structure: {data}\")\n",
+    "\n",
+    "img_bytes = base64.b64decode(b64)\n",
+    "\n",
+    "out_path = \"output.jpg\"\n",
+    "with open(out_path, \"wb\") as f:\n",
+    "    f.write(img_bytes)\n",
+    "\n",
+    "print(f\"Saved: {out_path}  ({len(img_bytes)} bytes)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "14a874c4",
+   "metadata": {},
+   "source": [
+    "## Text prompt refinement"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "485f6f46",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\"dark wooden battleaxe with bronze blade\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "MODEL = \"@cf/meta/llama-3.2-3b-instruct\"\n",
+    "URL = f\"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}\"\n",
+    "\n",
+    "instructions = \"\"\"\n",
+    "User is talking about some object. Your task is to generate a short and concise description of it. Use only user's own words, keep it as short as possible.\n",
+    "Example:\n",
+    "User: 'Umm, okay, I would like a really cool sword, with for example a bright orange crossguard. And also it should be slightly curved.'\n",
+    "You: 'a slightly curved sword with bright orange crossguard'\n",
+    "\"\"\"\n",
+    "prompt = \"Umm, alright, can you please give me an epic battleaxe? It should have a dark wooden shaft and bronze blade.\"\n",
+    "\n",
+    "response = requests.post(URL,\n",
+    "  headers={\"Authorization\": f\"Bearer {API_TOKEN}\"},\n",
+    "  json={\n",
+    "    \"messages\": [\n",
+    "      {\"role\": \"system\", \"content\": instructions},\n",
+    "      {\"role\": \"user\", \"content\": prompt}\n",
+    "    ]\n",
+    "  }\n",
+    ")\n",
+    "data = response.json()\n",
+    "result_text = data[\"result\"][\"response\"]\n",
+    "print(result_text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

3d-generation-pipeline/notebooks/local_model_test.ipynb

@@ -0,0 +1,105 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2c0da293",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\henrisel.DOMENIS\\DeltaVR3DModelGeneration\\3d-generation-pipeline\\.venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from diffusers import FluxPipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "51879ff1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Fetching 23 files:   0%|          | 0/23 [00:00<?, ?it/s]"
+     ]
+    }
+   ],
+   "source": [
+    "model_name = \"black-forest-labs/FLUX.1-schnell\"\n",
+    "\n",
+    "pipe = FluxPipeline.from_pretrained(model_name, torch_dtype=torch.bfloat16)\n",
+    "#pipe.enable_model_cpu_offload() #save some VRAM by offloading the model to CPU. Remove this if you have enough GPU power"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50a38bf4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"slightly curved sword, one side blue and other side green\"\n",
+    "pipe_result = pipe(\n",
+    "    prompt,\n",
+    "    guidance_scale=0.0,\n",
+    "    num_inference_steps=4,\n",
+    "    max_sequence_length=256,\n",
+    "    generator=torch.Generator(\"gpu\")\n",
+    ")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b692177a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe_result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d55eb3ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image = pipe_result[\"images\"][0]\n",
+    "image.save(\"flux-schnell.png\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

3d-generation-pipeline/notebooks/subprocess_test.ipynb

@@ -0,0 +1,165 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4826c91d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'2025-10-18-16-35-47'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datetime import datetime\n",
+    "\n",
+    "datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9419e692",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "STDOUT:\n",
+      " Device used:  cuda\n",
+      "After Remesh 9998 19996\n",
+      "\n",
+      "STDERR:\n",
+      " D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\timm\\models\\layers\\__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers\n",
+      "  warnings.warn(f\"Importing from {__name__} is deprecated, please import via timm.layers\", FutureWarning)\n",
+      "\n",
+      "  0%|          | 0/1 [00:00<?, ?it/s]\n",
+      "  0%|          | 0/1 [00:00<?, ?it/s]\n",
+      "Traceback (most recent call last):\n",
+      "  File \"D:\\users\\henrisel\\stable-fast-3d\\run.py\", line 122, in <module>\n",
+      "    mesh, glob_dict = model.run_image(\n",
+      "  File \"D:\\users\\henrisel\\stable-fast-3d\\sf3d\\system.py\", line 286, in run_image\n",
+      "    meshes, global_dict = self.generate_mesh(\n",
+      "  File \"D:\\users\\henrisel\\stable-fast-3d\\sf3d\\system.py\", line 369, in generate_mesh\n",
+      "    rast = self.baker.rasterize(\n",
+      "  File \"D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\texture_baker\\baker.py\", line 28, in rasterize\n",
+      "    return torch.ops.texture_baker_cpp.rasterize(\n",
+      "  File \"D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\torch\\_ops.py\", line 1243, in __call__\n",
+      "    return self._op(*args, **kwargs)\n",
+      "NotImplementedError: Could not run 'texture_baker_cpp::rasterize' with arguments from the 'CUDA' backend. This could be because the operator doesn't exist for this backend, or was omitted during the selective/custom build process (if using custom build). If you are a Facebook employee using PyTorch on mobile, please visit https://fburl.com/ptmfixes for possible resolutions. 'texture_baker_cpp::rasterize' is only available for these backends: [CPU, Meta, BackendSelect, Python, FuncTorchDynamicLayerBackMode, Functionalize, Named, Conjugate, Negative, ZeroTensor, ADInplaceOrView, AutogradOther, AutogradCPU, AutogradCUDA, AutogradXLA, AutogradMPS, AutogradXPU, AutogradHPU, AutogradLazy, AutogradMTIA, AutogradMAIA, AutogradMeta, Tracer, AutocastCPU, AutocastMTIA, AutocastMAIA, AutocastXPU, AutocastMPS, AutocastCUDA, FuncTorchBatched, BatchedNestedTensor, FuncTorchVmapMode, Batched, VmapMode, FuncTorchGradWrapper, PythonTLSSnapshot, FuncTorchDynamicLayerFrontMode, PreDispatch, PythonDispatcher].\n",
+      "\n",
+      "CPU: registered at texture_baker\\csrc\\baker.cpp:543 [kernel]\n",
+      "Meta: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\MetaFallbackKernel.cpp:23 [backend fallback]\n",
+      "BackendSelect: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\BackendSelectFallbackKernel.cpp:3 [backend fallback]\n",
+      "Python: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:194 [backend fallback]\n",
+      "FuncTorchDynamicLayerBackMode: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\DynamicLayer.cpp:479 [backend fallback]\n",
+      "Functionalize: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\FunctionalizeFallbackKernel.cpp:375 [backend fallback]\n",
+      "Named: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\NamedRegistrations.cpp:7 [backend fallback]\n",
+      "Conjugate: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\ConjugateFallback.cpp:17 [backend fallback]\n",
+      "Negative: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\native\\NegateFallback.cpp:18 [backend fallback]\n",
+      "ZeroTensor: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\ZeroTensorFallback.cpp:86 [backend fallback]\n",
+      "ADInplaceOrView: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:104 [backend fallback]\n",
+      "AutogradOther: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:63 [backend fallback]\n",
+      "AutogradCPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:67 [backend fallback]\n",
+      "AutogradCUDA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:75 [backend fallback]\n",
+      "AutogradXLA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:87 [backend fallback]\n",
+      "AutogradMPS: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:95 [backend fallback]\n",
+      "AutogradXPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:71 [backend fallback]\n",
+      "AutogradHPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:108 [backend fallback]\n",
+      "AutogradLazy: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:91 [backend fallback]\n",
+      "AutogradMTIA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:79 [backend fallback]\n",
+      "AutogradMAIA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:83 [backend fallback]\n",
+      "AutogradMeta: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:99 [backend fallback]\n",
+      "Tracer: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\torch\\csrc\\autograd\\TraceTypeManual.cpp:294 [backend fallback]\n",
+      "AutocastCPU: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:322 [backend fallback]\n",
+      "AutocastMTIA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:466 [backend fallback]\n",
+      "AutocastMAIA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:504 [backend fallback]\n",
+      "AutocastXPU: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:542 [backend fallback]\n",
+      "AutocastMPS: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:209 [backend fallback]\n",
+      "AutocastCUDA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:165 [backend fallback]\n",
+      "FuncTorchBatched: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\LegacyBatchingRegistrations.cpp:731 [backend fallback]\n",
+      "BatchedNestedTensor: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\LegacyBatchingRegistrations.cpp:758 [backend fallback]\n",
+      "FuncTorchVmapMode: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\VmapModeRegistrations.cpp:27 [backend fallback]\n",
+      "Batched: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\LegacyBatchingRegistrations.cpp:1075 [backend fallback]\n",
+      "VmapMode: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\VmapModeRegistrations.cpp:33 [backend fallback]\n",
+      "FuncTorchGradWrapper: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\TensorWrapper.cpp:210 [backend fallback]\n",
+      "PythonTLSSnapshot: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:202 [backend fallback]\n",
+      "FuncTorchDynamicLayerFrontMode: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\DynamicLayer.cpp:475 [backend fallback]\n",
+      "PreDispatch: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:206 [backend fallback]\n",
+      "PythonDispatcher: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:198 [backend fallback]\n",
+      "\n",
+      "\n",
+      "Return Code: 1\n"
+     ]
+    }
+   ],
+   "source": [
+    "import subprocess\n",
+    "\n",
+    "MODEL_FOLDER = r\"D:\\users\\henrisel\\stable-fast-3d\"\n",
+    "PROJECT_FOLDER = r\"D:\\users\\henrisel\\DeltaVR3DModelGeneration\\3d-generation-pipeline\"\n",
+    "\n",
+    "# Path to the Python interpreter in the other virtual environment\n",
+    "venv_python = MODEL_FOLDER + r\"\\.venv\\Scripts\\python.exe\"\n",
+    "\n",
+    "# Path to the .py file you want to run\n",
+    "script_path = MODEL_FOLDER + r\"\\run.py\"\n",
+    "\n",
+    "# Optional: arguments to pass to the script\n",
+    "args = [MODEL_FOLDER + r\"\\demo_files\\examples\\chair1.png\", \"--output-dir\", PROJECT_FOLDER + r\"\\images\"]\n",
+    "\n",
+    "# Build the command\n",
+    "command = [venv_python, script_path] + args\n",
+    "\n",
+    "try:\n",
+    "    # Run the subprocess\n",
+    "    result = subprocess.run(command, capture_output=True, text=True)\n",
+    "\n",
+    "    # Print output and errors\n",
+    "    print(\"STDOUT:\\n\", result.stdout)\n",
+    "    print(\"STDERR:\\n\", result.stderr)\n",
+    "    print(\"Return Code:\", result.returncode)\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"Error occurred: {e}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee480ba6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

3d-generation-pipeline/requirements.txt

@@ -0,0 +1,7 @@
+#torch==2.8.0+cu129 https://pytorch.org/get-started/previous-versions/
+transformers==4.57.0
+git+https://github.com/huggingface/diffusers.git
+accelerate==1.10.1
+huggingface_hub[hf_xet]==1.1.10
+sentencepiece==0.2.1
+protobuf==6.32.1

3d-generation-pipeline/start_pipeline.py

@@ -0,0 +1,111 @@
+import os
+import base64
+import requests
+import argparse
+import subprocess
+from pathlib import Path
+from datetime import datetime
+from dotenv import load_dotenv
+
+load_dotenv()
+
+ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"]
+API_TOKEN = os.environ["CLOUDFLARE_API_TOKEN"]
+PIPELINE_FOLDER = os.environ["PIPELINE_FOLDER"]
+MODEL_FOLDER = os.environ["MODEL_FOLDER"]
+
+
+def get_timestamp():
+    return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+
+
+def text_to_image(prompt, output_path):
+    MODEL = "@cf/black-forest-labs/flux-1-schnell"
+    URL = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}"
+
+    payload = {
+        "prompt": prompt,
+    }
+
+    headers = {
+        "Authorization": f"Bearer {API_TOKEN}",
+        "Content-Type": "application/json",
+    }
+
+    resp = requests.post(URL, json=payload, headers=headers, timeout=60)
+    resp.raise_for_status()
+
+    data = resp.json()
+    b64 = data["result"]["image"]
+    if not b64:
+        raise RuntimeError(f"Unexpected response structure: {data}")
+
+    img_bytes = base64.b64decode(b64)
+
+    with open(output_path, "wb") as f:
+        f.write(img_bytes)
+
+
+def refine_text_prompt(prompt):
+    MODEL = "@cf/meta/llama-3.2-3b-instruct"
+    URL = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}"
+
+    instructions = """
+    User is talking about some object. Your task is to generate a short and concise description of it. Use only user's own words, keep it as short as possible.
+    Example:
+    User: 'Umm, okay, I would like a really cool sword, with for example a bright orange crossguard. And also it should be slightly curved.'
+    You: 'a slightly curved sword with bright orange crossguard'
+    """
+
+    response = requests.post(URL,
+    headers={"Authorization": f"Bearer {API_TOKEN}"},
+    json={
+        "messages": [
+        {"role": "system", "content": instructions},
+        {"role": "user", "content": prompt}
+        ]
+    }
+    )
+    data = response.json()
+    return data["result"]["response"]
+
+def image_to_3d(image_path, output_path):
+    venv_python = MODEL_FOLDER + r"\.venv\Scripts\python.exe"
+    script_path = MODEL_FOLDER + r"\run.py"
+
+    args = [image_path, "--output-dir", output_path]
+    command = [venv_python, script_path] + args
+
+    try:
+        # Run the subprocess
+        result = subprocess.run(command, capture_output=True, text=True)
+
+        # Print output and errors
+        print("STDOUT:\n", result.stdout)
+        print("STDERR:\n", result.stderr)
+        print("Return Code:", result.returncode)
+
+    except Exception as e:
+        print(f"Error occurred: {e}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Text to 3D model pipeline")
+    parser.add_argument("--prompt", type=str, required=True, help="User text prompt")
+    args = parser.parse_args()
+
+    user_prompt = args.prompt
+    print(f"User prompt: {user_prompt}")
+    refined_prompt = refine_text_prompt(user_prompt)
+    print(f"Refined prompt: {refined_prompt}")
+    timestamp = get_timestamp()
+    pipeline_folder = Path(PIPELINE_FOLDER)
+    image_path = pipeline_folder / "images" / f"{timestamp}.jpg"
+    text_to_image(refined_prompt, image_path)
+    print(f"Generated image file: {image_path}")
+    model_path = pipeline_folder / "models" / timestamp
+    image_to_3d(image_path, model_path)
+    print(f"Generated 3D model file: {model_path}")
+
+
+if __name__ == "__main__":
+    main()

Assets/_PROJECT/Components/Bolt/Lexus2016RXTireNormal.png.png.meta

@@ -7,7 +7,7 @@ TextureImporter:
   mipmaps:
     mipMapMode: 0
     enableMipMap: 1
-    sRGBTexture: 1
+    sRGBTexture: 0
     linearTexture: 0
     fadeOut: 0
     borderMipMap: 0
@@ -54,7 +54,7 @@ TextureImporter:
   alphaUsage: 1
   alphaIsTransparency: 0
   spriteTessellationDetail: -1
-  textureType: 0
+  textureType: 1
   textureShape: 1
   singleChannelComponent: 0
   flipbookRows: 1

Assets/_PROJECT/Materials/Green.mat

@@ -0,0 +1,133 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!21 &2100000
+Material:
+  serializedVersion: 8
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_Name: Green
+  m_Shader: {fileID: 4800000, guid: 933532a4fcc9baf4fa0491de14d08ed7, type: 3}
+  m_Parent: {fileID: 0}
+  m_ModifiedSerializedProperties: 0
+  m_ValidKeywords: []
+  m_InvalidKeywords: []
+  m_LightmapFlags: 4
+  m_EnableInstancingVariants: 0
+  m_DoubleSidedGI: 0
+  m_CustomRenderQueue: -1
+  stringTagMap:
+    RenderType: Opaque
+  disabledShaderPasses: []
+  m_LockedProperties: 
+  m_SavedProperties:
+    serializedVersion: 3
+    m_TexEnvs:
+    - _BaseMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _BumpMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailAlbedoMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailMask:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailNormalMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _EmissionMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _MainTex:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _MetallicGlossMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _OcclusionMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _ParallaxMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _SpecGlossMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_Lightmaps:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_LightmapsInd:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_ShadowMasks:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    m_Ints: []
+    m_Floats:
+    - _AlphaClip: 0
+    - _AlphaToMask: 0
+    - _Blend: 0
+    - _BlendModePreserveSpecular: 1
+    - _BumpScale: 1
+    - _ClearCoatMask: 0
+    - _ClearCoatSmoothness: 0
+    - _Cull: 2
+    - _Cutoff: 0.5
+    - _DetailAlbedoMapScale: 1
+    - _DetailNormalMapScale: 1
+    - _DstBlend: 0
+    - _DstBlendAlpha: 0
+    - _EnvironmentReflections: 1
+    - _GlossMapScale: 0
+    - _Glossiness: 0
+    - _GlossyReflections: 0
+    - _Metallic: 0
+    - _OcclusionStrength: 1
+    - _Parallax: 0.005
+    - _QueueOffset: 0
+    - _ReceiveShadows: 1
+    - _Smoothness: 0.5
+    - _SmoothnessTextureChannel: 0
+    - _SpecularHighlights: 1
+    - _SrcBlend: 1
+    - _SrcBlendAlpha: 1
+    - _Surface: 0
+    - _WorkflowMode: 1
+    - _ZWrite: 1
+    m_Colors:
+    - _BaseColor: {r: 0, g: 1, b: 0.0381248, a: 1}
+    - _Color: {r: 0, g: 1, b: 0.0381248, a: 1}
+    - _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
+    - _SpecColor: {r: 0.19999996, g: 0.19999996, b: 0.19999996, a: 1}
+  m_BuildTextureStacks: []
+--- !u!114 &6221994712197478572
+MonoBehaviour:
+  m_ObjectHideFlags: 11
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d0353a89b1f911e48b9e16bdc9f2e058, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  version: 7

Assets/_PROJECT/Materials/Green.mat.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 937c5f357ed270843bd43d1f7d5d475b
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 2100000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/_PROJECT/Materials/Red.mat

@@ -0,0 +1,133 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &-7093071968994914494
+MonoBehaviour:
+  m_ObjectHideFlags: 11
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d0353a89b1f911e48b9e16bdc9f2e058, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  version: 7
+--- !u!21 &2100000
+Material:
+  serializedVersion: 8
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_Name: Red
+  m_Shader: {fileID: 4800000, guid: 933532a4fcc9baf4fa0491de14d08ed7, type: 3}
+  m_Parent: {fileID: 0}
+  m_ModifiedSerializedProperties: 0
+  m_ValidKeywords: []
+  m_InvalidKeywords: []
+  m_LightmapFlags: 4
+  m_EnableInstancingVariants: 0
+  m_DoubleSidedGI: 0
+  m_CustomRenderQueue: -1
+  stringTagMap:
+    RenderType: Opaque
+  disabledShaderPasses: []
+  m_LockedProperties: 
+  m_SavedProperties:
+    serializedVersion: 3
+    m_TexEnvs:
+    - _BaseMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _BumpMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailAlbedoMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailMask:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _DetailNormalMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _EmissionMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _MainTex:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _MetallicGlossMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _OcclusionMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _ParallaxMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - _SpecGlossMap:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_Lightmaps:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_LightmapsInd:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    - unity_ShadowMasks:
+        m_Texture: {fileID: 0}
+        m_Scale: {x: 1, y: 1}
+        m_Offset: {x: 0, y: 0}
+    m_Ints: []
+    m_Floats:
+    - _AlphaClip: 0
+    - _AlphaToMask: 0
+    - _Blend: 0
+    - _BlendModePreserveSpecular: 1
+    - _BumpScale: 1
+    - _ClearCoatMask: 0
+    - _ClearCoatSmoothness: 0
+    - _Cull: 2
+    - _Cutoff: 0.5
+    - _DetailAlbedoMapScale: 1
+    - _DetailNormalMapScale: 1
+    - _DstBlend: 0
+    - _DstBlendAlpha: 0
+    - _EnvironmentReflections: 1
+    - _GlossMapScale: 0
+    - _Glossiness: 0
+    - _GlossyReflections: 0
+    - _Metallic: 0
+    - _OcclusionStrength: 1
+    - _Parallax: 0.005
+    - _QueueOffset: 0
+    - _ReceiveShadows: 1
+    - _Smoothness: 0.5
+    - _SmoothnessTextureChannel: 0
+    - _SpecularHighlights: 1
+    - _SrcBlend: 1
+    - _SrcBlendAlpha: 1
+    - _Surface: 0
+    - _WorkflowMode: 1
+    - _ZWrite: 1
+    m_Colors:
+    - _BaseColor: {r: 1, g: 0, b: 0, a: 1}
+    - _Color: {r: 1, g: 0, b: 0, a: 1}
+    - _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
+    - _SpecColor: {r: 0.19999996, g: 0.19999996, b: 0.19999996, a: 1}
+  m_BuildTextureStacks: []

Assets/_PROJECT/Materials/Red.mat.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 707a698b0ec80454a8c68700bca72941
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 2100000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/_PROJECT/Scripts/3DModeGeneration.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 0ec3982ba49c4b84ea95332cb090e115
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/_PROJECT/Scripts/3DModeGeneration/ModelGenerationPipelineStarter.cs

@@ -0,0 +1,86 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using System.Diagnostics;
+using System.Threading.Tasks;
+
+public class ModelGenerationPipelineStarter : MonoBehaviour
+{
+    public Material activeMaterial;
+    public Material inactiveMaterial;
+
+    private MeshRenderer meshRenderer;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        meshRenderer = GetComponent<MeshRenderer>();
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+    void OnTriggerEnter(Collider other)
+    {
+        KbmController controller = other.GetComponent<KbmController>();
+        if (controller != null)
+        {
+            meshRenderer.material = activeMaterial;
+
+            StartModeGenerationPipeline();
+        }
+    }
+
+    private void OnTriggerExit(Collider other)
+    {
+        KbmController controller = other.GetComponent<KbmController>();
+        if (controller != null)
+        {
+            meshRenderer.material = inactiveMaterial;
+        }
+    }
+
+    private async void StartModeGenerationPipeline()
+    {
+        await Task.Run(() =>
+        {
+            string inputPrompt = "Uhm I want I think an epic broadsword with a fancy golden pommel";
+
+            // Path to your virtual environment's python.exe
+            string pythonExe = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\.venv\Scripts\python.exe";
+
+            // Path to your Python script
+            string scriptPath = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\start_pipeline.py";
+
+            // Arguments to pass to the script
+            string arguments = $"{scriptPath} --prompt \"{inputPrompt}\"";
+
+            ProcessStartInfo psi = new ProcessStartInfo
+            {
+                FileName = pythonExe,
+                Arguments = arguments,
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true
+            };
+
+            using (Process process = new Process())
+            {
+                process.StartInfo = psi;
+                process.OutputDataReceived += (sender, e) => UnityEngine.Debug.Log(e.Data);
+                process.ErrorDataReceived += (sender, e) => UnityEngine.Debug.LogError(e.Data);
+
+                process.Start();
+                process.BeginOutputReadLine();
+                process.BeginErrorReadLine();
+                process.WaitForExit();
+            }
+        });
+
+        UnityEngine.Debug.Log("Python script finished!");
+    }
+}

Assets/_PROJECT/Scripts/3DModeGeneration/ModelGenerationPipelineStarter.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 46e67223dce9b7a4783ed36b8ed65f19
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: