Compare commits

...

11 Commits

Author SHA1 Message Date
e25f1c75b5 update env example 2025-10-24 13:34:24 +03:00
c5c40f58ab Cube that starts generation pipeline on collision 2025-10-24 12:00:27 +03:00
Henri Sellis
2971027af2 fix start game UI 2025-10-24 10:22:53 +03:00
Henri Sellis
34a23ab94d make 3d model generation model folder loc configurable 2025-10-24 10:03:16 +03:00
Henri Sellis
8b3965d0b1 generated images and models into gitignore 2025-10-24 09:49:24 +03:00
42d7c0059b Add TODO Azure OpenAI API 2025-10-21 14:32:19 +00:00
Henri Sellis
6413d5a3d6 start pipeline script dynamically calculates absolute paths 2025-10-20 16:37:02 +03:00
Henri Sellis
e4f9423ca6 passing user prompt to pipeline script as cmd argument 2025-10-19 12:17:34 +03:00
Henri Sellis
1bb878848f combined pipeline running function 2025-10-18 16:44:38 +03:00
Henri Sellis
ae497eac6e test local text-to-image models and cloudflare API 2025-10-18 16:06:54 +03:00
Henri Sellis
2cf0a9f711 start of 3d generation pipelines 2025-10-11 18:37:46 +03:00
18 changed files with 969 additions and 4 deletions

View File

@ -0,0 +1,4 @@
CLOUDFLARE_ACCOUNT_ID=
CLOUDFLARE_API_TOKEN=
PIPELINE_FOLDER=
MODEL_FOLDER=

4
3d-generation-pipeline/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
.venv
.env
images/
models/

View File

@ -0,0 +1,2 @@
### TODO
* Azure OpenAI API: https://wiki.ut.ee/spaces/AA/pages/218073589/Azure+OpenAI+API+teenus

View File

@ -0,0 +1,28 @@
import torch
from diffusers import StableDiffusionPipeline, StableDiffusion3Pipeline
import time
start_timestamp = time.time()
#model = "stabilityai/stable-diffusion-3.5-medium" # generation time: 13 min
model = "stabilityai/stable-diffusion-3-medium-diffusers" # generation time: 10 min
#model = "stabilityai/stable-diffusion-2" # generation time: 4 sec
pipe = StableDiffusion3Pipeline.from_pretrained(model, torch_dtype=torch.float16)
#pipe = StableDiffusionPipeline.from_pretrained(model, torch_dtype=torch.float16)
pipe = pipe.to("cuda")
model_loaded_timestamp = time.time()
model_load_time = model_loaded_timestamp - start_timestamp
print(f"model load time: {round(model_load_time)} seconds")
prompt = "A majestic broadsword with a golden pommel, no background"
image = pipe(
prompt,
guidance_scale=3.0,
).images[0]
image_name = "image7.png"
image.save(f"images/{image_name}")
generation_time = time.time() - model_loaded_timestamp
print(f"image generation time: {round(generation_time)} seconds")

View File

@ -0,0 +1,152 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "1dc6faae",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import base64\n",
"import requests\n",
"from dotenv import load_dotenv"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b3107275",
"metadata": {},
"outputs": [],
"source": [
"load_dotenv()\n",
"\n",
"ACCOUNT_ID = os.environ[\"CLOUDFLARE_ACCOUNT_ID\"]\n",
"API_TOKEN = os.environ[\"CLOUDFLARE_API_TOKEN\"]"
]
},
{
"cell_type": "markdown",
"id": "999adf95",
"metadata": {},
"source": [
"## Text to image"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "40b35163",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Saved: output.jpg (263282 bytes)\n",
"Saved: image9.jpg (263282 bytes)\n"
]
}
],
"source": [
"MODEL = \"@cf/black-forest-labs/flux-1-schnell\"\n",
"URL = f\"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}\"\n",
"\n",
"payload = {\n",
" \"prompt\": \"a slightly curved broadsword with a fancy golden crossguard\",\n",
"}\n",
"\n",
"headers = {\n",
" \"Authorization\": f\"Bearer {API_TOKEN}\",\n",
" \"Content-Type\": \"application/json\",\n",
"}\n",
"\n",
"resp = requests.post(URL, json=payload, headers=headers, timeout=60)\n",
"resp.raise_for_status()\n",
"\n",
"data = resp.json()\n",
"b64 = data[\"result\"][\"image\"]\n",
"if not b64:\n",
" raise RuntimeError(f\"Unexpected response structure: {data}\")\n",
"\n",
"img_bytes = base64.b64decode(b64)\n",
"\n",
"out_path = \"output.jpg\"\n",
"with open(out_path, \"wb\") as f:\n",
" f.write(img_bytes)\n",
"\n",
"print(f\"Saved: {out_path} ({len(img_bytes)} bytes)\")"
]
},
{
"cell_type": "markdown",
"id": "14a874c4",
"metadata": {},
"source": [
"## Text prompt refinement"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "485f6f46",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\"dark wooden battleaxe with bronze blade\"\n"
]
}
],
"source": [
"MODEL = \"@cf/meta/llama-3.2-3b-instruct\"\n",
"URL = f\"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}\"\n",
"\n",
"instructions = \"\"\"\n",
"User is talking about some object. Your task is to generate a short and concise description of it. Use only user's own words, keep it as short as possible.\n",
"Example:\n",
"User: 'Umm, okay, I would like a really cool sword, with for example a bright orange crossguard. And also it should be slightly curved.'\n",
"You: 'a slightly curved sword with bright orange crossguard'\n",
"\"\"\"\n",
"prompt = \"Umm, alright, can you please give me an epic battleaxe? It should have a dark wooden shaft and bronze blade.\"\n",
"\n",
"response = requests.post(URL,\n",
" headers={\"Authorization\": f\"Bearer {API_TOKEN}\"},\n",
" json={\n",
" \"messages\": [\n",
" {\"role\": \"system\", \"content\": instructions},\n",
" {\"role\": \"user\", \"content\": prompt}\n",
" ]\n",
" }\n",
")\n",
"data = response.json()\n",
"result_text = data[\"result\"][\"response\"]\n",
"print(result_text)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -0,0 +1,105 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "2c0da293",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\henrisel.DOMENIS\\DeltaVR3DModelGeneration\\3d-generation-pipeline\\.venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import torch\n",
"from diffusers import FluxPipeline"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "51879ff1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Fetching 23 files: 0%| | 0/23 [00:00<?, ?it/s]"
]
}
],
"source": [
"model_name = \"black-forest-labs/FLUX.1-schnell\"\n",
"\n",
"pipe = FluxPipeline.from_pretrained(model_name, torch_dtype=torch.bfloat16)\n",
"#pipe.enable_model_cpu_offload() #save some VRAM by offloading the model to CPU. Remove this if you have enough GPU power"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "50a38bf4",
"metadata": {},
"outputs": [],
"source": [
"prompt = \"slightly curved sword, one side blue and other side green\"\n",
"pipe_result = pipe(\n",
" prompt,\n",
" guidance_scale=0.0,\n",
" num_inference_steps=4,\n",
" max_sequence_length=256,\n",
" generator=torch.Generator(\"gpu\")\n",
")\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b692177a",
"metadata": {},
"outputs": [],
"source": [
"pipe_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d55eb3ce",
"metadata": {},
"outputs": [],
"source": [
"image = pipe_result[\"images\"][0]\n",
"image.save(\"flux-schnell.png\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -0,0 +1,165 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4826c91d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2025-10-18-16-35-47'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from datetime import datetime\n",
"\n",
"datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "9419e692",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"STDOUT:\n",
" Device used: cuda\n",
"After Remesh 9998 19996\n",
"\n",
"STDERR:\n",
" D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\timm\\models\\layers\\__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers\n",
" warnings.warn(f\"Importing from {__name__} is deprecated, please import via timm.layers\", FutureWarning)\n",
"\n",
" 0%| | 0/1 [00:00<?, ?it/s]\n",
" 0%| | 0/1 [00:00<?, ?it/s]\n",
"Traceback (most recent call last):\n",
" File \"D:\\users\\henrisel\\stable-fast-3d\\run.py\", line 122, in <module>\n",
" mesh, glob_dict = model.run_image(\n",
" File \"D:\\users\\henrisel\\stable-fast-3d\\sf3d\\system.py\", line 286, in run_image\n",
" meshes, global_dict = self.generate_mesh(\n",
" File \"D:\\users\\henrisel\\stable-fast-3d\\sf3d\\system.py\", line 369, in generate_mesh\n",
" rast = self.baker.rasterize(\n",
" File \"D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\texture_baker\\baker.py\", line 28, in rasterize\n",
" return torch.ops.texture_baker_cpp.rasterize(\n",
" File \"D:\\users\\henrisel\\stable-fast-3d\\.venv\\lib\\site-packages\\torch\\_ops.py\", line 1243, in __call__\n",
" return self._op(*args, **kwargs)\n",
"NotImplementedError: Could not run 'texture_baker_cpp::rasterize' with arguments from the 'CUDA' backend. This could be because the operator doesn't exist for this backend, or was omitted during the selective/custom build process (if using custom build). If you are a Facebook employee using PyTorch on mobile, please visit https://fburl.com/ptmfixes for possible resolutions. 'texture_baker_cpp::rasterize' is only available for these backends: [CPU, Meta, BackendSelect, Python, FuncTorchDynamicLayerBackMode, Functionalize, Named, Conjugate, Negative, ZeroTensor, ADInplaceOrView, AutogradOther, AutogradCPU, AutogradCUDA, AutogradXLA, AutogradMPS, AutogradXPU, AutogradHPU, AutogradLazy, AutogradMTIA, AutogradMAIA, AutogradMeta, Tracer, AutocastCPU, AutocastMTIA, AutocastMAIA, AutocastXPU, AutocastMPS, AutocastCUDA, FuncTorchBatched, BatchedNestedTensor, FuncTorchVmapMode, Batched, VmapMode, FuncTorchGradWrapper, PythonTLSSnapshot, FuncTorchDynamicLayerFrontMode, PreDispatch, PythonDispatcher].\n",
"\n",
"CPU: registered at texture_baker\\csrc\\baker.cpp:543 [kernel]\n",
"Meta: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\MetaFallbackKernel.cpp:23 [backend fallback]\n",
"BackendSelect: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\BackendSelectFallbackKernel.cpp:3 [backend fallback]\n",
"Python: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:194 [backend fallback]\n",
"FuncTorchDynamicLayerBackMode: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\DynamicLayer.cpp:479 [backend fallback]\n",
"Functionalize: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\FunctionalizeFallbackKernel.cpp:375 [backend fallback]\n",
"Named: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\NamedRegistrations.cpp:7 [backend fallback]\n",
"Conjugate: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\ConjugateFallback.cpp:17 [backend fallback]\n",
"Negative: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\native\\NegateFallback.cpp:18 [backend fallback]\n",
"ZeroTensor: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\ZeroTensorFallback.cpp:86 [backend fallback]\n",
"ADInplaceOrView: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:104 [backend fallback]\n",
"AutogradOther: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:63 [backend fallback]\n",
"AutogradCPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:67 [backend fallback]\n",
"AutogradCUDA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:75 [backend fallback]\n",
"AutogradXLA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:87 [backend fallback]\n",
"AutogradMPS: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:95 [backend fallback]\n",
"AutogradXPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:71 [backend fallback]\n",
"AutogradHPU: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:108 [backend fallback]\n",
"AutogradLazy: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:91 [backend fallback]\n",
"AutogradMTIA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:79 [backend fallback]\n",
"AutogradMAIA: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:83 [backend fallback]\n",
"AutogradMeta: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\VariableFallbackKernel.cpp:99 [backend fallback]\n",
"Tracer: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\torch\\csrc\\autograd\\TraceTypeManual.cpp:294 [backend fallback]\n",
"AutocastCPU: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:322 [backend fallback]\n",
"AutocastMTIA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:466 [backend fallback]\n",
"AutocastMAIA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:504 [backend fallback]\n",
"AutocastXPU: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:542 [backend fallback]\n",
"AutocastMPS: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:209 [backend fallback]\n",
"AutocastCUDA: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\autocast_mode.cpp:165 [backend fallback]\n",
"FuncTorchBatched: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\LegacyBatchingRegistrations.cpp:731 [backend fallback]\n",
"BatchedNestedTensor: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\LegacyBatchingRegistrations.cpp:758 [backend fallback]\n",
"FuncTorchVmapMode: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\VmapModeRegistrations.cpp:27 [backend fallback]\n",
"Batched: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\LegacyBatchingRegistrations.cpp:1075 [backend fallback]\n",
"VmapMode: fallthrough registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\VmapModeRegistrations.cpp:33 [backend fallback]\n",
"FuncTorchGradWrapper: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\TensorWrapper.cpp:210 [backend fallback]\n",
"PythonTLSSnapshot: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:202 [backend fallback]\n",
"FuncTorchDynamicLayerFrontMode: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\functorch\\DynamicLayer.cpp:475 [backend fallback]\n",
"PreDispatch: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:206 [backend fallback]\n",
"PythonDispatcher: registered at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\aten\\src\\ATen\\core\\PythonFallbackKernel.cpp:198 [backend fallback]\n",
"\n",
"\n",
"Return Code: 1\n"
]
}
],
"source": [
"import subprocess\n",
"\n",
"MODEL_FOLDER = r\"D:\\users\\henrisel\\stable-fast-3d\"\n",
"PROJECT_FOLDER = r\"D:\\users\\henrisel\\DeltaVR3DModelGeneration\\3d-generation-pipeline\"\n",
"\n",
"# Path to the Python interpreter in the other virtual environment\n",
"venv_python = MODEL_FOLDER + r\"\\.venv\\Scripts\\python.exe\"\n",
"\n",
"# Path to the .py file you want to run\n",
"script_path = MODEL_FOLDER + r\"\\run.py\"\n",
"\n",
"# Optional: arguments to pass to the script\n",
"args = [MODEL_FOLDER + r\"\\demo_files\\examples\\chair1.png\", \"--output-dir\", PROJECT_FOLDER + r\"\\images\"]\n",
"\n",
"# Build the command\n",
"command = [venv_python, script_path] + args\n",
"\n",
"try:\n",
" # Run the subprocess\n",
" result = subprocess.run(command, capture_output=True, text=True)\n",
"\n",
" # Print output and errors\n",
" print(\"STDOUT:\\n\", result.stdout)\n",
" print(\"STDERR:\\n\", result.stderr)\n",
" print(\"Return Code:\", result.returncode)\n",
"\n",
"except Exception as e:\n",
" print(f\"Error occurred: {e}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ee480ba6",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -0,0 +1,7 @@
#torch==2.8.0+cu129 https://pytorch.org/get-started/previous-versions/
transformers==4.57.0
git+https://github.com/huggingface/diffusers.git
accelerate==1.10.1
huggingface_hub[hf_xet]==1.1.10
sentencepiece==0.2.1
protobuf==6.32.1

View File

@ -0,0 +1,111 @@
import os
import base64
import requests
import argparse
import subprocess
from pathlib import Path
from datetime import datetime
from dotenv import load_dotenv
load_dotenv()
ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"]
API_TOKEN = os.environ["CLOUDFLARE_API_TOKEN"]
PIPELINE_FOLDER = os.environ["PIPELINE_FOLDER"]
MODEL_FOLDER = os.environ["MODEL_FOLDER"]
def get_timestamp():
return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
def text_to_image(prompt, output_path):
MODEL = "@cf/black-forest-labs/flux-1-schnell"
URL = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}"
payload = {
"prompt": prompt,
}
headers = {
"Authorization": f"Bearer {API_TOKEN}",
"Content-Type": "application/json",
}
resp = requests.post(URL, json=payload, headers=headers, timeout=60)
resp.raise_for_status()
data = resp.json()
b64 = data["result"]["image"]
if not b64:
raise RuntimeError(f"Unexpected response structure: {data}")
img_bytes = base64.b64decode(b64)
with open(output_path, "wb") as f:
f.write(img_bytes)
def refine_text_prompt(prompt):
MODEL = "@cf/meta/llama-3.2-3b-instruct"
URL = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/run/{MODEL}"
instructions = """
User is talking about some object. Your task is to generate a short and concise description of it. Use only user's own words, keep it as short as possible.
Example:
User: 'Umm, okay, I would like a really cool sword, with for example a bright orange crossguard. And also it should be slightly curved.'
You: 'a slightly curved sword with bright orange crossguard'
"""
response = requests.post(URL,
headers={"Authorization": f"Bearer {API_TOKEN}"},
json={
"messages": [
{"role": "system", "content": instructions},
{"role": "user", "content": prompt}
]
}
)
data = response.json()
return data["result"]["response"]
def image_to_3d(image_path, output_path):
venv_python = MODEL_FOLDER + r"\.venv\Scripts\python.exe"
script_path = MODEL_FOLDER + r"\run.py"
args = [image_path, "--output-dir", output_path]
command = [venv_python, script_path] + args
try:
# Run the subprocess
result = subprocess.run(command, capture_output=True, text=True)
# Print output and errors
print("STDOUT:\n", result.stdout)
print("STDERR:\n", result.stderr)
print("Return Code:", result.returncode)
except Exception as e:
print(f"Error occurred: {e}")
def main():
parser = argparse.ArgumentParser(description="Text to 3D model pipeline")
parser.add_argument("--prompt", type=str, required=True, help="User text prompt")
args = parser.parse_args()
user_prompt = args.prompt
print(f"User prompt: {user_prompt}")
refined_prompt = refine_text_prompt(user_prompt)
print(f"Refined prompt: {refined_prompt}")
timestamp = get_timestamp()
pipeline_folder = Path(PIPELINE_FOLDER)
image_path = pipeline_folder / "images" / f"{timestamp}.jpg"
text_to_image(refined_prompt, image_path)
print(f"Generated image file: {image_path}")
model_path = pipeline_folder / "models" / timestamp
image_to_3d(image_path, model_path)
print(f"Generated 3D model file: {model_path}")
if __name__ == "__main__":
main()

View File

@ -7,7 +7,7 @@ TextureImporter:
mipmaps:
mipMapMode: 0
enableMipMap: 1
sRGBTexture: 1
sRGBTexture: 0
linearTexture: 0
fadeOut: 0
borderMipMap: 0
@ -54,7 +54,7 @@ TextureImporter:
alphaUsage: 1
alphaIsTransparency: 0
spriteTessellationDetail: -1
textureType: 0
textureType: 1
textureShape: 1
singleChannelComponent: 0
flipbookRows: 1

View File

@ -0,0 +1,133 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!21 &2100000
Material:
serializedVersion: 8
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_Name: Green
m_Shader: {fileID: 4800000, guid: 933532a4fcc9baf4fa0491de14d08ed7, type: 3}
m_Parent: {fileID: 0}
m_ModifiedSerializedProperties: 0
m_ValidKeywords: []
m_InvalidKeywords: []
m_LightmapFlags: 4
m_EnableInstancingVariants: 0
m_DoubleSidedGI: 0
m_CustomRenderQueue: -1
stringTagMap:
RenderType: Opaque
disabledShaderPasses: []
m_LockedProperties:
m_SavedProperties:
serializedVersion: 3
m_TexEnvs:
- _BaseMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _BumpMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailAlbedoMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailMask:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailNormalMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _EmissionMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _MainTex:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _MetallicGlossMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _OcclusionMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _ParallaxMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _SpecGlossMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_Lightmaps:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_LightmapsInd:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_ShadowMasks:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
m_Ints: []
m_Floats:
- _AlphaClip: 0
- _AlphaToMask: 0
- _Blend: 0
- _BlendModePreserveSpecular: 1
- _BumpScale: 1
- _ClearCoatMask: 0
- _ClearCoatSmoothness: 0
- _Cull: 2
- _Cutoff: 0.5
- _DetailAlbedoMapScale: 1
- _DetailNormalMapScale: 1
- _DstBlend: 0
- _DstBlendAlpha: 0
- _EnvironmentReflections: 1
- _GlossMapScale: 0
- _Glossiness: 0
- _GlossyReflections: 0
- _Metallic: 0
- _OcclusionStrength: 1
- _Parallax: 0.005
- _QueueOffset: 0
- _ReceiveShadows: 1
- _Smoothness: 0.5
- _SmoothnessTextureChannel: 0
- _SpecularHighlights: 1
- _SrcBlend: 1
- _SrcBlendAlpha: 1
- _Surface: 0
- _WorkflowMode: 1
- _ZWrite: 1
m_Colors:
- _BaseColor: {r: 0, g: 1, b: 0.0381248, a: 1}
- _Color: {r: 0, g: 1, b: 0.0381248, a: 1}
- _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
- _SpecColor: {r: 0.19999996, g: 0.19999996, b: 0.19999996, a: 1}
m_BuildTextureStacks: []
--- !u!114 &6221994712197478572
MonoBehaviour:
m_ObjectHideFlags: 11
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: d0353a89b1f911e48b9e16bdc9f2e058, type: 3}
m_Name:
m_EditorClassIdentifier:
version: 7

View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 937c5f357ed270843bd43d1f7d5d475b
NativeFormatImporter:
externalObjects: {}
mainObjectFileID: 2100000
userData:
assetBundleName:
assetBundleVariant:

View File

@ -0,0 +1,133 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!114 &-7093071968994914494
MonoBehaviour:
m_ObjectHideFlags: 11
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: d0353a89b1f911e48b9e16bdc9f2e058, type: 3}
m_Name:
m_EditorClassIdentifier:
version: 7
--- !u!21 &2100000
Material:
serializedVersion: 8
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_Name: Red
m_Shader: {fileID: 4800000, guid: 933532a4fcc9baf4fa0491de14d08ed7, type: 3}
m_Parent: {fileID: 0}
m_ModifiedSerializedProperties: 0
m_ValidKeywords: []
m_InvalidKeywords: []
m_LightmapFlags: 4
m_EnableInstancingVariants: 0
m_DoubleSidedGI: 0
m_CustomRenderQueue: -1
stringTagMap:
RenderType: Opaque
disabledShaderPasses: []
m_LockedProperties:
m_SavedProperties:
serializedVersion: 3
m_TexEnvs:
- _BaseMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _BumpMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailAlbedoMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailMask:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _DetailNormalMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _EmissionMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _MainTex:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _MetallicGlossMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _OcclusionMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _ParallaxMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- _SpecGlossMap:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_Lightmaps:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_LightmapsInd:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
- unity_ShadowMasks:
m_Texture: {fileID: 0}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0}
m_Ints: []
m_Floats:
- _AlphaClip: 0
- _AlphaToMask: 0
- _Blend: 0
- _BlendModePreserveSpecular: 1
- _BumpScale: 1
- _ClearCoatMask: 0
- _ClearCoatSmoothness: 0
- _Cull: 2
- _Cutoff: 0.5
- _DetailAlbedoMapScale: 1
- _DetailNormalMapScale: 1
- _DstBlend: 0
- _DstBlendAlpha: 0
- _EnvironmentReflections: 1
- _GlossMapScale: 0
- _Glossiness: 0
- _GlossyReflections: 0
- _Metallic: 0
- _OcclusionStrength: 1
- _Parallax: 0.005
- _QueueOffset: 0
- _ReceiveShadows: 1
- _Smoothness: 0.5
- _SmoothnessTextureChannel: 0
- _SpecularHighlights: 1
- _SrcBlend: 1
- _SrcBlendAlpha: 1
- _Surface: 0
- _WorkflowMode: 1
- _ZWrite: 1
m_Colors:
- _BaseColor: {r: 1, g: 0, b: 0, a: 1}
- _Color: {r: 1, g: 0, b: 0, a: 1}
- _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
- _SpecColor: {r: 0.19999996, g: 0.19999996, b: 0.19999996, a: 1}
m_BuildTextureStacks: []

View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 707a698b0ec80454a8c68700bca72941
NativeFormatImporter:
externalObjects: {}
mainObjectFileID: 2100000
userData:
assetBundleName:
assetBundleVariant:

Binary file not shown.

View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 0ec3982ba49c4b84ea95332cb090e115
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

View File

@ -0,0 +1,86 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Diagnostics;
using System.Threading.Tasks;
public class ModelGenerationPipelineStarter : MonoBehaviour
{
public Material activeMaterial;
public Material inactiveMaterial;
private MeshRenderer meshRenderer;
// Start is called before the first frame update
void Start()
{
meshRenderer = GetComponent<MeshRenderer>();
}
// Update is called once per frame
void Update()
{
}
void OnTriggerEnter(Collider other)
{
KbmController controller = other.GetComponent<KbmController>();
if (controller != null)
{
meshRenderer.material = activeMaterial;
StartModeGenerationPipeline();
}
}
private void OnTriggerExit(Collider other)
{
KbmController controller = other.GetComponent<KbmController>();
if (controller != null)
{
meshRenderer.material = inactiveMaterial;
}
}
private async void StartModeGenerationPipeline()
{
await Task.Run(() =>
{
string inputPrompt = "Uhm I want I think an epic broadsword with a fancy golden pommel";
// Path to your virtual environment's python.exe
string pythonExe = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\.venv\Scripts\python.exe";
// Path to your Python script
string scriptPath = @"D:\users\henrisel\DeltaVR3DModelGeneration\3d-generation-pipeline\start_pipeline.py";
// Arguments to pass to the script
string arguments = $"{scriptPath} --prompt \"{inputPrompt}\"";
ProcessStartInfo psi = new ProcessStartInfo
{
FileName = pythonExe,
Arguments = arguments,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};
using (Process process = new Process())
{
process.StartInfo = psi;
process.OutputDataReceived += (sender, e) => UnityEngine.Debug.Log(e.Data);
process.ErrorDataReceived += (sender, e) => UnityEngine.Debug.LogError(e.Data);
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
process.WaitForExit();
}
});
UnityEngine.Debug.Log("Python script finished!");
}
}

View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 46e67223dce9b7a4783ed36b8ed65f19
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: