implement cumprod

Need to store it as a image node.

.idea/dictionaries/bt.xml

@@ -31,6 +31,7 @@
       <w>hifigan</w>
       <w>hparams</w>
       <w>huggingface</w>
+      <w>kasisnu</w>
       <w>lipsync</w>
       <w>mocap</w>
       <w>mochi</w>
@@ -47,6 +48,7 @@
       <w>techcrunch</w>
       <w>transcoded</w>
       <w>transcodings</w>
+      <w>unet</w>
       <w>unpermitted</w>
       <w>upserting</w>
       <w>upserts</w>

app_config.yaml

@@ -0,0 +1,7 @@
+image_width: 512
+image_height: 512
+#scheduler_steps: 15
+# NB(bt): It looks like we can get down to 2 sampling steps, perhaps ?
+scheduler_steps: 2
+scheduler_samples: 1
+cfg_scale: 1.1

crates/desktop/src-tauri/Cargo.toml

@@ -18,11 +18,39 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 tauri-build = { version = "2.0.4", features = [] }
 
 [dependencies]
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-log = "0.4"
-tauri = { version = "2.2.4", features = [] }
-tauri-plugin-log = "2.0.0-rc"
-once_cell = "1.20.3"
-image = "0.25.5"
+
+# Internal
+filesys = { path = "../../lib/files/filesys" }
+memory_caching = { path = "../../lib/caching/memory_caching" }
+
+# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
+cudarc = "0.13.7"
+
+# External
+anyhow = "1.0"
+bytes = "1.10"
 base64 = "0.22.1"
+candle-core = { version = "0.8.2", features = ["cuda"] }
+candle-nn = { version = "0.8.2", features = ["cuda"] }
+candle-transformers = { version = "0.8.2", features = ["cuda"] }
+hf-hub = "0.3.2"
+image = "0.25.5"
+serde_yaml = "0.9"
+log = "0.4"
+once_cell = "1.20.3"
+rand = "0.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tauri = { version = "2.2.4", features = [] }
+tauri-plugin-log = "2"
+tokenizers = "0.15.2"
+
+[features]
+default = ["cuda"]
+cuda = [
+  "candle-core/cuda",
+  # TODO(bt,2025-02-22): cuDNN might speed up Stable Diffusion
+  # "candle-core/cudnn",
+  "candle-nn/cuda",
+  "candle-transformers/cuda",
+]

crates/desktop/src-tauri/capabilities/default.json

@@ -6,6 +6,7 @@
     "main"
   ],
   "permissions": [
-    "core:default"
+    "core:default",
+    "log:default"
   ]
-}
+}

crates/desktop/src-tauri/src/endpoints/flip_image.rs

@@ -1,11 +1,11 @@
-use std::io::Cursor;
+use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
-use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE};
 use image::codecs::png::{CompressionType, FilterType, PngEncoder};
 use image::ImageReader;
+use std::io::Cursor;
 
 #[tauri::command]
-pub fn infer_image(image: &str) -> Result<String, String> {
+pub fn flip_image(image: &str) -> Result<String, String> {
   println!("infer_image called; processing image...");
 
   let bytes = BASE64_STANDARD.decode(image)
@@ -18,17 +18,19 @@ pub fn infer_image(image: &str) -> Result<String, String> {
     .map_err(|err| format!("Image decode error: {}", err))?;
 
   let rotated_image = image.rotate180();
-  
+
   let mut buffer = Vec::new(); // TODO: Preallocate, better data structure
   let mut writer = Cursor::new(&mut buffer);
+
+  let encoder = PngEncoder::new_with_quality
+    (&mut writer, CompressionType::Fast, FilterType::Adaptive);
   
-  let encoder = PngEncoder::new_with_quality(&mut writer, CompressionType::Fast, FilterType::Adaptive);
   rotated_image.write_with_encoder(encoder)
     .map_err(|err| format!("Image encoding error: {}", err))?;
 
   let encoded = BASE64_STANDARD.encode(&buffer);
-  
+
   println!("Encoded: {:?}", encoded.split_at(10).0);
-  
+
   Ok(encoded)
 }

crates/desktop/src-tauri/src/endpoints/image_endpoint.rs

@@ -0,0 +1,117 @@
+use crate::ml::model_cache::ModelCache;
+use crate::ml::prompt_cache::PromptCache;
+use crate::ml::stable_diffusion::stable_diffusion_pipeline::{stable_diffusion_pipeline, Args};
+use crate::state::app_config::AppConfig;
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
+use bytes::BytesMut;
+use image::imageops::FilterType;
+use image::{DynamicImage, ImageFormat, ImageReader};
+use log::{error, info};
+use std::io::Cursor;
+use std::path::PathBuf;
+use tauri::State;
+
+const PROMPT_FILENAME : &str = "prompt.txt";
+const PROMPT: &str = "A beautiful landscape with mountains and a lake";
+
+const NEGATIVE_PROMPT: &str = "bad quality, bad faces, poor quality, blurry faces, watermark";
+
+/// This handler takes an image (as a base64 encoded string) and a prompt and returns
+/// an image (as a base64-encoded string).
+#[tauri::command]
+pub fn infer_image(
+  image: &str,
+  prompt: Option<String>,
+  strength: Option<u8>,
+  model_config: State<AppConfig>,
+  model_cache: State<ModelCache>,
+  prompt_cache: State<PromptCache>,
+) -> Result<String, String> {
+
+  let prompt = get_prompt_or_fallback(prompt);
+
+  info!("Prompt: {}", prompt);
+
+  let image = hydrate_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  let result = do_infer_image(&prompt, image, strength, &model_config, &model_cache, prompt_cache);
+  
+  if let Err(err) = result.as_deref() {
+    error!("There was an error: {:?}", err);
+  }
+  
+  result
+}
+
+fn do_infer_image(
+  prompt: &str,
+  image: DynamicImage,
+  strength: Option<u8>,
+  config: &AppConfig,
+  model_cache: &ModelCache,
+  prompt_cache: State<PromptCache>,
+) -> Result<String, String> {
+  println!("infer_image called; generating image with SDXL Turbo...");
+  
+  let args = Args {
+    image: &image,
+    prompt: prompt.to_string(),
+    uncond_prompt: NEGATIVE_PROMPT.to_string(),
+    //guidance_scale: Some(0.0),
+    model_cache,
+    configs: config,
+    prompt_cache: &prompt_cache,
+    i2i_strength: strength,
+    cfg_scale: config.cfg_scale,
+  };
+
+  match stable_diffusion_pipeline(args) {
+    Ok(image) => {
+      let mut bytes = Vec::with_capacity(1024*1024);
+      
+      image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
+        .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+      let bytes = BASE64_STANDARD.encode(bytes);
+
+      Ok(bytes)
+    },
+    Err(e) => Err(format!("Failed to generate image: {}", e)),
+  }
+}
+
+fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
+  let user_prompt = user_prompt.map(|prompt| prompt.trim().to_string())
+    .filter(|prompt| !prompt.is_empty());
+
+  if let Some(prompt) = user_prompt {
+    return prompt;
+  }
+
+  let prompt_file = PathBuf::from(PROMPT_FILENAME)
+    .canonicalize()
+    .unwrap_or_else(|_| PathBuf::from(PROMPT_FILENAME));
+
+  std::fs::read_to_string(&prompt_file)
+    .map_err(|err| format!("Failed to read prompt file: {}", err))
+    .unwrap_or_else(|_| {
+      error!("Failed to read prompt file: {:?}", prompt_file);
+      PROMPT.to_string()
+    })
+    .trim()
+    .to_string()
+}
+
+fn hydrate_base64_image(base64_image: &str) -> anyhow::Result<DynamicImage> {
+  let bytes = BASE64_STANDARD.decode(base64_image)?;
+
+  let image = ImageReader::new(Cursor::new(bytes))
+    .with_guessed_format()?
+    .decode()?;
+
+  // TODO(bt,2025-02-17): Running out of vram with full image buffer size
+  let image = image.resize(512, 512, FilterType::CatmullRom);
+
+  Ok(image)
+}

crates/desktop/src-tauri/src/endpoints/mod.rs

@@ -0,0 +1,3 @@
+pub mod flip_image;
+pub mod image_endpoint;
+pub mod test_counter;

crates/desktop/src-tauri/src/endpoints/test_counter.rs

@@ -0,0 +1,24 @@
+use std::sync::{Arc, RwLock};
+use once_cell::sync::Lazy;
+
+static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
+
+#[tauri::command]
+pub fn test_counter() -> u64 {
+  let value : u64;
+  {
+    match COUNTER.write() {
+      Ok(mut counter) => {
+        *counter += 1;
+        value = *counter;
+      },
+      Err(_e) => {
+        value = 0;
+      }
+    }
+  }
+
+  println!("I was invoked from JavaScript! {:?}", value);
+
+  value
+}

crates/desktop/src-tauri/src/lib.rs

@@ -1,55 +1,60 @@
-mod image_endpoint;
+pub mod endpoints;
+pub mod ml;
+pub mod state;
 
-use std::sync::{Arc, RwLock};
-use once_cell::sync::Lazy;
-use image_endpoint::infer_image;
+use crate::ml::model_cache::ModelCache;
+use crate::ml::prompt_cache::PromptCache;
+use crate::state::app_config::AppConfig;
+use endpoints::image_endpoint::infer_image;
+use endpoints::test_counter::test_counter;
+use tauri_plugin_log::Target;
+use tauri_plugin_log::TargetKind;
 
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
+  // NB: Tauri wants to install the logger itself, so we can't rely on the logger crate 
+  // until the tauri runtime begins.
+  println!("Loading model config...");
+
+  let config = AppConfig::init()
+    .expect("config should load");
+
+  let prompt_cache = PromptCache::with_capacity(8)
+    .expect("prompt cache should load");
+
+  println!("Creating model cache...");
+  
+  let model_cache = ModelCache::new(
+    config.device.clone(),
+    config.dtype,
+    config.sd_version.clone(),
+    config.sd_config.clone()
+  ).expect("Model cache should create");
+
+  println!("Initializing backend runtime...");
+
   tauri::Builder::default()
+    .plugin(tauri_plugin_log::Builder::new()
+      .level(log::LevelFilter::Info)
+      .targets(vec![Target::new(TargetKind::Stdout)])
+      .build())
     .setup(|app| {
-      if cfg!(debug_assertions) {
-        app.handle().plugin(
-          tauri_plugin_log::Builder::default()
-            .level(log::LevelFilter::Info)
-            .build(),
-        )?;
-      }
+      //if cfg!(debug_assertions) {
+      //  app.handle().plugin(
+      //    tauri_plugin_log::Builder::default()
+      //      .level(log::LevelFilter::Info)
+      //      .build(),
+      //  )?;
+      //}
       Ok(())
     })
+    .manage(config)
+    .manage(prompt_cache)
+    .manage(model_cache)
     .invoke_handler(tauri::generate_handler![
-      my_custom_command, 
-      test_round_trip, 
+      test_counter, 
       infer_image,
     ])
     .run(tauri::generate_context!())
     .expect("error while running tauri application");
 }
-
-#[tauri::command]
-fn my_custom_command() {
-  println!("I was invoked from JavaScript!");
-}
-
-static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
-
-#[tauri::command]
-fn test_round_trip() -> u64 {
-  let value : u64;
-  {
-    match COUNTER.write() {
-      Ok(mut counter) => {
-        *counter += 1;
-        value = *counter;
-      },
-      Err(_e) => {
-        value = 0;
-      }
-    }
-  }
-
-  println!("I was invoked from JavaScript! {:?}", value);
-
-  value
-}
-

crates/desktop/src-tauri/src/ml/cumprod.rs

@@ -0,0 +1,150 @@
+use candle_core::shape::Dim;
+use candle_core::Tensor;
+
+/// Tensor Cumulative Product
+/// 
+/// Candle doesn't implement cumprod, only cumsum.
+/// This implementation is adapted from cumsum in candle-ext:
+///  https://github.com/mokeyish/candle-ext/blob/main/tests/cumsum.rs
+///
+/// See Also: https://pytorch.org/docs/stable/generated/torch.cumprod.html
+/// See Also: https://numpy.org/doc/stable/reference/generated/numpy.cumprod.html
+/// See Also: https://github.com/huggingface/candle/issues/1646
+///
+pub fn cumprod<D: Dim>(input: &Tensor, dim: D) -> candle_core::Result<Tensor> {
+  let dim = dim.to_index(input.shape(), "cumprod")?;
+  let dim_size = input.dim(dim)?;
+  let rank = input.rank();
+  if rank == 0 {
+    return Ok(input.clone());
+  }
+
+  let mut tensors = Vec::with_capacity(dim_size);
+
+  let mut a = input.clone();
+  for i in 0..dim_size {
+    if i > 0 {
+      a = a.narrow(dim, 1, dim_size - i)?;
+      let b = input.narrow(dim, 0, dim_size - i)?;
+      a = (a * b)?;
+    }
+    tensors.push(a.narrow(dim, 0, 1)?);
+  }
+
+  let cumsum = Tensor::cat(&tensors, dim)?;
+  Ok(cumsum)
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::ml::cumprod::cumprod;
+  use candle_core::{Device, Tensor};
+
+  mod base_cases {
+    use super::*;
+
+    #[test]
+    fn zeroes_vector() -> anyhow::Result<()> {
+      let a = Tensor::new(&[0i64, 0, 0, 0], &Device::Cpu)?;
+      let b = cumprod(&a, 0)?;
+      assert_eq!(b.to_vec1::<i64>()?, &[0, 0, 0, 0]);
+      Ok(())
+    }
+
+    #[test]
+    fn zeroes_tensor() -> anyhow::Result<()> {
+      let a = Tensor::new(&[
+        [0i64, 0, 0],
+        [0i64, 0, 0],
+        [0i64, 0, 0],
+      ], &Device::Cpu)?;
+
+      let b = cumprod(&a, 0)?;
+
+      assert_eq!(b.to_vec2::<i64>()?, &[
+        [0, 0, 0],
+        [0, 0, 0],
+        [0, 0, 0],
+      ]);
+      Ok(())
+    }
+
+    #[test]
+    fn ones_vector() -> anyhow::Result<()> {
+      let a = Tensor::new(&[1i64, 1, 1, 1], &Device::Cpu)?;
+      let b = cumprod(&a, 0)?;
+      assert_eq!(b.to_vec1::<i64>()?, &[1, 1, 1, 1]);
+      Ok(())
+    }
+
+    #[test]
+    fn ones_tensor() -> anyhow::Result<()> {
+      let a = Tensor::new(&[
+        [1i64, 1, 1],
+        [1i64, 1, 1],
+        [1i64, 1, 1],
+      ], &Device::Cpu)?;
+
+      let b = cumprod(&a, 0)?;
+
+      assert_eq!(b.to_vec2::<i64>()?, &[
+        [1, 1, 1],
+        [1, 1, 1],
+        [1, 1, 1],
+      ]);
+      Ok(())
+    }
+  }
+
+  mod test_cases {
+    use super::*;
+
+    #[test]
+    fn vector() -> anyhow::Result<()> {
+      let a = Tensor::new(&[1i64, 2, 3, 4, 5], &Device::Cpu)?;
+      let b = cumprod(&a, 0)?;
+      assert_eq!(b.to_vec1::<i64>()?, &[1, 2, 6, 24, 120]);
+
+      let a = Tensor::new(&[0, 1i64, 2, 3, 4, 5], &Device::Cpu)?;
+      let b = cumprod(&a, 0)?;
+      assert_eq!(b.to_vec1::<i64>()?, &[0, 0, 0, 0, 0, 0]);
+      Ok(())
+    }
+
+    #[test]
+    fn tensor_dim_0() -> anyhow::Result<()> {
+      let a = Tensor::new(&[
+        [1i64, 2, 3],
+        [4i64, 5, 6],
+        [7i64, 8, 9],
+      ], &Device::Cpu)?;
+
+      let b = cumprod(&a, 0)?;
+
+      assert_eq!(b.to_vec2::<i64>()?, &[
+        [1, 2, 3],
+        [4, 10, 18],
+        [28, 80, 162],
+      ]);
+      Ok(())
+    }
+
+    #[test]
+    fn tensor_dim_1() -> anyhow::Result<()> {
+      let a = Tensor::new(&[
+        [1i64, 2, 3],
+        [4i64, 5, 6],
+        [7i64, 8, 9],
+      ], &Device::Cpu)?;
+
+      let b = cumprod(&a, 1)?;
+
+      assert_eq!(b.to_vec2::<i64>()?, &[
+        [1, 2, 6],
+        [4, 20, 120],
+        [7, 56, 504],
+      ]);
+      Ok(())
+    }
+  }
+}

crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs

@@ -0,0 +1,17 @@
+use anyhow::Result;
+use candle_core::{DType, Device, Tensor};
+use image::DynamicImage;
+
+pub fn dynamic_image_to_tensor(image: &DynamicImage, device: &Device, datatype: DType) -> Result<Tensor> {
+  let (height, width) = (image.height() as usize, image.width() as usize);
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let image = image.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom);
+  let image = image.to_rgb8();
+  let image = image.into_raw();
+  Ok(Tensor::from_vec(image, (height, width, 3), device)?
+    .permute((2, 0, 1))?
+    .to_dtype(datatype)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?)
+}

crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs

@@ -0,0 +1,28 @@
+use candle_core::{DType, Device, Tensor};
+
+pub fn load_image_file_to_tensor<T: AsRef<std::path::Path>>(path: T, device: &Device) -> anyhow::Result<Tensor> {
+  let img = image::ImageReader::open(path)?.decode()?;
+  let (height, width) = (img.height() as usize, img.width() as usize);
+  println!("INPUT IMAGE WIDTH AND HEIGHT: {}x{}", width, height);
+
+  // TODO: Downscale from 1024 -- YES! THIS HACK DOES THE TRICK OF GETTING THE TENSOR SHAPE TO [1,4,64,64] !!
+  let height = height / 2;
+  let width = width  / 2;
+  
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let img = img.resize_to_fill(
+    width as u32,
+    height as u32,
+    image::imageops::FilterType::CatmullRom,
+  );
+  println!("RESIZED IMAGE WIDTH AND HEIGHT: {}x{}", width, height);
+  let img = img.to_rgb8();
+  let img = img.into_raw();
+  let img = Tensor::from_vec(img, (height, width, 3), device)?
+    .permute((2, 0, 1))?
+    .to_dtype(DType::F32)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?;
+  Ok(img)
+}

crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs

@@ -0,0 +1,18 @@
+use anyhow::Result;
+use candle_core::{DType, Device, Tensor};
+
+pub fn load_image_file_to_tensor_2<T: AsRef<std::path::Path>>(path: T) -> Result<Tensor> {
+  let img = image::ImageReader::open(path)?.decode()?;
+  let (height, width) = (img.height() as usize, img.width() as usize);
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let img = img.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom);
+  let img = img.to_rgb8();
+  let img = img.into_raw();
+  let img = Tensor::from_vec(img, (height, width, 3), &Device::Cpu)?
+    .permute((2, 0, 1))?
+    .to_dtype(DType::F32)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?;
+  Ok(img)
+}

crates/desktop/src-tauri/src/ml/image/mod.rs

@@ -0,0 +1,5 @@
+pub mod dynamic_image_to_tensor;
+pub mod load_image_file_to_tensor;
+pub mod load_image_file_to_tensor_2;
+pub mod save_image_from_tensor;
+pub mod tensor_to_image_buffer;

crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs

@@ -0,0 +1,28 @@
+use anyhow::{Error as E, Result};
+use candle_core::Tensor;
+use image::{ImageBuffer, Rgb};
+
+pub fn save_image_from_tensor<P: AsRef<std::path::Path>>(img: &Tensor, p: P) -> Result<()> {
+  println!("Saving image to {:?}", p.as_ref());
+  let p = p.as_ref();
+  let (channel, height, width) = img.dims3()?;
+  println!("Image dimensions: {}x{} with {} channels", width, height, channel);
+
+  if channel != 3 {
+    anyhow::bail!("save_image expects an input of shape (3, height, width)")
+  }
+  let img = img.permute((1, 2, 0))?.flatten_all()?;
+  let pixels = img.to_vec1::<u8>()?;
+  println!("Converting tensor to image buffer...");
+
+  let image: ImageBuffer<Rgb<u8>, Vec<u8>> =
+    match ImageBuffer::from_raw(width as u32, height as u32, pixels) {
+      Some(image) => image,
+      None => anyhow::bail!("error saving image {p:?}"),
+    };
+  println!("Successfully created image buffer");
+
+  image.save(p).map_err(E::from)?;
+  println!("Successfully saved image to disk");
+  Ok(())
+}

crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs

@@ -0,0 +1,18 @@
+use candle_core::Tensor;
+use image::{ImageBuffer, Rgb};
+
+pub type RgbImage = ImageBuffer<Rgb<u8>, Vec<u8>>;
+
+pub fn tensor_to_image_buffer(img: &Tensor) -> anyhow::Result<RgbImage> {
+  let (channel, height, width) = img.dims3()?;
+
+  if channel != 3 {
+    anyhow::bail!("save_image expects an input of shape (3, height, width)")
+  }
+
+  let img = img.permute((1, 2, 0))?.flatten_all()?;
+  let pixels = img.to_vec1::<u8>()?;
+
+  ImageBuffer::from_raw(width as u32, height as u32, pixels)
+    .ok_or_else(|| anyhow::anyhow!("error loading image from raw tensor"))
+}

crates/desktop/src-tauri/src/ml/lcm_scheduler.rs

@@ -0,0 +1,474 @@
+use anyhow::anyhow;
+use candle_core::{DType, Device, Shape, Tensor, WithDType};
+use candle_transformers::models::stable_diffusion::schedulers::Scheduler;
+use candle_transformers::models::stable_diffusion::utils::linspace;
+use log::error;
+
+/*
+Notes on LCM implementation:
+
+  - Robust technical comparison of samplers: https://stable-diffusion-art.com/samplers/
+  - Apple Swift LCM Implementation ticket: https://github.com/apple/ml-stable-diffusion/issues/319
+  - Reference Swift Implementation: https://github.com/GuernikaCore/Schedulers/blob/main/Sources/Schedulers/LCMScheduler.swift
+
+Candle is missing several "batteries" we'll have to implement, and some we might want to upstream:
+
+  - t[::-1] (syntax to reverse the tensor)
+
+  - torch/numpy.cumprod :
+      - https://numpy.org/doc/stable/reference/generated/numpy.cumprod.html
+      - https://pytorch.org/docs/stable/generated/torch.cumprod.html
+
+Remaining work:
+
+  - [ ] implement add_noise
+  - [ ] implement cumprod
+  - [ ] initialize step_index / begin_index
+  - [ ] find out how `set_timestamps` is called in diffusers and see if we want to change it to CTOR param in candle
+  - [ ] try to plug this into candle and run
+  - [ ] fix timesteps() method to cache values
+  - [ ] compare with diffusers
+  - [ ] compare with candle schedulers
+  - [ ] compare with swift schedulers
+  - [ ] test + unit test
+*/
+
+struct LcmScheduler {
+  alphas: Tensor,
+  alphas_cumprod: Tensor,
+  final_alpha_cumprod: Tensor,
+
+  betas: Tensor,
+
+  timesteps: Tensor,
+
+  // Standard deviation of the initial noise distribution
+  // NB: Only in diffusers, and seems to only hold the value "1.0". Swift implementation omits it.
+  init_noise_sigma: f64,
+
+  // TODO: This is set by the set_timesteps() method after construction in python.
+  //  We might want to handle this upfront in the constructor ourselves
+  num_inference_steps: Option<usize>,
+
+  _step_index: Option<usize>,
+  _begin_index: Option<usize>,
+}
+
+impl LcmScheduler {
+  pub fn new(
+    num_train_timestamps: i64,
+    beta_start: f64,
+    beta_end: f64,
+    beta_schedule: &str,
+    trained_betas: Option<Tensor>,
+    set_alpha_to_one: bool,
+    rescale_betas_zero_snr: bool,
+    device: &Device,
+  ) -> anyhow::Result<Self> {
+    let betas;
+
+    if let Some(trained) = trained_betas {
+      betas = trained;
+    } else {
+      betas = match beta_schedule {
+        "scaled_linear" => {
+          // this schedule is very specific to the latent diffusion model.
+          let t = linspace(
+            beta_start.powf(0.5),
+            beta_end.powf(0.5),
+            num_train_timestamps as usize
+          )?;
+          t.to_device(device)?
+        }
+        _ => {
+          return Err(anyhow!("schedule {beta_schedule} is not implemented"))
+        }
+      };
+    }
+
+    if rescale_betas_zero_snr {
+      return Err(anyhow!("rescale_betas_zero_snr is not used upstream"));
+    }
+
+    let alphas = (1.0 - &betas)?;
+
+    // TODO: Implement cumprod: https://github.com/huggingface/candle/issues/1646
+    //  https://numpy.org/doc/stable/reference/generated/numpy.cumprod.html
+    error!("THIS IS USING CUMSUM, NOT CUMPROD. WE NEED TO IMPLEMENT CUMPROD.");
+    // self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)
+    let alphas_cumprod = alphas.cumsum(0)?;
+
+    // TODO: Check implementation
+    // At every step in ddim, we are looking into the previous alphas_cumprod
+    // For the final step, there is no previous alphas_cumprod because we are already at 0
+    // `set_alpha_to_one` decides whether we set this parameter simply to one or
+    // whether we use the final alpha of the "non-previous" one.
+    let final_alpha_cumprod = if set_alpha_to_one {
+      initialize_scalar_tensor(1.0, device)? // NB: `set_alpha_to_one` is false in python
+    } else {
+      alphas_cumprod.get(0)?
+    };
+
+    // standard deviation of the initial noise distribution
+    let init_noise_sigma = 1.0;
+
+    let timesteps = initialize_timesteps(num_train_timestamps, device)?;
+
+    Ok(Self {
+      alphas,
+      alphas_cumprod,
+      final_alpha_cumprod,
+      betas,
+      init_noise_sigma,
+      timesteps,
+      num_inference_steps: None,
+      _step_index: None,
+      _begin_index: None,
+    })
+  }
+
+  pub fn set_begin_index(&mut self, begin_index: usize) {
+    self._begin_index = Some(begin_index);
+  }
+
+  pub fn init_step_index(&mut self, timestep: usize) -> candle_core::Result<()> {
+    match self._begin_index {
+      None => {
+        todo!("implement begin index none branch")
+      }
+      Some(begin_index) => {
+        self._step_index = Some(begin_index);
+      }
+    }
+    Ok(())
+  }
+
+  pub fn set_timesteps(&mut self, num_inference_steps: usize, device: &Device) -> candle_core::Result<()> {
+    // NB: The diffusers version of this method passes `None` for `original_inference_steps` and `timesteps`.
+    // It passes strength = 1.0, and config.original_inference_steps = 50.
+    const STRENGTH : f64 = 1.0;
+    const CONFIG_ORIGINAL_INFERENCE_STEPS : usize = 50;
+    const CONFIG_NUM_TRAIN_TIMESTEPS : usize = 1000;
+
+    let original_steps = CONFIG_ORIGINAL_INFERENCE_STEPS;
+
+    // LCM Timesteps Setting
+    // The skipping step parameter k from the paper.
+    let k = (CONFIG_NUM_TRAIN_TIMESTEPS as f64 / original_steps as f64).floor();
+
+    let end = (original_steps as f64 * STRENGTH) as i64 + 1;
+    let lcm_origin_timesteps = ((Tensor::arange::<i64>(1, end, device)? * k)? - 1.0)?;
+
+    if num_inference_steps > CONFIG_NUM_TRAIN_TIMESTEPS {
+      return Err(candle_core::Error::Msg("num_inference_steps cannot be larger than train timesteps".to_string()));
+    }
+
+    let skipping_step = (lcm_origin_timesteps.dims().len() as f64 / num_inference_steps as f64).floor() as usize;
+
+    self.num_inference_steps = Some(num_inference_steps);
+    
+    // TODO: This is not efficient
+    let lcm_origin_timesteps = lcm_origin_timesteps.to_vec1::<i64>()?
+      .into_iter()
+      .map(i64::from)
+      .rev()
+      .collect::<Vec<i64>>();
+    
+    // TODO: This might not be exact. We should unit test this
+    let inference_indices = linspace(0.0, lcm_origin_timesteps.len() as f64, num_inference_steps)?;
+    
+    let inference_indices = inference_indices.floor()?
+      .to_dtype(DType::I64)?;
+    
+    let timestamps = {
+      // TODO: This code is awful
+      let inference_indices_t: Vec<usize> = inference_indices.to_vec1::<i64>()?
+        .into_iter()
+        .map(|n| n as usize)
+        .collect();
+
+      let mut timesteps: Vec<i64> = Vec::with_capacity(inference_indices.dims().len());
+      
+      for i in inference_indices_t {
+        if let Some(val) = lcm_origin_timesteps.get(i) {
+          timesteps.push(*val);
+        }
+      }
+      
+      let shape = Shape::from_dims(&[timesteps.len()]);
+      
+      Tensor::from_vec(timesteps, shape, device)?
+        .to_dtype(DType::I64)?
+        .to_device(device)?
+    };
+
+    self.timesteps = timestamps;
+
+    Ok(())
+  }
+
+}
+
+impl Scheduler for LcmScheduler {
+  fn timesteps(&self) -> &[usize] {
+    // TODO: We can't use the implementation from DDIM. This method requires
+    //  (1) an infallible response and (2) a reference we can't dynamically allocate. We must
+    //  cache this somewhere.
+    &[]
+  }
+
+  fn add_noise(&self, original_samples: &Tensor, noise: Tensor, timestep: usize) -> candle_core::Result<Tensor> {
+    // TODO: Make sure alphas_cumprod and timestep have same device and dtype as original_samples
+    //  Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement
+    //  for the subsequent add_noise calls
+    let mut alphas_cumprod = self.alphas_cumprod.to_device(original_samples.device())?;
+    alphas_cumprod = alphas_cumprod.to_dtype(original_samples.dtype())?;
+    //let timesteps = timesteps.to_device(original_samples.device())?;
+
+    let sqrt_alpha_prod = alphas_cumprod.get(timestep)?.powf(0.5)?;
+    let mut sqrt_alpha_prod = sqrt_alpha_prod.flatten_all()?; // TODO: Verify that flatten_all works here
+
+
+    while sqrt_alpha_prod.dims().len() < original_samples.dims().len() {
+      // TODO: Verify that this works for `unsqueeze(-1)`
+      sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(sqrt_alpha_prod.dims().len())?;
+    }
+
+    let sqrt_one_minus_alpha_prod = (1.0 - alphas_cumprod.get(timestep)?)?.powf(0.5)?;
+    let mut sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten_all()?; // TODO: Verify flatten all works here.
+
+    while sqrt_one_minus_alpha_prod.dims().len() < original_samples.dims().len() {
+      // TODO: Verify that this works for `unsqueeze(-1)`
+      sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(sqrt_one_minus_alpha_prod.dims().len())?;
+    }
+
+    let noisy_samples = ((sqrt_alpha_prod * original_samples)? + (sqrt_one_minus_alpha_prod * noise)?)?;
+
+    Ok(noisy_samples)
+  }
+
+  fn init_noise_sigma(&self) -> f64 {
+    self.init_noise_sigma
+  }
+
+  /// Ensures interchangeability with schedulers that need to scale the denoising model input
+  /// depending on the current timestep. (For LCM, this is a pass through no-op!)
+  fn scale_model_input(&self, sample: Tensor, _timestep: usize) -> candle_core::Result<Tensor> {
+    Ok(sample) // NB: The diffusers scheduling_lcm.py implementation is a pass through no-op !
+  }
+
+  /// Predict the sample from the previous timestep by reversing the SDE. This function
+  /// propagates the diffusion process from the learned model outputs (most often the
+  /// predicted noise).
+  fn step(&mut self, model_output: &Tensor, timestep: usize, sample: &Tensor) -> candle_core::Result<Tensor> {
+
+    if self.num_inference_steps.is_none() {
+      return Err(candle_core::Error::Msg(
+        "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
+          .to_string()));
+    }
+
+    if self._step_index.is_none() {
+      self.init_step_index(timestep)?;
+    }
+
+    let step_index = self._step_index.clone().expect("unsafe unwrap; should exist per above");
+
+    // 1. Get previous step value
+    let prev_step_index = step_index + 1;
+    let prev_timestep;
+
+    if prev_step_index < self.timesteps.elem_count() {
+      prev_timestep = self.timesteps.get(timestep)?.to_scalar::<i64>()? as usize;
+    } else {
+      prev_timestep = timestep;
+    }
+
+    // 2. Compute alphas, betas
+    let alpha_prod_t = self.alphas_cumprod.get(timestep)?;
+    let alpha_prod_t_prev = if prev_timestep >= 0 {
+      self.alphas_cumprod.get(prev_timestep)?
+    } else {
+      self.final_alpha_cumprod.clone()
+    };
+
+    let beta_prod_t = (1. - &alpha_prod_t)?;
+    let beta_prod_t_prev = (1. - &alpha_prod_t_prev)?;
+
+    // 3. Get scaling for boundary conditions
+    // NB: timestep_scaling seems to be always set to 10.0 in diffusers examples, but it is passed
+    // via a config object. Hardcoding this for now until we need to change it.
+    const TIMESTEP_SCALING : f64 = 10.0;
+    let (c_skip, c_out) = get_scalings_for_boundary_condition_discrete(timestep, TIMESTEP_SCALING);
+
+    // 4. Compute the predicted original sample x_0 based on the model parameterization
+    // NB: Prediction type is "epsilon" in the diffusers examples, but that is configurable.
+    // This is the only one we'll implement for now.
+    let predicted_original_sample = ((sample - (beta_prod_t.sqrt()? * model_output)?)? / &alpha_prod_t.sqrt()?)?;
+
+    // 5. (skip clip/threshold - not used in diffusers example)
+
+    // 6. Denoise model output using boundary conditions
+    let denoised = ((c_out * predicted_original_sample)? + (c_skip * sample)?)?;
+
+    // TODO: Step 7. We have to return the previous example somehow.
+    //  Diffusers tolerates it not being sent back, but the quality degrades as if one step is absent.
+    Ok(denoised)
+  }
+}
+
+fn initialize_scalar_tensor<D: WithDType>(value: D, device: &Device) -> anyhow::Result<Tensor> {
+  let scalar = Tensor::from_vec(vec![value], Shape::from(vec![]), device)?;
+  Ok(scalar)
+}
+
+// TODO(bt,2025-02-23): Use native implementations
+fn initialize_timesteps(num_train_timesteps: i64, device: &Device) -> anyhow::Result<Tensor> {
+  // Testing LCM scheduler assumptions:
+  // [::-1] is shorthand for reversing a numpy vector, eg. [0, 1,2,3,4] -> [4, 3, 2, 1, 0]
+  // self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64))
+  //let tensor = Tensor::arange::<i64>(0, 5, &Device::Cpu)?; // This works, but we can't reverse/flip it!
+
+  // https://github.com/tracel-ai/burn/pull/1468 flip impl? Will this work in place?
+  // https://github.com/huggingface/candle/issues/1875
+
+  let data = (0..num_train_timesteps)
+    .map(i64::from)
+    .rev()
+    .collect();
+
+  let shape = Shape::from_dims(&[num_train_timesteps as usize]);
+
+  let tensor = Tensor::from_vec(data, shape, device)?;
+
+  Ok(tensor)
+}
+
+fn get_scalings_for_boundary_condition_discrete(timestep: usize, timestep_scaling: f64) -> (f64, f64) {
+  let sigma_data = 0.5f64; // Default 0.5
+  let scaled_timestep = timestep as f64 * timestep_scaling;
+
+  let c_skip = sigma_data.powf(2.0) / (scaled_timestep.powf(2.0) + sigma_data.powf(2.0));
+  let c_out = scaled_timestep / (scaled_timestep.powf(2.0) + sigma_data.powf(2.0)).powf(0.5);
+
+  (c_skip, c_out)
+}
+
+// The Candle sources, and moreover, the Candle tests, are a great source of information for how the library works.
+//assert_eq!(t.i(..-1)?, 1.0);
+#[cfg(test)]
+mod tests {
+  use crate::ml::lcm_scheduler::{initialize_scalar_tensor, initialize_timesteps};
+  use crate::ml::lcm_scheduler::tests::test_helper_fn::*;
+  use candle_core::Device;
+
+  mod helper_function_unit_tests {
+    use crate::ml::lcm_scheduler::get_scalings_for_boundary_condition_discrete;
+    use super::*;
+
+    #[test]
+    fn test_initialize_scalar_tensor() -> anyhow::Result<()> {
+      let t = initialize_scalar_tensor(1.0, &Device::Cpu)?;
+      let empty: [usize; 0] = [];
+      assert_eq!(t.dims(), &empty);
+      assert_eq!(t.to_vec0::<f64>()?, 1.0);
+      Ok(())
+    }
+
+    #[test]
+    fn test_initialize_timesteps() -> anyhow::Result<()> {
+      let tensor = initialize_timesteps(5, &Device::Cpu)?;
+      assert_eq!(tensor.to_vec1::<i64>()?, &[4, 3, 2, 1,0]);
+      Ok(())
+    }
+
+    #[test]
+    fn test_get_scalings_for_boundary_condition_discrete() -> anyhow::Result<()> {
+      // Base cases
+      let v = get_scalings_for_boundary_condition_discrete(0, 0.0);
+      assert_eq!(v, (1.0, 0.0));
+      let v = get_scalings_for_boundary_condition_discrete(1, 0.0);
+      assert_eq!(v, (1.0, 0.0));
+      let v = get_scalings_for_boundary_condition_discrete(0, 1.0);
+      assert_eq!(v, (1.0, 0.0));
+
+      // Production code has "timestep_scaling" = 10.0
+      let v = get_scalings_for_boundary_condition_discrete(0, 10.0);
+      assert_eq!(v, (1.0, 0.0));
+
+      // Production code has "timestep_scaling" = 10.0
+      let v = get_scalings_for_boundary_condition_discrete(1, 10.0);
+      assert!((v.0 - 0.0024937655860349127).abs() < 0.01);
+      assert!((v.1 - 0.9987523388778445).abs() < 0.01);
+
+      // TODO: More test cases -
+      //let v = get_scalings_for_boundary_condition_discrete(10, 10.0);
+      //let v = get_scalings_for_boundary_condition_discrete(100, 10.0);
+
+      Ok(())
+    }
+  }
+
+  mod confirm_various_pytorch_behaviors {
+    use candle_core::IndexOp;
+    use super::*;
+
+    // NB: Non-unit tests, just to test understanding while porting code
+
+    #[test]
+    fn unsqueeze_zero() -> anyhow::Result<()> {
+      let t = tensor_1234()?;
+      let x = t.unsqueeze(0)?;
+      assert_eq!(x.to_vec2::<i64>()?, &[&[1, 2, 3, 4]]);
+      Ok(())
+    }
+
+    #[test]
+    fn unsqueeze_simulating_negative_one() -> anyhow::Result<()> {
+      let t = tensor_1234()?;
+      // NB: It looks like this matches the behavior of unsqueeze(-1)
+      // Per: https://pytorch.org/docs/stable/generated/torch.unsqueeze.html
+      let x = t.unsqueeze(t.dims().len())?;
+      assert_eq!(x.to_vec2::<i64>()?, &[&[1],&[2],&[3],&[4]]);
+      Ok(())
+    }
+
+    #[test]
+    fn one_minus_vector() -> anyhow::Result<()> {
+      let t = tensor_01234()?;
+      let x = (1. - t)?;
+      assert_eq!(x.rank(), 1); // NB: 1-dimensional output
+      assert_eq!(x.to_vec1::<i64>()?, &[1, 0, -1, -2, -3]);
+      Ok(())
+    }
+    
+    // #[test]
+    // fn reverse_vector() -> anyhow::Result<()> {
+    //   let t = tensor_1234()?;
+    //   let x = t.i(5..0)?;
+    //   assert_eq!(x.rank(), 1); // NB: 1-dimensional output
+    //   assert_eq!(x.to_vec1::<i64>()?, &[1, 0, -1, -2, -3]);
+    //   Ok(())
+    // }
+  }
+
+  pub mod test_helper_fn {
+    use candle_core::{Device, Shape, Tensor};
+
+    pub fn tensor_1234() -> anyhow::Result<Tensor> {
+      // Return an example tensor
+      let data: Vec<i64> = vec![1, 2, 3, 4];
+      let t = Tensor::from_vec(data, Shape::from(vec![4]), &Device::Cpu)?;
+      assert_eq!(t.to_vec1::<i64>()?, &[1, 2, 3, 4]);
+      Ok(t)
+    }
+
+    pub fn tensor_01234() -> anyhow::Result<Tensor> {
+      // Return an example tensor
+      let data: Vec<i64> = vec![0, 1, 2, 3, 4];
+      let t = Tensor::from_vec(data, Shape::from(vec![5]), &Device::Cpu)?;
+      assert_eq!(t.to_vec1::<i64>()?, &[0, 1, 2, 3, 4]);
+      Ok(t)
+    }
+  }
+}

crates/desktop/src-tauri/src/ml/mod.rs

@@ -0,0 +1,8 @@
+pub mod cumprod;
+pub mod image;
+pub mod lcm_scheduler;
+pub mod model_cache;
+pub mod model_file;
+pub mod models;
+pub mod prompt_cache;
+pub mod stable_diffusion;

crates/desktop/src-tauri/src/ml/model_cache.rs

@@ -0,0 +1,118 @@
+use crate::ml::models::unet_model::UNetModel;
+use crate::ml::model_file::StableDiffusionVersion;
+use anyhow::anyhow;
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use hf_hub::api::sync::Api;
+use std::sync::{Arc, RwLock};
+use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution;
+use crate::ml::models::lazy_load_vae_model::LazyLoadVaeModel;
+// TODO: This data structure is gross. Generalize the locking and loading semantics for reuse.
+
+// Simple registry for now. We can build complex machinery that aids in 
+// VRAM utilization, disk space saving, intelligent scheduling, etc. in the future.
+pub struct ModelCache {
+  // TODO(bt,2025-02-14): Support for more than one device in the future.
+  device: Device,
+  dtype: DType,
+  hf_api: Api,
+  sd_config: StableDiffusionConfig,
+  sd_version: StableDiffusionVersion,
+  
+  unet: Arc<RwLock<Option<UNetModel>>>,
+  lazy_vae: LazyLoadVaeModel,
+}
+
+impl ModelCache {
+  pub fn new(
+    device: Device, 
+    dtype: DType,
+    sd_version: StableDiffusionVersion, 
+    sd_config: StableDiffusionConfig,
+  ) -> anyhow::Result<Self> {
+    let api = Api::new()?;
+    
+    Ok(Self { 
+      device: device.clone(),
+      dtype,
+      hf_api: api.clone(),
+      unet: Arc::new(RwLock::new(None)),
+      lazy_vae: LazyLoadVaeModel::new(
+        &sd_config,
+        &sd_version,
+        &api,
+        &device,
+        dtype,
+      )?,
+      sd_config,
+      sd_version,
+    })
+  }
+
+  pub fn vae_encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
+    self.lazy_vae.encode(xs)
+  }
+
+  pub fn vae_decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
+    self.lazy_vae.decode(xs)
+  }
+  
+  pub fn unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
+    println!("unet_inference");
+    match self.try_unet_inference(latent_model_input, timestep, text_embeddings) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_unet()?;
+        self.try_unet_inference(latent_model_input, timestep, text_embeddings)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+  
+  fn load_unet(&self) -> anyhow::Result<()> {
+    println!("load_unet");
+    match self.unet.write() {
+      Err(err) => return Err(anyhow!("lock error: {:?}", err)),
+      Ok(mut maybe_model) => {
+        match &*maybe_model {
+          Some(_model) => {} // Fall through
+          None => {
+            let repo = self.sd_version.repo();
+            
+            println!("Downloading UNET model files from: {} ...", repo);
+
+            let unet_file = self.hf_api.model(repo.to_string())
+              .get("unet/diffusion_pytorch_model.safetensors")
+              .map_err(|err| anyhow!("error fetching model: {:?}", err))?;
+
+            println!("Building UNET model... (3)");
+            let unet = UNetModel::new(&self.sd_config, unet_file, &self.device, self.dtype)
+              .map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
+
+            println!("Built UNET...");
+            *maybe_model = Some(unet);
+          }
+        }
+      }
+    }
+    Ok(())
+  }
+
+  pub fn try_unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Option<Tensor>> {
+    println!("try_unet_inference");
+    match self.unet.read() {
+      Ok(mut model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.inference(latent_model_input, timestep, text_embeddings)?))
+          }
+        }
+      }
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+    }
+  }
+  
+}

crates/desktop/src-tauri/src/ml/model_file.rs

@@ -0,0 +1,83 @@
+#[cfg(feature = "accelerate")]
+extern crate accelerate_src;
+#[cfg(feature = "mkl")]
+extern crate intel_mkl_src;
+
+use candle_transformers::models::stable_diffusion;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ModelFile {
+  Tokenizer,
+  Tokenizer2,
+  Clip,
+  Clip2,
+  Unet,
+  Vae,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StableDiffusionVersion {
+  V1_5,
+  V1_5Inpaint,
+  V2_1,
+  V2Inpaint,
+  Xl,
+  XlInpaint,
+  Turbo,
+}
+
+impl StableDiffusionVersion {
+  pub fn repo(&self) -> &'static str {
+    match self {
+      Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
+      Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0",
+      Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting",
+      Self::V2_1 => "stabilityai/stable-diffusion-2-1",
+      Self::V1_5 => "runwayml/stable-diffusion-v1-5",
+      Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting",
+      Self::Turbo => "stabilityai/sdxl-turbo",
+    }
+  }
+}
+
+impl ModelFile {
+  pub fn get(
+    &self,
+    filename: Option<String>,
+    version: StableDiffusionVersion,
+    use_f16: bool,
+  ) -> anyhow::Result<std::path::PathBuf> {
+    use hf_hub::api::sync::Api;
+    match filename {
+      Some(filename) => Ok(std::path::PathBuf::from(filename)),
+      None => {
+        let (repo, path) = match self {
+          Self::Tokenizer => {
+            let tokenizer_repo = match version {
+              StableDiffusionVersion::V1_5
+              | StableDiffusionVersion::V2_1
+              | StableDiffusionVersion::V1_5Inpaint
+              | StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32",
+              StableDiffusionVersion::Xl
+              | StableDiffusionVersion::XlInpaint
+              | StableDiffusionVersion::Turbo => {
+                "openai/clip-vit-large-patch14"
+              }
+            };
+            (tokenizer_repo, "tokenizer.json")
+          }
+          Self::Tokenizer2 => {
+            ("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json")
+          }
+          Self::Clip => (version.repo(), "text_encoder/model.safetensors"),
+          Self::Clip2 => (version.repo(), "text_encoder_2/model.safetensors"),
+          Self::Unet => (version.repo(), "unet/diffusion_pytorch_model.safetensors"),
+          Self::Vae => (version.repo(), "vae/diffusion_pytorch_model.safetensors"),
+        };
+        let filename = Api::new()?.model(repo.to_string()).get(path)?;
+        Ok(filename)
+      }
+    }
+  }
+}
+

crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs

@@ -0,0 +1,130 @@
+use crate::ml::model_file::StableDiffusionVersion;
+use anyhow::anyhow;
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use hf_hub::api::sync::Api;
+use std::sync::{Arc, RwLock};
+
+pub struct LazyLoadVaeModel {
+  lazy_model: Arc<RwLock<Option<AutoEncoderKL>>>,
+  model_configs: VaeConfigs,
+}
+
+// TODO: Maybe we pass these via a central config read at startup, passed as an Arc<ConfigT>
+pub struct VaeConfigs {
+  sd_config: StableDiffusionConfig,
+  sd_version: StableDiffusionVersion,
+  device: Device,
+  dtype: DType,
+  hf_api: Api,
+}
+
+impl LazyLoadVaeModel {
+  pub fn new(
+    sd_config: &StableDiffusionConfig,
+    sd_version: &StableDiffusionVersion,
+    hf_api: &Api,
+    device: &Device,
+    dtype: DType,
+  ) -> anyhow::Result<Self> {
+    println!("building vae model... (1)");
+    
+    Ok(Self {
+      lazy_model: Arc::new(RwLock::new(None)),
+      model_configs: VaeConfigs {
+        sd_config: sd_config.clone(),
+        sd_version: sd_version.clone(),
+        device: device.clone(),
+        hf_api: hf_api.clone(),
+        dtype
+      }
+    })
+  }
+
+  pub fn encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
+    println!("vae encode...");
+    match self.try_encode(xs) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_model()?;
+        self.try_encode(xs)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+  
+  pub fn decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
+    println!("vae decode...");
+    match self.try_decode(xs) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_model()?;
+        self.try_decode(xs)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+
+  fn try_encode(&self, xs: &Tensor) -> anyhow::Result<Option<DiagonalGaussianDistribution>> {
+    match self.lazy_model.read() {
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+      Ok(model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.encode(xs)?))
+          }
+        }
+      }
+    }
+  }
+
+  fn try_decode(&self, xs: &Tensor) -> anyhow::Result<Option<Tensor>> {
+    match self.lazy_model.read() {
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+      Ok(model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.decode(xs)?))
+          }
+        }
+      }
+    }
+  }
+
+  fn load_model(&self) -> anyhow::Result<()> {
+    println!("load vae");
+    match self.lazy_model.write() {
+      Err(err) => return Err(anyhow!("lock error: {:?}", err)),
+      Ok(mut maybe_model) => {
+        match &*maybe_model {
+          Some(_model) => {} // Already loaded; fall through
+          None => {
+            let repo = self.model_configs.sd_version.repo();
+
+            println!("Building VAE model from : {:?} ... (3)", repo);
+            
+            let vae_file = self.model_configs.hf_api.model(repo.to_string())
+              .get("vae/diffusion_pytorch_model.safetensors")?;
+
+            println!("Building VAE model from file {:?}...", &vae_file);
+            let vae = self.model_configs
+              .sd_config
+              .build_vae(vae_file, &self.model_configs.device, self.model_configs.dtype)?;
+
+            println!("Built VAE...");
+            *maybe_model = Some(vae);
+          }
+        }
+      }
+    }
+    Ok(())
+  }
+ 
+}

crates/desktop/src-tauri/src/ml/models/mod.rs

@@ -0,0 +1,2 @@
+pub mod unet_model;
+pub mod lazy_load_vae_model;

crates/desktop/src-tauri/src/ml/models/unet_model.rs

@@ -0,0 +1,35 @@
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use std::path::Path;
+
+pub struct UNetModel {
+  model: UNet2DConditionModel,
+}
+
+impl UNetModel {
+  pub fn new<P: AsRef<Path>>(
+    sd_config: &StableDiffusionConfig,
+    unet_weights_file: P,
+    device: &Device,
+    dtype: DType,
+  ) -> anyhow::Result<Self> {
+    println!("building unet model... (1)");
+    
+    let model = sd_config.build_unet(
+      unet_weights_file, 
+      &device, 
+      4, 
+      false, 
+      dtype
+    )?;
+    
+    Ok(Self {
+      model,
+    })
+  }
+  
+  pub fn inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
+    Ok(self.model.forward(&latent_model_input, timestep, &text_embeddings)?)
+  }
+}

crates/desktop/src-tauri/src/ml/prompt_cache.rs

@@ -0,0 +1,22 @@
+use candle_core::Tensor;
+use memory_caching::arc_sieve::ArcSieve;
+
+pub struct PromptCache {
+  cache: ArcSieve<String, Tensor>,
+}
+
+impl PromptCache {
+  pub fn with_capacity(capacity: usize) -> anyhow::Result<Self> {
+    Ok(Self {
+      cache: ArcSieve::with_capacity(capacity)?,
+    })
+  }
+  
+  pub fn get_copy(&self, key: &String) -> anyhow::Result<Option<Tensor>> {
+    self.cache.get_copy(key)
+  }
+  
+  pub fn store_copy(&self, key: &String, value: &Tensor) -> anyhow::Result<()> {
+    self.cache.store_copy(key, value)
+  }
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs

@@ -0,0 +1,13 @@
+use crate::ml::model_file::StableDiffusionVersion;
+
+pub fn get_vae_scale(sd_version: StableDiffusionVersion) -> f64 {
+  match sd_version {
+    StableDiffusionVersion::V1_5
+    | StableDiffusionVersion::V1_5Inpaint
+    | StableDiffusionVersion::V2_1
+    | StableDiffusionVersion::V2Inpaint
+    | StableDiffusionVersion::XlInpaint
+    | StableDiffusionVersion::Xl => 0.18215,
+    StableDiffusionVersion::Turbo => 0.13025,
+  }
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/infer_clip_text_embeddings.rs

@@ -0,0 +1,153 @@
+#[cfg(feature = "accelerate")]
+extern crate accelerate_src;
+#[cfg(feature = "mkl")]
+extern crate intel_mkl_src;
+
+use candle_transformers::models::stable_diffusion;
+
+use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
+use anyhow::Error as E;
+use candle_core::{DType, Device, Module, Tensor, D};
+use log::info;
+use tokenizers::Tokenizer;
+
+#[allow(clippy::too_many_arguments)]
+pub fn infer_clip_text_embeddings(
+  prompt: &str,
+  uncond_prompt: &str,
+  tokenizer: Option<String>,
+  clip_weights: Option<String>,
+  clip2_weights: Option<String>,
+  sd_version: StableDiffusionVersion,
+  sd_config: &stable_diffusion::StableDiffusionConfig,
+  use_f16: bool,
+  device: &Device,
+  dtype: DType,
+  use_guide_scale: bool,
+) -> anyhow::Result<Tensor> {
+  info!("Preparing text embeddings...");
+
+  let which = match sd_version {
+    StableDiffusionVersion::Xl
+    | StableDiffusionVersion::XlInpaint
+    | StableDiffusionVersion::Turbo => vec![true, false],
+    _ => vec![true],
+  };
+
+  let text_embeddings = which
+    .iter()
+    .map(|first| {
+      do_infer_clip_text_embeddings(
+        &prompt,
+        &uncond_prompt,
+        None, // tokenizer
+        None, // clip_weights
+        None, // clip2_weights
+        sd_version,
+        &sd_config,
+        false, // use_f16
+        &device,
+        dtype,
+        use_guide_scale,
+        *first,
+      )
+    })
+    .collect::<anyhow::Result<Vec<_>>>()?;
+
+  let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?;
+  
+  Ok(text_embeddings)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn do_infer_clip_text_embeddings(
+    prompt: &str,
+    uncond_prompt: &str,
+    tokenizer: Option<String>,
+    clip_weights: Option<String>,
+    clip2_weights: Option<String>,
+    sd_version: StableDiffusionVersion,
+    sd_config: &stable_diffusion::StableDiffusionConfig,
+    use_f16: bool,
+    device: &Device,
+    dtype: DType,
+    use_guide_scale: bool,
+    first: bool,
+  ) -> anyhow::Result<Tensor> {
+  
+  let tokenizer_file = if first {
+    ModelFile::Tokenizer
+  } else {
+    ModelFile::Tokenizer2
+  };
+  let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
+  let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?;
+  let pad_id = match &sd_config.clip.pad_with {
+    Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(),
+    None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(),
+  };
+
+  println!("Running with prompt \"{prompt}\".");
+  let mut tokens = tokenizer
+    .encode(prompt, true)
+    .map_err(E::msg)?
+    .get_ids()
+    .to_vec();
+  if tokens.len() > sd_config.clip.max_position_embeddings {
+    anyhow::bail!(
+            "the prompt is too long, {} > max-tokens ({})",
+            tokens.len(),
+            sd_config.clip.max_position_embeddings
+        )
+  }
+  while tokens.len() < sd_config.clip.max_position_embeddings {
+    tokens.push(pad_id)
+  }
+  let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
+
+  println!("Building the Clip transformer.");
+  let clip_weights_file = if first {
+    ModelFile::Clip
+  } else {
+    ModelFile::Clip2
+  };
+  let clip_weights = if first {
+    clip_weights_file.get(clip_weights, sd_version, use_f16)?
+  } else {
+    clip_weights_file.get(clip2_weights, sd_version, use_f16)?
+  };
+  let clip_config = if first {
+    &sd_config.clip
+  } else {
+    sd_config.clip2.as_ref().unwrap()
+  };
+  let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
+
+  let text_embeddings = text_model.forward(&tokens)?;
+
+  let text_embeddings = if use_guide_scale {
+    let mut uncond_tokens = tokenizer
+      .encode(uncond_prompt, true)
+      .map_err(E::msg)?
+      .get_ids()
+      .to_vec();
+    if uncond_tokens.len() > sd_config.clip.max_position_embeddings {
+      anyhow::bail!(
+                "the negative prompt is too long, {} > max-tokens ({})",
+                uncond_tokens.len(),
+                sd_config.clip.max_position_embeddings
+            )
+    }
+    while uncond_tokens.len() < sd_config.clip.max_position_embeddings {
+      uncond_tokens.push(pad_id)
+    }
+
+    let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?;
+    let uncond_embeddings = text_model.forward(&uncond_tokens)?;
+
+    Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)?
+  } else {
+    text_embeddings.to_dtype(dtype)?
+  };
+  Ok(text_embeddings)
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs

@@ -0,0 +1,3 @@
+pub mod get_vae_scale;
+pub mod infer_clip_text_embeddings;
+pub mod stable_diffusion_pipeline;

crates/desktop/src-tauri/src/ml/stable_diffusion/stable_diffusion_pipeline.rs

@@ -0,0 +1,199 @@
+use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+
+use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
+use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
+use crate::ml::model_cache::ModelCache;
+use crate::ml::model_file::StableDiffusionVersion;
+use crate::ml::prompt_cache::PromptCache;
+use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
+use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
+use crate::state::app_config::AppConfig;
+use anyhow::{anyhow, Error as E, Result};
+use candle_core::{DType, IndexOp, Tensor, D};
+use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution;
+use image::DynamicImage;
+use log::info;
+use rand::Rng;
+
+pub struct Args<'a> {
+    pub image: &'a DynamicImage,
+    pub prompt: String,
+    pub uncond_prompt: String,
+    pub cfg_scale: Option<f64>,
+    pub i2i_strength: Option<u8>,
+    pub configs: &'a AppConfig,
+    pub model_cache: &'a ModelCache,
+    pub prompt_cache: &'a PromptCache,
+}
+
+pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
+    println!("Starting image generation with the following configuration:");
+    println!("  Model: {:?}", args.configs.sd_version);
+    println!("  Prompt: {}", args.prompt);
+    println!("  Steps: {}", args.configs.scheduler_steps);
+    println!("  Device: {:?}", args.configs.device);
+
+    println!("Model dimensions: {}x{}", args.configs.sd_config.width, args.configs.sd_config.height);
+
+    // TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
+    //  This is a target for performance improvement
+    //  See: https://github.com/huggingface/candle/issues/1331
+    let mut scheduler = args.configs.sd_config.build_scheduler(
+        args.configs.scheduler_steps)?;
+
+    let seed = args.configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
+    args.configs.device.set_seed(seed)?;
+
+    println!("Using seed: {}", seed);
+
+    let guidance_scale = match args.cfg_scale {
+        Some(guidance_scale) => guidance_scale,
+        None => match args.configs.sd_version {
+            StableDiffusionVersion::V1_5
+            | StableDiffusionVersion::V2_1
+            | StableDiffusionVersion::Xl => 7.5,
+            StableDiffusionVersion::Turbo => 0.,
+            _ => 0., // NB: Not sure what the other model families should use, so sticking with "0"
+        },
+    };
+
+    let use_guide_scale = guidance_scale > 1.0;
+
+    println!("Using guide scale: {}", use_guide_scale);
+
+    info!("Checking if prompt is cached");
+    let maybe_cached = args.prompt_cache.get_copy(&args.prompt)?;
+
+    let mut text_embeddings = if let Some(tensor) = maybe_cached {
+        tensor
+    } else {
+        info!("Prompt is NOT cached! Calculating embedding...");
+        let tensor = infer_clip_text_embeddings(
+            &args.prompt,
+            &args.uncond_prompt,
+            None, // tokenizer
+            None, // clip_weights
+            None, // clip2_weights
+            args.configs.sd_version,
+            &args.configs.sd_config,
+            false, // use_f16
+            &args.configs.device,
+            args.configs.dtype,
+            use_guide_scale,
+        )?;
+        args.prompt_cache.store_copy(&args.prompt, &tensor)?;
+        tensor
+    };
+
+    println!("Text embeddings shape: {:?}", text_embeddings.shape());
+
+    println!("Loading input image into tensor...");
+
+    let input_image = dynamic_image_to_tensor(
+        args.image,
+        &args.configs.device,
+        args.configs.dtype)?;
+
+    println!("Reference image shape: {:?}", input_image.shape());
+
+    let vae_scale = get_vae_scale(args.configs.sd_version);
+
+    let init_latent_dist : DiagonalGaussianDistribution = args.model_cache.vae_encode(&input_image)?;
+
+    // TODO(bt,2025-02-18): This takes a little bit to generate the sample.
+    //  This is a target for performance improvement
+    println!("Generating latents from input image...");
+    let latents = (init_latent_dist.sample()? * vae_scale)?;
+    
+    //.to_device(&args.configs.device)?;
+    
+    println!("Initial latents shape: {:?}", latents.shape());
+
+    println!("Calculating start step for diffusion process...");
+
+    let img2img_strength = match args.i2i_strength {
+        None => 0.75f64,
+        Some(strength) => (strength as f64) / 100.0f64,
+    };
+
+    let t_start = {
+        let start = args.configs.scheduler_steps - (args.configs.scheduler_steps as f64 * img2img_strength) as usize;
+
+        println!("Starting from step {} of {} (strength: {})", start, args.configs.scheduler_steps, img2img_strength);
+        start
+    };
+
+    let latents = if t_start < scheduler.timesteps().len() {
+        println!("Adding noise to latents...");
+        let noise = latents.randn_like(0f64, 1f64)?;
+        scheduler.add_noise(&latents, noise, scheduler.timesteps()[t_start])?
+    } else {
+        println!("Using latents directly (no noise addition needed)");
+        latents
+    };
+
+    println!("Latents initialized successfully");
+
+    let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();
+
+    // Initial noise shape: [1, 4, 64, 64]
+    println!("Initial noise shape: {:?}", latents.shape());
+
+    println!("Starting diffusion process...");
+    
+    let mut latents = latents;
+
+    for (timestep_index, &timestep) in timesteps.iter().enumerate() {
+        if timestep_index < t_start {
+            continue;
+        }
+
+        println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
+
+        let latent_model_input = if use_guide_scale {
+            Tensor::cat(&[&latents, &latents], 0)?
+        } else {
+            latents.clone() // TODO(bt,2025-08-18): Do we have to clone the tensor? `scale_model_input` requires ownership
+        };
+
+        let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;
+
+        let mut noise_pred = match args.model_cache.unet_inference(&latent_model_input, timestep as f64, &text_embeddings) {
+            Ok(pred) => pred,
+            Err(e) => {
+                println!("UNet inference failed with error: {}", e);
+                return Err(anyhow::anyhow!("UNet inference failed: {}", e));
+            },
+        };
+
+        if use_guide_scale {
+            let chunks = noise_pred.chunk(2, 0)?;
+            let (noise_pred_uncond, noise_pred_text) = (&chunks[0], &chunks[1]);
+            noise_pred = (noise_pred_uncond + (guidance_scale * (noise_pred_text - noise_pred_uncond)?)?)?;
+        }
+
+        latents = scheduler.step(&noise_pred, timestep, &latents)?;
+    }
+
+    println!("Diffusion process completed, decoding image...");
+    let image = args.model_cache.vae_decode(&(latents / vae_scale)?)?;
+    
+    println!("VAE decode completed");
+
+    println!("Scaling image back");
+    let image = ((image / 2.)? + 0.5)?;
+    
+    // TODO(bt,2025-08-18): This normalization is slow.
+    //  This is a target for performance improvement
+    println!("Normalized image values");
+    let image = (image.clamp(0f32, 1.)? * 255.)?;
+
+    println!("Convert to int8");
+    let image = image.to_dtype(DType::U8)?;
+    
+    println!("Converted to 8-bit format");
+
+    let image = tensor_to_image_buffer(&image.i(0)?)?;
+
+    Ok(image)
+}

crates/desktop/src-tauri/src/state/app_config.rs

@@ -0,0 +1,96 @@
+use crate::ml::model_file::StableDiffusionVersion;
+use crate::state::yaml_config::YamlConfig;
+use candle_core::{DType, Device};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use filesys::recursively_find_file_by_name::recursively_find_file_by_name;
+use hf_hub::api::sync::Api;
+use log::info;
+
+const CONFIG_FILENAME : &str = "app_config.yaml";
+
+const DEFAULT_SD_IMAGE_WIDTH: usize = 512;
+const DEFAULT_SD_IMAGE_HEIGHT: usize = 512;
+
+const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
+const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::Turbo;
+
+/// A central place to configure app-wide details.
+pub struct AppConfig {
+  /// We only support one device type currently, and we can't multiplex 
+  /// over several devices yet.
+  pub device: Device,
+  
+  /// The core numeric type to use for models.
+  pub dtype: DType,
+  
+  pub sd_version: StableDiffusionVersion,
+  pub sd_config: StableDiffusionConfig,
+
+  pub image_height: usize,
+  pub image_width: usize,
+  pub scheduler_steps: usize,
+  pub scheduler_samples: usize,
+  
+  pub seed: Option<u64>,
+
+  pub cfg_scale: Option<f64>,
+  
+  /// Probably shouldn't be here.
+  pub hf_api: Api,
+}
+
+impl AppConfig {
+  
+  pub fn init() -> anyhow::Result<Self> {
+    println!("Configuration filename: {}", CONFIG_FILENAME);
+    let yaml_configs = load_yaml_configs();
+    
+    println!("Configs: {:?}", yaml_configs);
+    
+    let device = Device::new_cuda(0)
+      .unwrap_or_else(|e| {
+          info!("CUDA not available ({}), falling back to CPU", e);
+          Device::Cpu
+        });
+    
+    let sd_config = match SD_VERSION {
+      StableDiffusionVersion::Turbo => {
+        StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+      _ => {
+        StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+    };
+    
+    let hf_api = Api::new()?;
+    
+    Ok(Self {
+      device,
+      dtype: DEFAULT_SD_NUMERIC_DATATYPE,
+      sd_version: SD_VERSION,
+      sd_config,
+      image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
+      image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
+      scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(1),
+      scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
+      seed: yaml_configs.seed,
+      cfg_scale: yaml_configs.cfg_scale,
+      hf_api,
+    })
+  }
+}
+
+fn load_yaml_configs() -> YamlConfig {
+  let maybe_path = recursively_find_file_by_name(CONFIG_FILENAME, "../../../", 5)
+    .ok()
+    .flatten();
+  if let Some(path) = maybe_path {
+    let path = path.canonicalize().unwrap_or(path);
+    println!("Attempting to load configs from: {:?}", &path);
+    if let Ok(config) = YamlConfig::read_from_file(path) {
+      return config;
+    }
+  }
+  println!("Loading default configs.");
+  YamlConfig::default()
+}

crates/desktop/src-tauri/src/state/mod.rs

@@ -0,0 +1,2 @@
+pub mod yaml_config;
+pub mod app_config;

crates/desktop/src-tauri/src/state/yaml_config.rs

@@ -0,0 +1,25 @@
+use std::fs::File;
+use std::io::BufReader;
+use std::path::Path;
+
+/// Configuration read or derived from a YAML config file at startup
+/// This is not meant to contain any runtime config values.
+/// This may not be exhaustive.
+#[derive(serde::Deserialize, serde::Serialize, Clone, Debug, Default)]
+pub struct YamlConfig {
+  pub image_height: Option<usize>,
+  pub image_width: Option<usize>,
+  pub scheduler_steps: Option<usize>,
+  pub scheduler_samples: Option<usize>,
+  pub seed: Option<u64>,
+  pub cfg_scale: Option<f64>,
+}
+
+impl YamlConfig {
+  pub fn read_from_file<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let config = serde_yaml::from_reader(reader)?;
+    Ok(config)
+  }
+}

crates/desktop/src-tauri/tauri.conf.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://schema.tauri.app/config/2.0.0-rc",
-  "productName": "Storyteller Desktop App",
+  "productName": "Realtime Desktop App",
   "version": "0.1.0",
   "identifier": "com.unique.value.thing.why",
   "build": {
@@ -12,9 +12,9 @@
   "app": {
     "windows": [
       {
-        "title": "Storyteller Desktop App Title Goes Here",
-        "width": 800,
-        "height": 600,
+        "title": "Realtime Desktop App Title Goes Here",
+        "width": 1200,
+        "height": 800,
         "resizable": true,
         "fullscreen": false
       }

crates/lib/files/filesys/src/lib.rs

@@ -31,4 +31,5 @@ pub mod file_size;
 pub mod filename_concat;
 pub mod is_filesystem_full_error;
 pub mod path_to_string;
+pub mod recursively_find_file_by_name;
 pub mod rename_across_devices;

crates/lib/files/filesys/src/recursively_find_file_by_name.rs

@@ -0,0 +1,56 @@
+use std::path::{Path, PathBuf};
+use walkdir::WalkDir;
+use errors::AnyhowResult;
+
+pub fn recursively_find_file_by_name<P: AsRef<Path>, Q: AsRef<Path>>(
+  search_filename: P,
+  search_root: Q,
+  max_depth: usize,
+) -> AnyhowResult<Option<PathBuf>> {
+
+  let walker = WalkDir::new(search_root)
+    .max_depth(max_depth);
+
+  for entry in walker {
+    let entry = entry?;
+    if entry.path().is_dir() {
+      let check_path = entry.path().join(search_filename.as_ref());
+      if check_path.exists() && check_path.is_file() {
+        return Ok(Some(check_path));
+      }
+    }
+  }
+
+  Ok(None)
+}
+
+#[cfg(test)]
+mod tests {
+  use testing::test_file_path::test_file_path;
+  use crate::recursively_find_file_by_name::recursively_find_file_by_name;
+
+  #[test]
+  fn test_path_exists() {
+    let test_data_path = test_file_path("test_data/").expect("should work");
+    let result = recursively_find_file_by_name(
+      "super_mario_rpg_beware_the_forests_mushrooms.mp3",
+      test_data_path,
+      5
+    );
+    let result = result.expect("should be ok");
+    let result = result.expect("should exist");
+    assert!(result.ends_with("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3"));
+  }
+
+  #[test]
+  fn test_path_does_not_exist() {
+    let test_data_path = test_file_path("test_data/").expect("should work");
+    let result = recursively_find_file_by_name(
+      "fake_name.foo",
+      test_data_path,
+      5
+    );
+    let result = result.expect("should be ok");
+    assert!(result.is_none());
+  }
+}

frontend/realtime/src/KonvaApp/Engine.ts

@@ -1,9 +1,8 @@
 import Konva from "konva";
 
-import { RenderEngine } from "./RenderingPrimitives/RenderEngine";
-import { ResponseType } from "./WorkerPrimitives/SharedWorkerBase";
 
 import { uiAccess, uiEvents } from "~/signals";
+import { ShapeNode } from "./Nodes";
 
 import { UndoStackManager } from "./UndoRedo";
 import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities";
@@ -20,22 +19,17 @@ import {
 import { ImageNode, VideoNode, TextNode } from "./Nodes";
 import { EngineOptions, TextNodeData, VideoNodeData } from "./types";
 
-import { SharedWorkerResponse } from "./WorkerPrimitives/SharedWorkerBase";
-import {
-  DiffusionSharedWorkerProgressData,
-  DiffusionSharedWorkerResponseData,
-} from "./SharedWorkers/Diffusion/DiffusionSharedWorker";
-
 import { AppModes, VideoResolutions } from "./constants";
 import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
 
 import { ToolbarNodeButtonNames } from "~/components/features/ToolbarNode/enums";
 import { NavigateFunction } from "react-router-dom";
 import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider";
-import { MediaFile } from "~/Classes/ApiManager/models/MediaFile";
+
 import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler";
 import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine";
-
+import { NodeColor } from "~/signals/uiEvents/toolbarNode";
+import { ShapeType } from "./Nodes";
 // for testing loading files from system
 // import { FileUtilities } from "./FileUtilities/FileUtilities";
 
@@ -61,7 +55,6 @@ export class Engine {
   private nodeIsolationLayer: Konva.Layer;
   private uiLayer: Konva.Layer;
   private previewLayer: Konva.Layer;
-  private renderEngine: RenderEngine;
   private offScreenCanvas: OffscreenCanvas;
   private realTimeDrawEngine: RealTimeDrawEngine;
   private nodesManager: NodesManager;
@@ -106,6 +99,7 @@ export class Engine {
     this.stage.add(this.nodeIsolationLayer);
     this.stage.add(this.uiLayer);
 
+
     // Konva Transformer
     this.nodeTransformer = new NodeTransformer();
     this.uiLayer.add(this.nodeTransformer.getKonvaNode());
@@ -126,10 +120,7 @@ export class Engine {
       height: VideoResolutions.SQUARE_1024.height,
       mediaLayerRef: this.mediaLayer,
       bgLayerRef: this.bgLayer,
-      offScreenCanvas: this.offScreenCanvas,
-      previewLayerRef: this.previewLayer,
-      onRenderingSystemMessageRecieved:
-        this.onRenderingSystemReceived.bind(this),
+      offScreenCanvas: this.offScreenCanvas
     });
 
     // Collection of all Nodes
@@ -189,79 +180,12 @@ export class Engine {
     // hence, lastly, setup these events
     this.setupEventSystem();
     this.setAppMode(AppModes.SELECT);
+
+
+
   }
 
-  // TODO write code to show error and retry.
 
-  onRenderingSystemReceived(
-    response: SharedWorkerResponse<
-      DiffusionSharedWorkerResponseData | MediaFile,
-      DiffusionSharedWorkerProgressData
-    >,
-  ) {
-    if (!response.data) {
-      // throw error to retry
-      uiAccess.dialogError.show({
-        title: "Generation Error",
-        message: response.data?.toString(),
-      });
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.error) {
-      console.log("Error Data?");
-      console.log(response.data, response);
-      uiAccess.dialogError.show({
-        title: "Generation Error Try again.",
-        message: response.data?.toString(),
-      });
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.result) {
-      console.log(response.data);
-      const data = response.data;
-      // create video node here.
-      // choose it to be the size of the rendering output, this case its mobile. (1560, 400)
-      if (typeof data === "string" || data === undefined) {
-        return;
-      }
-      if ("videoUrl" in data) {
-        console.log("Engine got stylized video: " + data.videoUrl);
-        this.addVideo({ mediaFileUrl: data.videoUrl });
-      } else if ("media_links" in data) {
-        console.log("Engine got rendered video: " + data.media_links.cdn_url);
-        downloadURI(data.media_links.cdn_url, "Download Video");
-      }
-
-      // hide the loader
-      //this.realTimeDrawEngine.videoLoadingCanvas.kNode.hide();
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.progress) {
-      const data = response.data as DiffusionSharedWorkerProgressData;
-      // TODO wil fix this ?!?! parameter issue
-      //this.realTimeDrawEngine.videoLoadingCanvas.kNode.show();
-      uiAccess.toolbarMain.loadingBar.update({
-        message: "Rendering Frames...",
-        progress: data.progress * 100,
-        isShowing: true,
-      });
-
-      // console.log(response);
-      // if (response.data.zipBlob) {
-      //   FileUtilities.downloadBlobZip(response.data.zipBlob);
-      // }
-      return;
-    }
-  }
 
   private setAppMode(newAppMode: AppModes) {
     this.appMode = newAppMode;
@@ -441,9 +365,12 @@ export class Engine {
     // Listen to Toolbar Main
     uiEvents.toolbarMain.UNDO.onClick(() => this.undoStackManager.undo());
     uiEvents.toolbarMain.REDO.onClick(() => this.undoStackManager.redo());
+
     uiEvents.toolbarMain.SAVE.onClick(async (/*event*/) => {
-      this.sceneManager.saveScene();
+      await this.realTimeDrawEngine.saveOutput();
+
     });
+
     uiEvents.toolbarMain.PREVIEW.onClick(async () => {
       console.log("Toolbar Main >> Preview");
       this.setAppMode(AppModes.PREVIEW);
@@ -472,10 +399,8 @@ export class Engine {
       this.addImage(image);
     });
 
-    uiEvents.onGetStagedVideo((videoData) => {
-      console.log("Engine got user video: " + videoData.mediaFileUrl);
-      this.addVideo(videoData);
-    });
+
+
     uiEvents.onAddTextToEngine((textdata) => {
       this.addText(textdata);
     });
@@ -506,21 +431,41 @@ export class Engine {
         }
       }
     });
-    uiEvents.aiStylize.onRequest(async (data) => {
-      console.log("Engine heard AI Stylize request: ", data);
 
-      try {
-        this.setAppMode(AppModes.RENDERING);
-        //await this.realTimeDrawEngine.startProcessing(data);
-      } catch (error) {
-        // throw error to retry
-        uiAccess.dialogError.show({
-          title: "Generation Error",
-          message: error?.toString() || "Unknown Error",
-        });
-        this.setAppMode(AppModes.SELECT);
+    uiEvents.promptEvents.onPromptStrengthChanged((strength) => {
+      this.realTimeDrawEngine.currentStrength = strength;
+    });
+
+    uiEvents.promptEvents.onPromptTextChanged((prompt) => {
+      this.realTimeDrawEngine.currentPrompt = prompt;
+    });
+
+    uiEvents.onAddShapeToEngine((shapeData) => {
+      switch (shapeData.shape) {
+        case "circle":
+          this.addShape(ShapeType.CIRCLE, 100);
+          break;
+        case "square":
+          this.addShape(ShapeType.SQUARE, 100);
+          break;
+        case "triangle":
+          this.addShape(ShapeType.TRIANGLE, 100);
+          break;
       }
     });
+
+    uiEvents.toolbarNode.color.onConfirmChanged((nodeColor) => {
+      console.log("Color change event triggered with:", nodeColor);
+      if (!nodeColor) {
+        console.warn("nodeColor is undefined or null");
+        return;
+      }
+      if (!nodeColor.kNodeId) {
+        console.warn("nodeColor.kNodeId is undefined or null");
+        return;
+      }
+      this.changeNodeColor(nodeColor);
+    })
   }
 
   disableAllButtons() {
@@ -556,7 +501,7 @@ export class Engine {
   }
 
   // Sandbox is quickly a way to test your idea.
-  public async sandbox() {}
+  public async sandbox() { }
 
   public onMessage(event: MessageEvent) {
     console.log("Message From Shared Worker");
@@ -603,6 +548,40 @@ export class Engine {
     this.addKeyboardShortcuts();
   }
 
+  private changeNodeColor(nodeColor: NodeColor) {
+    console.log('Changing node color:', nodeColor);
+    // Find the node that is selected with the id
+    const idSelector = '#' + nodeColor.kNodeId;
+    const selectedNode = this.mediaLayer.findOne(idSelector);
+    console.log('Selected node:', selectedNode);
+    console.log('ID selector:', idSelector);
+    console.log('Media layer:', this.mediaLayer);
+    // TODO: Change the color of the node
+    // Turns out shapes are image nodes... how do we change the image's fill??
+    if (selectedNode) {
+      // Get the shape node from the real time drawing engine's image nodes
+      const shapeNode = this.realTimeDrawEngine.findImageNodeById(nodeColor.kNodeId);
+      if (shapeNode && shapeNode instanceof ShapeNode) {
+        // Create new shape node with updated color
+        const newShapeNode = new ShapeNode({
+          canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+          canvasSize: this.realTimeDrawEngine.captureCanvas.size(), 
+          shapeType: shapeNode.shapeType,
+          size: shapeNode.size,
+          color: nodeColor.color,
+          mediaLayerRef: this.mediaLayer,
+          selectionManagerRef: this.selectionManager
+        });
+        newShapeNode.kNode.position(shapeNode.kNode.position());
+        newShapeNode.kNode.zIndex(shapeNode.kNode.zIndex());
+        
+        // Remove old node and add new one
+        shapeNode.kNode.destroy();
+        this.commandManager.createNode(newShapeNode);
+      }
+    }
+  }
+
   public addText(textNodeData: TextNodeData) {
     const textNode = new TextNode({
       textNodeData: textNodeData,
@@ -614,6 +593,24 @@ export class Engine {
     this.commandManager.createNode(textNode);
   }
 
+  public addShape(type: ShapeType, size: number, color?: string) {
+    const shapeNode = new ShapeNode({
+      canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+      canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
+      shapeType: type,
+      size: size,
+      color: color,
+      mediaLayerRef: this.mediaLayer,
+      selectionManagerRef: this.selectionManager
+    });
+    shapeNode.kNode.zIndex(1); // Replace desiredZIndex with a number
+
+    this.commandManager.createNode(shapeNode);
+    console.debug("Added shapenode:", shapeNode);
+    console.debug("Added node's ID:", shapeNode.kNode._id);
+    //this.realTimeDrawEngine.addNodes(shapeNode);
+  }
+
   public addImage(imageFile: File) {
     const imageNode = new ImageNode({
       mediaLayerRef: this.mediaLayer,
@@ -622,9 +619,9 @@ export class Engine {
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
     });
-
+    imageNode.kNode.zIndex(1);
     this.commandManager.createNode(imageNode);
-    this.realTimeDrawEngine.addNodes(imageNode);
+    //this.realTimeDrawEngine.addNodes(imageNode);
   }
 
   public addVideo(
@@ -668,7 +665,7 @@ export class Engine {
           if (
             longestVideoNode === undefined ||
             node.videoComponent.duration >
-              longestVideoNode.videoComponent.duration
+            longestVideoNode.videoComponent.duration
           ) {
             longestVideoNode = node;
           }

frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts

@@ -1,5 +1,5 @@
 import Konva from "konva";
-import { ImageNode, TextNode, VideoNode } from "../Nodes";
+import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes";
 import { MediaNode, Position, RGBColor, Transformation } from "../types";
 import {
   AddChromaKey,
@@ -48,7 +48,7 @@ export class CommandManager {
     this.undoStackManagerRef = engineRefs.undoStackManagerRef;
   }
 
-  createNode(node: VideoNode | ImageNode | TextNode) {
+  createNode(node: VideoNode | ImageNode | TextNode | ShapeNode) {
     const command = new CreateCommand({
       nodes: new Set<MediaNode>([node]),
       mediaLayerRef: this.mediaLayerRef,
@@ -58,6 +58,10 @@ export class CommandManager {
       renderEngineRef: this.renderEngineRef,
     });
     this.undoStackManagerRef.executeCommand(command);
+
+    // Set the kNode's manual id to same as kNode's internal _id
+    // This helps identify the kNode in the Konva layer with find
+    node.kNode.id(node.kNode._id.toString());
   }
   deleteNodes() {
     const nodes = this.selectionManagerRef.getSelectedNodes();

frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts

@@ -29,7 +29,7 @@ export abstract class BaseNode {
   }: {
     selectionManagerRef: SelectionManager;
     mediaLayerRef: Konva.Layer;
-    kNode: Konva.Image | Konva.Group;
+    kNode: Konva.Image | Konva.Group | Konva.Shape;
   }) {
     // console.log("Node constructed");
     // this.uuid = uuidv4();
@@ -67,8 +67,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.stroke(primaryOrange);
-      wrapperRect.strokeWidth(highlightStrokeWidth);
+      console.log("Wrapper rect:", wrapperRect);
+      if (wrapperRect !=null) {
+        wrapperRect.stroke(primaryOrange);
+        wrapperRect.strokeWidth(highlightStrokeWidth);
+      }
       return;
     }
     if (import.meta.env.DEV) {
@@ -86,7 +89,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.strokeWidth(0);
+      console.log("Wrapper rect:", wrapperRect); 
+      if (wrapperRect !=null) {
+        // Issue investigate
+        wrapperRect.strokeWidth(0);
+      }
       return;
     }
     if (import.meta.env.DEV) {

frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts

@@ -83,8 +83,10 @@ export class ImageNode extends NetworkedNode {
   }) {
     const imageComponent = new Image();
     imageComponent.crossOrigin = "anonymous";
+    // this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+
     imageComponent.onload = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
+      //this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
       this.imageSize = {
         width: imageComponent.width,
         height: imageComponent.height,
@@ -105,33 +107,33 @@ export class ImageNode extends NetworkedNode {
       this.kNode.fill(transparent);
       this.listenToBaseKNode();
       this.mediaLayerRef.draw();
-      this.uploadImage(imageFile);
+      //this.uploadImage(imageFile);
     };
     imageComponent.onerror = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
+      //this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
     };
     imageComponent.src = URL.createObjectURL(imageFile);
   }
   private async loadImageFromUrl(mediaFileUrl: string) {
-    this.setProgress({ progress: 75, status: UploadStatus.LOADING });
+    //this.setProgress({ progress: 75, status: UploadStatus.LOADING });
 
     const newImage = new Image();
     newImage.crossOrigin = "anonymous";
     newImage.onerror = () => {
-      this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
+      //this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
     };
     newImage.onload = () => {
       // console.log("network image", newImage);
       this.kNode.image(newImage);
       this.kNode.draw();
-      this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+      //this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
     };
     newImage.src = mediaFileUrl;
     this.listenToBaseKNode();
   }
 
   private async uploadImage(imageFile: File) {
-    this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
+    //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
 
     const mediaUploadApi = new MediaUploadApi();
     const uploadResponse = await mediaUploadApi.UploadImage({
@@ -149,7 +151,7 @@ export class ImageNode extends NetworkedNode {
   }
 
   private async retreiveImage(mediaFileToken: string) {
-    this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
+    //this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
     const mediaFileApi = new MediaFilesApi();
     const mediaFileResponse = await mediaFileApi.GetMediaFileByToken({
       mediaFileToken: mediaFileToken,

frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts

@@ -0,0 +1,202 @@
+import Konva from "konva";
+import { SelectionManager } from "../NodesManagers";
+import { Position, Size, NodeData, TransformationData } from "../types";
+import { NodeType } from "./constants";
+import { NodeUtilities } from "./NodeUtilities";
+import { NetworkedNode } from "./NetworkedNode";
+
+export enum ShapeType {
+  CIRCLE = "circle",
+  SQUARE = "square",
+  TRIANGLE = "triangle",
+}
+
+interface ShapeNodeConstructor {
+  canvasPosition: Position;
+  canvasSize: Size;
+  shapeType: ShapeType;
+  size: number;
+  color?: string; // Hex color string
+  transform?: TransformationData;
+  mediaLayerRef: Konva.Layer;
+  selectionManagerRef: SelectionManager;
+}
+
+export class ShapeNode extends NetworkedNode {
+  
+  retry(): void {
+    throw new Error("Method not implemented.");
+  }
+
+  public shapeType: ShapeType;
+  public size: number;
+
+  constructor({
+    canvasPosition,
+    canvasSize,
+    shapeType,
+    size,
+    color = "#ff0000", // Default red if no color provided
+    transform: existingTransform,
+    mediaLayerRef,
+    selectionManagerRef,
+  }: ShapeNodeConstructor) {
+    // Create the actual shape inside the group
+    const transform = NodeUtilities.getInitialTransform({
+      existingTransform,
+      canvasPosition,
+      canvasSize,
+    });
+
+    let shape = null;
+    
+    switch (shapeType) {
+      case ShapeType.CIRCLE:
+        const circleShape = new Konva.Circle({
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert circle to image
+        const dataURL = circleShape.toDataURL();
+        const imageObj = new Image();
+        imageObj.src = dataURL;
+
+        shape = new Konva.Image({
+          image: imageObj,
+          x: circleShape.x(),
+          y: circleShape.y(),
+          width: circleShape.width(),
+          height: circleShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        imageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary circle
+        circleShape.destroy();
+        break;
+
+      case ShapeType.SQUARE:
+        const squareShape = new Konva.Rect({
+          width: 100,
+          height: 100,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert square to image
+        const squareDataURL = squareShape.toDataURL();
+        const squareImageObj = new Image();
+        squareImageObj.src = squareDataURL;
+
+        shape = new Konva.Image({
+          image: squareImageObj,
+          x: squareShape.x(),
+          y: squareShape.y(),
+          width: squareShape.width(),
+          height: squareShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        squareImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary square
+        squareShape.destroy();
+        break;
+
+      case ShapeType.TRIANGLE:
+        const triangleShape = new Konva.RegularPolygon({
+          sides: 3,
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert triangle to image
+        const triangleDataURL = triangleShape.toDataURL();
+        const triangleImageObj = new Image();
+        triangleImageObj.src = triangleDataURL;
+
+        shape = new Konva.Image({
+          image: triangleImageObj,
+          x: triangleShape.x(),
+          y: triangleShape.y(),
+          width: triangleShape.width(),
+          height: triangleShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        triangleImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary triangle
+        triangleShape.destroy();
+        break;
+
+      default:
+        throw new Error("Invalid shape type");
+    }
+
+    super({
+      selectionManagerRef: selectionManagerRef,
+      mediaLayerRef: mediaLayerRef,
+      kNode: shape,
+    });
+
+    this.shapeType = shapeType;
+    this.size = size;
+
+    // Add shape to group
+    const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({
+      canvasOffset: canvasPosition,
+      componentSize: canvasSize,
+      maxSize: canvasSize,
+    });
+    this.kNode.setPosition(centerPosition);
+    this.mediaLayerRef.add(this.kNode);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+  }
+
+  public getNodeData(canvasPosition: Position) {
+    const data: NodeData = {
+      type: NodeType.SHAPE,
+      transform: {
+        position: {
+          x: this.kNode.position().x - canvasPosition.x,
+          y: this.kNode.position().y - canvasPosition.y,
+        },
+        size: this.kNode.size(),
+        rotation: this.kNode.rotation(),
+        scale: {
+          x: this.kNode.scaleX(),
+          y: this.kNode.scaleY(),
+        },
+        zIndex: this.kNode.getZIndex(),
+      },
+    };
+    return data;
+  }
+}

frontend/realtime/src/KonvaApp/Nodes/constants.ts

@@ -4,6 +4,7 @@ export enum NodeType {
   IMAGE = "image",
   TEXT = "text",
   VIDEO = "video",
+  SHAPE = "shape"
 }
 export const transparent = Colors.transparent;
 export const primaryOrange = Colors.primaryOrange;

frontend/realtime/src/KonvaApp/Nodes/index.ts

@@ -3,3 +3,4 @@ export { BaseNode } from "./BaseNode";
 export { ImageNode } from "./ImageNode";
 export { VideoNode } from "./VideoNode";
 export { TextNode } from "./TextNode";
+export { ShapeNode , ShapeType} from "./ShapeNode";

frontend/realtime/src/KonvaApp/NodesManagers/SelectionManager.ts

@@ -4,11 +4,12 @@ import { MediaNode, Position, Transformation } from "../types";
 import { uiAccess, uiEvents } from "~/signals";
 import { NetworkedNode } from "../Nodes/NetworkedNode";
 import { LoadingBarStatus } from "~/components/ui";
-import { ImageNode, TextNode, VideoNode } from "../Nodes";
+import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes";
 import {
   calculateContextualsPosition,
   getImageNodeButtonStates,
   getMultiSelectButtonStates,
+  getShapeNodeButtonStates,
   getTextNodeButtonStates,
   getVideoNodeButtonStates,
 } from "./ToolbarNodeUtilities";
@@ -273,17 +274,29 @@ export class SelectionManager {
     if (this.selectedNodes.size === 1) {
       if (node instanceof ImageNode) {
         showOrUpdate({
+          knodeIds: [node.kNode.id()],
           locked: node.isLocked(),
           buttonStates: getImageNodeButtonStates({ locked: node.isLocked() }),
         });
       }
-      if (node instanceof TextNode) {
+      else if (node instanceof ShapeNode) {
+        console.debug("Selected node is of type ShapeNode", node.kNode.id())
         showOrUpdate({
+          knodeIds: [node.kNode.id()],
           locked: node.isLocked(),
-          buttonStates: getTextNodeButtonStates({ locked: node.isLocked() }),
+          buttonStates: getShapeNodeButtonStates({ locked: node.isLocked() }),
+          color: node.getNodeData({ x: 0, y: 0 })?.textNodeData?.color ?? "#000000",
         });
       }
-      if (node instanceof VideoNode) {
+      else if (node instanceof TextNode) {
+        showOrUpdate({
+          knodeIds: [node.kNode.id()],
+          locked: node.isLocked(),
+          buttonStates: getTextNodeButtonStates({ locked: node.isLocked() }),
+          color: node.getNodeData({ x: 0, y: 0 })?.textNodeData?.color ?? "#000000",
+        });
+      }
+      else if (node instanceof VideoNode) {
         showOrUpdate({
           locked: node.isLocked(),
           buttonStates: getVideoNodeButtonStates({ locked: node.isLocked() }),

frontend/realtime/src/KonvaApp/NodesManagers/ToolbarNodeUtilities.ts

@@ -51,7 +51,8 @@ export function getImageNodeButtonStates(
         buttonName === ButtonNames.AI_STYLIZE ||
         buttonName === ButtonNames.SEGMENTATION ||
         buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA
+        buttonName === ButtonNames.CHROMA ||
+        buttonName === ButtonNames.COLOR
       ) {
         buttonStates[buttonName] = {
           disabled: true,
@@ -130,7 +131,9 @@ export function getVideoNodeButtonStates(
         return buttonStates;
       }
       // soon to come feature is disabled
-      if (buttonName === ButtonNames.AI_STYLIZE) {
+      if (buttonName === ButtonNames.AI_STYLIZE ||
+        buttonName === ButtonNames.COLOR
+      ) {
         buttonStates[buttonName] = {
           disabled: true,
           hidden: true,
@@ -149,7 +152,8 @@ export function getVideoNodeButtonStates(
     {} as ContextualToolbarProps["buttonStates"],
   );
 }
-export function getMultiSelectButtonStates(
+
+export function getShapeNodeButtonStates(
   props: { locked?: boolean | "unknown" } = {},
 ) {
   const ButtonNames = ToolbarNodeButtonNames;
@@ -157,7 +161,7 @@ export function getMultiSelectButtonStates(
     (buttonStates, buttonName) => {
       // transform button depends on locked state
       if (props.locked !== undefined && buttonName === ButtonNames.TRANSFORM) {
-        buttonStates[buttonName] = {
+        buttonStates.TRANSFORM = {
           disabled: props.locked === "unknown" || props.locked === true,
           hidden: false,
           active: true,
@@ -191,3 +195,47 @@ export function getMultiSelectButtonStates(
     {} as ContextualToolbarProps["buttonStates"],
   );
 }
+
+export function getMultiSelectButtonStates(
+  props: { locked?: boolean | "unknown" } = {},
+) {
+  const ButtonNames = ToolbarNodeButtonNames;
+  return Object.values(ButtonNames).reduce(
+    (buttonStates, buttonName) => {
+      // transform button depends on locked state
+      if (props.locked !== undefined && buttonName === ButtonNames.TRANSFORM) {
+        buttonStates[buttonName] = {
+          disabled: props.locked === "unknown" || props.locked === true,
+          hidden: false,
+          active: true,
+        };
+        return buttonStates;
+      }
+
+      // hidden buttons
+      if (
+        buttonName === ButtonNames.AI_STYLIZE ||
+        buttonName === ButtonNames.SEGMENTATION ||
+        buttonName === ButtonNames.DOWNLOAD ||
+        buttonName === ButtonNames.CHROMA ||
+        buttonName === ButtonNames.COLOR
+      ) {
+        buttonStates[buttonName] = {
+          disabled: true,
+          hidden: true,
+          active: false,
+        };
+        return buttonStates;
+      }
+
+      // all other buttons
+      buttonStates[buttonName] = {
+        disabled: false,
+        hidden: false,
+        active: false,
+      };
+      return buttonStates;
+    },
+    {} as ContextualToolbarProps["buttonStates"],
+  );
+}

frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts

@@ -6,7 +6,7 @@ import { Group } from "konva/lib/Group";
 import { invoke } from "@tauri-apps/api/core";
 
 import { FileUtilities } from "../FileUtilities/FileUtilities";
-import { ImageNode, VideoNode, TextNode } from "../Nodes";
+import { ImageNode, VideoNode, TextNode, ShapeNode, ShapeType } from "../Nodes";
 import { MediaNode } from "../types";
 
 import { RenderTask } from "./RenderTask";
@@ -16,14 +16,15 @@ import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas";
 
 export class RealTimeDrawEngine {
   private videoNodes: VideoNode[];
-  private imageNodes: (ImageNode | TextNode)[];
+  private imageNodes: (ImageNode | TextNode | ShapeNode)[];
 
   private offScreenCanvas: OffscreenCanvas;
+  private outputBitmap: ImageBitmap | undefined;
 
   // private frames: ImageBitmap[];
 
   // capturing composite within window
-  private bgLayerRef: Konva.Layer;
+
   private mediaLayerRef: Konva.Layer;
 
   private height: number;
@@ -41,18 +42,19 @@ export class RealTimeDrawEngine {
   public videoLoadingCanvas: VideoNode | undefined;
 
   public fps: number = 24;
+
+  public currentPrompt: string;
+  public currentStrength: number;
+
   constructor({
     width,
     height,
-    bgLayerRef,
     mediaLayerRef,
     offScreenCanvas,
   }: {
     width: number;
     height: number;
-    bgLayerRef: Konva.Layer;
     mediaLayerRef: Konva.Layer;
-    previewLayerRef: Konva.Layer;
     offScreenCanvas: OffscreenCanvas;
   }) {
     this.videoLoadingCanvas = undefined;
@@ -74,18 +76,17 @@ export class RealTimeDrawEngine {
     this.offScreenCanvas.width = this.width;
     this.offScreenCanvas.height = this.height;
 
-    this.bgLayerRef = bgLayerRef;
-
     // this is the whole canvas
     this.mediaLayerRef = mediaLayerRef;
 
-    // this.previewLayerRef = previewLayerRef;
     // Set background layer to red and media layer to green for visibility
 
     this.port = undefined;
 
     this.fps = 24;
 
+    this.currentPrompt = "";
+    this.currentStrength = 100;
     // This is captures a subset of the medialayer ref
     this.captureCanvas = new Konva.Rect({
       name: "CaptureCanvas",
@@ -99,20 +100,6 @@ export class RealTimeDrawEngine {
       draggable: false,
     });
 
-    // Add preview canvas with same dimensions but different style
-    // this.previewCanvas = new Konva.Rect({
-    //   name: "PreviewCanvas",
-    //   x: this.positionX,
-    //   y: this.positionY,
-    //   width: this.width,
-    //   height: this.height,
-    //   fill: "white",
-    //   stroke: "blue",
-    //   strokeWidth: 1,
-    //   dash: [5, 5],
-    //   draggable: false,
-    // });
-
     this.previewCanvas = new Konva.Image({
       name: "PreviewCanvas",
       x: this.positionX,
@@ -120,21 +107,90 @@ export class RealTimeDrawEngine {
       width: this.width,
       height: this.height,
       image: undefined,
-      stroke: "blue",
+      stroke: "black",
       strokeWidth: 1,
-      dash: [5, 5],
       draggable: false,
+      fill: "white",
     });
 
-    this.bgLayerRef.add(this.captureCanvas);
-    this.bgLayerRef.add(this.previewCanvas);
+    this.mediaLayerRef.add(this.captureCanvas);
+    this.mediaLayerRef.add(this.previewCanvas);
     // send back
     this.captureCanvas.setZIndex(0);
-    this.previewCanvas.setZIndex(0);
+    this.previewCanvas.setZIndex(1);
+    // Add mouse events for preview canvas copying
+    //this.previewCopyListener();
 
-    //this.debug();
   }
 
+  public previewCopyListener() {
+    this.previewCanvas.on('mousedown touchstart', () => {
+      if (!this.outputBitmap) {
+        console.log("No preview image to copy"); 
+        return;
+      }
+      
+      // TODO create it as a node.
+      
+      // const imageNode = new ImageNode({
+      //   mediaLayerRef: this.mediaLayer,
+      //   canvasPosition: this.renderEngine.captureCanvas.position(),
+      //   canvasSize: this.renderEngine.captureCanvas.size(),
+      //   imageFile: imageFile,
+      //   selectionManagerRef: this.selectionManager,
+      // });
+
+      // Create draggable preview copy
+      const previewCopy = new Konva.Image({
+        x: this.previewCanvas.x(),
+        y: this.previewCanvas.y(),
+        width: this.width,
+        height: this.height,
+        image: this.outputBitmap,
+        draggable: true,
+        listening: true
+      });
+
+      this.mediaLayerRef.add(previewCopy);
+      previewCopy.moveToTop();
+      this.mediaLayerRef.draw();
+
+      // Start dragging immediately
+      previewCopy.startDrag();
+
+      // Handle drag events
+      previewCopy.on('dragmove', () => {
+        this.mediaLayerRef.draw();
+      });
+
+      previewCopy.on('dragend', () => {
+        const previewBox = previewCopy.getClientRect();
+        const captureBox = this.captureCanvas.getClientRect();
+        
+        if (Konva.Util.haveIntersection(previewBox, captureBox)) {
+          // Snap to capture canvas position
+          previewCopy.position({
+            x: this.captureCanvas.x(),
+            y: this.captureCanvas.y()
+          });
+          this.mediaLayerRef.batchDraw();
+        } else {
+          // Remove if dropped outside capture area
+          previewCopy.destroy();
+          this.mediaLayerRef.batchDraw();
+        }
+      });
+    });
+  }
+
+  public findImageNodeById(id: string): (ImageNode | TextNode | ShapeNode | undefined) {
+    return this.imageNodes.find(node => {
+      if (node.kNode) {
+        return node.kNode.id() === id;
+      }
+      return false;
+    });
+  }
   async updateCaptureCanvas(
     width: number | undefined,
     height: number | undefined,
@@ -158,7 +214,7 @@ export class RealTimeDrawEngine {
     const oldPositionY = this.positionY;
 
     // recompute the position
-    const padBetweenCaptureAndPreview = 10;
+    const padBetweenCaptureAndPreview = 2;
     this.positionX =
       window.innerWidth / 2 -
       this.width / 2 -
@@ -266,6 +322,7 @@ export class RealTimeDrawEngine {
     });
     return canvas;
   }
+
   public isProcessing = false;
   private handleNodeDragEnd = async () => {
     // Clean up any existing state
@@ -276,12 +333,13 @@ export class RealTimeDrawEngine {
     this.isProcessing = true;
     await this.render();
   };
+  
   public addNodes(node: MediaNode) {
-    if (node instanceof VideoNode) {
-      this.videoNodes.push(node);
-      node.kNode.on("dragend", this.handleNodeDragEnd);
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
+   
+    if (node instanceof ImageNode || node instanceof TextNode || node instanceof ShapeNode) {
+      console.debug("Adding node:", node);
       this.imageNodes.push(node);
+      console.log(this.imageNodes)
       node.kNode.on("dragend", this.handleNodeDragEnd);
     }
   }
@@ -317,6 +375,7 @@ export class RealTimeDrawEngine {
     try {
       const box = config.layerOfInterest.getClientRect();
       const stage = config.layerOfInterest.getStage();
+      
 
       const x = config.x !== undefined ? config.x : Math.floor(box.x);
       const y = config.y !== undefined ? config.y : Math.floor(box.y);
@@ -427,10 +486,36 @@ export class RealTimeDrawEngine {
     });
   }
 
-  public async render() {
-    // only pick nodes that intersect with the canvas on screen bounds to freeze.
-    this.mediaLayerRef.draw();
+  public async saveOutput() {
+    if (!this.outputBitmap) {
+      console.error("No output bitmap available to save");
+      return;
+    }
 
+    try {
+      const canvas = document.createElement("canvas");
+      canvas.width = this.outputBitmap.width;
+      canvas.height = this.outputBitmap.height;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) throw new Error("Failed to get 2D context");
+
+      ctx.drawImage(this.outputBitmap, 0, 0);
+      const blob = await new Promise<Blob>((resolve) => {
+        canvas.toBlob((b) => resolve(b!), "image/png");
+      });
+
+      await FileUtilities.blobToFileJpeg(blob, "output");
+    } catch (error) {
+      console.error("Error saving output:", error);
+    }
+  }
+
+  public async render() {
+    // only pick nodes that intersect wi th the canvas on screen bounds to freeze.
+    this.mediaLayerRef.draw();
+    // Output all nodes in mediaLayerRef
+    const nodes = this.mediaLayerRef.getChildren();
+    console.log("All nodes in mediaLayer:", nodes);
     console.log(
       `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`,
     );
@@ -447,17 +532,29 @@ export class RealTimeDrawEngine {
       test: false,
     })) as ImageBitmap;
 
+    // Test code
+    if (false) {
+      this.outputBitmap = bitmap;
+      this.previewCanvas.image(bitmap);
+      this.isProcessing = false;
+      return;
+    }
+
     try {
       const base64Bitmap = await this.imageBitmapToBase64(bitmap);
-      console.log("Sending to server");
+
       const base64BitmapResponse = await invoke("infer_image", {
         image: base64Bitmap,
+        prompt: this.currentPrompt,
+        strength: this.currentStrength,
       });
-      console.log("RESPONSE IS");
+
       console.log(base64BitmapResponse);
       const decoded = await this.base64ToImageBitmap(
         base64BitmapResponse as string,
       );
+
+      this.outputBitmap = decoded;
       this.previewCanvas.image(decoded);
     } catch (error) {
       console.error("Error during image processing:", error);

frontend/realtime/src/KonvaApp/types/NodeData.ts

@@ -1,4 +1,4 @@
-import { NodeType, BaseNode, ImageNode, TextNode, VideoNode } from "../Nodes";
+import { NodeType, BaseNode, ImageNode, TextNode, VideoNode, ShapeNode } from "../Nodes";
 
 import { TransformationData } from "./Transformation";
 
@@ -6,7 +6,7 @@ import { TextNodeData } from "./Text";
 
 export type MediaNode =
   // | NetworkedNode
-  BaseNode | VideoNode | ImageNode | TextNode;
+  BaseNode | VideoNode | ImageNode | TextNode | ShapeNode;
 
 export type ImageNodeData = {
   mediaFileUrl: string;
@@ -26,6 +26,10 @@ export type VideoNodeData = {
   extractionUrl?: string;
 };
 
+export type ShapeNodeData = {
+  shape: "triangle" | "square" | "circle";
+}
+
 export type NodeData = {
   type: NodeType;
   transform: TransformationData;

frontend/realtime/src/KonvaRootComponent/ContextualToolbarNode.tsx

@@ -50,6 +50,20 @@ export const ContextualToolbarNode = () => {
     };
   }, []);
 
+  const handleColorChange = (color: string) => {
+    // Don't do anything if no node is known to be bound
+    if (uiAccess.toolbarNode.signal.value.knodeIds.length < 1) {
+      return;
+    }
+
+    // Update the UI component
+    uiAccess.toolbarNode.setColor(color);
+
+    // Signal the engine with the node ID (just grab the first -and expected only- node)
+    const nodeID = uiAccess.toolbarNode.signal.value.knodeIds[0];
+    dispatchUiEvents.toolbarNode.setSelectedColor({ kNodeId: nodeID, color });
+  }
+
   const boundedTopPosition = Math.min(position.y, winSize.height - 164);
   return (
     <Transition
@@ -71,6 +85,8 @@ export const ContextualToolbarNode = () => {
         onLockClicked={(e) => {
           dispatchUiEvents.toolbarNode.lock(e);
         }}
+        color={uiAccess.toolbarNode.signal.value.color}
+        onColorChange={handleColorChange}
       />
     </Transition>
   );

frontend/realtime/src/KonvaRootComponent/KonvaRootComponent.tsx

@@ -23,6 +23,8 @@ import { useLayoutContext } from "./contextSignals/layout";
 import { useRenderCounter } from "~/hooks/useRenderCounter";
 import { useNavigate } from "react-router-dom";
 import { SignaledMagicBox } from "./SignaledMagicBox";
+import { SignaledPromptSlider } from "./SignaledPromptSlider";
+import { SignaledPromptText } from "./SignaledPromptText";
 
 export const KonvaRootComponent = ({
   className,
@@ -54,18 +56,20 @@ export const KonvaRootComponent = ({
       <KonvaCanvasContainer
         ref={konvaContainerCallbackRef}
         className={className}
-        // retreive the classNames from the parent for sizing/styling
+      // retreive the classNames from the parent for sizing/styling
       />
-      <SignaledMagicBox />
+      {/* <SignaledMagicBox /> */}
       <SignaledCanvasDragDropFiles
         openAddImage={appUiContext.openAddImage}
         openAddVideo={appUiContext.openAddVideo}
       />
+      <SignaledPromptSlider />
+      <SignaledPromptText />
       <SignaledToolbarMain
         layoutSignal={layoutContext.signal}
         appUiContext={appUiContext}
       />
-      <SignaledToolbarVideoExtraction />
+      {/* <SignaledToolbarVideoExtraction /> */}
       <SignaledDialogs
         appUiSignal={appUiContext.signal}
         resetAll={appUiContext.resetAll}
@@ -73,7 +77,7 @@ export const KonvaRootComponent = ({
 
       <ContextualLoadingBar />
       <ContextualToolbarNode />
-      <ContextualButtonRetry />
+      {/* <ContextualButtonRetry /> */}
     </>
   );
 };

frontend/realtime/src/KonvaRootComponent/SignaledPromptSlider.tsx

@@ -0,0 +1,22 @@
+
+import { useSignalEffect } from "@preact/signals-react";
+import { Slider } from "~/components/ui"
+import { dispatchUiEvents } from "~/signals"
+import { promptStrength } from "~/signals/uiEvents/promptSettings";
+
+export const SignaledPromptSlider = () => {
+
+    const sliderValue = promptStrength.value
+    const onSliderValueChanged = dispatchUiEvents.promptSettings.setPromptStrength
+
+    // This makes component re-render whenever the signal value changes
+    useSignalEffect(() => {
+        promptStrength.value;
+    })
+
+    return (
+        <div className="fixed bottom-80 left-1/2 -translate-x-1/2 h-4 w-96">
+            <Slider min={0} max={100} value={sliderValue} onChange={onSliderValueChanged} step={1} />
+        </div>
+    )
+}

frontend/realtime/src/KonvaRootComponent/SignaledPromptText.tsx

@@ -0,0 +1,46 @@
+import { useSignalEffect } from "@preact/signals-react";
+import { useCallback, useEffect, useRef } from "react";
+import { ResizeType, Textarea } from "~/components/ui";
+import { dispatchUiEvents } from "~/signals";
+import { promptText } from "~/signals/uiEvents/promptSettings";
+
+export const SignaledPromptText = ({
+}) => {
+  const textAreaRef = useRef<HTMLTextAreaElement>(null);
+  const focusOnTextArea = useCallback(() => {
+    if (textAreaRef.current === null) {
+      return;
+    }
+    textAreaRef.current.focus();
+  }, []);
+  
+  useEffect(() => {
+    focusOnTextArea();
+  }, []);
+
+  const TextAreaStates = {
+    style: {
+      width: "530px",
+      height: "200px",
+    },
+    placeholder: "",
+    label: "",
+    resize: "none" as ResizeType,
+  };
+
+  const textContent = promptText.value;
+  const onTextChanged = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
+    dispatchUiEvents.promptSettings.setPrompt(event.target.value);
+  };
+
+  // This makes component re-render whenever the signal value changes
+  useSignalEffect(() => {
+    promptText.value;
+  })
+
+  return (
+    <div className="fixed bottom-24 left-1/2 -translate-x-1/2">
+      <Textarea {...TextAreaStates} ref={textAreaRef} onChange={onTextChanged} value={textContent} />
+    </div>
+  );
+};

frontend/realtime/src/KonvaRootComponent/SignaledToolbarMain.tsx

@@ -12,6 +12,7 @@ import { LayoutSignalType } from "./contextSignals/layout";
 import { AppUiContextInterface } from "./contextSignals/appUi";
 import { twMerge } from "tailwind-merge";
 import { paperWrapperStyles } from "~/components/styles";
+import { dispatchUiEvents } from "~/signals";
 
 export const SignaledToolbarMain = ({
   // layoutSignal,
@@ -43,6 +44,12 @@ export const SignaledToolbarMain = ({
         return () => appUiContext.openAddImage();
       case ToolbarMainButtonNames.ADD_VIDEO:
         return () => appUiContext.openAddVideo();
+      case ToolbarMainButtonNames.ADD_CIRCLE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "circle" });
+      case ToolbarMainButtonNames.ADD_TRIANGLE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "triangle" });
+      case ToolbarMainButtonNames.ADD_SQUARE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "square" });
       default:
         return dispatchers[buttonName];
     }

frontend/realtime/src/app/global.css

@@ -54,10 +54,10 @@
   .dotted-pattern {
     background-image: radial-gradient(
       circle at 1px 1px,
-      rgb(0 0 0 / 0.25) 1px,
+      rgb(255 255 255 / 0.25) 1px,
       transparent 0
     );
-    background-size: 32px 32px;
+    background-size: 40px 40px;
   }
 
   .gradient-text {

frontend/realtime/src/app/pages/landing/index.tsx

@@ -1,107 +1,169 @@
 import Navbar from "../../../components/landing/Navbar";
-import Hero from "../../../components/landing/Hero";
-import FeatureCard from "../../../components/landing/FeatureCard";
 import Footer from "../../../components/landing/Footer";
+import { Button } from "~/components/ui";
+import { faApple } from "@fortawesome/free-brands-svg-icons";
+import { faDesktop } from "@fortawesome/pro-solid-svg-icons";
+import { useIsMobile } from "~/hooks/useIsMobile";
 
 const Landing = () => {
+  const isMobile = useIsMobile();
   const features = [
     {
-      title: "Step 1: Compose Your Scene",
+      title: "Lorem ipsum dolor sit",
       description:
-        "Begin your story on our intuitive canvas. Drop and arrange photos and videos to create layered scenes. Position, scale, and transform elements with precision to build your perfect composition. Our dynamic canvas makes it easy to bring your vision to life.",
-      image: "/images/feature_compositing.png",
-      gradient: "bg-gradient-to-br from-purple-500/20 to-blue-500/20",
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt.",
+      image: "/images/features/ai-creation.jpg",
+      tag: "Core",
     },
     {
-      title: "Step 2: Smart Segmentation",
+      title: "Ut enim ad minim",
       description:
-        "Clean up your scenes with powerful AI segmentation. Remove green screens instantly, isolate subjects from any background, and perfect your layer masks with intelligent tools. Create professional-quality compositions with just a few clicks.",
-      image: "/images/feature_segmentation.png",
-      gradient: "bg-gradient-to-br from-emerald-500/20 to-cyan-500/20",
+        "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea.",
+      image: "/images/features/collaboration.jpg",
+      tag: "Workflow",
     },
     {
-      title: "Step 3: AI Style Transfer",
+      title: "Duis aute irure dolor",
       description:
-        "Transform your composition with AI style transfer magic. Choose from our curated collection of artistic styles. Apply consistent styling across your entire scene to create a cohesive visual masterpiece.",
-      image: "/images/feature_ai_stylize.png",
-      gradient: "bg-gradient-to-br from-rose-500/20 to-orange-500/20",
-    },
-    {
-      title: "Step 4: Render Your Video",
-      description:
-        "Render your styled composition into a stunning final movie. Export in high quality, ready to share with the world.",
-      image: "/images/feature_render.png",
-      gradient: "bg-gradient-to-br from-blue-500/20 to-indigo-500/20",
+        "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla.",
+      image: "/images/features/style-models.jpg",
+      tag: "Tools",
     },
   ];
 
-  const scrollToFeature = (index: number) => {
-    const element = document.getElementById(`feature-${index + 1}`);
-    if (element) {
-      element.scrollIntoView({ behavior: "smooth", block: "center" });
-    }
-  };
-
   return (
-    <div className="min-h-screen bg-white">
+    <div className="relative min-h-screen bg-[#101014] text-white">
       <Navbar />
 
-      <div>
-        <Hero onFeatureClick={scrollToFeature} />
-
-        <section className="relative overflow-hidden py-32">
-          {/* Decorative elements */}
-          <div className="dotted-pattern absolute inset-0 opacity-50" />
-
-          <div className="relative mx-auto max-w-7xl px-4 sm:px-6 lg:px-8">
-            {/* Section header with enhanced styling */}
-            <div className="mx-auto mb-24 max-w-3xl text-center">
-              <div className="mb-6 inline-flex items-center rounded-full bg-gradient-to-r from-purple-500/10 via-blue-500/10 to-purple-500/10 px-8 py-2.5">
-                <span className="whitespace-nowrap bg-gradient-to-r from-purple-600 via-blue-600 to-purple-600 bg-clip-text text-sm font-semibold text-transparent">
-                  AI-POWERED WORKFLOW
+      <div className="dotted-pattern absolute inset-0 z-[0] opacity-50" />
+      {/* Hero Section */}
+      <div className="relative flex overflow-hidden pt-24 xl:min-h-[1000px] xl:items-center xl:pt-0">
+        {/* Content Container */}
+        <div className="relative z-10 mx-auto grid h-full w-full max-w-[1920px] grid-cols-12 items-center px-6 md:px-16 xl:px-32">
+          <div className="z-10 col-span-12 w-full xl:absolute xl:left-0 xl:top-1/2 xl:max-w-4xl xl:-translate-y-1/2 xl:pl-32 before:xl:absolute before:xl:-inset-0 before:xl:-z-10 before:xl:mr-40 before:xl:rounded-full before:xl:bg-black/90 before:xl:to-transparent before:xl:blur-[180px]">
+            <div className="max-w-2xl">
+              <div className="mb-6">
+                <span className="text-lg font-semibold uppercase tracking-widest text-gray-400">
+                  MIRA
                 </span>
               </div>
-              <h2 className="mb-6 text-4xl font-bold tracking-tight text-gray-900">
-                Create Stunning Videos in
-                <span className="relative">
-                  <span className="gradient-text relative z-10">
-                    {" "}
-                    Four Simple Steps
-                  </span>
-                  <span className="bg-rose-200/50 absolute -bottom-2 left-0 z-0 h-3 w-full" />
-                </span>
-              </h2>
-              <p className="text-xl leading-relaxed text-gray-600">
-                From composition to final render, transform your media into
-                cinematic stories with our powerful AI-enhanced workflow.
+
+              <h1 className="mb-6 font-bold leading-tight lg:text-6xl">
+                Realtime AI editor.
+                <br />
+                On your local machine.
+              </h1>
+
+              <p className="mb-10 max-w-xl text-lg leading-relaxed">
+                Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do
+                eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
+                enim ad minim veniam, quis nostrud exercitation.
               </p>
-            </div>
 
-            {/* Features stack with enhanced visual separation */}
-            <div className="relative space-y-32">
-              {/* Vertical timeline line */}
-              <div className="to-rose-500/20 absolute left-1/2 top-0 h-full w-px bg-gradient-to-b from-purple-500/20 via-blue-500/20" />
-
-              {features.map((feature, index) => (
-                <div
-                  key={index}
-                  id={`feature-${index + 1}`}
-                  className="relative"
+              <div className="flex flex-col gap-4 sm:flex-row">
+                <Button
+                  className={`w-full transform rounded-lg px-8 py-4 font-semibold transition sm:w-auto ${
+                    isMobile
+                      ? "cursor-not-allowed bg-[#2D81FF]"
+                      : "bg-[#2D81FF] hover:scale-105 hover:bg-[#438AF6]"
+                  }`}
+                  disabled={isMobile}
+                  icon={isMobile ? faDesktop : faApple}
                 >
-                  {/* Step number bubble */}
-                  <div className="absolute left-1/2 -translate-x-1/2 -translate-y-16">
-                    <div className="flex h-12 w-12 items-center justify-center rounded-full border bg-white shadow-lg">
-                      <span className="bg-gradient-to-br from-purple-600 to-purple-600 bg-clip-text text-xl font-bold text-transparent">
-                        {index + 1}
-                      </span>
-                    </div>
-                  </div>
-                  <FeatureCard {...feature} isReversed={index % 2 === 1} />
-                </div>
-              ))}
+                  {isMobile ? "Download on desktop" : "Download for MacOS"}
+                </Button>
+                <button className="w-full transform rounded-lg border border-white/30 px-8 py-4 font-semibold transition hover:scale-105 hover:bg-white/10 sm:w-auto">
+                  Learn more
+                </button>
+              </div>
             </div>
           </div>
-        </section>
+          <div className="col-span-12 mt-12 aspect-video w-full transform overflow-hidden rounded-xl border border-white/[16%] bg-transparent md:mt-16 xl:col-span-9 xl:col-start-4 xl:mt-0">
+            <img
+              src="/images/landing_hero.png"
+              alt="AI Video Creation"
+              className="h-full w-full object-cover object-top opacity-90"
+            />
+          </div>
+        </div>
+      </div>
+
+      {/* Features Section */}
+      <div className="relative z-10 mx-auto mb-32 max-w-[1920px] px-4 py-20 sm:px-24 lg:px-32">
+        <div className="mb-8 flex items-center justify-between">
+          <h1 className="text-4xl font-bold">Features</h1>
+          {/* <button className="text-sm text-[#0096CE] hover:text-[#0084B5]">
+            View all
+          </button> */}
+        </div>
+
+        <div className="grid gap-6 md:grid-cols-3">
+          {features.map((item, index) => (
+            <div
+              key={index}
+              className="group cursor-pointer overflow-hidden rounded-xl bg-[#1C1C20] transition hover:bg-[#27272B]"
+            >
+              <div className="aspect-video overflow-hidden">
+                <img
+                  src={item.image}
+                  alt={item.title}
+                  className="h-full w-full object-cover transition group-hover:scale-105"
+                />
+              </div>
+              <div className="p-6">
+                {/* <div className="mb-4 text-sm text-[#0096CE]">{item.tag}</div> */}
+                <h3 className="mb-2 text-xl font-semibold">{item.title}</h3>
+                <p className="text-gray-400">{item.description}</p>
+              </div>
+            </div>
+          ))}
+        </div>
+
+        {/* Installation Steps Section */}
+        <div className="mt-36">
+          <h1 className="mb-12 text-5xl font-bold">
+            How to install Mira AI Editor
+          </h1>
+          <div className="grid gap-6 md:grid-cols-3">
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 1
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Download the app</h3>
+              <p className="mb-6 text-gray-400">
+                Download and install our application to start creating amazing
+                AI-powered content.
+              </p>
+              <Button className="rounded-lg bg-[#2D81FF] px-4 py-2 text-sm font-medium hover:bg-[#438AF6]">
+                Download App
+              </Button>
+            </div>
+
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 2
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Create an account</h3>
+              <p className="mb-6 text-gray-400">
+                Sign up for an account to access all features and start your
+                journey.
+              </p>
+              <Button className="text-sm font-medium" variant="secondary">
+                Sign up
+              </Button>
+            </div>
+
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 3
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Start creating</h3>
+              <p className="mb-6 text-gray-400">
+                Begin creating with our realtime AI editor!
+              </p>
+            </div>
+          </div>
+        </div>
       </div>
 
       <Footer />

frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx

@@ -42,12 +42,6 @@ export const DialogAddImage = ({
     closeCallback();
   }
 
-  function handleEnter() {
-    if (currFile) {
-      onAddImage(currFile);
-    }
-    handleClose();
-  }
   return (
     <Dialog open={isOpen} onClose={closeCallback} className="relative z-50">
       <div className={dialogBackgroundStyles}>
@@ -66,6 +60,10 @@ export const DialogAddImage = ({
               file={currFile}
               setFile={(file: File | null) => {
                 setAssetFile(file);
+                if (file) {
+                  onAddImage(file);
+                  handleClose();
+                }
               }}
             />
             {currFile && (
@@ -84,9 +82,6 @@ export const DialogAddImage = ({
             <Button onClick={handleClose} variant="secondary">
               Cancel
             </Button>
-            <Button onClick={handleEnter} disabled={currFile === null}>
-              Enter
-            </Button>
           </div>
         </DialogPanel>
       </div>

frontend/realtime/src/components/features/ToolbarMain/ButtonPreviewAndRender.tsx

@@ -11,8 +11,6 @@ export const ButtonPreviewAndRender = ({
 }) => {
   return (
     <div className="flex">
-      <ButtonPreview {...buttonPreviewProps} />
-      <ButtonRender {...buttonRenderProps} />
     </div>
   );
 };

frontend/realtime/src/components/features/ToolbarMain/ToolbarMain.tsx

@@ -5,13 +5,17 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import {
   faArrowRotateLeft,
   faArrowRotateRight,
+  faCircle,
   faFilePlus,
   faFilm,
   faFloppyDisk,
   faImage,
   faLocationArrow,
   faMagicWandSparkles,
+  faShapes,
+  faSquare,
   faText,
+  faTriangle,
 } from "@fortawesome/pro-solid-svg-icons";
 
 import { ToolbarButton } from "../ToolbarButton";
@@ -41,7 +45,7 @@ export const ToolbarMain = ({
           "flex w-fit items-center divide-x divide-ui-border",
           paperWrapperStyles,
           disabled &&
-            "pointer-events-none cursor-default bg-ui-border shadow-md",
+          "pointer-events-none cursor-default bg-ui-border shadow-md",
         )}
       >
         <div className="flex items-center gap-2 pr-2">
@@ -56,6 +60,33 @@ export const ToolbarMain = ({
             buttonProps={buttonProps.ADD_TEXT}
             tooltip="Add Text"
           />
+
+          <Popover className="relative">
+            <PopoverButton as={Fragment}>
+              <button
+                data-tooltip="Add Shapes"
+                className={twMerge(
+                  "size-10 rounded-lg p-2 hover:bg-gray-200/50",
+                  toolTipStyles,
+                )}
+              >
+                <FontAwesomeIcon icon={faShapes} />
+              </button>
+            </PopoverButton>
+            <PopoverPanel
+              anchor="bottom"
+              className={twMerge(
+                // "absolute bottom-full left-1/2 z-10 mb-2 -translate-x-1/2",
+                "flex flex-col [--anchor-gap:16px]",
+                paperWrapperStyles,
+              )}
+            >
+              <ToolbarButton icon={faSquare} buttonProps={buttonProps.ADD_SQUARE} />
+              <ToolbarButton icon={faCircle} buttonProps={buttonProps.ADD_CIRCLE} />
+              <ToolbarButton icon={faTriangle} buttonProps={buttonProps.ADD_TRIANGLE} />
+            </PopoverPanel>
+          </Popover>
+
           <Popover className="relative">
             <PopoverButton as={Fragment}>
               <button
@@ -79,19 +110,12 @@ export const ToolbarMain = ({
               <ToolbarButton icon={faImage} buttonProps={buttonProps.ADD_IMAGE}>
                 Add Image
               </ToolbarButton>
-              <ToolbarButton icon={faFilm} buttonProps={buttonProps.ADD_VIDEO}>
+              {/* <ToolbarButton icon={faFilm} buttonProps={buttonProps.ADD_VIDEO}>
                 Add Video
-              </ToolbarButton>
+              </ToolbarButton> */}
             </PopoverPanel>
           </Popover>
-          <ToolbarButton
-            icon={faMagicWandSparkles}
-            buttonProps={{
-              ...buttonProps.AI_STYLIZE,
-              prominent: true,
-            }}
-            tooltip="AI Stylize"
-          />
+
         </div>
         <div className="flex items-center gap-2 pl-2">
           <ToolbarButton

frontend/realtime/src/components/features/ToolbarMain/enum.ts

@@ -4,6 +4,9 @@ export enum ToolbarMainButtonNames {
   ADD_TEXT = "ADD_TEXT",
   ADD_IMAGE = "ADD_IMAGE",
   ADD_VIDEO = "ADD_VIDEO",
+  ADD_CIRCLE = "ADD_CIRCLE",
+  ADD_TRIANGLE = "ADD_TRIANGLE",
+  ADD_SQUARE = "ADD_SQUARE",
   AI_STYLIZE = "AI_STYLIZE",
   UNDO = "UNDO",
   REDO = "REDO",

frontend/realtime/src/components/features/ToolbarNode/ToolbarNode.tsx

@@ -4,14 +4,19 @@ import {
   faLockKeyholeOpen,
   faLockKeyhole,
   faUnlockKeyhole,
+  faPalette,
 } from "@fortawesome/pro-solid-svg-icons";
 import {
   ToolbarButton,
   ToolbarButtonProps,
 } from "~/components/features/ToolbarButton";
-import { paperWrapperStyles } from "~/components/styles";
+import { paperWrapperStyles, toolTipStyles } from "~/components/styles";
 import { ToolbarNodeButtonNames } from "./enums";
 import { ToolbarNodeButtonData } from "./data";
+import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
+import { Fragment } from "react/jsx-runtime";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { ColorPicker } from "~/components/ui/TextEditor/ColorPicker";
 
 export interface ToolbarNodeProps {
   disabled?: boolean;
@@ -24,6 +29,8 @@ export interface ToolbarNodeProps {
   buttonsProps?: {
     [key in ToolbarNodeButtonNames]: ToolbarButtonProps;
   };
+  color?: string;
+  onColorChange?: (color: string) => void;
 }
 export const ToolbarNode = ({
   disabled,
@@ -31,6 +38,8 @@ export const ToolbarNode = ({
   lockDisabled,
   onLockClicked,
   buttonsProps,
+  color = "#000000",
+  onColorChange = () => { },
 }: ToolbarNodeProps) => {
   const handleOnLockClicked: React.MouseEventHandler<HTMLButtonElement> = (
     e,
@@ -53,7 +62,7 @@ export const ToolbarNode = ({
           className: twMerge(
             locked && "text-primary hover:bg-primary hover:text-white",
             lockDisabledOrUnknown &&
-              "text-secondary-300 hover:text-secondary-300",
+            "text-secondary-300 hover:text-secondary-300",
           ),
           disabled: lockDisabledOrUnknown,
         }}
@@ -73,14 +82,24 @@ export const ToolbarNode = ({
       {ToolbarNodeButtonData.map((buttonDatum, idx) => {
         const buttonProps = buttonsProps?.[buttonDatum.name];
 
-        return (
-          <ToolbarButton
-            icon={buttonDatum.icon}
-            tooltip={buttonDatum.tooltip}
-            key={idx}
-            buttonProps={buttonProps}
-          />
-        );
+        if (!buttonProps || buttonProps.hidden) {
+          return;
+        }
+
+        if (buttonDatum.name !== ToolbarNodeButtonNames.COLOR) {
+          return (
+            <ToolbarButton
+              icon={buttonDatum.icon}
+              tooltip={buttonDatum.tooltip}
+              key={idx}
+              buttonProps={buttonProps}
+            />
+          );
+        } else {
+          return (
+            <ColorPicker color={color} onChange={onColorChange} faIcon={faPalette} borderStyle="" key={idx} />
+          )
+        }
       })}
     </div>
   );

frontend/realtime/src/components/features/ToolbarNode/data.ts

@@ -8,6 +8,7 @@ import {
   faTrashCan,
   faVectorSquare,
   faDownload,
+  faPalette,
 } from "@fortawesome/pro-solid-svg-icons";
 
 export const ToolbarNodeButtonData = [
@@ -16,6 +17,11 @@ export const ToolbarNodeButtonData = [
     icon: faVectorSquare,
     tooltip: "Move",
   },
+  {
+    name: ButtonNames.COLOR,
+    icon: faPalette,
+    tooltip: "Color",
+  },
   {
     name: ButtonNames.CHROMA,
     icon: faTransporter,

frontend/realtime/src/components/features/ToolbarNode/enums.ts

@@ -7,4 +7,5 @@ export enum ToolbarNodeButtonNames {
   MOVE_LAYER_UP = "MOVE_LAYER_UP",
   MOVE_LAYER_DOWN = "MOVE_LAYER_DOWN",
   DOWNLOAD = "DOWNLOAD",
+  COLOR = "COLOR",
 }

frontend/realtime/src/components/landing/Footer.tsx

@@ -9,18 +9,18 @@ const Footer = () => {
   const navigate = useNavigate();
 
   return (
-    <footer className="border-t border-gray-100 bg-gray-50">
+    <footer className="relative z-10 bg-[#18181C]">
       <div className="mx-auto max-w-7xl px-4 py-16 sm:px-6 lg:px-8">
         <div className="flex flex-col items-center space-y-10 text-center">
           <img
-            src="/brand/Storyteller-Logo-Black.png"
-            alt="Storyteller Board"
-            className="h-9"
+            src="/brand/mira-logo.png"
+            alt="Mira Realtime AI Editor"
+            className="h-[34px] pb-1"
           />
 
           <Button
             onClick={() => navigate("/signup")}
-            className="group rounded-xl px-8 py-4 text-lg"
+            className="text-md group rounded-xl bg-[#2D81FF] px-8 py-4 hover:bg-[#438AF6]"
           >
             <span className="relative flex items-center">
               Start Creating for Free

frontend/realtime/src/components/landing/Navbar.tsx

@@ -3,26 +3,31 @@ import { Button } from "../ui";
 
 const Navbar = () => {
   const navigate = useNavigate();
-
   return (
-    <nav className="fixed top-0 z-50 w-full border-b border-gray-100 bg-white/80 backdrop-blur-md">
-      <div className="mx-auto max-w-7xl px-4 sm:px-6 lg:px-8">
-        <div className="flex h-20 items-center justify-between">
+    <nav className="fixed top-0 z-50 w-full bg-[rgba(16,16,20,0.7)] backdrop-blur-3xl">
+      <div className="mx-auto max-w-[1920px] px-4 sm:px-24 lg:px-32">
+        <div className="flex h-[72px] items-center justify-between">
           <a href="/" className="flex items-center">
             <img
-              src="/brand/Storyteller-Logo-Black.png"
-              alt="Storyteller Board"
-              className="h-9"
+              src="/brand/mira-logo.png"
+              alt="Mira Realtime AI Editor"
+              className="h-[34px] pb-1"
             />
           </a>
-          <div className="flex items-center space-x-5">
-            <a
-              href="/login"
-              className="font-medium !text-gray-600 hover:!text-gray-900"
+          <div className="flex items-center space-x-3">
+            <Button
+              variant="secondary"
+              className=""
+              onClick={() => navigate("/signup")}
             >
-              Login
-            </a>
-            <Button onClick={() => navigate("/signup")}>Sign Up</Button>
+              Sign up
+            </Button>
+            <Button
+              className="hidden bg-[#2D81FF] hover:bg-[#438AF6] md:block"
+              onClick={() => []}
+            >
+              Download
+            </Button>
           </div>
         </div>
       </div>

frontend/realtime/src/components/ui/TextArea.tsx

@@ -7,10 +7,16 @@ export type ResizeType =
   | "both"
   | "horizontal"
   | "vertical"
-  | "block"
-  | "inline"
   | undefined;
 
+const resizeStyles = {
+  none: "resize-none",
+  both: "resize",
+  horizontal: "resize-x",
+  vertical: "resize-y",
+  undefined: "",
+}
+
 export interface TextareaInterface
   extends TextareaHTMLAttributes<HTMLTextAreaElement> {
   label?: string;
@@ -21,6 +27,9 @@ export const Textarea = forwardRef<HTMLTextAreaElement, TextareaInterface>(
     { className, label, resize = "vertical", id, ...rest }: TextareaInterface,
     ref,
   ) => {
+
+    const resizeStyle = resizeStyles[resize];
+
     return (
       <div className="flex flex-col">
         {label && <label htmlFor={id ? id : kebabCase(label)}>{label}</label>}
@@ -31,11 +40,11 @@ export const Textarea = forwardRef<HTMLTextAreaElement, TextareaInterface>(
           className={twMerge(
             "border-ui-panel-border bg-ui-controls rounded-lg border px-3 py-2",
             className,
+            resizeStyle,
           )}
           style={{
             outline: "2px solid transparent",
             transition: "outline-color 0.15s ease-in-out",
-            resize: resize,
           }}
           onFocus={(e) => {
             // disableHotkeyInput(DomLevels.INPUT);

frontend/realtime/src/components/ui/TextEditor/ColorPicker.tsx

@@ -1,6 +1,6 @@
 import { ChangeEvent, useState } from "react";
 import { twMerge } from "tailwind-merge";
-import { faFont, faHashtag } from "@fortawesome/pro-solid-svg-icons";
+import { faFont, faHashtag, IconDefinition } from "@fortawesome/pro-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
 import { HexColorPicker } from "react-colorful";
@@ -12,9 +12,15 @@ import { paperWrapperStyles } from "~/components/styles";
 export const ColorPicker = ({
   color: prevColor,
   onChange,
+  faIcon = faFont,
+  borderStyle = "border",
+  showBar = true,
 }: {
   color: string;
   onChange: (newColor: string) => void;
+  faIcon?: IconDefinition;
+  borderStyle?: string;
+  showBar?: boolean;
 }) => {
   const [{ currColor, textInput }, setStates] = useState<{
     currColor: string;
@@ -44,11 +50,15 @@ export const ColorPicker = ({
       }));
     }
   };
+
   return (
     <Popover className="relative">
-      <PopoverButton className="flex size-10 flex-col items-center gap-1 rounded-md border p-2">
-        <FontAwesomeIcon icon={faFont} />
-        <span className="h-1 w-full" style={{ backgroundColor: prevColor }} />
+      <PopoverButton className={twMerge(
+        "flex size-10 flex-col items-center gap-1 rounded-md p-2",
+        borderStyle
+      )}>
+        <FontAwesomeIcon icon={faIcon} color={prevColor} />
+        {showBar && <span className="h-1 w-full" style={{ backgroundColor: prevColor }} />}
       </PopoverButton>
       <PopoverPanel
         anchor="bottom"

frontend/realtime/src/hooks/useIsMobile.ts

@@ -0,0 +1,17 @@
+import { useEffect, useState } from "react";
+
+export const useIsMobile = () => {
+  const [isMobile, setIsMobile] = useState(false);
+
+  useEffect(() => {
+    const checkMobile = () => {
+      return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
+        navigator.userAgent,
+      );
+    };
+
+    setIsMobile(checkMobile());
+  }, []);
+
+  return isMobile;
+};

frontend/realtime/src/signals/uiAccess/toolbarNode.ts

@@ -10,12 +10,13 @@ type ButtonStates = {
   };
 };
 export interface ContextualToolbarProps extends ContextualUi {
-  knodeIds: number[];
+  knodeIds: string[];
   disabled: boolean;
   downloadUrl?: string;
   locked: boolean | "unknown";
   lockDisabled: boolean;
   buttonStates: ButtonStates;
+  color: string;
 }
 interface PartialContextualToolbarProps
   extends Partial<Omit<ContextualToolbarProps, "buttonStates">> {
@@ -32,6 +33,7 @@ const toolbarNodeSignal = signal<ContextualToolbarProps>({
   locked: false,
   lockDisabled: false,
   buttonStates: initButtonStates(),
+  color: "#000000",
 });
 
 export const toolbarNode = {
@@ -77,6 +79,11 @@ export const toolbarNode = {
           hidden: false,
           active: buttonStates[ButtonNames.TRANSFORM].active,
         },
+        [ButtonNames.COLOR]: {
+          disabled: locked,
+          hidden: false,
+          active: buttonStates[ButtonNames.COLOR].active,
+        },
       },
       locked: locked,
     };
@@ -135,6 +142,12 @@ export const toolbarNode = {
       },
     };
   },
+  setColor(color: string) {
+    toolbarNodeSignal.value = {
+      ...toolbarNodeSignal.value,
+      color,
+    }
+  },
   batchButtonStates({ buttonStates }: { buttonStates: Partial<ButtonStates> }) {
     const prevButtonStates = toolbarNodeSignal.value.buttonStates;
     toolbarNodeSignal.value = {

frontend/realtime/src/signals/uiEvents/addMedia.ts

@@ -1,11 +1,12 @@
 import { signal, effect } from "@preact/signals-react";
-import { TextNodeData } from "~/KonvaApp/types";
+import { ShapeNodeData, TextNodeData } from "~/KonvaApp/types";
 
 // ADDING IMAGES
 const stagedImage = signal<File | null>(null);
 const addImageToEngine = (image: File) => {
   stagedImage.value = image;
 };
+
 const onGetStagedImage = (callback: (file: File) => void) => {
   effect(() => {
     if (stagedImage.value) {
@@ -48,15 +49,30 @@ const onAddTextToEngine = (callback: (data: TextNodeData) => void) => {
   });
 };
 
+// Add shapes
+const stagedShape = signal<ShapeNodeData | null>(null);
+const addShapeToEngine = (data: ShapeNodeData) => {
+  stagedShape.value = data;
+}
+const onAddShapeToEngine = (callback: (data: ShapeNodeData) => void) => {
+  effect(() => {
+    if (stagedShape.value) {
+      callback(stagedShape.value);
+    }
+  });
+};
+
 //EXPORTS
 export const dispatchers = {
   addImageToEngine,
   addVideoToEngine,
   addTextToEngine,
+  addShapeToEngine,
 };
 
 export const events = {
   onGetStagedImage,
   onGetStagedVideo,
   onAddTextToEngine,
+  onAddShapeToEngine,
 };

frontend/realtime/src/signals/uiEvents/index.ts

@@ -24,6 +24,10 @@ import {
   dispatchers as toolbarVideoExtractionDispatchers,
   eventsHandlers as toolbarVideoExtractionEvents,
 } from "./toolbarVideoExtraction";
+import {
+  dispatchers as promptDispatchers,
+  events as promptEvents,
+} from "./promptSettings";
 
 export const uiEvents = {
   ...addMediaEvents,
@@ -34,6 +38,7 @@ export const uiEvents = {
   toolbarMain: toolbarMainEvents,
   toolbarNode: toolbarNodeEvents,
   toolbarVideoExtraction: toolbarVideoExtractionEvents,
+  promptEvents: promptEvents,
 };
 export const dispatchUiEvents = {
   ...addMediaDispatchers,
@@ -44,4 +49,5 @@ export const dispatchUiEvents = {
   toolbarMain: toolbarMainDispatchers,
   toolbarNode: toolbarNodeDispatchers,
   toolbarVideoExtraction: toolbarVideoExtractionDispatchers,
+  promptSettings: promptDispatchers,
 };

frontend/realtime/src/signals/uiEvents/promptSettings.ts

@@ -0,0 +1,44 @@
+import { effect, signal } from "@preact/signals-react";
+
+// Prompt settings
+// - Prompt text
+// - Prompt strength
+export const DEFAULT_PROMPT_TEXT: string = "";
+export const promptText = signal<string>(DEFAULT_PROMPT_TEXT);
+const setPrompt = (data: string) => {
+  promptText.value = data;
+};
+
+const onPromptTextChanged = (callback: (data: string) => void) => {
+  effect(() => {
+    if (promptText.value) {
+      callback(promptText.value);
+    }
+  });
+};
+
+export const DEFAULT_PROMPT_STRENGTH = 100;
+export const promptStrength = signal<number>(DEFAULT_PROMPT_STRENGTH);
+const setPromptStrength = (data: number) => {
+  // Round to integer, then clamp between 0 and 100
+  promptStrength.value = Math.max(0, Math.min(100, Math.round(data)));
+};
+
+const onPromptStrengthChanged = (callback: (data: number) => void) => {
+  effect(() => {
+    if (promptStrength.value) {
+      callback(promptStrength.value);
+    }
+  });
+};
+
+//EXPORTS
+export const dispatchers = {
+  setPrompt,
+  setPromptStrength,
+};
+
+export const events = {
+  onPromptTextChanged,
+  onPromptStrengthChanged,
+};

frontend/realtime/src/signals/uiEvents/toolbarNode.ts

@@ -44,6 +44,11 @@ const buttonEventsRecords = Object.values(ButtonNames).reduce(
 
 const buttonEventsHandlers = Object.values(ButtonNames).reduce(
   (acc, buttonName) => {
+    // We define custom events/handlers for color changes
+    if (buttonName === ButtonNames.COLOR) {
+      return acc;
+    }
+
     acc[buttonName] = {
       onClick: (callback: MouseEventHandler<HTMLButtonElement>) => {
         const { eventSignal, effectCleanup, lastEventTimestamp } =
@@ -111,9 +116,27 @@ const retryEventHandler = (callback: MouseEventHandler<HTMLButtonElement>) => {
 const retryDispatcher = (e: React.MouseEvent<HTMLButtonElement>) => {
   retryEvent.value = e;
 };
+
+export type NodeColor = {
+  kNodeId: string;
+  color: string;
+}
+const selectedColorEvent = signal<NodeColor | null>();
+const setSelectedColor = (nodeColor: NodeColor) => {
+  selectedColorEvent.value = nodeColor;
+}
+const onSelectedColorChanged = (callback: (color: NodeColor) => void) => {
+  effect(() => {
+    if (selectedColorEvent.value) {
+      callback(selectedColorEvent.value);
+    }
+  });
+};
+
 export const dispatchers = {
   lock: lockDispatcher,
   retry: retryDispatcher,
+  setSelectedColor,
   ...buttonDispatchers,
 };
 export const eventsHandlers = {
@@ -123,5 +146,8 @@ export const eventsHandlers = {
   retry: {
     onClick: retryEventHandler,
   },
+  color: {
+    onConfirmChanged: onSelectedColorChanged,
+  },
   ...buttonEventsHandlers,
 };

frontend/realtime/tailwind.config.js

@@ -54,6 +54,10 @@ export default {
         border: colors.zinc[200],
       },
     },
+    backgroundImage: {
+      "radial-gradient":
+        "radial-gradient(circle at left, var(--tw-gradient-stops))",
+    },
   },
   plugins: [],
 };

frontend/storyteller-board/src/components/features/ToolbarMain/ToolbarMain.tsx

@@ -79,9 +79,6 @@ export const ToolbarMain = ({
               <ToolbarButton icon={faImage} buttonProps={buttonProps.ADD_IMAGE}>
                 Add Image
               </ToolbarButton>
-              <ToolbarButton icon={faFilm} buttonProps={buttonProps.ADD_VIDEO}>
-                Add Video
-              </ToolbarButton>
             </PopoverPanel>
           </Popover>
           <ToolbarButton

script/build_production_realtime_desktop.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Tauri doesn't let you configure the frontend project directory statically, though they do provide an
+# environment variable to pass it to the CLI. Without doing this, the tauri cli randomly walks the
+# filesystem and finds the wrong frontend code.
+
+# TODO(bt,2025-02-13): This is not the correct way to get the root dir
+root_dir=$(pwd)
+export TAURI_FRONTEND_PATH="${root_dir}/frontend/realtime"
+
+cargo tauri build
+

script/launch_realtime_desktop.sh

@@ -8,4 +8,4 @@
 root_dir=$(pwd)
 export TAURI_FRONTEND_PATH="${root_dir}/frontend/realtime"
 
-cargo tauri dev
+cargo tauri dev --no-watch