render bug fix

- Both modes' menus have been separated for better maintenance
- Slider is generic, can be reused
- New signals added for engine to hook into
- Sliders tested working on the UI side

.idea/dictionaries/bt.xml

@@ -31,6 +31,7 @@
       <w>hifigan</w>
       <w>hparams</w>
       <w>huggingface</w>
+      <w>kasisnu</w>
       <w>lipsync</w>
       <w>mocap</w>
       <w>mochi</w>
@@ -47,6 +48,7 @@
       <w>techcrunch</w>
       <w>transcoded</w>
       <w>transcodings</w>
+      <w>unet</w>
       <w>unpermitted</w>
       <w>upserting</w>
       <w>upserts</w>

app_config.yaml

@@ -0,0 +1,7 @@
+image_width: 512
+image_height: 512
+#scheduler_steps: 15
+# NB(bt): It looks like we can get down to 2 sampling steps, perhaps ?
+scheduler_steps: 2
+scheduler_samples: 1
+cfg_scale: 1.1

crates/desktop/src-tauri/Cargo.toml

@@ -18,11 +18,33 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 tauri-build = { version = "2.0.4", features = [] }
 
 [dependencies]
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-log = "0.4"
-tauri = { version = "2.2.4", features = [] }
-tauri-plugin-log = "2.0.0-rc"
-once_cell = "1.20.3"
-image = "0.25.5"
+
+# Internal
+filesys = { path = "../../lib/files/filesys" }
+memory_caching = { path = "../../lib/caching/memory_caching" }
+
+# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
+cudarc = "0.13.7"
+
+# External
+anyhow = "1.0"
+bytes = "1.10"
 base64 = "0.22.1"
+candle-core = { version = "0.8.2", features = ["cuda"] }
+candle-nn = { version = "0.8.2", features = ["cuda"] }
+candle-transformers = { version = "0.8.2", features = ["cuda"] }
+hf-hub = "0.3.2"
+image = "0.25.5"
+serde_yaml = "0.9"
+log = "0.4"
+once_cell = "1.20.3"
+rand = "0.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tauri = { version = "2.2.4", features = [] }
+tauri-plugin-log = "2"
+tokenizers = "0.15.2"
+
+[features]
+default = ["cuda"]
+cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]

crates/desktop/src-tauri/capabilities/default.json

@@ -6,6 +6,7 @@
     "main"
   ],
   "permissions": [
-    "core:default"
+    "core:default",
+    "log:default"
   ]
-}
+}

crates/desktop/src-tauri/src/endpoints/flip_image.rs

@@ -1,11 +1,11 @@
-use std::io::Cursor;
+use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
-use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE};
 use image::codecs::png::{CompressionType, FilterType, PngEncoder};
 use image::ImageReader;
+use std::io::Cursor;
 
 #[tauri::command]
-pub fn infer_image(image: &str) -> Result<String, String> {
+pub fn flip_image(image: &str) -> Result<String, String> {
   println!("infer_image called; processing image...");
 
   let bytes = BASE64_STANDARD.decode(image)
@@ -18,17 +18,19 @@ pub fn infer_image(image: &str) -> Result<String, String> {
     .map_err(|err| format!("Image decode error: {}", err))?;
 
   let rotated_image = image.rotate180();
-  
+
   let mut buffer = Vec::new(); // TODO: Preallocate, better data structure
   let mut writer = Cursor::new(&mut buffer);
+
+  let encoder = PngEncoder::new_with_quality
+    (&mut writer, CompressionType::Fast, FilterType::Adaptive);
   
-  let encoder = PngEncoder::new_with_quality(&mut writer, CompressionType::Fast, FilterType::Adaptive);
   rotated_image.write_with_encoder(encoder)
     .map_err(|err| format!("Image encoding error: {}", err))?;
 
   let encoded = BASE64_STANDARD.encode(&buffer);
-  
+
   println!("Encoded: {:?}", encoded.split_at(10).0);
-  
+
   Ok(encoded)
 }

crates/desktop/src-tauri/src/endpoints/image_endpoint.rs

@@ -0,0 +1,117 @@
+use crate::ml::model_cache::ModelCache;
+use crate::ml::prompt_cache::PromptCache;
+use crate::ml::stable_diffusion::stable_diffusion_pipeline::{stable_diffusion_pipeline, Args};
+use crate::state::app_config::AppConfig;
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
+use bytes::BytesMut;
+use image::imageops::FilterType;
+use image::{DynamicImage, ImageFormat, ImageReader};
+use log::{error, info};
+use std::io::Cursor;
+use std::path::PathBuf;
+use tauri::State;
+
+const PROMPT_FILENAME : &str = "prompt.txt";
+const PROMPT: &str = "A beautiful landscape with mountains and a lake";
+
+const NEGATIVE_PROMPT: &str = "bad quality, bad faces, poor quality, blurry faces, watermark";
+
+/// This handler takes an image (as a base64 encoded string) and a prompt and returns
+/// an image (as a base64-encoded string).
+#[tauri::command]
+pub fn infer_image(
+  image: &str,
+  prompt: Option<String>,
+  strength: Option<u8>,
+  model_config: State<AppConfig>,
+  model_cache: State<ModelCache>,
+  prompt_cache: State<PromptCache>,
+) -> Result<String, String> {
+
+  let prompt = get_prompt_or_fallback(prompt);
+
+  info!("Prompt: {}", prompt);
+
+  let image = hydrate_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  let result = do_infer_image(&prompt, image, strength, &model_config, &model_cache, prompt_cache);
+  
+  if let Err(err) = result.as_deref() {
+    error!("There was an error: {:?}", err);
+  }
+  
+  result
+}
+
+fn do_infer_image(
+  prompt: &str,
+  image: DynamicImage,
+  strength: Option<u8>,
+  config: &AppConfig,
+  model_cache: &ModelCache,
+  prompt_cache: State<PromptCache>,
+) -> Result<String, String> {
+  println!("infer_image called; generating image with SDXL Turbo...");
+  
+  let args = Args {
+    image: &image,
+    prompt: prompt.to_string(),
+    uncond_prompt: NEGATIVE_PROMPT.to_string(),
+    //guidance_scale: Some(0.0),
+    model_cache,
+    configs: config,
+    prompt_cache: &prompt_cache,
+    i2i_strength: strength,
+    cfg_scale: config.cfg_scale,
+  };
+
+  match stable_diffusion_pipeline(args) {
+    Ok(image) => {
+      let mut bytes = Vec::with_capacity(1024*1024);
+      
+      image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
+        .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+      let bytes = BASE64_STANDARD.encode(bytes);
+
+      Ok(bytes)
+    },
+    Err(e) => Err(format!("Failed to generate image: {}", e)),
+  }
+}
+
+fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
+  let user_prompt = user_prompt.map(|prompt| prompt.trim().to_string())
+    .filter(|prompt| !prompt.is_empty());
+
+  if let Some(prompt) = user_prompt {
+    return prompt;
+  }
+
+  let prompt_file = PathBuf::from(PROMPT_FILENAME)
+    .canonicalize()
+    .unwrap_or_else(|_| PathBuf::from(PROMPT_FILENAME));
+
+  std::fs::read_to_string(&prompt_file)
+    .map_err(|err| format!("Failed to read prompt file: {}", err))
+    .unwrap_or_else(|_| {
+      error!("Failed to read prompt file: {:?}", prompt_file);
+      PROMPT.to_string()
+    })
+    .trim()
+    .to_string()
+}
+
+fn hydrate_base64_image(base64_image: &str) -> anyhow::Result<DynamicImage> {
+  let bytes = BASE64_STANDARD.decode(base64_image)?;
+
+  let image = ImageReader::new(Cursor::new(bytes))
+    .with_guessed_format()?
+    .decode()?;
+
+  // TODO(bt,2025-02-17): Running out of vram with full image buffer size
+  let image = image.resize(512, 512, FilterType::CatmullRom);
+
+  Ok(image)
+}

crates/desktop/src-tauri/src/endpoints/mod.rs

@@ -0,0 +1,3 @@
+pub mod flip_image;
+pub mod image_endpoint;
+pub mod test_counter;

crates/desktop/src-tauri/src/endpoints/test_counter.rs

@@ -0,0 +1,24 @@
+use std::sync::{Arc, RwLock};
+use once_cell::sync::Lazy;
+
+static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
+
+#[tauri::command]
+pub fn test_counter() -> u64 {
+  let value : u64;
+  {
+    match COUNTER.write() {
+      Ok(mut counter) => {
+        *counter += 1;
+        value = *counter;
+      },
+      Err(_e) => {
+        value = 0;
+      }
+    }
+  }
+
+  println!("I was invoked from JavaScript! {:?}", value);
+
+  value
+}

crates/desktop/src-tauri/src/lib.rs

@@ -1,55 +1,60 @@
-mod image_endpoint;
+pub mod endpoints;
+pub mod ml;
+pub mod state;
 
-use std::sync::{Arc, RwLock};
-use once_cell::sync::Lazy;
-use image_endpoint::infer_image;
+use crate::ml::model_cache::ModelCache;
+use crate::ml::prompt_cache::PromptCache;
+use crate::state::app_config::AppConfig;
+use endpoints::image_endpoint::infer_image;
+use endpoints::test_counter::test_counter;
+use tauri_plugin_log::Target;
+use tauri_plugin_log::TargetKind;
 
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
+  // NB: Tauri wants to install the logger itself, so we can't rely on the logger crate 
+  // until the tauri runtime begins.
+  println!("Loading model config...");
+
+  let config = AppConfig::init()
+    .expect("config should load");
+
+  let prompt_cache = PromptCache::with_capacity(8)
+    .expect("prompt cache should load");
+
+  println!("Creating model cache...");
+  
+  let model_cache = ModelCache::new(
+    config.device.clone(),
+    config.dtype,
+    config.sd_version.clone(),
+    config.sd_config.clone()
+  ).expect("Model cache should create");
+
+  println!("Initializing backend runtime...");
+
   tauri::Builder::default()
+    .plugin(tauri_plugin_log::Builder::new()
+      .level(log::LevelFilter::Info)
+      .targets(vec![Target::new(TargetKind::Stdout)])
+      .build())
     .setup(|app| {
-      if cfg!(debug_assertions) {
-        app.handle().plugin(
-          tauri_plugin_log::Builder::default()
-            .level(log::LevelFilter::Info)
-            .build(),
-        )?;
-      }
+      //if cfg!(debug_assertions) {
+      //  app.handle().plugin(
+      //    tauri_plugin_log::Builder::default()
+      //      .level(log::LevelFilter::Info)
+      //      .build(),
+      //  )?;
+      //}
       Ok(())
     })
+    .manage(config)
+    .manage(prompt_cache)
+    .manage(model_cache)
     .invoke_handler(tauri::generate_handler![
-      my_custom_command, 
-      test_round_trip, 
+      test_counter, 
       infer_image,
     ])
     .run(tauri::generate_context!())
     .expect("error while running tauri application");
 }
-
-#[tauri::command]
-fn my_custom_command() {
-  println!("I was invoked from JavaScript!");
-}
-
-static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
-
-#[tauri::command]
-fn test_round_trip() -> u64 {
-  let value : u64;
-  {
-    match COUNTER.write() {
-      Ok(mut counter) => {
-        *counter += 1;
-        value = *counter;
-      },
-      Err(_e) => {
-        value = 0;
-      }
-    }
-  }
-
-  println!("I was invoked from JavaScript! {:?}", value);
-
-  value
-}
-

crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs

@@ -0,0 +1,17 @@
+use anyhow::Result;
+use candle_core::{DType, Device, Tensor};
+use image::DynamicImage;
+
+pub fn dynamic_image_to_tensor(image: &DynamicImage, device: &Device, datatype: DType) -> Result<Tensor> {
+  let (height, width) = (image.height() as usize, image.width() as usize);
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let image = image.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom);
+  let image = image.to_rgb8();
+  let image = image.into_raw();
+  Ok(Tensor::from_vec(image, (height, width, 3), device)?
+    .permute((2, 0, 1))?
+    .to_dtype(datatype)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?)
+}

crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs

@@ -0,0 +1,28 @@
+use candle_core::{DType, Device, Tensor};
+
+pub fn load_image_file_to_tensor<T: AsRef<std::path::Path>>(path: T, device: &Device) -> anyhow::Result<Tensor> {
+  let img = image::ImageReader::open(path)?.decode()?;
+  let (height, width) = (img.height() as usize, img.width() as usize);
+  println!("INPUT IMAGE WIDTH AND HEIGHT: {}x{}", width, height);
+
+  // TODO: Downscale from 1024 -- YES! THIS HACK DOES THE TRICK OF GETTING THE TENSOR SHAPE TO [1,4,64,64] !!
+  let height = height / 2;
+  let width = width  / 2;
+  
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let img = img.resize_to_fill(
+    width as u32,
+    height as u32,
+    image::imageops::FilterType::CatmullRom,
+  );
+  println!("RESIZED IMAGE WIDTH AND HEIGHT: {}x{}", width, height);
+  let img = img.to_rgb8();
+  let img = img.into_raw();
+  let img = Tensor::from_vec(img, (height, width, 3), device)?
+    .permute((2, 0, 1))?
+    .to_dtype(DType::F32)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?;
+  Ok(img)
+}

crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs

@@ -0,0 +1,18 @@
+use anyhow::Result;
+use candle_core::{DType, Device, Tensor};
+
+pub fn load_image_file_to_tensor_2<T: AsRef<std::path::Path>>(path: T) -> Result<Tensor> {
+  let img = image::ImageReader::open(path)?.decode()?;
+  let (height, width) = (img.height() as usize, img.width() as usize);
+  let height = height - height % 32;
+  let width = width - width % 32;
+  let img = img.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom);
+  let img = img.to_rgb8();
+  let img = img.into_raw();
+  let img = Tensor::from_vec(img, (height, width, 3), &Device::Cpu)?
+    .permute((2, 0, 1))?
+    .to_dtype(DType::F32)?
+    .affine(2. / 255., -1.)?
+    .unsqueeze(0)?;
+  Ok(img)
+}

crates/desktop/src-tauri/src/ml/image/mod.rs

@@ -0,0 +1,5 @@
+pub mod dynamic_image_to_tensor;
+pub mod load_image_file_to_tensor;
+pub mod load_image_file_to_tensor_2;
+pub mod save_image_from_tensor;
+pub mod tensor_to_image_buffer;

crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs

@@ -0,0 +1,28 @@
+use anyhow::{Error as E, Result};
+use candle_core::Tensor;
+use image::{ImageBuffer, Rgb};
+
+pub fn save_image_from_tensor<P: AsRef<std::path::Path>>(img: &Tensor, p: P) -> Result<()> {
+  println!("Saving image to {:?}", p.as_ref());
+  let p = p.as_ref();
+  let (channel, height, width) = img.dims3()?;
+  println!("Image dimensions: {}x{} with {} channels", width, height, channel);
+
+  if channel != 3 {
+    anyhow::bail!("save_image expects an input of shape (3, height, width)")
+  }
+  let img = img.permute((1, 2, 0))?.flatten_all()?;
+  let pixels = img.to_vec1::<u8>()?;
+  println!("Converting tensor to image buffer...");
+
+  let image: ImageBuffer<Rgb<u8>, Vec<u8>> =
+    match ImageBuffer::from_raw(width as u32, height as u32, pixels) {
+      Some(image) => image,
+      None => anyhow::bail!("error saving image {p:?}"),
+    };
+  println!("Successfully created image buffer");
+
+  image.save(p).map_err(E::from)?;
+  println!("Successfully saved image to disk");
+  Ok(())
+}

crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs

@@ -0,0 +1,18 @@
+use candle_core::Tensor;
+use image::{ImageBuffer, Rgb};
+
+pub type RgbImage = ImageBuffer<Rgb<u8>, Vec<u8>>;
+
+pub fn tensor_to_image_buffer(img: &Tensor) -> anyhow::Result<RgbImage> {
+  let (channel, height, width) = img.dims3()?;
+
+  if channel != 3 {
+    anyhow::bail!("save_image expects an input of shape (3, height, width)")
+  }
+
+  let img = img.permute((1, 2, 0))?.flatten_all()?;
+  let pixels = img.to_vec1::<u8>()?;
+
+  ImageBuffer::from_raw(width as u32, height as u32, pixels)
+    .ok_or_else(|| anyhow::anyhow!("error loading image from raw tensor"))
+}

crates/desktop/src-tauri/src/ml/mod.rs

@@ -0,0 +1,6 @@
+pub mod image;
+pub mod model_cache;
+pub mod model_file;
+pub mod models;
+pub mod prompt_cache;
+pub mod stable_diffusion;

crates/desktop/src-tauri/src/ml/model_cache.rs

@@ -0,0 +1,118 @@
+use crate::ml::models::unet_model::UNetModel;
+use crate::ml::model_file::StableDiffusionVersion;
+use anyhow::anyhow;
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use hf_hub::api::sync::Api;
+use std::sync::{Arc, RwLock};
+use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution;
+use crate::ml::models::lazy_load_vae_model::LazyLoadVaeModel;
+// TODO: This data structure is gross. Generalize the locking and loading semantics for reuse.
+
+// Simple registry for now. We can build complex machinery that aids in 
+// VRAM utilization, disk space saving, intelligent scheduling, etc. in the future.
+pub struct ModelCache {
+  // TODO(bt,2025-02-14): Support for more than one device in the future.
+  device: Device,
+  dtype: DType,
+  hf_api: Api,
+  sd_config: StableDiffusionConfig,
+  sd_version: StableDiffusionVersion,
+  
+  unet: Arc<RwLock<Option<UNetModel>>>,
+  lazy_vae: LazyLoadVaeModel,
+}
+
+impl ModelCache {
+  pub fn new(
+    device: Device, 
+    dtype: DType,
+    sd_version: StableDiffusionVersion, 
+    sd_config: StableDiffusionConfig,
+  ) -> anyhow::Result<Self> {
+    let api = Api::new()?;
+    
+    Ok(Self { 
+      device: device.clone(),
+      dtype,
+      hf_api: api.clone(),
+      unet: Arc::new(RwLock::new(None)),
+      lazy_vae: LazyLoadVaeModel::new(
+        &sd_config,
+        &sd_version,
+        &api,
+        &device,
+        dtype,
+      )?,
+      sd_config,
+      sd_version,
+    })
+  }
+
+  pub fn vae_encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
+    self.lazy_vae.encode(xs)
+  }
+
+  pub fn vae_decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
+    self.lazy_vae.decode(xs)
+  }
+  
+  pub fn unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
+    println!("unet_inference");
+    match self.try_unet_inference(latent_model_input, timestep, text_embeddings) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_unet()?;
+        self.try_unet_inference(latent_model_input, timestep, text_embeddings)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+  
+  fn load_unet(&self) -> anyhow::Result<()> {
+    println!("load_unet");
+    match self.unet.write() {
+      Err(err) => return Err(anyhow!("lock error: {:?}", err)),
+      Ok(mut maybe_model) => {
+        match &*maybe_model {
+          Some(_model) => {} // Fall through
+          None => {
+            let repo = self.sd_version.repo();
+            
+            println!("Downloading UNET model files from: {} ...", repo);
+
+            let unet_file = self.hf_api.model(repo.to_string())
+              .get("unet/diffusion_pytorch_model.safetensors")
+              .map_err(|err| anyhow!("error fetching model: {:?}", err))?;
+
+            println!("Building UNET model... (3)");
+            let unet = UNetModel::new(&self.sd_config, unet_file, &self.device, self.dtype)
+              .map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
+
+            println!("Built UNET...");
+            *maybe_model = Some(unet);
+          }
+        }
+      }
+    }
+    Ok(())
+  }
+
+  pub fn try_unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Option<Tensor>> {
+    println!("try_unet_inference");
+    match self.unet.read() {
+      Ok(mut model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.inference(latent_model_input, timestep, text_embeddings)?))
+          }
+        }
+      }
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+    }
+  }
+  
+}

crates/desktop/src-tauri/src/ml/model_file.rs

@@ -0,0 +1,83 @@
+#[cfg(feature = "accelerate")]
+extern crate accelerate_src;
+#[cfg(feature = "mkl")]
+extern crate intel_mkl_src;
+
+use candle_transformers::models::stable_diffusion;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ModelFile {
+  Tokenizer,
+  Tokenizer2,
+  Clip,
+  Clip2,
+  Unet,
+  Vae,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StableDiffusionVersion {
+  V1_5,
+  V1_5Inpaint,
+  V2_1,
+  V2Inpaint,
+  Xl,
+  XlInpaint,
+  Turbo,
+}
+
+impl StableDiffusionVersion {
+  pub fn repo(&self) -> &'static str {
+    match self {
+      Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
+      Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0",
+      Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting",
+      Self::V2_1 => "stabilityai/stable-diffusion-2-1",
+      Self::V1_5 => "runwayml/stable-diffusion-v1-5",
+      Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting",
+      Self::Turbo => "stabilityai/sdxl-turbo",
+    }
+  }
+}
+
+impl ModelFile {
+  pub fn get(
+    &self,
+    filename: Option<String>,
+    version: StableDiffusionVersion,
+    use_f16: bool,
+  ) -> anyhow::Result<std::path::PathBuf> {
+    use hf_hub::api::sync::Api;
+    match filename {
+      Some(filename) => Ok(std::path::PathBuf::from(filename)),
+      None => {
+        let (repo, path) = match self {
+          Self::Tokenizer => {
+            let tokenizer_repo = match version {
+              StableDiffusionVersion::V1_5
+              | StableDiffusionVersion::V2_1
+              | StableDiffusionVersion::V1_5Inpaint
+              | StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32",
+              StableDiffusionVersion::Xl
+              | StableDiffusionVersion::XlInpaint
+              | StableDiffusionVersion::Turbo => {
+                "openai/clip-vit-large-patch14"
+              }
+            };
+            (tokenizer_repo, "tokenizer.json")
+          }
+          Self::Tokenizer2 => {
+            ("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json")
+          }
+          Self::Clip => (version.repo(), "text_encoder/model.safetensors"),
+          Self::Clip2 => (version.repo(), "text_encoder_2/model.safetensors"),
+          Self::Unet => (version.repo(), "unet/diffusion_pytorch_model.safetensors"),
+          Self::Vae => (version.repo(), "vae/diffusion_pytorch_model.safetensors"),
+        };
+        let filename = Api::new()?.model(repo.to_string()).get(path)?;
+        Ok(filename)
+      }
+    }
+  }
+}
+

crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs

@@ -0,0 +1,130 @@
+use crate::ml::model_file::StableDiffusionVersion;
+use anyhow::anyhow;
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use hf_hub::api::sync::Api;
+use std::sync::{Arc, RwLock};
+
+pub struct LazyLoadVaeModel {
+  lazy_model: Arc<RwLock<Option<AutoEncoderKL>>>,
+  model_configs: VaeConfigs,
+}
+
+// TODO: Maybe we pass these via a central config read at startup, passed as an Arc<ConfigT>
+pub struct VaeConfigs {
+  sd_config: StableDiffusionConfig,
+  sd_version: StableDiffusionVersion,
+  device: Device,
+  dtype: DType,
+  hf_api: Api,
+}
+
+impl LazyLoadVaeModel {
+  pub fn new(
+    sd_config: &StableDiffusionConfig,
+    sd_version: &StableDiffusionVersion,
+    hf_api: &Api,
+    device: &Device,
+    dtype: DType,
+  ) -> anyhow::Result<Self> {
+    println!("building vae model... (1)");
+    
+    Ok(Self {
+      lazy_model: Arc::new(RwLock::new(None)),
+      model_configs: VaeConfigs {
+        sd_config: sd_config.clone(),
+        sd_version: sd_version.clone(),
+        device: device.clone(),
+        hf_api: hf_api.clone(),
+        dtype
+      }
+    })
+  }
+
+  pub fn encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
+    println!("vae encode...");
+    match self.try_encode(xs) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_model()?;
+        self.try_encode(xs)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+  
+  pub fn decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
+    println!("vae decode...");
+    match self.try_decode(xs) {
+      Err(err) => Err(err),
+      Ok(Some(result)) => Ok(result),
+      Ok(None) => {
+        // Model wasn't previously loaded
+        self.load_model()?;
+        self.try_decode(xs)?
+          .ok_or(anyhow!("model was not previously loaded"))
+      }
+    }
+  }
+
+  fn try_encode(&self, xs: &Tensor) -> anyhow::Result<Option<DiagonalGaussianDistribution>> {
+    match self.lazy_model.read() {
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+      Ok(model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.encode(xs)?))
+          }
+        }
+      }
+    }
+  }
+
+  fn try_decode(&self, xs: &Tensor) -> anyhow::Result<Option<Tensor>> {
+    match self.lazy_model.read() {
+      Err(err) => Err(anyhow!("lock error: {:?}", err)),
+      Ok(model) => {
+        match &*model {
+          None => Ok(None), // Model is not yet loaded
+          Some(model) => {
+            Ok(Some(model.decode(xs)?))
+          }
+        }
+      }
+    }
+  }
+
+  fn load_model(&self) -> anyhow::Result<()> {
+    println!("load vae");
+    match self.lazy_model.write() {
+      Err(err) => return Err(anyhow!("lock error: {:?}", err)),
+      Ok(mut maybe_model) => {
+        match &*maybe_model {
+          Some(_model) => {} // Already loaded; fall through
+          None => {
+            let repo = self.model_configs.sd_version.repo();
+
+            println!("Building VAE model from : {:?} ... (3)", repo);
+            
+            let vae_file = self.model_configs.hf_api.model(repo.to_string())
+              .get("vae/diffusion_pytorch_model.safetensors")?;
+
+            println!("Building VAE model from file {:?}...", &vae_file);
+            let vae = self.model_configs
+              .sd_config
+              .build_vae(vae_file, &self.model_configs.device, self.model_configs.dtype)?;
+
+            println!("Built VAE...");
+            *maybe_model = Some(vae);
+          }
+        }
+      }
+    }
+    Ok(())
+  }
+ 
+}

crates/desktop/src-tauri/src/ml/models/mod.rs

@@ -0,0 +1,2 @@
+pub mod unet_model;
+pub mod lazy_load_vae_model;

crates/desktop/src-tauri/src/ml/models/unet_model.rs

@@ -0,0 +1,35 @@
+use candle_core::{DType, Device, Tensor};
+use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use std::path::Path;
+
+pub struct UNetModel {
+  model: UNet2DConditionModel,
+}
+
+impl UNetModel {
+  pub fn new<P: AsRef<Path>>(
+    sd_config: &StableDiffusionConfig,
+    unet_weights_file: P,
+    device: &Device,
+    dtype: DType,
+  ) -> anyhow::Result<Self> {
+    println!("building unet model... (1)");
+    
+    let model = sd_config.build_unet(
+      unet_weights_file, 
+      &device, 
+      4, 
+      false, 
+      dtype
+    )?;
+    
+    Ok(Self {
+      model,
+    })
+  }
+  
+  pub fn inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
+    Ok(self.model.forward(&latent_model_input, timestep, &text_embeddings)?)
+  }
+}

crates/desktop/src-tauri/src/ml/prompt_cache.rs

@@ -0,0 +1,22 @@
+use candle_core::Tensor;
+use memory_caching::arc_sieve::ArcSieve;
+
+pub struct PromptCache {
+  cache: ArcSieve<String, Tensor>,
+}
+
+impl PromptCache {
+  pub fn with_capacity(capacity: usize) -> anyhow::Result<Self> {
+    Ok(Self {
+      cache: ArcSieve::with_capacity(capacity)?,
+    })
+  }
+  
+  pub fn get_copy(&self, key: &String) -> anyhow::Result<Option<Tensor>> {
+    self.cache.get_copy(key)
+  }
+  
+  pub fn store_copy(&self, key: &String, value: &Tensor) -> anyhow::Result<()> {
+    self.cache.store_copy(key, value)
+  }
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs

@@ -0,0 +1,13 @@
+use crate::ml::model_file::StableDiffusionVersion;
+
+pub fn get_vae_scale(sd_version: StableDiffusionVersion) -> f64 {
+  match sd_version {
+    StableDiffusionVersion::V1_5
+    | StableDiffusionVersion::V1_5Inpaint
+    | StableDiffusionVersion::V2_1
+    | StableDiffusionVersion::V2Inpaint
+    | StableDiffusionVersion::XlInpaint
+    | StableDiffusionVersion::Xl => 0.18215,
+    StableDiffusionVersion::Turbo => 0.13025,
+  }
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/infer_clip_text_embeddings.rs

@@ -0,0 +1,153 @@
+#[cfg(feature = "accelerate")]
+extern crate accelerate_src;
+#[cfg(feature = "mkl")]
+extern crate intel_mkl_src;
+
+use candle_transformers::models::stable_diffusion;
+
+use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
+use anyhow::Error as E;
+use candle_core::{DType, Device, Module, Tensor, D};
+use log::info;
+use tokenizers::Tokenizer;
+
+#[allow(clippy::too_many_arguments)]
+pub fn infer_clip_text_embeddings(
+  prompt: &str,
+  uncond_prompt: &str,
+  tokenizer: Option<String>,
+  clip_weights: Option<String>,
+  clip2_weights: Option<String>,
+  sd_version: StableDiffusionVersion,
+  sd_config: &stable_diffusion::StableDiffusionConfig,
+  use_f16: bool,
+  device: &Device,
+  dtype: DType,
+  use_guide_scale: bool,
+) -> anyhow::Result<Tensor> {
+  info!("Preparing text embeddings...");
+
+  let which = match sd_version {
+    StableDiffusionVersion::Xl
+    | StableDiffusionVersion::XlInpaint
+    | StableDiffusionVersion::Turbo => vec![true, false],
+    _ => vec![true],
+  };
+
+  let text_embeddings = which
+    .iter()
+    .map(|first| {
+      do_infer_clip_text_embeddings(
+        &prompt,
+        &uncond_prompt,
+        None, // tokenizer
+        None, // clip_weights
+        None, // clip2_weights
+        sd_version,
+        &sd_config,
+        false, // use_f16
+        &device,
+        dtype,
+        use_guide_scale,
+        *first,
+      )
+    })
+    .collect::<anyhow::Result<Vec<_>>>()?;
+
+  let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?;
+  
+  Ok(text_embeddings)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn do_infer_clip_text_embeddings(
+    prompt: &str,
+    uncond_prompt: &str,
+    tokenizer: Option<String>,
+    clip_weights: Option<String>,
+    clip2_weights: Option<String>,
+    sd_version: StableDiffusionVersion,
+    sd_config: &stable_diffusion::StableDiffusionConfig,
+    use_f16: bool,
+    device: &Device,
+    dtype: DType,
+    use_guide_scale: bool,
+    first: bool,
+  ) -> anyhow::Result<Tensor> {
+  
+  let tokenizer_file = if first {
+    ModelFile::Tokenizer
+  } else {
+    ModelFile::Tokenizer2
+  };
+  let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
+  let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?;
+  let pad_id = match &sd_config.clip.pad_with {
+    Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(),
+    None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(),
+  };
+
+  println!("Running with prompt \"{prompt}\".");
+  let mut tokens = tokenizer
+    .encode(prompt, true)
+    .map_err(E::msg)?
+    .get_ids()
+    .to_vec();
+  if tokens.len() > sd_config.clip.max_position_embeddings {
+    anyhow::bail!(
+            "the prompt is too long, {} > max-tokens ({})",
+            tokens.len(),
+            sd_config.clip.max_position_embeddings
+        )
+  }
+  while tokens.len() < sd_config.clip.max_position_embeddings {
+    tokens.push(pad_id)
+  }
+  let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
+
+  println!("Building the Clip transformer.");
+  let clip_weights_file = if first {
+    ModelFile::Clip
+  } else {
+    ModelFile::Clip2
+  };
+  let clip_weights = if first {
+    clip_weights_file.get(clip_weights, sd_version, use_f16)?
+  } else {
+    clip_weights_file.get(clip2_weights, sd_version, use_f16)?
+  };
+  let clip_config = if first {
+    &sd_config.clip
+  } else {
+    sd_config.clip2.as_ref().unwrap()
+  };
+  let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
+
+  let text_embeddings = text_model.forward(&tokens)?;
+
+  let text_embeddings = if use_guide_scale {
+    let mut uncond_tokens = tokenizer
+      .encode(uncond_prompt, true)
+      .map_err(E::msg)?
+      .get_ids()
+      .to_vec();
+    if uncond_tokens.len() > sd_config.clip.max_position_embeddings {
+      anyhow::bail!(
+                "the negative prompt is too long, {} > max-tokens ({})",
+                uncond_tokens.len(),
+                sd_config.clip.max_position_embeddings
+            )
+    }
+    while uncond_tokens.len() < sd_config.clip.max_position_embeddings {
+      uncond_tokens.push(pad_id)
+    }
+
+    let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?;
+    let uncond_embeddings = text_model.forward(&uncond_tokens)?;
+
+    Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)?
+  } else {
+    text_embeddings.to_dtype(dtype)?
+  };
+  Ok(text_embeddings)
+}

crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs

@@ -0,0 +1,3 @@
+pub mod get_vae_scale;
+pub mod infer_clip_text_embeddings;
+pub mod stable_diffusion_pipeline;

crates/desktop/src-tauri/src/ml/stable_diffusion/stable_diffusion_pipeline.rs

@@ -0,0 +1,199 @@
+use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+
+use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
+use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
+use crate::ml::model_cache::ModelCache;
+use crate::ml::model_file::StableDiffusionVersion;
+use crate::ml::prompt_cache::PromptCache;
+use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
+use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
+use crate::state::app_config::AppConfig;
+use anyhow::{anyhow, Error as E, Result};
+use candle_core::{DType, IndexOp, Tensor, D};
+use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution;
+use image::DynamicImage;
+use log::info;
+use rand::Rng;
+
+pub struct Args<'a> {
+    pub image: &'a DynamicImage,
+    pub prompt: String,
+    pub uncond_prompt: String,
+    pub cfg_scale: Option<f64>,
+    pub i2i_strength: Option<u8>,
+    pub configs: &'a AppConfig,
+    pub model_cache: &'a ModelCache,
+    pub prompt_cache: &'a PromptCache,
+}
+
+pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
+    println!("Starting image generation with the following configuration:");
+    println!("  Model: {:?}", args.configs.sd_version);
+    println!("  Prompt: {}", args.prompt);
+    println!("  Steps: {}", args.configs.scheduler_steps);
+    println!("  Device: {:?}", args.configs.device);
+
+    println!("Model dimensions: {}x{}", args.configs.sd_config.width, args.configs.sd_config.height);
+
+    // TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
+    //  This is a target for performance improvement
+    //  See: https://github.com/huggingface/candle/issues/1331
+    let mut scheduler = args.configs.sd_config.build_scheduler(
+        args.configs.scheduler_steps)?;
+
+    let seed = args.configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
+    args.configs.device.set_seed(seed)?;
+
+    println!("Using seed: {}", seed);
+
+    let guidance_scale = match args.cfg_scale {
+        Some(guidance_scale) => guidance_scale,
+        None => match args.configs.sd_version {
+            StableDiffusionVersion::V1_5
+            | StableDiffusionVersion::V2_1
+            | StableDiffusionVersion::Xl => 7.5,
+            StableDiffusionVersion::Turbo => 0.,
+            _ => 0., // NB: Not sure what the other model families should use, so sticking with "0"
+        },
+    };
+
+    let use_guide_scale = guidance_scale > 1.0;
+
+    println!("Using guide scale: {}", use_guide_scale);
+
+    info!("Checking if prompt is cached");
+    let maybe_cached = args.prompt_cache.get_copy(&args.prompt)?;
+
+    let mut text_embeddings = if let Some(tensor) = maybe_cached {
+        tensor
+    } else {
+        info!("Prompt is NOT cached! Calculating embedding...");
+        let tensor = infer_clip_text_embeddings(
+            &args.prompt,
+            &args.uncond_prompt,
+            None, // tokenizer
+            None, // clip_weights
+            None, // clip2_weights
+            args.configs.sd_version,
+            &args.configs.sd_config,
+            false, // use_f16
+            &args.configs.device,
+            args.configs.dtype,
+            use_guide_scale,
+        )?;
+        args.prompt_cache.store_copy(&args.prompt, &tensor)?;
+        tensor
+    };
+
+    println!("Text embeddings shape: {:?}", text_embeddings.shape());
+
+    println!("Loading input image into tensor...");
+
+    let input_image = dynamic_image_to_tensor(
+        args.image,
+        &args.configs.device,
+        args.configs.dtype)?;
+
+    println!("Reference image shape: {:?}", input_image.shape());
+
+    let vae_scale = get_vae_scale(args.configs.sd_version);
+
+    let init_latent_dist : DiagonalGaussianDistribution = args.model_cache.vae_encode(&input_image)?;
+
+    // TODO(bt,2025-02-18): This takes a little bit to generate the sample.
+    //  This is a target for performance improvement
+    println!("Generating latents from input image...");
+    let latents = (init_latent_dist.sample()? * vae_scale)?;
+    
+    //.to_device(&args.configs.device)?;
+    
+    println!("Initial latents shape: {:?}", latents.shape());
+
+    println!("Calculating start step for diffusion process...");
+
+    let img2img_strength = match args.i2i_strength {
+        None => 0.75f64,
+        Some(strength) => (strength as f64) / 100.0f64,
+    };
+
+    let t_start = {
+        let start = args.configs.scheduler_steps - (args.configs.scheduler_steps as f64 * img2img_strength) as usize;
+
+        println!("Starting from step {} of {} (strength: {})", start, args.configs.scheduler_steps, img2img_strength);
+        start
+    };
+
+    let latents = if t_start < scheduler.timesteps().len() {
+        println!("Adding noise to latents...");
+        let noise = latents.randn_like(0f64, 1f64)?;
+        scheduler.add_noise(&latents, noise, scheduler.timesteps()[t_start])?
+    } else {
+        println!("Using latents directly (no noise addition needed)");
+        latents
+    };
+
+    println!("Latents initialized successfully");
+
+    let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();
+
+    // Initial noise shape: [1, 4, 64, 64]
+    println!("Initial noise shape: {:?}", latents.shape());
+
+    println!("Starting diffusion process...");
+    
+    let mut latents = latents;
+
+    for (timestep_index, &timestep) in timesteps.iter().enumerate() {
+        if timestep_index < t_start {
+            continue;
+        }
+
+        println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
+
+        let latent_model_input = if use_guide_scale {
+            Tensor::cat(&[&latents, &latents], 0)?
+        } else {
+            latents.clone() // TODO(bt,2025-08-18): Do we have to clone the tensor? `scale_model_input` requires ownership
+        };
+
+        let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;
+
+        let mut noise_pred = match args.model_cache.unet_inference(&latent_model_input, timestep as f64, &text_embeddings) {
+            Ok(pred) => pred,
+            Err(e) => {
+                println!("UNet inference failed with error: {}", e);
+                return Err(anyhow::anyhow!("UNet inference failed: {}", e));
+            },
+        };
+
+        if use_guide_scale {
+            let chunks = noise_pred.chunk(2, 0)?;
+            let (noise_pred_uncond, noise_pred_text) = (&chunks[0], &chunks[1]);
+            noise_pred = (noise_pred_uncond + (guidance_scale * (noise_pred_text - noise_pred_uncond)?)?)?;
+        }
+
+        latents = scheduler.step(&noise_pred, timestep, &latents)?;
+    }
+
+    println!("Diffusion process completed, decoding image...");
+    let image = args.model_cache.vae_decode(&(latents / vae_scale)?)?;
+    
+    println!("VAE decode completed");
+
+    println!("Scaling image back");
+    let image = ((image / 2.)? + 0.5)?;
+    
+    // TODO(bt,2025-08-18): This normalization is slow.
+    //  This is a target for performance improvement
+    println!("Normalized image values");
+    let image = (image.clamp(0f32, 1.)? * 255.)?;
+
+    println!("Convert to int8");
+    let image = image.to_dtype(DType::U8)?;
+    
+    println!("Converted to 8-bit format");
+
+    let image = tensor_to_image_buffer(&image.i(0)?)?;
+
+    Ok(image)
+}

crates/desktop/src-tauri/src/state/app_config.rs

@@ -0,0 +1,96 @@
+use crate::ml::model_file::StableDiffusionVersion;
+use crate::state::yaml_config::YamlConfig;
+use candle_core::{DType, Device};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use filesys::recursively_find_file_by_name::recursively_find_file_by_name;
+use hf_hub::api::sync::Api;
+use log::info;
+
+const CONFIG_FILENAME : &str = "app_config.yaml";
+
+const DEFAULT_SD_IMAGE_WIDTH: usize = 512;
+const DEFAULT_SD_IMAGE_HEIGHT: usize = 512;
+
+const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
+const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::Turbo;
+
+/// A central place to configure app-wide details.
+pub struct AppConfig {
+  /// We only support one device type currently, and we can't multiplex 
+  /// over several devices yet.
+  pub device: Device,
+  
+  /// The core numeric type to use for models.
+  pub dtype: DType,
+  
+  pub sd_version: StableDiffusionVersion,
+  pub sd_config: StableDiffusionConfig,
+
+  pub image_height: usize,
+  pub image_width: usize,
+  pub scheduler_steps: usize,
+  pub scheduler_samples: usize,
+  
+  pub seed: Option<u64>,
+
+  pub cfg_scale: Option<f64>,
+  
+  /// Probably shouldn't be here.
+  pub hf_api: Api,
+}
+
+impl AppConfig {
+  
+  pub fn init() -> anyhow::Result<Self> {
+    println!("Configuration filename: {}", CONFIG_FILENAME);
+    let yaml_configs = load_yaml_configs();
+    
+    println!("Configs: {:?}", yaml_configs);
+    
+    let device = Device::new_cuda(0)
+      .unwrap_or_else(|e| {
+          info!("CUDA not available ({}), falling back to CPU", e);
+          Device::Cpu
+        });
+    
+    let sd_config = match SD_VERSION {
+      StableDiffusionVersion::Turbo => {
+        StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+      _ => {
+        StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+    };
+    
+    let hf_api = Api::new()?;
+    
+    Ok(Self {
+      device,
+      dtype: DEFAULT_SD_NUMERIC_DATATYPE,
+      sd_version: SD_VERSION,
+      sd_config,
+      image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
+      image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
+      scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(1),
+      scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
+      seed: yaml_configs.seed,
+      cfg_scale: yaml_configs.cfg_scale,
+      hf_api,
+    })
+  }
+}
+
+fn load_yaml_configs() -> YamlConfig {
+  let maybe_path = recursively_find_file_by_name(CONFIG_FILENAME, "../../../", 5)
+    .ok()
+    .flatten();
+  if let Some(path) = maybe_path {
+    let path = path.canonicalize().unwrap_or(path);
+    println!("Attempting to load configs from: {:?}", &path);
+    if let Ok(config) = YamlConfig::read_from_file(path) {
+      return config;
+    }
+  }
+  println!("Loading default configs.");
+  YamlConfig::default()
+}

crates/desktop/src-tauri/src/state/mod.rs

@@ -0,0 +1,2 @@
+pub mod yaml_config;
+pub mod app_config;

crates/desktop/src-tauri/src/state/yaml_config.rs

@@ -0,0 +1,25 @@
+use std::fs::File;
+use std::io::BufReader;
+use std::path::Path;
+
+/// Configuration read or derived from a YAML config file at startup
+/// This is not meant to contain any runtime config values.
+/// This may not be exhaustive.
+#[derive(serde::Deserialize, serde::Serialize, Clone, Debug, Default)]
+pub struct YamlConfig {
+  pub image_height: Option<usize>,
+  pub image_width: Option<usize>,
+  pub scheduler_steps: Option<usize>,
+  pub scheduler_samples: Option<usize>,
+  pub seed: Option<u64>,
+  pub cfg_scale: Option<f64>,
+}
+
+impl YamlConfig {
+  pub fn read_from_file<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let config = serde_yaml::from_reader(reader)?;
+    Ok(config)
+  }
+}

crates/desktop/src-tauri/tauri.conf.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://schema.tauri.app/config/2.0.0-rc",
-  "productName": "Storyteller Desktop App",
+  "productName": "Realtime Desktop App",
   "version": "0.1.0",
   "identifier": "com.unique.value.thing.why",
   "build": {
@@ -12,9 +12,9 @@
   "app": {
     "windows": [
       {
-        "title": "Storyteller Desktop App Title Goes Here",
-        "width": 800,
-        "height": 600,
+        "title": "Realtime Desktop App Title Goes Here",
+        "width": 1200,
+        "height": 800,
         "resizable": true,
         "fullscreen": false
       }

crates/lib/files/filesys/src/lib.rs

@@ -31,4 +31,5 @@ pub mod file_size;
 pub mod filename_concat;
 pub mod is_filesystem_full_error;
 pub mod path_to_string;
+pub mod recursively_find_file_by_name;
 pub mod rename_across_devices;

crates/lib/files/filesys/src/recursively_find_file_by_name.rs

@@ -0,0 +1,56 @@
+use std::path::{Path, PathBuf};
+use walkdir::WalkDir;
+use errors::AnyhowResult;
+
+pub fn recursively_find_file_by_name<P: AsRef<Path>, Q: AsRef<Path>>(
+  search_filename: P,
+  search_root: Q,
+  max_depth: usize,
+) -> AnyhowResult<Option<PathBuf>> {
+
+  let walker = WalkDir::new(search_root)
+    .max_depth(max_depth);
+
+  for entry in walker {
+    let entry = entry?;
+    if entry.path().is_dir() {
+      let check_path = entry.path().join(search_filename.as_ref());
+      if check_path.exists() && check_path.is_file() {
+        return Ok(Some(check_path));
+      }
+    }
+  }
+
+  Ok(None)
+}
+
+#[cfg(test)]
+mod tests {
+  use testing::test_file_path::test_file_path;
+  use crate::recursively_find_file_by_name::recursively_find_file_by_name;
+
+  #[test]
+  fn test_path_exists() {
+    let test_data_path = test_file_path("test_data/").expect("should work");
+    let result = recursively_find_file_by_name(
+      "super_mario_rpg_beware_the_forests_mushrooms.mp3",
+      test_data_path,
+      5
+    );
+    let result = result.expect("should be ok");
+    let result = result.expect("should exist");
+    assert!(result.ends_with("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3"));
+  }
+
+  #[test]
+  fn test_path_does_not_exist() {
+    let test_data_path = test_file_path("test_data/").expect("should work");
+    let result = recursively_find_file_by_name(
+      "fake_name.foo",
+      test_data_path,
+      5
+    );
+    let result = result.expect("should be ok");
+    assert!(result.is_none());
+  }
+}

frontend/realtime/.gitignore

@@ -12,7 +12,7 @@ dist
 dist-ssr
 *.local
 backup
-
+.venv
 # Editor directories and files
 ~
 .vscode

frontend/realtime/paint_py_server/server.py

@@ -0,0 +1,527 @@
+import asyncio
+import base64
+import json
+import logging
+import pathlib
+import time
+from io import BytesIO
+from typing import Dict, Any, Optional, Tuple, Callable, List
+
+import torch
+import websockets
+from PIL import Image
+from diffusers import (
+    StableDiffusionXLPipeline,
+    StableDiffusionXLImg2ImgPipeline,
+    AutoPipelineForImage2Image,
+    LCMScheduler,
+    UNet2DConditionModel,
+)
+from tqdm import tqdm
+from huggingface_hub import HfApi, hf_hub_download
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Global variables
+connected = False
+pipe = None
+model_initialized = False
+current_settings = {}
+
+# Custom TQDMProgressCallback for tracking download progress
+class ProgressCallback:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.total = 0
+        self.current = 0
+        self.last_update_time = 0
+        self.last_progress = 0
+        self.stage = "downloading"
+        self.current_file = ""
+    
+    async def __call__(self, current, total, file_name=""):
+        self.total = total
+        self.current = current
+        self.current_file = file_name or self.current_file
+        
+        # Only send updates at most once every 500ms to avoid flooding the connection
+        current_time = time.time()
+        progress = int(100 * current / total) if total > 0 else 0
+        
+        if (current_time - self.last_update_time > 0.5 and progress != self.last_progress) or current >= total:
+            self.last_update_time = current_time
+            self.last_progress = progress
+            
+            await self.websocket.send(json.dumps({
+                'type': 'progress_update',
+                'stage': self.stage,
+                'progress': progress,
+                'file': self.current_file
+            }))
+
+# Pure functions
+def decode_base64_to_image(base64_string: str) -> Image.Image:
+    """Decode a base64 string to a PIL Image."""
+    try:
+        image_data = base64.b64decode(base64_string)
+        return Image.open(BytesIO(image_data))
+    except Exception as e:
+        logger.error(f"Error decoding base64 image: {e}")
+        raise ValueError(f"Invalid base64 image: {e}")
+
+def encode_image_to_base64(image: Image.Image) -> str:
+    """Encode a PIL Image to a base64 string."""
+    try:
+        buffered = BytesIO()
+        image.save(buffered, format="PNG")
+        return base64.b64encode(buffered.getvalue()).decode('utf-8')
+    except Exception as e:
+        logger.error(f"Error encoding image to base64: {e}")
+        raise ValueError(f"Error encoding image: {e}")
+
+async def load_model(sdxl_checkpoint_path: pathlib.Path, progress_callback: Optional[ProgressCallback] = None) -> Tuple[AutoPipelineForImage2Image, bool]:
+    """Load the SDXL model with LCM scheduler and report progress."""
+    try:
+        logger.info(f"Loading model from {sdxl_checkpoint_path}")
+        
+        if progress_callback:
+            progress_callback.stage = "loading_unet"
+            progress_callback.current_file = "latent-consistency/lcm-sdxl"
+            await progress_callback(0, 100)
+        
+        # Load the UNet with LCM
+        unet = UNet2DConditionModel.from_pretrained(
+            "latent-consistency/lcm-sdxl",
+            torch_dtype=torch.float16,
+            variant="fp16",
+            use_safetensors=True,
+        )
+        
+        if progress_callback:
+            progress_callback.stage = "loading_base_model"
+            progress_callback.current_file = str(sdxl_checkpoint_path)
+            await progress_callback(25, 100)
+        
+        # Load the base model using the checkpoint path
+        pipe = AutoPipelineForImage2Image.from_pretrained(
+            "stabilityai/stable-diffusion-xl-base-1.0",
+            unet=unet,
+            torch_dtype=torch.float16,
+            variant="fp16",
+           use_safetensors=True
+        ).to("cuda")
+        
+        if progress_callback:
+            progress_callback.stage = "configuring_scheduler"
+            await progress_callback(75, 100)
+        
+        # Replace the scheduler with LCM scheduler
+        pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+        
+        if progress_callback:
+            progress_callback.stage = "loading_lcm_adapter"
+            progress_callback.current_file = "latent-consistency/lcm-lora-sdxl"
+            await progress_callback(85, 100)
+        
+        # Load the default LCM adapter
+        pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl", adapter_name="lcm",from_safetensors=True)
+        
+        if progress_callback:
+            await progress_callback(100, 100)
+        
+        return pipe, True
+    except Exception as e:
+        logger.error(f"Error loading model: {e}")
+        if progress_callback:
+            progress_callback.stage = "error"
+            progress_callback.current_file = str(e)
+            await progress_callback(0, 100)
+        return None, False
+
+async def load_lora(pipe: AutoPipelineForImage2Image, lora_path: pathlib.Path, adapter_name: str = "lora", progress_callback: Optional[ProgressCallback] = None) -> bool:
+    """Load a LoRA adapter into the pipeline and report progress."""
+    try:
+        logger.info(f"Loading LoRA from {lora_path} as {adapter_name}")
+        
+        if progress_callback:
+            progress_callback.stage = "loading_lora"
+            progress_callback.current_file = str(lora_path)
+            await progress_callback(0, 100)
+        
+        # Load the LoRA weights
+        pipe.load_lora_weights(lora_path, adapter_name=adapter_name)
+        
+        if progress_callback:
+            await progress_callback(100, 100)
+            
+        return True
+    except Exception as e:
+        logger.error(f"Error loading LoRA: {e}")
+        if progress_callback:
+            progress_callback.stage = "error"
+            progress_callback.current_file = str(e)
+            await progress_callback(0, 100)
+        return False
+
+def generate_image(
+    pipe: AutoPipelineForImage2Image,
+    image: Image.Image,
+    prompt: str,
+    lora_strength: float = 1.0,
+    image_to_image_strength: float = 1.0,
+    width: int = 1024,
+    height: int = 1024,
+    num_inference_steps: int = 4,
+    guidance_scale: float = 1.0,
+    seed: Optional[int] = None
+) -> Optional[Image.Image]:
+    """Generate an image using the LCM model."""
+    try:
+        # Resize input image to match desired dimensions
+        image = image.resize((width, height))
+        
+        # Set up generator for reproducibility if seed is provided
+        generator = torch.manual_seed(seed) if seed is not None else None
+        
+        # Set lora adapter weights if available
+        if hasattr(pipe, 'set_adapters') and len(pipe.adapters) > 0:
+            # If we have both lcm and lora adapters
+            if "lcm" in pipe.adapters and "lora" in pipe.adapters:
+                pipe.set_adapters(["lcm", "lora"], adapter_weights=[1.0, lora_strength])
+            # If we only have lcm adapter
+            elif "lcm" in pipe.adapters:
+                pipe.set_adapters(["lcm"], adapter_weights=[1.0])
+        
+        # Generate the image
+        result = pipe(
+            prompt=prompt,
+            image=image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=image_to_image_strength,
+            width=width,
+            height=height,
+            generator=generator
+        )
+        
+        return result.images[0]
+    except Exception as e:
+        logger.error(f"Error generating image: {e}")
+        return None
+
+async def handle_client(websocket):
+    """Handle a client connection."""
+    global connected, pipe, model_initialized, current_settings
+    
+    if connected:
+        await websocket.send(json.dumps({
+            'type': 'connection_error',
+            'error': "Another client is already connected"
+        }))
+        await websocket.close(1008, "Another client is already connected")
+        return
+    
+    connected = True
+    logger.info("Client connected")
+    
+    # Create a progress callback for this connection
+    progress_callback = ProgressCallback(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                data = json.loads(message)
+                logger.info(f"Received message type: {data.get('type', 'unknown')}")
+                logger.info(f"Full payload: {json.dumps(data, indent=2)}")
+                
+                # Check message type
+                message_type = data.get('type', '')
+                
+                # Handle setup request - initializes the model and settings
+                if message_type == 'setup':
+                    logger.info("Processing setup message")
+                    model_data = data.get('model', {})
+                    lora_data = data.get('lora')
+                    device = data.get('device', 'cuda')
+                    
+                    logger.info(f"Model data: {json.dumps(model_data, indent=2)}")
+                    logger.info(f"LoRA data: {json.dumps(lora_data, indent=2) if lora_data else 'None'}")
+                    logger.info(f"Device: {device}")
+                    
+                    # Extract setup parameters with detailed logging
+                    model_path = model_data.get('path', '')
+                    logger.info(f"Raw model path from message: '{model_path}'")
+                    
+                    # Check if the path is valid
+                    if not model_path:
+                        logger.error("Model path is empty")
+                        await websocket.send(json.dumps({
+                            'type': 'setup_response',
+                            'success': False,
+                            'error': 'Model path is required'
+                        }))
+                        continue
+                    
+                    # Convert to Path object
+                    sdxl_checkpoint_path = pathlib.Path(model_path)
+                    logger.info(f"Converted to pathlib.Path: '{sdxl_checkpoint_path}'")
+                    logger.info(f"Path is absolute: {sdxl_checkpoint_path.is_absolute()}")
+                    logger.info(f"Path exists check: {sdxl_checkpoint_path.exists()}")
+                    
+                    # If the path exists as a file
+                    if sdxl_checkpoint_path.is_file():
+                        logger.info(f"Path is a file: Yes")
+                        logger.info(f"File size: {sdxl_checkpoint_path.stat().st_size / (1024*1024):.2f} MB")
+                    else:
+                        logger.info(f"Path is a file: No")
+                    
+                    # Store current settings
+                    current_settings = {
+                        'sdxl_checkpoint_path': str(sdxl_checkpoint_path),
+                        'lora_path': str(lora_data['path']) if lora_data else None,
+                        'lora_alpha': float(lora_data['alpha']) if lora_data else 1.0,
+                        'device': device
+                    }
+                    logger.info(f"Current settings: {json.dumps(current_settings, indent=2)}")
+                    
+                    # Validate required parameters
+                    if not sdxl_checkpoint_path.exists():
+                        logger.error(f"Path does not exist: {sdxl_checkpoint_path}")
+                        # Try alternative path format
+                        alt_path = pathlib.Path(model_path.replace('\\', '/'))
+                        logger.info(f"Trying alternative path format: {alt_path}")
+                        logger.info(f"Alternative path exists check: {alt_path.exists()}")
+                        
+                        if alt_path.exists():
+                            logger.info("Using alternative path format")
+                            sdxl_checkpoint_path = alt_path
+                        else:
+                            await websocket.send(json.dumps({
+                                'type': 'setup_response',
+                                'success': False,
+                                'error': f'SDXL checkpoint path does not exist: {sdxl_checkpoint_path}'
+                            }))
+                            continue
+                    
+                    # Load the model with progress reporting
+                    logger.info(f"Loading model from: {sdxl_checkpoint_path}")
+                    pipe, success = await load_model(sdxl_checkpoint_path, progress_callback)
+                    
+                    if not success:
+                        await websocket.send(json.dumps({
+                            'type': 'setup_response',
+                            'success': False,
+                            'error': 'Failed to load model'
+                        }))
+                        continue
+                    
+                    # Load LoRA if specified
+                    lora_success = True
+                    if lora_data and lora_data['path'] and pathlib.Path(lora_data['path']).exists():
+                        lora_success = await load_lora(pipe, pathlib.Path(lora_data['path']), progress_callback=progress_callback)
+                        if not lora_success:
+                            await websocket.send(json.dumps({
+                                'type': 'setup_response',
+                                'success': False,
+                                'error': f'Failed to load LoRA from {pathlib.Path(lora_data["path"])}'
+                            }))
+                            continue
+                    
+                    model_initialized = True
+                    await websocket.send(json.dumps({
+                        'type': 'setup_response',
+                        'success': True,
+                        'message': 'Model initialized successfully',
+                        'settings': current_settings
+                    }))
+                
+                # Handle settings update - changes settings without full reinitialization
+                elif message_type == 'update_settings':
+                    if not model_initialized:
+                        await websocket.send(json.dumps({
+                            'type': 'update_settings_response',
+                            'success': False,
+                            'error': 'Model not initialized. Send a setup request first.'
+                        }))
+                        continue
+                    
+                    settings_data = data.get('settings', {})
+                    updates_applied = []
+                    errors = []
+
+                    # Check for LoRA path update
+                    if 'lora_path' in settings_data and settings_data['lora_path'] != current_settings.get('lora_path'):
+                        new_lora_path = pathlib.Path(settings_data['lora_path']) if settings_data['lora_path'] else None
+                        
+                        if new_lora_path and new_lora_path.exists():
+                            # Load the new LoRA with progress reporting
+                            lora_success = await load_lora(pipe, new_lora_path, progress_callback=progress_callback)
+                            if lora_success:
+                                current_settings['lora_path'] = str(new_lora_path)
+                                updates_applied.append('lora_path')
+                            else:
+                                errors.append(f'Failed to load LoRA from {new_lora_path}')
+                        elif new_lora_path:
+                            errors.append(f'LoRA path does not exist: {new_lora_path}')
+                        else:
+                            # If LoRA path is None, we'll just update the setting
+                            current_settings['lora_path'] = None
+                            updates_applied.append('lora_path')
+                    
+                    # Handle any other settings that can be updated without reloading the model
+                    # For example, default generation parameters
+                    for param in ['default_lora_strength', 'default_image_to_image_strength',
+                                 'default_width', 'default_height', 'default_steps',
+                                 'default_guidance_scale']:
+                        if param in settings_data:
+                            current_settings[param] = settings_data[param]
+                            updates_applied.append(param)
+                    
+                    # Send response
+                    if errors:
+                        await websocket.send(json.dumps({
+                            'type': 'update_settings_response',
+                            'success': False,
+                            'error': ', '.join(errors),
+                            'updates_applied': updates_applied,
+                            'current_settings': current_settings
+                        }))
+                    else:
+                        await websocket.send(json.dumps({
+                            'type': 'update_settings_response',
+                            'success': True,
+                            'message': 'Settings updated successfully',
+                            'updates_applied': updates_applied,
+                            'current_settings': current_settings
+                        }))
+                
+                # Handle generation request
+                elif message_type == 'generate':
+                    if not model_initialized:
+                        await websocket.send(json.dumps({
+                            'type': 'generate_response',
+                            'success': False,
+                            'error': 'Model not initialized. Send a setup request first.'
+                        }))
+                        continue
+                    
+                    generate_data = data.get('generate', {})
+                    
+                    # Extract generation parameters
+                    base64_image = generate_data.get('image')
+                    prompt = generate_data.get('prompt', '')
+                    
+                    # Use default settings if not provided
+                    lora_strength = float(generate_data.get('lora_strength', current_settings.get('default_lora_strength', 1.0)))
+                    image_to_image_strength = float(generate_data.get('image_to_image_strength', current_settings.get('default_image_to_image_strength', 1.0)))
+                    width = int(generate_data.get('generated_image_width', current_settings.get('default_width', 1024)))
+                    height = int(generate_data.get('generated_image_height', current_settings.get('default_height', 1024)))
+                    num_inference_steps = int(generate_data.get('num_inference_steps', current_settings.get('default_steps', 4)))
+                    guidance_scale = float(generate_data.get('guidance_scale', current_settings.get('default_guidance_scale', 1.0)))
+                    seed = int(generate_data.get('seed')) if 'seed' in generate_data else None
+                    
+                    # Validate required parameters
+                    if not base64_image:
+                        await websocket.send(json.dumps({
+                            'type': 'generate_response',
+                            'success': False,
+                            'error': 'Missing base64 image'
+                        }))
+                        continue
+                    
+                    # Decode the input image
+                    input_image = decode_base64_to_image(base64_image)
+                    
+                    # Update for generation in progress
+                    await websocket.send(json.dumps({
+                        'type': 'progress_update',
+                        'stage': 'generating',
+                        'progress': 0,
+                        'file': ""
+                    }))
+                    
+                    # Generate the image
+                    output_image = generate_image(
+                        pipe=pipe,
+                        image=input_image,
+                        prompt=prompt,
+                        lora_strength=lora_strength,
+                        image_to_image_strength=image_to_image_strength,
+                        width=width,
+                        height=height,
+                        num_inference_steps=num_inference_steps,
+                        guidance_scale=guidance_scale,
+                        seed=seed
+                    )
+                    
+                    # Update for generation complete
+                    await websocket.send(json.dumps({
+                        'type': 'progress_update',
+                        'stage': 'generating',
+                        'progress': 100,
+                        'file': ""
+                    }))
+                    
+                    if output_image is None:
+                        await websocket.send(json.dumps({
+                            'type': 'generate_response',
+                            'success': False,
+                            'error': 'Failed to generate image'
+                        }))
+                        continue
+                    
+                    # Encode the output image to base64
+                    output_base64 = encode_image_to_base64(output_image)
+                    
+                    # Send the result back to the client
+                    await websocket.send(json.dumps({
+                        'type': 'generate_response',
+                        'success': True,
+                        'image': output_base64
+                    }))
+                
+                # Handle status request
+                elif message_type == 'status':
+                    await websocket.send(json.dumps({
+                        'type': 'status_response',
+                        'success': True,
+                        'model_initialized': model_initialized,
+                        'current_settings': current_settings
+                    }))
+                
+                else:
+                    await websocket.send(json.dumps({
+                        'type': 'error',
+                        'error': f'Unknown message type: {message_type}'
+                    }))
+                    
+            except json.JSONDecodeError:
+                await websocket.send(json.dumps({
+                    'type': 'error',
+                    'error': 'Invalid JSON'
+                }))
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await websocket.send(json.dumps({
+                    'type': 'error',
+                    'error': str(e)
+                }))
+    except websockets.exceptions.ConnectionClosed:
+        logger.info("Connection closed")
+    finally:
+        connected = False
+        logger.info("Client disconnected")
+
+async def main():
+    """Start the WebSocket server."""
+    server = await websockets.serve(handle_client, "localhost", 8765)
+    logger.info("Server started on ws://localhost:8765")
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    asyncio.run(main()) 

frontend/realtime/src/KonvaApp/Engine.ts

@@ -1,10 +1,9 @@
 import Konva from "konva";
 
-import { RenderEngine } from "./RenderingPrimitives/RenderEngine";
-import { ResponseType } from "./WorkerPrimitives/SharedWorkerBase";
-
 import { uiAccess, uiEvents } from "~/signals";
-
+import { ShapeNode } from "./Nodes";
+import { PaintNode } from "./Nodes/PaintNode";
+import { PreviewCopyNode } from "./Nodes/PreviewCopy";
 import { UndoStackManager } from "./UndoRedo";
 import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities";
 import {
@@ -20,22 +19,17 @@ import {
 import { ImageNode, VideoNode, TextNode } from "./Nodes";
 import { EngineOptions, TextNodeData, VideoNodeData } from "./types";
 
-import { SharedWorkerResponse } from "./WorkerPrimitives/SharedWorkerBase";
-import {
-  DiffusionSharedWorkerProgressData,
-  DiffusionSharedWorkerResponseData,
-} from "./SharedWorkers/Diffusion/DiffusionSharedWorker";
-
 import { AppModes, VideoResolutions } from "./constants";
 import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
 
 import { ToolbarNodeButtonNames } from "~/components/features/ToolbarNode/enums";
 import { NavigateFunction } from "react-router-dom";
 import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider";
-import { MediaFile } from "~/Classes/ApiManager/models/MediaFile";
+
 import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler";
 import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine";
-
+import { NodeColor } from "~/signals/uiEvents/toolbarNode";
+import { ShapeType } from "./Nodes";
 // for testing loading files from system
 // import { FileUtilities } from "./FileUtilities/FileUtilities";
 
@@ -61,7 +55,6 @@ export class Engine {
   private nodeIsolationLayer: Konva.Layer;
   private uiLayer: Konva.Layer;
   private previewLayer: Konva.Layer;
-  private renderEngine: RenderEngine;
   private offScreenCanvas: OffscreenCanvas;
   private realTimeDrawEngine: RealTimeDrawEngine;
   private nodesManager: NodesManager;
@@ -118,20 +111,8 @@ export class Engine {
     });
 
     // core layer for all the work done.
-
     this.offScreenCanvas = new OffscreenCanvas(0, 0);
 
-    this.realTimeDrawEngine = new RealTimeDrawEngine({
-      width: VideoResolutions.SQUARE_1024.width,
-      height: VideoResolutions.SQUARE_1024.height,
-      mediaLayerRef: this.mediaLayer,
-      bgLayerRef: this.bgLayer,
-      offScreenCanvas: this.offScreenCanvas,
-      previewLayerRef: this.previewLayer,
-      onRenderingSystemMessageRecieved:
-        this.onRenderingSystemReceived.bind(this),
-    });
-
     // Collection of all Nodes
     this.nodesManager = new NodesManager();
     // Partial Collection of selected Nodes
@@ -139,6 +120,20 @@ export class Engine {
       nodeTransformerRef: this.nodeTransformer,
       mediaLayerRef: this.mediaLayer,
     });
+
+    this.realTimeDrawEngine = new RealTimeDrawEngine({
+      width: VideoResolutions.SQUARE_1024.width,
+      height: VideoResolutions.SQUARE_1024.height,
+      mediaLayerRef: this.mediaLayer,
+      offScreenCanvas: this.offScreenCanvas,
+      onDraw: async (canvas, lineBounds) => {
+        await this.addPaintNode(canvas, lineBounds);
+      },
+      onPreviewCopy: async (image) => {
+        await this.addPreviewCopy(image);
+      },
+    });
+
     // Selector Square to select Nodes
     this.selectorSquare = new SelectorSquare({
       captureCanvasRef: this.realTimeDrawEngine.captureCanvas,
@@ -150,7 +145,9 @@ export class Engine {
     this.uiLayer.add(this.selectorSquare.getKonvaNode());
 
     //Collection of commands for undo-redo
-    this.undoStackManager = new UndoStackManager();
+    this.undoStackManager = new UndoStackManager(() => {
+      this.realTimeDrawEngine.render();
+    });
     this.commandManager = new CommandManager({
       mediaLayerRef: this.mediaLayer,
       nodesManagerRef: this.nodesManager,
@@ -188,86 +185,31 @@ export class Engine {
     // some of the managers has events
     // hence, lastly, setup these events
     this.setupEventSystem();
-    this.setAppMode(AppModes.SELECT);
-  }
-
-  // TODO write code to show error and retry.
-
-  onRenderingSystemReceived(
-    response: SharedWorkerResponse<
-      DiffusionSharedWorkerResponseData | MediaFile,
-      DiffusionSharedWorkerProgressData
-    >,
-  ) {
-    if (!response.data) {
-      // throw error to retry
-      uiAccess.dialogError.show({
-        title: "Generation Error",
-        message: response.data?.toString(),
-      });
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.error) {
-      console.log("Error Data?");
-      console.log(response.data, response);
-      uiAccess.dialogError.show({
-        title: "Generation Error Try again.",
-        message: response.data?.toString(),
-      });
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.result) {
-      console.log(response.data);
-      const data = response.data;
-      // create video node here.
-      // choose it to be the size of the rendering output, this case its mobile. (1560, 400)
-      if (typeof data === "string" || data === undefined) {
-        return;
-      }
-      if ("videoUrl" in data) {
-        console.log("Engine got stylized video: " + data.videoUrl);
-        this.addVideo({ mediaFileUrl: data.videoUrl });
-      } else if ("media_links" in data) {
-        console.log("Engine got rendered video: " + data.media_links.cdn_url);
-        downloadURI(data.media_links.cdn_url, "Download Video");
-      }
-
-      // hide the loader
-      //this.realTimeDrawEngine.videoLoadingCanvas.kNode.hide();
-      uiAccess.toolbarMain.loadingBar.hide();
-      this.setAppMode(AppModes.SELECT);
-      return;
-    }
-
-    if (response.responseType === ResponseType.progress) {
-      const data = response.data as DiffusionSharedWorkerProgressData;
-      // TODO wil fix this ?!?! parameter issue
-      //this.realTimeDrawEngine.videoLoadingCanvas.kNode.show();
-      uiAccess.toolbarMain.loadingBar.update({
-        message: "Rendering Frames...",
-        progress: data.progress * 100,
-        isShowing: true,
-      });
-
-      // console.log(response);
-      // if (response.data.zipBlob) {
-      //   FileUtilities.downloadBlobZip(response.data.zipBlob);
-      // }
-      return;
-    }
+    this.setAppMode(this.appMode);
   }
 
   private setAppMode(newAppMode: AppModes) {
     this.appMode = newAppMode;
     switch (this.appMode) {
+      case AppModes.PAINT: {
+        console.log("APPMODE: PAINT");
+        this.selectorSquare.disable(); // this breaks the paint if you use it in the wrong spot
+        this.selectionManager.disable(); // this prevents selection.
+        this.realTimeDrawEngine.enablePaintMode();
+        this.realTimeDrawEngine.disableDragging();
+
+        uiAccess.toolbarMain.enable();
+        uiAccess.toolbarMain.changeButtonState(ToolbarMainButtonNames.SELECT, {
+          active: false,
+        });
+        this.matteBox.disable();
+        return;
+      }
       case AppModes.SELECT: {
         console.log("APPMODE: SELECT");
+        this.realTimeDrawEngine.disablePaintMode();
+        this.realTimeDrawEngine.enableDragging();
+
         this.selectorSquare.enable();
         this.selectionManager.enable();
         uiAccess.toolbarMain.enable();
@@ -439,11 +381,32 @@ export class Engine {
     );
 
     // Listen to Toolbar Main
-    uiEvents.toolbarMain.UNDO.onClick(() => this.undoStackManager.undo());
-    uiEvents.toolbarMain.REDO.onClick(() => this.undoStackManager.redo());
-    uiEvents.toolbarMain.SAVE.onClick(async (/*event*/) => {
-      this.sceneManager.saveScene();
+    uiEvents.toolbarMain.UNDO.onClick(() => {
+      this.undoStackManager.undo();
     });
+    uiEvents.toolbarMain.REDO.onClick(() => {
+      this.undoStackManager.redo();
+      this.realTimeDrawEngine.render();
+    });
+
+    uiEvents.toolbarMain.SAVE.onClick(async (/*event*/) => {
+      await this.realTimeDrawEngine.saveOutput();
+    });
+
+    uiEvents.toolbarMain.SELECT.onClick(() => {
+      console.log("Toolbar Main >> Select");
+      this.setAppMode(AppModes.SELECT);
+    });
+
+    uiEvents.toolbarMain.PAINT.onClick(() => {
+      console.log("Toolbar Main >> Paint");
+      this.setAppMode(AppModes.PAINT);
+    });
+
+    uiEvents.toolbarMain.ERASER.onClick(() => {
+      console.log("Toolbar Main >> Eraser");
+    });
+
     uiEvents.toolbarMain.PREVIEW.onClick(async () => {
       console.log("Toolbar Main >> Preview");
       this.setAppMode(AppModes.PREVIEW);
@@ -472,10 +435,6 @@ export class Engine {
       this.addImage(image);
     });
 
-    uiEvents.onGetStagedVideo((videoData) => {
-      console.log("Engine got user video: " + videoData.mediaFileUrl);
-      this.addVideo(videoData);
-    });
     uiEvents.onAddTextToEngine((textdata) => {
       this.addText(textdata);
     });
@@ -506,21 +465,44 @@ export class Engine {
         }
       }
     });
-    uiEvents.aiStylize.onRequest(async (data) => {
-      console.log("Engine heard AI Stylize request: ", data);
 
-      try {
-        this.setAppMode(AppModes.RENDERING);
-        //await this.realTimeDrawEngine.startProcessing(data);
-      } catch (error) {
-        // throw error to retry
-        uiAccess.dialogError.show({
-          title: "Generation Error",
-          message: error?.toString() || "Unknown Error",
-        });
-        this.setAppMode(AppModes.SELECT);
+    uiEvents.promptEvents.onPromptStrengthChanged((strength) => {
+      this.realTimeDrawEngine.currentStrength = strength;
+    });
+
+    uiEvents.promptEvents.onPromptTextChanged((prompt) => {
+      this.realTimeDrawEngine.currentPrompt = prompt;
+    });
+
+    uiEvents.onAddShapeToEngine((shapeData) => {
+      switch (shapeData.shape) {
+        case "circle":
+          this.addShape(ShapeType.CIRCLE, 100);
+          break;
+        case "square":
+          this.addShape(ShapeType.SQUARE, 100);
+          break;
+        case "triangle":
+          this.addShape(ShapeType.TRIANGLE, 100);
+          break;
       }
     });
+
+    uiEvents.toolbarNode.color.onConfirmChanged((nodeColor) => {
+      console.log("Color change event triggered with:", nodeColor);
+      if (!nodeColor) {
+        console.warn("nodeColor is undefined or null");
+        return;
+      }
+      if (!nodeColor.kNodeId) {
+        console.warn("nodeColor.kNodeId is undefined or null");
+        return;
+      }
+      this.changeNodeColor(nodeColor);
+    });
+    uiEvents.toolbarMain.onPaintColorChanged((color) => {
+      this.realTimeDrawEngine.paintColor = color;
+    });
   }
 
   disableAllButtons() {
@@ -584,9 +566,9 @@ export class Engine {
       x: 10,
       y: 80,
       text: "",
-      fontSize: 24,
+      fontSize: 18,
       fontFamily: "Source Sans 3",
-      fill: "black",
+      fill: "white",
     });
     const anim = new Konva.Animation((frame) => {
       if (frame && import.meta.env.DEV) {
@@ -603,6 +585,45 @@ export class Engine {
     this.addKeyboardShortcuts();
   }
 
+  private changeNodeColor(nodeColor: NodeColor) {
+    console.log("Changing node color:", nodeColor);
+    // Find the node that is selected with the id
+    const idSelector = "#" + nodeColor.kNodeId;
+    const selectedNode = this.mediaLayer.findOne(idSelector);
+    console.log("Selected node:", selectedNode);
+    console.log("ID selector:", idSelector);
+    console.log("Media layer:", this.mediaLayer);
+    // TODO: Change the color of the node
+    // Turns out shapes are image nodes... how do we change the image's fill??
+    if (selectedNode) {
+      // Get the shape node from the real time drawing engine's image nodes
+      const shapeNode = this.realTimeDrawEngine.findImageNodeById(
+        nodeColor.kNodeId,
+      );
+      if (shapeNode && shapeNode instanceof ShapeNode) {
+        // Create new shape node with updated color
+        const newShapeNode = new ShapeNode({
+          canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+          canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
+          shapeType: shapeNode.shapeType,
+          size: shapeNode.size,
+          color: nodeColor.color,
+          mediaLayerRef: this.mediaLayer,
+          selectionManagerRef: this.selectionManager,
+          loaded: async () => {
+            this.realTimeDrawEngine.render();
+          },
+        });
+        newShapeNode.kNode.position(shapeNode.kNode.position());
+        newShapeNode.kNode.zIndex(shapeNode.kNode.zIndex());
+
+        // Remove old node and add new one
+        shapeNode.kNode.destroy();
+        this.commandManager.createNode(newShapeNode);
+      }
+    }
+  }
+
   public addText(textNodeData: TextNodeData) {
     const textNode = new TextNode({
       textNodeData: textNodeData,
@@ -614,6 +635,38 @@ export class Engine {
     this.commandManager.createNode(textNode);
   }
 
+  public addShape(type: ShapeType, size: number, color?: string) {
+    const shapeNode = new ShapeNode({
+      canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+      canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
+      shapeType: type,
+      size: size,
+      color: color,
+      mediaLayerRef: this.mediaLayer,
+      selectionManagerRef: this.selectionManager,
+      loaded: async () => {
+        await this.realTimeDrawEngine.render();
+      },
+    });
+    shapeNode.kNode.zIndex(1); // Replace desiredZIndex with a number
+
+    this.commandManager.createNode(shapeNode);
+
+    console.debug("Added shapenode:", shapeNode);
+    console.debug("Added node's ID:", shapeNode.kNode._id);
+  }
+  public addPreviewCopy(image: Image.Konva) {
+    // Start of Selection
+    const copyNode = new PreviewCopyNode({
+      image: image,
+      mediaLayerRef: this.mediaLayer,
+      selectionManagerRef: this.selectionManager,
+      loaded: async () => {
+        this.realTimeDrawEngine.render();
+      },
+    });
+    this.commandManager.createNode(copyNode);
+  }
   public addImage(imageFile: File) {
     const imageNode = new ImageNode({
       mediaLayerRef: this.mediaLayer,
@@ -621,10 +674,32 @@ export class Engine {
       canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
+      loaded: async () => {
+        await this.realTimeDrawEngine.render();
+      },
     });
-
+    imageNode.kNode.zIndex(1);
     this.commandManager.createNode(imageNode);
-    this.realTimeDrawEngine.addNodes(imageNode);
+  }
+  public addPaintNode(
+    canvas: HTMLCanvasElement,
+    lineBounds: {
+      width: number;
+      height: number;
+      x: number;
+      y: number;
+    },
+  ) {
+    var node = new PaintNode({
+      canvasElement: canvas,
+      lineBounds: lineBounds,
+      mediaLayerRef: this.mediaLayer,
+      selectionManagerRef: this.selectionManager,
+      loaded: async () => {
+        await this.realTimeDrawEngine.render();
+      },
+    });
+    this.commandManager.createNode(node);
   }
 
   public addVideo(

frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts

@@ -1,5 +1,5 @@
 import Konva from "konva";
-import { ImageNode, TextNode, VideoNode } from "../Nodes";
+import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes";
 import { MediaNode, Position, RGBColor, Transformation } from "../types";
 import {
   AddChromaKey,
@@ -20,14 +20,15 @@ import {
   NodeTransformer,
   SelectionManager,
 } from "../NodesManagers";
-import { RenderEngine } from "../RenderingPrimitives/RenderEngine";
+import { RealTimeDrawEngine } from "../RenderingPrimitives/RealTimeDrawEngine";
+import { PaintNode } from "../Nodes/PaintNode";
 
 interface EngineReferences {
   mediaLayerRef: Konva.Layer;
   nodesManagerRef: NodesManager;
   nodeTransformerRef: NodeTransformer;
   selectionManagerRef: SelectionManager;
-  renderEngineRef: RenderEngine;
+  renderEngineRef: RealTimeDrawEngine;
   undoStackManagerRef: UndoStackManager;
 }
 
@@ -36,7 +37,7 @@ export class CommandManager {
   private nodesManagerRef: NodesManager;
   private nodeTransformerRef: NodeTransformer;
   private selectionManagerRef: SelectionManager;
-  private renderEngineRef: RenderEngine;
+  private renderEngineRef: RealTimeDrawEngine;
   private undoStackManagerRef: UndoStackManager;
 
   constructor(engineRefs: EngineReferences) {
@@ -48,7 +49,7 @@ export class CommandManager {
     this.undoStackManagerRef = engineRefs.undoStackManagerRef;
   }
 
-  createNode(node: VideoNode | ImageNode | TextNode) {
+  createNode(node: MediaNode) {
     const command = new CreateCommand({
       nodes: new Set<MediaNode>([node]),
       mediaLayerRef: this.mediaLayerRef,
@@ -58,6 +59,11 @@ export class CommandManager {
       renderEngineRef: this.renderEngineRef,
     });
     this.undoStackManagerRef.executeCommand(command);
+
+    // Set the kNode's manual id to same as kNode's internal _id
+    // This helps identify the kNode in the Konva layer with find
+    node.kNode.id(node.kNode._id.toString());
+    this.renderEngineRef.render();
   }
   deleteNodes() {
     const nodes = this.selectionManagerRef.getSelectedNodes();
@@ -70,6 +76,7 @@ export class CommandManager {
       renderEngineRef: this.renderEngineRef,
     });
     this.undoStackManagerRef.executeCommand(command);
+    this.renderEngineRef.render();
   }
   toggleLockNodes() {
     const nodes = this.selectionManagerRef.getSelectedNodes();
@@ -97,6 +104,7 @@ export class CommandManager {
       mediaLayerRef: this.mediaLayerRef,
     });
     this.undoStackManagerRef.executeCommand(command);
+    this.renderEngineRef.render();
   }
   moveNodesDown() {
     const command = new MoveLayerDown({
@@ -105,6 +113,7 @@ export class CommandManager {
       mediaLayerRef: this.mediaLayerRef,
     });
     this.undoStackManagerRef.executeCommand(command);
+    this.renderEngineRef.render();
   }
   translateNodes(props: {
     nodes: Set<MediaNode>;
@@ -116,6 +125,7 @@ export class CommandManager {
       layerRef: this.mediaLayerRef,
     });
     this.undoStackManagerRef.pushCommand(command);
+    this.renderEngineRef.render();
   }
   transformNodes(props: {
     nodes: Set<MediaNode>;
@@ -127,6 +137,7 @@ export class CommandManager {
       layerRef: this.mediaLayerRef,
     });
     this.undoStackManagerRef.pushCommand(command);
+    this.renderEngineRef.render();
   }
   addChromaKey(props: { videoNode: VideoNode; newChromaColor: RGBColor }) {
     const command = new AddChromaKey(props);

frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts

@@ -29,7 +29,7 @@ export abstract class BaseNode {
   }: {
     selectionManagerRef: SelectionManager;
     mediaLayerRef: Konva.Layer;
-    kNode: Konva.Image | Konva.Group;
+    kNode: Konva.Image | Konva.Group | Konva.Shape;
   }) {
     // console.log("Node constructed");
     // this.uuid = uuidv4();
@@ -67,8 +67,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.stroke(primaryOrange);
-      wrapperRect.strokeWidth(highlightStrokeWidth);
+      console.log("Wrapper rect:", wrapperRect);
+      if (wrapperRect != null) {
+        wrapperRect.stroke(primaryOrange);
+        wrapperRect.strokeWidth(highlightStrokeWidth);
+      }
       return;
     }
     if (import.meta.env.DEV) {
@@ -86,7 +89,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.strokeWidth(0);
+      console.log("Wrapper rect:", wrapperRect);
+      if (wrapperRect != null) {
+        // Issue investigate
+        wrapperRect.strokeWidth(0);
+      }
       return;
     }
     if (import.meta.env.DEV) {
@@ -131,7 +138,7 @@ export abstract class BaseNode {
       // console.log("handle select");
       if (!isMultiSelect) {
         // clear selection if not multislect
-        //console.log("No Shift >> no multiselect");
+        console.log("No Shift >> no multiselect");
         this.selectionManagerRef.clearSelection();
       }
       if (this.selectionManagerRef.isNodeSelected(this)) {
@@ -144,8 +151,25 @@ export abstract class BaseNode {
       }
     };
 
+    const handleRightClick = () => {
+      this.selectionManagerRef.clearSelection();
+      this.selectionManagerRef.selectNode(this, undefined, true);
+    }
+
+    this.kNode.on("contextmenu", (e) => {
+      console.log("CONTEXT MENU", e);
+      e.evt.preventDefault();
+      handleRightClick();
+    });
+
     this.kNode.on("mousedown", (e) => {
-      // console.log("MOUSE DOWN");
+      // https://developer.mozilla.org/en-US/docs/Web/API/UIEvent/which
+      // 3: Right mouse button
+      // If right click, let context event handle it
+      if (e.evt.button === 3) {
+        return;
+      }
+
       // Selection of Node
       if (!this.selectionManagerRef.isNodeSelected(this)) {
         this.isSelecting = true;
@@ -155,7 +179,13 @@ export abstract class BaseNode {
     });
 
     this.kNode.on("mouseup", (e) => {
-      // console.log("MOUSE UP");
+      // https://developer.mozilla.org/en-US/docs/Web/API/UIEvent/which
+      // 3: Right mouse button
+      // If right click, let context event handle it
+      if (e.evt.button === 3) {
+        return;
+      }
+
       // just coming out of dragging or selecting mode
       // no need to handle select
       if (this.selectionManagerRef.isDragging() || this.isSelecting) {

frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts

@@ -17,12 +17,13 @@ interface ImageNodeContructor {
   transform?: TransformationData;
   mediaLayerRef: Konva.Layer;
   selectionManagerRef: SelectionManager;
+  loaded: () => Promise<void>;
 }
 
 export class ImageNode extends NetworkedNode {
   public kNode: Konva.Image;
   public imageSize?: Size;
-
+  private loaded!: () => Promise<void>;
   constructor({
     canvasPosition,
     canvasSize,
@@ -32,6 +33,7 @@ export class ImageNode extends NetworkedNode {
     transform: existingTransform,
     mediaLayerRef,
     selectionManagerRef,
+    loaded,
   }: ImageNodeContructor) {
     // kNodes need to be created first to guaruntee
     // that it is not undefined in parent's context
@@ -53,6 +55,7 @@ export class ImageNode extends NetworkedNode {
       localFile: imageFile,
     });
     this.kNode = kNode;
+    this.loaded = loaded;
     // this.imageSize = minNodeSize;
     this.mediaLayerRef.add(this.kNode);
 
@@ -83,8 +86,10 @@ export class ImageNode extends NetworkedNode {
   }) {
     const imageComponent = new Image();
     imageComponent.crossOrigin = "anonymous";
+    // this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+
     imageComponent.onload = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
+      //this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
       this.imageSize = {
         width: imageComponent.width,
         height: imageComponent.height,
@@ -105,33 +110,34 @@ export class ImageNode extends NetworkedNode {
       this.kNode.fill(transparent);
       this.listenToBaseKNode();
       this.mediaLayerRef.draw();
-      this.uploadImage(imageFile);
+      //this.uploadImage(imageFile);
+      this.loaded();
     };
     imageComponent.onerror = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
+      //this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
     };
     imageComponent.src = URL.createObjectURL(imageFile);
   }
   private async loadImageFromUrl(mediaFileUrl: string) {
-    this.setProgress({ progress: 75, status: UploadStatus.LOADING });
+    //this.setProgress({ progress: 75, status: UploadStatus.LOADING });
 
     const newImage = new Image();
     newImage.crossOrigin = "anonymous";
     newImage.onerror = () => {
-      this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
+      //this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
     };
     newImage.onload = () => {
       // console.log("network image", newImage);
       this.kNode.image(newImage);
       this.kNode.draw();
-      this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+      //this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
     };
     newImage.src = mediaFileUrl;
     this.listenToBaseKNode();
   }
 
   private async uploadImage(imageFile: File) {
-    this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
+    //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
 
     const mediaUploadApi = new MediaUploadApi();
     const uploadResponse = await mediaUploadApi.UploadImage({
@@ -149,7 +155,7 @@ export class ImageNode extends NetworkedNode {
   }
 
   private async retreiveImage(mediaFileToken: string) {
-    this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
+    //this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
     const mediaFileApi = new MediaFilesApi();
     const mediaFileResponse = await mediaFileApi.GetMediaFileByToken({
       mediaFileToken: mediaFileToken,

frontend/realtime/src/KonvaApp/Nodes/PaintNode.ts

@@ -0,0 +1,92 @@
+import Konva from "konva";
+import { SelectionManager } from "../NodesManagers";
+import { Position, NodeData } from "../types";
+import { NodeType } from "./constants";
+
+import { NetworkedNode } from "./NetworkedNode";
+
+interface PaintNodeConstructor {
+  canvasElement:HTMLCanvasElement;
+  lineBounds: {
+    width: number;
+    height: number;
+    x: number;
+    y: number;
+    };
+  mediaLayerRef: Konva.Layer;
+  selectionManagerRef: SelectionManager;
+  
+  loaded: () => Promise<void>;
+}
+
+export class PaintNode extends NetworkedNode {
+  retry(): void {
+    throw new Error("Method not implemented.");
+  }
+
+  constructor({
+    canvasElement,
+    lineBounds,
+    mediaLayerRef,
+    selectionManagerRef,
+    loaded,
+  }: PaintNodeConstructor) {
+
+    const cl = canvasElement
+
+    // Convert circle to image
+    const dataURL = cl.toDataURL();
+    const imageObj = new Image();
+    imageObj.src = dataURL;
+
+    const paintShape = new Konva.Image({
+        x: lineBounds.x,
+        y: lineBounds.y,
+        width: lineBounds.width,
+        height: lineBounds.height,
+        image: imageObj,
+        listening: false, // Disable event listening to allow captureCanvas to receive events
+        draggable: true,
+        // stroke: '#000000',
+        // strokeWidth: 2,
+        // dash: [5, 5], // Creates dotted outline THIS IS FOR DEBUG
+        globalCompositeOperation: 'source-over',
+        fill: 'transparent' 
+      });
+
+    imageObj.onload = () => {
+        this.mediaLayerRef.draw();
+        loaded();
+    };
+  
+    super({
+      selectionManagerRef: selectionManagerRef,
+      mediaLayerRef: mediaLayerRef,
+      kNode: paintShape,
+    });
+
+    this.mediaLayerRef.add(this.kNode);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+  }
+
+  public getNodeData(canvasPosition: Position) {
+    const data: NodeData = {
+      type: NodeType.SHAPE,
+      transform: {
+        position: {
+          x: this.kNode.position().x - canvasPosition.x,
+          y: this.kNode.position().y - canvasPosition.y,
+        },
+        size: this.kNode.size(),
+        rotation: this.kNode.rotation(),
+        scale: {
+          x: this.kNode.scaleX(),
+          y: this.kNode.scaleY(),
+        },
+        zIndex: this.kNode.getZIndex(),
+      },
+    };
+    return data;
+  }
+}

frontend/realtime/src/KonvaApp/Nodes/PreviewCopy.ts

@@ -0,0 +1,58 @@
+import Konva from "konva";
+import { SelectionManager } from "../NodesManagers";
+import { Position, Size, NodeData, TransformationData } from "../types";
+import { NodeType } from "./constants";
+import { NodeUtilities } from "./NodeUtilities";
+import { NetworkedNode } from "./NetworkedNode";
+
+interface PreviewCopyConstructor {
+  image: Konva.Image;
+  mediaLayerRef: Konva.Layer;
+  selectionManagerRef: SelectionManager;
+  loaded: () => Promise<void>;
+}
+
+export class PreviewCopyNode extends NetworkedNode {
+  retry(): void {
+    throw new Error("Method not implemented.");
+  }
+
+  constructor({
+    image,
+    mediaLayerRef,
+    selectionManagerRef,
+    loaded,
+  }: PreviewCopyConstructor) {
+    // Create the actual shape inside the group
+
+    super({
+      selectionManagerRef: selectionManagerRef,
+      mediaLayerRef: mediaLayerRef,
+      kNode: image,
+    });
+
+    this.mediaLayerRef.add(this.kNode);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+  }
+
+  public getNodeData(canvasPosition: Position) {
+    const data: NodeData = {
+      type: NodeType.SHAPE,
+      transform: {
+        position: {
+          x: this.kNode.position().x - canvasPosition.x,
+          y: this.kNode.position().y - canvasPosition.y,
+        },
+        size: this.kNode.size(),
+        rotation: this.kNode.rotation(),
+        scale: {
+          x: this.kNode.scaleX(),
+          y: this.kNode.scaleY(),
+        },
+        zIndex: this.kNode.getZIndex(),
+      },
+    };
+    return data;
+  }
+}

frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts

@@ -0,0 +1,206 @@
+import Konva from "konva";
+import { SelectionManager } from "../NodesManagers";
+import { Position, Size, NodeData, TransformationData } from "../types";
+import { NodeType } from "./constants";
+import { NodeUtilities } from "./NodeUtilities";
+import { NetworkedNode } from "./NetworkedNode";
+
+export enum ShapeType {
+  CIRCLE = "circle",
+  SQUARE = "square",
+  TRIANGLE = "triangle",
+}
+
+interface ShapeNodeConstructor {
+  canvasPosition: Position;
+  canvasSize: Size;
+  shapeType: ShapeType;
+  size: number;
+  color?: string; // Hex color string
+  transform?: TransformationData;
+  mediaLayerRef: Konva.Layer;
+  selectionManagerRef: SelectionManager;
+  loaded: () => Promise<void>;
+}
+
+export class ShapeNode extends NetworkedNode {
+  retry(): void {
+    throw new Error("Method not implemented.");
+  }
+
+  public shapeType: ShapeType;
+  public size: number;
+
+  constructor({
+    canvasPosition,
+    canvasSize,
+    shapeType,
+    size,
+    color = "#2D81FF", // Default red if no color provided
+    transform: existingTransform,
+    mediaLayerRef,
+    selectionManagerRef,
+    loaded,
+  }: ShapeNodeConstructor) {
+    // Create the actual shape inside the group
+    const transform = NodeUtilities.getInitialTransform({
+      existingTransform,
+      canvasPosition,
+      canvasSize,
+    });
+
+    let shape = null;
+
+    switch (shapeType) {
+      case ShapeType.CIRCLE:
+        const circleShape = new Konva.Circle({
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert circle to image
+        const dataURL = circleShape.toDataURL();
+        const imageObj = new Image();
+        imageObj.src = dataURL;
+
+        shape = new Konva.Image({
+          image: imageObj,
+          x: circleShape.x(),
+          y: circleShape.y(),
+          width: circleShape.width(),
+          height: circleShape.height(),
+          ...transform,
+          fill: "transparent",
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        imageObj.onload = () => {
+          this.mediaLayerRef.draw();
+          loaded();
+        };
+
+        // Clean up the temporary circle
+        circleShape.destroy();
+        break;
+
+      case ShapeType.SQUARE:
+        const squareShape = new Konva.Rect({
+          width: 100,
+          height: 100,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert square to image
+        const squareDataURL = squareShape.toDataURL();
+        const squareImageObj = new Image();
+        squareImageObj.src = squareDataURL;
+
+        shape = new Konva.Image({
+          image: squareImageObj,
+          x: squareShape.x(),
+          y: squareShape.y(),
+          width: squareShape.width(),
+          height: squareShape.height(),
+          ...transform,
+          fill: "transparent",
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        squareImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+          loaded()
+        };
+
+        // Clean up the temporary square
+        squareShape.destroy();
+        break;
+
+      case ShapeType.TRIANGLE:
+        const triangleShape = new Konva.RegularPolygon({
+          sides: 3,
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert triangle to image
+        const triangleDataURL = triangleShape.toDataURL();
+        const triangleImageObj = new Image();
+        triangleImageObj.src = triangleDataURL;
+
+        shape = new Konva.Image({
+          image: triangleImageObj,
+          x: triangleShape.x(),
+          y: triangleShape.y(),
+          width: triangleShape.width(),
+          height: triangleShape.height(),
+          ...transform,
+          fill: "transparent",
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        triangleImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+          loaded()
+        };
+
+        // Clean up the temporary triangle
+        triangleShape.destroy();
+        break;
+
+      default:
+        throw new Error("Invalid shape type");
+    }
+
+    super({
+      selectionManagerRef: selectionManagerRef,
+      mediaLayerRef: mediaLayerRef,
+      kNode: shape,
+    });
+
+    this.shapeType = shapeType;
+    this.size = size;
+
+    // Add shape to group
+    const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({
+      canvasOffset: canvasPosition,
+      componentSize: canvasSize,
+      maxSize: canvasSize,
+    });
+    this.kNode.setPosition(centerPosition);
+    this.mediaLayerRef.add(this.kNode);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+  }
+
+  public getNodeData(canvasPosition: Position) {
+    const data: NodeData = {
+      type: NodeType.SHAPE,
+      transform: {
+        position: {
+          x: this.kNode.position().x - canvasPosition.x,
+          y: this.kNode.position().y - canvasPosition.y,
+        },
+        size: this.kNode.size(),
+        rotation: this.kNode.rotation(),
+        scale: {
+          x: this.kNode.scaleX(),
+          y: this.kNode.scaleY(),
+        },
+        zIndex: this.kNode.getZIndex(),
+      },
+    };
+    return data;
+  }
+}

frontend/realtime/src/KonvaApp/Nodes/constants.ts

@@ -4,11 +4,12 @@ export enum NodeType {
   IMAGE = "image",
   TEXT = "text",
   VIDEO = "video",
+  SHAPE = "shape",
 }
 export const transparent = Colors.transparent;
 export const primaryOrange = Colors.primaryOrange;
 
-export const highlightStrokeWidth = 10;
+export const highlightStrokeWidth = 2;
 export const minNodeSize = {
   width: 200,
   height: 200,

frontend/realtime/src/KonvaApp/Nodes/index.ts

@@ -3,3 +3,4 @@ export { BaseNode } from "./BaseNode";
 export { ImageNode } from "./ImageNode";
 export { VideoNode } from "./VideoNode";
 export { TextNode } from "./TextNode";
+export { ShapeNode , ShapeType} from "./ShapeNode";

frontend/realtime/src/KonvaApp/NodesManagers/SelectionManager.ts

@@ -4,11 +4,12 @@ import { MediaNode, Position, Transformation } from "../types";
 import { uiAccess, uiEvents } from "~/signals";
 import { NetworkedNode } from "../Nodes/NetworkedNode";
 import { LoadingBarStatus } from "~/components/ui";
-import { ImageNode, TextNode, VideoNode } from "../Nodes";
+import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes";
 import {
   calculateContextualsPosition,
   getImageNodeButtonStates,
   getMultiSelectButtonStates,
+  getShapeNodeButtonStates,
   getTextNodeButtonStates,
   getVideoNodeButtonStates,
 } from "./ToolbarNodeUtilities";
@@ -79,7 +80,7 @@ export class SelectionManager {
   public isDisabled() {
     return this.disable;
   }
-  public selectNode(node: MediaNode, doNotDraw?: boolean): boolean {
+  public selectNode(node: MediaNode, doNotDraw?: boolean, showContextMenu = false): boolean {
     if (
       this.disabled || // no-op when disabled
       this.selectedNodes.has(node) || // if the node is already selected
@@ -99,7 +100,11 @@ export class SelectionManager {
     this.selectedNodes.add(node);
     this.updateNodeTransformer();
     this.updateContextComponents();
-    this.showContextComponents();
+
+    console.log("SelectionManager > selectNode > showContextMenu", showContextMenu);
+    if (showContextMenu) {
+      this.showContextComponents();
+    }
 
     if (!doNotDraw) {
       this.mediaLayerRef.batchDraw();
@@ -188,8 +193,8 @@ export class SelectionManager {
     // done, clear data and show menu
     this.initialPositions.clear();
     this.updateContextComponents();
-    this.showContextComponents();
   }
+
   private getKNodeTransformation(kNode: Konva.Node) {
     return {
       kNodeId: kNode._id,
@@ -273,17 +278,29 @@ export class SelectionManager {
     if (this.selectedNodes.size === 1) {
       if (node instanceof ImageNode) {
         showOrUpdate({
+          knodeIds: [node.kNode.id()],
           locked: node.isLocked(),
           buttonStates: getImageNodeButtonStates({ locked: node.isLocked() }),
         });
       }
-      if (node instanceof TextNode) {
+      else if (node instanceof ShapeNode) {
+        console.debug("Selected node is of type ShapeNode", node.kNode.id())
         showOrUpdate({
+          knodeIds: [node.kNode.id()],
           locked: node.isLocked(),
-          buttonStates: getTextNodeButtonStates({ locked: node.isLocked() }),
+          buttonStates: getShapeNodeButtonStates({ locked: node.isLocked() }),
+          color: node.getNodeData({ x: 0, y: 0 })?.textNodeData?.color ?? "#000000",
         });
       }
-      if (node instanceof VideoNode) {
+      else if (node instanceof TextNode) {
+        showOrUpdate({
+          knodeIds: [node.kNode.id()],
+          locked: node.isLocked(),
+          buttonStates: getTextNodeButtonStates({ locked: node.isLocked() }),
+          color: node.getNodeData({ x: 0, y: 0 })?.textNodeData?.color ?? "#000000",
+        });
+      }
+      else if (node instanceof VideoNode) {
         showOrUpdate({
           locked: node.isLocked(),
           buttonStates: getVideoNodeButtonStates({ locked: node.isLocked() }),

frontend/realtime/src/KonvaApp/NodesManagers/SelectorSquare.ts

@@ -40,8 +40,9 @@ export class SelectorSquare {
     stageRef: Konva.Stage;
   }) {
     this.kSquare = new Konva.Rect({
-      fill: "	rgb(250, 128, 114, 0.3)",
-      stroke: "salmon",
+      fill: "rgba(45, 129, 255, 0.3)",
+      stroke: "#00a1ff",
+      strokeWidth: 1,
       visible: false,
       // disable events to not interrupt with events
       listening: false,

frontend/realtime/src/KonvaApp/NodesManagers/ToolbarNodeUtilities.ts

@@ -51,7 +51,8 @@ export function getImageNodeButtonStates(
         buttonName === ButtonNames.AI_STYLIZE ||
         buttonName === ButtonNames.SEGMENTATION ||
         buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA
+        buttonName === ButtonNames.CHROMA ||
+        buttonName === ButtonNames.COLOR
       ) {
         buttonStates[buttonName] = {
           disabled: true,
@@ -130,7 +131,9 @@ export function getVideoNodeButtonStates(
         return buttonStates;
       }
       // soon to come feature is disabled
-      if (buttonName === ButtonNames.AI_STYLIZE) {
+      if (buttonName === ButtonNames.AI_STYLIZE ||
+        buttonName === ButtonNames.COLOR
+      ) {
         buttonStates[buttonName] = {
           disabled: true,
           hidden: true,
@@ -149,7 +152,8 @@ export function getVideoNodeButtonStates(
     {} as ContextualToolbarProps["buttonStates"],
   );
 }
-export function getMultiSelectButtonStates(
+
+export function getShapeNodeButtonStates(
   props: { locked?: boolean | "unknown" } = {},
 ) {
   const ButtonNames = ToolbarNodeButtonNames;
@@ -157,7 +161,7 @@ export function getMultiSelectButtonStates(
     (buttonStates, buttonName) => {
       // transform button depends on locked state
       if (props.locked !== undefined && buttonName === ButtonNames.TRANSFORM) {
-        buttonStates[buttonName] = {
+        buttonStates.TRANSFORM = {
           disabled: props.locked === "unknown" || props.locked === true,
           hidden: false,
           active: true,
@@ -191,3 +195,47 @@ export function getMultiSelectButtonStates(
     {} as ContextualToolbarProps["buttonStates"],
   );
 }
+
+export function getMultiSelectButtonStates(
+  props: { locked?: boolean | "unknown" } = {},
+) {
+  const ButtonNames = ToolbarNodeButtonNames;
+  return Object.values(ButtonNames).reduce(
+    (buttonStates, buttonName) => {
+      // transform button depends on locked state
+      if (props.locked !== undefined && buttonName === ButtonNames.TRANSFORM) {
+        buttonStates[buttonName] = {
+          disabled: props.locked === "unknown" || props.locked === true,
+          hidden: false,
+          active: true,
+        };
+        return buttonStates;
+      }
+
+      // hidden buttons
+      if (
+        buttonName === ButtonNames.AI_STYLIZE ||
+        buttonName === ButtonNames.SEGMENTATION ||
+        buttonName === ButtonNames.DOWNLOAD ||
+        buttonName === ButtonNames.CHROMA ||
+        buttonName === ButtonNames.COLOR
+      ) {
+        buttonStates[buttonName] = {
+          disabled: true,
+          hidden: true,
+          active: false,
+        };
+        return buttonStates;
+      }
+
+      // all other buttons
+      buttonStates[buttonName] = {
+        disabled: false,
+        hidden: false,
+        active: false,
+      };
+      return buttonStates;
+    },
+    {} as ContextualToolbarProps["buttonStates"],
+  );
+}

frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts

@@ -6,25 +6,36 @@ import { Group } from "konva/lib/Group";
 import { invoke } from "@tauri-apps/api/core";
 
 import { FileUtilities } from "../FileUtilities/FileUtilities";
-import { ImageNode, VideoNode, TextNode } from "../Nodes";
+import { ImageNode, VideoNode, TextNode, ShapeNode, ShapeType } from "../Nodes";
 import { MediaNode } from "../types";
 
 import { RenderTask } from "./RenderTask";
 import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas";
+import { Image } from "@tauri-apps/api/image";
+
+import { PaintNode } from "../Nodes/PaintNode";
+
+import {
+  ServerSetupPayload,
+  ServerSettingsPayload,
+  ServerResponse,
+} from "../types/ServerTypes";
 
 // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
 
 export class RealTimeDrawEngine {
   private videoNodes: VideoNode[];
-  private imageNodes: (ImageNode | TextNode)[];
+  private imageNodes: (ImageNode | TextNode | ShapeNode | PaintNode)[];
 
   private offScreenCanvas: OffscreenCanvas;
+  private outputBitmap: ImageBitmap | undefined;
 
   // private frames: ImageBitmap[];
 
   // capturing composite within window
-  private bgLayerRef: Konva.Layer;
+
   private mediaLayerRef: Konva.Layer;
+  private drawingsLayer: Konva.Layer; // New Layer for Drawings
 
   private height: number;
   private width: number;
@@ -41,23 +52,59 @@ export class RealTimeDrawEngine {
   public videoLoadingCanvas: VideoNode | undefined;
 
   public fps: number = 24;
+
+  public currentPrompt: string;
+  public currentStrength: number;
+
+  // Paint Color
+  // paint Brush Size
+  // has to exit out of paint mode when shape or image are used.
+  public paintColor: string = "#000000";
+  public paintBrushSize: number = 5;
+  public isPaintMode: boolean = false;
+
+  private onDrawCallback?: (
+    canvas: HTMLCanvasElement,
+    lineBounds: {
+      width: number;
+      height: number;
+      x: number;
+      y: number;
+    },
+  ) => void;
+
+  private onPreviewCopyCallback?: (previewCopy: Konva.Image) => void; // New Callback
+
+  private serverSocket: WebSocket | null = null;
+
   constructor({
     width,
     height,
-    bgLayerRef,
     mediaLayerRef,
     offScreenCanvas,
+    onDraw,
+    onPreviewCopy, // New Parameter
   }: {
     width: number;
     height: number;
-    bgLayerRef: Konva.Layer;
     mediaLayerRef: Konva.Layer;
-    previewLayerRef: Konva.Layer;
     offScreenCanvas: OffscreenCanvas;
+    onDraw?: (
+      canvas: HTMLCanvasElement,
+      lineBounds: {
+        width: number;
+        height: number;
+        x: number;
+        y: number;
+      },
+    ) => void;
+    onPreviewCopy?: (previewCopy: Konva.Image) => void; // New Parameter
   }) {
     this.videoLoadingCanvas = undefined;
     this.videoNodes = [];
     this.imageNodes = [];
+    this.onDrawCallback = onDraw;
+    this.onPreviewCopyCallback = onPreviewCopy; // Assign Callback
 
     // TODO: Make this dynamic and update this on change of canvas.
 
@@ -74,18 +121,23 @@ export class RealTimeDrawEngine {
     this.offScreenCanvas.width = this.width;
     this.offScreenCanvas.height = this.height;
 
-    this.bgLayerRef = bgLayerRef;
-
     // this is the whole canvas
     this.mediaLayerRef = mediaLayerRef;
 
-    // this.previewLayerRef = previewLayerRef;
+    // Create a separate layer for drawings
+    this.drawingsLayer = new Konva.Layer({
+      clearBeforeDraw: true, // Ensures transparent background
+    });
+    this.mediaLayerRef.getStage()?.add(this.drawingsLayer); // to od pass in stage
+
     // Set background layer to red and media layer to green for visibility
 
     this.port = undefined;
 
     this.fps = 24;
 
+    this.currentPrompt = "";
+    this.currentStrength = 100;
     // This is captures a subset of the medialayer ref
     this.captureCanvas = new Konva.Rect({
       name: "CaptureCanvas",
@@ -99,20 +151,6 @@ export class RealTimeDrawEngine {
       draggable: false,
     });
 
-    // Add preview canvas with same dimensions but different style
-    // this.previewCanvas = new Konva.Rect({
-    //   name: "PreviewCanvas",
-    //   x: this.positionX,
-    //   y: this.positionY,
-    //   width: this.width,
-    //   height: this.height,
-    //   fill: "white",
-    //   stroke: "blue",
-    //   strokeWidth: 1,
-    //   dash: [5, 5],
-    //   draggable: false,
-    // });
-
     this.previewCanvas = new Konva.Image({
       name: "PreviewCanvas",
       x: this.positionX,
@@ -120,21 +158,336 @@ export class RealTimeDrawEngine {
       width: this.width,
       height: this.height,
       image: undefined,
-      stroke: "blue",
+      stroke: "black",
       strokeWidth: 1,
-      dash: [5, 5],
       draggable: false,
+      fill: "white",
     });
 
-    this.bgLayerRef.add(this.captureCanvas);
-    this.bgLayerRef.add(this.previewCanvas);
+    this.mediaLayerRef.add(this.captureCanvas);
+    this.mediaLayerRef.add(this.previewCanvas);
     // send back
     this.captureCanvas.setZIndex(0);
-    this.previewCanvas.setZIndex(0);
+    this.previewCanvas.setZIndex(1);
+    // Add mouse events for preview canvas copying
+    //this.previewCopyListener();
 
-    //this.debug();
+    //this.startServer();
   }
 
+  private isEnabled: boolean = false;
+  private cleanupFunction: (() => void) | null = null;
+
+  // this starts the python server
+  public startServer() {
+    // Create WebSocket connection
+    const socket = new WebSocket("ws://localhost:8765");
+
+    // Setup event handlers
+    socket.onopen = () => {
+      console.log("Connected to inference server");
+      // Send initial setup payload to load models
+      const setupPayload: ServerSetupPayload = {
+        type: "setup",
+        model: {
+          name: "stabilityai/sdxl-turbo",
+          path: "F:/ComfyUI_windows_portable_nvidia/ComfyUI_windows_portable/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors",
+          precision: "fp16",
+        },
+        lora: {
+          path: "F:/ComfyUI_windows_portable_nvidia/ComfyUI_windows_portable/ComfyUI/models/loras/LCM_LoRA_Weights_SDXL.safetensors",
+          alpha: 0.75,
+        },
+        device: "cuda",
+      };
+
+      socket.send(JSON.stringify(setupPayload));
+    };
+
+    // Handle incoming messages from server
+    socket.onmessage = (event) => {
+      try {
+        const response: ServerResponse = JSON.parse(event.data);
+        console.log("Server response:", response);
+        // Handle model loading progress updates
+        if (response.type === "progress") {
+          console.log(`Loading progress: ${response.percent}%`);
+          // You might want to update UI with loading progress here
+        }
+        // Handle successful model load
+        else if (
+          response.type === "success" &&
+          response.status === "model_loaded"
+        ) {
+          console.log("Model loaded successfully");
+          // Enable UI elements that depend on model being loaded
+        }
+        // Handle image generation results
+        else if (
+          response.type === "success" &&
+          response.status === "generation_complete"
+        ) {
+          if (response.image) {
+            // Convert base64 image to ImageBitmap for preview
+            this.base64ToImageBitmap(response.image).then((bitmap) => {
+              this.outputBitmap = bitmap;
+              this.previewCanvas.image(bitmap);
+              this.mediaLayerRef.draw();
+            });
+          }
+        }
+      } catch (error) {
+        console.error("Error parsing server response:", error);
+      }
+    };
+
+    socket.onerror = (error) => {
+      console.error("WebSocket error:", error);
+    };
+
+    socket.onclose = () => {
+      console.log("Disconnected from inference server");
+    };
+
+    // Store socket reference for later use
+    this.serverSocket = socket;
+  }
+
+  // Add method to update model settings
+  public updateServerSettings(settings: ServerSettingsPayload) {
+    if (this.serverSocket && this.serverSocket.readyState === WebSocket.OPEN) {
+      this.serverSocket.send(JSON.stringify(settings));
+    } else {
+      console.error(
+        "Cannot update settings: Server connection not established",
+      );
+    }
+  }
+
+  public paintMode() {
+    let isDrawing = false;
+    let currentLine: Konva.Line | null = null;
+
+    const startDrawing = (pos: { x: number; y: number }) => {
+      if (!this.isEnabled) return;
+      const stage = this.mediaLayerRef.getStage();
+      if (!stage) return;
+
+      // Convert pointer position to relative position within capture canvas
+      const captureBox = this.captureCanvas.getClientRect();
+      const relativeX = pos.x - captureBox.x;
+      const relativeY = pos.y - captureBox.y;
+
+      currentLine = new Konva.Line({
+        points: [relativeX, relativeY],
+        stroke: this.paintColor,
+        strokeWidth: 5,
+        lineCap: "round",
+        lineJoin: "round",
+        x: this.captureCanvas.x(),
+        y: this.captureCanvas.y(),
+        draggable: false,
+      });
+
+      this.drawingsLayer.add(currentLine); // Add to drawingsLayer
+      isDrawing = true;
+    };
+
+    const draw = (pos: { x: number; y: number }) => {
+      if (!this.isEnabled) return;
+      if (!isDrawing || !currentLine) return;
+
+      // Convert pointer position to relative position
+      const captureBox = this.captureCanvas.getClientRect();
+      const relativeX = pos.x - captureBox.x;
+      const relativeY = pos.y - captureBox.y;
+
+      const newPoints = currentLine.points().concat([relativeX, relativeY]);
+      currentLine.points(newPoints);
+      this.drawingsLayer.batchDraw();
+    };
+
+    const stopDrawing = () => {
+      if (!this.isEnabled) return;
+      if (!isDrawing || !currentLine) return;
+
+      // Store current line before resetting state
+      const lineToConvert = currentLine;
+
+      // Reset drawing state immediately so we can start a new stroke
+      isDrawing = false;
+      currentLine = null;
+
+      // Create a temporary layer to render the line
+      const tempLayer = new Konva.Layer({
+        clearBeforeDraw: true,
+      });
+      this.mediaLayerRef.getStage()?.add(tempLayer);
+      tempLayer.add(lineToConvert);
+
+      // Get the line canvas with transparent background
+      // Get the bounding box of the line
+      const lineBounds = lineToConvert.getClientRect();
+
+      // Create canvas with just enough size to contain the line
+      const lineCanvas = tempLayer.toCanvas({
+        x: lineBounds.x,
+        y: lineBounds.y,
+        width: lineBounds.width,
+        height: lineBounds.height,
+        pixelRatio: 1,
+      });
+
+      if (this.onDrawCallback) {
+        this.onDrawCallback(lineCanvas, lineBounds);
+      }
+
+      // Clean up
+      lineToConvert.destroy();
+      tempLayer.destroy();
+
+      this.drawingsLayer.batchDraw();
+    };
+
+    // Check if point is within capture canvas bounds
+    const isWithinCaptureCanvas = (pos: { x: number; y: number }) => {
+      const captureBox = this.captureCanvas.getClientRect();
+      return (
+        pos.x >= captureBox.x &&
+        pos.x <= captureBox.x + captureBox.width &&
+        pos.y >= captureBox.y &&
+        pos.y <= captureBox.y + captureBox.height
+      );
+    };
+
+    // Add event listeners
+    const stage = this.mediaLayerRef.getStage();
+    if (!stage) return;
+
+    stage.on("mousedown touchstart", (e) => {
+      const pos = stage.getPointerPosition();
+      if (pos && isWithinCaptureCanvas(pos)) {
+        startDrawing(pos);
+      }
+    });
+
+    stage.on("mousemove touchmove", (e) => {
+      const pos = stage.getPointerPosition();
+      if (pos && isWithinCaptureCanvas(pos)) {
+        draw(pos);
+      }
+    });
+
+    stage.on("mouseup touchend", async () => {
+      stopDrawing();
+      await this.render();
+    });
+
+    // Store cleanup function
+    this.cleanupFunction = () => {
+      stage.off("mousedown touchstart");
+      stage.off("mousemove touchmove");
+      stage.off("mouseup touchend");
+    };
+  }
+
+  public enablePaintMode() {
+    this.isEnabled = true;
+    if (!this.cleanupFunction) {
+      this.paintMode();
+    }
+  }
+
+  public enableDragging() {
+    // Enable dragging for all nodes in media layer
+    this.imageNodes?.forEach((node) => {
+      node.kNode.draggable(true);
+      node.kNode.listening(true);
+    });
+    this.mediaLayerRef.batchDraw();
+  }
+
+  public disableDragging() {
+    // Disable dragging for all nodes in media layer
+    this.imageNodes?.forEach((node) => {
+      node.kNode.draggable(false);
+      node.kNode.listening(false);
+    });
+    this.mediaLayerRef.batchDraw();
+  }
+
+  public disablePaintMode() {
+    this.isEnabled = false;
+    if (this.cleanupFunction) {
+      this.cleanupFunction();
+      this.cleanupFunction = null;
+    }
+  }
+
+  public previewCopyListener() {
+    this.previewCanvas.on("mousedown touchstart", () => {
+      if (!this.outputBitmap) {
+        console.log("No preview image to copy");
+        return;
+      }
+
+      // Create draggable preview copy
+      const previewCopy = new Konva.Image({
+        x: this.previewCanvas.x(),
+        y: this.previewCanvas.y(),
+        width: this.width,
+        height: this.height,
+        image: this.outputBitmap,
+        draggable: true,
+        listening: true,
+      });
+
+      // this.drawingsLayer.add(previewCopy); // Add to drawingsLayer instead of mediaLayerRef
+      // this.drawingsLayer.batchDraw();
+
+      // Invoke the callback with the preview copy
+      if (this.onPreviewCopyCallback) {
+        this.onPreviewCopyCallback(previewCopy);
+      }
+
+      // Start dragging immediately
+      previewCopy.startDrag();
+
+      // Handle drag events
+      previewCopy.on("dragmove", () => {
+        // this.drawingsLayer.draw();
+      });
+
+      previewCopy.on("dragend", () => {
+        const previewBox = previewCopy.getClientRect();
+        const captureBox = this.captureCanvas.getClientRect();
+
+        if (Konva.Util.haveIntersection(previewBox, captureBox)) {
+          // Snap to capture canvas position
+          previewCopy.position({
+            x: this.captureCanvas.x(),
+            y: this.captureCanvas.y(),
+          });
+          //this.drawingsLayer.batchDraw();
+        } else {
+          // Remove if dropped outside capture area
+          //previewCopy.destroy();
+          //this.drawingsLayer.batchDraw();
+        }
+      });
+    });
+  }
+
+  public findImageNodeById(
+    id: string,
+  ): ImageNode | TextNode | ShapeNode | PaintNode | undefined {
+    return this.imageNodes.find((node) => {
+      if (node.kNode) {
+        return node.kNode.id() === id;
+      }
+      return false;
+    });
+  }
   async updateCaptureCanvas(
     width: number | undefined,
     height: number | undefined,
@@ -158,7 +511,7 @@ export class RealTimeDrawEngine {
     const oldPositionY = this.positionY;
 
     // recompute the position
-    const padBetweenCaptureAndPreview = 10;
+    const padBetweenCaptureAndPreview = 2;
     this.positionX =
       window.innerWidth / 2 -
       this.width / 2 -
@@ -266,24 +619,38 @@ export class RealTimeDrawEngine {
     });
     return canvas;
   }
+
   public isProcessing = false;
   private handleNodeDragEnd = async () => {
     // Clean up any existing state
     if (this.isProcessing) {
+      console.log("isProcessing Returning");
       return;
     }
     console.log("Node drag ended");
     this.isProcessing = true;
     await this.render();
   };
-  public addNodes(node: MediaNode) {
-    if (node instanceof VideoNode) {
-      this.videoNodes.push(node);
-      node.kNode.on("dragend", this.handleNodeDragEnd);
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
+
+  public async addNodes(node: MediaNode) {
+    if (
+      node instanceof ImageNode ||
+      node instanceof TextNode ||
+      node instanceof ShapeNode ||
+      node instanceof PaintNode
+    ) {
+      console.debug("Adding node:", node);
       this.imageNodes.push(node);
-      node.kNode.on("dragend", this.handleNodeDragEnd);
+      console.log(this.imageNodes);
+      //node.kNode.on("dragend", this.handleNodeDragEnd);
     }
+
+    // ensure the layer doesn't move if added while painting.
+    if (this.isEnabled) {
+      this.disableDragging();
+    }
+
+    await this.render();
   }
 
   public removeNodes(node: MediaNode) {
@@ -293,7 +660,12 @@ export class RealTimeDrawEngine {
         node.kNode.off("dragend", this.handleNodeDragEnd);
         this.videoNodes.splice(index, 1);
       }
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
+    } else if (
+      node instanceof ImageNode ||
+      node instanceof TextNode ||
+      node instanceof ShapeNode ||
+      node instanceof PaintNode
+    ) {
       const index = this.imageNodes.indexOf(node);
       if (index > -1) {
         node.kNode.off("dragend", this.handleNodeDragEnd);
@@ -402,7 +774,7 @@ export class RealTimeDrawEngine {
     base64String: string,
   ): Promise<ImageBitmap> {
     // Create an image element
-    const img = new Image();
+    const img = document.createElement("img");
 
     // Convert base64 to data URL if it doesn't include the prefix
     const dataUrl = base64String.startsWith("data:")
@@ -427,10 +799,37 @@ export class RealTimeDrawEngine {
     });
   }
 
-  public async render() {
-    // only pick nodes that intersect with the canvas on screen bounds to freeze.
-    this.mediaLayerRef.draw();
+  public async saveOutput() {
+    if (!this.outputBitmap) {
+      console.error("No output bitmap available to save");
+      return;
+    }
 
+    try {
+      const canvas = document.createElement("canvas");
+      canvas.width = this.outputBitmap.width;
+      canvas.height = this.outputBitmap.height;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) throw new Error("Failed to get 2D context");
+
+      ctx.drawImage(this.outputBitmap, 0, 0);
+      const blob = await new Promise<Blob>((resolve) => {
+        canvas.toBlob((b) => resolve(b!), "image/png");
+      });
+
+      await FileUtilities.blobToFileJpeg(blob, "output");
+    } catch (error) {
+      console.error("Error saving output:", error);
+    }
+  }
+
+  public async render() {
+    // only pick nodes that intersect wi th the canvas on screen bounds to freeze.
+
+    this.mediaLayerRef.draw();
+    // Output all nodes in mediaLayerRef
+    const nodes = this.mediaLayerRef.getChildren();
+    console.log("All nodes in mediaLayer:", nodes);
     console.log(
       `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`,
     );
@@ -447,17 +846,29 @@ export class RealTimeDrawEngine {
       test: false,
     })) as ImageBitmap;
 
+    // Test code
+    if (true) {
+      this.outputBitmap = bitmap;
+      this.previewCanvas.image(bitmap);
+      this.isProcessing = false;
+      return;
+    }
+
     try {
       const base64Bitmap = await this.imageBitmapToBase64(bitmap);
-      console.log("Sending to server");
+
       const base64BitmapResponse = await invoke("infer_image", {
         image: base64Bitmap,
+        prompt: this.currentPrompt,
+        strength: this.currentStrength,
       });
-      console.log("RESPONSE IS");
+
       console.log(base64BitmapResponse);
       const decoded = await this.base64ToImageBitmap(
         base64BitmapResponse as string,
       );
+
+      this.outputBitmap = decoded;
       this.previewCanvas.image(decoded);
     } catch (error) {
       console.error("Error during image processing:", error);

frontend/realtime/src/KonvaApp/UndoRedo/CreateCommand.ts

@@ -6,7 +6,7 @@ import {
   NodeTransformer,
   SelectionManager,
 } from "../NodesManagers";
-import { RenderEngine } from "../RenderingPrimitives/RenderEngine";
+import { RealTimeDrawEngine } from "../RenderingPrimitives/RealTimeDrawEngine";
 
 import { toolbarNode } from "~/signals/uiAccess/toolbarNode";
 
@@ -19,7 +19,7 @@ export class CreateCommand implements ICommand {
   private nodeTransformerRef: NodeTransformer;
   private selectionManagerRef: SelectionManager;
 
-  private renderEngineRef: RenderEngine;
+  private renderEngineRef: RealTimeDrawEngine;
 
   constructor({
     nodes,
@@ -33,7 +33,7 @@ export class CreateCommand implements ICommand {
     mediaLayerRef: Konva.Layer;
     nodesManagerRef: NodesManager;
     nodeTransformerRef: NodeTransformer;
-    renderEngineRef: RenderEngine;
+    renderEngineRef: RealTimeDrawEngine;
     selectionManagerRef: SelectionManager;
   }) {
     this.nodes = nodes;
@@ -53,6 +53,7 @@ export class CreateCommand implements ICommand {
       this.renderEngineRef.addNodes(node);
     });
     this.mediaLayerRef.draw();
+
   }
 
   undo() {
@@ -71,5 +72,7 @@ export class CreateCommand implements ICommand {
     this.nodeTransformerRef.clear();
     toolbarNode.hide();
     this.mediaLayerRef.draw();
+    this.renderEngineRef.render();
+ 
   }
 }

frontend/realtime/src/KonvaApp/UndoRedo/UndoRedoManager.ts

@@ -5,6 +5,11 @@ export class UndoStackManager {
   private undoStack: ICommand[] = [];
   private redoStack: ICommand[] = [];
   private readonly STACK_LIMIT = 20;
+  private onCommandExecuted: () => void;
+
+  constructor(onCommandExecuted: () => void) {
+    this.onCommandExecuted = onCommandExecuted;
+  }
 
   setDisabled(newState: boolean) {
     this.disabled = newState;
@@ -50,6 +55,7 @@ export class UndoStackManager {
       }
       console.log("undo:", this.undoStack);
     }
+    this.onCommandExecuted();
   }
 
   redo() {
@@ -68,5 +74,6 @@ export class UndoStackManager {
 
       console.log("Redo:", this.redoStack);
     }
+    this.onCommandExecuted();
   }
 }

frontend/realtime/src/KonvaApp/constants.ts

@@ -2,6 +2,8 @@ export enum AppModes {
   INIT = "INIT",
   PREVIEW = "PREVIEW",
   SELECT = "SELECT",
+  PAINT = "PAINT",
+  ERASER = "ERASER",
   RENDERING = "RENDERING",
 }
 
@@ -14,5 +16,5 @@ export const VideoResolutions = {
 export const Colors = {
   transparent: "rgba(0,0,0,0)",
   translucentBlack: "rgba(0,0,0,0.3)",
-  primaryOrange: "rgba(230,100,98,1)",
+  primaryOrange: "rgba(45, 129, 255, 1)",
 };

frontend/realtime/src/KonvaApp/types/NodeData.ts

@@ -1,12 +1,13 @@
-import { NodeType, BaseNode, ImageNode, TextNode, VideoNode } from "../Nodes";
+import { NodeType, BaseNode, ImageNode, TextNode, VideoNode, ShapeNode } from "../Nodes";
 
 import { TransformationData } from "./Transformation";
 
 import { TextNodeData } from "./Text";
+import { PaintNode } from "../Nodes/PaintNode";
 
 export type MediaNode =
   // | NetworkedNode
-  BaseNode | VideoNode | ImageNode | TextNode;
+  BaseNode | VideoNode | ImageNode | TextNode | ShapeNode | PaintNode;
 
 export type ImageNodeData = {
   mediaFileUrl: string;
@@ -26,6 +27,10 @@ export type VideoNodeData = {
   extractionUrl?: string;
 };
 
+export type ShapeNodeData = {
+  shape: "triangle" | "square" | "circle";
+}
+
 export type NodeData = {
   type: NodeType;
   transform: TransformationData;

frontend/realtime/src/KonvaRootComponent/ContextualToolbarNode.tsx

@@ -50,6 +50,20 @@ export const ContextualToolbarNode = () => {
     };
   }, []);
 
+  const handleColorChange = (color: string) => {
+    // Don't do anything if no node is known to be bound
+    if (uiAccess.toolbarNode.signal.value.knodeIds.length < 1) {
+      return;
+    }
+
+    // Update the UI component
+    uiAccess.toolbarNode.setColor(color);
+
+    // Signal the engine with the node ID (just grab the first -and expected only- node)
+    const nodeID = uiAccess.toolbarNode.signal.value.knodeIds[0];
+    dispatchUiEvents.toolbarNode.setSelectedColor({ kNodeId: nodeID, color });
+  }
+
   const boundedTopPosition = Math.min(position.y, winSize.height - 164);
   return (
     <Transition
@@ -71,6 +85,8 @@ export const ContextualToolbarNode = () => {
         onLockClicked={(e) => {
           dispatchUiEvents.toolbarNode.lock(e);
         }}
+        color={uiAccess.toolbarNode.signal.value.color}
+        onColorChange={handleColorChange}
       />
     </Transition>
   );

frontend/realtime/src/KonvaRootComponent/KonvaRootComponent.tsx

@@ -23,6 +23,8 @@ import { useLayoutContext } from "./contextSignals/layout";
 import { useRenderCounter } from "~/hooks/useRenderCounter";
 import { useNavigate } from "react-router-dom";
 import { SignaledMagicBox } from "./SignaledMagicBox";
+import { SignaledPromptSlider } from "./SignaledPromptSlider";
+import { SignaledPromptText } from "./SignaledPromptText";
 
 export const KonvaRootComponent = ({
   className,
@@ -54,18 +56,20 @@ export const KonvaRootComponent = ({
       <KonvaCanvasContainer
         ref={konvaContainerCallbackRef}
         className={className}
-        // retreive the classNames from the parent for sizing/styling
+      // retreive the classNames from the parent for sizing/styling
       />
-      <SignaledMagicBox />
+      {/* <SignaledMagicBox /> */}
       <SignaledCanvasDragDropFiles
         openAddImage={appUiContext.openAddImage}
         openAddVideo={appUiContext.openAddVideo}
       />
+      <SignaledPromptSlider />
+      <SignaledPromptText />
       <SignaledToolbarMain
         layoutSignal={layoutContext.signal}
         appUiContext={appUiContext}
       />
-      <SignaledToolbarVideoExtraction />
+      {/* <SignaledToolbarVideoExtraction /> */}
       <SignaledDialogs
         appUiSignal={appUiContext.signal}
         resetAll={appUiContext.resetAll}
@@ -73,7 +77,7 @@ export const KonvaRootComponent = ({
 
       <ContextualLoadingBar />
       <ContextualToolbarNode />
-      <ContextualButtonRetry />
+      {/* <ContextualButtonRetry /> */}
     </>
   );
 };

frontend/realtime/src/KonvaRootComponent/SignaledPromptSlider.tsx

@@ -0,0 +1,28 @@
+import { useSignalEffect } from "@preact/signals-react";
+import { Slider } from "~/components/ui";
+import { dispatchUiEvents } from "~/signals";
+import { promptStrength } from "~/signals/uiEvents/promptSettings";
+
+export const SignaledPromptSlider = () => {
+  const sliderValue = promptStrength.value;
+  const onSliderValueChanged =
+    dispatchUiEvents.promptSettings.setPromptStrength;
+
+  // This makes component re-render whenever the signal value changes
+  useSignalEffect(() => {
+    promptStrength.value;
+  });
+
+  return (
+    <div className="fixed bottom-[138px] left-1/2 h-4 w-[400px] -translate-x-1/2 shadow-lg">
+      <Slider
+        min={0}
+        max={100}
+        value={sliderValue}
+        onChange={onSliderValueChanged}
+        step={1}
+        innerLabel={`Strength - ${sliderValue}%`}
+      />
+    </div>
+  );
+};

frontend/realtime/src/KonvaRootComponent/SignaledPromptText.tsx

@@ -0,0 +1,39 @@
+import { useSignalEffect } from "@preact/signals-react";
+import { useRef } from "react";
+import { ResizeType, Textarea } from "~/components/ui";
+import { dispatchUiEvents } from "~/signals";
+import { promptText } from "~/signals/uiEvents/promptSettings";
+
+export const SignaledPromptText = ({}) => {
+  const textAreaRef = useRef<HTMLTextAreaElement>(null);
+
+  const TextAreaStates = {
+    placeholder: "Type your prompt here...",
+    label: "",
+    resize: "none" as ResizeType,
+    rows: 3,
+  };
+
+  const textContent = promptText.value;
+  const onTextChanged = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
+    dispatchUiEvents.promptSettings.setPrompt(event.target.value);
+  };
+
+  // This makes component re-render whenever the signal value changes
+  useSignalEffect(() => {
+    promptText.value;
+  });
+
+  return (
+    <div className="fixed bottom-5 left-1/2 w-[400px] -translate-x-1/2 text-[16px] font-medium shadow-lg">
+      <Textarea
+        className="glass rounded-xl"
+        {...TextAreaStates}
+        ref={textAreaRef}
+        onChange={onTextChanged}
+        value={textContent}
+        spellCheck={false}
+      />
+    </div>
+  );
+};

frontend/realtime/src/KonvaRootComponent/SignaledToolbarMain.tsx

@@ -12,6 +12,7 @@ import { LayoutSignalType } from "./contextSignals/layout";
 import { AppUiContextInterface } from "./contextSignals/appUi";
 import { twMerge } from "tailwind-merge";
 import { paperWrapperStyles } from "~/components/styles";
+import { dispatchUiEvents } from "~/signals";
 
 export const SignaledToolbarMain = ({
   // layoutSignal,
@@ -43,6 +44,19 @@ export const SignaledToolbarMain = ({
         return () => appUiContext.openAddImage();
       case ToolbarMainButtonNames.ADD_VIDEO:
         return () => appUiContext.openAddVideo();
+      case ToolbarMainButtonNames.ADD_CIRCLE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "circle" });
+      case ToolbarMainButtonNames.ADD_TRIANGLE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "triangle" });
+      case ToolbarMainButtonNames.ADD_SQUARE:
+        return () => dispatchUiEvents.addShapeToEngine({ shape: "square" });
+      case ToolbarMainButtonNames.SELECT:
+      case ToolbarMainButtonNames.ERASER:
+      case ToolbarMainButtonNames.PAINT:
+        return (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
+          toolbarMain.setMode(buttonName);
+          dispatchers[buttonName](e);
+        };
       default:
         return dispatchers[buttonName];
     }
@@ -69,7 +83,7 @@ export const SignaledToolbarMain = ({
     dispatchers.loadingBarRetry(e);
   };
   return (
-    <div className="fixed bottom-4 left-1/2 -translate-x-1/2">
+    <div className="fixed left-4 top-1/2 -translate-y-1/2">
       <Transition
         as="div"
         className={twMerge(

frontend/realtime/src/PyServer/ServerClient.ts

@@ -0,0 +1,260 @@
+import {
+  ServerMessage,
+  WebSocketHandlers,
+  SetupRequest,
+  UpdateSettingsRequest,
+  GenerateRequest,
+  StatusRequest,
+  GenerationParameters,
+  ServerSettings,
+  createSetupRequest,
+  createUpdateSettingsRequest,
+  createGenerateRequest,
+  createStatusRequest,
+} from "./ServerTypes";
+
+/**
+ * ServerClient provides a typed interface for communicating with the Python WebSocket server
+ */
+export class ServerClient {
+  private ws: WebSocket | null = null;
+  private handlers: WebSocketHandlers = {};
+  private connectionPromise: Promise<void> | null = null;
+  private connectionResolve: (() => void) | null = null;
+  private connectionReject: ((error: Error) => void) | null = null;
+
+  /**
+   * Create a new ServerClient
+   * @param url The WebSocket URL to connect to
+   * @param handlers Optional event handlers for different message types
+   */
+  constructor(
+    private url: string,
+    handlers: Partial<WebSocketHandlers> = {},
+  ) {
+    this.handlers = { ...handlers };
+  }
+
+  /**
+   * Connect to the WebSocket server
+   * @returns A promise that resolves when connected
+   */
+  public connect(): Promise<void> {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      return Promise.resolve();
+    }
+
+    if (this.connectionPromise) {
+      return this.connectionPromise;
+    }
+
+    this.connectionPromise = new Promise<void>((resolve, reject) => {
+      this.connectionResolve = resolve;
+      this.connectionReject = reject;
+
+      try {
+        this.ws = new WebSocket(this.url);
+
+        this.ws.onopen = () => {
+          if (this.connectionResolve) {
+            this.connectionResolve();
+          }
+          if (this.handlers.onOpen) {
+            this.handlers.onOpen();
+          }
+        };
+
+        this.ws.onclose = () => {
+          if (this.handlers.onClose) {
+            this.handlers.onClose();
+          }
+          this.ws = null;
+          this.connectionPromise = null;
+        };
+
+        this.ws.onerror = (event) => {
+          const error = new Error(`WebSocket error: ${event}`);
+          if (this.connectionReject) {
+            this.connectionReject(error);
+          }
+          if (this.handlers.onError) {
+            this.handlers.onError({
+              type: "error",
+              error: `Connection error: ${event}`,
+            });
+          }
+        };
+
+        this.ws.onmessage = (event) => {
+          try {
+            const message = JSON.parse(event.data) as ServerMessage;
+            this.handleMessage(message);
+          } catch (error) {
+            console.error("Error parsing message:", error);
+            if (this.handlers.onError) {
+              this.handlers.onError({
+                type: "error",
+                error: `Failed to parse message: ${error}`,
+              });
+            }
+          }
+        };
+      } catch (error) {
+        if (this.connectionReject) {
+          this.connectionReject(error as Error);
+        }
+        this.connectionPromise = null;
+      }
+    });
+
+    return this.connectionPromise;
+  }
+
+  /**
+   * Disconnect from the WebSocket server
+   */
+  public disconnect(): void {
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+    }
+    this.connectionPromise = null;
+  }
+
+  /**
+   * Send a message to the server
+   * @param message The message to send
+   * @returns A promise that resolves when the message is sent
+   */
+  private async send(message: ServerMessage): Promise<void> {
+    await this.connect();
+
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("WebSocket is not connected");
+    }
+
+    this.ws.send(JSON.stringify(message));
+  }
+
+  /**
+   * Handle an incoming message from the server
+   * @param message The message received from the server
+   */
+  private handleMessage(message: ServerMessage): void {
+    switch (message.type) {
+      case "setup_response":
+        if (this.handlers.onSetupResponse) {
+          this.handlers.onSetupResponse(message);
+        }
+        break;
+
+      case "update_settings_response":
+        if (this.handlers.onUpdateSettingsResponse) {
+          this.handlers.onUpdateSettingsResponse(message);
+        }
+        break;
+
+      case "generate_response":
+        if (this.handlers.onGenerateResponse) {
+          this.handlers.onGenerateResponse(message);
+        }
+        break;
+
+      case "status_response":
+        if (this.handlers.onStatusResponse) {
+          this.handlers.onStatusResponse(message);
+        }
+        break;
+
+      case "progress_update":
+        if (this.handlers.onProgressUpdate) {
+          this.handlers.onProgressUpdate(message);
+        }
+        break;
+
+      case "error":
+      case "connection_error":
+        if (this.handlers.onError) {
+          this.handlers.onError(message);
+        }
+        break;
+    }
+  }
+
+  /**
+   * Set event handlers
+   * @param handlers The event handlers to set
+   */
+  public setHandlers(handlers: Partial<WebSocketHandlers>): void {
+    this.handlers = { ...this.handlers, ...handlers };
+  }
+
+  /**
+   * Send a setup request to initialize the model
+   * @param sdxl_checkpoint_path Path to the SDXL checkpoint
+   * @param lora_path Optional path to a LoRA file
+   */
+  public async setup(
+    sdxl_checkpoint_path: string,
+    lora_path?: string,
+  ): Promise<void> {
+    const request = createSetupRequest(sdxl_checkpoint_path, lora_path);
+    await this.send(request);
+  }
+
+  /**
+   * Send an update settings request
+   * @param settings The settings to update
+   */
+  public async updateSettings(
+    settings: Partial<ServerSettings>,
+  ): Promise<void> {
+    const request = createUpdateSettingsRequest(settings);
+    await this.send(request);
+  }
+
+  /**
+   * Send a generate request
+   * @param params The generation parameters
+   */
+  public async generate(params: GenerationParameters): Promise<void> {
+    const request = createGenerateRequest(params);
+    await this.send(request);
+  }
+
+  /**
+   * Send a status request
+   */
+  public async requestStatus(): Promise<void> {
+    const request = createStatusRequest();
+    await this.send(request);
+  }
+}
+
+/**
+ * Create a base64 string from an image
+ * @param file The image file
+ * @returns A promise that resolves to the base64 string
+ */
+export function fileToBase64(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = () => {
+      const result = reader.result as string;
+      // Remove the data URL prefix (e.g., "data:image/png;base64,")
+      const base64 = result.split(",")[1];
+      resolve(base64);
+    };
+    reader.onerror = reject;
+  });
+}
+
+/**
+ * Create an image from a base64 string
+ * @param base64 The base64 string
+ * @returns The image URL
+ */
+export function base64ToImageUrl(base64: string): string {
+  return `data:image/png;base64,${base64}`;
+}

frontend/realtime/src/PyServer/ServerTypes.ts

@@ -0,0 +1,216 @@
+// Server Message Types - Common base type for all messages
+export type MessageType =
+  | "setup"
+  | "setup_response"
+  | "update_settings"
+  | "update_settings_response"
+  | "generate"
+  | "generate_response"
+  | "status"
+  | "status_response"
+  | "progress_update"
+  | "error"
+  | "connection_error";
+
+// Base interface for all messages
+interface BaseMessage {
+  type: MessageType;
+}
+
+// ===== Setup Messages =====
+
+// Setup Request
+export interface SetupRequest extends BaseMessage {
+  type: "setup";
+  setup: {
+    sdxl_checkpoint_path: string;
+    lora_path?: string;
+  };
+}
+
+// Setup Response
+export interface SetupResponse extends BaseMessage {
+  type: "setup_response";
+  success: boolean;
+  message?: string;
+  error?: string;
+  settings?: ServerSettings;
+}
+
+// ===== Update Settings Messages =====
+
+// Default generation settings that can be configured
+export interface GenerationSettings {
+  default_lora_strength?: number;
+  default_image_to_image_strength?: number;
+  default_width?: number;
+  default_height?: number;
+  default_steps?: number;
+  default_guidance_scale?: number;
+}
+
+// Server settings - includes both paths and generation defaults
+export interface ServerSettings extends GenerationSettings {
+  sdxl_checkpoint_path: string;
+  lora_path?: string | null;
+}
+
+// Update Settings Request
+export interface UpdateSettingsRequest extends BaseMessage {
+  type: "update_settings";
+  settings: Partial<ServerSettings>;
+}
+
+// Update Settings Response
+export interface UpdateSettingsResponse extends BaseMessage {
+  type: "update_settings_response";
+  success: boolean;
+  message?: string;
+  error?: string;
+  updates_applied?: string[];
+  current_settings: ServerSettings;
+}
+
+// ===== Generate Messages =====
+
+// Generation parameters
+export interface GenerationParameters {
+  image: string; // base64 encoded image
+  prompt: string;
+  lora_strength?: number;
+  image_to_image_strength?: number;
+  generated_image_width?: number;
+  generated_image_height?: number;
+  num_inference_steps?: number;
+  guidance_scale?: number;
+  seed?: number;
+}
+
+// Generate Request
+export interface GenerateRequest extends BaseMessage {
+  type: "generate";
+  generate: GenerationParameters;
+}
+
+// Generate Response
+export interface GenerateResponse extends BaseMessage {
+  type: "generate_response";
+  success: boolean;
+  image?: string; // base64 encoded image
+  error?: string;
+}
+
+// ===== Status Messages =====
+
+// Status Request
+export interface StatusRequest extends BaseMessage {
+  type: "status";
+}
+
+// Status Response
+export interface StatusResponse extends BaseMessage {
+  type: "status_response";
+  success: boolean;
+  model_initialized: boolean;
+  current_settings: ServerSettings;
+}
+
+// ===== Progress Update =====
+
+// Progress stages
+export type ProgressStage =
+  | "downloading"
+  | "loading_unet"
+  | "loading_base_model"
+  | "configuring_scheduler"
+  | "loading_lcm_adapter"
+  | "loading_lora"
+  | "generating"
+  | "error";
+
+// Progress Update Message
+export interface ProgressUpdateMessage extends BaseMessage {
+  type: "progress_update";
+  stage: ProgressStage;
+  progress: number; // 0-100
+  file: string;
+}
+
+// ===== Error Messages =====
+
+// Error Message
+export interface ErrorMessage extends BaseMessage {
+  type: "error";
+  error: string;
+}
+
+// Connection Error Message
+export interface ConnectionErrorMessage extends BaseMessage {
+  type: "connection_error";
+  error: string;
+}
+
+// Union type for all possible server messages
+export type ServerMessage =
+  | SetupRequest
+  | SetupResponse
+  | UpdateSettingsRequest
+  | UpdateSettingsResponse
+  | GenerateRequest
+  | GenerateResponse
+  | StatusRequest
+  | StatusResponse
+  | ProgressUpdateMessage
+  | ErrorMessage
+  | ConnectionErrorMessage;
+
+// WebSocket event handlers for different message types
+export interface WebSocketHandlers {
+  onSetupResponse?: (response: SetupResponse) => void;
+  onUpdateSettingsResponse?: (response: UpdateSettingsResponse) => void;
+  onGenerateResponse?: (response: GenerateResponse) => void;
+  onStatusResponse?: (response: StatusResponse) => void;
+  onProgressUpdate?: (update: ProgressUpdateMessage) => void;
+  onError?: (error: ErrorMessage | ConnectionErrorMessage) => void;
+  onOpen?: () => void;
+  onClose?: () => void;
+}
+
+// Helper function to create a typed setup request
+export function createSetupRequest(
+  sdxl_checkpoint_path: string,
+  lora_path?: string,
+): SetupRequest {
+  return {
+    type: "setup",
+    setup: {
+      sdxl_checkpoint_path,
+      lora_path,
+    },
+  };
+}
+
+// Helper function to create a typed update settings request
+export function createUpdateSettingsRequest(
+  settings: Partial<ServerSettings>,
+): UpdateSettingsRequest {
+  return {
+    type: "update_settings",
+    settings,
+  };
+}
+
+// Helper function to create a typed generate request
+export function createGenerateRequest(
+  params: GenerationParameters,
+): GenerateRequest {
+  return {
+    type: "generate",
+    generate: params,
+  };
+}
+
+// Helper function to create a typed status request
+export function createStatusRequest(): StatusRequest {
+  return { type: "status" };
+}

frontend/realtime/src/app/global.css

@@ -4,10 +4,11 @@
 
 @layer base {
   :root {
-    @apply bg-ui-background font-sans text-[#222222];
+    @apply bg-ui-background font-sans text-white;
   }
   .pegboard {
-    background-image: radial-gradient(#3f3f466b 1px, transparent 0);
+    background-image: radial-gradient(#ffffff25 1px, transparent 0);
+    background-color: #1b1b1f;
     background-size: 36px 36px;
     @apply bg-center;
   }
@@ -41,7 +42,24 @@
   }
 
   input {
-    @apply border-0 px-2 py-1 outline-none -outline-offset-2 focus:outline-primary-500 focus:ring-0;
+    @apply bg-ui-controls border-0 px-2 py-1 outline-none -outline-offset-2 focus:outline-primary-500 focus:ring-0;
+  }
+
+  textarea {
+    @apply bg-ui-controls placeholder:text-white/50;
+
+    /* Webkit scrollbar styles */
+    &::-webkit-scrollbar {
+      @apply w-1;
+    }
+
+    &::-webkit-scrollbar-track {
+      @apply bg-transparent;
+    }
+
+    &::-webkit-scrollbar-thumb {
+      @apply rounded-full bg-white/20 hover:bg-white/30;
+    }
   }
 }
 
@@ -54,10 +72,10 @@
   .dotted-pattern {
     background-image: radial-gradient(
       circle at 1px 1px,
-      rgb(0 0 0 / 0.25) 1px,
+      rgb(255 255 255 / 0.25) 1px,
       transparent 0
     );
-    background-size: 32px 32px;
+    background-size: 40px 40px;
   }
 
   .gradient-text {
@@ -87,6 +105,11 @@
     background-position: 100% 0;
     animation: highlight-sweep 3s infinite linear;
   }
+
+  .glass {
+    @apply relative border border-ui-border shadow-lg backdrop-blur-xl transition-colors duration-300;
+    @apply bg-ui-controls/50 hover:bg-ui-controls/90 !important;
+  }
 }
 
 @keyframes glow {

frontend/realtime/src/app/pages/landing/index.tsx

@@ -1,107 +1,169 @@
 import Navbar from "../../../components/landing/Navbar";
-import Hero from "../../../components/landing/Hero";
-import FeatureCard from "../../../components/landing/FeatureCard";
 import Footer from "../../../components/landing/Footer";
+import { Button } from "~/components/ui";
+import { faApple } from "@fortawesome/free-brands-svg-icons";
+import { faDesktop } from "@fortawesome/pro-solid-svg-icons";
+import { useIsMobile } from "~/hooks/useIsMobile";
 
 const Landing = () => {
+  const isMobile = useIsMobile();
   const features = [
     {
-      title: "Step 1: Compose Your Scene",
+      title: "Lorem ipsum dolor sit",
       description:
-        "Begin your story on our intuitive canvas. Drop and arrange photos and videos to create layered scenes. Position, scale, and transform elements with precision to build your perfect composition. Our dynamic canvas makes it easy to bring your vision to life.",
-      image: "/images/feature_compositing.png",
-      gradient: "bg-gradient-to-br from-purple-500/20 to-blue-500/20",
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt.",
+      image: "/images/features/ai-creation.jpg",
+      tag: "Core",
     },
     {
-      title: "Step 2: Smart Segmentation",
+      title: "Ut enim ad minim",
       description:
-        "Clean up your scenes with powerful AI segmentation. Remove green screens instantly, isolate subjects from any background, and perfect your layer masks with intelligent tools. Create professional-quality compositions with just a few clicks.",
-      image: "/images/feature_segmentation.png",
-      gradient: "bg-gradient-to-br from-emerald-500/20 to-cyan-500/20",
+        "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea.",
+      image: "/images/features/collaboration.jpg",
+      tag: "Workflow",
     },
     {
-      title: "Step 3: AI Style Transfer",
+      title: "Duis aute irure dolor",
       description:
-        "Transform your composition with AI style transfer magic. Choose from our curated collection of artistic styles. Apply consistent styling across your entire scene to create a cohesive visual masterpiece.",
-      image: "/images/feature_ai_stylize.png",
-      gradient: "bg-gradient-to-br from-rose-500/20 to-orange-500/20",
-    },
-    {
-      title: "Step 4: Render Your Video",
-      description:
-        "Render your styled composition into a stunning final movie. Export in high quality, ready to share with the world.",
-      image: "/images/feature_render.png",
-      gradient: "bg-gradient-to-br from-blue-500/20 to-indigo-500/20",
+        "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla.",
+      image: "/images/features/style-models.jpg",
+      tag: "Tools",
     },
   ];
 
-  const scrollToFeature = (index: number) => {
-    const element = document.getElementById(`feature-${index + 1}`);
-    if (element) {
-      element.scrollIntoView({ behavior: "smooth", block: "center" });
-    }
-  };
-
   return (
-    <div className="min-h-screen bg-white">
+    <div className="relative min-h-screen bg-[#101014] text-white">
       <Navbar />
 
-      <div>
-        <Hero onFeatureClick={scrollToFeature} />
-
-        <section className="relative overflow-hidden py-32">
-          {/* Decorative elements */}
-          <div className="dotted-pattern absolute inset-0 opacity-50" />
-
-          <div className="relative mx-auto max-w-7xl px-4 sm:px-6 lg:px-8">
-            {/* Section header with enhanced styling */}
-            <div className="mx-auto mb-24 max-w-3xl text-center">
-              <div className="mb-6 inline-flex items-center rounded-full bg-gradient-to-r from-purple-500/10 via-blue-500/10 to-purple-500/10 px-8 py-2.5">
-                <span className="whitespace-nowrap bg-gradient-to-r from-purple-600 via-blue-600 to-purple-600 bg-clip-text text-sm font-semibold text-transparent">
-                  AI-POWERED WORKFLOW
+      <div className="dotted-pattern absolute inset-0 z-[0] opacity-50" />
+      {/* Hero Section */}
+      <div className="relative flex overflow-hidden pt-24 xl:min-h-[1000px] xl:items-center xl:pt-0">
+        {/* Content Container */}
+        <div className="relative z-10 mx-auto grid h-full w-full max-w-[1920px] grid-cols-12 items-center px-6 md:px-16 xl:px-32">
+          <div className="z-10 col-span-12 w-full xl:absolute xl:left-0 xl:top-1/2 xl:max-w-4xl xl:-translate-y-1/2 xl:pl-32 before:xl:absolute before:xl:-inset-0 before:xl:-z-10 before:xl:mr-40 before:xl:rounded-full before:xl:bg-black/90 before:xl:to-transparent before:xl:blur-[180px]">
+            <div className="max-w-2xl">
+              <div className="mb-6">
+                <span className="text-lg font-semibold uppercase tracking-widest text-gray-400">
+                  MIRA
                 </span>
               </div>
-              <h2 className="mb-6 text-4xl font-bold tracking-tight text-gray-900">
-                Create Stunning Videos in
-                <span className="relative">
-                  <span className="gradient-text relative z-10">
-                    {" "}
-                    Four Simple Steps
-                  </span>
-                  <span className="bg-rose-200/50 absolute -bottom-2 left-0 z-0 h-3 w-full" />
-                </span>
-              </h2>
-              <p className="text-xl leading-relaxed text-gray-600">
-                From composition to final render, transform your media into
-                cinematic stories with our powerful AI-enhanced workflow.
+
+              <h1 className="mb-6 font-bold leading-tight lg:text-6xl">
+                Realtime AI editor.
+                <br />
+                On your local machine.
+              </h1>
+
+              <p className="mb-10 max-w-xl text-lg leading-relaxed">
+                Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do
+                eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
+                enim ad minim veniam, quis nostrud exercitation.
               </p>
-            </div>
 
-            {/* Features stack with enhanced visual separation */}
-            <div className="relative space-y-32">
-              {/* Vertical timeline line */}
-              <div className="to-rose-500/20 absolute left-1/2 top-0 h-full w-px bg-gradient-to-b from-purple-500/20 via-blue-500/20" />
-
-              {features.map((feature, index) => (
-                <div
-                  key={index}
-                  id={`feature-${index + 1}`}
-                  className="relative"
+              <div className="flex flex-col gap-4 sm:flex-row">
+                <Button
+                  className={`w-full transform rounded-lg px-8 py-4 font-semibold transition sm:w-auto ${
+                    isMobile
+                      ? "cursor-not-allowed bg-[#2D81FF]"
+                      : "bg-[#2D81FF] hover:scale-105 hover:bg-[#438AF6]"
+                  }`}
+                  disabled={isMobile}
+                  icon={isMobile ? faDesktop : faApple}
                 >
-                  {/* Step number bubble */}
-                  <div className="absolute left-1/2 -translate-x-1/2 -translate-y-16">
-                    <div className="flex h-12 w-12 items-center justify-center rounded-full border bg-white shadow-lg">
-                      <span className="bg-gradient-to-br from-purple-600 to-purple-600 bg-clip-text text-xl font-bold text-transparent">
-                        {index + 1}
-                      </span>
-                    </div>
-                  </div>
-                  <FeatureCard {...feature} isReversed={index % 2 === 1} />
-                </div>
-              ))}
+                  {isMobile ? "Download on desktop" : "Download for MacOS"}
+                </Button>
+                <button className="w-full transform rounded-lg border border-white/30 px-8 py-4 font-semibold transition hover:scale-105 hover:bg-white/10 sm:w-auto">
+                  Learn more
+                </button>
+              </div>
             </div>
           </div>
-        </section>
+          <div className="col-span-12 mt-12 aspect-video w-full transform overflow-hidden rounded-xl border border-white/[16%] bg-transparent md:mt-16 xl:col-span-9 xl:col-start-4 xl:mt-0">
+            <img
+              src="/images/landing_hero.png"
+              alt="AI Video Creation"
+              className="h-full w-full object-cover object-top opacity-90"
+            />
+          </div>
+        </div>
+      </div>
+
+      {/* Features Section */}
+      <div className="relative z-10 mx-auto mb-32 max-w-[1920px] px-4 py-20 sm:px-24 lg:px-32">
+        <div className="mb-8 flex items-center justify-between">
+          <h1 className="text-4xl font-bold">Features</h1>
+          {/* <button className="text-sm text-[#0096CE] hover:text-[#0084B5]">
+            View all
+          </button> */}
+        </div>
+
+        <div className="grid gap-6 md:grid-cols-3">
+          {features.map((item, index) => (
+            <div
+              key={index}
+              className="group cursor-pointer overflow-hidden rounded-xl bg-[#1C1C20] transition hover:bg-[#27272B]"
+            >
+              <div className="aspect-video overflow-hidden">
+                <img
+                  src={item.image}
+                  alt={item.title}
+                  className="h-full w-full object-cover transition group-hover:scale-105"
+                />
+              </div>
+              <div className="p-6">
+                {/* <div className="mb-4 text-sm text-[#0096CE]">{item.tag}</div> */}
+                <h3 className="mb-2 text-xl font-semibold">{item.title}</h3>
+                <p className="text-gray-400">{item.description}</p>
+              </div>
+            </div>
+          ))}
+        </div>
+
+        {/* Installation Steps Section */}
+        <div className="mt-36">
+          <h1 className="mb-12 text-5xl font-bold">
+            How to install Mira AI Editor
+          </h1>
+          <div className="grid gap-6 md:grid-cols-3">
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 1
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Download the app</h3>
+              <p className="mb-6 text-gray-400">
+                Download and install our application to start creating amazing
+                AI-powered content.
+              </p>
+              <Button className="rounded-lg bg-[#2D81FF] px-4 py-2 text-sm font-medium hover:bg-[#438AF6]">
+                Download App
+              </Button>
+            </div>
+
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 2
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Create an account</h3>
+              <p className="mb-6 text-gray-400">
+                Sign up for an account to access all features and start your
+                journey.
+              </p>
+              <Button className="text-sm font-medium" variant="secondary">
+                Sign up
+              </Button>
+            </div>
+
+            <div className="rounded-xl bg-[#1C1C20] p-8">
+              <div className="mb-4 w-fit rounded-full bg-white/15 px-3 py-1 text-sm font-medium text-white">
+                STEP 3
+              </div>
+              <h3 className="mb-4 text-2xl font-semibold">Start creating</h3>
+              <p className="mb-6 text-gray-400">
+                Begin creating with our realtime AI editor!
+              </p>
+            </div>
+          </div>
+        </div>
       </div>
 
       <Footer />

frontend/realtime/src/app/pages/main/index.tsx

@@ -8,6 +8,7 @@ import { ToolbarUserProfile } from "~/components/features";
 //Components of the Konva App are all in the KonvaComponent
 import { KonvaRootComponent } from "~/KonvaRootComponent";
 import { ToolbarTopLeft } from "~/components/features/ToolbarTopLeft";
+import { ToolbarTopRight } from "~/components/features/ToolbarTopRight";
 
 export const Main = withProtectionRoute(() => {
   // This is a hook that will log the number of times the component has rerendered
@@ -25,6 +26,7 @@ export const Main = withProtectionRoute(() => {
       />
       <div className="absolute top-0 flex w-full justify-between px-2">
         <ToolbarTopLeft />
+        <ToolbarTopRight />
         {/* <ToolbarUserProfile /> */}
       </div>
     </div>

frontend/realtime/src/components/ImageGenerator.tsx

@@ -0,0 +1,524 @@
+import React, { useEffect, useState, useRef } from "react";
+import {
+  ServerClient,
+  fileToBase64,
+  base64ToImageUrl,
+} from "../PyServer/ServerClient";
+import {
+  ProgressUpdateMessage,
+  SetupResponse,
+  GenerateResponse,
+  StatusResponse,
+  UpdateSettingsResponse,
+} from "../PyServer/ServerTypes";
+
+// Default server URL
+const DEFAULT_SERVER_URL = "ws://localhost:8765";
+
+export const ImageGenerator: React.FC = () => {
+  // Server connection state
+  const [isConnected, setIsConnected] = useState(false);
+  const [serverUrl, setServerUrl] = useState(DEFAULT_SERVER_URL);
+  const clientRef = useRef<ServerClient | null>(null);
+
+  // Model state
+  const [modelPath, setModelPath] = useState("");
+  const [loraPath, setLoraPath] = useState("");
+  const [isModelInitialized, setIsModelInitialized] = useState(false);
+
+  // Generation parameters
+  const [sourceImage, setSourceImage] = useState<File | null>(null);
+  const [sourceImageUrl, setSourceImageUrl] = useState<string | null>(null);
+  const [prompt, setPrompt] = useState("");
+  const [loraStrength, setLoraStrength] = useState(1.0);
+  const [imageToImageStrength, setImageToImageStrength] = useState(0.75);
+  const [steps, setSteps] = useState(4);
+  const [guidanceScale, setGuidanceScale] = useState(1.0);
+  const [width, setWidth] = useState(1024);
+  const [height, setHeight] = useState(1024);
+
+  // Results and UI state
+  const [generatedImageUrl, setGeneratedImageUrl] = useState<string | null>(
+    null,
+  );
+  const [isLoading, setIsLoading] = useState(false);
+  const [progress, setProgress] = useState<ProgressUpdateMessage | null>(null);
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+
+  // Initialize the client
+  useEffect(() => {
+    const client = new ServerClient(serverUrl, {
+      onOpen: () => {
+        console.log("Connected to server");
+        setIsConnected(true);
+        setErrorMessage(null);
+
+        // Request status to see if model is already initialized
+        client.requestStatus().catch((error) => {
+          console.error("Error requesting status:", error);
+        });
+      },
+
+      onClose: () => {
+        console.log("Disconnected from server");
+        setIsConnected(false);
+        setIsModelInitialized(false);
+      },
+
+      onError: (error) => {
+        console.error("Error:", error);
+        setErrorMessage(error.error);
+        setIsLoading(false);
+      },
+
+      onProgressUpdate: (update) => {
+        console.log("Progress:", update);
+        setProgress(update);
+
+        if (update.stage !== "error") {
+          setErrorMessage(null);
+        }
+      },
+
+      onSetupResponse: (response) => {
+        console.log("Setup response:", response);
+        setIsLoading(false);
+
+        if (response.success) {
+          setIsModelInitialized(true);
+          setErrorMessage(null);
+        } else {
+          setErrorMessage(response.error || "Unknown error during setup");
+        }
+      },
+
+      onUpdateSettingsResponse: (response) => {
+        console.log("Update settings response:", response);
+        setIsLoading(false);
+
+        if (response.success) {
+          setErrorMessage(null);
+        } else {
+          setErrorMessage(response.error || "Unknown error updating settings");
+        }
+      },
+
+      onGenerateResponse: (response) => {
+        console.log("Generate response:", response);
+        setIsLoading(false);
+
+        if (response.success && response.image) {
+          setGeneratedImageUrl(base64ToImageUrl(response.image));
+          setErrorMessage(null);
+        } else {
+          setErrorMessage(response.error || "Unknown error generating image");
+        }
+      },
+
+      onStatusResponse: (response) => {
+        console.log("Status response:", response);
+
+        if (response.success) {
+          setIsModelInitialized(response.model_initialized);
+
+          if (response.current_settings) {
+            // Update UI with current settings
+            setModelPath(response.current_settings.sdxl_checkpoint_path || "");
+            setLoraPath(response.current_settings.lora_path || "");
+
+            // Update generation parameters if they exist
+            if (response.current_settings.default_lora_strength !== undefined) {
+              setLoraStrength(response.current_settings.default_lora_strength);
+            }
+
+            if (
+              response.current_settings.default_image_to_image_strength !==
+              undefined
+            ) {
+              setImageToImageStrength(
+                response.current_settings.default_image_to_image_strength,
+              );
+            }
+
+            if (response.current_settings.default_steps !== undefined) {
+              setSteps(response.current_settings.default_steps);
+            }
+
+            if (
+              response.current_settings.default_guidance_scale !== undefined
+            ) {
+              setGuidanceScale(
+                response.current_settings.default_guidance_scale,
+              );
+            }
+
+            if (response.current_settings.default_width !== undefined) {
+              setWidth(response.current_settings.default_width);
+            }
+
+            if (response.current_settings.default_height !== undefined) {
+              setHeight(response.current_settings.default_height);
+            }
+          }
+        }
+      },
+    });
+
+    clientRef.current = client;
+
+    // Connect to the server
+    client.connect().catch((error) => {
+      console.error("Error connecting to server:", error);
+      setErrorMessage(`Failed to connect to server: ${error.message}`);
+    });
+
+    return () => {
+      client.disconnect();
+      clientRef.current = null;
+    };
+  }, [serverUrl]);
+
+  // Handle source image selection
+  const handleImageChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (file) {
+      setSourceImage(file);
+      setSourceImageUrl(URL.createObjectURL(file));
+    }
+  };
+
+  // Handle setup submission
+  const handleSetup = async () => {
+    if (!clientRef.current) return;
+
+    setIsLoading(true);
+    setErrorMessage(null);
+
+    try {
+      await clientRef.current.setup(modelPath, loraPath || undefined);
+    } catch (error) {
+      console.error("Error setting up model:", error);
+      setErrorMessage(`Error setting up model: ${(error as Error).message}`);
+      setIsLoading(false);
+    }
+  };
+
+  // Handle update settings
+  const handleUpdateSettings = async () => {
+    if (!clientRef.current) return;
+
+    setIsLoading(true);
+    setErrorMessage(null);
+
+    try {
+      await clientRef.current.updateSettings({
+        lora_path: loraPath || null,
+        default_lora_strength: loraStrength,
+        default_image_to_image_strength: imageToImageStrength,
+        default_steps: steps,
+        default_guidance_scale: guidanceScale,
+        default_width: width,
+        default_height: height,
+      });
+    } catch (error) {
+      console.error("Error updating settings:", error);
+      setErrorMessage(`Error updating settings: ${(error as Error).message}`);
+      setIsLoading(false);
+    }
+  };
+
+  // Handle generation
+  const handleGenerate = async () => {
+    if (!clientRef.current || !sourceImage) {
+      setErrorMessage("Please select a source image first");
+      return;
+    }
+
+    setIsLoading(true);
+    setErrorMessage(null);
+    setGeneratedImageUrl(null);
+
+    try {
+      const base64Image = await fileToBase64(sourceImage);
+
+      await clientRef.current.generate({
+        image: base64Image,
+        prompt,
+        lora_strength: loraStrength,
+        image_to_image_strength: imageToImageStrength,
+        num_inference_steps: steps,
+        guidance_scale: guidanceScale,
+        generated_image_width: width,
+        generated_image_height: height,
+      });
+    } catch (error) {
+      console.error("Error generating image:", error);
+      setErrorMessage(`Error generating image: ${(error as Error).message}`);
+      setIsLoading(false);
+    }
+  };
+
+  return (
+    <div className="mx-auto max-w-4xl p-6">
+      <h1 className="mb-6 text-2xl font-bold">SDXL Image Generator</h1>
+
+      {/* Connection and Setup */}
+      <div className="mb-6 rounded-lg bg-gray-100 p-4">
+        <h2 className="mb-2 text-xl font-semibold">Server Connection</h2>
+        <div className="mb-4 flex gap-2">
+          <input
+            type="text"
+            value={serverUrl}
+            onChange={(e) => setServerUrl(e.target.value)}
+            className="flex-grow rounded border p-2"
+            placeholder="WebSocket URL"
+          />
+          <button
+            onClick={() => clientRef.current?.connect()}
+            disabled={isConnected}
+            className="rounded bg-blue-500 px-4 py-2 text-white disabled:bg-gray-400"
+          >
+            Connect
+          </button>
+          <button
+            onClick={() => clientRef.current?.disconnect()}
+            disabled={!isConnected}
+            className="rounded bg-red-500 px-4 py-2 text-white disabled:bg-gray-400"
+          >
+            Disconnect
+          </button>
+        </div>
+
+        <div className="mb-2 flex items-center">
+          <span className="mr-2">Status:</span>
+          <span
+            className={`rounded px-2 py-1 text-white ${isConnected ? "bg-green-500" : "bg-red-500"}`}
+          >
+            {isConnected ? "Connected" : "Disconnected"}
+          </span>
+          <span className="ml-4 mr-2">Model:</span>
+          <span
+            className={`rounded px-2 py-1 text-white ${isModelInitialized ? "bg-green-500" : "bg-yellow-500"}`}
+          >
+            {isModelInitialized ? "Initialized" : "Not Initialized"}
+          </span>
+        </div>
+      </div>
+
+      {/* Model Setup */}
+      <div className="mb-6 rounded-lg bg-gray-100 p-4">
+        <h2 className="mb-2 text-xl font-semibold">Model Setup</h2>
+        <div className="space-y-4">
+          <div>
+            <label className="mb-1 block">SDXL Checkpoint Path:</label>
+            <input
+              type="text"
+              value={modelPath}
+              onChange={(e) => setModelPath(e.target.value)}
+              className="w-full rounded border p-2"
+              placeholder="/path/to/sdxl/model"
+            />
+          </div>
+          <div>
+            <label className="mb-1 block">LoRA Path (optional):</label>
+            <input
+              type="text"
+              value={loraPath}
+              onChange={(e) => setLoraPath(e.target.value)}
+              className="w-full rounded border p-2"
+              placeholder="/path/to/lora"
+            />
+          </div>
+          <div className="flex gap-2">
+            <button
+              onClick={handleSetup}
+              disabled={!isConnected || !modelPath || isLoading}
+              className="rounded bg-green-500 px-4 py-2 text-white disabled:bg-gray-400"
+            >
+              Initialize Model
+            </button>
+            <button
+              onClick={handleUpdateSettings}
+              disabled={!isConnected || !isModelInitialized || isLoading}
+              className="rounded bg-purple-500 px-4 py-2 text-white disabled:bg-gray-400"
+            >
+              Update Settings
+            </button>
+          </div>
+        </div>
+      </div>
+
+      {/* Generation Parameters */}
+      <div className="mb-6 rounded-lg bg-gray-100 p-4">
+        <h2 className="mb-2 text-xl font-semibold">Generation Parameters</h2>
+        <div className="grid grid-cols-1 gap-4 md:grid-cols-2">
+          <div>
+            <label className="mb-1 block">Prompt:</label>
+            <textarea
+              value={prompt}
+              onChange={(e) => setPrompt(e.target.value)}
+              className="h-20 w-full rounded border p-2"
+              placeholder="Enter your prompt..."
+            />
+          </div>
+
+          <div>
+            <label className="mb-1 block">Source Image:</label>
+            <input
+              type="file"
+              accept="image/*"
+              onChange={handleImageChange}
+              className="mb-2"
+            />
+            {sourceImageUrl && (
+              <img
+                src={sourceImageUrl}
+                alt="Source"
+                className="max-h-40 rounded border"
+              />
+            )}
+          </div>
+
+          <div>
+            <label className="mb-1 block">
+              LoRA Strength: {loraStrength.toFixed(2)}
+            </label>
+            <input
+              type="range"
+              min="0"
+              max="1"
+              step="0.05"
+              value={loraStrength}
+              onChange={(e) => setLoraStrength(parseFloat(e.target.value))}
+              className="w-full"
+            />
+          </div>
+
+          <div>
+            <label className="mb-1 block">
+              Image-to-Image Strength: {imageToImageStrength.toFixed(2)}
+            </label>
+            <input
+              type="range"
+              min="0"
+              max="1"
+              step="0.05"
+              value={imageToImageStrength}
+              onChange={(e) =>
+                setImageToImageStrength(parseFloat(e.target.value))
+              }
+              className="w-full"
+            />
+          </div>
+
+          <div>
+            <label className="mb-1 block">Steps: {steps}</label>
+            <input
+              type="range"
+              min="1"
+              max="20"
+              step="1"
+              value={steps}
+              onChange={(e) => setSteps(parseInt(e.target.value))}
+              className="w-full"
+            />
+          </div>
+
+          <div>
+            <label className="mb-1 block">
+              Guidance Scale: {guidanceScale.toFixed(2)}
+            </label>
+            <input
+              type="range"
+              min="0"
+              max="10"
+              step="0.1"
+              value={guidanceScale}
+              onChange={(e) => setGuidanceScale(parseFloat(e.target.value))}
+              className="w-full"
+            />
+          </div>
+
+          <div>
+            <label className="mb-1 block">Width:</label>
+            <select
+              value={width}
+              onChange={(e) => setWidth(parseInt(e.target.value))}
+              className="w-full rounded border p-2"
+            >
+              <option value="512">512</option>
+              <option value="768">768</option>
+              <option value="1024">1024</option>
+              <option value="1280">1280</option>
+            </select>
+          </div>
+
+          <div>
+            <label className="mb-1 block">Height:</label>
+            <select
+              value={height}
+              onChange={(e) => setHeight(parseInt(e.target.value))}
+              className="w-full rounded border p-2"
+            >
+              <option value="512">512</option>
+              <option value="768">768</option>
+              <option value="1024">1024</option>
+              <option value="1280">1280</option>
+            </select>
+          </div>
+        </div>
+
+        <button
+          onClick={handleGenerate}
+          disabled={
+            !isConnected || !isModelInitialized || !sourceImage || isLoading
+          }
+          className="mt-4 rounded bg-blue-500 px-6 py-2 text-white disabled:bg-gray-400"
+        >
+          Generate Image
+        </button>
+      </div>
+
+      {/* Progress and Results */}
+      {isLoading && progress && (
+        <div className="mb-6 rounded-lg bg-gray-100 p-4">
+          <h2 className="mb-2 text-xl font-semibold">Progress</h2>
+          <div className="mb-2">
+            <div className="mb-1 flex justify-between">
+              <span>{progress.stage}</span>
+              <span>{progress.progress}%</span>
+            </div>
+            <div className="h-2.5 w-full rounded-full bg-gray-300">
+              <div
+                className="h-2.5 rounded-full bg-blue-600"
+                style={{ width: `${progress.progress}%` }}
+              ></div>
+            </div>
+            {progress.file && (
+              <div className="mt-1 text-sm text-gray-600">{progress.file}</div>
+            )}
+          </div>
+        </div>
+      )}
+
+      {errorMessage && (
+        <div className="mb-6 rounded border border-red-400 bg-red-100 px-4 py-3 text-red-700">
+          <p>{errorMessage}</p>
+        </div>
+      )}
+
+      {generatedImageUrl && (
+        <div className="mb-6 rounded-lg bg-gray-100 p-4">
+          <h2 className="mb-2 text-xl font-semibold">Generated Image</h2>
+          <div className="flex justify-center">
+            <img
+              src={generatedImageUrl}
+              alt="Generated"
+              className="max-w-full rounded border"
+            />
+          </div>
+        </div>
+      )}
+    </div>
+  );
+};

frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx

@@ -42,12 +42,6 @@ export const DialogAddImage = ({
     closeCallback();
   }
 
-  function handleEnter() {
-    if (currFile) {
-      onAddImage(currFile);
-    }
-    handleClose();
-  }
   return (
     <Dialog open={isOpen} onClose={closeCallback} className="relative z-50">
       <div className={dialogBackgroundStyles}>
@@ -58,7 +52,7 @@ export const DialogAddImage = ({
             "w-full max-w-2xl",
           )}
         >
-          <DialogTitle className="text-3xl font-bold">Upload Image</DialogTitle>
+          <DialogTitle className="text-2xl font-bold">Upload Image</DialogTitle>
           <div className="flex flex-col rounded-lg border-2 border-dashed border-ui-border">
             <FileUploader
               title=""
@@ -66,6 +60,10 @@ export const DialogAddImage = ({
               file={currFile}
               setFile={(file: File | null) => {
                 setAssetFile(file);
+                if (file) {
+                  onAddImage(file);
+                  handleClose();
+                }
               }}
             />
             {currFile && (
@@ -84,9 +82,6 @@ export const DialogAddImage = ({
             <Button onClick={handleClose} variant="secondary">
               Cancel
             </Button>
-            <Button onClick={handleEnter} disabled={currFile === null}>
-              Enter
-            </Button>
           </div>
         </DialogPanel>
       </div>

frontend/realtime/src/components/features/DialogEditText/DialogEditText.tsx

@@ -73,7 +73,7 @@ export const DialogEditText = ({
     <Dialog open={isOpen} onClose={closeCallback} className="relative z-50">
       <div className={dialogBackgroundStyles}>
         <DialogPanel className={twMerge(paperWrapperStyles, dialogPanelStyles)}>
-          <DialogTitle className="text-3xl font-bold">Edit Text</DialogTitle>
+          <DialogTitle className="text-2xl font-bold">Edit Text</DialogTitle>
           <TextEditor
             text={text}
             formatData={textFormatData}

frontend/realtime/src/components/features/FileUploader/DragAndDropZone.tsx

@@ -24,7 +24,7 @@ export const DragAndDropZone = ({ file, fileTypes }: Props) => {
 
   // const fileName = file && getFileName(file).toUpperCase();
   const wrapperClassName = twMerge(
-    "group cursor-pointer px-6 py-12 bg-gray-100 hover:bg-gray-200/60 transition-colors duration-150 ease-in-out",
+    "group cursor-pointer px-6 py-12 bg-ui-controls hover:bg-white/25 transition-colors duration-150 ease-in-out rounded-lg",
     !file && "flex flex-col items-center justify-center gap-",
     file && "flex items-center gap-2 py-6",
     // "rounded-lg border-2 border-dashed border-ui-border",

frontend/realtime/src/components/features/ToolbarButton.tsx

@@ -44,12 +44,12 @@ export const ToolbarButton = ({
   }
 
   const mergedButtonClasses = twMerge(
-    "rounded-lg py-2 px-3 hover:bg-gray-200/50 transition-all duration-100",
-    children ? "w-fit flex items-center gap-2 text-nowrap" : "size-10",
-    active && "pointer-events-none text-primary ",
+    "rounded-lg hover:bg-white/15 transition-all duration-100 border-2 border-transparent text-white/80 px-2",
+    children ? "w-fit flex items-center gap-2.5 text-nowrap" : "size-10",
+    active && "bg-primary/30 border-2 border-primary hover:bg-primary/30",
     disabled && "pointer-events-none text-secondary-300",
     buttonProps.prominent &&
-      "highlight-button border border-primary-400/30 text-primary-500 hover:bg-primary-100/40 hover:border-primary-400/60",
+      "border border-primary-400/30 text-primary-500 hover:bg-primary-100/40 hover:border-primary-400/60",
     customButtonClassNames,
   );
 
@@ -57,6 +57,7 @@ export const ToolbarButton = ({
     <button
       className={mergedButtonClasses}
       disabled={disabled}
+      autoFocus={false}
       {...restButtonProps}
       onClick={(e) => {
         e.preventDefault();
@@ -73,7 +74,11 @@ export const ToolbarButton = ({
     </button>
   );
   if (tooltip) {
-    return <Tooltip tip={tooltip}>{Button}</Tooltip>;
+    return (
+      <Tooltip position="right" tip={tooltip}>
+        {Button}
+      </Tooltip>
+    );
   }
   return Button;
 };

frontend/realtime/src/components/features/ToolbarMain/ButtonPreviewAndRender.tsx

@@ -11,8 +11,6 @@ export const ButtonPreviewAndRender = ({
 }) => {
   return (
     <div className="flex">
-      <ButtonPreview {...buttonPreviewProps} />
-      <ButtonRender {...buttonRenderProps} />
     </div>
   );
 };

frontend/realtime/src/components/features/ToolbarMain/ToolbarMain.tsx

@@ -1,25 +1,32 @@
-import { Fragment, MouseEventHandler } from "react";
-import { twMerge } from "tailwind-merge";
-import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
-import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import {
   faArrowRotateLeft,
   faArrowRotateRight,
+  faCircle,
+  faEraser,
   faFilePlus,
-  faFilm,
-  faFloppyDisk,
   faImage,
   faLocationArrow,
-  faMagicWandSparkles,
+  faPaintbrush,
+  faShapes,
+  faSquare,
   faText,
+  faTriangle,
 } from "@fortawesome/pro-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
+import { Fragment, MouseEventHandler } from "react";
+import { twMerge } from "tailwind-merge";
 
 import { ToolbarButton } from "../ToolbarButton";
 
 // style and constants
 import { paperWrapperStyles, toolTipStyles } from "~/components/styles";
 import { ToolbarMainButtonNames } from "./enum";
-import { ButtonPreviewAndRender } from "./ButtonPreviewAndRender";
+import { dispatchUiEvents, uiEvents } from "~/signals/uiEvents";
+import { paintColor } from "~/signals/uiEvents/toolbarMain/paintMode";
+import { ColorPicker } from "~/components/ui/TextEditor/ColorPicker";
+import { PaintModeMenu } from "~/components/ui/ToolbarMain/PaintModeMenu";
+import { EraseModeMenu } from "~/components/ui/ToolbarMain/EraseModeMenu";
 
 export const ToolbarMain = ({
   disabled = false,
@@ -38,13 +45,14 @@ export const ToolbarMain = ({
     <>
       <div
         className={twMerge(
-          "flex w-fit items-center divide-x divide-ui-border",
+          "flex h-fit flex-col items-center gap-y-2",
           paperWrapperStyles,
+          "glass",
           disabled &&
-            "pointer-events-none cursor-default bg-ui-border shadow-md",
+          "pointer-events-none cursor-default bg-ui-border shadow-md",
         )}
       >
-        <div className="flex items-center gap-2 pr-2">
+        <div className="flex flex-col items-center gap-2">
           <ToolbarButton
             icon={faLocationArrow}
             iconProps={{ className: "fa-flip-horizontal" }}
@@ -56,44 +64,152 @@ export const ToolbarMain = ({
             buttonProps={buttonProps.ADD_TEXT}
             tooltip="Add Text"
           />
+
+          <Popover className="relative">
+            {({ close }) => (
+              <>
+                <PopoverButton as={Fragment}>
+                  <button
+                    data-tooltip="Add Shapes"
+                    className={twMerge(
+                      "size-10 rounded-lg border-2 border-transparent text-white/80 transition-all duration-100 hover:bg-white/15",
+                      toolTipStyles.base,
+                      toolTipStyles.right,
+                    )}
+                  >
+                    <FontAwesomeIcon icon={faShapes} />
+                  </button>
+                </PopoverButton>
+                <PopoverPanel
+                  anchor="right"
+                  className={twMerge(
+                    "flex flex-row [--anchor-gap:12px]",
+                    paperWrapperStyles,
+                  )}
+                >
+                  <ToolbarButton
+                    icon={faSquare}
+                    buttonProps={{
+                      ...buttonProps.ADD_SQUARE,
+                      onClick: (e) => {
+                        buttonProps.ADD_SQUARE.onClick?.(e);
+                        close();
+                      },
+                    }}
+                  />
+                  <ToolbarButton
+                    icon={faCircle}
+                    buttonProps={{
+                      ...buttonProps.ADD_CIRCLE,
+                      onClick: (e) => {
+                        buttonProps.ADD_CIRCLE.onClick?.(e);
+                        close();
+                      },
+                    }}
+                  />
+                  <ToolbarButton
+                    icon={faTriangle}
+                    buttonProps={{
+                      ...buttonProps.ADD_TRIANGLE,
+                      onClick: (e) => {
+                        buttonProps.ADD_TRIANGLE.onClick?.(e);
+                        close();
+                      },
+                    }}
+                  />
+                </PopoverPanel>
+              </>
+            )}
+          </Popover>
+
           <Popover className="relative">
             <PopoverButton as={Fragment}>
               <button
-                data-tooltip="Add..."
+                data-tooltip="Add Media"
                 className={twMerge(
-                  "size-10 rounded-lg p-2 hover:bg-gray-200/50",
-                  toolTipStyles,
+                  "size-10 rounded-lg border-2 border-transparent text-white/80 transition-all duration-100 hover:bg-white/15",
+                  toolTipStyles.base,
+                  toolTipStyles.right,
                 )}
               >
                 <FontAwesomeIcon icon={faFilePlus} />
               </button>
             </PopoverButton>
             <PopoverPanel
-              anchor="bottom"
+              anchor="right"
               className={twMerge(
                 // "absolute bottom-full left-1/2 z-10 mb-2 -translate-x-1/2",
-                "flex flex-col [--anchor-gap:16px]",
+                "flex [--anchor-gap:12px]",
                 paperWrapperStyles,
               )}
             >
               <ToolbarButton icon={faImage} buttonProps={buttonProps.ADD_IMAGE}>
-                Add Image
+                <span className="text-[16px]">Add Image</span>
               </ToolbarButton>
-              <ToolbarButton icon={faFilm} buttonProps={buttonProps.ADD_VIDEO}>
+              {/* <ToolbarButton icon={faFilm} buttonProps={buttonProps.ADD_VIDEO}>
                 Add Video
-              </ToolbarButton>
+              </ToolbarButton> */}
             </PopoverPanel>
           </Popover>
-          <ToolbarButton
-            icon={faMagicWandSparkles}
-            buttonProps={{
-              ...buttonProps.AI_STYLIZE,
-              prominent: true,
-            }}
-            tooltip="AI Stylize"
-          />
         </div>
-        <div className="flex items-center gap-2 pl-2">
+
+        <hr className="w-full border-t border-white/15" />
+
+        <div className="flex flex-col items-center gap-2">
+          <div className="flex flex-col items-center gap-2">
+            { /* Conditionally show paint menu on mode selection */}
+            {buttonProps.PAINT.active ? (
+              <div className="relative">
+                <PaintModeMenu
+                  color={paintColor.value}
+                  onChange={dispatchUiEvents.toolbarMain.setPaintColor}
+                  faIcon={faPaintbrush}
+                  borderStyle="border-2  bg-primary/30 border-2 border-primary hover:bg-primary/30 text-white"
+                  showBar={false}
+                  staticIconColor="white"
+                  streamChanges
+                  defaultOpen={true}
+                  anchor="right"
+                  anchorGap={12}
+                  closeOnMouseLeave={true}
+                />
+                <div
+                  className="pointer-events-none absolute -bottom-1.5 -right-1.5 h-[18px] w-[18px] rounded-full border border-gray-400"
+                  style={{ backgroundColor: paintColor.value }}
+                />
+              </div>
+            ) : (
+              <ToolbarButton
+                icon={faPaintbrush}
+                buttonProps={buttonProps.PAINT}
+                tooltip="Paint Brush"
+              />
+            )}
+
+            { /* Conditionally show eraser menu on mode selection */}
+            {buttonProps.ERASER.active ? (
+              <EraseModeMenu
+                faIcon={faEraser}
+                borderStyle="border-2  bg-primary/30 border-2 border-primary hover:bg-primary/30 text-white"
+                defaultOpen={true}
+                anchor="right"
+                anchorGap={12}
+                closeOnMouseLeave={true}
+              />
+            )
+              : (
+                <ToolbarButton
+                  icon={faEraser}
+                  buttonProps={buttonProps.ERASER}
+                  tooltip="Eraser"
+                />
+              )}
+          </div>
+        </div>
+
+        <hr className="w-full border-t border-white/15" />
+
+        <div className="flex flex-col items-center gap-2">
           <ToolbarButton
             icon={faArrowRotateLeft}
             buttonProps={buttonProps.UNDO}
@@ -104,15 +220,11 @@ export const ToolbarMain = ({
             buttonProps={buttonProps.REDO}
             tooltip="Redo"
           />
-          <ToolbarButton
+          {/* <ToolbarButton
             icon={faFloppyDisk}
             buttonProps={buttonProps.SAVE}
             tooltip="Save"
-          />
-          <ButtonPreviewAndRender
-            buttonPreviewProps={buttonProps.PREVIEW}
-            buttonRenderProps={buttonProps.DOWNLOAD}
-          />
+          /> */}
         </div>
       </div>
     </>

frontend/realtime/src/components/features/ToolbarMain/enum.ts

@@ -4,6 +4,11 @@ export enum ToolbarMainButtonNames {
   ADD_TEXT = "ADD_TEXT",
   ADD_IMAGE = "ADD_IMAGE",
   ADD_VIDEO = "ADD_VIDEO",
+  ADD_CIRCLE = "ADD_CIRCLE",
+  ADD_TRIANGLE = "ADD_TRIANGLE",
+  ADD_SQUARE = "ADD_SQUARE",
+  PAINT = "PAINT",
+  ERASER = "ERASER",
   AI_STYLIZE = "AI_STYLIZE",
   UNDO = "UNDO",
   REDO = "REDO",

frontend/realtime/src/components/features/ToolbarNode/ToolbarNode.tsx

@@ -4,14 +4,16 @@ import {
   faLockKeyholeOpen,
   faLockKeyhole,
   faUnlockKeyhole,
+  faPalette,
 } from "@fortawesome/pro-solid-svg-icons";
 import {
   ToolbarButton,
   ToolbarButtonProps,
 } from "~/components/features/ToolbarButton";
-import { paperWrapperStyles } from "~/components/styles";
+import { paperWrapperStyles, toolTipStyles } from "~/components/styles";
 import { ToolbarNodeButtonNames } from "./enums";
 import { ToolbarNodeButtonData } from "./data";
+import { ColorPicker } from "~/components/ui/TextEditor/ColorPicker";
 
 export interface ToolbarNodeProps {
   disabled?: boolean;
@@ -24,6 +26,8 @@ export interface ToolbarNodeProps {
   buttonsProps?: {
     [key in ToolbarNodeButtonNames]: ToolbarButtonProps;
   };
+  color?: string;
+  onColorChange?: (color: string) => void;
 }
 export const ToolbarNode = ({
   disabled,
@@ -31,6 +35,8 @@ export const ToolbarNode = ({
   lockDisabled,
   onLockClicked,
   buttonsProps,
+  color = "#000000",
+  onColorChange = () => { },
 }: ToolbarNodeProps) => {
   const handleOnLockClicked: React.MouseEventHandler<HTMLButtonElement> = (
     e,
@@ -45,7 +51,7 @@ export const ToolbarNode = ({
       className={twMerge(
         paperWrapperStyles,
         disabled && "pointer-events-none cursor-default bg-ui-border shadow-md",
-        "flex gap-2 transition",
+        "flex flex-col gap-2 transition",
       )}
     >
       <ToolbarButton
@@ -53,7 +59,7 @@ export const ToolbarNode = ({
           className: twMerge(
             locked && "text-primary hover:bg-primary hover:text-white",
             lockDisabledOrUnknown &&
-              "text-secondary-300 hover:text-secondary-300",
+            "text-secondary-300 hover:text-secondary-300",
           ),
           disabled: lockDisabledOrUnknown,
         }}
@@ -73,14 +79,24 @@ export const ToolbarNode = ({
       {ToolbarNodeButtonData.map((buttonDatum, idx) => {
         const buttonProps = buttonsProps?.[buttonDatum.name];
 
-        return (
-          <ToolbarButton
-            icon={buttonDatum.icon}
-            tooltip={buttonDatum.tooltip}
-            key={idx}
-            buttonProps={buttonProps}
-          />
-        );
+        if (!buttonProps || buttonProps.hidden) {
+          return;
+        }
+
+        if (buttonDatum.name !== ToolbarNodeButtonNames.COLOR) {
+          return (
+            <ToolbarButton
+              icon={buttonDatum.icon}
+              tooltip={buttonDatum.tooltip}
+              key={idx}
+              buttonProps={buttonProps}
+            />
+          );
+        } else {
+          return (
+            <ColorPicker color={color} onChange={onColorChange} faIcon={faPalette} borderStyle="" key={idx} />
+          )
+        }
       })}
     </div>
   );

frontend/realtime/src/components/features/ToolbarNode/data.ts

@@ -8,6 +8,7 @@ import {
   faTrashCan,
   faVectorSquare,
   faDownload,
+  faPalette,
 } from "@fortawesome/pro-solid-svg-icons";
 
 export const ToolbarNodeButtonData = [
@@ -16,6 +17,11 @@ export const ToolbarNodeButtonData = [
     icon: faVectorSquare,
     tooltip: "Move",
   },
+  {
+    name: ButtonNames.COLOR,
+    icon: faPalette,
+    tooltip: "Color",
+  },
   {
     name: ButtonNames.CHROMA,
     icon: faTransporter,

frontend/realtime/src/components/features/ToolbarNode/enums.ts

@@ -7,4 +7,5 @@ export enum ToolbarNodeButtonNames {
   MOVE_LAYER_UP = "MOVE_LAYER_UP",
   MOVE_LAYER_DOWN = "MOVE_LAYER_DOWN",
   DOWNLOAD = "DOWNLOAD",
+  COLOR = "COLOR",
 }

frontend/realtime/src/components/features/ToolbarTopLeft/index.tsx

@@ -1,28 +1,27 @@
 import { twMerge } from "tailwind-merge";
 
-import { paperWrapperStyles } from "~/components/styles";
-import { faPlus, faQuestion } from "@fortawesome/pro-solid-svg-icons";
-import { ToolbarButton } from "~/components/features/ToolbarButton";
+// import { paperWrapperStyles } from "~/components/styles";
+// import { faPlus, faQuestion } from "@fortawesome/pro-solid-svg-icons";
+// import { ToolbarButton } from "~/components/features/ToolbarButton";
 
 export const ToolbarTopLeft = () => {
   return (
     <div
       className={twMerge(
-        paperWrapperStyles,
         "z-20 mt-2 flex h-fit w-fit items-center gap-1 py-2.5 pl-4",
       )}
     >
       <img
-        src="/brand/Storyteller-Logo-Black.png"
+        src="/brand/mira-logo.png"
         alt="logo"
-        className="mr-4 h-8"
+        className="h-[30px] select-none pb-1"
       />
-      <ToolbarButton icon={faPlus}>
+      {/* <ToolbarButton icon={faPlus}>
         <span className="font-semibold">New Board</span>
       </ToolbarButton>
       <ToolbarButton icon={faQuestion}>
         <span className="font-semibold">Help</span>
-      </ToolbarButton>
+      </ToolbarButton> */}
     </div>
   );
 };

frontend/realtime/src/components/features/ToolbarTopRight/index.tsx

@@ -0,0 +1,31 @@
+import { faSave } from "@fortawesome/pro-solid-svg-icons";
+import { twMerge } from "tailwind-merge";
+import { Button } from "~/components/ui/Button";
+import { toolbarMain } from "~/signals/uiAccess/toolbarMain";
+import { dispatchers } from "~/signals/uiEvents/toolbarMain";
+import { ToolbarMainButtonNames } from "../ToolbarMain/enum";
+
+// import { paperWrapperStyles } from "~/components/styles";
+// import { faPlus, faQuestion } from "@fortawesome/pro-solid-svg-icons";
+// import { ToolbarButton } from "~/components/features/ToolbarButton";
+
+export const ToolbarTopRight = () => {
+  const saveButtonState =
+    toolbarMain.signal.value.buttonStates[ToolbarMainButtonNames.SAVE];
+
+  return (
+    <div
+      className={twMerge(
+        "z-20 mt-2 flex h-fit w-fit items-center gap-1 p-4 py-2.5",
+      )}
+    >
+      <Button
+        icon={faSave}
+        onClick={dispatchers[ToolbarMainButtonNames.SAVE]}
+        disabled={saveButtonState?.disabled}
+      >
+        Save Image
+      </Button>
+    </div>
+  );
+};

frontend/realtime/src/components/landing/Footer.tsx

@@ -9,18 +9,18 @@ const Footer = () => {
   const navigate = useNavigate();
 
   return (
-    <footer className="border-t border-gray-100 bg-gray-50">
+    <footer className="relative z-10 bg-[#18181C]">
       <div className="mx-auto max-w-7xl px-4 py-16 sm:px-6 lg:px-8">
         <div className="flex flex-col items-center space-y-10 text-center">
           <img
-            src="/brand/Storyteller-Logo-Black.png"
-            alt="Storyteller Board"
-            className="h-9"
+            src="/brand/mira-logo.png"
+            alt="Mira Realtime AI Editor"
+            className="h-[34px] pb-1"
           />
 
           <Button
             onClick={() => navigate("/signup")}
-            className="group rounded-xl px-8 py-4 text-lg"
+            className="text-md group rounded-xl bg-[#2D81FF] px-8 py-4 hover:bg-[#438AF6]"
           >
             <span className="relative flex items-center">
               Start Creating for Free

frontend/realtime/src/components/landing/Navbar.tsx

@@ -3,26 +3,31 @@ import { Button } from "../ui";
 
 const Navbar = () => {
   const navigate = useNavigate();
-
   return (
-    <nav className="fixed top-0 z-50 w-full border-b border-gray-100 bg-white/80 backdrop-blur-md">
-      <div className="mx-auto max-w-7xl px-4 sm:px-6 lg:px-8">
-        <div className="flex h-20 items-center justify-between">
+    <nav className="fixed top-0 z-50 w-full bg-[rgba(16,16,20,0.7)] backdrop-blur-3xl">
+      <div className="mx-auto max-w-[1920px] px-4 sm:px-24 lg:px-32">
+        <div className="flex h-[72px] items-center justify-between">
           <a href="/" className="flex items-center">
             <img
-              src="/brand/Storyteller-Logo-Black.png"
-              alt="Storyteller Board"
-              className="h-9"
+              src="/brand/mira-logo.png"
+              alt="Mira Realtime AI Editor"
+              className="h-[34px] pb-1"
             />
           </a>
-          <div className="flex items-center space-x-5">
-            <a
-              href="/login"
-              className="font-medium !text-gray-600 hover:!text-gray-900"
+          <div className="flex items-center space-x-3">
+            <Button
+              variant="secondary"
+              className=""
+              onClick={() => navigate("/signup")}
             >
-              Login
-            </a>
-            <Button onClick={() => navigate("/signup")}>Sign Up</Button>
+              Sign up
+            </Button>
+            <Button
+              className="hidden bg-[#2D81FF] hover:bg-[#438AF6] md:block"
+              onClick={() => []}
+            >
+              Download
+            </Button>
           </div>
         </div>
       </div>

frontend/realtime/src/components/styles.ts

@@ -1,26 +1,35 @@
 import { twMerge } from "tailwind-merge";
 
 export const paperWrapperStyles =
-  "rounded-xl border border-ui-border bg-ui-panel p-2 shadow-lg";
+  "rounded-xl border border-ui-border p-1.5 shadow-lg bg-ui-panel";
 
 export const transitionTimingStyles = "transition duration-150 ease-in-out";
 
 export const dialogBackgroundStyles =
   "fixed inset-0 flex w-screen items-center justify-center bg-black/50";
 
-export const toolTipStyles = twMerge(
-  "relative",
-  "before:w-0 before:h-0 before:absolute",
-  "before:left-1/2 before:bottom-full before:mb-[1px] before:-translate-x-1/2 before:z-50",
-  "before:border-l-8 before:border-l-transparent",
-  "before:border-r-8 before:border-r-transparent",
-  "before:border-t-8 before:border-t-white",
-  "after:absolute after:left-1/2 after:bottom-full after:-translate-x-1/2 after:z-40",
-  "after:content-[attr(data-tooltip)] after:text-black after:text-nowrap",
-  "after:rounded-lg after:border after:border-ui-border after:bg-ui-panel after:px-2.5 after:py-1.5 after:mb-2 after:shadow-xl",
-  "after:hidden before:hidden hover:after:block hover:before:block",
-);
+export const toolTipStyles = {
+  base: twMerge(
+    "relative",
+    "after:absolute after:z-40",
+    "after:content-[attr(data-tooltip)] after:text-black after:text-nowrap after:text-sm after:font-medium after:text-white/80 after:glass",
+    "after:rounded-lg after:border after:border-ui-border after:bg-ui-panel after:px-2.5 after:py-1 after:shadow-xl",
+    "after:hidden hover:after:block",
+  ),
+  top: twMerge(
+    "after:left-1/2 after:bottom-full after:-translate-x-1/2 after:mb-3.5",
+  ),
+  bottom: twMerge(
+    "after:left-1/2 after:top-full after:-translate-x-1/2 after:mt-3.5",
+  ),
+  left: twMerge(
+    "after:top-1/2 after:right-full after:-translate-y-1/2 after:mr-3.5",
+  ),
+  right: twMerge(
+    "after:top-1/2 after:left-full after:-translate-y-1/2 after:ml-3.5",
+  ),
+};
 
 export const dialogPanelStyles = twMerge(
-  "flex max-w-5xl flex-col justify-between gap-4 p-6",
+  "flex max-w-5xl flex-col justify-between gap-4 p-6 bg-[#1C1C20]",
 );

frontend/realtime/src/components/ui/Button.tsx

@@ -31,14 +31,14 @@ export const Button = ({
         return "bg-tertiary hover:bg-tertiary-400 text-white focus-visible:outline-tertiary-600";
       }
       case "secondary": {
-        return "bg-gray-600 hover:bg-gray-500 text-white focus-visible:outline-secondary";
+        return "bg-ui-controls hover:bg-ui-controls/80 text-white focus-visible:outline-secondary";
       }
       case "action": {
         return " bg-action hover:bg-action-500 text-white focus-visible:outline-action";
       }
       case "primary":
       default: {
-        return "bg-primary hover:bg-primary-400 text-white focus-visible:outline-primary-600";
+        return "bg-primary hover:bg-primary-600 text-white focus-visible:outline-primary-600";
       }
     }
   }

frontend/realtime/src/components/ui/ComboBox.tsx

@@ -29,25 +29,25 @@ export const Combobox = ({ options, value, onChange }: ComboBoxInterface) => {
       onChange={onChange}
       onClose={() => setQuery("")}
     >
-      <div className="relative flex h-10 rounded-md border">
+      <div className="relative flex h-10 rounded-md border border-ui-border">
         <ComboboxInput
-          className="rounded-md"
+          className="rounded-lg px-3"
           displayValue={(font: string) => font}
           onChange={(event) => setQuery(event.target.value)}
         />
-        <ComboboxButton className="absolute right-0 top-0 flex h-10 items-center justify-center hover:text-primary">
-          <FontAwesomeIcon icon={faChevronDown} className="size-4 p-2" />
+        <ComboboxButton className="absolute right-0 top-0 flex h-10 items-center justify-center px-3 transition-colors duration-300 hover:text-primary">
+          <FontAwesomeIcon icon={faChevronDown} className="size-4" />
         </ComboboxButton>
       </div>
       <ComboboxOptions
         anchor="bottom"
-        className="mt-1 w-[var(--input-width)] rounded-lg border empty:invisible"
+        className="mt-1 w-[var(--input-width)] rounded-lg border border-ui-border empty:invisible"
       >
         {filteredOptions.map((option, idx) => (
           <ComboboxOption
             key={idx}
             value={option}
-            className="cursor-pointer bg-white p-2 data-[focus]:bg-primary-200 data-[selected]:font-medium"
+            className="bg-ui-controls cursor-pointer px-3 py-2 data-[focus]:bg-gray-600 data-[selected]:font-medium"
           >
             {option}
           </ComboboxOption>

frontend/realtime/src/components/ui/Slider.tsx

@@ -1,6 +1,8 @@
 import { Transition } from "@headlessui/react";
 import React, { useState, useRef, useEffect, ReactNode } from "react";
 import { twMerge } from "tailwind-merge";
+import { Button } from "./Button";
+import { faMinus, faPlus } from "@fortawesome/pro-solid-svg-icons";
 
 interface SliderProps {
   min: number;
@@ -12,6 +14,9 @@ interface SliderProps {
   showTooltip?: boolean;
   tooltipContent?: ReactNode;
   suffix?: string;
+  innerLabel?: string;
+  showDecrement?: boolean,
+  showIncrement?: boolean,
 }
 
 export const Slider = ({
@@ -24,6 +29,9 @@ export const Slider = ({
   showTooltip,
   tooltipContent,
   suffix,
+  innerLabel,
+  showDecrement,
+  showIncrement
 }: SliderProps) => {
   const [localValue, setLocalValue] = useState(value);
   const sliderRef = useRef<HTMLDivElement>(null);
@@ -34,15 +42,19 @@ export const Slider = ({
     setLocalValue(value);
   }, [value]);
 
+  const setClampedValue = (value: number) => {
+    const roundedValue = Math.round(value / step) * step;
+    const clampedValue = Math.min(Math.max(roundedValue, min), max);
+    setLocalValue(clampedValue);
+    onChange(clampedValue);
+  }
+
   const updatePosition = (x: number) => {
     if (!sliderRef.current) return;
     const rect = sliderRef.current.getBoundingClientRect();
     const fraction = (x - rect.left) / rect.width;
     const newValue = fraction * (max - min) + min;
-    const roundedValue = Math.round(newValue / step) * step;
-    const clampedValue = Math.min(Math.max(roundedValue, min), max);
-    setLocalValue(clampedValue);
-    onChange(clampedValue);
+    setClampedValue(newValue);
   };
 
   const handleMouseDown = (e: React.MouseEvent<HTMLDivElement>) => {
@@ -79,34 +91,75 @@ export const Slider = ({
     }
   };
 
+  const handleDecrement = () => {
+    setClampedValue(localValue - step);
+  }
+
+  const handleIncrement = () => {
+    setClampedValue(localValue + step);
+  }
+
   const percentage = ((localValue - min) / (max - min)) * 100;
   const displayValue =
     localValue.toFixed(step.toString().split(".")[1]?.length || 0) +
     (suffix || "");
 
   return (
-    <div
-      ref={sliderRef}
-      className={twMerge(
-        "relative h-2.5 w-full cursor-pointer rounded-full bg-gray-500",
-        className,
+    <div className="flex w-full">
+      {showDecrement && (
+        <Button
+          icon={faMinus}
+          className="size-6 text-white/80 bg-transparent rounded-full hover:bg-white/10 focus-visible:outline-primary active:bg-primary/30 my-auto mr-1"
+          onClick={handleDecrement}
+        />
       )}
-      onMouseDown={handleMouseDown}
-    >
-      <div
-        className="absolute h-2.5 rounded-full bg-primary"
-        style={{ width: `${percentage}%` }}
-      ></div>
+
       <div
+        ref={sliderRef}
         className={twMerge(
-          "absolute -top-1 h-[18px] w-[18px] -translate-x-1/2 transform rounded-full bg-gray-200 transition-colors",
-          isDragging ? "bg-white" : "bg-gray-300 hover:bg-white",
+          "glass group relative h-7 w-full cursor-pointer overflow-hidden rounded-lg border border-ui-border",
+          isDragging && "!bg-ui-controls/90",
+          className,
         )}
-        style={{ left: `${percentage}%` }}
         onMouseDown={handleMouseDown}
-        onMouseEnter={handleMouseEnter}
-        onMouseLeave={handleMouseLeave}
-      />
+      >
+        <div
+          className={twMerge(
+            "absolute h-7 bg-primary/30 transition-colors duration-300 group-hover:bg-primary/50",
+            isDragging && "!bg-primary/50",
+          )}
+          style={{ width: `${percentage}%` }}
+        >
+          {innerLabel && (
+            <span
+              className={twMerge(
+                "absolute top-1/2 ml-2.5 -translate-y-1/2 text-nowrap text-sm font-medium text-white/60 transition-colors duration-300 group-hover:text-white",
+                isDragging && "!text-white",
+              )}
+            >
+              {innerLabel}
+            </span>
+          )}
+          <div
+            className={twMerge(
+              "absolute right-0 top-1/2 mr-1.5 h-3.5 w-0.5 -translate-y-1/2",
+              isDragging ? "bg-white" : "bg-white/50",
+            )}
+            onMouseDown={handleMouseDown}
+            onMouseEnter={handleMouseEnter}
+            onMouseLeave={handleMouseLeave}
+          />
+        </div>
+      </div>
+
+      {showIncrement && (
+        <Button
+          icon={faPlus}
+          className="size-6 text-white/80 bg-transparent rounded-full hover:bg-white/10 focus-visible:outline-primary active:bg-primary/30 my-auto ml-1"
+          onClick={handleIncrement}
+        />
+      )}
+
       {showTooltip && (
         <Transition
           as={"div"}

frontend/realtime/src/components/ui/TextArea.tsx

@@ -7,10 +7,16 @@ export type ResizeType =
   | "both"
   | "horizontal"
   | "vertical"
-  | "block"
-  | "inline"
   | undefined;
 
+const resizeStyles = {
+  none: "resize-none",
+  both: "resize",
+  horizontal: "resize-x",
+  vertical: "resize-y",
+  undefined: "",
+};
+
 export interface TextareaInterface
   extends TextareaHTMLAttributes<HTMLTextAreaElement> {
   label?: string;
@@ -21,6 +27,8 @@ export const Textarea = forwardRef<HTMLTextAreaElement, TextareaInterface>(
     { className, label, resize = "vertical", id, ...rest }: TextareaInterface,
     ref,
   ) => {
+    const resizeStyle = resizeStyles[resize];
+
     return (
       <div className="flex flex-col">
         {label && <label htmlFor={id ? id : kebabCase(label)}>{label}</label>}
@@ -29,17 +37,16 @@ export const Textarea = forwardRef<HTMLTextAreaElement, TextareaInterface>(
           ref={ref}
           id={id ? id : label ? kebabCase(label) : undefined}
           className={twMerge(
-            "border-ui-panel-border bg-ui-controls rounded-lg border px-3 py-2",
+            "rounded-lg border border-ui-border px-4 py-2.5 text-white outline-none focus:outline-none",
             className,
+            resizeStyle,
           )}
           style={{
             outline: "2px solid transparent",
-            transition: "outline-color 0.15s ease-in-out",
-            resize: resize,
           }}
           onFocus={(e) => {
             // disableHotkeyInput(DomLevels.INPUT);
-            e.currentTarget.style.outlineColor = "#e66462";
+            e.currentTarget.style.outlineColor = "#2d81ff";
           }}
           onBlur={(e) => {
             // enableHotkeyInput(DomLevels.INPUT);

frontend/realtime/src/components/ui/TextEditor/ButtonsAlignment.tsx

@@ -1,6 +1,6 @@
 import {
   faAlignCenter,
-  faAlignJustify,
+  // faAlignJustify,
   faAlignLeft,
   faAlignRight,
 } from "@fortawesome/pro-solid-svg-icons";

frontend/realtime/src/components/ui/TextEditor/ColorPicker.tsx

@@ -1,21 +1,46 @@
-import { ChangeEvent, useState } from "react";
-import { twMerge } from "tailwind-merge";
-import { faFont, faHashtag } from "@fortawesome/pro-solid-svg-icons";
+import {
+  faFont,
+  faHashtag,
+  IconDefinition,
+} from "@fortawesome/pro-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
-import { HexColorPicker } from "react-colorful";
+import { ChangeEvent, useEffect, useState } from "react";
+import { HexAlphaColorPicker } from "react-colorful";
+import { twMerge } from "tailwind-merge";
 
+import { paperWrapperStyles } from "~/components/styles";
 import { Button } from "../Button";
 import { Input } from "../Input";
-import { paperWrapperStyles } from "~/components/styles";
 
 export const ColorPicker = ({
   color: prevColor,
   onChange,
+  faIcon = faFont,
+  borderStyle,
+  showBar = true,
+  anchor = "bottom",
+  anchorGap = 0,
+  fillBg = false,
+  staticIconColor,
+  streamChanges = false,
+  defaultOpen = false,
+  closeOnMouseLeave = false,
 }: {
   color: string;
   onChange: (newColor: string) => void;
+  faIcon?: IconDefinition;
+  borderStyle?: string;
+  showBar?: boolean;
+  anchor?: "top" | "right" | "bottom" | "left";
+  anchorGap?: number;
+  fillBg?: boolean;
+  staticIconColor?: string;
+  streamChanges?: boolean;
+  defaultOpen?: boolean;
+  closeOnMouseLeave?: boolean;
 }) => {
+  const [isOpen, setIsOpen] = useState(defaultOpen);
   const [{ currColor, textInput }, setStates] = useState<{
     currColor: string;
     textInput: string;
@@ -24,13 +49,18 @@ export const ColorPicker = ({
     textInput: prevColor.substring(1),
   });
 
-  const handleSelectColor = (newColor: string) => {
-    setStates({ currColor: newColor, textInput: newColor.substring(1) });
-  };
-  const isHexColorCode = (color: string) => {
-    const hexColorRegex = /^(?:[0-9a-fA-F]{3}){1,2}$/;
-    return hexColorRegex.test(color);
+  useEffect(() => {
+    setIsOpen(defaultOpen);
+  }, [defaultOpen]);
+
+  const handleMouseLeave = (close: () => void) => {
+    if (closeOnMouseLeave) {
+      onChange(currColor);
+      setIsOpen(false);
+      close();
+    }
   };
+
   const handleTextInput = (e: ChangeEvent<HTMLInputElement>) => {
     if (isHexColorCode(e.target.value)) {
       setStates({
@@ -44,55 +74,101 @@ export const ColorPicker = ({
       }));
     }
   };
+
+  const isHexColorCode = (color: string) => {
+    const hexColorRegex = /^(?:[0-9a-fA-F]{3}){1,2}$/;
+    return hexColorRegex.test(color);
+  };
+
   return (
     <Popover className="relative">
-      <PopoverButton className="flex size-10 flex-col items-center gap-1 rounded-md border p-2">
-        <FontAwesomeIcon icon={faFont} />
-        <span className="h-1 w-full" style={{ backgroundColor: prevColor }} />
-      </PopoverButton>
-      <PopoverPanel
-        anchor="bottom"
-        className={twMerge(
-          paperWrapperStyles,
-          "flex flex-col items-center gap-2",
-        )}
-      >
-        {({ close }) => {
-          return (
-            <>
-              <HexColorPicker color={currColor} onChange={handleSelectColor} />
-              <Input
-                style={{ width: "198px" }}
-                icon={faHashtag}
-                value={textInput}
-                onChange={handleTextInput}
+      {({ open, close }) => (
+        <>
+          <PopoverButton
+            className={twMerge(
+              "flex size-10 items-center gap-1 rounded-lg bg-ui-controls p-2",
+              borderStyle,
+              showBar ? "flex-col" : "justify-center",
+            )}
+            style={fillBg ? { backgroundColor: prevColor } : {}}
+            onMouseEnter={() => defaultOpen && setIsOpen(true)}
+          >
+            <FontAwesomeIcon
+              icon={faIcon}
+              color={staticIconColor ?? (fillBg ? undefined : prevColor)}
+            />
+            {showBar && (
+              <span
+                className="h-1 w-full"
+                style={{ backgroundColor: prevColor }}
               />
-              <div className="flex w-full justify-center gap-2">
-                <Button
-                  variant="secondary"
-                  onClick={() => {
+            )}
+          </PopoverButton>
+          {(open || isOpen) && (
+            <div onMouseLeave={() => handleMouseLeave(close)}>
+              <PopoverPanel
+                anchor={anchor}
+                className={twMerge(
+                  paperWrapperStyles,
+                  "flex flex-col items-center gap-2 overflow-hidden",
+                )}
+                style={
+                  {
+                    "--anchor-gap": `${anchorGap}px`,
+                  } as React.CSSProperties
+                }
+                static
+              >
+                <HexAlphaColorPicker
+                  className="overflow-hidden"
+                  color={currColor}
+                  onChange={(color) => {
                     setStates({
-                      currColor: prevColor,
-                      textInput: prevColor.substring(1),
+                      currColor: color,
+                      textInput: color.substring(1),
                     });
-                    close();
+                    if (streamChanges) {
+                      onChange(color);
+                    }
                   }}
-                >
-                  Cancel
-                </Button>
-                <Button
-                  onClick={() => {
-                    onChange(currColor);
-                    close();
-                  }}
-                >
-                  OK
-                </Button>
-              </div>
-            </>
-          );
-        }}
-      </PopoverPanel>
+                />
+                <Input
+                  style={{ width: "198px" }}
+                  icon={faHashtag}
+                  value={textInput}
+                  onChange={handleTextInput}
+                />
+                {!closeOnMouseLeave && (
+                  <div className="flex w-full justify-center gap-2">
+                    <Button
+                      variant="secondary"
+                      onClick={() => {
+                        setStates({
+                          currColor: prevColor,
+                          textInput: prevColor.substring(1),
+                        });
+                        setIsOpen(false);
+                        close();
+                      }}
+                    >
+                      Cancel
+                    </Button>
+                    <Button
+                      onClick={() => {
+                        onChange(currColor);
+                        setIsOpen(false);
+                        close();
+                      }}
+                    >
+                      OK
+                    </Button>
+                  </div>
+                )}
+              </PopoverPanel>
+            </div>
+          )}
+        </>
+      )}
     </Popover>
   );
 };

frontend/realtime/src/components/ui/TextEditor/TextEditor.tsx

@@ -151,7 +151,11 @@ export const TextEditor = ({
           onChangeTextDecoration={onChangeTextDecoration}
         />
       </div>
-      <Textarea {...unionedTextAreaProps} ref={textAreaRef} />
+      <Textarea
+        className="bg-white"
+        {...unionedTextAreaProps}
+        ref={textAreaRef}
+      />
     </div>
   );
 };

frontend/realtime/src/components/ui/TextEditor/styles.ts

@@ -1,4 +1,4 @@
 export const buttonGroupButton =
-  "size-10 rounded-none bg-white text-black hover:bg-gray-200 data-[selected=true]:text-primary-400 disabled:hover:bg-transparent disabled:text-gray-400";
+  "size-10 rounded-none bg-ui-controls text-white/80 hover:bg-white/30 data-[selected=true]:text-white data-[selected=true]:bg-primary/30 disabled:hover:bg-transparent disabled:text-gray-400 border border-ui-border";
 export const buttonGroupGrouper =
-  "flex divide-x overflow-hidden rounded-md border";
+  "flex divide-x overflow-hidden rounded-md border border-ui-border";

frontend/realtime/src/components/ui/ToolbarMain/EraseModeMenu.tsx

@@ -0,0 +1,102 @@
+import {
+  faFont,
+  faHashtag,
+  IconDefinition,
+} from "@fortawesome/pro-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
+import { ChangeEvent, useEffect, useState } from "react";
+import { HexColorPicker } from "react-colorful";
+import { twMerge } from "tailwind-merge";
+
+import { useSignals } from "@preact/signals-react/runtime";
+import { paperWrapperStyles } from "~/components/styles";
+import { BRUSH_MAX_SIZE, BRUSH_MIN_SIZE, paintBrushSize, setPaintBrushSize } from "~/signals/uiEvents/toolbarMain/paintMode";
+import { Button } from "../Button";
+import { Input } from "../Input";
+import { Slider } from "../Slider";
+import { eraseBrushSize, setEraseBrushSize } from "~/signals/uiEvents/toolbarMain/eraseMode";
+
+export const EraseModeMenu = ({
+  faIcon = faFont,
+  borderStyle,
+  anchor = "bottom",
+  anchorGap = 0,
+  defaultOpen = false,
+  closeOnMouseLeave = false,
+}: {
+  faIcon?: IconDefinition;
+  borderStyle?: string;
+  anchor?: "top" | "right" | "bottom" | "left";
+  anchorGap?: number;
+  defaultOpen?: boolean;
+  closeOnMouseLeave?: boolean;
+}) => {
+  const [isOpen, setIsOpen] = useState(defaultOpen);
+
+  useEffect(() => {
+    setIsOpen(defaultOpen);
+  }, [defaultOpen]);
+
+  const handleMouseLeave = (close: () => void) => {
+    if (closeOnMouseLeave) {
+      setIsOpen(false);
+      close();
+    }
+  };
+
+  // Rerender the component when these signals change:
+  useSignals();
+  const brushSize = eraseBrushSize.value;
+
+  return (
+    <Popover className="relative">
+      {({ open, close }) => (
+        <>
+          <PopoverButton
+            className={twMerge(
+              "flex size-10 items-center justify-center gap-1 rounded-lg bg-ui-controls p-2",
+              borderStyle,
+            )}
+            onMouseEnter={() => defaultOpen && setIsOpen(true)}
+          >
+            <FontAwesomeIcon
+              icon={faIcon}
+            />
+          </PopoverButton>
+          {(open || isOpen) && (
+            <div onMouseLeave={() => handleMouseLeave(close)}>
+              <PopoverPanel
+                anchor={anchor}
+                className={twMerge(
+                  paperWrapperStyles,
+                  "flex flex-col items-center gap-2 overflow-hidden w-[198px]",
+                )}
+                style={
+                  {
+                    "--anchor-gap": `${anchorGap}px`,
+                  } as React.CSSProperties
+                }
+                static
+              >
+                <div className="flex flex-col w-full gap-2">
+                  <p className="text-white font-medium text-sm w-full justify-start">Eraser Size:</p>
+                  <Slider
+                    min={BRUSH_MIN_SIZE}
+                    max={BRUSH_MAX_SIZE}
+                    value={brushSize}
+                    onChange={setEraseBrushSize}
+                    step={1}
+                    innerLabel={brushSize.toString()}
+                    showDecrement
+                    showIncrement
+                  />
+                </div>
+              </PopoverPanel>
+            </div>
+          )}
+        </>
+      )}
+    </Popover>
+  );
+};

frontend/realtime/src/components/ui/ToolbarMain/PaintModeMenu.tsx

@@ -0,0 +1,194 @@
+import {
+  faFont,
+  faHashtag,
+  IconDefinition,
+} from "@fortawesome/pro-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
+import { ChangeEvent, useEffect, useState } from "react";
+import { HexAlphaColorPicker } from "react-colorful";
+import { twMerge } from "tailwind-merge";
+
+import { useSignals } from "@preact/signals-react/runtime";
+import { paperWrapperStyles } from "~/components/styles";
+import { BRUSH_MAX_SIZE, BRUSH_MIN_SIZE, paintBrushSize, setPaintBrushSize } from "~/signals/uiEvents/toolbarMain/paintMode";
+import { Button } from "../Button";
+import { Input } from "../Input";
+import { Slider } from "../Slider";
+
+export const PaintModeMenu = ({
+  color: prevColor,
+  onChange,
+  faIcon = faFont,
+  borderStyle,
+  showBar = true,
+  anchor = "bottom",
+  anchorGap = 0,
+  fillBg = false,
+  staticIconColor,
+  streamChanges = false,
+  defaultOpen = false,
+  closeOnMouseLeave = false,
+}: {
+  color: string;
+  onChange: (newColor: string) => void;
+  faIcon?: IconDefinition;
+  borderStyle?: string;
+  showBar?: boolean;
+  anchor?: "top" | "right" | "bottom" | "left";
+  anchorGap?: number;
+  fillBg?: boolean;
+  staticIconColor?: string;
+  streamChanges?: boolean;
+  defaultOpen?: boolean;
+  closeOnMouseLeave?: boolean;
+}) => {
+  const [isOpen, setIsOpen] = useState(defaultOpen);
+  const [{ currColor, textInput }, setStates] = useState<{
+    currColor: string;
+    textInput: string;
+  }>({
+    currColor: prevColor,
+    textInput: prevColor.substring(1),
+  });
+
+  useEffect(() => {
+    setIsOpen(defaultOpen);
+  }, [defaultOpen]);
+
+  const handleMouseLeave = (close: () => void) => {
+    if (closeOnMouseLeave) {
+      onChange(currColor);
+      setIsOpen(false);
+      close();
+    }
+  };
+
+  const handleTextInput = (e: ChangeEvent<HTMLInputElement>) => {
+    if (isHexColorCode(e.target.value)) {
+      setStates({
+        currColor: `#${e.target.value}`,
+        textInput: e.target.value,
+      });
+    } else {
+      setStates((curr) => ({
+        ...curr,
+        textInput: e.target.value,
+      }));
+    }
+  };
+
+  const isHexColorCode = (color: string) => {
+    const hexColorRegex = /^(?:[0-9a-fA-F]{3}){1,2}$/;
+    return hexColorRegex.test(color);
+  };
+
+  // Rerender the component when these signals change:
+  useSignals();
+  const brushSize = paintBrushSize.value;
+
+  return (
+    <Popover className="relative">
+      {({ open, close }) => (
+        <>
+          <PopoverButton
+            className={twMerge(
+              "flex size-10 items-center gap-1 rounded-lg bg-ui-controls p-2",
+              borderStyle,
+              showBar ? "flex-col" : "justify-center",
+            )}
+            style={fillBg ? { backgroundColor: prevColor } : {}}
+            onMouseEnter={() => defaultOpen && setIsOpen(true)}
+          >
+            <FontAwesomeIcon
+              icon={faIcon}
+              color={staticIconColor ?? (fillBg ? undefined : prevColor)}
+            />
+            {showBar && (
+              <span
+                className="h-1 w-full"
+                style={{ backgroundColor: prevColor }}
+              />
+            )}
+          </PopoverButton>
+          {(open || isOpen) && (
+            <div onMouseLeave={() => handleMouseLeave(close)}>
+              <PopoverPanel
+                anchor={anchor}
+                className={twMerge(
+                  paperWrapperStyles,
+                  "flex flex-col items-center gap-2 overflow-hidden",
+                )}
+                style={
+                  {
+                    "--anchor-gap": `${anchorGap}px`,
+                  } as React.CSSProperties
+                }
+                static
+              >
+                <HexAlphaColorPicker
+                  className="overflow-hidden"
+                  color={currColor}
+                  onChange={(color) => {
+                    setStates({
+                      currColor: color,
+                      textInput: color.substring(1),
+                    });
+                    if (streamChanges) {
+                      onChange(color);
+                    }
+                  }}
+                />
+                <Input
+                  style={{ width: "198px" }}
+                  icon={faHashtag}
+                  value={textInput}
+                  onChange={handleTextInput}
+                />
+                <div className="flex flex-col w-full mt-2 gap-2">
+                  <p className="text-white font-medium text-sm w-full justify-start">Brush Size:</p>
+                  <Slider
+                    min={BRUSH_MIN_SIZE}
+                    max={BRUSH_MAX_SIZE}
+                    value={brushSize}
+                    onChange={setPaintBrushSize}
+                    step={1}
+                    innerLabel={brushSize.toString()}
+                    showDecrement
+                    showIncrement
+                  />
+                </div>
+                {!streamChanges && (
+                  <div className="flex w-full justify-center gap-2">
+                    <Button
+                      variant="secondary"
+                      onClick={() => {
+                        setStates({
+                          currColor: prevColor,
+                          textInput: prevColor.substring(1),
+                        });
+                        setIsOpen(false);
+                        close();
+                      }}
+                    >
+                      Cancel
+                    </Button>
+                    <Button
+                      onClick={() => {
+                        onChange(currColor);
+                        setIsOpen(false);
+                        close();
+                      }}
+                    >
+                      OK
+                    </Button>
+                  </div>
+                )}
+              </PopoverPanel>
+            </div>
+          )}
+        </>
+      )}
+    </Popover>
+  );
+};

frontend/realtime/src/components/ui/Tooltip.tsx

@@ -7,15 +7,18 @@ export const Tooltip = ({
   tip,
   children,
   forceShow,
+  position = "top",
 }: {
   tip: string;
   children: ReactElement<any, string | JSXElementConstructor<any>>;
   forceShow?: boolean;
+  position?: "top" | "bottom" | "left" | "right";
 }) => {
   const clonedChildren = cloneElement(children, {
     "data-tooltip": tip,
     className: twMerge(
-      toolTipStyles,
+      toolTipStyles.base,
+      toolTipStyles[position],
       forceShow && "after:block before:block",
       children.props.className,
     ),

frontend/realtime/src/hooks/useIsMobile.ts

@@ -0,0 +1,17 @@
+import { useEffect, useState } from "react";
+
+export const useIsMobile = () => {
+  const [isMobile, setIsMobile] = useState(false);
+
+  useEffect(() => {
+    const checkMobile = () => {
+      return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
+        navigator.userAgent,
+      );
+    };
+
+    setIsMobile(checkMobile());
+  }, []);
+
+  return isMobile;
+};

frontend/realtime/src/signals/uiAccess/toolbarMain/toolbarMain.ts

@@ -1,5 +1,6 @@
 import { signal } from "@preact/signals-react";
 import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
+import { isToolbarMode } from "~/signals/uiEvents/toolbarMain/toolbarMain";
 
 interface ToolbarMainSignalInterface {
   disabled: boolean;
@@ -42,6 +43,17 @@ export const toolbarMain = {
       },
     };
   },
+
+  setMode(mode: ToolbarMainButtonNames) {
+    if (!isToolbarMode(mode)) {
+      return;
+    }
+
+    this.changeButtonState(ToolbarMainButtonNames.SELECT, { active: false });
+    this.changeButtonState(ToolbarMainButtonNames.ERASER, { active: false });
+    this.changeButtonState(ToolbarMainButtonNames.PAINT, { active: false });
+    this.changeButtonState(mode, { active: true });
+  }
 };
 
 function initButtonStates() {

frontend/realtime/src/signals/uiAccess/toolbarNode.ts

@@ -10,12 +10,13 @@ type ButtonStates = {
   };
 };
 export interface ContextualToolbarProps extends ContextualUi {
-  knodeIds: number[];
+  knodeIds: string[];
   disabled: boolean;
   downloadUrl?: string;
   locked: boolean | "unknown";
   lockDisabled: boolean;
   buttonStates: ButtonStates;
+  color: string;
 }
 interface PartialContextualToolbarProps
   extends Partial<Omit<ContextualToolbarProps, "buttonStates">> {
@@ -32,6 +33,7 @@ const toolbarNodeSignal = signal<ContextualToolbarProps>({
   locked: false,
   lockDisabled: false,
   buttonStates: initButtonStates(),
+  color: "#000000",
 });
 
 export const toolbarNode = {
@@ -77,6 +79,11 @@ export const toolbarNode = {
           hidden: false,
           active: buttonStates[ButtonNames.TRANSFORM].active,
         },
+        [ButtonNames.COLOR]: {
+          disabled: locked,
+          hidden: false,
+          active: buttonStates[ButtonNames.COLOR].active,
+        },
       },
       locked: locked,
     };
@@ -135,6 +142,12 @@ export const toolbarNode = {
       },
     };
   },
+  setColor(color: string) {
+    toolbarNodeSignal.value = {
+      ...toolbarNodeSignal.value,
+      color,
+    }
+  },
   batchButtonStates({ buttonStates }: { buttonStates: Partial<ButtonStates> }) {
     const prevButtonStates = toolbarNodeSignal.value.buttonStates;
     toolbarNodeSignal.value = {

frontend/realtime/src/signals/uiEvents/addMedia.ts

@@ -1,11 +1,12 @@
 import { signal, effect } from "@preact/signals-react";
-import { TextNodeData } from "~/KonvaApp/types";
+import { ShapeNodeData, TextNodeData } from "~/KonvaApp/types";
 
 // ADDING IMAGES
 const stagedImage = signal<File | null>(null);
 const addImageToEngine = (image: File) => {
   stagedImage.value = image;
 };
+
 const onGetStagedImage = (callback: (file: File) => void) => {
   effect(() => {
     if (stagedImage.value) {
@@ -48,15 +49,30 @@ const onAddTextToEngine = (callback: (data: TextNodeData) => void) => {
   });
 };
 
+// Add shapes
+const stagedShape = signal<ShapeNodeData | null>(null);
+const addShapeToEngine = (data: ShapeNodeData) => {
+  stagedShape.value = data;
+}
+const onAddShapeToEngine = (callback: (data: ShapeNodeData) => void) => {
+  effect(() => {
+    if (stagedShape.value) {
+      callback(stagedShape.value);
+    }
+  });
+};
+
 //EXPORTS
 export const dispatchers = {
   addImageToEngine,
   addVideoToEngine,
   addTextToEngine,
+  addShapeToEngine,
 };
 
 export const events = {
   onGetStagedImage,
   onGetStagedVideo,
   onAddTextToEngine,
+  onAddShapeToEngine,
 };

frontend/realtime/src/signals/uiEvents/index.ts

@@ -24,6 +24,10 @@ import {
   dispatchers as toolbarVideoExtractionDispatchers,
   eventsHandlers as toolbarVideoExtractionEvents,
 } from "./toolbarVideoExtraction";
+import {
+  dispatchers as promptDispatchers,
+  events as promptEvents,
+} from "./promptSettings";
 
 export const uiEvents = {
   ...addMediaEvents,
@@ -34,6 +38,7 @@ export const uiEvents = {
   toolbarMain: toolbarMainEvents,
   toolbarNode: toolbarNodeEvents,
   toolbarVideoExtraction: toolbarVideoExtractionEvents,
+  promptEvents: promptEvents,
 };
 export const dispatchUiEvents = {
   ...addMediaDispatchers,
@@ -44,4 +49,5 @@ export const dispatchUiEvents = {
   toolbarMain: toolbarMainDispatchers,
   toolbarNode: toolbarNodeDispatchers,
   toolbarVideoExtraction: toolbarVideoExtractionDispatchers,
+  promptSettings: promptDispatchers,
 };

frontend/realtime/src/signals/uiEvents/promptSettings.ts

@@ -0,0 +1,44 @@
+import { effect, signal } from "@preact/signals-react";
+
+// Prompt settings
+// - Prompt text
+// - Prompt strength
+export const DEFAULT_PROMPT_TEXT: string = "";
+export const promptText = signal<string>(DEFAULT_PROMPT_TEXT);
+const setPrompt = (data: string) => {
+  promptText.value = data;
+};
+
+const onPromptTextChanged = (callback: (data: string) => void) => {
+  effect(() => {
+    if (promptText.value) {
+      callback(promptText.value);
+    }
+  });
+};
+
+export const DEFAULT_PROMPT_STRENGTH = 100;
+export const promptStrength = signal<number>(DEFAULT_PROMPT_STRENGTH);
+const setPromptStrength = (data: number) => {
+  // Round to integer, then clamp between 0 and 100
+  promptStrength.value = Math.max(0, Math.min(100, Math.round(data)));
+};
+
+const onPromptStrengthChanged = (callback: (data: number) => void) => {
+  effect(() => {
+    if (promptStrength.value) {
+      callback(promptStrength.value);
+    }
+  });
+};
+
+//EXPORTS
+export const dispatchers = {
+  setPrompt,
+  setPromptStrength,
+};
+
+export const events = {
+  onPromptTextChanged,
+  onPromptStrengthChanged,
+};