pull configs and ml deps out of tauri app

seed is static
testing
2026-10-24 10:39:28 -04:00 · 2025-03-24 20:17:28 -04:00 · 2025-03-24 18:48:12 -04:00 · 2025-03-24 18:34:59 -04:00 · 2025-03-24 00:23:03 -04:00 · 2025-03-24 00:09:00 -04:00
70 changed files with 1593 additions and 868 deletions
@@ -66,3 +66,7 @@ scraps/
 # These should be built by the platform
 windows/sqlite/sqlite3.exp
 windows/sqlite/sqlite3.lib
+
+# ArtCraft Windows DLLs
+crates/desktop/src-tauri/*.dll
+windows_dlls/
@@ -658,23 +658,22 @@ name = "app"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "assertor",
 "base64 0.22.1",
 "bytes 1.10.0",
- "candle-core",
- "candle-nn",
- "candle-transformers",
- "cudarc",
+ "const_format",
 "directories 6.0.0",
 "dirs 6.0.0",
 "filesys",
 "futures 0.3.29",
 "half",
- "hf-hub 0.3.2",
 "image",
 "libloading 0.8.6",
 "log",
 "memory_caching",
+ "ml_models",
 "once_cell",
+ "primitives",
 "rand 0.8.5",
 "reqwest 0.12.12",
 "serde",
@@ -686,7 +685,6 @@ dependencies = [
 "tauri-plugin-log",
 "tempdir",
 "tempfile",
- "tokenizers 0.15.2",
 "tokio",
 "tokio-util",
 ]
@@ -855,6 +853,15 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "10d2119f741b79fe9907f5396d19bffcb46568cfcc315e78677d731972ac7085"

+[[package]]
+name = "assertor"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b02f2af4042fbbed4d5732c517666629512c574c8e2edb5cfe2cf8a86564d52"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "async-channel"
 version = "1.9.0"
@@ -1869,6 +1876,18 @@ dependencies = [
 "toml 0.8.14",
 ]

+[[package]]
+name = "carrot_utils"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f34277d609ff503187aa8e8604913b103e2eb8f893222a4948748f22527252e"
+dependencies = [
+ "chrono",
+ "colored",
+ "log",
+ "simplelog",
+]
+
 [[package]]
 name = "cast"
 version = "0.3.0"
@@ -2046,9 +2065,9 @@ dependencies = [

 [[package]]
 name = "clap"
-version = "4.4.8"
+version = "4.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64"
+checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
 dependencies = [
 "clap_builder",
 "clap_derive",
@@ -2056,23 +2075,23 @@ dependencies = [

 [[package]]
 name = "clap_builder"
-version = "4.4.8"
+version = "4.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc"
+checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
 dependencies = [
 "anstream",
 "anstyle",
 "clap_lex",
- "strsim 0.10.0",
+ "strsim 0.11.1",
 ]

 [[package]]
 name = "clap_derive"
-version = "4.4.7"
+version = "4.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.87",
@@ -2080,9 +2099,9 @@ dependencies = [

 [[package]]
 name = "clap_lex"
-version = "0.6.0"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"

 [[package]]
 name = "cloud_storage"
@@ -2175,6 +2194,16 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"

+[[package]]
+name = "colored"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
+dependencies = [
+ "lazy_static",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "combine"
 version = "3.8.1"
@@ -2300,6 +2329,26 @@ dependencies = [
 "tiny-keccak",
 ]

+[[package]]
+name = "const_format"
+version = "0.2.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd"
+dependencies = [
+ "const_format_proc_macros",
+]
+
+[[package]]
+name = "const_format_proc_macros"
+version = "0.2.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -6747,6 +6796,49 @@ dependencies = [
 "ws2_32-sys",
 ]

+[[package]]
+name = "ml_models"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "assertor",
+ "base64 0.22.1",
+ "bytes 1.10.0",
+ "candle-core",
+ "candle-nn",
+ "candle-transformers",
+ "const_format",
+ "cudarc",
+ "directories 6.0.0",
+ "dirs 6.0.0",
+ "filesys",
+ "futures 0.3.29",
+ "half",
+ "hf-hub 0.3.2",
+ "image",
+ "libloading 0.8.6",
+ "log",
+ "memory_caching",
+ "ndarray",
+ "nvidia-checker",
+ "once_cell",
+ "ort",
+ "primitives",
+ "rand 0.8.5",
+ "reqwest 0.12.12",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "serial_test",
+ "tauri",
+ "tauri-plugin-log",
+ "tempdir",
+ "tempfile",
+ "tokenizers 0.15.2",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "mockall"
 version = "0.12.1"
@@ -7011,6 +7103,19 @@ dependencies = [
 "tempfile",
 ]

+[[package]]
+name = "ndarray"
+version = "0.15.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "rawpointer",
+]
+
 [[package]]
 name = "ndk"
 version = "0.8.0"
@@ -7288,6 +7393,23 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"

+[[package]]
+name = "nvidia-checker"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8b584e4b0be14d4e3f8b779d79477aa0f36658d7a535c42abc725013d0c8951"
+dependencies = [
+ "carrot_utils",
+ "chrono",
+ "clap",
+ "colored",
+ "dirs 5.0.1",
+ "log",
+ "serde",
+ "serde_derive",
+ "toml 0.8.14",
+]
+
 [[package]]
 name = "objc"
 version = "0.2.7"
@@ -7793,6 +7915,26 @@ dependencies = [
 "hashbrown 0.14.5",
 ]

+[[package]]
+name = "ort"
+version = "1.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "889dca4c98efa21b1ba54ddb2bde44fd4920d910f492b618351f839d8428d79d"
+dependencies = [
+ "flate2",
+ "half",
+ "lazy_static",
+ "libc",
+ "ndarray",
+ "tar",
+ "thiserror 1.0.63",
+ "tracing",
+ "ureq",
+ "vswhom",
+ "winapi 0.3.9",
+ "zip 0.6.6",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -8360,6 +8502,7 @@ dependencies = [
 name = "primitives"
 version = "0.0.1"
 dependencies = [
+ "assertor",
 "my-workspace-hack",
 "testing",
 ]
@@ -10382,6 +10525,17 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"

+[[package]]
+name = "simplelog"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0"
+dependencies = [
+ "log",
+ "termcolor",
+ "time",
+]
+
 [[package]]
 name = "siphasher"
 version = "0.3.11"
@@ -11912,6 +12066,15 @@ dependencies = [
 "safetensors",
 ]

+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "termtree"
 version = "0.4.1"
@@ -12573,9 +12736,9 @@ dependencies = [

 [[package]]
 name = "tracing-subscriber"
-version = "0.3.19"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
+checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
 dependencies = [
 "nu-ansi-term",
 "sharded-slab",
@@ -12884,6 +13047,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"

+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"
@@ -0,0 +1,78 @@
+[package]
+name = "ml_models"
+edition = "2021"
+version = "0.0.1"
+authors = ["storyteller"]
+publish = false
+description = ""
+license = ""
+repository = ""
+
+[lib]
+name = "ml_models"
+path = "src/lib.rs"
+crate-type = ["staticlib", "cdylib", "rlib"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["my-workspace-hack"]
+
+[dependencies]
+
+# Internal
+filesys = { path = "../../lib/files/filesys" }
+memory_caching = { path = "../../lib/caching/memory_caching" }
+primitives = { path = "../../lib/data/primitives" }
+
+# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
+cudarc = "0.13.7"
+
+# External
+anyhow = "1.0"
+base64 = "0.22.1"
+bytes = "1.10"
+candle-core = { path = "../../vendor/candle/candle-core", features = ["cuda"] }
+candle-nn = { path = "../../vendor/candle/candle-nn", features = ["cuda"] }
+candle-transformers = { path = "../../vendor/candle/candle-transformers", features = ["cuda"] }
+const_format = "0.2.34"
+directories = "6.0.0"
+dirs = "6.0.0"
+futures = "0.3"
+half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
+hf-hub = "0.3.2"
+image = "0.25.5"
+log = "0.4"
+ndarray = "0.15.6"
+nvidia-checker = "0.1.2"
+once_cell = "1.20.3"
+ort = { version = "1.16.2", features = ["cuda"] }
+rand = "0.8.5"
+# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
+reqwest = { version = "0.12.12", features = ["stream"], default-features = false }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9"
+tauri = { version = "2.2.4", features = [] }
+tauri-plugin-log = "2"
+tempdir = "0.3.7"
+tempfile = "3.18.0"
+tokenizers = "0.15.2"
+tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
+tokio-util = { version = "0.7", features = ["codec"] }
+
+# tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
+
+[features]
+default = ["cuda"]
+cuda = [
+    "candle-core/cuda",
+    "candle-nn/cuda",
+    "candle-transformers/cuda",
+    # NB(bt,2025-03-10): Building without Flash attention on Windows for now to bypass linker issues
+    "candle-transformers/flash-attn",
+    "candle-core/cudnn"
+]
+
+[dev-dependencies]
+libloading = "0.8.6"
+serial_test = "3.2.0"
+assertor = "0.0.3"
@@ -0,0 +1 @@
+pub mod ml;
@@ -0,0 +1,2 @@
+pub mod onnx_session;
+pub mod remove_image_background;
@@ -0,0 +1,19 @@
+//! Adapted from https://github.com/dnanhkhoa/rust-background-removal/blob/main/src/onnx.rs
+
+use std::path::Path;
+use ort::{Environment, ExecutionProvider, GraphOptimizationLevel, SessionBuilder};
+
+pub fn onnx_session(onnx_model_file: &Path) -> Result<ort::Session, anyhow::Error> {
+  let environment = Environment::default().into_arc();
+  let session = SessionBuilder::new(&environment)?
+    .with_optimization_level(GraphOptimizationLevel::Level1)? // Configure model optimization level
+    .with_intra_threads(1)? // Configure the number of threads used for inference
+    .with_execution_providers([
+      // Configure execution providers (e.g., CUDA, CoreML, CPU)
+      ExecutionProvider::CUDA(Default::default()),
+      ExecutionProvider::CoreML(Default::default()),
+      ExecutionProvider::CPU(Default::default()),
+    ])?
+    .with_model_from_file(onnx_model_file)?;
+  Ok(session)
+}
@@ -0,0 +1,61 @@
+use crate::ml::background_removal::onnx_session::onnx_session;
+use image::{imageops, DynamicImage};
+use ndarray::{Array, CowArray};
+use ort::Value;
+use std::path::Path;
+
+pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: DynamicImage) -> anyhow::Result<DynamicImage> {
+  // NB: Real time needed cuDNN 9, but this uses cuDNN 8:
+  // sudo apt-get install libcudnn8-dev libcudnn8
+  // update-alternatives: warning: forcing reinstallation of alternative /usr/include/x86_64-linux-gnu/cudnn_v9.h because link group libcudnn is broken
+  // update-alternatives: using /usr/include/x86_64-linux-gnu/cudnn_v8.h to provide /usr/include/cudnn.h (libcudnn) in manual mode
+  let session = onnx_session(model_path.as_ref())?;
+  let image = process_dynamic_image(&session, image)?;
+  Ok(image)
+}
+
+fn process_dynamic_image(
+  session: &ort::Session,
+  dynamic_img: DynamicImage,
+) -> Result<DynamicImage, anyhow::Error> {
+  let input_shape = session.inputs[0]
+    .dimensions()
+    .map(|dim| dim.unwrap())
+    .collect::<Vec<usize>>();
+  let input_img = dynamic_img.into_rgba8();
+  let scaling_factor = f32::min(
+    1., // Avoid upscaling
+    f32::min(
+      input_shape[3] as f32 / input_img.width() as f32, // Width ratio
+      input_shape[2] as f32 / input_img.height() as f32, // Height ratio
+    ),
+  );
+  let mut resized_img = imageops::resize(
+    &input_img,
+    input_shape[3] as u32,
+    input_shape[2] as u32,
+    imageops::FilterType::Triangle,
+  );
+  let input_tensor = CowArray::from(
+    Array::from_shape_fn(input_shape, |indices| {
+      let mean = 128.;
+      let std = 256.;
+
+      (resized_img[(indices[3] as u32, indices[2] as u32)][indices[1]] as f32 - mean) / std
+    })
+      .into_dyn(),
+  );
+  let inputs = vec![Value::from_array(session.allocator(), &input_tensor)?];
+  let outputs = session.run(inputs)?;
+  let output_tensor = outputs[0].try_extract::<f32>()?;
+  for (indices, alpha) in output_tensor.view().indexed_iter() {
+    resized_img[(indices[3] as u32, indices[2] as u32)][3] = (alpha * 255.) as u8;
+  }
+  let output_img = imageops::resize(
+    &resized_img,
+    (input_img.width() as f32 * scaling_factor) as u32,
+    (input_img.height() as f32 * scaling_factor) as u32,
+    imageops::FilterType::Triangle,
+  );
+  Ok(DynamicImage::ImageRgba8(output_img))
+}
@@ -1,8 +1,8 @@
+pub mod background_removal;
 pub mod image;
 pub mod model_cache;
+pub mod model_config;
 pub mod model_file;
-pub mod models;
 pub mod prompt_cache;
 pub mod stable_diffusion;
-pub mod model_type;
-pub mod downloads;
+pub mod weights_registry;
@@ -1,15 +1,14 @@
-use crate::ml::models::unet_model::UNetModel;
 use anyhow::anyhow;
+use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
 use candle_transformers::models::stable_diffusion::vae::AutoEncoderKL;
 use std::sync::{Arc, RwLock};
-
 // TODO: This data structure is gross. Generalize the locking and loading semantics for reuse.

 // Simple registry for now. We can build complex machinery that aids in 
 // VRAM utilization, disk space saving, intelligent scheduling, etc. in the future.
 pub struct ModelCache {
  vae: RwLock<Option<Arc<AutoEncoderKL>>>,
-  unet2: RwLock<Option<Arc<UNetModel>>>,
+  unet2: RwLock<Option<Arc<UNet2DConditionModel>>>,
 }

 impl ModelCache {
@@ -39,7 +38,7 @@ impl ModelCache {
    }
  }

-  pub fn set_unet(&self, vae: Arc<UNetModel>) -> anyhow::Result<()> {
+  pub fn set_unet(&self, vae: Arc<UNet2DConditionModel>) -> anyhow::Result<()> {
    match self.unet2.write() {
      Err(err) => Err(anyhow!("{}", err)),
      Ok(mut lock) => {
@@ -49,7 +48,7 @@ impl ModelCache {
    }
  }

-  pub fn get_unet(&self) -> anyhow::Result<Option<Arc<UNetModel>>> {
+  pub fn get_unet(&self) -> anyhow::Result<Option<Arc<UNet2DConditionModel>>> {
    match self.unet2.read() {
      Err(err) => Err(anyhow!("{}", err)),
      Ok(ref lock) => {
@@ -0,0 +1,61 @@
+use crate::ml::model_file::StableDiffusionVersion;
+use candle_core::{DType, Device};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
+use hf_hub::api::sync::Api;
+use log::info;
+
+const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
+const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;
+const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
+const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::V1_5;
+
+pub struct ModelConfig {
+
+  /// We only support one device type currently, and we can't multiplex
+  /// over several devices yet.
+  pub device: Device,
+
+  /// The core numeric type to use for models.
+  pub dtype: DType,
+
+  pub sd_version: StableDiffusionVersion,
+  pub sd_config: StableDiffusionConfig,
+
+  /// Probably shouldn't be here.
+  pub hf_api: Api,
+}
+
+impl ModelConfig {
+  pub fn init() -> anyhow::Result<Self> {
+    let device = Device::new_cuda(0)
+      .unwrap_or_else(|e| {
+          info!("CUDA not available ({}), falling back to CPU", e);
+          Device::Cpu
+        });
+
+    let sd_config = match SD_VERSION {
+      StableDiffusionVersion::Turbo => {
+        StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+      StableDiffusionVersion::V1_5 | StableDiffusionVersion::V1_5Inpaint => {
+        StableDiffusionConfig::v1_5(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+      StableDiffusionVersion::V2_1 | StableDiffusionVersion::V2Inpaint => {
+        StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+      StableDiffusionVersion::Xl | StableDiffusionVersion::XlInpaint => {
+        StableDiffusionConfig::sdxl(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
+      }
+    };
+
+    let hf_api = Api::new()?;
+
+    Ok(Self {
+      device,
+      dtype: DEFAULT_SD_NUMERIC_DATATYPE,
+      sd_version: SD_VERSION,
+      sd_config,
+      hf_api,
+    })
+  }
+}
@@ -3,19 +3,51 @@ extern crate accelerate_src;
 #[cfg(feature = "mkl")]
 extern crate intel_mkl_src;

-use std::path::PathBuf;
 use candle_transformers::models::stable_diffusion;
+use std::path::{Path, PathBuf};

 use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
+use crate::ml::weights_registry::weights::{CLIP_JSON, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2};
 use anyhow::Error as E;
 use candle_core::{DType, Device, Module, Tensor, D};
 use log::info;
 use tokenizers::Tokenizer;
-use crate::ml::model_type::ModelType;
-use crate::state::app_dir::AppWeightsDir;
+
+
+pub struct ClipArgs<'a, P: AsRef<Path>,  Q: AsRef<Path>> {
+  pub prompt: &'a str, 
+  pub uncond_prompt: &'a str, 
+  pub tokenizer: Option<String>, 
+  pub clip_weights: Option<String>, 
+  pub clip2_weights: Option<String>, 
+  pub sd_version: StableDiffusionVersion, 
+  pub sd_config: &'a stable_diffusion::StableDiffusionConfig, 
+  pub use_f16: bool, 
+  pub device: &'a Device, 
+  pub dtype: DType, 
+  pub use_guide_scale: bool, 
+  pub clip_json_path: P,
+  pub clip_weights_path: Q,
+}

 #[allow(clippy::too_many_arguments)]
-pub fn infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: Option<String>, clip_weights: Option<String>, clip2_weights: Option<String>, sd_version: StableDiffusionVersion, sd_config: &stable_diffusion::StableDiffusionConfig, use_f16: bool, device: &Device, dtype: DType, use_guide_scale: bool, weights_dir: &AppWeightsDir) -> anyhow::Result<Tensor> {
+pub fn infer_clip_text_embeddings<P: AsRef<Path>, Q: AsRef<Path>>(args: ClipArgs<'_, P, Q>) -> anyhow::Result<Tensor> {
+  let ClipArgs {
+    prompt, 
+    uncond_prompt, 
+    tokenizer, 
+    clip_weights, 
+    clip2_weights, 
+    sd_version, 
+    sd_config, 
+    use_f16, 
+    device, 
+    dtype, 
+    use_guide_scale, 
+    clip_json_path,
+    clip_weights_path,
+  } = args;
+
  info!("Preparing text embeddings... for {:?}", sd_version);

  let which = match sd_version {
@@ -39,7 +71,8 @@ pub fn infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer:
        dtype,
        use_guide_scale,
        *first,
-        weights_dir,
+        clip_json_path.as_ref(),
+        clip_weights_path.as_ref(),
      )
    })
    .collect::<anyhow::Result<Vec<_>>>()?;
@@ -50,20 +83,25 @@ pub fn infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer:
 }

 #[allow(clippy::too_many_arguments)]
-fn do_infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: Option<String>, clip_weights: Option<String>, clip2_weights: Option<String>, sd_version: StableDiffusionVersion, sd_config: &stable_diffusion::StableDiffusionConfig, use_f16: bool, device: &Device, dtype: DType, use_guide_scale: bool, first: bool, weights_dir: &AppWeightsDir) -> anyhow::Result<Tensor> {
+fn do_infer_clip_text_embeddings(
+  prompt: &str, 
+  uncond_prompt: &str, 
+  tokenizer: Option<String>, 
+  clip_weights: Option<String>, 
+  clip2_weights: Option<String>, 
+  sd_version: StableDiffusionVersion, 
+  sd_config: &stable_diffusion::StableDiffusionConfig, 
+  use_f16: bool, 
+  device: &Device, 
+  dtype: DType, 
+  use_guide_scale: bool, 
+  first: bool, 
+  clip_json_path: &Path,
+  clip_weights_path: &Path,
+) -> anyhow::Result<Tensor> {
  info!("do_infer_clip_text_embeddings called with args {:?}", (prompt, uncond_prompt, tokenizer.clone().unwrap_or_else(|| "None".to_string()), clip_weights.clone().unwrap_or_else(|| "None".to_string()), clip2_weights.clone().unwrap_or_else(|| "None".to_string()), sd_version, sd_config, use_f16, device, dtype, use_guide_scale, first));

-  //let tokenizer_file = if first {
-  //  info!("ModelFile::Tokenizer");
-  //  ModelFile::Tokenizer
-  //} else {
-  //  info!("ModelFile::Tokenizer2");
-  //  ModelFile::Tokenizer2
-  //};
-
-  //let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
-
-  let tokenizer = weights_dir.model_path(&ModelType::ClipJson);
+  let tokenizer = clip_json_path;

  info!("Loading Clip Tokenizer path: {:?}", tokenizer);

@@ -85,14 +123,16 @@ fn do_infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: O
  let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;

  println!("Building the Clip transformer.");
-  let clip_weights_file = if first { ModelFile::Clip } else { ModelFile::Clip2 };
+  
+  // TODO(bt,2025-03-23): These used to be two different clip models. We should only need one.
  let clip_weights = if first {
-    //clip_weights_file.get(clip_weights, sd_version, use_f16)?
-    weights_dir.model_path(&ModelType::SdxlTurboClipEncoder)
+    println!(">>>>> CLIP 1");
+    clip_weights_path
  } else {
-    //clip_weights_file.get(clip2_weights, sd_version, use_f16)?
-    weights_dir.model_path(&ModelType::SdxlTurboClipEncoder2)
+    println!(">>>>> CLIP 2");
+    clip_weights_path
  };
+  
  let clip_config = if first { &sd_config.clip } else { sd_config.clip2.as_ref().unwrap() };
  let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;

@@ -1,72 +1,99 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};

-use crate::events::notification_event::{NotificationEvent, NotificationModelType};
 use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
 use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
 use crate::ml::model_cache::ModelCache;
+use crate::ml::model_config::ModelConfig;
 use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
-use crate::ml::model_type::ModelType;
-use crate::ml::models::unet_model::UNetModel;
 use crate::ml::prompt_cache::PromptCache;
 use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
-use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
-use crate::state::app_config::AppConfig;
-use crate::state::app_dir::AppDataRoot;
+use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
+use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
+use crate::ml::weights_registry::weights::{LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
 use anyhow::{anyhow, Error as E, Result};
-use candle_core::{DType, IndexOp, Tensor, D};
+use candle_core::{DType, Device, IndexOp, Tensor, D};
 use candle_nn::VarBuilder;
-use candle_transformers::models::stable_diffusion::lcm::LCMScheduler;
+use candle_transformers::models::stable_diffusion::lcm::{LCMScheduler, LCMSchedulerConfig};
 use candle_transformers::models::stable_diffusion::unet_2d::BlockConfig;
 use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
 use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModelConfig;
 use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
 use image::DynamicImage;
 use log::info;
 use rand::Rng;
-use tauri::{AppHandle, Emitter};

-pub struct Args<'a> {
+const DEFAULT_STRENGTH : f64 = 75.0;
+
+pub struct Args<'a, P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>> {
  pub image: &'a DynamicImage,
  pub prompt: String,
  pub uncond_prompt: String,
  pub cfg_scale: Option<f64>,
-  pub i2i_strength: Option<u8>,
-  pub configs: &'a AppConfig,
+  pub img2img_strength: Option<f64>,
  pub model_cache: &'a ModelCache,
  pub prompt_cache: &'a PromptCache,
-  pub app_data_root: &'a AppDataRoot,
-  pub app: &'a AppHandle,
  pub use_flash_attn: bool,
+  pub model_config: &'a ModelConfig,
+  pub maybe_seed: Option<u64>,
+  pub scheduler_steps: usize,
+  pub vae_path: P1,
+  pub unet_path: P2,
+  pub clip_json_path: P3,
+  pub clip_weights_path: P4,
 }

-pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
-  let Args { prompt, uncond_prompt, cfg_scale, i2i_strength, configs, model_cache, prompt_cache, app, image, app_data_root, use_flash_attn } = args;
+pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<RgbImage> 
+  where P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>
+{
+  let Args { 
+    prompt, 
+    uncond_prompt, 
+    cfg_scale,
+    img2img_strength, 
+    model_cache, 
+    prompt_cache, 
+    image, 
+    use_flash_attn ,
+    model_config: ModelConfig {
+      device,
+      dtype,
+      sd_version,
+      sd_config,
+      hf_api,
+    },
+    maybe_seed,
+    scheduler_steps,
+    vae_path,
+    unet_path,
+    clip_json_path,
+    clip_weights_path,
+  } = args;
+  
+  let dtype = *dtype;
+  let sd_version = *sd_version;

-  let weights_dir = app_data_root.weights_dir();
+  let img2img_strength = img2img_strength
+    .map(|s| remap_lcm_strength_range(s))
+    .unwrap_or(DEFAULT_STRENGTH);
+  
+  let img2img_strength = img2img_strength / 100.0;

-  let img2img_strength = args.i2i_strength.unwrap_or(75) as f64 / 100.0;
  let use_f16 = true;

-  //println!("Starting image generation with the following configuration:");
-  //println!("  Model: {:?}", configs.sd_version);
-  //println!("  Prompt: {}", prompt);
-  //println!("  Steps: {}", configs.scheduler_steps);
-  //println!("  Device: {:?}", configs.device);
-
-  //println!("Model dimensions: {}x{}", configs.sd_config.width, configs.sd_config.height);
-
  // Use LCM Scheduler instead of Euler Ancestral for better speed and quality
-  let mut scheduler = LCMScheduler::new(configs.scheduler_steps, img2img_strength, candle_transformers::models::stable_diffusion::lcm::LCMSchedulerConfig::default())?;
+  let mut scheduler = LCMScheduler::new(scheduler_steps, img2img_strength, LCMSchedulerConfig::default())?;

-  let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
-  configs.device.set_seed(seed)?;
+  let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());

-  //info!("Using seed: {}", seed);
+  let seed = 42;
+  
+  device.set_seed(seed)?;

  let guidance_scale = match cfg_scale {
    Some(guidance_scale) => guidance_scale,
-    None => match configs.sd_version {
+    None => match sd_version {
      StableDiffusionVersion::V1_5 | StableDiffusionVersion::V2_1 | StableDiffusionVersion::Xl => 7.5,
      StableDiffusionVersion::Turbo => 0.,
      _ => 0., // NB: Not sure what the other model families should use, so sticking with "0"
@@ -76,72 +103,50 @@ pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
  // let use_guide_scale = guidance_scale > 1.0;
  let use_guide_scale = false;

-  //info!("Using guide scale: {}", use_guide_scale);
-
-  //info!("Checking if prompt is cached");
  let maybe_cached = prompt_cache.get_copy(&prompt)?;

  let mut text_embeddings = if let Some(tensor) = maybe_cached {
    tensor
  } else {
-    info!("Loading clip weights...");
-    //println!("Loading clip weights for {:?} with model {:?}", configs.sd_version, configs.sd_version.repo());
-    //let api = configs.hf_api.clone();
-
-    //let clip_path = api.model(configs.sd_version.repo().to_string()).get(if use_f16 { "text_encoder/model.fp16.safetensors" } else { "text_encoder/model.safetensors" })?;
-
-    //println!(">>>> CLIP REPO = {:?}", configs.sd_version.repo());
-    //println!(">>>> CLIP PATH = {:?}", clip_path);
-    //println!(">>>> USE_FP16 = {:?}", use_f16);
-    //let clip_weights = Some(clip_path.to_string_lossy().to_string());
-
-    let clip_weights = weights_dir.model_path(&ModelType::LykonDreamshaper7TextEncoderFp16);
-    let clip_weights = Some(clip_weights.to_string_lossy().to_string());
-
    info!("Prompt is NOT cached! Calculating embedding...");
-    let tensor = infer_clip_text_embeddings(
-      &prompt,
-      &uncond_prompt,
-      None, // tokenizer
-      clip_weights,
-      None, // clip2_weights
-      configs.sd_version,
-      &configs.sd_config,
+    let tensor = infer_clip_text_embeddings(ClipArgs {
+      prompt: & prompt,
+      uncond_prompt: &uncond_prompt,
+      tokenizer: None, // tokenizer
+      clip_weights: None,
+      clip2_weights: None, // clip2_weights
+      sd_version,
+      sd_config: &sd_config,
      use_f16, // use_f16
-      &configs.device,
-      configs.dtype,
+      device,
+      dtype,
      use_guide_scale,
-      weights_dir,
-    )?;
+      clip_json_path,
+      clip_weights_path,
+    })?;
+
    prompt_cache.store_copy(&prompt, &tensor)?;
    tensor
  };

-  println!("Text embeddings shape: {:?}", text_embeddings.shape());
+  info!("Text embeddings shape: {:?}", text_embeddings.shape());

-  println!("Loading input image into tensor...");
+  info!("Loading input image into tensor...");

-  let input_image = dynamic_image_to_tensor(image, &configs.device, configs.dtype)?;
+  let input_image = dynamic_image_to_tensor(image, &device, dtype)?;

-  println!("Reference image shape: {:?}", input_image.shape());
+  info!("Reference image shape: {:?}", input_image.shape());

-  let vae_scale = get_vae_scale(configs.sd_version);
+  let vae_scale = get_vae_scale(sd_version);

  let maybe_vae = model_cache.get_vae()?;

  let vae = match maybe_vae {
    Some(vae) => vae,
    None => {
-      //let repo = configs.sd_version.repo();
-      //println!(">>>>> VAE REPO = {:?}", repo);
-      //println!("Building VAE model from : {:?} ... (3)", repo);
-      //let vae_file = configs.hf_api.model(repo.to_string()).get("vae/diffusion_pytorch_model.safetensors")?;
-
-      let vae_file = weights_dir.model_path(&ModelType::LykonDreamshaper7Vae);
-
-      println!("Building VAE model from file {:?}...", &vae_file);
-
-      let vae = configs.sd_config.build_vae(vae_file, &configs.device, configs.dtype)?;
+      info!("Building VAE model from file {:?}...", vae_path.as_ref());
+      
+      let vae = sd_config.build_vae(vae_path, &device, dtype)?;

      let vae = Arc::new(vae);

@@ -158,9 +163,7 @@ pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
    None => {
      info!("No unet found in cache; loading...");

-      let unet_weights = weights_dir.model_path(&ModelType::SimianLuoLcmDreamshaperV7Unet);
-
-      let in_channels = match configs.sd_version {
+      let in_channels = match sd_version {
        StableDiffusionVersion::XlInpaint | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::V1_5Inpaint => 9,
        _ => 4,
      };
@@ -195,12 +198,10 @@ pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
      };

      // Load the model directly from the safetensors file
-      let vs_unet = unsafe { VarBuilder::from_mmaped_safetensors(&[unet_weights], configs.dtype, &configs.device)? };
+      let vs_unet = unsafe { VarBuilder::from_mmaped_safetensors(&[unet_path], dtype, &device)? };

      let unet = UNet2DConditionModel::new(vs_unet, in_channels, 4, use_flash_attn, unet_config)?;

-      let unet = UNetModel::new_with_inner(unet);
-
      let unet = Arc::new(unet);

      model_cache.set_unet(unet.clone())?;
@@ -211,50 +212,39 @@ pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {

  let init_latent_dist: DiagonalGaussianDistribution = vae.encode(&input_image)?;

-  // Parse and calculate the img2img strength
-  let img2img_strength = match i2i_strength {
-    None => 0.75f64,
-    Some(strength) => (strength as f64) / 100.0f64,
-  };
-
  // Create LCM guidance scale embeddings
  let embedding_dim = 1280;
-  let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &configs.device, configs.dtype)?;
+  let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &device, dtype)?;

  // Generate latents from input image using the LCM approach
-  println!("Generating latents from input image...");
+  info!("Generating latents from input image...");
  let init_latents = (init_latent_dist.sample()? * vae_scale)?;
-  println!("Initial latents shape: {:?}", init_latents.shape());
+  info!("Initial latents shape: {:?}", init_latents.shape());

  // Calculate timesteps for LCM with proper img2img handling
-  let timesteps = LCMScheduler::get_timesteps_for_steps(configs.scheduler_steps, img2img_strength);
+  let timesteps = LCMScheduler::get_timesteps_for_steps(scheduler_steps, img2img_strength);
  let t_start = timesteps[0]; // First denoising step

  // Add noise at the appropriate timestep for img2img
-  println!("Adding noise to latents for t_start={}", t_start);
+  info!("Adding noise to latents for t_start={}", t_start);
  let noise = init_latents.randn_like(0f64, 1f64)?;
  let latents = scheduler.add_noise(&init_latents, noise, t_start)?;

-  println!("Latents initialized successfully");
-  println!("Initial noise shape: {:?}", latents.shape());
-
-  println!("Starting diffusion process...");
-
  let mut latents = latents;
  let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();

  for (timestep_index, &timestep) in timesteps.iter().enumerate() {
-    println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
+    info!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);

    let latent_model_input = if use_guide_scale { Tensor::cat(&[&latents, &latents], 0)? } else { latents.clone() };

    let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep);

    // Use the guidance scale embedding in the UNet inference with the proper method name
-    let mut noise_pred = match unet.forward_with_guidance(&latent_model_input, timestep as f64, &text_embeddings, &guidance_scale_embedding) {
+    let mut noise_pred = match unet.forward_with_guidance(&latent_model_input, timestep as f64, &text_embeddings, Some(&guidance_scale_embedding)) {
      Ok(pred) => pred,
      Err(e) => {
-        println!("UNet inference failed with error: {}", e);
+        info!("UNet inference failed with error: {}", e);
        return Err(anyhow::anyhow!("UNet inference failed: {}", e));
      },
    };
@@ -266,24 +256,24 @@ pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
    }

    // Apply the LCM scheduler step
-    latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, configs.scheduler_steps)?;
+    latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, scheduler_steps)?;
  }

-  println!("Diffusion process completed, decoding image...");
+  info!("Diffusion process completed, decoding image...");
  let image = vae.decode(&(latents / vae_scale)?)?;

-  println!("VAE decode completed");
+  info!("VAE decode completed");

-  println!("Scaling image back");
+  info!("Scaling image back");
  let image = ((image / 2.)? + 0.5)?;

-  println!("Normalized image values");
+  info!("Normalized image values");
  let image = (image.clamp(0f32, 1.)? * 255.)?;

-  println!("Convert to int8");
+  info!("Convert to int8");
  let image = image.to_dtype(DType::U8)?;

-  println!("Converted to 8-bit format");
+  info!("Converted to 8-bit format");

  let image = tensor_to_image_buffer(&image.i(0)?)?;

@@ -1,4 +1,5 @@
 pub mod get_vae_scale;
 pub mod infer_clip_text_embeddings;
 pub mod stable_diffusion_pipeline;
-pub mod lcm_pipeline;
+pub mod lcm_pipeline;
+mod remap_lcm_strength_range;
@@ -0,0 +1,54 @@
+use primitives::numerics::remap_range_f64::{remap_range_f64, Args};
+
+/// Our LCM implementation uses strength [13, 100]. Strength below 13 will take 8 steps, 
+/// so we need to limit ourselves to this range from an input range of [0, 100].
+pub fn remap_lcm_strength_range(value: f64) -> f64 {
+  remap_range_f64(Args {
+    old_min: 0.0,
+    old_max: 100.0,
+    new_min: 13.0,
+    new_max: 100.0,
+    value,
+  })
+}
+
+#[cfg(test)]
+mod test {
+  use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
+  use assertor::{assert_that, FloatAssertion};
+
+  #[test]
+  fn old_max() {
+    assert_eq!(remap_lcm_strength_range(100.0), 100.0);
+  }
+  
+  #[test]
+  fn old_min() {
+    assert_eq!(remap_lcm_strength_range(0.0), 13.0);
+  }
+
+  #[test]
+  fn under_bounds() {
+    assert_eq!(remap_lcm_strength_range(-100.0), 13.0);
+  }
+  
+  #[test]
+  fn over_bounds() {
+    assert_eq!(remap_lcm_strength_range(200.0), 100.0);
+  }
+
+  #[test]
+  fn other_values() {
+    assert_that!(remap_lcm_strength_range(0.0)).with_abs_tol(0.001).is_approx_equal_to(13.000);
+    assert_that!(remap_lcm_strength_range(5.0)).with_abs_tol(0.001).is_approx_equal_to(17.350);
+    assert_that!(remap_lcm_strength_range(11.0)).with_abs_tol(0.001).is_approx_equal_to(22.570);
+    assert_that!(remap_lcm_strength_range(13.0)).with_abs_tol(0.001).is_approx_equal_to(24.310);
+    assert_that!(remap_lcm_strength_range(14.0)).with_abs_tol(0.001).is_approx_equal_to(25.180);
+    assert_that!(remap_lcm_strength_range(20.0)).with_abs_tol(0.001).is_approx_equal_to(30.400);
+    assert_that!(remap_lcm_strength_range(25.0)).with_abs_tol(0.001).is_approx_equal_to(34.750);
+    assert_that!(remap_lcm_strength_range(50.0)).with_abs_tol(0.001).is_approx_equal_to(56.500);
+    assert_that!(remap_lcm_strength_range(75.0)).with_abs_tol(0.001).is_approx_equal_to(78.250);
+    assert_that!(remap_lcm_strength_range(99.0)).with_abs_tol(0.001).is_approx_equal_to(99.130);
+    assert_that!(remap_lcm_strength_range(100.0)).with_abs_tol(0.001).is_approx_equal_to(100.000);
+  }
+}
@@ -1,76 +1,86 @@
+use std::path::Path;
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};

-use crate::events::notification_event::{NotificationModelType, NotificationEvent};
 use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
 use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
 use crate::ml::model_cache::ModelCache;
 use crate::ml::model_file::StableDiffusionVersion;
-use crate::ml::model_type::ModelType;
-use crate::ml::models::unet_model::UNetModel;
 use crate::ml::prompt_cache::PromptCache;
 use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
-use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
-use crate::state::app_config::AppConfig;
+use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
 use anyhow::{anyhow, Error as E, Result};
-use candle_core::{DType, IndexOp, Tensor, D};
+use candle_core::{DType, Device, IndexOp, Tensor, D};
 use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
+use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
 use image::DynamicImage;
 use log::info;
 use rand::Rng;
-use tauri::{AppHandle, Emitter};
-use crate::state::app_dir::AppDataRoot;

-pub struct Args<'a> {
+pub struct Args<'a, P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>> {
    pub image: &'a DynamicImage,
    pub prompt: String,
    pub uncond_prompt: String,
    pub cfg_scale: Option<f64>,
    pub i2i_strength: Option<u8>,
-    pub configs: &'a AppConfig,
    pub model_cache: &'a ModelCache,
    pub prompt_cache: &'a PromptCache,
-    pub app_data_root: &'a AppDataRoot,
-    pub app: &'a AppHandle,
+    pub sd_version: StableDiffusionVersion,
+    pub sd_config: StableDiffusionConfig,
+    pub device: &'a Device,
+    pub dtype: DType,
+    pub maybe_seed: Option<u64>,
+    pub scheduler_steps: usize,
+    pub sdxl_turbo_vae_path: P,
+    pub sdxl_turbo_unet_path: Q,
+    pub clip_json_path: R,
+    pub clip_weights_path: S,
 }

-pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
+pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<RgbImage> 
+  where P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>
+{
    let Args { 
        prompt, 
        uncond_prompt, 
        cfg_scale, 
        i2i_strength, 
-        configs, 
        model_cache, 
        prompt_cache, 
-        app, 
        image,
-        app_data_root,
+        sd_version,
+        sd_config,
+        device,
+        dtype,
+        maybe_seed,
+        scheduler_steps,
+        sdxl_turbo_vae_path,
+        sdxl_turbo_unet_path,
+        clip_json_path,
+        clip_weights_path,
    } = args;
    
-    let weights_dir = app_data_root.weights_dir();
-    
    println!("Starting image generation with the following configuration:");
-    println!("  Model: {:?}", configs.sd_version);
+    println!("  Model: {:?}", sd_version);
    println!("  Prompt: {}", prompt);
-    println!("  Steps: {}", configs.scheduler_steps);
-    println!("  Device: {:?}", configs.device);
+    println!("  Steps: {}", scheduler_steps);
+    println!("  Device: {:?}", device);

-    println!("Model dimensions: {}x{}", configs.sd_config.width, configs.sd_config.height);
+    println!("Model dimensions: {}x{}", sd_config.width, sd_config.height);

    // TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
    //  This is a target for performance improvement
    //  See: https://github.com/huggingface/candle/issues/1331
-    let mut scheduler = configs.sd_config.build_scheduler(
-        configs.scheduler_steps)?;
+    let mut scheduler = sd_config.build_scheduler(scheduler_steps)?;

-    let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
-    configs.device.set_seed(seed)?;
+    let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
+    
+    device.set_seed(seed)?;

    info!("Using seed: {}", seed);

    let guidance_scale = match cfg_scale {
        Some(guidance_scale) => guidance_scale,
-        None => match configs.sd_version {
+        None => match sd_version {
            StableDiffusionVersion::V1_5
            | StableDiffusionVersion::V2_1
            | StableDiffusionVersion::Xl => 7.5,
@@ -91,20 +101,21 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
    } else {
        
        info!("Prompt is NOT cached! Calculating embedding...");
-        let tensor = infer_clip_text_embeddings(
-            &prompt,
-            &uncond_prompt,
-            None, // tokenizer
-            None, // clip_weights
-            None, // clip2_weights
-            configs.sd_version,
-            &configs.sd_config,
-            false, // use_f16
-            &configs.device,
-            configs.dtype,
+        let tensor = infer_clip_text_embeddings(ClipArgs {
+            prompt: &prompt,
+            uncond_prompt: &uncond_prompt,
+            tokenizer: None, // tokenizer
+            clip_weights: None, // clip_weights
+            clip2_weights: None, // clip2_weights
+            sd_version: sd_version,
+            sd_config: &sd_config,
+            use_f16: false, // use_f16
+            device: device,
+            dtype: dtype,
            use_guide_scale,
-            weights_dir,
-        )?;
+            clip_json_path: clip_json_path.as_ref(),
+            clip_weights_path: clip_weights_path.as_ref(),
+        })?;
        prompt_cache.store_copy(&prompt, &tensor)?;
        tensor
    };
@@ -113,14 +124,11 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {

    println!("Loading input image into tensor...");

-    let input_image = dynamic_image_to_tensor(
-        image,
-        &configs.device,
-        configs.dtype)?;
+    let input_image = dynamic_image_to_tensor(image, device, dtype)?;

    println!("Reference image shape: {:?}", input_image.shape());

-    let vae_scale = get_vae_scale(configs.sd_version);
+    let vae_scale = get_vae_scale(sd_version);

    let maybe_vae = model_cache.get_vae()?;
    
@@ -129,33 +137,14 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
        None => {
            info!("No vae found in cache; loading...");
            
-            let mut notify_download_complete = false;
-            //if !vae_file.exists() {
-            if true {
-                notify_download_complete = true;
-                app.emit("notification", NotificationEvent::ModelDownloadStarted {
-                    model_name: "sdxl-turbo-vae",
-                    model_type: NotificationModelType::Vae,
-                })?;
-            }
+            let vae_file = sdxl_turbo_vae_path.as_ref();
            
-            let vae_file = weights_dir.model_path(&ModelType::SdxlTurboVae);
-            
-            let vae = configs
-              .sd_config
-              .build_vae(vae_file, &configs.device, configs.dtype)?;
+            let vae = sd_config.build_vae(vae_file, &device, dtype)?;

            let vae = Arc::new(vae);
            
            model_cache.set_vae(vae.clone())?;
-            
-            if notify_download_complete {
-                app.emit("notification", NotificationEvent::ModelDownloadComplete {
-                    model_name: "sdxl-turbo-vae",
-                    model_type: NotificationModelType::Vae,
-                })?;
-            }
-            
+
            vae
        }
    };
@@ -167,32 +156,15 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
        None => {
            info!("No unet found in cache; loading...");

-            let mut notify_download_complete = false;
-            //if !unet_file.exists() {
-            if true {
-                notify_download_complete = true;
-                app.emit("notification", NotificationEvent::ModelDownloadStarted {
-                    model_name: "sdxl-turbo-unet",
-                    model_type: NotificationModelType::Unet,
-                })?;
-            }
-            
-            let unet_file = weights_dir.model_path(&ModelType::SdxlTurboUnet);
+            let unet_weights_file = sdxl_turbo_unet_path.as_ref();

-            let unet = UNetModel::new(&configs.sd_config, unet_file, &configs.device, configs.dtype)
+            let unet = sd_config.build_unet(unet_weights_file, &device, 4, true, dtype)
              .map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
-            
+
            let unet = Arc::new(unet);

            model_cache.set_unet(unet.clone())?;

-            if notify_download_complete {
-                app.emit("notification", NotificationEvent::ModelDownloadComplete {
-                    model_name: "sdxl-turbo-unet",
-                    model_type: NotificationModelType::Unet,
-                })?;
-            }
-
            unet
        }
    };
@@ -216,9 +188,9 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
    };

    let t_start = {
-        let start = configs.scheduler_steps - (configs.scheduler_steps as f64 * img2img_strength) as usize;
+        let start = scheduler_steps - (scheduler_steps as f64 * img2img_strength) as usize;

-        println!("Starting from step {} of {} (strength: {})", start, configs.scheduler_steps, img2img_strength);
+        println!("Starting from step {} of {} (strength: {})", start, scheduler_steps, img2img_strength);
        start
    };

@@ -257,7 +229,7 @@ pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {

        let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;

-        let mut noise_pred = match unet.inference(&latent_model_input, timestep as f64, &text_embeddings) {
+        let mut noise_pred = match unet.forward(&latent_model_input, timestep as f64, &text_embeddings) {
            Ok(pred) => pred,
            Err(e) => {
                println!("UNet inference failed with error: {}", e);
@@ -0,0 +1,3 @@
+mod weight_descriptor_builder;
+pub mod weight_descriptor;
+pub mod weights;
@@ -0,0 +1,45 @@
+
+// TODO(bt,2025-03-13): When it's time for dynamic, non-const allocation, refer to
+//  http::header::name::HeaderName: https://github.com/hyperium/http/blob/master/src/header/name.rs
+//  as a pattern that will support both compile-time constants and dynamically allocated String-based variants.
+
+#[derive(Clone, Copy)]
+pub enum WeightFunction {
+  ImageSegmentation,
+  TextEncoder,
+  TextTokenizer,
+  Unet,
+  Vae,
+}
+
+pub struct WeightDescriptor {
+  /// Name of the weight, eg. "Lykon Dreamshaper 7"
+  pub name: &'static str,
+
+  /// Name of the file on the filesystem, eg. lykon_dreamshaper_7_text_encoder.fp16.safetensors
+  pub filename: &'static str,
+
+  /// URL of the file in the R2 bucket,
+  /// eg. https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/lykon_dreamshaper_7_text_encoder.fp16.safetensors
+  pub r2_download_url: &'static str,
+
+  pub function: WeightFunction,
+}
+
+impl WeightFunction {
+  pub fn as_descriptive_str(&self) -> &'static str {
+    match self {
+      WeightFunction::ImageSegmentation => "image segmentation",
+      WeightFunction::TextEncoder => "text encoder",
+      WeightFunction::TextTokenizer => "text tokenizer",
+      WeightFunction::Unet => "UNET",
+      WeightFunction::Vae => "VAE",
+    }
+  }
+}
+
+impl WeightDescriptor {
+  pub fn to_descriptive_name(&self) -> String {
+    format!("{} ({})", self.name, self.function.as_descriptive_str())
+  }
+}
@@ -0,0 +1,17 @@
+pub (super) const R2_BUCKET_URL : &str = "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/";
+
+// TODO(bt,2025-03-13): Try to use keyword args from the call site.
+// NB: We're using a macro because we can't use a builder pattern and then concat string literals
+//  and variables (even if they're known to be &'static) and keep allocations static.
+macro_rules! weight {
+    ($name: literal, $file_name:literal, $weight_function:expr) => {
+      $crate::ml::weights_registry::weight_descriptor::WeightDescriptor {
+        name: $name,
+        filename: $file_name,
+        r2_download_url: const_format::concatcp!(crate::ml::weights_registry::weight_descriptor_builder::R2_BUCKET_URL, $file_name),
+        function: $weight_function,
+      }
+    };
+}
+
+pub(crate) use weight;
@@ -0,0 +1,65 @@
+use crate::ml::weights_registry::weight_descriptor::{WeightDescriptor, WeightFunction};
+use crate::ml::weights_registry::weight_descriptor_builder::weight;
+
+pub const CLIP_JSON : WeightDescriptor = weight!(
+  "CLIP",
+  "clip_vit_bigg_14_lion2b_39b_b160k.tokenizer.json",
+  WeightFunction::TextTokenizer
+);
+
+/// For LCM-based image generation
+pub const LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16 : WeightDescriptor = weight!(
+  "Lykon Dreamshaper 7",
+  "lykon_dreamshaper_7_text_encoder.fp16.safetensors",
+  WeightFunction::TextEncoder
+);
+
+/// For LCM-based image generation
+pub const LYKON_DEAMSHAPER_7_VAE : WeightDescriptor = weight!(
+  "Lykon Dreamshaper 7",
+  "lykon_dreamshaper_7_vae.safetensors",
+  WeightFunction::Vae
+);
+
+/// For LCM-based image generation
+pub const SIMIANLUO_LCM_DREAMSHAPER_V7_UNET : WeightDescriptor = weight!(
+  "SimianLuo LCM Dreamshaper v7",
+  "simianluo_lcm_dreamshaper_v7_unet.safetensors",
+  WeightFunction::Unet
+);
+
+/// Image segmentation and background removal
+/// dis_medium.onnx
+pub const DIS_MEDIUM_ONNX : WeightDescriptor = weight!(
+  "Dichotomous Image Segmentation Medium",
+  "dis_medium.onnx",
+  WeightFunction::ImageSegmentation
+);
+
+#[deprecated]
+pub const SDXL_TURBO_CLIP_TEXT_ENCODER : WeightDescriptor = weight!(
+  "SDXL Turbo CLIP",
+  "clip_text_encoder.safetensors",
+  WeightFunction::TextEncoder
+);
+
+#[deprecated]
+pub const SDXL_TURBO_CLIP_TEXT_ENCODER_2 : WeightDescriptor = weight!(
+  "SDXL Turbo CLIP #2",
+  "clip_text_encoder_2.safetensors",
+  WeightFunction::TextEncoder
+);
+
+#[deprecated]
+pub const SDXL_TURBO_VAE : WeightDescriptor = weight!(
+  "SDXL Turbo",
+  "diffusion_pytorch_model.vae.safetensors",
+  WeightFunction::Vae
+);
+
+#[deprecated]
+pub const SDXL_TURBO_UNET : WeightDescriptor = weight!(
+  "SDXL Turbo",
+  "diffusion_pytorch_model.unet.safetensors",
+  WeightFunction::Unet
+);
@@ -7,6 +7,7 @@ license = ""
 repository = ""
 edition = "2021"
 rust-version = "1.77.2"
+publish = false

 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

@@ -22,22 +23,22 @@ tauri-build = { version = "2.0.4", features = [] }
 # Internal
 filesys = { path = "../../lib/files/filesys" }
 memory_caching = { path = "../../lib/caching/memory_caching" }
+primitives = { path = "../../lib/data/primitives" }

-# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
-cudarc = "0.13.7"
+# Internal ML
+# TODO: Only on Windows and Linux
+ml_models = { path = "../ml_models" }

 # External
 #openssl = { version = "0.10", features = ["vendored"] }
 anyhow = "1.0"
 base64 = "0.22.1"
 bytes = "1.10"
-candle-core = { path = "../../vendor/candle/candle-core", features = ["cuda"] }
-candle-nn = { path = "../../vendor/candle/candle-nn", features = ["cuda"] }
-candle-transformers = { path = "../../vendor/candle/candle-transformers", features = ["cuda"] }
-dirs = "6.0.0"
+const_format = "0.2.34"
 directories = "6.0.0"
+dirs = "6.0.0"
 futures = "0.3"
-hf-hub = "0.3.2"
+half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
 image = "0.25.5"
 log = "0.4"
 once_cell = "1.20.3"
@@ -51,25 +52,15 @@ tauri = { version = "2.2.4", features = [] }
 tauri-plugin-log = "2"
 tempdir = "0.3.7"
 tempfile = "3.18.0"
-tokenizers = "0.15.2"
 tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
 tokio-util = { version = "0.7", features = ["codec"] }
-half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }

-[features]
-default = ["cuda"]
-cuda = [
-    "candle-core/cuda",
-    "candle-nn/cuda",
-    "candle-transformers/cuda",
-# NB(bt,2025-03-10): Building without Flash attention on Windows for now to bypass linker issues
-    "candle-transformers/flash-attn",
-    "candle-core/cudnn"
-]
+# tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }

 [dev-dependencies]
 libloading = "0.8.6"
 serial_test = "3.2.0"
+assertor = "0.0.3"

 [profile.dev]
 incremental = true # Compile your binary in smaller steps.
@@ -1,16 +1,12 @@
-use crate::ml::downloads::download_all_models::download_all_models;
 use crate::state::app_dir::AppDataRoot;
 use tauri::State;

 #[tauri::command]
 pub async fn download_models(
-  app_data_root: State<'_, AppDataRoot>,
+  _app_data_root: State<'_, AppDataRoot>,
 ) -> Result<String, String> {
-  
-  download_all_models(&app_data_root).await
-    .map_err(|err| {
-      "error downloading models".to_string()
-    })?;
-  
+
+  // NB: This endpoint has been removed for now as we now download models in the app lifecycle.
+
  Ok(("downloaded".to_string()))
 }
@@ -1,4 +1,6 @@
 pub mod download_models;
 pub mod flip_image;
-pub mod image_endpoint;
+pub mod realtime_image_endpoint;
+pub mod remove_background_endpoint;
 pub mod test_counter;
+pub mod text_to_image_endpoint;
@@ -1,27 +1,24 @@
-use crate::ml::downloads::download_all_models::download_all_models;
-use crate::ml::model_cache::ModelCache;
-use crate::ml::prompt_cache::PromptCache;
-use crate::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
 use crate::state::app_config::AppConfig;
-use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
-use bytes::BytesMut;
+use crate::state::app_dir::AppDataRoot;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use crate::utils::image::encode_rgb_image_base64_png::encode_rgb_image_base64_png;
 use image::imageops::FilterType;
-use image::{DynamicImage, ImageFormat, ImageReader};
+use image::{DynamicImage, RgbImage};
 use log::{error, info};
-use std::io::Cursor;
+use ml_models::ml::model_cache::ModelCache;
+use ml_models::ml::prompt_cache::PromptCache;
+use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
+use ml_models::ml::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
 use std::path::PathBuf;
 use tauri::{AppHandle, State};
-use crate::state::app_dir::AppDataRoot;

 const PROMPT_FILENAME : &str = "prompt.txt";
-const PROMPT: &str = "green goblin, blood dripping from lips, detailed, photorealistic, 8k";
+const PROMPT: &str = "shiba inu, cute dog, detailed, walking in a wooded forest, photorealistic, 8k";

 const NEGATIVE_PROMPT: &str = "bad quality, bad faces, poor quality, blurry faces, watermark";

 const RANDOM_SEED: u32 = 42;

-// TODO: Mark async https://github.com/tauri-apps/tauri/discussions/7737
-

 /// This handler takes an image (as a base64 encoded string) and a prompt and returns
 /// an image (as a base64-encoded string).
@@ -29,28 +26,24 @@ const RANDOM_SEED: u32 = 42;
 pub async fn infer_image(
  image: &str,
  prompt: Option<String>,
-  strength: Option<u8>,
+  strength: Option<f64>,
  model_config: State<'_, AppConfig>,
  model_cache: State<'_, ModelCache>,
  prompt_cache: State<'_, PromptCache>,
  app_data_root: State<'_, AppDataRoot>,
  app: AppHandle,
 ) -> Result<String, String> {
+  info!("infer_image endpoint called.");

  let prompt = get_prompt_or_fallback(prompt);

-  info!("Prompt: {}", prompt);
-  info!("Strength: {:?}", strength);
+  info!("Strength: {:?}; Prompt: {}", strength, prompt);

-  let image = hydrate_base64_image(image)
+  let image = decode_base64_image(image)
    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
-
-  // TODO(bt): Move this to another endpoint
-  download_all_models(&app_data_root).await
-    .map_err(|err| {
-      error!("couldn't download models: {:?}", err);
-      "Couldn't download models".to_string()
-    })?;
+  
+  // TODO(bt,2025-02-17): Running out of vram with full image buffer size
+  let image = image.resize(512, 512, FilterType::CatmullRom);

  let result = do_infer_image(
    &prompt, 
@@ -59,55 +52,64 @@ pub async fn infer_image(
    &model_config, 
    &model_cache, 
    prompt_cache, 
-    app, 
    &app_data_root
  ).await;
  
-  if let Err(err) = result.as_deref() {
-    error!("There was an error: {:?}", err);
-  }
-  
-  result
+  let image = match result {
+    Ok(image) => image,
+    Err(err) => {
+      error!("There was an error generating the image: {:?}", err);
+      return Err(format!("There was an error generating the image: {}", err));
+    }
+  };
+
+  let bytes = encode_rgb_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  info!("Inference successful; image converted to base64, serving back to browser...");
+
+  Ok(bytes)
 }

 async fn do_infer_image(
  prompt: &str,
  image: DynamicImage,
-  strength: Option<u8>,
+  strength: Option<f64>,
  config: &AppConfig,
  model_cache: &ModelCache,
  prompt_cache: State<'_, PromptCache>,
-  app: AppHandle,
  app_data_root: &AppDataRoot,
-) -> Result<String, String> {
+) -> Result<RgbImage, String> {
+  
+  let weights_dir = app_data_root.weights_dir();
+  let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
+  let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
+  let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
+  //let clip_weights_path= weights_dir.weight_path(&LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16);
+  let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);

  let args = Args {
    image: &image,
    prompt: prompt.to_string(),
    uncond_prompt: NEGATIVE_PROMPT.to_string(),
    model_cache,
-    configs: config,
    prompt_cache: &prompt_cache,
-    i2i_strength: strength,
+    img2img_strength: strength,
    cfg_scale: config.cfg_scale,
-    app: &app,
-    app_data_root,
    use_flash_attn: true,
+    model_config: &config.model_config,
+    maybe_seed: None,
+    scheduler_steps: config.scheduler_steps,
+    vae_path: &vae_path,
+    unet_path: &unet_path,
+    clip_json_path: &clip_json_path,
+    clip_weights_path: &clip_weights_path,
  };

-  match lcm_pipeline(args) {
-    Ok(image) => {
-      let mut bytes = Vec::with_capacity(1024*1024);
-      
-      image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
-        .map_err(|err| format!("failure to encode image: {:?}", err))?;
+  let image = lcm_pipeline(args)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;

-      let bytes = BASE64_STANDARD.encode(bytes);
-
-      Ok(bytes)
-    },
-    Err(e) => Err(format!("Failed to generate image: {}", e)),
-  }
+  Ok(image)
 }

 fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
@@ -131,16 +133,3 @@ fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
    .trim()
    .to_string()
 }
-
-fn hydrate_base64_image(base64_image: &str) -> anyhow::Result<DynamicImage> {
-  let bytes = BASE64_STANDARD.decode(base64_image)?;
-
-  let image = ImageReader::new(Cursor::new(bytes))
-    .with_guessed_format()?
-    .decode()?;
-
-  // TODO(bt,2025-02-17): Running out of vram with full image buffer size
-  let image = image.resize(512, 512, FilterType::CatmullRom);
-
-  Ok(image)
-}
@@ -0,0 +1,37 @@
+//! Adapted from https://github.com/dnanhkhoa/rust-background-removal
+
+use crate::state::app_dir::AppDataRoot;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
+use image::{imageops, DynamicImage};
+use log::{error, info};
+use ml_models::ml::background_removal::onnx_session::onnx_session;
+use ml_models::ml::background_removal::remove_image_background::remove_image_background;
+use ml_models::ml::model_cache::ModelCache;
+use ml_models::ml::weights_registry::weights::{DIS_MEDIUM_ONNX, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
+use tauri::{AppHandle, State};
+
+/// This handler removes the background from an image.
+#[tauri::command]
+pub async fn remove_background(
+  image: &str,
+  model_cache: State<'_, ModelCache>,
+  app_data_root: State<'_, AppDataRoot>,
+  app: AppHandle,
+) -> Result<String, String> {
+  info!("remove_background endpoint called.");
+
+  let image = decode_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  let model_path = app_data_root.weights_dir().weight_path(&DIS_MEDIUM_ONNX);
+  
+  let image = remove_image_background(model_path, image)
+    .await
+    .map_err(|err| format!("Couldn't remove image from image: {}", err))?;
+
+  let image = encode_dynamic_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  Ok(image)
+}
@@ -0,0 +1,110 @@
+use crate::state::app_config::AppConfig;
+use crate::state::app_dir::AppDataRoot;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use crate::utils::image::encode_rgb_image_base64_png::encode_rgb_image_base64_png;
+use image::imageops::FilterType;
+use image::{DynamicImage, ImageReader, RgbImage};
+use log::{error, info};
+use ml_models::ml::model_cache::ModelCache;
+use ml_models::ml::prompt_cache::PromptCache;
+use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
+use once_cell::sync::Lazy;
+use std::io::Cursor;
+use std::path::PathBuf;
+use tauri::{AppHandle, State};
+use ml_models::ml::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
+
+const RANDOM_SEED: u32 = 42;
+
+const STRENGTH: f64 = 1.0;
+const PNG_BYTES : &[u8] = include_bytes!("../../binary_includes/1024.png");
+
+// NB: We're just hacking a temporary "text to image" test endpoint leveraging the existing image to image modality.
+static IMAGE : Lazy<DynamicImage> = Lazy::new(||{
+  ImageReader::new(Cursor::new(PNG_BYTES))
+    .with_guessed_format()
+    .expect("failed to detect png format")
+    .decode()
+    .expect("failed to decode image")
+});
+
+#[tauri::command]
+pub async fn text_to_image(
+  prompt: String,
+  model_config: State<'_, AppConfig>,
+  model_cache: State<'_, ModelCache>,
+  prompt_cache: State<'_, PromptCache>,
+  app_data_root: State<'_, AppDataRoot>,
+  app: AppHandle,
+) -> Result<String, String> {
+  info!("text_to_image endpoint called.");
+
+  let image = IMAGE.resize(512, 512, FilterType::CatmullRom);
+
+  let result = do_infer_image(
+    &prompt, 
+    image, 
+    Some(STRENGTH), 
+    &model_config, 
+    &model_cache, 
+    prompt_cache, 
+    app, 
+    &app_data_root
+  ).await;
+  
+  let image = match result {
+    Ok(image) => image,
+    Err(err) => {
+      error!("There was an error generating the image: {:?}", err);
+      return Err(format!("There was an error generating the image: {}", err));
+    }
+  };
+
+  let bytes = encode_rgb_image_base64_png(image)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  info!("Inference successful; image converted to base64, serving back to browser...");
+
+  Ok(bytes)
+}
+
+async fn do_infer_image(
+  prompt: &str,
+  image: DynamicImage,
+  strength: Option<f64>,
+  config: &AppConfig,
+  model_cache: &ModelCache,
+  prompt_cache: State<'_, PromptCache>,
+  app: AppHandle,
+  app_data_root: &AppDataRoot,
+) -> Result<RgbImage, String> {
+  
+  let weights_dir = app_data_root.weights_dir();
+  let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
+  let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
+  let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
+  let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);
+
+  let args = Args {
+    image: &image,
+    prompt: prompt.to_string(),
+    uncond_prompt: "".to_string(),
+    model_cache,
+    prompt_cache: &prompt_cache,
+    img2img_strength: strength,
+    cfg_scale: config.cfg_scale,
+    use_flash_attn: true,
+    model_config: &config.model_config,
+    maybe_seed: None,
+    scheduler_steps: config.scheduler_steps,
+    vae_path: &vae_path,
+    unet_path: &unet_path,
+    clip_json_path: &clip_json_path,
+    clip_weights_path: &clip_weights_path,
+  };
+
+  let image = lcm_pipeline(args)
+    .map_err(|err| format!("failure to encode image: {:?}", err))?;
+
+  Ok(image)
+}
@@ -1,13 +1,13 @@
 use serde::Serialize;

-
 #[derive(Clone, Copy, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum NotificationModelType {
+  ImageSegmentation,
+  TextEncoder,
+  TextTokenizer,
  Unet,
  Vae,
-  Json,
-  ClipEncoder,
 }

 #[derive(Clone, Serialize)]
@@ -1,17 +1,20 @@
 pub mod endpoints;
 pub mod events;
-pub mod ml;
 pub mod state;
 pub mod threads;
 pub mod transfer;
+pub mod utils;

 use crate::endpoints::download_models::download_models;
-use crate::endpoints::image_endpoint::infer_image;
+use crate::endpoints::realtime_image_endpoint::infer_image;
+use crate::endpoints::remove_background_endpoint::remove_background;
 use crate::endpoints::test_counter::test_counter;
-use crate::ml::model_cache::ModelCache;
-use crate::ml::prompt_cache::PromptCache;
+use crate::endpoints::text_to_image_endpoint::text_to_image;
 use crate::state::app_config::AppConfig;
 use crate::threads::downloader_thread::downloader_thread;
+use crate::utils::log_environment_details::log_environment_details;
+use ml_models::ml::model_cache::ModelCache;
+use ml_models::ml::prompt_cache::PromptCache;
 use tauri_plugin_log::Target;
 use tauri_plugin_log::TargetKind;

@@ -42,6 +45,9 @@ pub fn run() {
      .targets(vec![Target::new(TargetKind::Stdout)])
      .build())
    .setup(|app| {
+      // TODO(bt): This is broken on windows
+      // log_environment_details();
+
      //if cfg!(debug_assertions) {
      //  app.handle().plugin(
      //    tauri_plugin_log::Builder::default()
@@ -60,9 +66,11 @@ pub fn run() {
    .manage(model_cache)
    .manage(app_data_root)
    .invoke_handler(tauri::generate_handler![
-      test_counter, 
-      infer_image,
      download_models,
+      infer_image,
+      remove_background,
+      test_counter,
+      text_to_image,
    ])
    .run(tauri::generate_context!())
    .expect("error while running tauri application");
@@ -1,15 +0,0 @@
-use crate::ml::downloads::download_model_registry_file::download_model_registry_file;
-use crate::ml::model_type::ModelType;
-use crate::state::app_dir::AppDataRoot;
-
-pub async fn download_all_models(app_data_root: &AppDataRoot) -> anyhow::Result<()> {
-  let weights_dir = app_data_root.weights_dir();
-  
-  //download_model_registry_file(ModelType::ClipJson, weights_dir).await?;
-  //download_model_registry_file(ModelType::SdxlTurboUnet, weights_dir).await?;
-  //download_model_registry_file(ModelType::SdxlTurboVae, weights_dir).await?;
-  //download_model_registry_file(ModelType::SdxlTurboClipEncoder, weights_dir).await?;
-  //download_model_registry_file(ModelType::SdxlTurboClipEncoder2, weights_dir).await?;
-  
-  Ok(())
-}
@@ -1,30 +0,0 @@
-use crate::ml::model_type::ModelType;
-use crate::state::app_dir::AppWeightsDir;
-use log::info;
-use std::fs::File;
-use std::path::{Path, PathBuf};
-
-pub async fn download_model_registry_file(
-  model_type: ModelType,
-  weights_dir: &AppWeightsDir,
-) -> anyhow::Result<PathBuf> {
-  
-  let path = weights_dir.model_path(&model_type);
-
-  if path.exists() {
-    return Ok(path);
-  }
-
-  info!("downloading model: {:?} to {:?}", model_type, path);
-
-  let body = reqwest::get(model_type.get_download_url())
-    .await?
-    .bytes()
-    .await?;
-  
-  let _file = File::create(&path)?;
-  
-  std::fs::write(&path, body.as_ref())?;
-  
-  Ok(path)
-}
@@ -1,2 +0,0 @@
-pub mod download_model_registry_file;
-pub mod download_all_models;
@@ -1,86 +0,0 @@
-use crate::events::notification_event::NotificationModelType;
-use crate::state::app_dir::AppWeightsDir;
-use std::path::PathBuf;
-
-#[derive(Copy, Clone, Debug)]
-pub enum ModelType {
-  ClipJson,
-  SdxlTurboUnet,
-  SdxlTurboVae,
-  SdxlTurboClipEncoder,
-  SdxlTurboClipEncoder2,
-  SimianLuoLcmDreamshaperV7Unet,
-  LykonDreamshaper7Vae,
-  LykonDreamshaper7TextEncoderFp16,
-}
-
-impl ModelType {
-  pub fn get_name(&self) -> &'static str {
-    match self {
-      Self::ClipJson => "CLIP (JSON)",
-      Self::SdxlTurboUnet => "SDXL Turbo UNET",
-      Self::SdxlTurboVae => "SDXL Turbo VAE",
-      Self::SdxlTurboClipEncoder => "Clip Encoder #1",
-      Self::SdxlTurboClipEncoder2 => "Clip Encoder #2",
-      Self::SimianLuoLcmDreamshaperV7Unet => "SimianLuo LCM Dreamshaper v7 UNET",
-      Self::LykonDreamshaper7Vae => "Lykon Dreamshaper 7 VAE",
-      Self::LykonDreamshaper7TextEncoderFp16 => "Lykon Dreamshaper Text Encoder (fp16)",
-    }
-  }
-
-  pub fn get_notification_type(&self) -> NotificationModelType {
-    match self {
-      Self::ClipJson => NotificationModelType::Json,
-      Self::SdxlTurboUnet => NotificationModelType::Unet,
-      Self::SdxlTurboVae => NotificationModelType::Vae,
-      Self::SdxlTurboClipEncoder => NotificationModelType::ClipEncoder,
-      Self::SdxlTurboClipEncoder2 => NotificationModelType::ClipEncoder,
-      Self::SimianLuoLcmDreamshaperV7Unet => NotificationModelType::Unet,
-      Self::LykonDreamshaper7Vae => NotificationModelType::Vae,
-      Self::LykonDreamshaper7TextEncoderFp16 => NotificationModelType::ClipEncoder,
-    }
-  }
-
-  pub fn get_hf_id(&self) -> Option<&'static str> {
-    match self {
-      Self::ClipJson => Some("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k"),
-      Self::SdxlTurboUnet 
-      | Self::SdxlTurboVae
-      | Self::SdxlTurboClipEncoder
-      | Self::SdxlTurboClipEncoder2 => Some("stabilityai/sdxl-turbo"),
-      Self::SimianLuoLcmDreamshaperV7Unet => Some("SimianLuo/LCM_Dreamshaper_v7"),
-      Self::LykonDreamshaper7Vae
-      | Self::LykonDreamshaper7TextEncoderFp16 => Some("Lykon/dreamshaper-7"),
-    }
-  }
-
-  pub fn get_download_url(&self) -> &'static str {
-    match self {
-      Self::ClipJson => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/tokenizer.json",
-      Self::SdxlTurboUnet => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/diffusion_pytorch_model.unet.safetensors",
-      Self::SdxlTurboVae => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/diffusion_pytorch_model.vae.safetensors",
-      Self::SdxlTurboClipEncoder => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/clip_text_encoder.safetensors",
-      Self::SdxlTurboClipEncoder2 => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/clip_text_encoder_2.safetensors",
-      Self::SimianLuoLcmDreamshaperV7Unet => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/simianluo_lcm_dreamshaper_v7_unet.safetensors",
-      Self::LykonDreamshaper7Vae => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/lykon_dreamshaper_7_vae.safetensors",
-      Self::LykonDreamshaper7TextEncoderFp16 => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/lykon_dreamshaper_7_text_encoder.fp16.safetensors",
-    }
-  }
-
-  pub fn get_filename(&self) -> &'static str {
-    match self {
-      Self::ClipJson => "clip_vit_bigg_14_lion2b_39b_b160k.tokenizer.json",
-      Self::SdxlTurboUnet => "diffusion_pytorch_model.unet.safetensors",
-      Self::SdxlTurboVae => "diffusion_pytorch_model.vae.safetensors",
-      Self::SdxlTurboClipEncoder => "clip_text_encoder.safetensors",
-      Self::SdxlTurboClipEncoder2 => "clip_text_encoder_2.safetensors",
-      Self::SimianLuoLcmDreamshaperV7Unet => "simianluo_lcm_dreamshaper_v7_unet.safetensors",
-      Self::LykonDreamshaper7Vae => "lykon_dreamshaper_7_vae.safetensors",
-      Self::LykonDreamshaper7TextEncoderFp16 => "lykon_dreamshaper_7_text_encoder.fp16.safetensors",
-    }
-  }
-  
-  pub fn get_path(&self, weights_dir: &AppWeightsDir) -> PathBuf {
-    weights_dir.model_path(self)
-  }
-}
@@ -1,130 +0,0 @@
-use crate::ml::model_file::StableDiffusionVersion;
-use anyhow::anyhow;
-use candle_core::{DType, Device, Tensor};
-use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
-use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
-use hf_hub::api::sync::Api;
-use std::sync::{Arc, RwLock};
-
-pub struct LazyLoadVaeModel {
-  lazy_model: Arc<RwLock<Option<AutoEncoderKL>>>,
-  model_configs: VaeConfigs,
-}
-
-// TODO: Maybe we pass these via a central config read at startup, passed as an Arc<ConfigT>
-pub struct VaeConfigs {
-  sd_config: StableDiffusionConfig,
-  sd_version: StableDiffusionVersion,
-  device: Device,
-  dtype: DType,
-  hf_api: Api,
-}
-
-impl LazyLoadVaeModel {
-  pub fn new(
-    sd_config: &StableDiffusionConfig,
-    sd_version: &StableDiffusionVersion,
-    hf_api: &Api,
-    device: &Device,
-    dtype: DType,
-  ) -> anyhow::Result<Self> {
-    println!("building vae model... (1)");
-    
-    Ok(Self {
-      lazy_model: Arc::new(RwLock::new(None)),
-      model_configs: VaeConfigs {
-        sd_config: sd_config.clone(),
-        sd_version: sd_version.clone(),
-        device: device.clone(),
-        hf_api: hf_api.clone(),
-        dtype
-      }
-    })
-  }
-
-  pub fn encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
-    println!("vae encode...");
-    match self.try_encode(xs) {
-      Err(err) => Err(err),
-      Ok(Some(result)) => Ok(result),
-      Ok(None) => {
-        // Model wasn't previously loaded
-        self.load_model()?;
-        self.try_encode(xs)?
-          .ok_or(anyhow!("model was not previously loaded"))
-      }
-    }
-  }
-  
-  pub fn decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
-    println!("vae decode...");
-    match self.try_decode(xs) {
-      Err(err) => Err(err),
-      Ok(Some(result)) => Ok(result),
-      Ok(None) => {
-        // Model wasn't previously loaded
-        self.load_model()?;
-        self.try_decode(xs)?
-          .ok_or(anyhow!("model was not previously loaded"))
-      }
-    }
-  }
-
-  fn try_encode(&self, xs: &Tensor) -> anyhow::Result<Option<DiagonalGaussianDistribution>> {
-    match self.lazy_model.read() {
-      Err(err) => Err(anyhow!("lock error: {:?}", err)),
-      Ok(model) => {
-        match &*model {
-          None => Ok(None), // Model is not yet loaded
-          Some(model) => {
-            Ok(Some(model.encode(xs)?))
-          }
-        }
-      }
-    }
-  }
-
-  fn try_decode(&self, xs: &Tensor) -> anyhow::Result<Option<Tensor>> {
-    match self.lazy_model.read() {
-      Err(err) => Err(anyhow!("lock error: {:?}", err)),
-      Ok(model) => {
-        match &*model {
-          None => Ok(None), // Model is not yet loaded
-          Some(model) => {
-            Ok(Some(model.decode(xs)?))
-          }
-        }
-      }
-    }
-  }
-
-  fn load_model(&self) -> anyhow::Result<()> {
-    println!("load vae");
-    match self.lazy_model.write() {
-      Err(err) => return Err(anyhow!("lock error: {:?}", err)),
-      Ok(mut maybe_model) => {
-        match &*maybe_model {
-          Some(_model) => {} // Already loaded; fall through
-          None => {
-            let repo = self.model_configs.sd_version.repo();
-
-            println!("Building VAE model from : {:?} ... (3)", repo);
-            
-            let vae_file = self.model_configs.hf_api.model(repo.to_string())
-              .get("vae/diffusion_pytorch_model.safetensors")?;
-
-            println!("Building VAE model from file {:?}...", &vae_file);
-            let vae = self.model_configs
-              .sd_config
-              .build_vae(vae_file, &self.model_configs.device, self.model_configs.dtype)?;
-
-            println!("Built VAE...");
-            *maybe_model = Some(vae);
-          }
-        }
-      }
-    }
-    Ok(())
-  }
- 
-}
@@ -1,2 +0,0 @@
-pub mod unet_model;
-pub mod lazy_load_vae_model;
@@ -1,30 +0,0 @@
-use candle_core::{DType, Device, Tensor};
-use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
-use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
-use std::path::Path;
-
-pub struct UNetModel {
-  model: UNet2DConditionModel,
-}
-
-impl UNetModel {
-  pub fn new<P: AsRef<Path>>(sd_config: &StableDiffusionConfig, unet_weights_file: P, device: &Device, dtype: DType) -> anyhow::Result<Self> {
-    println!("building unet model... (1)");
-
-    let model = sd_config.build_unet(unet_weights_file, &device, 4, true, dtype)?;
-
-    Ok(Self { model })
-  }
-
-  pub fn new_with_inner(model: UNet2DConditionModel) -> Self {
-    Self { model }
-  }
-
-  pub fn inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
-    Ok(self.model.forward(&latent_model_input, timestep, &text_embeddings)?)
-  }
-
-  pub fn forward_with_guidance(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor, guidance_scale_embedding: &Tensor) -> anyhow::Result<Tensor> {
-    Ok(self.model.forward_with_guidance(&latent_model_input, timestep, &text_embeddings, Some(guidance_scale_embedding))?)
-  }
-}
@@ -1,32 +1,14 @@
-use crate::ml::model_file::StableDiffusionVersion;
 use crate::state::app_dir::AppDataRoot;
 use crate::state::os_platform::OsPlatform;
 use crate::state::yaml_config::YamlConfig;
-use candle_core::{DType, Device};
-use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
-use filesys::recursively_find_file_by_name::recursively_find_file_by_name;
-use hf_hub::api::sync::Api;
-use log::info;
-use std::path::PathBuf;
+use ml_models::ml::model_config::ModelConfig;

 const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
 const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;

-const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
-const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::V1_5;

 /// A central place to configure app-wide details.
 pub struct AppConfig {
-  /// We only support one device type currently, and we can't multiplex 
-  /// over several devices yet.
-  pub device: Device,
-  
-  /// The core numeric type to use for models.
-  pub dtype: DType,
-  
-  pub sd_version: StableDiffusionVersion,
-  pub sd_config: StableDiffusionConfig,
-
  pub image_height: usize,
  pub image_width: usize,
  pub scheduler_steps: usize,
@@ -36,11 +18,10 @@ pub struct AppConfig {

  pub cfg_scale: Option<f64>,
  
-  /// Probably shouldn't be here.
-  pub hf_api: Api,
-  
  /// Root location for application data.
  pub app_data_root: AppDataRoot,
+  
+  pub model_config: ModelConfig,
 }

 impl AppConfig {
@@ -50,30 +31,6 @@ impl AppConfig {
    
    println!("Configs: {:?}", yaml_configs);
    
-    let device = Device::new_cuda(0)
-      .unwrap_or_else(|e| {
-          info!("CUDA not available ({}), falling back to CPU", e);
-          Device::Cpu
-        });
-
-    
-    let sd_config = match SD_VERSION {
-      StableDiffusionVersion::Turbo => {
-        StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
-      }
-      StableDiffusionVersion::V1_5 | StableDiffusionVersion::V1_5Inpaint => {
-        StableDiffusionConfig::v1_5(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
-      }
-      StableDiffusionVersion::V2_1 | StableDiffusionVersion::V2Inpaint => {
-        StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
-      }
-      StableDiffusionVersion::Xl | StableDiffusionVersion::XlInpaint => {
-        StableDiffusionConfig::sdxl(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
-      }
-    };
-    
-    let hf_api = Api::new()?;
-    
    let os_platform = OsPlatform::get();

    let default_data_root = match os_platform {
@@ -92,18 +49,14 @@ impl AppConfig {
    println!("App data root: {:?}", app_data_root.path());

    Ok(Self {
-      device,
-      dtype: DEFAULT_SD_NUMERIC_DATATYPE,
-      sd_version: SD_VERSION,
-      sd_config,
      image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
      image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
      scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
      scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
      seed: yaml_configs.seed,
      cfg_scale: yaml_configs.cfg_scale,
-      hf_api,
      app_data_root,
+      model_config: ModelConfig::init()?,
    })
  }
 }
@@ -1,11 +1,12 @@
-use crate::ml::model_type::ModelType;
 use crate::state::expanduser::expanduser;
 use crate::state::os_platform::OsPlatform;
 use anyhow::anyhow;
 use directories::UserDirs;
+use ml_models::ml::weights_registry::weight_descriptor::WeightDescriptor;
 use std::path::{Path, PathBuf};
 use tempdir::TempDir;
 use tempfile::{Builder, NamedTempFile};
+
 const DEFAULT_DATA_DIR : &str = "artcraft";
 const ASSETS_SUBDIRECTORY : &str = "assets";
 const WEIGHTS_SUBDIRECTORY : &str = "weights";
@@ -139,9 +140,9 @@ impl AppWeightsDir {
  pub fn path(&self) -> &Path {
    &self.path
  }
-  
-  pub fn model_path(&self, model_registry: &ModelType) -> PathBuf {
-    self.path.join(model_registry.get_filename())
+
+  pub fn weight_path(&self, descriptor: &WeightDescriptor) -> PathBuf {
+    self.path.join(descriptor.filename)
  }
 }

@@ -1,10 +1,11 @@
 use crate::events::notification_event::{NotificationEvent, NotificationModelType};
-use crate::ml::model_type::ModelType;
 use crate::state::app_dir::AppDataRoot;
 use crate::transfer::download::{download_async, ProgressUpdate};
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use log::{debug, error, info};
+use ml_models::ml::weights_registry::weight_descriptor::WeightFunction;
+use ml_models::ml::weights_registry::weights::{CLIP_JSON, DIS_MEDIUM_ONNX, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2, SDXL_TURBO_UNET, SDXL_TURBO_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
 use std::collections::VecDeque;
 use std::path::PathBuf;
 use std::sync::mpsc::Receiver;
@@ -39,22 +40,22 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !

  let mut download_queue = VecDeque::new();

-  // TODO: Automatic enqueue of known important models + enqueue new models on-demand
-  download_queue.push_back(ModelType::ClipJson);
-  //download_queue.push_back(ModelType::SdxlTurboUnet);
-  //download_queue.push_back(ModelType::SdxlTurboVae);
-  download_queue.push_back(ModelType::SdxlTurboClipEncoder); // TODO(bt): Why is this still needed?
-  //download_queue.push_back(ModelType::SdxlTurboClipEncoder2);
-  download_queue.push_back(ModelType::SimianLuoLcmDreamshaperV7Unet);
-  download_queue.push_back(ModelType::LykonDreamshaper7Vae);
-  download_queue.push_back(ModelType::LykonDreamshaper7TextEncoderFp16);
+  download_queue.push_back(CLIP_JSON);
+  download_queue.push_back(LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16);
+  download_queue.push_back(LYKON_DEAMSHAPER_7_VAE);
+  download_queue.push_back(SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
+  download_queue.push_back(SDXL_TURBO_CLIP_TEXT_ENCODER); // TODO(bt): Why is this still needed?
+  download_queue.push_back(DIS_MEDIUM_ONNX);
+  //download_queue.push_back(SDXL_TURBO_CLIP_TEXT_ENCODER_2); // TODO(bt): REMOVE
+  //download_queue.push_back(SDXL_TURBO_VAE); // TODO(bt): REMOVE
+  //download_queue.push_back(SDXL_TURBO_UNET); // TODO(bt): REMOVE

  let mut handles = FuturesUnordered::new();

  while let Some(model) = download_queue.pop_front() {
-    let url = model.get_download_url().to_string();
+    let url = model.r2_download_url.to_string();

-    let filename = app_data_root.weights_dir().model_path(&model);
+    let filename = app_data_root.weights_dir().weight_path(&model);

    if filename.exists() {
      info!("File already exists: {:?}", &filename);
@@ -75,9 +76,17 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
      }
    };

+    let model_type = match model.function {
+      WeightFunction::TextEncoder => NotificationModelType::TextEncoder,
+      WeightFunction::TextTokenizer => NotificationModelType::TextTokenizer,
+      WeightFunction::Unet => NotificationModelType::Unet,
+      WeightFunction::Vae => NotificationModelType::Vae,
+      WeightFunction::ImageSegmentation => NotificationModelType::ImageSegmentation,
+    };
+
    let result = app.emit(NOTIFICATION_CHANNEL_NAME, NotificationEvent::ModelDownloadStarted {
-      model_name: model.get_name(),
-      model_type: model.get_notification_type(),
+      model_name: model.name,
+      model_type,
    });

    if let Err(err) = result {
@@ -102,8 +111,8 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
        lock.push(ModelDownloadStatusChannel {
          receiver: rx,
          final_file_path: filename.clone(),
-          model_name: model.get_name().to_string(),
-          model_type: model.get_notification_type(),
+          model_name: model.to_descriptive_name(),
+          model_type,
        })
      }
    }
@@ -0,0 +1,14 @@
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::{DynamicImage, ImageReader};
+use std::io::Cursor;
+
+pub fn decode_base64_image(base64_image: &str) -> anyhow::Result<DynamicImage> {
+  let bytes = BASE64_STANDARD.decode(base64_image)?;
+
+  let image = ImageReader::new(Cursor::new(bytes))
+    .with_guessed_format()?
+    .decode()?;
+
+  Ok(image)
+}
@@ -0,0 +1,14 @@
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::{DynamicImage, ImageFormat};
+use std::io::Cursor;
+
+pub fn encode_dynamic_image_base64_png(image: DynamicImage) -> anyhow::Result<String> {
+  let mut bytes = Vec::with_capacity(1024*1024);
+
+  image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)?;
+
+  let bytes = BASE64_STANDARD.encode(bytes);
+
+  Ok(bytes)
+}
@@ -0,0 +1,16 @@
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::{ImageFormat, RgbImage};
+use std::io::Cursor;
+
+pub fn encode_rgb_image_base64_png(
+  image: RgbImage
+) -> anyhow::Result<String> {
+  let mut bytes = Vec::with_capacity(1024*1024);
+
+  image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)?;
+
+  let bytes = BASE64_STANDARD.encode(bytes);
+
+  Ok(bytes)
+}
@@ -0,0 +1,3 @@
+pub mod decode_base64_image;
+pub mod encode_dynamic_image_base64_png;
+pub mod encode_rgb_image_base64_png;
@@ -0,0 +1,11 @@
+use log::info;
+//use nvidia_checker::{get_cuda_version, get_cudnn_version, get_kernel_version, get_nvidia_driver_version, get_os_version, get_tensorrt_version};
+
+pub fn log_environment_details() {
+  //info!("NVIDIA driver version: {}", get_nvidia_driver_version(false));
+  //info!("CUDA version: {}", get_cuda_version(false));
+  //info!("cuDNN version: {}", get_cudnn_version(false));
+  //info!("TensorRT version: {}", get_tensorrt_version(false));
+  //info!("OS version: {}", get_os_version(false));
+  //info!("Kernel version: {}", get_kernel_version(false));
+}
@@ -0,0 +1,2 @@
+pub mod image;
+pub mod log_environment_details;
@@ -32,3 +32,4 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

 [dev-dependencies]
 testing = { path = "../../testing" }
+assertor = "0.0.3"
@@ -1,5 +1,6 @@
 pub mod i32_to_u32_zero_clamped;
 pub mod i64_to_u64_zero_clamped;
+pub mod remap_range_f64;
 pub mod u64_to_i32_saturating;
 pub mod u64_to_i64_saturating;
 pub mod u64_to_u32_saturating;
@@ -0,0 +1,131 @@
+
+pub struct Args {
+  pub old_min: f64,
+  pub old_max: f64,
+  pub new_min: f64,
+  pub new_max: f64,
+  
+  /// The value to remap to the new range.
+  pub value: f64,
+}
+
+pub fn remap_range_f64(args: Args) -> f64 {
+  let old_range = args.old_max - args.old_min;
+  let new_range = args.new_max -  args.new_min;
+
+  let new_value = (args.value -  args.old_min) * new_range / old_range + args.new_min;
+  
+  if new_value > args.new_max {
+    args.new_max
+  } else if new_value < args.new_min { 
+    args.new_min
+  } else {
+    new_value
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::numerics::remap_range_f64::{remap_range_f64, Args};
+  use assertor::{assert_that, FloatAssertion};
+
+  #[test]
+  fn test_base_cases() {
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: 0.0,
+      new_max: 1.0,
+      value: 0.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(0.0);
+    
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: 0.0,
+      new_max: 1.0,
+      value: 1.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(1.0);
+  }
+
+  #[test]
+  fn test_maxima() {
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 100.0,
+      new_min: 0.0,
+      new_max: 1.0,
+      value: 100.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(1.0);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: 0.0,
+      new_max: 50.0,
+      value: 1.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(50.0);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: -100.0,
+      new_max: 100.0,
+      value: 1.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(100.0);
+  }
+
+  #[test]
+  fn test_minima() {
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 100.0,
+      new_min: 0.0,
+      new_max: 1.0,
+      value: 0.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(0.0);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: 0.0,
+      new_max: 50.0,
+      value: 0.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(0.0);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: -100.0,
+      new_max: 100.0,
+      value: 0.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(-100.0);
+  }
+
+  #[test]
+  fn test_other_values() {
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 100.0,
+      new_min: 0.0,
+      new_max: 1.0,
+      value: 50.0,
+    })).with_abs_tol(0.01).is_approx_equal_to(0.50);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: 0.0,
+      new_max: 50.0,
+      value: 0.10,
+    })).with_abs_tol(0.01).is_approx_equal_to(5.0);
+
+    assert_that!(remap_range_f64(Args {
+      old_min: 0.0,
+      old_max: 1.0,
+      new_min: -100.0,
+      new_max: 100.0,
+      value: 0.25,
+    })).with_abs_tol(0.01).is_approx_equal_to(-50.0);
+  }
+}
@@ -1,35 +1,36 @@
 import Konva from "konva";

-import { uiAccess, uiEvents } from "~/signals";
-import { ShapeNode } from "./Nodes";
-import { PaintNode } from "./Nodes/PaintNode";
-import { PreviewCopyNode } from "./Nodes/PreviewCopy";
-import { UndoStackManager } from "./UndoRedo";
-import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities";
+import {uiAccess, uiEvents} from "~/signals";
+import {ImageNode, ShapeNode, ShapeType, TextNode, VideoNode} from "./Nodes";
+import {PaintNode} from "./Nodes/PaintNode";
+import {PreviewCopyNode} from "./Nodes/PreviewCopy";
+import {UndoStackManager} from "./UndoRedo";
+import {CommandManager, MatteBox, SceneManager} from "./EngineUtitlities";
 import {
-  NodesManager,
  NodeIsolator,
-  NodeTransformer,
+  NodesManager,
  NodesTranslationEventDetails,
  NodeTransformationEventDetails,
+  NodeTransformer,
  SelectionManager,
  SelectionManagerEvents,
  SelectorSquare,
 } from "./NodesManagers";
-import { ImageNode, VideoNode, TextNode } from "./Nodes";
-import { EngineOptions, TextNodeData, VideoNodeData } from "./types";
+import {EngineOptions, TextNodeData, VideoNodeData} from "./types";

-import { AppModes, VideoResolutions } from "./constants";
-import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
+import {AppModes, VideoResolutions} from "./constants";
+import {ToolbarMainButtonNames} from "~/components/features/ToolbarMain/enum";

-import { ToolbarNodeButtonNames } from "~/components/features/ToolbarNode/enums";
-import { NavigateFunction } from "react-router-dom";
-import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider";
+import {ToolbarNodeButtonNames} from "~/components/features/ToolbarNode/enums";
+import {NavigateFunction} from "react-router-dom";
+import {LoadingVideosProvider} from "./EngineUtitlities/LoadingVideosProvider";

-import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler";
-import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine";
-import { NodeColor } from "~/signals/uiEvents/toolbarNode";
-import { ShapeType } from "./Nodes";
+import {VideoExtractionHandler} from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler";
+import {RealTimeDrawEngine} from "./RenderingPrimitives/RealTimeDrawEngine";
+import {NodeColor} from "~/signals/uiEvents/toolbarNode";
+import {invoke} from "@tauri-apps/api/core";
+import {DecodeBase64ToImage} from "~/utilities/DecodeBase64ToImage.ts";
+import {EncodeImageBitmapToBase64} from "~/utilities/EncodeImageBitmapToBase64.ts";


 export interface RenderingOptions {
@@ -326,6 +327,9 @@ export class Engine {
    uiEvents.toolbarNode.MOVE_LAYER_UP.onClick(() =>
      this.commandManager.moveNodesUp(),
    );
+    uiEvents.toolbarNode.REMOVE_BACKGROUND.onClick(() =>
+      this.removeBackground(),
+    );

    // Listen to Toolbar Main
    uiEvents.toolbarMain.UNDO.onClick(() => {
@@ -445,6 +449,38 @@ export class Engine {
    });
  }

+  removeBackground() {
+    const selectedNodeSet = this.selectionManager.getSelectedNodes();
+
+    if (selectedNodeSet.size !== 1) {
+      return; // Can only remove background from a single image at a time.
+    }
+
+    const [ selectedNode ] = selectedNodeSet;
+
+    const removeBgInvocation
+        = async (selectedNode: ImageNode, callback: (existingImageNode: ImageNode, createdImage: ImageBitmap) => void) => {
+
+      const image : HTMLImageElement = selectedNode.kNode.image();
+      const base64Bitmap = await EncodeImageBitmapToBase64(image);
+      const base64BitmapResponse = await invoke("remove_background", {
+        image: base64Bitmap,
+      });
+
+      const createdImage = await DecodeBase64ToImage(base64BitmapResponse as string);
+
+      callback(selectedNode, createdImage);
+    };
+
+    const _promise = removeBgInvocation(selectedNode, (selectedNode: ImageNode, createdImage: ImageBitmap) => {
+      if (!this.nodesManager.hasNode(selectedNode)) {
+        return;
+      }
+      this.commandManager.deleteSpecificNodes([selectedNode]);
+      this.addImageFromImageBitmap(createdImage);
+    })
+  }
+
  disableAllButtons() {
    const buttonNames = Object.values(ToolbarNodeButtonNames);
    for (const name of buttonNames) {
@@ -624,6 +660,23 @@ export class Engine {

    this.commandManager.createNode(imageNode);
  }
+
+  public addImageFromImageBitmap(imageBitmap: ImageBitmap) {
+    const imageNode = new ImageNode({
+      mediaLayerRef: this.mediaLayer,
+      canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+      canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
+      imageBitmap: imageBitmap,
+      selectionManagerRef: this.selectionManager,
+      loaded: async () => {
+        await this.realTimeDrawEngine.render();
+        this.setAppMode(AppModes.SELECT);
+      },
+    });
+
+    this.commandManager.createNode(imageNode);
+  }
+
  public addPaintNode(
    canvas: HTMLCanvasElement,
    lineBounds: {
@@ -78,6 +78,20 @@ export class CommandManager {
    this.undoStackManagerRef.executeCommand(command);
    this.renderEngineRef.render();
  }
+
+  deleteSpecificNodes(nodes: MediaNode[]) {
+    const command = new DeleteCommand({
+      nodes: nodes,
+      mediaLayerRef: this.mediaLayerRef,
+      nodesManagerRef: this.nodesManagerRef,
+      nodeTransformerRef: this.nodeTransformerRef,
+      selectionManagerRef: this.selectionManagerRef,
+      renderEngineRef: this.renderEngineRef,
+    });
+    this.undoStackManagerRef.executeCommand(command);
+    this.renderEngineRef.render();
+  }
+
  toggleLockNodes() {
    const nodes = this.selectionManagerRef.getSelectedNodes();
    const node = nodes.values().next().value;
@@ -12,6 +12,7 @@ interface ImageNodeContructor {
  canvasPosition: Position;
  canvasSize: Size;
  imageFile?: File;
+  imageBitmap?: ImageBitmap;
  mediaFileToken?: string;
  mediaFileUrl?: string;
  transform?: TransformationData;
@@ -34,6 +35,7 @@ export class ImageNode extends NetworkedNode {
    mediaLayerRef,
    selectionManagerRef,
    loaded,
+    imageBitmap,
  }: ImageNodeContructor) {
    // kNodes need to be created first to guaruntee
    // that it is not undefined in parent's context
@@ -73,6 +75,14 @@ export class ImageNode extends NetworkedNode {
      this.loadImageFromUrl(mediaFileUrl);
      return;
    }
+    if (imageBitmap) {
+      this.loadImageFromImageBitmap({
+        imageBitmap: imageBitmap,
+        maxSize: canvasSize,
+        refPosition: canvasPosition,
+      });
+      return;
+    }
    console.log("image node creation is fucked");
  }
  private loadImageFromFile({
@@ -136,6 +146,40 @@ export class ImageNode extends NetworkedNode {
    this.listenToBaseKNode();
  }

+  private loadImageFromImageBitmap({
+    imageBitmap,
+    maxSize,
+    refPosition,
+  }: {
+    imageBitmap: ImageBitmap;
+    maxSize: Size;
+    refPosition: Position;
+  }) {
+    this.imageSize = {
+      width: imageBitmap.width,
+      height: imageBitmap.height,
+    };
+    const adjustedSize = NodeUtilities.adjustNodeSizeToCanvas({
+      componentSize: this.imageSize,
+      maxSize: maxSize, // TODO
+    });
+    const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({
+      canvasOffset: refPosition, // TODO
+      componentSize: adjustedSize, // TODO
+      maxSize: maxSize, // TODO
+    });
+    this.kNode.image(imageBitmap);
+    this.kNode.setSize(adjustedSize);
+    this.kNode.setPosition(centerPosition);
+    this.kNode.moveToTop();
+    this.kNode.fill(transparent);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+    //this.uploadImage(imageFile);
+    this.loaded();
+    this.listenToBaseKNode();
+  }
+
  private async uploadImage(imageFile: File) {
    //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });

@@ -44,6 +44,10 @@ export class NodesManager {
    // this.printNodesAndZIndices();
  }

+  public hasNode(node: MediaNode): boolean {
+    return this.allNodes.has(node);
+  }
+
  /*************************
   *  For Debugging use
   */
@@ -46,29 +46,29 @@ export function getImageNodeButtonStates(
        return buttonStates;
      }

-      // hidden buttons
-      if (
-        buttonName === ButtonNames.AI_STYLIZE ||
-        buttonName === ButtonNames.SEGMENTATION ||
-        buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA ||
-        buttonName === ButtonNames.COLOR
-      ) {
-        buttonStates[buttonName] = {
-          disabled: true,
-          hidden: true,
-          active: false,
-        };
-        return buttonStates;
-      }
+      switch (buttonName) {
+        case ButtonNames.AI_STYLIZE:
+        case ButtonNames.SEGMENTATION:
+        case ButtonNames.DOWNLOAD:
+        case ButtonNames.CHROMA:
+        case ButtonNames.COLOR:
+          // hidden buttons
+          buttonStates[buttonName] = {
+            disabled: true,
+            hidden: true,
+            active: false,
+          };
+          return buttonStates;

-      // all other buttons
-      buttonStates[buttonName] = {
-        disabled: false,
-        hidden: false,
-        active: false,
-      };
-      return buttonStates;
+        default:
+          // all other buttons
+          buttonStates[buttonName] = {
+            disabled: false,
+            hidden: false,
+            active: false,
+          };
+          return buttonStates;
+      }
    },
    {} as ContextualToolbarProps["buttonStates"],
  );
@@ -89,28 +89,29 @@ export function getTextNodeButtonStates(
        return buttonStates;
      }

-      // hidden buttons
-      if (
-        buttonName === ButtonNames.AI_STYLIZE ||
-        buttonName === ButtonNames.SEGMENTATION ||
-        buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA
-      ) {
-        buttonStates[buttonName] = {
-          disabled: true,
-          hidden: true,
-          active: false,
-        };
-        return buttonStates;
-      }
+      switch (buttonName) {
+        case ButtonNames.AI_STYLIZE:
+        case ButtonNames.SEGMENTATION:
+        case ButtonNames.DOWNLOAD:
+        case ButtonNames.CHROMA:
+        case ButtonNames.REMOVE_BACKGROUND:
+          // hidden buttons
+          buttonStates[buttonName] = {
+            disabled: true,
+            hidden: true,
+            active: false,
+          };
+          return buttonStates;

-      // all other buttons
-      buttonStates[buttonName] = {
-        disabled: false,
-        hidden: false,
-        active: false,
-      };
-      return buttonStates;
+        default:
+          // all other buttons
+          buttonStates[buttonName] = {
+            disabled: false,
+            hidden: false,
+            active: false,
+          };
+          return buttonStates;
+      }
    },
    {} as ContextualToolbarProps["buttonStates"],
  );
@@ -130,24 +131,26 @@ export function getVideoNodeButtonStates(
        };
        return buttonStates;
      }
-      // soon to come feature is disabled
-      if (buttonName === ButtonNames.AI_STYLIZE ||
-        buttonName === ButtonNames.COLOR
-      ) {
-        buttonStates[buttonName] = {
-          disabled: true,
-          hidden: true,
-          active: false,
-        };
-        return buttonStates;
+      switch (buttonName) {
+        case ButtonNames.AI_STYLIZE:
+        case ButtonNames.COLOR:
+        case ButtonNames.REMOVE_BACKGROUND:
+          // soon to come feature is disabled
+          buttonStates[buttonName] = {
+              disabled: true,
+              hidden: true,
+              active: false,
+          };
+          return buttonStates;
+        default:
+          // all other buttons
+          buttonStates[buttonName] = {
+              disabled: false,
+              hidden: false,
+              active: false,
+          };
+          return buttonStates;
      }
-      // all other buttons
-      buttonStates[buttonName] = {
-        disabled: false,
-        hidden: false,
-        active: false,
-      };
-      return buttonStates;
    },
    {} as ContextualToolbarProps["buttonStates"],
  );
@@ -169,28 +172,28 @@ export function getShapeNodeButtonStates(
        return buttonStates;
      }

-      // hidden buttons
-      if (
-        buttonName === ButtonNames.AI_STYLIZE ||
-        buttonName === ButtonNames.SEGMENTATION ||
-        buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA
-      ) {
-        buttonStates[buttonName] = {
-          disabled: true,
-          hidden: true,
-          active: false,
-        };
-        return buttonStates;
+      switch (buttonName) {
+        case ButtonNames.AI_STYLIZE:
+        case ButtonNames.SEGMENTATION:
+        case ButtonNames.DOWNLOAD:
+        case ButtonNames.CHROMA:
+        case ButtonNames.REMOVE_BACKGROUND:
+          // hidden buttons
+          buttonStates[buttonName] = {
+              disabled: true,
+              hidden: true,
+              active: false,
+          };
+          return buttonStates;
+        default:
+          // all other buttons
+          buttonStates[buttonName] = {
+              disabled: false,
+              hidden: false,
+              active: false,
+          };
+          return buttonStates;
      }
-
-      // all other buttons
-      buttonStates[buttonName] = {
-        disabled: false,
-        hidden: false,
-        active: false,
-      };
-      return buttonStates;
    },
    {} as ContextualToolbarProps["buttonStates"],
  );
@@ -212,29 +215,29 @@ export function getMultiSelectButtonStates(
        return buttonStates;
      }

-      // hidden buttons
-      if (
-        buttonName === ButtonNames.AI_STYLIZE ||
-        buttonName === ButtonNames.SEGMENTATION ||
-        buttonName === ButtonNames.DOWNLOAD ||
-        buttonName === ButtonNames.CHROMA ||
-        buttonName === ButtonNames.COLOR
-      ) {
-        buttonStates[buttonName] = {
-          disabled: true,
-          hidden: true,
-          active: false,
-        };
-        return buttonStates;
+      switch (buttonName) {
+        case ButtonNames.AI_STYLIZE:
+        case ButtonNames.SEGMENTATION:
+        case ButtonNames.DOWNLOAD:
+        case ButtonNames.CHROMA:
+        case ButtonNames.COLOR:
+        case ButtonNames.REMOVE_BACKGROUND:
+          // hidden buttons
+          buttonStates[buttonName] = {
+              disabled: true,
+              hidden: true,
+              active: false,
+          };
+          return buttonStates;
+        default:
+          // all other buttons
+          buttonStates[buttonName] = {
+              disabled: false,
+              hidden: false,
+              active: false,
+          };
+          return buttonStates;
      }
-
-      // all other buttons
-      buttonStates[buttonName] = {
-        disabled: false,
-        hidden: false,
-        active: false,
-      };
-      return buttonStates;
    },
    {} as ContextualToolbarProps["buttonStates"],
  );
@@ -25,6 +25,7 @@ import {
  isLoadingVisible,
  loadingProgress,
 } from "~/signals/uiEvents/loadingIndicator";
+import {DecodeBase64ToImage} from "~/utilities/DecodeBase64ToImage.ts";

 interface ServerSettings {
  model_path: string;
@@ -285,7 +286,7 @@ export class RealTimeDrawEngine {
      this.client.onResult(async (response) => {
        try {
          // Convert the base64 image to ImageBitmap and update the preview
-          const bitmap = await this.base64ToImageBitmap(response.image);
+          const bitmap = await DecodeBase64ToImage(response.image);
          this.outputBitmap = bitmap;
          this.previewCanvas.image(bitmap);
          this.mediaLayerRef.batchDraw();
@@ -895,35 +896,6 @@ export class RealTimeDrawEngine {
    return base64String.split(",")[1];
  }

-  private async base64ToImageBitmap(
-    base64String: string,
-  ): Promise<ImageBitmap> {
-    // Create an image element
-    const img = document.createElement("img");
-
-    // Convert base64 to data URL if it doesn't include the prefix
-    const dataUrl = base64String.startsWith("data:")
-      ? base64String
-      : `data:image/png;base64,${base64String}`;
-
-    // Create a promise to handle the image loading
-    return new Promise((resolve, reject) => {
-      img.onload = async () => {
-        try {
-          const bitmap = await createImageBitmap(img);
-          resolve(bitmap);
-        } catch (error) {
-          reject(error);
-        }
-      };
-
-      img.onerror = () => reject(new Error("Failed to load image"));
-
-      // Set the source to trigger loading
-      img.src = dataUrl;
-    });
-  }
-
  public async saveOutput() {
    if (!this.outputBitmap) {
      console.error("No output bitmap available to save");
@@ -1007,7 +979,7 @@ export class RealTimeDrawEngine {
      });

      //console.log(base64BitmapResponse);
-      const decoded = await this.base64ToImageBitmap(
+      const decoded = await DecodeBase64ToImage(
        base64BitmapResponse as string,
      );

@@ -22,6 +22,11 @@ export const ToolbarNodeButtonData = [
    icon: faPalette,
    tooltip: "Color",
  },
+  {
+    name: ButtonNames.REMOVE_BACKGROUND,
+    icon: faScalpelLineDashed,
+    tooltip: "Remove Background",
+  },
  {
    name: ButtonNames.CHROMA,
    icon: faTransporter,
@@ -8,4 +8,5 @@ export enum ToolbarNodeButtonNames {
  MOVE_LAYER_DOWN = "MOVE_LAYER_DOWN",
  DOWNLOAD = "DOWNLOAD",
  COLOR = "COLOR",
+  REMOVE_BACKGROUND = "REMOVE_BACKGROUND",
 }
@@ -3,7 +3,7 @@ import { SignaledModelButtons } from "~/KonvaRootComponent/SignaledModelButtons"

 export const ToolbarTopCenter = () => {
  return (
-    <div className="flex flex-col gap-3">
+    <div className="flex flex-col items-center gap-3">
      {/* Model selection buttons (only shown in dev mode for now) */}
      {import.meta.env.DEV && <SignaledModelButtons />}

@@ -25,7 +25,9 @@ export const ToolbarTopLeft = () => {
  console.log(appMode.value);

  return (
-    <div className={twMerge("z-20 flex h-fit w-fit items-center gap-8 px-4")}>
+    <div
+      className={twMerge("z-20 flex h-fit w-fit items-center gap-8 pl-1 pr-4")}
+    >
      <img
        src="/brand/mira-logo.png"
        alt="logo"
@@ -55,7 +55,7 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
      <TabList className="glass relative inline-flex min-w-fit overflow-x-auto rounded-lg p-1 shadow-lg">
        {/* Animated indicator */}
        <div
-          className="absolute top-1 z-10 h-[calc(100%-8px)] rounded-md border-2 border-primary bg-primary/30 transition-all duration-300 ease-in-out"
+          className="absolute top-1 z-10 h-[calc(100%-8px)] rounded-md border-2 border-primary bg-primary/30 transition-all duration-200 ease-in-out"
          style={{
            left: indicatorStyle.left,
            width: indicatorStyle.width,
@@ -68,7 +68,7 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
            ref={(el) => (tabsRef.current[index] = el)}
            className={({ selected }) =>
              twMerge(
-                "relative z-20 mx-0.5 min-w-max rounded-md border-2 border-transparent px-4 py-1 text-center font-semibold transition-all duration-300 ease-in-out",
+                "relative z-20 mx-0.5 min-w-max rounded-md border-2 border-transparent px-4 py-0.5 text-center font-semibold transition-all duration-200 ease-in-out",
                selected ? "text-white" : "text-gray-300 hover:text-white",
              )
            }
@@ -0,0 +1,30 @@
+/**
+ * This decodes Base64 encoded PNG images into an image element.
+ * @param base64String a standard (not "web safe") base64-encoded string
+ */
+export const DecodeBase64ToImage = async (base64String: string) : Promise<ImageBitmap> => {
+  // Create an image element
+  const img = document.createElement("img");
+
+  // Convert base64 to data URL if it doesn't include the prefix
+  const dataUrl = base64String.startsWith("data:")
+    ? base64String
+    : `data:image/png;base64,${base64String}`;
+
+  // Create a promise to handle the image loading
+  return new Promise((resolve, reject) => {
+    img.onload = async () => {
+      try {
+        const bitmap = await createImageBitmap(img);
+        resolve(bitmap);
+      } catch (error) {
+        reject(error);
+      }
+    };
+
+    img.onerror = () => reject(new Error("Failed to load image"));
+
+    // Set the source to trigger loading
+    img.src = dataUrl;
+  });
+}
@@ -0,0 +1,22 @@
+
+export const EncodeImageBitmapToBase64 = async (imageBitmap: ImageBitmap): Promise<string> => {
+  // Create a temporary canvas
+  const canvas = document.createElement("canvas");
+  canvas.width = imageBitmap.width;
+  canvas.height = imageBitmap.height;
+
+  // Draw the ImageBitmap onto the canvas
+  const ctx = canvas.getContext("2d");
+  if (!ctx) throw new Error("Failed to get 2D context");
+
+  ctx.drawImage(imageBitmap, 0, 0);
+
+  // Convert to base64
+  const base64String = canvas.toDataURL("image/png");
+
+  // Clean up
+  canvas.remove();
+
+  // Remove the data:image/png;base64, prefix if you want just the base64 string
+  return base64String.split(",")[1];
+}
Author	SHA1	Message	Date
Brandon Thomas	899707d2d9	pull configs and ml deps out of tauri app	2025-03-24 20:17:28 -04:00
Brandon Thomas	cc0e9ae97c	seed is static	2025-03-24 18:48:12 -04:00
Brandon Thomas	e4312b80d1	testing	2025-03-24 18:34:59 -04:00
Brandon Thomas	4ceaac9411	test old clip code	2025-03-24 00:23:03 -04:00
Brandon Thomas	adfed2b616	debug	2025-03-24 00:09:00 -04:00
Brandon Thomas	54e6b8314f	split packages to work on mac	2025-03-23 23:01:53 -04:00
Brandon Thomas	e3245dc593	fix on windows	2025-03-22 00:58:04 -04:00
Brandon Thomas	08590d0c61	log environment details	2025-03-21 01:11:34 -04:00
Brandon Thomas	32cb39a643	text_to_image endpoint	2025-03-20 23:28:59 -04:00
Brandon Thomas	5fdb4d1024	disable msi builds due to 2.0 gb limit for 32bit	2025-03-20 11:59:43 -04:00
Brandon Thomas	d11945b685	download DIS model	2025-03-20 11:07:34 -04:00
Brandon Thomas	f3babbcb19	if statements -> switch statements	2025-03-20 10:53:34 -04:00
Brandon Thomas	f37011ffc6	remove background: frontend UX (#458 )	2025-03-20 01:41:43 -04:00
Brandon Thomas	de683b750a	remove background from images	2025-03-18 22:10:18 -04:00
Brandon Thomas	4603c65d2a	fix strength slider range	2025-03-18 18:21:39 -04:00
Brandon Thomas	c5b0d837a9	fix bug with float strengths from frontend	2025-03-18 01:48:54 -04:00
Brandon Thomas	652785224a	easier windows builds. put dlls in the windows_dlls folder	2025-03-14 04:22:18 -04:00
Brandon Thomas	8d3ed947d1	change default prompt, log end of endpoint logic	2025-03-14 03:20:46 -04:00
Brandon Thomas	243583f0b7	kill more dead code	2025-03-13 23:39:04 -04:00
Brandon Thomas	150223c48c	refactor weights (#455 )	2025-03-13 23:29:20 -04:00
bflatastic	ddf6a7f54e	top center ui bug fix and padding	2025-03-13 22:28:26 +07:00
Brandon Thomas	17c6e549b5	ignore windows dlls	2025-03-13 00:38:10 -04:00
Brandon Thomas	031d9d8f20	Download models from Cloudflare instead of HF (#454 )	2025-03-13 00:32:43 -04:00