Show only image records

- for selecting existing images from library (+ in prompt box)
- for viewing media in library (my library button on top right)

Cargo.toml

@@ -1,6 +1,8 @@
 [workspace]
 resolver = "2"
 
+# This block defines the workspace members.
+# When we add a library or binary crate, it needs an entry here.
 members = [
   # Types: common strongly-typed tokens, ids, payloads, etc.
   "crates/_types/composite_identifiers",
@@ -22,9 +24,10 @@ members = [
   "crates/cli/production/migration-tool",
 
   # Desktop apps
-  "crates/desktop/src-tauri",
   "crates/desktop/ml_models",
   "crates/desktop/ml_weights_registry",
+  "crates/desktop/tauri-4o",
+  "crates/desktop/tauri-realtime",
 
   # Shared libraries
   "crates/lib/actix_cors_configs",
@@ -32,6 +35,7 @@ members = [
   "crates/lib/api_clients/cloud_storage",
   "crates/lib/api_clients/google_drive_common",
   "crates/lib/api_clients/google_sign_in",
+  "crates/lib/api_clients/openai_sora_client",
   "crates/lib/api_clients/thumbnail_generator",
   "crates/lib/bootstrap",
   "crates/lib/bucket_paths",
@@ -58,6 +62,7 @@ members = [
   "crates/lib/files/mimetypes",
   "crates/lib/files/videos",
   "crates/lib/idempotency",
+  "crates/lib/image_algorithms",
   "crates/lib/jobs_common",
   "crates/lib/logging",
   "crates/lib/markdown",
@@ -117,3 +122,35 @@ members = [
   # see usage: https://crates.io/crates/cargo-hakari
   "crates/my-workspace-hack",
 ]
+
+[workspace.package]
+# Common inheritable settings for monorepo packages
+publish = false
+
+[workspace.dependencies]
+# Hakari workspace hack (faster compile times)
+my-workspace-hack = { path = "crates/my-workspace-hack" }
+
+# External libs
+# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
+# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
+anyhow = "1.0.97"
+chrono = "0.4.40"
+log = "0.4.27"
+once_cell = "1.21.3"
+rand = "0.9.0"
+reqwest = { version = "0.12.15", features = ["json", "cookies"] }
+serde = {  version = "1.0.219", features = ["derive"] }
+serde_derive = "1.0.219"
+serde_json = "1.0.140"
+serde_yaml = "0.9.1"
+toml = "0.7.4"
+
+# Internal libs
+crockford = { path = "crates/lib/data/crockford" }
+errors = { path = "crates/lib/errors" }
+filesys = { path = "crates/lib/files/filesys" }
+memory_caching = { path = "crates/lib/caching/memory_caching" }
+openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
+primitives = { path = "crates/lib/data/primitives" }
+subprocess_common = { path = "crates/lib/subprocess_common" }

crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_type.rs

@@ -40,6 +40,10 @@ pub enum InferenceJobType {
   #[serde(rename = "studio_gen2")]
   StudioGen2,
 
+  /// Sora GPT 4o image gen
+  #[serde(rename = "image_gen_api")]
+  ImageGenApi,
+
   /// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
   #[serde(rename = "convert_fbx_gltf")]
   ConvertFbxToGltf,
@@ -108,6 +112,7 @@ impl InferenceJobType {
       Self::GptSovits => "gpt_sovits",
       Self::ComfyUi => "comfy_ui",
       Self::StudioGen2 => "studio_gen2",
+      Self::ImageGenApi => "image_gen_api",
       Self::ConvertFbxToGltf => "convert_fbx_gltf",
       Self::MocapNet => "mocap_net",
       Self::RvcV2 => "rvc_v2",
@@ -132,6 +137,7 @@ impl InferenceJobType {
       "gpt_sovits" => Ok(Self::GptSovits),
       "comfy_ui" => Ok(Self::ComfyUi),
       "studio_gen2" => Ok(Self::StudioGen2),
+      "image_gen_api" => Ok(Self::ImageGenApi),
       "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
       "mocap_net" => Ok(Self::MocapNet),
       "rvc_v2" => Ok(Self::RvcV2),
@@ -159,6 +165,7 @@ impl InferenceJobType {
       Self::GptSovits,
       Self::ComfyUi,
       Self::StudioGen2,
+      Self::ImageGenApi,
       Self::ConvertFbxToGltf,
       Self::MocapNet,
       Self::RvcV2,
@@ -197,6 +204,7 @@ mod tests {
       assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
       assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
       assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
+      assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
       assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
       assert_serialization(InferenceJobType::MocapNet, "mocap_net");
       assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
@@ -220,6 +228,7 @@ mod tests {
       assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
       assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
       assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
+      assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
       assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
       assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
       assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
@@ -243,6 +252,7 @@ mod tests {
       assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
       assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
       assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
+      assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
       assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
       assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
       assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
@@ -261,7 +271,7 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceJobType::all_variants();
-      assert_eq!(variants.len(), 16);
+      assert_eq!(variants.len(), 17);
       assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
@@ -269,6 +279,7 @@ mod tests {
       assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));

crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs

@@ -33,6 +33,10 @@ pub enum InferenceModelType {
   #[serde(rename = "seed_vc")]
   SeedVc,
 
+  /// NB: This is for Sora GPT 4o image gen
+  #[serde(rename = "image_gen_api")]
+  ImageGenApi,
+
   #[serde(rename = "tacotron2")]
   Tacotron2,
   #[serde(rename = "vits")]
@@ -70,6 +74,7 @@ impl InferenceModelType {
       Self::VallEX => "vall_e_x",
       Self::RerenderAVideo => "rerender_a_video",
       Self::StableDiffusion => "stable_diffusion",
+      Self::ImageGenApi => "image_gen_api",
       Self::SeedVc => "seed_vc",
       Self::MocapNet => "mocap_net",
       Self::StyleTTS2 => "styletts2",
@@ -90,6 +95,7 @@ impl InferenceModelType {
       "vall_e_x" => Ok(Self::VallEX),
       "rerender_a_video" => Ok(Self::RerenderAVideo),
       "stable_diffusion" => Ok(Self::StableDiffusion),
+      "image_gen_api" => Ok(Self::ImageGenApi),
       "mocap_net" => Ok(Self::MocapNet),
       "styletts2" => Ok(Self::StyleTTS2),
       "comfy_ui" => Ok(Self::ComfyUi),
@@ -113,6 +119,7 @@ impl InferenceModelType {
       Self::VallEX,
       Self::RerenderAVideo,
       Self::StableDiffusion,
+      Self::ImageGenApi,
       Self::MocapNet,
       Self::StyleTTS2,
       Self::ConvertFbxToGltf,
@@ -140,6 +147,7 @@ mod tests {
       assert_serialization(InferenceModelType::VallEX, "vall_e_x");
       assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
       assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
+      assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
       assert_serialization(InferenceModelType::MocapNet, "mocap_net");
       assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
       assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
@@ -159,6 +167,7 @@ mod tests {
       assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
       assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
       assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
+      assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
       assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
       assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
       assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
@@ -176,6 +185,7 @@ mod tests {
       assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
       assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
       assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
+      assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
       assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
       assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
       assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
@@ -187,17 +197,18 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceModelType::all_variants();
-      assert_eq!(variants.len(), 13);
+      assert_eq!(variants.len(), 15);
       assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
-      assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
+      assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
+      assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));

crates/_types/enums_public/src/by_table/generic_inference_jobs/public_inference_job_type.rs

@@ -91,6 +91,10 @@ pub enum PublicInferenceJobType {
   /// Re-render a video is a video style transfer algorithm. We developed code
   /// around it, but chose to develop AnimateDiff / ComfyUI support instead.
   RerenderAVideo,
+
+  /// Sora GPT 4o image gen
+  #[serde(rename = "image_gen_api")]
+  ImageGenApi,
 }
 
 /// NB: Legacy API for older code.
@@ -119,6 +123,7 @@ impl PublicInferenceJobType {
       InferenceJobType::Unknown => Self::Unknown,
       InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
       InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
+      InferenceJobType::ImageGenApi => Self::ImageGenApi,
     }
   }
   
@@ -140,6 +145,7 @@ impl PublicInferenceJobType {
       Self::SeedVc => InferenceJobType::SeedVc,
       Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
       Self::StableDiffusion => InferenceJobType::StableDiffusion,
+      Self::ImageGenApi => InferenceJobType::ImageGenApi,
       Self::StyleTTS2 => InferenceJobType::StyleTTS2,
       Self::Tacotron2 => InferenceJobType::Tacotron2,
       Self::Unknown => InferenceJobType::Unknown,

crates/desktop/ml_models/Cargo.toml

@@ -20,6 +20,7 @@ normal = ["my-workspace-hack"]
 
 # Internal
 filesys = { path = "../../lib/files/filesys" }
+image_algorithms = { path = "../../lib/image_algorithms" }
 memory_caching = { path = "../../lib/caching/memory_caching" }
 ml_weights_registry = { path = "../ml_weights_registry" }
 primitives = { path = "../../lib/data/primitives" }

crates/desktop/ml_models/src/ml/background_removal/remove_image_background.rs

@@ -3,6 +3,7 @@ use image::{imageops, DynamicImage};
 use ndarray::{Array, CowArray};
 use ort::Value;
 use std::path::Path;
+use image_algorithms::remove_image_border::remove_image_border;
 
 pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: DynamicImage) -> anyhow::Result<DynamicImage> {
   // NB: Real time needed cuDNN 9, but this uses cuDNN 8:
@@ -11,6 +12,10 @@ pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: Dynam
   // update-alternatives: using /usr/include/x86_64-linux-gnu/cudnn_v8.h to provide /usr/include/cudnn.h (libcudnn) in manual mode
   let session = onnx_session(model_path.as_ref())?;
   let image = process_dynamic_image(&session, image)?;
+  
+  // Crop transparent border cruft away.
+  let image = remove_image_border(&image)?;
+  
   Ok(image)
 }
 

crates/desktop/ml_models/src/ml/flux/flux_pipeline.rs

@@ -0,0 +1,297 @@
+use std::path::Path;
+use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
+
+use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
+use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
+use crate::ml::model_cache::ModelCache;
+use crate::ml::prompt_cache::PromptCache;
+use candle_nn::VarBuilder;
+use crate::ml::model_config::ModelConfig;
+use anyhow::{anyhow, Error as E, Result};
+use candle_core::{DType, IndexOp, Tensor, D, Device, Module};
+use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
+use candle_transformers::models::stable_diffusion::lcm::LCMScheduler;
+use image::DynamicImage;
+use log::{debug, info};
+use rand::Rng;
+use tauri::{AppHandle, Emitter};
+use candle_transformers::models::{clip, t5, flux};
+use tokenizers::Tokenizer;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FluxModel {
+  Schnell,
+  Dev,
+}
+
+pub struct FluxPipelineArgs<'a> {
+  pub image: &'a DynamicImage,
+  pub prompt: String,
+  pub model_cache: &'a ModelCache,
+  pub prompt_cache: &'a PromptCache,
+  pub app: &'a AppHandle,
+  pub flux_model: FluxModel,
+  pub active_blocks: Option<usize>,
+  pub use_cuda_stream: Option<bool>,
+  pub prefetch_next_batch: Option<bool>,
+  pub use_full_gpu: Option<bool>,
+  pub img2img_strength: Option<f64>,
+  pub seed: Option<u64>,
+  pub use_quantized_model: Option<bool>,
+  pub model_config: &'a ModelConfig,
+  pub flux_dev_or_schnell_path: &'a Path,
+  pub flux_schnell_quantized_gguf: &'a Path,
+  pub flux_autoencoder_path: &'a Path,
+  pub google_t5_model_path: &'a Path,
+  pub google_t5_config_path: &'a Path,
+  pub lmz_t5_tokenizer_path: &'a Path,
+  pub openai_clip_model_path: &'a Path,
+  pub openai_clip_tokenizer_json_path: &'a Path,
+}
+
+pub fn flux_pipeline(args: FluxPipelineArgs<'_>) -> Result<RgbImage> {
+  let FluxPipelineArgs { 
+    prompt, 
+    model_cache, 
+    prompt_cache, 
+    app, 
+    image, 
+    flux_model, 
+    active_blocks, 
+    use_cuda_stream, 
+    prefetch_next_batch, 
+    use_full_gpu, 
+    img2img_strength, 
+    seed, 
+    use_quantized_model, 
+    model_config ,
+    flux_dev_or_schnell_path,
+    flux_schnell_quantized_gguf,
+    flux_autoencoder_path,
+    google_t5_model_path,
+    google_t5_config_path,
+    lmz_t5_tokenizer_path,
+    openai_clip_model_path,
+    openai_clip_tokenizer_json_path,
+  } = args;
+
+  if use_quantized_model.unwrap_or(false) {
+    #[cfg(feature = "cuda")]
+    candle_core::quantized::cuda::set_force_dmmv(false);
+  }
+
+  let height = 1024;
+  let width = 1024;
+
+  let quantized = use_quantized_model.unwrap_or(false);
+  let device = model_config.device.clone();
+  let dtype = device.bf16_default_to_f32();
+
+  info!("Starting image generation with FLUX model:");
+  info!("  Model: {:?}", flux_model);
+  info!("  Prompt: {}", prompt);
+  info!("  Dimensions: {}x{}", width, height);
+  info!("  Device: {:?}", model_config.device);
+
+  let seed_value = seed.unwrap_or_else(|| rand::thread_rng().gen());
+  device.set_seed(seed_value)?;
+  
+  info!("Using seed: {}", seed_value);
+
+  // Setup FLUX model configuration
+  let flux_config = match flux_model {
+    FluxModel::Dev => flux::model::Config::dev(),
+    FluxModel::Schnell => flux::model::Config::schnell(),
+  };
+
+  // Get T5 embeddings
+  let t5_emb = {
+    info!("Generating T5 embeddings for prompt...");
+    let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[google_t5_model_path], dtype , &device)? };
+    let config = std::fs::read_to_string(google_t5_config_path)?;
+    let config: t5::Config = serde_json::from_str(&config)?;
+
+    let mut model = t5::T5EncoderModel::load(vb, &config)?;
+    let tokenizer = Tokenizer::from_file(lmz_t5_tokenizer_path).map_err(E::msg)?;
+
+    let mut tokens = tokenizer.encode(prompt.as_str(), true).map_err(E::msg)?.get_ids().to_vec();
+    tokens.resize(256, 0);
+    let input_token_ids = Tensor::new(&tokens[..], &device)?.unsqueeze(0)?;
+
+    model.forward(&input_token_ids)?
+  };
+  
+  info!("T5 embeddings generated");
+
+  // Get CLIP embeddings
+  let clip_emb = {
+    info!("Generating CLIP embeddings for prompt...");
+    let vb =
+        unsafe { VarBuilder::from_mmaped_safetensors(&[openai_clip_model_path], dtype, &device)? };
+    // https://huggingface.co/openai/clip-vit-large-patch14/blob/main/config.json
+    let config = clip::text_model::ClipTextConfig {
+        vocab_size: 49408,
+        projection_dim: 768,
+        activation: clip::text_model::Activation::QuickGelu,
+        intermediate_size: 3072,
+        embed_dim: 768,
+        max_position_embeddings: 77,
+        pad_with: None,
+        num_hidden_layers: 12,
+        num_attention_heads: 12,
+    };
+    let model =
+        clip::text_model::ClipTextTransformer::new(vb.pp("text_model"), &config)?;
+    let tokenizer = Tokenizer::from_file(openai_clip_tokenizer_json_path).map_err(E::msg)?;
+    let tokens = tokenizer
+        .encode(prompt.as_str(), true)
+        .map_err(E::msg)?
+        .get_ids()
+        .to_vec();
+    let input_token_ids = Tensor::new(&tokens[..], &device)?.unsqueeze(0)?;
+    model.forward(&input_token_ids)?
+  };
+  info!("CLIP embeddings generated");
+
+  // Process input image to tensor
+  info!("Converting input image to tensor...");
+  let img_tensor = dynamic_image_to_tensor(image, &device, dtype)?;
+  info!("Image tensor shape: {:?}", img_tensor.shape());
+
+  // Create initial noise - this is important to match the expected format
+  info!("Generating initial noise for FLUX...");
+  let initial_tensor = flux::sampling::get_noise(1, height, width, &device)?.to_dtype(dtype)?;
+  info!("Initial tensor shape: {:?}", initial_tensor.shape());
+  
+  // Blend initial noise with input image if img2img_strength is specified
+  let initial_tensor = if let Some(strength) = img2img_strength {
+    if strength > 0.0 && strength < 1.0 {
+      info!("Using img2img with strength: {}", strength);
+      // Reshape the input tensor to match the noise tensor dimensions
+      let adjusted_img = if img_tensor.shape() != initial_tensor.shape() {
+        info!("Reshaping input image to match noise dimensions...");
+        // You may need more complex reshaping logic here depending on your use case
+        let (_, c, h, w) = initial_tensor.dims4()?;
+        img_tensor.upsample_nearest2d(h, w)?
+      } else {
+        img_tensor
+      };
+      
+      // Blend the noise and the image according to strength
+      let strength_tensor = Tensor::new(strength as f32, &device)?;
+      let inverse_strength = Tensor::new(1.0 - strength as f32, &device)?;
+      let weighted_noise = (&initial_tensor * &strength_tensor)?;
+      let weighted_img = (&adjusted_img * &inverse_strength)?;
+      (&weighted_noise + &weighted_img)?
+    } else {
+      initial_tensor
+    }
+  } else {
+    initial_tensor
+  };
+  info!("Final initial tensor shape: {:?}", initial_tensor.shape());
+
+  // Create FLUX state
+  info!("Creating FLUX state...");
+  let state = if quantized {
+    flux::sampling::State::new(
+        &t5_emb.to_dtype(DType::F32)?,
+        &clip_emb.to_dtype(DType::F32)?,
+        &initial_tensor.to_dtype(DType::F32)?,
+    )?
+  } else {
+    flux::sampling::State::new(&t5_emb, &clip_emb, &initial_tensor)?
+  };
+  info!("{state:?}");
+
+  // Set appropriate timesteps based on model
+  let timesteps = match flux_model {
+    FluxModel::Dev => flux::sampling::get_schedule(28, Some((state.img.dim(1)?, 0.5, 1.15))),
+    FluxModel::Schnell => flux::sampling::get_schedule(4, None),
+  };
+
+  info!("Starting FLUX denoising with timesteps: {:?}", timesteps);
+
+  // Actual model loading and inference
+  let img_latent = if quantized {
+    // Quantized model path
+    info!("Loading quantized FLUX model...");
+    let model_file = match flux_model {
+      FluxModel::Schnell => flux_schnell_quantized_gguf,
+      FluxModel::Dev => return Err(anyhow!("Quantized model not available for FLUX.1-dev")),
+    };
+
+    info!("Loading quantized model from {}", model_file.to_str().unwrap());
+
+    let vb = candle_transformers::quantized_var_builder::VarBuilder::from_gguf(model_file, &device)?;
+    let mut model = flux::quantized_model::Flux::new(&flux_config, vb)?;
+    info!("Quantized model loaded on {:?}", device);
+
+    // Run denoising
+    info!("Running denoising...");
+    let img_latent = flux::sampling::denoise(&mut model, &state.img, &state.img_ids, &state.txt, &state.txt_ids, &state.vec, &timesteps, 4.)?.to_dtype(dtype)?;
+    info!("Denoising completed");
+    img_latent
+  } else {
+    // Full precision model
+    info!("Loading FLUX model...");
+
+    let mut model = if use_full_gpu.unwrap_or(false) {
+      // Load entire model on GPU
+      info!("Loading entire model directly to GPU...");
+      let gpu_vb = unsafe { VarBuilder::from_mmaped_safetensors(&[flux_dev_or_schnell_path], dtype, &device)? };
+      let mut model = flux::model::Flux::new(&flux_config, gpu_vb)?;
+      model.set_use_device_management(false);
+      model
+    } else {
+      // Use memory-efficient approach with blocks on CPU
+      info!("Using memory-efficient approach with blocks on CPU...");
+      let cpu_device = candle_core::Device::Cpu;
+      let cpu_vb = unsafe { VarBuilder::from_mmaped_safetensors(&[flux_dev_or_schnell_path], dtype, &cpu_device)? };
+      let mut model = flux::model::Flux::new_with_gpu_core(&flux_config, cpu_vb, &device)?;
+
+      // Set active blocks
+      let active_blocks_count = active_blocks.unwrap_or(1);
+      model.set_max_active_blocks(active_blocks_count);
+      info!("Using {} active blocks on GPU", active_blocks_count);
+
+      // Enable prefetching if requested
+      if prefetch_next_batch.unwrap_or(false) {
+        model.set_prefetch_next_batch(true);
+        info!("Prefetching enabled - will overlap transfers with computation");
+      }
+
+      model
+    };
+
+    // Run denoising
+    flux::sampling::denoise(&mut model, &state.img, &state.img_ids, &state.txt, &state.txt_ids, &state.vec, &timesteps, 4.)?
+  };
+
+  info!("FLUX denoising completed, unpacking latent image...");
+  let img_unpacked = flux::sampling::unpack(&img_latent, height, width)?;
+
+  // Load autoencoder for final decoding
+  info!("Loading autoencoder for final decoding...");
+  
+  let ae_config = match flux_model {
+    FluxModel::Dev => flux::autoencoder::Config::dev(),
+    FluxModel::Schnell => flux::autoencoder::Config::schnell(),
+  };
+
+  let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[flux_autoencoder_path], dtype, &device)? };
+  let ae_model = flux::autoencoder::AutoEncoder::new(&ae_config, vb)?;
+
+  info!("Decoding final image...");
+  let img = ae_model.decode(&img_unpacked)?;
+
+  // Convert tensor to image
+  info!("Converting tensor to image...");
+  let img = ((img.clamp(-1f32, 1f32)? + 1.0)? * 127.5)?.to_dtype(candle_core::DType::U8)?;
+
+  // Convert to RGB image
+  let image = tensor_to_image_buffer(&img.i(0)?)?;
+
+  info!("FLUX pipeline completed successfully");
+  Ok(image)
+}

crates/desktop/ml_models/src/ml/flux/mod.rs

@@ -0,0 +1 @@
+pub mod flux_pipeline;

crates/desktop/ml_models/src/ml/mod.rs

@@ -1,4 +1,5 @@
 pub mod background_removal;
+pub mod flux;
 pub mod image;
 pub mod model_cache;
 pub mod model_config;

crates/desktop/ml_models/src/ml/stable_diffusion/lcm_pipeline.rs

@@ -20,12 +20,14 @@ use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
 use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
 use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
 use image::DynamicImage;
-use log::info;
+use log::{debug, error, info};
 use ml_weights_registry::weights_registry::weights::{LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
 use rand::Rng;
 
 const DEFAULT_STRENGTH : f64 = 75.0;
 
+const DEFAULT_SEED : u64 = 42;
+
 pub struct Args<'a, P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>> {
   pub image: &'a DynamicImage,
   pub prompt: String,
@@ -85,10 +87,9 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
   // Use LCM Scheduler instead of Euler Ancestral for better speed and quality
   let mut scheduler = LCMScheduler::new(scheduler_steps, img2img_strength, LCMSchedulerConfig::default())?;
 
-  let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
+  //let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
+  let seed = maybe_seed.unwrap_or(DEFAULT_SEED);
 
-  let seed = 42;
-  
   device.set_seed(seed)?;
 
   let guidance_scale = match cfg_scale {
@@ -129,13 +130,13 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
     tensor
   };
 
-  info!("Text embeddings shape: {:?}", text_embeddings.shape());
+  debug!("Text embeddings shape: {:?}", text_embeddings.shape());
 
-  info!("Loading input image into tensor...");
+  debug!("Loading input image into tensor...");
 
   let input_image = dynamic_image_to_tensor(image, &device, dtype)?;
 
-  info!("Reference image shape: {:?}", input_image.shape());
+  debug!("Reference image shape: {:?}", input_image.shape());
 
   let vae_scale = get_vae_scale(sd_version);
 
@@ -161,7 +162,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
   let unet = match maybe_unet {
     Some(unet) => unet,
     None => {
-      info!("No unet found in cache; loading...");
+      info!("Building UNET model from file {:?}...", unet_path.as_ref());
 
       let in_channels = match sd_version {
         StableDiffusionVersion::XlInpaint | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::V1_5Inpaint => 9,
@@ -217,16 +218,17 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
   let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &device, dtype)?;
 
   // Generate latents from input image using the LCM approach
-  info!("Generating latents from input image...");
+  debug!("Generating latents from input image...");
   let init_latents = (init_latent_dist.sample()? * vae_scale)?;
-  info!("Initial latents shape: {:?}", init_latents.shape());
+
+  debug!("Initial latents shape: {:?}", init_latents.shape());
 
   // Calculate timesteps for LCM with proper img2img handling
   let timesteps = LCMScheduler::get_timesteps_for_steps(scheduler_steps, img2img_strength);
   let t_start = timesteps[0]; // First denoising step
 
   // Add noise at the appropriate timestep for img2img
-  info!("Adding noise to latents for t_start={}", t_start);
+  debug!("Adding noise to latents for t_start={}", t_start);
   let noise = init_latents.randn_like(0f64, 1f64)?;
   let latents = scheduler.add_noise(&init_latents, noise, t_start)?;
 
@@ -244,7 +246,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
     let mut noise_pred = match unet.forward_with_guidance(&latent_model_input, timestep as f64, &text_embeddings, Some(&guidance_scale_embedding)) {
       Ok(pred) => pred,
       Err(e) => {
-        info!("UNet inference failed with error: {}", e);
+        error!("UNet inference failed with error: {}", e);
         return Err(anyhow::anyhow!("UNet inference failed: {}", e));
       },
     };
@@ -259,21 +261,19 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
     latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, scheduler_steps)?;
   }
 
-  info!("Diffusion process completed, decoding image...");
+  debug!("Diffusion process completed, decoding image...");
   let image = vae.decode(&(latents / vae_scale)?)?;
 
-  info!("VAE decode completed");
-
-  info!("Scaling image back");
+  debug!("Scaling image back");
   let image = ((image / 2.)? + 0.5)?;
 
-  info!("Normalized image values");
+  debug!("Normalized image values");
   let image = (image.clamp(0f32, 1.)? * 255.)?;
 
-  info!("Convert to int8");
+  debug!("Convert to int8");
   let image = image.to_dtype(DType::U8)?;
 
-  info!("Converted to 8-bit format");
+  debug!("Converted to 8-bit format");
 
   let image = tensor_to_image_buffer(&image.i(0)?)?;
 

crates/desktop/ml_models/src/ml/stable_diffusion/stable_diffusion_pipeline.rs

@@ -59,14 +59,6 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
         clip_weights_path,
     } = args;
     
-    println!("Starting image generation with the following configuration:");
-    println!("  Model: {:?}", sd_version);
-    println!("  Prompt: {}", prompt);
-    println!("  Steps: {}", scheduler_steps);
-    println!("  Device: {:?}", device);
-
-    println!("Model dimensions: {}x{}", sd_config.width, sd_config.height);
-
     // TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
     //  This is a target for performance improvement
     //  See: https://github.com/huggingface/candle/issues/1331

crates/desktop/ml_weights_registry/src/weights_registry/flux_weights.rs

@@ -0,0 +1,80 @@
+use crate::weights_registry::weight_descriptor::{WeightDescriptor, WeightFunction};
+use crate::weights_registry::weight_descriptor_builder::weight;
+
+
+/// https://huggingface.co/black-forest-labs/FLUX.1-dev
+pub const FLUX_DEV : WeightDescriptor = weight!(
+  "Flux-1 Dev",
+  "flux1-dev.safetensors",
+  WeightFunction::ImageGeneration
+);
+
+// TODO(bt): I think the autoencoder for Dev and Schnell are the same.
+//  They have the same checksum unless I mixed up the files when downloading.
+
+/// https://huggingface.co/black-forest-labs/FLUX.1-dev
+pub const FLUX_DEV_AUTOENCODER : WeightDescriptor = weight!(
+  "Flux-1 Dev (Autoencoder)",
+  "flux1-dev.ae.safetensors",
+  WeightFunction::ImageGeneration
+);
+
+/// https://huggingface.co/black-forest-labs/FLUX.1-schnell
+pub const FLUX_SCHNELL : WeightDescriptor = weight!(
+  "Flux-1 Schnell",
+  "flux1-schnell.safetensors",
+  WeightFunction::ImageGeneration
+);
+
+/// https://huggingface.co/black-forest-labs/FLUX.1-schnell
+pub const FLUX_SCHNELL_AUTOENCODER : WeightDescriptor = weight!(
+  "Flux-1 Schnell (Autoencoder)",
+  "flux1-schnell.ae.safetensors",
+  WeightFunction::ImageGeneration
+);
+
+/// https://huggingface.co/lmz/candle-flux
+pub const LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF : WeightDescriptor = weight!(
+  "LMZ Flux Schnell Quantized",
+  "lmz_candle_flux_flux1-schnell.gguf",
+  WeightFunction::ImageGeneration
+);
+
+/// https://huggingface.co/google/t5-v1_1-xxl/tree/main
+/// NB: I couldn't find the safetensors file here
+/// NB: This should live under "refs/pr/2"
+pub const GOOGLE_T5_V1_1_XXL_MODEL : WeightDescriptor = weight!(
+  "Google T5 v1.1 (XXL)",
+  "google_t5_v1_1_xxl_model.safetensors",
+  WeightFunction::TextEncoder
+);
+
+/// https://huggingface.co/google/t5-v1_1-xxl/tree/main
+/// NB: I couldn't find the safetensors file here
+/// NB: This should live under "refs/pr/2"
+pub const GOOGLE_T5_V1_1_XXL_CONFIG : WeightDescriptor = weight!(
+  "Google T5 v1.1 (XXL) Config",
+  "google_t5_v1_1_xxl_config.json",
+  WeightFunction::TextEncoder
+);
+
+/// https://huggingface.co/lmz/mt5-tokenizers
+pub const LMZ_T5_TOKENIZER_JSON : WeightDescriptor = weight!(
+  "LMZ T5 Tokenizer",
+  "lmz_mt5_tokenizers_t5-v1_1-xxl.tokenizer.json",
+  WeightFunction::TextTokenizer
+);
+
+/// https://huggingface.co/openai/clip-vit-large-patch14/tree/main
+pub const OPENAI_CLIP_VIT_P14_MODEL : WeightDescriptor = weight!(
+  "OpenAI CLIP VIT (patch 14) model",
+  "openai_clip-vit-large-patch14.model.safetensors",
+  WeightFunction::TextTokenizer
+);
+
+/// https://huggingface.co/openai/clip-vit-large-patch14/tree/main
+pub const OPENAI_CLIP_VIT_P14_TOKENIZER_JSON: WeightDescriptor = weight!(
+  "OpenAI CLIP VIT (patch 14) tokenizer",
+  "openai_clip-vit-large-patch14.tokenizer.json",
+  WeightFunction::TextTokenizer
+);

crates/desktop/ml_weights_registry/src/weights_registry/mod.rs

@@ -1,3 +1,4 @@
 mod weight_descriptor_builder;
+pub mod flux_weights;
 pub mod weight_descriptor;
 pub mod weights;

crates/desktop/ml_weights_registry/src/weights_registry/weight_descriptor.rs

@@ -5,6 +5,7 @@
 
 #[derive(Clone, Copy)]
 pub enum WeightFunction {
+  ImageGeneration,
   ImageSegmentation,
   TextEncoder,
   TextTokenizer,
@@ -29,6 +30,7 @@ pub struct WeightDescriptor {
 impl WeightFunction {
   pub fn as_descriptive_str(&self) -> &'static str {
     match self {
+      WeightFunction::ImageGeneration => "image generation",
       WeightFunction::ImageSegmentation => "image segmentation",
       WeightFunction::TextEncoder => "text encoder",
       WeightFunction::TextTokenizer => "text tokenizer",

crates/desktop/tauri-4o/Cargo.toml

@@ -0,0 +1,74 @@
+[package]
+name = "artcraft"
+version = "0.1.0"
+description = "Artcraft"
+authors = ["you"]
+license = ""
+repository = ""
+edition = "2021"
+rust-version = "1.77.2"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+name = "app_lib"
+crate-type = ["staticlib", "cdylib", "rlib"]
+
+[build-dependencies]
+tauri-build = { version = "2.1.1", features = [] }
+
+[dependencies]
+# Workspace hack (faster compile times)
+my-workspace-hack = { workspace =  true }
+
+# Internal
+crockford = { workspace = true }
+errors = { workspace = true }
+filesys = { workspace = true }
+memory_caching = { workspace = true }
+openai_sora_client = { workspace = true }
+primitives = { workspace = true }
+subprocess_common = { workspace = true }
+
+# External
+#openssl = { version = "0.10", features = ["vendored"] }
+# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
+anyhow = { workspace = true }
+base64 = "0.22.1"
+bytes = "1.10"
+chrono = { workspace = true }
+const_format = "0.2.34"
+directories = "6.0.0"
+dirs = "6.0.0"
+futures = "0.3"
+image = "0.25.5"
+log = { workspace = true }
+once_cell = { workspace = true }
+rand = { workspace = true }
+reqwest = { workspace = true, features = ["multipart", "stream"], default-features = false }
+serde = { workspace = true }
+serde_derive = { workspace = true }
+serde_json = { workspace = true }
+serde_yaml = { workspace = true }
+tauri = { version = "2.4.1", features = ["unstable"] }
+tauri-plugin-log = "2.3.1"
+tempdir = "0.3.7"
+tempfile = "3.18.0"
+tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
+tokio-util = { version = "0.7", features = ["codec"] }
+
+[dev-dependencies]
+libloading = "0.8.6"
+serial_test = "3.2.0"
+assertor = "0.0.3"
+
+[profile.dev]
+incremental = true # Compile your binary in smaller steps.
+
+[profile.release]
+codegen-units = 1 # Allows LLVM to perform better optimization.
+lto = true # Enables link-time-optimizations.
+opt-level = 3 # `s` prioritizes small binary size. Use `3` if you prefer speed.
+panic = "abort" # Higher performance by disabling panic handlers.
+strip = true # Ensures debug symbols are removed.

crates/desktop/tauri-4o/src/commands/image_generation_command.rs

@@ -0,0 +1,90 @@
+use std::fs::read_to_string;
+use base64::prelude::BASE64_STANDARD;
+use errors::AnyhowResult;
+use base64::Engine;
+use image::codecs::png::{CompressionType, FilterType, PngEncoder};
+use image::{DynamicImage, ImageReader};
+use std::io::Cursor;
+use log::{error, info};
+use tauri::{AppHandle, Manager, State};
+use openai_sora_client::credentials::SoraCredentials;
+use openai_sora_client::image_gen::common::{ImageSize, NumImages};
+use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
+use openai_sora_client::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
+use openai_sora_client::upload::upload_media_from_file::SoraMediaUploadRequest;
+use crate::state::app_dir::AppDataRoot;
+
+#[tauri::command]
+pub async fn image_generation_command(
+  image: &str,
+  prompt: &str,
+  app: AppHandle,
+  app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+  info!("image_generation_command called; processing image...");
+
+  let bytes = BASE64_STANDARD.decode(image)
+    .map_err(|err| format!("Base64 decode error: {}", err))?;
+
+  generate_image(bytes, prompt, &app_data_root)
+    .await
+    .map_err(|err| {
+      error!("error: {:?}", err);
+      "there was an error".to_string()
+    })?;
+
+  Ok("success".to_string())
+}
+
+pub async fn generate_image(file_bytes: Vec<u8>, prompt: &str, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
+  let sora_credentials = get_credentials(app_data_root)?;
+
+  let filename = "image.png".to_string();
+
+  let response= sora_media_upload_from_bytes(file_bytes, filename, &sora_credentials)
+      .await
+      .map_err(|err| {
+        error!("Failed to upload image to Sora: {:?}", err);
+        err
+      })?;
+
+  let sora_media_tokens = vec![response.id];
+
+  let response = sora_image_gen_remix(SoraImageGenRemixRequest {
+    prompt: prompt.to_string(),
+    num_images: NumImages::One,
+    image_size: ImageSize::Square,
+    sora_media_tokens: sora_media_tokens.clone(),
+    credentials: &sora_credentials,
+  }).await
+      .map_err(|err| {
+        error!("Failed to call Sora image generation: {:?}", err);
+        err
+      })?;
+
+  println!(">> TASK ID: {:?} ", response.task_id);
+
+  Ok(())
+}
+
+fn get_credentials(app_data_root: &AppDataRoot) -> AnyhowResult<SoraCredentials> {
+  let cookie_file = app_data_root.get_sora_cookie_file_path();
+
+  let bearer = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/bearer.txt")?
+      .trim()
+      .to_string();
+
+  let cookie= read_to_string(cookie_file)?
+      .trim()
+      .to_string();
+
+  let sentinel = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/sentinel.txt")?
+      .trim()
+      .to_string();
+
+  Ok(SoraCredentials {
+    bearer_token: bearer,
+    cookie: cookie,
+    sentinel: Some(sentinel),
+  })
+}

crates/desktop/tauri-4o/src/commands/mod.rs

@@ -0,0 +1,3 @@
+pub mod flip_image;
+pub mod image_generation_command;
+pub mod open_login_command;

crates/desktop/tauri-4o/src/commands/open_login_command.rs

@@ -0,0 +1,72 @@
+use std::time::Duration;
+use log::{error, info};
+use once_cell::sync::Lazy;
+use tauri::{AppHandle, Manager, Url, WebviewUrl, WebviewWindowBuilder, WindowBuilder};
+use tokio::time::sleep_until;
+use errors::AnyhowResult;
+use crate::threads::login_thread::{LOGIN_WINDOW_NAME, SORA_LOGIN_URL};
+use crate::utils::clear_all_webview_cookies::clear_all_webview_cookies;
+
+pub const START_URL_STR: &str = "https://storyteller.ai/";
+
+pub static START_URL: Lazy<Url> = Lazy::new(|| {
+  Url::parse(START_URL_STR).expect("URL should parse")
+});
+
+pub const SORA_HOMEPAGE_URL_STR: &str = "https://sora.com/";
+
+pub static SORA_HOMEPAGE_URL : Lazy<Url> = Lazy::new(|| {
+  Url::parse(SORA_HOMEPAGE_URL_STR).expect("URL should parse")
+});
+
+#[tauri::command]
+pub async fn open_login_command(
+  app: AppHandle,
+) -> Result<String, String> {
+  info!("open_login_command called");
+
+  do_open_login(&app)
+    .await
+    .map_err(|err| {
+      error!("Error opening login: {:?}", err);
+      format!("Error opening login: {:?}", err)
+    })?;
+
+  Ok("result".to_string())
+}
+
+async fn do_open_login(app: &AppHandle) -> AnyhowResult<()> {
+  info!("Building login window...");
+
+  let url = WebviewUrl::External(START_URL.clone());
+
+  let window = WebviewWindowBuilder::new(app, LOGIN_WINDOW_NAME, url)
+      //.user_agent(openai_sora_client::credentials::USER_AGENT)
+      .always_on_top(true)
+      .title("Login to OpenAI")
+      .center()
+      .resizable(true)
+      .visible(true)
+      .closable(true)
+      .focused(true)
+      .devtools(true)
+      .build()?;
+
+  for (name, webview) in window.webviews() {
+    if name != LOGIN_WINDOW_NAME {
+      continue;
+    }
+
+    clear_all_webview_cookies(&webview)?;
+
+    // NB: We're starting to get Cloudflare protection screens. Let's try to avoid.
+    webview.navigate(SORA_HOMEPAGE_URL.clone())?;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    webview.navigate(SORA_LOGIN_URL.clone())?;
+    break;
+  }
+
+  info!("Done.");
+  Ok(())
+}

crates/desktop/tauri-4o/src/events/notification_event.rs

@@ -8,6 +8,9 @@ pub enum NotificationModelType {
   TextTokenizer,
   Unet,
   Vae,
+  Json,
+  ClipEncoder,
+  FluxModel,
 }
 
 #[derive(Clone, Serialize)]
@@ -26,5 +29,17 @@ pub enum NotificationEvent<'a> {
   ModelDownloadComplete {
     model_name: &'a str,
     model_type: NotificationModelType,
-  }
+  },
+  ModelLoading {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInference {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInferenceComplete {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
 }

crates/desktop/tauri-4o/src/lib.rs

@@ -0,0 +1,65 @@
+pub mod commands;
+pub mod events;
+pub mod state;
+pub mod threads;
+pub mod transfer;
+pub mod utils;
+
+use crate::commands::flip_image::flip_image;
+use crate::commands::image_generation_command::image_generation_command;
+use crate::commands::open_login_command::open_login_command;
+use crate::state::app_config::AppConfig;
+use crate::threads::login_thread::login_thread;
+
+use tauri_plugin_log::Target;
+use tauri_plugin_log::TargetKind;
+
+#[cfg_attr(mobile, tauri::mobile_entry_point)]
+pub fn run() {
+  // NB: Tauri wants to install the logger itself, so we can't rely on the logger crate 
+  // until the tauri runtime begins.
+  println!("Loading model config...");
+
+  let config = AppConfig::init()
+    .expect("config should load");
+
+  let app_data_root = config.app_data_root.clone();
+  let app_data_root2 = config.app_data_root.clone();
+
+  println!("Initializing backend runtime...");
+
+  tauri::Builder::default()
+    .plugin(tauri_plugin_log::Builder::new()
+      .level(log::LevelFilter::Info)
+      .targets(vec![
+        Target::new(TargetKind::Stdout),
+        Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
+      ])
+      .build())
+    .setup(|app| {
+      // TODO(bt): This is broken on windows
+      // log_environment_details();
+
+      //if cfg!(debug_assertions) {
+      //  app.handle().plugin(
+      //    tauri_plugin_log::Builder::default()
+      //      .level(log::LevelFilter::Info)
+      //      .build(),
+      //  )?;
+      //}
+      let app = app.handle().clone();
+
+      tauri::async_runtime::spawn(login_thread(app, app_data_root2));
+
+      Ok(())
+    })
+    .manage(config)
+    .manage(app_data_root)
+    .invoke_handler(tauri::generate_handler![
+      flip_image,
+      image_generation_command,
+      open_login_command,
+    ])
+    .run(tauri::generate_context!("tauri.conf.json"))
+    .expect("error while running tauri application");
+}

crates/desktop/tauri-4o/src/main.rs

@@ -0,0 +1,6 @@
+// Prevents additional console window on Windows in release, DO NOT REMOVE!!
+#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
+
+fn main() {
+  app_lib::run();
+}

crates/desktop/tauri-4o/src/state/app_config.rs

@@ -0,0 +1,58 @@
+use crate::state::app_dir::AppDataRoot;
+use crate::state::os_platform::OsPlatform;
+use crate::state::yaml_config::YamlConfig;
+
+const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
+const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;
+
+
+/// A central place to configure app-wide details.
+pub struct AppConfig {
+  pub image_height: usize,
+  pub image_width: usize,
+  pub scheduler_steps: usize,
+  pub scheduler_samples: usize,
+  
+  pub seed: Option<u64>,
+
+  pub cfg_scale: Option<f64>,
+  
+  /// Root location for application data.
+  pub app_data_root: AppDataRoot,
+}
+
+impl AppConfig {
+  
+  pub fn init() -> anyhow::Result<Self> {
+    let yaml_configs = YamlConfig::load_from_config_file_recursive();
+    
+    println!("Configs: {:?}", yaml_configs);
+    
+    let os_platform = OsPlatform::get();
+
+    let default_data_root = match os_platform {
+      OsPlatform::Linux => yaml_configs.linux_default_data_path.as_deref(),
+      OsPlatform::MacOs => yaml_configs.mac_default_data_path.as_deref(),
+      OsPlatform::Windows => yaml_configs.windows_default_data_path.as_deref(),
+    };
+    
+    println!("Possible user-defined data root: {:?}", default_data_root);
+
+    let app_data_root = match default_data_root {
+      None => AppDataRoot::create_default()?,
+      Some(path) => AppDataRoot::create_existing(path)?,
+    };
+
+    println!("App data root: {:?}", app_data_root.path());
+
+    Ok(Self {
+      image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
+      image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
+      scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
+      scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
+      seed: yaml_configs.seed,
+      cfg_scale: yaml_configs.cfg_scale,
+      app_data_root,
+    })
+  }
+}

crates/desktop/tauri-4o/src/state/app_dir.rs

@@ -0,0 +1,219 @@
+use crate::state::expanduser::expanduser;
+use crate::state::os_platform::OsPlatform;
+use anyhow::anyhow;
+use directories::UserDirs;
+use std::path::{Path, PathBuf};
+use chrono::{DateTime, Local};
+use tempdir::TempDir;
+use tempfile::{Builder, NamedTempFile};
+
+const DEFAULT_DATA_DIR : &str = "artcraft";
+const ASSETS_SUBDIRECTORY : &str = "assets";
+const WEIGHTS_SUBDIRECTORY : &str = "weights";
+
+const TEMPORARY_SUBDIRECTORY : &str = "temp";
+
+/// Note: Tauri appends ".log" to the end of the filename.
+const LOG_FILE_NAME : &str = "application_debug";
+
+/// The path to the application data directory, which includes "asset" and "weights" data.
+#[derive(Clone)]
+pub struct AppDataRoot {
+  path: PathBuf,
+  assets_dir: AppAssetsDir,
+  weights_dir: AppWeightsDir,
+  temp_dir: TemporaryDir,
+  log_file_name: PathBuf,
+  log_file_name_string: String,
+}
+
+#[derive(Clone)]
+pub struct AppAssetsDir {
+  path: PathBuf,
+}
+
+#[derive(Clone)]
+pub struct AppWeightsDir {
+  path: PathBuf,
+}
+
+#[derive(Clone)]
+pub struct TemporaryDir {
+  path: PathBuf,
+}
+
+impl AppDataRoot {
+  pub fn create_default() -> anyhow::Result<Self> {
+    let directory = get_default_data_dir()?;
+    Self::create_existing(directory)
+  }
+
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    
+    match OsPlatform::get() {
+      OsPlatform::Linux | OsPlatform::MacOs => {
+        if let Some(d) = dir.as_os_str().to_str() {
+          dir = expanduser(d)?;
+        }
+      },
+      OsPlatform::Windows => {}
+    }
+    
+    if !dir.is_dir() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir_all(&dir)?;
+    }
+
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    
+    let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
+    let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
+    let temp_dir = TemporaryDir::create_existing(dir.join(TEMPORARY_SUBDIRECTORY))?;
+    let log_file_name = dir.join(LOG_FILE_NAME);
+    let log_file_name_string = log_file_name
+        .to_str()
+        .ok_or(anyhow!("couldn't convert log path to str"))?
+        .to_string();
+
+    Ok(Self {
+      path: dir,
+      assets_dir,
+      weights_dir,
+      temp_dir,
+      log_file_name,
+      log_file_name_string,
+    })
+  }
+  
+  pub fn assets_dir(&self) -> &AppAssetsDir {
+    &self.assets_dir
+  }
+  
+  pub fn weights_dir(&self) -> &AppWeightsDir {
+    &self.weights_dir
+  }
+
+  pub fn temp_dir(&self) -> &TemporaryDir {
+    &self.temp_dir
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+
+  pub fn log_file_name(&self) -> &Path {
+    &self.log_file_name
+  }
+
+  pub fn log_file_name_str(&self) -> &str {
+    &self.log_file_name_string
+  }
+
+  pub fn get_sora_cookie_file_path(&self) -> PathBuf {
+    self.path.join("sora_cookies.txt")
+  }
+}
+
+impl AppAssetsDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+  
+  /// Return the current date asset directory
+  /// If it doesn't exist, create it.
+  pub fn make_or_get_current_date_dir(&self) -> anyhow::Result<PathBuf> {
+    let format: DateTime<Local> = Local::now();
+    let date_directory_name = format.format("%Y-%d-%m").to_string();
+    let full_path = self.path.join(date_directory_name);
+    if !full_path.exists() {
+      std::fs::create_dir(&full_path)?;
+    }
+    Ok(full_path)
+  }
+}
+
+impl AppWeightsDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn path(&self) -> &Path {
+    &self.path
+  }
+}
+
+impl TemporaryDir {
+  pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
+    let mut dir = dir.as_ref().to_path_buf();
+    match dir.canonicalize() {
+      Ok(d) => dir = d,
+      Err(err) => {
+        println!("Error canonicalizing {:?}: {}", dir, err);
+      }
+    }
+    if !dir.exists() {
+      println!("Creating directory {:?}", dir);
+      std::fs::create_dir(&dir)?;
+    }
+    Ok(Self {
+      path: dir,
+    })
+  }
+
+  pub fn with_prefix(&self, prefix: &str) -> anyhow::Result<TempDir> {
+    let tempdir = TempDir::new_in(&self.path, prefix)?;
+    Ok(tempdir)
+  }
+
+  pub fn new_named_temp_file(&self) -> anyhow::Result<NamedTempFile> {
+    let tempfile = Builder::new()
+      .prefix("temp_")
+      .suffix(".bin")
+      .tempfile_in(&self.path)?;
+    Ok(tempfile)
+  }
+}
+
+// eg. /home/bob/artcraft, /Users/bob/artcraft, or C:\Users\Alice\artcraft
+fn get_default_data_dir() -> anyhow::Result<PathBuf> {
+  Ok(UserDirs::new()
+      .ok_or_else(|| anyhow!("could not determine user home directory"))?
+      .home_dir()
+      .join(DEFAULT_DATA_DIR))
+}

crates/desktop/tauri-4o/src/threads/login_thread.rs

@@ -0,0 +1,103 @@
+use std::fs::OpenOptions;
+use std::io::Write;
+use anyhow::anyhow;
+use log::{error, info};
+use once_cell::sync::Lazy;
+use reqwest::Url;
+use tauri::{AppHandle, Manager, Webview};
+use errors::AnyhowResult;
+use crate::state::app_dir::AppDataRoot;
+use crate::utils::sora_webview_cookies::get_all_sora_cookies_as_string;
+
+pub const LOGIN_WINDOW_NAME: &str = "login_window";
+
+pub const SORA_LOGIN_URL_STR: &str = "https://sora.com/auth/login?callback_path=/";
+
+pub static SORA_LOGIN_URL: Lazy<Url> = Lazy::new(|| {
+  Url::parse(SORA_LOGIN_URL_STR).expect("URL should parse")
+});
+
+pub const SORA_ROOT_URL_STR: &str = "https://sora.com/";
+
+pub static SORA_ROOT_URL: Lazy<Url> = Lazy::new(|| {
+  Url::parse(SORA_ROOT_URL_STR).expect("URL should parse")
+});
+
+pub async fn login_thread(app: AppHandle, app_data_root: AppDataRoot) -> ! {
+  loop {
+    for (window_name, webview) in app.webviews() {
+      if window_name == LOGIN_WINDOW_NAME {
+        let result = check_login_window(&webview, &app_data_root).await;
+        if let Err(err) = result {
+          error!("Error checking login window: {:?}", err);
+        }
+        break;
+      }
+    }
+    tokio::time::sleep(std::time::Duration::from_millis(2_000)).await;
+  }
+}
+
+async fn check_login_window(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
+  clear_browsing_data_on_test_domain(webview)?;
+  //keep_on_task(webview)?;
+  extract_cookies_to_file(webview, app_data_root)?;
+  // TODO: Other maintenance tasks ...
+  Ok(())
+}
+
+fn keep_on_task(webview: &Webview) -> AnyhowResult<()> {
+  let url = webview.url()?;
+  let hostname= url.host()
+      .ok_or(anyhow!("no host in url"))?
+      .to_string();
+  match hostname.as_str() {
+    "auth.openai.com" => {},
+    "openai.com" => {},
+    "sora.com" => {},
+    // Third party SSO
+    "accounts.google.com" => {},
+    "accounts.youtube.com" => {},
+    "login.live.com" => {},
+    "appleid.apple.com" => {},
+    _ => {
+      info!("Non login hostname: {}", hostname);
+      webview.navigate(SORA_LOGIN_URL.clone())?;
+    }
+  }
+  Ok(())
+}
+
+/// This is just so we have a way to clear browsing data.
+fn clear_browsing_data_on_test_domain(webview: &Webview) -> AnyhowResult<()> {
+  let url = webview.url()?;
+  let hostname= url.host()
+      .ok_or(anyhow!("no host in url"))?
+      .to_string();
+  match hostname.as_str() {
+    "storyteller.ai" => {
+      info!("Clearing all browsing data...");
+      webview.clear_all_browsing_data()?;
+    }
+    _ => {}
+  }
+  Ok(())
+}
+
+// TODO(bt,2025-04-07): Heuristic to detect when logged in. Only write when logged in.
+fn extract_cookies_to_file(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
+  let cookies = get_all_sora_cookies_as_string(webview)?;
+
+  let cookie_file = app_data_root.get_sora_cookie_file_path();
+
+  let mut file = OpenOptions::new()
+      .create(true)
+      .write(true)
+      .truncate(true)
+      .open(cookie_file)?;
+
+  file.write_all(cookies.as_bytes())?;
+  file.flush()?;
+
+  Ok(())
+}

crates/desktop/tauri-4o/src/threads/mod.rs

@@ -0,0 +1 @@
+pub mod login_thread;

crates/desktop/tauri-4o/src/utils/clear_all_webview_cookies.rs

@@ -0,0 +1,7 @@
+use tauri::Webview;
+use errors::AnyhowResult;
+
+pub fn clear_all_webview_cookies(webview: &Webview) -> AnyhowResult<()> {
+  webview.clear_all_browsing_data()?;
+  Ok(())
+}

crates/desktop/tauri-4o/src/utils/mod.rs

@@ -0,0 +1,3 @@
+pub mod clear_all_webview_cookies;
+pub mod image;
+pub mod sora_webview_cookies;

crates/desktop/tauri-4o/src/utils/sora_webview_cookies.rs

@@ -0,0 +1,26 @@
+use once_cell::sync::Lazy;
+use reqwest::Url;
+use tauri::Webview;
+use tauri::webview::Cookie;
+use errors::AnyhowResult;
+
+const SORA_ROOT_COOKIE_URL_STR: &str = "https://sora.com/";
+
+static SORA_ROOT_COOKIE_URL: Lazy<Url> = Lazy::new(|| {
+  Url::parse(SORA_ROOT_COOKIE_URL_STR).expect("URL should parse")
+});
+
+pub fn get_all_sora_cookies(webview: &Webview) -> AnyhowResult<Vec<Cookie>> {
+  let cookies = webview.cookies_for_url(SORA_ROOT_COOKIE_URL.clone())?;
+  Ok(cookies)
+}
+
+pub fn get_all_sora_cookies_as_string(webview: &Webview) -> AnyhowResult<String> {
+  let cookies = get_all_sora_cookies(webview)?;
+  let cookie_string = cookies
+    .iter()
+    .map(|cookie| format!("{}={}", cookie.name(), cookie.value()))
+    .collect::<Vec<String>>()
+    .join("; ");
+  Ok(cookie_string)
+}

crates/desktop/tauri-realtime/.gitignore

@@ -0,0 +1,4 @@
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+/gen/schemas

crates/desktop/tauri-realtime/.taurignore

@@ -0,0 +1,4 @@
+# Don't restart the dev server on Rust changes
+crates/
+Cargo.toml
+Cargo.lock

crates/desktop/tauri-realtime/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "app"
+name = "tauri-realtime"
 version = "0.1.0"
 description = "A Tauri App"
 authors = ["you"]
@@ -16,11 +16,12 @@ name = "app_lib"
 crate-type = ["staticlib", "cdylib", "rlib"]
 
 [build-dependencies]
-tauri-build = { version = "2.0.4", features = [] }
+tauri-build = { version = "2.1.1", features = [] }
 
 [dependencies]
 
 # Internal
+crockford = { path = "../../lib/data/crockford" }
 filesys = { path = "../../lib/files/filesys" }
 memory_caching = { path = "../../lib/caching/memory_caching" }
 ml_weights_registry = { path = "../ml_weights_registry" }
@@ -29,12 +30,16 @@ primitives = { path = "../../lib/data/primitives" }
 # Internal ML
 # TODO: Only on Windows and Linux
 #ml_models = { path = "../ml_models", optional = true }
+# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
+cudarc = { version = "0.13.5", features = ["std", "cublas", "cublaslt", "curand", "driver", "nvrtc", "f16", "cuda-version-from-build-system", "dynamic-linking"], default-features=false }
 
 # External
 #openssl = { version = "0.10", features = ["vendored"] }
+# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
 anyhow = "1.0"
 base64 = "0.22.1"
 bytes = "1.10"
+chrono = "0.4.40"
 const_format = "0.2.34"
 directories = "6.0.0"
 dirs = "6.0.0"
@@ -43,14 +48,14 @@ half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_di
 image = "0.25.5"
 log = "0.4"
 once_cell = "1.20.3"
+ort = { version = "1.16.2", features = ["cuda", "load-dynamic"] }
 rand = "0.8.5"
-# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
 reqwest = { version = "0.12.12", features = ["stream"], default-features = false }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
-tauri = { version = "2.2.4", features = [] }
-tauri-plugin-log = "2"
+tauri = { version = "2.4.1", features = [] }
+tauri-plugin-log = "2.3.1"
 tempdir = "0.3.7"
 tempfile = "3.18.0"
 tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }

crates/desktop/tauri-realtime/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+  tauri_build::build()
+}

crates/desktop/tauri-realtime/capabilities/default.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "../gen/schemas/desktop-schema.json",
+  "identifier": "default",
+  "description": "enables the default permissions",
+  "windows": [
+    "main"
+  ],
+  "permissions": [
+    "core:default",
+    "log:default"
+  ]
+}

crates/desktop/tauri-realtime/src/endpoints/flip_image.rs

@@ -0,0 +1,36 @@
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::codecs::png::{CompressionType, FilterType, PngEncoder};
+use image::ImageReader;
+use std::io::Cursor;
+
+#[tauri::command]
+pub fn flip_image(image: &str) -> Result<String, String> {
+  println!("infer_image called; processing image...");
+
+  let bytes = BASE64_STANDARD.decode(image)
+    .map_err(|err| format!("Base64 decode error: {}", err))?;
+
+  let image = ImageReader::new(Cursor::new(bytes))
+    .with_guessed_format()
+    .map_err(|err| format!("Image format error: {}", err))?
+    .decode()
+    .map_err(|err| format!("Image decode error: {}", err))?;
+
+  let rotated_image = image.rotate180();
+
+  let mut buffer = Vec::new(); // TODO: Preallocate, better data structure
+  let mut writer = Cursor::new(&mut buffer);
+
+  let encoder = PngEncoder::new_with_quality
+    (&mut writer, CompressionType::Fast, FilterType::Adaptive);
+  
+  rotated_image.write_with_encoder(encoder)
+    .map_err(|err| format!("Image encoding error: {}", err))?;
+
+  let encoded = BASE64_STANDARD.encode(&buffer);
+
+  println!("Encoded: {:?}", encoded.split_at(10).0);
+
+  Ok(encoded)
+}

crates/desktop/tauri-realtime/src/endpoints/inpaint_image_endpoint.rs

@@ -0,0 +1,13 @@
+use log::info;
+
+
+#[tauri::command]
+pub async fn inpaint_image(
+  image: &str,
+  mask: &str,
+  prompt: String,
+) -> Result<String, String> {
+  info!("inpaint_endpoint endpoint called.");
+
+  Ok(mask.to_string())
+}

crates/desktop/tauri-realtime/src/endpoints/mod.rs

@@ -1,5 +1,7 @@
 pub mod download_models;
 pub mod flip_image;
+pub mod inpaint_image_endpoint;
 pub mod realtime_image_endpoint;
 pub mod remove_background_endpoint;
+pub mod save_image_endpoint;
 pub mod text_to_image_endpoint;

crates/desktop/tauri-realtime/src/endpoints/realtime_image_endpoint.rs

@@ -31,6 +31,7 @@ pub async fn infer_image(
   image: &str,
   prompt: Option<String>,
   strength: Option<f64>,
+  seed: Option<u64>,
   model_config: State<'_, AppConfig>,
   model_cache: State<'_, ModelCache>,
   prompt_cache: State<'_, PromptCache>,
@@ -49,12 +50,15 @@ pub async fn infer_image(
   // TODO(bt,2025-02-17): Running out of vram with full image buffer size
   let mut image = image.resize(512, 512, FilterType::CatmullRom);
 
+  let seed = seed.or(model_config.seed);
+
   #[cfg(not(target_os = "macos"))]
   {
     let result = infer_image_impl(
       &prompt,
       image,
       strength,
+      seed,
       &model_config,
       &model_cache,
       prompt_cache,
@@ -83,6 +87,7 @@ async fn infer_image_impl(
   prompt: &str,
   image: DynamicImage,
   strength: Option<f64>,
+  seed: Option<u64>,
   config: &AppConfig,
   model_cache: &ModelCache,
   prompt_cache: State<'_, PromptCache>,
@@ -106,7 +111,7 @@ async fn infer_image_impl(
     cfg_scale: config.cfg_scale,
     use_flash_attn: true,
     model_config: &config.model_config,
-    maybe_seed: None,
+    maybe_seed: seed,
     scheduler_steps: config.scheduler_steps,
     vae_path: &vae_path,
     unet_path: &unet_path,

crates/desktop/tauri-realtime/src/endpoints/save_image_endpoint.rs

@@ -0,0 +1,38 @@
+use crate::state::app_config::AppConfig;
+use crate::state::app_dir::AppDataRoot;
+use crate::stubs::model_cache::ModelCache;
+use crate::stubs::prompt_cache::PromptCache;
+use crate::utils::image::decode_base64_image::decode_base64_image;
+use chrono::Local;
+use crockford::crockford_entropy_lower;
+use image::{ImageFormat, RgbImage};
+use log::info;
+use tauri::{AppHandle, State};
+
+const FILENAME_LENGTH : usize = 10;
+
+/// This handler takes an image (as a base64 encoded string) and saves it.
+#[tauri::command]
+pub async fn save_image(
+  image: &str,
+  app_data_root: State<'_, AppDataRoot>,
+) -> Result<String, String> {
+  info!("save_image endpoint called.");
+
+  let image = decode_base64_image(image)
+    .map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
+
+  let assets_dir = app_data_root.assets_dir();
+  
+  let date_save_dir = assets_dir.make_or_get_current_date_dir()
+    .map_err(|err| format!("Couldn't create assets save directory: {}", err))?;
+
+  let entropy = crockford_entropy_lower(FILENAME_LENGTH); // TODO: These are ugly. A monotonic counter would be better.
+  let filename = format!("{}.png", entropy);
+  let filename = date_save_dir.join(filename);
+  
+  image.save_with_format(filename,  ImageFormat::Png)
+    .map_err(|err| format!("Couldn't save image: {}", err))?;
+
+  Ok("".to_string())
+}

crates/desktop/tauri-realtime/src/endpoints/text_to_image_endpoint.rs

@@ -1,23 +1,21 @@
 #[cfg(not(target_os = "macos"))]
-use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
+use ml_models::ml::flux::flux_pipeline::{flux_pipeline, FluxModel, FluxPipelineArgs};
 
 use crate::state::app_config::AppConfig;
-use crate::state::app_dir::AppDataRoot;
+use crate::state::app_dir::{AppDataRoot, AppWeightsDir};
 use crate::stubs::model_cache::ModelCache;
 use crate::stubs::prompt_cache::PromptCache;
 use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
 use image::imageops::FilterType;
 use image::{DynamicImage, ImageReader, RgbImage};
 use log::{error, info};
-use ml_weights_registry::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
 
+use ml_weights_registry::weights_registry::flux_weights::{FLUX_DEV, FLUX_SCHNELL, FLUX_SCHNELL_AUTOENCODER, GOOGLE_T5_V1_1_XXL_CONFIG, GOOGLE_T5_V1_1_XXL_MODEL, LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF, LMZ_T5_TOKENIZER_JSON, OPENAI_CLIP_VIT_P14_MODEL, OPENAI_CLIP_VIT_P14_TOKENIZER_JSON};
 use once_cell::sync::Lazy;
 use std::io::Cursor;
 use tauri::{AppHandle, State};
 
-const RANDOM_SEED: u32 = 42;
-
-const STRENGTH: f64 = 1.0;
+const STRENGTH: f64 = 100.0;
 const PNG_BYTES : &[u8] = include_bytes!("../../binary_includes/1024.png");
 
 // NB: We're just hacking a temporary "text to image" test endpoint leveraging the existing image to image modality.
@@ -35,8 +33,8 @@ pub async fn text_to_image(
   model_config: State<'_, AppConfig>,
   model_cache: State<'_, ModelCache>,
   prompt_cache: State<'_, PromptCache>,
-  app_data_root: State<'_, AppDataRoot>,
   app: AppHandle,
+  app_data_root: State<'_, AppDataRoot>,
 ) -> Result<String, String> {
   info!("text_to_image endpoint called.");
 
@@ -51,8 +49,8 @@ pub async fn text_to_image(
       &model_config,
       &model_cache,
       prompt_cache,
+      app_data_root.weights_dir(),
       app,
-      &app_data_root
     ).await;
 
     image = match result {
@@ -80,35 +78,36 @@ async fn text_to_image_impl(
   config: &AppConfig,
   model_cache: &ModelCache,
   prompt_cache: State<'_, PromptCache>,
+  weights_dir: &AppWeightsDir,
   app: AppHandle,
-  app_data_root: &AppDataRoot,
 ) -> Result<RgbImage, String> {
   
-  let weights_dir = app_data_root.weights_dir();
-  let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
-  let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
-  let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
-  let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);
-
-  let args = Args {
+  let args = FluxPipelineArgs {
     image: &image,
     prompt: prompt.to_string(),
-    uncond_prompt: "".to_string(),
     model_cache,
     prompt_cache: &prompt_cache,
+    app: &app,
+    flux_model: FluxModel::Schnell,
+    active_blocks: None,
+    use_cuda_stream: Some(true),
+    prefetch_next_batch: Some(false),
+    use_full_gpu: Some(true),
     img2img_strength: strength,
-    cfg_scale: config.cfg_scale,
-    use_flash_attn: true,
+    seed: None,
+    use_quantized_model: Some(true),
     model_config: &config.model_config,
-    maybe_seed: None,
-    scheduler_steps: config.scheduler_steps,
-    vae_path: &vae_path,
-    unet_path: &unet_path,
-    clip_json_path: &clip_json_path,
-    clip_weights_path: &clip_weights_path,
+    flux_dev_or_schnell_path: &weights_dir.weight_path(&FLUX_SCHNELL),
+    flux_schnell_quantized_gguf: &weights_dir.weight_path(&LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF),
+    flux_autoencoder_path: &weights_dir.weight_path(&FLUX_SCHNELL_AUTOENCODER),
+    google_t5_model_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_MODEL),
+    google_t5_config_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_CONFIG),
+    lmz_t5_tokenizer_path: &weights_dir.weight_path(&LMZ_T5_TOKENIZER_JSON),
+    openai_clip_model_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_MODEL),
+    openai_clip_tokenizer_json_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_TOKENIZER_JSON),
   };
 
-  let image = lcm_pipeline(args)
+  let image = flux_pipeline(args)
     .map_err(|err| format!("failure to encode image: {:?}", err))?;
 
   Ok(image)

crates/desktop/tauri-realtime/src/events/mod.rs

@@ -0,0 +1 @@
+pub mod notification_event;

crates/desktop/tauri-realtime/src/events/notification_event.rs

@@ -0,0 +1,45 @@
+use serde::Serialize;
+
+#[derive(Clone, Copy, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum NotificationModelType {
+  ImageSegmentation,
+  TextEncoder,
+  TextTokenizer,
+  Unet,
+  Vae,
+  Json,
+  ClipEncoder,
+  FluxModel,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum NotificationEvent<'a> {
+  ModelDownloadStarted {
+    model_name: &'a str, 
+    model_type: NotificationModelType,
+  },
+  ModelDownloadProgress {
+    model_name: &'a str,
+    model_type: NotificationModelType,
+    currently_downloaded_bytes: usize,
+    total_file_bytes: usize,
+  },
+  ModelDownloadComplete {
+    model_name: &'a str,
+    model_type: NotificationModelType,
+  },
+  ModelLoading {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInference {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+  ModelInferenceComplete {
+    model_name: String,
+    model_type: NotificationModelType,
+  },
+}

crates/desktop/tauri-realtime/src/lib.rs

@@ -8,8 +8,10 @@ pub mod utils;
 
 use crate::endpoints::download_models::download_models;
 use crate::endpoints::flip_image::flip_image;
+use crate::endpoints::inpaint_image_endpoint::inpaint_image;
 use crate::endpoints::realtime_image_endpoint::infer_image;
 use crate::endpoints::remove_background_endpoint::remove_background;
+use crate::endpoints::save_image_endpoint::save_image;
 use crate::endpoints::text_to_image_endpoint::text_to_image;
 use crate::state::app_config::AppConfig;
 use crate::stubs::model_cache::ModelCache;
@@ -28,7 +30,6 @@ pub fn run() {
   let config = AppConfig::init()
     .expect("config should load");
 
-
   let prompt_cache = PromptCache::with_capacity(8)
     .expect("prompt cache should load");
 
@@ -44,7 +45,10 @@ pub fn run() {
   tauri::Builder::default()
     .plugin(tauri_plugin_log::Builder::new()
       .level(log::LevelFilter::Info)
-      .targets(vec![Target::new(TargetKind::Stdout)])
+      .targets(vec![
+        Target::new(TargetKind::Stdout),
+        Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
+      ])
       .build())
     .setup(|app| {
       // TODO(bt): This is broken on windows
@@ -71,7 +75,10 @@ pub fn run() {
       download_models,
       flip_image,
       infer_image,
+      inpaint_image,
       remove_background,
+      save_image,
+      text_to_image,
       text_to_image,
     ])
     .run(tauri::generate_context!())