Compare commits

..
Author SHA1 Message Date
Brandon Thomas be184d740c remove background: frontend UX 2025-03-20 01:40:17 -04:00
3577 changed files with 2846 additions and 175324 deletions
+2 -2
View File
@@ -1,4 +1,4 @@
hakari-package = "workspace-build-acceleration"
hakari-package = "my-workspace-hack"
resolver = "1"
@@ -11,4 +11,4 @@ platforms = [
"x86_64-pc-windows-msvc",
]
exact-versions = true
exact-versions = true
-3
View File
@@ -32,9 +32,6 @@ target/
crates/lib/database_queries/target/
crates/lib/sqlite_queries/target/
# Javascript build directories
node_modules/
# Vim noise
*.swo
*.swp
Generated
+551 -1125
View File
File diff suppressed because it is too large Load Diff
+3 -46
View File
@@ -1,8 +1,6 @@
[workspace]
resolver = "2"
# This block defines the workspace members.
# When we add a library or binary crate, it needs an entry here.
members = [
# Types: common strongly-typed tokens, ids, payloads, etc.
"crates/_types/composite_identifiers",
@@ -24,8 +22,7 @@ members = [
"crates/cli/production/migration-tool",
# Desktop apps
"crates/desktop/tauri-4o",
"crates/desktop/tauri-realtime",
"crates/desktop/src-tauri",
# Shared libraries
"crates/lib/actix_cors_configs",
@@ -33,7 +30,6 @@ members = [
"crates/lib/api_clients/cloud_storage",
"crates/lib/api_clients/google_drive_common",
"crates/lib/api_clients/google_sign_in",
"crates/lib/api_clients/openai_sora_client",
"crates/lib/api_clients/thumbnail_generator",
"crates/lib/bootstrap",
"crates/lib/bucket_paths",
@@ -60,20 +56,16 @@ members = [
"crates/lib/files/mimetypes",
"crates/lib/files/videos",
"crates/lib/idempotency",
"crates/lib/image_algorithms",
"crates/lib/jobs_common",
"crates/lib/logging",
"crates/lib/machine_learning/ml_models",
"crates/lib/machine_learning/ml_weights_registry",
"crates/lib/machine_learning/prompt_engineering",
"crates/lib/markdown",
"crates/lib/migration",
"crates/lib/mysql_queries",
"crates/lib/password",
"crates/lib/prompt_engineering",
"crates/lib/redis_common",
"crates/lib/redis_schema",
"crates/lib/server_environment",
"crates/lib/shared_service_components",
"crates/lib/sqlx_mysql_helpers",
"crates/lib/storyteller_root",
"crates/lib/subprocess_common",
@@ -121,40 +113,5 @@ members = [
# cargo-hakari workspace hack (compilation speedup hack)
# see usage: https://crates.io/crates/cargo-hakari
"crates/workspace-build-acceleration",
"crates/my-workspace-hack",
]
[workspace.package]
# Common inheritable settings for monorepo packages
publish = false
[workspace.dependencies]
# Hakari workspace hack (faster compile times)
workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
# External libs
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
anyhow = "1.0.97"
chrono = { version = "0.4.40", features = ["serde"] }
log = "0.4.27"
once_cell = "1.21.3"
rand = "0.9.0"
regex = "1.11.1"
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
serde = { version = "1.0.219", features = ["derive"] }
serde_derive = "1.0.219"
serde_json = "1.0.140"
serde_yaml = "0.9.1"
toml = "0.7.4"
url = "2.5.4"
# Internal libs
crockford = { path = "crates/lib/data/crockford" }
errors = { path = "crates/lib/errors" }
filesys = { path = "crates/lib/files/filesys" }
memory_caching = { path = "crates/lib/caching/memory_caching" }
shared_service_components = { path = "crates/lib/shared_service_components" }
openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
primitives = { path = "crates/lib/data/primitives" }
subprocess_common = { path = "crates/lib/subprocess_common" }
@@ -13,7 +13,7 @@ name = "composite_identifiers"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -21,7 +21,7 @@ enums = { path = "../enums" }
tokens = { path = "../tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Serialization
serde = "1.0.125"
+2 -2
View File
@@ -13,13 +13,13 @@ name = "enums"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Serialization
serde = "1.0.125"
@@ -40,10 +40,6 @@ pub enum InferenceJobType {
#[serde(rename = "studio_gen2")]
StudioGen2,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
#[serde(rename = "convert_fbx_gltf")]
ConvertFbxToGltf,
@@ -112,7 +108,6 @@ impl InferenceJobType {
Self::GptSovits => "gpt_sovits",
Self::ComfyUi => "comfy_ui",
Self::StudioGen2 => "studio_gen2",
Self::ImageGenApi => "image_gen_api",
Self::ConvertFbxToGltf => "convert_fbx_gltf",
Self::MocapNet => "mocap_net",
Self::RvcV2 => "rvc_v2",
@@ -137,7 +132,6 @@ impl InferenceJobType {
"gpt_sovits" => Ok(Self::GptSovits),
"comfy_ui" => Ok(Self::ComfyUi),
"studio_gen2" => Ok(Self::StudioGen2),
"image_gen_api" => Ok(Self::ImageGenApi),
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
"mocap_net" => Ok(Self::MocapNet),
"rvc_v2" => Ok(Self::RvcV2),
@@ -165,7 +159,6 @@ impl InferenceJobType {
Self::GptSovits,
Self::ComfyUi,
Self::StudioGen2,
Self::ImageGenApi,
Self::ConvertFbxToGltf,
Self::MocapNet,
Self::RvcV2,
@@ -204,7 +197,6 @@ mod tests {
assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
assert_serialization(InferenceJobType::MocapNet, "mocap_net");
assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
@@ -228,7 +220,6 @@ mod tests {
assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
@@ -252,7 +243,6 @@ mod tests {
assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
@@ -271,7 +261,7 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceJobType::all_variants();
assert_eq!(variants.len(), 17);
assert_eq!(variants.len(), 16);
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
@@ -279,7 +269,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
@@ -33,10 +33,6 @@ pub enum InferenceModelType {
#[serde(rename = "seed_vc")]
SeedVc,
/// NB: This is for Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
#[serde(rename = "tacotron2")]
Tacotron2,
#[serde(rename = "vits")]
@@ -74,7 +70,6 @@ impl InferenceModelType {
Self::VallEX => "vall_e_x",
Self::RerenderAVideo => "rerender_a_video",
Self::StableDiffusion => "stable_diffusion",
Self::ImageGenApi => "image_gen_api",
Self::SeedVc => "seed_vc",
Self::MocapNet => "mocap_net",
Self::StyleTTS2 => "styletts2",
@@ -95,7 +90,6 @@ impl InferenceModelType {
"vall_e_x" => Ok(Self::VallEX),
"rerender_a_video" => Ok(Self::RerenderAVideo),
"stable_diffusion" => Ok(Self::StableDiffusion),
"image_gen_api" => Ok(Self::ImageGenApi),
"mocap_net" => Ok(Self::MocapNet),
"styletts2" => Ok(Self::StyleTTS2),
"comfy_ui" => Ok(Self::ComfyUi),
@@ -119,7 +113,6 @@ impl InferenceModelType {
Self::VallEX,
Self::RerenderAVideo,
Self::StableDiffusion,
Self::ImageGenApi,
Self::MocapNet,
Self::StyleTTS2,
Self::ConvertFbxToGltf,
@@ -147,7 +140,6 @@ mod tests {
assert_serialization(InferenceModelType::VallEX, "vall_e_x");
assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
@@ -167,7 +159,6 @@ mod tests {
assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
@@ -185,7 +176,6 @@ mod tests {
assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
@@ -197,18 +187,17 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceModelType::all_variants();
assert_eq!(variants.len(), 15);
assert_eq!(variants.len(), 13);
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
@@ -57,9 +57,6 @@ pub enum MediaFileOriginModelType {
#[serde(rename = "studio")]
StorytellerStudio,
#[serde(rename = "studio_ig")]
StorytellerStudioImageGen,
#[serde(rename = "vst")]
VideoStyleTransfer,
@@ -96,7 +93,6 @@ impl MediaFileOriginModelType {
Self::MocapNet => "mocap_net",
Self::StyleTTS2 => "styletts2",
Self::StableDiffusion15 => "stable_diffusion_1_5",
Self::StorytellerStudioImageGen => "studio_ig",
Self::GptSovits => "gpt_sovits",
Self::StorytellerStudio => "studio",
Self::VideoStyleTransfer => "vst",
@@ -119,7 +115,6 @@ impl MediaFileOriginModelType {
"mocap_net" => Ok(Self::MocapNet),
"styletts2" => Ok(Self::StyleTTS2),
"stable_diffusion_1_5" => Ok(Self::StableDiffusion15),
"studio_ig" => Ok(Self::StorytellerStudioImageGen),
"gpt_sovits" => Ok(Self::GptSovits),
"studio" => Ok(Self::StorytellerStudio),
"vst" => Ok(Self::VideoStyleTransfer),
@@ -147,7 +142,6 @@ impl MediaFileOriginModelType {
Self::StableDiffusion15,
Self::GptSovits,
Self::StorytellerStudio,
Self::StorytellerStudioImageGen,
Self::VideoStyleTransfer,
Self::ComfyUi,
Self::VallEX,
+2 -2
View File
@@ -13,11 +13,11 @@ name = "enums_public"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Internal
enums = { path = "../enums" }
@@ -91,10 +91,6 @@ pub enum PublicInferenceJobType {
/// Re-render a video is a video style transfer algorithm. We developed code
/// around it, but chose to develop AnimateDiff / ComfyUI support instead.
RerenderAVideo,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
}
/// NB: Legacy API for older code.
@@ -123,7 +119,6 @@ impl PublicInferenceJobType {
InferenceJobType::Unknown => Self::Unknown,
InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
InferenceJobType::ImageGenApi => Self::ImageGenApi,
}
}
@@ -145,7 +140,6 @@ impl PublicInferenceJobType {
Self::SeedVc => InferenceJobType::SeedVc,
Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
Self::StableDiffusion => InferenceJobType::StableDiffusion,
Self::ImageGenApi => InferenceJobType::ImageGenApi,
Self::StyleTTS2 => InferenceJobType::StyleTTS2,
Self::Tacotron2 => InferenceJobType::Tacotron2,
Self::Unknown => InferenceJobType::Unknown,
@@ -64,10 +64,6 @@ pub enum PublicMediaFileModelType {
#[serde(rename = "studio")]
StorytellerStudio,
/// NB: This is GPT4o image generation
#[serde(rename = "studio_ig")]
StorytellerStudioImageGen,
#[serde(rename = "vst")]
VideoStyleTransfer,
@@ -93,7 +89,6 @@ impl PublicMediaFileModelType {
MediaFileOriginModelType::LivePortrait => Self::FaceMirror,
MediaFileOriginModelType::SadTalker => Self::FaceAnimator,
MediaFileOriginModelType::StyleTTS2 => Self::VoiceDesigner,
MediaFileOriginModelType::StorytellerStudioImageGen => Self::StorytellerStudioImageGen,
// Conserved variants
MediaFileOriginModelType::RvcV2 => Self::RvcV2,
MediaFileOriginModelType::SoVitsSvc => Self::SoVitsSvc,
@@ -118,7 +113,6 @@ impl PublicMediaFileModelType {
Self::FaceAnimator => MediaFileOriginModelType::SadTalker,
Self::Lipsync => MediaFileOriginModelType::FaceFusion,
Self::VoiceDesigner => MediaFileOriginModelType::StyleTTS2,
Self::StorytellerStudioImageGen => MediaFileOriginModelType::StorytellerStudioImageGen,
// Conserved variants
Self::RvcV2 => MediaFileOriginModelType::RvcV2,
Self::SoVitsSvc => MediaFileOriginModelType::SoVitsSvc,
+2 -2
View File
@@ -13,7 +13,7 @@ name = "reusable_types"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
# (none)
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
chrono = { version = "0.4.22", features = ["serde"] }
+2 -2
View File
@@ -13,7 +13,7 @@ name = "tokens"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
# (none)
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Serialization
serde = "1.0.125"
@@ -13,7 +13,7 @@ name = "dev-database-seed"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -32,7 +32,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
log = "0.4.14"
@@ -13,7 +13,7 @@ name = "dev-upload-media-file"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -29,7 +29,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = "4.4.8"
+2 -2
View File
@@ -13,7 +13,7 @@ name = "scrape-test"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
@@ -22,7 +22,7 @@ errors = { path = "../../../lib/errors" }
web_scrapers = { path = "../../../lib/web_scrapers" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = "4.4.8"
+2 -2
View File
@@ -13,11 +13,11 @@ name = "db-backfill"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Internal
datetimes = { path = "../../../lib/data/datetimes" }
+2 -2
View File
@@ -13,7 +13,7 @@ name = "db-backup"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -21,7 +21,7 @@ easyenv = { path = "../../../lib/easyenv" }
errors = { path = "../../../lib/errors" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.86"
+2 -2
View File
@@ -13,7 +13,7 @@ name = "db-cleanup"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -26,7 +26,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = "4.4.8"
@@ -13,7 +13,7 @@ name = "elasticsearch-cli"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -29,7 +29,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = "4.4.8"
@@ -13,7 +13,7 @@ name = "kubectl-pod-cleanup"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -22,7 +22,7 @@ easyenv = { path = "../../../lib/easyenv" }
errors = { path = "../../../lib/errors" }
# Workspace hack (faster compile yytimes)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
log = "0.4.20"
@@ -13,7 +13,7 @@ name = "lru-delete-models"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" }
logging = { path = "../../../lib/logging" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = { version = "4.4.8", features = ["derive"] }
@@ -13,7 +13,7 @@ name = "migration-tool"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
normal = ["my-workspace-hack"]
[dependencies]
# Internal
@@ -30,7 +30,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
clap = "4.4.8"
@@ -1,5 +1,5 @@
[package]
name = "tauri-realtime"
name = "app"
version = "0.1.0"
description = "A Tauri App"
authors = ["you"]
@@ -7,7 +7,6 @@ license = ""
repository = ""
edition = "2021"
rust-version = "1.77.2"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,65 +15,70 @@ name = "app_lib"
crate-type = ["staticlib", "cdylib", "rlib"]
[build-dependencies]
tauri-build = { version = "2.1.1", features = [] }
tauri-build = { version = "2.0.4", features = [] }
[dependencies]
# Internal
crockford = { path = "../../lib/data/crockford" }
filesys = { path = "../../lib/files/filesys" }
memory_caching = { path = "../../lib/caching/memory_caching" }
ml_weights_registry = { path = "../../lib/machine_learning/ml_weights_registry" }
primitives = { path = "../../lib/data/primitives" }
# Internal ML
# TODO: Only on Windows and Linux
#ml_models = { path = "../ml_models", optional = true }
# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
cudarc = { version = "0.13.5", features = ["std", "cublas", "cublaslt", "curand", "driver", "nvrtc", "f16", "cuda-version-from-build-system", "dynamic-linking"], default-features=false }
cudarc = "0.13.7"
# External
#openssl = { version = "0.10", features = ["vendored"] }
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
anyhow = "1.0"
base64 = "0.22.1"
bytes = "1.10"
chrono = "0.4.40"
candle-core = { path = "../../vendor/candle/candle-core", features = ["cuda"] }
candle-nn = { path = "../../vendor/candle/candle-nn", features = ["cuda"] }
candle-transformers = { path = "../../vendor/candle/candle-transformers", features = ["cuda"] }
const_format = "0.2.34"
directories = "6.0.0"
dirs = "6.0.0"
futures = "0.3"
half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
hf-hub = "0.3.2"
image = "0.25.5"
log = "0.4"
once_cell = "1.20.3"
ort = { version = "1.16.2", features = ["cuda", "load-dynamic"] }
rand = "0.8.5"
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
reqwest = { version = "0.12.12", features = ["stream"], default-features = false }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde_yaml = "0.9"
tauri = { version = "2.4.1", features = [] }
tauri-plugin-log = "2.3.1"
tauri = { version = "2.2.4", features = [] }
tauri-plugin-log = "2"
tempdir = "0.3.7"
tempfile = "3.18.0"
tokenizers = "0.15.2"
tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
tokio-util = { version = "0.7", features = ["codec"] }
half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
# NB: From `rust-background-removal`
ndarray = "0.15.6"
ort = { version = "1.16.2", features = ["cuda"] }
# tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
[target.'cfg(not(target_os = "macos"))'.dependencies]
ml_models = { path = "../../lib/machine_learning/ml_models" }
[features]
default = ["cuda"]
cuda = [
"candle-core/cuda",
"candle-nn/cuda",
"candle-transformers/cuda",
# NB(bt,2025-03-10): Building without Flash attention on Windows for now to bypass linker issues
"candle-transformers/flash-attn",
"candle-core/cudnn"
]
[dev-dependencies]
libloading = "0.8.6"
serial_test = "3.2.0"
assertor = "0.0.3"
#[target.'cfg(target_os = "macos")'.features]
#default = []
#ml_models = ["dep:ml_models"]
[profile.dev]
incremental = true # Compile your binary in smaller steps.

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

Before

Width:  |  Height:  |  Size: 2.2 KiB

After

Width:  |  Height:  |  Size: 2.2 KiB

Before

Width:  |  Height:  |  Size: 9.0 KiB

After

Width:  |  Height:  |  Size: 9.0 KiB

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 28 KiB

Before

Width:  |  Height:  |  Size: 3.3 KiB

After

Width:  |  Height:  |  Size: 3.3 KiB

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

Before

Width:  |  Height:  |  Size: 3.9 KiB

After

Width:  |  Height:  |  Size: 3.9 KiB

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

Before

Width:  |  Height:  |  Size: 49 KiB

After

Width:  |  Height:  |  Size: 49 KiB

@@ -1,21 +1,16 @@
#[cfg(not(target_os = "macos"))]
use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
use crate::ml::model_cache::ModelCache;
use crate::ml::prompt_cache::PromptCache;
use crate::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
use crate::state::app_config::AppConfig;
use crate::state::app_dir::AppDataRoot;
use crate::stubs::model_cache::ModelCache;
use crate::stubs::prompt_cache::PromptCache;
use crate::utils::image::decode_base64_image::decode_base64_image;
use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
use crate::utils::image::encode_rgb_image_base64_png::encode_rgb_image_base64_png;
use image::imageops::FilterType;
use image::{DynamicImage, RgbImage};
use log::{error, info};
use ml_weights_registry::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
use std::path::PathBuf;
use tauri::{AppHandle, State};
const PROMPT_FILENAME : &str = "prompt.txt";
const PROMPT: &str = "shiba inu, cute dog, detailed, walking in a wooded forest, photorealistic, 8k";
@@ -31,7 +26,6 @@ pub async fn infer_image(
image: &str,
prompt: Option<String>,
strength: Option<f64>,
seed: Option<u64>,
model_config: State<'_, AppConfig>,
model_cache: State<'_, ModelCache>,
prompt_cache: State<'_, PromptCache>,
@@ -48,33 +42,28 @@ pub async fn infer_image(
.map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
// TODO(bt,2025-02-17): Running out of vram with full image buffer size
let mut image = image.resize(512, 512, FilterType::CatmullRom);
let image = image.resize(512, 512, FilterType::CatmullRom);
let seed = seed.or(model_config.seed);
let result = do_infer_image(
&prompt,
image,
strength,
&model_config,
&model_cache,
prompt_cache,
app,
&app_data_root
).await;
let image = match result {
Ok(image) => image,
Err(err) => {
error!("There was an error generating the image: {:?}", err);
return Err(format!("There was an error generating the image: {}", err));
}
};
#[cfg(not(target_os = "macos"))]
{
let result = infer_image_impl(
&prompt,
image,
strength,
seed,
&model_config,
&model_cache,
prompt_cache,
&app_data_root
).await;
image = match result {
Ok(img) => DynamicImage::ImageRgb8(img),
Err(err) => {
error!("There was an error generating the image: {:?}", err);
return Err(format!("There was an error generating the image: {}", err));
}
};
}
let bytes = encode_dynamic_image_base64_png(image)
let bytes = encode_rgb_image_base64_png(image)
.map_err(|err| format!("failure to encode image: {:?}", err))?;
info!("Inference successful; image converted to base64, serving back to browser...");
@@ -82,41 +71,29 @@ pub async fn infer_image(
Ok(bytes)
}
#[cfg(not(target_os = "macos"))]
async fn infer_image_impl(
async fn do_infer_image(
prompt: &str,
image: DynamicImage,
strength: Option<f64>,
seed: Option<u64>,
config: &AppConfig,
model_cache: &ModelCache,
prompt_cache: State<'_, PromptCache>,
app: AppHandle,
app_data_root: &AppDataRoot,
) -> Result<RgbImage, String> {
let weights_dir = app_data_root.weights_dir();
let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
//let clip_weights_path= weights_dir.weight_path(&LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16);
let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);
let args = Args {
image: &image,
prompt: prompt.to_string(),
uncond_prompt: NEGATIVE_PROMPT.to_string(),
model_cache,
configs: config,
prompt_cache: &prompt_cache,
img2img_strength: strength,
i2i_strength: strength,
cfg_scale: config.cfg_scale,
app: &app,
app_data_root,
use_flash_attn: true,
model_config: &config.model_config,
maybe_seed: seed,
scheduler_steps: config.scheduler_steps,
vae_path: &vae_path,
unet_path: &unet_path,
clip_json_path: &clip_json_path,
clip_weights_path: &clip_weights_path,
};
let image = lcm_pipeline(args)
@@ -0,0 +1,5 @@
pub mod download_models;
pub mod flip_image;
pub mod image_endpoint;
pub mod remove_background_endpoint;
pub mod test_counter;
@@ -1,21 +1,46 @@
//! Adapted from https://github.com/dnanhkhoa/rust-background-removal
use crate::ml::background_removal::onnx_session::onnx_session;
use crate::ml::model_cache::ModelCache;
use crate::state::app_dir::AppDataRoot;
use crate::utils::image::decode_base64_image::decode_base64_image;
use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
use image::{imageops, DynamicImage};
use log::{error, info};
use ndarray::{Array, CowArray};
use ort::Value;
use std::path::Path;
use image_algorithms::remove_image_border::remove_image_border;
use tauri::{AppHandle, State};
/// This handler removes the background from an image.
#[tauri::command]
pub async fn remove_background(
image: &str,
model_cache: State<'_, ModelCache>,
app_data_root: State<'_, AppDataRoot>,
app: AppHandle,
) -> Result<String, String> {
info!("remove_background endpoint called.");
let image = decode_base64_image(image)
.map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: DynamicImage) -> anyhow::Result<DynamicImage> {
// NB: Real time needed cuDNN 9, but this uses cuDNN 8:
// sudo apt-get install libcudnn8-dev libcudnn8
// update-alternatives: warning: forcing reinstallation of alternative /usr/include/x86_64-linux-gnu/cudnn_v9.h because link group libcudnn is broken
// update-alternatives: using /usr/include/x86_64-linux-gnu/cudnn_v8.h to provide /usr/include/cudnn.h (libcudnn) in manual mode
let session = onnx_session(model_path.as_ref())?;
let image = process_dynamic_image(&session, image)?;
// Crop transparent border cruft away.
let image = remove_image_border(&image)?;
let model_path = app_data_root.weights_dir().path().join("medium.onnx"); // TODO
let session = onnx_session(model_path.to_str().unwrap()) // TODO
.map_err(|err| format!("failure to start onnx session: {:?}", err))?;
let image = process_dynamic_image(&session, image)
.map_err(|err| {
error!("Couldn't remove background from image: {}", err);
format!("Couldn't remove background from image: {}", err)
})?;
let image = encode_dynamic_image_base64_png(image)
.map_err(|err| format!("failure to encode image: {:?}", err))?;
Ok(image)
}
@@ -63,4 +88,4 @@ fn process_dynamic_image(
imageops::FilterType::Triangle,
);
Ok(DynamicImage::ImageRgba8(output_img))
}
}
@@ -0,0 +1,24 @@
use std::sync::{Arc, RwLock};
use once_cell::sync::Lazy;
static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
#[tauri::command]
pub fn test_counter() -> u64 {
let value : u64;
{
match COUNTER.write() {
Ok(mut counter) => {
*counter += 1;
value = *counter;
},
Err(_e) => {
value = 0;
}
}
}
println!("I was invoked from JavaScript! {:?}", value);
value
}
@@ -1,16 +1,13 @@
use serde::Serialize;
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum NotificationModelType {
ImageSegmentation,
TextEncoder,
TextTokenizer,
Unet,
Vae,
Json,
ClipEncoder,
FluxModel,
TextEncoder,
TextTokenizer,
}
#[derive(Clone, Serialize)]
@@ -29,17 +26,5 @@ pub enum NotificationEvent<'a> {
ModelDownloadComplete {
model_name: &'a str,
model_type: NotificationModelType,
},
ModelLoading {
model_name: String,
model_type: NotificationModelType,
},
ModelInference {
model_name: String,
model_type: NotificationModelType,
},
ModelInferenceComplete {
model_name: String,
model_type: NotificationModelType,
},
}
}
@@ -1,23 +1,19 @@
pub mod endpoints;
pub mod events;
pub mod ml;
pub mod state;
pub mod stubs;
pub mod threads;
pub mod transfer;
pub mod utils;
use crate::endpoints::download_models::download_models;
use crate::endpoints::flip_image::flip_image;
use crate::endpoints::inpaint_image_endpoint::inpaint_image;
use crate::endpoints::realtime_image_endpoint::infer_image;
use crate::endpoints::image_endpoint::infer_image;
use crate::endpoints::remove_background_endpoint::remove_background;
use crate::endpoints::save_image_endpoint::save_image;
use crate::endpoints::text_to_image_endpoint::text_to_image;
use crate::endpoints::test_counter::test_counter;
use crate::ml::model_cache::ModelCache;
use crate::ml::prompt_cache::PromptCache;
use crate::state::app_config::AppConfig;
use crate::stubs::model_cache::ModelCache;
use crate::stubs::prompt_cache::PromptCache;
use crate::threads::downloader_thread::downloader_thread;
use tauri_plugin_log::Target;
use tauri_plugin_log::TargetKind;
@@ -45,15 +41,9 @@ pub fn run() {
tauri::Builder::default()
.plugin(tauri_plugin_log::Builder::new()
.level(log::LevelFilter::Info)
.targets(vec![
Target::new(TargetKind::Stdout),
Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
])
.targets(vec![Target::new(TargetKind::Stdout)])
.build())
.setup(|app| {
// TODO(bt): This is broken on windows
// log_environment_details();
//if cfg!(debug_assertions) {
// app.handle().plugin(
// tauri_plugin_log::Builder::default()
@@ -73,13 +63,9 @@ pub fn run() {
.manage(app_data_root)
.invoke_handler(tauri::generate_handler![
download_models,
flip_image,
infer_image,
inpaint_image,
remove_background,
save_image,
text_to_image,
text_to_image,
test_counter,
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
@@ -0,0 +1 @@
pub mod onnx_session;
@@ -1,9 +1,8 @@
//! Adapted from https://github.com/dnanhkhoa/rust-background-removal/blob/main/src/onnx.rs
use std::path::Path;
use ort::{Environment, ExecutionProvider, GraphOptimizationLevel, SessionBuilder};
pub fn onnx_session(onnx_model_file: &Path) -> Result<ort::Session, anyhow::Error> {
pub fn onnx_session(onnx_model_file: &str) -> Result<ort::Session, anyhow::Error> {
let environment = Environment::default().into_arc();
let session = SessionBuilder::new(&environment)?
.with_optimization_level(GraphOptimizationLevel::Level1)? // Configure model optimization level
@@ -1,8 +1,7 @@
pub mod background_removal;
pub mod flux;
pub mod image;
pub mod model_cache;
pub mod model_config;
pub mod model_file;
pub mod prompt_cache;
pub mod stable_diffusion;
pub mod stable_diffusion;
pub mod weights_registry;
@@ -4,50 +4,18 @@ extern crate accelerate_src;
extern crate intel_mkl_src;
use candle_transformers::models::stable_diffusion;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
use ml_weights_registry::weights_registry::weights::{CLIP_JSON, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2};
use crate::ml::weights_registry::weights::{CLIP_JSON, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2};
use crate::state::app_dir::AppWeightsDir;
use anyhow::Error as E;
use candle_core::{DType, Device, Module, Tensor, D};
use log::info;
use tokenizers::Tokenizer;
pub struct ClipArgs<'a, P: AsRef<Path>, Q: AsRef<Path>> {
pub prompt: &'a str,
pub uncond_prompt: &'a str,
pub tokenizer: Option<String>,
pub clip_weights: Option<String>,
pub clip2_weights: Option<String>,
pub sd_version: StableDiffusionVersion,
pub sd_config: &'a stable_diffusion::StableDiffusionConfig,
pub use_f16: bool,
pub device: &'a Device,
pub dtype: DType,
pub use_guide_scale: bool,
pub clip_json_path: P,
pub clip_weights_path: Q,
}
#[allow(clippy::too_many_arguments)]
pub fn infer_clip_text_embeddings<P: AsRef<Path>, Q: AsRef<Path>>(args: ClipArgs<'_, P, Q>) -> anyhow::Result<Tensor> {
let ClipArgs {
prompt,
uncond_prompt,
tokenizer,
clip_weights,
clip2_weights,
sd_version,
sd_config,
use_f16,
device,
dtype,
use_guide_scale,
clip_json_path,
clip_weights_path,
} = args;
pub fn infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: Option<String>, clip_weights: Option<String>, clip2_weights: Option<String>, sd_version: StableDiffusionVersion, sd_config: &stable_diffusion::StableDiffusionConfig, use_f16: bool, device: &Device, dtype: DType, use_guide_scale: bool, weights_dir: &AppWeightsDir) -> anyhow::Result<Tensor> {
info!("Preparing text embeddings... for {:?}", sd_version);
let which = match sd_version {
@@ -71,8 +39,7 @@ pub fn infer_clip_text_embeddings<P: AsRef<Path>, Q: AsRef<Path>>(args: ClipArgs
dtype,
use_guide_scale,
*first,
clip_json_path.as_ref(),
clip_weights_path.as_ref(),
weights_dir,
)
})
.collect::<anyhow::Result<Vec<_>>>()?;
@@ -83,25 +50,10 @@ pub fn infer_clip_text_embeddings<P: AsRef<Path>, Q: AsRef<Path>>(args: ClipArgs
}
#[allow(clippy::too_many_arguments)]
fn do_infer_clip_text_embeddings(
prompt: &str,
uncond_prompt: &str,
tokenizer: Option<String>,
clip_weights: Option<String>,
clip2_weights: Option<String>,
sd_version: StableDiffusionVersion,
sd_config: &stable_diffusion::StableDiffusionConfig,
use_f16: bool,
device: &Device,
dtype: DType,
use_guide_scale: bool,
first: bool,
clip_json_path: &Path,
clip_weights_path: &Path,
) -> anyhow::Result<Tensor> {
fn do_infer_clip_text_embeddings(prompt: &str, uncond_prompt: &str, tokenizer: Option<String>, clip_weights: Option<String>, clip2_weights: Option<String>, sd_version: StableDiffusionVersion, sd_config: &stable_diffusion::StableDiffusionConfig, use_f16: bool, device: &Device, dtype: DType, use_guide_scale: bool, first: bool, weights_dir: &AppWeightsDir) -> anyhow::Result<Tensor> {
info!("do_infer_clip_text_embeddings called with args {:?}", (prompt, uncond_prompt, tokenizer.clone().unwrap_or_else(|| "None".to_string()), clip_weights.clone().unwrap_or_else(|| "None".to_string()), clip2_weights.clone().unwrap_or_else(|| "None".to_string()), sd_version, sd_config, use_f16, device, dtype, use_guide_scale, first));
let tokenizer = clip_json_path;
let tokenizer = weights_dir.weight_path(&CLIP_JSON);
info!("Loading Clip Tokenizer path: {:?}", tokenizer);
@@ -123,16 +75,11 @@ fn do_infer_clip_text_embeddings(
let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
println!("Building the Clip transformer.");
// TODO(bt,2025-03-23): These used to be two different clip models. We should only need one.
let clip_weights = if first {
println!(">>>>> CLIP 1");
clip_weights_path
weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER)
} else {
println!(">>>>> CLIP 2");
clip_weights_path
weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER_2)
};
let clip_config = if first { &sd_config.clip } else { sd_config.clip2.as_ref().unwrap() };
let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
@@ -1,82 +1,53 @@
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::events::notification_event::{NotificationEvent, NotificationModelType};
use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
use crate::ml::model_cache::ModelCache;
use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
use crate::ml::prompt_cache::PromptCache;
use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
use crate::ml::weights_registry::weights::{LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
use crate::state::app_config::AppConfig;
use crate::state::app_dir::AppDataRoot;
use anyhow::{anyhow, Error as E, Result};
use candle_core::{DType, Device, IndexOp, Tensor, D};
use candle_core::{DType, IndexOp, Tensor, D};
use candle_nn::VarBuilder;
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
use candle_transformers::models::stable_diffusion::lcm::{LCMScheduler, LCMSchedulerConfig};
use candle_transformers::models::stable_diffusion::lcm::LCMScheduler;
use candle_transformers::models::stable_diffusion::unet_2d::BlockConfig;
use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModelConfig;
use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
use crate::ml::model_cache::ModelCache;
use crate::ml::model_config::ModelConfig;
use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
use crate::ml::prompt_cache::PromptCache;
use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
use image::DynamicImage;
use log::{debug, error, info};
use ml_weights_registry::weights_registry::weights::{LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
use log::info;
use rand::Rng;
use tauri::{AppHandle, Emitter};
const DEFAULT_STRENGTH : f64 = 75.0;
const DEFAULT_SEED : u64 = 42;
pub struct Args<'a, P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>> {
pub struct Args<'a> {
pub image: &'a DynamicImage,
pub prompt: String,
pub uncond_prompt: String,
pub cfg_scale: Option<f64>,
pub img2img_strength: Option<f64>,
pub i2i_strength: Option<f64>,
pub configs: &'a AppConfig,
pub model_cache: &'a ModelCache,
pub prompt_cache: &'a PromptCache,
pub app_data_root: &'a AppDataRoot,
pub app: &'a AppHandle,
pub use_flash_attn: bool,
pub model_config: &'a ModelConfig,
pub maybe_seed: Option<u64>,
pub scheduler_steps: usize,
pub vae_path: P1,
pub unet_path: P2,
pub clip_json_path: P3,
pub clip_weights_path: P4,
}
pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<RgbImage>
where P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>
{
let Args {
prompt,
uncond_prompt,
cfg_scale,
img2img_strength,
model_cache,
prompt_cache,
image,
use_flash_attn ,
model_config: ModelConfig {
device,
dtype,
sd_version,
sd_config,
hf_api,
},
maybe_seed,
scheduler_steps,
vae_path,
unet_path,
clip_json_path,
clip_weights_path,
} = args;
let dtype = *dtype;
let sd_version = *sd_version;
pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
let Args { prompt, uncond_prompt, cfg_scale, i2i_strength, configs, model_cache, prompt_cache, app, image, app_data_root, use_flash_attn } = args;
let img2img_strength = img2img_strength
let weights_dir = app_data_root.weights_dir();
let img2img_strength = args.i2i_strength
.map(|s| remap_lcm_strength_range(s))
.unwrap_or(DEFAULT_STRENGTH);
@@ -85,16 +56,14 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
let use_f16 = true;
// Use LCM Scheduler instead of Euler Ancestral for better speed and quality
let mut scheduler = LCMScheduler::new(scheduler_steps, img2img_strength, LCMSchedulerConfig::default())?;
let mut scheduler = LCMScheduler::new(configs.scheduler_steps, img2img_strength, candle_transformers::models::stable_diffusion::lcm::LCMSchedulerConfig::default())?;
//let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
let seed = maybe_seed.unwrap_or(DEFAULT_SEED);
device.set_seed(seed)?;
let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
configs.device.set_seed(seed)?;
let guidance_scale = match cfg_scale {
Some(guidance_scale) => guidance_scale,
None => match sd_version {
None => match configs.sd_version {
StableDiffusionVersion::V1_5 | StableDiffusionVersion::V2_1 | StableDiffusionVersion::Xl => 7.5,
StableDiffusionVersion::Turbo => 0.,
_ => 0., // NB: Not sure what the other model families should use, so sticking with "0"
@@ -109,45 +78,50 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
let mut text_embeddings = if let Some(tensor) = maybe_cached {
tensor
} else {
info!("Prompt is NOT cached! Calculating embedding...");
let tensor = infer_clip_text_embeddings(ClipArgs {
prompt: & prompt,
uncond_prompt: &uncond_prompt,
tokenizer: None, // tokenizer
clip_weights: None,
clip2_weights: None, // clip2_weights
sd_version,
sd_config: &sd_config,
use_f16, // use_f16
device,
dtype,
use_guide_scale,
clip_json_path,
clip_weights_path,
})?;
info!("Loading clip weights...");
let clip_weights = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16);
let clip_weights = Some(clip_weights.to_string_lossy().to_string());
info!("Prompt is NOT cached! Calculating embedding...");
let tensor = infer_clip_text_embeddings(
&prompt,
&uncond_prompt,
None, // tokenizer
clip_weights,
None, // clip2_weights
configs.sd_version,
&configs.sd_config,
use_f16, // use_f16
&configs.device,
configs.dtype,
use_guide_scale,
weights_dir,
)?;
prompt_cache.store_copy(&prompt, &tensor)?;
tensor
};
debug!("Text embeddings shape: {:?}", text_embeddings.shape());
println!("Text embeddings shape: {:?}", text_embeddings.shape());
debug!("Loading input image into tensor...");
println!("Loading input image into tensor...");
let input_image = dynamic_image_to_tensor(image, &device, dtype)?;
let input_image = dynamic_image_to_tensor(image, &configs.device, configs.dtype)?;
debug!("Reference image shape: {:?}", input_image.shape());
println!("Reference image shape: {:?}", input_image.shape());
let vae_scale = get_vae_scale(sd_version);
let vae_scale = get_vae_scale(configs.sd_version);
let maybe_vae = model_cache.get_vae()?;
let vae = match maybe_vae {
Some(vae) => vae,
None => {
info!("Building VAE model from file {:?}...", vae_path.as_ref());
let vae = sd_config.build_vae(vae_path, &device, dtype)?;
let vae_file = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
println!("Building VAE model from file {:?}...", &vae_file);
let vae = configs.sd_config.build_vae(vae_file, &configs.device, configs.dtype)?;
let vae = Arc::new(vae);
@@ -162,9 +136,11 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
let unet = match maybe_unet {
Some(unet) => unet,
None => {
info!("Building UNET model from file {:?}...", unet_path.as_ref());
info!("No unet found in cache; loading...");
let in_channels = match sd_version {
let unet_weights = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
let in_channels = match configs.sd_version {
StableDiffusionVersion::XlInpaint | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::V1_5Inpaint => 9,
_ => 4,
};
@@ -199,7 +175,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
};
// Load the model directly from the safetensors file
let vs_unet = unsafe { VarBuilder::from_mmaped_safetensors(&[unet_path], dtype, &device)? };
let vs_unet = unsafe { VarBuilder::from_mmaped_safetensors(&[unet_weights], configs.dtype, &configs.device)? };
let unet = UNet2DConditionModel::new(vs_unet, in_channels, 4, use_flash_attn, unet_config)?;
@@ -215,20 +191,19 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
// Create LCM guidance scale embeddings
let embedding_dim = 1280;
let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &device, dtype)?;
let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &configs.device, configs.dtype)?;
// Generate latents from input image using the LCM approach
debug!("Generating latents from input image...");
println!("Generating latents from input image...");
let init_latents = (init_latent_dist.sample()? * vae_scale)?;
debug!("Initial latents shape: {:?}", init_latents.shape());
println!("Initial latents shape: {:?}", init_latents.shape());
// Calculate timesteps for LCM with proper img2img handling
let timesteps = LCMScheduler::get_timesteps_for_steps(scheduler_steps, img2img_strength);
let timesteps = LCMScheduler::get_timesteps_for_steps(configs.scheduler_steps, img2img_strength);
let t_start = timesteps[0]; // First denoising step
// Add noise at the appropriate timestep for img2img
debug!("Adding noise to latents for t_start={}", t_start);
println!("Adding noise to latents for t_start={}", t_start);
let noise = init_latents.randn_like(0f64, 1f64)?;
let latents = scheduler.add_noise(&init_latents, noise, t_start)?;
@@ -236,7 +211,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();
for (timestep_index, &timestep) in timesteps.iter().enumerate() {
info!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
let latent_model_input = if use_guide_scale { Tensor::cat(&[&latents, &latents], 0)? } else { latents.clone() };
@@ -246,7 +221,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
let mut noise_pred = match unet.forward_with_guidance(&latent_model_input, timestep as f64, &text_embeddings, Some(&guidance_scale_embedding)) {
Ok(pred) => pred,
Err(e) => {
error!("UNet inference failed with error: {}", e);
println!("UNet inference failed with error: {}", e);
return Err(anyhow::anyhow!("UNet inference failed: {}", e));
},
};
@@ -258,22 +233,24 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
}
// Apply the LCM scheduler step
latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, scheduler_steps)?;
latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, configs.scheduler_steps)?;
}
debug!("Diffusion process completed, decoding image...");
println!("Diffusion process completed, decoding image...");
let image = vae.decode(&(latents / vae_scale)?)?;
debug!("Scaling image back");
println!("VAE decode completed");
println!("Scaling image back");
let image = ((image / 2.)? + 0.5)?;
debug!("Normalized image values");
println!("Normalized image values");
let image = (image.clamp(0f32, 1.)? * 255.)?;
debug!("Convert to int8");
println!("Convert to int8");
let image = image.to_dtype(DType::U8)?;
debug!("Converted to 8-bit format");
println!("Converted to 8-bit format");
let image = tensor_to_image_buffer(&image.i(0)?)?;
@@ -1,78 +1,75 @@
use std::path::Path;
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::events::notification_event::{NotificationEvent, NotificationModelType};
use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
use crate::ml::model_cache::ModelCache;
use crate::ml::model_file::StableDiffusionVersion;
use crate::ml::prompt_cache::PromptCache;
use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
use crate::ml::weights_registry::weights::{SDXL_TURBO_UNET, SDXL_TURBO_VAE};
use crate::state::app_config::AppConfig;
use crate::state::app_dir::AppDataRoot;
use anyhow::{anyhow, Error as E, Result};
use candle_core::{DType, Device, IndexOp, Tensor, D};
use candle_core::{DType, IndexOp, Tensor, D};
use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
use image::DynamicImage;
use log::info;
use rand::Rng;
use tauri::{AppHandle, Emitter};
pub struct Args<'a, P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>> {
pub struct Args<'a> {
pub image: &'a DynamicImage,
pub prompt: String,
pub uncond_prompt: String,
pub cfg_scale: Option<f64>,
pub i2i_strength: Option<u8>,
pub configs: &'a AppConfig,
pub model_cache: &'a ModelCache,
pub prompt_cache: &'a PromptCache,
pub sd_version: StableDiffusionVersion,
pub sd_config: StableDiffusionConfig,
pub device: &'a Device,
pub dtype: DType,
pub maybe_seed: Option<u64>,
pub scheduler_steps: usize,
pub sdxl_turbo_vae_path: P,
pub sdxl_turbo_unet_path: Q,
pub clip_json_path: R,
pub clip_weights_path: S,
pub app_data_root: &'a AppDataRoot,
pub app: &'a AppHandle,
}
pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<RgbImage>
where P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>
{
pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
let Args {
prompt,
uncond_prompt,
cfg_scale,
i2i_strength,
configs,
model_cache,
prompt_cache,
app,
image,
sd_version,
sd_config,
device,
dtype,
maybe_seed,
scheduler_steps,
sdxl_turbo_vae_path,
sdxl_turbo_unet_path,
clip_json_path,
clip_weights_path,
app_data_root,
} = args;
let weights_dir = app_data_root.weights_dir();
println!("Starting image generation with the following configuration:");
println!(" Model: {:?}", configs.sd_version);
println!(" Prompt: {}", prompt);
println!(" Steps: {}", configs.scheduler_steps);
println!(" Device: {:?}", configs.device);
println!("Model dimensions: {}x{}", configs.sd_config.width, configs.sd_config.height);
// TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
// This is a target for performance improvement
// See: https://github.com/huggingface/candle/issues/1331
let mut scheduler = sd_config.build_scheduler(scheduler_steps)?;
let mut scheduler = configs.sd_config.build_scheduler(
configs.scheduler_steps)?;
let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
device.set_seed(seed)?;
let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
configs.device.set_seed(seed)?;
info!("Using seed: {}", seed);
let guidance_scale = match cfg_scale {
Some(guidance_scale) => guidance_scale,
None => match sd_version {
None => match configs.sd_version {
StableDiffusionVersion::V1_5
| StableDiffusionVersion::V2_1
| StableDiffusionVersion::Xl => 7.5,
@@ -93,21 +90,20 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
} else {
info!("Prompt is NOT cached! Calculating embedding...");
let tensor = infer_clip_text_embeddings(ClipArgs {
prompt: &prompt,
uncond_prompt: &uncond_prompt,
tokenizer: None, // tokenizer
clip_weights: None, // clip_weights
clip2_weights: None, // clip2_weights
sd_version: sd_version,
sd_config: &sd_config,
use_f16: false, // use_f16
device: device,
dtype: dtype,
let tensor = infer_clip_text_embeddings(
&prompt,
&uncond_prompt,
None, // tokenizer
None, // clip_weights
None, // clip2_weights
configs.sd_version,
&configs.sd_config,
false, // use_f16
&configs.device,
configs.dtype,
use_guide_scale,
clip_json_path: clip_json_path.as_ref(),
clip_weights_path: clip_weights_path.as_ref(),
})?;
weights_dir,
)?;
prompt_cache.store_copy(&prompt, &tensor)?;
tensor
};
@@ -116,11 +112,14 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
println!("Loading input image into tensor...");
let input_image = dynamic_image_to_tensor(image, device, dtype)?;
let input_image = dynamic_image_to_tensor(
image,
&configs.device,
configs.dtype)?;
println!("Reference image shape: {:?}", input_image.shape());
let vae_scale = get_vae_scale(sd_version);
let vae_scale = get_vae_scale(configs.sd_version);
let maybe_vae = model_cache.get_vae()?;
@@ -129,9 +128,11 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
None => {
info!("No vae found in cache; loading...");
let vae_file = sdxl_turbo_vae_path.as_ref();
let vae_file = weights_dir.weight_path(&SDXL_TURBO_VAE);
let vae = sd_config.build_vae(vae_file, &device, dtype)?;
let vae = configs
.sd_config
.build_vae(vae_file, &configs.device, configs.dtype)?;
let vae = Arc::new(vae);
@@ -148,9 +149,9 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
None => {
info!("No unet found in cache; loading...");
let unet_weights_file = sdxl_turbo_unet_path.as_ref();
let unet_weights_file = weights_dir.weight_path(&SDXL_TURBO_UNET);
let unet = sd_config.build_unet(unet_weights_file, &device, 4, true, dtype)
let unet = configs.sd_config.build_unet(unet_weights_file, &configs.device, 4, true, configs.dtype)
.map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
let unet = Arc::new(unet);
@@ -180,9 +181,9 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
};
let t_start = {
let start = scheduler_steps - (scheduler_steps as f64 * img2img_strength) as usize;
let start = configs.scheduler_steps - (configs.scheduler_steps as f64 * img2img_strength) as usize;
println!("Starting from step {} of {} (strength: {})", start, scheduler_steps, img2img_strength);
println!("Starting from step {} of {} (strength: {})", start, configs.scheduler_steps, img2img_strength);
start
};
@@ -1,4 +1,3 @@
mod weight_descriptor_builder;
pub mod flux_weights;
pub mod weight_descriptor;
pub mod weights;
@@ -5,8 +5,6 @@
#[derive(Clone, Copy)]
pub enum WeightFunction {
ImageGeneration,
ImageSegmentation,
TextEncoder,
TextTokenizer,
Unet,
@@ -30,8 +28,6 @@ pub struct WeightDescriptor {
impl WeightFunction {
pub fn as_descriptive_str(&self) -> &'static str {
match self {
WeightFunction::ImageGeneration => "image generation",
WeightFunction::ImageSegmentation => "image segmentation",
WeightFunction::TextEncoder => "text encoder",
WeightFunction::TextTokenizer => "text tokenizer",
WeightFunction::Unet => "UNET",
@@ -5,10 +5,10 @@ pub (super) const R2_BUCKET_URL : &str = "https://pub-bc5e2bc0cdee4bb5ae8fca9d64
// and variables (even if they're known to be &'static) and keep allocations static.
macro_rules! weight {
($name: literal, $file_name:literal, $weight_function:expr) => {
$crate::weights_registry::weight_descriptor::WeightDescriptor {
$crate::ml::weights_registry::weight_descriptor::WeightDescriptor {
name: $name,
filename: $file_name,
r2_download_url: const_format::concatcp!(crate::weights_registry::weight_descriptor_builder::R2_BUCKET_URL, $file_name),
r2_download_url: const_format::concatcp!(crate::ml::weights_registry::weight_descriptor_builder::R2_BUCKET_URL, $file_name),
function: $weight_function,
}
};
@@ -1,5 +1,5 @@
use crate::weights_registry::weight_descriptor::{WeightDescriptor, WeightFunction};
use crate::weights_registry::weight_descriptor_builder::weight;
use crate::ml::weights_registry::weight_descriptor::{WeightDescriptor, WeightFunction};
use crate::ml::weights_registry::weight_descriptor_builder::weight;
pub const CLIP_JSON : WeightDescriptor = weight!(
"CLIP",
@@ -28,14 +28,6 @@ pub const SIMIANLUO_LCM_DREAMSHAPER_V7_UNET : WeightDescriptor = weight!(
WeightFunction::Unet
);
/// Image segmentation and background removal
/// dis_medium.onnx
pub const DIS_MEDIUM_ONNX : WeightDescriptor = weight!(
"Dichotomous Image Segmentation Medium",
"dis_medium.onnx",
WeightFunction::ImageSegmentation
);
#[deprecated]
pub const SDXL_TURBO_CLIP_TEXT_ENCODER : WeightDescriptor = weight!(
"SDXL Turbo CLIP",
@@ -1,38 +1,62 @@
use crate::ml::model_file::StableDiffusionVersion;
use crate::state::app_dir::AppDataRoot;
use crate::state::os_platform::OsPlatform;
use crate::state::yaml_config::YamlConfig;
use candle_core::{DType, Device};
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
use filesys::recursively_find_file_by_name::recursively_find_file_by_name;
use hf_hub::api::sync::Api;
use log::info;
use std::path::PathBuf;
const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;
const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::V1_5;
pub struct ModelConfig {
/// We only support one device type currently, and we can't multiplex
/// A central place to configure app-wide details.
pub struct AppConfig {
/// We only support one device type currently, and we can't multiplex
/// over several devices yet.
pub device: Device,
/// The core numeric type to use for models.
pub dtype: DType,
pub sd_version: StableDiffusionVersion,
pub sd_config: StableDiffusionConfig,
pub image_height: usize,
pub image_width: usize,
pub scheduler_steps: usize,
pub scheduler_samples: usize,
pub seed: Option<u64>,
pub cfg_scale: Option<f64>,
/// Probably shouldn't be here.
pub hf_api: Api,
/// Root location for application data.
pub app_data_root: AppDataRoot,
}
impl ModelConfig {
impl AppConfig {
pub fn init() -> anyhow::Result<Self> {
let yaml_configs = YamlConfig::load_from_config_file_recursive();
println!("Configs: {:?}", yaml_configs);
let device = Device::new_cuda(0)
.unwrap_or_else(|e| {
info!("CUDA not available ({}), falling back to CPU", e);
Device::Cpu
});
let sd_config = match SD_VERSION {
StableDiffusionVersion::Turbo => {
StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
@@ -47,15 +71,39 @@ impl ModelConfig {
StableDiffusionConfig::sdxl(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
}
};
let hf_api = Api::new()?;
let os_platform = OsPlatform::get();
let default_data_root = match os_platform {
OsPlatform::Linux => yaml_configs.linux_default_data_path.as_deref(),
OsPlatform::MacOs => yaml_configs.mac_default_data_path.as_deref(),
OsPlatform::Windows => yaml_configs.windows_default_data_path.as_deref(),
};
println!("Possible user-defined data root: {:?}", default_data_root);
let app_data_root = match default_data_root {
None => AppDataRoot::create_default()?,
Some(path) => AppDataRoot::create_existing(path)?,
};
println!("App data root: {:?}", app_data_root.path());
Ok(Self {
device,
dtype: DEFAULT_SD_NUMERIC_DATATYPE,
sd_version: SD_VERSION,
sd_config,
image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
seed: yaml_configs.seed,
cfg_scale: yaml_configs.cfg_scale,
hf_api,
app_data_root,
})
}
}
}
@@ -1,10 +1,9 @@
use crate::ml::weights_registry::weight_descriptor::WeightDescriptor;
use crate::state::expanduser::expanduser;
use crate::state::os_platform::OsPlatform;
use anyhow::anyhow;
use directories::UserDirs;
use ml_weights_registry::weights_registry::weight_descriptor::WeightDescriptor;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Local};
use tempdir::TempDir;
use tempfile::{Builder, NamedTempFile};
@@ -14,9 +13,6 @@ const WEIGHTS_SUBDIRECTORY : &str = "weights";
const TEMPORARY_SUBDIRECTORY : &str = "temp";
/// Note: Tauri appends ".log" to the end of the filename.
const LOG_FILE_NAME : &str = "application_debug";
/// The path to the application data directory, which includes "asset" and "weights" data.
#[derive(Clone)]
pub struct AppDataRoot {
@@ -24,8 +20,6 @@ pub struct AppDataRoot {
assets_dir: AppAssetsDir,
weights_dir: AppWeightsDir,
temp_dir: TemporaryDir,
log_file_name: PathBuf,
log_file_name_string: String,
}
#[derive(Clone)]
@@ -76,19 +70,12 @@ impl AppDataRoot {
let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
let temp_dir = TemporaryDir::create_existing(dir.join(TEMPORARY_SUBDIRECTORY))?;
let log_file_name = dir.join(LOG_FILE_NAME);
let log_file_name_string = log_file_name
.to_str()
.ok_or(anyhow!("couldn't convert log path to str"))?
.to_string();
Ok(Self {
path: dir,
assets_dir,
weights_dir,
temp_dir,
log_file_name,
log_file_name_string,
})
}
@@ -107,14 +94,6 @@ impl AppDataRoot {
pub fn path(&self) -> &Path {
&self.path
}
pub fn log_file_name(&self) -> &Path {
&self.log_file_name
}
pub fn log_file_name_str(&self) -> &str {
&self.log_file_name_string
}
}
impl AppAssetsDir {
@@ -138,18 +117,6 @@ impl AppAssetsDir {
pub fn path(&self) -> &Path {
&self.path
}
/// Return the current date asset directory
/// If it doesn't exist, create it.
pub fn make_or_get_current_date_dir(&self) -> anyhow::Result<PathBuf> {
let format: DateTime<Local> = Local::now();
let date_directory_name = format.format("%Y-%d-%m").to_string();
let full_path = self.path.join(date_directory_name);
if !full_path.exists() {
std::fs::create_dir(&full_path)?;
}
Ok(full_path)
}
}
impl AppWeightsDir {
@@ -1,11 +1,9 @@
use crate::events::notification_event::{NotificationEvent, NotificationModelType};
use crate::state::app_dir::AppDataRoot;
use crate::transfer::download::{download_async, ProgressUpdate};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use futures::StreamExt;
use log::{debug, error, info};
use ml_weights_registry::weights_registry::weight_descriptor::WeightFunction;
use ml_weights_registry::weights_registry::weights::{CLIP_JSON, DIS_MEDIUM_ONNX, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2, SDXL_TURBO_UNET, SDXL_TURBO_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
use std::collections::VecDeque;
use std::path::PathBuf;
use std::sync::mpsc::Receiver;
@@ -13,7 +11,8 @@ use std::sync::{Arc, Mutex};
use tauri::{AppHandle, Emitter};
use tempfile::NamedTempFile;
use tokio::task::JoinHandle;
use ml_weights_registry::weights_registry::flux_weights::{FLUX_SCHNELL, FLUX_SCHNELL_AUTOENCODER, GOOGLE_T5_V1_1_XXL_CONFIG, GOOGLE_T5_V1_1_XXL_MODEL, LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF, LMZ_T5_TOKENIZER_JSON, OPENAI_CLIP_VIT_P14_MODEL, OPENAI_CLIP_VIT_P14_TOKENIZER_JSON};
use crate::ml::weights_registry::weight_descriptor::WeightFunction;
use crate::ml::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SDXL_TURBO_CLIP_TEXT_ENCODER_2, SDXL_TURBO_UNET, SDXL_TURBO_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
const NOTIFICATION_CHANNEL_NAME : &str = "notification";
const MAX_FILES : usize = 8;
@@ -46,21 +45,10 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
download_queue.push_back(LYKON_DEAMSHAPER_7_VAE);
download_queue.push_back(SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
download_queue.push_back(SDXL_TURBO_CLIP_TEXT_ENCODER); // TODO(bt): Why is this still needed?
download_queue.push_back(DIS_MEDIUM_ONNX);
//download_queue.push_back(SDXL_TURBO_CLIP_TEXT_ENCODER_2); // TODO(bt): REMOVE
//download_queue.push_back(SDXL_TURBO_VAE); // TODO(bt): REMOVE
//download_queue.push_back(SDXL_TURBO_UNET); // TODO(bt): REMOVE
// Flux:
download_queue.push_back(FLUX_SCHNELL);
download_queue.push_back(FLUX_SCHNELL_AUTOENCODER);
download_queue.push_back(LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF);
download_queue.push_back(GOOGLE_T5_V1_1_XXL_MODEL);
download_queue.push_back(GOOGLE_T5_V1_1_XXL_CONFIG);
download_queue.push_back(LMZ_T5_TOKENIZER_JSON);
download_queue.push_back(OPENAI_CLIP_VIT_P14_MODEL);
download_queue.push_back(OPENAI_CLIP_VIT_P14_TOKENIZER_JSON);
let mut handles = FuturesUnordered::new();
while let Some(model) = download_queue.pop_front() {
@@ -92,8 +80,6 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
WeightFunction::TextTokenizer => NotificationModelType::TextTokenizer,
WeightFunction::Unet => NotificationModelType::Unet,
WeightFunction::Vae => NotificationModelType::Vae,
WeightFunction::ImageSegmentation => NotificationModelType::ImageSegmentation,
WeightFunction::ImageGeneration => NotificationModelType::ImageSegmentation, // TODO FIX
};
let result = app.emit(NOTIFICATION_CHANNEL_NAME, NotificationEvent::ModelDownloadStarted {
@@ -1,6 +1,7 @@
use crate::ml::image::tensor_to_image_buffer::RgbImage;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use image::{ImageFormat, RgbImage};
use image::ImageFormat;
use std::io::Cursor;
pub fn encode_rgb_image_base64_png(
@@ -0,0 +1 @@
pub mod image;
@@ -32,6 +32,9 @@
"icons/128x128@2x.png",
"icons/icon.icns",
"icons/icon.ico"
]
],
"resources": {
"../../../windows_dlls/*.dll": "."
}
}
}
-74
View File
@@ -1,74 +0,0 @@
[package]
name = "artcraft"
version = "0.1.0"
description = "Artcraft"
authors = ["you"]
license = ""
repository = ""
edition = "2021"
rust-version = "1.77.2"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "app_lib"
crate-type = ["staticlib", "cdylib", "rlib"]
[build-dependencies]
tauri-build = { version = "2.1.1", features = [] }
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { workspace = true }
# Internal
crockford = { workspace = true }
errors = { workspace = true }
filesys = { workspace = true }
memory_caching = { workspace = true }
openai_sora_client = { workspace = true }
primitives = { workspace = true }
subprocess_common = { workspace = true }
# External
#openssl = { version = "0.10", features = ["vendored"] }
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
anyhow = { workspace = true }
base64 = "0.22.1"
bytes = "1.10"
chrono = { workspace = true }
const_format = "0.2.34"
directories = "6.0.0"
dirs = "6.0.0"
futures = "0.3"
image = "0.25.5"
log = { workspace = true }
once_cell = { workspace = true }
rand = { workspace = true }
reqwest = { workspace = true, features = ["multipart", "stream"], default-features = false }
serde = { workspace = true }
serde_derive = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tauri = { version = "2.4.1", features = ["unstable"] }
tauri-plugin-log = "2.3.1"
tempdir = "0.3.7"
tempfile = "3.18.0"
tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
tokio-util = { version = "0.7", features = ["codec"] }
[dev-dependencies]
libloading = "0.8.6"
serial_test = "3.2.0"
assertor = "0.0.3"
[profile.dev]
incremental = true # Compile your binary in smaller steps.
[profile.release]
codegen-units = 1 # Allows LLVM to perform better optimization.
lto = true # Enables link-time-optimizations.
opt-level = 3 # `s` prioritizes small binary size. Use `3` if you prefer speed.
panic = "abort" # Higher performance by disabling panic handlers.
strip = true # Ensures debug symbols are removed.
@@ -1,90 +0,0 @@
use std::fs::read_to_string;
use base64::prelude::BASE64_STANDARD;
use errors::AnyhowResult;
use base64::Engine;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, ImageReader};
use std::io::Cursor;
use log::{error, info};
use tauri::{AppHandle, Manager, State};
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::upload::upload_media_from_file::SoraMediaUploadRequest;
use crate::state::app_dir::AppDataRoot;
#[tauri::command]
pub async fn image_generation_command(
image: &str,
prompt: &str,
app: AppHandle,
app_data_root: State<'_, AppDataRoot>,
) -> Result<String, String> {
info!("image_generation_command called; processing image...");
let bytes = BASE64_STANDARD.decode(image)
.map_err(|err| format!("Base64 decode error: {}", err))?;
generate_image(bytes, prompt, &app_data_root)
.await
.map_err(|err| {
error!("error: {:?}", err);
"there was an error".to_string()
})?;
Ok("success".to_string())
}
pub async fn generate_image(file_bytes: Vec<u8>, prompt: &str, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
let sora_credentials = get_credentials(app_data_root)?;
let filename = "image.png".to_string();
let response= sora_media_upload_from_bytes(file_bytes, filename, &sora_credentials)
.await
.map_err(|err| {
error!("Failed to upload image to Sora: {:?}", err);
err
})?;
let sora_media_tokens = vec![response.id];
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
prompt: prompt.to_string(),
num_images: NumImages::One,
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: &sora_credentials,
}).await
.map_err(|err| {
error!("Failed to call Sora image generation: {:?}", err);
err
})?;
println!(">> TASK ID: {:?} ", response.task_id);
Ok(())
}
fn get_credentials(app_data_root: &AppDataRoot) -> AnyhowResult<SoraCredentials> {
let cookie_file = app_data_root.get_sora_cookie_file_path();
let bearer = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/bearer.txt")?
.trim()
.to_string();
let cookie= read_to_string(cookie_file)?
.trim()
.to_string();
let sentinel = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/sentinel.txt")?
.trim()
.to_string();
Ok(SoraCredentials {
bearer_token: bearer,
cookie: cookie,
sentinel: Some(sentinel),
})
}
@@ -1,3 +0,0 @@
pub mod flip_image;
pub mod image_generation_command;
pub mod open_login_command;

Some files were not shown because too many files have changed in this diff Show More