Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14feaccb91 | ||
|
|
e1dbf4f2fe | ||
|
|
d826db599d | ||
|
|
a642732c59 | ||
|
|
435c5aa0b4 | ||
|
|
90b4aff846 | ||
|
|
4d3f097432 | ||
|
|
e5de0e4731 | ||
|
|
04498b0457 | ||
|
|
fbfc1e15f2 | ||
|
|
4a65317729 | ||
|
|
e7ed0693a3 | ||
|
|
99ac8faf33 | ||
|
|
f0213d4a63 | ||
|
|
58f684b1e0 | ||
|
|
e83389a874 | ||
|
|
3602f8fc2b | ||
|
|
115c4fc60c |
@@ -1,4 +1,4 @@
|
||||
hakari-package = "workspace-build-acceleration"
|
||||
hakari-package = "my-workspace-hack"
|
||||
|
||||
resolver = "1"
|
||||
|
||||
@@ -11,4 +11,4 @@ platforms = [
|
||||
"x86_64-pc-windows-msvc",
|
||||
]
|
||||
|
||||
exact-versions = true
|
||||
exact-versions = true
|
||||
@@ -32,9 +32,6 @@ target/
|
||||
crates/lib/database_queries/target/
|
||||
crates/lib/sqlite_queries/target/
|
||||
|
||||
# Javascript build directories
|
||||
node_modules/
|
||||
|
||||
# Vim noise
|
||||
*.swo
|
||||
*.swp
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
|
||||
# This block defines the workspace members.
|
||||
# When we add a library or binary crate, it needs an entry here.
|
||||
members = [
|
||||
# Types: common strongly-typed tokens, ids, payloads, etc.
|
||||
"crates/_types/composite_identifiers",
|
||||
@@ -24,8 +22,9 @@ members = [
|
||||
"crates/cli/production/migration-tool",
|
||||
|
||||
# Desktop apps
|
||||
"crates/desktop/tauri-4o",
|
||||
"crates/desktop/tauri-realtime",
|
||||
"crates/desktop/src-tauri",
|
||||
"crates/desktop/ml_models",
|
||||
"crates/desktop/ml_weights_registry",
|
||||
|
||||
# Shared libraries
|
||||
"crates/lib/actix_cors_configs",
|
||||
@@ -33,7 +32,6 @@ members = [
|
||||
"crates/lib/api_clients/cloud_storage",
|
||||
"crates/lib/api_clients/google_drive_common",
|
||||
"crates/lib/api_clients/google_sign_in",
|
||||
"crates/lib/api_clients/openai_sora_client",
|
||||
"crates/lib/api_clients/thumbnail_generator",
|
||||
"crates/lib/bootstrap",
|
||||
"crates/lib/bucket_paths",
|
||||
@@ -60,20 +58,16 @@ members = [
|
||||
"crates/lib/files/mimetypes",
|
||||
"crates/lib/files/videos",
|
||||
"crates/lib/idempotency",
|
||||
"crates/lib/image_algorithms",
|
||||
"crates/lib/jobs_common",
|
||||
"crates/lib/logging",
|
||||
"crates/lib/machine_learning/ml_models",
|
||||
"crates/lib/machine_learning/ml_weights_registry",
|
||||
"crates/lib/machine_learning/prompt_engineering",
|
||||
"crates/lib/markdown",
|
||||
"crates/lib/migration",
|
||||
"crates/lib/mysql_queries",
|
||||
"crates/lib/password",
|
||||
"crates/lib/prompt_engineering",
|
||||
"crates/lib/redis_common",
|
||||
"crates/lib/redis_schema",
|
||||
"crates/lib/server_environment",
|
||||
"crates/lib/shared_service_components",
|
||||
"crates/lib/sqlx_mysql_helpers",
|
||||
"crates/lib/storyteller_root",
|
||||
"crates/lib/subprocess_common",
|
||||
@@ -121,40 +115,5 @@ members = [
|
||||
|
||||
# cargo-hakari workspace hack (compilation speedup hack)
|
||||
# see usage: https://crates.io/crates/cargo-hakari
|
||||
"crates/workspace-build-acceleration",
|
||||
"crates/my-workspace-hack",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
# Common inheritable settings for monorepo packages
|
||||
publish = false
|
||||
|
||||
[workspace.dependencies]
|
||||
# Hakari workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
|
||||
|
||||
# External libs
|
||||
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
|
||||
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
|
||||
anyhow = "1.0.97"
|
||||
chrono = { version = "0.4.40", features = ["serde"] }
|
||||
log = "0.4.27"
|
||||
once_cell = "1.21.3"
|
||||
rand = "0.9.0"
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = "1.0.140"
|
||||
serde_yaml = "0.9.1"
|
||||
toml = "0.7.4"
|
||||
url = "2.5.4"
|
||||
|
||||
# Internal libs
|
||||
crockford = { path = "crates/lib/data/crockford" }
|
||||
errors = { path = "crates/lib/errors" }
|
||||
filesys = { path = "crates/lib/files/filesys" }
|
||||
memory_caching = { path = "crates/lib/caching/memory_caching" }
|
||||
shared_service_components = { path = "crates/lib/shared_service_components" }
|
||||
openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
|
||||
primitives = { path = "crates/lib/data/primitives" }
|
||||
subprocess_common = { path = "crates/lib/subprocess_common" }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "composite_identifiers"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ enums = { path = "../enums" }
|
||||
tokens = { path = "../tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -13,13 +13,13 @@ name = "enums"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -40,10 +40,6 @@ pub enum InferenceJobType {
|
||||
#[serde(rename = "studio_gen2")]
|
||||
StudioGen2,
|
||||
|
||||
/// Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
|
||||
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
|
||||
#[serde(rename = "convert_fbx_gltf")]
|
||||
ConvertFbxToGltf,
|
||||
@@ -112,7 +108,6 @@ impl InferenceJobType {
|
||||
Self::GptSovits => "gpt_sovits",
|
||||
Self::ComfyUi => "comfy_ui",
|
||||
Self::StudioGen2 => "studio_gen2",
|
||||
Self::ImageGenApi => "image_gen_api",
|
||||
Self::ConvertFbxToGltf => "convert_fbx_gltf",
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::RvcV2 => "rvc_v2",
|
||||
@@ -137,7 +132,6 @@ impl InferenceJobType {
|
||||
"gpt_sovits" => Ok(Self::GptSovits),
|
||||
"comfy_ui" => Ok(Self::ComfyUi),
|
||||
"studio_gen2" => Ok(Self::StudioGen2),
|
||||
"image_gen_api" => Ok(Self::ImageGenApi),
|
||||
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"rvc_v2" => Ok(Self::RvcV2),
|
||||
@@ -165,7 +159,6 @@ impl InferenceJobType {
|
||||
Self::GptSovits,
|
||||
Self::ComfyUi,
|
||||
Self::StudioGen2,
|
||||
Self::ImageGenApi,
|
||||
Self::ConvertFbxToGltf,
|
||||
Self::MocapNet,
|
||||
Self::RvcV2,
|
||||
@@ -204,7 +197,6 @@ mod tests {
|
||||
assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
|
||||
assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
|
||||
assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
|
||||
assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
|
||||
assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
|
||||
assert_serialization(InferenceJobType::MocapNet, "mocap_net");
|
||||
assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
|
||||
@@ -228,7 +220,6 @@ mod tests {
|
||||
assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
|
||||
assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
|
||||
assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
|
||||
assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
|
||||
assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
|
||||
assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
|
||||
assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
|
||||
@@ -252,7 +243,6 @@ mod tests {
|
||||
assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
|
||||
assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
|
||||
assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
|
||||
assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
|
||||
assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
|
||||
assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
|
||||
assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
|
||||
@@ -271,7 +261,7 @@ mod tests {
|
||||
fn all_variants() {
|
||||
// Static check
|
||||
let mut variants = InferenceJobType::all_variants();
|
||||
assert_eq!(variants.len(), 17);
|
||||
assert_eq!(variants.len(), 16);
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
|
||||
@@ -279,7 +269,6 @@ mod tests {
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
|
||||
|
||||
@@ -33,10 +33,6 @@ pub enum InferenceModelType {
|
||||
#[serde(rename = "seed_vc")]
|
||||
SeedVc,
|
||||
|
||||
/// NB: This is for Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
|
||||
#[serde(rename = "tacotron2")]
|
||||
Tacotron2,
|
||||
#[serde(rename = "vits")]
|
||||
@@ -74,7 +70,6 @@ impl InferenceModelType {
|
||||
Self::VallEX => "vall_e_x",
|
||||
Self::RerenderAVideo => "rerender_a_video",
|
||||
Self::StableDiffusion => "stable_diffusion",
|
||||
Self::ImageGenApi => "image_gen_api",
|
||||
Self::SeedVc => "seed_vc",
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::StyleTTS2 => "styletts2",
|
||||
@@ -95,7 +90,6 @@ impl InferenceModelType {
|
||||
"vall_e_x" => Ok(Self::VallEX),
|
||||
"rerender_a_video" => Ok(Self::RerenderAVideo),
|
||||
"stable_diffusion" => Ok(Self::StableDiffusion),
|
||||
"image_gen_api" => Ok(Self::ImageGenApi),
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"styletts2" => Ok(Self::StyleTTS2),
|
||||
"comfy_ui" => Ok(Self::ComfyUi),
|
||||
@@ -119,7 +113,6 @@ impl InferenceModelType {
|
||||
Self::VallEX,
|
||||
Self::RerenderAVideo,
|
||||
Self::StableDiffusion,
|
||||
Self::ImageGenApi,
|
||||
Self::MocapNet,
|
||||
Self::StyleTTS2,
|
||||
Self::ConvertFbxToGltf,
|
||||
@@ -147,7 +140,6 @@ mod tests {
|
||||
assert_serialization(InferenceModelType::VallEX, "vall_e_x");
|
||||
assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
|
||||
assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
|
||||
assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
|
||||
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
|
||||
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
|
||||
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
|
||||
@@ -167,7 +159,6 @@ mod tests {
|
||||
assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
|
||||
assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
|
||||
assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
|
||||
assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
|
||||
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
|
||||
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
|
||||
assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
|
||||
@@ -185,7 +176,6 @@ mod tests {
|
||||
assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
|
||||
assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
|
||||
assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
|
||||
assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
|
||||
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
|
||||
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
|
||||
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
|
||||
@@ -197,18 +187,17 @@ mod tests {
|
||||
fn all_variants() {
|
||||
// Static check
|
||||
let mut variants = InferenceModelType::all_variants();
|
||||
assert_eq!(variants.len(), 15);
|
||||
assert_eq!(variants.len(), 13);
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
|
||||
|
||||
@@ -57,9 +57,6 @@ pub enum MediaFileOriginModelType {
|
||||
#[serde(rename = "studio")]
|
||||
StorytellerStudio,
|
||||
|
||||
#[serde(rename = "studio_ig")]
|
||||
StorytellerStudioImageGen,
|
||||
|
||||
#[serde(rename = "vst")]
|
||||
VideoStyleTransfer,
|
||||
|
||||
@@ -96,7 +93,6 @@ impl MediaFileOriginModelType {
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::StyleTTS2 => "styletts2",
|
||||
Self::StableDiffusion15 => "stable_diffusion_1_5",
|
||||
Self::StorytellerStudioImageGen => "studio_ig",
|
||||
Self::GptSovits => "gpt_sovits",
|
||||
Self::StorytellerStudio => "studio",
|
||||
Self::VideoStyleTransfer => "vst",
|
||||
@@ -119,7 +115,6 @@ impl MediaFileOriginModelType {
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"styletts2" => Ok(Self::StyleTTS2),
|
||||
"stable_diffusion_1_5" => Ok(Self::StableDiffusion15),
|
||||
"studio_ig" => Ok(Self::StorytellerStudioImageGen),
|
||||
"gpt_sovits" => Ok(Self::GptSovits),
|
||||
"studio" => Ok(Self::StorytellerStudio),
|
||||
"vst" => Ok(Self::VideoStyleTransfer),
|
||||
@@ -147,7 +142,6 @@ impl MediaFileOriginModelType {
|
||||
Self::StableDiffusion15,
|
||||
Self::GptSovits,
|
||||
Self::StorytellerStudio,
|
||||
Self::StorytellerStudioImageGen,
|
||||
Self::VideoStyleTransfer,
|
||||
Self::ComfyUi,
|
||||
Self::VallEX,
|
||||
|
||||
@@ -13,11 +13,11 @@ name = "enums_public"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Internal
|
||||
enums = { path = "../enums" }
|
||||
|
||||
@@ -91,10 +91,6 @@ pub enum PublicInferenceJobType {
|
||||
/// Re-render a video is a video style transfer algorithm. We developed code
|
||||
/// around it, but chose to develop AnimateDiff / ComfyUI support instead.
|
||||
RerenderAVideo,
|
||||
|
||||
/// Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
}
|
||||
|
||||
/// NB: Legacy API for older code.
|
||||
@@ -123,7 +119,6 @@ impl PublicInferenceJobType {
|
||||
InferenceJobType::Unknown => Self::Unknown,
|
||||
InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
|
||||
InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
|
||||
InferenceJobType::ImageGenApi => Self::ImageGenApi,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,7 +140,6 @@ impl PublicInferenceJobType {
|
||||
Self::SeedVc => InferenceJobType::SeedVc,
|
||||
Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
|
||||
Self::StableDiffusion => InferenceJobType::StableDiffusion,
|
||||
Self::ImageGenApi => InferenceJobType::ImageGenApi,
|
||||
Self::StyleTTS2 => InferenceJobType::StyleTTS2,
|
||||
Self::Tacotron2 => InferenceJobType::Tacotron2,
|
||||
Self::Unknown => InferenceJobType::Unknown,
|
||||
|
||||
@@ -64,10 +64,6 @@ pub enum PublicMediaFileModelType {
|
||||
#[serde(rename = "studio")]
|
||||
StorytellerStudio,
|
||||
|
||||
/// NB: This is GPT4o image generation
|
||||
#[serde(rename = "studio_ig")]
|
||||
StorytellerStudioImageGen,
|
||||
|
||||
#[serde(rename = "vst")]
|
||||
VideoStyleTransfer,
|
||||
|
||||
@@ -93,7 +89,6 @@ impl PublicMediaFileModelType {
|
||||
MediaFileOriginModelType::LivePortrait => Self::FaceMirror,
|
||||
MediaFileOriginModelType::SadTalker => Self::FaceAnimator,
|
||||
MediaFileOriginModelType::StyleTTS2 => Self::VoiceDesigner,
|
||||
MediaFileOriginModelType::StorytellerStudioImageGen => Self::StorytellerStudioImageGen,
|
||||
// Conserved variants
|
||||
MediaFileOriginModelType::RvcV2 => Self::RvcV2,
|
||||
MediaFileOriginModelType::SoVitsSvc => Self::SoVitsSvc,
|
||||
@@ -118,7 +113,6 @@ impl PublicMediaFileModelType {
|
||||
Self::FaceAnimator => MediaFileOriginModelType::SadTalker,
|
||||
Self::Lipsync => MediaFileOriginModelType::FaceFusion,
|
||||
Self::VoiceDesigner => MediaFileOriginModelType::StyleTTS2,
|
||||
Self::StorytellerStudioImageGen => MediaFileOriginModelType::StorytellerStudioImageGen,
|
||||
// Conserved variants
|
||||
Self::RvcV2 => MediaFileOriginModelType::RvcV2,
|
||||
Self::SoVitsSvc => MediaFileOriginModelType::SoVitsSvc,
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "reusable_types"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
# (none)
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
chrono = { version = "0.4.22", features = ["serde"] }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "tokens"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
# (none)
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "dev-database-seed"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -32,7 +32,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
log = "0.4.14"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "dev-upload-media-file"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -29,7 +29,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "scrape-test"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -22,7 +22,7 @@ errors = { path = "../../../lib/errors" }
|
||||
web_scrapers = { path = "../../../lib/web_scrapers" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,11 +13,11 @@ name = "db-backfill"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# Internal
|
||||
datetimes = { path = "../../../lib/data/datetimes" }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "db-backup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ easyenv = { path = "../../../lib/easyenv" }
|
||||
errors = { path = "../../../lib/errors" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
anyhow = "1.0.86"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "db-cleanup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -26,7 +26,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "elasticsearch-cli"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -29,7 +29,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "kubectl-pod-cleanup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -22,7 +22,7 @@ easyenv = { path = "../../../lib/easyenv" }
|
||||
errors = { path = "../../../lib/errors" }
|
||||
|
||||
# Workspace hack (faster compile yytimes)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
log = "0.4.20"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "lru-delete-models"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" }
|
||||
logging = { path = "../../../lib/logging" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = { version = "4.4.8", features = ["derive"] }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "migration-tool"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -30,7 +30,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -14,19 +14,15 @@ path = "src/lib.rs"
|
||||
crate-type = ["staticlib", "cdylib", "rlib"]
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
filesys = { path = "../../files/filesys" }
|
||||
image_algorithms = { path = "../../image_algorithms" }
|
||||
memory_caching = { path = "../../caching/memory_caching" }
|
||||
filesys = { path = "../../lib/files/filesys" }
|
||||
memory_caching = { path = "../../lib/caching/memory_caching" }
|
||||
ml_weights_registry = { path = "../ml_weights_registry" }
|
||||
primitives = { path = "../../data/primitives" }
|
||||
primitives = { path = "../../lib/data/primitives" }
|
||||
|
||||
# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
|
||||
cudarc = "0.13.7"
|
||||
@@ -35,9 +31,9 @@ cudarc = "0.13.7"
|
||||
anyhow = "1.0"
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.10"
|
||||
candle-core = { path = "../../../vendor/candle/candle-core", features = ["cuda"] }
|
||||
candle-nn = { path = "../../../vendor/candle/candle-nn", features = ["cuda"] }
|
||||
candle-transformers = { path = "../../../vendor/candle/candle-transformers", features = ["cuda"] }
|
||||
candle-core = { path = "../../vendor/candle/candle-core", features = ["cuda"] }
|
||||
candle-nn = { path = "../../vendor/candle/candle-nn", features = ["cuda"] }
|
||||
candle-transformers = { path = "../../vendor/candle/candle-transformers", features = ["cuda"] }
|
||||
const_format = "0.2.34"
|
||||
directories = "6.0.0"
|
||||
dirs = "6.0.0"
|
||||
@@ -3,7 +3,6 @@ use image::{imageops, DynamicImage};
|
||||
use ndarray::{Array, CowArray};
|
||||
use ort::Value;
|
||||
use std::path::Path;
|
||||
use image_algorithms::remove_image_border::remove_image_border;
|
||||
|
||||
pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: DynamicImage) -> anyhow::Result<DynamicImage> {
|
||||
// NB: Real time needed cuDNN 9, but this uses cuDNN 8:
|
||||
@@ -12,10 +11,6 @@ pub async fn remove_image_background<P: AsRef<Path>>(model_path: P, image: Dynam
|
||||
// update-alternatives: using /usr/include/x86_64-linux-gnu/cudnn_v8.h to provide /usr/include/cudnn.h (libcudnn) in manual mode
|
||||
let session = onnx_session(model_path.as_ref())?;
|
||||
let image = process_dynamic_image(&session, image)?;
|
||||
|
||||
// Crop transparent border cruft away.
|
||||
let image = remove_image_border(&image)?;
|
||||
|
||||
Ok(image)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
pub mod background_removal;
|
||||
pub mod flux;
|
||||
pub mod image;
|
||||
pub mod model_cache;
|
||||
pub mod model_config;
|
||||
@@ -20,14 +20,12 @@ use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
|
||||
use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
|
||||
use crate::ml::stable_diffusion::remap_lcm_strength_range::remap_lcm_strength_range;
|
||||
use image::DynamicImage;
|
||||
use log::{debug, error, info};
|
||||
use log::info;
|
||||
use ml_weights_registry::weights_registry::weights::{LYKON_DEAMSHAPER_7_TEXT_ENCODER_FP16, LYKON_DEAMSHAPER_7_VAE, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
|
||||
use rand::Rng;
|
||||
|
||||
const DEFAULT_STRENGTH : f64 = 75.0;
|
||||
|
||||
const DEFAULT_SEED : u64 = 42;
|
||||
|
||||
pub struct Args<'a, P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>> {
|
||||
pub image: &'a DynamicImage,
|
||||
pub prompt: String,
|
||||
@@ -87,9 +85,10 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
// Use LCM Scheduler instead of Euler Ancestral for better speed and quality
|
||||
let mut scheduler = LCMScheduler::new(scheduler_steps, img2img_strength, LCMSchedulerConfig::default())?;
|
||||
|
||||
//let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
let seed = maybe_seed.unwrap_or(DEFAULT_SEED);
|
||||
let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
|
||||
let seed = 42;
|
||||
|
||||
device.set_seed(seed)?;
|
||||
|
||||
let guidance_scale = match cfg_scale {
|
||||
@@ -130,13 +129,13 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
tensor
|
||||
};
|
||||
|
||||
debug!("Text embeddings shape: {:?}", text_embeddings.shape());
|
||||
info!("Text embeddings shape: {:?}", text_embeddings.shape());
|
||||
|
||||
debug!("Loading input image into tensor...");
|
||||
info!("Loading input image into tensor...");
|
||||
|
||||
let input_image = dynamic_image_to_tensor(image, &device, dtype)?;
|
||||
|
||||
debug!("Reference image shape: {:?}", input_image.shape());
|
||||
info!("Reference image shape: {:?}", input_image.shape());
|
||||
|
||||
let vae_scale = get_vae_scale(sd_version);
|
||||
|
||||
@@ -162,7 +161,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
let unet = match maybe_unet {
|
||||
Some(unet) => unet,
|
||||
None => {
|
||||
info!("Building UNET model from file {:?}...", unet_path.as_ref());
|
||||
info!("No unet found in cache; loading...");
|
||||
|
||||
let in_channels = match sd_version {
|
||||
StableDiffusionVersion::XlInpaint | StableDiffusionVersion::V2Inpaint | StableDiffusionVersion::V1_5Inpaint => 9,
|
||||
@@ -218,17 +217,16 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(guidance_scale, embedding_dim, &device, dtype)?;
|
||||
|
||||
// Generate latents from input image using the LCM approach
|
||||
debug!("Generating latents from input image...");
|
||||
info!("Generating latents from input image...");
|
||||
let init_latents = (init_latent_dist.sample()? * vae_scale)?;
|
||||
|
||||
debug!("Initial latents shape: {:?}", init_latents.shape());
|
||||
info!("Initial latents shape: {:?}", init_latents.shape());
|
||||
|
||||
// Calculate timesteps for LCM with proper img2img handling
|
||||
let timesteps = LCMScheduler::get_timesteps_for_steps(scheduler_steps, img2img_strength);
|
||||
let t_start = timesteps[0]; // First denoising step
|
||||
|
||||
// Add noise at the appropriate timestep for img2img
|
||||
debug!("Adding noise to latents for t_start={}", t_start);
|
||||
info!("Adding noise to latents for t_start={}", t_start);
|
||||
let noise = init_latents.randn_like(0f64, 1f64)?;
|
||||
let latents = scheduler.add_noise(&init_latents, noise, t_start)?;
|
||||
|
||||
@@ -246,7 +244,7 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
let mut noise_pred = match unet.forward_with_guidance(&latent_model_input, timestep as f64, &text_embeddings, Some(&guidance_scale_embedding)) {
|
||||
Ok(pred) => pred,
|
||||
Err(e) => {
|
||||
error!("UNet inference failed with error: {}", e);
|
||||
info!("UNet inference failed with error: {}", e);
|
||||
return Err(anyhow::anyhow!("UNet inference failed: {}", e));
|
||||
},
|
||||
};
|
||||
@@ -261,19 +259,21 @@ pub fn lcm_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<Rg
|
||||
latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, scheduler_steps)?;
|
||||
}
|
||||
|
||||
debug!("Diffusion process completed, decoding image...");
|
||||
info!("Diffusion process completed, decoding image...");
|
||||
let image = vae.decode(&(latents / vae_scale)?)?;
|
||||
|
||||
debug!("Scaling image back");
|
||||
info!("VAE decode completed");
|
||||
|
||||
info!("Scaling image back");
|
||||
let image = ((image / 2.)? + 0.5)?;
|
||||
|
||||
debug!("Normalized image values");
|
||||
info!("Normalized image values");
|
||||
let image = (image.clamp(0f32, 1.)? * 255.)?;
|
||||
|
||||
debug!("Convert to int8");
|
||||
info!("Convert to int8");
|
||||
let image = image.to_dtype(DType::U8)?;
|
||||
|
||||
debug!("Converted to 8-bit format");
|
||||
info!("Converted to 8-bit format");
|
||||
|
||||
let image = tensor_to_image_buffer(&image.i(0)?)?;
|
||||
|
||||
@@ -59,6 +59,14 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
clip_weights_path,
|
||||
} = args;
|
||||
|
||||
println!("Starting image generation with the following configuration:");
|
||||
println!(" Model: {:?}", sd_version);
|
||||
println!(" Prompt: {}", prompt);
|
||||
println!(" Steps: {}", scheduler_steps);
|
||||
println!(" Device: {:?}", device);
|
||||
|
||||
println!("Model dimensions: {}x{}", sd_config.width, sd_config.height);
|
||||
|
||||
// TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
|
||||
// This is a target for performance improvement
|
||||
// See: https://github.com/huggingface/candle/issues/1331
|
||||
@@ -1,4 +1,3 @@
|
||||
mod weight_descriptor_builder;
|
||||
pub mod flux_weights;
|
||||
pub mod weight_descriptor;
|
||||
pub mod weights;
|
||||
@@ -5,7 +5,6 @@
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum WeightFunction {
|
||||
ImageGeneration,
|
||||
ImageSegmentation,
|
||||
TextEncoder,
|
||||
TextTokenizer,
|
||||
@@ -30,7 +29,6 @@ pub struct WeightDescriptor {
|
||||
impl WeightFunction {
|
||||
pub fn as_descriptive_str(&self) -> &'static str {
|
||||
match self {
|
||||
WeightFunction::ImageGeneration => "image generation",
|
||||
WeightFunction::ImageSegmentation => "image segmentation",
|
||||
WeightFunction::TextEncoder => "text encoder",
|
||||
WeightFunction::TextTokenizer => "text tokenizer",
|
||||
@@ -1,5 +1,5 @@
|
||||
[package]
|
||||
name = "tauri-realtime"
|
||||
name = "app"
|
||||
version = "0.1.0"
|
||||
description = "A Tauri App"
|
||||
authors = ["you"]
|
||||
@@ -16,30 +16,25 @@ name = "app_lib"
|
||||
crate-type = ["staticlib", "cdylib", "rlib"]
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2.1.1", features = [] }
|
||||
tauri-build = { version = "2.0.4", features = [] }
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
crockford = { path = "../../lib/data/crockford" }
|
||||
filesys = { path = "../../lib/files/filesys" }
|
||||
memory_caching = { path = "../../lib/caching/memory_caching" }
|
||||
ml_weights_registry = { path = "../../lib/machine_learning/ml_weights_registry" }
|
||||
ml_weights_registry = { path = "../ml_weights_registry" }
|
||||
primitives = { path = "../../lib/data/primitives" }
|
||||
|
||||
# Internal ML
|
||||
# TODO: Only on Windows and Linux
|
||||
#ml_models = { path = "../ml_models", optional = true }
|
||||
# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
|
||||
cudarc = { version = "0.13.5", features = ["std", "cublas", "cublaslt", "curand", "driver", "nvrtc", "f16", "cuda-version-from-build-system", "dynamic-linking"], default-features=false }
|
||||
|
||||
# External
|
||||
#openssl = { version = "0.10", features = ["vendored"] }
|
||||
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
|
||||
anyhow = "1.0"
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.10"
|
||||
chrono = "0.4.40"
|
||||
const_format = "0.2.34"
|
||||
directories = "6.0.0"
|
||||
dirs = "6.0.0"
|
||||
@@ -48,14 +43,14 @@ half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_di
|
||||
image = "0.25.5"
|
||||
log = "0.4"
|
||||
once_cell = "1.20.3"
|
||||
ort = { version = "1.16.2", features = ["cuda", "load-dynamic"] }
|
||||
rand = "0.8.5"
|
||||
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
|
||||
reqwest = { version = "0.12.12", features = ["stream"], default-features = false }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
serde_yaml = "0.9"
|
||||
tauri = { version = "2.4.1", features = [] }
|
||||
tauri-plugin-log = "2.3.1"
|
||||
tauri = { version = "2.2.4", features = [] }
|
||||
tauri-plugin-log = "2"
|
||||
tempdir = "0.3.7"
|
||||
tempfile = "3.18.0"
|
||||
tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
|
||||
@@ -64,7 +59,7 @@ tokio-util = { version = "0.7", features = ["codec"] }
|
||||
# tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
|
||||
|
||||
[target.'cfg(not(target_os = "macos"))'.dependencies]
|
||||
ml_models = { path = "../../lib/machine_learning/ml_models" }
|
||||
ml_models = { path = "../ml_models" }
|
||||
|
||||
[dev-dependencies]
|
||||
libloading = "0.8.6"
|
||||
|
Before Width: | Height: | Size: 8.4 KiB After Width: | Height: | Size: 8.4 KiB |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 23 KiB After Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 2.2 KiB After Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 9.0 KiB After Width: | Height: | Size: 9.0 KiB |
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 13 KiB |
|
Before Width: | Height: | Size: 25 KiB After Width: | Height: | Size: 25 KiB |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 28 KiB |
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 3.3 KiB |
|
Before Width: | Height: | Size: 5.9 KiB After Width: | Height: | Size: 5.9 KiB |
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 3.9 KiB After Width: | Height: | Size: 3.9 KiB |
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 49 KiB After Width: | Height: | Size: 49 KiB |
@@ -0,0 +1,24 @@
|
||||
use crate::ml::model_cache::ModelCache;
|
||||
use crate::ml::prompt_cache::PromptCache;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use image::{ImageReader, RgbImage};
|
||||
use log::info;
|
||||
use tauri::{AppHandle, State};
|
||||
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn inpaint_image(
|
||||
image: &str,
|
||||
mask: &str,
|
||||
prompt: String,
|
||||
model_config: State<'_, AppConfig>,
|
||||
model_cache: State<'_, ModelCache>,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
app: AppHandle,
|
||||
) -> Result<String, String> {
|
||||
info!("inpaint_endpoint endpoint called.");
|
||||
|
||||
Ok(mask.to_string())
|
||||
}
|
||||
@@ -3,5 +3,4 @@ pub mod flip_image;
|
||||
pub mod inpaint_image_endpoint;
|
||||
pub mod realtime_image_endpoint;
|
||||
pub mod remove_background_endpoint;
|
||||
pub mod save_image_endpoint;
|
||||
pub mod text_to_image_endpoint;
|
||||
@@ -31,7 +31,6 @@ pub async fn infer_image(
|
||||
image: &str,
|
||||
prompt: Option<String>,
|
||||
strength: Option<f64>,
|
||||
seed: Option<u64>,
|
||||
model_config: State<'_, AppConfig>,
|
||||
model_cache: State<'_, ModelCache>,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
@@ -50,15 +49,12 @@ pub async fn infer_image(
|
||||
// TODO(bt,2025-02-17): Running out of vram with full image buffer size
|
||||
let mut image = image.resize(512, 512, FilterType::CatmullRom);
|
||||
|
||||
let seed = seed.or(model_config.seed);
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
let result = infer_image_impl(
|
||||
&prompt,
|
||||
image,
|
||||
strength,
|
||||
seed,
|
||||
&model_config,
|
||||
&model_cache,
|
||||
prompt_cache,
|
||||
@@ -87,7 +83,6 @@ async fn infer_image_impl(
|
||||
prompt: &str,
|
||||
image: DynamicImage,
|
||||
strength: Option<f64>,
|
||||
seed: Option<u64>,
|
||||
config: &AppConfig,
|
||||
model_cache: &ModelCache,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
@@ -111,7 +106,7 @@ async fn infer_image_impl(
|
||||
cfg_scale: config.cfg_scale,
|
||||
use_flash_attn: true,
|
||||
model_config: &config.model_config,
|
||||
maybe_seed: seed,
|
||||
maybe_seed: None,
|
||||
scheduler_steps: config.scheduler_steps,
|
||||
vae_path: &vae_path,
|
||||
unet_path: &unet_path,
|
||||
@@ -1,20 +1,22 @@
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
use ml_models::ml::flux::flux_pipeline::{flux_pipeline, FluxModel, FluxPipelineArgs};
|
||||
use ml_models::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
|
||||
|
||||
use crate::state::app_config::AppConfig;
|
||||
use crate::state::app_dir::{AppDataRoot, AppWeightsDir};
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use crate::stubs::model_cache::ModelCache;
|
||||
use crate::stubs::prompt_cache::PromptCache;
|
||||
use crate::utils::image::encode_dynamic_image_base64_png::encode_dynamic_image_base64_png;
|
||||
use image::imageops::FilterType;
|
||||
use image::{DynamicImage, ImageReader, RgbImage};
|
||||
use log::{error, info};
|
||||
use ml_weights_registry::weights_registry::weights::{CLIP_JSON, LYKON_DEAMSHAPER_7_VAE, SDXL_TURBO_CLIP_TEXT_ENCODER, SIMIANLUO_LCM_DREAMSHAPER_V7_UNET};
|
||||
|
||||
use ml_weights_registry::weights_registry::flux_weights::{FLUX_DEV, FLUX_SCHNELL, FLUX_SCHNELL_AUTOENCODER, GOOGLE_T5_V1_1_XXL_CONFIG, GOOGLE_T5_V1_1_XXL_MODEL, LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF, LMZ_T5_TOKENIZER_JSON, OPENAI_CLIP_VIT_P14_MODEL, OPENAI_CLIP_VIT_P14_TOKENIZER_JSON};
|
||||
use once_cell::sync::Lazy;
|
||||
use std::io::Cursor;
|
||||
use tauri::{AppHandle, State};
|
||||
|
||||
const RANDOM_SEED: u32 = 42;
|
||||
|
||||
const STRENGTH: f64 = 100.0;
|
||||
const PNG_BYTES : &[u8] = include_bytes!("../../binary_includes/1024.png");
|
||||
|
||||
@@ -33,8 +35,8 @@ pub async fn text_to_image(
|
||||
model_config: State<'_, AppConfig>,
|
||||
model_cache: State<'_, ModelCache>,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
app: AppHandle,
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
app: AppHandle,
|
||||
) -> Result<String, String> {
|
||||
info!("text_to_image endpoint called.");
|
||||
|
||||
@@ -49,8 +51,8 @@ pub async fn text_to_image(
|
||||
&model_config,
|
||||
&model_cache,
|
||||
prompt_cache,
|
||||
app_data_root.weights_dir(),
|
||||
app,
|
||||
&app_data_root
|
||||
).await;
|
||||
|
||||
image = match result {
|
||||
@@ -78,36 +80,35 @@ async fn text_to_image_impl(
|
||||
config: &AppConfig,
|
||||
model_cache: &ModelCache,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
weights_dir: &AppWeightsDir,
|
||||
app: AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
) -> Result<RgbImage, String> {
|
||||
|
||||
let args = FluxPipelineArgs {
|
||||
let weights_dir = app_data_root.weights_dir();
|
||||
let vae_path = weights_dir.weight_path(&LYKON_DEAMSHAPER_7_VAE);
|
||||
let unet_path = weights_dir.weight_path(&SIMIANLUO_LCM_DREAMSHAPER_V7_UNET);
|
||||
let clip_json_path = weights_dir.weight_path(&CLIP_JSON);
|
||||
let clip_weights_path= weights_dir.weight_path(&SDXL_TURBO_CLIP_TEXT_ENCODER);
|
||||
|
||||
let args = Args {
|
||||
image: &image,
|
||||
prompt: prompt.to_string(),
|
||||
uncond_prompt: "".to_string(),
|
||||
model_cache,
|
||||
prompt_cache: &prompt_cache,
|
||||
app: &app,
|
||||
flux_model: FluxModel::Schnell,
|
||||
active_blocks: None,
|
||||
use_cuda_stream: Some(true),
|
||||
prefetch_next_batch: Some(false),
|
||||
use_full_gpu: Some(true),
|
||||
img2img_strength: strength,
|
||||
seed: None,
|
||||
use_quantized_model: Some(true),
|
||||
cfg_scale: config.cfg_scale,
|
||||
use_flash_attn: true,
|
||||
model_config: &config.model_config,
|
||||
flux_dev_or_schnell_path: &weights_dir.weight_path(&FLUX_SCHNELL),
|
||||
flux_schnell_quantized_gguf: &weights_dir.weight_path(&LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF),
|
||||
flux_autoencoder_path: &weights_dir.weight_path(&FLUX_SCHNELL_AUTOENCODER),
|
||||
google_t5_model_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_MODEL),
|
||||
google_t5_config_path: &weights_dir.weight_path(&GOOGLE_T5_V1_1_XXL_CONFIG),
|
||||
lmz_t5_tokenizer_path: &weights_dir.weight_path(&LMZ_T5_TOKENIZER_JSON),
|
||||
openai_clip_model_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_MODEL),
|
||||
openai_clip_tokenizer_json_path: &weights_dir.weight_path(&OPENAI_CLIP_VIT_P14_TOKENIZER_JSON),
|
||||
maybe_seed: None,
|
||||
scheduler_steps: config.scheduler_steps,
|
||||
vae_path: &vae_path,
|
||||
unet_path: &unet_path,
|
||||
clip_json_path: &clip_json_path,
|
||||
clip_weights_path: &clip_weights_path,
|
||||
};
|
||||
|
||||
let image = flux_pipeline(args)
|
||||
let image = lcm_pipeline(args)
|
||||
.map_err(|err| format!("failure to encode image: {:?}", err))?;
|
||||
|
||||
Ok(image)
|
||||
@@ -8,9 +8,6 @@ pub enum NotificationModelType {
|
||||
TextTokenizer,
|
||||
Unet,
|
||||
Vae,
|
||||
Json,
|
||||
ClipEncoder,
|
||||
FluxModel,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
@@ -29,17 +26,5 @@ pub enum NotificationEvent<'a> {
|
||||
ModelDownloadComplete {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelLoading {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelInference {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelInferenceComplete {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -7,11 +7,10 @@ pub mod transfer;
|
||||
pub mod utils;
|
||||
|
||||
use crate::endpoints::download_models::download_models;
|
||||
use crate::endpoints::flip_image::flip_image;
|
||||
use crate::endpoints::inpaint_image_endpoint::inpaint_image;
|
||||
use crate::endpoints::flip_image::flip_image;
|
||||
use crate::endpoints::realtime_image_endpoint::infer_image;
|
||||
use crate::endpoints::remove_background_endpoint::remove_background;
|
||||
use crate::endpoints::save_image_endpoint::save_image;
|
||||
use crate::endpoints::text_to_image_endpoint::text_to_image;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use crate::stubs::model_cache::ModelCache;
|
||||
@@ -30,6 +29,7 @@ pub fn run() {
|
||||
let config = AppConfig::init()
|
||||
.expect("config should load");
|
||||
|
||||
|
||||
let prompt_cache = PromptCache::with_capacity(8)
|
||||
.expect("prompt cache should load");
|
||||
|
||||
@@ -45,10 +45,7 @@ pub fn run() {
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_log::Builder::new()
|
||||
.level(log::LevelFilter::Info)
|
||||
.targets(vec![
|
||||
Target::new(TargetKind::Stdout),
|
||||
Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
|
||||
])
|
||||
.targets(vec![Target::new(TargetKind::Stdout)])
|
||||
.build())
|
||||
.setup(|app| {
|
||||
// TODO(bt): This is broken on windows
|
||||
@@ -77,8 +74,6 @@ pub fn run() {
|
||||
infer_image,
|
||||
inpaint_image,
|
||||
remove_background,
|
||||
save_image,
|
||||
text_to_image,
|
||||
text_to_image,
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
@@ -4,7 +4,6 @@ use anyhow::anyhow;
|
||||
use directories::UserDirs;
|
||||
use ml_weights_registry::weights_registry::weight_descriptor::WeightDescriptor;
|
||||
use std::path::{Path, PathBuf};
|
||||
use chrono::{DateTime, Local};
|
||||
use tempdir::TempDir;
|
||||
use tempfile::{Builder, NamedTempFile};
|
||||
|
||||
@@ -14,9 +13,6 @@ const WEIGHTS_SUBDIRECTORY : &str = "weights";
|
||||
|
||||
const TEMPORARY_SUBDIRECTORY : &str = "temp";
|
||||
|
||||
/// Note: Tauri appends ".log" to the end of the filename.
|
||||
const LOG_FILE_NAME : &str = "application_debug";
|
||||
|
||||
/// The path to the application data directory, which includes "asset" and "weights" data.
|
||||
#[derive(Clone)]
|
||||
pub struct AppDataRoot {
|
||||
@@ -24,8 +20,6 @@ pub struct AppDataRoot {
|
||||
assets_dir: AppAssetsDir,
|
||||
weights_dir: AppWeightsDir,
|
||||
temp_dir: TemporaryDir,
|
||||
log_file_name: PathBuf,
|
||||
log_file_name_string: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -76,19 +70,12 @@ impl AppDataRoot {
|
||||
let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
|
||||
let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
|
||||
let temp_dir = TemporaryDir::create_existing(dir.join(TEMPORARY_SUBDIRECTORY))?;
|
||||
let log_file_name = dir.join(LOG_FILE_NAME);
|
||||
let log_file_name_string = log_file_name
|
||||
.to_str()
|
||||
.ok_or(anyhow!("couldn't convert log path to str"))?
|
||||
.to_string();
|
||||
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
assets_dir,
|
||||
weights_dir,
|
||||
temp_dir,
|
||||
log_file_name,
|
||||
log_file_name_string,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -107,14 +94,6 @@ impl AppDataRoot {
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn log_file_name(&self) -> &Path {
|
||||
&self.log_file_name
|
||||
}
|
||||
|
||||
pub fn log_file_name_str(&self) -> &str {
|
||||
&self.log_file_name_string
|
||||
}
|
||||
}
|
||||
|
||||
impl AppAssetsDir {
|
||||
@@ -138,18 +117,6 @@ impl AppAssetsDir {
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Return the current date asset directory
|
||||
/// If it doesn't exist, create it.
|
||||
pub fn make_or_get_current_date_dir(&self) -> anyhow::Result<PathBuf> {
|
||||
let format: DateTime<Local> = Local::now();
|
||||
let date_directory_name = format.format("%Y-%d-%m").to_string();
|
||||
let full_path = self.path.join(date_directory_name);
|
||||
if !full_path.exists() {
|
||||
std::fs::create_dir(&full_path)?;
|
||||
}
|
||||
Ok(full_path)
|
||||
}
|
||||
}
|
||||
|
||||
impl AppWeightsDir {
|
||||
@@ -13,7 +13,6 @@ use std::sync::{Arc, Mutex};
|
||||
use tauri::{AppHandle, Emitter};
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::task::JoinHandle;
|
||||
use ml_weights_registry::weights_registry::flux_weights::{FLUX_SCHNELL, FLUX_SCHNELL_AUTOENCODER, GOOGLE_T5_V1_1_XXL_CONFIG, GOOGLE_T5_V1_1_XXL_MODEL, LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF, LMZ_T5_TOKENIZER_JSON, OPENAI_CLIP_VIT_P14_MODEL, OPENAI_CLIP_VIT_P14_TOKENIZER_JSON};
|
||||
|
||||
const NOTIFICATION_CHANNEL_NAME : &str = "notification";
|
||||
const MAX_FILES : usize = 8;
|
||||
@@ -51,16 +50,6 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
|
||||
//download_queue.push_back(SDXL_TURBO_VAE); // TODO(bt): REMOVE
|
||||
//download_queue.push_back(SDXL_TURBO_UNET); // TODO(bt): REMOVE
|
||||
|
||||
// Flux:
|
||||
download_queue.push_back(FLUX_SCHNELL);
|
||||
download_queue.push_back(FLUX_SCHNELL_AUTOENCODER);
|
||||
download_queue.push_back(LMZ_CANDLE_FLUX_SCHNELL_QUANTIZED_GGUF);
|
||||
download_queue.push_back(GOOGLE_T5_V1_1_XXL_MODEL);
|
||||
download_queue.push_back(GOOGLE_T5_V1_1_XXL_CONFIG);
|
||||
download_queue.push_back(LMZ_T5_TOKENIZER_JSON);
|
||||
download_queue.push_back(OPENAI_CLIP_VIT_P14_MODEL);
|
||||
download_queue.push_back(OPENAI_CLIP_VIT_P14_TOKENIZER_JSON);
|
||||
|
||||
let mut handles = FuturesUnordered::new();
|
||||
|
||||
while let Some(model) = download_queue.pop_front() {
|
||||
@@ -93,7 +82,6 @@ pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> !
|
||||
WeightFunction::Unet => NotificationModelType::Unet,
|
||||
WeightFunction::Vae => NotificationModelType::Vae,
|
||||
WeightFunction::ImageSegmentation => NotificationModelType::ImageSegmentation,
|
||||
WeightFunction::ImageGeneration => NotificationModelType::ImageSegmentation, // TODO FIX
|
||||
};
|
||||
|
||||
let result = app.emit(NOTIFICATION_CHANNEL_NAME, NotificationEvent::ModelDownloadStarted {
|
||||