Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
becbfa06f5 | ||
|
|
018a77d113 | ||
|
|
d5df7dbca1 | ||
|
|
a605a2e51d | ||
|
|
3e2016a7af | ||
|
|
a77f9c3eec |
@@ -1,4 +1,4 @@
|
||||
hakari-package = "workspace-build-acceleration"
|
||||
hakari-package = "my-workspace-hack"
|
||||
|
||||
resolver = "1"
|
||||
|
||||
@@ -11,4 +11,4 @@ platforms = [
|
||||
"x86_64-pc-windows-msvc",
|
||||
]
|
||||
|
||||
exact-versions = true
|
||||
exact-versions = true
|
||||
@@ -32,9 +32,6 @@ target/
|
||||
crates/lib/database_queries/target/
|
||||
crates/lib/sqlite_queries/target/
|
||||
|
||||
# Javascript build directories
|
||||
node_modules/
|
||||
|
||||
# Vim noise
|
||||
*.swo
|
||||
*.swp
|
||||
@@ -69,7 +66,3 @@ scraps/
|
||||
# These should be built by the platform
|
||||
windows/sqlite/sqlite3.exp
|
||||
windows/sqlite/sqlite3.lib
|
||||
|
||||
# ArtCraft Windows DLLs
|
||||
crates/desktop/src-tauri/*.dll
|
||||
windows_dlls/
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
|
||||
# This block defines the workspace members.
|
||||
# When we add a library or binary crate, it needs an entry here.
|
||||
members = [
|
||||
# Types: common strongly-typed tokens, ids, payloads, etc.
|
||||
"crates/_types/composite_identifiers",
|
||||
@@ -24,8 +22,7 @@ members = [
|
||||
"crates/cli/production/migration-tool",
|
||||
|
||||
# Desktop apps
|
||||
"crates/desktop/tauri-4o",
|
||||
"crates/desktop/tauri-realtime",
|
||||
"crates/desktop/src-tauri",
|
||||
|
||||
# Shared libraries
|
||||
"crates/lib/actix_cors_configs",
|
||||
@@ -33,7 +30,6 @@ members = [
|
||||
"crates/lib/api_clients/cloud_storage",
|
||||
"crates/lib/api_clients/google_drive_common",
|
||||
"crates/lib/api_clients/google_sign_in",
|
||||
"crates/lib/api_clients/openai_sora_client",
|
||||
"crates/lib/api_clients/thumbnail_generator",
|
||||
"crates/lib/bootstrap",
|
||||
"crates/lib/bucket_paths",
|
||||
@@ -60,20 +56,16 @@ members = [
|
||||
"crates/lib/files/mimetypes",
|
||||
"crates/lib/files/videos",
|
||||
"crates/lib/idempotency",
|
||||
"crates/lib/image_algorithms",
|
||||
"crates/lib/jobs_common",
|
||||
"crates/lib/logging",
|
||||
"crates/lib/machine_learning/ml_models",
|
||||
"crates/lib/machine_learning/ml_weights_registry",
|
||||
"crates/lib/machine_learning/prompt_engineering",
|
||||
"crates/lib/markdown",
|
||||
"crates/lib/migration",
|
||||
"crates/lib/mysql_queries",
|
||||
"crates/lib/password",
|
||||
"crates/lib/prompt_engineering",
|
||||
"crates/lib/redis_common",
|
||||
"crates/lib/redis_schema",
|
||||
"crates/lib/server_environment",
|
||||
"crates/lib/shared_service_components",
|
||||
"crates/lib/sqlx_mysql_helpers",
|
||||
"crates/lib/storyteller_root",
|
||||
"crates/lib/subprocess_common",
|
||||
@@ -121,40 +113,5 @@ members = [
|
||||
|
||||
# cargo-hakari workspace hack (compilation speedup hack)
|
||||
# see usage: https://crates.io/crates/cargo-hakari
|
||||
"crates/workspace-build-acceleration",
|
||||
"crates/my-workspace-hack",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
# Common inheritable settings for monorepo packages
|
||||
publish = false
|
||||
|
||||
[workspace.dependencies]
|
||||
# Hakari workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
|
||||
|
||||
# External libs
|
||||
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
|
||||
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
|
||||
anyhow = "1.0.97"
|
||||
chrono = { version = "0.4.40", features = ["serde"] }
|
||||
log = "0.4.27"
|
||||
once_cell = "1.21.3"
|
||||
rand = "0.9.0"
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = "1.0.140"
|
||||
serde_yaml = "0.9.1"
|
||||
toml = "0.7.4"
|
||||
url = "2.5.4"
|
||||
|
||||
# Internal libs
|
||||
crockford = { path = "crates/lib/data/crockford" }
|
||||
errors = { path = "crates/lib/errors" }
|
||||
filesys = { path = "crates/lib/files/filesys" }
|
||||
memory_caching = { path = "crates/lib/caching/memory_caching" }
|
||||
shared_service_components = { path = "crates/lib/shared_service_components" }
|
||||
openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
|
||||
primitives = { path = "crates/lib/data/primitives" }
|
||||
subprocess_common = { path = "crates/lib/subprocess_common" }
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
image_width: 1024
|
||||
image_height: 1024
|
||||
image_width: 512
|
||||
image_height: 512
|
||||
#scheduler_steps: 15
|
||||
# NB(bt): It looks like we can get down to 2 sampling steps, perhaps ?
|
||||
scheduler_steps: 8
|
||||
scheduler_steps: 4
|
||||
scheduler_samples: 1
|
||||
cfg_scale: 15.0
|
||||
seed: 42
|
||||
cfg_scale: 1.1
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "composite_identifiers"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ enums = { path = "../enums" }
|
||||
tokens = { path = "../tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -13,13 +13,13 @@ name = "enums"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -40,10 +40,6 @@ pub enum InferenceJobType {
|
||||
#[serde(rename = "studio_gen2")]
|
||||
StudioGen2,
|
||||
|
||||
/// Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
|
||||
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
|
||||
#[serde(rename = "convert_fbx_gltf")]
|
||||
ConvertFbxToGltf,
|
||||
@@ -112,7 +108,6 @@ impl InferenceJobType {
|
||||
Self::GptSovits => "gpt_sovits",
|
||||
Self::ComfyUi => "comfy_ui",
|
||||
Self::StudioGen2 => "studio_gen2",
|
||||
Self::ImageGenApi => "image_gen_api",
|
||||
Self::ConvertFbxToGltf => "convert_fbx_gltf",
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::RvcV2 => "rvc_v2",
|
||||
@@ -137,7 +132,6 @@ impl InferenceJobType {
|
||||
"gpt_sovits" => Ok(Self::GptSovits),
|
||||
"comfy_ui" => Ok(Self::ComfyUi),
|
||||
"studio_gen2" => Ok(Self::StudioGen2),
|
||||
"image_gen_api" => Ok(Self::ImageGenApi),
|
||||
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"rvc_v2" => Ok(Self::RvcV2),
|
||||
@@ -165,7 +159,6 @@ impl InferenceJobType {
|
||||
Self::GptSovits,
|
||||
Self::ComfyUi,
|
||||
Self::StudioGen2,
|
||||
Self::ImageGenApi,
|
||||
Self::ConvertFbxToGltf,
|
||||
Self::MocapNet,
|
||||
Self::RvcV2,
|
||||
@@ -204,7 +197,6 @@ mod tests {
|
||||
assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
|
||||
assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
|
||||
assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
|
||||
assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
|
||||
assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
|
||||
assert_serialization(InferenceJobType::MocapNet, "mocap_net");
|
||||
assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
|
||||
@@ -228,7 +220,6 @@ mod tests {
|
||||
assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
|
||||
assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
|
||||
assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
|
||||
assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
|
||||
assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
|
||||
assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
|
||||
assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
|
||||
@@ -252,7 +243,6 @@ mod tests {
|
||||
assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
|
||||
assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
|
||||
assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
|
||||
assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
|
||||
assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
|
||||
assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
|
||||
assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
|
||||
@@ -271,7 +261,7 @@ mod tests {
|
||||
fn all_variants() {
|
||||
// Static check
|
||||
let mut variants = InferenceJobType::all_variants();
|
||||
assert_eq!(variants.len(), 17);
|
||||
assert_eq!(variants.len(), 16);
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
|
||||
@@ -279,7 +269,6 @@ mod tests {
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
|
||||
|
||||
@@ -33,10 +33,6 @@ pub enum InferenceModelType {
|
||||
#[serde(rename = "seed_vc")]
|
||||
SeedVc,
|
||||
|
||||
/// NB: This is for Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
|
||||
#[serde(rename = "tacotron2")]
|
||||
Tacotron2,
|
||||
#[serde(rename = "vits")]
|
||||
@@ -74,7 +70,6 @@ impl InferenceModelType {
|
||||
Self::VallEX => "vall_e_x",
|
||||
Self::RerenderAVideo => "rerender_a_video",
|
||||
Self::StableDiffusion => "stable_diffusion",
|
||||
Self::ImageGenApi => "image_gen_api",
|
||||
Self::SeedVc => "seed_vc",
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::StyleTTS2 => "styletts2",
|
||||
@@ -95,7 +90,6 @@ impl InferenceModelType {
|
||||
"vall_e_x" => Ok(Self::VallEX),
|
||||
"rerender_a_video" => Ok(Self::RerenderAVideo),
|
||||
"stable_diffusion" => Ok(Self::StableDiffusion),
|
||||
"image_gen_api" => Ok(Self::ImageGenApi),
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"styletts2" => Ok(Self::StyleTTS2),
|
||||
"comfy_ui" => Ok(Self::ComfyUi),
|
||||
@@ -119,7 +113,6 @@ impl InferenceModelType {
|
||||
Self::VallEX,
|
||||
Self::RerenderAVideo,
|
||||
Self::StableDiffusion,
|
||||
Self::ImageGenApi,
|
||||
Self::MocapNet,
|
||||
Self::StyleTTS2,
|
||||
Self::ConvertFbxToGltf,
|
||||
@@ -147,7 +140,6 @@ mod tests {
|
||||
assert_serialization(InferenceModelType::VallEX, "vall_e_x");
|
||||
assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
|
||||
assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
|
||||
assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
|
||||
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
|
||||
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
|
||||
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
|
||||
@@ -167,7 +159,6 @@ mod tests {
|
||||
assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
|
||||
assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
|
||||
assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
|
||||
assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
|
||||
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
|
||||
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
|
||||
assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
|
||||
@@ -185,7 +176,6 @@ mod tests {
|
||||
assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
|
||||
assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
|
||||
assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
|
||||
assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
|
||||
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
|
||||
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
|
||||
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
|
||||
@@ -197,18 +187,17 @@ mod tests {
|
||||
fn all_variants() {
|
||||
// Static check
|
||||
let mut variants = InferenceModelType::all_variants();
|
||||
assert_eq!(variants.len(), 15);
|
||||
assert_eq!(variants.len(), 13);
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
|
||||
assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
|
||||
|
||||
@@ -57,9 +57,6 @@ pub enum MediaFileOriginModelType {
|
||||
#[serde(rename = "studio")]
|
||||
StorytellerStudio,
|
||||
|
||||
#[serde(rename = "studio_ig")]
|
||||
StorytellerStudioImageGen,
|
||||
|
||||
#[serde(rename = "vst")]
|
||||
VideoStyleTransfer,
|
||||
|
||||
@@ -96,7 +93,6 @@ impl MediaFileOriginModelType {
|
||||
Self::MocapNet => "mocap_net",
|
||||
Self::StyleTTS2 => "styletts2",
|
||||
Self::StableDiffusion15 => "stable_diffusion_1_5",
|
||||
Self::StorytellerStudioImageGen => "studio_ig",
|
||||
Self::GptSovits => "gpt_sovits",
|
||||
Self::StorytellerStudio => "studio",
|
||||
Self::VideoStyleTransfer => "vst",
|
||||
@@ -119,7 +115,6 @@ impl MediaFileOriginModelType {
|
||||
"mocap_net" => Ok(Self::MocapNet),
|
||||
"styletts2" => Ok(Self::StyleTTS2),
|
||||
"stable_diffusion_1_5" => Ok(Self::StableDiffusion15),
|
||||
"studio_ig" => Ok(Self::StorytellerStudioImageGen),
|
||||
"gpt_sovits" => Ok(Self::GptSovits),
|
||||
"studio" => Ok(Self::StorytellerStudio),
|
||||
"vst" => Ok(Self::VideoStyleTransfer),
|
||||
@@ -147,7 +142,6 @@ impl MediaFileOriginModelType {
|
||||
Self::StableDiffusion15,
|
||||
Self::GptSovits,
|
||||
Self::StorytellerStudio,
|
||||
Self::StorytellerStudioImageGen,
|
||||
Self::VideoStyleTransfer,
|
||||
Self::ComfyUi,
|
||||
Self::VallEX,
|
||||
|
||||
@@ -13,11 +13,11 @@ name = "enums_public"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Internal
|
||||
enums = { path = "../enums" }
|
||||
|
||||
@@ -91,10 +91,6 @@ pub enum PublicInferenceJobType {
|
||||
/// Re-render a video is a video style transfer algorithm. We developed code
|
||||
/// around it, but chose to develop AnimateDiff / ComfyUI support instead.
|
||||
RerenderAVideo,
|
||||
|
||||
/// Sora GPT 4o image gen
|
||||
#[serde(rename = "image_gen_api")]
|
||||
ImageGenApi,
|
||||
}
|
||||
|
||||
/// NB: Legacy API for older code.
|
||||
@@ -123,7 +119,6 @@ impl PublicInferenceJobType {
|
||||
InferenceJobType::Unknown => Self::Unknown,
|
||||
InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
|
||||
InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
|
||||
InferenceJobType::ImageGenApi => Self::ImageGenApi,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,7 +140,6 @@ impl PublicInferenceJobType {
|
||||
Self::SeedVc => InferenceJobType::SeedVc,
|
||||
Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
|
||||
Self::StableDiffusion => InferenceJobType::StableDiffusion,
|
||||
Self::ImageGenApi => InferenceJobType::ImageGenApi,
|
||||
Self::StyleTTS2 => InferenceJobType::StyleTTS2,
|
||||
Self::Tacotron2 => InferenceJobType::Tacotron2,
|
||||
Self::Unknown => InferenceJobType::Unknown,
|
||||
|
||||
@@ -64,10 +64,6 @@ pub enum PublicMediaFileModelType {
|
||||
#[serde(rename = "studio")]
|
||||
StorytellerStudio,
|
||||
|
||||
/// NB: This is GPT4o image generation
|
||||
#[serde(rename = "studio_ig")]
|
||||
StorytellerStudioImageGen,
|
||||
|
||||
#[serde(rename = "vst")]
|
||||
VideoStyleTransfer,
|
||||
|
||||
@@ -93,7 +89,6 @@ impl PublicMediaFileModelType {
|
||||
MediaFileOriginModelType::LivePortrait => Self::FaceMirror,
|
||||
MediaFileOriginModelType::SadTalker => Self::FaceAnimator,
|
||||
MediaFileOriginModelType::StyleTTS2 => Self::VoiceDesigner,
|
||||
MediaFileOriginModelType::StorytellerStudioImageGen => Self::StorytellerStudioImageGen,
|
||||
// Conserved variants
|
||||
MediaFileOriginModelType::RvcV2 => Self::RvcV2,
|
||||
MediaFileOriginModelType::SoVitsSvc => Self::SoVitsSvc,
|
||||
@@ -118,7 +113,6 @@ impl PublicMediaFileModelType {
|
||||
Self::FaceAnimator => MediaFileOriginModelType::SadTalker,
|
||||
Self::Lipsync => MediaFileOriginModelType::FaceFusion,
|
||||
Self::VoiceDesigner => MediaFileOriginModelType::StyleTTS2,
|
||||
Self::StorytellerStudioImageGen => MediaFileOriginModelType::StorytellerStudioImageGen,
|
||||
// Conserved variants
|
||||
Self::RvcV2 => MediaFileOriginModelType::RvcV2,
|
||||
Self::SoVitsSvc => MediaFileOriginModelType::SoVitsSvc,
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "reusable_types"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
# (none)
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
chrono = { version = "0.4.22", features = ["serde"] }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "tokens"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
# (none)
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "dev-database-seed"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -32,7 +32,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
log = "0.4.14"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "dev-upload-media-file"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -29,7 +29,7 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "scrape-test"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -22,7 +22,7 @@ errors = { path = "../../../lib/errors" }
|
||||
web_scrapers = { path = "../../../lib/web_scrapers" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,11 +13,11 @@ name = "db-backfill"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# Internal
|
||||
datetimes = { path = "../../../lib/data/datetimes" }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "db-backup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ easyenv = { path = "../../../lib/easyenv" }
|
||||
errors = { path = "../../../lib/errors" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
anyhow = "1.0.86"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "db-cleanup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -26,7 +26,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "elasticsearch-cli"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -29,7 +29,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "kubectl-pod-cleanup"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -22,7 +22,7 @@ easyenv = { path = "../../../lib/easyenv" }
|
||||
errors = { path = "../../../lib/errors" }
|
||||
|
||||
# Workspace hack (faster compile yytimes)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
log = "0.4.20"
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "lru-delete-models"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -21,7 +21,7 @@ errors = { path = "../../../lib/errors" }
|
||||
logging = { path = "../../../lib/logging" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = { version = "4.4.8", features = ["derive"] }
|
||||
|
||||
@@ -13,7 +13,7 @@ name = "migration-tool"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
normal = ["my-workspace-hack"]
|
||||
|
||||
[dependencies]
|
||||
# Internal
|
||||
@@ -30,7 +30,7 @@ storyteller_root = { path = "../../../lib/storyteller_root" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" }
|
||||
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
|
||||
|
||||
# External
|
||||
clap = "4.4.8"
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
[package]
|
||||
name = "app"
|
||||
version = "0.1.0"
|
||||
description = "A Tauri App"
|
||||
authors = ["you"]
|
||||
license = ""
|
||||
repository = ""
|
||||
edition = "2021"
|
||||
rust-version = "1.77.2"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[lib]
|
||||
name = "app_lib"
|
||||
crate-type = ["staticlib", "cdylib", "rlib"]
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2.0.4", features = [] }
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
filesys = { path = "../../lib/files/filesys" }
|
||||
memory_caching = { path = "../../lib/caching/memory_caching" }
|
||||
|
||||
# Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332
|
||||
cudarc = "0.13.7"
|
||||
|
||||
# External
|
||||
anyhow = "1.0"
|
||||
bytes = "1.10"
|
||||
base64 = "0.22.1"
|
||||
candle-core = { path = "../../vendor/candle/candle-core", features = ["cuda"] }
|
||||
candle-nn = { path = "../../vendor/candle/candle-nn", features = ["cuda"] }
|
||||
candle-transformers = { path = "../../vendor/candle/candle-transformers", features = ["cuda"] }
|
||||
hf-hub = "0.3.2"
|
||||
image = "0.25.5"
|
||||
serde_yaml = "0.9"
|
||||
log = "0.4"
|
||||
once_cell = "1.20.3"
|
||||
expanduser = "1.2.2"
|
||||
rand = "0.8"
|
||||
reqwest = "0.12.12"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
tauri = { version = "2.2.4", features = [] }
|
||||
tauri-plugin-log = "2"
|
||||
tokenizers = "0.15.2"
|
||||
|
||||
[features]
|
||||
default = ["cuda"]
|
||||
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda", "candle-transformers/flash-attn", "candle-core/cudnn"]
|
||||
|
||||
[dev-dependencies]
|
||||
libloading = "0.8.6"
|
||||
|
||||
[profile.dev]
|
||||
incremental = true # Compile your binary in smaller steps.
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1 # Allows LLVM to perform better optimization.
|
||||
lto = true # Enables link-time-optimizations.
|
||||
opt-level = 3 # `s` prioritizes small binary size. Use `3` if you prefer speed.
|
||||
panic = "abort" # Higher performance by disabling panic handlers.
|
||||
strip = true # Ensures debug symbols are removed.
|
||||
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 23 KiB After Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 2.2 KiB After Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 9.0 KiB After Width: | Height: | Size: 9.0 KiB |
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 13 KiB |
|
Before Width: | Height: | Size: 25 KiB After Width: | Height: | Size: 25 KiB |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 28 KiB |
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 3.3 KiB |
|
Before Width: | Height: | Size: 5.9 KiB After Width: | Height: | Size: 5.9 KiB |
|
Before Width: | Height: | Size: 7.4 KiB After Width: | Height: | Size: 7.4 KiB |
|
Before Width: | Height: | Size: 3.9 KiB After Width: | Height: | Size: 3.9 KiB |
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 49 KiB After Width: | Height: | Size: 49 KiB |
@@ -0,0 +1,16 @@
|
||||
use crate::ml::downloads::download_all_models::download_all_models;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use tauri::State;
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn download_models(
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
) -> Result<String, String> {
|
||||
|
||||
download_all_models(&app_data_root).await
|
||||
.map_err(|err| {
|
||||
"error downloading models".to_string()
|
||||
})?;
|
||||
|
||||
Ok(("downloaded".to_string()))
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use crate::ml::downloads::download_all_models::download_all_models;
|
||||
use crate::ml::model_cache::ModelCache;
|
||||
use crate::ml::prompt_cache::PromptCache;
|
||||
use crate::ml::stable_diffusion::lcm_pipeline::{lcm_pipeline, Args};
|
||||
use crate::state::app_config::AppConfig;
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
|
||||
use bytes::BytesMut;
|
||||
use image::imageops::FilterType;
|
||||
use image::{DynamicImage, ImageFormat, ImageReader};
|
||||
use log::{error, info};
|
||||
use std::io::Cursor;
|
||||
use std::path::PathBuf;
|
||||
use tauri::{AppHandle, State};
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
const PROMPT_FILENAME : &str = "prompt.txt";
|
||||
const PROMPT: &str = "A beautiful landscape with mountains and a lake";
|
||||
|
||||
const NEGATIVE_PROMPT: &str = "bad quality, bad faces, poor quality, blurry faces, watermark";
|
||||
|
||||
// TODO: Mark async https://github.com/tauri-apps/tauri/discussions/7737
|
||||
|
||||
|
||||
/// This handler takes an image (as a base64 encoded string) and a prompt and returns
|
||||
/// an image (as a base64-encoded string).
|
||||
#[tauri::command]
|
||||
pub async fn infer_image(
|
||||
image: &str,
|
||||
prompt: Option<String>,
|
||||
strength: Option<u8>,
|
||||
model_config: State<'_, AppConfig>,
|
||||
model_cache: State<'_, ModelCache>,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
app: AppHandle,
|
||||
) -> Result<String, String> {
|
||||
|
||||
let prompt = get_prompt_or_fallback(prompt);
|
||||
|
||||
info!("Prompt: {}", prompt);
|
||||
info!("Strength: {:?}", strength);
|
||||
|
||||
let image = hydrate_base64_image(image)
|
||||
.map_err(|err| format!("Couldn't hydrate image from base64: {}", err))?;
|
||||
|
||||
// TODO(bt): Move this to another endpoint
|
||||
download_all_models(&app_data_root).await
|
||||
.map_err(|err| {
|
||||
error!("couldn't download models: {:?}", err);
|
||||
"Couldn't download models".to_string()
|
||||
})?;
|
||||
|
||||
let result = do_infer_image(
|
||||
&prompt,
|
||||
image,
|
||||
strength,
|
||||
&model_config,
|
||||
&model_cache,
|
||||
prompt_cache,
|
||||
app,
|
||||
&app_data_root
|
||||
).await;
|
||||
|
||||
if let Err(err) = result.as_deref() {
|
||||
error!("There was an error: {:?}", err);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn do_infer_image(
|
||||
prompt: &str,
|
||||
image: DynamicImage,
|
||||
strength: Option<u8>,
|
||||
config: &AppConfig,
|
||||
model_cache: &ModelCache,
|
||||
prompt_cache: State<'_, PromptCache>,
|
||||
app: AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
) -> Result<String, String> {
|
||||
|
||||
let args = Args {
|
||||
image: &image,
|
||||
prompt: prompt.to_string(),
|
||||
uncond_prompt: NEGATIVE_PROMPT.to_string(),
|
||||
model_cache,
|
||||
configs: config,
|
||||
prompt_cache: &prompt_cache,
|
||||
i2i_strength: strength,
|
||||
cfg_scale: config.cfg_scale,
|
||||
app: &app,
|
||||
app_data_root,
|
||||
};
|
||||
|
||||
match lcm_pipeline(args) {
|
||||
Ok(image) => {
|
||||
let mut bytes = Vec::with_capacity(1024*1024);
|
||||
|
||||
image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
|
||||
.map_err(|err| format!("failure to encode image: {:?}", err))?;
|
||||
|
||||
let bytes = BASE64_STANDARD.encode(bytes);
|
||||
|
||||
Ok(bytes)
|
||||
},
|
||||
Err(e) => Err(format!("Failed to generate image: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_prompt_or_fallback(user_prompt: Option<String>) -> String {
|
||||
let user_prompt = user_prompt.map(|prompt| prompt.trim().to_string())
|
||||
.filter(|prompt| !prompt.is_empty());
|
||||
|
||||
if let Some(prompt) = user_prompt {
|
||||
return prompt;
|
||||
}
|
||||
|
||||
let prompt_file = PathBuf::from(PROMPT_FILENAME)
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| PathBuf::from(PROMPT_FILENAME));
|
||||
|
||||
std::fs::read_to_string(&prompt_file)
|
||||
.map_err(|err| format!("Failed to read prompt file: {}", err))
|
||||
.unwrap_or_else(|_| {
|
||||
error!("Failed to read prompt file: {:?}", prompt_file);
|
||||
PROMPT.to_string()
|
||||
})
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn hydrate_base64_image(base64_image: &str) -> anyhow::Result<DynamicImage> {
|
||||
let bytes = BASE64_STANDARD.decode(base64_image)?;
|
||||
|
||||
let image = ImageReader::new(Cursor::new(bytes))
|
||||
.with_guessed_format()?
|
||||
.decode()?;
|
||||
|
||||
// TODO(bt,2025-02-17): Running out of vram with full image buffer size
|
||||
let image = image.resize(512, 512, FilterType::CatmullRom);
|
||||
|
||||
Ok(image)
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod download_models;
|
||||
pub mod flip_image;
|
||||
pub mod image_endpoint;
|
||||
pub mod test_counter;
|
||||
@@ -0,0 +1,24 @@
|
||||
use std::sync::{Arc, RwLock};
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
static COUNTER : Lazy<Arc<RwLock<u64>>> = Lazy::new(|| Arc::new(RwLock::new(0)));
|
||||
|
||||
#[tauri::command]
|
||||
pub fn test_counter() -> u64 {
|
||||
let value : u64;
|
||||
{
|
||||
match COUNTER.write() {
|
||||
Ok(mut counter) => {
|
||||
*counter += 1;
|
||||
value = *counter;
|
||||
},
|
||||
Err(_e) => {
|
||||
value = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("I was invoked from JavaScript! {:?}", value);
|
||||
|
||||
value
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
use serde::Serialize;
|
||||
|
||||
|
||||
#[derive(Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NotificationModelType {
|
||||
Unet,
|
||||
Vae,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NotificationEvent<'a> {
|
||||
ModelDownloadStarted {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelDownloadComplete {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,16 @@
|
||||
pub mod endpoints;
|
||||
pub mod events;
|
||||
pub mod ml;
|
||||
pub mod state;
|
||||
pub mod stubs;
|
||||
pub mod threads;
|
||||
pub mod transfer;
|
||||
pub mod utils;
|
||||
|
||||
use crate::endpoints::download_models::download_models;
|
||||
use crate::endpoints::flip_image::flip_image;
|
||||
use crate::endpoints::inpaint_image_endpoint::inpaint_image;
|
||||
use crate::endpoints::realtime_image_endpoint::infer_image;
|
||||
use crate::endpoints::remove_background_endpoint::remove_background;
|
||||
use crate::endpoints::save_image_endpoint::save_image;
|
||||
use crate::endpoints::text_to_image_endpoint::text_to_image;
|
||||
use crate::endpoints::image_endpoint::infer_image;
|
||||
use crate::endpoints::test_counter::test_counter;
|
||||
use crate::ml::model_cache::ModelCache;
|
||||
use crate::ml::prompt_cache::PromptCache;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use crate::stubs::model_cache::ModelCache;
|
||||
use crate::stubs::prompt_cache::PromptCache;
|
||||
use crate::threads::downloader_thread::downloader_thread;
|
||||
|
||||
use tauri_plugin_log::Target;
|
||||
use tauri_plugin_log::TargetKind;
|
||||
|
||||
@@ -45,15 +38,9 @@ pub fn run() {
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_log::Builder::new()
|
||||
.level(log::LevelFilter::Info)
|
||||
.targets(vec![
|
||||
Target::new(TargetKind::Stdout),
|
||||
Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
|
||||
])
|
||||
.targets(vec![Target::new(TargetKind::Stdout)])
|
||||
.build())
|
||||
.setup(|app| {
|
||||
// TODO(bt): This is broken on windows
|
||||
// log_environment_details();
|
||||
|
||||
//if cfg!(debug_assertions) {
|
||||
// app.handle().plugin(
|
||||
// tauri_plugin_log::Builder::default()
|
||||
@@ -72,14 +59,9 @@ pub fn run() {
|
||||
.manage(model_cache)
|
||||
.manage(app_data_root)
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
download_models,
|
||||
flip_image,
|
||||
test_counter,
|
||||
infer_image,
|
||||
inpaint_image,
|
||||
remove_background,
|
||||
save_image,
|
||||
text_to_image,
|
||||
text_to_image,
|
||||
download_models,
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
.expect("error while running tauri application");
|
||||
@@ -0,0 +1,15 @@
|
||||
use crate::ml::downloads::download_model_registry_file::download_model_registry_file;
|
||||
use crate::ml::model_type::ModelType;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
pub async fn download_all_models(app_data_root: &AppDataRoot) -> anyhow::Result<()> {
|
||||
let weights_dir = app_data_root.weights_dir();
|
||||
|
||||
download_model_registry_file(ModelType::ClipJson, weights_dir).await?;
|
||||
download_model_registry_file(ModelType::SdxlTurboUnet, weights_dir).await?;
|
||||
download_model_registry_file(ModelType::SdxlTurboVae, weights_dir).await?;
|
||||
download_model_registry_file(ModelType::SdxlTurboClipEncoder, weights_dir).await?;
|
||||
download_model_registry_file(ModelType::SdxlTurboClipEncoder2, weights_dir).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
use crate::ml::model_type::ModelType;
|
||||
use crate::state::app_dir::AppWeightsDir;
|
||||
use log::info;
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub async fn download_model_registry_file(
|
||||
model_type: ModelType,
|
||||
weights_dir: &AppWeightsDir,
|
||||
) -> anyhow::Result<PathBuf> {
|
||||
|
||||
let path = weights_dir.model_path(&model_type);
|
||||
|
||||
if path.exists() {
|
||||
return Ok(path);
|
||||
}
|
||||
|
||||
info!("downloading model: {:?} to {:?}", model_type, path);
|
||||
|
||||
let body = reqwest::get(model_type.get_download_url())
|
||||
.await?
|
||||
.bytes()
|
||||
.await?;
|
||||
|
||||
let _file = File::create(&path)?;
|
||||
|
||||
std::fs::write(&path, body.as_ref())?;
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod download_model_registry_file;
|
||||
pub mod download_all_models;
|
||||
@@ -0,0 +1,8 @@
|
||||
pub mod image;
|
||||
pub mod model_cache;
|
||||
pub mod model_file;
|
||||
pub mod models;
|
||||
pub mod prompt_cache;
|
||||
pub mod stable_diffusion;
|
||||
pub mod model_type;
|
||||
pub mod downloads;
|
||||
@@ -1,14 +1,15 @@
|
||||
use crate::ml::models::unet_model::UNetModel;
|
||||
use anyhow::anyhow;
|
||||
use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
|
||||
use candle_transformers::models::stable_diffusion::vae::AutoEncoderKL;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
// TODO: This data structure is gross. Generalize the locking and loading semantics for reuse.
|
||||
|
||||
// Simple registry for now. We can build complex machinery that aids in
|
||||
// VRAM utilization, disk space saving, intelligent scheduling, etc. in the future.
|
||||
pub struct ModelCache {
|
||||
vae: RwLock<Option<Arc<AutoEncoderKL>>>,
|
||||
unet2: RwLock<Option<Arc<UNet2DConditionModel>>>,
|
||||
unet2: RwLock<Option<Arc<UNetModel>>>,
|
||||
}
|
||||
|
||||
impl ModelCache {
|
||||
@@ -38,7 +39,7 @@ impl ModelCache {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_unet(&self, vae: Arc<UNet2DConditionModel>) -> anyhow::Result<()> {
|
||||
pub fn set_unet(&self, vae: Arc<UNetModel>) -> anyhow::Result<()> {
|
||||
match self.unet2.write() {
|
||||
Err(err) => Err(anyhow!("{}", err)),
|
||||
Ok(mut lock) => {
|
||||
@@ -48,7 +49,7 @@ impl ModelCache {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_unet(&self) -> anyhow::Result<Option<Arc<UNet2DConditionModel>>> {
|
||||
pub fn get_unet(&self) -> anyhow::Result<Option<Arc<UNetModel>>> {
|
||||
match self.unet2.read() {
|
||||
Err(err) => Err(anyhow!("{}", err)),
|
||||
Ok(ref lock) => {
|
||||
@@ -0,0 +1,83 @@
|
||||
#[cfg(feature = "accelerate")]
|
||||
extern crate accelerate_src;
|
||||
#[cfg(feature = "mkl")]
|
||||
extern crate intel_mkl_src;
|
||||
|
||||
use candle_transformers::models::stable_diffusion;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ModelFile {
|
||||
Tokenizer,
|
||||
Tokenizer2,
|
||||
Clip,
|
||||
Clip2,
|
||||
Unet,
|
||||
Vae,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum StableDiffusionVersion {
|
||||
V1_5,
|
||||
V1_5Inpaint,
|
||||
V2_1,
|
||||
V2Inpaint,
|
||||
Xl,
|
||||
XlInpaint,
|
||||
Turbo,
|
||||
}
|
||||
|
||||
impl StableDiffusionVersion {
|
||||
pub fn repo(&self) -> &'static str {
|
||||
match self {
|
||||
Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting",
|
||||
Self::V2_1 => "stabilityai/stable-diffusion-2-1",
|
||||
Self::V1_5 => "runwayml/stable-diffusion-v1-5",
|
||||
Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting",
|
||||
Self::Turbo => "stabilityai/sdxl-turbo",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelFile {
|
||||
pub fn get(
|
||||
&self,
|
||||
filename: Option<String>,
|
||||
version: StableDiffusionVersion,
|
||||
use_f16: bool,
|
||||
) -> anyhow::Result<std::path::PathBuf> {
|
||||
use hf_hub::api::sync::Api;
|
||||
match filename {
|
||||
Some(filename) => Ok(std::path::PathBuf::from(filename)),
|
||||
None => {
|
||||
let (repo, path) = match self {
|
||||
Self::Tokenizer => {
|
||||
let tokenizer_repo = match version {
|
||||
StableDiffusionVersion::V1_5
|
||||
| StableDiffusionVersion::V2_1
|
||||
| StableDiffusionVersion::V1_5Inpaint
|
||||
| StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32",
|
||||
StableDiffusionVersion::Xl
|
||||
| StableDiffusionVersion::XlInpaint
|
||||
| StableDiffusionVersion::Turbo => {
|
||||
"openai/clip-vit-large-patch14"
|
||||
}
|
||||
};
|
||||
(tokenizer_repo, "tokenizer.json")
|
||||
}
|
||||
Self::Tokenizer2 => {
|
||||
("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json")
|
||||
}
|
||||
Self::Clip => (version.repo(), "text_encoder/model.safetensors"),
|
||||
Self::Clip2 => (version.repo(), "text_encoder_2/model.safetensors"),
|
||||
Self::Unet => (version.repo(), "unet/diffusion_pytorch_model.safetensors"),
|
||||
Self::Vae => (version.repo(), "vae/diffusion_pytorch_model.safetensors"),
|
||||
};
|
||||
let filename = Api::new()?.model(repo.to_string()).get(path)?;
|
||||
Ok(filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
use std::path::PathBuf;
|
||||
use crate::state::app_dir::AppWeightsDir;
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub enum ModelType {
|
||||
ClipJson,
|
||||
SdxlTurboUnet,
|
||||
SdxlTurboVae,
|
||||
SdxlTurboClipEncoder,
|
||||
SdxlTurboClipEncoder2,
|
||||
}
|
||||
|
||||
impl ModelType {
|
||||
pub fn get_hf_id(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::ClipJson => Some("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k"),
|
||||
Self::SdxlTurboUnet
|
||||
| Self::SdxlTurboVae
|
||||
| Self::SdxlTurboClipEncoder
|
||||
| Self::SdxlTurboClipEncoder2 => Some("stabilityai/sdxl-turbo"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_download_url(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ClipJson => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/tokenizer.json",
|
||||
Self::SdxlTurboUnet => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/diffusion_pytorch_model.unet.safetensors",
|
||||
Self::SdxlTurboVae => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/diffusion_pytorch_model.vae.safetensors",
|
||||
Self::SdxlTurboClipEncoder => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/clip_text_encoder.safetensors",
|
||||
Self::SdxlTurboClipEncoder2 => "https://pub-bc5e2bc0cdee4bb5ae8fca9d641ca0d6.r2.dev/clip_text_encoder_2.safetensors",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_filename(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ClipJson => "clip_vit_bigg_14_lion2b_39b_b160k.tokenizer.json",
|
||||
Self::SdxlTurboUnet => "diffusion_pytorch_model.unet.safetensors",
|
||||
Self::SdxlTurboVae => "diffusion_pytorch_model.vae.safetensors",
|
||||
Self::SdxlTurboClipEncoder => "clip_text_encoder.safetensors",
|
||||
Self::SdxlTurboClipEncoder2 => "clip_text_encoder_2.safetensors",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_path(&self, weights_dir: &AppWeightsDir) -> PathBuf {
|
||||
weights_dir.model_path(self)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
use crate::ml::model_file::StableDiffusionVersion;
|
||||
use anyhow::anyhow;
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
|
||||
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
|
||||
use hf_hub::api::sync::Api;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
pub struct LazyLoadVaeModel {
|
||||
lazy_model: Arc<RwLock<Option<AutoEncoderKL>>>,
|
||||
model_configs: VaeConfigs,
|
||||
}
|
||||
|
||||
// TODO: Maybe we pass these via a central config read at startup, passed as an Arc<ConfigT>
|
||||
pub struct VaeConfigs {
|
||||
sd_config: StableDiffusionConfig,
|
||||
sd_version: StableDiffusionVersion,
|
||||
device: Device,
|
||||
dtype: DType,
|
||||
hf_api: Api,
|
||||
}
|
||||
|
||||
impl LazyLoadVaeModel {
|
||||
pub fn new(
|
||||
sd_config: &StableDiffusionConfig,
|
||||
sd_version: &StableDiffusionVersion,
|
||||
hf_api: &Api,
|
||||
device: &Device,
|
||||
dtype: DType,
|
||||
) -> anyhow::Result<Self> {
|
||||
println!("building vae model... (1)");
|
||||
|
||||
Ok(Self {
|
||||
lazy_model: Arc::new(RwLock::new(None)),
|
||||
model_configs: VaeConfigs {
|
||||
sd_config: sd_config.clone(),
|
||||
sd_version: sd_version.clone(),
|
||||
device: device.clone(),
|
||||
hf_api: hf_api.clone(),
|
||||
dtype
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn encode(&self, xs: &Tensor) -> anyhow::Result<DiagonalGaussianDistribution> {
|
||||
println!("vae encode...");
|
||||
match self.try_encode(xs) {
|
||||
Err(err) => Err(err),
|
||||
Ok(Some(result)) => Ok(result),
|
||||
Ok(None) => {
|
||||
// Model wasn't previously loaded
|
||||
self.load_model()?;
|
||||
self.try_encode(xs)?
|
||||
.ok_or(anyhow!("model was not previously loaded"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(&self, xs: &Tensor) -> anyhow::Result<Tensor> {
|
||||
println!("vae decode...");
|
||||
match self.try_decode(xs) {
|
||||
Err(err) => Err(err),
|
||||
Ok(Some(result)) => Ok(result),
|
||||
Ok(None) => {
|
||||
// Model wasn't previously loaded
|
||||
self.load_model()?;
|
||||
self.try_decode(xs)?
|
||||
.ok_or(anyhow!("model was not previously loaded"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_encode(&self, xs: &Tensor) -> anyhow::Result<Option<DiagonalGaussianDistribution>> {
|
||||
match self.lazy_model.read() {
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
Ok(model) => {
|
||||
match &*model {
|
||||
None => Ok(None), // Model is not yet loaded
|
||||
Some(model) => {
|
||||
Ok(Some(model.encode(xs)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_decode(&self, xs: &Tensor) -> anyhow::Result<Option<Tensor>> {
|
||||
match self.lazy_model.read() {
|
||||
Err(err) => Err(anyhow!("lock error: {:?}", err)),
|
||||
Ok(model) => {
|
||||
match &*model {
|
||||
None => Ok(None), // Model is not yet loaded
|
||||
Some(model) => {
|
||||
Ok(Some(model.decode(xs)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn load_model(&self) -> anyhow::Result<()> {
|
||||
println!("load vae");
|
||||
match self.lazy_model.write() {
|
||||
Err(err) => return Err(anyhow!("lock error: {:?}", err)),
|
||||
Ok(mut maybe_model) => {
|
||||
match &*maybe_model {
|
||||
Some(_model) => {} // Already loaded; fall through
|
||||
None => {
|
||||
let repo = self.model_configs.sd_version.repo();
|
||||
|
||||
println!("Building VAE model from : {:?} ... (3)", repo);
|
||||
|
||||
let vae_file = self.model_configs.hf_api.model(repo.to_string())
|
||||
.get("vae/diffusion_pytorch_model.safetensors")?;
|
||||
|
||||
println!("Building VAE model from file {:?}...", &vae_file);
|
||||
let vae = self.model_configs
|
||||
.sd_config
|
||||
.build_vae(vae_file, &self.model_configs.device, self.model_configs.dtype)?;
|
||||
|
||||
println!("Built VAE...");
|
||||
*maybe_model = Some(vae);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod unet_model;
|
||||
pub mod lazy_load_vae_model;
|
||||
@@ -0,0 +1,26 @@
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel;
|
||||
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
|
||||
use std::path::Path;
|
||||
|
||||
pub struct UNetModel {
|
||||
model: UNet2DConditionModel,
|
||||
}
|
||||
|
||||
impl UNetModel {
|
||||
pub fn new<P: AsRef<Path>>(sd_config: &StableDiffusionConfig, unet_weights_file: P, device: &Device, dtype: DType) -> anyhow::Result<Self> {
|
||||
println!("building unet model... (1)");
|
||||
|
||||
let model = sd_config.build_unet(unet_weights_file, &device, 4, true, dtype)?;
|
||||
|
||||
Ok(Self { model })
|
||||
}
|
||||
|
||||
pub fn inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result<Tensor> {
|
||||
Ok(self.model.forward(&latent_model_input, timestep, &text_embeddings)?)
|
||||
}
|
||||
|
||||
pub fn forward_with_guidance(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor, guidance_scale_embedding: &Tensor) -> anyhow::Result<Tensor> {
|
||||
Ok(self.model.forward_with_guidance(&latent_model_input, timestep, &text_embeddings, Some(guidance_scale_embedding))?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
#[cfg(feature = "accelerate")]
|
||||
extern crate accelerate_src;
|
||||
#[cfg(feature = "mkl")]
|
||||
extern crate intel_mkl_src;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use candle_transformers::models::stable_diffusion;
|
||||
|
||||
use crate::ml::model_file::{ModelFile, StableDiffusionVersion};
|
||||
use anyhow::Error as E;
|
||||
use candle_core::{DType, Device, Module, Tensor, D};
|
||||
use log::info;
|
||||
use tokenizers::Tokenizer;
|
||||
use crate::ml::model_type::ModelType;
|
||||
use crate::state::app_dir::AppWeightsDir;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn infer_clip_text_embeddings(
|
||||
prompt: &str,
|
||||
uncond_prompt: &str,
|
||||
tokenizer: Option<String>,
|
||||
clip_weights: Option<String>,
|
||||
clip2_weights: Option<String>,
|
||||
sd_version: StableDiffusionVersion,
|
||||
sd_config: &stable_diffusion::StableDiffusionConfig,
|
||||
use_f16: bool,
|
||||
device: &Device,
|
||||
dtype: DType,
|
||||
use_guide_scale: bool,
|
||||
weights_dir: &AppWeightsDir,
|
||||
) -> anyhow::Result<Tensor> {
|
||||
info!("Preparing text embeddings...");
|
||||
|
||||
let which = match sd_version {
|
||||
StableDiffusionVersion::Xl
|
||||
| StableDiffusionVersion::XlInpaint
|
||||
| StableDiffusionVersion::Turbo => vec![true, false],
|
||||
_ => vec![true],
|
||||
};
|
||||
|
||||
let text_embeddings = which
|
||||
.iter()
|
||||
.map(|first| {
|
||||
do_infer_clip_text_embeddings(
|
||||
&prompt,
|
||||
&uncond_prompt,
|
||||
None, // tokenizer
|
||||
None, // clip_weights
|
||||
None, // clip2_weights
|
||||
sd_version,
|
||||
&sd_config,
|
||||
false, // use_f16
|
||||
&device,
|
||||
dtype,
|
||||
use_guide_scale,
|
||||
*first,
|
||||
weights_dir,
|
||||
)
|
||||
})
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
|
||||
let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?;
|
||||
|
||||
Ok(text_embeddings)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn do_infer_clip_text_embeddings(
|
||||
prompt: &str,
|
||||
uncond_prompt: &str,
|
||||
tokenizer: Option<String>,
|
||||
clip_weights: Option<String>,
|
||||
clip2_weights: Option<String>,
|
||||
sd_version: StableDiffusionVersion,
|
||||
sd_config: &stable_diffusion::StableDiffusionConfig,
|
||||
use_f16: bool,
|
||||
device: &Device,
|
||||
dtype: DType,
|
||||
use_guide_scale: bool,
|
||||
first: bool,
|
||||
weights_dir: &AppWeightsDir,
|
||||
) -> anyhow::Result<Tensor> {
|
||||
|
||||
let tokenizer_file = if first {
|
||||
info!("ModelFile::Tokenizer");
|
||||
ModelFile::Tokenizer
|
||||
} else {
|
||||
info!("ModelFile::Tokenizer2");
|
||||
ModelFile::Tokenizer2
|
||||
};
|
||||
|
||||
info!("Loading clip from download");
|
||||
//let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?;
|
||||
let tokenizer = weights_dir.model_path(&ModelType::ClipJson);
|
||||
|
||||
info!("Tokenizer path: {:?}", tokenizer);
|
||||
|
||||
let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?;
|
||||
|
||||
let pad_id = match &sd_config.clip.pad_with {
|
||||
Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(),
|
||||
None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(),
|
||||
};
|
||||
|
||||
println!("Running with prompt \"{prompt}\".");
|
||||
let mut tokens = tokenizer
|
||||
.encode(prompt, true)
|
||||
.map_err(E::msg)?
|
||||
.get_ids()
|
||||
.to_vec();
|
||||
if tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!(
|
||||
"the prompt is too long, {} > max-tokens ({})",
|
||||
tokens.len(),
|
||||
sd_config.clip.max_position_embeddings
|
||||
)
|
||||
}
|
||||
while tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
tokens.push(pad_id)
|
||||
}
|
||||
let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
|
||||
println!("Building the Clip transformer.");
|
||||
let clip_weights_file = if first {
|
||||
ModelFile::Clip
|
||||
} else {
|
||||
ModelFile::Clip2
|
||||
};
|
||||
let clip_weights = if first {
|
||||
//clip_weights_file.get(clip_weights, sd_version, use_f16)?
|
||||
weights_dir.model_path(&ModelType::SdxlTurboClipEncoder)
|
||||
} else {
|
||||
//clip_weights_file.get(clip2_weights, sd_version, use_f16)?
|
||||
weights_dir.model_path(&ModelType::SdxlTurboClipEncoder2)
|
||||
};
|
||||
let clip_config = if first {
|
||||
&sd_config.clip
|
||||
} else {
|
||||
sd_config.clip2.as_ref().unwrap()
|
||||
};
|
||||
let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?;
|
||||
|
||||
let text_embeddings = text_model.forward(&tokens)?;
|
||||
|
||||
let text_embeddings = if use_guide_scale {
|
||||
let mut uncond_tokens = tokenizer
|
||||
.encode(uncond_prompt, true)
|
||||
.map_err(E::msg)?
|
||||
.get_ids()
|
||||
.to_vec();
|
||||
if uncond_tokens.len() > sd_config.clip.max_position_embeddings {
|
||||
anyhow::bail!(
|
||||
"the negative prompt is too long, {} > max-tokens ({})",
|
||||
uncond_tokens.len(),
|
||||
sd_config.clip.max_position_embeddings
|
||||
)
|
||||
}
|
||||
while uncond_tokens.len() < sd_config.clip.max_position_embeddings {
|
||||
uncond_tokens.push(pad_id)
|
||||
}
|
||||
|
||||
let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?;
|
||||
let uncond_embeddings = text_model.forward(&uncond_tokens)?;
|
||||
|
||||
Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)?
|
||||
} else {
|
||||
text_embeddings.to_dtype(dtype)?
|
||||
};
|
||||
Ok(text_embeddings)
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
use crate::events::notification_event::{NotificationModelType, NotificationEvent};
|
||||
use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
|
||||
use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
|
||||
use crate::ml::model_cache::ModelCache;
|
||||
use crate::ml::model_file::StableDiffusionVersion;
|
||||
use crate::ml::model_type::ModelType;
|
||||
use crate::ml::models::unet_model::UNetModel;
|
||||
use crate::ml::prompt_cache::PromptCache;
|
||||
use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
|
||||
use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use anyhow::{anyhow, Error as E, Result};
|
||||
use candle_core::{DType, IndexOp, Tensor, D};
|
||||
use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
|
||||
use candle_transformers::models::stable_diffusion::lcm::LCMScheduler;
|
||||
use image::DynamicImage;
|
||||
use log::info;
|
||||
use rand::Rng;
|
||||
use tauri::{AppHandle, Emitter};
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
pub struct Args<'a> {
|
||||
pub image: &'a DynamicImage,
|
||||
pub prompt: String,
|
||||
pub uncond_prompt: String,
|
||||
pub cfg_scale: Option<f64>,
|
||||
pub i2i_strength: Option<u8>,
|
||||
pub configs: &'a AppConfig,
|
||||
pub model_cache: &'a ModelCache,
|
||||
pub prompt_cache: &'a PromptCache,
|
||||
pub app_data_root: &'a AppDataRoot,
|
||||
pub app: &'a AppHandle,
|
||||
}
|
||||
|
||||
pub fn lcm_pipeline(args: Args<'_>) -> Result<RgbImage> {
|
||||
let Args {
|
||||
prompt,
|
||||
uncond_prompt,
|
||||
cfg_scale,
|
||||
i2i_strength,
|
||||
configs,
|
||||
model_cache,
|
||||
prompt_cache,
|
||||
app,
|
||||
image,
|
||||
app_data_root,
|
||||
} = args;
|
||||
|
||||
let weights_dir = app_data_root.weights_dir();
|
||||
|
||||
println!("Starting image generation with the following configuration:");
|
||||
println!(" Model: {:?}", configs.sd_version);
|
||||
println!(" Prompt: {}", prompt);
|
||||
println!(" Steps: {}", configs.scheduler_steps);
|
||||
println!(" Device: {:?}", configs.device);
|
||||
|
||||
println!("Model dimensions: {}x{}", configs.sd_config.width, configs.sd_config.height);
|
||||
|
||||
// Use LCM Scheduler instead of Euler Ancestral for better speed and quality
|
||||
let mut scheduler = LCMScheduler::new(
|
||||
configs.scheduler_steps,
|
||||
candle_transformers::models::stable_diffusion::lcm::LCMSchedulerConfig::default(),
|
||||
)?;
|
||||
|
||||
let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
configs.device.set_seed(seed)?;
|
||||
|
||||
info!("Using seed: {}", seed);
|
||||
|
||||
let guidance_scale = match cfg_scale {
|
||||
Some(guidance_scale) => guidance_scale,
|
||||
None => match configs.sd_version {
|
||||
StableDiffusionVersion::V1_5
|
||||
| StableDiffusionVersion::V2_1
|
||||
| StableDiffusionVersion::Xl => 7.5,
|
||||
StableDiffusionVersion::Turbo => 0.,
|
||||
_ => 0., // NB: Not sure what the other model families should use, so sticking with "0"
|
||||
},
|
||||
};
|
||||
|
||||
let use_guide_scale = guidance_scale > 1.0;
|
||||
|
||||
info!("Using guide scale: {}", use_guide_scale);
|
||||
|
||||
info!("Checking if prompt is cached");
|
||||
let maybe_cached = prompt_cache.get_copy(&prompt)?;
|
||||
|
||||
let mut text_embeddings = if let Some(tensor) = maybe_cached {
|
||||
tensor
|
||||
} else {
|
||||
|
||||
info!("Prompt is NOT cached! Calculating embedding...");
|
||||
let tensor = infer_clip_text_embeddings(
|
||||
&prompt,
|
||||
&uncond_prompt,
|
||||
None, // tokenizer
|
||||
None, // clip_weights
|
||||
None, // clip2_weights
|
||||
configs.sd_version,
|
||||
&configs.sd_config,
|
||||
false, // use_f16
|
||||
&configs.device,
|
||||
configs.dtype,
|
||||
use_guide_scale,
|
||||
weights_dir,
|
||||
)?;
|
||||
prompt_cache.store_copy(&prompt, &tensor)?;
|
||||
tensor
|
||||
};
|
||||
|
||||
println!("Text embeddings shape: {:?}", text_embeddings.shape());
|
||||
|
||||
println!("Loading input image into tensor...");
|
||||
|
||||
let input_image = dynamic_image_to_tensor(
|
||||
image,
|
||||
&configs.device,
|
||||
configs.dtype)?;
|
||||
|
||||
println!("Reference image shape: {:?}", input_image.shape());
|
||||
|
||||
let vae_scale = get_vae_scale(configs.sd_version);
|
||||
|
||||
let maybe_vae = model_cache.get_vae()?;
|
||||
|
||||
let vae = match maybe_vae {
|
||||
Some(vae) => vae,
|
||||
None => {
|
||||
info!("No vae found in cache; loading...");
|
||||
|
||||
let mut notify_download_complete = false;
|
||||
//if !vae_file.exists() {
|
||||
if true {
|
||||
notify_download_complete = true;
|
||||
app.emit("notification", NotificationEvent::ModelDownloadStarted {
|
||||
model_name: "sdxl-turbo-vae",
|
||||
model_type: NotificationModelType::Vae,
|
||||
})?;
|
||||
}
|
||||
|
||||
let vae_file = weights_dir.model_path(&ModelType::SdxlTurboVae);
|
||||
|
||||
let vae = configs
|
||||
.sd_config
|
||||
.build_vae(vae_file, &configs.device, configs.dtype)?;
|
||||
|
||||
let vae = Arc::new(vae);
|
||||
|
||||
model_cache.set_vae(vae.clone())?;
|
||||
|
||||
if notify_download_complete {
|
||||
app.emit("notification", NotificationEvent::ModelDownloadComplete {
|
||||
model_name: "sdxl-turbo-vae",
|
||||
model_type: NotificationModelType::Vae,
|
||||
})?;
|
||||
}
|
||||
|
||||
vae
|
||||
}
|
||||
};
|
||||
|
||||
let maybe_unet = model_cache.get_unet()?;
|
||||
|
||||
let unet = match maybe_unet {
|
||||
Some(unet) => unet,
|
||||
None => {
|
||||
info!("No unet found in cache; loading...");
|
||||
|
||||
let mut notify_download_complete = false;
|
||||
//if !unet_file.exists() {
|
||||
if true {
|
||||
notify_download_complete = true;
|
||||
app.emit("notification", NotificationEvent::ModelDownloadStarted {
|
||||
model_name: "sdxl-turbo-unet",
|
||||
model_type: NotificationModelType::Unet,
|
||||
})?;
|
||||
}
|
||||
|
||||
let unet_file = weights_dir.model_path(&ModelType::SdxlTurboUnet);
|
||||
|
||||
let unet = UNetModel::new(&configs.sd_config, unet_file, &configs.device, configs.dtype)
|
||||
.map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
|
||||
|
||||
let unet = Arc::new(unet);
|
||||
|
||||
model_cache.set_unet(unet.clone())?;
|
||||
|
||||
if notify_download_complete {
|
||||
app.emit("notification", NotificationEvent::ModelDownloadComplete {
|
||||
model_name: "sdxl-turbo-unet",
|
||||
model_type: NotificationModelType::Unet,
|
||||
})?;
|
||||
}
|
||||
|
||||
unet
|
||||
}
|
||||
};
|
||||
|
||||
let init_latent_dist : DiagonalGaussianDistribution = vae.encode(&input_image)?;
|
||||
|
||||
// Parse and calculate the img2img strength
|
||||
let img2img_strength = match i2i_strength {
|
||||
None => 0.75f64,
|
||||
Some(strength) => (strength as f64) / 100.0f64,
|
||||
};
|
||||
|
||||
// Create LCM guidance scale embeddings
|
||||
let embedding_dim = 256;
|
||||
let guidance_scale_embedding = scheduler.get_guidance_scale_embedding(
|
||||
guidance_scale,
|
||||
embedding_dim,
|
||||
&configs.device,
|
||||
configs.dtype
|
||||
)?;
|
||||
|
||||
// Generate latents from input image using the LCM approach
|
||||
println!("Generating latents from input image...");
|
||||
let init_latents = (init_latent_dist.sample()? * vae_scale)?;
|
||||
println!("Initial latents shape: {:?}", init_latents.shape());
|
||||
|
||||
// Calculate timesteps for LCM with proper img2img handling
|
||||
let timesteps = LCMScheduler::get_timesteps_for_steps(configs.scheduler_steps, img2img_strength);
|
||||
let t_start = timesteps[0]; // First denoising step
|
||||
|
||||
// Add noise at the appropriate timestep for img2img
|
||||
println!("Adding noise to latents for t_start={}", t_start);
|
||||
let noise = init_latents.randn_like(0f64, 1f64)?;
|
||||
let latents = scheduler.add_noise(&init_latents, noise, t_start)?;
|
||||
|
||||
println!("Latents initialized successfully");
|
||||
println!("Initial noise shape: {:?}", latents.shape());
|
||||
|
||||
println!("Starting diffusion process...");
|
||||
|
||||
let mut latents = latents;
|
||||
let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect();
|
||||
|
||||
for (timestep_index, ×tep) in timesteps.iter().enumerate() {
|
||||
println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep);
|
||||
|
||||
let latent_model_input = if use_guide_scale {
|
||||
Tensor::cat(&[&latents, &latents], 0)?
|
||||
} else {
|
||||
latents.clone()
|
||||
};
|
||||
|
||||
let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep);
|
||||
|
||||
// Use the guidance scale embedding in the UNet inference with the proper method name
|
||||
let mut noise_pred = match unet.forward_with_guidance(
|
||||
&latent_model_input,
|
||||
timestep as f64,
|
||||
&text_embeddings,
|
||||
&guidance_scale_embedding) {
|
||||
Ok(pred) => pred,
|
||||
Err(e) => {
|
||||
println!("UNet inference failed with error: {}", e);
|
||||
return Err(anyhow::anyhow!("UNet inference failed: {}", e));
|
||||
},
|
||||
};
|
||||
|
||||
if use_guide_scale {
|
||||
let chunks = noise_pred.chunk(2, 0)?;
|
||||
let (noise_pred_uncond, noise_pred_text) = (&chunks[0], &chunks[1]);
|
||||
noise_pred = (noise_pred_uncond + (guidance_scale * (noise_pred_text - noise_pred_uncond)?)?)?;
|
||||
}
|
||||
|
||||
// Apply the LCM scheduler step
|
||||
latents = scheduler.step(&noise_pred, timestep, &latents, timestep_index, configs.scheduler_steps)?;
|
||||
}
|
||||
|
||||
println!("Diffusion process completed, decoding image...");
|
||||
let image = vae.decode(&(latents / vae_scale)?)?;
|
||||
|
||||
println!("VAE decode completed");
|
||||
|
||||
println!("Scaling image back");
|
||||
let image = ((image / 2.)? + 0.5)?;
|
||||
|
||||
println!("Normalized image values");
|
||||
let image = (image.clamp(0f32, 1.)? * 255.)?;
|
||||
|
||||
println!("Convert to int8");
|
||||
let image = image.to_dtype(DType::U8)?;
|
||||
|
||||
println!("Converted to 8-bit format");
|
||||
|
||||
let image = tensor_to_image_buffer(&image.i(0)?)?;
|
||||
|
||||
Ok(image)
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
pub mod get_vae_scale;
|
||||
pub mod infer_clip_text_embeddings;
|
||||
pub mod stable_diffusion_pipeline;
|
||||
pub mod lcm_pipeline;
|
||||
mod remap_lcm_strength_range;
|
||||
pub mod lcm_pipeline;
|
||||
@@ -1,78 +1,76 @@
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
use crate::events::notification_event::{NotificationModelType, NotificationEvent};
|
||||
use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor;
|
||||
use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage};
|
||||
use crate::ml::model_cache::ModelCache;
|
||||
use crate::ml::model_file::StableDiffusionVersion;
|
||||
use crate::ml::model_type::ModelType;
|
||||
use crate::ml::models::unet_model::UNetModel;
|
||||
use crate::ml::prompt_cache::PromptCache;
|
||||
use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale;
|
||||
use crate::ml::stable_diffusion::infer_clip_text_embeddings::{infer_clip_text_embeddings, ClipArgs};
|
||||
use crate::ml::stable_diffusion::infer_clip_text_embeddings::infer_clip_text_embeddings;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use anyhow::{anyhow, Error as E, Result};
|
||||
use candle_core::{DType, Device, IndexOp, Tensor, D};
|
||||
use candle_core::{DType, IndexOp, Tensor, D};
|
||||
use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution};
|
||||
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
|
||||
use image::DynamicImage;
|
||||
use log::info;
|
||||
use rand::Rng;
|
||||
use tauri::{AppHandle, Emitter};
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
pub struct Args<'a, P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>> {
|
||||
pub struct Args<'a> {
|
||||
pub image: &'a DynamicImage,
|
||||
pub prompt: String,
|
||||
pub uncond_prompt: String,
|
||||
pub cfg_scale: Option<f64>,
|
||||
pub i2i_strength: Option<u8>,
|
||||
pub configs: &'a AppConfig,
|
||||
pub model_cache: &'a ModelCache,
|
||||
pub prompt_cache: &'a PromptCache,
|
||||
pub sd_version: StableDiffusionVersion,
|
||||
pub sd_config: StableDiffusionConfig,
|
||||
pub device: &'a Device,
|
||||
pub dtype: DType,
|
||||
pub maybe_seed: Option<u64>,
|
||||
pub scheduler_steps: usize,
|
||||
pub sdxl_turbo_vae_path: P,
|
||||
pub sdxl_turbo_unet_path: Q,
|
||||
pub clip_json_path: R,
|
||||
pub clip_weights_path: S,
|
||||
pub app_data_root: &'a AppDataRoot,
|
||||
pub app: &'a AppHandle,
|
||||
}
|
||||
|
||||
pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>) -> Result<RgbImage>
|
||||
where P1: AsRef<Path>, P2: AsRef<Path>, P3: AsRef<Path>, P4: AsRef<Path>
|
||||
{
|
||||
pub fn stable_diffusion_pipeline(args: Args<'_>) -> Result<RgbImage> {
|
||||
let Args {
|
||||
prompt,
|
||||
uncond_prompt,
|
||||
cfg_scale,
|
||||
i2i_strength,
|
||||
configs,
|
||||
model_cache,
|
||||
prompt_cache,
|
||||
app,
|
||||
image,
|
||||
sd_version,
|
||||
sd_config,
|
||||
device,
|
||||
dtype,
|
||||
maybe_seed,
|
||||
scheduler_steps,
|
||||
sdxl_turbo_vae_path,
|
||||
sdxl_turbo_unet_path,
|
||||
clip_json_path,
|
||||
clip_weights_path,
|
||||
app_data_root,
|
||||
} = args;
|
||||
|
||||
let weights_dir = app_data_root.weights_dir();
|
||||
|
||||
println!("Starting image generation with the following configuration:");
|
||||
println!(" Model: {:?}", configs.sd_version);
|
||||
println!(" Prompt: {}", prompt);
|
||||
println!(" Steps: {}", configs.scheduler_steps);
|
||||
println!(" Device: {:?}", configs.device);
|
||||
|
||||
println!("Model dimensions: {}x{}", configs.sd_config.width, configs.sd_config.height);
|
||||
|
||||
// TODO(bt,2025-02-18): The scheduler is `EulerAncestralDiscreteScheduler`, but we may want to port an LCM scheduler.
|
||||
// This is a target for performance improvement
|
||||
// See: https://github.com/huggingface/candle/issues/1331
|
||||
let mut scheduler = sd_config.build_scheduler(scheduler_steps)?;
|
||||
let mut scheduler = configs.sd_config.build_scheduler(
|
||||
configs.scheduler_steps)?;
|
||||
|
||||
let seed = maybe_seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
|
||||
device.set_seed(seed)?;
|
||||
let seed = configs.seed.unwrap_or_else(|| rand::thread_rng().gen());
|
||||
configs.device.set_seed(seed)?;
|
||||
|
||||
info!("Using seed: {}", seed);
|
||||
|
||||
let guidance_scale = match cfg_scale {
|
||||
Some(guidance_scale) => guidance_scale,
|
||||
None => match sd_version {
|
||||
None => match configs.sd_version {
|
||||
StableDiffusionVersion::V1_5
|
||||
| StableDiffusionVersion::V2_1
|
||||
| StableDiffusionVersion::Xl => 7.5,
|
||||
@@ -93,21 +91,20 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
} else {
|
||||
|
||||
info!("Prompt is NOT cached! Calculating embedding...");
|
||||
let tensor = infer_clip_text_embeddings(ClipArgs {
|
||||
prompt: &prompt,
|
||||
uncond_prompt: &uncond_prompt,
|
||||
tokenizer: None, // tokenizer
|
||||
clip_weights: None, // clip_weights
|
||||
clip2_weights: None, // clip2_weights
|
||||
sd_version: sd_version,
|
||||
sd_config: &sd_config,
|
||||
use_f16: false, // use_f16
|
||||
device: device,
|
||||
dtype: dtype,
|
||||
let tensor = infer_clip_text_embeddings(
|
||||
&prompt,
|
||||
&uncond_prompt,
|
||||
None, // tokenizer
|
||||
None, // clip_weights
|
||||
None, // clip2_weights
|
||||
configs.sd_version,
|
||||
&configs.sd_config,
|
||||
false, // use_f16
|
||||
&configs.device,
|
||||
configs.dtype,
|
||||
use_guide_scale,
|
||||
clip_json_path: clip_json_path.as_ref(),
|
||||
clip_weights_path: clip_weights_path.as_ref(),
|
||||
})?;
|
||||
weights_dir,
|
||||
)?;
|
||||
prompt_cache.store_copy(&prompt, &tensor)?;
|
||||
tensor
|
||||
};
|
||||
@@ -116,11 +113,14 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
|
||||
println!("Loading input image into tensor...");
|
||||
|
||||
let input_image = dynamic_image_to_tensor(image, device, dtype)?;
|
||||
let input_image = dynamic_image_to_tensor(
|
||||
image,
|
||||
&configs.device,
|
||||
configs.dtype)?;
|
||||
|
||||
println!("Reference image shape: {:?}", input_image.shape());
|
||||
|
||||
let vae_scale = get_vae_scale(sd_version);
|
||||
let vae_scale = get_vae_scale(configs.sd_version);
|
||||
|
||||
let maybe_vae = model_cache.get_vae()?;
|
||||
|
||||
@@ -129,14 +129,33 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
None => {
|
||||
info!("No vae found in cache; loading...");
|
||||
|
||||
let vae_file = sdxl_turbo_vae_path.as_ref();
|
||||
let mut notify_download_complete = false;
|
||||
//if !vae_file.exists() {
|
||||
if true {
|
||||
notify_download_complete = true;
|
||||
app.emit("notification", NotificationEvent::ModelDownloadStarted {
|
||||
model_name: "sdxl-turbo-vae",
|
||||
model_type: NotificationModelType::Vae,
|
||||
})?;
|
||||
}
|
||||
|
||||
let vae = sd_config.build_vae(vae_file, &device, dtype)?;
|
||||
let vae_file = weights_dir.model_path(&ModelType::SdxlTurboVae);
|
||||
|
||||
let vae = configs
|
||||
.sd_config
|
||||
.build_vae(vae_file, &configs.device, configs.dtype)?;
|
||||
|
||||
let vae = Arc::new(vae);
|
||||
|
||||
model_cache.set_vae(vae.clone())?;
|
||||
|
||||
|
||||
if notify_download_complete {
|
||||
app.emit("notification", NotificationEvent::ModelDownloadComplete {
|
||||
model_name: "sdxl-turbo-vae",
|
||||
model_type: NotificationModelType::Vae,
|
||||
})?;
|
||||
}
|
||||
|
||||
vae
|
||||
}
|
||||
};
|
||||
@@ -148,15 +167,32 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
None => {
|
||||
info!("No unet found in cache; loading...");
|
||||
|
||||
let unet_weights_file = sdxl_turbo_unet_path.as_ref();
|
||||
let mut notify_download_complete = false;
|
||||
//if !unet_file.exists() {
|
||||
if true {
|
||||
notify_download_complete = true;
|
||||
app.emit("notification", NotificationEvent::ModelDownloadStarted {
|
||||
model_name: "sdxl-turbo-unet",
|
||||
model_type: NotificationModelType::Unet,
|
||||
})?;
|
||||
}
|
||||
|
||||
let unet_file = weights_dir.model_path(&ModelType::SdxlTurboUnet);
|
||||
|
||||
let unet = sd_config.build_unet(unet_weights_file, &device, 4, true, dtype)
|
||||
let unet = UNetModel::new(&configs.sd_config, unet_file, &configs.device, configs.dtype)
|
||||
.map_err(|err| anyhow!("error initializing unet model: {:?}", err))?;
|
||||
|
||||
|
||||
let unet = Arc::new(unet);
|
||||
|
||||
model_cache.set_unet(unet.clone())?;
|
||||
|
||||
if notify_download_complete {
|
||||
app.emit("notification", NotificationEvent::ModelDownloadComplete {
|
||||
model_name: "sdxl-turbo-unet",
|
||||
model_type: NotificationModelType::Unet,
|
||||
})?;
|
||||
}
|
||||
|
||||
unet
|
||||
}
|
||||
};
|
||||
@@ -180,9 +216,9 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
};
|
||||
|
||||
let t_start = {
|
||||
let start = scheduler_steps - (scheduler_steps as f64 * img2img_strength) as usize;
|
||||
let start = configs.scheduler_steps - (configs.scheduler_steps as f64 * img2img_strength) as usize;
|
||||
|
||||
println!("Starting from step {} of {} (strength: {})", start, scheduler_steps, img2img_strength);
|
||||
println!("Starting from step {} of {} (strength: {})", start, configs.scheduler_steps, img2img_strength);
|
||||
start
|
||||
};
|
||||
|
||||
@@ -221,7 +257,7 @@ pub fn stable_diffusion_pipeline<P1, P2, P3, P4>(args: Args<'_, P1, P2, P3, P4>)
|
||||
|
||||
let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?;
|
||||
|
||||
let mut noise_pred = match unet.forward(&latent_model_input, timestep as f64, &text_embeddings) {
|
||||
let mut noise_pred = match unet.inference(&latent_model_input, timestep as f64, &text_embeddings) {
|
||||
Ok(pred) => pred,
|
||||
Err(e) => {
|
||||
println!("UNet inference failed with error: {}", e);
|
||||
@@ -0,0 +1,108 @@
|
||||
use crate::ml::model_file::StableDiffusionVersion;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use crate::state::os_platform::OsPlatform;
|
||||
use crate::state::yaml_config::YamlConfig;
|
||||
use candle_core::{DType, Device};
|
||||
use candle_transformers::models::stable_diffusion::StableDiffusionConfig;
|
||||
use expanduser::expanduser;
|
||||
use filesys::recursively_find_file_by_name::recursively_find_file_by_name;
|
||||
use hf_hub::api::sync::Api;
|
||||
use log::info;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const DEFAULT_SD_IMAGE_WIDTH: usize = 512;
|
||||
const DEFAULT_SD_IMAGE_HEIGHT: usize = 512;
|
||||
|
||||
const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32;
|
||||
const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::Turbo;
|
||||
|
||||
const DEFAULT_DATA_DIR_UNIX : &str = "~/artcraft";
|
||||
|
||||
/// A central place to configure app-wide details.
|
||||
pub struct AppConfig {
|
||||
/// We only support one device type currently, and we can't multiplex
|
||||
/// over several devices yet.
|
||||
pub device: Device,
|
||||
|
||||
/// The core numeric type to use for models.
|
||||
pub dtype: DType,
|
||||
|
||||
pub sd_version: StableDiffusionVersion,
|
||||
pub sd_config: StableDiffusionConfig,
|
||||
|
||||
pub image_height: usize,
|
||||
pub image_width: usize,
|
||||
pub scheduler_steps: usize,
|
||||
pub scheduler_samples: usize,
|
||||
|
||||
pub seed: Option<u64>,
|
||||
|
||||
pub cfg_scale: Option<f64>,
|
||||
|
||||
/// Probably shouldn't be here.
|
||||
pub hf_api: Api,
|
||||
|
||||
/// Root location for application data.
|
||||
pub app_data_root: AppDataRoot,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
|
||||
pub fn init() -> anyhow::Result<Self> {
|
||||
let yaml_configs = YamlConfig::load_from_config_file_recursive();
|
||||
|
||||
println!("Configs: {:?}", yaml_configs);
|
||||
|
||||
let device = Device::new_cuda(0)
|
||||
.unwrap_or_else(|e| {
|
||||
info!("CUDA not available ({}), falling back to CPU", e);
|
||||
Device::Cpu
|
||||
});
|
||||
|
||||
let sd_config = match SD_VERSION {
|
||||
StableDiffusionVersion::Turbo => {
|
||||
StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
|
||||
}
|
||||
_ => {
|
||||
StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH))
|
||||
}
|
||||
};
|
||||
|
||||
let hf_api = Api::new()?;
|
||||
|
||||
let os_platform = OsPlatform::get();
|
||||
|
||||
let default_data_path = match os_platform {
|
||||
OsPlatform::Linux => yaml_configs.linux_default_data_path.as_deref(),
|
||||
OsPlatform::MacOs => yaml_configs.mac_default_data_path.as_deref(),
|
||||
OsPlatform::Windows => yaml_configs.windows_default_data_path.as_deref(),
|
||||
};
|
||||
|
||||
let data_root_path = match default_data_path {
|
||||
Some(default_data_path) => default_data_path.to_string(),
|
||||
None => match os_platform {
|
||||
OsPlatform::Linux | OsPlatform::MacOs => DEFAULT_DATA_DIR_UNIX.to_string(),
|
||||
OsPlatform::Windows => ".".to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
println!("Using data root: {}", data_root_path);
|
||||
|
||||
let app_data_root = AppDataRoot::create_existing(data_root_path)?;
|
||||
|
||||
Ok(Self {
|
||||
device,
|
||||
dtype: DEFAULT_SD_NUMERIC_DATATYPE,
|
||||
sd_version: SD_VERSION,
|
||||
sd_config,
|
||||
image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
|
||||
image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
|
||||
scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
|
||||
scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
|
||||
seed: yaml_configs.seed,
|
||||
cfg_scale: yaml_configs.cfg_scale,
|
||||
hf_api,
|
||||
app_data_root,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use crate::state::os_platform::OsPlatform;
|
||||
use expanduser::expanduser;
|
||||
use std::path::{Path, PathBuf};
|
||||
use crate::ml::model_type::ModelType;
|
||||
|
||||
const ASSETS_SUBDIRECTORY : &str = "assets";
|
||||
const WEIGHTS_SUBDIRECTORY : &str = "weights";
|
||||
|
||||
/// The path to the application data directory, which includes "asset" and "weights" data.
|
||||
#[derive(Clone)]
|
||||
pub struct AppDataRoot {
|
||||
path: PathBuf,
|
||||
assets_dir: AppAssetsDir,
|
||||
weights_dir: AppWeightsDir,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppAssetsDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppWeightsDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl AppDataRoot {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
|
||||
match OsPlatform::get() {
|
||||
OsPlatform::Linux | OsPlatform::MacOs => {
|
||||
if let Some(d) = dir.as_os_str().to_str() {
|
||||
dir = expanduser(d)?;
|
||||
}
|
||||
},
|
||||
OsPlatform::Windows => {}
|
||||
}
|
||||
|
||||
if !dir.is_dir() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
}
|
||||
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
|
||||
let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
|
||||
let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
|
||||
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
assets_dir,
|
||||
weights_dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn assets_dir(&self) -> &AppAssetsDir {
|
||||
&self.assets_dir
|
||||
}
|
||||
|
||||
pub fn weights_dir(&self) -> &AppWeightsDir {
|
||||
&self.weights_dir
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl AppAssetsDir {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
if !dir.exists() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir(&dir)?;
|
||||
}
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl AppWeightsDir {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
if !dir.exists() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir(&dir)?;
|
||||
}
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn model_path(&self, model_registry: &ModelType) -> PathBuf {
|
||||
self.path.join(model_registry.get_filename())
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
pub mod app_config;
|
||||
pub mod app_dir;
|
||||
pub mod os_platform;
|
||||
pub mod yaml_config;
|
||||
pub mod expanduser;
|
||||
pub mod yaml_config;
|
||||
@@ -0,0 +1,11 @@
|
||||
use tauri::{AppHandle, Emitter};
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
pub async fn downloader_thread(app_data_root: AppDataRoot, app: AppHandle) -> ! {
|
||||
loop {
|
||||
app.emit("testing", 1);
|
||||
|
||||
println!(">>> Testing: {:?}", app_data_root.path());
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
@@ -13,8 +13,8 @@
|
||||
"windows": [
|
||||
{
|
||||
"title": "ArtCraft",
|
||||
"width": 2400,
|
||||
"height": 1300,
|
||||
"width": 1200,
|
||||
"height": 800,
|
||||
"resizable": true,
|
||||
"fullscreen": false
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
# Don't restart the dev server on Rust changes
|
||||
crates/
|
||||
Cargo.toml
|
||||
Cargo.lock
|
||||
@@ -1,74 +0,0 @@
|
||||
[package]
|
||||
name = "artcraft"
|
||||
version = "0.1.0"
|
||||
description = "Artcraft"
|
||||
authors = ["you"]
|
||||
license = ""
|
||||
repository = ""
|
||||
edition = "2021"
|
||||
rust-version = "1.77.2"
|
||||
publish = false
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[lib]
|
||||
name = "app_lib"
|
||||
crate-type = ["staticlib", "cdylib", "rlib"]
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2.1.1", features = [] }
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
crockford = { workspace = true }
|
||||
errors = { workspace = true }
|
||||
filesys = { workspace = true }
|
||||
memory_caching = { workspace = true }
|
||||
openai_sora_client = { workspace = true }
|
||||
primitives = { workspace = true }
|
||||
subprocess_common = { workspace = true }
|
||||
|
||||
# External
|
||||
#openssl = { version = "0.10", features = ["vendored"] }
|
||||
# NB(bt,2025-03-10): default-features = false on Windows while I debug openssl / rustls
|
||||
anyhow = { workspace = true }
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.10"
|
||||
chrono = { workspace = true }
|
||||
const_format = "0.2.34"
|
||||
directories = "6.0.0"
|
||||
dirs = "6.0.0"
|
||||
futures = "0.3"
|
||||
image = "0.25.5"
|
||||
log = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["multipart", "stream"], default-features = false }
|
||||
serde = { workspace = true }
|
||||
serde_derive = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
tauri = { version = "2.4.1", features = ["unstable"] }
|
||||
tauri-plugin-log = "2.3.1"
|
||||
tempdir = "0.3.7"
|
||||
tempfile = "3.18.0"
|
||||
tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
|
||||
tokio-util = { version = "0.7", features = ["codec"] }
|
||||
|
||||
[dev-dependencies]
|
||||
libloading = "0.8.6"
|
||||
serial_test = "3.2.0"
|
||||
assertor = "0.0.3"
|
||||
|
||||
[profile.dev]
|
||||
incremental = true # Compile your binary in smaller steps.
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1 # Allows LLVM to perform better optimization.
|
||||
lto = true # Enables link-time-optimizations.
|
||||
opt-level = 3 # `s` prioritizes small binary size. Use `3` if you prefer speed.
|
||||
panic = "abort" # Higher performance by disabling panic handlers.
|
||||
strip = true # Ensures debug symbols are removed.
|
||||
@@ -1,90 +0,0 @@
|
||||
use std::fs::read_to_string;
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
use errors::AnyhowResult;
|
||||
use base64::Engine;
|
||||
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
|
||||
use image::{DynamicImage, ImageReader};
|
||||
use std::io::Cursor;
|
||||
use log::{error, info};
|
||||
use tauri::{AppHandle, Manager, State};
|
||||
use openai_sora_client::credentials::SoraCredentials;
|
||||
use openai_sora_client::image_gen::common::{ImageSize, NumImages};
|
||||
use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
|
||||
use openai_sora_client::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
|
||||
use openai_sora_client::upload::upload_media_from_file::SoraMediaUploadRequest;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn image_generation_command(
|
||||
image: &str,
|
||||
prompt: &str,
|
||||
app: AppHandle,
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
) -> Result<String, String> {
|
||||
info!("image_generation_command called; processing image...");
|
||||
|
||||
let bytes = BASE64_STANDARD.decode(image)
|
||||
.map_err(|err| format!("Base64 decode error: {}", err))?;
|
||||
|
||||
generate_image(bytes, prompt, &app_data_root)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("error: {:?}", err);
|
||||
"there was an error".to_string()
|
||||
})?;
|
||||
|
||||
Ok("success".to_string())
|
||||
}
|
||||
|
||||
pub async fn generate_image(file_bytes: Vec<u8>, prompt: &str, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
|
||||
let sora_credentials = get_credentials(app_data_root)?;
|
||||
|
||||
let filename = "image.png".to_string();
|
||||
|
||||
let response= sora_media_upload_from_bytes(file_bytes, filename, &sora_credentials)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Failed to upload image to Sora: {:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
let sora_media_tokens = vec![response.id];
|
||||
|
||||
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
|
||||
prompt: prompt.to_string(),
|
||||
num_images: NumImages::One,
|
||||
image_size: ImageSize::Square,
|
||||
sora_media_tokens: sora_media_tokens.clone(),
|
||||
credentials: &sora_credentials,
|
||||
}).await
|
||||
.map_err(|err| {
|
||||
error!("Failed to call Sora image generation: {:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
println!(">> TASK ID: {:?} ", response.task_id);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_credentials(app_data_root: &AppDataRoot) -> AnyhowResult<SoraCredentials> {
|
||||
let cookie_file = app_data_root.get_sora_cookie_file_path();
|
||||
|
||||
let bearer = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/bearer.txt")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let cookie= read_to_string(cookie_file)?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let sentinel = read_to_string("/Users/bt/dev/storyteller/storyteller-rust/test_data/temp/sentinel.txt")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(SoraCredentials {
|
||||
bearer_token: bearer,
|
||||
cookie: cookie,
|
||||
sentinel: Some(sentinel),
|
||||
})
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
pub mod flip_image;
|
||||
pub mod image_generation_command;
|
||||
pub mod open_login_command;
|
||||
@@ -1,72 +0,0 @@
|
||||
use std::time::Duration;
|
||||
use log::{error, info};
|
||||
use once_cell::sync::Lazy;
|
||||
use tauri::{AppHandle, Manager, Url, WebviewUrl, WebviewWindowBuilder, WindowBuilder};
|
||||
use tokio::time::sleep_until;
|
||||
use errors::AnyhowResult;
|
||||
use crate::threads::login_thread::{LOGIN_WINDOW_NAME, SORA_LOGIN_URL};
|
||||
use crate::utils::clear_all_webview_cookies::clear_all_webview_cookies;
|
||||
|
||||
pub const START_URL_STR: &str = "https://storyteller.ai/";
|
||||
|
||||
pub static START_URL: Lazy<Url> = Lazy::new(|| {
|
||||
Url::parse(START_URL_STR).expect("URL should parse")
|
||||
});
|
||||
|
||||
pub const SORA_HOMEPAGE_URL_STR: &str = "https://sora.com/";
|
||||
|
||||
pub static SORA_HOMEPAGE_URL : Lazy<Url> = Lazy::new(|| {
|
||||
Url::parse(SORA_HOMEPAGE_URL_STR).expect("URL should parse")
|
||||
});
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn open_login_command(
|
||||
app: AppHandle,
|
||||
) -> Result<String, String> {
|
||||
info!("open_login_command called");
|
||||
|
||||
do_open_login(&app)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Error opening login: {:?}", err);
|
||||
format!("Error opening login: {:?}", err)
|
||||
})?;
|
||||
|
||||
Ok("result".to_string())
|
||||
}
|
||||
|
||||
async fn do_open_login(app: &AppHandle) -> AnyhowResult<()> {
|
||||
info!("Building login window...");
|
||||
|
||||
let url = WebviewUrl::External(START_URL.clone());
|
||||
|
||||
let window = WebviewWindowBuilder::new(app, LOGIN_WINDOW_NAME, url)
|
||||
//.user_agent(openai_sora_client::credentials::USER_AGENT)
|
||||
.always_on_top(true)
|
||||
.title("Login to OpenAI")
|
||||
.center()
|
||||
.resizable(true)
|
||||
.visible(true)
|
||||
.closable(true)
|
||||
.focused(true)
|
||||
.devtools(true)
|
||||
.build()?;
|
||||
|
||||
for (name, webview) in window.webviews() {
|
||||
if name != LOGIN_WINDOW_NAME {
|
||||
continue;
|
||||
}
|
||||
|
||||
clear_all_webview_cookies(&webview)?;
|
||||
|
||||
// NB: We're starting to get Cloudflare protection screens. Let's try to avoid.
|
||||
webview.navigate(SORA_HOMEPAGE_URL.clone())?;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
webview.navigate(SORA_LOGIN_URL.clone())?;
|
||||
break;
|
||||
}
|
||||
|
||||
info!("Done.");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NotificationModelType {
|
||||
ImageSegmentation,
|
||||
TextEncoder,
|
||||
TextTokenizer,
|
||||
Unet,
|
||||
Vae,
|
||||
Json,
|
||||
ClipEncoder,
|
||||
FluxModel,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NotificationEvent<'a> {
|
||||
ModelDownloadStarted {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelDownloadProgress {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
currently_downloaded_bytes: usize,
|
||||
total_file_bytes: usize,
|
||||
},
|
||||
ModelDownloadComplete {
|
||||
model_name: &'a str,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelLoading {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelInference {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
ModelInferenceComplete {
|
||||
model_name: String,
|
||||
model_type: NotificationModelType,
|
||||
},
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
pub mod commands;
|
||||
pub mod events;
|
||||
pub mod state;
|
||||
pub mod threads;
|
||||
pub mod transfer;
|
||||
pub mod utils;
|
||||
|
||||
use crate::commands::flip_image::flip_image;
|
||||
use crate::commands::image_generation_command::image_generation_command;
|
||||
use crate::commands::open_login_command::open_login_command;
|
||||
use crate::state::app_config::AppConfig;
|
||||
use crate::threads::login_thread::login_thread;
|
||||
|
||||
use tauri_plugin_log::Target;
|
||||
use tauri_plugin_log::TargetKind;
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
// NB: Tauri wants to install the logger itself, so we can't rely on the logger crate
|
||||
// until the tauri runtime begins.
|
||||
println!("Loading model config...");
|
||||
|
||||
let config = AppConfig::init()
|
||||
.expect("config should load");
|
||||
|
||||
let app_data_root = config.app_data_root.clone();
|
||||
let app_data_root2 = config.app_data_root.clone();
|
||||
|
||||
println!("Initializing backend runtime...");
|
||||
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_log::Builder::new()
|
||||
.level(log::LevelFilter::Info)
|
||||
.targets(vec![
|
||||
Target::new(TargetKind::Stdout),
|
||||
Target::new(TargetKind::LogDir { file_name: Some(app_data_root.log_file_name_str().to_string()) }),
|
||||
])
|
||||
.build())
|
||||
.setup(|app| {
|
||||
// TODO(bt): This is broken on windows
|
||||
// log_environment_details();
|
||||
|
||||
//if cfg!(debug_assertions) {
|
||||
// app.handle().plugin(
|
||||
// tauri_plugin_log::Builder::default()
|
||||
// .level(log::LevelFilter::Info)
|
||||
// .build(),
|
||||
// )?;
|
||||
//}
|
||||
let app = app.handle().clone();
|
||||
|
||||
tauri::async_runtime::spawn(login_thread(app, app_data_root2));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.manage(config)
|
||||
.manage(app_data_root)
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
flip_image,
|
||||
image_generation_command,
|
||||
open_login_command,
|
||||
])
|
||||
.run(tauri::generate_context!("tauri.conf.json"))
|
||||
.expect("error while running tauri application");
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
fn main() {
|
||||
app_lib::run();
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use crate::state::os_platform::OsPlatform;
|
||||
use crate::state::yaml_config::YamlConfig;
|
||||
|
||||
const DEFAULT_SD_IMAGE_WIDTH: usize = 1024;
|
||||
const DEFAULT_SD_IMAGE_HEIGHT: usize = 1024;
|
||||
|
||||
|
||||
/// A central place to configure app-wide details.
|
||||
pub struct AppConfig {
|
||||
pub image_height: usize,
|
||||
pub image_width: usize,
|
||||
pub scheduler_steps: usize,
|
||||
pub scheduler_samples: usize,
|
||||
|
||||
pub seed: Option<u64>,
|
||||
|
||||
pub cfg_scale: Option<f64>,
|
||||
|
||||
/// Root location for application data.
|
||||
pub app_data_root: AppDataRoot,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
|
||||
pub fn init() -> anyhow::Result<Self> {
|
||||
let yaml_configs = YamlConfig::load_from_config_file_recursive();
|
||||
|
||||
println!("Configs: {:?}", yaml_configs);
|
||||
|
||||
let os_platform = OsPlatform::get();
|
||||
|
||||
let default_data_root = match os_platform {
|
||||
OsPlatform::Linux => yaml_configs.linux_default_data_path.as_deref(),
|
||||
OsPlatform::MacOs => yaml_configs.mac_default_data_path.as_deref(),
|
||||
OsPlatform::Windows => yaml_configs.windows_default_data_path.as_deref(),
|
||||
};
|
||||
|
||||
println!("Possible user-defined data root: {:?}", default_data_root);
|
||||
|
||||
let app_data_root = match default_data_root {
|
||||
None => AppDataRoot::create_default()?,
|
||||
Some(path) => AppDataRoot::create_existing(path)?,
|
||||
};
|
||||
|
||||
println!("App data root: {:?}", app_data_root.path());
|
||||
|
||||
Ok(Self {
|
||||
image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT),
|
||||
image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH),
|
||||
scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(4),
|
||||
scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15),
|
||||
seed: yaml_configs.seed,
|
||||
cfg_scale: yaml_configs.cfg_scale,
|
||||
app_data_root,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
use crate::state::expanduser::expanduser;
|
||||
use crate::state::os_platform::OsPlatform;
|
||||
use anyhow::anyhow;
|
||||
use directories::UserDirs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use chrono::{DateTime, Local};
|
||||
use tempdir::TempDir;
|
||||
use tempfile::{Builder, NamedTempFile};
|
||||
|
||||
const DEFAULT_DATA_DIR : &str = "artcraft";
|
||||
const ASSETS_SUBDIRECTORY : &str = "assets";
|
||||
const WEIGHTS_SUBDIRECTORY : &str = "weights";
|
||||
|
||||
const TEMPORARY_SUBDIRECTORY : &str = "temp";
|
||||
|
||||
/// Note: Tauri appends ".log" to the end of the filename.
|
||||
const LOG_FILE_NAME : &str = "application_debug";
|
||||
|
||||
/// The path to the application data directory, which includes "asset" and "weights" data.
|
||||
#[derive(Clone)]
|
||||
pub struct AppDataRoot {
|
||||
path: PathBuf,
|
||||
assets_dir: AppAssetsDir,
|
||||
weights_dir: AppWeightsDir,
|
||||
temp_dir: TemporaryDir,
|
||||
log_file_name: PathBuf,
|
||||
log_file_name_string: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppAssetsDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppWeightsDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TemporaryDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl AppDataRoot {
|
||||
pub fn create_default() -> anyhow::Result<Self> {
|
||||
let directory = get_default_data_dir()?;
|
||||
Self::create_existing(directory)
|
||||
}
|
||||
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
|
||||
match OsPlatform::get() {
|
||||
OsPlatform::Linux | OsPlatform::MacOs => {
|
||||
if let Some(d) = dir.as_os_str().to_str() {
|
||||
dir = expanduser(d)?;
|
||||
}
|
||||
},
|
||||
OsPlatform::Windows => {}
|
||||
}
|
||||
|
||||
if !dir.is_dir() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
}
|
||||
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
|
||||
let assets_dir = AppAssetsDir::create_existing(dir.join(ASSETS_SUBDIRECTORY))?;
|
||||
let weights_dir = AppWeightsDir::create_existing(dir.join(WEIGHTS_SUBDIRECTORY))?;
|
||||
let temp_dir = TemporaryDir::create_existing(dir.join(TEMPORARY_SUBDIRECTORY))?;
|
||||
let log_file_name = dir.join(LOG_FILE_NAME);
|
||||
let log_file_name_string = log_file_name
|
||||
.to_str()
|
||||
.ok_or(anyhow!("couldn't convert log path to str"))?
|
||||
.to_string();
|
||||
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
assets_dir,
|
||||
weights_dir,
|
||||
temp_dir,
|
||||
log_file_name,
|
||||
log_file_name_string,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn assets_dir(&self) -> &AppAssetsDir {
|
||||
&self.assets_dir
|
||||
}
|
||||
|
||||
pub fn weights_dir(&self) -> &AppWeightsDir {
|
||||
&self.weights_dir
|
||||
}
|
||||
|
||||
pub fn temp_dir(&self) -> &TemporaryDir {
|
||||
&self.temp_dir
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn log_file_name(&self) -> &Path {
|
||||
&self.log_file_name
|
||||
}
|
||||
|
||||
pub fn log_file_name_str(&self) -> &str {
|
||||
&self.log_file_name_string
|
||||
}
|
||||
|
||||
pub fn get_sora_cookie_file_path(&self) -> PathBuf {
|
||||
self.path.join("sora_cookies.txt")
|
||||
}
|
||||
}
|
||||
|
||||
impl AppAssetsDir {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
if !dir.exists() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir(&dir)?;
|
||||
}
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Return the current date asset directory
|
||||
/// If it doesn't exist, create it.
|
||||
pub fn make_or_get_current_date_dir(&self) -> anyhow::Result<PathBuf> {
|
||||
let format: DateTime<Local> = Local::now();
|
||||
let date_directory_name = format.format("%Y-%d-%m").to_string();
|
||||
let full_path = self.path.join(date_directory_name);
|
||||
if !full_path.exists() {
|
||||
std::fs::create_dir(&full_path)?;
|
||||
}
|
||||
Ok(full_path)
|
||||
}
|
||||
}
|
||||
|
||||
impl AppWeightsDir {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
if !dir.exists() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir(&dir)?;
|
||||
}
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl TemporaryDir {
|
||||
pub fn create_existing<P: AsRef<Path>>(dir: P) -> anyhow::Result<Self> {
|
||||
let mut dir = dir.as_ref().to_path_buf();
|
||||
match dir.canonicalize() {
|
||||
Ok(d) => dir = d,
|
||||
Err(err) => {
|
||||
println!("Error canonicalizing {:?}: {}", dir, err);
|
||||
}
|
||||
}
|
||||
if !dir.exists() {
|
||||
println!("Creating directory {:?}", dir);
|
||||
std::fs::create_dir(&dir)?;
|
||||
}
|
||||
Ok(Self {
|
||||
path: dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_prefix(&self, prefix: &str) -> anyhow::Result<TempDir> {
|
||||
let tempdir = TempDir::new_in(&self.path, prefix)?;
|
||||
Ok(tempdir)
|
||||
}
|
||||
|
||||
pub fn new_named_temp_file(&self) -> anyhow::Result<NamedTempFile> {
|
||||
let tempfile = Builder::new()
|
||||
.prefix("temp_")
|
||||
.suffix(".bin")
|
||||
.tempfile_in(&self.path)?;
|
||||
Ok(tempfile)
|
||||
}
|
||||
}
|
||||
|
||||
// eg. /home/bob/artcraft, /Users/bob/artcraft, or C:\Users\Alice\artcraft
|
||||
fn get_default_data_dir() -> anyhow::Result<PathBuf> {
|
||||
Ok(UserDirs::new()
|
||||
.ok_or_else(|| anyhow!("could not determine user home directory"))?
|
||||
.home_dir()
|
||||
.join(DEFAULT_DATA_DIR))
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
use once_cell::sync::Lazy;
|
||||
use std::io;
|
||||
use std::path::{PathBuf, MAIN_SEPARATOR};
|
||||
|
||||
static PREFIX: Lazy<String> = Lazy::new(|| format!("~{}", MAIN_SEPARATOR));
|
||||
|
||||
/// The "expanduser" crate doesn't compile on Windows, so we replace its functionality slightly
|
||||
pub fn expanduser<P: AsRef<str>>(path: P) -> io::Result<PathBuf> {
|
||||
|
||||
Ok(match path.as_ref() {
|
||||
// matches an exact "~"
|
||||
s if s == "~" => {
|
||||
home_dir()?
|
||||
},
|
||||
// matches paths that start with `~/`
|
||||
s if s.starts_with(&*PREFIX) => {
|
||||
let home = home_dir()?;
|
||||
home.join(&s[2..])
|
||||
},
|
||||
// // matches paths that start with `~` but not `~/`, might be a `~username/` path
|
||||
// s if s.starts_with("~") => {
|
||||
// let mut parts = s[1..].splitn(2, MAIN_SEPARATOR);
|
||||
// let user = parts.next()
|
||||
// .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "malformed path"))?;
|
||||
// let user = Passwd::from_name(&user)
|
||||
// .map_err(|_| io::Error::new(io::ErrorKind::Other, "error searching for user"))?
|
||||
// .ok_or_else(|| io::Error::new(io::ErrorKind::Other, format!("user '{}', does not exist", &user)))?;
|
||||
// if let Some(ref path) = parts.next() {
|
||||
// PathBuf::from(user.dir).join(&path)
|
||||
// } else {
|
||||
// PathBuf::from(user.dir)
|
||||
// }
|
||||
// },
|
||||
// nothing to expand, just make a PathBuf
|
||||
s => PathBuf::from(s)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn home_dir() -> io::Result<PathBuf> {
|
||||
dirs::home_dir()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "no home directory is set"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::state::expanduser::expanduser;
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn test_success() {
|
||||
let old_home = env::var("HOME").expect("no home dir set");
|
||||
let new_home = "/home/foo";
|
||||
env::set_var("HOME", new_home);
|
||||
let path = expanduser("~/path/to/directory");
|
||||
env::set_var("HOME", old_home);
|
||||
assert_eq!(path.expect("io error"), PathBuf::from("/home/foo/path/to/directory"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn test_only_tilde() {
|
||||
let old_home = env::var("HOME").expect("no home dir set");
|
||||
let new_home = "/home/foo";
|
||||
env::set_var("HOME", new_home);
|
||||
let pathstr = "~";
|
||||
let path = expanduser(pathstr);
|
||||
env::set_var("HOME", old_home);
|
||||
assert_eq!(path.expect("io error"), PathBuf::from("/home/foo"));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use anyhow::anyhow;
|
||||
use log::{error, info};
|
||||
use once_cell::sync::Lazy;
|
||||
use reqwest::Url;
|
||||
use tauri::{AppHandle, Manager, Webview};
|
||||
use errors::AnyhowResult;
|
||||
use crate::state::app_dir::AppDataRoot;
|
||||
use crate::utils::sora_webview_cookies::get_all_sora_cookies_as_string;
|
||||
|
||||
pub const LOGIN_WINDOW_NAME: &str = "login_window";
|
||||
|
||||
pub const SORA_LOGIN_URL_STR: &str = "https://sora.com/auth/login?callback_path=/";
|
||||
|
||||
pub static SORA_LOGIN_URL: Lazy<Url> = Lazy::new(|| {
|
||||
Url::parse(SORA_LOGIN_URL_STR).expect("URL should parse")
|
||||
});
|
||||
|
||||
pub const SORA_ROOT_URL_STR: &str = "https://sora.com/";
|
||||
|
||||
pub static SORA_ROOT_URL: Lazy<Url> = Lazy::new(|| {
|
||||
Url::parse(SORA_ROOT_URL_STR).expect("URL should parse")
|
||||
});
|
||||
|
||||
pub async fn login_thread(app: AppHandle, app_data_root: AppDataRoot) -> ! {
|
||||
loop {
|
||||
for (window_name, webview) in app.webviews() {
|
||||
if window_name == LOGIN_WINDOW_NAME {
|
||||
let result = check_login_window(&webview, &app_data_root).await;
|
||||
if let Err(err) = result {
|
||||
error!("Error checking login window: {:?}", err);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2_000)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_login_window(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
|
||||
clear_browsing_data_on_test_domain(webview)?;
|
||||
//keep_on_task(webview)?;
|
||||
extract_cookies_to_file(webview, app_data_root)?;
|
||||
// TODO: Other maintenance tasks ...
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn keep_on_task(webview: &Webview) -> AnyhowResult<()> {
|
||||
let url = webview.url()?;
|
||||
let hostname= url.host()
|
||||
.ok_or(anyhow!("no host in url"))?
|
||||
.to_string();
|
||||
match hostname.as_str() {
|
||||
"auth.openai.com" => {},
|
||||
"openai.com" => {},
|
||||
"sora.com" => {},
|
||||
// Third party SSO
|
||||
"accounts.google.com" => {},
|
||||
"accounts.youtube.com" => {},
|
||||
"login.live.com" => {},
|
||||
"appleid.apple.com" => {},
|
||||
_ => {
|
||||
info!("Non login hostname: {}", hostname);
|
||||
webview.navigate(SORA_LOGIN_URL.clone())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// This is just so we have a way to clear browsing data.
|
||||
fn clear_browsing_data_on_test_domain(webview: &Webview) -> AnyhowResult<()> {
|
||||
let url = webview.url()?;
|
||||
let hostname= url.host()
|
||||
.ok_or(anyhow!("no host in url"))?
|
||||
.to_string();
|
||||
match hostname.as_str() {
|
||||
"storyteller.ai" => {
|
||||
info!("Clearing all browsing data...");
|
||||
webview.clear_all_browsing_data()?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// TODO(bt,2025-04-07): Heuristic to detect when logged in. Only write when logged in.
|
||||
fn extract_cookies_to_file(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
|
||||
let cookies = get_all_sora_cookies_as_string(webview)?;
|
||||
|
||||
let cookie_file = app_data_root.get_sora_cookie_file_path();
|
||||
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(cookie_file)?;
|
||||
|
||||
file.write_all(cookies.as_bytes())?;
|
||||
file.flush()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
pub mod login_thread;
|
||||
@@ -1,241 +0,0 @@
|
||||
//! Implemented from https://github.com/huggingface/hf_transfer/blob/main/src/lib.rs
|
||||
|
||||
use crate::transfer::error::Error;
|
||||
use crate::transfer::util::{exponential_backoff, BASE_WAIT_TIME, MAX_WAIT_TIME};
|
||||
use futures::stream::FuturesUnordered;
|
||||
use futures::StreamExt;
|
||||
use reqwest::header::{
|
||||
HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_RANGE,
|
||||
RANGE,
|
||||
};
|
||||
use reqwest::Url;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::fs::remove_file;
|
||||
use std::io::SeekFrom;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::time::Duration;
|
||||
use log::info;
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::fs::OpenOptions;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::io::AsyncSeekExt;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::time::sleep;
|
||||
|
||||
pub struct ProgressUpdate {
|
||||
pub complete: usize,
|
||||
pub total_length: usize,
|
||||
}
|
||||
|
||||
/// max_files: Number of open file handles, which determines the maximum number of parallel downloads
|
||||
/// parallel_failures: Number of maximum failures of different chunks in parallel (cannot exceed max_files)
|
||||
/// max_retries: Number of maximum attempts per chunk. (Retries are exponentially backed off + jitter)
|
||||
///
|
||||
/// The number of threads can be tuned by the environment variable `TOKIO_WORKER_THREADS` as documented in
|
||||
/// https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.worker_threads
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn download_async<P: AsRef<Path>>(
|
||||
url: String,
|
||||
filename: NamedTempFile,
|
||||
final_filename: P,
|
||||
max_files: usize,
|
||||
chunk_size: usize,
|
||||
parallel_failures: usize,
|
||||
max_retries: usize,
|
||||
input_headers: Option<HashMap<String, String>>,
|
||||
maybe_progress_sender: Option<Sender<ProgressUpdate>>,
|
||||
) -> Result<(), Error> {
|
||||
info!(">>> Downloading: {}", filename.as_ref().display());
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
// https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526
|
||||
.http2_keep_alive_timeout(Duration::from_secs(15))
|
||||
.build()?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
let mut auth_token = None;
|
||||
if let Some(input_headers) = input_headers {
|
||||
headers.reserve(input_headers.len());
|
||||
for (k, v) in input_headers {
|
||||
let name: HeaderName = k
|
||||
.try_into()
|
||||
.map_err(|err| Error::new_err(format!("Invalid header: {err}")))?;
|
||||
let value: HeaderValue = AsRef::<str>::as_ref(&v)
|
||||
.try_into()
|
||||
.map_err(|err| Error::new_err(format!("Invalid header value: {err}")))?;
|
||||
if name == AUTHORIZATION {
|
||||
auth_token = Some(value);
|
||||
} else {
|
||||
headers.insert(name, value);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let response = if let Some(token) = auth_token.as_ref() {
|
||||
client.get(&url).header(AUTHORIZATION, token)
|
||||
} else {
|
||||
client.get(&url)
|
||||
}
|
||||
.headers(headers.clone())
|
||||
.header(RANGE, "bytes=0-0")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| Error::new_err(format!("Error while downloading: {err}")))?
|
||||
.error_for_status()
|
||||
.map_err(|err| Error::new_err(err.to_string()))?;
|
||||
|
||||
// Only call the final redirect URL to avoid overloading the Hub with requests and also
|
||||
// altering the download count
|
||||
let redirected_url = response.url();
|
||||
if Url::parse(&url)
|
||||
.map_err(|err| Error::new_err(format!("failed to parse url: {err}")))?
|
||||
.host()
|
||||
== redirected_url.host()
|
||||
{
|
||||
if let Some(token) = auth_token {
|
||||
headers.insert(AUTHORIZATION, token);
|
||||
}
|
||||
}
|
||||
|
||||
let content_range = response
|
||||
.headers()
|
||||
.get(CONTENT_RANGE)
|
||||
.ok_or(Error::new_err("No content length".to_string()))?
|
||||
.to_str()
|
||||
.map_err(|err| Error::new_err(format!("Error while downloading: {err}")))?;
|
||||
|
||||
let size: Vec<&str> = content_range.split('/').collect();
|
||||
// Content-Range: bytes 0-0/702517648
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
|
||||
let length: usize = size
|
||||
.last()
|
||||
.ok_or(Error::new_err(
|
||||
"Error while downloading: No size was detected".to_string(),
|
||||
))?
|
||||
.parse()
|
||||
.map_err(|err| Error::new_err(format!("Error while downloading: {err}")))?;
|
||||
|
||||
|
||||
info!("File {:?} is {:?} length", filename.as_ref(), length);
|
||||
|
||||
let mut handles = FuturesUnordered::new();
|
||||
let semaphore = Arc::new(Semaphore::new(max_files));
|
||||
let parallel_failures_semaphore = Arc::new(Semaphore::new(parallel_failures));
|
||||
let max_chunk_complete = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
for start in (0..length).step_by(chunk_size) {
|
||||
let url = redirected_url.to_string();
|
||||
let filename = filename.as_ref().to_owned();
|
||||
let client = client.clone();
|
||||
let headers = headers.clone();
|
||||
|
||||
let stop = std::cmp::min(start + chunk_size - 1, length);
|
||||
let semaphore = semaphore.clone();
|
||||
let parallel_failures_semaphore = parallel_failures_semaphore.clone();
|
||||
let maybe_progress = maybe_progress_sender.clone();
|
||||
let max_chunk_complete = max_chunk_complete.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let permit = semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|err| Error::new_err(format!("Error while downloading: {err}")))?;
|
||||
let mut chunk = download_chunk(&client, &url, &filename, start, stop, headers.clone()).await;
|
||||
let mut i = 0;
|
||||
if parallel_failures > 0 {
|
||||
while let Err(dlerr) = chunk {
|
||||
if i >= max_retries {
|
||||
return Err(Error::new_err(format!(
|
||||
"Failed after too many retries ({max_retries}): {dlerr}"
|
||||
)));
|
||||
}
|
||||
let parallel_failure_permit = parallel_failures_semaphore.clone().try_acquire_owned().map_err(|err| {
|
||||
Error::new_err(format!(
|
||||
"Failed too many failures in parallel ({parallel_failures}): {dlerr} ({err})"
|
||||
))
|
||||
})?;
|
||||
|
||||
let wait_time = exponential_backoff(BASE_WAIT_TIME, i, MAX_WAIT_TIME);
|
||||
sleep(Duration::from_millis(wait_time as u64)).await;
|
||||
|
||||
chunk = download_chunk(&client, &url, &filename, start, stop, headers.clone()).await;
|
||||
i += 1;
|
||||
drop(parallel_failure_permit);
|
||||
|
||||
if let Some(sender) = maybe_progress.as_ref() {
|
||||
// NB: Calling fetch_max twice. First call will return the lower value.
|
||||
let _old = max_chunk_complete.fetch_max(stop, Ordering::SeqCst);
|
||||
let max_chunk = max_chunk_complete.fetch_max(stop, Ordering::SeqCst);
|
||||
let progress_update = ProgressUpdate {
|
||||
complete: max_chunk,
|
||||
total_length: length,
|
||||
};
|
||||
if let Err(_err) = sender.send(progress_update) {
|
||||
// Fail silently, not essential to issue progress updates
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(permit);
|
||||
chunk.map_err(|e| Error::new_err(format!("Downloading error {e}"))).and(Ok(stop - start))
|
||||
}));
|
||||
}
|
||||
|
||||
// Output the chained result
|
||||
while let Some(result) = handles.next().await {
|
||||
match result {
|
||||
Ok(Ok(size)) => {
|
||||
//if let Some(ref callback) = callback {
|
||||
// callback.call((size,), None)?;
|
||||
//}
|
||||
}
|
||||
Ok(Err(py_err)) => {
|
||||
return Err(py_err);
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(Error::new_err(format!(
|
||||
"Error while downloading: {err}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let file = filename.persist(final_filename)
|
||||
.map_err(|e| Error::new_err(format!("Could not rename to final filename: {:?}", e)))?;
|
||||
|
||||
file.sync_all()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_chunk<P: AsRef<Path>>(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
filename: P,
|
||||
start: usize,
|
||||
stop: usize,
|
||||
headers: HeaderMap,
|
||||
) -> Result<(), Error> {
|
||||
// Process each socket concurrently.
|
||||
let range = format!("bytes={start}-{stop}");
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.truncate(false)
|
||||
.create(true)
|
||||
.open(filename)
|
||||
.await?;
|
||||
file.seek(SeekFrom::Start(start as u64)).await?;
|
||||
let response = client
|
||||
.get(url)
|
||||
.headers(headers)
|
||||
.header(RANGE, range)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
let content = response.bytes().await?;
|
||||
file.write_all(&content).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
//! Implemented from https://github.com/huggingface/hf_transfer/blob/main/src/lib.rs
|
||||
|
||||
use std::fmt::Display;
|
||||
use reqwest::header::ToStrError;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
Io(std::io::Error),
|
||||
Request(reqwest::Error),
|
||||
ToStrError(ToStrError),
|
||||
Misc(String),
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(value: std::io::Error) -> Self {
|
||||
Self::Io(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<reqwest::Error> for Error {
|
||||
fn from(value: reqwest::Error) -> Self {
|
||||
Self::Request(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ToStrError> for Error {
|
||||
fn from(value: ToStrError) -> Self {
|
||||
Self::ToStrError(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Io(io) => write!(f, "Io: {io}"),
|
||||
Self::Request(req) => write!(f, "Request: {req}"),
|
||||
Self::ToStrError(req) => write!(f, "Response non ascii: {req}"),
|
||||
Self::Misc(err) => write!(f, "Misc error: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl Error {
|
||||
pub fn new_err(reason: String) -> Self {
|
||||
Self::Misc(reason)
|
||||
}
|
||||
}
|
||||