server/artcraft
1
0
Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-10-20 17:02:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

add fal hunyuan image-to-3d (#614)

Browse Source
This commit is contained in:
Brandon Thomas
2025-05-18 15:37:55 -04:00
committed by GitHub
parent cb64f45b48
commit 76cd00cc13
41 changed files with 823 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+18 -14
View File
@@ -8806,7 +8806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_macros 0.11.2",
"phf_shared 0.11.2",
"phf_shared 0.11.3",
]
[[package]]
@@ -8855,7 +8855,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
dependencies = [
"phf_shared 0.11.2",
"phf_shared 0.11.3",
"rand 0.8.5",
]
@@ -8880,7 +8880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
dependencies = [
"phf_generator 0.11.2",
"phf_shared 0.11.2",
"phf_shared 0.11.3",
"proc-macro2",
"quote",
"syn 2.0.87",
@@ -8892,7 +8892,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
dependencies = [
"siphasher",
"siphasher 0.3.11",
]
[[package]]
@@ -8901,16 +8901,16 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
"siphasher 0.3.11",
]
[[package]]
name = "phf_shared"
version = "0.11.2"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
"siphasher 1.0.1",
]
[[package]]
@@ -11282,6 +11282,12 @@ version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "siphasher"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "slab"
version = "0.4.9"
@@ -11840,6 +11846,7 @@ dependencies = [
"errors",
"idempotency",
"log",
"mimetypes",
"reqwest 0.12.15",
"serde",
"serde_derive",
@@ -13899,12 +13906,9 @@ dependencies = [
[[package]]
name = "unicase"
version = "2.7.0"
version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check",
]
checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
[[package]]
name = "unicode-bidi"
@@ -15321,7 +15325,7 @@ dependencies = [
"openssl-sys",
"parking_lot 0.12.2",
"phf 0.11.2",
"phf_shared 0.11.2",
"phf_shared 0.11.3",
"proc-macro2",
"prost",
"pyo3",
crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs
+1 -1
View File
@@ -36,7 +36,7 @@ impl TemporaryDir {
Ok(tempfile)
}
pub fn new_named_temp_file_with_extension(&self, extension: &str) -> anyhow::Result<NamedTempFile> {
pub fn new_named_temp_file_with_extension(&self, extension: &str) -> Result<NamedTempFile, std::io::Error> {
let extension = if extension.starts_with(".") {
extension.to_string()
} else {
crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs
+29
View File
@@ -0,0 +1,29 @@
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use errors::AnyhowResult;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tempfile::NamedTempFile;
use tokens::tokens::media_files::MediaFileToken;
// TODO: Better more concrete error handling
pub async fn download_media_file_to_temp_dir(app_data_root: &AppDataRoot, token: &MediaFileToken) -> AnyhowResult<NamedTempFile> {
let response = get_media_file(&ApiHost::Storyteller, token).await?;
let media_file_url = &response.media_file.media_links.cdn_url;
let extension_with_dot = get_url_file_extension(media_file_url)
.map(|ext| format!(".{}", ext))
.unwrap_or_else(|| ".png".to_string()); // TODO: Better default extension passed by caller as heuristic.
//let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot);
//let filename = app_data_root.downloads_dir().path().join(&filename);
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?;
simple_http_download_to_tempfile(&media_file_url, &mut file).await?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
crates/desktop/artcraft/src/core/utils/download_url_to_temp_dir.rs
+20
View File
@@ -0,0 +1,20 @@
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use errors::AnyhowResult;
use std::io::Write;
use tempfile::NamedTempFile;
use url::Url;
pub async fn download_url_to_temp_dir(url: &str, app_data_root: &AppDataRoot) -> AnyhowResult<NamedTempFile> {
let url = Url::parse(&url)?;
let response = reqwest::get(url.clone()).await?;
let response_bytes = response.bytes().await?;
let ext = url.path().split(".").last().unwrap_or("png");
let mut temp_file = app_data_root.temp_dir().new_named_temp_file_with_extension(ext)?;
temp_file.write_all(&response_bytes)?;
Ok(temp_file)
}
crates/desktop/artcraft/src/core/utils/mod.rs
+3
View File
@@ -1,9 +1,12 @@
pub mod best_window_size_heuristic;
pub mod clear_all_webview_cookies;
pub mod download_media_file_to_temp_dir;
pub mod download_url_to_temp_dir;
pub mod get_url_file_extension;
pub mod image;
pub mod large_file_transfer;
pub mod object_store;
pub mod save_base64_image_to_temp_dir;
pub mod simple_http_download;
pub mod simple_http_download_to_tempfile;
pub mod webview_unsafe;
crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs
+25
View File
@@ -0,0 +1,25 @@
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use errors::AnyhowResult;
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use std::io::Write;
use tempfile::NamedTempFile;
// TODO: Retain error types (io/decode)
pub async fn save_base64_image_to_temp_dir(app_data_root: &AppDataRoot, base64_image: String) -> AnyhowResult<NamedTempFile> {
let bytes = BASE64_STANDARD.decode(base64_image)?;
let extension = MimetypeInfo::get_for_bytes(&bytes)
.map(|info| info.file_extension())
.flatten()
.map(|ext| ext.extension_with_period().to_string())
.unwrap_or_else(|| ".png".to_string());
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?;
file.write_all(bytes.as_ref())?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
crates/desktop/artcraft/src/lib.rs
+2
View File
@@ -15,6 +15,7 @@ use crate::core::threads::discord_presence_thread::discord_presence_thread;
use crate::core::threads::main_window_thread::main_window_thread::main_window_thread;
use crate::core::utils::webview_unsafe::webview_unsafe_for_app;
use crate::services::fal::commands::fal_background_removal_command::fal_background_removal_command;
use crate::services::fal::commands::fal_hunyuan_image_to_3d_command::fal_hunyuan_image_to_3d_command;
use crate::services::fal::commands::fal_kling_image_to_video_command::fal_kling_image_to_video_command;
use crate::services::fal::commands::get_fal_api_key_command::get_fal_api_key_command;
use crate::services::fal::commands::set_fal_api_key_command::set_fal_api_key_command;
@@ -159,6 +160,7 @@ pub fn run() {
.invoke_handler(tauri::generate_handler![
check_sora_session_command,
fal_background_removal_command,
fal_hunyuan_image_to_3d_command,
fal_kling_image_to_video_command,
flip_image,
get_app_preferences_command,
crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs
+16 -48
View File
@@ -1,18 +1,20 @@
use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker};
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
@@ -41,8 +43,8 @@ use std::io::{Cursor, Read, Write};
use std::ops::Add;
use std::path::PathBuf;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
@@ -73,7 +75,7 @@ impl SerializeMarker for FalBackgroundRemovalSuccessResponse {}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum SoraImageRemixErrorType {
pub enum BackgroundRemovalErrorType {
/// Generic server error
ServerError,
/// No Fal API key available
@@ -87,7 +89,7 @@ pub async fn fal_background_removal_command(
app_data_root: State<'_, AppDataRoot>,
fal_creds_manager: State<'_, FalCredentialManager>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
) -> CommandResult<FalBackgroundRemovalSuccessResponse, SoraImageRemixErrorType, ()> {
) -> CommandResult<FalBackgroundRemovalSuccessResponse, BackgroundRemovalErrorType, ()> {
info!("fal_background_removal_command called; image media token: {:?}", request.image_media_token);
@@ -100,7 +102,7 @@ pub async fn fal_background_removal_command(
return Err(CommandErrorResponseWrapper {
status: CommandErrorStatus::Unauthorized,
error_message: Some("You need to set a FAL api key".to_string()),
error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey),
error_type: Some(BackgroundRemovalErrorType::NeedsFalApiKey),
error_details: None,
});
}
@@ -123,7 +125,7 @@ pub async fn fal_background_removal_command(
//}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = SoraImageRemixErrorType::ServerError;
let mut error_type = BackgroundRemovalErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
//match (err) {
@@ -205,10 +207,10 @@ pub async fn remove_background(
let mut temp_download;
if let Some(media_token) = request.image_media_token {
temp_download = download_media_file(&app_data_root, &media_token).await?;
temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?;
} else if let Some(base64_bytes) = request.base64_image {
temp_download = persist_base64(&app_data_root, base64_bytes).await?;
temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?;
} else {
return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided")));
}
@@ -253,37 +255,3 @@ pub async fn remove_background(
Ok((response, base64_bytes))
}
async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result<NamedTempFile, InnerError> {
let response = get_media_file(&ApiHost::Storyteller, token).await?;
let media_file_url = &response.media_file.media_links.cdn_url;
let extension_with_dot = get_url_file_extension(media_file_url)
.map(|ext| format!(".{}", ext))
.unwrap_or_else(|| ".png".to_string());
//let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot);
//let filename = app_data_root.downloads_dir().path().join(&filename);
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?;
simple_http_download_to_tempfile(&media_file_url, &mut file).await?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result<NamedTempFile, InnerError> {
let bytes = BASE64_STANDARD.decode(base64_image)?;
let extension = MimetypeInfo::get_for_bytes(&bytes)
.map(|info| info.file_extension())
.flatten()
.map(|ext| ext.extension_with_period().to_string())
.unwrap_or_else(|| ".png".to_string());
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?;
file.write_all(bytes.as_ref())?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs
+220
View File
@@ -0,0 +1,220 @@
use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker};
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use fal_client::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
use log::{debug, error, info, warn};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::creds::credential_migration::CredentialMigrationRef;
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
use openai_sora_client::sora_error::SoraError;
use reqwest::Url;
use serde_derive::{Deserialize, Serialize};
use std::fs::read_to_string;
use std::io::{Cursor, Read, Write};
use std::ops::Add;
use std::path::PathBuf;
use std::time::Duration;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
use tempfile::NamedTempFile;
use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize)]
pub struct FalHunyuanImageTo3dRequest {
/// Image media file; the image to remove the background from.
pub image_media_token: Option<MediaFileToken>,
/// Base64-encoded image
pub base64_image: Option<String>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum HunyuanErrorType {
/// Generic server error
ServerError,
/// No Fal API key available
NeedsFalApiKey,
}
#[tauri::command]
pub async fn fal_hunyuan_image_to_3d_command(
request: FalHunyuanImageTo3dRequest,
app_data_root: State<'_, AppDataRoot>,
fal_creds_manager: State<'_, FalCredentialManager>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
fal_task_queue: State<'_, FalTaskQueue>,
) -> CommandResult<(), HunyuanErrorType, ()> {
info!("fal_hunyuan_image_to_3d_command called; image media token: {:?}", request.image_media_token);
let has_credentials = fal_creds_manager
.has_apparent_api_token()
.unwrap_or(true); // NB: Failures would be lock issues
if !has_credentials {
warn!("No API key found");
return Err(CommandErrorResponseWrapper {
status: CommandErrorStatus::Unauthorized,
error_message: Some("You need to set a FAL api key".to_string()),
error_type: Some(HunyuanErrorType::NeedsFalApiKey),
error_details: None,
});
}
let result = image_to_3d(
request,
&app_data_root,
&fal_creds_manager,
&storyteller_creds_manager,
&fal_task_queue,
).await;
if let Err(err) = result {
error!("error: {:?}", err);
//let event = SoraImageEnqueueFailureEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = HunyuanErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
//match (err) {
// _ => {},
//}
return Err(CommandErrorResponseWrapper {
status,
error_message: Some(error_message.to_string()),
error_type: Some(error_type),
error_details: None,
})
}
//let event = SoraImageEnqueueSuccessEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
Ok(().into())
}
#[derive(Debug)]
enum InnerError {
FalError(FalErrorPlus),
AnyhowError(AnyhowError),
StorytellerApiError(ApiError),
DecodeError(DecodeError),
IoError(std::io::Error),
}
impl From<AnyhowError> for InnerError {
fn from(value: AnyhowError) -> Self {
Self::AnyhowError(value)
}
}
impl From<FalErrorPlus> for InnerError {
fn from(value: FalErrorPlus) -> Self {
Self::FalError(value)
}
}
impl From<ApiError> for InnerError {
fn from(value: ApiError) -> Self {
Self::StorytellerApiError(value)
}
}
impl From<DecodeError> for InnerError {
fn from(value: DecodeError) -> Self {
Self::DecodeError(value)
}
}
impl From<std::io::Error> for InnerError {
fn from(value: std::io::Error) -> Self {
Self::IoError(value)
}
}
pub async fn image_to_3d(
request: FalHunyuanImageTo3dRequest,
app_data_root: &AppDataRoot,
fal_creds_manager: &FalCredentialManager,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_task_queue: &FalTaskQueue,
) -> Result<(), InnerError> {
let api_key = fal_creds_manager.get_key_required()?;
let creds = storyteller_creds_manager.get_credentials_required()?;
let mut temp_download;
if let Some(media_token) = request.image_media_token {
temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?;
} else if let Some(base64_bytes) = request.base64_image {
temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?;
} else {
return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided")));
}
info!("Calling FAL image to 3d ...");
let filename = temp_download.path().to_path_buf();
let enqueued = enqueue_hunyuan2_image_to_3d(Hunyuan2Args {
image_path: filename,
api_key: &api_key,
}).await?;
if let Err(err) = fal_task_queue.insert(&enqueued) {
error!("Failed to enqueue task: {:?}", err);
return Err(InnerError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err)));
}
Ok(())
}
crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs
+17 -50
View File
@@ -1,19 +1,21 @@
use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker};
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
@@ -43,8 +45,8 @@ use std::io::{Cursor, Read, Write};
use std::ops::Add;
use std::path::PathBuf;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
@@ -63,7 +65,7 @@ pub struct FalKlingImageToVideoRequest {
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum SoraImageRemixErrorType {
pub enum KlingImageToVideoErrorType {
/// Generic server error
ServerError,
/// No Fal API key available
@@ -78,7 +80,7 @@ pub async fn fal_kling_image_to_video_command(
fal_creds_manager: State<'_, FalCredentialManager>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
fal_task_queue: State<'_, FalTaskQueue>,
) -> CommandResult<(), SoraImageRemixErrorType, ()> {
) -> CommandResult<(), KlingImageToVideoErrorType, ()> {
info!("fal_kling_image_to_video_command called; image media token: {:?}", request.image_media_token);
@@ -91,7 +93,7 @@ pub async fn fal_kling_image_to_video_command(
return Err(CommandErrorResponseWrapper {
status: CommandErrorStatus::Unauthorized,
error_message: Some("You need to set a FAL api key".to_string()),
error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey),
error_type: Some(KlingImageToVideoErrorType::NeedsFalApiKey),
error_details: None,
});
}
@@ -114,7 +116,7 @@ pub async fn fal_kling_image_to_video_command(
//}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = SoraImageRemixErrorType::ServerError;
let mut error_type = KlingImageToVideoErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
//match (err) {
@@ -191,10 +193,10 @@ pub async fn image_to_video(
let mut temp_download;
if let Some(media_token) = request.image_media_token {
temp_download = download_media_file(&app_data_root, &media_token).await?;
temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?;
} else if let Some(base64_bytes) = request.base64_image {
temp_download = persist_base64(&app_data_root, base64_bytes).await?;
temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?;
} else {
return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided")));
}
@@ -216,38 +218,3 @@ pub async fn image_to_video(
Ok(())
}
async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result<NamedTempFile, InnerError> {
let response = get_media_file(&ApiHost::Storyteller, token).await?;
let media_file_url = &response.media_file.media_links.cdn_url;
let extension_with_dot = get_url_file_extension(media_file_url)
.map(|ext| format!(".{}", ext))
.unwrap_or_else(|| ".png".to_string());
//let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot);
//let filename = app_data_root.downloads_dir().path().join(&filename);
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?;
simple_http_download_to_tempfile(&media_file_url, &mut file).await?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result<NamedTempFile, InnerError> {
let bytes = BASE64_STANDARD.decode(base64_image)?;
let extension = MimetypeInfo::get_for_bytes(&bytes)
.map(|info| info.file_extension())
.flatten()
.map(|ext| ext.extension_with_period().to_string())
.unwrap_or_else(|| ".png".to_string());
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?;
file.write_all(bytes.as_ref())?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
crates/desktop/artcraft/src/services/fal/commands/mod.rs
+1
View File
@@ -1,4 +1,5 @@
pub mod fal_background_removal_command;
pub mod fal_hunyuan_image_to_3d_command;
pub mod fal_kling_image_to_video_command;
pub mod get_fal_api_key_command;
pub mod set_fal_api_key_command;
crates/desktop/artcraft/src/services/fal/threads/fal_task_polling_thread.rs
+22 -29
View File
@@ -1,32 +1,38 @@
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::services::sora::events::sora_image_generation_complete_event::SoraImageGenerationCompleteEvent;
use crate::services::sora::events::sora_image_generation_failed_event::SoraImageGenerationFailedEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::download_url_to_temp_dir::download_url_to_temp_dir;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::events::sora_image_generation_complete_event::SoraImageGenerationCompleteEvent;
use crate::services::sora::events::sora_image_generation_failed_event::SoraImageGenerationFailedEvent;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use errors::AnyhowResult;
use fal_client::export::queue::Status;
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::model::fal_request_id::FalRequestId;
use fal_client::utils::queue_status_checker::QueueStatusChecker;
use log::{error, info, warn};
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mimetypes::mimetype_info::file_extension::FileExtension;
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use openai_sora_client::requests::image_gen::image_gen_status::{Generation, TaskId, TaskStatus};
use serde_derive::Serialize;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::media_files::upload_new_engine_asset_from_file::upload_new_engine_asset_from_file;
use storyteller_client::media_files::upload_video_media_file_from_file::{upload_video_media_file_from_file, UploadVideoMediaFileSuccessResponse};
use storyteller_client::recipes::upload_media_file_from_file::upload_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter};
use tempdir::TempDir;
use tempfile::NamedTempFile;
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
use fal_client::model::fal_request_id::FalRequestId;
use storyteller_client::api_error::ApiError;
use storyteller_client::media_files::upload_video_media_file_from_file::{upload_video_media_file_from_file, UploadVideoMediaFileSuccessResponse};
pub async fn fal_task_polling_thread(
app_handle: AppHandle,
@@ -125,20 +131,21 @@ async fn polling_loop(
}
}
};
let download_file = download_generation(&url, &app_data_root).await?;
let result = upload_video_media_file_from_file(
&ApiHost::Storyteller,
Some(&storyteller_creds),
// TODO: Clean up
let download_file = download_url_to_temp_dir(&url, &app_data_root).await?;
let result = upload_media_file_from_file(
&ApiHost::Storyteller,
Some(&storyteller_creds),
download_file
).await;
match result {
Ok(success) => {
info!("Uploaded to API backend: {:?}", success.media_file_token);
//let event = SoraImageGenerationCompleteEvent {
// media_file_token: success.media_file_token,
//};
@@ -147,12 +154,12 @@ async fn polling_loop(
succeeded_tasks.push(task_id);
}
Err(err) => {
warn!("Failed to upload video media file: {:?}", err);
warn!("Failed to upload media file: {:?}", err);
continue;
}
}
}
}
// // TODO: The cursoring logic likely needs to improve.
@@ -225,17 +232,3 @@ async fn polling_loop(
}
}
async fn download_generation(url: &str, app_data_root: &AppDataRoot) -> AnyhowResult<NamedTempFile> {
let url = Url::parse(&url)?;
let response = reqwest::get(url.clone()).await?;
let response_bytes = response.bytes().await?;
let ext = url.path().split(".").last().unwrap_or("png");
let mut temp_file = app_data_root.temp_dir().new_named_temp_file_with_extension(ext)?;
temp_file.write_all(&response_bytes)?;
Ok(temp_file)
}
crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs
+6 -6
View File
@@ -1,15 +1,15 @@
use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult};
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download::simple_http_download;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use errors::{AnyhowError, AnyhowResult};
@@ -31,8 +31,8 @@ use std::fs::read_to_string;
use std::io::Cursor;
use std::ops::Add;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
crates/lib/api_clients/fal_client/src/model/fal_endpoint.rs
+6
View File
@@ -1,22 +1,28 @@
use fal::queue::Queue;
use serde::de::DeserializeOwned;
const HUNYUAN_3D_2_BASE: &str = "fal-ai/hunyuan3d/v2";
const KLING16_IMAGE_TO_VIDEO: &str = "fal-ai/kling-video/v1.6/pro/image-to-video";
#[derive(Clone, Debug)]
pub enum FalEndpoint {
Kling16ImageToVideo,
Hunyuan3d2Base,
Other(String),
}
impl FalEndpoint {
pub fn from_queue_response<R: DeserializeOwned>(queue: &Queue<R>) -> Self {
match queue.endpoint.as_str() {
HUNYUAN_3D_2_BASE => FalEndpoint::Hunyuan3d2Base,
KLING16_IMAGE_TO_VIDEO => FalEndpoint::Kling16ImageToVideo,
_ => FalEndpoint::Other(queue.endpoint.clone()),
}
}
pub fn url(&self) -> &str {
match self {
Self::Hunyuan3d2Base => HUNYUAN_3D_2_BASE,
Self::Kling16ImageToVideo => KLING16_IMAGE_TO_VIDEO,
Self::Other(url) => url,
}
crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs
+69
View File
@@ -0,0 +1,69 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::hunyuan3d::v2::v2;
use fal::endpoints::fal_ai::hunyuan3d::v2::Hunyuan3DInput;
use fal_client::file_to_base64_url::file_to_base64_url;
use futures::StreamExt;
use std::io::Write;
use std::path::Path;
pub struct Hunyuan2Args<'a, P: AsRef<Path>> {
pub image_path: P,
pub api_key: &'a FalApiKey,
}
pub async fn enqueue_hunyuan2_image_to_3d<P: AsRef<Path>>(args: Hunyuan2Args<'_, P>) -> Result<EnqueuedRequest, FalErrorPlus> {
let image_url = file_to_base64_url(args.image_path)?;
/*
TODO: Handle error messages -
FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected '<key_id>:<key_secret>'.\"}")))
FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}")))
*/
let request = Hunyuan3DInput {
input_image_url: image_url,
textured_mesh: Some(true),
guidance_scale: None,
num_inference_steps: None,
octree_resolution: None,
seed: None,
};
let result = v2(request)
.with_api_key(&args.api_key.0)
.queue()
.await?;
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
let image = test_file_path("test_data/image/juno.jpg")?;
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = Hunyuan2Args {
image_path: image,
api_key: &api_key,
};
let result = enqueue_hunyuan2_image_to_3d(args).await?;
Ok(())
}
}
crates/lib/api_clients/fal_client/src/requests/mod.rs
+1
View File
@@ -1,2 +1,3 @@
pub mod enqueue_hunyuan2_image_to_3d;
pub mod enqueue_kling_16_image_to_video;
pub mod remove_background_rembg;
crates/lib/api_clients/fal_client/src/utils/queue_status_checker.rs
+16 -8
View File
@@ -1,15 +1,13 @@
use fal::endpoints::fal_ai::kling_video::v1_6::pro::effects::I2VOutput;
use fal::FalError;
use fal::prelude::fal_ai::imageutils::sam::RemoveBackgroundOutput;
use fal::prelude::QueueResponse;
use fal::queue::{Queue, QueueStatus};
use reqwest::Client;
use serde::de::DeserializeOwned;
use url::Url;
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use crate::model::fal_endpoint::FalEndpoint;
use fal::endpoints::fal_ai::hunyuan3d::v2::ObjectOutput;
use fal::endpoints::fal_ai::kling_video::v1_6::pro::effects::I2VOutput;
use fal::prelude::QueueResponse;
use fal::queue::{Queue, QueueStatus};
use reqwest::Client;
use serde::de::DeserializeOwned;
pub struct QueueStatusChecker {
api_key: FalApiKey,
@@ -22,6 +20,11 @@ impl QueueStatusChecker {
pub async fn check_status(&self, request: &EnqueuedRequest) -> Result<QueueStatus, FalErrorPlus> {
match &request.fal_endpoint {
FalEndpoint::Hunyuan3d2Base => {
let queue = self.make_queue::<ObjectOutput>(&request);
let status = queue.status(false).await?;
Ok(status)
}
FalEndpoint::Kling16ImageToVideo => {
let queue = self.make_queue::<I2VOutput>(&request);
let status = queue.status(false).await?;
@@ -35,6 +38,11 @@ impl QueueStatusChecker {
pub async fn get_download_url(&self, request: &EnqueuedRequest) -> Result<String, FalErrorPlus> {
match &request.fal_endpoint {
FalEndpoint::Hunyuan3d2Base => {
let queue = self.make_queue::<ObjectOutput>(&request);
let status = queue.response().await?;
Ok(status.model_mesh.url)
}
FalEndpoint::Kling16ImageToVideo => {
let queue = self.make_queue::<I2VOutput>(&request);
let status = queue.response().await?;
crates/lib/api_clients/storyteller_client/Cargo.toml
+1
View File
@@ -22,6 +22,7 @@ enums_public = { workspace = true }
errors = { workspace = true }
idempotency = { workspace = true }
tokens = { workspace = true }
mimetypes = { workspace = true }
# External
anyhow = { workspace = true }
crates/lib/api_clients/storyteller_client/src/api_error.rs → crates/lib/api_clients/storyteller_client/src/error/api_error.rs
View File
crates/lib/api_clients/storyteller_client/src/error/client_error.rs
+35
View File
@@ -0,0 +1,35 @@
use std::error::Error;
use std::fmt::{Display, Formatter};
use std::io;
use std::path::PathBuf;
/// An error arising from the client, either before or after the request is sent.
#[derive(Debug)]
pub enum ClientError {
/// Could not determine the filetype for the file.
FileTypeNotKnown(PathBuf),
/// Could not handle the file type.
FileTypeNotHandled(PathBuf),
/// An error doing file I/O (on our side)
IoError(io::Error),
}
impl Error for ClientError {}
impl Display for ClientError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::FileTypeNotKnown(path) => write!(f, "Could not determine the filetype for the file: {:?}", path),
Self::FileTypeNotHandled(path) => write!(f, "Could not handle the file type for the file: {:?}", path),
Self::IoError(err) => write!(f, "IO error: {}", err),
}
}
}
impl From<io::Error> for ClientError {
fn from(error: io::Error) -> Self {
ClientError::IoError(error)
}
}
crates/lib/api_clients/storyteller_client/src/error/mod.rs
+3
View File
@@ -0,0 +1,3 @@
pub mod api_error;
pub mod client_error;
pub mod storyteller_error;
crates/lib/api_clients/storyteller_client/src/error/storyteller_error.rs
+32
View File
@@ -0,0 +1,32 @@
use crate::error::api_error::ApiError;
use crate::error::client_error::ClientError;
use std::error::Error;
#[derive(Debug)]
pub enum StorytellerError {
Api(ApiError),
Client(ClientError),
}
impl Error for StorytellerError {}
impl std::fmt::Display for StorytellerError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StorytellerError::Api(e) => write!(f, "API error: {:?}", e),
StorytellerError::Client(e) => write!(f, "Client error: {:?}", e),
}
}
}
impl From<ApiError> for StorytellerError {
fn from(error: ApiError) -> Self {
StorytellerError::Api(error)
}
}
impl From<ClientError> for StorytellerError {
fn from(error: ClientError) -> Self {
StorytellerError::Client(error)
}
}
crates/lib/api_clients/storyteller_client/src/lib.rs
+2 -1
View File
@@ -1,7 +1,8 @@
//! An API client for Storyteller / FakeYou
pub mod api_error;
pub mod credentials;
pub mod error;
pub mod media_files;
pub mod recipes;
pub mod shared_response_types;
pub mod utils;
crates/lib/api_clients/storyteller_client/src/media_files/get_media_file.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::api_error::ApiError;
use crate::error::api_error::ApiError;
use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails;
use crate::shared_response_types::media_links::MediaLinks;
use crate::shared_response_types::simple_entity_stats::SimpleEntityStats;
crates/lib/api_clients/storyteller_client/src/media_files/mod.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod get_media_file;
pub mod upload_image_media_file_from_file;
pub mod upload_new_engine_asset_from_file;
pub mod upload_video_media_file_from_file;
crates/lib/api_clients/storyteller_client/src/media_files/upload_image_media_file_from_file.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::api_error::ApiError;
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::api_error::ApiError;
use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails;
use crate::shared_response_types::media_links::MediaLinks;
use crate::shared_response_types::simple_entity_stats::SimpleEntityStats;
crates/lib/api_clients/storyteller_client/src/media_files/upload_new_engine_asset_from_file.rs
+93
View File
@@ -0,0 +1,93 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::api_error::ApiError;
use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails;
use crate::shared_response_types::media_links::MediaLinks;
use crate::shared_response_types::simple_entity_stats::SimpleEntityStats;
use crate::shared_response_types::user_details_light::UserDetailsLight;
use crate::utils::api_host::ApiHost;
use crate::utils::constants::{APPLICATION_JSON, USER_AGENT};
use crate::utils::filter_bad_response::filter_bad_response;
use crate::utils::http_get_anonymous::http_get_anonymous;
use chrono::{DateTime, Utc};
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::common::visibility::Visibility;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType;
use idempotency::uuid::generate_random_uuid;
use log::debug;
use reqwest::multipart::Form;
use reqwest::Client;
use serde_derive::{Deserialize, Serialize};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use tokens::tokens::batch_generations::BatchGenerationToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::prompts::PromptToken;
use uuid::uuid;
// TODO: The naming of these methods is ridiculous. It needs to be cleaned up.
/// Upload an image media file from a file.
pub async fn upload_new_engine_asset_from_file<P: AsRef<Path>>(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
path: P,
) -> Result<UploadImageMediaFileSuccessResponse, ApiError> {
let url = get_route(api_host);
debug!("Requesting {:?}", &url);
//let mut file = File::open(path)?;
let client = Client::builder()
.gzip(true)
.build()?;
let form = Form::new()
.text("uuid_idempotency_token", generate_random_uuid())
.text("engine_category", "object") // TODO: Which type should we use?
.file("file", path)
.await?;
let mut request_builder = client.post(url)
.header("User-Agent", USER_AGENT)
.header("Accept", APPLICATION_JSON);
if let Some(creds) = maybe_creds {
if let Some(header) = &creds.maybe_as_cookie_header() {
request_builder = request_builder.header("Cookie", header);
}
}
let response = request_builder
.multipart(form)
.send()
.await?;
let response = filter_bad_response(response).await?;
let response_body = &response.text().await?;
let media_file = serde_json::from_str(&response_body)?;
Ok(media_file)
}
fn get_route(api_host: &ApiHost) -> String {
let api_hostname = api_host.to_api_hostname();
format!("https://{}/v1/media_files/upload/new_engine_asset", api_hostname)
}
// TODO(bt,2025-04-22): Share API definitions between client and server in common crate.
#[derive(Deserialize, Debug)]
pub struct UploadImageMediaFileSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
crates/lib/api_clients/storyteller_client/src/media_files/upload_video_media_file_from_file.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::api_error::ApiError;
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::api_error::ApiError;
use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails;
use crate::shared_response_types::media_links::MediaLinks;
use crate::shared_response_types::simple_entity_stats::SimpleEntityStats;
crates/lib/api_clients/storyteller_client/src/recipes/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod upload_media_file_from_file;
crates/lib/api_clients/storyteller_client/src/recipes/upload_media_file_from_file.rs
+63
View File
@@ -0,0 +1,63 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::client_error::ClientError;
use crate::error::storyteller_error::StorytellerError;
use crate::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use crate::media_files::upload_new_engine_asset_from_file::upload_new_engine_asset_from_file;
use crate::media_files::upload_video_media_file_from_file::upload_video_media_file_from_file;
use crate::utils::api_host::ApiHost;
use mimetypes::mimetype_info::file_extension::FileExtension;
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use serde_derive::Deserialize;
use std::path::Path;
use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize, Debug)]
pub struct UploadMediaFileSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
/// Upload an image media file from a file.
pub async fn upload_media_file_from_file<P: AsRef<Path>>(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
path: P,
) -> Result<UploadMediaFileSuccessResponse, StorytellerError> {
let maybe_type = MimetypeInfo::get_for_path(path.as_ref())
.map_err(|err| ClientError::from(err))?
.map(|mime| mime.file_extension())
.flatten();
match maybe_type {
Some(FileExtension::Glb) => {
match upload_new_engine_asset_from_file(api_host, maybe_creds, path).await {
Ok(result) => Ok(UploadMediaFileSuccessResponse {
success: result.success,
media_file_token: result.media_file_token,
}),
Err(err) => Err(StorytellerError::Api(err)),
}
}
Some(FileExtension::Mp4) => {
match upload_video_media_file_from_file(api_host, maybe_creds, path).await {
Ok(result) => Ok(UploadMediaFileSuccessResponse {
success: result.success,
media_file_token: result.media_file_token,
}),
Err(err) => Err(StorytellerError::Api(err)),
}
}
Some(FileExtension::Png) | Some(FileExtension::Jpg) | Some(FileExtension::Gif) | Some(FileExtension::Webp) => {
match upload_image_media_file_from_file(api_host, maybe_creds, path).await {
Ok(result) => Ok(UploadMediaFileSuccessResponse {
success: result.success,
media_file_token: result.media_file_token,
}),
Err(err) => Err(StorytellerError::Api(err)),
}
}
Some(_) => Err(StorytellerError::Client(ClientError::FileTypeNotHandled(path.as_ref().to_path_buf()))),
None => Err(StorytellerError::Client(ClientError::FileTypeNotKnown(path.as_ref().to_path_buf()))),
}
}
crates/lib/api_clients/storyteller_client/src/utils/filter_bad_response.rs
+2 -3
View File
@@ -1,7 +1,6 @@
use anyhow::anyhow;
use crate::api_error::ApiError;
use crate::error::api_error::ApiError;
use crate::utils::status_codes::*;
use reqwest::StatusCode;
use anyhow::anyhow;
/// Pass request errors to this method for standard error handling behavior.
/// This function needs to take temporary ownership, but will return it to the caller.
crates/lib/api_clients/storyteller_client/src/utils/http_get_anonymous.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::error::api_error::ApiError;
use reqwest::Client;
use crate::api_error::ApiError;
const USER_AGENT: &str = "storyteller-client/1.0";
pub async fn http_get_anonymous(url: String) -> Result<reqwest::Response, ApiError> {
crates/lib/files/mimetypes/src/custom/custom_infer.rs
+32
View File
@@ -0,0 +1,32 @@
use infer::Infer;
use std::sync::LazyLock;
pub (crate) static CUSTOM_INFER: LazyLock<Infer> = LazyLock::new(|| {
let mut infer = Infer::new();
add_custom_types(&mut infer);
infer
});
fn add_custom_types(infer: &mut Infer) {
infer.add("model/gltf-binary", "glb", gltf_matcher);
}
fn gltf_matcher(buf: &[u8]) -> bool {
// Magic is 'glTF' (103 108 84 70)
// https://github.com/KhronosGroup/glTF/blob/main/extensions/1.0/Khronos/KHR_binary_glTF/README.md
buf.len() >= 4 && buf[0] == 103 && buf[1] == 108 && buf[2] == 84 && buf[3] == 70
}
#[cfg(test)]
mod tests {
use crate::custom::custom_infer::CUSTOM_INFER;
use testing::test_file_path::test_file_path;
#[test]
fn test_gltf() -> anyhow::Result<()> {
let path = test_file_path("test_data/3d/hanashi.glb")?;
let maybe_type = CUSTOM_INFER.get_from_path(path)?.unwrap();
assert_eq!(maybe_type.mime_type(), "model/gltf-binary");
Ok(())
}
}
crates/lib/files/mimetypes/src/custom/mod.rs
+1
View File
@@ -0,0 +1 @@
pub (crate) mod custom_infer;
crates/lib/files/mimetypes/src/lib.rs
+1
View File
@@ -19,6 +19,7 @@
#![allow(dead_code)]
#![allow(non_snake_case)]
pub (crate) mod custom;
pub mod mimetype_for_bytes;
pub mod mimetype_for_file;
pub mod mimetype_info;
crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs
+3
View File
@@ -8,6 +8,7 @@ pub enum FileExtension {
Aac,
Flac,
Gif,
Glb,
Jpg,
Mov,
Mp3,
@@ -26,6 +27,7 @@ impl FileExtension {
"audio/aac" => Some(FileExtension::Aac),
"audio/flac" => Some(FileExtension::Flac),
"image/gif" => Some(FileExtension::Gif),
"model/gltf-binary" => Some(FileExtension::Glb),
"image/jpeg" => Some(FileExtension::Jpg),
"video/quicktime" => Some(FileExtension::Mov),
"audio/mpeg" => Some(FileExtension::Mp3),
@@ -53,6 +55,7 @@ impl FileExtension {
FileExtension::Aac => (".aac", "aac"),
FileExtension::Flac => (".flac", "flac"),
FileExtension::Gif => (".gif", "gif"),
FileExtension::Glb => (".glb", "glb"),
FileExtension::Jpg => (".jpg", "jpg"),
FileExtension::Mov => (".mov", "mov"),
FileExtension::Mp3 => (".mp3", "mp3"),
crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs
+16 -3
View File
@@ -1,3 +1,4 @@
use crate::custom::custom_infer::CUSTOM_INFER;
use crate::mimetype_info::file_extension::FileExtension;
use infer::Type;
use std::io;
@@ -14,12 +15,12 @@ pub struct MimetypeInfo {
impl MimetypeInfo {
pub fn get_for_bytes(bytes: &[u8]) -> Option<Self> {
infer::get(bytes)
CUSTOM_INFER.get(bytes)
.map(|t| Self::for_type(t))
}
pub fn get_for_path<P: AsRef<Path>>(path: P) -> io::Result<Option<Self>> {
infer::get_from_path(path)
CUSTOM_INFER.get_from_path(path)
.map(|maybe_type| maybe_type.map(|t| Self::for_type(t)))
}
@@ -62,8 +63,8 @@ mod tests {
}
mod for_files {
use testing::test_file_path::test_file_path;
use super::*;
use testing::test_file_path::test_file_path;
#[test]
fn mp3() -> anyhow::Result<()> {
@@ -76,5 +77,17 @@ mod tests {
assert_eq!(ext.extension_without_period(), "mp3");
Ok(())
}
#[test]
fn glb() -> anyhow::Result<()> {
let path = test_file_path("test_data/3d/hanashi.glb")?;
let t = MimetypeInfo::get_for_path(path)?.expect("should detect type");
assert_eq!(t.mime_type(), "model/gltf-binary");
let ext = t.file_extension.expect("should have extension");
assert_eq!(ext, FileExtension::Glb);
assert_eq!(ext.extension_with_period(), ".glb");
assert_eq!(ext.extension_without_period(), "glb");
Ok(())
}
}
}
frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx
+17 -2
View File
@@ -4,7 +4,7 @@ import { Transition, TransitionChild } from "@headlessui/react";
import { createPortal } from "react-dom";
import dayjs from "dayjs";
import { faDownToLine } from "@fortawesome/pro-solid-svg-icons";
import { FalKlingImageToVideo } from "@storyteller/tauri-api"
import { FalKlingImageToVideo, FalHunyuanImageTo3d } from "@storyteller/tauri-api"
interface LightboxModalProps {
isOpen: boolean;
@@ -130,7 +130,22 @@ export function LightboxModal({
//onCloseGallery(); // close the gallery
}}
>
Generate Video
Video
</Button>
<Button
onClick={async (e) => {
let _result = await FalHunyuanImageTo3d({
image_media_token: mediaId,
//base64_image: downloadUrl,
});
//e.stopPropagation();
//await onAddToSceneClicked(downloadUrl, mediaId);
//onClose(); // close the lightbox
//onCloseGallery(); // close the gallery
}}
>
3D
</Button>
{onAddToSceneClicked && downloadUrl && (
frontend/libs/tauri-api/src/index.ts
+5 -4
View File
@@ -1,11 +1,12 @@
export * from "./lib/common/CommandStatus.js";
export * from "./lib/fal/FalBackgroundRemoval.js";
export * from "./lib/fal/FalHunyuanImageTo3d.js";
export * from "./lib/fal/FalKlingImageToVideo.js";
export * from "./lib/fal/GetFalApiKey.js";
export * from "./lib/fal/SetFalApiKey.js";
export * from "./lib/settings/GetAppPreferences.js";
export * from "./lib/settings/UpdateAppPreference.js";
export * from "./lib/sora/CheckSoraSession.js";
export * from "./lib/sora/LogoutSoraSession.js";
export * from "./lib/sora/SoraImageRemix.js";
export * from "./lib/sora/useSoraLoginListener.js";
export * from "./lib/fal/FalBackgroundRemoval.js";
export * from "./lib/fal/FalKlingImageToVideo.js";
export * from "./lib/fal/GetFalApiKey.js";
export * from "./lib/fal/SetFalApiKey.js";
frontend/libs/tauri-api/src/lib/fal/FalHunyuanImageTo3d.ts
+39
View File
@@ -0,0 +1,39 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
export enum FalHunyuanImageTo3dErrorType {
ServerError = "server_error",
NeedsFalApiKey = "needs_fal_api_key",
}
export interface FalHunyuanImageTo3dRequest {
image_media_token?: string;
base64_image?: string;
}
export interface FalHunyuanImageTo3dError extends CommandResult {
error_type: FalHunyuanImageTo3dErrorType;
error_message?: string;
}
export interface FalHunyuanImageTo3dPayload {
media_token: string;
cdn_url: string;
base64_bytes: string;
}
export interface FalHunyuanImageTo3dSuccess extends CommandResult {
payload: FalHunyuanImageTo3dPayload;
}
export type FalHunyuanImageTo3dResult = FalHunyuanImageTo3dSuccess | FalHunyuanImageTo3dError;
export const FalHunyuanImageTo3d = async (request: FalHunyuanImageTo3dRequest) : Promise<FalHunyuanImageTo3dResult> => {
let result = await invoke("fal_hunyuan_image_to_3d_command", {
request: {
image_media_token: request.image_media_token,
base64_image: request.base64_image,
}
});
return (result as FalHunyuanImageTo3dResult);
}
test_data/3d/hanashi.glb
BIN
View File
Binary file not shown.