Compare commits

..
Author SHA1 Message Date
Brandon Thomas 2feefcb75b end frame support in frontend 2025-08-26 19:12:37 -04:00
Brandon Thomas b54953bc04 seedance end frame 2025-08-26 19:02:25 -04:00
Brandon Thomas 29ade5fbbc wip 2025-08-26 12:27:53 -04:00
Brandon Thomas d6ddf0f693 kling 2.1 pro ending frame 2025-08-26 02:18:32 -04:00
Brandon Thomas e7e816b3c2 start adding video end frames: kling 1.6 2025-08-26 02:04:01 -04:00
96 changed files with 588 additions and 1286 deletions
@@ -11,6 +11,8 @@ use crate::core::events::generation_events::common::{GenerationAction, Generatio
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
@@ -6,6 +6,8 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -10,6 +10,8 @@ use crate::core::events::generation_events::common::{GenerationAction, Generatio
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
@@ -25,7 +27,6 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use errors::AnyhowError;
use log::{error, info, warn};
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
@@ -40,20 +41,9 @@ use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use tokens::tokens::media_files::MediaFileToken;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ContextualImageEditModel {
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
}
#[derive(Deserialize, Debug)]
pub struct EnqueueContextualEditImageCommand {
/// The model to use.
@@ -4,6 +4,8 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -5,6 +5,8 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -8,6 +8,8 @@ use crate::core::events::functional_events::show_provider_login_modal_event::Sho
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
@@ -1,9 +1,7 @@
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint;
use crate::core::commands::enqueue::image_inpaint::flux_pro_1_inpaint::handle_flux_pro_1_inpaint::handle_flux_pro_1_inpaint;
use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType};
@@ -13,6 +11,8 @@ use crate::core::events::generation_events::common::{GenerationAction, Generatio
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
@@ -44,20 +44,15 @@ use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint;
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ImageInpaintModel {
// Mask-based inpainting models
#[serde(rename = "flux_dev_juggernaut")]
FluxDevJuggernaut,
#[serde(rename = "flux_pro_1")]
FluxPro1,
// Non-inpainting, instructive editing models
// NB: We're supporting these to keep the app simple and not over-complicate the javascript
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
}
#[derive(Deserialize, Debug)]
@@ -249,21 +244,6 @@ pub async fn handle_request(
sora_task_queue,
).await?
}
Some(ImageInpaintModel::FluxProKontextMax) => {
// Redirect to contextual image edit handler.
handle_flux_pro_kontext_inpaint(
request,
app,
app_data_root,
app_env_configs,
provider_priority_store,
storyteller_creds_manager,
fal_creds_manager,
fal_task_queue,
sora_creds_manager,
sora_task_queue,
).await?
}
};
let result = success_event
@@ -7,6 +7,8 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -7,6 +7,8 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -130,7 +132,7 @@ pub async fn handle_flux_pro_1_inpaint_artcraft(
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::FluxPro1),
model: Some(GenerationModel::FluxProKontextMax),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageInpaintEdit,
})
@@ -1,59 +0,0 @@
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint_artcraft::handle_flux_pro_kontext_inpaint_artcraft;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use log::info;
use tauri::AppHandle;
pub async fn handle_flux_pro_kontext_inpaint(
request: &EnqueueInpaintImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_creds_manager: &FalCredentialManager,
fal_task_queue: &FalTaskQueue,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, InternalImageInpaintError> {
let priority = provider_priority_store.get_priority()?;
// TODO: Check if providers are available before proceeding.
info!("Providers by priority: {:?}", priority);
for provider in priority.iter() {
match provider {
Provider::Fal => {
// Fallthrough (for now)
}
Provider::Sora => {
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching flux pro kontext inpaint via Artcraft...");
return handle_flux_pro_kontext_inpaint_artcraft(
request,
app,
app_data_root,
app_env_configs,
storyteller_creds_manager
).await;
}
}
}
Err(InternalImageInpaintError::NoProviderAvailable)
}
@@ -1,183 +0,0 @@
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest};
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest};
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest};
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality;
use idempotency::uuid::generate_random_uuid;
use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes;
use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions;
use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask;
use log::{error, info};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image;
use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image;
use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
use storyteller_client::generate::image::inpaint::flux_pro_1_inpaint_image::flux_pro_1_inpaint_image;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::media_files::upload_image_media_file_from_bytes::{upload_image_media_file_from_bytes, ImageType, UploadImageBytesArgs};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use tauri::AppHandle;
use tokens::tokens::media_files::MediaFileToken;
pub async fn handle_flux_pro_kontext_inpaint_artcraft(
request: &EnqueueInpaintImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<TaskEnqueueSuccess, InternalImageInpaintError> {
let creds = match storyteller_creds_manager.get_credentials()? {
Some(creds) => creds,
None => {
return Err(InternalImageInpaintError::NeedsStorytellerCredentials);
},
};
let image_media_token = match &request.image_media_token {
Some(token) => token.clone(),
None => {
return Err(InternalImageInpaintError::NoSourceImageSpecified);
},
};
//let image_media_file = get_media_file(
// &app_env_configs.storyteller_host,
// &image_media_token,
//).await.map_err(|err| {
// error!("Failed to get media file for image: {:?}", err);
// InternalImageInpaintError::StorytellerError(err)
//})?;
// TODO: Experiment in seeing if we can use the mask in any useful way.
// let mask_media_token = get_mask(
// request,
// app_env_configs,
// &creds,
// ).await?;
info!("Calling Artcraft flux pro kontext inpaint ...");
let uuid_idempotency_token = generate_random_uuid();
let num_images = match request.image_count {
None => None,
Some(1) => Some(FluxProKontextMaxEditImageNumImages::One),
Some(2) => Some(FluxProKontextMaxEditImageNumImages::Two),
Some(3) => Some(FluxProKontextMaxEditImageNumImages::Three),
Some(4) => Some(FluxProKontextMaxEditImageNumImages::Four),
Some(other) => {
return Err(InternalImageInpaintError::InvalidNumberOfRequestedImages {
min: 1,
max: 4,
requested: other,
});
},
};
let request = FluxProKontextMaxEditImageRequest {
uuid_idempotency_token,
prompt: Some(request.prompt.clone()),
image_media_token,
num_images,
};
let result = flux_pro_kontext_max_edit_image(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft flux pro kontext inpaint. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft flux pro kontext inpaint: {:?}", err);
return Err(InternalImageInpaintError::StorytellerError(err));
}
};
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::FluxProKontextMax),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageInpaintEdit,
})
}
// async fn get_mask(
// request: &EnqueueInpaintImageCommand,
// app_env_configs: &AppEnvConfigs,
// storyteller_creds: &StorytellerCredentialSet,
// ) -> Result<MediaFileToken, InternalImageInpaintError> {
//
// if request.mask_image_media_token.is_some() && request.mask_image_raw_bytes.is_some() {
// return Err(InternalImageInpaintError::MaskMediaTokenAndBytesSupplied);
// }
//
// if let Some(token) = request.mask_image_media_token.as_ref() {
// return Ok(token.clone());
// };
//
// let image_bytes = request.mask_image_raw_bytes.as_ref()
// .ok_or(InternalImageInpaintError::NoMaskImageSpecified)?;
//
// let image_bytes = normalize_image_bytes_to_flux_mask(image_bytes)
// .map_err(|err| {
// error!("Failed to convert image bytes to png: {:?}", err);
// InternalImageInpaintError::CouldNotEncodeMask
// })?;
//
// info!("Uploading image media file from bytes...");
//
// let result = upload_image_media_file_from_bytes(UploadImageBytesArgs {
// api_host: &app_env_configs.storyteller_host,
// maybe_creds: Some(&storyteller_creds),
// image_bytes: image_bytes.0,
// image_type: ImageType::Png,
// is_intermediate_system_file: true,
// }).await
// .map_err(|err| {
// error!("Failed to upload image media file: {:?}", err);
// InternalImageInpaintError::StorytellerError(err)
// })?;
//
// Ok(result.media_file_token)
// }
@@ -1,2 +0,0 @@
pub (super) mod handle_flux_pro_kontext_inpaint;
pub (super) mod handle_flux_pro_kontext_inpaint_artcraft;
@@ -1,5 +1,4 @@
mod flux_dev_juggernaut_inpaint;
mod flux_pro_1_inpaint;
mod flux_pro_kontext_inpaint;
pub mod enqueue_image_inpaint_command;
pub mod errors;
@@ -6,6 +6,7 @@ use crate::core::commands::response::shorthand::Response;
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::model::video_models::VideoModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
@@ -20,27 +21,6 @@ use serde_derive::{Deserialize, Serialize};
use tauri::{AppHandle, State};
use tokens::tokens::media_files::MediaFileToken;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum VideoModel {
#[serde(rename = "kling_1.6_pro")]
Kling16Pro,
#[serde(rename = "kling_2.1_pro")]
Kling21Pro,
#[serde(rename = "kling_2.1_master")]
Kling21Master,
#[serde(rename = "seedance_1.0_lite")]
Seedance10Lite,
#[serde(rename = "veo_2")]
Veo2,
}
#[derive(Deserialize)]
pub struct EnqueueImageToVideoRequest {
/// REQUIRED.
@@ -1,10 +1,11 @@
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::{EnqueueImageToVideoRequest, VideoModel};
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::EnqueueImageToVideoRequest;
use crate::core::commands::enqueue::image_to_video::internal_video_error::InternalVideoError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::model::video_models::VideoModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
@@ -1,10 +1,11 @@
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::{EnqueueImageToVideoRequest, VideoModel};
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::EnqueueImageToVideoRequest;
use crate::core::commands::enqueue::image_to_video::internal_video_error::InternalVideoError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::model::video_models::VideoModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir;
@@ -12,6 +12,7 @@ use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
@@ -25,35 +26,12 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
use log::{error, info, warn};
use serde::{Deserialize, Serialize};
use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs};
use tauri::{AppHandle, State};
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ImageModel {
#[serde(rename = "flux_1_dev")]
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
#[serde(rename = "flux_pro_11")]
FluxPro11,
#[serde(rename = "flux_pro_11_ultra")]
FluxPro11Ultra,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "recraft_3")]
Recraft3,
// Generic Midjourney model, version unknown.
#[serde(rename = "midjourney")]
Midjourney,
}
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
#[derive(Deserialize)]
pub struct EnqueueTextToImageRequest {
@@ -1,9 +1,10 @@
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, ImageModel, TextToImageSize};
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::commands::enqueue::text_to_image::internal_image_error::InternalImageError;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
@@ -1,9 +1,10 @@
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, ImageModel};
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::EnqueueTextToImageRequest;
use crate::core::commands::enqueue::text_to_image::internal_image_error::InternalImageError;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::image_models::ImageModel;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use anyhow::anyhow;
@@ -9,6 +9,7 @@ use crate::core::commands::enqueue::text_to_image::internal_image_error::Interna
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
@@ -6,6 +6,7 @@ use crate::core::commands::enqueue::text_to_image::internal_image_error::Interna
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
@@ -5,6 +5,7 @@ use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::functional_events::show_provider_login_modal_event::ShowProviderLoginModalEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::image_models::ImageModel;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use enums::common::generation_provider::GenerationProvider;
@@ -9,6 +9,7 @@ use crate::core::events::functional_events::show_provider_login_modal_event::Sho
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::warning_events::flash_user_input_error_event::FlashUserInputErrorEvent;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
+1
View File
@@ -2,6 +2,7 @@ pub mod artcraft_error;
pub mod commands;
pub mod events;
pub mod lifecycle;
pub mod model;
pub mod state;
pub mod threads;
pub mod utils;
@@ -0,0 +1,15 @@
use serde_derive::Deserialize;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ContextualImageEditModel {
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
// TODO: Flux Kontext models.
}
@@ -0,0 +1,24 @@
use serde_derive::Deserialize;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ImageModel {
#[serde(rename = "flux_1_dev")]
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
#[serde(rename = "flux_pro_11")]
FluxPro11,
#[serde(rename = "flux_pro_11_ultra")]
FluxPro11Ultra,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "recraft_3")]
Recraft3,
// Generic Midjourney model, version unknown.
#[serde(rename = "midjourney")]
Midjourney,
}
@@ -0,0 +1,3 @@
pub mod contextual_image_edit_models;
pub mod image_models;
pub mod video_models;
@@ -0,0 +1,22 @@
use serde_derive::Deserialize;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum VideoModel {
#[serde(rename = "kling_1.6_pro")]
Kling16Pro,
#[serde(rename = "kling_2.1_pro")]
Kling21Pro,
#[serde(rename = "kling_2.1_master")]
Kling21Master,
#[serde(rename = "seedance_1.0_lite")]
Seedance10Lite,
#[serde(rename = "veo_2")]
Veo2,
}
@@ -40,6 +40,7 @@ pub async fn maybe_handle_text_to_image_complete_event(
batch_token,
).await?;
if let Err(err) = event.send(&app) {
error!("Failed to send TextToImageGenerationCompleteEvent: {:?}", err); // Fail open
}
@@ -27,7 +27,6 @@ use errors::AnyhowResult;
pub mod billing;
pub mod configs;
pub mod error;
pub mod http_server;
pub mod state;
pub mod threads;
@@ -30,7 +30,6 @@ use crate::configs::static_api_tokens::StaticApiTokenSet;
pub mod billing;
pub mod configs;
pub mod error;
pub mod http_server;
pub mod state;
pub mod threads;
@@ -1,37 +0,0 @@
use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
/// Any generic errors arising internally.
/// An error class that we do not expose via the API.
/// This should be massaged before we send it to the user.
#[derive(Debug)]
pub enum InternalError {
/// Issue with HMAC decoding
/// This is typically when using development cookies against production,
/// or vice versa.
VisitorCookieError(String),
/// Decode error with AVT cookie missing a required field.
VisitorCookieMissingField(&'static str),
/// Catch instances of anyhow::Error.
AnyhowError(anyhow::Error),
}
impl Error for InternalError {}
impl Display for InternalError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
InternalError::VisitorCookieError(reason) => write!(f, "Visitor cookie error: {:?}", reason),
InternalError::VisitorCookieMissingField(field) => write!(f, "Visitor cookie missing field: {}", field),
InternalError::AnyhowError(err) => write!(f, "Anyhow error: {}", err),
}
}
}
impl From<anyhow::Error> for InternalError {
fn from(err: anyhow::Error) -> Self {
InternalError::AnyhowError(err)
}
}
@@ -1 +0,0 @@
pub mod internal_error;
@@ -16,6 +16,21 @@ const QUALITY : u8 = 95;
pub struct MediaLinksBuilder {}
impl MediaLinksBuilder {
#[deprecated(note = "we need to start passing the domain and the server environment to generate the CDN")]
pub fn from_media_path(
domain: MediaDomain,
bucket_path: &MediaFileBucketPath,
) -> MediaLinks {
Self::from_media_path_and_env(domain, ServerEnvironment::Production, bucket_path)
}
#[deprecated(note = "we need to start passing the domain and the server environment to generate the CDN")]
pub fn from_rooted_path(
domain: MediaDomain,
rooted_path: &str,
) -> MediaLinks {
Self::from_rooted_path_and_env(domain, ServerEnvironment::Production, rooted_path)
}
pub fn from_media_path_and_env(
domain: MediaDomain,
@@ -133,7 +148,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn wav_file() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.wav");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.wav");
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/foo/bar.wav");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -141,7 +156,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn glb_file() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.glb");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.glb");
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/foo/bar.glb");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -149,7 +164,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn jpg_image() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.jpg");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.jpg");
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/foo/bar.jpg");
assert_eq!(links.maybe_thumbnail_template, Some("https://cdn-2.fakeyou.com/cdn-cgi/image/width={WIDTH},quality=95/foo/bar.jpg".to_string()));
assert_eq!(links.maybe_video_previews, None);
@@ -157,7 +172,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn mp4_video() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.mp4");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.mp4");
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/foo/bar.mp4");
assert_eq!(links.maybe_thumbnail_template, None);
let video_previews = links.maybe_video_previews.expect("should have previews");
@@ -175,7 +190,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
fn wav_file() {
// https://storage.googleapis.com/vocodes-public/media/9/4/a/2/7/94a27nmbd0bqmd10tg0pp3hz45zytf67/fakeyou_94a27nmbd0bqmd10tg0pp3hz45zytf67.wav
let media_path = MediaFileBucketPath::from_object_hash("94a27nmbd0bqmd10tg0pp3hz45zytf67", Some("fakeyou_"), Some(".wav"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/media/9/4/a/2/7/94a27nmbd0bqmd10tg0pp3hz45zytf67/fakeyou_94a27nmbd0bqmd10tg0pp3hz45zytf67.wav");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -185,7 +200,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
fn png_image() {
/// https://storage.googleapis.com/vocodes-public/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png
let media_path = MediaFileBucketPath::from_object_hash("37mb3gh8fmj85y21thvbv08bzv24atjt", Some("upload_"), Some(".png"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png");
assert_eq!(links.maybe_thumbnail_template, Some("https://cdn-2.fakeyou.com/cdn-cgi/image/width={WIDTH},quality=95/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png".to_string()));
assert_eq!(links.maybe_video_previews, None);
@@ -195,7 +210,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
fn mp4_video() {
// https://storage.googleapis.com/vocodes-public/media/t/6/c/n/y/t6cnyw4g3e8k7carkk2bvrt6nd3fycjv/storyteller_t6cnyw4g3e8k7carkk2bvrt6nd3fycjv.mp4
let media_path = MediaFileBucketPath::from_object_hash("t6cnyw4g3e8k7carkk2bvrt6nd3fycjv", Some("storyteller_"), Some(".mp4"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn-2.fakeyou.com/media/t/6/c/n/y/t6cnyw4g3e8k7carkk2bvrt6nd3fycjv/storyteller_t6cnyw4g3e8k7carkk2bvrt6nd3fycjv.mp4");
assert_eq!(links.maybe_thumbnail_template, None);
let video_previews = links.maybe_video_previews.expect("should have previews");
@@ -214,11 +229,10 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
mod rooted_path {
use super::*;
use server_environment::ServerEnvironment;
#[test]
fn wav_file() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.wav");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.wav");
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/foo/bar.wav");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -226,7 +240,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn glb_file() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.glb");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.glb");
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/foo/bar.glb");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -234,7 +248,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn jpg_image() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.jpg");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.jpg");
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/foo/bar.jpg");
assert_eq!(links.maybe_thumbnail_template, Some("https://cdn.storyteller.ai/cdn-cgi/image/width={WIDTH},quality=95/foo/bar.jpg".to_string()));
assert_eq!(links.maybe_video_previews, None);
@@ -242,7 +256,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
#[test]
fn mp4_video() {
let links = MediaLinksBuilder::from_rooted_path_and_env(DOMAIN, ServerEnvironment::Production, "/foo/bar.mp4");
let links = MediaLinksBuilder::from_rooted_path(DOMAIN, "/foo/bar.mp4");
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/foo/bar.mp4");
assert_eq!(links.maybe_thumbnail_template, None);
let video_previews = links.maybe_video_previews.expect("should have previews");
@@ -255,13 +269,12 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
mod media_path {
use super::*;
use server_environment::ServerEnvironment;
#[test]
fn wav_file() {
// https://storage.googleapis.com/vocodes-public/media/9/4/a/2/7/94a27nmbd0bqmd10tg0pp3hz45zytf67/fakeyou_94a27nmbd0bqmd10tg0pp3hz45zytf67.wav
let media_path = MediaFileBucketPath::from_object_hash("94a27nmbd0bqmd10tg0pp3hz45zytf67", Some("fakeyou_"), Some(".wav"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/media/9/4/a/2/7/94a27nmbd0bqmd10tg0pp3hz45zytf67/fakeyou_94a27nmbd0bqmd10tg0pp3hz45zytf67.wav");
assert_eq!(links.maybe_thumbnail_template, None);
assert_eq!(links.maybe_video_previews, None);
@@ -271,7 +284,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
fn png_image() {
/// https://storage.googleapis.com/vocodes-public/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png
let media_path = MediaFileBucketPath::from_object_hash("37mb3gh8fmj85y21thvbv08bzv24atjt", Some("upload_"), Some(".png"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png");
assert_eq!(links.maybe_thumbnail_template, Some("https://cdn.storyteller.ai/cdn-cgi/image/width={WIDTH},quality=95/media/3/7/m/b/3/37mb3gh8fmj85y21thvbv08bzv24atjt/upload_37mb3gh8fmj85y21thvbv08bzv24atjt.png".to_string()));
assert_eq!(links.maybe_video_previews, None);
@@ -281,7 +294,7 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks
fn mp4_video() {
// https://storage.googleapis.com/vocodes-public/media/t/6/c/n/y/t6cnyw4g3e8k7carkk2bvrt6nd3fycjv/storyteller_t6cnyw4g3e8k7carkk2bvrt6nd3fycjv.mp4
let media_path = MediaFileBucketPath::from_object_hash("t6cnyw4g3e8k7carkk2bvrt6nd3fycjv", Some("storyteller_"), Some(".mp4"));
let links = MediaLinksBuilder::from_media_path_and_env(DOMAIN, ServerEnvironment::Production, &media_path);
let links = MediaLinksBuilder::from_media_path(DOMAIN, &media_path);
assert_eq!(links.cdn_url.as_str(), "https://cdn.storyteller.ai/media/t/6/c/n/y/t6cnyw4g3e8k7carkk2bvrt6nd3fycjv/storyteller_t6cnyw4g3e8k7carkk2bvrt6nd3fycjv.mp4");
assert_eq!(links.maybe_thumbnail_template, None);
let video_previews = links.maybe_video_previews.expect("should have previews");
@@ -4,12 +4,12 @@ use actix_web::HttpRequest;
use anyhow::anyhow;
use log::warn;
use crate::error::internal_error::InternalError;
use crate::http_server::cookies::anonymous_visitor_tracking::avt_cookie_payload::AvtCookiePayload;
use cookies::jwt_signer::JwtSigner;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use crate::http_server::cookies::anonymous_visitor_tracking::avt_cookie_payload::AvtCookiePayload;
const VISITOR_COOKIE_NAME : &str = "visitor";
/// Handle "anonymous visitor tracking" cookies.
@@ -70,7 +70,7 @@ impl AvtCookieManager {
Ok(payload)
}
pub fn decode_cookie_payload_from_request(&self, request: &HttpRequest) -> Result<Option<AvtCookiePayload>, InternalError> {
pub fn decode_cookie_payload_from_request(&self, request: &HttpRequest) -> AnyhowResult<Option<AvtCookiePayload>> {
let cookie = match request.cookie(VISITOR_COOKIE_NAME) {
None => return Ok(None),
Some(cookie) => cookie,
@@ -79,7 +79,7 @@ impl AvtCookieManager {
match self.decode_cookie_payload(&cookie) {
Err(e) => {
warn!("Visitor cookie decode error: {:?}", e);
Err(InternalError::VisitorCookieError(e.to_string()))
Err(anyhow!("Could not decode visitor cookie: {:?}", e))
},
Ok(payload) => Ok(Some(payload)),
}
@@ -1,7 +1,9 @@
use std::collections::BTreeMap;
use std::str::FromStr;
use crate::error::internal_error::InternalError;
use anyhow::anyhow;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
/**
@@ -36,19 +38,18 @@ impl AvtCookiePayload {
}
}
pub fn from_map(map: BTreeMap<String, String>) -> Result<Self, InternalError> {
pub fn from_map(map: BTreeMap<String, String>) -> AnyhowResult<Self> {
let avt_token = map.get("avt_token")
.ok_or_else(|| InternalError::VisitorCookieMissingField("avt_token"))?;
.ok_or(anyhow!("avt_token not present"))?
.to_string();
let cookie_version = map
.get("cookie_version")
.ok_or_else(|| InternalError::VisitorCookieMissingField("cookie_version"))?;
.ok_or(anyhow!("avt_token not present"))?;
let avt_token = AnonymousVisitorTrackingToken::new_from_str(avt_token);
let avt_token = AnonymousVisitorTrackingToken::new_from_str(&avt_token);
let cookie_version = u32::from_str(cookie_version)
.map_err(|e| InternalError::VisitorCookieError(
format!("invalid integer for cookie_version: {:?}, version: {}", e, cookie_version)))?;
let cookie_version = u32::from_str(cookie_version)?;
Ok(AvtCookiePayload {
avt_token,
@@ -31,7 +31,6 @@ use mysql_queries::queries::generic_inference::web::batch_get_inference_job_stat
use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
use r2d2_redis::redis::Commands;
use redis_common::redis_keys::RedisKeys;
use server_environment::ServerEnvironment;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use utoipa::{IntoParams, ToSchema};
@@ -204,11 +203,7 @@ pub async fn batch_get_inference_job_status_handler(
if tokens.is_empty() {
// NB: MediaDomain doesn't matter since it's an empty list.
return Ok(records_to_response(
Vec::new(),
server_state.server_environment,
MediaDomain::Storyteller
));
return Ok(records_to_response(Vec::new(), MediaDomain::Storyteller));
}
// NB: Since this is publicly exposed, we don't query sensitive data.
@@ -254,19 +249,18 @@ pub async fn batch_get_inference_job_status_handler(
let media_domain = get_media_domain(&http_request);
Ok(records_to_response(records, server_state.server_environment, media_domain))
Ok(records_to_response(records, media_domain))
}
fn records_to_response(
records: Vec<GenericInferenceJobStatus>,
server_environment: ServerEnvironment,
media_domain: MediaDomain,
) -> Json<BatchGetInferenceJobStatusSuccessResponse> {
Json(BatchGetInferenceJobStatusSuccessResponse {
success: true,
job_states: records.into_iter()
.map(|record| {
db_record_to_response_payload(record, None, server_environment, media_domain)
db_record_to_response_payload(record, None, media_domain)
})
.collect::<Vec<_>>(),
})
@@ -275,7 +269,6 @@ fn records_to_response(
fn db_record_to_response_payload(
record: GenericInferenceJobStatus,
maybe_extra_status_description: Option<String>,
server_environment: ServerEnvironment,
media_domain: MediaDomain,
) -> BatchInferenceJobStatusResponsePayload {
let inference_category = record.request_details.inference_category;
@@ -381,11 +374,7 @@ fn db_record_to_response_payload(
BatchResultDetailsResponse {
entity_type: result_details.entity_type,
entity_token: result_details.entity_token,
media_links: MediaLinksBuilder::from_rooted_path_and_env(
media_domain,
server_environment,
&public_bucket_media_path
),
media_links: MediaLinksBuilder::from_rooted_path(media_domain, &public_bucket_media_path),
maybe_public_bucket_media_path: Some(public_bucket_media_path),
maybe_successfully_completed_at: result_details.maybe_successfully_completed_at,
}
@@ -29,7 +29,6 @@ use mysql_queries::queries::generic_inference::web::get_inference_job_status::ge
use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
use r2d2_redis::redis::{Commands, RedisResult};
use redis_common::redis_keys::RedisKeys;
use server_environment::ServerEnvironment;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use utoipa::ToSchema;
@@ -252,7 +251,6 @@ pub async fn get_inference_job_status_handler(
let record_for_response = record_to_payload(
record,
maybe_extra_status_description,
server_state.server_environment,
media_domain,
);
@@ -265,7 +263,6 @@ pub async fn get_inference_job_status_handler(
fn record_to_payload(
record: GenericInferenceJobStatus,
maybe_extra_status_description: Option<String>,
server_environment: ServerEnvironment,
media_domain: MediaDomain,
) -> InferenceJobStatusResponsePayload {
let inference_category = record.request_details.inference_category;
@@ -371,11 +368,7 @@ fn record_to_payload(
ResultDetailsResponse {
entity_type: result_details.entity_type,
entity_token: result_details.entity_token,
media_links: MediaLinksBuilder::from_rooted_path_and_env(
media_domain,
server_environment,
&public_bucket_media_path
),
media_links: MediaLinksBuilder::from_rooted_path(media_domain, &public_bucket_media_path),
maybe_public_bucket_media_path: Some(public_bucket_media_path),
maybe_successfully_completed_at: result_details.maybe_successfully_completed_at,
}
@@ -391,7 +384,6 @@ mod tests {
use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::record_to_payload;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use mysql_queries::queries::generic_inference::web::job_status::{GenericInferenceJobStatus, RequestDetails, ResultDetails};
use server_environment::ServerEnvironment;
use url::Url;
#[test]
@@ -415,7 +407,7 @@ mod tests {
};
let payload =
record_to_payload(status, None, ServerEnvironment::Production, MediaDomain::Storyteller);
record_to_payload(status, None, MediaDomain::Storyteller);
assert!(payload.maybe_result.is_some());
@@ -454,7 +446,7 @@ mod tests {
};
let payload =
record_to_payload(status, None, ServerEnvironment::Production, MediaDomain::FakeYou);
record_to_payload(status, None, MediaDomain::FakeYou);
assert!(payload.maybe_result.is_some());
@@ -31,7 +31,6 @@ use mysql_queries::queries::generic_inference::web::list_session_jobs::{list_ses
use primitives::numerics::i64_to_u64_zero_clamped::i64_to_u64_zero_clamped;
use r2d2_redis::redis::Commands;
use redis_common::redis_keys::RedisKeys;
use server_environment::ServerEnvironment;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::{IntoParams, ToSchema};
@@ -198,17 +197,16 @@ pub async fn list_session_jobs_handler(
let media_domain = get_media_domain(&http_request);
records_to_response(records, server_state.server_environment, media_domain)
records_to_response(records, media_domain)
}
fn records_to_response(
records: Vec<GenericInferenceJobStatus>,
server_environment: ServerEnvironment,
media_domain: MediaDomain,
) -> Result<Json<ListSessionJobsSuccessResponse>, ListSessionJobsError> {
let mut records = records.into_iter()
.map(|record| {
db_record_to_response_payload(record, None, server_environment, media_domain)
db_record_to_response_payload(record, None, media_domain)
})
.collect::<Vec<_>>();
@@ -242,7 +240,6 @@ fn records_to_response(
fn db_record_to_response_payload(
record: GenericInferenceJobStatus,
maybe_extra_status_description: Option<String>,
server_environment: ServerEnvironment,
media_domain: MediaDomain,
) -> ListSessionJobsItem {
let inference_category = record.request_details.inference_category;
@@ -362,11 +359,7 @@ fn db_record_to_response_payload(
entity_type: result_details.entity_type,
entity_token: result_details.entity_token,
maybe_batch_token: result_details.maybe_batch_token,
media_links: MediaLinksBuilder::from_rooted_path_and_env(
media_domain,
server_environment,
&public_bucket_media_path
),
media_links: MediaLinksBuilder::from_rooted_path(media_domain, &public_bucket_media_path),
maybe_public_bucket_media_path: Some(public_bucket_media_path),
maybe_successfully_completed_at: result_details.maybe_successfully_completed_at,
}
@@ -333,11 +333,7 @@ pub async fn batch_get_media_files_handler(
maybe_media_subtype: result.maybe_media_subtype,
maybe_engine_extension,
maybe_batch_token: result.maybe_batch_token,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -112,11 +112,7 @@ pub async fn list_batch_generated_redux_media_files_handler(
media_class: result.media_class,
media_type: result.media_type,
maybe_batch_token: result.maybe_batch_token,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
cover_image: MediaFileCoverImageDetailsBuilder::from_optional_db_fields(
&result.token,
media_domain,
@@ -288,11 +288,7 @@ pub async fn list_featured_media_files_handler(
media_type: m.media_type,
maybe_engine_category: m.maybe_engine_category,
maybe_animation_type: m.maybe_animation_type,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -277,11 +277,7 @@ pub async fn list_media_files_by_batch_token_handler(
maybe_origin_model_type: record.maybe_origin_model_type
.map(|t| PublicMediaFileModelType::from_enum(t)),
maybe_origin_model_token: record.maybe_origin_model_token,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -345,11 +345,7 @@ pub async fn list_media_files_handler(
maybe_origin_model_type: record.maybe_origin_model_type
.map(|m| PublicMediaFileModelType::from_enum(m)),
maybe_origin_model_token: record.maybe_origin_model_token,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -215,11 +215,7 @@ pub async fn list_pinned_media_files_handler(
media_type: m.media_type,
maybe_engine_category: m.maybe_engine_category,
maybe_animation_type: m.maybe_animation_type,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -227,11 +227,7 @@ pub async fn search_featured_media_files_handler(
media_type: result.media_type,
maybe_engine_category: result.maybe_engine_category,
maybe_animation_type: result.maybe_animation_type,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -254,11 +254,7 @@ pub async fn search_session_media_files_handler(
media_type: result.media_type,
maybe_engine_category: result.maybe_engine_category,
maybe_animation_type: result.maybe_animation_type,
media_links: MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path
),
media_links: MediaLinksBuilder::from_media_path(media_domain, &public_bucket_path),
public_bucket_path: public_bucket_path
.get_full_object_path_str()
.to_string(),
@@ -230,10 +230,7 @@ pub async fn list_user_bookmarks_for_user_handler(
});
let maybe_media_file_media_links = maybe_media_file_bucket_path.as_ref()
.map(|bucket_path| MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
bucket_path));
.map(|bucket_path| MediaLinksBuilder::from_media_path(media_domain, bucket_path));
let mut maybe_media_file_cover = None;
let mut maybe_model_weight_cover = None;
@@ -92,7 +92,6 @@ use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
pub mod billing;
pub mod configs;
pub mod error;
pub mod http_server;
pub mod state;
pub mod threads;
@@ -13,17 +13,3 @@ export const normalizeCanvas = (canvas: HTMLCanvasElement, width: number, height
ctx.drawImage(canvas, 0, 0, width, height);
return newCanvas;
}
export const drawAlphaMask = (canvas: OffscreenCanvas, width: number, height: number) => {
const ctx = canvas.getContext("2d");
if (!ctx) {
throw new Error("Failed to get canvas context");
}
// Fill the entire screen with black and composite to keep non-transparent pixels
ctx.save();
ctx.globalCompositeOperation = "source-in";
ctx.fillStyle = "black";
ctx.fillRect(0, 0, width, height);
ctx.restore();
}
@@ -36,7 +36,7 @@ import { useSignals } from "@preact/signals-react/runtime";
import { setLogoutStates } from "~/signals/authentication/utilities";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { TabId, useTabStore } from "~/pages/Stores/TabState";
import { useTabStore } from "~/pages/Stores/TabState";
import {
is3DEditorInitialized,
is3DSceneLoaded,
@@ -59,23 +59,7 @@ interface Props {
const SWITCHER_THROTTLE_TIME = 500; // milliseconds
// NB: See `TabState` for the default tab.
const appMenuTabs: MenuIconItem<TabId>[] = [
{
id: "IMAGE",
label: "Text to Image",
icon: <FontAwesomeIcon icon={faImage} />,
},
{
id: "VIDEO",
label: "Image to Video",
icon: <FontAwesomeIcon icon={faFilm} />,
},
{
id: "EDIT",
label: "Edit Image",
icon: <FontAwesomeIcon icon={faPenNib} />,
},
const appMenuTabs: MenuIconItem[] = [
{
id: "2D",
label: "2D Canvas",
@@ -92,6 +76,21 @@ const appMenuTabs: MenuIconItem<TabId>[] = [
description: "Precision control. Great for AI film.",
large: true,
},
{
id: "IMAGE",
label: "Prompt to Image",
icon: <FontAwesomeIcon icon={faImage} />,
},
{
id: "VIDEO",
label: "Image to Video",
icon: <FontAwesomeIcon icon={faFilm} />,
},
{
id: "EDIT",
label: "Edit Image",
icon: <FontAwesomeIcon icon={faPenNib} />,
},
];
export const topNavMediaId = signal<string>("");
@@ -154,9 +153,6 @@ export const TopBar = ({ pageName, loginSignUpPressed }: Props) => {
url,
mediaToken: mediaId || "",
};
// Add it to state history
useEditStore.getState().addHistoryImageBundle({ images: [baseImage] });
useEditStore.getState().setBaseImageInfo(baseImage);
// Close gallery modal and lightbox if open
@@ -11,14 +11,14 @@ import Konva from "konva";
import { setCanvasRenderBitmap } from "../../signals/canvasRenderBitmap";
import { captureStageImageBitmap } from "./hooks/useUpdateSnapshot";
import { ContextMenuContainer } from "./components/ui/ContextMenu";
import { ImageModel } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
import {
CANVAS_2D_PAGE_MODEL_LIST,
ClassyModelSelector,
ModelPage,
ModelSelector,
useModelSelectorStore,
} from "@storyteller/ui-model-selector";
import { useCanvasBgRemovedEvent } from "@storyteller/tauri-api";
import { getSelectedImageModel } from "@storyteller/ui-model-selector";
const PAGE_ID : ModelPage = ModelPage.Canvas2D;
@@ -50,12 +50,20 @@ export const DecodeBase64ToImage = async (
const PageDraw = () => {
const canvasWidth = useRef<number>(1024);
const canvasHeight = useRef<number>(1024);
const { selectedModels } = useModelSelectorStore();
const [isSelecting, setIsSelecting] = useState<boolean>(false);
const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
const store = useSceneStore();
const selectedImageModel : ImageModel | undefined = getSelectedImageModel(PAGE_ID);
const selectedModel =
selectedModels[PAGE_ID] ||
CANVAS_2D_PAGE_MODEL_LIST[0]?.label;
const selectedModelInfo: ModelInfo | undefined =
CANVAS_2D_PAGE_MODEL_LIST.find(
(m) => m.label === selectedModel,
)?.modelInfo;
useDeleteHotkeys({ onDelete: store.deleteSelectedItems });
useUndoRedoHotkeys({ undo: store.undo, redo: store.redo });
@@ -297,8 +305,7 @@ const PageDraw = () => {
}}
onEnqueuePressed={onEnqueuedPressed}
onFitPressed={onFitPressed}
//selectedModelInfo={selectedModelInfo}
selectedImageModel={selectedImageModel}
selectedModelInfo={selectedModelInfo}
/>
</div>
<SideToolbar
@@ -439,7 +446,7 @@ const PageDraw = () => {
</ContextMenuContainer>
</div>
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<ClassyModelSelector
<ModelSelector
items={CANVAS_2D_PAGE_MODEL_LIST}
page={PAGE_ID}
panelTitle="Select Model"
@@ -9,14 +9,12 @@ import { JobProvider, useJobContext } from "../JobContext";
// Set this value on when enqueue is pressed nasty global variable.
import { getCanvasRenderBitmap } from "../../../signals/canvasRenderBitmap";
const PromptEditor: React.FC<PromptEditorProps> = ({
onImageStyleChange,
onEnqueuePressed,
onAspectRatioChange,
onFitPressed,
//selectedModelInfo,
selectedImageModel,
selectedModelInfo,
}) => {
const [images, setImages] = useState<ImageStyle[]>([]);
@@ -46,8 +44,7 @@ const PromptEditor: React.FC<PromptEditorProps> = ({
onEnqueuePressed={onEnqueuePressed}
onAspectRatioChange={onAspectRatioChange}
onFitPressed={onFitPressed}
//selectedModelInfo={selectedModelInfo}
selectedImageModel={selectedImageModel}
selectedModelInfo={selectedModelInfo}
/>
</JobProvider>
@@ -1,5 +1,4 @@
import { ImageModel } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
import { ModelInfo } from "libs/model-list/src/lib/ModelInfo";
export interface PromptEditorProps {
initialPrompt?: string;
@@ -12,7 +11,6 @@ export interface PromptEditorProps {
onEnqueuePressed?: () => void;
onFitPressed?: () => void;
selectedModelInfo?: ModelInfo;
selectedImageModel?: ImageModel;
}
export type AspectRatio = "1:1" | "3:2" | "2:3";
@@ -1,19 +1,11 @@
import { Button } from "@storyteller/ui-button";
import { useImageEditCompleteEvent } from "@storyteller/tauri-events";
import {
faClockRotateLeft,
faTrash,
faTrashXmark,
faXmark,
} from "@fortawesome/pro-solid-svg-icons";
import { Fragment, useRef, useState } from "react";
import { faClockRotateLeft, faTrash, faTrashXmark, faXmark } from "@fortawesome/pro-solid-svg-icons";
import { useRef, useState } from "react";
import { twMerge } from "tailwind-merge";
import { BaseSelectorImage } from "./BaseImageSelector";
import { Tooltip } from "@storyteller/ui-tooltip";
import {
isActionReminderOpen,
showActionReminder,
} from "@storyteller/ui-action-reminder-modal";
import { isActionReminderOpen, showActionReminder } from "@storyteller/ui-action-reminder-modal";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
export interface ImageBundle {
@@ -23,18 +15,21 @@ export interface ImageBundle {
interface HistoryStackProps {
onClear: () => void;
onImageSelect?: (image: BaseSelectorImage) => void;
imageBundles: ImageBundle[];
onNewImageBundle?: (newBundle: ImageBundle) => void;
selectedImageToken?: string;
startingBundles: ImageBundle[];
}
export const HistoryStack = ({
onClear,
onImageSelect = () => { },
imageBundles,
onNewImageBundle = () => { },
selectedImageToken
startingBundles = [],
}: HistoryStackProps) => {
const [imageBundles, setImageBundles] =
useState<ImageBundle[]>(startingBundles);
const [selectedImageToken, setSelectedImageToken] = useState<string | null>(
startingBundles.length > 0 && startingBundles[0].images.length > 0
? startingBundles[0].images[0].mediaToken
: null,
);
useImageEditCompleteEvent(async (event) => {
const newBundle: ImageBundle = {
images: event.edited_images.map(
@@ -46,7 +41,12 @@ export const HistoryStack = ({
),
};
onNewImageBundle(newBundle);
setImageBundles([...imageBundles, newBundle]);
const newlySelected = newBundle.images[0];
if (newlySelected) {
setSelectedImageToken(newlySelected.mediaToken);
onImageSelect(newlySelected);
}
});
// This is used to force image reloads in different sessions
@@ -54,23 +54,27 @@ export const HistoryStack = ({
const sessionRandBuster = useRef(Math.random());
const handleClear = () => {
setImageBundles(startingBundles);
setSelectedImageToken(null);
onClear();
};
return (
<div className="h-auto w-20 rounded-lg">
<div className={"glass max-h-[50vh] flex flex-col-reverse items-center justify-center gap-2 overflow-y-auto rounded-lg p-1.5"}>
<div className="max-h-1/2 h-auto w-20 overflow-y-auto rounded-lg">
<div className="flex glass rounded-lg flex-col-reverse items-center justify-center gap-2 p-1">
{imageBundles.map((bundle, index) => (
<Fragment key={index}>
{bundle.images.map((image) => (
<>
{index < imageBundles.length - 1 && <hr className="h-0.5 w-3/4 rounded-md border-none bg-white/10" />}
{bundle.images.map((image, imgIndex) => (
<Button
key={image.mediaToken}
key={imgIndex}
className={twMerge(
"relative aspect-square h-full w-full border-2 bg-transparent p-0 hover:bg-transparent hover:opacity-80",
"aspect-square relative h-full w-full border-2 bg-transparent p-0 m-1 hover:bg-transparent hover:opacity-80",
selectedImageToken === image.mediaToken &&
"border-primary hover:opacity-100",
)}
onClick={() => {
setSelectedImageToken(image.mediaToken);
onImageSelect(image);
}}
>
@@ -83,54 +87,35 @@ export const HistoryStack = ({
/>
</Button>
))}
{index < imageBundles.length - 1 && (
<hr className="min-h-0.5 h-0.5 w-3/4 rounded-md border-none bg-white/10" key={"hr" + index} />
)}
</Fragment>
</>
))}
<FontAwesomeIcon
icon={faClockRotateLeft}
className="p-1 text-gray-400"
/>
<FontAwesomeIcon icon={faClockRotateLeft} className="p-1 text-gray-400" />
</div>
<div className="mt-3 flex justify-center">
<div className="glass w-fit rounded-xl border-2 border-red/50 shadow-lg hover:border-red/80">
<div className="relative h-full">
<Tooltip
content="Reset All"
position="left"
closeOnClick={true}
className="ms-1 rounded-md bg-red px-3 py-1"
delay={100}
>
<button
className="flex h-10 w-10 items-center justify-center rounded-lg border-2 border-transparent text-white transition-colors hover:bg-red/50"
onClick={() =>
showActionReminder({
reminderType: "default",
title: "Reset All",
primaryActionIcon: faTrashXmark,
primaryActionBtnClassName: "bg-red hover:bg-red/80",
message: (
<p className="text-sm text-white/70">
Are you sure you want to reset all? This will clear all
your work and cannot be undone.
</p>
),
primaryActionText: "Reset all",
onPrimaryAction: () => {
handleClear();
isActionReminderOpen.value = false;
},
})
}
>
<FontAwesomeIcon icon={faXmark} className="h-5 w-5 text-xl" />
</button>
</Tooltip>
</div>
</div>
<div className="glass mt-4 rounded-full border-red/50 border-2 size-12 flex justify-center items-center mx-auto">
<button
className="flex h-full w-full aspect-square rounded-full items-center justify-center text-white transition-colors hover:bg-red/50"
onClick={() =>
showActionReminder({
reminderType: "default",
title: "Reset All",
primaryActionIcon: faTrashXmark,
primaryActionBtnClassName: "bg-red hover:bg-red/80",
message: (
<p className="text-sm text-white/70">
Are you sure you want to reset all? This will clear all your
work and cannot be undone.
</p>
),
primaryActionText: "Reset all",
onPrimaryAction: () => {
handleClear();
isActionReminderOpen.value = false;
},
})
}
>
<FontAwesomeIcon icon={faXmark} className="h-5 w-5 text-xl" />
</button>
</div>
</div>
);
@@ -2,6 +2,7 @@ import { useState, useRef, useEffect, useCallback } from "react";
import Konva from "konva"; // just for types
import {
EnqueueImageInpaint,
EnqueueImageInpaintModel,
} from "@storyteller/tauri-api";
import { ContextMenuContainer } from "../PageDraw/components/ui/ContextMenu";
import { useCopyPasteHotkeys } from "../PageDraw/hooks/useCopyPasteHotkeys";
@@ -10,24 +11,34 @@ import { useUndoRedoHotkeys } from "../PageDraw/hooks/useUndoRedoHotkeys";
import PromptEditor from "./PromptEditor/PromptEditor";
import { ActiveEditTool, useEditStore } from "./stores/EditState";
import { EditPaintSurface } from "./EditPaintSurface";
import { drawAlphaMask, normalizeCanvas } from "../../Helpers/CanvasHelpers";
import { normalizeCanvas } from "../../Helpers/CanvasHelpers";
import { BaseImageSelector, BaseSelectorImage } from "./BaseImageSelector";
import DrawToolControlBar from "./DrawToolControlBar";
import {
ClassyModelSelector,
getSelectedImageModel,
IMAGE_EDITOR_PAGE_MODEL_LIST,
ModelPage,
ModelSelector,
useModelSelectorStore,
} from "@storyteller/ui-model-selector";
import { ImageModel } from "@storyteller/model-list";
import { lookupModelByTauriId, ModelInfo } from "@storyteller/model-list";
import { HistoryStack, ImageBundle } from "./HistoryStack";
import { ModelTag } from "libs/model-list/src/lib/ModelTag";
const PAGE_ID: ModelPage = ModelPage.ImageEditor;
const PageEdit = () => {
//useStateSceneLoader();
const { selectedModels } = useModelSelectorStore();
const selectedImageModel: ImageModel | undefined = getSelectedImageModel(PAGE_ID);
const selectedModelLabel =
selectedModels[PAGE_ID] || IMAGE_EDITOR_PAGE_MODEL_LIST[0]?.label;
const selectedModel = IMAGE_EDITOR_PAGE_MODEL_LIST.find(
(m) => m.label === selectedModelLabel,
);
const selectedModelInfo: ModelInfo | undefined =
selectedModel?.modelInfo;
// State for canvas dimensions
const canvasWidth = useRef<number>(1024);
@@ -46,8 +57,6 @@ const PageEdit = () => {
// Use the Zustand store
const store = useEditStore();
const addHistoryImageBundle = useEditStore((state) => state.addHistoryImageBundle);
const historyImageBundles = useEditStore((state) => state.historyImageBundles)
// Pass store actions directly as callbacks
useDeleteHotkeys({ onDelete: store.deleteSelectedItems });
@@ -156,6 +165,18 @@ const PageEdit = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
// Auto-fit the canvas when base image is loaded
useEffect(() => {
if (!store.baseImageBitmap) {
console.log("No base image bitmap available, skipping auto-fit");
return;
}
// When baseImageBitmap changes from null to an actual image, call onFitPressed
// Use a slight delay to ensure the image dimensions are properly loaded
onFitPressed();
}, [onFitPressed, store.baseImageBitmap]);
// Create a function to use the left layer ref and download the bitmap from it
const getMaskArrayBuffer = async (): Promise<Uint8Array> => {
if (
@@ -187,10 +208,6 @@ const PageEdit = () => {
rect.height(),
);
// Convert colored canvas to alpha mask
// NOTE: This isn't needed because the tauri backend uses the alpha channel anyway
// drawAlphaMask(fittedCanvas, rect.width(), rect.height());
const blob = await fittedCanvas.convertToBlob({ type: "image/png" });
const arrayBuffer = await blob.arrayBuffer();
@@ -218,7 +235,7 @@ const PageEdit = () => {
try {
await EnqueueImageInpaint({
model: selectedImageModel,
model: selectedModelInfo,
image_media_token: editedImageToken,
mask_image_raw_bytes: arrayBuffer,
prompt: prompt,
@@ -232,23 +249,19 @@ const PageEdit = () => {
[
generationCount,
isEnqueuing,
selectedImageModel,
selectedModelInfo,
store.baseImageInfo?.mediaToken,
],
);
//const modelConfig = lookupModelByTauriId(selectedImageModel!.tauriId);
//const supportsMaskedInpainting = modelConfig?.tags?.includes(ModelTag.MaskedInpainting) ?? false;
const supportsMaskedInpainting = selectedImageModel?.usesInpaintingMask ?? false;
useEffect(() => {
if (!supportsMaskedInpainting && (store.activeTool !== "select" || store.lineNodes.length > 0)) {
// TODO: Implement a new mode for unsupported masking and hide the nodes layer instead of clearing
store.setActiveTool("select");
store.clearLineNodes();
}
}, [store, supportsMaskedInpainting]);
const modelConfig = lookupModelByTauriId(selectedModelInfo!.tauri_id);
const supportsMaskedInpainting = modelConfig?.tags?.includes(ModelTag.MaskedInpainting) ?? false;
if (!supportsMaskedInpainting && (store.activeTool !== "select" || store.lineNodes.length > 0)) {
// TODO: Implement a new mode for unsupported masking and hide the nodes layer instead of clearing
store.setActiveTool("select");
store.clearLineNodes();
}
/*
*
@@ -270,7 +283,6 @@ const PageEdit = () => {
>
<BaseImageSelector
onImageSelect={(image: BaseSelectorImage) => {
addHistoryImageBundle({ images: [image] });
store.setBaseImageInfo(image);
}}
showLoading={
@@ -303,13 +315,11 @@ const PageEdit = () => {
onClear={() => {
store.RESET();
}}
imageBundles={historyImageBundles}
startingBundles={[{ images: [store.baseImageInfo] } as ImageBundle]}
onImageSelect={(baseImage) => {
store.clearLineNodes();
store.setBaseImageInfo(baseImage);
}}
onNewImageBundle={addHistoryImageBundle}
selectedImageToken={store.baseImageInfo?.mediaToken}
/>
</div>
<div
@@ -317,7 +327,7 @@ const PageEdit = () => {
}`}
>
<PromptEditor
selectedImageModel={selectedImageModel}
modelInfo={selectedModelInfo}
onModeChange={(mode: string) => {
store.setActiveTool(mode as ActiveEditTool);
}}
@@ -372,9 +382,9 @@ const PageEdit = () => {
canvasWidth.current = width;
canvasHeight.current = height;
}}
//fillColor={store.fillColor}
fillColor={store.fillColor}
activeTool={store.activeTool}
brushColor={"rgba(39, 187, 245, 0.54)"}
brushColor={store.brushColor}
brushSize={store.brushSize}
onSelectionChange={setIsSelecting}
stageRef={stageRef}
@@ -385,7 +395,7 @@ const PageEdit = () => {
</ContextMenuContainer>
</div>
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<ClassyModelSelector
<ModelSelector
items={IMAGE_EDITOR_PAGE_MODEL_LIST}
page={PAGE_ID}
panelTitle="Select Model"
@@ -7,7 +7,7 @@ const PromptEditor: React.FC<PromptBoxEditProps> = ({
onGenerateClick,
isDisabled,
onFitPressed,
selectedImageModel,
modelInfo,
generationCount,
onGenerationCountChange,
supportsMaskedInpainting
@@ -20,7 +20,7 @@ const PromptEditor: React.FC<PromptBoxEditProps> = ({
onGenerateClick={onGenerateClick}
isDisabled={isDisabled}
onFitPressed={onFitPressed}
selectedImageModel={selectedImageModel}
modelInfo={modelInfo}
generationCount={generationCount}
onGenerationCountChange={onGenerationCountChange}
supportsMaskedInpainting={supportsMaskedInpainting}
@@ -3,7 +3,6 @@ import { LineNode, generateId } from "~/pages/PageDraw/stores/SceneState";
import { Node } from "~/pages/PageDraw/Node";
import { BaseSelectorImage } from "../BaseImageSelector";
import { FetchProxy } from "libs/tauri-utils/src/lib/FetchProxy";
import { ImageBundle } from "../HistoryStack";
export type ActiveEditTool = "select" | "edit" | "expand";
export type EditOperation = "add" | "minus";
@@ -127,11 +126,6 @@ interface EditState {
// pushHistory: () => void;
// clearHistory: () => void;
RESET: () => void;
// For the history stack
historyImageBundles: ImageBundle[];
clearHistoryImages: () => void;
addHistoryImageBundle: (bundle: ImageBundle) => void;
}
// Add a flag to prevent saving state during restoration
@@ -178,9 +172,6 @@ export const useEditStore = create<EditState>((set, get, store) => ({
baseImageInfo: null,
baseImageBitmap: null,
// History stack state
historyImageBundles: [],
// Actions
addNode: (node: Node) => {
set((state) => {
@@ -1018,13 +1009,5 @@ export const useEditStore = create<EditState>((set, get, store) => ({
RESET: () => {
set(store.getInitialState());
},
clearHistoryImages: () => {
set({ historyImageBundles: [] });
},
addHistoryImageBundle(bundle) {
set((state) => ({ historyImageBundles: [...state.historyImageBundles, bundle] }));
},
}
}));
@@ -68,20 +68,21 @@ import {
import {
STAGE_3D_PAGE_MODEL_LIST,
ModelPage,
getSelectedImageModel,
ClassyModelSelector,
ModelSelector,
useModelSelectorStore,
} from "@storyteller/ui-model-selector";
import { LoginModal, useLoginModalStore } from "@storyteller/ui-login-modal";
import PageDraw from "../PageDraw/PageDraw";
import { useTabStore } from "../Stores/TabState";
import PageEdit from "../PageEdit/PageEdit";
import { ImageModel } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
const PAGE_ID : ModelPage = ModelPage.Stage3D;
export const PageEditor = () => {
useSignals();
const { triggerRecheck } = useLoginModalStore();
const { selectedModels } = useModelSelectorStore();
const { status } = authentication;
useEffect(() => {
@@ -95,6 +96,12 @@ export const PageEditor = () => {
event.stopPropagation();
};
// TODO: For 3d Promptbox to accept this later on
// const { selectedModels } = useModelSelectorStore();
// const selectedModel =
// selectedModels[ModelCategory.Editor3D] ||
// videoGenerationModels[0]?.label;
useEffect(() => {
timelineHeight.value = 0; //timelineHeight.value = 208;
sidePanelWidth.value = 340;
@@ -103,7 +110,14 @@ export const PageEditor = () => {
};
}, []);
const selectedImageModel : ImageModel | undefined = getSelectedImageModel(PAGE_ID);
const selectedModel =
selectedModels[PAGE_ID] ||
STAGE_3D_PAGE_MODEL_LIST[0]?.label;
const selectedModelInfo: ModelInfo | undefined =
STAGE_3D_PAGE_MODEL_LIST.find(
(m) => m.label === selectedModel,
)?.modelInfo;
const height =
dndTimelineHeight.value > -1
@@ -512,7 +526,7 @@ export const PageEditor = () => {
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
selectedImageModel={selectedImageModel}
selectedModelInfo={selectedModelInfo}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
@@ -534,7 +548,7 @@ export const PageEditor = () => {
/>
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<ClassyModelSelector
<ModelSelector
items={STAGE_3D_PAGE_MODEL_LIST}
page={PAGE_ID}
panelTitle="Select Model"
@@ -5,14 +5,15 @@ import BackgroundGallery from "./BackgroundGallery";
import {
TEXT_TO_IMAGE_PAGE_MODEL_LIST,
ModelPage,
ClassyModelSelector,
getSelectedImageModel,
ModelSelector,
useModelSelectorStore,
} from "@storyteller/ui-model-selector";
import { ImageModel } from "@storyteller/model-list";
interface TextToImageProps {
imageMediaId?: string;
imageUrl?: string;
}
import { ModelInfo } from "@storyteller/model-list";
import { useTextToImageGenerationCompleteEvent } from "@storyteller/tauri-events";
import { useTextToImageStore } from "./TextToImageStore";
import { animated, useSpring } from "@react-spring/web";
@@ -26,12 +27,19 @@ const PAGE_ID: ModelPage = ModelPage.TextToImage;
const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
const containerRef = useRef<HTMLDivElement>(null);
const { selectedModels } = useModelSelectorStore();
const batches = useTextToImageStore((s) => s.batches);
const startBatch = useTextToImageStore((s) => s.startBatch);
const completeBatch = useTextToImageStore((s) => s.completeBatch);
const resetBatches = useTextToImageStore((s) => s.reset);
const selectedImageModel : ImageModel | undefined = getSelectedImageModel(PAGE_ID);
const selectedModel =
selectedModels[PAGE_ID] || TEXT_TO_IMAGE_PAGE_MODEL_LIST[0]?.label;
const selectedModelInfo: ModelInfo | undefined =
TEXT_TO_IMAGE_PAGE_MODEL_LIST.find(
(m) => m.label === selectedModel,
)?.modelInfo;
const jobContext: JobContextType = {
jobTokens: [],
@@ -68,10 +76,6 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
config: { tension: 200, friction: 28, mass: 1.1 },
});
// Show the batches in reverse order, with the newest items at top.
// Like Midjourney instead of a "chat history" style.
const inverseBatch = batches.toReversed();
return (
<div
ref={containerRef}
@@ -91,8 +95,19 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
{hasAnyBatches && (
<div className="h-full w-full overflow-y-auto pb-40">
<div className="mx-auto flex max-w-screen-2xl flex-col gap-8 pr-2">
{inverseBatch.map((batch) => (
{batches.map((batch) => (
<div key={batch.id} className="flex items-start gap-4">
<div>
<div className="glass inline-block w-[320px] shrink-0 rounded-xl px-4 py-3 text-left text-sm text-white/90">
<div>{batch.prompt}</div>
</div>
<div className="mt-2 flex justify-end">
<Badge
label={batch.modelLabel}
className="px-2 py-1 text-xs opacity-70"
/>
</div>
</div>
<div className="grid flex-1 grid-cols-4 gap-4">
{batch.status === "pending" && batch.images.length === 0
? Array.from({
@@ -132,17 +147,6 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
</button>
))}
</div>
<div>
<div className="glass inline-block w-[320px] shrink-0 rounded-xl px-4 py-3 text-left text-sm text-white/90">
<div>{batch.prompt}</div>
</div>
<div className="mt-2 flex justify-end">
<Badge
label={batch.modelLabel}
className="px-2 py-1 text-xs opacity-70"
/>
</div>
</div>
</div>
))}
</div>
@@ -167,12 +171,12 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
useJobContext={() => {
return jobContext;
}}
selectedModel={selectedImageModel}
model={selectedModel}
modelInfo={selectedModelInfo}
imageMediaId={imageMediaId}
url={imageUrl ?? undefined}
onEnqueuePressed={async (prompt, count, subscriberId) => {
const modelLabel = selectedImageModel?.fullName ?? "";
startBatch(prompt, count, modelLabel, subscriberId);
startBatch(prompt, count, selectedModel, subscriberId);
}}
/>
</animated.div>
@@ -180,7 +184,7 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
{!showPromptAtBottom && <BackgroundGallery />}
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<ClassyModelSelector
<ModelSelector
items={TEXT_TO_IMAGE_PAGE_MODEL_LIST}
page={PAGE_ID}
mode="hoverSelect"
@@ -3,12 +3,12 @@ import { JobContextType } from "@storyteller/common";
import { PromptBoxVideo } from "@storyteller/ui-promptbox";
import BackgroundGallery from "./BackgroundGallery";
import {
ClassyModelSelector,
IMAGE_TO_VIDEO_PAGE_MODEL_LIST,
ModelPage,
ModelSelector,
useModelSelectorStore,
IMAGE_TO_VIDEO_PAGE_MODEL_LIST,
} from "@storyteller/ui-model-selector";
import { VideoModel } from "@storyteller/model-list";
import { getSelectedVideoModel } from "@storyteller/ui-model-selector";
import { ModelInfo } from "@storyteller/model-list";
const PAGE_ID : ModelPage = ModelPage.ImageToVideo;
@@ -19,8 +19,15 @@ interface ImageToVideoProps {
const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
const containerRef = useRef<HTMLDivElement>(null);
const { selectedModels } = useModelSelectorStore();
const selectedVideoModel : VideoModel | undefined = getSelectedVideoModel(PAGE_ID);
const selectedModel =
selectedModels[PAGE_ID] ||
IMAGE_TO_VIDEO_PAGE_MODEL_LIST[0]?.label;
const selectedModelInfo: ModelInfo | undefined = IMAGE_TO_VIDEO_PAGE_MODEL_LIST.find(
(m) => m.label === selectedModel,
)?.modelInfo;
const jobContext: JobContextType = {
jobTokens: [],
@@ -49,7 +56,8 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
console.log("Using job context");
return jobContext;
}}
selectedModel={selectedVideoModel}
model={selectedModel}
modelInfo={selectedModelInfo}
imageMediaId={imageMediaId}
url={imageUrl ?? undefined}
onEnqueuePressed={async () => {
@@ -60,7 +68,7 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
<BackgroundGallery />
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<ClassyModelSelector
<ModelSelector
items={IMAGE_TO_VIDEO_PAGE_MODEL_LIST}
page={PAGE_ID}
panelTitle="Select Model"
@@ -3,7 +3,7 @@ import { useSceneStore } from '../PageDraw/stores/SceneState';
export type TabId = '2D' | '3D' | 'VIDEO' | 'IMAGE' | 'EDIT';
const DEFAULT_TAB : TabId = 'IMAGE';
const DEFAULT_TAB : TabId = '2D';
interface TabState {
// Current active tab
@@ -1,5 +1,4 @@
export * from "./lib/model-selectors-for-pages";
export * from "./lib/model-selector";
export * from "./lib/model-selector-store";
export * from "./lib/model-lists";
export * from "./lib/model-pages";
export * from "./lib/classy-model-selector";
export * from "./lib/classy-model-selector-store";
@@ -1,37 +0,0 @@
import { create } from "zustand";
import { ModelPage } from "./model-pages";
import { ImageModel, Model, VideoModel } from "@storyteller/model-list";
interface ClassyModelSelectorState {
selectedModels: { [page in ModelPage]?: Model };
setSelectedModel: (page: ModelPage, model: Model) => void;
}
export const useClassyModelSelectorStore = create<ClassyModelSelectorState>((set) => ({
selectedModels: {},
setSelectedModel: (page, model) =>
set((state) => ({
selectedModels: {
...state.selectedModels,
[page]: model,
},
})),
}));
export const getSelectedImageModel = (page: ModelPage) : ImageModel | undefined => {
const { selectedModels } = useClassyModelSelectorStore();
const maybeModel = selectedModels[page];
if (!maybeModel || !(maybeModel instanceof ImageModel)) {
return undefined;
}
return maybeModel;
};
export const getSelectedVideoModel = (page: ModelPage) : VideoModel | undefined => {
const { selectedModels } = useClassyModelSelectorStore();
const maybeModel = selectedModels[page];
if (!maybeModel || !(maybeModel instanceof VideoModel)) {
return undefined;
}
return maybeModel;
};
@@ -0,0 +1,57 @@
import type { PopoverItem } from "@storyteller/ui-popover";
import { faClock, faFilm, faImage } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
getCreatorIcon,
ModelConfig,
TEXT_TO_IMAGE_PAGE_MODELS,
IMAGE_TO_VIDEO_PAGE_MODELS,
CANVAS_2D_PAGE_MODELS,
STAGE_3D_PAGE_MODELS,
IMAGE_EDITOR_PAGE_MODELS,
} from "@storyteller/model-list";
export type ModelList = Omit<PopoverItem, "selected">[];
const withIcon = (creatorIcon: any, fallback: any) => creatorIcon || fallback;
const buildItems = (
models: ModelConfig[],
fallbackIcon: any
) =>
models.map((m) => ({
label: m.label,
icon: withIcon(getCreatorIcon(m.info.creator), fallbackIcon),
description: m.description,
badges: m.badges?.map((b) => ({
label: b.label,
icon: <FontAwesomeIcon icon={faClock} />,
})),
modelConfig: m, // Access to full object.
modelInfo: m.info,
}));
export const TEXT_TO_IMAGE_PAGE_MODEL_LIST : ModelList = buildItems(
TEXT_TO_IMAGE_PAGE_MODELS,
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const IMAGE_TO_VIDEO_PAGE_MODEL_LIST : ModelList = buildItems(
IMAGE_TO_VIDEO_PAGE_MODELS,
<FontAwesomeIcon icon={faFilm} className="h-4 w-4" />
);
export const CANVAS_2D_PAGE_MODEL_LIST : ModelList = buildItems(
CANVAS_2D_PAGE_MODELS,
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const STAGE_3D_PAGE_MODEL_LIST : ModelList = buildItems(
STAGE_3D_PAGE_MODELS,
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const IMAGE_EDITOR_PAGE_MODEL_LIST : ModelList = buildItems(
IMAGE_EDITOR_PAGE_MODELS,
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
@@ -0,0 +1,18 @@
import { create } from "zustand";
import { ModelPage } from "./model-pages";
interface ModelSelectorState {
selectedModels: { [page in ModelPage]?: string };
setSelectedModel: (page: ModelPage, modelLabel: string) => void;
}
export const useModelSelectorStore = create<ModelSelectorState>((set) => ({
selectedModels: {},
setSelectedModel: (page, modelLabel) =>
set((state) => ({
selectedModels: {
...state.selectedModels,
[page]: modelLabel,
},
})),
}));
@@ -0,0 +1,12 @@
import { render } from '@testing-library/react';
import ModelSelector from './model-selector';
describe('ModelSelector', () => {
it('should render successfully', () => {
const { baseElement } = render(<ModelSelector />);
expect(baseElement).toBeTruthy();
});
});
@@ -1,9 +1,9 @@
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
import { useClassyModelSelectorStore } from "./classy-model-selector-store";
import { useModelSelectorStore } from "./model-selector-store";
import { useEffect, useMemo } from "react";
import { ModelPage } from "./model-pages";
interface ClassyModelSelectorProps {
interface ModelSelectorProps {
items: Omit<PopoverItem, "selected">[];
page: ModelPage;
mode?: "hoverSelect" | "default" | "toggle" | "button";
@@ -14,31 +14,31 @@ interface ClassyModelSelectorProps {
triggerLabel?: string;
}
export function ClassyModelSelector({
export function ModelSelector({
items,
page,
...popoverProps
}: ClassyModelSelectorProps) {
const { selectedModels, setSelectedModel } = useClassyModelSelectorStore();
const selectedModel = selectedModels[page] || items[0]?.model;
}: ModelSelectorProps) {
const { selectedModels, setSelectedModel } = useModelSelectorStore();
const selectedModel = selectedModels[page] || items[0]?.label;
// For the first mount, make sure the selected model is set for other components to listen
useEffect(() => {
// Initialize selected model if not set
if (!selectedModels[page] && items[0]) {
setSelectedModel(page, items[0].model!);
setSelectedModel(page, items[0].label);
}
}, []);
const handleModelSelect = (item: PopoverItem) => {
setSelectedModel(page, item.model!);
setSelectedModel(page, item.label);
};
const modelList = useMemo(
() =>
items.map((item) => ({
...item,
selected: item.model === selectedModel,
selected: item.label === selectedModel,
})),
[items, selectedModel]
);
@@ -53,4 +53,4 @@ export function ClassyModelSelector({
);
}
export default ClassyModelSelector;
export default ModelSelector;
@@ -1,71 +0,0 @@
import type { PopoverItem } from "@storyteller/ui-popover";
import { faClock, faFilm, faImage } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
getCreatorIcon,
Model,
VIDEO_MODELS,
IMAGE_MODELS,
ALL_MODELS_BY_ID,
} from "@storyteller/model-list";
import { ModelTag } from "@storyteller/model-list";
export type ModelList = Omit<PopoverItem, "selected">[];
const withIcon = (creatorIcon: any, fallback: any) => creatorIcon || fallback;
const buildItems2 = (
models: Model[],
fallbackIcon: any
) =>
models.map((model : Model) => ({
label: model.selectorName,
icon: withIcon(getCreatorIcon(model.creator), fallbackIcon),
description: model.selectorDescription,
badges: model.toLegacyBadges()?.map((b) => ({
label: b.label,
icon: <FontAwesomeIcon icon={faClock} />,
})),
modelConfig: model.toLegacyModelConfig(), // Access to full object.
model: model,
}));
/**
* IN-PROGRESS MIGRATION (messy for now)
* We're gradually going to phase out ModelList, ModelConfig, etc.
* We won't index by name, but rather id, or simply will always have full
* access to the object directly.
*/
export const CANVAS_2D_PAGE_MODEL_LIST : ModelList = buildItems2(
IMAGE_MODELS.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit)),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const STAGE_3D_PAGE_MODEL_LIST : ModelList = buildItems2(
IMAGE_MODELS.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit)),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const IMAGE_EDITOR_PAGE_MODEL_LIST : ModelList = buildItems2(
//[
// ALL_MODELS_BY_ID.get("flux_pro_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_dev_juggernaut_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_pro_kontext_max")!,
//],
IMAGE_MODELS.filter((m) => m.canEditImages),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
export const IMAGE_TO_VIDEO_PAGE_MODEL_LIST : ModelList = buildItems2(
VIDEO_MODELS,
<FontAwesomeIcon icon={faFilm} className="h-4 w-4" />
);
export const TEXT_TO_IMAGE_PAGE_MODEL_LIST : ModelList = buildItems2(
IMAGE_MODELS
.filter((model) => !model.usesInpaintingMask) // We can't use masked inpainting models here
.filter((model) => model.id !== "flux_pro_kontext_max") // NB: Flux Pro Kontext Max requires an input image
,
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
@@ -13,7 +13,7 @@ import {
faChevronUp,
faCircleCheck,
} from "@fortawesome/pro-solid-svg-icons";
import { Model, ModelInfo } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
export interface PopoverItem {
label: string;
@@ -28,7 +28,6 @@ export interface PopoverItem {
icon?: ReactNode;
}>;
modelInfo?: ModelInfo;
model?: Model; // NB: Let's migrate to using this.
}
interface PopoverMenuProps {
@@ -35,13 +35,19 @@ import { IsDesktopApp } from "@storyteller/tauri-utils";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { PromptsApi } from "@storyteller/api";
import {
// SoraImageRemix,
// SoraImageRemixAspectRatio,
// CheckSoraSession,
// SoraSessionState,
// waitForSoraLogin,
EnqueueContextualEditImage,
EnqueueContextualEditImageSize,
} from "@storyteller/tauri-api";
// import { showActionReminder } from "@storyteller/ui-action-reminder-modal";
// import { invoke } from "@tauri-apps/api/core";
import { usePrompt2DStore, RefImage } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
import { getCapabilitiesForModel } from "@storyteller/model-list";
import { ImageModel } from "@storyteller/model-list";
import { ModelInfo, getCapabilitiesForModel } from "@storyteller/model-list";
export type AspectRatio = "1:1" | "3:2" | "2:3";
@@ -55,7 +61,7 @@ interface PromptBox2DProps {
assetFile: File;
progressCallback: (newState: UploaderState) => void;
}) => Promise<void>;
selectedImageModel?: ImageModel;
selectedModelInfo?: ModelInfo;
getCanvasRenderBitmap: () => MaybeCanvasRenderBitmapType;
EncodeImageBitmapToBase64: (imageBitmap: ImageBitmap) => Promise<string>;
useJobContext: () => JobContextType;
@@ -71,7 +77,7 @@ export const PromptBox2D = ({
useJobContext,
onEnqueuePressed,
onAspectRatioChange,
selectedImageModel,
selectedModelInfo,
onFitPressed,
}: PromptBox2DProps) => {
useSignals();
@@ -123,7 +129,7 @@ export const PromptBox2D = ({
// Update generation count options from selected model capabilities
useEffect(() => {
const caps = getCapabilitiesForModel(selectedImageModel);
const caps = getCapabilitiesForModel(selectedModelInfo);
const items: PopoverItem[] = Array.from(
{ length: caps.maxGenerationCount },
(_, i) => i + 1
@@ -140,7 +146,7 @@ export const PromptBox2D = ({
);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedImageModel]);
}, [selectedModelInfo]);
// Keep UI selection in sync when store value changes
useEffect(() => {
@@ -312,7 +318,31 @@ export const PromptBox2D = ({
setPrompt(e.target.value);
};
// Helper to show Sora login reminder and wait for login
// const handleSoraLoginReminder = async () => {
// return new Promise<void>((resolve) => {
// showActionReminder({
// reminderType: "soraLogin",
// onPrimaryAction: async () => {
// await invoke("open_sora_login_command");
// await waitForSoraLogin();
// toast.success("Logged in to Sora!");
// resolve();
// },
// });
// });
// };
const handleTauriEnqueue = async () => {
// NB(bt): This needs to move to an error handler.
// // Check if the Sora session is valid
// const soraSession = await CheckSoraSession();
// if (soraSession.state !== SoraSessionState.Valid) {
// setIsEnqueueing(false);
// await handleSoraLoginReminder();
// return;
// }
const api = new PromptsApi();
let image = getCanvasRenderBitmap();
if (image === undefined) {
@@ -349,7 +379,7 @@ export const PromptBox2D = ({
const aspectRatio = getCurrentAspectRatio();
const generateResponse = await EnqueueContextualEditImage({
model: selectedImageModel,
model: selectedModelInfo,
scene_image_media_token: snapshotMediaToken.data!,
image_media_tokens: referenceImages.map((image) => image.mediaToken),
disable_system_prompt: !useSystemPrompt,
@@ -53,7 +53,7 @@ import {
// import { showActionReminder } from "@storyteller/ui-action-reminder-modal";
import { usePrompt3DStore, RefImage } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
import { ImageModel } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
interface PromptBox3DProps {
cameras: Signal<Camera[]>;
@@ -73,7 +73,7 @@ interface PromptBox3DProps {
handleCameraFocalLengthChange: (id: string, value: number) => void;
onAspectRatioSelect: (newRatio: CameraAspectRatio) => void;
setEnginePrompt: (prompt: string) => void;
selectedImageModel?: ImageModel;
selectedModelInfo?: ModelInfo;
snapshotCurrentFrame:
| ((shouldDownload?: boolean) => {
base64Snapshot: string;
@@ -100,7 +100,7 @@ export const PromptBox3D = ({
handleCameraFocalLengthChange,
onAspectRatioSelect,
setEnginePrompt,
selectedImageModel,
selectedModelInfo,
snapshotCurrentFrame,
}: PromptBox3DProps) => {
useSignals();
@@ -450,7 +450,7 @@ export const PromptBox3D = ({
const aspectRatio = getCurrentAspectRatio();
const generateResponse = await EnqueueContextualEditImage({
model: selectedImageModel,
model: selectedModelInfo,
scene_image_media_token: snapshotResult.data!,
image_media_tokens: referenceImages.map((image) => image.mediaToken),
disable_system_prompt: !useSystemPrompt,
@@ -15,7 +15,7 @@ import { ButtonIconSelect } from "@storyteller/ui-button-icon-select";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { useEffect, useRef, useState } from "react";
import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list";
import { ModelInfo, getCapabilitiesForModel } from "@storyteller/model-list";
export interface PromptBoxEditProps {
onModeChange?: (mode: string) => void;
@@ -23,7 +23,7 @@ export interface PromptBoxEditProps {
onGenerateClick: (prompt: string) => void;
isDisabled?: boolean;
onFitPressed?: () => void | Promise<void>;
selectedImageModel?: ImageModel;
modelInfo?: ModelInfo;
generationCount?: number;
onGenerationCountChange?: (count: number) => void;
supportsMaskedInpainting?: boolean;
@@ -35,7 +35,7 @@ export const PromptBoxEdit = ({
onGenerateClick,
isDisabled,
onFitPressed,
selectedImageModel,
modelInfo,
generationCount: generationCountProp,
onGenerationCountChange,
supportsMaskedInpainting = false,
@@ -69,7 +69,7 @@ export const PromptBoxEdit = ({
// Build generation count options based on selected model
useEffect(() => {
const caps = getCapabilitiesForModel(selectedImageModel);
const caps = getCapabilitiesForModel(modelInfo);
console.log(">>> caps", caps);
const items: PopoverItem[] = Array.from(
{ length: caps.maxGenerationCount },
@@ -91,7 +91,7 @@ export const PromptBoxEdit = ({
onGenerationCountChange?.(clamped);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedImageModel]);
}, [modelInfo]);
// Keep UI selection in sync when store/value changes
useEffect(() => {
@@ -28,7 +28,7 @@ import {
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IconProp } from "@fortawesome/fontawesome-svg-core";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { ImageModel, ModelInfo, getCapabilitiesForModel } from "@storyteller/model-list";
import { ModelInfo, getCapabilitiesForModel } from "@storyteller/model-list";
import { usePromptImageStore, RefImage } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
@@ -39,7 +39,8 @@ interface PromptBoxImageProps {
count: number,
subscriberId: string
) => void | Promise<void>;
selectedModel?: ImageModel;
model: string;
modelInfo?: ModelInfo;
imageMediaId?: string;
url?: string;
}
@@ -47,7 +48,8 @@ interface PromptBoxImageProps {
export const PromptBoxImage = ({
useJobContext,
onEnqueuePressed,
selectedModel,
model,
modelInfo,
imageMediaId,
url,
}: PromptBoxImageProps) => {
@@ -131,7 +133,7 @@ export const PromptBoxImage = ({
useEffect(() => {
// Build generation count options based on selected model
const caps = getCapabilitiesForModel(selectedModel);
const caps = getCapabilitiesForModel(modelInfo);
const defaultCount = Math.min(
Math.max(1, caps.defaultGenerationCount ?? 1),
caps.maxGenerationCount
@@ -149,7 +151,7 @@ export const PromptBoxImage = ({
}));
setGenerationCountList(items);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedModel]);
}, [modelInfo]);
useEffect(() => {
setGenerationCountList((prev) =>
@@ -283,7 +285,7 @@ export const PromptBoxImage = ({
const generateResponse = await EnqueueTextToImage({
prompt: prompt,
model: selectedModel,
model: modelInfo,
aspect_ratio: aspectRatio,
number_images: generationCount,
frontend_caller: "text_to_image",
@@ -21,14 +21,15 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IconProp } from "@fortawesome/fontawesome-svg-core";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { VideoModel } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
import { usePromptVideoStore, RefImage } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
interface PromptBoxVideoProps {
useJobContext: () => JobContextType;
onEnqueuePressed?: () => void | Promise<void>;
selectedModel?: VideoModel;
model: string;
modelInfo?: ModelInfo;
imageMediaId?: string;
url?: string;
}
@@ -36,7 +37,8 @@ interface PromptBoxVideoProps {
export const PromptBoxVideo = ({
useJobContext,
onEnqueuePressed,
selectedModel,
model,
modelInfo,
imageMediaId,
url,
}: PromptBoxVideoProps) => {
@@ -192,7 +194,7 @@ export const PromptBoxVideo = ({
const handleEnqueue = async () => {
setIsEnqueueing(true);
console.log("PromptBoxVideo - Prompting with model", selectedModel);
console.log("PromptBoxVideo - Prompting with model", modelInfo);
gtagEvent("enqueue_video");
@@ -204,7 +206,7 @@ export const PromptBoxVideo = ({
}, 10000);
const generateResponse = await EnqueueImageToVideo({
model: selectedModel,
model: modelInfo,
image_media_token: referenceImages[0].mediaToken,
prompt: prompt,
});
+7 -16
View File
@@ -1,16 +1,7 @@
export * from "./lib/classes/ImageModel.js";
export * from "./lib/classes/Model.js";
export * from "./lib/classes/VideoModel.js";
export * from "./lib/classes/metadata/ModelCreator.js";
export * from "./lib/classes/metadata/ModelCreatorIcons.js";
export * from "./lib/classes/metadata/ModelMapping.js";
export * from "./lib/classes/metadata/ModelTag.js";
export * from "./lib/legacy/ModelConfig.js";
export * from "./lib/legacy/ModelInfo.js";
export * from "./lib/legacy/Models.js";
export * from "./lib/legacy/ModelsByPage.js";
export * from "./lib/lists/AllModels.js";
export * from "./lib/lists/AllModelsById.js";
export * from "./lib/lists/ImageModels.js";
export * from "./lib/lists/VideoModels.js";
export * from "./lib/ModelConfig.js";
export * from "./lib/ModelCreator.js";
export * from "./lib/ModelCreatorIcons.js";
export * from "./lib/ModelInfo.js";
export * from "./lib/ModelMapping.js";
export * from "./lib/Models.js";
export * from "./lib/ModelsByPage.js";
@@ -1,5 +1,5 @@
import { ModelInfo } from "./ModelInfo.js";
import { ModelTag } from "../classes/metadata/ModelTag.js";
import { ModelTag } from "./ModelTag.js";
export type ModelCategory = "image" | "video";
@@ -1,4 +1,4 @@
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import { ModelCreator } from "./ModelCreator.js";
export interface ModelInfo {
// Human-readible name
@@ -1,6 +1,4 @@
// TODO: It might be better to have top-level boolean fields on the models.
export enum ModelTag {
// Models that show up in the 2D and 3D editors
InstructiveEdit = "instructiveEdit",
@@ -10,4 +8,4 @@ export enum ModelTag {
// Models that show up in the inpainting editor, but that do not use masking.
NonMaskedInpainting = "nonMaskedInpainting",
}
}
@@ -1,12 +1,11 @@
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import { ModelCreator } from "./ModelCreator.js";
import { ModelInfo } from "./ModelInfo.js";
import { ModelTag } from "../classes/metadata/ModelTag.js";
import { ModelTag } from "./ModelTag.js";
import {
ModelConfig,
ModelCapabilities,
ModelCategory,
} from "./ModelConfig.js";
import { Model } from "../classes/Model.js";
const mc = ModelCreator;
@@ -30,7 +29,7 @@ const cfg = (
...m,
});
const ALL_MODELS: ModelConfig[] = [
export const ALL_MODELS: ModelConfig[] = [
//////////////////////////////
// Image models
//////////////////////////////
@@ -214,9 +213,9 @@ const ALL_MODELS: ModelConfig[] = [
}),
];
// export const ALL_MODELS_BY_ID: Map<string, ModelConfig> = new Map(
// ALL_MODELS.map((model) => [model.id, model])
// );
export const ALL_MODELS_BY_ID: Map<string, ModelConfig> = new Map(
ALL_MODELS.map((model) => [model.id, model])
);
export const getAllModels = (): ModelConfig[] => ALL_MODELS;
@@ -240,10 +239,9 @@ export const lookupModelByTauriId = (
// Single exported capability resolver so callers never need to touch anything else
export const getCapabilitiesForModel = (
model?: Model | ModelInfo
model?: ModelInfo
): ModelCapabilities => {
if (!model) return DEFAULT_CAPABILITIES;
const tauriId = model instanceof Model ? model.tauriId : model.tauri_id;
const cfg = lookupModelByTauriId(tauriId);
const cfg = lookupModelByTauriId(model.tauri_id);
return cfg?.capabilities ?? DEFAULT_CAPABILITIES;
};
@@ -0,0 +1,36 @@
import { ALL_MODELS_BY_ID } from "./Models.js";
import { ModelConfig } from "./ModelConfig.js";
import { ModelTag } from "./ModelTag.js";
import { getModelsByCategory } from "./Models.js";
/**
* We might not want to always programatically build the list of models, because
* we likely want to control the order of models in the UI to prioritize
* more popular models. We may also want to selectively prune non-useful models.
*/
export const TEXT_TO_IMAGE_PAGE_MODELS : ModelConfig[] =
getModelsByCategory("image")
// Remove inpainting and editing models.
.filter((m) => m.id !== "flux_pro_inpaint")
.filter((m) => m.id !== "flux_pro_kontext_max");
export const IMAGE_TO_VIDEO_PAGE_MODELS : ModelConfig[] =
getModelsByCategory("video");
export const CANVAS_2D_PAGE_MODELS : ModelConfig[] =
getModelsByCategory("image")
// Remove non-instructive models.
.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
export const STAGE_3D_PAGE_MODELS : ModelConfig[] =
getModelsByCategory("image")
// Remove non-instructive models.
.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
export const IMAGE_EDITOR_PAGE_MODELS : ModelConfig[] = [
// Explicit list for now.
ALL_MODELS_BY_ID.get("flux_pro_inpaint")!,
ALL_MODELS_BY_ID.get("flux_dev_juggernaut_inpaint")!,
ALL_MODELS_BY_ID.get("flux_pro_kontext_max")!,
];
@@ -1,49 +0,0 @@
import { ModelCreator } from "src/index.js";
import { Model } from "./Model.js";
import { ModelCategory } from "../legacy/ModelConfig.js";
import { ModelTag } from "./metadata/ModelTag.js";
export class ImageModel extends Model {
// Maximum number of images that can be generated at once
readonly maxGenerationCount: number;
// Default number of images that can be generated at once
readonly defaultGenerationCount: number;
// Signals image editing models that focus on editing a single image.
readonly canEditImages: boolean;
// For inpainting models, does it require sending a mask?
readonly usesInpaintingMask: boolean;
constructor(args: {
id: string;
tauriId: string;
fullName: string;
category: ModelCategory;
creator: ModelCreator;
selectorName: string;
selectorDescription: string;
selectorBadges: string[];
maxGenerationCount: number;
defaultGenerationCount: number;
canEditImages?: boolean;
usesInpaintingMask?: boolean;
tags?: ModelTag[];
}) {
if (args.maxGenerationCount < 1) {
throw new Error("maxGenerationCount must be at least 1");
}
if (args.defaultGenerationCount < 1) {
throw new Error("defaultGenerationCount must be at least 1");
}
if (args.defaultGenerationCount > args.maxGenerationCount) {
throw new Error("defaultGenerationCount must be less than or equal to maxGenerationCount");
}
super(args);
this.maxGenerationCount = args.maxGenerationCount;
this.defaultGenerationCount = args.defaultGenerationCount;
this.canEditImages = args.canEditImages ?? false;
this.usesInpaintingMask = args.usesInpaintingMask ?? false;
}
}
@@ -1,82 +0,0 @@
import { ModelCreator } from "./metadata/ModelCreator.js";
import { ModelCategory, ModelConfig } from "../legacy/ModelConfig.js";
import { ModelTag } from "./metadata/ModelTag.js";
// NB: Do not create instances of this class directly, use subclasses.
export class Model {
// A unique frontend-only string for the model
readonly id: string;
// A unique identifier that Tauri uses for the model (this
// might differ from our backend or other systems)
readonly tauriId: string;
// A long name for the model that might need to be abbreviated.
readonly fullName: string;
// The type of model (image, video, etc.)
// TODO: Not sure that this is used for anything
readonly category: ModelCategory;
// What company made the model.
readonly creator: ModelCreator;
// Name for the selector
readonly selectorName: string;
// Description for the selector
readonly selectorDescription: string;
// Labels for the selector
readonly selectorBadges: string[];
// A list of filterable "capabilities" that can be used to filter models.
readonly tags: ModelTag[];
protected constructor(args: {
id: string;
tauriId: string;
fullName: string;
category: ModelCategory;
creator: ModelCreator;
selectorName: string;
selectorDescription: string;
selectorBadges: string[];
tags?: ModelTag[];
}) {
this.id = args.id;
this.tauriId = args.tauriId;
this.fullName = args.fullName;
this.category = args.category;
this.creator = args.creator;
this.selectorName = args.selectorName;
this.selectorDescription = args.selectorDescription;
this.selectorBadges = args.selectorBadges;
this.tags = args.tags ?? [];
}
toLegacyBadges() : { label: string }[] {
return this.selectorBadges.map((b) => ({ label: b }));
}
// TODO: This is a method to support migration. Kill it after we no longer need it.
toLegacyModelConfig(): ModelConfig {
return {
id: this.id,
label: this.selectorName,
description: this.selectorDescription,
badges: this.toLegacyBadges(),
category: this.category,
info: {
name: this.fullName,
tauri_id: this.tauriId,
creator: this.creator,
},
capabilities: {
maxGenerationCount: 9, // NB: Sentinel value to detect continued use
defaultGenerationCount: 9, // NB: Sentinel value to detect continued use
},
tags: [],
};
}
}
@@ -1,30 +0,0 @@
import { ModelCreator } from "src/index.js";
import { Model } from "./Model.js";
import { ModelCategory } from "../legacy/ModelConfig.js";
import { ModelTag } from "./metadata/ModelTag.js";
export class VideoModel extends Model {
// Whether the model supports image starting frames
readonly startFrame: boolean;
// Whether the model supports image ending frames
readonly endFrame: boolean;
constructor(args: {
id: string;
tauriId: string;
fullName: string;
category: ModelCategory;
creator: ModelCreator;
selectorName: string;
selectorDescription: string;
selectorBadges: string[];
startFrame: boolean;
endFrame: boolean;
tags?: ModelTag[];
}) {
super(args);
this.startFrame = args.startFrame;
this.endFrame = args.endFrame;
}
}
@@ -1,36 +0,0 @@
/**
* We might not want to always programatically build the list of models, because
* we likely want to control the order of models in the UI to prioritize
* more popular models. We may also want to selectively prune non-useful models.
*/
// export const TEXT_TO_IMAGE_PAGE_MODELS : ModelConfig[] =
// getModelsByCategory("image")
// // Remove inpainting and editing models.
// .filter((m) => m.id !== "flux_pro_inpaint")
// .filter((m) => m.id !== "flux_pro_kontext_max");
// export const IMAGE_TO_VIDEO_PAGE_MODELS : ModelConfig[] =
// getModelsByCategory("video");
// export const CANVAS_2D_PAGE_MODELS : ModelConfig[] =
// getModelsByCategory("image")
// // Remove non-instructive models.
// .filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
// export const STAGE_3D_PAGE_MODELS : ModelConfig[] =
// getModelsByCategory("image")
// // Remove non-instructive models.
// .filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
// export const IMAGE_EDITOR_PAGE_MODELS : ModelConfig[] = [
// // Explicit list for now.
// ALL_MODELS_BY_ID.get("flux_pro_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_dev_juggernaut_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_pro_kontext_max")!,
// ];
// export const CANVAS_2D_PAGE_MODELS_NEW : ImageModel[] = IMAGE_MODELS
// // Remove non-instructive models.
// .filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
@@ -1,5 +0,0 @@
import { IMAGE_MODELS } from "./ImageModels.js";
import { VIDEO_MODELS } from "./VideoModels.js";
import { Model } from "../classes/Model.js";
export const ALL_MODELS_LIST: Model[] = [...IMAGE_MODELS, ...VIDEO_MODELS];
@@ -1,10 +0,0 @@
import { Model } from "../classes/Model.js";
import { ALL_MODELS_LIST } from "./AllModels.js";
export const ALL_MODELS_BY_ID : Map<string, Model> = new Map(
ALL_MODELS_LIST.map((model) => [model.id, model])
);
if (ALL_MODELS_BY_ID.size !== ALL_MODELS_LIST.length) {
throw new Error("All models must have unique IDs");
}
@@ -1,121 +0,0 @@
import { ImageModel } from "../classes/ImageModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import { ModelTag } from "../classes/metadata/ModelTag.js";
export const IMAGE_MODELS: ImageModel[] = [
new ImageModel({
id: "midjourney",
tauriId: "midjourney",
fullName: "Midjourney",
category: "image",
creator: ModelCreator.Midjourney,
selectorName: "Midjourney",
selectorDescription: "Incredible style and quality",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
}),
new ImageModel({
id: "flux_pro_1_1_ultra",
tauriId: "flux_pro_11_ultra",
fullName: "Flux Pro 1.1 Ultra",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Pro 1.1 Ultra",
selectorDescription: "High quality model",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
}),
new ImageModel({
id: "flux_pro_1_1",
tauriId: "flux_pro_11",
fullName: "Flux Pro 1.1",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Pro 1.1",
selectorDescription: "High quality model",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
}),
new ImageModel({
id: "flux_1_dev",
tauriId: "flux_1_dev",
fullName: "Flux 1 Dev",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux 1 Dev",
selectorDescription: "High quality model",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
}),
new ImageModel({
id: "flux_1_schnell",
tauriId: "flux_1_schnell",
fullName: "Flux 1 Schnell",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux 1 Schnell",
selectorDescription: "High quality model",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
}),
new ImageModel({
id: "gpt_image_1",
tauriId: "gpt_image_1",
fullName: "GPT Image 1 (GPT-4o)",
category: "image",
creator: ModelCreator.OpenAi,
selectorName: "GPT Image 1 (GPT-4o)",
selectorDescription: "Slow, ultra instructive model",
selectorBadges: ["45 sec."],
maxGenerationCount: 1,
defaultGenerationCount: 1,
tags: [ModelTag.InstructiveEdit],
}),
new ImageModel({
id: "flux_pro_kontext_max",
tauriId: "flux_pro_kontext_max",
fullName: "Flux Pro Kontext Max",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Pro Kontext Max",
selectorDescription: "Fast instructive editing",
selectorBadges: ["20 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
canEditImages: true,
tags: [ModelTag.InstructiveEdit],
}),
new ImageModel({
id: "flux_pro_inpaint",
tauriId: "flux_pro_1",
fullName: "Flux Pro Inpaint",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Pro",
selectorDescription: "Fast inpainting",
selectorBadges: ["30 sec."],
maxGenerationCount: 1, // NB: Fal only allows one image for some reason!
defaultGenerationCount: 1, // NB: Fal only allows one image for some reason!
canEditImages: true,
usesInpaintingMask: true,
}),
new ImageModel({
id: "flux_dev_juggernaut_inpaint",
tauriId: "flux_dev_juggernaut",
fullName: "Flux Dev Juggernaut Inpaint",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Dev Juggernaut",
selectorDescription: "Fast inpainting",
selectorBadges: ["15 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 4,
canEditImages: true,
usesInpaintingMask: true,
}),
];
@@ -1,65 +0,0 @@
import { VideoModel } from "../classes/VideoModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
export const VIDEO_MODELS: VideoModel[] = [
new VideoModel({
id: "kling_1_6_pro",
tauriId: "kling_1.6_pro",
fullName: "Kling 1.6 Pro",
category: "video",
creator: ModelCreator.Kling,
selectorName: "Kling 1.6 Pro",
selectorDescription: "Fast video model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: true,
}),
new VideoModel({
id: "kling_2_1_pro",
tauriId: "kling_2.1_pro",
fullName: "Kling 2.1 Pro",
category: "video",
creator: ModelCreator.Kling,
selectorName: "Kling 2.1 Pro",
selectorDescription: "Fast video model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: true,
}),
new VideoModel({
id: "kling_2_1_master",
tauriId: "kling_2.1_master",
fullName: "Kling 2.1 Master",
category: "video",
creator: ModelCreator.Kling,
selectorName: "Kling 2.1 Master",
selectorDescription: "Fast video model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: false,
}),
new VideoModel({
id: "seedance_1_0_lite",
tauriId: "seedance_1.0_lite",
fullName: "Seedance 1.0 Lite",
category: "video",
creator: ModelCreator.Bytedance,
selectorName: "Seedance 1.0 Lite",
selectorDescription: "Fast video model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: false,
}),
new VideoModel({
id: "veo_2",
tauriId: "veo_2",
fullName: "Google Veo 2",
category: "video",
creator: ModelCreator.Google,
selectorName: "Google Veo 2",
selectorDescription: "Fast video model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: false,
}),
];
@@ -1,11 +1,11 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { ImageModel, ModelInfo } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
export interface EnqueueContextualEditImageRequest {
/// The model to use.
model?: ImageModel | ModelInfo | EnqueueContextualEditImageModel;
model?: ModelInfo | EnqueueContextualEditImageModel;
/// If set, this is the first image in the contextual image set.
/// This gets submitted along with `image_media_tokens` and will
@@ -95,8 +95,6 @@ export const EnqueueContextualEditImage = async (request: EnqueueContextualEditI
if (!!request.model) {
if (typeof request.model === "string") {
modelName = request.model;
} else if (request.model instanceof ImageModel) {
modelName = request.model.tauriId;
} else {
modelName = request.model.tauri_id;
}
@@ -1,10 +1,10 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { ImageModel, ModelInfo } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
export interface EnqueueImageInpaintRequest {
// The model to use.
model?: ImageModel | ModelInfo | EnqueueImageInpaintModel;
model?: ModelInfo | EnqueueImageInpaintModel;
// The image we're editing.
image_media_token?: string;
@@ -74,8 +74,6 @@ export const EnqueueImageInpaint = async (request: EnqueueImageInpaintRequest) :
if (!!request.model) {
if (typeof request.model === "string") {
modelName = request.model;
} else if (request.model instanceof ImageModel) {
modelName = request.model.tauriId;
} else {
modelName = request.model.tauri_id;
}
@@ -1,6 +1,6 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { Model } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
export enum EnqueueImageToVideoErrorType {
/// Caller didn't specify a model
@@ -15,7 +15,7 @@ export enum EnqueueImageToVideoErrorType {
export interface EnqueueImageToVideoRequest {
// Required. The model to use.
model?: Model;
model?: ModelInfo;
// Required. Starting frame.
image_media_token?: string;
@@ -49,9 +49,10 @@ export interface EnqueueImageToVideoSuccess extends CommandResult {
export type EnqueueImageToVideoResult = EnqueueImageToVideoSuccess | EnqueueImageToVideoError;
export const EnqueueImageToVideo = async (request: EnqueueImageToVideoRequest) : Promise<EnqueueImageToVideoResult> => {
const modelName = request.model?.tauri_id;
let mutableRequest : RawEnqueueImageToVideoRequest = {
model: request.model?.tauriId,
model: modelName,
image_media_token: request.image_media_token,
};
@@ -1,6 +1,6 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { Model, ModelInfo } from "@storyteller/model-list";
import { ModelInfo } from "@storyteller/model-list";
export enum EnqueueTextToImageErrorType {
/// Caller didn't specify a model
@@ -15,7 +15,7 @@ export enum EnqueueTextToImageErrorType {
export interface EnqueueTextToImageRequest {
prompt?: string;
model?: Model | ModelInfo | EnqueueTextToImageModel;
model?: ModelInfo | EnqueueTextToImageModel;
aspect_ratio?: EnqueueTextToImageSize;
number_images?: number;
@@ -73,8 +73,6 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P
if (!!request.model) {
if (typeof request.model === "string") {
modelName = request.model;
} else if (request.model instanceof Model) {
modelName = request.model.tauriId;
} else {
modelName = request.model.tauri_id;
}