Compare commits

..
Author SHA1 Message Date
Brandon Thomas 2c447f3641 rebuild package-lock.json from scratch 2025-12-05 02:21:35 -05:00
95 changed files with 607 additions and 2352 deletions
-1
View File
@@ -152,7 +152,6 @@ CREATE TABLE media_files (
-- * `vmd`
-- * `pmx`
-- * `csv`
-- * `spz` Gaussian splats
media_type VARCHAR(16) NOT NULL,
-- Broad class of media:
@@ -5,5 +5,4 @@ pub mod generate_flux_pro_11_text_to_image;
pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod multi_function;
pub mod remove_image_background;
@@ -1 +0,0 @@
pub mod nano_banana_pro_multi_function_image_gen;
@@ -1,71 +0,0 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const NANO_BANANA_PRO_MULTI_FUNCTION_IMAGE_GEN_PATH: &str = "/v1/generate/image/multi_function/nano_banana_pro";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct NanoBananaProMultiFunctionImageGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// Image media tokens to include in the editing context.
/// If present, we're doing image editing (image-to-image / image-editing)
/// If absent, we're doing image generation (text-to-image)
pub image_media_tokens: Option<Vec<MediaFileToken>>,
/// Number of images to generate. Default is one.
pub num_images: Option<NanoBananaProMultiFunctionImageGenNumImages>,
/// Resolution of the image to generate. Default is OneK (1K).
pub resolution: Option<NanoBananaProMultiFunctionImageGenImageResolution>,
/// Aspect ratio of the images to generate. Default is "1:1"
pub aspect_ratio: Option<NanoBananaProMultiFunctionImageGenAspectRatio>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaProMultiFunctionImageGenNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaProMultiFunctionImageGenImageResolution {
OneK,
TwoK,
FourK,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaProMultiFunctionImageGenAspectRatio {
// Square
OneByOne, // Default
// Wide
FiveByFour,
FourByThree,
ThreeByTwo,
SixteenByNine,
TwentyOneByNine,
// Tall
FourByFive,
ThreeByFour,
TwoByThree,
NineBySixteen, // NB: No NineByTwentyOne ?
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct NanoBananaProMultiFunctionImageGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -5,5 +5,4 @@ pub mod generate_flux_pro_11_text_to_image;
pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod multi_function;
pub mod remove_image_background;
@@ -1 +0,0 @@
pub mod nano_banana_pro_multi_function_image_gen_image;
@@ -1,18 +0,0 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenRequest, NanoBananaProMultiFunctionImageGenResponse, NANO_BANANA_PRO_MULTI_FUNCTION_IMAGE_GEN_PATH};
pub async fn nano_banana_pro_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: NanoBananaProMultiFunctionImageGenRequest,
) -> Result<NanoBananaProMultiFunctionImageGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
NANO_BANANA_PRO_MULTI_FUNCTION_IMAGE_GEN_PATH,
maybe_creds,
request,
).await?)
}
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageSuccessResponse;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageSuccessResponse;
use crate::core::commands::response::shorthand::ResponseOrErrorMessage;
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
use crate::core::state::app_preferences::app_preferences_manager::AppPreferencesManager;
@@ -2,7 +2,7 @@ use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_bg_removal::generic::handle_generic_bg_removal_artcraft::handle_generic_bg_removal_artcraft;
use crate::core::commands::enqueue::image_bg_removal::generic::handle_generic_bg_removal_fal::handle_generic_bg_removal_fal;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -1,6 +1,6 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason, ProviderFailureReason};
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
@@ -1,9 +1,8 @@
use crate::core::commands::enqueue::common::notify_frontend_of_errors::notify_frontend_of_errors;
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::flux_kontext::handle_flux_kontext_edit::handle_flux_kontext_edit;
use crate::core::commands::enqueue::image_edit::gemini_25_flash::handle_gemini_25_flash_edit::handle_gemini_25_flash_edit;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit;
use crate::core::commands::enqueue::image_edit::nano_banana::handle_nano_banana_edit::handle_nano_banana_edit;
use crate::core::commands::enqueue::image_edit::nano_banana_pro::handle_nano_banana_pro_edit::handle_nano_banana_pro_edit;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType};
@@ -37,8 +36,8 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use serde_derive::{Deserialize, Serialize};
use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs};
use std::time::Duration;
use storyteller_client::endpoints::media_files::get_media_file::get_media_file;
use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::endpoints::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
@@ -47,19 +46,13 @@ use tokens::tokens::media_files::MediaFileToken;
/// Don't change the serializations without coordinating with the frontend.
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ImageEditModel {
pub enum ContextualImageEditModel {
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
#[serde(rename = "gemini_25_flash")]
Gemini25Flash,
#[serde(rename = "nano_banana")]
NanoBanana,
#[serde(rename = "nano_banana_pro")]
NanoBananaPro,
#[serde(rename = "gpt_image_1")]
GptImage1,
@@ -71,9 +64,9 @@ pub enum ImageEditModel {
}
#[derive(Deserialize, Debug)]
pub struct EnqueueEditImageCommand {
pub struct EnqueueContextualEditImageCommand {
/// The model to use.
pub model: Option<ImageEditModel>,
pub model: Option<ContextualImageEditModel>,
/// Images to use for the image edit.
/// The first image is typically a 2D canvas or 3D stage, but doesn't have to be.
@@ -100,9 +93,6 @@ pub struct EnqueueEditImageCommand {
/// Image quality.
pub image_quality: Option<EditImageQuality>,
/// Image resolution
pub image_resolution: Option<EditImageResolution>,
/// OPTIONAL.
/// Name of the frontend caller.
/// We'll use this to selectively trigger events.
@@ -137,20 +127,9 @@ pub enum EditImageQuality {
Low,
}
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum EditImageResolution {
/// 1K - nano banana pro
OneK,
/// 2K - nano banana pro
TwoK,
/// 4K - nano banana pro
FourK,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum EnqueueEditImageErrorType {
pub enum EnqueueContextualEditImageErrorType {
/// Caller didn't specify a model
ModelNotSpecified,
@@ -174,15 +153,15 @@ pub enum EnqueueEditImageErrorType {
}
#[derive(Serialize)]
pub struct EnqueueEditImageSuccessResponse {
pub struct EnqueueContextualEditImageSuccessResponse {
}
impl SerializeMarker for EnqueueEditImageSuccessResponse {}
impl SerializeMarker for EnqueueContextualEditImageSuccessResponse {}
#[tauri::command]
pub async fn enqueue_edit_image_command(
pub async fn enqueue_contextual_edit_image_command(
app: AppHandle,
request: EnqueueEditImageCommand,
request: EnqueueContextualEditImageCommand,
app_data_root: State<'_, AppDataRoot>,
app_env_configs: State<'_, AppEnvConfigs>,
provider_priority_store: State<'_, ProviderPriorityStore>,
@@ -190,9 +169,9 @@ pub async fn enqueue_edit_image_command(
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
sora_creds_manager: State<'_, SoraCredentialManager>,
sora_task_queue: State<'_, SoraTaskQueue>,
) -> ResponseOrErrorType<EnqueueEditImageSuccessResponse, EnqueueEditImageErrorType> {
) -> ResponseOrErrorType<EnqueueContextualEditImageSuccessResponse, EnqueueContextualEditImageErrorType> {
info!("enqueue_edit_image_command called; image media tokens : {:?}, full request: {:?}",
info!("enqueue_contextual_edit_image_command called; image media tokens : {:?}, full request: {:?}",
&request.image_media_tokens, &request);
let result = handle_request(
@@ -240,13 +219,13 @@ pub async fn enqueue_edit_image_command(
CreditsBalanceChangedEvent{}.send_infallible(&app);
Ok(EnqueueEditImageSuccessResponse {}.into())
Ok(EnqueueContextualEditImageSuccessResponse {}.into())
}
}
}
pub async fn handle_request(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -256,11 +235,11 @@ pub async fn handle_request(
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let mut success_event= match request.model {
let success_event= match request.model {
None => {
return Err(GenerateError::no_model_specified())
}
Some(ImageEditModel::FluxProKontextMax) => {
Some(ContextualImageEditModel::FluxProKontextMax) => {
handle_flux_kontext_edit(
request,
app,
@@ -272,8 +251,8 @@ pub async fn handle_request(
sora_task_queue,
).await?
}
Some(ImageEditModel::Gemini25Flash | ImageEditModel::NanoBanana) => {
handle_nano_banana_edit(
Some(ContextualImageEditModel::Gemini25Flash) => {
handle_gemini_25_flash_edit(
request,
app,
app_data_root,
@@ -284,19 +263,7 @@ pub async fn handle_request(
sora_task_queue,
).await?
}
Some(ImageEditModel::NanoBananaPro) => {
handle_nano_banana_pro_edit(
request,
app,
app_data_root,
app_env_configs,
provider_priority_store,
storyteller_creds_manager,
sora_creds_manager,
sora_task_queue,
).await?
}
Some(ImageEditModel::GptImage1) => {
Some(ContextualImageEditModel::GptImage1) => {
handle_gpt_image_1_edit(
request,
app,
@@ -309,18 +276,7 @@ pub async fn handle_request(
).await?
}
};
match request.frontend_caller {
Some(TauriCommandCaller::ImageEditor) => {
// NB: Certain image generations enqueued in the inpainting editor are classed
// as "image_generation" and won't update the inpainting UI. This is a hack to
// temporarily fix the UI event binding.
// TODO: The frontend shouldn't require us to change the type to inpainting.
success_event.task_type = TaskType::ImageInpaintEdit;
}
_ => {} // fall-through
}
let result = success_event
.insert_into_task_database_with_frontend_payload(
task_database,
@@ -338,31 +294,31 @@ pub async fn handle_request(
Ok(success_event)
}
fn error_to_tauri_response(error: GenerateError) -> CommandErrorResponseWrapper<EnqueueEditImageErrorType, ()> {
fn error_to_tauri_response(error: GenerateError) -> CommandErrorResponseWrapper<EnqueueContextualEditImageErrorType, ()> {
let mut status = CommandErrorStatus::ServerError;
let mut error_type = EnqueueEditImageErrorType::ServerError;
let mut error_type = EnqueueContextualEditImageErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.".to_string();
match error {
GenerateError::BadInput(BadInputReason::NoModelSpecified) => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueEditImageErrorType::ModelNotSpecified;
error_type = EnqueueContextualEditImageErrorType::ModelNotSpecified;
error_message = "No model specified for image generation".to_string();
}
GenerateError::NoProviderAvailable => {
status = CommandErrorStatus::ServerError;
error_type = EnqueueEditImageErrorType::NoProviderAvailable;
error_type = EnqueueContextualEditImageErrorType::NoProviderAvailable;
error_message = "No configured provider available for image generation".to_string();
}
GenerateError::BadInput(BadInputReason::InvalidNumberOfRequestedImages { min, max, requested }) => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueEditImageErrorType::BadRequest;
error_type = EnqueueContextualEditImageErrorType::BadRequest;
error_message = format!("Invalid number of images requested ({}). Must be between {} and {}", requested, min, max);
}
GenerateError::BadInput(BadInputReason::InvalidNumberOfInputImages{ min, max, provided }) => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueEditImageErrorType::BadRequest;
error_type = EnqueueContextualEditImageErrorType::BadRequest;
error_message = format!("Invalid number of input images ({}). Must be between {} and {}", provided, min, max);
}
_ => {} // Fall-through for now
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::flux_kontext::handle_flux_kontext_edit_artcraft::handle_flux_kontext_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
@@ -14,7 +14,7 @@ use log::info;
use tauri::AppHandle;
pub async fn handle_flux_kontext_edit(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
@@ -30,7 +30,7 @@ use storyteller_client::endpoints::generate::image::generate_flux_pro_11_ultra_t
use tauri::AppHandle;
pub async fn handle_flux_kontext_edit_artcraft(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -1,5 +1,6 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::gemini_25_flash::handle_gemini_25_flash_edit_artcraft::handle_gemini_25_flash_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -11,10 +12,9 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use log::info;
use tauri::AppHandle;
use crate::core::commands::enqueue::image_edit::nano_banana::handle_nano_banana_edit_artcraft::handle_nano_banana_edit_artcraft;
pub async fn handle_nano_banana_edit(
request: &EnqueueEditImageCommand,
pub async fn handle_gemini_25_flash_edit(
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -39,8 +39,8 @@ pub async fn handle_nano_banana_edit(
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching Nano Banana (edit) via Artcraft...");
return handle_nano_banana_edit_artcraft(
info!("Dispatching gemini 2.5 flash (edit) via Artcraft...");
return handle_gemini_25_flash_edit_artcraft(
request,
app,
app_data_root,
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
@@ -29,8 +29,8 @@ use tauri::AppHandle;
pub(super) const MAX_IMAGES: usize = 10;
pub async fn handle_nano_banana_edit_artcraft(
request: &EnqueueEditImageCommand,
pub async fn handle_gemini_25_flash_edit_artcraft(
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -44,7 +44,7 @@ pub async fn handle_nano_banana_edit_artcraft(
},
};
info!("Calling Artcraft Nano Banana (edit) ...");
info!("Calling Artcraft gemini 2.5 flash (edit) ...");
let uuid_idempotency_token = generate_random_uuid();
@@ -98,12 +98,12 @@ pub async fn handle_nano_banana_edit_artcraft(
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft Nano Banana. Job token: {}",
info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft Nano Banana: {:?}", err);
error!("Failed to use Artcraft gpt-image-1: {:?}", err);
return Err(GenerateError::from(err));
}
};
@@ -0,0 +1,2 @@
pub (super) mod handle_gemini_25_flash_edit;
pub (super) mod handle_gemini_25_flash_edit_artcraft;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -15,7 +15,7 @@ use tauri::AppHandle;
pub(super) const MAX_IMAGES: usize = 10;
pub async fn handle_gpt_image_1_edit(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::MAX_IMAGES;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -29,7 +29,7 @@ use storyteller_client::endpoints::generate::image::generate_flux_pro_11_ultra_t
use tauri::AppHandle;
pub async fn handle_gpt_image_1_edit_artcraft(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageSize, EnqueueEditImageCommand, EnqueueEditImageErrorType};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageSize, EnqueueContextualEditImageCommand, EnqueueContextualEditImageErrorType};
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::MAX_IMAGES;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
@@ -42,7 +42,7 @@ const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); //
const DEFAULT_ASPECT_RATIO : EditImageSize = EditImageSize::Square;
pub async fn handle_gpt_image_1_edit_sora(
request: &EnqueueEditImageCommand,
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
@@ -1,5 +1,4 @@
pub mod enqueue_edit_image_command;
pub mod enqueue_contextual_edit_image_command;
pub mod gpt_image_1;
mod flux_kontext;
mod nano_banana;
mod nano_banana_pro;
mod gemini_25_flash;
@@ -1,2 +0,0 @@
pub (super) mod handle_nano_banana_edit;
pub (super) mod handle_nano_banana_edit_artcraft;
@@ -1,53 +0,0 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::nano_banana_pro::handle_nano_banana_pro_edit_artcraft::handle_nano_banana_pro_edit_artcraft;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use log::info;
use tauri::AppHandle;
pub async fn handle_nano_banana_pro_edit(
request: &EnqueueEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
storyteller_creds_manager: &StorytellerCredentialManager,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let priority = provider_priority_store.get_priority()?;
// TODO: Check if providers are available before proceeding.
info!("Providers by priority: {:?}", priority);
for provider in priority.iter() {
match provider {
Provider::Fal => {
// Fallthrough (TODO)
}
Provider::Sora => {
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching Nano Banana Pro (edit) via Artcraft...");
return handle_nano_banana_pro_edit_artcraft(
request,
app,
app_data_root,
app_env_configs,
storyteller_creds_manager
).await;
}
}
}
Err(GenerateError::NoProviderAvailable)
}
@@ -1,137 +0,0 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest};
use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest};
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest};
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use idempotency::uuid::generate_random_uuid;
use log::{error, info};
use storyteller_client::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image;
use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image;
use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
use tauri::AppHandle;
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest};
use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::ImageEditModel::NanoBanana;
pub(super) const MAX_IMAGES: usize = 10;
pub async fn handle_nano_banana_pro_edit_artcraft(
request: &EnqueueEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let creds = match storyteller_creds_manager.get_credentials()? {
Some(creds) => creds,
None => {
return Err(GenerateError::needs_storyteller_credentials());
},
};
info!("Calling Artcraft Nano Banana (edit) ...");
let uuid_idempotency_token = generate_random_uuid();
let num_images = match request.image_count {
None => None,
Some(1) => Some(NanoBananaProMultiFunctionImageGenNumImages::One),
Some(2) => Some(NanoBananaProMultiFunctionImageGenNumImages::Two),
Some(3) => Some(NanoBananaProMultiFunctionImageGenNumImages::Three),
Some(4) => Some(NanoBananaProMultiFunctionImageGenNumImages::Four),
Some(other) => {
return Err(GenerateError::BadInput(BadInputReason::InvalidNumberOfRequestedImages {
min: 1,
max: 4,
requested: other,
}));
},
};
let aspect_ratio = request.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EditImageSize::Auto => None,
EditImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
EditImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
EditImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
})
.flatten();
let resolution = request.image_resolution
.map(|image_resolution| match image_resolution {
EditImageResolution::OneK => NanoBananaProMultiFunctionImageGenImageResolution::OneK,
EditImageResolution::TwoK => NanoBananaProMultiFunctionImageGenImageResolution::TwoK,
EditImageResolution::FourK => NanoBananaProMultiFunctionImageGenImageResolution::FourK,
});
let mut media_tokens = Vec::with_capacity(10);
if let Some(scene_image_media_token) = request.scene_image_media_token.clone() {
media_tokens.push(scene_image_media_token);
}
if let Some(image_media_tokens) = request.image_media_tokens.as_ref() {
media_tokens.extend_from_slice(image_media_tokens);
}
if media_tokens.len() > MAX_IMAGES {
return Err(GenerateError::BadInput(BadInputReason::InvalidNumberOfInputImages {
min: 1,
max: MAX_IMAGES as u32,
provided: media_tokens.len() as u32,
}));
}
let request = NanoBananaProMultiFunctionImageGenRequest {
uuid_idempotency_token,
prompt: Some(request.prompt.clone()),
image_media_tokens: Some(media_tokens),
num_images,
resolution,
aspect_ratio,
};
let result = nano_banana_pro_multi_function_image_gen(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft Nano Banana. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft Nano Banana: {:?}", err);
return Err(GenerateError::from(err));
}
};
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::NanoBananaPro),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageGeneration,
})
}
@@ -1,2 +0,0 @@
pub (super) mod handle_nano_banana_pro_edit;
pub (super) mod handle_nano_banana_pro_edit_artcraft;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
@@ -1,6 +1,6 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
@@ -1,6 +1,6 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint_artcraft::handle_flux_pro_kontext_inpaint_artcraft;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -1,6 +1,6 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint_artcraft::handle_flux_pro_kontext_inpaint_artcraft;
use crate::core::commands::enqueue::image_inpaint::gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint_artcraft::handle_gemini_25_flash_inpaint_artcraft;
@@ -1,6 +1,6 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::{EnqueueImageToVideoRequest, GrokAspectRatio, SoraOrientation};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageCommand;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::EnqueueImageToVideoRequest;
use crate::core::commands::enqueue::image_to_video::sora2::handle_sora2_video_sora::handle_sora2_video_sora;
@@ -66,8 +66,6 @@ impl TaskEnqueueSuccess{
Some(GenerationModel::FluxPro11Ultra) => Some(TaskModelType::FluxPro11Ultra),
Some(GenerationModel::FluxProKontextMax) => Some(TaskModelType::FluxProKontextMax),
Some(GenerationModel::Gemini25Flash) => Some(TaskModelType::Gemini25Flash),
Some(GenerationModel::NanoBanana) => Some(TaskModelType::NanoBanana),
Some(GenerationModel::NanoBananaPro) => Some(TaskModelType::NanoBananaPro),
Some(GenerationModel::GptImage1) => Some(TaskModelType::GptImage1),
Some(GenerationModel::GrokImage) => Some(TaskModelType::GrokImage),
Some(GenerationModel::Recraft3) => Some(TaskModelType::Recraft3),
@@ -1,14 +1,13 @@
use crate::core::commands::enqueue::common::notify_frontend_of_errors::notify_frontend_of_errors;
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::gemini_25_flash::handle_gemini_25_flash::handle_gemini_25_flash;
use crate::core::commands::enqueue::text_to_image::generic::handle_image_artcraft::handle_image_artcraft;
use crate::core::commands::enqueue::text_to_image::generic::handle_image_fal::handle_image_fal;
use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1;
use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora;
use crate::core::commands::enqueue::text_to_image::grok::handle_grok::handle_grok;
use crate::core::commands::enqueue::text_to_image::midjourney::handle_midjourney::handle_midjourney;
use crate::core::commands::enqueue::text_to_image::nano_banana::handle_nano_banana::handle_nano_banana;
use crate::core::commands::enqueue::text_to_image::nano_banana_pro::handle_nano_banana_pro::handle_nano_banana_pro;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
use crate::core::commands::response::shorthand::Response;
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
@@ -59,10 +58,6 @@ pub enum ImageModel {
#[serde(rename = "gemini_25_flash")]
Gemini25Flash,
#[serde(rename = "nano_banana")]
NanoBanana,
#[serde(rename = "nano_banana_pro")]
NanoBananaPro,
// Generic Midjourney model, version unknown.
#[serde(rename = "midjourney")]
@@ -80,9 +75,6 @@ pub struct EnqueueTextToImageRequest {
/// Aspect ratio.
pub aspect_ratio: Option<TextToImageSize>,
/// Aspect ratio.
pub image_resolution: Option<TextToImageResolution>,
/// The number of images to generate.
pub number_images: Option<u32>,
@@ -124,17 +116,6 @@ pub enum TextToImageSize {
Tall,
}
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum TextToImageResolution {
/// 1K - nano banana pro
OneK,
/// 2K - nano banana pro
TwoK,
/// 4K - nano banana pro
FourK,
}
#[derive(Serialize)]
pub struct EnqueueTextToImageSuccessResponse {
}
@@ -316,20 +297,8 @@ pub async fn dispatch_request(
None => {
return Err(GenerateError::BadInput(BadInputReason::NoModelSpecified));
}
Some(ImageModel::NanoBanana | ImageModel::Gemini25Flash) => {
return handle_nano_banana(
request,
app,
app_data_root,
app_env_configs,
provider_priority_store,
storyteller_creds_manager,
sora_creds_manager,
sora_task_queue,
).await;
}
Some(ImageModel::NanoBananaPro) => {
return handle_nano_banana_pro(
Some(ImageModel::Gemini25Flash) => {
return handle_gemini_25_flash(
request,
app,
app_data_root,
@@ -3,9 +3,9 @@ use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::commands::enqueue::text_to_image::gemini_25_flash::handle_gemini_25_flash_artcraft::handle_gemini_25_flash_artcraft;
use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft;
use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora;
use crate::core::commands::enqueue::text_to_image::nano_banana::handle_nano_banana_artcraft::handle_nano_banana_artcraft;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
@@ -24,7 +24,7 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use std::time::Duration;
use tauri::AppHandle;
pub async fn handle_nano_banana(
pub async fn handle_gemini_25_flash(
request: &EnqueueTextToImageRequest,
app: &AppHandle,
app_data_root: &AppDataRoot,
@@ -50,8 +50,8 @@ pub async fn handle_nano_banana(
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching Nano Banana via Artcraft...");
return handle_nano_banana_artcraft(
info!("Dispatching gemini 2.5 flash via Artcraft...");
return handle_gemini_25_flash_artcraft(
request,
app_env_configs,
storyteller_creds_manager
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{GenerateError, MissingCredentialsReason, ProviderFailureReason};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -26,7 +26,7 @@ use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image
use storyteller_client::endpoints::generate::image::generate_gpt_image_1_text_to_image::generate_gpt_image_1_text_to_image;
use tauri::AppHandle;
pub async fn handle_nano_banana_artcraft(
pub async fn handle_gemini_25_flash_artcraft(
request: &EnqueueTextToImageRequest,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
@@ -39,7 +39,7 @@ pub async fn handle_nano_banana_artcraft(
},
};
info!("Calling Artcraft Nano Banana...");
info!("Calling Artcraft gemini 2.5 flash ...");
let uuid_idempotency_token = generate_random_uuid();
@@ -69,12 +69,12 @@ pub async fn handle_nano_banana_artcraft(
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft Nano Banana. Job token: {}",
info!("Successfully enqueued Artcraft gemini 2.5 flash. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft Nano Banana: {:?}", err);
error!("Failed to use Artcraft gemini 2.5 flash: {:?}", err);
return Err(GenerateError::ProviderFailure(ProviderFailureReason::StorytellerError(err)));
}
};
@@ -0,0 +1,2 @@
pub (crate) mod handle_gemini_25_flash_artcraft;
pub (super) mod handle_gemini_25_flash;
@@ -57,8 +57,6 @@ pub async fn handle_image_artcraft(
}
Some(
ImageModel::Gemini25Flash |
ImageModel::NanoBanana |
ImageModel::NanoBananaPro |
ImageModel::GptImage1 |
ImageModel::GrokImage |
ImageModel::Midjourney
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,5 +1,5 @@
use crate::core::commands::enqueue::generate_error::{GenerateError, MissingCredentialsReason, ProviderFailureReason};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
@@ -1,7 +1,6 @@
pub mod enqueue_text_to_image_command;
pub mod generic;
mod nano_banana;
mod nano_banana_pro;
mod gemini_25_flash;
mod gpt_image_1;
mod midjourney;
mod grok;
@@ -1,2 +0,0 @@
pub (crate) mod handle_nano_banana_artcraft;
pub (super) mod handle_nano_banana;
@@ -1,59 +0,0 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::commands::enqueue::text_to_image::nano_banana_pro::handle_nano_banana_pro_artcraft::handle_nano_banana_pro_artcraft;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use log::{error, info};
use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest};
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use std::time::Duration;
use tauri::AppHandle;
pub async fn handle_nano_banana_pro(
request: &EnqueueTextToImageRequest,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
storyteller_creds_manager: &StorytellerCredentialManager,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let priority = provider_priority_store.get_priority()?;
// TODO: Check if providers are available before proceeding.
info!("Providers by priority: {:?}", priority);
for provider in priority.iter() {
match provider {
Provider::Fal => {
// Fallthrough (for now)
}
Provider::Sora => {
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching Nano Banana Pro via Artcraft...");
return handle_nano_banana_pro_artcraft(
request,
app_env_configs,
storyteller_creds_manager
).await;
}
}
}
Err(GenerateError::NoProviderAvailable)
}
@@ -1,111 +0,0 @@
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason, ProviderFailureReason};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageResolution, TextToImageSize};
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest};
use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest};
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest};
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use idempotency::uuid::generate_random_uuid;
use log::{error, info};
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest};
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use std::time::Duration;
use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image;
use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen;
use tauri::AppHandle;
pub async fn handle_nano_banana_pro_artcraft(
request: &EnqueueTextToImageRequest,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let creds = match storyteller_creds_manager.get_credentials()? {
Some(creds) => creds,
None => {
return Err(GenerateError::MissingCredentials(MissingCredentialsReason::NeedsStorytellerCredentials));
},
};
info!("Calling Artcraft Nano Banana Pro...");
let uuid_idempotency_token = generate_random_uuid();
let num_images = match request.number_images {
None => None,
Some(1) => Some(NanoBananaProMultiFunctionImageGenNumImages::One),
Some(2) => Some(NanoBananaProMultiFunctionImageGenNumImages::Two),
Some(3) => Some(NanoBananaProMultiFunctionImageGenNumImages::Three),
Some(4) => Some(NanoBananaProMultiFunctionImageGenNumImages::Four),
Some(other) => {
return Err(GenerateError::BadInput(BadInputReason::InvalidNumberOfRequestedImages {
min: 1,
max: 4,
requested: other,
}));
},
};
let aspect_ratio = request.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
TextToImageSize::Auto => None,
TextToImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
TextToImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
TextToImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
})
.flatten();
let resolution = request.image_resolution
.map(|image_resolution| match image_resolution {
TextToImageResolution::OneK => NanoBananaProMultiFunctionImageGenImageResolution::OneK,
TextToImageResolution::TwoK => NanoBananaProMultiFunctionImageGenImageResolution::TwoK,
TextToImageResolution::FourK => NanoBananaProMultiFunctionImageGenImageResolution::FourK,
});
let request = NanoBananaProMultiFunctionImageGenRequest {
uuid_idempotency_token,
prompt: request.prompt.clone(),
image_media_tokens: None,
num_images,
resolution,
aspect_ratio,
};
let result = nano_banana_pro_multi_function_image_gen(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft Nano Banana. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft Nano Banana: {:?}", err);
return Err(GenerateError::ProviderFailure(ProviderFailureReason::StorytellerError(err)));
}
};
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::NanoBananaPro),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageGeneration,
})
}
@@ -1,2 +0,0 @@
pub (crate) mod handle_nano_banana_pro_artcraft;
pub (super) mod handle_nano_banana_pro;
@@ -34,11 +34,7 @@ pub enum GenerationModel {
GrokImage,
#[serde(rename = "gemini_25_flash")]
Gemini25Flash,
#[serde(rename = "nano_banana")]
NanoBanana,
#[serde(rename = "nano_banana_pro")]
NanoBananaPro,
#[serde(rename = "hunyuan_3d_2_0")]
Hunyuan3d2_0,
#[serde(rename = "hunyuan_3d_2_1")]
+2 -2
View File
@@ -6,7 +6,7 @@ use tauri::Manager;
use crate::core::commands::app_preferences::get_app_preferences_command::get_app_preferences_command;
use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command;
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::enqueue_image_bg_removal_command;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::enqueue_edit_image_command;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::enqueue_contextual_edit_image_command;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::enqueue_image_inpaint_command;
use crate::core::commands::enqueue::image_to_object::enqueue_image_to_3d_object_command::enqueue_image_to_3d_object_command;
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::enqueue_image_to_video_command;
@@ -172,7 +172,7 @@ pub fn run() {
// My first attempt at naively doing this didn't work because the macros can't find their codegen'd targets.
let builder = builder.invoke_handler(tauri::generate_handler![
check_sora_session_command,
enqueue_edit_image_command,
enqueue_contextual_edit_image_command,
enqueue_image_bg_removal_command,
enqueue_image_inpaint_command,
enqueue_image_to_3d_object_command,
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::response::shorthand::ResponseOrErrorMessage;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
@@ -1,5 +1,5 @@
use chrono::{DateTime, Utc};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::providers::get_provider_order_command::GetProviderOrderResponse;
use crate::core::commands::response::shorthand::ResponseOrErrorMessage;
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::EnqueueEditImageSuccessResponse;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageSuccessResponse;
use crate::core::commands::response::shorthand::ResponseOrErrorMessage;
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
use crate::core::state::app_preferences::app_preferences_manager::AppPreferencesManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -1,4 +1,4 @@
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{ImageEditModel, EditImageQuality, EditImageSize};
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{ContextualImageEditModel, EditImageQuality, EditImageSize};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
@@ -47,19 +47,10 @@
"img-src": "* data: blob: 'unsafe-inline'",
"script-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"'wasm-unsafe-eval'",
"asset:",
"blob:",
"data:",
"tauri:",
"https://*.clarity.ms",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://*.unpkg.com",
"https://clarity.ms",
"https://unpkg.com",
"ipc://localhost",
"tauri://localhost"
],
-9
View File
@@ -47,19 +47,10 @@
"img-src": "* data: blob: 'unsafe-inline'",
"script-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"'wasm-unsafe-eval'",
"asset:",
"blob:",
"data:",
"tauri:",
"https://*.clarity.ms",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://*.unpkg.com",
"https://clarity.ms",
"https://unpkg.com",
"ipc://localhost",
"tauri://localhost"
],
@@ -15,5 +15,4 @@ pub fn add_artcraft_3d(cors: Cors, _is_production: bool) -> Cors {
.allowed_origin("http://localhost:5174") // If already started
.allowed_origin("http://localhost:5175") // If already started
.allowed_origin("https://macaroni-1.tailce84f.ts.net") // TODO
.allowed_origin("https://halide.tailce84f.ts.net") // TODO
}
@@ -40,6 +40,7 @@ pub struct MediaFileInsertBuilder {
origin_product_category: MediaFileOriginProductCategory, // NB: Non-nullable field
// maybe_origin_model_type: Option<MediaFileOriginModelType>,
// maybe_origin_model_token: Option<&'a ModelWeightToken>,
// maybe_origin_filename: Option<String>,
// Media info
maybe_mime_type: Option<String>,
@@ -50,10 +51,6 @@ pub struct MediaFileInsertBuilder {
maybe_frame_width: Option<u32>,
maybe_frame_height: Option<u32>,
checksum_sha2: Option<String>, // NB: Non-nullable field
// File name
maybe_title: Option<String>,
maybe_origin_filename: Option<String>,
// Media info for certain product areas
maybe_engine_category: Option<MediaFileEngineCategory>, // TODO: Deprecate
@@ -103,8 +100,6 @@ impl MediaFileInsertBuilder {
maybe_frame_width: None,
maybe_frame_height: None,
checksum_sha2: None,
maybe_title: None,
maybe_origin_filename: None,
maybe_engine_category: None,
maybe_batch_generation_token: None,
public_bucket_directory_hash: None,
@@ -181,6 +176,7 @@ impl MediaFileInsertBuilder {
// TODO: Method for maybe_origin_model_type
// TODO: Method for maybe_origin_model_token
// TODO: Method for maybe_origin_filename
pub fn mime_type(mut self, mime_type: &str) -> Self {
self.maybe_mime_type = Some(mime_type.to_string());
@@ -230,16 +226,7 @@ impl MediaFileInsertBuilder {
self
}
pub fn maybe_title(mut self, maybe_title: Option<&str>) -> Self {
self.maybe_title = maybe_title.map(|title| title.to_string());
self
}
pub fn maybe_origin_filename(mut self, maybe_origin_filename: Option<&str>) -> Self {
self.maybe_origin_filename = maybe_origin_filename.map(|s| s.to_string());
self
}
// TODO: maybe_title
// TODO: maybe_text_transcript
// TODO: maybe_scene_source_media_file_token
// TODO: maybe_prompt_token
@@ -287,7 +274,7 @@ impl MediaFileInsertBuilder {
origin_product_category: self.origin_product_category,
maybe_origin_model_type: None, // TODO
maybe_origin_model_token: None, // TODO
maybe_origin_filename: self.maybe_origin_filename,
maybe_origin_filename: None, // TODO
maybe_mime_type: self.maybe_mime_type.as_deref(),
file_size_bytes: self.file_size_bytes,
maybe_duration_millis: self.maybe_duration_millis,
@@ -297,7 +284,7 @@ impl MediaFileInsertBuilder {
maybe_frame_height: self.maybe_frame_height,
maybe_engine_category: self.maybe_engine_category,
checksum_sha2: &checksum_sha2,
maybe_title: self.maybe_title.as_deref(),
maybe_title: None, // TODO
maybe_text_transcript: None, // TODO
maybe_scene_source_media_file_token: None, // TODO
maybe_prompt_token: self.maybe_prompt_token.as_ref(),
@@ -172,7 +172,6 @@ impl RawUserBookmarkRecord {
MediaFileType::Fbx => Some("3d file".to_string()),
MediaFileType::Glb => Some("3d file".to_string()),
MediaFileType::Gltf => Some("3d file".to_string()),
MediaFileType::Spz => Some("3d splat".to_string()),
MediaFileType::SceneRon => Some("3d scene".to_string()),
MediaFileType::SceneJson => Some("3d scene".to_string()),
MediaFileType::Pmd => Some("polygon data".to_string()),
@@ -41,11 +41,6 @@ pub enum MediaFileType {
/// glTF files (for Bevy)
Gltf,
/// Spz Gaussian splats
/// This is a standard file format for Gaussian Splats, used by World Labs.
/// The mime type is application/gzip.
Spz,
/// Bevy's scene files (in RON; Rusty Object Notation)
/// This will be replaced with another format in future versions of Bevy
SceneRon,
@@ -135,7 +130,6 @@ impl MediaFileType {
Self::Fbx => "fbx",
Self::Glb => "glb",
Self::Gltf => "gltf",
Self::Spz => "spz",
Self::SceneRon => "scene_ron",
Self::SceneJson => "scene_json",
Self::Pmd => "pmd",
@@ -160,7 +154,6 @@ impl MediaFileType {
"fbx" => Ok(Self::Fbx),
"glb" => Ok(Self::Glb),
"gltf" => Ok(Self::Gltf),
"spz" => Ok(Self::Spz),
"scene_ron" => Ok(Self::SceneRon),
"scene_json" => Ok(Self::SceneJson),
"pmd" => Ok(Self::Pmd),
@@ -188,7 +181,6 @@ impl MediaFileType {
Self::Fbx,
Self::Glb,
Self::Gltf,
Self::Spz,
Self::SceneRon,
Self::SceneJson,
Self::Pmd,
@@ -245,7 +237,6 @@ mod tests {
assert_serialization(MediaFileType::Fbx, "fbx");
assert_serialization(MediaFileType::Glb, "glb");
assert_serialization(MediaFileType::Gltf, "gltf");
assert_serialization(MediaFileType::Spz, "spz");
assert_serialization(MediaFileType::SceneRon, "scene_ron");
assert_serialization(MediaFileType::SceneJson, "scene_json");
assert_serialization(MediaFileType::Pmd, "pmd");
@@ -273,7 +264,6 @@ mod tests {
assert_eq!(MediaFileType::Fbx.to_str(), "fbx");
assert_eq!(MediaFileType::Glb.to_str(), "glb");
assert_eq!(MediaFileType::Gltf.to_str(), "gltf");
assert_eq!(MediaFileType::Spz.to_str(), "spz");
assert_eq!(MediaFileType::SceneRon.to_str(), "scene_ron");
assert_eq!(MediaFileType::SceneJson.to_str(), "scene_json");
assert_eq!(MediaFileType::Pmd.to_str(), "pmd");
@@ -297,7 +287,6 @@ mod tests {
assert_eq!(MediaFileType::from_str("fbx").unwrap(), MediaFileType::Fbx);
assert_eq!(MediaFileType::from_str("glb").unwrap(), MediaFileType::Glb);
assert_eq!(MediaFileType::from_str("gltf").unwrap(), MediaFileType::Gltf);
assert_eq!(MediaFileType::from_str("spz").unwrap(), MediaFileType::Spz);
assert_eq!(MediaFileType::from_str("scene_ron").unwrap(), MediaFileType::SceneRon);
assert_eq!(MediaFileType::from_str("scene_json").unwrap(), MediaFileType::SceneJson);
assert_eq!(MediaFileType::from_str("pmd").unwrap(), MediaFileType::Pmd);
@@ -320,7 +309,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = MediaFileType::all_variants();
assert_eq!(variants.len(), 20);
assert_eq!(variants.len(), 19);
assert_eq!(variants.pop_first(), Some(MediaFileType::Audio));
assert_eq!(variants.pop_first(), Some(MediaFileType::Image));
assert_eq!(variants.pop_first(), Some(MediaFileType::Video));
@@ -328,7 +317,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(MediaFileType::Fbx));
assert_eq!(variants.pop_first(), Some(MediaFileType::Glb));
assert_eq!(variants.pop_first(), Some(MediaFileType::Gltf));
assert_eq!(variants.pop_first(), Some(MediaFileType::Spz));
assert_eq!(variants.pop_first(), Some(MediaFileType::SceneRon));
assert_eq!(variants.pop_first(), Some(MediaFileType::SceneJson));
assert_eq!(variants.pop_first(), Some(MediaFileType::Pmd));
@@ -38,13 +38,8 @@ pub enum ModelType {
SeedEdit3,
#[serde(rename = "qwen")]
Qwen,
/// Gemini 2.5 Flash, AKA "Nano Banana"
#[serde(rename = "gemini_25_flash")]
Gemini25Flash,
#[serde(rename = "nano_banana")]
NanoBanana,
#[serde(rename = "nano_banana_pro")]
NanoBananaPro,
/// Midjourney without distinguishing a model type or version
#[serde(rename = "midjourney")]
@@ -122,8 +117,6 @@ impl ModelType {
Self::SeedEdit3 => "seededit_3",
Self::Qwen => "qwen",
Self::Gemini25Flash => "gemini_25_flash",
Self::NanoBanana => "nano_banana",
Self::NanoBananaPro => "nano_banana_pro",
Self::Midjourney => "midjourney",
Self::MidjourneyV6 => "midjourney_v6",
Self::MidjourneyV6p1 => "midjourney_v6p1",
@@ -167,8 +160,6 @@ impl ModelType {
"seededit_3" => Ok(Self::SeedEdit3),
"qwen" => Ok(Self::Qwen),
"gemini_25_flash" => Ok(Self::Gemini25Flash),
"nano_banana" => Ok(Self::NanoBanana),
"nano_banana_pro" => Ok(Self::NanoBananaPro),
"midjourney" => Ok(Self::Midjourney),
"midjourney_v6" => Ok(Self::MidjourneyV6),
"midjourney_v6p1" => Ok(Self::MidjourneyV6p1),
@@ -216,8 +207,6 @@ impl ModelType {
Self::SeedEdit3,
Self::Qwen,
Self::Gemini25Flash,
Self::NanoBanana,
Self::NanoBananaPro,
Self::Midjourney,
Self::MidjourneyV6,
Self::MidjourneyV6p1,
@@ -270,8 +259,6 @@ mod tests {
assert_serialization(ModelType::SeedEdit3, "seededit_3");
assert_serialization(ModelType::Qwen, "qwen");
assert_serialization(ModelType::Gemini25Flash, "gemini_25_flash");
assert_serialization(ModelType::NanoBanana, "nano_banana");
assert_serialization(ModelType::NanoBananaPro, "nano_banana_pro");
assert_serialization(ModelType::Midjourney, "midjourney");
assert_serialization(ModelType::MidjourneyV6, "midjourney_v6");
assert_serialization(ModelType::MidjourneyV6p1, "midjourney_v6p1");
@@ -312,8 +299,6 @@ mod tests {
assert_eq!(ModelType::SeedEdit3.to_str(), "seededit_3");
assert_eq!(ModelType::Qwen.to_str(), "qwen");
assert_eq!(ModelType::Gemini25Flash.to_str(), "gemini_25_flash");
assert_eq!(ModelType::NanoBanana.to_str(), "nano_banana");
assert_eq!(ModelType::NanoBananaPro.to_str(), "nano_banana_pro");
assert_eq!(ModelType::Midjourney.to_str(), "midjourney");
assert_eq!(ModelType::MidjourneyV6.to_str(), "midjourney_v6");
assert_eq!(ModelType::MidjourneyV6p1.to_str(), "midjourney_v6p1");
@@ -356,8 +341,6 @@ mod tests {
assert_eq!(ModelType::from_str("seededit_3").unwrap(), ModelType::SeedEdit3);
assert_eq!(ModelType::from_str("qwen").unwrap(), ModelType::Qwen);
assert_eq!(ModelType::from_str("gemini_25_flash").unwrap(), ModelType::Gemini25Flash);
assert_eq!(ModelType::from_str("nano_banana").unwrap(), ModelType::NanoBanana);
assert_eq!(ModelType::from_str("nano_banana_pro").unwrap(), ModelType::NanoBananaPro);
assert_eq!(ModelType::from_str("midjourney").unwrap(), ModelType::Midjourney);
assert_eq!(ModelType::from_str("midjourney_v6").unwrap(), ModelType::MidjourneyV6);
assert_eq!(ModelType::from_str("midjourney_v6p1").unwrap(), ModelType::MidjourneyV6p1);
@@ -385,7 +368,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = ModelType::all_variants();
assert_eq!(variants.len(), 35);
assert_eq!(variants.len(), 33);
// Image models
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell));
@@ -400,8 +383,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(ModelType::SeedEdit3));
assert_eq!(variants.pop_first(), Some(ModelType::Qwen));
assert_eq!(variants.pop_first(), Some(ModelType::Gemini25Flash));
assert_eq!(variants.pop_first(), Some(ModelType::NanoBanana));
assert_eq!(variants.pop_first(), Some(ModelType::NanoBananaPro));
assert_eq!(variants.pop_first(), Some(ModelType::Midjourney));
assert_eq!(variants.pop_first(), Some(ModelType::MidjourneyV6));
assert_eq!(variants.pop_first(), Some(ModelType::MidjourneyV6p1));
@@ -29,10 +29,6 @@ pub enum TaskModelType {
FluxProKontextMax,
#[serde(rename = "gemini_25_flash")]
Gemini25Flash,
#[serde(rename = "nano_banana")]
NanoBanana,
#[serde(rename = "nano_banana_pro")]
NanoBananaPro,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "grok_image")]
@@ -89,8 +85,6 @@ impl TaskModelType {
Self::FluxPro11Ultra => "flux_pro_1.1_ultra",
Self::FluxProKontextMax => "flux_pro_kontext_max",
Self::Gemini25Flash => "gemini_25_flash",
Self::NanoBanana => "nano_banana",
Self::NanoBananaPro => "nano_banana_pro",
Self::GptImage1 => "gpt_image_1",
Self::GrokImage => "grok_image",
Self::Recraft3 => "recraft_3",
@@ -122,8 +116,6 @@ impl TaskModelType {
"flux_pro_1.1_ultra" => Ok(Self::FluxPro11Ultra),
"flux_pro_kontext_max" => Ok(Self::FluxProKontextMax),
"gemini_25_flash" => Ok(Self::Gemini25Flash),
"nano_banana" => Ok(Self::NanoBanana),
"nano_banana_pro" => Ok(Self::NanoBananaPro),
"gpt_image_1" => Ok(Self::GptImage1),
"grok_image" => Ok(Self::GrokImage),
"recraft_3" => Ok(Self::Recraft3),
@@ -158,8 +150,6 @@ impl TaskModelType {
Self::FluxPro11Ultra,
Self::FluxProKontextMax,
Self::Gemini25Flash,
Self::NanoBanana,
Self::NanoBananaPro,
Self::GptImage1,
Self::GrokImage,
Self::Recraft3,
@@ -201,8 +191,6 @@ mod tests {
assert_serialization(TaskModelType::FluxPro11Ultra, "flux_pro_1.1_ultra");
assert_serialization(TaskModelType::FluxProKontextMax, "flux_pro_kontext_max");
assert_serialization(TaskModelType::Gemini25Flash, "gemini_25_flash");
assert_serialization(TaskModelType::NanoBanana, "nano_banana");
assert_serialization(TaskModelType::NanoBananaPro, "nano_banana_pro");
assert_serialization(TaskModelType::GptImage1, "gpt_image_1");
assert_serialization(TaskModelType::GrokImage, "grok_image");
assert_serialization(TaskModelType::Recraft3, "recraft_3");
@@ -233,8 +221,6 @@ mod tests {
assert_eq!(TaskModelType::FluxPro11Ultra.to_str(), "flux_pro_1.1_ultra");
assert_eq!(TaskModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max");
assert_eq!(TaskModelType::Gemini25Flash.to_str(), "gemini_25_flash");
assert_eq!(TaskModelType::NanoBanana.to_str(), "nano_banana");
assert_eq!(TaskModelType::NanoBananaPro.to_str(), "nano_banana_pro");
assert_eq!(TaskModelType::GptImage1.to_str(), "gpt_image_1");
assert_eq!(TaskModelType::GrokImage.to_str(), "grok_image");
assert_eq!(TaskModelType::Recraft3.to_str(), "recraft_3");
@@ -265,8 +251,6 @@ mod tests {
assert_eq!(TaskModelType::from_str("flux_pro_1.1_ultra").unwrap(), TaskModelType::FluxPro11Ultra);
assert_eq!(TaskModelType::from_str("flux_pro_kontext_max").unwrap(), TaskModelType::FluxProKontextMax);
assert_eq!(TaskModelType::from_str("gemini_25_flash").unwrap(), TaskModelType::Gemini25Flash);
assert_eq!(TaskModelType::from_str("nano_banana").unwrap(), TaskModelType::NanoBanana);
assert_eq!(TaskModelType::from_str("nano_banana_pro").unwrap(), TaskModelType::NanoBananaPro);
assert_eq!(TaskModelType::from_str("gpt_image_1").unwrap(), TaskModelType::GptImage1);
assert_eq!(TaskModelType::from_str("grok_image").unwrap(), TaskModelType::GrokImage);
assert_eq!(TaskModelType::from_str("recraft_3").unwrap(), TaskModelType::Recraft3);
@@ -300,7 +284,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = TaskModelType::all_variants();
assert_eq!(variants.len(), 25);
assert_eq!(variants.len(), 23);
// Image models
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell));
@@ -310,8 +294,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11Ultra));
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxProKontextMax));
assert_eq!(variants.pop_first(), Some(TaskModelType::Gemini25Flash));
assert_eq!(variants.pop_first(), Some(TaskModelType::NanoBanana));
assert_eq!(variants.pop_first(), Some(TaskModelType::NanoBananaPro));
assert_eq!(variants.pop_first(), Some(TaskModelType::GptImage1));
assert_eq!(variants.pop_first(), Some(TaskModelType::GrokImage));
assert_eq!(variants.pop_first(), Some(TaskModelType::Recraft3));
@@ -42,8 +42,7 @@ GC_ENABLED_PUBLIC_BUCKET_NAME="REPLACE_ME"
AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
# Fake API key for FAL
#FAL_WEBHOOK_URL="https://macaroni-1.tailce84f.ts.net/v1/webhooks/fal"
FAL_WEBHOOK_URL="https://halide.tailce84f.ts.net/v1/webhooks/fal"
FAL_WEBHOOK_URL="https://macaroni-1.tailce84f.ts.net/v1/webhooks/fal"
#FAL_WEBHOOK_URL="https://s1.pancake-chimaera.ts.net/api/fal/webhook"
STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL="https://getartcraft.com/checkout_success"
@@ -90,28 +90,30 @@ pub async fn generate_flux_1_dev_text_to_image_handler(
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
// // TODO: This is test code
// let credits = match request.num_images {
// Some(GenerateFlux1DevTextToImageNumImages::One) => 100,
// Some(GenerateFlux1DevTextToImageNumImages::Two) => 200,
// Some(GenerateFlux1DevTextToImageNumImages::Three) => 300,
// Some(GenerateFlux1DevTextToImageNumImages::Four) => 400,
// None => 100,
// };
//
// // TODO: This is test code
// let result = temporary_test_wallet_deduction(
// user_token,
// Some("todo-reference-token"),
// credits,
// &mut mysql_connection,
// ).await;
//
// // TODO: This is test code
// if let Err(err) = result {
// warn!("Temporary wallet deduction failed: {:?}", err); // Infallible for now.
// }
// TODO: This is test code
let credits = match request.num_images {
Some(GenerateFlux1DevTextToImageNumImages::One) => 100,
Some(GenerateFlux1DevTextToImageNumImages::Two) => 200,
Some(GenerateFlux1DevTextToImageNumImages::Three) => 300,
Some(GenerateFlux1DevTextToImageNumImages::Four) => 400,
None => 100,
};
// TODO: This is test code
let result = temporary_test_wallet_deduction(
user_token,
Some("todo-reference-token"),
credits,
&mut mysql_connection,
).await;
// TODO: This is test code
if let Err(err) = result {
warn!("Temporary wallet deduction failed: {:?}", err); // Infallible for now.
}
const IS_MOD : bool = false;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
@@ -5,5 +5,4 @@ pub mod generate_flux_pro_11_text_to_image_handler;
pub mod generate_flux_pro_11_ultra_text_to_image_handler;
pub mod generate_gpt_image_1_text_to_image_handler;
pub mod inpaint;
pub mod remove_image_background_handler;
pub mod multi_function;
pub mod remove_image_background_handler;
@@ -1 +0,0 @@
pub mod nano_banana_pro_multi_function_image_gen_handler;
@@ -1,372 +0,0 @@
use std::collections::HashSet;
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest, NanoBananaProMultiFunctionImageGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::edit::enqueue_nano_banana_pro_edit_image_webhook::{enqueue_nano_banana_pro_image_edit_webhook, EnqueueNanoBananaProEditImageArgs, EnqueueNanoBananaProEditImageAspectRatio, EnqueueNanoBananaProEditImageNumImages, EnqueueNanoBananaProEditImageResolution};
use fal_client::requests::webhook::image::text::enqueue_nano_banana_pro_text_to_image_webhook::{enqueue_nano_banana_pro_text_to_image_webhook, EnqueueNanoBananaProTextToImageArgs, EnqueueNanoBananaProTextToImageAspectRatio, EnqueueNanoBananaProTextToImageNumImages, EnqueueNanoBananaProTextToImageResolution};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Nano Banana Pro Multi-Function (generate + edit)
#[utoipa::path(
post,
tag = "Generate Images (Multi-Function)",
path = "/v1/generate/image/multi_function/nano_banana_pro",
responses(
(status = 200, description = "Success", body = NanoBananaProMultiFunctionImageGenResponse),
),
params(
("request" = NanoBananaProMultiFunctionImageGenRequest, description = "Payload for Request"),
)
)]
pub async fn nano_banana_pro_multi_function_image_gen_handler(
http_request: HttpRequest,
request: Json<NanoBananaProMultiFunctionImageGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<NanoBananaProMultiFunctionImageGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let image_urls = match request.image_media_tokens.as_ref() {
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls(
&http_request,
&mut mysql_connection,
server_state.server_environment,
media_tokens,
).await?
},
None => None,
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(input_image_urls) = image_urls.as_deref() {
info!("nano banana pro edit image");
let num_images = match request.num_images {
Some(NanoBananaProMultiFunctionImageGenNumImages::One) => EnqueueNanoBananaProEditImageNumImages::One,
Some(NanoBananaProMultiFunctionImageGenNumImages::Two) => EnqueueNanoBananaProEditImageNumImages::Two,
Some(NanoBananaProMultiFunctionImageGenNumImages::Three) => EnqueueNanoBananaProEditImageNumImages::Three,
Some(NanoBananaProMultiFunctionImageGenNumImages::Four) => EnqueueNanoBananaProEditImageNumImages::Four,
None => EnqueueNanoBananaProEditImageNumImages::One, // Default to One
};
let resolution = match request.resolution {
Some(NanoBananaProMultiFunctionImageGenImageResolution::OneK) => EnqueueNanoBananaProEditImageResolution::OneK,
Some(NanoBananaProMultiFunctionImageGenImageResolution::TwoK) => EnqueueNanoBananaProEditImageResolution::TwoK,
Some(NanoBananaProMultiFunctionImageGenImageResolution::FourK) => EnqueueNanoBananaProEditImageResolution::FourK,
None => EnqueueNanoBananaProEditImageResolution::OneK,
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProEditImageAspectRatio::OneByOne,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProEditImageAspectRatio::FiveByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProEditImageAspectRatio::FourByThree,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByTwo) => EnqueueNanoBananaProEditImageAspectRatio::ThreeByTwo,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine) => EnqueueNanoBananaProEditImageAspectRatio::SixteenByNine,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::TwentyOneByNine) => EnqueueNanoBananaProEditImageAspectRatio::TwentyOneByNine,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByFive) => EnqueueNanoBananaProEditImageAspectRatio::FourByFive,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByFour) => EnqueueNanoBananaProEditImageAspectRatio::ThreeByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::TwoByThree) => EnqueueNanoBananaProEditImageAspectRatio::TwoByThree,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen) => EnqueueNanoBananaProEditImageAspectRatio::NineBySixteen,
None => EnqueueNanoBananaProEditImageAspectRatio::OneByOne,
};
let args = EnqueueNanoBananaProEditImageArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
image_urls: input_image_urls.to_owned(),
num_images,
resolution: Some(resolution),
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_nano_banana_pro_image_edit_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_nano_banana_pro_image_edit_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("nano banana pro text-to-image");
let num_images = match request.num_images {
Some(NanoBananaProMultiFunctionImageGenNumImages::One) => EnqueueNanoBananaProTextToImageNumImages::One,
Some(NanoBananaProMultiFunctionImageGenNumImages::Two) => EnqueueNanoBananaProTextToImageNumImages::Two,
Some(NanoBananaProMultiFunctionImageGenNumImages::Three) => EnqueueNanoBananaProTextToImageNumImages::Three,
Some(NanoBananaProMultiFunctionImageGenNumImages::Four) => EnqueueNanoBananaProTextToImageNumImages::Four,
None => EnqueueNanoBananaProTextToImageNumImages::One, // Default to One
};
let resolution = match request.resolution {
Some(NanoBananaProMultiFunctionImageGenImageResolution::OneK) => EnqueueNanoBananaProTextToImageResolution::OneK,
Some(NanoBananaProMultiFunctionImageGenImageResolution::TwoK) => EnqueueNanoBananaProTextToImageResolution::TwoK,
Some(NanoBananaProMultiFunctionImageGenImageResolution::FourK) => EnqueueNanoBananaProTextToImageResolution::FourK,
None => EnqueueNanoBananaProTextToImageResolution::OneK,
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProTextToImageAspectRatio::FiveByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProTextToImageAspectRatio::FourByThree,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByTwo) => EnqueueNanoBananaProTextToImageAspectRatio::ThreeByTwo,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine) => EnqueueNanoBananaProTextToImageAspectRatio::SixteenByNine,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::TwentyOneByNine) => EnqueueNanoBananaProTextToImageAspectRatio::TwentyOneByNine,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByFive) => EnqueueNanoBananaProTextToImageAspectRatio::FourByFive,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByFour) => EnqueueNanoBananaProTextToImageAspectRatio::ThreeByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::TwoByThree) => EnqueueNanoBananaProTextToImageAspectRatio::TwoByThree,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen) => EnqueueNanoBananaProTextToImageAspectRatio::NineBySixteen,
None => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne,
};
let args = EnqueueNanoBananaProTextToImageArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
num_images,
resolution: Some(resolution),
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_nano_banana_pro_text_to_image_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_nano_banana_pro_text_to_image_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::NanoBananaPro),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(media_tokens) = &request.image_media_tokens {
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: media_tokens.iter().map(|token| {
PromptContextItem {
media_token: token.clone(),
context_semantic_type: PromptContextSemanticType::Imgref,
}
}).collect(),
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(NanoBananaProMultiFunctionImageGenResponse {
success: true,
inference_job_token: job_token,
}))
}
async fn lookup_image_urls(
http_request: &HttpRequest,
mysql_connection: &mut PoolConnection<MySql>,
server_environment: ServerEnvironment,
tokens: &[MediaFileToken],
) -> Result<Option<Vec<String>>, CommonWebError> {
const CAN_SEE_DELETED: bool = false;
let result = batch_get_media_files_by_tokens_with_connection(
mysql_connection,
tokens,
CAN_SEE_DELETED,
).await;
let media_files = match result {
Ok(files) => files,
Err(err) => {
error!("Error getting media files by tokens: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
if media_files.len() != tokens.len() {
warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), tokens.len());
let requested : HashSet<&MediaFileToken> = HashSet::from_iter(tokens.iter());
let returned : HashSet<&MediaFileToken> = HashSet::from_iter(media_files.iter().map(|m| &m.token));
let diff = requested.difference(&returned)
.cloned()
.collect::<Vec<&MediaFileToken>>();
return Err(CommonWebError::BadInputWithSimpleMessage(
format!("Not all media files could be found. Media files found: {}, tokens provided: {}, in original: {:?}, req {:?}, ret {:?}",
media_files.len(), tokens.len(), diff, requested, returned)));
}
let media_domain = get_media_domain(&http_request);
let image_urls = media_files.iter()
.map(|file| {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&file.public_bucket_directory_hash,
file.maybe_public_bucket_prefix.as_deref(),
file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
})
.collect::<Vec<_>>();
if image_urls.is_empty() {
return Ok(None)
}
Ok(Some(image_urls))
}
@@ -7,7 +7,6 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
@@ -16,7 +15,6 @@ use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mimetypes::mimetype_to_extension::mimetype_to_extension;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use tokens::tokens::media_files::MediaFileToken;
@@ -28,8 +26,6 @@ use crate::state::server_state::ServerState;
// TODO(bt,2023-12-20): THIS CODE NEEDS CLEANUP. This has been cargo culted three+ times.
// It's ridiculous and complicated.
const GZIP_MIME_TYPE : &str = "application/gzip";
pub enum SuccessCase {
MediaAlreadyUploaded {
existing_media_file_token: MediaFileToken,
@@ -172,21 +168,9 @@ pub async fn process_upload_media_file(
let mut maybe_codec_name = None;
let mut media_file_type = None;
let mut is_spz = false;
if let Some(mimetype) = maybe_mimetype.as_deref() {
// `.spz` files are Gaussian splats
let is_spz_extension = upload_media_request.file_name
.as_deref()
.map(|filename| filename.trim().to_ascii_lowercase().ends_with(".spz"))
.unwrap_or(false);
is_spz = mimetype == GZIP_MIME_TYPE && is_spz_extension;
let is_allowed_mimetype_bypass = is_spz;
if !allowed_mimetypes.contains(mimetype) && !is_allowed_mimetype_bypass {
if !allowed_mimetypes.contains(mimetype) {
// NB: Don't let our error message inject malicious strings
let filtered_mimetype = mimetype
.chars()
@@ -256,8 +240,6 @@ pub async fn process_upload_media_file(
"image/jpeg" => false,
"image/png" => false,
"image/webp" => false,
// Don't decode gzip, gaussian splats (spz)
"application/gzip" => false,
_ => true,
};
@@ -276,10 +258,6 @@ pub async fn process_upload_media_file(
}
}
if is_spz {
media_file_type = Some(MediaFileType::Spz);
}
if media_file_type.is_none() && maybe_mimetype.is_none() {
// https://research.cs.wisc.edu/graphics/Courses/cs-838-1999/Jeff/BVH.html
const BVH_HEADER : &[u8] = "HIERARCHY".as_bytes();
@@ -320,7 +298,6 @@ pub async fn process_upload_media_file(
MediaFileType::Fbx => MediaFileClass::Dimensional,
MediaFileType::Glb => MediaFileClass::Dimensional,
MediaFileType::Gltf => MediaFileClass::Dimensional,
MediaFileType::Spz => MediaFileClass::Dimensional,
MediaFileType::SceneRon => MediaFileClass::Dimensional,
MediaFileType::SceneJson => MediaFileClass::Dimensional,
MediaFileType::Pmd => MediaFileClass::Dimensional,
@@ -358,23 +335,10 @@ pub async fn process_upload_media_file(
MediaFileType::Bvh => Some(".bvh".to_string()),
MediaFileType::Fbx => Some(".fbx".to_string()),
MediaFileType::Gltf => Some(".gltf".to_string()),
MediaFileType::Spz => Some(".spz".to_string()),
_ => None,
};
}
// For WorldLabs spz files.
let is_world_labs_spz= upload_media_request.file_name
.as_deref()
.map(|filename| filename.trim().to_ascii_lowercase())
.filter(|filename| filename.contains("ceramic"))
.filter(|filename| filename.ends_with(".spz"))
.is_some();
if is_world_labs_spz {
extension = Some(".ceramic.spz".to_string());
}
const PREFIX : Option<&str> = Some("upload_");
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, extension.as_deref());
@@ -391,61 +355,39 @@ pub async fn process_upload_media_file(
MediaFileUploadError::ServerError
})?;
let media_token = MediaFileInsertBuilder::new()
.maybe_creator_user(maybe_user_token.as_ref())
.maybe_creator_anonymous_visitor(maybe_avt_token.as_ref())
.creator_ip_address(&ip_address)
.public_bucket_directory_hash(&public_upload_path)
.media_file_class(media_file_class)
.media_file_type(media_file_type)
.media_file_origin_category(MediaFileOriginCategory::Upload)
.creator_set_visibility(creator_set_visibility)
.mime_type(mime_type)
.file_size_bytes(file_size_bytes as u64)
.checksum_sha2(&hash)
.maybe_title(upload_media_request.title.as_deref())
.maybe_origin_filename(upload_media_request.file_name.as_deref())
.is_intermediate_system_file(false)
.insert_pool(&server_state.mysql_pool)
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(media_file_class),
media_file_type,
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type,
maybe_engine_category: None,
maybe_prompt_token: None,
maybe_batch_token: None,
maybe_animation_type: None,
maybe_mime_type: Some(mime_type),
file_size_bytes: file_size_bytes as u64,
maybe_duration_millis: None, // TODO: Fix this?
sha256_checksum: &hash,
maybe_title: upload_media_request.title.as_deref(),
maybe_scene_source_media_file_token: None,
is_intermediate_system_file: false, // NB: is_user_upload = TRUE
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: extension.as_deref(),
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New generic download creation DB error: {:?}", err);
MediaFileUploadError::ServerError
})?;
//let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
// maybe_media_class: Some(media_file_class), // DONE
// media_file_type, // DONE
// maybe_creator_user_token: maybe_user_token.as_ref(), // DONE
// maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), // DONE
// creator_ip_address: &ip_address, // DONE
// creator_set_visibility, // DONE
// upload_type, // DONE - weird type
// maybe_engine_category: None, // DONE
// maybe_prompt_token: None, // DONE
// maybe_batch_token: None, // DONE
// maybe_animation_type: None, // DONE
// maybe_mime_type: Some(mime_type), // DONE
// file_size_bytes: file_size_bytes as u64, // DONE
// maybe_duration_millis: None, // TODO: Fix this? // DONE
// sha256_checksum: &hash, // DONE
// maybe_title: upload_media_request.title.as_deref(), // DONE
// maybe_scene_source_media_file_token: None, // DONE
// is_intermediate_system_file: false, // NB: is_user_upload = TRUE // DONE
// public_bucket_directory_hash: public_upload_path.get_object_hash(), // DONE
// maybe_public_bucket_prefix: PREFIX, // DONE
// maybe_public_bucket_extension: extension.as_deref(), // DONE
// pool: &server_state.mysql_pool,
//})
// .await
// .map_err(|err| {
// warn!("New generic download creation DB error: {:?}", err);
// MediaFileUploadError::ServerError
// })?;
info!("new media file - token: {:?}", &media_token);
info!("new media file id: {} token: {:?}", record_id, &token);
Ok(SuccessCase::MediaSuccessfullyUploaded {
media_file_token: media_token,
media_file_token: token,
})
}
@@ -10,7 +10,6 @@ use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_t
use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler;
use crate::http_server::endpoints::generate::image::inpaint::flux_dev_juggernaut_inpaint_handler::flux_dev_juggernaut_inpaint_image_handler;
use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_handler::flux_pro_1_inpaint_image_handler;
use crate::http_server::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_handler::nano_banana_pro_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_1_image_to_3d_handler::generate_hunyuan_2_1_image_to_3d_handler;
@@ -40,12 +39,6 @@ where
{
app.service(web::scope("/v1/generate")
.service(web::scope("/image")
.service(web::scope("/multi_function")
.service(web::resource("/nano_banana_pro")
.route(web::post().to(nano_banana_pro_multi_function_image_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::scope("/edit")
.service(web::resource("/flux_pro_kontext_max")
.route(web::post().to(flux_pro_kontext_max_edit_image_handler))
@@ -4,10 +4,10 @@ import {
faFilm,
faPalette,
faImage,
faPenNib,
faDroplet,
faPhotoFilm,
faGlobe,
faPencil,
} from "@fortawesome/pro-solid-svg-icons";
import { useTabStore, TabId } from "~/pages/Stores/TabState";
import { set3DPageMounted } from "~/pages/PageEnigma/Editor/editor";
@@ -47,7 +47,7 @@ export const APP_DESCRIPTORS: AppDescriptor[] = [
{
id: "EDIT",
label: "Edit Image",
icon: faPencil,
icon: faPenNib,
imageSrc: "/resources/gifs/INPAINT_CANVAS_DEMO.gif",
description: "Modify your image with AI",
large: true,
@@ -123,7 +123,7 @@ export const ALL_APPS: FullAppItem[] = [
id: "edit-image",
label: "Edit Image",
description: "Change with inpainting",
icon: faPencil,
icon: faPenNib,
category: "edit",
action: "EDIT",
color: "bg-purple-600/40",
@@ -283,11 +283,11 @@ const PageDraw = () => {
onAspectRatioChange={async (ratio: string) => {
const ratioToType = (ratio: string): AspectRatioType => {
switch (ratio) {
case "tall":
case "2:3":
return AspectRatioType.PORTRAIT;
case "wide":
case "3:2":
return AspectRatioType.LANDSCAPE;
case "square":
case "1:1":
return AspectRatioType.SQUARE;
default:
return AspectRatioType.NONE;
@@ -1,4 +1,4 @@
import { faImages, faPencil } from "@fortawesome/pro-solid-svg-icons";
import { faImage, faImages } from "@fortawesome/pro-solid-svg-icons";
import { downloadFileFromUrl } from "libs/api/src/lib/LocalApi";
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
import { useState } from "react";
@@ -153,7 +153,7 @@ export const BaseImageSelector = ({
<div className="flex h-full w-full items-center justify-center overflow-hidden bg-ui-panel text-base-fg">
<div className="aspect-video w-full max-w-5xl bg-ui-background">
<UploadEntryCard
icon={faPencil}
icon={faImage}
title="Edit Image"
description="Click to upload or drag and drop an image here to edit"
accentBackgroundClass="bg-blue-500/40"
@@ -1,13 +1,6 @@
import { useState, useRef, useEffect, useCallback } from "react";
import Konva from "konva"; // just for types
import {
EnqueueEditImage,
EnqueueImageInpaint,
EnqueueEditImageSize,
EnqueueEditImageResolution,
} from "@storyteller/tauri-api";
import { PromptsApi } from "@storyteller/api";
import { RefImage, usePromptEditStore } from "@storyteller/ui-promptbox";
import { EnqueueImageInpaint } from "@storyteller/tauri-api";
import { ContextMenuContainer } from "../PageDraw/components/ui/ContextMenu";
import { useCopyPasteHotkeys } from "../PageDraw/hooks/useCopyPasteHotkeys";
import { useDeleteHotkeys } from "../PageDraw/hooks/useDeleteHotkeys";
@@ -65,7 +58,6 @@ const PageEdit = () => {
const historyImageBundles = useEditStore(
(state) => state.historyImageBundles,
);
const referenceImages = usePromptEditStore((s) => s.referenceImages);
// Pass store actions directly as callbacks
useDeleteHotkeys({ onDelete: store.deleteSelectedItems });
@@ -154,16 +146,17 @@ const PageEdit = () => {
// Top toolbar (MarkerToolControlBar): ~140px from top (toolbar + padding)
// Bottom prompt box: ~200px from bottom (can vary with reference images)
// Mode selector above prompt: ~60px
const hasTopToolbar =
const topToolbarHeight =
store.activeTool === "marker" ||
store.activeTool === "eraser" ||
store.activeTool === "edit";
const topToolbarHeight = hasTopToolbar ? 140 : 80;
store.activeTool === "edit"
? 140
: 0;
// Check if model supports masking or is nano banana (has mode selector)
const hasModeSelectorUI =
selectedImageModel?.usesInpaintingMask ||
selectedImageModel?.isNanoBananaModel();
selectedImageModel?.id === "gemini_25_flash";
const bottomUIHeight = hasModeSelectorUI ? 260 : 200;
const totalVerticalReserved = topToolbarHeight + bottomUIHeight;
@@ -276,54 +269,8 @@ const PageEdit = () => {
return new Uint8Array(arrayBuffer);
};
const getCompositeCanvasFile = async (): Promise<File | null> => {
if (
!stageRef.current ||
!leftPanelRef.current ||
!baseImageKonvaRef.current ||
!store.baseImageBitmap
) {
return null;
}
const rect = baseImageKonvaRef.current;
const width = rect.width();
const height = rect.height();
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d");
if (!ctx) return null;
ctx.drawImage(store.baseImageBitmap, 0, 0, width, height);
const markerLayerCanvas = leftPanelRef.current.toCanvas({
x: stageRef.current.x(),
y: stageRef.current.y(),
width: rect.width() * stageRef.current.scaleX(),
height: rect.height() * stageRef.current.scaleY(),
pixelRatio: 1 / stageRef.current.scaleX(),
});
const fittedMarkerCanvas = normalizeCanvas(
markerLayerCanvas,
width,
height,
);
ctx.drawImage(fittedMarkerCanvas, 0, 0, width, height);
const blob = await canvas.convertToBlob({ type: "image/png" });
const uuid = crypto.randomUUID();
return new File([blob], `${uuid}.png`, { type: "image/png" });
};
const handleGenerate = useCallback(
async (
prompt: string,
options?: {
aspectRatio?: string;
resolution?: string;
images?: RefImage[];
},
) => {
async (prompt: string) => {
const editedImageToken = store.baseImageInfo?.mediaToken;
if (!editedImageToken) {
@@ -331,106 +278,22 @@ const PageEdit = () => {
return;
}
// Helper to map aspect ratio string to enum
const mapAspectRatio = (
ratio?: string,
): EnqueueEditImageSize | undefined => {
switch (ratio) {
case "auto":
return EnqueueEditImageSize.Auto;
case "wide":
return EnqueueEditImageSize.Wide;
case "tall":
return EnqueueEditImageSize.Tall;
case "square":
return EnqueueEditImageSize.Square;
default:
return undefined;
}
};
// Helper to map resolution string to enum
const mapResolution = (
res?: string,
): EnqueueEditImageResolution | undefined => {
switch (res) {
case "1k":
return EnqueueEditImageResolution.OneK;
case "2k":
return EnqueueEditImageResolution.TwoK;
case "4k":
return EnqueueEditImageResolution.FourK;
default:
return undefined;
}
};
const arrayBuffer = await getMaskArrayBuffer();
const subscriberId: string =
crypto?.randomUUID?.() ??
`inpaint-${Date.now()}-${Math.random().toString(36).slice(2)}`;
try {
let result;
if (selectedImageModel?.editingIsInpainting) {
const arrayBuffer = await getMaskArrayBuffer();
result = await EnqueueImageInpaint({
model: selectedImageModel,
image_media_token: editedImageToken,
mask_image_raw_bytes: arrayBuffer,
prompt: prompt,
image_count: generationCount,
frontend_caller: "image_editor",
frontend_subscriber_id: subscriberId,
});
} else if (selectedImageModel?.isNanoBananaModel()) {
const compositeFile = await getCompositeCanvasFile();
if (!compositeFile) {
console.error("Failed to create composite canvas");
return;
}
const api = new PromptsApi();
const snapshotResult = await api.uploadSceneSnapshot({
screenshot: compositeFile,
});
if (!snapshotResult.data) {
console.error("Failed to upload scene snapshot");
return;
}
const imgs = options?.images || [];
result = await EnqueueEditImage({
model: selectedImageModel,
scene_image_media_token: snapshotResult.data,
image_media_tokens: imgs
.map((img) => img.mediaToken)
.filter((t) => t.length > 0),
prompt: prompt,
image_count: generationCount,
frontend_caller: "image_editor",
frontend_subscriber_id: subscriberId,
aspect_ratio: mapAspectRatio(options?.aspectRatio),
image_resolution: mapResolution(options?.resolution),
});
} else {
const imgs = options?.images || [];
result = await EnqueueEditImage({
model: selectedImageModel,
image_media_tokens: [
editedImageToken,
...imgs
.filter((img) => img.mediaToken !== editedImageToken)
.map((img) => img.mediaToken),
].filter((t) => t.length > 0),
disable_system_prompt: true,
prompt: prompt,
image_count: generationCount,
frontend_caller: "image_editor",
frontend_subscriber_id: subscriberId,
aspect_ratio: mapAspectRatio(options?.aspectRatio),
image_resolution: mapResolution(options?.resolution),
});
}
const result = await EnqueueImageInpaint({
model: selectedImageModel,
image_media_token: editedImageToken,
mask_image_raw_bytes: arrayBuffer,
prompt: prompt,
image_count: generationCount,
frontend_caller: "image_editor",
frontend_subscriber_id: subscriberId,
});
if (result.status === "success") {
setPendingGenerations((prev) => [
...prev,
@@ -444,17 +307,10 @@ const PageEdit = () => {
throw error;
}
},
// eslint-disable-next-line react-hooks/exhaustive-deps
[
generationCount,
selectedImageModel,
store.baseImageInfo?.mediaToken,
referenceImages,
],
[generationCount, selectedImageModel, store.baseImageInfo?.mediaToken],
);
const isNanoBananaModel = selectedImageModel?.isNanoBananaModel();
const isNanoBananaModel = selectedImageModel?.id === "gemini_25_flash";
const supportsMaskedInpainting =
selectedImageModel?.usesInpaintingMask ?? false;
@@ -1,11 +1,10 @@
import React from "react";
import { PromptBoxEdit, PromptBoxEditProps } from "@storyteller/ui-promptbox";
import { uploadImage } from "../../../components/reusable/UploadModalMedia/uploadImage";
const PromptEditor: React.FC<PromptBoxEditProps> = (props) => {
return (
<div className="mx-auto flex w-full max-w-3xl flex-col space-y-2">
<PromptBoxEdit {...props} uploadImage={uploadImage} />
<PromptBoxEdit {...props} />
</div>
);
};
@@ -1,6 +1,6 @@
import * as THREE from "three";
import { TransformControls } from "three/examples/jsm/Addons";
import { FKBoneBlacklistStrings } from "./FKBoneFilter";
import { TransformControls } from "../TransformControls";
export const FKHelperSphereName = "__FKHelperSphere__";
@@ -1335,10 +1335,10 @@ class Editor {
this.rawRenderer &&
this.render_composer
) {
//const then = performance.now();
const then = performance.now();
this.composer.render();
//const now = performance.now();
//console.log("Performance: ", now - then);
const now = performance.now();
console.log("Performance: ", now - then);
//this.rawRenderer.render(this.activeScene.scene, this.render_camera!);
// this.render_composer.render();
} else if (this.renderer && this.render_camera && !this.rendering) {
@@ -59,14 +59,14 @@ interface LightboxModalProps {
onDownloadClicked?: (url: string, mediaClass?: string) => Promise<void>;
onAddToSceneClicked?: (
url: string,
media_id: string | undefined,
media_id: string | undefined
) => Promise<void>;
mediaClass?: string;
onPromptCopy?: (prompt: string) => void;
onEditClicked?: (url: string, media_id?: string) => Promise<void> | void;
onTurnIntoVideoClicked?: (
url: string,
media_id?: string,
media_id?: string
) => Promise<void> | void;
batchImageToken?: string;
}
@@ -104,7 +104,7 @@ export function LightboxModal({
const [hasPromptToken, setHasPromptToken] = useState<boolean>(false);
const [isPromptHovered, setIsPromptHovered] = useState<boolean>(false);
const [generationProvider, setGenerationProvider] = useState<string | null>(
null,
null
);
const [modelType, setModelType] = useState<string | null>(null);
const [contextImages, setContextImages] = useState<Array<{
@@ -121,7 +121,7 @@ export function LightboxModal({
const shareCopiedTimeoutRef = useRef<number | null>(null);
const [currentMediaId, setCurrentMediaId] = useState<string | undefined>(
mediaId,
mediaId
);
useEffect(() => {
setCurrentMediaId(mediaId);
@@ -236,7 +236,7 @@ export function LightboxModal({
}))
.filter(
(item): item is { url: string; token: string } =>
Boolean(item.url) && Boolean(item.token),
Boolean(item.url) && Boolean(item.token)
);
if (items.length > 0) {
@@ -293,7 +293,7 @@ export function LightboxModal({
if (!emblaMainApi || !emblaThumbsApi) return;
emblaMainApi.scrollTo(index);
},
[emblaMainApi, emblaThumbsApi],
[emblaMainApi, emblaThumbsApi]
);
const onSelect = useCallback(() => {
@@ -427,9 +427,7 @@ export function LightboxModal({
PLACEHOLDER_IMAGES.DEFAULT;
e.currentTarget.style.opacity = "0.3";
// Set the `data-brokenurl` property for debugging the broken images:
(
e.currentTarget as HTMLImageElement
).dataset.brokenurl = url || "";
(e.currentTarget as HTMLImageElement).dataset.brokenurl = url || "";
}
}}
onLoad={() => {
@@ -462,7 +460,7 @@ export function LightboxModal({
"embla-thumbs__slide relative h-20 w-20 flex-[0_0_5rem] overflow-hidden rounded-md border-2 transition-all",
isSelected
? "border-brand-primary-400 opacity-100"
: "border-transparent opacity-60 hover:border-white/40 hover:opacity-100",
: "border-transparent opacity-60 hover:border-white/40 hover:opacity-100"
)}
>
<img
@@ -480,13 +478,11 @@ export function LightboxModal({
</div>
)}
{!mediaLoaded &&
selectedImageUrl &&
mediaClass !== "dimensional" && (
<div className="absolute inset-0 bg-ui-panel flex items-center justify-center">
<LoadingSpinner className="h-12 w-12 text-base-fg" />
</div>
)}
{!mediaLoaded && selectedImageUrl && mediaClass !== "dimensional" && (
<div className="absolute inset-0 bg-ui-panel flex items-center justify-center">
<LoadingSpinner className="h-12 w-12 text-base-fg" />
</div>
)}
</div>
{/* info + actions */}
@@ -516,7 +512,7 @@ export function LightboxModal({
</div>
<div
className={twMerge(
"relative text-sm text-base-fg break-words p-3 rounded-lg cursor-pointer transition-colors duration-100 leading-relaxed",
"relative text-sm text-base-fg break-words p-3 rounded-lg cursor-pointer transition-colors duration-100 leading-relaxed"
)}
style={{
background: isPromptHovered
@@ -551,7 +547,7 @@ export function LightboxModal({
<div
className={twMerge(
"pointer-events-none absolute inset-0 flex items-end justify-end opacity-0 transition-opacity duration-50",
isPromptHovered && "opacity-100",
isPromptHovered && "opacity-100"
)}
>
<div
@@ -572,7 +568,7 @@ export function LightboxModal({
<div className="text-sm font-medium text-base-fg/90">
Reference Images
</div>
<div className="grid grid-cols-6 gap-2 w-fit">
<div className="grid grid-cols-6 gap-2">
{contextImages.map((contextImage, index) => {
const { thumbnail, fullSize } =
getContextImageThumbnail(contextImage, {
@@ -582,7 +578,7 @@ export function LightboxModal({
return (
<div
key={contextImage.media_token}
className="glass relative aspect-square overflow-hidden rounded-lg w-12 border-2 border-white/30 hover:border-white/80 transition-all group cursor-pointer hover:cursor-zoom-in"
className="glass relative aspect-square overflow-hidden rounded-lg w-14 border-2 border-white/30 hover:border-white/80 transition-all group cursor-pointer hover:cursor-zoom-in"
onClick={() => setRefPreviewUrl(fullSize)}
>
<img
@@ -635,7 +631,7 @@ export function LightboxModal({
<div className="flex items-center gap-2">
{getProviderIconByName(
generationProvider,
"h-4 w-4 invert",
"h-4 w-4 invert"
)}
<span className="text-sm text-base-fg rounded">
{getProviderDisplayName(generationProvider)}
@@ -685,7 +681,7 @@ export function LightboxModal({
setShareCopied(true);
if (shareCopiedTimeoutRef.current) {
window.clearTimeout(
shareCopiedTimeoutRef.current,
shareCopiedTimeoutRef.current
);
}
shareCopiedTimeoutRef.current = window.setTimeout(
@@ -693,7 +689,7 @@ export function LightboxModal({
setShareCopied(false);
shareCopiedTimeoutRef.current = null;
},
1500,
1500
);
} catch (err) {
toast.error("Unable to copy link");
@@ -716,7 +712,7 @@ export function LightboxModal({
gtagEvent("edit_image_clicked");
await onEditClicked(
actionUrl,
selectedMediaToken,
selectedMediaToken
);
}}
>
@@ -735,7 +731,7 @@ export function LightboxModal({
gtagEvent("turn_into_video_clicked");
await onTurnIntoVideoClicked(
actionUrl,
selectedMediaToken,
selectedMediaToken
);
}}
>
@@ -752,7 +748,7 @@ export function LightboxModal({
gtagEvent("add_to_scene_clicked");
await onAddToSceneClicked(
actionUrl,
selectedMediaToken,
selectedMediaToken
);
onClose();
onCloseGallery();
@@ -17,7 +17,6 @@ import {
faSpinnerThird,
faFrame,
faCopy,
faExpand,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangleVertical,
@@ -29,9 +28,8 @@ import { IconProp } from "@fortawesome/fontawesome-svg-core";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { PromptsApi } from "@storyteller/api";
import {
EnqueueEditImage,
EnqueueEditImageSize,
EnqueueEditImageResolution,
EnqueueContextualEditImage,
EnqueueContextualEditImageSize,
} from "@storyteller/tauri-api";
import { usePrompt2DStore } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
@@ -40,7 +38,7 @@ import { ImageModel } from "@storyteller/model-list";
import { ImagePromptRow } from "./ImagePromptRow";
import { twMerge } from "tailwind-merge";
export type AspectRatio = "wide" | "tall" | "square";
export type AspectRatio = "1:1" | "3:2" | "2:3";
interface PromptBox2DProps {
uploadImage: ({
@@ -84,8 +82,6 @@ export const PromptBox2D = ({
const setUseSystemPrompt = usePrompt2DStore((s) => s.setUseSystemPrompt);
const aspectRatio = usePrompt2DStore((s) => s.aspectRatio);
const setAspectRatio = usePrompt2DStore((s) => s.setAspectRatio);
const resolution = usePrompt2DStore((s) => s.resolution);
const setResolution = usePrompt2DStore((s) => s.setResolution);
const generationCount = usePrompt2DStore((s) => s.generationCount);
const setGenerationCount = usePrompt2DStore((s) => s.setGenerationCount);
@@ -95,42 +91,24 @@ export const PromptBox2D = ({
const [showImagePrompts, setShowImagePrompts] = useState(false);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "Wide",
selected: aspectRatio === "wide",
label: "3:2",
selected: aspectRatio === "3:2",
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "Tall",
selected: aspectRatio === "tall",
label: "2:3",
selected: aspectRatio === "2:3",
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "Square",
selected: aspectRatio === "square",
label: "1:1",
selected: aspectRatio === "1:1",
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
{
label: "1K",
selected: resolution === "1k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "2K",
selected: resolution === "2k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "4K",
selected: resolution === "4k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
]);
const [generationCountList, setGenerationCountList] = useState<PopoverItem[]>(
[],
[]
);
// Update generation count options from selected model capabilities
@@ -138,7 +116,7 @@ export const PromptBox2D = ({
const caps = getCapabilitiesForModel(selectedImageModel);
const items: PopoverItem[] = Array.from(
{ length: caps.maxGenerationCount },
(_, i) => i + 1,
(_, i) => i + 1
).map((count) => ({
label: String(count),
selected: count === generationCount,
@@ -148,7 +126,7 @@ export const PromptBox2D = ({
// Clamp selection to allowed range
if (generationCount < 1 || generationCount > caps.maxGenerationCount) {
setGenerationCount(
Math.min(Math.max(1, generationCount), caps.maxGenerationCount),
Math.min(Math.max(1, generationCount), caps.maxGenerationCount)
);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -160,19 +138,10 @@ export const PromptBox2D = ({
prev.map((item) => ({
...item,
selected: item.label === String(generationCount),
})),
}))
);
}, [generationCount]);
useEffect(() => {
setResolutionList((prev) =>
prev.map((item) => ({
...item,
selected: item.label.toLowerCase() === resolution,
})),
);
}, [resolution]);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const isImageRowVisible =
showImagePrompts ||
@@ -187,21 +156,16 @@ export const PromptBox2D = ({
});
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
const value = selectedItem.label.toLowerCase() as AspectRatio;
onAspectRatioChange?.(value);
setAspectRatio(value);
onAspectRatioChange?.(selectedItem.label as AspectRatio);
setAspectRatio(selectedItem.label as AspectRatio);
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
const handleResolutionSelect = (selectedItem: PopoverItem) => {
setResolution(selectedItem.label.toLowerCase() as any);
};
// Image prompt row replaces legacy upload/gallery UI
const handleGenerationCountSelect = (selectedItem: PopoverItem) => {
@@ -211,7 +175,7 @@ export const PromptBox2D = ({
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
@@ -270,19 +234,15 @@ export const PromptBox2D = ({
console.log("Snapshot media token:", snapshotMediaToken.data);
const aspectRatio = getCurrentAspectRatio();
const resolution = getCurrentResolution();
const generateResponse = await EnqueueEditImage({
const generateResponse = await EnqueueContextualEditImage({
model: selectedImageModel,
scene_image_media_token: snapshotMediaToken.data!,
image_media_tokens: referenceImages
.map((image) => image.mediaToken)
.filter((t) => t.length > 0),
image_media_tokens: referenceImages.map((image) => image.mediaToken),
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
image_count: generationCount,
aspect_ratio: aspectRatio,
image_resolution: resolution,
});
console.log("generateResponse", generateResponse);
@@ -296,7 +256,7 @@ export const PromptBox2D = ({
let image = getCanvasRenderBitmap();
if (image === undefined) {
toast.error(
"Error: Unable to generate image. Please check the input and try again.",
"Error: Unable to generate image. Please check the input and try again."
);
return;
}
@@ -330,9 +290,7 @@ export const PromptBox2D = ({
disableSystemPrompt: !useSystemPrompt,
prompt: prompt,
snapshotMediaToken: snapshotMediaToken.data,
additionalImages: referenceImages
.map((image) => image.mediaToken)
.filter((t) => t.length > 0),
additionalImages: referenceImages.map((image) => image.mediaToken),
});
if (response.success === true) {
@@ -367,7 +325,7 @@ export const PromptBox2D = ({
"(4) Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
referenceImages
);
const isDesktop = IsDesktopApp();
@@ -381,7 +339,7 @@ export const PromptBox2D = ({
} catch (error) {
console.error("Error during image generation:", error);
toast.error(
"An error occurred while generating the image. Please try again.",
"An error occurred while generating the image. Please try again."
);
} finally {
await new Promise((resolve) => setTimeout(resolve, 100));
@@ -396,30 +354,16 @@ export const PromptBox2D = ({
return iconElement.props.icon;
};
const getCurrentAspectRatio = (): EnqueueEditImageSize => {
const getCurrentAspectRatio = (): EnqueueContextualEditImageSize => {
const selected = aspectRatioList.find((item) => item.selected);
switch (selected?.label.toLowerCase()) {
case "wide":
return EnqueueEditImageSize.Wide;
case "tall":
return EnqueueEditImageSize.Tall;
case "square":
default:
return EnqueueEditImageSize.Square;
}
};
const getCurrentResolution = (): EnqueueEditImageResolution | undefined => {
const selected = resolutionList.find((item) => item.selected);
switch (selected?.label) {
case "1k":
return EnqueueEditImageResolution.OneK;
case "2k":
return EnqueueEditImageResolution.TwoK;
case "4k":
return EnqueueEditImageResolution.FourK;
case "3:2":
return EnqueueContextualEditImageSize.Wide;
case "2:3":
return EnqueueContextualEditImageSize.Tall;
case "1:1":
default:
return undefined;
return EnqueueContextualEditImageSize.Square;
}
};
@@ -450,7 +394,7 @@ export const PromptBox2D = ({
visible={true}
maxImagePromptCount={Math.max(
1,
selectedImageModel?.maxImagePromptCount ?? 1,
selectedImageModel?.maxImagePromptCount ?? 1
)}
allowUpload={true}
referenceImages={referenceImages}
@@ -462,7 +406,7 @@ export const PromptBox2D = ({
src={image.url}
alt="Reference preview"
className="w-full h-full object-contain"
/>,
/>
);
setIsModalOpen(true);
}}
@@ -473,7 +417,7 @@ export const PromptBox2D = ({
"glass w-[730px] rounded-xl p-4",
selectedImageModel?.canUseImagePrompt &&
isImageRowVisible &&
"rounded-t-none",
"rounded-t-none"
)}
>
<div className="flex justify-center gap-2">
@@ -488,7 +432,7 @@ export const PromptBox2D = ({
variant="action"
className={twMerge(
"h-8 w-8 p-0 bg-transparent hover:bg-transparent group transition-all border-0 shadow-none",
isImageRowVisible && "text-primary",
isImageRowVisible && "text-primary"
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
>
@@ -525,47 +469,26 @@ export const PromptBox2D = ({
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
{selectedImageModel?.canChangeAspectRatio && (
<Tooltip
content="Aspect ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
)}
{selectedImageModel?.canChangeResolution && (
<Tooltip
content="Resolution"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={resolutionList}
onSelect={handleResolutionSelect}
mode="toggle"
panelTitle="Resolution"
showIconsInList
triggerIcon={
<FontAwesomeIcon icon={faExpand} className="h-4 w-4" />
}
/>
</Tooltip>
)}
<Tooltip
content="Aspect ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
<Tooltip
content={
useSystemPrompt
@@ -9,7 +9,6 @@ import {
faSpinnerThird,
faMessageCheck,
faMessageXmark,
faExpand,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangleWide,
@@ -39,9 +38,8 @@ import { Modal } from "@storyteller/ui-modal";
import { Signal } from "@preact/signals-react";
import {
CommandSuccessStatus,
EnqueueEditImage,
EnqueueEditImageSize,
EnqueueEditImageResolution,
EnqueueContextualEditImage,
EnqueueContextualEditImageSize,
} from "@storyteller/tauri-api";
import { usePrompt3DStore } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
@@ -106,46 +104,27 @@ export const PromptBox3D = ({
const setPrompt = usePrompt3DStore((s) => s.setPrompt);
const useSystemPrompt = usePrompt3DStore((s) => s.useSystemPrompt);
const setUseSystemPrompt = usePrompt3DStore((s) => s.setUseSystemPrompt);
const resolution = usePrompt3DStore((s) => s.resolution);
const setResolution = usePrompt3DStore((s) => s.setResolution);
const [isEnqueueing, setIsEnqueueing] = useState(false);
const referenceImages = usePrompt3DStore((s) => s.referenceImages);
const setReferenceImages = usePrompt3DStore((s) => s.setReferenceImages);
const [showImagePrompts, setShowImagePrompts] = useState(false);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "Wide",
label: "3:2",
selected: true,
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "Tall",
label: "2:3",
selected: false,
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "Square",
label: "1:1",
selected: false,
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
{
label: "1K",
selected: resolution === "1k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "2K",
selected: resolution === "2k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "4K",
selected: resolution === "4k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
]);
const [isCameraSettingsOpen, setIsCameraSettingsOpen] = useState(false);
const isImageRowVisible =
showImagePrompts ||
@@ -159,26 +138,19 @@ export const PromptBox3D = ({
}
}, [prompt]);
useEffect(() => {
setResolutionList((prev) =>
prev.map((item) => ({
...item,
selected: item.label.toLowerCase() === resolution,
}))
);
}, [resolution]);
// Update aspect ratio list based on the current cameraAspectRatio signal
useEffect(() => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected:
(item.label === "Wide" &&
(item.label === "16:9" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) ||
(item.label === "3:2" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) ||
(item.label === "Tall" &&
(item.label === "2:3" &&
cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) ||
(item.label === "Square" &&
(item.label === "1:1" &&
cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1),
}))
);
@@ -195,27 +167,26 @@ export const PromptBox3D = ({
// Map the selected label to the corresponding CameraAspectRatio enum value
let newRatio: CameraAspectRatio;
switch (selectedItem.label.toLowerCase()) {
case "wide":
switch (selectedItem.label) {
case "16:9":
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
break;
case "3:2":
newRatio = CameraAspectRatio.HORIZONTAL_3_2;
break;
case "tall":
case "2:3":
newRatio = CameraAspectRatio.VERTICAL_2_3;
break;
case "square":
case "1:1":
newRatio = CameraAspectRatio.SQUARE_1_1;
break;
default:
newRatio = CameraAspectRatio.HORIZONTAL_3_2;
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
}
onAspectRatioSelect(newRatio);
};
const handleResolutionSelect = (selectedItem: PopoverItem) => {
setResolution(selectedItem.label.toLowerCase() as any);
};
// ImagePromptRow will handle uploads and gallery internally
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
@@ -364,9 +335,8 @@ export const PromptBox3D = ({
console.log("snapshotResult", snapshotResult);
const aspectRatio = getCurrentAspectRatio();
const resolution = getCurrentResolution();
const generateResponse = await EnqueueEditImage({
const generateResponse = await EnqueueContextualEditImage({
model: selectedImageModel,
scene_image_media_token: snapshotResult.data!,
image_media_tokens: referenceImages.map((image) => image.mediaToken),
@@ -374,7 +344,6 @@ export const PromptBox3D = ({
prompt: prompt,
image_count: 1,
aspect_ratio: aspectRatio,
image_resolution: resolution,
});
console.log("generateResponse", generateResponse);
@@ -431,30 +400,16 @@ export const PromptBox3D = ({
}
};
const getCurrentAspectRatio = (): EnqueueEditImageSize => {
const getCurrentAspectRatio = (): EnqueueContextualEditImageSize => {
switch (cameraAspectRatio.value) {
case CameraAspectRatio.HORIZONTAL_3_2:
return EnqueueEditImageSize.Wide;
return EnqueueContextualEditImageSize.Wide;
case CameraAspectRatio.VERTICAL_2_3:
case CameraAspectRatio.VERTICAL_9_16:
return EnqueueEditImageSize.Tall;
return EnqueueContextualEditImageSize.Tall;
case CameraAspectRatio.SQUARE_1_1:
default:
return EnqueueEditImageSize.Square;
}
};
const getCurrentResolution = (): EnqueueEditImageResolution | undefined => {
const selected = resolutionList.find((item) => item.selected);
switch (selected?.label) {
case "1k":
return EnqueueEditImageResolution.OneK;
case "2k":
return EnqueueEditImageResolution.TwoK;
case "4k":
return EnqueueEditImageResolution.FourK;
default:
return undefined;
return EnqueueContextualEditImageSize.Square;
}
};
@@ -584,47 +539,26 @@ export const PromptBox3D = ({
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
{selectedImageModel?.canChangeAspectRatio && (
<Tooltip
content="Aspect ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
)}
{selectedImageModel?.canChangeResolution && (
<Tooltip
content="Resolution"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={resolutionList}
onSelect={handleResolutionSelect}
mode="toggle"
panelTitle="Resolution"
showIconsInList
triggerIcon={
<FontAwesomeIcon icon={faExpand} className="h-4 w-4" />
}
/>
</Tooltip>
)}
<Tooltip
content="Aspect ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
<Tooltip
content="Camera"
position="top"
@@ -1,10 +1,6 @@
import {
faRectangle,
faRectangleVertical,
faSquare,
} from "@fortawesome/pro-regular-svg-icons";
import {
faEdit,
// faExpand,
faMessageCheck,
faMessageXmark,
faMousePointer,
@@ -16,8 +12,6 @@ import {
faEraser,
faUndo,
faRedo,
faExpand,
faArrowsUpDownLeftRight,
} from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { Button, ToggleButton } from "@storyteller/ui-button";
@@ -28,19 +22,12 @@ import { useEffect, useRef, useState } from "react";
import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list";
import { twMerge } from "tailwind-merge";
import { ImagePromptRow, type UploadImageFn } from "./ImagePromptRow";
import { RefImage, usePromptEditStore } from "./promptStore";
import { usePromptEditStore } from "./promptStore";
export interface PromptBoxEditProps {
onModeChange?: (mode: string) => void;
selectedMode?: string;
onGenerateClick: (
prompt: string,
options?: {
aspectRatio?: string;
resolution?: string;
images?: RefImage[];
},
) => void | Promise<void>;
onGenerateClick: (prompt: string) => void | Promise<void>;
isDisabled?: boolean;
onFitPressed?: () => void | Promise<void>;
selectedImageModel?: ImageModel;
@@ -74,102 +61,17 @@ export const PromptBoxEdit = ({
const [useSystemPrompt, setUseSystemPrompt] = useState(true);
const [isFocused, setIsFocused] = useState(false);
const [generationCount, setGenerationCount] = useState<number>(
typeof generationCountProp === "number" ? generationCountProp : 1,
typeof generationCountProp === "number" ? generationCountProp : 1
);
const [internalEnqueueing, setInternalEnqueueing] = useState(false);
const referenceImages = usePromptEditStore((s) => s.referenceImages);
const setReferenceImages = usePromptEditStore((s) => s.setReferenceImages);
const aspectRatio = usePromptEditStore((s) => s.aspectRatio);
const setAspectRatio = usePromptEditStore((s) => s.setAspectRatio);
const resolution = usePromptEditStore((s) => s.resolution);
const setResolution = usePromptEditStore((s) => s.setResolution);
const [showImagePrompts, setShowImagePrompts] = useState(false);
const [generationCountList, setGenerationCountList] = useState<PopoverItem[]>(
[],
[]
);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "Auto",
selected: aspectRatio === "auto",
icon: (
<FontAwesomeIcon icon={faArrowsUpDownLeftRight} className="h-4 w-4" />
),
},
{
label: "Wide",
selected: aspectRatio === "wide",
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "Tall",
selected: aspectRatio === "tall",
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "Square",
selected: aspectRatio === "square",
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
{
label: "1K",
selected: resolution === "1k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "2K",
selected: resolution === "2k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "4K",
selected: resolution === "4k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
]);
useEffect(() => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected:
item.label.toLowerCase() === aspectRatio ||
(item.label === "Auto" && aspectRatio === "auto"),
})),
);
}, [aspectRatio]);
useEffect(() => {
setResolutionList((prev) =>
prev.map((item) => ({
...item,
selected: item.label.toLowerCase() === resolution,
})),
);
}, [resolution]);
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
const value =
selectedItem.label === "Auto" ? "auto" : selectedItem.label.toLowerCase();
setAspectRatio(value as any);
};
const handleResolutionSelect = (selectedItem: PopoverItem) => {
setResolution(selectedItem.label.toLowerCase() as any);
};
const getCurrentAspectRatioIcon = () => {
const selected = aspectRatioList.find((item) => item.selected);
if (!selected || !selected.icon) return faRectangle;
const iconElement = selected.icon as React.ReactElement<{ icon: any }>;
return iconElement.props.icon;
};
const textareaRef = useRef<HTMLTextAreaElement>(null);
const isImageRowVisible =
(selectedImageModel?.canUseImagePrompt &&
@@ -183,17 +85,20 @@ export const PromptBoxEdit = ({
}
});
// Sync internal state if a controlled prop is provided
useEffect(() => {
if (typeof generationCountProp === "number") {
setGenerationCount(generationCountProp);
}
}, [generationCountProp]);
// Build generation count options based on selected model
useEffect(() => {
const caps = getCapabilitiesForModel(selectedImageModel);
console.log(">>> caps", caps);
const items: PopoverItem[] = Array.from(
{ length: caps.maxGenerationCount },
(_, i) => i + 1,
(_, i) => i + 1
).map((count) => ({
label: String(count),
selected: count === generationCount,
@@ -201,10 +106,11 @@ export const PromptBoxEdit = ({
}));
setGenerationCountList(items);
// Clamp selection
if (generationCount < 1 || generationCount > caps.maxGenerationCount) {
const clamped = Math.min(
Math.max(1, generationCount),
caps.maxGenerationCount,
caps.maxGenerationCount
);
setGenerationCount(clamped);
onGenerationCountChange?.(clamped);
@@ -212,12 +118,13 @@ export const PromptBoxEdit = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedImageModel]);
// Keep UI selection in sync when store/value changes
useEffect(() => {
setGenerationCountList((prev) =>
prev.map((item) => ({
...item,
selected: item.label === String(generationCount),
})),
}))
);
}, [generationCount]);
@@ -230,6 +137,7 @@ export const PromptBoxEdit = ({
const next =
value.slice(0, selectionStart) + pastedText + value.slice(selectionEnd);
setPrompt(next);
// Restore caret after React updates the value
requestAnimationFrame(() => {
const pos = selectionStart + pastedText.length;
textareaRef.current?.setSelectionRange(pos, pos);
@@ -242,10 +150,12 @@ export const PromptBoxEdit = ({
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
// Stop propagation of keyboard events to prevent them from reaching the canvas
e.stopPropagation();
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
// Block Enter if requirements are not met
const busy = Boolean(isEnqueueing ?? internalEnqueueing);
if (!prompt.trim() || isDisabled || busy) return;
void handleGenerate();
@@ -260,7 +170,7 @@ export const PromptBoxEdit = ({
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
@@ -272,13 +182,7 @@ export const PromptBoxEdit = ({
setInternalEnqueueing(false);
}, 10000);
try {
await Promise.resolve(
onGenerateClick(prompt, {
aspectRatio,
resolution,
images: referenceImages,
}),
);
await Promise.resolve(onGenerateClick(prompt));
} finally {
clearTimeout(timeout);
setInternalEnqueueing(false);
@@ -299,43 +203,50 @@ export const PromptBoxEdit = ({
},
]
: supportsMaskedInpainting
? [
{
value: "edit",
icon: faEdit,
text: "Edit Region",
},
{
value: "eraser",
icon: faEraser,
text: "Eraser",
},
]
: [
{
value: "edit",
icon: faEdit,
text: "Edit Region",
},
{
value: "select",
icon: faMousePointer,
text: "Select",
},
];
? [
{
value: "edit",
icon: faEdit,
text: "Edit Region",
},
{
value: "eraser",
icon: faEraser,
text: "Eraser",
},
]
: [
{
value: "edit",
icon: faEdit,
text: "Edit Region",
},
{
value: "select",
icon: faMousePointer,
text: "Select",
},
];
return (
<>
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
{(supportsMaskedInpainting || isNanoBananaModel) && (
<div
className={twMerge(
"glass w-fit mx-auto rounded-xl px-2 py-2 flex items-center gap-3",
selectedImageModel?.canUseImagePrompt &&
isImageRowVisible &&
"mb-[72px]",
{selectedImageModel?.canUseImagePrompt && isImageRowVisible && (
<ImagePromptRow
visible={true}
maxImagePromptCount={Math.max(
1,
selectedImageModel?.maxImagePromptCount ?? 1
)}
>
allowUpload={true}
referenceImages={referenceImages}
setReferenceImages={setReferenceImages}
uploadImage={uploadImage}
onImageClick={() => {}}
/>
)}
{(supportsMaskedInpainting || isNanoBananaModel) && (
<div className="glass w-fit mx-auto rounded-xl px-2 py-2 flex items-center gap-3">
<ButtonIconSelect
options={modes}
onOptionChange={onModeSelectionChange}
@@ -364,199 +275,140 @@ export const PromptBoxEdit = ({
</div>
</div>
)}
<div className="relative z-20 flex flex-col">
{selectedImageModel?.canUseImagePrompt && isImageRowVisible && (
<ImagePromptRow
visible={true}
maxImagePromptCount={Math.max(
1,
selectedImageModel?.maxImagePromptCount ?? 1,
)}
allowUpload={true}
referenceImages={referenceImages}
setReferenceImages={setReferenceImages}
uploadImage={uploadImage}
onImageClick={() => {}}
className=""
/>
<div
className={twMerge(
"glass w-[730px] rounded-xl p-4",
isFocused && "ring-1 ring-primary border-primary",
selectedImageModel?.canUseImagePrompt &&
isImageRowVisible &&
"rounded-t-none"
)}
<div
className={twMerge(
"glass w-[730px] rounded-xl p-4",
isFocused && "ring-1 ring-primary border-primary",
selectedImageModel?.canUseImagePrompt &&
isImageRowVisible &&
"rounded-t-none",
)}
>
<div className="flex justify-center gap-2">
{selectedImageModel?.canUseImagePrompt && (
<Tooltip
content="Add Image"
position="top"
closeOnClick={true}
className={twMerge(isImageRowVisible && "hidden opacity-0")}
>
<div className="flex justify-center gap-2">
{selectedImageModel?.canUseImagePrompt && (
<Tooltip
content="Add Image"
position="top"
closeOnClick={true}
className={twMerge(isImageRowVisible && "hidden opacity-0")}
>
<Button
variant="action"
className={twMerge(
"h-8 w-8 p-0 bg-transparent hover:bg-transparent group transition-all border-0 shadow-none",
isImageRowVisible && "text-primary"
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
>
<Button
variant="action"
className={twMerge(
"h-8 w-8 p-0 bg-transparent hover:bg-transparent group transition-all border-0 shadow-none",
isImageRowVisible && "text-primary",
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
<svg
width="24"
height="20"
viewBox="0 0 24 20"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className="group-hover:opacity-100 opacity-80 transition-all"
>
<svg
width="24"
height="20"
viewBox="0 0 24 20"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className="group-hover:opacity-100 opacity-80 transition-all"
>
<path
opacity="1"
d="M2.66667 2H16C16.3667 2 16.6667 2.3 16.6667 2.66667V6.1125C17.1 6.04167 17.5458 6 18 6C18.225 6 18.4458 6.00833 18.6667 6.02917V2.66667C18.6667 1.19583 17.4708 0 16 0H2.66667C1.19583 0 0 1.19583 0 2.66667V16C0 17.4708 1.19583 18.6667 2.66667 18.6667H11.5C11.0625 18.0583 10.7083 17.3875 10.4542 16.6667H2.66667C2.3 16.6667 2 16.3667 2 16V2.66667C2 2.3 2.3 2 2.66667 2ZM11.8625 7.49167C11.6833 7.1875 11.3542 7 11 7C10.6458 7 10.3167 7.1875 10.1375 7.49167L8.2 10.7833L7.48333 9.75833C7.29583 9.49167 6.99167 9.33333 6.6625 9.33333C6.33333 9.33333 6.02917 9.49167 5.84167 9.75833L3.50833 13.0917C3.29583 13.3958 3.26667 13.7958 3.44167 14.125C3.61667 14.4542 3.9625 14.6667 4.33333 14.6667H10.0292C10.0125 14.4458 10 14.225 10 14C10 11.7833 10.9 9.77917 12.3542 8.33333L11.8625 7.49583V7.49167ZM5.33333 6.66667C6.07083 6.66667 6.66667 6.07083 6.66667 5.33333C6.66667 4.59583 6.07083 4 5.33333 4C4.59583 4 4 4.59583 4 5.33333C4 6.07083 4.59583 6.66667 5.33333 6.66667ZM18 20C21.3125 20 24 17.3125 24 14C24 10.6875 21.3125 8 18 8C14.6875 8 12 10.6875 12 14C12 17.3125 14.6875 20 18 20ZM18.6667 11.3333V13.3333H20.6667C21.0333 13.3333 21.3333 13.6333 21.3333 14C21.3333 14.3667 21.0333 14.6667 20.6667 14.6667H18.6667V16.6667C18.6667 17.0333 18.3667 17.3333 18 17.3333C17.6333 17.3333 17.3333 17.0333 17.3333 16.6667V14.6667H15.3333C14.9667 14.6667 14.6667 14.3667 14.6667 14C14.6667 13.6333 14.9667 13.3333 15.3333 13.3333H17.3333V11.3333C17.3333 10.9667 17.6333 10.6667 18 10.6667C18.3667 10.6667 18.6667 10.9667 18.6667 11.3333Z"
fill="currentColor"
/>
</svg>
</Button>
</Tooltip>
)}
<textarea
ref={textareaRef}
rows={1}
placeholder="Write what you want to change in your image and click generate..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
/>
<path
opacity="1"
d="M2.66667 2H16C16.3667 2 16.6667 2.3 16.6667 2.66667V6.1125C17.1 6.04167 17.5458 6 18 6C18.225 6 18.4458 6.00833 18.6667 6.02917V2.66667C18.6667 1.19583 17.4708 0 16 0H2.66667C1.19583 0 0 1.19583 0 2.66667V16C0 17.4708 1.19583 18.6667 2.66667 18.6667H11.5C11.0625 18.0583 10.7083 17.3875 10.4542 16.6667H2.66667C2.3 16.6667 2 16.3667 2 16V2.66667C2 2.3 2.3 2 2.66667 2ZM11.8625 7.49167C11.6833 7.1875 11.3542 7 11 7C10.6458 7 10.3167 7.1875 10.1375 7.49167L8.2 10.7833L7.48333 9.75833C7.29583 9.49167 6.99167 9.33333 6.6625 9.33333C6.33333 9.33333 6.02917 9.49167 5.84167 9.75833L3.50833 13.0917C3.29583 13.3958 3.26667 13.7958 3.44167 14.125C3.61667 14.4542 3.9625 14.6667 4.33333 14.6667H10.0292C10.0125 14.4458 10 14.225 10 14C10 11.7833 10.9 9.77917 12.3542 8.33333L11.8625 7.49583V7.49167ZM5.33333 6.66667C6.07083 6.66667 6.66667 6.07083 6.66667 5.33333C6.66667 4.59583 6.07083 4 5.33333 4C4.59583 4 4 4.59583 4 5.33333C4 6.07083 4.59583 6.66667 5.33333 6.66667ZM18 20C21.3125 20 24 17.3125 24 14C24 10.6875 21.3125 8 18 8C14.6875 8 12 10.6875 12 14C12 17.3125 14.6875 20 18 20ZM18.6667 11.3333V13.3333H20.6667C21.0333 13.3333 21.3333 13.6333 21.3333 14C21.3333 14.3667 21.0333 14.6667 20.6667 14.6667H18.6667V16.6667C18.6667 17.0333 18.3667 17.3333 18 17.3333C17.6333 17.3333 17.3333 17.0333 17.3333 16.6667V14.6667H15.3333C14.9667 14.6667 14.6667 14.3667 14.6667 14C14.6667 13.6333 14.9667 13.3333 15.3333 13.3333H17.3333V11.3333C17.3333 10.9667 17.6333 10.6667 18 10.6667C18.3667 10.6667 18.6667 10.9667 18.6667 11.3333Z"
fill="currentColor"
/>
</svg>
</Button>
</Tooltip>
)}
<textarea
ref={textareaRef}
rows={1}
placeholder="Write what you want to change in your image and click generate..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Tooltip
content={
useSystemPrompt
? "Use system prompt: ON"
: "Use system prompt: OFF"
}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={useSystemPrompt}
icon={faMessageXmark}
activeIcon={faMessageCheck}
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
/>
</Tooltip>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
{selectedImageModel?.canChangeAspectRatio && (
<Tooltip
content="Aspect Ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
)}
{selectedImageModel?.canChangeResolution && (
<Tooltip
content="Resolution"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={resolutionList}
onSelect={handleResolutionSelect}
mode="toggle"
panelTitle="Resolution"
showIconsInList
triggerIcon={
<FontAwesomeIcon icon={faExpand} className="h-4 w-4" />
}
/>
</Tooltip>
)}
<div className="flex items-center gap-2">
{onFitPressed && (
<Tooltip
content={
useSystemPrompt
? "Use system prompt: ON"
: "Use system prompt: OFF"
}
content={"Fit canvas to screen"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={useSystemPrompt}
icon={faMessageXmark}
activeIcon={faMessageCheck}
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
/>
</Tooltip>
</div>
<div className="flex items-center gap-2">
{onFitPressed && (
<Tooltip
content={"Fit canvas to screen"}
position="top"
className="z-50"
delay={200}
<Button
variant="secondary"
className="h-9 bg-ui-controls/60 px-3 text-base-fg hover:bg-ui-controls/90"
onClick={onFitPressed}
>
<Button
variant="secondary"
className="h-9 bg-ui-controls/60 px-3 text-base-fg hover:bg-ui-controls/90"
onClick={onFitPressed}
>
<FontAwesomeIcon icon={faFrame} className="h-4 w-4" />
Fit
</Button>
</Tooltip>
)}
<Tooltip
content="Number of generations"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={generationCountList}
onSelect={handleGenerationCountSelect}
mode="toggle"
panelTitle="No. of images"
triggerIcon={
<FontAwesomeIcon icon={faCopy} className="h-4 w-4" />
}
panelClassName="min-w-28"
buttonClassName="h-9"
/>
<FontAwesomeIcon icon={faFrame} className="h-4 w-4" />
Fit
</Button>
</Tooltip>
<Button
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={
!(isEnqueueing ?? internalEnqueueing) && !isDisabled
? faSparkles
: undefined
)}
<Tooltip
content="Number of generations"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={generationCountList}
onSelect={handleGenerationCountSelect}
mode="toggle"
triggerIcon={
<FontAwesomeIcon icon={faCopy} className="h-4 w-4" />
}
onClick={handleGenerate}
disabled={
(isEnqueueing ?? internalEnqueueing) ||
isDisabled ||
!prompt.trim()
}
>
{(isEnqueueing ?? internalEnqueueing) ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
</div>
panelClassName="min-w-28"
panelTitle="No. of images"
buttonClassName="h-9"
/>
</Tooltip>
<Button
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={
!(isEnqueueing ?? internalEnqueueing) && !isDisabled
? faSparkles
: undefined
}
onClick={handleGenerate}
disabled={
(isEnqueueing ?? internalEnqueueing) ||
isDisabled ||
!prompt.trim()
}
>
{isEnqueueing ?? internalEnqueueing ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
</div>
</div>
</div>
@@ -10,7 +10,6 @@ import {
EnqueueTextToImage,
EnqueueTextToImageRequest,
EnqueueTextToImageSize,
EnqueueTextToImageResolution,
} from "@storyteller/tauri-api";
import {
faMessageXmark,
@@ -18,7 +17,6 @@ import {
faSparkles,
faSpinnerThird,
faCopy,
faExpand,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangle,
@@ -47,7 +45,7 @@ interface PromptBoxImageProps {
onEnqueuePressed?: (
prompt: string,
count: number,
subscriberId: string,
subscriberId: string
) => void | Promise<void>;
selectedModel?: ImageModel;
imageMediaId?: string;
@@ -89,8 +87,6 @@ export const PromptBoxImage = ({
const setUseSystemPrompt = usePromptImageStore((s) => s.setUseSystemPrompt);
const aspectRatio = usePromptImageStore((s) => s.aspectRatio);
const setAspectRatio = usePromptImageStore((s) => s.setAspectRatio);
const resolution = usePromptImageStore((s) => s.resolution);
const setResolution = usePromptImageStore((s) => s.setResolution);
const generationCount = usePromptImageStore((s) => s.generationCount);
const setGenerationCount = usePromptImageStore((s) => s.setGenerationCount);
const [isEnqueueing, setIsEnqueueing] = useState(false);
@@ -111,40 +107,23 @@ export const PromptBoxImage = ({
}, [isImageRowVisible, onImageRowVisibilityChange]);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "Wide",
selected: aspectRatio === "wide",
label: "3:2",
selected: aspectRatio === "3:2",
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "Tall",
selected: aspectRatio === "tall",
label: "2:3",
selected: aspectRatio === "2:3",
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "Square",
selected: aspectRatio === "square",
label: "1:1",
selected: aspectRatio === "1:1",
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
{
label: "1K",
selected: resolution === "1k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "2K",
selected: resolution === "2k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
{
label: "4K",
selected: resolution === "4k",
icon: <FontAwesomeIcon icon={faExpand} className="h-4 w-4" />,
},
]);
const [generationCountList, setGenerationCountList] = useState<PopoverItem[]>(
[],
[]
);
const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -172,14 +151,14 @@ export const PromptBoxImage = ({
const caps = getCapabilitiesForModel(selectedModel);
const defaultCount = Math.min(
Math.max(1, caps.defaultGenerationCount ?? 1),
caps.maxGenerationCount,
caps.maxGenerationCount
);
setGenerationCount(defaultCount);
const items: PopoverItem[] = Array.from(
{ length: caps.maxGenerationCount },
(_, i) => i + 1,
(_, i) => i + 1
).map((count) => ({
label: String(count),
selected: count === defaultCount,
@@ -193,7 +172,7 @@ export const PromptBoxImage = ({
prev.map((item) => ({
...item,
selected: item.label === String(generationCount),
})),
}))
);
}, [generationCount]);
@@ -201,34 +180,21 @@ export const PromptBoxImage = ({
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected: item.label.toLowerCase() === aspectRatio,
})),
selected: item.label === aspectRatio,
}))
);
}, [aspectRatio]);
useEffect(() => {
setResolutionList((prev) =>
prev.map((item) => ({
...item,
selected: item.label.toLowerCase() === resolution,
})),
);
}, [resolution]);
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatio(selectedItem.label.toLowerCase() as any);
setAspectRatio(selectedItem.label as any);
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
const handleResolutionSelect = (selectedItem: PopoverItem) => {
setResolution(selectedItem.label.toLowerCase() as any);
};
const handleGenerationCountSelect = (selectedItem: PopoverItem) => {
const count = parseInt(selectedItem.label, 10);
setGenerationCount(count);
@@ -236,7 +202,7 @@ export const PromptBoxImage = ({
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
@@ -269,8 +235,8 @@ export const PromptBoxImage = ({
return;
}
console.debug("Selected model:", selectedModel);
console.debug("Prompt:", prompt);
console.debug("Selected model:", selectedModel)
console.debug("Prompt:", prompt)
setIsEnqueueing(true);
@@ -287,13 +253,11 @@ export const PromptBoxImage = ({
try {
const aspectRatio = getCurrentAspectRatio();
const resolution = getCurrentResolution();
const request: EnqueueTextToImageRequest = {
prompt: prompt,
model: selectedModel,
aspect_ratio: aspectRatio,
image_resolution: resolution,
number_images: generationCount,
frontend_caller: "text_to_image",
frontend_subscriber_id: subscriberId,
@@ -304,9 +268,9 @@ export const PromptBoxImage = ({
!!referenceImages &&
referenceImages.length > 0
) {
request.image_media_tokens = referenceImages
.map((image) => image.mediaToken)
.filter((t) => t.length > 0);
request.image_media_tokens = referenceImages.map(
(image) => image.mediaToken
);
}
const generateResponse = await EnqueueTextToImage(request);
@@ -330,31 +294,17 @@ export const PromptBoxImage = ({
const getCurrentAspectRatio = (): EnqueueTextToImageSize => {
const selected = aspectRatioList.find((item) => item.selected);
switch (selected?.label.toLowerCase()) {
case "wide":
switch (selected?.label) {
case "3:2":
return EnqueueTextToImageSize.Wide;
case "tall":
case "2:3":
return EnqueueTextToImageSize.Tall;
case "square":
case "1:1":
default:
return EnqueueTextToImageSize.Square;
}
};
const getCurrentResolution = (): EnqueueTextToImageResolution | undefined => {
const selected = resolutionList.find((item) => item.selected);
switch (selected?.label) {
case "1k":
return EnqueueTextToImageResolution.OneK;
case "2k":
return EnqueueTextToImageResolution.TwoK;
case "4k":
return EnqueueTextToImageResolution.FourK;
default:
return undefined;
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
@@ -381,7 +331,7 @@ export const PromptBoxImage = ({
visible={true}
maxImagePromptCount={Math.max(
1,
selectedModel?.maxImagePromptCount ?? 1,
selectedModel?.maxImagePromptCount ?? 1
)}
allowUpload={true}
referenceImages={referenceImages}
@@ -395,7 +345,7 @@ export const PromptBoxImage = ({
src={image.url}
alt="Reference preview"
className="w-full h-full object-contain"
/>,
/>
);
setIsModalOpen(true);
}}
@@ -410,7 +360,7 @@ export const PromptBoxImage = ({
"rounded-t-none",
isFocused
? "ring-1 ring-primary border-primary"
: "ring-1 ring-transparent",
: "ring-1 ring-transparent"
)}
>
<div className="flex justify-center gap-2">
@@ -425,7 +375,7 @@ export const PromptBoxImage = ({
variant="action"
className={twMerge(
"h-8 w-8 p-0 bg-transparent hover:bg-transparent group transition-all border-0 shadow-none",
isImageRowVisible && "text-primary",
isImageRowVisible && "text-primary"
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
>
@@ -462,47 +412,26 @@ export const PromptBoxImage = ({
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
{selectedModel?.canChangeAspectRatio && (
<Tooltip
content="Aspect Ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentResolutionIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
)}
{selectedModel?.canChangeResolution && (
<Tooltip
content="Resolution"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={resolutionList}
onSelect={handleResolutionSelect}
mode="toggle"
panelTitle="Resolution"
showIconsInList
triggerIcon={
<FontAwesomeIcon icon={faExpand} className="h-4 w-4" />
}
/>
</Tooltip>
)}
<Tooltip
content="Aspect Ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentResolutionIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
<Tooltip
content={
useSystemPrompt
@@ -15,13 +15,11 @@ import {
faMessageCheck,
faSparkles,
faSpinnerThird,
faSquare,
faPortrait,
faWaveformLines,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangle,
faSquare,
faRectangleVertical,
} from "@fortawesome/pro-regular-svg-icons";
import { faRectangle } from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IconProp } from "@fortawesome/fontawesome-svg-core";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
@@ -56,7 +54,7 @@ interface PromptBoxVideoProps {
useJobContext: () => JobContextType;
onEnqueuePressed?: (
prompt: string,
subscriberId: string,
subscriberId: string
) => void | Promise<void>;
selectedModel?: VideoModel;
imageMediaId?: string;
@@ -97,14 +95,14 @@ export const PromptBoxVideo = ({
const setUseSystemPrompt = usePromptVideoStore((s) => s.setUseSystemPrompt);
const generateWithSound = usePromptVideoStore((s) => s.generateWithSound);
const setGenerateWithSound = usePromptVideoStore(
(s) => s.setGenerateWithSound,
(s) => s.setGenerateWithSound
);
const resolution = usePromptVideoStore((s) => s.resolution);
const setResolution = usePromptVideoStore((s) => s.setResolution);
const [isEnqueueing, setIsEnqueueing] = useState(false);
const [isFocused, setIsFocused] = useState(false);
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[],
[]
);
const referenceImages = usePromptVideoStore((s) => s.referenceImages);
const setReferenceImages = usePromptVideoStore((s) => s.setReferenceImages);
@@ -126,7 +124,7 @@ export const PromptBoxVideo = ({
// When switching to a new model, the existing resolution might not be correct.
// This is a gross and nasty hack to handle the case where the resolution is not found.
const resolutionExists = selectedModel.sizeOptions.some(
(option) => option.textLabel === resolution,
(option) => option.textLabel === resolution
);
const useFirstOption = !resolutionExists;
@@ -137,7 +135,7 @@ export const PromptBoxVideo = ({
faIcon = faRectangle;
break;
case SizeIconOption.Portrait:
faIcon = faRectangleVertical;
faIcon = faPortrait;
break;
case SizeIconOption.Square:
faIcon = faSquare;
@@ -153,7 +151,7 @@ export const PromptBoxVideo = ({
});
} else {
const resolutionExists = DEFAULT_RESOLUTIONS.some(
(option) => option.textLabel === resolution,
(option) => option.textLabel === resolution
);
const useFirstOption = !resolutionExists;
@@ -164,7 +162,7 @@ export const PromptBoxVideo = ({
faIcon = faRectangle;
break;
case SizeIconOption.Portrait:
faIcon = faRectangleVertical;
faIcon = faPortrait;
break;
case SizeIconOption.Square:
faIcon = faSquare;
@@ -215,7 +213,7 @@ export const PromptBoxVideo = ({
resolutionOptions.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
}))
);
};
@@ -331,11 +329,12 @@ export const PromptBoxVideo = ({
};
const getGrokAspectRatio = (): GROK_ASPECT_RATIO => {
// NB: This function was just written to give us better type safety.
// NB: This function was just written to give us better type safety.
// There has to be a cleaner appraoach.
const maybeAspectRatio = selectedModel?.sizeOptions?.find(
(option) => option.textLabel === resolution,
)?.tauriValue;
const maybeAspectRatio = selectedModel?.
sizeOptions?.
find((option) => option.textLabel === resolution)?.
tauriValue;
switch (maybeAspectRatio) {
case "landscape":
@@ -347,7 +346,7 @@ export const PromptBoxVideo = ({
default:
return "landscape";
}
};
}
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
@@ -387,7 +386,7 @@ export const PromptBoxVideo = ({
src={image.url}
alt="Reference preview"
className="w-full h-full object-contain"
/>,
/>
);
setIsModalOpen(true);
}}
@@ -404,7 +403,7 @@ export const PromptBoxVideo = ({
isImageRowVisible && "rounded-t-none",
isFocused
? "ring-1 ring-primary border-primary"
: "ring-1 ring-transparent",
: "ring-1 ring-transparent"
)}
>
<div className="flex justify-center gap-2">
@@ -8,19 +8,16 @@ export interface RefImage {
}
// ----- 2D Prompt Box Store -----
type AspectRatio = "wide" | "tall" | "square";
type Resolution = "1k" | "2k" | "4k";
type AspectRatio = "3:2" | "2:3" | "1:1";
interface Prompt2DStore {
prompt: string;
aspectRatio: AspectRatio;
resolution: Resolution;
useSystemPrompt: boolean;
referenceImages: RefImage[];
generationCount: number;
setPrompt: (prompt: string) => void;
setAspectRatio: (ratio: AspectRatio) => void;
setResolution: (resolution: Resolution) => void;
setUseSystemPrompt: (value: boolean) => void;
setReferenceImages: (images: RefImage[]) => void;
setGenerationCount: (count: number) => void;
@@ -28,14 +25,12 @@ interface Prompt2DStore {
export const usePrompt2DStore = create<Prompt2DStore>()((set) => ({
prompt: "",
aspectRatio: "wide",
resolution: "1k",
aspectRatio: "3:2",
useSystemPrompt: true,
referenceImages: [],
generationCount: 1,
setPrompt: (prompt) => set({ prompt }),
setAspectRatio: (aspectRatio) => set({ aspectRatio }),
setResolution: (resolution) => set({ resolution }),
setUseSystemPrompt: (useSystemPrompt) => set({ useSystemPrompt }),
setReferenceImages: (referenceImages) => set({ referenceImages }),
setGenerationCount: (generationCount) => set({ generationCount }),
@@ -46,22 +41,18 @@ export { usePrompt2DStore as usePromptStore };
// ----- 3D Prompt Box Store -----
interface Prompt3DStore {
prompt: string;
resolution: Resolution;
useSystemPrompt: boolean;
referenceImages: RefImage[];
setPrompt: (prompt: string) => void;
setResolution: (resolution: Resolution) => void;
setUseSystemPrompt: (value: boolean) => void;
setReferenceImages: (images: RefImage[]) => void;
}
export const usePrompt3DStore = create<Prompt3DStore>()((set) => ({
prompt: "",
resolution: "1k",
useSystemPrompt: true,
referenceImages: [],
setPrompt: (prompt) => set({ prompt }),
setResolution: (resolution) => set({ resolution }),
setUseSystemPrompt: (useSystemPrompt) => set({ useSystemPrompt }),
setReferenceImages: (referenceImages) => set({ referenceImages }),
}));
@@ -70,13 +61,11 @@ export const usePrompt3DStore = create<Prompt3DStore>()((set) => ({
interface PromptImageStore {
prompt: string;
aspectRatio: AspectRatio;
resolution: Resolution;
useSystemPrompt: boolean;
referenceImages: RefImage[];
generationCount: number;
setPrompt: (prompt: string) => void;
setAspectRatio: (ratio: AspectRatio) => void;
setResolution: (resolution: Resolution) => void;
setUseSystemPrompt: (value: boolean) => void;
setReferenceImages: (images: RefImage[]) => void;
setGenerationCount: (count: number) => void;
@@ -84,14 +73,12 @@ interface PromptImageStore {
export const usePromptImageStore = create<PromptImageStore>()((set) => ({
prompt: "",
aspectRatio: "wide",
resolution: "1k",
aspectRatio: "3:2",
useSystemPrompt: true,
referenceImages: [],
generationCount: 1,
setPrompt: (prompt) => set({ prompt }),
setAspectRatio: (aspectRatio) => set({ aspectRatio }),
setResolution: (resolution) => set({ resolution }),
setUseSystemPrompt: (useSystemPrompt) => set({ useSystemPrompt }),
setReferenceImages: (referenceImages) => set({ referenceImages }),
setGenerationCount: (generationCount) => set({ generationCount }),
@@ -131,22 +118,12 @@ export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
}));
// ----- Edit Prompt Box Store -----
type EditAspectRatio = "auto" | "wide" | "tall" | "square";
interface PromptEditStore {
referenceImages: RefImage[];
aspectRatio: EditAspectRatio;
resolution: Resolution;
setReferenceImages: (images: RefImage[]) => void;
setAspectRatio: (ratio: EditAspectRatio) => void;
setResolution: (resolution: Resolution) => void;
}
export const usePromptEditStore = create<PromptEditStore>()((set) => ({
referenceImages: [],
aspectRatio: "auto",
resolution: "1k",
setReferenceImages: (referenceImages) => set({ referenceImages }),
setAspectRatio: (aspectRatio) => set({ aspectRatio }),
setResolution: (resolution) => set({ resolution }),
}));
@@ -1,19 +1,13 @@
/// <reference types='vitest' />
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
import dts from "vite-plugin-dts";
import * as path from "path";
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
import dts from 'vite-plugin-dts';
import * as path from 'path';
export default defineConfig(() => ({
root: __dirname,
cacheDir: "../../../node_modules/.vite/libs/components/settings-modal",
plugins: [
react(),
dts({
entryRoot: "src",
tsconfigPath: path.join(__dirname, "tsconfig.lib.json"),
}),
],
cacheDir: '../../../node_modules/.vite/libs/components/settings-modal',
plugins: [react(), dts({ entryRoot: 'src', tsconfigPath: path.join(__dirname, 'tsconfig.lib.json') })],
// Uncomment this if you are using workers.
// worker: {
// plugins: [ nxViteTsPaths() ],
@@ -21,7 +15,7 @@ export default defineConfig(() => ({
// Configuration for building your library.
// See: https://vitejs.dev/guide/build.html#library-mode
build: {
outDir: "./dist",
outDir: './dist',
emptyOutDir: true,
reportCompressedSize: true,
commonjsOptions: {
@@ -29,27 +23,27 @@ export default defineConfig(() => ({
},
lib: {
// Could also be a dictionary or array of multiple entry points.
entry: "src/index.ts",
name: "@storyteller/ui-settings-modal",
fileName: "index",
entry: 'src/index.ts',
name: '@storyteller/ui-settings-modal',
fileName: 'index',
// Change this to the formats you want to support.
// Don't forget to update your package.json as well.
formats: ["es" as const],
formats: ['es' as const]
},
rollupOptions: {
// External packages that should not be bundled into your library.
external: ["react", "react-dom", "react/jsx-runtime", /@storyteller\/.*/],
external: ['react','react-dom','react/jsx-runtime']
},
},
test: {
watch: false,
globals: true,
environment: "jsdom",
include: ["{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"],
reporters: ["default"],
environment: 'jsdom',
include: ['{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
reporters: ['default'],
coverage: {
reportsDirectory: "./test-output/vitest/coverage",
provider: "v8" as const,
},
reportsDirectory: './test-output/vitest/coverage',
provider: 'v8' as const,
}
},
}));
@@ -21,11 +21,6 @@ export class ImageModel extends Model {
// For inpainting models, does it require sending a mask?
readonly usesInpaintingMask: boolean;
// For editing models, is "editing" == "inpainting"?
// Examples of "true" are "flux dev juggernaut"
// Examples of "false" are "nano_banana_pro", which is just "editing".
readonly editingIsInpainting: boolean;
// Whether this model supports additional image prompts (reference images)
readonly canUseImagePrompt: boolean;
@@ -36,12 +31,6 @@ export class ImageModel extends Model {
// If true, it'll be displayed on the text-to-image page.
readonly canTextToImage: boolean;
// Whether the model supports changing resolution (e.g. 1k, 2k, 4k)
readonly canChangeResolution: boolean;
// Whether the model supports changing aspect ratio
readonly canChangeAspectRatio: boolean;
constructor(args: {
id: string;
tauriId: string;
@@ -55,12 +44,9 @@ export class ImageModel extends Model {
defaultGenerationCount: number;
canEditImages?: boolean;
usesInpaintingMask?: boolean;
editingIsInpainting?: boolean;
canUseImagePrompt?: boolean;
maxImagePromptCount?: number;
canTextToImage?: boolean;
canChangeResolution?: boolean;
canChangeAspectRatio?: boolean;
tags?: ModelTag[];
progressBarTime?: number;
}) {
@@ -80,26 +66,8 @@ export class ImageModel extends Model {
this.defaultGenerationCount = args.defaultGenerationCount;
this.canEditImages = args.canEditImages ?? false;
this.usesInpaintingMask = args.usesInpaintingMask ?? false;
this.editingIsInpainting = args.editingIsInpainting ?? false;
this.canUseImagePrompt = args.canUseImagePrompt ?? false;
this.maxImagePromptCount = Math.max(0, args.maxImagePromptCount ?? 1);
this.canTextToImage = args.canTextToImage === false ? false : true; // Default to true !
this.canChangeResolution = args.canChangeResolution ?? false;
this.canChangeAspectRatio = args.canChangeAspectRatio ?? false;
}
// If the model is a "Nano Banana"-type model, we may want to enable certain features.
// For example, in the editor, we may want to use the marker.
// TODO: Rather than "isNanoBananaModel()", we should have: "enableEditorMarker" as it's
// more semantic.
isNanoBananaModel(): boolean {
switch(this.id) {
case "gemini_25_flash":
case "nano_banana":
case "nano_banana_pro":
return true;
default:
return false;
}
}
}
@@ -20,7 +20,7 @@ const cfg = (
id: string;
category: ModelCategory;
info: ModelInfo;
},
}
): ModelConfig => ({
label: m.label ?? m.info.name,
description: m.description,
@@ -134,36 +134,6 @@ const ALL_MODELS: ModelConfig[] = [
},
tags: [ModelTag.InstructiveEdit, ModelTag.NonMaskedInpainting],
}),
cfg({
id: "gemini_25_flash",
category: "image",
info: {
name: "Nano Banana",
tauri_id: "gemini_25_flash",
creator: mc.Google,
},
description: "Incredibly instructive editing",
badges: [{ label: "25 sec." }],
capabilities: {
maxGenerationCount: 4,
},
tags: [ModelTag.InstructiveEdit, ModelTag.NonMaskedInpainting],
}),
cfg({
id: "nano_banana_pro",
category: "image",
info: {
name: "Nano Banana Pro",
tauri_id: "nano_banana_pro",
creator: mc.Google,
},
description: "Extremely powerful",
badges: [{ label: "35 sec." }],
capabilities: {
maxGenerationCount: 4,
},
tags: [ModelTag.InstructiveEdit, ModelTag.NonMaskedInpainting],
}),
cfg({
id: "flux_pro_inpaint",
category: "image",
@@ -267,22 +237,23 @@ export const getModelsByCategory = (category: ModelCategory): ModelConfig[] =>
export const getInstructiveImageEditModels = (): ModelConfig[] =>
ALL_MODELS.filter(
(m) => m.category === "image" && m.tags?.includes("instructiveEdit"),
(m) => m.category === "image" && m.tags?.includes("instructiveEdit")
);
export const getMaskedInpaintModels = (): ModelConfig[] =>
ALL_MODELS.filter(
(m) =>
m.category === "image" && m.tags?.includes(ModelTag.MaskedInpainting),
(m) => m.category === "image" && m.tags?.includes(ModelTag.MaskedInpainting)
);
export const lookupModelByTauriId = (
tauriId: string,
tauriId: string
): ModelConfig | undefined =>
ALL_MODELS.find((m) => m.info.tauri_id === tauriId);
// Single exported capability resolver so callers never need to touch anything else
export const getCapabilitiesForModel = (model?: Model): ModelCapabilities => {
export const getCapabilitiesForModel = (
model?: Model
): ModelCapabilities => {
if (!model) return DEFAULT_CAPABILITIES;
const cfg = lookupModelByTauriId(model.tauriId);
return cfg?.capabilities ?? DEFAULT_CAPABILITIES;
@@ -2,8 +2,6 @@ import { ImageModel } from "../classes/ImageModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import { ModelTag } from "../classes/metadata/ModelTag.js";
// TODO: Some of the model configs, such as generation counts, are authoritatively controlled in `legacy/Models.ts`
export const IMAGE_MODELS: ImageModel[] = [
new ImageModel({
id: "grok_image",
@@ -19,7 +17,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: false,
maxImagePromptCount: 6,
progressBarTime: 10000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "midjourney",
@@ -35,7 +32,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: false,
maxImagePromptCount: 6,
progressBarTime: 45000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_pro_1_1_ultra",
@@ -51,7 +47,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: true,
maxImagePromptCount: 6,
progressBarTime: 35000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_pro_1_1",
@@ -67,7 +62,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: true,
maxImagePromptCount: 6,
progressBarTime: 10000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_1_dev",
@@ -83,7 +77,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: true,
maxImagePromptCount: 6,
progressBarTime: 10000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_1_schnell",
@@ -99,7 +92,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canUseImagePrompt: true,
maxImagePromptCount: 6,
progressBarTime: 10000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "gemini_25_flash",
@@ -118,28 +110,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canTextToImage: false,
tags: [ModelTag.InstructiveEdit],
progressBarTime: 25000,
// canChangeResolution: true,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "nano_banana_pro",
tauriId: "nano_banana_pro",
fullName: "Nano Banana Pro",
category: "image",
creator: ModelCreator.Google,
selectorName: "Nano Banana Pro",
selectorDescription: "Extremely powerful",
selectorBadges: ["30 sec."],
maxGenerationCount: 4,
defaultGenerationCount: 1,
canEditImages: true,
canUseImagePrompt: true,
maxImagePromptCount: 4,
canTextToImage: true,
tags: [ModelTag.InstructiveEdit],
progressBarTime: 25000,
canChangeResolution: true,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "gpt_image_1",
@@ -150,14 +120,13 @@ export const IMAGE_MODELS: ImageModel[] = [
selectorName: "GPT Image 1 (GPT-4o)",
selectorDescription: "Slow, but super smart",
selectorBadges: ["60 sec."],
maxGenerationCount: 4,
maxGenerationCount: 1,
defaultGenerationCount: 1,
tags: [ModelTag.InstructiveEdit],
canEditImages: true,
canUseImagePrompt: true,
maxImagePromptCount: 6,
progressBarTime: 60000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_pro_kontext_max",
@@ -174,7 +143,6 @@ export const IMAGE_MODELS: ImageModel[] = [
canTextToImage: false,
tags: [ModelTag.InstructiveEdit],
progressBarTime: 20000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_pro_inpaint",
@@ -182,17 +150,15 @@ export const IMAGE_MODELS: ImageModel[] = [
fullName: "Flux Pro Inpaint",
category: "image",
creator: ModelCreator.BlackForestLabs,
selectorName: "Flux Pro (Inpainting)",
selectorName: "Flux Pro",
selectorDescription: "Fast inpainting",
selectorBadges: ["30 sec."],
maxGenerationCount: 1, // NB: Fal only allows one image for some reason!
defaultGenerationCount: 1, // NB: Fal only allows one image for some reason!
canEditImages: true,
usesInpaintingMask: true,
editingIsInpainting: true,
canTextToImage: false,
progressBarTime: 30000,
canChangeAspectRatio: true,
}),
new ImageModel({
id: "flux_dev_juggernaut_inpaint",
@@ -207,15 +173,13 @@ export const IMAGE_MODELS: ImageModel[] = [
defaultGenerationCount: 4,
canEditImages: true,
usesInpaintingMask: true,
editingIsInpainting: true,
canTextToImage: false,
progressBarTime: 10000,
canChangeAspectRatio: true,
}),
];
export const IMAGE_MODELS_BY_ID: Map<string, ImageModel> = new Map(
IMAGE_MODELS.map((model) => [model.id, model]),
IMAGE_MODELS.map((model) => [model.id, model])
);
if (IMAGE_MODELS_BY_ID.size !== IMAGE_MODELS.length) {
+1 -1
View File
@@ -2,7 +2,7 @@ export * from "./lib/artcraft/ArtcraftGetCredits.js";
export * from "./lib/artcraft/ArtcraftGetSubscription.js";
export * from "./lib/artcraft/media_files/MediaFileDelete.js";
export * from "./lib/common/CommandStatus.js";
export * from "./lib/enqueue/EnqueueEditImage.js";
export * from "./lib/enqueue/EnqueueContextualEditImage.js";
export * from "./lib/enqueue/EnqueueImageBgRemovalCommand.js";
export * from "./lib/enqueue/EnqueueImageInpaint.js";
export * from "./lib/enqueue/EnqueueImageTo3dObject.js";
@@ -2,10 +2,10 @@ import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { ImageModel } from "@storyteller/model-list";
export interface EnqueueEditImageRequest {
export interface EnqueueContextualEditImageRequest {
/// The model to use.
model?: ImageModel | EnqueueEditImageModel;
model?: ImageModel | EnqueueContextualEditImageModel;
/// If set, this is the first image in the contextual image set.
/// This gets submitted along with `image_media_tokens` and will
@@ -26,37 +26,26 @@ export interface EnqueueEditImageRequest {
/// Number of images to generate.
image_count?: number;
aspect_ratio?: EnqueueEditImageSize;
aspect_ratio?: EnqueueContextualEditImageSize;
image_quality?: EnqueueEditImageQuality;
/// Image resolution. Support: Nano Banana Pro
image_resolution?: EnqueueEditImageResolution;
// TODO: Actual enum.
frontend_caller?: string;
// Optional frontend state to return later.
frontend_subscriber_id?: string;
image_quality?: EnqueueContextualEditImageQuality;
}
// Shape of request sent to Tauri
// (We do some transformations from the public-facing request object.)
interface RawEnqueueEditImageRequest {
model?: EnqueueEditImageModel | string;
interface RawEnqueueContextualEditImageRequest {
model?: EnqueueContextualEditImageModel | string;
scene_image_media_token?: string;
image_media_tokens?: string[];
prompt?: string;
disable_system_prompt?: boolean;
image_count?: number;
aspect_ratio?: EnqueueEditImageSize;
image_quality?: EnqueueEditImageQuality;
frontend_caller?: string;
frontend_subscriber_id?: string;
aspect_ratio?: EnqueueContextualEditImageSize;
image_quality?: EnqueueContextualEditImageQuality;
}
export enum EnqueueEditImageErrorType {
export enum EnqueueContextualEditImageErrorType {
ModelNotSpecified = "model_not_specified",
NoProviderAvailable = "no_provider_available",
BadRequest = "bad_request",
@@ -67,46 +56,40 @@ export enum EnqueueEditImageErrorType {
SoraIsHavingProblems = "sora_is_having_problems",
}
export enum EnqueueEditImageModel {
export enum EnqueueContextualEditImageModel {
GptImage1 = "gpt_image_1",
FluxProKontextMax = "flux_pro_kontext_max",
}
export enum EnqueueEditImageSize {
export enum EnqueueContextualEditImageSize {
Auto = "auto",
Square = "square",
Wide = "wide",
Tall = "tall",
}
export enum EnqueueEditImageQuality {
export enum EnqueueContextualEditImageQuality {
Auto = "auto",
High = "high",
Medium = "medium",
Low = "low",
}
export enum EnqueueEditImageResolution {
OneK = "one_k",
TwoK = "two_k",
FourK = "four_k",
}
export interface EnqueueEditImageError extends CommandResult {
error_type: EnqueueEditImageErrorType;
export interface EnqueueContextualEditImageError extends CommandResult {
error_type: EnqueueContextualEditImageErrorType;
error_message?: string;
}
export interface EnqueueEditImagePayload {
export interface EnqueueContextualEditImagePayload {
}
export interface EnqueueEditImageSuccess extends CommandResult {
payload: EnqueueEditImagePayload;
export interface EnqueueContextualEditImageSuccess extends CommandResult {
payload: EnqueueContextualEditImagePayload;
}
export type EnqueueEditImageResult = EnqueueEditImageSuccess | EnqueueEditImageError;
export type EnqueueContextualEditImageResult = EnqueueContextualEditImageSuccess | EnqueueContextualEditImageError;
export const EnqueueEditImage = async (request: EnqueueEditImageRequest) : Promise<EnqueueEditImageResult> => {
export const EnqueueContextualEditImage = async (request: EnqueueContextualEditImageRequest) : Promise<EnqueueContextualEditImageResult> => {
let modelName = undefined;
if (!!request.model) {
@@ -122,7 +105,7 @@ export const EnqueueEditImage = async (request: EnqueueEditImageRequest) : Promi
throw new Error("No model specified in request: " + JSON.stringify(request));
}
let mutableRequest : RawEnqueueEditImageRequest = {
let mutableRequest : RawEnqueueContextualEditImageRequest = {
model: modelName,
prompt: request.prompt,
};
@@ -151,17 +134,9 @@ export const EnqueueEditImage = async (request: EnqueueEditImageRequest) : Promi
mutableRequest.disable_system_prompt = request.disable_system_prompt;
}
if (!!request.frontend_caller) {
mutableRequest.frontend_caller = request.frontend_caller;
}
if (!!request.frontend_subscriber_id) {
mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id;
}
const result = await invoke("enqueue_edit_image_command", {
const result = await invoke("enqueue_contextual_edit_image_command", {
request: mutableRequest,
});
return (result as EnqueueEditImageResult);
return (result as EnqueueContextualEditImageResult);
}
@@ -23,9 +23,6 @@ export interface EnqueueTextToImageRequest {
// The desired output aspect ratio.
aspect_ratio?: EnqueueTextToImageSize;
// Image resolution. Support: Nano Banana Pro
image_resolution?: EnqueueTextToImageResolution;
// The number of images to generate.
number_images?: number;
@@ -49,7 +46,6 @@ interface EnqueueTextToImageRawRequest {
prompt?: string;
model?: EnqueueTextToImageModel | string; // TODO: Shouldn't allow string
aspect_ratio?: EnqueueTextToImageSize;
image_resolution?: EnqueueTextToImageResolution;
number_images?: number;
image_media_tokens?: string[];
frontend_caller?: string;
@@ -69,12 +65,6 @@ export enum EnqueueTextToImageSize {
Tall = "tall",
}
export enum EnqueueTextToImageResolution {
OneK = "one_k",
TwoK = "two_k",
FourK = "four_k",
}
export interface EnqueueTextToImageError extends CommandResult {
error_type: EnqueueTextToImageErrorType;
error_message?: string;
@@ -117,10 +107,6 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P
mutableRequest.aspect_ratio = request.aspect_ratio;
}
if (!!request.image_resolution) {
mutableRequest.image_resolution = request.image_resolution;
}
if (!!request.number_images) {
mutableRequest.number_images = request.number_images;
}
-5
View File
@@ -45,11 +45,6 @@ sqlx_offline() {
#sqlx_migrate
sqlx_offline
# Issue with Linux performance
# https://github.com/tauri-apps/tauri/issues/13498
export WEBKIT_DISABLE_DMABUF_RENDERER=1
export WEBKIT_DISABLE_COMPOSITING_MODE=1
echo "Build and run..."
RUSTFLAGS="-Awarnings" cargo tauri dev \
--no-dev-server \