Compare commits

...
127 changed files with 9005 additions and 521 deletions
@@ -0,0 +1,12 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO tasks (\n id,\n task_status,\n task_type,\n model_type,\n provider,\n provider_job_id,\n frontend_caller,\n frontend_subscriber_id,\n frontend_subscriber_payload\n )\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 9
},
"nullable": []
},
"hash": "01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c"
}
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO tasks (\n id,\n task_status,\n task_type,\n model_type,\n provider,\n provider_job_id,\n frontend_subscriber_id,\n frontend_subscriber_payload\n )\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 8
},
"nullable": []
},
"hash": "22b91a21cbb88abc9bc17b338cf2f9418e08e414667d44476623eb141ab17b87"
}
Generated
+1
View File
@@ -663,6 +663,7 @@ dependencies = [
"futures 0.3.31",
"idempotency",
"image",
"images",
"indexmap 2.9.0",
"log",
"memory_store",
@@ -4,17 +4,43 @@
-- tasks_v2.sqlite - added model_type (nullable)
CREATE TABLE tasks (
-- Task auto-incrementing primary key.
-- We don't use this.
id TEXT NOT NULL PRIMARY KEY,
-- TaskStatus enum
-- This is how the job system manages job states.
-- e.g. pending, started, complete_success, dead, etc.
task_status TEXT NOT NULL,
-- TaskType enum
-- e.g. image_generation, video_generation, etc.
task_type TEXT NOT NULL,
-- TaskModelType enum
-- e.g. flux_1_dev, veo_2, etc.
model_type TEXT,
-- GenerationProvider enum
-- Together with `provider_job_id`, this creates a foreign key lookup to the job.
-- e.g. 'artcraft', 'fal', 'sora', etc.
provider TEXT,
-- The primary key for the job in the provider's system.
provider_job_id TEXT,
-- OPTIONAL.
-- Tell the job system which caller initiated the task.
-- e.g. 'canvas'
frontend_caller TEXT,
-- OPTIONAL.
-- An arbitrary opaque identifier that frontend can set.
frontend_subscriber_id TEXT,
-- OPTIONAL.
-- An opaque JSON payload set by the frontend subscriber that
-- will be re-emitted back to the frontend.
frontend_subscriber_payload TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
@@ -0,0 +1,89 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::prelude::fal_ai::flux_pro::kontext::max::{max, FluxProKontextMaxInput};
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct FluxProKontextMaxArgs<'a, U: IntoUrl, R: IntoUrl> {
// Request
pub prompt: &'a str,
pub image_url: U,
pub num_images: FluxProKontextMaxNumImages,
// Fulfillment
pub webhook_url: R,
pub api_key: &'a FalApiKey,
}
#[derive(Copy, Clone, Debug)]
pub enum FluxProKontextMaxNumImages {
One, // Default
Two,
Three,
Four,
}
pub async fn enqueue_flux_pro_kontext_max_edit_webhook<U: IntoUrl, R: IntoUrl>(
args: FluxProKontextMaxArgs<'_, U, R>
) -> Result<WebhookResponse, FalErrorPlus> {
let num_images = match args.num_images {
FluxProKontextMaxNumImages::One => 1,
FluxProKontextMaxNumImages::Two => 2,
FluxProKontextMaxNumImages::Three => 3,
FluxProKontextMaxNumImages::Four => 4,
};
let request = FluxProKontextMaxInput {
prompt: args.prompt.to_string(),
image_url: args.image_url.as_str().to_string(),
num_images: Some(num_images),
// Maybe expose
aspect_ratio: None,
safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant
output_format: Some("png".to_string()), // png or jpeg
seed: None,
// Constants
sync_mode: None, // Synchronous / slow
};
let result = max(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages};
use errors::AnyhowResult;
use std::fs::read_to_string;
use test_data::web::image_urls::TALL_MOCHI_WITH_GLASSES_IMAGE_URL;
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = FluxProKontextMaxArgs {
image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL,
prompt: "turn the glasses into sunglasses, make them sleek sunglasses with black rims, square shaped",
api_key: &api_key,
webhook_url: "https://example.com/webhook",
num_images: FluxProKontextMaxNumImages::One,
};
let result = enqueue_flux_pro_kontext_max_edit_webhook(args).await?;
Ok(())
}
}
@@ -0,0 +1 @@
pub mod enqueue_flux_pro_kontext_max_edit_webhook;
@@ -0,0 +1,92 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::prelude::fal_ai::flux_pro::v1::fill::fill;
use fal::prelude::fal_ai::flux_pro::v1::fill::FluxProFillInput;
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct FluxPro1InfillArgs<'a, U: IntoUrl, R: IntoUrl, L: IntoUrl> {
// Request
pub prompt: &'a str,
pub image_url: U,
pub mask_url: R,
pub num_images: FluxPro1InfillNumImages,
// Fulfillment
pub webhook_url: L,
pub api_key: &'a FalApiKey,
}
#[derive(Copy, Clone, Debug)]
pub enum FluxPro1InfillNumImages {
One, // Default
Two,
Three,
Four,
}
pub async fn enqueue_flux_pro_1_infill_webhook<U: IntoUrl, R: IntoUrl, L: IntoUrl>(
args: FluxPro1InfillArgs<'_, U, R, L>
) -> Result<WebhookResponse, FalErrorPlus> {
let num_images = match args.num_images {
FluxPro1InfillNumImages::One => 1,
FluxPro1InfillNumImages::Two => 2,
FluxPro1InfillNumImages::Three => 3,
FluxPro1InfillNumImages::Four => 4,
};
let request = FluxProFillInput {
prompt: args.prompt.to_string(),
image_url: args.image_url.as_str().to_string(),
mask_url: args.mask_url.as_str().to_string(),
num_images: Some(num_images),
// Maybe expose
safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant
output_format: Some("png".to_string()), // png or jpeg
seed: None,
// Constants
sync_mode: None, // Synchronous / slow
};
let result = fill(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages};
use errors::AnyhowResult;
use std::fs::read_to_string;
use test_data::web::image_urls::{TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, TALL_MOCHI_WITH_GLASSES_IMAGE_URL};
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = FluxPro1InfillArgs {
image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL,
mask_url: TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL,
prompt: "slick sunglasses, cool glasses, reflection in glasses lenses",
api_key: &api_key,
webhook_url: "https://example.com/webhook",
num_images: FluxPro1InfillNumImages::One,
};
let result = enqueue_flux_pro_1_infill_webhook(args).await?;
Ok(())
}
}
@@ -0,0 +1 @@
pub mod enqueue_flux_pro_1_infill_webhook;
@@ -1,7 +1,9 @@
pub mod edit;
pub mod enqueue_flux_1_dev_text_to_image_webhook;
pub mod enqueue_flux_1_schnell_text_to_image_webhook;
pub mod enqueue_flux_pro_11_text_to_image_webhook;
pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook;
pub mod enqueue_gpt_image_1_edit_image_webhook;
pub mod enqueue_gpt_image_1_text_to_image_webhook;
pub mod infill;
pub mod remove_background_rembg_webhook;
@@ -0,0 +1,19 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageRequest, FluxProKontextMaxEditImageResponse, FLUX_PRO_KONTEXT_MAX_EDIT_IMAGE_PATH};
pub async fn flux_pro_kontext_max_edit_image(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: FluxProKontextMaxEditImageRequest,
) -> Result<FluxProKontextMaxEditImageResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
FLUX_PRO_KONTEXT_MAX_EDIT_IMAGE_PATH,
maybe_creds,
request,
).await?)
}
@@ -1 +1,2 @@
pub mod flux_pro_kontext_max_edit_image;
pub mod gpt_image_1_edit_image;
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageRequest, FluxPro1InpaintImageResponse, FLUX_PRO_1_INPAINT_PATH};
pub async fn flux_pro_1_inpaint_image(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: FluxPro1InpaintImageRequest,
) -> Result<FluxPro1InpaintImageResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
FLUX_PRO_1_INPAINT_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1 @@
pub mod flux_pro_1_inpaint_image;
@@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image;
pub mod generate_flux_pro_11_text_to_image;
pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod remove_image_background;
@@ -1,4 +1,5 @@
pub mod get_media_file;
pub mod upload_image_media_file_from_bytes;
pub mod upload_image_media_file_from_file;
pub mod upload_new_engine_asset_from_file;
pub mod upload_video_media_file_from_file;
@@ -0,0 +1,138 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::api_error::ApiError;
use crate::error::client_error::ClientError;
use crate::error::storyteller_error::StorytellerError;
use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails;
use crate::shared_response_types::media_links::MediaLinks;
use crate::shared_response_types::simple_entity_stats::SimpleEntityStats;
use crate::shared_response_types::user_details_light::UserDetailsLight;
use crate::utils::api_host::ApiHost;
use crate::utils::constants::{APPLICATION_JSON, USER_AGENT};
use crate::utils::filter_bad_response::filter_bad_response;
use crate::utils::http_get_anonymous::http_get_anonymous;
use chrono::{DateTime, Utc};
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::common::visibility::Visibility;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType;
use idempotency::uuid::generate_random_uuid;
use log::debug;
use reqwest::multipart::{Form, Part};
use reqwest::Client;
use serde_derive::{Deserialize, Serialize};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use tokens::tokens::batch_generations::BatchGenerationToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::prompts::PromptToken;
use uuid::uuid;
pub struct UploadImageBytesArgs<'a> {
pub api_host: &'a ApiHost,
pub maybe_creds: Option<&'a StorytellerCredentialSet>,
// NB: Bytes need to be owned for the request.
pub image_bytes: Vec<u8>,
pub image_type: ImageType,
/// If true, we should hide the image from the user's gallery.
pub is_intermediate_system_file: bool,
}
#[derive(Clone, Copy, Debug)]
pub enum ImageType {
Png,
Jpeg,
Gif,
}
impl ImageType {
pub fn mime_type(&self) -> &str {
match self {
ImageType::Png => "image/png",
ImageType::Jpeg => "image/jpeg",
ImageType::Gif => "image/gif",
}
}
pub fn file_extension(&self) -> &str {
match self {
ImageType::Png => "png",
ImageType::Jpeg => "jpg",
ImageType::Gif => "gif",
}
}
}
/// Upload an image media file from a file.
/// NB: File type is not checked, so caller needs to enforce.
/// NB: We need owned bytes for the request.
pub async fn upload_image_media_file_from_bytes(
args: UploadImageBytesArgs<'_>
) -> Result<UploadImageMediaFileSuccessResponse, StorytellerError> {
let url = get_route(args.api_host);
debug!("Requesting {:?}", &url);
let client = Client::builder()
.gzip(true)
.build()
.map_err(|err| StorytellerError::Client(ClientError::from(err)))?;
let part = Part::bytes(args.image_bytes)
.file_name(format!("image.{}", args.image_type.file_extension()))
.mime_str(args.image_type.mime_type())
.map_err(|err| StorytellerError::Client(ClientError::from(err)))?;
let mut form = Form::new()
.text("uuid_idempotency_token", generate_random_uuid())
.part("file", part);
if args.is_intermediate_system_file {
form = form.text("is_intermediate_system_file", "true");
}
let mut request_builder = client.post(url)
.header("User-Agent", USER_AGENT)
.header("Accept", APPLICATION_JSON);
if let Some(creds) = args.maybe_creds {
if let Some(header) = &creds.maybe_as_cookie_header() {
request_builder = request_builder.header("Cookie", header);
}
}
let response = request_builder
.multipart(form)
.send()
.await
.map_err(|err| StorytellerError::Api(ApiError::from(err)))?;
let response = filter_bad_response(response).await?;
let response_body = &response.text().await
.map_err(|err| StorytellerError::Api(ApiError::from(err)))?;
let media_file = serde_json::from_str(&response_body)
.map_err(|err| StorytellerError::Api(ApiError::from(err)))?;
Ok(media_file)
}
fn get_route(api_host: &ApiHost) -> String {
let api_hostname_and_scheme = api_host.to_api_hostname_and_scheme();
format!("{}/v1/media_files/upload/image", api_hostname_and_scheme)
}
// TODO(bt,2025-04-22): Share API definitions between client and server in common crate.
#[derive(Deserialize, Debug)]
pub struct UploadImageMediaFileSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
@@ -33,14 +33,24 @@ use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::prompts::PromptToken;
use uuid::uuid;
pub struct UploadImageFromFileArgs<'a, P: AsRef<Path>> {
pub api_host: &'a ApiHost,
pub maybe_creds: Option<&'a StorytellerCredentialSet>,
// NB: Path needs to be owned for the request.
pub path: P,
/// If true, we should hide the image from the user's gallery.
pub is_intermediate_system_file: bool,
}
/// Upload an image media file from a file.
pub async fn upload_image_media_file_from_file<P: AsRef<Path>>(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
path: P,
args: UploadImageFromFileArgs<'_, P>,
) -> Result<UploadImageMediaFileSuccessResponse, StorytellerError> {
let url = get_route(api_host);
let url = get_route(args.api_host);
debug!("Requesting {:?}", &url);
@@ -51,17 +61,21 @@ pub async fn upload_image_media_file_from_file<P: AsRef<Path>>(
.build()
.map_err(|err| StorytellerError::Client(ClientError::from(err)))?;
let form = Form::new()
let mut form = Form::new()
.text("uuid_idempotency_token", generate_random_uuid())
.file("file", path)
.file("file", args.path)
.await
.map_err(|err| StorytellerError::Client(ClientError::from(err)))?;
if args.is_intermediate_system_file {
form = form.text("is_intermediate_system_file", "true");
}
let mut request_builder = client.post(url)
.header("User-Agent", USER_AGENT)
.header("Accept", APPLICATION_JSON);
if let Some(creds) = maybe_creds {
if let Some(creds) = args.maybe_creds {
if let Some(header) = &creds.maybe_as_cookie_header() {
request_builder = request_builder.header("Cookie", header);
}
@@ -1,7 +1,7 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::client_error::ClientError;
use crate::error::storyteller_error::StorytellerError;
use crate::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use crate::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
use crate::media_files::upload_new_engine_asset_from_file::upload_new_engine_asset_from_file;
use crate::media_files::upload_video_media_file_from_file::upload_video_media_file_from_file;
use crate::utils::api_host::ApiHost;
@@ -49,7 +49,13 @@ pub async fn upload_media_file_from_file<P: AsRef<Path>>(
}
}
Some(FileExtension::Png) | Some(FileExtension::Jpg) | Some(FileExtension::Gif) | Some(FileExtension::Webp) => {
match upload_image_media_file_from_file(api_host, maybe_creds, path).await {
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host,
maybe_creds,
path,
is_intermediate_system_file: false,
}).await;
match result {
Ok(result) => Ok(UploadMediaFileSuccessResponse {
success: result.success,
media_file_token: result.media_file_token,
+1
View File
@@ -33,6 +33,7 @@ errors = { workspace = true }
fal_client = { workspace = true }
filesys = { workspace = true }
idempotency = { workspace = true }
images = { workspace = true }
memory_store = { workspace = true }
mimetypes = { workspace = true }
openai_sora_client = { workspace = true }
@@ -43,6 +43,7 @@ use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize, Debug)]
@@ -55,6 +56,21 @@ pub struct EnqueueImageBgRemovalCommand {
/// Base64-encoded image
pub base64_image: Option<String>,
/// OPTIONAL.
/// Name of the frontend caller.
/// We'll use this to selectively trigger events.
pub frontend_caller: Option<TauriCommandCaller>,
/// OPTIONAL.
/// A frontend-defined identifier that we'll send back to the frontend
/// as a Tauri event on task completion.
pub frontend_subscriber_id: Option<String>,
/// OPTIONAL.
/// A frontend-defined payload that we'll send back to the frontend
/// as a Tauri event on task completion.
pub frontend_subscriber_payload: Option<String>,
}
#[derive(Serialize, Debug)]
@@ -169,7 +185,12 @@ pub async fn handle_request(
).await?;
let result = success_event
.insert_into_task_database(task_database)
.insert_into_task_database_with_frontend_payload(
task_database,
request.frontend_caller,
request.frontend_subscriber_id.as_deref(),
request.frontend_subscriber_payload.as_deref()
)
.await;
if let Err(err) = result {
@@ -39,7 +39,7 @@ use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
use storyteller_client::generate::image::remove_image_background::remove_image_background;
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
use tauri::AppHandle;
use tokens::tokens::media_files::MediaFileToken;
@@ -123,9 +123,13 @@ async fn upload_image_from_base64_bytes(
info!("Uploading image media file from temp file: {:?}", temp_file.path());
let result =
upload_image_media_file_from_file(&app_env_configs.storyteller_host, Some(&creds), temp_file)
.await
let result =
upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&creds),
path: temp_file,
is_intermediate_system_file: true, // NB: Probably not essential to keep this.
}).await
.map_err(|err| {
error!("Failed to upload image media file: {:?}", err);
InternalBgRemovalError::StorytellerError(err)
@@ -43,6 +43,7 @@ use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
use crate::core::commands::enqueue::image_edit::flux_kontext::handle_flux_kontext_edit::handle_flux_kontext_edit;
#[derive(Deserialize, Debug)]
pub struct EnqueueContextualEditImageCommand {
@@ -207,6 +208,20 @@ pub async fn handle_request(
None => {
return Err(InternalContextualEditImageError::NoModelSpecified)
}
Some(ContextualImageEditModel::FluxProKontextMax) => {
handle_flux_kontext_edit(
request,
app,
app_data_root,
app_env_configs,
provider_priority_store,
storyteller_creds_manager,
fal_creds_manager,
fal_task_queue,
sora_creds_manager,
sora_task_queue,
).await?
}
Some(ContextualImageEditModel::GptImage1) => {
handle_gpt_image_1_edit(
request,
@@ -221,7 +236,6 @@ pub async fn handle_request(
sora_task_queue,
).await?
}
// TODO(bt,2025-07-05): Flux Kontext, etc.
};
let result = success_event
@@ -30,6 +30,12 @@ pub enum InternalContextualEditImageError {
requested: u32,
},
// TODO: Maybe just reuse the other error enum variant
/// An invalid number of input images was requested for Flux Kontext.
InvalidNumberOfInputImagesForFluxKontext {
message: String,
},
/// No sora credentials are set.
NeedsSoraCredentials,
@@ -84,6 +90,11 @@ impl InternalContextualEditImageError {
error_type = EnqueueContextualEditImageErrorType::BadRequest;
error_message = format!("Invalid number of input images ({}). Must be between {} and {}", requested, min, max);
}
InternalContextualEditImageError::InvalidNumberOfInputImagesForFluxKontext { message } => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueContextualEditImageErrorType::BadRequest;
error_message = message.clone();
}
InternalContextualEditImageError::SoraError(_) => {}
InternalContextualEditImageError::AnyhowError(_) => {}
InternalContextualEditImageError::StorytellerError(_) => {}
@@ -0,0 +1,59 @@
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_edit::flux_kontext::handle_flux_kontext_edit_artcraft::handle_flux_kontext_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use log::info;
use tauri::AppHandle;
pub async fn handle_flux_kontext_edit(
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_creds_manager: &FalCredentialManager,
fal_task_queue: &FalTaskQueue,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, InternalContextualEditImageError> {
let priority = provider_priority_store.get_priority()?;
// TODO: Check if providers are available before proceeding.
info!("Providers by priority: {:?}", priority);
for provider in priority.iter() {
match provider {
Provider::Fal => {
// Fallthrough (TODO)
}
Provider::Sora => {
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching gpt-image-1 (edit) via Artcraft...");
return handle_flux_kontext_edit_artcraft(
request,
app,
app_data_root,
app_env_configs,
storyteller_creds_manager
).await;
}
}
}
Err(InternalContextualEditImageError::NoProviderAvailable)
}
@@ -0,0 +1,148 @@
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest;
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest};
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest};
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
use fal_client::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::FluxProKontextMaxNumImages;
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality;
use idempotency::uuid::generate_random_uuid;
use log::{error, info};
use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image;
use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image;
use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
use tauri::AppHandle;
pub async fn handle_flux_kontext_edit_artcraft(
request: &EnqueueContextualEditImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<TaskEnqueueSuccess, InternalContextualEditImageError> {
let creds = match storyteller_creds_manager.get_credentials()? {
Some(creds) => creds,
None => {
return Err(InternalContextualEditImageError::NeedsStorytellerCredentials);
},
};
info!("Calling Artcraft flux kontext max edit (edit) ...");
let uuid_idempotency_token = generate_random_uuid();
let num_images = match request.image_count {
None => None,
Some(1) => Some(FluxProKontextMaxEditImageNumImages::One),
Some(2) => Some(FluxProKontextMaxEditImageNumImages::Two),
Some(3) => Some(FluxProKontextMaxEditImageNumImages::Three),
Some(4) => Some(FluxProKontextMaxEditImageNumImages::Four),
Some(other) => {
return Err(InternalContextualEditImageError::InvalidNumberOfRequestedImages {
min: 1,
max: 4,
requested: other,
});
},
};
let mut maybe_media_token = None;
let list_count = request.image_media_tokens
.as_ref()
.map(|tokens| tokens.len())
.unwrap_or(0);
if request.scene_image_media_token.is_some() && list_count > 0 {
return Err(InternalContextualEditImageError::InvalidNumberOfInputImagesForFluxKontext {
message: "Cannot specify both scene_image_media_token and image_media_tokens for Flux Kontext Max".to_string()
});
}
if list_count > 1 {
return Err(InternalContextualEditImageError::InvalidNumberOfInputImagesForFluxKontext {
message: "Cannot specify more than one image in image_media_tokens for Flux Kontext Max".to_string()
});
}
if let Some(media_token) = &request.scene_image_media_token {
maybe_media_token = Some(media_token.clone());
}
let maybe_list_media_token = request.image_media_tokens
.as_ref()
.map(|tokens| tokens.first())
.flatten();
if let Some(media_token) = maybe_list_media_token {
maybe_media_token = Some(media_token.clone());
}
let media_token = match maybe_media_token {
Some(token) => token,
None => {
return Err(InternalContextualEditImageError::InvalidNumberOfInputImagesForFluxKontext {
message: "No input image specified for Flux Kontext Max edit".to_string(),
});
},
};
let request = FluxProKontextMaxEditImageRequest {
uuid_idempotency_token,
prompt: Some(request.prompt.clone()),
image_media_token: media_token,
num_images,
};
let result = flux_pro_kontext_max_edit_image(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft gpt-image-1: {:?}", err);
return Err(InternalContextualEditImageError::StorytellerError(err));
}
};
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::FluxProKontextMax),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageGeneration,
})
}
@@ -0,0 +1,2 @@
pub (super) mod handle_flux_kontext_edit;
mod handle_flux_kontext_edit_artcraft;
@@ -1,3 +1,4 @@
pub mod enqueue_contextual_edit_image_command;
pub mod errors;
pub mod gpt_image_1;
pub mod gpt_image_1;
mod flux_kontext;
@@ -0,0 +1,249 @@
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, EnqueueTextToImageSuccessResponse};
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::image_inpaint::flux_pro_1_inpaint::handle_flux_pro_1_inpaint::handle_flux_pro_1_inpaint;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType};
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::core::state::task_database::TaskDatabase;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use errors::AnyhowError;
use log::{error, info, warn};
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::sora_error::SoraError;
use serde_derive::{Deserialize, Serialize};
use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs};
use std::time::Duration;
use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize, Debug, Copy, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ImageInpaintModel {
#[serde(rename = "flux_pro_1")]
FluxPro1,
}
#[derive(Deserialize, Debug)]
pub struct EnqueueInpaintImageCommand {
/// REQUIRED (Option<T> is just for error messages).
/// The model to use.
pub model: Option<ImageInpaintModel>,
/// REQUIRED (Option<T> is just for error messages).
/// The source image to edit.
pub image_media_token: Option<MediaFileToken>,
/// REQUIRED: Supply this *XOR* `mask_image_raw_bytes`.
/// The mask to focus the edit (already uploaded).
pub mask_image_media_token: Option<MediaFileToken>,
/// REQUIRED: Supply this *XOR* `mask_image_media_token`.
/// The mask to focus the edit (raw bytes).
pub mask_image_raw_bytes: Option<Vec<u8>>,
/// REQUIRED.
/// The user's image generation prompt.
pub prompt: String,
/// Number of images to generate.
pub image_count: Option<u32>,
/// If true, force the dimensions of the source image and mask image to match.
pub require_matching_dimensions: Option<bool>,
/// OPTIONAL.
/// Name of the frontend caller.
/// We'll use this to selectively trigger events.
pub frontend_caller: Option<TauriCommandCaller>,
/// OPTIONAL.
/// A frontend-defined identifier that we'll send back to the frontend
/// as a Tauri event on task completion.
pub frontend_subscriber_id: Option<String>,
/// OPTIONAL.
/// A frontend-defined payload that we'll send back to the frontend
/// as a Tauri event on task completion.
pub frontend_subscriber_payload: Option<String>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum EnqueueInpaintImageErrorType {
/// Caller didn't specify a model
ModelNotSpecified,
/// No model available for image generation
NoProviderAvailable,
/// No source image was supplied.
NoSourceImageSpecified,
/// No mask image was supplied.
NoMaskImageSpecified,
/// Too many mask images were supplied.
MultipleMaskImagesSpecified,
/// Bad mask image was supplied.
BadMaskImage,
/// Generic bad request error
BadRequest,
/// Generic server error
ServerError,
}
#[derive(Serialize)]
pub struct EnqueueImageInpaintSuccessResponse {
}
impl SerializeMarker for EnqueueImageInpaintSuccessResponse {}
#[tauri::command]
pub async fn enqueue_image_inpaint_command(
app: AppHandle,
request: EnqueueInpaintImageCommand,
app_data_root: State<'_, AppDataRoot>,
app_env_configs: State<'_, AppEnvConfigs>,
provider_priority_store: State<'_, ProviderPriorityStore>,
task_database: State<'_, TaskDatabase>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
fal_creds_manager: State<'_, FalCredentialManager>,
fal_task_queue: State<'_, FalTaskQueue>,
sora_creds_manager: State<'_, SoraCredentialManager>,
sora_task_queue: State<'_, SoraTaskQueue>,
) -> ResponseOrErrorType<EnqueueImageInpaintSuccessResponse, EnqueueInpaintImageErrorType> {
info!("enqueue_image_inpaint_command called; model: {:?}, image_media_token: {:?}, prompt: {:?}",
&request.model, &request.image_media_token, &request.prompt);
let result = handle_request(
&request,
&app,
&app_data_root,
&app_env_configs,
&provider_priority_store,
&task_database,
&storyteller_creds_manager,
&fal_creds_manager,
&fal_task_queue,
&sora_creds_manager,
&sora_task_queue,
).await;
match result {
Err(err) => {
error!("Error enqueuing inpaint image: {:?}", err);
// TODO: Derive from err. Make service provider optional.
let event = GenerationEnqueueFailureEvent {
action: GenerationAction::GenerateImage,
service: GenerationServiceProvider::Artcraft, // FIXME: This is wrong.
model: None,
reason: None,
};
if let Err(err) = event.send(&app) {
error!("Failed to emit event: {:?}", err); // Fail open.
}
Err(err.to_tauri_response())
}
Ok(event) => {
let event = GenerationEnqueueSuccessEvent {
action: event.to_frontend_event_action(),
service: event.to_frontend_event_service(),
model: event.model,
};
if let Err(err) = event.send(&app) {
error!("Failed to emit event: {:?}", err); // Fail open.
}
Ok(EnqueueImageInpaintSuccessResponse {}.into())
}
}
}
pub async fn handle_request(
request: &EnqueueInpaintImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
task_database: &TaskDatabase,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_creds_manager: &FalCredentialManager,
fal_task_queue: &FalTaskQueue,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, InternalImageInpaintError> {
let success_event= match request.model {
None => {
return Err(InternalImageInpaintError::NoModelSpecified)
}
Some(ImageInpaintModel::FluxPro1) => {
handle_flux_pro_1_inpaint(
request,
app,
app_data_root,
app_env_configs,
provider_priority_store,
storyteller_creds_manager,
fal_creds_manager,
fal_task_queue,
sora_creds_manager,
sora_task_queue,
).await?
}
};
let result = success_event
.insert_into_task_database_with_frontend_payload(
task_database,
request.frontend_caller,
request.frontend_subscriber_id.as_deref(),
request.frontend_subscriber_payload.as_deref()
)
.await;
if let Err(err) = result {
error!("Failed to create task in database: {:?}", err);
// NB: Fail open, but find a way to flag this.
}
Ok(success_event)
}
@@ -0,0 +1,128 @@
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageErrorType;
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
use errors::AnyhowError;
use serde::Serialize;
use storyteller_client::error::storyteller_error::StorytellerError;
#[derive(Debug)]
pub enum InternalImageInpaintError {
/// Request did not specify a model.
NoModelSpecified,
/// No provider is configured for the model type.
NoProviderAvailable,
/// A source image was not provided
NoSourceImageSpecified,
/// A mask image was not provided
NoMaskImageSpecified,
/// Both mask image media token and raw bytes were supplied.
MaskMediaTokenAndBytesSupplied,
/// Mask image has an unknown MIME type.
CouldNotDetermineMaskMimeType,
/// Could not encode the mask image to PNG.
CouldNotEncodeMask,
/// No storyteller credentials are set.
NeedsStorytellerCredentials,
/// An invalid number of images was requested.
InvalidNumberOfRequestedImages {
min: u32,
max: u32,
requested: u32,
},
/// An invalid number of input images was requested.
InvalidNumberOfInputImages {
min: u32,
max: u32,
requested: u32,
},
AnyhowError(AnyhowError),
StorytellerError(StorytellerError),
}
impl From<AnyhowError> for InternalImageInpaintError {
fn from(value: AnyhowError) -> Self {
Self::AnyhowError(value)
}
}
impl From<StorytellerError> for InternalImageInpaintError {
fn from(value: StorytellerError) -> Self {
Self::StorytellerError(value)
}
}
impl InternalImageInpaintError {
pub fn to_tauri_response<E: Serialize>(&self) -> CommandErrorResponseWrapper<EnqueueInpaintImageErrorType, E> {
let mut status = CommandErrorStatus::ServerError;
let mut error_type = EnqueueInpaintImageErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.".to_string();
match self {
InternalImageInpaintError::NoModelSpecified => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::ModelNotSpecified;
error_message = "No model specified for image generation".to_string();
}
InternalImageInpaintError::NoProviderAvailable => {
status = CommandErrorStatus::ServerError;
error_type = EnqueueInpaintImageErrorType::NoProviderAvailable;
error_message = "No configured provider available for image generation".to_string();
}
InternalImageInpaintError::NoSourceImageSpecified => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::NoSourceImageSpecified;
error_message = "No source image was provided".to_string();
}
InternalImageInpaintError::NoMaskImageSpecified => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::NoMaskImageSpecified;
error_message = "No mask image was provided".to_string();
}
InternalImageInpaintError::MaskMediaTokenAndBytesSupplied => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::MultipleMaskImagesSpecified;
error_message = "multiple mask images provided".to_string();
}
InternalImageInpaintError::CouldNotDetermineMaskMimeType => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::BadMaskImage;
error_message = "bad mask image (mime)".to_string();
}
InternalImageInpaintError::CouldNotEncodeMask => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::BadMaskImage;
error_message = "bad mask image (encode)".to_string();
}
InternalImageInpaintError::InvalidNumberOfRequestedImages { min, max, requested } => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::BadRequest;
error_message = format!("Invalid number of images requested ({}). Must be between {} and {}", requested, min, max);
}
InternalImageInpaintError::InvalidNumberOfInputImages { min, max, requested } => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueInpaintImageErrorType::BadRequest;
error_message = format!("Invalid number of input images ({}). Must be between {} and {}", requested, min, max);
}
InternalImageInpaintError::AnyhowError(_) => {}
InternalImageInpaintError::StorytellerError(_) => {}
InternalImageInpaintError::NeedsStorytellerCredentials => {}
}
CommandErrorResponseWrapper {
status,
error_message: Some(error_message.to_string()),
error_type: Some(error_type),
error_details: None,
}
}
}
@@ -0,0 +1,63 @@
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft;
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::image_inpaint::flux_pro_1_inpaint::handle_flux_pro_1_inpaint_artcraft::handle_flux_pro_1_inpaint_artcraft;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use log::info;
use tauri::AppHandle;
pub(super) const MAX_IMAGES: usize = 10;
pub async fn handle_flux_pro_1_inpaint(
request: &EnqueueInpaintImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
provider_priority_store: &ProviderPriorityStore,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_creds_manager: &FalCredentialManager,
fal_task_queue: &FalTaskQueue,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> Result<TaskEnqueueSuccess, InternalImageInpaintError> {
let priority = provider_priority_store.get_priority()?;
// TODO: Check if providers are available before proceeding.
info!("Providers by priority: {:?}", priority);
for provider in priority.iter() {
match provider {
Provider::Fal => {
// Fallthrough (for now)
}
Provider::Sora => {
// Fallthrough
}
Provider::Artcraft => {
info!("Dispatching flux pro 1 inpaint via Artcraft...");
return handle_flux_pro_1_inpaint_artcraft(
request,
app,
app_data_root,
app_env_configs,
storyteller_creds_manager
).await;
}
}
}
Err(InternalImageInpaintError::NoProviderAvailable)
}
@@ -0,0 +1,181 @@
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest;
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand;
use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand;
use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError;
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
use crate::core::model::image_models::ImageModel;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::provider_priority::ProviderPriorityStore;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest};
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest};
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality;
use idempotency::uuid::generate_random_uuid;
use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes;
use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions;
use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask;
use log::{error, info};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image;
use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
use storyteller_client::generate::image::inpaint::flux_pro_1_inpaint_image::flux_pro_1_inpaint_image;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::media_files::upload_image_media_file_from_bytes::{upload_image_media_file_from_bytes, ImageType, UploadImageBytesArgs};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use tauri::AppHandle;
use tokens::tokens::media_files::MediaFileToken;
pub async fn handle_flux_pro_1_inpaint_artcraft(
request: &EnqueueInpaintImageCommand,
app: &AppHandle,
app_data_root: &AppDataRoot,
app_env_configs: &AppEnvConfigs,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<TaskEnqueueSuccess, InternalImageInpaintError> {
let creds = match storyteller_creds_manager.get_credentials()? {
Some(creds) => creds,
None => {
return Err(InternalImageInpaintError::NeedsStorytellerCredentials);
},
};
let image_media_token = match &request.image_media_token {
Some(token) => token.clone(),
None => {
return Err(InternalImageInpaintError::NoSourceImageSpecified);
},
};
//let image_media_file = get_media_file(
// &app_env_configs.storyteller_host,
// &image_media_token,
//).await.map_err(|err| {
// error!("Failed to get media file for image: {:?}", err);
// InternalImageInpaintError::StorytellerError(err)
//})?;
let mask_media_token = get_mask(
request,
app_env_configs,
&creds,
).await?;
info!("Calling Artcraft flux pro 1 inpaint ...");
let uuid_idempotency_token = generate_random_uuid();
let num_images = match request.image_count {
None => None,
Some(1) => Some(FluxPro1InpaintImageNumImages::One),
Some(2) => Some(FluxPro1InpaintImageNumImages::Two),
Some(3) => Some(FluxPro1InpaintImageNumImages::Three),
Some(4) => Some(FluxPro1InpaintImageNumImages::Four),
Some(other) => {
return Err(InternalImageInpaintError::InvalidNumberOfRequestedImages {
min: 1,
max: 4,
requested: other,
});
},
};
let request = FluxPro1InpaintImageRequest {
uuid_idempotency_token,
prompt: Some(request.prompt.clone()),
image_media_token,
mask_media_token,
num_images,
};
let result = flux_pro_1_inpaint_image(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_id = match result {
Ok(enqueued) => {
// TODO(bt,2025-07-05): Enqueue job token?
info!("Successfully enqueued Artcraft flux pro 1 inpaint. Job token: {}",
enqueued.inference_job_token);
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to use Artcraft flux pro 1 inpaint: {:?}", err);
return Err(InternalImageInpaintError::StorytellerError(err));
}
};
Ok(TaskEnqueueSuccess {
provider: GenerationProvider::Artcraft,
model: Some(GenerationModel::FluxProKontextMax),
provider_job_id: Some(job_id.to_string()),
task_type: TaskType::ImageGeneration,
})
}
async fn get_mask(
request: &EnqueueInpaintImageCommand,
app_env_configs: &AppEnvConfigs,
storyteller_creds: &StorytellerCredentialSet,
) -> Result<MediaFileToken, InternalImageInpaintError> {
if request.mask_image_media_token.is_some() && request.mask_image_raw_bytes.is_some() {
return Err(InternalImageInpaintError::MaskMediaTokenAndBytesSupplied);
}
if let Some(token) = request.mask_image_media_token.as_ref() {
return Ok(token.clone());
};
let image_bytes = request.mask_image_raw_bytes.as_ref()
.ok_or(InternalImageInpaintError::NoMaskImageSpecified)?;
let image_bytes = normalize_image_bytes_to_flux_mask(image_bytes)
.map_err(|err| {
error!("Failed to convert image bytes to png: {:?}", err);
InternalImageInpaintError::CouldNotEncodeMask
})?;
info!("Uploading image media file from bytes...");
let result = upload_image_media_file_from_bytes(UploadImageBytesArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&storyteller_creds),
image_bytes: image_bytes.0,
image_type: ImageType::Png,
is_intermediate_system_file: true,
}).await
.map_err(|err| {
error!("Failed to upload image media file: {:?}", err);
InternalImageInpaintError::StorytellerError(err)
})?;
Ok(result.media_file_token)
}
@@ -0,0 +1,2 @@
pub (super) mod handle_flux_pro_1_inpaint;
mod handle_flux_pro_1_inpaint_artcraft;
@@ -0,0 +1,3 @@
pub mod enqueue_image_inpaint_command;
pub mod errors;
mod flux_pro_1_inpaint;
@@ -1,6 +1,7 @@
pub mod image;
pub mod image_bg_removal;
pub mod image_edit;
pub mod task_enqueue_success;
pub mod image_inpaint;
pub mod object;
pub mod task_enqueue_success;
pub mod video;
@@ -4,6 +4,7 @@ use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_model_type::TaskModelType;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use sqlite_tasks::error::SqliteTasksError;
use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs};
use tokens::tokens::sqlite::tasks::TaskId;
@@ -35,13 +36,31 @@ impl TaskEnqueueSuccess{
}
pub async fn insert_into_task_database(&self, task_database: &TaskDatabase) -> Result<TaskId, SqliteTasksError> {
self.insert_into_task_database_with_frontend_payload(
task_database,
None,
None,
None,
).await
}
// TODO: This belongs somewhere else, not as a method of an event struct.
pub async fn insert_into_task_database_with_frontend_payload(
&self,
task_database: &TaskDatabase,
frontend_caller: Option<TauriCommandCaller>,
frontend_subscriber_id: Option<&str>,
frontend_subscriber_payload: Option<&str>,
) -> Result<TaskId, SqliteTasksError> {
// TODO: Move this mapping elsewhere, or remove the other models.
let model_type = match self.model {
None => None,
Some(GenerationModel::Flux1Dev) => Some(TaskModelType::Flux1Dev),
Some(GenerationModel::Flux1Schnell) => Some(TaskModelType::Flux1Schnell),
Some(GenerationModel::FluxPro1) => Some(TaskModelType::FluxPro1), // NB: This is for inpainting.
Some(GenerationModel::FluxPro11) => Some(TaskModelType::FluxPro11),
Some(GenerationModel::FluxPro11Ultra) => Some(TaskModelType::FluxPro11Ultra),
Some(GenerationModel::FluxProKontextMax) => Some(TaskModelType::FluxProKontextMax),
Some(GenerationModel::GptImage1) => Some(TaskModelType::GptImage1),
Some(GenerationModel::Recraft3) => Some(TaskModelType::Recraft3),
Some(GenerationModel::Kling21Pro) => Some(TaskModelType::Kling21Pro),
@@ -61,23 +80,12 @@ impl TaskEnqueueSuccess{
db: task_database.get_connection(),
status: TaskStatus::Pending,
task_type: self.task_type,
model_type: model_type,
model_type,
provider: self.provider,
provider_job_id: self.provider_job_id.as_deref(),
frontend_subscriber_id: None,
frontend_subscriber_payload: None,
frontend_caller,
frontend_subscriber_id,
frontend_subscriber_payload,
}).await
}
// pub fn tauri_event_model(&self) -> GenerationModel {
// match self.model {
// ImageModel::Flux1Dev => GenerationModel::Flux1Dev,
// ImageModel::Flux1Schnell => GenerationModel::Flux1Schnell,
// ImageModel::FluxPro11 => GenerationModel::FluxPro11,
// ImageModel::FluxPro11Ultra => GenerationModel::FluxPro11Ultra,
// ImageModel::GptImage1 => GenerationModel::GptImage1,
// ImageModel::Recraft3 => GenerationModel::Recraft3,
// }
// }
}
@@ -1,4 +1,5 @@
use crate::core::events::sendable_event_error::SendableEventError;
use enums::tauri::ux::tauri_event_name::TauriEventName;
use log::info;
use serde::Serialize;
use std::fmt::Debug;
@@ -29,7 +30,7 @@ struct BasicEventWrapper<T: Serialize + Debug> {
pub trait BasicSendableEvent : Clone + Debug + Serialize {
/// This is the name of the event that the frontend subscribes to.
const FRONTEND_EVENT_NAME: &'static str;
const FRONTEND_EVENT_NAME: TauriEventName;
/// This is the status of the event: Success, Failure, etc.
const EVENT_STATUS: BasicEventStatus;
@@ -41,8 +42,8 @@ pub trait BasicSendableEvent : Clone + Debug + Serialize {
status: Self::EVENT_STATUS,
data: self.clone(),
};
info!("Sending event to frontend: {} ; payload = {:?}", Self::FRONTEND_EVENT_NAME, wrapped);
let result = app.emit(Self::FRONTEND_EVENT_NAME, wrapped);
info!("Sending event to frontend: {} ; payload = {:?}", Self::FRONTEND_EVENT_NAME.to_str(), wrapped);
let result = app.emit(Self::FRONTEND_EVENT_NAME.to_str(), wrapped);
if let Err(err) = result {
return Err(SendableEventError::from(err));
}
@@ -0,0 +1,19 @@
use crate::core::events::basic_sendable_event_trait::{BasicEventStatus, BasicSendableEvent};
use enums::tauri::ux::tauri_event_name::TauriEventName;
use serde_derive::Serialize;
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub struct CanvasBackgroundRemovalCompleteEvent {
pub media_token: MediaFileToken,
pub image_cdn_url: Url,
pub maybe_frontend_subscriber_id: Option<String>,
pub maybe_frontend_subscriber_payload: Option<String>,
}
impl BasicSendableEvent for CanvasBackgroundRemovalCompleteEvent {
const FRONTEND_EVENT_NAME: TauriEventName = TauriEventName::CanvasBgRemovedEvent;
const EVENT_STATUS: BasicEventStatus = BasicEventStatus::Success;
}
@@ -0,0 +1 @@
pub mod canvas_background_removal_complete_event;
@@ -15,10 +15,14 @@ pub enum GenerationModel {
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
#[serde(rename = "flux_pro_1")]
FluxPro1,
#[serde(rename = "flux_pro_1.1")]
FluxPro11,
#[serde(rename = "flux_pro_1.1_ultra")]
FluxPro11Ultra,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "hunyuan_3d_2_0")]
@@ -1,5 +1,6 @@
use crate::core::events::basic_sendable_event_trait::{BasicEventStatus, BasicSendableEvent};
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use enums::tauri::ux::tauri_event_name::TauriEventName;
use serde_derive::Serialize;
#[derive(Clone, Debug, Serialize)]
@@ -11,6 +12,6 @@ pub struct GenerationCompleteEvent {
}
impl BasicSendableEvent for GenerationCompleteEvent {
const FRONTEND_EVENT_NAME: &'static str = "generation-complete-event";
const FRONTEND_EVENT_NAME: TauriEventName = TauriEventName::GenerationCompleteEvent;
const EVENT_STATUS: BasicEventStatus = BasicEventStatus::Success;
}
@@ -1,5 +1,6 @@
use crate::core::events::basic_sendable_event_trait::{BasicEventStatus, BasicSendableEvent};
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use enums::tauri::ux::tauri_event_name::TauriEventName;
use serde_derive::Serialize;
#[derive(Clone, Debug, Serialize)]
@@ -14,7 +15,7 @@ pub struct GenerationEnqueueFailureEvent {
}
impl BasicSendableEvent for GenerationEnqueueFailureEvent {
const FRONTEND_EVENT_NAME: &'static str = "generation-enqueue-failure-event";
const FRONTEND_EVENT_NAME: TauriEventName = TauriEventName::GenerationEnqueueFailureEvent;
const EVENT_STATUS: BasicEventStatus = BasicEventStatus::Failure;
}
@@ -1,5 +1,6 @@
use crate::core::events::basic_sendable_event_trait::{BasicEventStatus, BasicSendableEvent};
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use enums::tauri::ux::tauri_event_name::TauriEventName;
use serde_derive::Serialize;
#[derive(Clone, Debug, Serialize)]
@@ -11,6 +12,6 @@ pub struct GenerationEnqueueSuccessEvent {
}
impl BasicSendableEvent for GenerationEnqueueSuccessEvent {
const FRONTEND_EVENT_NAME: &'static str = "generation-enqueue-success-event";
const FRONTEND_EVENT_NAME: TauriEventName = TauriEventName::GenerationEnqueueSuccessEvent;
const EVENT_STATUS: BasicEventStatus = BasicEventStatus::Success;
}
@@ -1,5 +1,6 @@
use crate::core::events::basic_sendable_event_trait::{BasicEventStatus, BasicSendableEvent};
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use enums::tauri::ux::tauri_event_name::TauriEventName;
use serde_derive::Serialize;
#[derive(Clone, Debug, Serialize)]
@@ -14,6 +15,6 @@ pub struct GenerationFailedEvent {
}
impl BasicSendableEvent for GenerationFailedEvent {
const FRONTEND_EVENT_NAME: &'static str = "generation-failed-event";
const FRONTEND_EVENT_NAME: TauriEventName = TauriEventName::GenerationFailedEvent;
const EVENT_STATUS: BasicEventStatus = BasicEventStatus::Failure;
}
@@ -1,4 +1,5 @@
pub mod basic_sendable_event_trait;
pub mod generation_events;
pub mod sendable_event_error;
pub mod sendable_event_trait;
pub mod sendable_event_trait;
pub mod functional_events;
@@ -7,6 +7,9 @@ use serde_derive::Deserialize;
pub enum ContextualImageEditModel {
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
// TODO: Flux Kontext models.
}
+2
View File
@@ -7,6 +7,7 @@ use crate::core::commands::app_preferences::get_app_preferences_command::get_app
use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command;
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::enqueue_text_to_image_command;
use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::enqueue_image_bg_removal_command;
use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::enqueue_image_inpaint_command;
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::enqueue_contextual_edit_image_command;
use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::enqueue_image_to_3d_object_command;
use crate::core::commands::enqueue::video::enqueue_image_to_video_command::enqueue_image_to_video_command;
@@ -144,6 +145,7 @@ pub fn run() {
check_sora_session_command,
enqueue_contextual_edit_image_command,
enqueue_image_bg_removal_command,
enqueue_image_inpaint_command,
enqueue_image_to_3d_object_command,
enqueue_image_to_video_command,
enqueue_text_to_image_command,
@@ -29,7 +29,7 @@ use reqwest::Url;
use serde_derive::{Deserialize, Serialize};
use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
@@ -256,11 +256,12 @@ pub async fn remove_background(
info!("Uploading image media file...");
let upload_result = upload_image_media_file_from_file(
&app_env_configs.storyteller_host,
Some(&creds),
result_filename,
).await?;
let upload_result = upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&creds),
path: result_filename,
is_intermediate_system_file: true, // NB: Probably not essential to keep this.
}).await?;
// TODO: Don't re-request to simply build MediaLinks (or CDN URL). Get those from the upload API in one turn.
info!("Re-requesting media file...");
@@ -17,7 +17,7 @@ use reqwest::Url;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
use tauri::AppHandle;
use tempdir::TempDir;
@@ -134,12 +134,14 @@ async fn polling_loop(
let download_path = download_generation(generation, &app_data_root).await?;
info!("Uploading to backend...");
let result = upload_image_media_file_from_file(
&app_env_configs.storyteller_host,
Some(&creds),
download_path
).await?;
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&creds),
path: download_path,
is_intermediate_system_file: false,
}).await?;
info!("Uploaded to API backend: {:?}", result.media_file_token);
let event = GenerationCompleteEvent {
@@ -0,0 +1,48 @@
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::functional_events::canvas_background_removal_complete_event::CanvasBackgroundRemovalCompleteEvent;
use crate::core::events::generation_events::generation_complete_event::GenerationCompleteEvent;
use artcraft_api_defs::jobs::list_session_jobs::ListSessionJobsItem;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use errors::AnyhowResult;
use log::error;
use sqlite_tasks::queries::list_tasks_by_provider_and_tokens::{list_tasks_by_provider_and_tokens, ListTasksArgs, Task};
use tokens::tokens::media_files::MediaFileToken;
pub async fn maybe_send_background_removal_complete_event(
app: &tauri::AppHandle,
task: &Task,
job: &ListSessionJobsItem,
) -> AnyhowResult<()> {
match task.task_type {
TaskType::BackgroundRemoval => {} // NB: Fall-through
_ => return Ok(()),
}
match task.frontend_caller {
Some(TauriCommandCaller::Canvas) => {} // NB: Fall-through
_ => return Ok(()),
}
let result = match job.maybe_result {
Some(ref res) => res,
None => {
error!("Job result is None for task: {:?}", task);
return Ok(()); // No result, nothing to do
},
};
let event = CanvasBackgroundRemovalCompleteEvent {
media_token: MediaFileToken::new_from_str(&result.entity_token),
image_cdn_url: result.media_links.cdn_url.clone(),
maybe_frontend_subscriber_id: task.frontend_subscriber_id.clone(),
maybe_frontend_subscriber_payload: task.frontend_subscriber_payload.clone(),
};
if let Err(err) = event.send(&app) {
error!("Failed to send CanvasBackgroundRemovalCompleteEvent: {:?}", err); // Fail open
}
Ok(())
}
@@ -0,0 +1 @@
pub mod maybe_send_background_removal_complete_event;
@@ -1 +1,2 @@
pub mod events;
pub mod storyteller_task_polling_thread;
@@ -12,10 +12,12 @@ use crate::core::utils::enum_conversion::task_type::to_generation_action;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use crate::services::storyteller::threads::events::maybe_send_background_removal_complete_event::maybe_send_background_removal_complete_event;
use anyhow::anyhow;
use enums::common::generation_provider::GenerationProvider;
use enums::common::job_status_plus::JobStatusPlus;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use errors::AnyhowResult;
use log::{error, info};
use reqwest::Url;
@@ -137,7 +139,17 @@ async fn polling_loop(
if !updated {
continue; // If anything breaks with queries, don't spam events.
}
let result = maybe_send_background_removal_complete_event(
&app_handle,
&task,
&job,
).await;
if let Err(err) = result {
error!("Failed to send background removal complete event: {:?}", err);
}
let service = to_generation_service_provider(task.provider);
let action = to_generation_action(task.task_type);
@@ -0,0 +1,18 @@
use crate::error::ImagesError;
use image::{ImageFormat, ImageReader};
use std::io::Cursor;
pub fn image_bytes_to_png_bytes(
arbitrary_image_bytes: &[u8],
) -> Result<Vec<u8>, ImagesError> {
let reader = ImageReader::new(Cursor::new(arbitrary_image_bytes));
let image = reader
.with_guessed_format()? // NB: Can raise IoError.
.decode()?;
let mut output_bytes: Vec<u8> = Vec::new();
image.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png)?;
Ok(output_bytes)
}
@@ -0,0 +1,29 @@
use std::io::Cursor;
use image::{ImageFormat, ImageReader};
use crate::error::ImagesError;
pub struct PngBytesWithDimensions {
pub png_bytes: Vec<u8>,
pub width: u32,
pub height: u32,
}
pub fn image_bytes_to_png_bytes_with_dimensions(
arbitrary_image_bytes: &[u8],
) -> Result<PngBytesWithDimensions, ImagesError> {
let reader = ImageReader::new(Cursor::new(arbitrary_image_bytes));
let image = reader
.with_guessed_format()? // NB: Can raise IoError.
.decode()?;
let mut output_bytes: Vec<u8> = Vec::new();
image.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png)?;
Ok(PngBytesWithDimensions {
png_bytes: output_bytes,
width: image.width(),
height: image.height(),
})
}
@@ -1 +1,3 @@
pub mod image_bytes_to_png_bytes;
pub mod image_bytes_to_png_bytes_with_dimensions;
pub mod webp_bytes_to_png_bytes;
+1
View File
@@ -26,6 +26,7 @@ pub use image;
pub mod encoding;
pub mod error;
pub mod image_info;
pub mod mask_images;
pub mod resize_image_file_preserving_aspect;
pub mod resize_preserving_aspect;
@@ -0,0 +1 @@
pub mod normalize_image_bytes_to_flux_mask;
@@ -0,0 +1,40 @@
use crate::error::ImagesError;
use image::{GenericImage, GenericImageView, ImageFormat, ImageReader, Rgba};
use std::io::Cursor;
pub struct PngBytes(pub Vec<u8>);
/// Normalize arbitrary image bytes to a PNG format image.
/// Make sure the bitmap is in the correct mask format.
pub fn normalize_image_bytes_to_flux_mask(
arbitrary_image_bytes: &[u8],
) -> Result<PngBytes, ImagesError> {
let reader = ImageReader::new(Cursor::new(arbitrary_image_bytes));
let mut image = reader
.with_guessed_format()? // NB: Can raise IoError.
.decode()?;
let (width, height) = image.dimensions();
for y in 0..height {
for x in 0..width {
let rgba_pixel = image.get_pixel(x, y);
if rgba_pixel[3] == 0 {
// Alpha pixel is transparent
// Set to black. Mask ignore region.
image.put_pixel(x, y, Rgba([0, 0, 0, 255]));
} else {
// Set to white. Mask interest region.
image.put_pixel(x, y, Rgba([255, 255, 255, 255]));
}
}
}
let mut output_bytes: Vec<u8> = Vec::new();
image.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png)?;
Ok(PngBytes(output_bytes))
}
@@ -23,3 +23,12 @@ pub const WHITE_HOUSE_SUNSET_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media
// media token: m_z61yg5w7dnnvhb1f9sqt8jf02y61bv
pub const WIDE_FALL_MOUNTAINS_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/w/6/m/j/p/w6mjpcqkky7f2f3xkyvkcv3qeedkmejs/asset_w6mjpcqkky7f2f3xkyvkcv3qeedkmejs.jpg";
// media token: m_m1bz02z1kkzanxy6rb4vk1kvq9de9g
pub const JUNO_AT_LAKE_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/s/v/c/p/p/svcpp441gyb6mjtt8413kf5zxw1ehy4t/upload_svcpp441gyb6mjtt8413kf5zxw1ehy4t.jpg";
// media token: m_5y81rcnpnrbey55da6cc2s9z4d2f45
pub const TALL_MOCHI_WITH_GLASSES_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/n/m/f/h/a/nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y/upload_nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y.jpg";
// media token: m_7x18h6qnyay4a42kyyyw2xctasx5k9
pub const TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/r/e/j/b/p/rejbp61m56hqx4a498xh7j47vkjd7jnh/upload_rejbp61m56hqx4a498xh7j47vkjd7jnh.png";
@@ -1,4 +1,6 @@
use errors::AnyhowResult;
use log::{error, info};
use sqlx::migrate::MigrateError;
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode};
use sqlx::{Pool, Sqlite, SqlitePool};
use std::path::Path;
@@ -10,19 +12,20 @@ pub struct TaskDbConnection {
impl TaskDbConnection {
pub async fn connect_and_migrate<P: AsRef<Path>>(database_file: P) -> AnyhowResult<Self> {
let connection_options = SqliteConnectOptions::new()
.filename(database_file)
.create_if_missing(true)
.journal_mode(SqliteJournalMode::Wal);
match run_migrations(&database_file).await {
Ok(pool) => return Ok(Self { pool }),
Err(err) => {
error!("Error running SQLite migrations: {:?}", err);
}
}
let pool = SqlitePool::connect_with(connection_options).await?;
info!("Deleting and recreating SQLite database at {:?}", database_file.as_ref());
// Run migrations regardless of whether the database is new, SQLx will track which migrations
// have been run.
// The migrations text get compiled into the binary, so no worries about build inclusion.
// Since the task database is being treated as ephemeral, we can always run migrations without
// worrying about previous state if we simply blow away old versions of the schema.
sqlx::migrate!("../../../../_sql/artcraft_migrations").run(&pool).await?;
if let Err(err) = std::fs::remove_file(&database_file) {
error!("Error deleting SQLite database file: {:?}", err);
}
let pool = run_migrations(database_file).await?;
Ok(Self { pool })
}
@@ -31,3 +34,21 @@ impl TaskDbConnection {
&self.pool
}
}
async fn run_migrations<P: AsRef<Path>>(database_file: P) -> Result<SqlitePool, MigrateError> {
let connection_options = SqliteConnectOptions::new()
.filename(database_file)
.create_if_missing(true)
.journal_mode(SqliteJournalMode::Wal);
let pool = SqlitePool::connect_with(connection_options).await?;
// Run migrations regardless of whether the database is new, SQLx will track which migrations
// have been run.
// The migrations text get compiled into the binary, so no worries about build inclusion.
// Since the task database is being treated as ephemeral, we can always run migrations without
// worrying about previous state if we simply blow away old versions of the schema.
sqlx::migrate!("../../../../_sql/artcraft_migrations").run(&pool).await?;
Ok(pool)
}
@@ -4,6 +4,7 @@ use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_model_type::TaskModelType;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use tokens::tokens::sqlite::tasks::TaskId;
pub struct CreateTaskArgs<'a> {
@@ -13,6 +14,7 @@ pub struct CreateTaskArgs<'a> {
pub model_type: Option<TaskModelType>,
pub provider: GenerationProvider,
pub provider_job_id: Option<&'a str>,
pub frontend_caller: Option<TauriCommandCaller>,
pub frontend_subscriber_id: Option<&'a str>,
pub frontend_subscriber_payload: Option<&'a str>,
}
@@ -28,6 +30,7 @@ pub async fn create_task(
let task_type_temp = args.task_type.to_str();
let model_type_temp = args.model_type.map(|s| s.to_str());
let provider_temp = args.provider.to_string();
let frontend_caller_temp = args.frontend_caller.map(|s| s.to_str());
let query = sqlx::query!(r#"
INSERT INTO tasks (
@@ -37,10 +40,11 @@ pub async fn create_task(
model_type,
provider,
provider_job_id,
frontend_caller,
frontend_subscriber_id,
frontend_subscriber_payload
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
"#,
task_id_temp,
status_temp,
@@ -48,6 +52,7 @@ pub async fn create_task(
model_type_temp,
provider_temp,
args.provider_job_id,
frontend_caller_temp,
args.frontend_subscriber_id,
args.frontend_subscriber_payload
);
@@ -4,6 +4,7 @@ use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_model_type::TaskModelType;
use enums::tauri::tasks::task_status::TaskStatus;
use enums::tauri::tasks::task_type::TaskType;
use enums::tauri::ux::tauri_command_caller::TauriCommandCaller;
use sqlx::{QueryBuilder, Sqlite};
use tokens::tokens::sqlite::tasks::TaskId;
@@ -22,9 +23,10 @@ pub struct Task {
pub id: TaskId,
pub status: TaskStatus,
pub task_type: TaskType,
pub model_type: TaskModelType,
pub model_type: Option<TaskModelType>,
pub provider: GenerationProvider,
pub provider_job_id: Option<String>,
pub frontend_caller: Option<TauriCommandCaller>,
pub frontend_subscriber_id: Option<String>,
pub frontend_subscriber_payload: Option<String>,
}
@@ -35,9 +37,10 @@ struct RawTask {
id: String,
task_status: String,
task_type: String,
model_type: String,
model_type: Option<String>,
provider: String,
provider_job_id: Option<String>,
frontend_caller: Option<String>,
frontend_subscriber_id: Option<String>,
frontend_subscriber_payload: Option<String>,
}
@@ -54,6 +57,7 @@ pub async fn list_tasks_by_provider_and_tokens(
model_type,
provider,
provider_job_id,
frontend_caller,
frontend_subscriber_id,
frontend_subscriber_payload
FROM tasks
@@ -91,11 +95,17 @@ pub async fn list_tasks_by_provider_and_tokens(
.map_err(|err| SqliteTasksError::TaskParseError(err))?,
task_type: TaskType::from_str(&task.task_type)
.map_err(|err| SqliteTasksError::TaskParseError(err))?,
model_type: TaskModelType::from_str(&task.model_type)
.map_err(|err| SqliteTasksError::TaskParseError(err))?,
model_type: task.model_type
.map(|model| TaskModelType::from_str(&model)
.map_err(|err| SqliteTasksError::TaskParseError(err)))
.transpose()?,
provider: GenerationProvider::from_str(&task.provider)
.map_err(|err| SqliteTasksError::TaskParseError(err))?,
provider_job_id: task.provider_job_id,
frontend_caller: task.frontend_caller
.map(|caller| TauriCommandCaller::from_str(&caller)
.map_err(|err| SqliteTasksError::TaskParseError(err)))
.transpose()?,
frontend_subscriber_id: task.frontend_subscriber_id,
frontend_subscriber_payload: task.frontend_subscriber_payload,
});
@@ -0,0 +1,36 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const FLUX_PRO_KONTEXT_MAX_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/flux_pro_kontext_max";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxProKontextMaxEditImageRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// The image we're editing.
pub image_media_token: MediaFileToken,
/// Number of images to generate. Default is one.
pub num_images: Option<FluxProKontextMaxEditImageNumImages>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum FluxProKontextMaxEditImageNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxProKontextMaxEditImageResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -1,2 +1,3 @@
pub mod flux_pro_kontext_max_edit_image;
pub mod gpt_image_1_edit_image;
@@ -0,0 +1,39 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const FLUX_PRO_1_INPAINT_PATH: &str = "/v1/generate/image/inpaint/flux_pro_1";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxPro1InpaintImageRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// The image we're editing.
pub image_media_token: MediaFileToken,
/// The mask to use against the image.
pub mask_media_token: MediaFileToken,
/// Number of images to generate. Default is one.
pub num_images: Option<FluxPro1InpaintImageNumImages>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum FluxPro1InpaintImageNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxPro1InpaintImageResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1 @@
pub mod flux_pro_1_inpaint_image;
@@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image;
pub mod generate_flux_pro_11_text_to_image;
pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod remove_image_background;
@@ -19,6 +19,12 @@ pub enum PromptContextSemanticType {
/// Image-to-Video ending frame
VidEndFrame,
/// Source image, eg. for inpainting.
Imgsrc,
/// Image mask, eg. for inpainting.
Imgmask,
/// Standard image reference without a semantic type (e.g. Sora/ChatGPT 4o/gpt-image-1)
Imgref,
@@ -38,6 +44,8 @@ impl PromptContextSemanticType {
match self {
Self::VidStartFrame => "vid_start_frame",
Self::VidEndFrame => "vid_end_frame",
Self::Imgsrc => "imgsrc",
Self::Imgmask => "imgmask",
Self::Imgref => "imgref",
Self::ImgrefCharacter => "imgref_character",
Self::ImgrefStyle => "imgref_style",
@@ -49,6 +57,8 @@ impl PromptContextSemanticType {
match value {
"vid_start_frame" => Ok(Self::VidStartFrame),
"vid_end_frame" => Ok(Self::VidEndFrame),
"imgsrc" => Ok(Self::Imgsrc),
"imgmask" => Ok(Self::Imgmask),
"imgref" => Ok(Self::Imgref),
"imgref_character" => Ok(Self::ImgrefCharacter),
"imgref_style" => Ok(Self::ImgrefStyle),
@@ -63,6 +73,8 @@ impl PromptContextSemanticType {
BTreeSet::from([
Self::VidStartFrame,
Self::VidEndFrame,
Self::Imgsrc,
Self::Imgmask,
Self::Imgref,
Self::ImgrefCharacter,
Self::ImgrefStyle,
@@ -83,6 +95,8 @@ mod tests {
fn test_serialization() {
assert_serialization(PromptContextSemanticType::VidStartFrame, "vid_start_frame");
assert_serialization(PromptContextSemanticType::VidEndFrame, "vid_end_frame");
assert_serialization(PromptContextSemanticType::Imgsrc, "imgsrc");
assert_serialization(PromptContextSemanticType::Imgmask, "imgmask");
assert_serialization(PromptContextSemanticType::Imgref, "imgref");
assert_serialization(PromptContextSemanticType::ImgrefCharacter, "imgref_character");
assert_serialization(PromptContextSemanticType::ImgrefStyle, "imgref_style");
@@ -97,6 +111,8 @@ mod tests {
fn to_str() {
assert_eq!(PromptContextSemanticType::VidStartFrame.to_str(), "vid_start_frame");
assert_eq!(PromptContextSemanticType::VidEndFrame.to_str(), "vid_end_frame");
assert_eq!(PromptContextSemanticType::Imgsrc.to_str(), "imgsrc");
assert_eq!(PromptContextSemanticType::Imgmask.to_str(), "imgmask");
assert_eq!(PromptContextSemanticType::Imgref.to_str(), "imgref");
assert_eq!(PromptContextSemanticType::ImgrefCharacter.to_str(), "imgref_character");
assert_eq!(PromptContextSemanticType::ImgrefStyle.to_str(), "imgref_style");
@@ -107,6 +123,8 @@ mod tests {
fn from_str() {
assert_eq!(PromptContextSemanticType::from_str("vid_start_frame").unwrap(), PromptContextSemanticType::VidStartFrame);
assert_eq!(PromptContextSemanticType::from_str("vid_end_frame").unwrap(), PromptContextSemanticType::VidEndFrame);
assert_eq!(PromptContextSemanticType::from_str("imgsrc").unwrap(), PromptContextSemanticType::Imgsrc);
assert_eq!(PromptContextSemanticType::from_str("imgmask").unwrap(), PromptContextSemanticType::Imgmask);
assert_eq!(PromptContextSemanticType::from_str("imgref").unwrap(), PromptContextSemanticType::Imgref);
assert_eq!(PromptContextSemanticType::from_str("imgref_character").unwrap(), PromptContextSemanticType::ImgrefCharacter);
assert_eq!(PromptContextSemanticType::from_str("imgref_style").unwrap(), PromptContextSemanticType::ImgrefStyle);
@@ -121,9 +139,11 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = PromptContextSemanticType::all_variants();
assert_eq!(variants.len(), 6);
assert_eq!(variants.len(), 8);
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidStartFrame));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidEndFrame));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgsrc));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgmask));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgref));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefCharacter));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefStyle));
@@ -37,7 +37,7 @@ impl GenerationProvider {
"artcraft" => Ok(Self::Artcraft),
"fal" => Ok(Self::Fal),
"sora" => Ok(Self::Sora),
_ => Err(format!("invalid task_state: {:?}", job_status)),
_ => Err(format!("invalid generation_provider: {:?}", job_status)),
}
}
@@ -17,15 +17,23 @@ pub enum ModelType {
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
#[serde(rename = "flux_pro_1")]
FluxPro1,
#[serde(rename = "flux_pro_1p1")]
FluxPro11,
#[serde(rename = "flux_pro_1p1_ultra")]
FluxPro11Ultra,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "recraft_3")]
Recraft3,
//// Image Infill models
//#[serde(rename = "flux_pro_1_infill")]
//FluxPro1Infill,
// Video models
#[serde(rename = "kling_1p6_pro")]
Kling16Pro,
@@ -57,8 +65,10 @@ impl ModelType {
// Image models
Self::Flux1Dev => "flux_1_dev",
Self::Flux1Schnell => "flux_1_schnell",
Self::FluxPro1 => "flux_pro_1",
Self::FluxPro11 => "flux_pro_1p1",
Self::FluxPro11Ultra => "flux_pro_1p1_ultra",
Self::FluxProKontextMax => "flux_pro_kontext_max",
Self::GptImage1 => "gpt_image_1",
Self::Recraft3 => "recraft_3",
@@ -80,8 +90,10 @@ impl ModelType {
// Image models
"flux_1_dev" => Ok(Self::Flux1Dev),
"flux_1_schnell" => Ok(Self::Flux1Schnell),
"flux_pro_1" => Ok(Self::FluxPro1),
"flux_pro_1p1" => Ok(Self::FluxPro11),
"flux_pro_1p1_ultra" => Ok(Self::FluxPro11Ultra),
"flux_pro_kontext_max" => Ok(Self::FluxProKontextMax),
"gpt_image_1" => Ok(Self::GptImage1),
"recraft_3" => Ok(Self::Recraft3),
@@ -96,7 +108,7 @@ impl ModelType {
"hunyuan_3d_2p0" => Ok(Self::Hunyuan3d2_0),
"hunyuan_3d_2p1" => Ok(Self::Hunyuan3d2_1),
_ => Err(format!("invalid task_state: {:?}", job_status)),
_ => Err(format!("invalid model_type: {:?}", job_status)),
}
}
@@ -107,8 +119,10 @@ impl ModelType {
// Image models
Self::Flux1Dev,
Self::Flux1Schnell,
Self::FluxPro1,
Self::FluxPro11,
Self::FluxPro11Ultra,
Self::FluxProKontextMax,
Self::GptImage1,
Self::Recraft3,
@@ -139,8 +153,10 @@ mod tests {
// Image models
assert_serialization(ModelType::Flux1Dev, "flux_1_dev");
assert_serialization(ModelType::Flux1Schnell, "flux_1_schnell");
assert_serialization(ModelType::FluxPro1, "flux_pro_1");
assert_serialization(ModelType::FluxPro11, "flux_pro_1p1");
assert_serialization(ModelType::FluxPro11Ultra, "flux_pro_1p1_ultra");
assert_serialization(ModelType::FluxProKontextMax, "flux_pro_kontext_max");
assert_serialization(ModelType::GptImage1, "gpt_image_1");
assert_serialization(ModelType::Recraft3, "recraft_3");
// Video models
@@ -159,8 +175,10 @@ mod tests {
// Image models
assert_eq!(ModelType::Flux1Dev.to_str(), "flux_1_dev");
assert_eq!(ModelType::Flux1Schnell.to_str(), "flux_1_schnell");
assert_eq!(ModelType::FluxPro1.to_str(), "flux_pro_1");
assert_eq!(ModelType::FluxPro11.to_str(), "flux_pro_1p1");
assert_eq!(ModelType::FluxPro11Ultra.to_str(), "flux_pro_1p1_ultra");
assert_eq!(ModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max");
assert_eq!(ModelType::GptImage1.to_str(), "gpt_image_1");
assert_eq!(ModelType::Recraft3.to_str(), "recraft_3");
@@ -181,8 +199,10 @@ mod tests {
// Image models
assert_eq!(ModelType::from_str("flux_1_dev").unwrap(), ModelType::Flux1Dev);
assert_eq!(ModelType::from_str("flux_1_schnell").unwrap(), ModelType::Flux1Schnell);
assert_eq!(ModelType::from_str("flux_pro_1").unwrap(), ModelType::FluxPro1);
assert_eq!(ModelType::from_str("flux_pro_1p1").unwrap(), ModelType::FluxPro11);
assert_eq!(ModelType::from_str("flux_pro_1p1_ultra").unwrap(), ModelType::FluxPro11Ultra);
assert_eq!(ModelType::from_str("flux_pro_kontext_max").unwrap(), ModelType::FluxProKontextMax);
assert_eq!(ModelType::from_str("gpt_image_1").unwrap(), ModelType::GptImage1);
assert_eq!(ModelType::from_str("recraft_3").unwrap(), ModelType::Recraft3);
// Video models
@@ -199,12 +219,14 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = ModelType::all_variants();
assert_eq!(variants.len(), 13);
assert_eq!(variants.len(), 15);
// Image models
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro1));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11Ultra));
assert_eq!(variants.pop_first(), Some(ModelType::FluxProKontextMax));
assert_eq!(variants.pop_first(), Some(ModelType::GptImage1));
assert_eq!(variants.pop_first(), Some(ModelType::Recraft3));
// Video models
+2 -1
View File
@@ -1 +1,2 @@
pub mod tasks;
pub mod tasks;
pub mod ux;
@@ -15,10 +15,15 @@ pub enum TaskModelType {
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
// NB: For inpainting for now
#[serde(rename = "flux_pro_1")]
FluxPro1,
#[serde(rename = "flux_pro_1.1")]
FluxPro11,
#[serde(rename = "flux_pro_1.1_ultra")]
FluxPro11Ultra,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "recraft_3")]
@@ -55,8 +60,10 @@ impl TaskModelType {
// Image models
Self::Flux1Dev => "flux_1_dev",
Self::Flux1Schnell => "flux_1_schnell",
Self::FluxPro1 => "flux_pro_1",
Self::FluxPro11 => "flux_pro_1.1",
Self::FluxPro11Ultra => "flux_pro_1.1_ultra",
Self::FluxProKontextMax => "flux_pro_kontext_max",
Self::GptImage1 => "gpt_image_1",
Self::Recraft3 => "recraft_3",
// Video models
@@ -76,8 +83,10 @@ impl TaskModelType {
// Image models
"flux_1_dev" => Ok(Self::Flux1Dev),
"flux_1_schnell" => Ok(Self::Flux1Schnell),
"flux_pro_1" => Ok(Self::FluxPro1),
"flux_pro_1.1" => Ok(Self::FluxPro11),
"flux_pro_1.1_ultra" => Ok(Self::FluxPro11Ultra),
"flux_pro_kontext_max" => Ok(Self::FluxProKontextMax),
"gpt_image_1" => Ok(Self::GptImage1),
"recraft_3" => Ok(Self::Recraft3),
// Video models
@@ -89,7 +98,7 @@ impl TaskModelType {
// 3D Object generation models
"hunyuan_3d_2.0" => Ok(Self::Hunyuan3d2_0),
"hunyuan_3d_2.1" => Ok(Self::Hunyuan3d2_1),
_ => Err(format!("invalid task_state: {:?}", job_status)),
_ => Err(format!("invalid task_model_type: {:?}", job_status)),
}
}
@@ -100,8 +109,10 @@ impl TaskModelType {
// Image models
Self::Flux1Dev,
Self::Flux1Schnell,
Self::FluxPro1,
Self::FluxPro11,
Self::FluxPro11Ultra,
Self::FluxProKontextMax,
Self::GptImage1,
Self::Recraft3,
// Video models
@@ -119,8 +130,8 @@ impl TaskModelType {
#[cfg(test)]
mod tests {
use crate::test_helpers::assert_serialization;
use crate::tauri::tasks::task_model_type::TaskModelType;
use crate::test_helpers::assert_serialization;
mod explicit_checks {
use super::*;
@@ -130,8 +141,10 @@ mod tests {
// Image models
assert_serialization(TaskModelType::Flux1Dev, "flux_1_dev");
assert_serialization(TaskModelType::Flux1Schnell, "flux_1_schnell");
assert_serialization(TaskModelType::FluxPro1, "flux_pro_1");
assert_serialization(TaskModelType::FluxPro11, "flux_pro_1.1");
assert_serialization(TaskModelType::FluxPro11Ultra, "flux_pro_1.1_ultra");
assert_serialization(TaskModelType::FluxProKontextMax, "flux_pro_kontext_max");
assert_serialization(TaskModelType::GptImage1, "gpt_image_1");
assert_serialization(TaskModelType::Recraft3, "recraft_3");
// Video models
@@ -150,8 +163,10 @@ mod tests {
// Image models
assert_eq!(TaskModelType::Flux1Dev.to_str(), "flux_1_dev");
assert_eq!(TaskModelType::Flux1Schnell.to_str(), "flux_1_schnell");
assert_eq!(TaskModelType::FluxPro1.to_str(), "flux_pro_1");
assert_eq!(TaskModelType::FluxPro11.to_str(), "flux_pro_1.1");
assert_eq!(TaskModelType::FluxPro11Ultra.to_str(), "flux_pro_1.1_ultra");
assert_eq!(TaskModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max");
assert_eq!(TaskModelType::GptImage1.to_str(), "gpt_image_1");
assert_eq!(TaskModelType::Recraft3.to_str(), "recraft_3");
// Video models
@@ -170,8 +185,10 @@ mod tests {
// Image models
assert_eq!(TaskModelType::from_str("flux_1_dev").unwrap(), TaskModelType::Flux1Dev);
assert_eq!(TaskModelType::from_str("flux_1_schnell").unwrap(), TaskModelType::Flux1Schnell);
assert_eq!(TaskModelType::from_str("flux_pro_1").unwrap(), TaskModelType::FluxPro1);
assert_eq!(TaskModelType::from_str("flux_pro_1.1").unwrap(), TaskModelType::FluxPro11);
assert_eq!(TaskModelType::from_str("flux_pro_1.1_ultra").unwrap(), TaskModelType::FluxPro11Ultra);
assert_eq!(TaskModelType::from_str("flux_pro_kontext_max").unwrap(), TaskModelType::FluxProKontextMax);
assert_eq!(TaskModelType::from_str("gpt_image_1").unwrap(), TaskModelType::GptImage1);
assert_eq!(TaskModelType::from_str("recraft_3").unwrap(), TaskModelType::Recraft3);
// Video models
@@ -188,12 +205,14 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = TaskModelType::all_variants();
assert_eq!(variants.len(), 13);
assert_eq!(variants.len(), 15);
// Image models
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell));
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro1));
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11));
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11Ultra));
assert_eq!(variants.pop_first(), Some(TaskModelType::FluxProKontextMax));
assert_eq!(variants.pop_first(), Some(TaskModelType::GptImage1));
assert_eq!(variants.pop_first(), Some(TaskModelType::Recraft3));
// Video models
@@ -59,7 +59,7 @@ impl TaskStatus {
"cancelled_by_user" => Ok(Self::CancelledByUser),
"cancelled_by_provider" => Ok(Self::CancelledByProvider),
"cancelled_by_us" => Ok(Self::CancelledByUs),
_ => Err(format!("invalid task_state: {:?}", job_status)),
_ => Err(format!("invalid task_status: {:?}", job_status)),
}
}
@@ -38,7 +38,7 @@ impl TaskType {
"video_generation" => Ok(Self::VideoGeneration),
"object_generation" => Ok(Self::ObjectGeneration),
"background_removal" => Ok(Self::BackgroundRemoval),
_ => Err(format!("invalid task_state: {:?}", job_status)),
_ => Err(format!("invalid task_type: {:?}", job_status)),
}
}
@@ -0,0 +1,2 @@
pub mod tauri_command_caller;
pub mod tauri_event_name;
@@ -0,0 +1,109 @@
use std::collections::BTreeSet;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Defines the names of the Tauri-sent events that the frontend subscribes to.
/// These event names are also stored in the database, so keep them short-ish.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum TauriCommandCaller {
/// The 2D canvas
Canvas,
}
impl_enum_display_and_debug_using_to_str!(TauriCommandCaller);
impl_mysql_enum_coders!(TauriCommandCaller);
impl_mysql_from_row!(TauriCommandCaller);
// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders`
impl TauriCommandCaller {
pub fn to_str(&self) -> &'static str {
match self {
Self::Canvas => "canvas",
}
}
pub fn from_str(job_status: &str) -> Result<Self, String> {
match job_status {
"canvas" => Ok(Self::Canvas),
_ => Err(format!("invalid tauri_command_caller: {:?}", job_status)),
}
}
pub fn all_variants() -> BTreeSet<Self> {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::Canvas,
])
}
}
#[cfg(test)]
mod tests {
use crate::test_helpers::assert_serialization;
use crate::tauri::ux::tauri_command_caller::TauriCommandCaller;
mod explicit_checks {
use super::*;
#[test]
fn test_serialization() {
assert_serialization(TauriCommandCaller::Canvas, "canvas");
}
#[test]
fn to_str() {
assert_eq!(TauriCommandCaller::Canvas.to_str(), "canvas");
}
#[test]
fn from_str() {
assert_eq!(TauriCommandCaller::from_str("canvas").unwrap(), TauriCommandCaller::Canvas);
}
#[test]
fn all_variants() {
let mut variants = TauriCommandCaller::all_variants();
assert_eq!(variants.len(), 1);
assert_eq!(variants.pop_first(), Some(TauriCommandCaller::Canvas));
assert_eq!(variants.pop_first(), None);
}
}
mod mechanical_checks {
use super::*;
use strum::IntoEnumIterator;
#[test]
fn variant_length() {
assert_eq!(TauriCommandCaller::all_variants().len(), TauriCommandCaller::iter().len());
}
#[test]
fn round_trip() {
for variant in TauriCommandCaller::all_variants() {
// Test to_str(), from_str(), Display, and Debug.
assert_eq!(variant, TauriCommandCaller::from_str(variant.to_str()).unwrap());
assert_eq!(variant, TauriCommandCaller::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, TauriCommandCaller::from_str(&format!("{:?}", variant)).unwrap());
}
}
//#[test]
//fn serialized_length_ok_for_database() {
// const MAX_LENGTH : usize = 16;
// for variant in TauriCommandCaller::all_variants() {
// let serialized = variant.to_str();
// assert!(serialized.len() > 0, "variant {:?} is too short", variant);
// assert!(serialized.len() <= MAX_LENGTH, "variant {:?} is too long", variant);
// }
//}
}
}
@@ -0,0 +1,168 @@
use std::collections::BTreeSet;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Defines the names of the Tauri-sent events that the frontend subscribes to.
/// These event names are also stored in the database, so keep them short-ish.
///
/// NB: Events should end in "_event" so they're easy to grep for in Javascript.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum TauriEventName {
// TODO: Get rid of kebab case.
/// General purpose event:
/// Generation enqueued
#[serde(rename = "generation-enqueue-success-event")]
GenerationEnqueueSuccessEvent,
// TODO: Get rid of kebab case.
/// General purpose event:
/// Generation failed to enqueue
#[serde(rename = "generation-enqueue-failure-event")]
GenerationEnqueueFailureEvent,
// TODO: Get rid of kebab case.
/// General purpose event:
/// Generation completed successfully
#[serde(rename = "generation-complete-event")]
GenerationCompleteEvent,
// TODO: Get rid of kebab case.
/// General purpose event:
/// Generation failed
#[serde(rename = "generation-failed-event")]
GenerationFailedEvent,
/// Special event:
/// Background removal complete
#[serde(rename = "canvas_bg_removed_event")]
CanvasBgRemovedEvent,
}
impl_enum_display_and_debug_using_to_str!(TauriEventName);
impl_mysql_enum_coders!(TauriEventName);
impl_mysql_from_row!(TauriEventName);
// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders`
impl TauriEventName {
pub fn to_str(&self) -> &'static str {
match self {
Self::GenerationEnqueueSuccessEvent => "generation-enqueue-success-event",
Self::GenerationEnqueueFailureEvent => "generation-enqueue-failure-event",
Self::GenerationCompleteEvent => "generation-complete-event",
Self::GenerationFailedEvent => "generation-failed-event",
Self::CanvasBgRemovedEvent => "canvas_bg_removed_event",
}
}
pub fn from_str(job_status: &str) -> Result<Self, String> {
match job_status {
"generation-enqueue-success-event" => Ok(Self::GenerationEnqueueSuccessEvent),
"generation-enqueue-failure-event" => Ok(Self::GenerationEnqueueFailureEvent),
"generation-complete-event" => Ok(Self::GenerationCompleteEvent),
"generation-failed-event" => Ok(Self::GenerationFailedEvent),
"canvas_bg_removed_event" => Ok(Self::CanvasBgRemovedEvent),
_ => Err(format!("invalid tauri_event_name: {:?}", job_status)),
}
}
pub fn all_variants() -> BTreeSet<Self> {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::GenerationEnqueueSuccessEvent,
Self::GenerationEnqueueFailureEvent,
Self::GenerationCompleteEvent,
Self::GenerationFailedEvent,
Self::CanvasBgRemovedEvent,
])
}
}
#[cfg(test)]
mod tests {
use crate::tauri::ux::tauri_event_name::TauriEventName;
use crate::test_helpers::assert_serialization;
mod explicit_checks {
use super::*;
#[test]
fn test_serialization() {
assert_serialization(TauriEventName::GenerationEnqueueSuccessEvent, "generation-enqueue-success-event");
assert_serialization(TauriEventName::GenerationEnqueueFailureEvent, "generation-enqueue-failure-event");
assert_serialization(TauriEventName::GenerationCompleteEvent, "generation-complete-event");
assert_serialization(TauriEventName::GenerationFailedEvent, "generation-failed-event");
assert_serialization(TauriEventName::CanvasBgRemovedEvent, "canvas_bg_removed_event");
}
#[test]
fn to_str() {
assert_eq!(TauriEventName::GenerationEnqueueSuccessEvent.to_str(), "generation-enqueue-success-event");
assert_eq!(TauriEventName::GenerationEnqueueFailureEvent.to_str(), "generation-enqueue-failure-event");
assert_eq!(TauriEventName::GenerationCompleteEvent.to_str(), "generation-complete-event");
assert_eq!(TauriEventName::GenerationFailedEvent.to_str(), "generation-failed-event");
assert_eq!(TauriEventName::CanvasBgRemovedEvent.to_str(), "canvas_bg_removed_event");
}
#[test]
fn from_str() {
assert_eq!(TauriEventName::from_str("generation-enqueue-success-event").unwrap(), TauriEventName::GenerationEnqueueSuccessEvent);
assert_eq!(TauriEventName::from_str("generation-enqueue-failure-event").unwrap(), TauriEventName::GenerationEnqueueFailureEvent);
assert_eq!(TauriEventName::from_str("generation-complete-event").unwrap(), TauriEventName::GenerationCompleteEvent);
assert_eq!(TauriEventName::from_str("generation-failed-event").unwrap(), TauriEventName::GenerationFailedEvent);
assert_eq!(TauriEventName::from_str("canvas_bg_removed_event").unwrap(), TauriEventName::CanvasBgRemovedEvent);
}
#[test]
fn all_variants() {
let mut variants = TauriEventName::all_variants();
assert_eq!(variants.len(), 5);
assert_eq!(variants.pop_first(), Some(TauriEventName::GenerationEnqueueSuccessEvent));
assert_eq!(variants.pop_first(), Some(TauriEventName::GenerationEnqueueFailureEvent));
assert_eq!(variants.pop_first(), Some(TauriEventName::GenerationCompleteEvent));
assert_eq!(variants.pop_first(), Some(TauriEventName::GenerationFailedEvent));
assert_eq!(variants.pop_first(), Some(TauriEventName::CanvasBgRemovedEvent));
assert_eq!(variants.pop_first(), None);
}
}
mod mechanical_checks {
use super::*;
use strum::IntoEnumIterator;
#[test]
fn variant_length() {
assert_eq!(TauriEventName::all_variants().len(), TauriEventName::iter().len());
}
#[test]
fn round_trip() {
for variant in TauriEventName::all_variants() {
// Test to_str(), from_str(), Display, and Debug.
assert_eq!(variant, TauriEventName::from_str(variant.to_str()).unwrap());
assert_eq!(variant, TauriEventName::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, TauriEventName::from_str(&format!("{:?}", variant)).unwrap());
}
}
//#[test]
//fn serialized_length_ok_for_database() {
// const MAX_LENGTH : usize = 16;
// for variant in TauriEventName::all_variants() {
// let serialized = variant.to_str();
// assert!(serialized.len() > 0, "variant {:?} is too short", variant);
// assert!(serialized.len() <= MAX_LENGTH, "variant {:?} is too long", variant);
// }
//}
}
}
@@ -0,0 +1,258 @@
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest, FluxProKontextMaxEditImageResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use sqlx::Acquire;
use utoipa::ToSchema;
/// Gpt Image 1 Image Editing
#[utoipa::path(
post,
tag = "Generate Images (Edit)",
path = "/v1/generate/image/edit/flux_pro_kontext_max",
responses(
(status = 200, description = "Success", body = FluxProKontextMaxEditImageResponse),
),
params(
("request" = FluxProKontextMaxEditImageRequest, description = "Payload for Request"),
)
)]
pub async fn flux_pro_kontext_max_edit_image_handler(
http_request: HttpRequest,
request: Json<FluxProKontextMaxEditImageRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<FluxProKontextMaxEditImageResponse>, CommonWebError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
const CAN_SEE_DELETED: bool = false;
// TODO: Maybe don't use the batch endpoint.
let media_tokens = vec![
request.image_media_token.clone(),
];
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let media_files = match result {
Ok(files) => files,
Err(err) => {
error!("Error getting media files by tokens: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let image_media_file = match media_files.first() {
Some(file) => file,
None => {
warn!("No media files found for tokens: {:?}", media_tokens);
return Err(CommonWebError::BadInputWithSimpleMessage(
"No media files found for provided tokens".to_string()));
}
};
let media_domain = get_media_domain(&http_request);
let image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&image_media_file.public_bucket_directory_hash,
image_media_file.maybe_public_bucket_prefix.as_deref(),
image_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let num_images = match request.num_images {
Some(FluxProKontextMaxEditImageNumImages::One) => FluxProKontextMaxNumImages::One,
Some(FluxProKontextMaxEditImageNumImages::Two) => FluxProKontextMaxNumImages::Two,
Some(FluxProKontextMaxEditImageNumImages::Three) => FluxProKontextMaxNumImages::Three,
Some(FluxProKontextMaxEditImageNumImages::Four) => FluxProKontextMaxNumImages::Four,
None => FluxProKontextMaxNumImages::One, // Default to One
};
let args = FluxProKontextMaxArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
image_url: image_url,
num_images,
};
let fal_result = enqueue_flux_pro_kontext_max_edit_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_flux_pro_kontext_max_edit_webook: {:?}", err);
CommonWebError::ServerError
})?;
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::FluxProKontextMax),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: vec![
PromptContextItem {
media_token: image_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgsrc,
}
],
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(FluxProKontextMaxEditImageResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -1 +1,2 @@
pub mod flux_pro_kontext_max_edit_image_handler;
pub mod gpt_image_1_edit_image_handler;
@@ -0,0 +1,286 @@
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest, FluxPro1InpaintImageResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook;
use fal_client::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use sqlx::Acquire;
use utoipa::ToSchema;
/// Flux Pro-1 Inpainting
#[utoipa::path(
post,
tag = "Generate Images (Inpaint)",
path = "/v1/generate/image/inpaint/flux_pro_1",
responses(
(status = 200, description = "Success", body = FluxPro1InpaintImageResponse),
),
params(
("request" = FluxPro1InpaintImageRequest, description = "Payload for Request"),
)
)]
pub async fn flux_pro_1_inpaint_image_handler(
http_request: HttpRequest,
request: Json<FluxPro1InpaintImageRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<FluxPro1InpaintImageResponse>, CommonWebError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
const CAN_SEE_DELETED: bool = false;
let media_tokens = vec![
request.image_media_token.clone(),
request.mask_media_token.clone(),
];
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let media_files = match result {
Ok(files) => files,
Err(err) => {
error!("Error getting media files by tokens: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
if media_files.len() != media_tokens.len() {
warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), media_tokens.len());
return Err(CommonWebError::BadInputWithSimpleMessage(
format!("Not all media files could be found. Media files found: {}, tokens provided: {}",
media_files.len(), media_tokens.len())));
}
let media_domain = get_media_domain(&http_request);
let image_media_file = media_files.iter()
.find(|file| file.token == request.image_media_token)
.ok_or_else(|| {
warn!("Image media file not found for token: {}", request.image_media_token);
CommonWebError::BadInputWithSimpleMessage("Image media file not found".to_string())
})?;
let mask_media_file = media_files.iter()
.find(|file| file.token == request.mask_media_token)
.ok_or_else(|| {
warn!("Mask media file not found for token: {}", request.mask_media_token);
CommonWebError::BadInputWithSimpleMessage("Mask media file not found".to_string())
})?;
let image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&image_media_file.public_bucket_directory_hash,
image_media_file.maybe_public_bucket_prefix.as_deref(),
image_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
let mask_image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&mask_media_file.public_bucket_directory_hash,
mask_media_file.maybe_public_bucket_prefix.as_deref(),
mask_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let num_images = match request.num_images {
Some(FluxPro1InpaintImageNumImages::One) => FluxPro1InfillNumImages::One,
Some(FluxPro1InpaintImageNumImages::Two) => FluxPro1InfillNumImages::Two,
Some(FluxPro1InpaintImageNumImages::Three) => FluxPro1InfillNumImages::Three,
Some(FluxPro1InpaintImageNumImages::Four) => FluxPro1InfillNumImages::Four,
None => FluxPro1InfillNumImages::One, // Default to One
};
let args = FluxPro1InfillArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
image_url: image_url,
mask_url: mask_image_url,
num_images,
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
let fal_result = enqueue_flux_pro_1_infill_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_flux_pro_1_infill_webhook: {:?}", err);
CommonWebError::ServerError
})?;
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
// TODO(bt,2025-07-31): Should we have an "inpaint" specific variant?
// Depends on how we want to model provider / feature matrices, routing, and results going forward.
maybe_model_type: Some(ModelType::FluxPro1),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: vec![
PromptContextItem {
media_token: image_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgsrc,
},
PromptContextItem {
media_token: mask_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgmask,
},
],
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(FluxPro1InpaintImageResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1 @@
pub mod flux_pro_1_inpaint_handler;
@@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image_handler;
pub mod generate_flux_pro_11_text_to_image_handler;
pub mod generate_flux_pro_11_ultra_text_to_image_handler;
pub mod generate_gpt_image_1_text_to_image_handler;
pub mod inpaint;
pub mod remove_image_background_handler;
@@ -4,7 +4,8 @@ use std::sync::Arc;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::{web, HttpRequest, HttpResponse};
use chrono::NaiveDateTime;
use actix_web::web::Json;
use chrono::{DateTime, NaiveDateTime, Utc};
use log::error;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
@@ -22,6 +23,8 @@ pub struct HealthCheckResponse {
pub unhealthy_check_consecutive_count: Option<u64>,
pub server_build_sha: String,
pub server_hostname: String,
pub startup_time: DateTime<Utc>,
pub seconds_since_startup: u64,
}
@@ -65,6 +68,11 @@ pub async fn get_health_check_handler(
let is_healthy = health_check_status.is_healthy;
let now = Utc::now();
let seconds_since_startup = now.signed_duration_since(server_state.startup_time)
.num_seconds()
.unsigned_abs();
let response = HealthCheckResponse {
success: true,
is_healthy: health_check_status.is_healthy,
@@ -73,6 +81,8 @@ pub async fn get_health_check_handler(
unhealthy_check_consecutive_count: health_check_status.unhealthy_check_consecutive_count,
server_build_sha: server_state.server_info.build_sha.clone(),
server_hostname: server_state.hostname.clone(),
startup_time: server_state.startup_time,
seconds_since_startup,
};
let body = serde_json::to_string(&response)
@@ -1,9 +1,11 @@
use crate::http_server::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image_handler::flux_pro_kontext_max_edit_image_handler;
use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler;
use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_handler::flux_pro_1_inpaint_image_handler;
use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_1_image_to_3d_handler::generate_hunyuan_2_1_image_to_3d_handler;
@@ -31,10 +33,21 @@ where
app.service(web::scope("/v1/generate")
.service(web::scope("/image")
.service(web::scope("/edit")
.service(web::resource("/flux_pro_kontext_max")
.route(web::post().to(flux_pro_kontext_max_edit_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/gpt_image_1")
.route(web::post().to(gpt_image_1_edit_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::scope("/inpaint")
.service(web::resource("/flux_pro_1")
.route(web::post().to(flux_pro_1_inpaint_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::resource("/flux_1_dev_text_to_image")
.route(web::post().to(generate_flux_1_dev_text_to_image_handler))
@@ -25,6 +25,7 @@ use actix_multipart::form::MultipartFormConfig;
use actix_web::middleware::{DefaultHeaders, Logger};
use actix_web::{middleware, web, App, HttpServer};
use anyhow::anyhow;
use chrono::Utc;
use elasticsearch::http::transport::Transport;
use elasticsearch::Elasticsearch;
use log::info;
@@ -392,6 +393,8 @@ async fn main() -> AnyhowResult<()> {
let openai_api_key= easyenv::get_env_string_required("OPENAI_API_KEY")?;
let startup_time = Utc::now();
let server_state = ServerState {
env_config: EnvConfig {
num_workers,
@@ -409,6 +412,7 @@ async fn main() -> AnyhowResult<()> {
client: stripe_client,
},
hostname: server_hostname,
startup_time,
server_environment_old: server_environment,
server_environment: server_environment_typed,
flags: service_feature_flags,
@@ -1,3 +1,4 @@
use chrono::{DateTime, Utc};
use elasticsearch::Elasticsearch;
use r2d2_redis::{r2d2, RedisConnectionManager};
use sqlx::MySqlPool;
@@ -51,6 +52,9 @@ pub struct ServerState {
pub hostname: String,
/// When the server starts.
pub startup_time: DateTime<Utc>,
#[deprecated(note = "Use `server_environment` instead")]
/// Knowing if we're in production will allow us to turn off development-only functionalities.
pub server_environment_old: ServerEnvironment,
@@ -0,0 +1,65 @@
use crate::prelude::{Deserialize, FalRequest, Serialize};
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct FluxProKontextMaxInput {
/// The prompt to fill the masked part of the image.
/// "A knight in shining armour holding a greatshield with \"FAL\" on it"
pub prompt: String,
/// The image URL to generate an image from. Needs to match the dimensions of the mask.
/// "https://storage.googleapis.com/falserverless/flux-lora/example-images/knight.jpeg"
pub image_url: String,
/// The number of images to generate.
#[serde(skip_serializing_if = "Option::is_none")]
pub num_images: Option<i64>,
/// The format of the generated image.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_format: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aspect_ratio: Option<String>,
/// The safety tolerance level for the generated image. 1 being the most strict and 5 being the most permissive.
#[serde(skip_serializing_if = "Option::is_none")]
pub safety_tolerance: Option<String>,
/// The same seed and the same prompt given to the same version of the model
/// will output the same image every time.
#[serde(skip_serializing_if = "Option::is_none")]
pub seed: Option<i64>,
/// If set to true, the function will wait for the image to be generated and uploaded
/// before returning the response. This will increase the latency of the function but
/// it allows you to get the image directly in the response without going through the CDN.
#[serde(skip_serializing_if = "Option::is_none")]
pub sync_mode: Option<bool>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct Image {
#[serde(skip_serializing_if = "Option::is_none")]
pub content_type: Option<String>,
pub height: i64,
pub url: String,
pub width: i64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Output {
/// Whether the generated images contain NSFW concepts.
pub has_nsfw_concepts: Vec<bool>,
/// The generated image files info.
pub images: Vec<Image>,
/// The prompt used for generating the image.
pub prompt: String,
/// Seed of the generated Image. It will be the same value of the one passed in the
/// input or the randomly generated that was used in case none was passed.
pub seed: i64,
//pub timings: Timings,
}
pub fn max(params: FluxProKontextMaxInput) -> FalRequest<FluxProKontextMaxInput, Output> {
FalRequest::new("fal-ai/flux-pro/kontext/max", params)
}
@@ -0,0 +1,2 @@
pub mod max;
pub mod max_multi;
@@ -19,6 +19,22 @@ use std::collections::HashMap;
)))
)]
pub mod finetuned;
#[cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",
feature = "endpoints_fal-ai_flux-kontext"
))]
#[cfg_attr(
docsrs,
doc(cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",
feature = "endpoints_fal-ai_flux-kontext"
)))
)]
pub mod kontext;
#[cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",
@@ -0,0 +1,15 @@
export const normalizeCanvas = (canvas: HTMLCanvasElement, width: number, height: number): HTMLCanvasElement => {
const newCanvas = document.createElement("canvas");
newCanvas.width = width;
newCanvas.height = height;
const ctx = newCanvas.getContext("2d");
if (!ctx) {
throw new Error("Failed to get canvas context");
}
ctx.imageSmoothingEnabled = true;
ctx.drawImage(canvas, 0, 0, width, height);
return newCanvas;
}
@@ -6,6 +6,7 @@ import {
faFilm,
faPaintbrush,
faImage,
faPenNib,
// faGem,
} from "@fortawesome/pro-solid-svg-icons";
import { Button } from "@storyteller/ui-button";
@@ -73,6 +74,11 @@ const appMenuTabs: MenuIconItem[] = [
label: "Image to Video",
icon: <FontAwesomeIcon icon={faFilm} />,
},
{
id: "EDIT",
label: "Edit Image",
icon: <FontAwesomeIcon icon={faPenNib} />,
}
];
export const topNavMediaId = signal<string>("");
@@ -15,7 +15,11 @@ import { setCanvasRenderBitmap } from "../../signals/canvasRenderBitmap";
import { captureStageImageBitmap } from "./hooks/useUpdateSnapshot";
import { ContextMenuContainer } from "./components/ui/ContextMenu";
import { FalBackgroundRemoval } from "@storyteller/tauri-api";
import { getCreatorIcon, ModelCreator } from "@storyteller/model-list";
import { EnqueueImageBgRemoval } from "@storyteller/tauri-api";
import { getCreatorIcon, IMAGE_MODELS_BY_LABEL, ModelCreator, ModelInfo } from "@storyteller/model-list";
import { instructiveImageEditModels, ModelCategory, ModelSelector, useModelSelectorStore } from "@storyteller/ui-model-selector";
import { useCanvasBgRemovedEvent } from "@storyteller/tauri-api";
export const DecodeBase64ToImage = async (
base64String: string,
@@ -45,12 +49,19 @@ export const DecodeBase64ToImage = async (
const PageDraw = () => {
const canvasWidth = useRef<number>(1024);
const canvasHeight = useRef<number>(1024);
const { selectedModels } = useModelSelectorStore();
const [isSelecting, setIsSelecting] = useState<boolean>(false);
const [selectedModel, setSelectedModel] = useState<string>("GPT-4o");
const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
const store = useSceneStore();
const selectedModel =
selectedModels[ModelCategory.Canvas2D] ||
instructiveImageEditModels[0]?.label;
const selectedModelInfo: ModelInfo | undefined =
IMAGE_MODELS_BY_LABEL[selectedModel];
useDeleteHotkeys({ onDelete: store.deleteSelectedItems });
useUndoRedoHotkeys({ undo: store.undo, redo: store.redo });
useCopyPasteHotkeys({
@@ -58,6 +69,27 @@ const PageDraw = () => {
onPaste: store.pasteItems,
});
useCanvasBgRemovedEvent(async (event) => {
console.log("Canvas bg removed event received:", event);
const nodeId = event.maybe_frontend_subscriber_id;
if (!nodeId) {
console.error("No node ID received from background removal");
return;
}
// const base64String = response.payload?.base64_bytes as string;
// const binaryString = atob(base64String);
// const bytes = Uint8Array.from(binaryString, (c) => c.charCodeAt(0));
// const blob = new Blob([bytes], { type: "image/png" });
// const file = new File([blob], "generated_image.png", {
// type: blob.type,
// });
store.finishRemoveBackground(
nodeId,
event.media_token,
event.image_cdn_url
);
});
// Listen for gallery drag and drop events
useEffect(() => {
const handleGallery2DDrop = (event: CustomEvent) => {
@@ -144,31 +176,6 @@ const PageDraw = () => {
}
};
const modelList: PopoverItem[] = [
{
label: "GPT-4o",
icon: getCreatorIcon(ModelCreator.OpenAi) || (
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
),
selected: selectedModel === "GPT-4o",
description: "High quality model",
badges: [{ label: "2 min.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
{
label: "FLUX.1 Kontext",
icon: getCreatorIcon(ModelCreator.BlackForestLabs) || (
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
),
selected: selectedModel === "FLUX.1 Kontext",
description: "Fast and high-quality model",
badges: [{ label: "20 sec.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
];
const handleModelSelect = (item: PopoverItem) => {
setSelectedModel(item.label);
};
const onEnqueuedPressed = async () => {
const { width, height } = store.getAspectRatioDimensions();
@@ -295,6 +302,7 @@ const PageDraw = () => {
}}
onEnqueuePressed={onEnqueuedPressed}
onFitPressed={onFitPressed}
selectedModelInfo={selectedModelInfo}
/>
</div>
<SideToolbar
@@ -386,46 +394,7 @@ const PageDraw = () => {
store.toggleLock(store.selectedNodeIds);
break;
case "REMOVE_BACKGROUND":
await store.removeBackground(
store.selectedNodeIds,
async (
success: boolean,
image_base64: string,
message: string,
) => {
if (!success) {
console.error(message);
return { success: false };
}
try {
const response = await FalBackgroundRemoval({
base64_image: image_base64,
});
if (
response.status !== "success" ||
!("payload" in response)
) {
console.error("Failed to remove background", response);
return { success: false };
}
const base64String = response.payload
?.base64_bytes as string;
const binaryString = atob(base64String);
const bytes = Uint8Array.from(binaryString, (c) =>
c.charCodeAt(0),
);
const blob = new Blob([bytes], { type: "image/png" });
const file = new File([blob], "generated_image.png", {
type: blob.type,
});
return { success: true, file };
} catch (error) {
console.error("Failed to remove background", error);
return { success: false };
}
},
);
await store.beginRemoveBackground(store.selectedNodeIds);
break;
case "BRING_TO_FRONT":
store.bringToFront(store.selectedNodeIds);
@@ -474,10 +443,9 @@ const PageDraw = () => {
</ContextMenuContainer>
</div>
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<PopoverMenu
items={modelList}
onSelect={handleModelSelect}
mode="hoverSelect"
<ModelSelector
items={instructiveImageEditModels}
category={ModelCategory.Canvas2D}
panelTitle="Select Model"
panelClassName="min-w-[280px]"
buttonClassName="bg-transparent p-0 text-lg hover:bg-transparent text-white/80 hover:text-white"
@@ -19,7 +19,7 @@ import { useRightPanelLayoutManagement } from "./hooks/useRightPanelLayoutManage
import { useStageCentering } from "./hooks/useCenteredStage";
import { useGlobalMouseUp } from "./hooks/useGlobalMouseUp";
type MiraiProps = {
export type MiraiProps = {
nodes: Node[];
selectedNodeIds: string[];
onCanvasSizeChange?: (width: number, height: number) => void;
@@ -14,6 +14,7 @@ const PromptEditor: React.FC<PromptEditorProps> = ({
onEnqueuePressed,
onAspectRatioChange,
onFitPressed,
selectedModelInfo,
}) => {
const [images, setImages] = useState<ImageStyle[]>([]);
@@ -43,6 +44,7 @@ const PromptEditor: React.FC<PromptEditorProps> = ({
onEnqueuePressed={onEnqueuePressed}
onAspectRatioChange={onAspectRatioChange}
onFitPressed={onFitPressed}
selectedModelInfo={selectedModelInfo}
/>
</JobProvider>
@@ -1,3 +1,5 @@
import { ModelInfo } from "libs/model-list/src/lib/ModelInfo";
export interface PromptEditorProps {
initialPrompt?: string;
onPromptChange?: (prompt: string) => void;
@@ -8,6 +10,7 @@ export interface PromptEditorProps {
onImageStyleChange?: (images: ImageStyle[]) => void;
onEnqueuePressed?: () => void;
onFitPressed?: () => void;
selectedModelInfo?: ModelInfo;
}
export type AspectRatio = "1:1" | "3:2" | "2:3";
@@ -1,5 +1,6 @@
import { create } from "zustand";
import { Node, NodeType } from "../Node";
import { EnqueueImageBgRemoval } from "libs/tauri-api/src/lib/enqueue/EnqueueImageBgRemovalCommand";
// Add LineNode type
export type LineNode = {
@@ -19,6 +20,7 @@ export type LineNode = {
offsetY?: number;
zIndex: number;
locked?: boolean; // Add locked property
globalCompositeOperation?: string; // Add globalCompositeOperation property
};
// Add this enum at the top of the file with other types
@@ -30,7 +32,7 @@ export enum AspectRatioType {
}
// Logic to remove background from image nodes would go here
const convertFileToBase64 = (file: File): Promise<string> => {
export const convertFileToBase64 = (file: File): Promise<string> => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
@@ -114,6 +116,7 @@ interface SceneState {
shouldSaveState: boolean,
) => void;
selectNode: (id: string | null, isMultiSelect?: boolean) => void;
moveNode: (
id: string,
x: number,
@@ -135,6 +138,7 @@ interface SceneState {
fill?: string,
) => void;
createCircle: (x: number, y: number, radius?: number, fill?: string) => void;
createTriangle: (
x: number,
y: number,
@@ -228,13 +232,17 @@ interface SceneState {
sendToBack: (nodeIds: string[]) => void;
bringForward: (nodeIds: string[]) => void;
sendBackward: (nodeIds: string[]) => void;
removeBackground: (
// Start background removal (enqueues task with Tauri)
beginRemoveBackground: (
nodeIds: string[],
operation: (
success: boolean,
base64_image: string,
message: string,
) => Promise<{ success: boolean; file?: File }>,
) => Promise<void>;
// Finish background removal (handoff back from a Tauri-sent event)
finishRemoveBackground: (
nodeId: string,
mediaToken: string,
imageCdnUrl: string,
) => Promise<void>;
// Add new lock action
@@ -248,7 +256,7 @@ interface SceneState {
getAspectRatioDimensions: () => { width: number; height: number };
}
const generateId = (): string => {
export const generateId = (): string => {
return Math.random().toString(36).substring(2, 9);
};
@@ -273,6 +281,7 @@ export const useSceneStore = create<SceneState>((set, get) => ({
clipboard: [],
history: [],
historyIndex: -1,
activeTool: "select",
brushColor: "#000000",
brushSize: 5,
@@ -280,6 +289,8 @@ export const useSceneStore = create<SceneState>((set, get) => ({
fillColor: "white",
currentShape: null,
shapeColor: "#4d79b3",
// Cursor initial state
cursorPosition: null,
cursorVisible: false,
@@ -296,6 +307,7 @@ export const useSceneStore = create<SceneState>((set, get) => ({
zIndex: nextZ,
});
// console.log("New Node with ID:", newNode.id);
const nodes = [...state.nodes, newNode];
// console.log("Nodes after update");
// console.log(nodes);
@@ -307,6 +319,7 @@ export const useSceneStore = create<SceneState>((set, get) => ({
removeNode: (id: string, shouldSaveState: boolean = true) => {
set((state) => {
// Remove the node
const newNodes = state.nodes.filter((node) => node.id !== id);
// Update selection state
@@ -324,6 +337,7 @@ export const useSceneStore = create<SceneState>((set, get) => ({
}
},
updateNode: (
id: string,
updates: Partial<Node>,
@@ -1423,61 +1437,71 @@ export const useSceneStore = create<SceneState>((set, get) => ({
});
get().saveState();
},
removeBackground: async (
beginRemoveBackground: async (
nodeIds: string[],
operation: (
success: boolean,
base64_image: string,
message: string,
) => Promise<{ success: boolean; file?: File }>,
) => {
const hasImageNodes = nodeIds.some((id) => {
const node = get().nodes.find((n) => n.id === id);
return node ? node.type === "image" : false;
});
if (hasImageNodes) {
const firstNode = get().nodes.find(
(node) => nodeIds.includes(node.id) && node.type === "image",
);
if (firstNode && firstNode.imageFile) {
try {
const base64Image = await convertFileToBase64(firstNode.imageFile);
const { success, file } = await operation(
true,
base64Image,
"Removing Background.",
);
if (success && file) {
// Create a new node instance
const updatedNode = new Node({
...firstNode,
imageFile: file,
});
// Load the new image
await updatedNode.setImageFromFile(file);
// Update the store with the fully loaded node
set((state: SceneState) => ({
nodes: state.nodes.map((node: Node) => {
if (node.id === firstNode.id) {
return updatedNode;
}
return node;
}),
}));
get().saveState();
if (!hasImageNodes) {
return;
}
const firstNode = get().nodes.find(
(node) => nodeIds.includes(node.id) && node.type === "image",
);
if (!firstNode || !firstNode.imageFile) {
return;
}
try {
const base64Image = await convertFileToBase64(firstNode.imageFile);
// NB: This handler is async, so we subscribe (externally) and
// wait for the bg removal to complete.
const _response = await EnqueueImageBgRemoval({
base64_image: base64Image,
frontend_caller: "canvas",
frontend_subscriber_id: firstNode.id,
});
} catch (error) {
console.error("Error starting background removal:", error);
}
},
finishRemoveBackground: async (
nodeId: string,
mediaToken: string,
imageCdnUrl: string,
) => {
if (!nodeId || !mediaToken || !imageCdnUrl) {
return;
}
const firstNode = get().nodes.find(
(node) => node.id === nodeId && node.type === "image",
);
if (!firstNode || !firstNode.imageFile) {
return;
}
try {
// Create a new node instance from the old node
const updatedNode = new Node({
...firstNode,
imageUrl: imageCdnUrl,
});
// Load the new image
await updatedNode.setImageFromUrl(imageCdnUrl);
// Update the store with the fully loaded node
set((state: SceneState) => ({
nodes: state.nodes.map((node: Node) => {
if (node.id === firstNode.id) {
return updatedNode;
}
} catch (error) {
console.error("Error updating image:", error);
}
}
return node;
}),
}));
get().saveState();
} catch (error) {
console.error("Error completing background removal:", error);
}
},
// Add toggleLock action
toggleLock: (nodeIds: string[]) => {
set((state) => {
@@ -0,0 +1,16 @@
export const EncodeImageBitmapToArray = async (imageBitmap: ImageBitmap): Promise<Uint8Array> => {
const canvas = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
const ctx = canvas.getContext('2d');
if (!ctx) {
throw new Error("Failed to get 2D context");
}
ctx.drawImage(imageBitmap, 0, 0);
const blob = await canvas.convertToBlob({ type: 'image/png' });
const arrayBuffer = await blob.arrayBuffer();
return new Uint8Array(arrayBuffer);
}
@@ -0,0 +1,171 @@
import { faImage } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { downloadFileFromUrl } from "libs/api/src/lib/LocalApi";
import { Button } from "@storyteller/ui-button";
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { useRef, useState } from "react";
import toast from "react-hot-toast";
import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
import { UploaderStates } from "~/enums";
export interface BaseSelectorImage {
id: string;
url: string;
mediaToken: string;
}
export type BaseImageSelectorProps = {
onImageSelect: (imageUrl: BaseSelectorImage) => void;
showLoading?: boolean;
}
const MAX_GALLERY_SELECTIONS = 1;
export const BaseImageSelector = ({
onImageSelect,
showLoading = false,
}: BaseImageSelectorProps) => {
const fileInputRef = useRef<HTMLInputElement>(null);
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[]
);
const [isLoading, setIsLoading] = useState(false);
const handleGalleryClick = () => setIsGalleryModalOpen(true);
const handleGalleryClose = () => {
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleImageSelect = (id: string) => {
// If already selected, deselect it
// Else if not selected and under max limit, select it
if (selectedGalleryImages.includes(id)) {
setSelectedGalleryImages([]);
} else {
setSelectedGalleryImages([id]);
}
}
const handleUseGalleryImages = (selectedItems: GalleryItem[]) => {
// We only want one file
if (selectedItems.length !== 1) {
return;
}
const item = selectedItems[0];
if (!item.fullImage) return;
const referenceImage: BaseSelectorImage = {
id: Math.random().toString(36).substring(7),
url: item.fullImage,
mediaToken: item.id,
};
sendImageEvent(referenceImage);
}
const handleUploadClick = () => {
if (fileInputRef.current) {
fileInputRef.current.click();
}
}
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const files = event.target.files;
if (files) {
setIsLoading(true);
Array.from(files).forEach((file) => {
const reader = new FileReader(); reader.onloadend = () => {
uploadImage({
title: `reference-image-${Math.random()
.toString(36)
.substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
console.debug("Upload progress:", newState.data);
if (newState.status === UploaderStates.success && newState.data) {
const referenceImage: BaseSelectorImage = {
id: Math.random().toString(36).substring(7),
url: reader.result as string,
mediaToken: newState.data || "",
};
toast.success("Image uploaded successfully!");
sendImageEvent(referenceImage);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
toast.error("Upload failed. Please try again.");
setIsLoading(false);
}
},
});
};
reader.readAsDataURL(file);
});
}
};
const sendImageEvent = (image: BaseSelectorImage) => {
handleGalleryClose();
onImageSelect(image);
}
return (
<>
<div className="h-1/2 w-1/2 flex items-center justify-center bg-ui-background border-ui-panel-border border-8">
{isLoading || showLoading ? (
<div className="flex flex-col items-center gap-4">
<span>Uploading image...</span>
<LoadingSpinner />
</div>
) :
<div className="flex flex-col gap-8">
<FontAwesomeIcon icon={faImage} className="text-6xl text-ui-controls" />
<span className="text-ui-controls text-lg ml-2">Click to upload or drag and drop an image here to edit</span>
<div className="mt-4 flex justify-center gap-2">
<input
type="file"
ref={fileInputRef}
className="hidden"
accept="image/*"
onChange={handleFileUpload}
multiple
/>
<Button
variant="primary"
onClick={handleUploadClick}
>
Upload an image
</Button>
<Button
variant="secondary"
onClick={handleGalleryClick}
>
Select from library
</Button>
</div>
</div>
}
</div>
<GalleryModal
isOpen={!!isGalleryModalOpen}
onClose={handleGalleryClose}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={handleImageSelect}
maxSelections={MAX_GALLERY_SELECTIONS}
onUseSelected={handleUseGalleryImages}
onDownloadClicked={downloadFileFromUrl}
forceFilter="image"
/>
</>
)
}
@@ -0,0 +1,51 @@
import { faPlusCircle, faMinusCircle, faDrawCircle, faEraser, faPaintbrush } from "@fortawesome/pro-regular-svg-icons";
import { EditOperation } from "./stores/EditState";
import { ButtonIconSelect } from "@storyteller/ui-button-icon-select";
import { SliderV2 } from "@storyteller/ui-sliderv2";
import { faPaintBrush } from "@fortawesome/pro-solid-svg-icons";
export interface DrawToolControlBarProps {
currentMode: EditOperation;
currentSize: number;
onModeChange?: (mode: EditOperation) => void;
onSizeChange?: (size: number) => void;
}
const DrawToolControlBar = ({
currentMode,
currentSize,
onModeChange,
onSizeChange = () => { }
}: DrawToolControlBarProps) => {
const modes = [
{
value: "add",
icon: faPaintBrush,
text: "Draw",
tooltip: "Draw strokes",
},
{
value: "minus",
icon: faEraser,
text: "Erase",
tooltip: "Erases strokes",
},
];
return (
<div className="absolute top-16 left-1/2 flex -translate-x-1/2 flex-col gap-3">
<div className="glass w-[400px] flex gap-2 rounded-xl p-2 items-center">
<ButtonIconSelect
options={modes}
// @ts-expect-error - TypeScript doesn't recognize the value as EditOperation
onOptionChange={onModeChange}
selectedOption={currentMode}
/>
<SliderV2 min={1} max={50} value={currentSize} onChange={onSizeChange} step={1} innerLabel={"Size " + currentSize + "pt"} showDecrement showIncrement />
</div>
</div>
)
}
export default DrawToolControlBar;

Some files were not shown because too many files have changed in this diff Show More