Compare commits

...
Author SHA1 Message Date
Brandon Thomas 105eee5bdb handle fal image download and bucket upload 2025-06-03 14:44:11 -04:00
Brandon Thomas 9e32196ca2 decode image 2025-06-03 12:59:47 -04:00
Brandon Thomas c8c857e3d0 read webhook request json 2025-06-03 12:12:02 -04:00
Brandon Thomas f480bc032e webhook debugging 2025-06-03 10:36:32 -04:00
Brandon Thomas c3e41a1a3e fal webhook endpoint 2025-06-03 10:26:47 -04:00
Brandon Thomas 377acb3e4a add routes 2025-06-03 09:31:42 -04:00
Brandon Thomas bfd864c082 idempotency token 2025-06-03 09:23:04 -04:00
Brandon Thomas bd059643f0 enum type, wip 2025-06-03 01:16:44 -04:00
Brandon Thomas d39c6c94e5 storyteller-web builds 2025-06-03 00:53:39 -04:00
Brandon Thomas 0d6ec74611 wip 2025-06-02 14:25:02 -04:00
Brandon Thomas 4c26309d4e refactor queue vs webhook endpoints 2025-06-02 14:25:02 -04:00
Brandon Thomas c6d3b92d42 revert changes to old fal methods 2025-06-02 14:25:02 -04:00
Brandon Thomas 3f85ef0884 enqueuing to webhook works 2025-06-02 14:25:02 -04:00
Brandon Thomas d730ffb7bd wip 2025-06-02 14:25:02 -04:00
Brandon Thomas 15bc87e025 server impl wip 2025-06-02 14:25:01 -04:00
Brandon Thomas 0af30604a6 endpoint wip 2025-06-02 14:25:01 -04:00
Brandon Thomas 01dfb1e301 crate for shared api defs 2025-06-02 14:25:01 -04:00
Brandon Thomas 3ea554f322 wip fal backend 2025-06-02 14:25:01 -04:00
70 changed files with 1404 additions and 117 deletions
Generated
+26 -3
View File
@@ -689,6 +689,19 @@ dependencies = [
"workspace-build-acceleration",
]
[[package]]
name = "artcraft_api_defs"
version = "0.0.1"
dependencies = [
"enums",
"serde",
"serde_derive",
"serde_json",
"tokens",
"utoipa",
"workspace-build-acceleration",
]
[[package]]
name = "ascii"
version = "0.9.3"
@@ -3554,6 +3567,8 @@ dependencies = [
"fal-derive",
"futures 0.3.31",
"image",
"log",
"percent-encoding-rfc3986",
"reqwest 0.12.15",
"serde",
"serde_json",
@@ -7230,6 +7245,12 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "percent-encoding-rfc3986"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3637c05577168127568a64e9dc5a6887da720efef07b3d9472d45f63ab191166"
[[package]]
name = "phf"
version = "0.8.0"
@@ -9876,12 +9897,14 @@ dependencies = [
"actix_cors_configs",
"actix_helpers",
"anyhow",
"artcraft_api_defs",
"async-stripe",
"async-trait",
"base64 0.22.1",
"billing_component",
"bootstrap",
"bucket_paths",
"bytes 1.10.1",
"chrono",
"cloud_storage",
"collections",
@@ -9899,6 +9922,7 @@ dependencies = [
"enums",
"enums_public",
"errors",
"fal_client",
"filesys",
"futures 0.1.31",
"futures 0.3.31",
@@ -10796,11 +10820,10 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.18.0"
version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c317e0a526ee6120d8dabad239c8dadca62b24b6f168914bbbc8e2fb1f0e567"
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"cfg-if 1.0.0",
"fastrand 2.3.0",
"getrandom 0.3.1",
"once_cell",
+6
View File
@@ -88,6 +88,7 @@ members = [
#"crates/schema/database/sqlite_queries",
# Schema (public): common strongly-typed tokens, ids, payloads, etc.
"crates/schema/public/api/artcraft_api_defs",
"crates/schema/public/composite_identifiers",
"crates/schema/public/enums",
"crates/schema/public/enums_public",
@@ -181,6 +182,7 @@ actix-web-actors = "4.3.1+deprecated"
actix-web-lab = "0.20.2"
anyhow = "1.0.98"
base64 = "0.22.1"
bytes = "1.10.1"
chrono = { version = "0.4.41", features = ["serde"] }
cookie = "0.18.1"
futures = { version = "0.3.31" }
@@ -197,18 +199,22 @@ serde_json = "1.0.140"
serde_yaml = "0.9.34+deprecated"
sha3 = "0.10.8"
tauri = { version = "2.4.1", features = ["unstable", "devtools"] } # NB: "devtools" prevents App Store distribution
tempdir = "0.3.7"
tempfile = "3.20.0"
thiserror = "2.0.12"
tokio = { version = "1.45.1", features = ["macros"] }
toml = "0.8.22"
url = { version = "2.5.4", features = ["serde"] }
# Internal libs
artcraft_api_defs = { path = "crates/schema/public/api/artcraft_api_defs" }
build_metadata = { path = "crates/lib/build_metadata" }
crockford = { path = "crates/lib/data/crockford" }
enums = { path = "crates/schema/public/enums" }
enums_public = { path = "crates/schema/public/enums_public" }
errors = { path = "crates/lib/errors" }
fal_client = { path = "crates/lib/api_clients/fal_client" }
file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
filesys = { path = "crates/lib/files/filesys" }
idempotency = { path = "crates/lib/idempotency" }
images = { path = "crates/lib/files/images" }
@@ -39,6 +39,9 @@ CREATE TABLE generic_inference_jobs (
-- Examples (may not be up to date):
-- * text_to_speech
-- * voice_conversion
-- * image_generation
-- * video_generation
-- * background_removal
inference_category VARCHAR(32) NOT NULL,
-- If the job is externally enqueued, this is the third-party service
+2 -2
View File
@@ -54,8 +54,8 @@ serde_json = { workspace = true }
tauri = { workspace = true }
tauri-plugin-log = "2.3.1"
tauri-plugin-upload = "2.2.1"
tempdir = "0.3.7"
tempfile = "3.18.0"
tempdir = { workspace = true }
tempfile = { workspace = true }
tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
tokio-util = { version = "0.7", features = ["codec"] }
url = { workspace = true }
@@ -24,9 +24,8 @@ use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
@@ -12,8 +12,8 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use fal_client::creds::fal_api_key::FalApiKey;
use fal_client::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use log::{error, info, warn};
use tauri::AppHandle;
@@ -11,8 +11,8 @@ use crate::services::sora::state::sora_credential_manager::SoraCredentialManager
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use fal_client::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use fal_client::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use log::{error, info, warn};
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
@@ -24,7 +24,7 @@ use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use fal_client::requests::queue::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
@@ -24,8 +24,7 @@ use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
@@ -24,8 +24,7 @@ use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use fal_client::requests::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use fal_client::requests::queue::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
@@ -14,4 +14,5 @@ pub fn add_artcraft_3d(cors: Cors, _is_production: bool) -> Cors {
.allowed_origin("http://localhost:5173")
.allowed_origin("http://localhost:5174") // If already started
.allowed_origin("http://localhost:5175") // If already started
.allowed_origin("https://macaroni-1.tailce84f.ts.net") // TODO
}
+1 -1
View File
@@ -17,7 +17,7 @@ path = "src/lib.rs"
workspace-build-acceleration = { workspace = true }
# Internal
file_to_base64_url = { path = "../../files/file_to_base64_url" }
file_to_base64_url = { workspace = true }
errors = { workspace = true }
# External
@@ -15,6 +15,8 @@ pub enum FalErrorPlus {
UrlParseError(url::ParseError),
/// An endpoint we don't support yet.
UnhandledEndpoint(String),
/// Error from the `reqwest` crate.
ReqwestError(reqwest::Error),
}
impl Display for FalErrorPlus {
@@ -25,6 +27,7 @@ impl Display for FalErrorPlus {
Self::AnyhowError(err) => write!(f, "FalErrorPlus::AnyhowError: {:?}", err),
Self::UrlParseError(err) => write!(f, "FalErrorPlus::UrlParseError: {:?}", err),
Self::UnhandledEndpoint(endpoint) => write!(f, "FalErrorPlus::UnhandledEndpoint: {:?}", endpoint),
Self::ReqwestError(err) => write!(f, "FalErrorPlus::ReqwestError: {:?}", err),
}
}
}
@@ -48,3 +51,9 @@ impl From<url::ParseError> for FalErrorPlus {
FalErrorPlus::UrlParseError(err)
}
}
impl From<reqwest::Error> for FalErrorPlus {
fn from(err: reqwest::Error) -> Self {
FalErrorPlus::ReqwestError(err)
}
}
@@ -1,4 +1,2 @@
pub mod enqueue_hunyuan2_image_to_3d;
pub mod remove_background_rembg;
pub mod video_gen;
pub mod image_gen;
pub mod queue;
pub mod webhook;
@@ -42,7 +42,7 @@ pub async fn enqueue_hunyuan2_image_to_3d<P: AsRef<Path>>(args: Hunyuan2Args<'_,
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use crate::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
@@ -42,7 +42,7 @@ pub async fn enqueue_flux_pro_ultra_text_to_image(args: FluxProUltraTextToImageA
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use crate::requests::queue::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use errors::AnyhowResult;
use std::fs::read_to_string;
@@ -39,7 +39,7 @@ pub async fn enqueue_recraft3_text_to_image(args: Recraft3TextToImageArgs<'_>) -
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use crate::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use errors::AnyhowResult;
use std::fs::read_to_string;
@@ -0,0 +1,4 @@
pub mod enqueue_hunyuan2_image_to_3d;
pub mod image_gen;
pub mod remove_background_rembg;
pub mod video_gen;
@@ -9,11 +9,6 @@ use futures::StreamExt;
use std::path::Path;
use url::Url;
pub struct RemBgArgs<'a, P: AsRef<Path>> {
pub image_path: P,
pub api_key: &'a FalApiKey,
}
pub struct RemBgResponse {
pub image_url: Url,
}
@@ -59,7 +54,7 @@ pub async fn remove_background_rembg<P: AsRef<Path>>(image_path: P, api_key: &Fa
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::remove_background_rembg::remove_background_rembg;
use crate::requests::queue::remove_background_rembg::remove_background_rembg;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
@@ -57,7 +57,7 @@ pub async fn enqueue_kling_16_image_to_video<P: AsRef<Path>>(args: Kling16Args<'
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use crate::requests::queue::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
@@ -3,7 +3,6 @@ use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::minimax::image_to_video::image_to_video;
use fal::endpoints::fal_ai::minimax::image_to_video::VideoOutput;
use fal::endpoints::fal_ai::minimax::image_to_video::ImageToVideoRequest;
use fal_client::file_to_base64_url::file_to_base64_url;
use futures::StreamExt;
@@ -41,10 +40,10 @@ pub async fn enqueue_minimax_01_image_to_video<P: AsRef<Path>>(args: Minimax01Ar
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::queue::video_gen::enqueue_minimax_01_image_to_video::{enqueue_minimax_01_image_to_video, Minimax01Args};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
use crate::requests::video_gen::enqueue_minimax_01_image_to_video::{enqueue_minimax_01_image_to_video, Minimax01Args};
#[tokio::test]
#[ignore]
@@ -0,0 +1 @@
pub mod remove_background_rembg_webhook;
@@ -0,0 +1,62 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::endpoints::fal_ai::imageutils::rembg::{rembg, RemoveBackgroundInput};
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct RemoveBackgroundRembgWebhookArgs<'a, U: IntoUrl, V: IntoUrl> {
pub image_url: U,
pub webhook_url: V,
pub api_key: &'a FalApiKey
}
pub async fn remove_background_rembg_webhook<U: IntoUrl, V: IntoUrl>(
args: RemoveBackgroundRembgWebhookArgs<'_, U, V>
) -> Result<WebhookResponse, FalErrorPlus> {
let image_url = args.image_url.into_url()?;
let request = RemoveBackgroundInput {
image_url: image_url.to_string(),
crop_to_bbox: None,
sync_mode: None
};
let result = rembg(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::remove_background_rembg_webhook::remove_background_rembg_webhook;
use crate::requests::webhook::image::remove_background_rembg_webhook::RemoveBackgroundRembgWebhookArgs;
use std::fs::read_to_string;
#[tokio::test]
#[ignore] // NB: Manually test, don't run in CI!
async fn test_rembg_request() -> anyhow::Result<()> {
// XXX: Don't commit secrets!
let api_key = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&api_key);
let args = RemoveBackgroundRembgWebhookArgs {
image_url: "https://cdn-2.fakeyou.com/media/b/2/p/b/0/b2pb09bcbxdqjzgbnsapvfs1t897wdnx/image_b2pb09bcbxdqjzgbnsapvfs1t897wdnx.png",
webhook_url: "https://api.storyteller.ai/webhook",
api_key: &api_key,
};
let response = remove_background_rembg_webhook(args).await?;
println!("{:?}", response);
assert_eq!(1, 2);
Ok(())
}
}
@@ -0,0 +1 @@
pub mod image;
@@ -1,10 +1,17 @@
use actix_web::HttpResponse;
use actix_web::web::Json;
#[derive(Serialize)]
pub struct SimpleGenericJsonSuccess {
pub success: bool,
}
impl SimpleGenericJsonSuccess {
pub fn wrapped(success: bool) -> Json<Self> {
Json(SimpleGenericJsonSuccess { success })
}
}
pub fn simple_json_success() -> HttpResponse {
let response = SimpleGenericJsonSuccess {
success: true,
@@ -174,6 +174,8 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
InferenceCategory::VideoFilter => Some("Video Filter"),
InferenceCategory::ImageGeneration => Some("Image Generation"),
InferenceCategory::VideoGeneration => Some("Video Generation"),
InferenceCategory::BackgroundRemoval => Some("Background Removal"),
InferenceCategory::Mocap => Some("Mocap"),
InferenceCategory::Workflow => Some("Workflow"),
InferenceCategory::FormatConversion => Some("format conversion"),
@@ -194,6 +196,8 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::VideoGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::BackgroundRemoval => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
@@ -0,0 +1,133 @@
use anyhow::anyhow;
use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
use log::warn;
use sqlx::pool::PoolConnection;
use sqlx::{MySql, MySqlPool};
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
use enums::common::job_status_plus::JobStatusPlus;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::users::UserToken;
use crate::helpers::boolean_converters::i8_to_bool;
use crate::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
#[derive(Debug, Default)]
pub struct FalJobDetails {
pub job_token: InferenceJobToken,
pub status: JobStatusPlus,
pub external_third_party: InferenceJobExternalThirdParty,
pub external_third_party_id: String,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
pub creator_ip_address: String,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
/// Returns Ok(None) when the record cannot be found.
pub async fn get_inference_job_by_fal_id(fal_id: &str, mysql_pool: &MySqlPool)
-> AnyhowResult<Option<FalJobDetails>>
{
let mut connection = mysql_pool.acquire().await?;
get_inference_job_by_fal_id_from_connection(fal_id, &mut connection).await
}
/// Returns Ok(None) when the record cannot be found.
pub async fn get_inference_job_by_fal_id_from_connection(fal_id: &str, mysql_connection: &mut PoolConnection<MySql>)
-> AnyhowResult<Option<FalJobDetails>>
{
// NB(bt): jobs.uuid_idempotency_token is the current way to reconstruct the hash of the
// TTS result since we don't store a bucket hash on the table. This is an ugly hack :(
// TODO(bt,2023-10-12): ^^^ Is this comment still accurate? I don't see that field referenced below.
let maybe_status = sqlx::query_as!(
RawJobRecord,
r#"
SELECT
jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,
jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,
jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,
jobs.maybe_external_third_party_id as `external_third_party_id`,
jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,
jobs.creator_ip_address,
jobs.created_at,
jobs.updated_at
FROM generic_inference_jobs as jobs
WHERE jobs.maybe_external_third_party = ?
AND jobs.maybe_external_third_party_id = ?
"#,
InferenceJobExternalThirdParty::Fal.to_str(),
fal_id
)
.fetch_one(&mut **mysql_connection)
.await;
let record = match maybe_status {
Ok(record) => record,
Err(err) => match err {
sqlx::Error::RowNotFound => return Ok(None),
_ => {
warn!("error querying job record: {:?}", err);
return Err(anyhow!("error querying job record: {:?}", err));
}
}
};
let record = raw_record_to_public_result(record)?;
Ok(Some(record))
}
fn raw_record_to_public_result(record: RawJobRecord) -> AnyhowResult<FalJobDetails> {
Ok(FalJobDetails {
job_token: record.job_token,
status: record.status,
external_third_party: record.external_third_party.ok_or_else(|| anyhow!("missing external_third_party"))?,
external_third_party_id: record.external_third_party_id.ok_or_else(|| anyhow!("missing external_third_party_id"))?,
maybe_creator_user_token: record.maybe_creator_user_token,
maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token,
creator_ip_address: record.creator_ip_address,
created_at: record.created_at,
updated_at: record.updated_at,
})
}
#[derive(Debug, Default)]
struct RawJobRecord {
pub job_token: InferenceJobToken,
pub status: JobStatusPlus,
// NB: Nullable, but required to be filled for this query
pub external_third_party: Option<InferenceJobExternalThirdParty>,
// NB: Nullable, but required to be filled for this query
pub external_third_party_id: Option<String>,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
pub creator_ip_address: String,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -150,6 +150,8 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
InferenceCategory::VideoFilter => Some("Video Filter"),
InferenceCategory::ImageGeneration => Some("Image Generation"),
InferenceCategory::VideoGeneration => Some("Video Generation"),
InferenceCategory::BackgroundRemoval => Some("Background Removal"),
InferenceCategory::Mocap => Some("Mocap"),
InferenceCategory::Workflow => Some("Workflow"),
InferenceCategory::FormatConversion => Some("format conversion"),
@@ -170,6 +172,8 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::VideoGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::BackgroundRemoval => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
@@ -0,0 +1,154 @@
use anyhow::anyhow;
use log::info;
use sqlx::MySqlPool;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use enums::common::job_status_plus::JobStatusPlus;
use enums::common::visibility::Visibility;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
use crate::errors::database_query_error::DatabaseQueryError;
use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
use crate::queries::tts::tts_inference_jobs::kill_tts_inference_jobs::JobStatus;
#[derive(Debug, Clone, Copy)]
pub enum FalCategory {
ImageGeneration,
VideoGeneration,
BackgroundRemoval,
}
pub struct InsertGenericInferenceForFalArgs<'a> {
pub uuid_idempotency_token: &'a str,
/// The external primary key identifier for the job.
pub maybe_external_third_party_id: &'a str,
pub fal_category: FalCategory,
pub maybe_inference_args: Option<GenericInferenceArgs>,
pub maybe_creator_user_token: Option<&'a UserToken>,
pub maybe_avt_token: Option<&'a AnonymousVisitorTrackingToken>,
pub creator_ip_address: &'a str,
pub creator_set_visibility: Visibility,
pub mysql_pool: &'a MySqlPool,
}
pub async fn insert_generic_inference_job_for_fal_queue(args: InsertGenericInferenceForFalArgs<'_>)
-> Result<InferenceJobToken, DatabaseQueryError>
{
let job_token = InferenceJobToken::generate();
let serialized_args_payload = serde_json::ser::to_string(&args.maybe_inference_args)
.map_err(|_e| anyhow!("could not encode inference args"))?;
const JOB_TYPE : InferenceJobType = InferenceJobType::FalQueue;
let (inference_category, product_category) =
match args.fal_category {
FalCategory::ImageGeneration => (
InferenceCategory::ImageGeneration,
InferenceJobProductCategory::FalImage
),
FalCategory::VideoGeneration => (
InferenceCategory::VideoGeneration,
InferenceJobProductCategory::FalVideo
),
FalCategory::BackgroundRemoval => (
InferenceCategory::BackgroundRemoval,
InferenceJobProductCategory::FalBgRemoval
),
};
let maybe_external_third_party = InferenceJobExternalThirdParty::Fal;
let maybe_external_third_party_id = "";
const STATUS : JobStatusPlus = JobStatusPlus::Pending;
let query = sqlx::query!(
r#"
INSERT INTO generic_inference_jobs
SET
token = ?,
uuid_idempotency_token = ?,
job_type = ?,
maybe_external_third_party = ?,
maybe_external_third_party_id = ?,
product_category = ?,
inference_category = ?,
maybe_model_type = NULL,
maybe_model_token = NULL,
maybe_input_source_token = NULL,
maybe_input_source_token_type = NULL,
maybe_download_url = NULL,
maybe_cover_image_media_file_token = NULL,
maybe_raw_inference_text = NULL,
maybe_inference_args = ?,
maybe_creator_user_token = ?,
maybe_creator_anonymous_visitor_token = ?,
creator_ip_address = ?,
creator_set_visibility = ?,
priority_level = 0,
is_keepalive_required = FALSE,
max_duration_seconds = 0,
is_debug_request = FALSE,
maybe_routing_tag = NULL,
status = ?
"#,
job_token.as_str(),
args.uuid_idempotency_token,
JOB_TYPE.to_str(),
maybe_external_third_party.to_str(),
args.maybe_external_third_party_id,
product_category.to_str(),
inference_category.to_str(),
serialized_args_payload,
args.maybe_creator_user_token.map(|t| t.to_string()),
args.maybe_avt_token.map(|t| t.to_string()),
args.creator_ip_address,
args.creator_set_visibility.to_str(),
STATUS.to_str(),
);
let query_result = query.execute(args.mysql_pool)
.await;
let record_id = match query_result {
Err(err) => return Err(DatabaseQueryError::from(err)),
Ok(res) => res.last_insert_id(),
};
info!("Insert generic inference job for FAL queue: {job_token} with record ID {record_id}");
Ok(job_token)
}
@@ -241,6 +241,8 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
InferenceCategory::VideoFilter => Some("Video Filter"),
InferenceCategory::ImageGeneration => Some("Image Generation"),
InferenceCategory::VideoGeneration => Some("Video Generation"),
InferenceCategory::BackgroundRemoval => Some("Background Removal"),
InferenceCategory::Mocap => Some("Mocap"),
InferenceCategory::Workflow => Some("Workflow"),
InferenceCategory::FormatConversion => Some("format conversion"),
@@ -260,6 +262,8 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::VideoGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::BackgroundRemoval => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
@@ -3,9 +3,11 @@ pub mod dismiss_finished_jobs_for_user;
pub mod get_inference_job_status;
pub mod get_pending_inference_job_count;
pub mod insert_generic_inference_job;
pub mod insert_generic_inference_job_for_fal_queue;
pub mod job_status;
pub mod kill_generic_inference_jobs;
pub mod kill_jobs_in_development;
pub mod list_session_jobs;
pub mod mark_generic_inference_job_cancelled_by_user;
pub mod mark_generic_inference_job_successfully_done_by_token;
pub mod mark_generic_inference_job_successfully_done_by_token;
pub mod get_inference_job_by_fal_id;
@@ -0,0 +1,33 @@
[package]
name = "artcraft_api_defs"
edition = "2024"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
name = "artcraft_api_defs"
path = "src/lib.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
[dependencies]
# Hackari workspace (faster compile times)
workspace-build-acceleration = { workspace = true }
# Internal
enums = { workspace = true }
tokens = { workspace = true }
# Serialization
serde = { workspace = true }
serde_derive = { workspace = true }
serde_json = { workspace = true }
utoipa = { version = "4", features = ["chrono", "actix_extras"] }
[dev-dependencies]
@@ -0,0 +1 @@
pub mod remove_image_background;
@@ -0,0 +1,20 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
#[derive(Serialize, Deserialize, ToSchema)]
pub struct RemoveImageBackgroundRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Source of the file to remove the background from.
/// It must be an image.
pub media_file_token: Option<MediaFileToken>,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct RemoveImageBackgroundResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1 @@
pub mod image;
@@ -0,0 +1 @@
pub mod generate;
@@ -35,10 +35,18 @@ pub enum InferenceCategory {
#[serde(rename = "voice_conversion")]
VoiceConversion,
/// Image generation (eg. Stable Diffusion 1.5)
/// Image generation (eg. Stable Diffusion 1.5), FAL-powered image generation, etc.
#[serde(rename = "image_generation")]
ImageGeneration,
/// FAL-powered video generation
#[serde(rename = "video_generation")]
VideoGeneration,
/// FAL-powered image background removal
#[serde(rename = "background_removal")]
BackgroundRemoval,
/// Turn video into animation data with mocap processing (eg. Mocapnet).
#[serde(rename = "mocap")]
Mocap,
@@ -89,6 +97,8 @@ impl InferenceCategory {
Self::TextToSpeech => "text_to_speech",
Self::VoiceConversion => "voice_conversion",
Self::ImageGeneration => "image_generation",
Self::VideoGeneration => "video_generation",
Self::BackgroundRemoval => "background_removal",
Self::Mocap => "mocap",
Self::Workflow => "workflow",
Self::F5TTS => "f5_tts",
@@ -107,6 +117,8 @@ impl InferenceCategory {
"text_to_speech" => Ok(Self::TextToSpeech),
"voice_conversion" => Ok(Self::VoiceConversion),
"image_generation" => Ok(Self::ImageGeneration),
"video_generation" => Ok(Self::VideoGeneration),
"background_removal" => Ok(Self::BackgroundRemoval),
"f5_tts" => Ok(Self::F5TTS),
"mocap" => Ok(Self::Mocap),
"workflow" => Ok(Self::Workflow),
@@ -128,6 +140,8 @@ impl InferenceCategory {
Self::TextToSpeech,
Self::VoiceConversion,
Self::ImageGeneration,
Self::VideoGeneration,
Self::BackgroundRemoval,
Self::Mocap,
Self::F5TTS,
Self::SeedVc,
@@ -155,6 +169,8 @@ mod tests {
assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech");
assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
assert_serialization(InferenceCategory::ImageGeneration, "image_generation");
assert_serialization(InferenceCategory::VideoGeneration, "video_generation");
assert_serialization(InferenceCategory::BackgroundRemoval, "background_removal");
assert_serialization(InferenceCategory::Mocap, "mocap");
assert_serialization(InferenceCategory::F5TTS, "f5_tts");
assert_serialization(InferenceCategory::SeedVc, "seed_vc");
@@ -172,6 +188,8 @@ mod tests {
assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
assert_eq!(InferenceCategory::ImageGeneration.to_str(), "image_generation");
assert_eq!(InferenceCategory::VideoGeneration.to_str(), "video_generation");
assert_eq!(InferenceCategory::BackgroundRemoval.to_str(), "background_removal");
assert_eq!(InferenceCategory::F5TTS.to_str(), "f5_tts");
assert_eq!(InferenceCategory::SeedVc.to_str(), "seed_vc");
assert_eq!(InferenceCategory::Mocap.to_str(), "mocap");
@@ -189,6 +207,8 @@ mod tests {
assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
assert_eq!(InferenceCategory::from_str("image_generation").unwrap(), InferenceCategory::ImageGeneration);
assert_eq!(InferenceCategory::from_str("video_generation").unwrap(), InferenceCategory::VideoGeneration);
assert_eq!(InferenceCategory::from_str("background_removal").unwrap(), InferenceCategory::BackgroundRemoval);
assert_eq!(InferenceCategory::from_str("f5_tts").unwrap(), InferenceCategory::F5TTS);
assert_eq!(InferenceCategory::from_str("seed_vc").unwrap(), InferenceCategory::SeedVc);
assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap);
@@ -202,22 +222,10 @@ mod tests {
#[test]
fn all_variants() {
// Static check
let mut variants = InferenceCategory::all_variants();
assert_eq!(variants.len(), 12);
assert_eq!(variants.pop_first(), Some(InferenceCategory::DeprecatedField));
assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));
assert_eq!(variants.pop_first(), Some(InferenceCategory::ImageGeneration));
assert_eq!(variants.pop_first(), Some(InferenceCategory::F5TTS));
assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap));
assert_eq!(variants.pop_first(), Some(InferenceCategory::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow));
assert_eq!(variants.pop_first(), Some(InferenceCategory::FormatConversion));
assert_eq!(variants.pop_first(), Some(InferenceCategory::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceCategory::VideoFilter));
assert_eq!(variants.pop_first(), Some(InferenceCategory::ConvertBvhToWorkflow));
assert_eq!(variants.pop_first(), None);
const EXPECTED_COUNT : usize = 15;
assert_eq!(InferenceCategory::all_variants().len(), EXPECTED_COUNT);
assert_eq!(InferenceCategory::iter().len(), EXPECTED_COUNT);
// Generated check
use strum::IntoEnumIterator;
@@ -0,0 +1,114 @@
use std::collections::BTreeSet;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
/// Used in the `generic_inference_jobs` table in `VARCHAR(16)` field `maybe_external_third_party`.
///
/// YOU CAN ADD NEW VALUES, BUT DO NOT CHANGE EXISTING VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum InferenceJobExternalThirdParty {
/// Fal jobs
#[serde(rename = "fal")]
#[default]
Fal,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
impl_enum_display_and_debug_using_to_str!(InferenceJobExternalThirdParty);
impl_mysql_enum_coders!(InferenceJobExternalThirdParty);
/// NB: Legacy API for older code.
impl InferenceJobExternalThirdParty {
pub fn to_str(&self) -> &'static str {
match self {
Self::Fal => "fal",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"fal" => Ok(Self::Fal),
_ => Err(format!("invalid value: {:?}", value)),
}
}
pub fn all_variants() -> BTreeSet<Self> {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::Fal,
])
}
}
#[cfg(test)]
mod tests {
use crate::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
use crate::test_helpers::assert_serialization;
mod explicit_checks {
use super::*;
#[test]
fn test_serialization() {
assert_serialization(InferenceJobExternalThirdParty::Fal, "fal");
}
#[test]
fn to_str() {
assert_eq!(InferenceJobExternalThirdParty::Fal.to_str(), "fal");
}
#[test]
fn from_str() {
assert_eq!(InferenceJobExternalThirdParty::from_str("fal").unwrap(), InferenceJobExternalThirdParty::Fal);
}
#[test]
fn all_variants() {
// Static check
const EXPECTED_COUNT : usize = 1;
assert_eq!(InferenceJobExternalThirdParty::all_variants().len(), EXPECTED_COUNT);
assert_eq!(InferenceJobExternalThirdParty::iter().len(), EXPECTED_COUNT);
// Generated check
use strum::IntoEnumIterator;
assert_eq!(InferenceJobExternalThirdParty::all_variants().len(), InferenceJobExternalThirdParty::iter().len());
}
}
mod mechanical_checks {
use super::*;
#[test]
fn variant_length() {
use strum::IntoEnumIterator;
assert_eq!(InferenceJobExternalThirdParty::all_variants().len(), InferenceJobExternalThirdParty::iter().len());
}
#[test]
fn round_trip() {
for variant in InferenceJobExternalThirdParty::all_variants() {
assert_eq!(variant, InferenceJobExternalThirdParty::from_str(variant.to_str()).unwrap());
assert_eq!(variant, InferenceJobExternalThirdParty::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, InferenceJobExternalThirdParty::from_str(&format!("{:?}", variant)).unwrap());
}
}
#[test]
fn serialized_length_ok_for_database() {
const MAX_LENGTH : usize = 32;
for variant in InferenceJobExternalThirdParty::all_variants() {
let serialized = variant.to_str();
assert!(serialized.len() > 0, "variant {:?} is too short", variant);
assert!(serialized.len() <= MAX_LENGTH, "variant {:?} is too long", variant);
}
}
}
}
@@ -21,6 +21,12 @@ pub enum InferenceJobProductCategory {
/// Download: GptSoVits
#[default]
DownloadGptSoVits,
// =============== FAL ===============
FalImage,
FalVideo,
FalBgRemoval,
// =============== TEXT TO SPEECH ===============
@@ -113,6 +119,9 @@ impl InferenceJobProductCategory {
pub fn to_str(&self) -> &'static str {
match self {
Self::DownloadGptSoVits => "download_gpt_so_vits",
Self::FalImage => "fal_image",
Self::FalVideo => "fal_video",
Self::FalBgRemoval => "fal_bg_removal",
Self::TtsGptSoVits => "tts_gpt_so_vits",
Self::TtsStyleTts2 => "tts_style_tts2",
Self::TtsTacotron2 => "tts_tacotron2",
@@ -141,6 +150,9 @@ impl InferenceJobProductCategory {
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"download_gpt_so_vits" => Ok(Self::DownloadGptSoVits),
"fal_image" => Ok(Self::FalImage),
"fal_video" => Ok(Self::FalVideo),
"fal_bg_removal" => Ok(Self::FalBgRemoval),
"tts_gpt_so_vits" => Ok(Self::TtsGptSoVits),
"tts_style_tts2" => Ok(Self::TtsStyleTts2),
"tts_tacotron2" => Ok(Self::TtsTacotron2),
@@ -172,6 +184,9 @@ impl InferenceJobProductCategory {
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::DownloadGptSoVits,
Self::FalImage,
Self::FalVideo,
Self::FalBgRemoval,
Self::TtsGptSoVits,
Self::TtsStyleTts2,
Self::TtsTacotron2,
@@ -209,6 +224,9 @@ mod tests {
#[test]
fn test_serialization() {
assert_serialization(InferenceJobProductCategory::DownloadGptSoVits, "download_gpt_so_vits");
assert_serialization(InferenceJobProductCategory::FalImage, "fal_image");
assert_serialization(InferenceJobProductCategory::FalVideo, "fal_video");
assert_serialization(InferenceJobProductCategory::FalBgRemoval, "fal_bg_removal");
assert_serialization(InferenceJobProductCategory::TtsGptSoVits, "tts_gpt_so_vits");
assert_serialization(InferenceJobProductCategory::TtsStyleTts2, "tts_style_tts2");
assert_serialization(InferenceJobProductCategory::TtsTacotron2, "tts_tacotron2");
@@ -236,6 +254,9 @@ mod tests {
#[test]
fn to_str() {
assert_eq!(InferenceJobProductCategory::DownloadGptSoVits.to_str(), "download_gpt_so_vits");
assert_eq!(InferenceJobProductCategory::FalImage.to_str(), "fal_image");
assert_eq!(InferenceJobProductCategory::FalVideo.to_str(), "fal_video");
assert_eq!(InferenceJobProductCategory::FalBgRemoval.to_str(), "fal_bg_removal");
assert_eq!(InferenceJobProductCategory::TtsGptSoVits.to_str(), "tts_gpt_so_vits");
assert_eq!(InferenceJobProductCategory::TtsStyleTts2.to_str(), "tts_style_tts2");
assert_eq!(InferenceJobProductCategory::TtsTacotron2.to_str(), "tts_tacotron2");
@@ -262,6 +283,9 @@ mod tests {
#[test]
fn from_str() {
assert_eq!(InferenceJobProductCategory::from_str("download_gpt_so_vits").unwrap(), InferenceJobProductCategory::DownloadGptSoVits);
assert_eq!(InferenceJobProductCategory::from_str("fal_image").unwrap(), InferenceJobProductCategory::FalImage);
assert_eq!(InferenceJobProductCategory::from_str("fal_video").unwrap(), InferenceJobProductCategory::FalVideo);
assert_eq!(InferenceJobProductCategory::from_str("fal_bg_removal").unwrap(), InferenceJobProductCategory::FalBgRemoval);
assert_eq!(InferenceJobProductCategory::from_str("tts_gpt_so_vits").unwrap(), InferenceJobProductCategory::TtsGptSoVits);
assert_eq!(InferenceJobProductCategory::from_str("tts_style_tts2").unwrap(), InferenceJobProductCategory::TtsStyleTts2);
assert_eq!(InferenceJobProductCategory::from_str("tts_tacotron2").unwrap(), InferenceJobProductCategory::TtsTacotron2);
@@ -288,32 +312,11 @@ mod tests {
#[test]
fn all_variants() {
// Static check
let mut variants = InferenceJobProductCategory::all_variants();
assert_eq!(variants.len(), 21);
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::DownloadGptSoVits));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::TtsGptSoVits));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::TtsStyleTts2));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::TtsTacotron2));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VcRvc2));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VcSeedVc));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VcSvc));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidLipsyncFaceFusion));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidLipsyncSadTalker));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidLivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidLivePortraitWebcam));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidStudio));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidStudioGen2));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidStyleTransfer));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::LipsyncFaceFusion));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::LipsyncSadTalker));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::LivePortraitWebcam));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::StableDiffusion));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::Studio));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::VidFaceFusion));
assert_eq!(variants.pop_first(), Some(InferenceJobProductCategory::Vst));
assert_eq!(variants.pop_first(), None);
const EXPECTED_COUNT : usize = 26;
assert_eq!(InferenceJobProductCategory::all_variants().len(), EXPECTED_COUNT);
assert_eq!(InferenceJobProductCategory::iter().len(), EXPECTED_COUNT);
// Generated check
use strum::IntoEnumIterator;
assert_eq!(InferenceJobProductCategory::all_variants().len(), InferenceJobProductCategory::iter().len());
@@ -15,6 +15,10 @@ use strum::EnumIter;
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum InferenceJobType {
/// Fal jobs do not run on our job runners.
/// Instead, we receive web hook updates from Fal.
FalQueue,
/// Storyteller Studio and Video Style Transfer Jobs (which we may want to split).
/// These run in Comfy.
/// TODO(bt,2024-07-15): We may segregate these two job types in the future
@@ -105,6 +109,7 @@ impl_mysql_enum_coders!(InferenceJobType);
impl InferenceJobType {
pub fn to_str(&self) -> &'static str {
match self {
Self::FalQueue => "fal_queue",
Self::VideoRender => "video_render",
Self::LivePortrait => "live_portrait",
Self::FaceFusion => "face_fusion",
@@ -130,6 +135,7 @@ impl InferenceJobType {
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"fal_queue" => Ok(Self::FalQueue),
"video_render" => Ok(Self::VideoRender),
"live_portrait" => Ok(Self::LivePortrait),
"face_fusion" => Ok(Self::FaceFusion),
@@ -158,6 +164,7 @@ impl InferenceJobType {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::FalQueue,
Self::VideoRender,
Self::LivePortrait,
Self::FaceFusion,
@@ -197,6 +204,7 @@ mod tests {
#[test]
fn test_serialization() {
assert_serialization(InferenceJobType::FalQueue, "fal_queue");
assert_serialization(InferenceJobType::VideoRender, "video_render");
assert_serialization(InferenceJobType::LivePortrait, "live_portrait");
assert_serialization(InferenceJobType::FaceFusion, "face_fusion");
@@ -221,6 +229,7 @@ mod tests {
#[test]
fn to_str() {
assert_eq!(InferenceJobType::FalQueue.to_str(), "fal_queue");
assert_eq!(InferenceJobType::VideoRender.to_str(), "video_render");
assert_eq!(InferenceJobType::LivePortrait.to_str(), "live_portrait");
assert_eq!(InferenceJobType::FaceFusion.to_str(), "face_fusion");
@@ -245,6 +254,7 @@ mod tests {
#[test]
fn from_str() {
assert_eq!(InferenceJobType::from_str("fal_queue").unwrap(), InferenceJobType::FalQueue);
assert_eq!(InferenceJobType::from_str("video_render").unwrap(), InferenceJobType::VideoRender);
assert_eq!(InferenceJobType::from_str("live_portrait").unwrap(), InferenceJobType::LivePortrait);
assert_eq!(InferenceJobType::from_str("face_fusion").unwrap(), InferenceJobType::FaceFusion);
@@ -270,29 +280,10 @@ mod tests {
#[test]
fn all_variants() {
// Static check
let mut variants = InferenceJobType::all_variants();
assert_eq!(variants.len(), 17);
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
assert_eq!(variants.pop_first(), Some(InferenceJobType::F5TTS));
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::SadTalker));
assert_eq!(variants.pop_first(), Some(InferenceJobType::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceJobType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StableDiffusion));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StyleTTS2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::Tacotron2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::Unknown));
assert_eq!(variants.pop_first(), Some(InferenceJobType::BevyToWorkflow));
assert_eq!(variants.pop_first(), Some(InferenceJobType::RerenderAVideo));
assert_eq!(variants.pop_first(), None);
const EXPECTED_COUNT : usize = 21;
assert_eq!(InferenceJobType::all_variants().len(), EXPECTED_COUNT);
assert_eq!(InferenceJobType::iter().len(), EXPECTED_COUNT);
// Generated check
use strum::IntoEnumIterator;
@@ -1,6 +1,7 @@
pub mod frontend_failure_category;
pub mod inference_category;
pub mod inference_input_source_token_type;
pub mod inference_job_external_third_party;
pub mod inference_job_product_category;
pub mod inference_job_type;
pub mod inference_model_type;
@@ -91,6 +91,36 @@ impl_mysql_from_row!(MediaFileType);
/// NB: Legacy API for older code.
impl MediaFileType {
/// Jpeg and Png are the most widely supported static image formats for AI inference.
/// Webp, Webm, and Gif aren't as widely supported, so we don't include them here.
pub fn is_jpg_or_png(&self) -> bool {
matches!(self, Self::Jpg | Self::Png)
}
/// A little bit less strict than `is_jpg_or_png`, this includes the legacy `Image` type.
pub fn is_jpg_or_png_or_legacy_image(&self) -> bool {
matches!(self, Self::Jpg | Self::Png | Self::Image)
}
/// Returns the `MediaFileType` if the mime type matches one of the known types.
/// This is not exhaustive.
/// (NB: In general we shouldn't keep much application logic in the `enums` crate,
/// but this is a good case for it.)
pub fn try_from_mime_type(mime_type: &str) -> Option<Self> {
match mime_type {
"image/jpeg" => Some(Self::Jpg),
"image/png" => Some(Self::Png),
//"audio/wav" => Some(Self::Wav),
//"audio/mpeg" => Some(Self::Mp3),
//"image/gif" => Some(Self::Gif),
//"video/mp4" => Some(Self::Mp4),
//"model/gltf-binary" => Some(Self::Glb),
//"model/gltf+json" => Some(Self::Gltf),
_ => None,
}
}
pub fn to_str(&self) -> &'static str {
match self {
Self::Audio => "audio",
@@ -171,6 +201,29 @@ impl MediaFileType {
mod tests {
use crate::by_table::media_files::media_file_type::MediaFileType;
use crate::test_helpers::assert_serialization;
mod utility {
use super::*;
#[test]
fn test_jpg_or_png() {
// True
assert!(MediaFileType::Jpg.is_jpg_or_png());
assert!(MediaFileType::Png.is_jpg_or_png());
// Assert these image types are false
assert!(!MediaFileType::Gif.is_jpg_or_png());
assert!(!MediaFileType::Image.is_jpg_or_png());
// Everything else is false
for variant in MediaFileType::all_variants() {
if matches!(variant, MediaFileType::Jpg | MediaFileType::Png) {
continue;
}
assert!(!variant.is_jpg_or_png(), "Expected {:?} to not be jpg or png", variant);
}
}
}
mod serde {
use super::*;
@@ -32,8 +32,13 @@ pub enum PublicInferenceJobType {
#[serde(rename = "f5_tts")]
F5TTS,
// ======= Everything else is the same =======
/// No need to hide this.
#[serde(rename = "fal_queue")]
FalQueue,
/// Jobs that run ComfyUI workflows
/// This is actually just for Video Style Transfer and Storyteller Studio.
#[deprecated(note = "Use VideoRender instead.")]
@@ -106,6 +111,7 @@ impl PublicInferenceJobType {
InferenceJobType::LivePortrait => Self::ActingFace,
InferenceJobType::FaceFusion => Self::Lipsync,
// Conserved variants
InferenceJobType::FalQueue => Self::FalQueue,
InferenceJobType::VideoRender => Self::VideoRender,
InferenceJobType::GptSovits => Self::GptSovits,
InferenceJobType::F5TTS => Self::F5TTS,
@@ -133,6 +139,7 @@ impl PublicInferenceJobType {
Self::ActingFace => InferenceJobType::LivePortrait,
Self::Lipsync => InferenceJobType::FaceFusion,
// Conserved variants
Self::FalQueue => InferenceJobType::FalQueue,
Self::VideoRender => InferenceJobType::VideoRender,
Self::GptSovits => InferenceJobType::GptSovits,
Self::F5TTS => InferenceJobType::F5TTS,
@@ -360,6 +360,11 @@ async fn old_dispatch(
return Err(ProcessSingleJobError::InvalidJob(
anyhow!("invalid job category for dispatch: {:?}", job.inference_category)))
}
InferenceCategory::VideoGeneration | InferenceCategory::BackgroundRemoval => {
// These are handled by the new dispatch
return Err(ProcessSingleJobError::InvalidJob(
anyhow!("these job types do not support the old-style dispatch: {:?}", job.inference_category)))
}
};
Ok(job_success_result)
@@ -31,6 +31,7 @@ workspace-build-acceleration = { workspace = true }
# Internal
actix_cors_configs = { path = "../../../lib/actix_cors_configs" }
actix_helpers = { path = "../../../lib/actix_helpers" }
artcraft_api_defs = { workspace = true }
bootstrap = { path = "../../../lib/bootstrap" }
bucket_paths = { path = "../../../lib/bucket_paths" }
cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
@@ -46,6 +47,7 @@ email_sender= { path = "../../../lib/email_sender" }
enums = { workspace = true }
enums_public = { workspace = true }
errors = { workspace = true }
fal_client = { workspace = true }
filesys = { workspace = true }
google_sign_in = { path = "../../../lib/api_clients/google_sign_in" }
hashing = { path = "../../../lib/files/hashing" }
@@ -122,7 +124,9 @@ elasticsearch = "8.5.0-alpha.1"
# Files
infer = "0.16.0"
tempdir = "0.3.7"
bytes = { workspace = true }
tempdir = { workspace = true }
tempfile = { workspace = true }
# The "never-encode-slash" feature seems to have been removed:
# https://github.com/durch/rust-s3/pull/254
@@ -41,3 +41,6 @@ GC_ENABLED_PUBLIC_BUCKET_NAME="REPLACE_ME"
# Legacy configs
AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
# Fake API key for FAL
FAL_WEBHOOK_URL="https://macaroni-1.tailce84f.ts.net/v1/webhooks/fal"
#FAL_WEBHOOK_URL="https://s1.pancake-chimaera.ts.net/api/fal/webhook"
@@ -0,0 +1 @@
pub mod remove_image_background_handler;
@@ -0,0 +1,213 @@
use std::fmt;
use std::sync::Arc;
use crate::http_server::common_requests::media_file_token_path_info::MediaFileTokenPathInfo;
use crate::http_server::common_responses::media::media_links::MediaLinks;
use crate::http_server::deprecated_endpoints::engine::create_scene_handler::CreateSceneError;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::web_utils::response_success_helpers::simple_json_success;
use crate::state::server_state::ServerState;
use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Json;
use actix_web::web::Path;
use actix_web::{web, HttpRequest, HttpResponse};
use artcraft_api_defs::generate::image::remove_image_background::RemoveImageBackgroundRequest;
use artcraft_api_defs::generate::image::remove_image_background::RemoveImageBackgroundResponse;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::common::visibility::Visibility;
use fal_client::requests::webhook::image::remove_background_rembg_webhook::{remove_background_rembg_webhook, RemoveBackgroundRembgWebhookArgs};
use http_server_common::request::get_request_ip::get_request_ip;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use idempotency::uuid::generate_random_uuid;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
// =============== Error Response ===============
#[derive(Debug, Serialize, ToSchema)]
pub enum RemoveImageBackgroundError {
BadInput(String),
NotFound,
NotAuthorized,
ServerError,
}
impl ResponseError for RemoveImageBackgroundError {
fn status_code(&self) -> StatusCode {
match *self {
RemoveImageBackgroundError::BadInput(_) => StatusCode::BAD_REQUEST,
RemoveImageBackgroundError::NotFound => StatusCode::NOT_FOUND,
RemoveImageBackgroundError::NotAuthorized => StatusCode::UNAUTHORIZED,
RemoveImageBackgroundError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for RemoveImageBackgroundError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// =============== Handler ===============
/// Background removal
#[utoipa::path(
post,
tag = "Generate Images",
path = "/v1/generate/image/remove_background",
responses(
(status = 200, description = "Success", body = RemoveImageBackgroundResponse),
(status = 400, description = "Bad input", body = RemoveImageBackgroundError),
(status = 401, description = "Not authorized", body = RemoveImageBackgroundError),
(status = 500, description = "Server error", body = RemoveImageBackgroundError),
),
params(
("request" = RemoveImageBackgroundRequest, description = "Payload for Request"),
)
)]
pub async fn remove_image_background_handler(
http_request: HttpRequest,
request: Json<RemoveImageBackgroundRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<RemoveImageBackgroundResponse>, RemoveImageBackgroundError> {
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session(&http_request, &server_state.mysql_pool)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
RemoveImageBackgroundError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(RemoveImageBackgroundError::NotAuthorized);
// }
//};
let media_file_token = match &request.media_file_token {
Some(token) => token,
None => {
warn!("No media file token provided");
return Err(RemoveImageBackgroundError::BadInput("No media file token provided".to_string()));
}
};
insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
RemoveImageBackgroundError::BadInput("invalid idempotency token".to_string())
})?;
const IS_MOD : bool = false;
let media_file_lookup_result = get_media_file(
media_file_token,
IS_MOD,
&server_state.mysql_pool,
).await;
let media_file = match media_file_lookup_result {
Ok(Some(media_file)) => media_file,
Ok(None) => {
warn!("MediaFile not found: {:?}", media_file_token);
return Err(RemoveImageBackgroundError::NotFound);
},
Err(err) => {
warn!("Error looking up media_file: {:?}", err);
return Err(RemoveImageBackgroundError::ServerError);
}
};
if !media_file.media_type.is_jpg_or_png_or_legacy_image() {
return Err(RemoveImageBackgroundError::BadInput("Media file must be a JPG or PNG image".to_string()));
}
let media_domain = get_media_domain(&http_request);
let bucket_path = MediaFileBucketPath::from_object_hash(
&media_file.public_bucket_directory_hash,
media_file.maybe_public_bucket_prefix.as_deref(),
media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinks::from_media_path_and_env(
media_domain,
server_state.server_environment,
&bucket_path);
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let args = RemoveBackgroundRembgWebhookArgs {
image_url: media_links.cdn_url,
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
let fal_result = remove_background_rembg_webhook(args)
.await
.map_err(|err| {
warn!("Error calling remove_background_rembg_webhook: {:?}", err);
RemoveImageBackgroundError::ServerError
})?;
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
RemoveImageBackgroundError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let uuid_idempotency_token = generate_random_uuid();
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::BackgroundRemoval,
maybe_inference_args: None,
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_pool: &server_state.mysql_pool,
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(RemoveImageBackgroundError::ServerError);
}
};
Ok(Json(RemoveImageBackgroundResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,2 @@
pub mod image;
pub mod video;
@@ -70,6 +70,10 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option
InferenceCategory::F5TTS => percent(duration_seconds, F5_TTS_JOB_AVERAGE_SECONDS),
InferenceCategory::SeedVc => percent(duration_seconds, SEED_VC_JOB_AVERAGE_SECONDS),
// TODO: Better estimates for FAL, etc.
InferenceCategory::VideoGeneration => percent(duration_seconds, 60*5),
InferenceCategory::BackgroundRemoval => percent(duration_seconds, 10),
// TODO: Better estimate using video duration, params, etc.
InferenceCategory::Workflow => comfy_workflow_estimate(maybe_args, duration_seconds),
@@ -4,6 +4,7 @@ pub mod beta_keys;
pub mod comments;
pub mod download_job;
pub mod featured_items;
pub mod generate;
pub mod image_studio;
pub mod inference_job;
pub mod media_files;
@@ -18,10 +19,11 @@ pub mod subscriptions;
pub mod tags;
pub mod trending;
pub mod tts;
pub mod users;
pub mod user_bookmarks;
pub mod user_ratings;
pub mod users;
pub mod voice_clone_requests;
pub mod voice_conversion;
pub mod voice_designer;
pub mod webhooks;
pub mod weights;
@@ -0,0 +1,147 @@
use std::fmt;
use std::sync::Arc;
use crate::http_server::endpoints::webhooks::handle_image_payload::handle_image_payload;
use crate::state::server_state::ServerState;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use anyhow::Error;
use errors::AnyhowResult;
use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use log::{info, warn};
use serde_json::Value;
use utoipa::ToSchema;
use mysql_queries::queries::generic_inference::web::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
// 1. tauri --> hit endpoint to enqueue
//
// 2. webhook
// - upload media file
// - update job record with media token + status
//
// 3. tauri --> storyteller jobs endpoint polls
// - alert frontend of completion
//
// 4. javascript polls tauri (backend removal)
// TODO(bt, 2025-06-03): Handle webhook crypto authentication
#[derive(Debug, Deserialize, ToSchema)]
pub struct FalWebhookRequest {
pub status: FalWebhookStatus,
pub request_id: Option<String>,
pub gateway_request_id: Option<String>,
pub error: Option<String>,
/// Payload of the webhook, if any.
pub payload: Option<Value>,
}
#[derive(Debug, Deserialize, ToSchema)]
pub enum FalWebhookStatus {
#[serde(alias = "OK")]
Ok,
#[serde(alias = "ERROR")]
Error
}
// =============== Error Response ===============
#[derive(Debug, Serialize, ToSchema)]
pub enum FalWebhookError {
BadInput(String),
NotFound,
NotAuthorized,
ServerError,
}
impl ResponseError for FalWebhookError {
fn status_code(&self) -> StatusCode {
match *self {
FalWebhookError::BadInput(_) => StatusCode::BAD_REQUEST,
FalWebhookError::NotFound => StatusCode::NOT_FOUND,
FalWebhookError::NotAuthorized => StatusCode::UNAUTHORIZED,
FalWebhookError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for FalWebhookError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl From<anyhow::Error> for FalWebhookError {
fn from(value: Error) -> Self {
info!("Converting anyhow::Error to FalWebhookError: {:?}", value);
FalWebhookError::ServerError
}
}
// =============== Handler ===============
/// Fal webhook
#[utoipa::path(
post,
tag = "Webooks",
path = "/v1/webhooks/fal",
responses(
(status = 200, description = "Success", body = SimpleGenericJsonSuccess),
(status = 400, description = "Bad input", body = FalWebhookError),
(status = 401, description = "Not authorized", body = FalWebhookError),
(status = 500, description = "Server error", body = FalWebhookError),
),
params(
("request" = FalWebhookRequest, description = "Payload for Request"),
)
)]
pub async fn fal_webhook_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
request: Json<FalWebhookRequest>,
) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
info!("Received FAL webhook body: {:?}", request);
let request_id = request.request_id
.as_deref()
.ok_or_else(|| FalWebhookError::BadInput("Missing request_id".to_string()))?;
let payload = request.payload
.as_ref()
.ok_or_else(|| FalWebhookError::BadInput("Missing payload".to_string()))?;
let db_result = get_inference_job_by_fal_id(
request_id,
&server_state.mysql_pool,
).await;
let job = match db_result {
Ok(Some(record)) => record,
Ok(None) => {
warn!("Could not find job record by fal request id: {:?} ; request payload = {:?}", request_id, request);
return Err(FalWebhookError::NotFound)
},
Err(err) => {
warn!("Error querying job record: {:?}", err);
return Err(FalWebhookError::ServerError);
}
};
if let Some(payload_obj) = payload.as_object() {
if payload_obj.contains_key("image") {
handle_image_payload(payload_obj, &job, &server_state).await?;
}
}
Ok(SimpleGenericJsonSuccess::wrapped(true))
}
@@ -0,0 +1,105 @@
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::state::server_state::ServerState;
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
use crate::util::http_download_url_to_tempfile::http_download_url_to_tempfile;
use actix_web::web;
use anyhow::anyhow;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use errors::AnyhowResult;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use images::image_info::image_info::ImageInfo;
use log::{info, warn};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use mysql_queries::queries::generic_inference::web::get_inference_job_by_fal_id::FalJobDetails;
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
use serde_json::{Map, Value};
use std::sync::Arc;
const PREFIX : Option<&str> = Some("image_");
#[derive(Deserialize, Debug)]
pub struct FalWebhookImage {
pub content_type: Option<String>,
pub file_name: Option<String>,
pub file_size: Option<usize>,
pub height: Option<usize>,
pub width: Option<usize>,
pub url: Option<String>,
}
pub async fn handle_image_payload(
payload: &Map<String, Value>,
job: &FalJobDetails,
server_state: &ServerState,
) -> AnyhowResult<()> {
let image_value = payload.get("image")
.ok_or_else(|| anyhow!("no `image` key in payload"))?;
let image: FalWebhookImage = serde_json::from_value(image_value.clone())?;
info!("Image payload received: {:?}", image);
let image_url = image.url
.as_deref()
.ok_or_else(|| anyhow!("no `url` in image payload"))?;
//let mime_type = image.content_type
// .as_deref()
// .ok_or_else(|| anyhow!("no `content_type` in image payload"))?;
let file_bytes = http_download_url_to_bytes(image_url)
.await
.map_err(|e| anyhow!("Failed to download image: {:?}", e))?;
let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes)
.ok_or_else(|| anyhow!("Failed to get mimetype info"))?;
let mime_type = mimetype_info.mime_type();
let media_file_type = MediaFileType::try_from_mime_type(mime_type)
.ok_or_else(|| anyhow!("Unsupported media file type: {}", mime_type))?;
let extension_with_period = mimetype_info.file_extension()
.map(|ext| ext.extension_with_period())
.ok_or_else(|| anyhow!("Failed to get file extension from mimetype info"))?;
let file_size_bytes = file_bytes.len();
let file_hash = sha256_hash_bytes(&file_bytes)?;
let image_info = ImageInfo::decode_image_from_bytes(&file_bytes)?;
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension_with_period));
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type_process(
public_upload_path.get_full_object_path_str(),
file_bytes.as_ref(),
&mime_type)
.await?;
let media_token = MediaFileInsertBuilder::new()
.maybe_creator_user(job.maybe_creator_user_token.as_ref())
.maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref())
.creator_ip_address(&job.creator_ip_address)
.public_bucket_directory_hash(&public_upload_path)
.media_file_class(MediaFileClass::Image)
.media_file_type(media_file_type)
.media_file_origin_category(MediaFileOriginCategory::Inference)
//.media_file_origin_product_category(MediaFileOriginProductCategory::Unknown)
.mime_type(mime_type)
.file_size_bytes(file_size_bytes as u64)
.frame_width(image_info.width())
.frame_height(image_info.height())
.checksum_sha2(&file_hash)
.insert_pool(&server_state.mysql_pool)
.await?;
info!("Media file uploaded with token: {}", media_token);
Ok(())
}
@@ -0,0 +1,2 @@
pub mod fal_webhook_handler;
pub mod handle_image_payload;
@@ -0,0 +1,26 @@
use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler;
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
pub fn add_generate_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/generate")
.service(web::scope("/image")
.service(web::resource("/remove_background")
.route(web::post().to(remove_image_background_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
)
}
@@ -124,8 +124,10 @@ use crate::http_server::endpoints::voice_designer::voices::list_voices_by_user::
use crate::http_server::endpoints::voice_designer::voices::search_voices::search_voices;
use crate::http_server::endpoints::voice_designer::voices::update_voice::update_voice_handler;
use crate::http_server::routes::add_control_plane_routes::add_control_plane_routes;
use crate::http_server::routes::add_generate_routes::add_generate_routes;
use crate::http_server::routes::add_image_studio_routes::add_image_studio_routes;
use crate::http_server::routes::add_studio_gen2_routes::add_studio_gen2_routes;
use crate::http_server::routes::add_webhook_routes::add_webhook_routes;
use crate::http_server::routes::beta_key_routes::add_beta_key_routes;
use crate::http_server::routes::job_routes::add_job_routes;
use crate::http_server::routes::media_files_routes::add_media_file_routes;
@@ -177,6 +179,8 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
app = add_job_routes(app);
app = add_engine_routes(app); /* /v1/engine/... */
app = add_control_plane_routes(app); /* /v1/control_plane/... */
app = add_generate_routes(app); /* /v1/generate/... */
app = add_webhook_routes(app); /* /v1/webhooks/... */
// app = add_image_gen_routes(app);
@@ -0,0 +1,24 @@
use crate::http_server::endpoints::webhooks::fal_webhook_handler::fal_webhook_handler;
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
pub fn add_webhook_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/webhooks")
.service(web::resource("/fal")
.route(web::post().to(fal_webhook_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -1,13 +1,15 @@
pub mod add_routes;
pub mod add_studio_gen2_routes;
pub mod beta_key_routes;
pub mod job_routes;
pub mod media_files_routes;
pub mod model_download_routes;
pub mod moderation_routes;
pub mod user_routes;
pub mod weights_routes;
pub mod workflow_routes;
mod tag_routes;
mod add_control_plane_routes;
mod add_generate_routes;
mod add_image_studio_routes;
mod add_control_plane_routes;
mod add_studio_gen2_routes;
mod add_webhook_routes;
mod beta_key_routes;
mod job_routes;
mod media_files_routes;
mod model_download_routes;
mod moderation_routes;
mod tag_routes;
mod user_routes;
mod weights_routes;
mod workflow_routes;
pub mod add_routes;
@@ -60,6 +60,7 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
use config::shared_constants::DEFAULT_RUST_LOG;
use email_sender::smtp_email_sender::SmtpEmailSender;
use errors::AnyhowResult;
use fal_client::creds::fal_api_key::FalApiKey;
use memory_caching::arc_ttl_sieve::ArcTtlSieve;
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
use mysql_queries::mediators::badge_granter::BadgeGranter;
@@ -85,7 +86,7 @@ use crate::http_server::web_utils::handle_multipart_error::handle_multipart_erro
use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator;
use crate::state::certs::google_sign_in_cert::GoogleSignInCert;
use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache;
use crate::state::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, InMemoryCaches, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans};
use crate::state::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, InMemoryCaches, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans};
use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread;
use crate::threads::poll_ip_banlist_thread::poll_ip_bans;
@@ -391,6 +392,9 @@ async fn main() -> AnyhowResult<()> {
ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
};
let fal_api_key = FalApiKey::new(easyenv::get_env_string_required("FAL_API_KEY")?);
let fal_webhook_url = easyenv::get_env_string_required("FAL_WEBHOOK_URL")?;
let server_state = ServerState {
env_config: EnvConfig {
@@ -431,6 +435,10 @@ async fn main() -> AnyhowResult<()> {
sort_key_crypto,
static_api_token_set,
email_sender,
fal: FalData {
api_key: fal_api_key,
webhook_url: fal_webhook_url,
},
caches: InMemoryCaches {
durable: DurableInMemoryCaches {
model_token_info: model_token_info_cache,
@@ -7,6 +7,7 @@ use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBan
use billing_component::stripe::stripe_config::StripeConfig;
use cloud_storage::bucket_client::BucketClient;
use email_sender::smtp_email_sender::SmtpEmailSender;
use fal_client::creds::fal_api_key::FalApiKey;
use memory_caching::arc_ttl_sieve::ArcTtlSieve;
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
use mysql_queries::mediators::badge_granter::BadgeGranter;
@@ -85,6 +86,8 @@ pub struct ServerState {
pub public_bucket_client: BucketClient,
pub auto_gc_bucket_client: BucketClient,
pub fal: FalData,
/// Where to store audio uploads for w2l
pub audio_uploads_bucket_root: String,
@@ -283,3 +286,10 @@ pub struct TrollBans {
pub user_tokens: TrollUserBanList,
pub ip_addresses: IpBanList,
}
/// Fal integration
#[derive(Clone)]
pub struct FalData {
pub api_key: FalApiKey,
pub webhook_url: String,
}
@@ -0,0 +1,24 @@
use bytes::Bytes;
use errors::AnyhowResult;
use reqwest::{Client, IntoUrl};
const USER_AGENT: &str = "storyteller-web/1.0";
// TODO(bt, 2025-06-03): Don't load the entire file into memory!
/// Downloads a (binary) file to memory. Good for images, etc. Not great for large files.
pub async fn http_download_url_to_bytes<U: IntoUrl>(url: U) -> AnyhowResult<Bytes> {
let client = Client::builder()
.gzip(true)
.build()?;
let response = client.get(url) // NB: No IntoUrl for &Url.
.header("User-Agent", USER_AGENT)
.header("Accept", "*/*")
.send()
.await?;
let bytes = response.bytes().await?;
Ok(bytes)
}
@@ -0,0 +1,14 @@
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
use errors::AnyhowResult;
use reqwest::IntoUrl;
use std::io::Write;
use tempfile::NamedTempFile;
const USER_AGENT: &str = "storyteller-web/1.0";
/// Downloads a (binary) file to a filesystem path. Good for images, etc. Not great for large files.
pub async fn http_download_url_to_tempfile<U: IntoUrl>(url: U, temp_file: &mut NamedTempFile) -> AnyhowResult<()> {
let bytes = http_download_url_to_bytes(url).await?;
temp_file.write_all(&bytes)?;
Ok(())
}
@@ -9,6 +9,8 @@ pub mod email_to_gravatar;
pub mod encrypted_sort_id;
pub mod enroll_in_studio;
pub mod generate_random_username;
pub mod http_download_url_to_bytes;
pub mod http_download_url_to_tempfile;
pub mod placeholder_images;
pub mod read_toml_file_to_struct;
pub mod title_to_url_slug;
+2
View File
@@ -33,6 +33,8 @@ all-features = true
eventsource-stream = "0.2"
smart-default = "0.7"
thiserror = "1"
percent-encoding-rfc3986 = "0.1.3"
log = "0.4.27"
[dependencies.base64]
version = "0.22"
+1
View File
@@ -7,6 +7,7 @@ pub mod endpoints;
pub mod image;
pub mod queue;
pub mod request;
pub mod webhook;
use eventsource_stream::EventStreamError;
pub use fal_derive::endpoint;
+35 -2
View File
@@ -1,11 +1,13 @@
use std::marker::PhantomData;
use percent_encoding_rfc3986::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::IntoUrl;
use serde::{de::DeserializeOwned, Serialize};
use log::info;
use crate::{
queue::{Queue, QueueResponse},
FalError,
};
use crate::webhook::WebhookResponse;
/// A request to the FAL API
///
@@ -129,4 +131,35 @@ impl<Params: Serialize, Response: DeserializeOwned> FalRequest<Params, Response>
Ok(Queue::new(self.client, self.endpoint, key, payload))
}
pub async fn queue_webhook<U: IntoUrl>(self, url: U) -> Result<WebhookResponse, FalError> {
let key = self
.api_key
.expect("No fal API key provided, and FAL_API_KEY environment variable is not set");
let url_encoded = url.into_url()?;
//let url_encoded = utf8_percent_encode(url_encoded.as_str(), NON_ALPHANUMERIC).to_string();
let request_url = format!("https://queue.fal.run/{}?fal_webhook={}", self.endpoint, url_encoded);
info!("Sending request to FAL queue webhook: {}", request_url);
let response = self
.client
.post(request_url)
.json(&self.params)
.header("Authorization", format!("Key {}", &key))
.header("Content-Type", "application/json")
.send()
.await?;
if response.status() != 200 {
let error = response.text().await?;
return Err(error.into());
}
let payload: WebhookResponse = response.error_for_status()?.json().await?;
Ok(payload)
}
}
+9
View File
@@ -0,0 +1,9 @@
use crate::prelude::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct WebhookResponse {
pub status: Option<String>,
pub request_id: Option<String>,
pub gateway_request_id: Option<String>,
}