Compare commits

...
Author SHA1 Message Date
bflatastic f1678c97b4 expand tooltip on hover 2026-04-16 10:59:53 +07:00
bflatastic 7c6cc86399 mention text area expand fixes 2026-04-16 10:57:19 +07:00
bflatastic b96300f0a5 website: expandable and resizable promptbox 2026-04-16 10:52:49 +07:00
Brandon Thomas 49e49efa3c worldlabs marble 1.1 client library (#1379) 2026-04-15 22:01:13 -04:00
BFlat d51c001835 video ref select video from library support (#1378)
## Video reference library picker

- Added "Pick from library" option to the Video Ref slot, with gallery
filtered to videos
- Added loading state to "Use selected" button in gallery modal
- Enforced seedance-2 limits (max 3 videos / 15s total) for both upload
and library flows, with a deduped video-ref-limit toast so users see a
warning on count overflow, duration overflow, or submitting when already
at max
2026-04-16 07:49:48 +07:00
BFlat f94fc5bf07 Fix @mention tagging in prompts with CJK characters (#1376)
Two bugs were preventing @image / @video / @character references from
being tagged in prompts written with Chinese, Japanese, or Korean text:

1. The mention-highlight regex was built with just the "g" flag, so a
user-typed "@image6" never matched the canonical label "@Image6" and the
reference was rendered as plain text. Make the regex case-insensitive
and do the colorMap lookup via a lowercase-keyed table so "@image6"
still finds the color for "@Image6".

2. detectMention required the character immediately before "@" to match
/\s/, which never matches CJK characters. Typing "@" right after a
Chinese character like "从" therefore never opened the autocomplete
dropdown, leaving users to type the mention by hand (which then hit bug
1). Replace the whitespace check with a "not an ASCII identifier char"
check so CJK, quotes, and punctuation all count as valid word boundaries
before an @mention.
2026-04-16 07:05:21 +07:00
Brandon Thomas f397ad825c worldlabs fixes for nsfw 500ing (#1377) 2026-04-15 20:05:07 -04:00
Brandon Thomas 7eea797883 more logging (endpoint failing without debugging info) (#1373) 2026-04-15 17:49:21 -04:00
bflatastic 73856fca85 loading spinner to wait for video thumbnail to return 2026-04-16 01:48:29 +07:00
bflatastic 3d27c620e4 site video time progress update 2026-04-16 01:35:28 +07:00
Brandon Thomas 062b23d3a0 omni seedance 2 (#1372) 2026-04-15 13:15:20 -04:00
BFlat 129fc3d593 enable omni gen api on website (#1370) 2026-04-15 01:58:56 -04:00
Brandon Thomas ad40798a12 chrome launch script (#1371) 2026-04-15 01:44:39 -04:00
bflatastic 3db8e66204 gallery to clear cache and reset after logout/login with new account 2026-04-14 23:35:52 +07:00
54 changed files with 2428 additions and 487 deletions
Generated
+1
View File
@@ -14793,6 +14793,7 @@ dependencies = [
"serde",
"serde_json",
"strum",
"test_data",
"tokio",
"uuid 1.22.0",
"workspace-build-acceleration",
@@ -15,7 +15,7 @@ body:
type: json
content: |
{
"user_token": "user_jg7ng4kdt2x5y", // newuser946
"user_token": "user_rtdhen9kz73kx", //
"payments_namespace": "artcraft"
}
order: 4100053395280358
@@ -34,3 +34,4 @@ env_logger.workspace = true
env.workspace = true
chrono.workspace = true
strum.workspace = true
test_data.workspace = true
@@ -4,12 +4,26 @@ pub enum WorldLabsModel {
/// `Marble 0.1-mini` is good for quick drafts
/// Generation time: 30-45 seconds
/// Cost: 150-330 credits
#[deprecated(note="Marble 0.1-mini is deprecated. Use `marble-1.0-draft` instead.")]
Marble0p1Mini,
/// `Marble 0.1-plus` is best for final renders
/// Generation time: ~5 minutes,
/// Cost: 1500-1600 credits
#[deprecated(note="Marble 0.1-plus is deprecated. Use `marble-1.0` instead.")]
Marble0p1Plus,
/// marble-1.0
Marble1p0,
/// marble-1.0-draft
Marble1p0Draft,
/// marble-1.1
Marble1p1,
/// marble-1.1-plus
Marble1p1Plus,
}
impl WorldLabsModel {
@@ -18,6 +32,159 @@ impl WorldLabsModel {
match self {
Self::Marble0p1Mini => "Marble 0.1-mini",
Self::Marble0p1Plus => "Marble 0.1-plus",
Self::Marble1p0 => "marble-1.0",
Self::Marble1p0Draft => "marble-1.0-draft",
Self::Marble1p1 => "marble-1.1",
Self::Marble1p1Plus => "marble-1.1-plus",
}
}
pub fn is_deprecated(&self) -> bool {
match self {
Self::Marble0p1Mini | Self::Marble0p1Plus => true,
_ => false,
}
}
pub fn to_new_value(self) -> Self {
match self {
Self::Marble0p1Mini => Self::Marble1p0Draft,
Self::Marble0p1Plus => Self::Marble1p0,
_ => self,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
mod api_name {
use super::*;
#[allow(deprecated)]
#[test]
fn marble_0p1_mini() {
assert_eq!(WorldLabsModel::Marble0p1Mini.get_model_api_name_str(), "Marble 0.1-mini");
}
#[allow(deprecated)]
#[test]
fn marble_0p1_plus() {
assert_eq!(WorldLabsModel::Marble0p1Plus.get_model_api_name_str(), "Marble 0.1-plus");
}
#[test]
fn marble_1p0() {
assert_eq!(WorldLabsModel::Marble1p0.get_model_api_name_str(), "marble-1.0");
}
#[test]
fn marble_1p0_draft() {
assert_eq!(WorldLabsModel::Marble1p0Draft.get_model_api_name_str(), "marble-1.0-draft");
}
#[test]
fn marble_1p1() {
assert_eq!(WorldLabsModel::Marble1p1.get_model_api_name_str(), "marble-1.1");
}
#[test]
fn marble_1p1_plus() {
assert_eq!(WorldLabsModel::Marble1p1Plus.get_model_api_name_str(), "marble-1.1-plus");
}
}
mod is_deprecated {
use super::*;
#[allow(deprecated)]
#[test]
fn marble_0p1_mini_is_deprecated() {
assert!(WorldLabsModel::Marble0p1Mini.is_deprecated());
}
#[allow(deprecated)]
#[test]
fn marble_0p1_plus_is_deprecated() {
assert!(WorldLabsModel::Marble0p1Plus.is_deprecated());
}
#[test]
fn marble_1p0_is_not_deprecated() {
assert!(!WorldLabsModel::Marble1p0.is_deprecated());
}
#[test]
fn marble_1p0_draft_is_not_deprecated() {
assert!(!WorldLabsModel::Marble1p0Draft.is_deprecated());
}
#[test]
fn marble_1p1_is_not_deprecated() {
assert!(!WorldLabsModel::Marble1p1.is_deprecated());
}
#[test]
fn marble_1p1_plus_is_not_deprecated() {
assert!(!WorldLabsModel::Marble1p1Plus.is_deprecated());
}
}
mod to_new_value {
use super::*;
#[allow(deprecated)]
#[test]
fn marble_0p1_mini_maps_to_1p0_draft() {
let new = WorldLabsModel::Marble0p1Mini.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.0-draft");
assert!(!new.is_deprecated());
}
#[allow(deprecated)]
#[test]
fn marble_0p1_plus_maps_to_1p0() {
let new = WorldLabsModel::Marble0p1Plus.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.0");
assert!(!new.is_deprecated());
}
#[test]
fn marble_1p0_stays_as_1p0() {
let new = WorldLabsModel::Marble1p0.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.0");
}
#[test]
fn marble_1p0_draft_stays_as_1p0_draft() {
let new = WorldLabsModel::Marble1p0Draft.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.0-draft");
}
#[test]
fn marble_1p1_stays_as_1p1() {
let new = WorldLabsModel::Marble1p1.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.1");
}
#[test]
fn marble_1p1_plus_stays_as_1p1_plus() {
let new = WorldLabsModel::Marble1p1Plus.to_new_value();
assert_eq!(new.get_model_api_name_str(), "marble-1.1-plus");
}
#[test]
fn new_models_are_not_deprecated_after_conversion() {
for model in [
WorldLabsModel::Marble1p0,
WorldLabsModel::Marble1p0Draft,
WorldLabsModel::Marble1p1,
WorldLabsModel::Marble1p1Plus,
] {
let new = model.to_new_value();
assert!(!new.is_deprecated(), "{:?} should not be deprecated after to_new_value", model);
}
}
}
}
@@ -33,10 +33,15 @@ pub struct GenerateWorldResponse {
pub async fn generate_world(args: GenerateWorldArgs<'_>) -> Result<GenerateWorldResponse, WorldLabsError> {
let client = Client::new();
let model_name = args.model
.to_new_value()
.get_model_api_name_str()
.to_string();
let payload = RawRequest {
world_prompt: args.world_prompt,
display_name: args.display_name,
model: Some(args.model.get_model_api_name_str().to_string()),
model: Some(model_name),
seed: args.seed,
tags: args.tags,
permission: args.permission,
@@ -90,8 +95,11 @@ pub async fn generate_world(args: GenerateWorldArgs<'_>) -> Result<GenerateWorld
mod tests {
use super::*;
use crate::api::requests::generate_world::http_request::{ContentReference, WorldPrompt};
use crate::api::requests::prepare_upload::prepare_upload::{prepare_upload, PrepareUploadArgs, MediaAssetKind};
use crate::api::requests::upload_to_signed_url::upload_to_signed_url::{upload_to_signed_url, UploadToSignedUrlArgs};
use crate::test_utils::get_test_api_key::get_test_api_key;
use crate::test_utils::setup_test_logging::setup_test_logging;
use filesys::file_read_bytes::file_read_bytes;
use log::LevelFilter;
#[tokio::test]
@@ -151,4 +159,291 @@ mod tests {
assert_eq!(1, 2);
}
/// End-to-end: upload a safe image step by step (prepare → upload →
/// generate_world), printing each step's result for debugging.
///
/// Uses a synthetic gradient PNG from test_data/.
#[tokio::test]
#[ignore]
async fn test_generate_world_for_safe_image() {
setup_test_logging(LevelFilter::Debug);
let creds = get_test_api_key().unwrap();
// --- Read file ---
let file_bytes = file_read_bytes("/Users/bt/dev/storyteller/artcraft/test_data/image/mochi.jpg")
.expect("failed to read safe test image");
println!("Read {} bytes from safe test image", file_bytes.len());
// --- Step 1: prepare_upload ---
println!("\n=== Step 1: prepare_upload ===");
let prepare_result = prepare_upload(PrepareUploadArgs {
creds: &creds,
file_name: "mochi.jpg",
kind: MediaAssetKind::Image,
request_timeout: None,
}).await;
println!("prepare_upload result: {:?}", prepare_result.as_ref().map(|r| &r.media_asset_id));
let prepare_response = match prepare_result {
Ok(r) => {
println!(" Media Asset ID: {}", r.media_asset_id.as_str());
println!(" Upload URL: {}", r.upload_url);
println!(" Required headers: {:?}", r.required_headers);
r
}
Err(err) => {
println!(" ERROR at prepare_upload: {:?}", err);
panic!("prepare_upload failed: {:?}", err);
}
};
// --- Step 2: upload_to_signed_url ---
println!("\n=== Step 2: upload_to_signed_url ===");
let upload_result = upload_to_signed_url(UploadToSignedUrlArgs {
upload_url: &prepare_response.upload_url,
file_bytes,
required_headers: &prepare_response.required_headers,
content_type: "image/jpeg",
request_timeout: None,
}).await;
match &upload_result {
Ok(()) => println!(" Upload succeeded."),
Err(err) => {
println!(" ERROR at upload_to_signed_url: {:?}", err);
panic!("upload_to_signed_url failed: {:?}", err);
}
}
// --- Step 3: generate_world ---
println!("\n=== Step 3: generate_world ===");
let generate_result = generate_world(GenerateWorldArgs {
creds: &creds,
world_prompt: WorldPrompt::Image {
image_prompt: ContentReference::MediaAsset {
media_asset_id: prepare_response.media_asset_id.as_str().to_string(),
},
text_prompt: Some("A colorful gradient landscape".to_string()),
is_pano: None,
disable_recaption: None,
},
display_name: None,
model: WorldLabsModel::Marble0p1Mini,
seed: None,
tags: None,
permission: None,
request_timeout: None,
}).await;
match &generate_result {
Ok(r) => {
println!(" SUCCESS — safe image");
println!(" Operation ID: {}", r.operation_id.as_str());
println!(" Done: {}", r.done);
}
Err(err) => {
println!(" ERROR at generate_world: {:?}", err);
}
}
assert_eq!(1, 2, "Inspect output above");
}
/// End-to-end: upload an NSFW image step by step (prepare → upload →
/// generate_world), printing each step's result for debugging.
///
/// We expect the NSFW rejection to happen at one of these steps.
/// This test helps us identify *which* step returns the moderation error
/// and what the error response body looks like.
///
/// NB: Place the NSFW test image at `test_data/images/nsfw_test_image.png`.
/// This file is NOT checked into git.
#[tokio::test]
#[ignore]
async fn test_generate_world_for_nsfw_image() {
setup_test_logging(LevelFilter::Debug);
let creds = get_test_api_key().unwrap();
// --- Read file ---
let file_bytes = file_read_bytes("/Users/bt/Pictures/nsfw_test/nsfw_test_1.jpg")
.expect("failed to read NSFW test image");
println!("Read {} bytes from NSFW test image", file_bytes.len());
// --- Step 1: prepare_upload ---
println!("\n=== Step 1: prepare_upload ===");
let prepare_result = prepare_upload(PrepareUploadArgs {
creds: &creds,
file_name: "nsfw_test_1.jpg",
kind: MediaAssetKind::Image,
request_timeout: None,
}).await;
let prepare_response = match prepare_result {
Ok(r) => {
println!(" OK — Media Asset ID: {}", r.media_asset_id.as_str());
println!(" Upload URL: {}", r.upload_url);
r
}
Err(err) => {
println!(" NSFW CAUGHT AT STEP 1 (prepare_upload)!");
println!(" Error (Debug): {:?}", err);
println!(" Error (Display): {}", err);
assert_eq!(1, 2, "NSFW caught at prepare_upload — inspect output above");
unreachable!();
}
};
// --- Step 2: upload_to_signed_url ---
println!("\n=== Step 2: upload_to_signed_url ===");
let upload_result = upload_to_signed_url(UploadToSignedUrlArgs {
upload_url: &prepare_response.upload_url,
file_bytes,
required_headers: &prepare_response.required_headers,
content_type: "image/jpeg",
request_timeout: None,
}).await;
match &upload_result {
Ok(()) => println!(" OK — upload succeeded."),
Err(err) => {
println!(" NSFW CAUGHT AT STEP 2 (upload_to_signed_url)!");
println!(" Error (Debug): {:?}", err);
println!(" Error (Display): {}", err);
assert_eq!(1, 2, "NSFW caught at upload — inspect output above");
}
}
// --- Step 3: generate_world ---
println!("\n=== Step 3: generate_world ===");
let generate_result = generate_world(GenerateWorldArgs {
creds: &creds,
world_prompt: WorldPrompt::Image {
image_prompt: ContentReference::MediaAsset {
media_asset_id: prepare_response.media_asset_id.as_str().to_string(),
},
text_prompt: None,
is_pano: None,
disable_recaption: None,
},
display_name: None,
model: WorldLabsModel::Marble0p1Mini,
seed: None,
tags: None,
permission: None,
request_timeout: None,
}).await;
match &generate_result {
Ok(r) => {
println!(" UNEXPECTED SUCCESS — NSFW image was NOT rejected at generate_world!");
println!(" Operation ID: {}", r.operation_id.as_str());
println!(" Done: {}", r.done);
}
Err(err) => {
println!(" NSFW CAUGHT AT STEP 3 (generate_world)!");
println!(" Error (Debug): {:?}", err);
println!(" Error (Display): {}", err);
}
}
assert_eq!(1, 2, "Inspect output above");
}
/// Marble 1.1: generate world from a mountain/tree image via URI.
#[tokio::test]
#[ignore]
async fn test_generate_world_marble_1p1_image_uri() {
setup_test_logging(LevelFilter::Debug);
let creds = get_test_api_key().unwrap();
let image_url = test_data::web::image_urls::MOUNTAIN_TREE_IMAGE_URL;
println!("=== generate_world with Marble 1.1 ===");
println!("Image URL: {}", image_url);
let result = generate_world(GenerateWorldArgs {
creds: &creds,
world_prompt: WorldPrompt::Image {
image_prompt: ContentReference::Uri {
uri: image_url.to_string(),
},
text_prompt: Some("Mountain landscape with a tree".to_string()),
is_pano: None,
disable_recaption: None,
},
display_name: None,
model: WorldLabsModel::Marble1p1,
seed: None,
tags: None,
permission: None,
request_timeout: None,
}).await;
match &result {
Ok(r) => {
println!("SUCCESS — Marble 1.1");
println!(" Operation ID: {}", r.operation_id.as_str());
println!(" Done: {}", r.done);
}
Err(err) => {
println!("ERROR — Marble 1.1");
println!(" Error (Debug): {:?}", err);
println!(" Error (Display): {}", err);
}
}
assert_eq!(1, 2, "Inspect output above");
}
/// Marble 1.1-plus: generate world from a wide fall mountains image via URI.
#[tokio::test]
#[ignore]
async fn test_generate_world_marble_1p1_plus_image_uri() {
setup_test_logging(LevelFilter::Debug);
let creds = get_test_api_key().unwrap();
let image_url = test_data::web::image_urls::SUPER_WIDE_FALL_MOUNTAINS_IMAGE_URL;
println!("=== generate_world with Marble 1.1-plus ===");
println!("Image URL: {}", image_url);
let result = generate_world(GenerateWorldArgs {
creds: &creds,
world_prompt: WorldPrompt::Image {
image_prompt: ContentReference::Uri {
uri: image_url.to_string(),
},
text_prompt: Some("Autumn mountain panorama".to_string()),
is_pano: None,
disable_recaption: None,
},
display_name: None,
model: WorldLabsModel::Marble1p1Plus,
seed: None,
tags: None,
permission: None,
request_timeout: None,
}).await;
match &result {
Ok(r) => {
println!("SUCCESS — Marble 1.1-plus");
println!(" Operation ID: {}", r.operation_id.as_str());
println!(" Done: {}", r.done);
}
Err(err) => {
println!("ERROR — Marble 1.1-plus");
println!(" Error (Debug): {:?}", err);
println!(" Error (Display): {}", err);
}
}
assert_eq!(1, 2, "Inspect output above");
}
}
@@ -195,7 +195,8 @@ mod tests {
let creds = get_test_api_key().unwrap();
let operation_id = OperationId("1fab3bf1-05a1-4929-907e-c6df07c539e2".to_string());
//let operation_id = OperationId("1fab3bf1-05a1-4929-907e-c6df07c539e2".to_string());
let operation_id = OperationId("a0894fa6-594d-4680-8f3a-5109f22664b1".to_string());
let response = get_operation(GetOperationArgs {
creds: &creds,
@@ -16,6 +16,11 @@ pub fn filter_world_labs_http_error(status_code: StatusCode, maybe_body: Option<
_ => {},
}
// Check for NSFW content policy rejection (403 with nsfw/sexuality keywords).
if let Some(nsfw_error) = check_nsfw_content_rejection(status_code, maybe_body) {
return Err(nsfw_error.into());
}
if let Some(body) = maybe_body {
return Err(WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody {
status_code,
@@ -28,3 +33,163 @@ pub fn filter_world_labs_http_error(status_code: StatusCode, maybe_body: Option<
body: String::new(),
}.into())
}
/// Check if a 403 response is an NSFW content policy rejection.
/// Returns `Some(WorldLabsSpecificApiError)` if it matches, `None` otherwise.
fn check_nsfw_content_rejection(
status_code: StatusCode,
maybe_body: Option<&str>,
) -> Option<WorldLabsSpecificApiError> {
if status_code != StatusCode::FORBIDDEN {
return None;
}
let body = maybe_body?;
let body_lower = body.to_lowercase();
if body_lower.contains("nsfw") || body_lower.contains("sexuality") {
Some(WorldLabsSpecificApiError::NsfwContentPolicyRejected {
message: Some(body.to_string()),
})
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn success_200_returns_ok() {
let result = filter_world_labs_http_error(StatusCode::OK, None);
assert!(result.is_ok());
}
#[test]
fn success_200_with_body_returns_ok() {
let result = filter_world_labs_http_error(StatusCode::OK, Some("{}"));
assert!(result.is_ok());
}
// --- NSFW 403 ---
#[test]
fn nsfw_403_real_response() {
let body = r#"{"detail":"NSFW Content is not permitted by policy. Sexuality"}"#;
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, Some(body));
let err = result.unwrap_err();
assert!(err.is_403_forbidden());
assert!(matches!(err, WorldLabsError::ApiSpecific(
WorldLabsSpecificApiError::NsfwContentPolicyRejected { .. }
)));
}
#[test]
fn nsfw_403_preserves_message() {
let body = r#"{"detail":"NSFW Content is not permitted by policy. Sexuality"}"#;
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, Some(body));
let err = result.unwrap_err();
if let WorldLabsError::ApiSpecific(WorldLabsSpecificApiError::NsfwContentPolicyRejected { message }) = err {
assert_eq!(message.as_deref(), Some(body));
} else {
panic!("expected NsfwContentPolicyRejected, got: {:?}", err);
}
}
#[test]
fn nsfw_403_case_insensitive_nsfw() {
let body = r#"{"detail":"nsfw content detected"}"#;
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, Some(body));
assert!(matches!(result.unwrap_err(), WorldLabsError::ApiSpecific(
WorldLabsSpecificApiError::NsfwContentPolicyRejected { .. }
)));
}
#[test]
fn nsfw_403_case_insensitive_sexuality() {
let body = r#"{"detail":"Rejected due to Sexuality content"}"#;
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, Some(body));
assert!(matches!(result.unwrap_err(), WorldLabsError::ApiSpecific(
WorldLabsSpecificApiError::NsfwContentPolicyRejected { .. }
)));
}
// --- Non-NSFW 403 ---
#[test]
fn non_nsfw_403_is_generic_error() {
let body = r#"{"detail":"Forbidden: invalid API key"}"#;
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, Some(body));
let err = result.unwrap_err();
assert!(err.is_403_forbidden());
assert!(matches!(err, WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { .. }
)));
}
#[test]
fn non_nsfw_403_no_body_is_generic_error() {
let result = filter_world_labs_http_error(StatusCode::FORBIDDEN, None);
let err = result.unwrap_err();
assert!(err.is_403_forbidden());
assert!(matches!(err, WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { .. }
)));
}
// --- 402 Payment Required ---
#[test]
fn payment_required_402() {
let result = filter_world_labs_http_error(StatusCode::PAYMENT_REQUIRED, Some("{}"));
let err = result.unwrap_err();
assert!(matches!(err, WorldLabsError::ApiSpecific(
WorldLabsSpecificApiError::InsufficientCredits
)));
assert!(!err.is_403_forbidden());
}
// --- Other errors ---
#[test]
fn server_error_500_is_generic() {
let body = r#"{"detail":"Internal server error"}"#;
let result = filter_world_labs_http_error(StatusCode::INTERNAL_SERVER_ERROR, Some(body));
let err = result.unwrap_err();
assert!(!err.is_403_forbidden());
assert!(matches!(err, WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { .. }
)));
}
#[test]
fn bad_request_400_is_generic() {
let body = r#"{"detail":"Bad request"}"#;
let result = filter_world_labs_http_error(StatusCode::BAD_REQUEST, Some(body));
assert!(matches!(result.unwrap_err(), WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { .. }
)));
}
#[test]
fn nsfw_keyword_in_non_403_is_not_nsfw_error() {
// Even if the body mentions "nsfw", a 400 should NOT be classified as NSFW rejection.
let body = r#"{"detail":"nsfw check failed"}"#;
let result = filter_world_labs_http_error(StatusCode::BAD_REQUEST, Some(body));
assert!(matches!(result.unwrap_err(), WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { .. }
)));
}
#[test]
fn error_500_no_body() {
let result = filter_world_labs_http_error(StatusCode::INTERNAL_SERVER_ERROR, None);
assert!(matches!(result.unwrap_err(), WorldLabsError::ApiGeneric(
WorldLabsGenericApiError::UncategorizedBadResponseWithStatusAndBody { body, .. }
) if body.is_empty()));
}
}
@@ -10,6 +10,16 @@ pub enum WorldLabsError {
ApiGeneric(WorldLabsGenericApiError),
}
impl WorldLabsError {
pub fn is_403_forbidden(&self) -> bool {
match self {
Self::ApiSpecific(e) => e.is_403_forbidden(),
Self::ApiGeneric(e) => e.is_403_forbidden(),
Self::Client(_) => false,
}
}
}
impl Error for WorldLabsError {}
impl std::fmt::Display for WorldLabsError {
@@ -24,6 +24,18 @@ pub enum WorldLabsGenericApiError {
WreqError(wreq::Error),
}
impl WorldLabsGenericApiError {
pub fn is_403_forbidden(&self) -> bool {
match self {
Self::UncategorizedBadResponseWithStatusAndBody { status_code, .. }
| Self::GoogleUploadFailed { status_code, .. } => {
*status_code == StatusCode::FORBIDDEN
}
_ => false,
}
}
}
impl Error for WorldLabsGenericApiError {}
impl Display for WorldLabsGenericApiError {
@@ -5,6 +5,20 @@ use std::fmt::{Display, Formatter};
pub enum WorldLabsSpecificApiError {
/// There aren't sufficient funds for generation.
InsufficientCredits,
/// The content was rejected by WorldLabs' NSFW / content policy filter.
NsfwContentPolicyRejected {
message: Option<String>,
},
}
impl WorldLabsSpecificApiError {
pub fn is_403_forbidden(&self) -> bool {
match self {
Self::NsfwContentPolicyRejected { .. } => true,
_ => false,
}
}
}
impl Error for WorldLabsSpecificApiError {}
@@ -13,6 +27,13 @@ impl Display for WorldLabsSpecificApiError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::InsufficientCredits => write!(f, "Insufficient credits"),
Self::NsfwContentPolicyRejected { message } => {
write!(f, "NSFW content policy rejection")?;
if let Some(msg) = message {
write!(f, ": {}", msg)?;
}
Ok(())
}
}
}
}
@@ -42,6 +42,38 @@ fn credits_for(model: WorldLabsModel, input_type: InputType) -> u32 {
(WorldLabsModel::Marble0p1Plus, InputType::ImageNonPanorama) => 1580,
(WorldLabsModel::Marble0p1Plus, InputType::MultiImage) => 1600,
(WorldLabsModel::Marble0p1Plus, InputType::Video) => 1600,
// Marble 1.0 (successor to 0.1-plus)
(WorldLabsModel::Marble1p0, InputType::ImagePanorama) => 1500,
(WorldLabsModel::Marble1p0, InputType::Text) => 1580,
(WorldLabsModel::Marble1p0, InputType::ImageNonPanorama) => 1580,
(WorldLabsModel::Marble1p0, InputType::MultiImage) => 1600,
(WorldLabsModel::Marble1p0, InputType::Video) => 1600,
// Marble 1.0-draft (successor to 0.1-mini)
(WorldLabsModel::Marble1p0Draft, InputType::ImagePanorama) => 150,
(WorldLabsModel::Marble1p0Draft, InputType::Text) => 230,
(WorldLabsModel::Marble1p0Draft, InputType::ImageNonPanorama) => 230,
(WorldLabsModel::Marble1p0Draft, InputType::MultiImage) => 250,
(WorldLabsModel::Marble1p0Draft, InputType::Video) => 250,
// Marble 1.1
(WorldLabsModel::Marble1p1, InputType::ImagePanorama) => 1500,
(WorldLabsModel::Marble1p1, InputType::Text) => 1580,
(WorldLabsModel::Marble1p1, InputType::ImageNonPanorama) => 1580,
(WorldLabsModel::Marble1p1, InputType::MultiImage) => 1600,
(WorldLabsModel::Marble1p1, InputType::Video) => 1600,
// TODO: Looks like there's no way for us to know what the price will be
// for us to bill this accurately, we'll have to bill this after the fact.
// also - they don't have an API for us to ascertain this??! Wat.
// Marble 1.1-plus (range pricing — we use the max to charge enough upfront)
// Actual range: 1500–3000 / 1580–3080 / 1600–3100
(WorldLabsModel::Marble1p1Plus, InputType::ImagePanorama) => 3000,
(WorldLabsModel::Marble1p1Plus, InputType::Text) => 3080,
(WorldLabsModel::Marble1p1Plus, InputType::ImageNonPanorama) => 3080,
(WorldLabsModel::Marble1p1Plus, InputType::MultiImage) => 3100,
(WorldLabsModel::Marble1p1Plus, InputType::Video) => 3100,
}
}
@@ -64,6 +96,10 @@ mod tests {
let models = [
WorldLabsModel::Marble0p1Mini,
WorldLabsModel::Marble0p1Plus,
WorldLabsModel::Marble1p0,
WorldLabsModel::Marble1p0Draft,
WorldLabsModel::Marble1p1,
WorldLabsModel::Marble1p1Plus,
];
println!("\n{:<20} {:<20} {:>10} {:>12}", "Model", "Input Type", "Credits", "USD Cents");
@@ -101,6 +137,59 @@ mod tests {
assert_eq!(calculate_cost(WorldLabsModel::Marble0p1Plus, InputType::Video).worldlabs_credits, 1600);
}
#[test]
fn test_marble_1p0_draft_credits() {
// Same as Marble 0.1-mini (its successor)
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0Draft, InputType::Text).worldlabs_credits, 230);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0Draft, InputType::ImageNonPanorama).worldlabs_credits, 230);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0Draft, InputType::ImagePanorama).worldlabs_credits, 150);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0Draft, InputType::MultiImage).worldlabs_credits, 250);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0Draft, InputType::Video).worldlabs_credits, 250);
}
#[test]
fn test_marble_1p0_credits() {
// Standard model, same pricing as Marble 0.1-plus
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0, InputType::Text).worldlabs_credits, 1580);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0, InputType::ImageNonPanorama).worldlabs_credits, 1580);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0, InputType::ImagePanorama).worldlabs_credits, 1500);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0, InputType::MultiImage).worldlabs_credits, 1600);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p0, InputType::Video).worldlabs_credits, 1600);
}
#[test]
fn test_marble_1p1_credits() {
// Standard model, same pricing tier as 1.0
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1, InputType::Text).worldlabs_credits, 1580);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1, InputType::ImageNonPanorama).worldlabs_credits, 1580);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1, InputType::ImagePanorama).worldlabs_credits, 1500);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1, InputType::MultiImage).worldlabs_credits, 1600);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1, InputType::Video).worldlabs_credits, 1600);
}
#[test]
fn test_marble_1p1_plus_credits() {
// Plus model uses max of the range pricing (1500–3000, 1580–3080, 1600–3100)
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1Plus, InputType::Text).worldlabs_credits, 3080);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1Plus, InputType::ImageNonPanorama).worldlabs_credits, 3080);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1Plus, InputType::ImagePanorama).worldlabs_credits, 3000);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1Plus, InputType::MultiImage).worldlabs_credits, 3100);
assert_eq!(calculate_cost(WorldLabsModel::Marble1p1Plus, InputType::Video).worldlabs_credits, 3100);
}
#[test]
fn test_marble_1p1_plus_is_more_expensive_than_standard() {
for input_type in InputType::iter() {
let standard = calculate_cost(WorldLabsModel::Marble1p1, input_type);
let plus = calculate_cost(WorldLabsModel::Marble1p1Plus, input_type);
assert!(
plus.worldlabs_credits > standard.worldlabs_credits,
"1.1-plus should cost more than 1.1 for {:?}: {} vs {}",
input_type, plus.worldlabs_credits, standard.worldlabs_credits,
);
}
}
#[test]
fn test_dollar_conversion() {
// 1250 credits = $1.00 = 100 cents
@@ -120,6 +209,10 @@ mod tests {
let models = [
WorldLabsModel::Marble0p1Mini,
WorldLabsModel::Marble0p1Plus,
WorldLabsModel::Marble1p0,
WorldLabsModel::Marble1p0Draft,
WorldLabsModel::Marble1p1,
WorldLabsModel::Marble1p1Plus,
];
for model in &models {
@@ -24,6 +24,10 @@ pub struct NotificationDetails {
/// Whether this notification originated from an error.
pub(crate) is_from_error: bool,
/// Extra side-channel messages, eg. if we know we're sending an error/exception,
/// but want to include an additional message. (The "description" field is messy atm.)
pub(crate) extra_message: Option<String>,
/// HTTP method associated with the event, if any.
pub(crate) http_method: Option<String>,
@@ -90,25 +94,30 @@ impl NotificationDetails {
sections.push(desc.clone());
}
// 2. Error chain
// 2. Extra message
if let Some(message) = &self.extra_message {
sections.push(message.clone());
}
// 3. Error chain
if let Some(error_section) = self.format_error_chain() {
sections.push(error_section);
}
// 3. Event time
// 4. Event time
sections.push(self.format_event_time());
// 4. HTTP context
// 5. HTTP context
if let Some(http_section) = self.format_http_context() {
sections.push(http_section);
}
// 5. Service tokens
// 6. Service tokens
if let Some(tokens_section) = self.format_service_tokens() {
sections.push(tokens_section);
}
// 6. Application/service identity and hostname
// 7. Application/service identity and hostname
if let Some(identity_section) = Self::format_app_identity(application_name, service_id, hostname) {
sections.push(identity_section);
}
@@ -14,6 +14,10 @@ pub struct NotificationDetailsBuilder {
pub(crate) maybe_error: Option<Arc<dyn std::error::Error + Send + Sync + 'static>>,
pub(crate) is_from_error: bool,
/// Extra side-channel messages, eg. if we know we're sending an error/exception,
/// but want to include an additional message. (The "description" field is messy atm.)
pub(crate) extra_message: Option<String>,
pub(crate) http_method: Option<String>,
pub(crate) http_path: Option<String>,
pub(crate) http_status_code: Option<u16>,
@@ -41,6 +45,7 @@ impl NotificationDetailsBuilder {
event_time: Utc::now(),
maybe_error: None,
is_from_error: false,
extra_message: None,
http_method: None,
http_path: None,
http_status_code: None,
@@ -70,6 +75,7 @@ impl NotificationDetailsBuilder {
event_time: Utc::now(),
maybe_error: Some(error),
is_from_error: true,
extra_message: None,
http_method: None,
http_path: None,
http_status_code: None,
@@ -106,6 +112,11 @@ impl NotificationDetailsBuilder {
self
}
pub fn set_extra_message(mut self, extra_message: Option<String>) -> Self {
self.extra_message = extra_message;
self
}
pub fn set_http_method(mut self, http_method: Option<String>) -> Self {
self.http_method = http_method;
self
@@ -176,6 +187,7 @@ impl NotificationDetailsBuilder {
event_time: self.event_time,
maybe_error: self.maybe_error,
is_from_error: self.is_from_error,
extra_message: self.extra_message,
http_method: self.http_method,
http_path: self.http_path,
http_status_code: self.http_status_code,
@@ -9,3 +9,9 @@ ENABLE_PAGING=true
# Doesn't make sense for jobs:
# ENABLE_PAGING_FOR_500S=true
# We don't need to look back that far in development
MAX_JOB_AGE_THRESHOLD_HOURS=5
# Don't collide with storyteller-web in development
HTTP_BIND_ADDRESS=0.0.0.0:11223
@@ -2,3 +2,6 @@
# Development
SERVER_ENVIRONMENT='development'
# Don't collide with storyteller-web in development
HTTP_BIND_ADDRESS=0.0.0.0:13333
@@ -9,8 +9,8 @@ COOKIE_DOMAIN=.jungle.horse
WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
# Paging
ENABLE_PAGING=true
ENABLE_PAGING_FOR_500S=true
ENABLE_PAGING=false
ENABLE_PAGING_FOR_500S=false
# Test the rate limiter bypass (obviously these are dummy values)
AI_STREAMER_USERNAMES="PewDiePie,xQc"
@@ -48,7 +48,6 @@ fn build_omni_gen_video_models() -> Vec<OmniGenVideoModelDetails> {
});
models.push(OmniGenVideoModelDetails {
is_disabled: Some(true), // TODO: Temporarily disable
model: CommonVideoModel::Seedance2p0,
full_name: None,
text_prompt_supported: Some(true),
@@ -36,13 +36,25 @@ use anyhow::anyhow;
pub enum AdvancedCommonWebError {
/// 400 Bad Request with a user-facing message.
BadInputWithSimpleMessage(String),
/// 401 Unauthorized.
NotAuthorized,
/// 404 Not Found.
NotFound,
/// 402 Payment Required.
PaymentRequired,
/// 403 Forbidden.
Forbidden,
/// 403 Forbidden — content was rejected by a content policy filter (e.g. NSFW).
ContentPolicyRejected,
/// 403 Forbidden — content was rejected, with a user-facing message.
ContentPolicyRejectedWithMessage(String),
/// Uncaught errors are always 500 Internal Server Error.
/// The user will never see the error cause or message, but our
/// middleware will handle alerting, logging, etc.
@@ -50,6 +62,12 @@ pub enum AdvancedCommonWebError {
/// Stored in `Arc` so the error alerting middleware can clone it for paging
/// without consuming the original.
UncaughtServerError(Arc<dyn std::error::Error + Send + Sync + 'static>),
/// Same as above, with a specified extra message string
UncaughtServerErrorWithInternalMessage {
internal_message: String,
error: Arc<dyn std::error::Error + Send + Sync + 'static>
}
}
// =============== Public accessors ===============
@@ -60,6 +78,13 @@ impl AdvancedCommonWebError {
Self::UncaughtServerError(Arc::new(error))
}
pub fn from_error_with_message(message: String, error: impl std::error::Error + Send + Sync + 'static) -> Self {
Self::UncaughtServerErrorWithInternalMessage {
internal_message: message,
error: Arc::new(error)
}
}
/// Wrap an `anyhow::Error` as an `UncaughtServerError`.
/// (`anyhow::Error` doesn't implement `std::error::Error`, so `from_error` can't accept it.)
pub fn from_anyhow_error(error: anyhow::Error) -> Self {
@@ -75,6 +100,7 @@ impl AdvancedCommonWebError {
pub fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
match self {
Self::UncaughtServerError(err) => Some(err.as_ref()),
Self::UncaughtServerErrorWithInternalMessage { error, .. } => Some(error.as_ref()),
_ => None,
}
}
@@ -84,13 +110,18 @@ impl AdvancedCommonWebError {
pub fn clone_cause_arc(&self) -> Option<Arc<dyn std::error::Error + Send + Sync + 'static>> {
match self {
Self::UncaughtServerError(err) => Some(Arc::clone(err)),
Self::UncaughtServerErrorWithInternalMessage { error, .. } => Some(Arc::clone(error)),
_ => None,
}
}
/// Whether this is a server error (500).
pub fn is_server_error(&self) -> bool {
matches!(self, Self::UncaughtServerError(_))
match self {
Self::UncaughtServerError(_) => true,
Self::UncaughtServerErrorWithInternalMessage { .. } => true,
_ => false,
}
}
}
@@ -103,7 +134,13 @@ impl Display for AdvancedCommonWebError {
Self::NotAuthorized => write!(f, "Not authorized"),
Self::NotFound => write!(f, "Not found"),
Self::PaymentRequired => write!(f, "Payment required"),
Self::Forbidden => write!(f, "Forbidden"),
Self::ContentPolicyRejected => write!(f, "Content policy rejected"),
Self::ContentPolicyRejectedWithMessage(msg) => write!(f, "Content policy rejected: {}", msg),
Self::UncaughtServerError(err) => write!(f, "Server error: {}", err),
Self::UncaughtServerErrorWithInternalMessage { internal_message, error } => {
write!(f, "Server error: {}: {}", internal_message, error)
}
}
}
}
@@ -115,7 +152,13 @@ impl std::fmt::Debug for AdvancedCommonWebError {
Self::NotAuthorized => write!(f, "NotAuthorized"),
Self::NotFound => write!(f, "NotFound"),
Self::PaymentRequired => write!(f, "PaymentRequired"),
Self::Forbidden => write!(f, "Forbidden"),
Self::ContentPolicyRejected => write!(f, "ContentPolicyRejected"),
Self::ContentPolicyRejectedWithMessage(msg) => write!(f, "ContentPolicyRejectedWithMessage({:?})", msg),
Self::UncaughtServerError(err) => write!(f, "UncaughtServerError({:?})", err),
Self::UncaughtServerErrorWithInternalMessage { internal_message, error } => {
write!(f, "UncaughtServerErrorWithInternalMessage({:?}, {:?})", internal_message, error)
}
}
}
}
@@ -124,6 +167,7 @@ impl std::error::Error for AdvancedCommonWebError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::UncaughtServerError(err) => Some(err.as_ref()),
Self::UncaughtServerErrorWithInternalMessage { error, .. } => Some(error.as_ref()),
_ => None,
}
}
@@ -138,7 +182,11 @@ impl ResponseError for AdvancedCommonWebError {
Self::NotAuthorized => StatusCode::UNAUTHORIZED,
Self::NotFound => StatusCode::NOT_FOUND,
Self::PaymentRequired => StatusCode::PAYMENT_REQUIRED,
Self::Forbidden => StatusCode::FORBIDDEN,
Self::ContentPolicyRejected => StatusCode::FORBIDDEN,
Self::ContentPolicyRejectedWithMessage(_) => StatusCode::FORBIDDEN,
Self::UncaughtServerError(_) => StatusCode::INTERNAL_SERVER_ERROR,
Self::UncaughtServerErrorWithInternalMessage { .. } => StatusCode::INTERNAL_SERVER_ERROR,
}
}
@@ -155,6 +203,24 @@ impl ResponseError for AdvancedCommonWebError {
message: msg,
})
}
Self::ContentPolicyRejected => {
HttpResponseBuilder::new(status)
.json(JsonErrorWithMessage {
success: false,
error_code: status.as_u16(),
error_code_str: status.canonical_reason(),
message: "Content rejected by policy",
})
}
Self::ContentPolicyRejectedWithMessage(msg) => {
HttpResponseBuilder::new(status)
.json(JsonErrorWithMessage {
success: false,
error_code: status.as_u16(),
error_code_str: status.canonical_reason(),
message: msg,
})
}
_ => {
HttpResponseBuilder::new(status)
.json(JsonErrorWithoutMessage {
@@ -432,5 +498,35 @@ mod tests {
assert!(std::error::Error::source(&AdvancedCommonWebError::NotFound).is_none());
assert!(std::error::Error::source(&AdvancedCommonWebError::NotAuthorized).is_none());
assert!(std::error::Error::source(&AdvancedCommonWebError::PaymentRequired).is_none());
assert!(std::error::Error::source(&AdvancedCommonWebError::Forbidden).is_none());
assert!(std::error::Error::source(&AdvancedCommonWebError::ContentPolicyRejected).is_none());
assert!(std::error::Error::source(&AdvancedCommonWebError::ContentPolicyRejectedWithMessage("test".to_string())).is_none());
}
#[test]
fn forbidden_returns_403() {
let error = AdvancedCommonWebError::Forbidden;
assert_eq!(error.status_code(), StatusCode::FORBIDDEN);
assert!(!error.is_server_error());
}
#[test]
fn content_policy_rejected_returns_403() {
let error = AdvancedCommonWebError::ContentPolicyRejected;
assert_eq!(error.status_code(), StatusCode::FORBIDDEN);
assert!(!error.is_server_error());
}
#[test]
fn content_policy_rejected_with_message_returns_403() {
let error = AdvancedCommonWebError::ContentPolicyRejectedWithMessage("NSFW detected".to_string());
assert_eq!(error.status_code(), StatusCode::FORBIDDEN);
assert!(!error.is_server_error());
let response = error.error_response();
let bytes = response.into_body().try_into_bytes().unwrap();
let body = String::from_utf8(bytes.to_vec()).unwrap();
assert!(body.contains("\"message\":\"NSFW detected\""));
assert!(body.contains("\"error_code\":403"));
}
}
@@ -114,7 +114,10 @@ pub async fn edit_character_handler(
.await
.map_err(|err| {
error!("Error updating character on Kinovi: {:?}", err);
AdvancedCommonWebError::from_error(err)
AdvancedCommonWebError::from_error_with_message(
"Error Updating Kinovi Character API".to_string(),
err,
)
})?;
info!("Updated character {} on Kinovi (name='{}')", kinovi_id, final_name);
@@ -1,7 +1,7 @@
use std::sync::Arc;
use crate::billing::wallets::attempt_wallet_deduction::attempt_wallet_deduction_else_common_web_error;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
use crate::http_server::endpoint_helpers::refund_wallet_after_api_failure::refund_wallet_after_api_failure;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
@@ -12,10 +12,10 @@ use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::splat::generate_worldlabs_marble_0p1_mini_splat::{GenerateWorldlabsMarble0p1MiniSplatRequest, GenerateWorldlabsMarble0p1MiniSplatResponse};
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::generation::common_model_type::CommonModelType;
use enums::common::visibility::Visibility;
use enums::common::generation::common_generation_mode::CommonGenerationMode;
use enums::common::generation::common_model_type::CommonModelType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::worldlabs::insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token::{insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token, InsertGenericInferenceForWorldlabsWithAprioriJobTokenArgs};
@@ -28,6 +28,8 @@ use worldlabs_api_client::api::api_types::world_labs_model::WorldLabsModel;
use worldlabs_api_client::api::requests::generate_world::generate_world::{generate_world, GenerateWorldArgs};
use worldlabs_api_client::api::requests::generate_world::http_request::{ContentReference, WorldPrompt};
use worldlabs_api_client::credentials::world_labs_api_creds::WorldLabsApiCreds;
use worldlabs_api_client::error::world_labs_error::WorldLabsError;
use worldlabs_api_client::error::world_labs_specific_api_error::WorldLabsSpecificApiError;
use worldlabs_api_client::pricing::check_pricing::{calculate_cost, InputType};
/// World Labs Marble 0.1-mini Splat Generation
@@ -46,18 +48,18 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
http_request: HttpRequest,
request: Json<GenerateWorldlabsMarble0p1MiniSplatRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<GenerateWorldlabsMarble0p1MiniSplatResponse>, CommonWebError> {
) -> Result<Json<GenerateWorldlabsMarble0p1MiniSplatResponse>, AdvancedCommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
if request.image_media_file_token.is_none() && request.prompt.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage(
return Err(AdvancedCommonWebError::BadInputWithSimpleMessage(
"At least one of image_media_file_token or prompt must be provided".to_string()
));
}
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
return Err(AdvancedCommonWebError::BadInputWithSimpleMessage(reason));
}
let mut mysql_connection = server_state.mysql_pool
@@ -70,7 +72,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
AdvancedCommonWebError::from(e)
})?;
let maybe_avt_token = server_state
@@ -80,7 +82,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
let user_token = match maybe_user_session.as_ref() {
Some(session) => &session.user_token,
None => {
return Err(CommonWebError::NotAuthorized);
return Err(AdvancedCommonWebError::NotAuthorized);
}
};
@@ -88,7 +90,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
AdvancedCommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
let apriori_job_token = InferenceJobToken::generate();
@@ -124,7 +126,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
let cdn_url = image_urls.get(image_token)
.ok_or_else(|| {
warn!("Image token not found in lookup results: {:?}", image_token);
CommonWebError::ServerError
AdvancedCommonWebError::server_error_with_message("Image token was not found in lookup results")
})?;
WorldPrompt::Image {
@@ -157,7 +159,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
Err(err) => {
warn!("World Labs generate_world error: {:?}", err);
refund_wallet_after_api_failure(&wallet_deduction.ledger_entry_token, &mut mysql_connection).await?;
return Err(CommonWebError::ServerError);
return Err(classify_worldlabs_error(err));
}
};
@@ -168,7 +170,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
AdvancedCommonWebError::from_error(err)
})?;
let prompt_result = insert_prompt(InsertPromptArgs {
@@ -237,7 +239,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for WorldLabs queue: {:?}", err);
return Err(CommonWebError::ServerError);
return Err(AdvancedCommonWebError::from_error(err));
}
};
@@ -246,7 +248,7 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
AdvancedCommonWebError::from_error(err)
})?;
Ok(Json(GenerateWorldlabsMarble0p1MiniSplatResponse {
@@ -254,3 +256,17 @@ pub async fn generate_worldlabs_marble_0p1_mini_splat_handler(
inference_job_token: job_token,
}))
}
fn classify_worldlabs_error(err: WorldLabsError) -> AdvancedCommonWebError {
if let WorldLabsError::ApiSpecific(WorldLabsSpecificApiError::NsfwContentPolicyRejected { message }) = &err {
return AdvancedCommonWebError::ContentPolicyRejectedWithMessage(
message.clone().unwrap_or_else(|| "Content rejected by policy".to_string())
);
}
if err.is_403_forbidden() {
return AdvancedCommonWebError::Forbidden;
}
AdvancedCommonWebError::from_error(err)
}
@@ -1,7 +1,7 @@
use std::sync::Arc;
use crate::billing::wallets::attempt_wallet_deduction::attempt_wallet_deduction_else_common_web_error;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
use crate::http_server::endpoint_helpers::refund_wallet_after_api_failure::refund_wallet_after_api_failure;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
@@ -12,10 +12,10 @@ use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::splat::generate_worldlabs_marble_0p1_plus_splat::{GenerateWorldlabsMarble0p1PlusSplatRequest, GenerateWorldlabsMarble0p1PlusSplatResponse};
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::generation::common_model_type::CommonModelType;
use enums::common::visibility::Visibility;
use enums::common::generation::common_generation_mode::CommonGenerationMode;
use enums::common::generation::common_model_type::CommonModelType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::worldlabs::insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token::{insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token, InsertGenericInferenceForWorldlabsWithAprioriJobTokenArgs};
@@ -28,6 +28,8 @@ use worldlabs_api_client::api::api_types::world_labs_model::WorldLabsModel;
use worldlabs_api_client::api::requests::generate_world::generate_world::{generate_world, GenerateWorldArgs};
use worldlabs_api_client::api::requests::generate_world::http_request::{ContentReference, WorldPrompt};
use worldlabs_api_client::credentials::world_labs_api_creds::WorldLabsApiCreds;
use worldlabs_api_client::error::world_labs_error::WorldLabsError;
use worldlabs_api_client::error::world_labs_specific_api_error::WorldLabsSpecificApiError;
use worldlabs_api_client::pricing::check_pricing::{calculate_cost, InputType};
/// World Labs Marble 0.1-plus Splat Generation
@@ -46,18 +48,18 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
http_request: HttpRequest,
request: Json<GenerateWorldlabsMarble0p1PlusSplatRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<GenerateWorldlabsMarble0p1PlusSplatResponse>, CommonWebError> {
) -> Result<Json<GenerateWorldlabsMarble0p1PlusSplatResponse>, AdvancedCommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
if request.image_media_file_token.is_none() && request.prompt.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage(
return Err(AdvancedCommonWebError::BadInputWithSimpleMessage(
"At least one of image_media_file_token or prompt must be provided".to_string()
));
}
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
return Err(AdvancedCommonWebError::BadInputWithSimpleMessage(reason));
}
let mut mysql_connection = server_state.mysql_pool
@@ -70,7 +72,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
AdvancedCommonWebError::from(e)
})?;
let maybe_avt_token = server_state
@@ -80,7 +82,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
let user_token = match maybe_user_session.as_ref() {
Some(session) => &session.user_token,
None => {
return Err(CommonWebError::NotAuthorized);
return Err(AdvancedCommonWebError::NotAuthorized);
}
};
@@ -88,7 +90,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
AdvancedCommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
let apriori_job_token = InferenceJobToken::generate();
@@ -124,7 +126,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
let cdn_url = image_urls.get(image_token)
.ok_or_else(|| {
warn!("Image token not found in lookup results: {:?}", image_token);
CommonWebError::ServerError
AdvancedCommonWebError::server_error_with_message("Image token no found in lookup result")
})?;
WorldPrompt::Image {
@@ -157,7 +159,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
Err(err) => {
warn!("World Labs generate_world error: {:?}", err);
refund_wallet_after_api_failure(&wallet_deduction.ledger_entry_token, &mut mysql_connection).await?;
return Err(CommonWebError::ServerError);
return Err(classify_worldlabs_error(err));
}
};
@@ -168,7 +170,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
AdvancedCommonWebError::from_error(err)
})?;
let prompt_result = insert_prompt(InsertPromptArgs {
@@ -237,7 +239,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for WorldLabs queue: {:?}", err);
return Err(CommonWebError::ServerError);
return Err(AdvancedCommonWebError::from_error(err));
}
};
@@ -246,7 +248,7 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
AdvancedCommonWebError::from_error(err)
})?;
Ok(Json(GenerateWorldlabsMarble0p1PlusSplatResponse {
@@ -254,3 +256,18 @@ pub async fn generate_worldlabs_marble_0p1_plus_splat_handler(
inference_job_token: job_token,
}))
}
fn classify_worldlabs_error(err: WorldLabsError) -> AdvancedCommonWebError {
if let WorldLabsError::ApiSpecific(WorldLabsSpecificApiError::NsfwContentPolicyRejected { message }) = &err {
return AdvancedCommonWebError::ContentPolicyRejectedWithMessage(
message.clone().unwrap_or_else(|| "Content rejected by policy".to_string())
);
}
if err.is_403_forbidden() {
return AdvancedCommonWebError::Forbidden;
}
AdvancedCommonWebError::from_error(err)
}
@@ -54,8 +54,11 @@ pub struct DistilledVideoRequest {
/// Cost estimate as computed by the Artcraft provider — this is what we bill on.
pub cost: VideoGenerationCostEstimate,
/// Execution plan as computed by the Fal provider — what we hand to the router.
/// Execution plan as computed by the execution provider — what we hand to the router.
plan: VideoGenerationPlan<'static>,
/// The provider used for execution (Fal, Seedance2Pro, etc.).
pub execution_provider: Provider,
}
impl DistilledVideoRequest {
@@ -76,6 +79,7 @@ impl DistilledVideoRequest {
pub fn distill_video_request(
request: &OmniGenVideoCostAndGenerateRequest,
media_file_hydration_map: Option<&HashMap<MediaFileToken, Url>>,
execution_provider: Provider,
) -> Result<DistilledVideoRequest, AdvancedCommonWebError> {
// 1. Convert the raw API request into a router request.
let initial = hydrate_to_router_request(request)?;
@@ -109,9 +113,8 @@ pub fn distill_video_request(
media_file_hydration_map,
)?;
// 3. Build a `GenerateVideoRequest<'static>` whose borrows point into the
// owned boxes. SAFETY: same invariant as the image version — each box
// lives in `Self` for the full lifetime of `DistilledVideoRequest`.
// 3. Build static lifetime references for text fields.
// SAFETY: each box lives in `Self` for the full lifetime of `DistilledVideoRequest`.
let prompt_static: Option<&'static str> = owned_prompt.as_deref().map(|s: &String| {
let raw: *const str = s.as_str();
unsafe { &*raw }
@@ -127,6 +130,23 @@ pub fn distill_video_request(
unsafe { &*raw }
});
// 4. Cost estimate (Artcraft provider).
// `initial` already has media fields in token form (from hydrate_to_router_request),
// which is exactly what the Artcraft cost builder needs. Just swap the provider.
let cost: VideoGenerationCostEstimate = {
let cost_request = GenerateVideoRequest {
provider: Provider::Artcraft,
..initial
};
let cost_plan = cost_request.build().map_err(|e| {
warn!("Failed to build cost plan during video distillation: {}", e);
AdvancedCommonWebError::from_error(e)
})?;
cost_plan.estimate_costs()
};
// 5. Execution plan (execution provider).
// The execution plan uses URL-form media references (resolved from the hydration map).
let start_frame_static: Option<ImageRef<'static>> =
owned_start_frame_url.as_deref().map(|s: &String| {
let raw: *const str = s.as_str();
@@ -155,7 +175,7 @@ pub fn distill_video_request(
let request_static: GenerateVideoRequest<'static> = GenerateVideoRequest {
model: initial.model,
provider: Provider::Fal,
provider: execution_provider,
prompt: prompt_static,
negative_prompt: negative_prompt_static,
start_frame: start_frame_static,
@@ -173,20 +193,6 @@ pub fn distill_video_request(
idempotency_token: idempotency_static,
};
// 4. Cost estimate (Artcraft provider).
let cost_request = GenerateVideoRequest {
provider: Provider::Artcraft,
..request_static
};
let cost_plan = cost_request.build().map_err(|e| {
warn!("Failed to build cost plan during video distillation: {}", e);
AdvancedCommonWebError::from_error(e)
})?;
let cost: VideoGenerationCostEstimate = cost_plan.estimate_costs();
drop(cost_plan);
drop(cost_request);
// 5. Execution plan (Fal provider).
let plan: VideoGenerationPlan<'static> = {
let request_ref: &'static GenerateVideoRequest<'static> = unsafe {
let raw: *const GenerateVideoRequest<'static> = &request_static;
@@ -210,6 +216,7 @@ pub fn distill_video_request(
request: request_static,
cost,
plan,
execution_provider,
})
}
@@ -0,0 +1,305 @@
//! Execution helpers for the omni-gen video generation handler.
//!
//! Routes generation to the appropriate provider (Fal or Seedance2Pro/Kinovi)
//! based on the distilled request's `execution_provider`.
use std::collections::HashMap;
use log::{info, warn};
use url::Url;
use artcraft_api_defs::omni_gen::cost_and_generate_requests::omni_gen_video_cost_and_generate_request::OmniGenVideoCostAndGenerateRequest;
use artcraft_router::api::provider::Provider;
use artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse;
use enums::common::generation::common_generation_mode::CommonGenerationMode;
use seedance2pro_client::creds::seedance2pro_session::Seedance2ProSession;
use seedance2pro_client::requests::generate_video::generate_video::{
generate_video, BatchCount, GenerateVideoArgs, ModelType, Resolution,
};
use seedance2pro_client::requests::prepare_file_upload::prepare_file_upload::{
prepare_file_upload, PrepareFileUploadArgs,
};
use seedance2pro_client::requests::upload_file::upload_file::{upload_file, UploadFileArgs};
use tokens::tokens::media_files::MediaFileToken;
use url_utils::extension::extract_extension_from_url::{
extract_extension_from_url, ExtractExtensions,
};
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
use crate::state::server_state::ServerState;
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
use super::distill_video_request::DistilledVideoRequest;
/// Result of a successful generation, regardless of provider.
pub struct GenerationResult {
/// The external job ID used to track the generation (first order_id for Seedance).
pub external_job_id: String,
/// Whether this is a Seedance2Pro generation (changes DB insertion path).
pub is_seedance2pro: bool,
/// For Seedance2Pro batch jobs, the list of all order IDs.
pub maybe_seedance_order_ids: Option<Vec<String>>,
/// The generation mode (Text, Keyframe, Reference).
pub generation_mode: CommonGenerationMode,
}
/// Execute video generation via the appropriate provider.
pub async fn execute_generation(
distilled: &DistilledVideoRequest,
request: &OmniGenVideoCostAndGenerateRequest,
server_state: &ServerState,
media_file_hydration_map: Option<&HashMap<MediaFileToken, Url>>,
) -> Result<GenerationResult, AdvancedCommonWebError> {
match distilled.execution_provider {
Provider::Seedance2Pro => {
execute_generation_kinovi(
distilled,
request,
server_state,
media_file_hydration_map,
).await
}
_ => {
execute_generation_fal(
distilled,
request,
server_state,
).await
}
}
}
/// Execute generation via Fal (the existing path).
async fn execute_generation_fal(
distilled: &DistilledVideoRequest,
request: &OmniGenVideoCostAndGenerateRequest,
server_state: &ServerState,
) -> Result<GenerationResult, AdvancedCommonWebError> {
let fal_client = artcraft_router::client::router_fal_client::RouterFalClient::new(
server_state.fal.api_key.clone(),
server_state.fal.webhook_url.clone(),
);
let router_client = artcraft_router::client::router_client::RouterClient::Fal(fal_client);
let generation_response = distilled.plan().generate_video(&router_client)
.await
.map_err(|e| {
warn!("Video generation failed (Fal): {:?}", e);
AdvancedCommonWebError::from_error(e)
})?;
let external_job_id = match &generation_response {
GenerateVideoResponse::Artcraft(p) => p.inference_job_token.as_str().to_string(),
GenerateVideoResponse::Muapi(p) => p.request_id.as_str().to_string(),
GenerateVideoResponse::Seedance2Pro(p) => p.order_id.clone(),
GenerateVideoResponse::Fal(p) => p.request_id.clone().unwrap_or_default(),
};
let generation_mode = if request.start_frame_image_media_token.is_some() {
CommonGenerationMode::Keyframe
} else {
CommonGenerationMode::Text
};
Ok(GenerationResult {
external_job_id,
is_seedance2pro: false,
maybe_seedance_order_ids: None,
generation_mode,
})
}
/// Execute generation via Seedance2Pro/Kinovi.
///
/// Downloads all referenced media from our CDN, re-uploads to Seedance2Pro's
/// CDN, then calls the Seedance2Pro generate endpoint.
async fn execute_generation_kinovi(
distilled: &DistilledVideoRequest,
request: &OmniGenVideoCostAndGenerateRequest,
server_state: &ServerState,
media_file_hydration_map: Option<&HashMap<MediaFileToken, Url>>,
) -> Result<GenerationResult, AdvancedCommonWebError> {
let session = Seedance2ProSession::from_cookies_string(
server_state.seedance2pro.cookies.clone()
);
let empty_map = HashMap::new();
let hydration_map = media_file_hydration_map.unwrap_or(&empty_map);
// Upload each media reference to Seedance2Pro CDN.
let start_frame_url = upload_token_to_seedance2pro(
&session, hydration_map, request.start_frame_image_media_token.as_ref(),
).await?;
let end_frame_url = upload_token_to_seedance2pro(
&session, hydration_map, request.end_frame_image_media_token.as_ref(),
).await?;
let reference_image_urls = upload_tokens_to_seedance2pro(
&session, hydration_map, request.reference_image_media_tokens.as_deref(),
).await?;
let reference_video_urls = upload_tokens_to_seedance2pro(
&session, hydration_map, request.reference_video_media_tokens.as_deref(),
).await?;
let reference_audio_urls = upload_tokens_to_seedance2pro(
&session, hydration_map, request.reference_audio_media_tokens.as_deref(),
).await?;
// Determine generation mode.
let is_keyframe = request.start_frame_image_media_token.is_some()
|| request.end_frame_image_media_token.is_some();
let is_reference = request.reference_image_media_tokens.as_ref().map_or(false, |t| !t.is_empty())
|| request.reference_video_media_tokens.as_ref().map_or(false, |t| !t.is_empty())
|| request.reference_audio_media_tokens.as_ref().map_or(false, |t| !t.is_empty());
let generation_mode = if is_keyframe {
CommonGenerationMode::Keyframe
} else if is_reference {
CommonGenerationMode::Reference
} else {
CommonGenerationMode::Text
};
// Map aspect ratio / duration / batch from the request.
let resolution = Resolution::Landscape16x9; // TODO: Map from request.aspect_ratio
let duration_seconds = request.duration_seconds.unwrap_or(5).clamp(4, 15) as u8;
let batch_count = match request.video_batch_count {
Some(2) => BatchCount::Two,
Some(4) => BatchCount::Four,
_ => BatchCount::One,
};
let prompt = request.prompt.clone().unwrap_or_default();
let video_gen_args = GenerateVideoArgs {
session: &session,
model_type: ModelType::Seedance2Pro,
prompt,
resolution,
duration_seconds,
batch_count,
start_frame_url,
end_frame_url,
reference_image_urls,
reference_video_urls,
reference_audio_urls,
character_ids: None, // TODO: resolve_kinovi_character_ids when character support is added
use_face_blur_hack: None,
host_override: None,
};
let gen_response = generate_video(video_gen_args).await
.map_err(|err| {
warn!("Seedance2Pro generate_video failed: {:?}", err);
AdvancedCommonWebError::from_error(err)
})?;
info!(
"Seedance2Pro generation: task_id={}, order_id={}",
gen_response.task_id, gen_response.order_id
);
let order_ids: Vec<String> = match gen_response.order_ids {
Some(ids) if !ids.is_empty() => ids,
_ => vec![gen_response.order_id.clone()],
};
Ok(GenerationResult {
external_job_id: gen_response.order_id,
is_seedance2pro: true,
maybe_seedance_order_ids: Some(order_ids),
generation_mode,
})
}
// --- Upload helpers ---
/// Upload a single media token (if present) to Seedance2Pro CDN.
async fn upload_token_to_seedance2pro(
session: &Seedance2ProSession,
hydration_map: &HashMap<MediaFileToken, Url>,
maybe_token: Option<&MediaFileToken>,
) -> Result<Option<String>, AdvancedCommonWebError> {
let token = match maybe_token {
None => return Ok(None),
Some(t) => t,
};
let url = hydration_map.get(token).ok_or_else(|| {
AdvancedCommonWebError::BadInputWithSimpleMessage(
format!("Media token not found in hydration map: {:?}", token))
})?;
let uploaded = upload_url_to_seedance2pro(session, url).await?;
Ok(Some(uploaded))
}
/// Upload a list of media tokens (if present) to Seedance2Pro CDN.
async fn upload_tokens_to_seedance2pro(
session: &Seedance2ProSession,
hydration_map: &HashMap<MediaFileToken, Url>,
maybe_tokens: Option<&[MediaFileToken]>,
) -> Result<Option<Vec<String>>, AdvancedCommonWebError> {
let tokens = match maybe_tokens {
None => return Ok(None),
Some(tokens) if tokens.is_empty() => return Ok(None),
Some(tokens) => tokens,
};
let mut urls = Vec::with_capacity(tokens.len());
for token in tokens {
let url = hydration_map.get(token).ok_or_else(|| {
AdvancedCommonWebError::BadInputWithSimpleMessage(
format!("Media token not found in hydration map: {:?}", token))
})?;
urls.push(upload_url_to_seedance2pro(session, url).await?);
}
Ok(Some(urls))
}
/// Download from our CDN and re-upload to Seedance2Pro.
async fn upload_url_to_seedance2pro(
session: &Seedance2ProSession,
our_cdn_url: &Url,
) -> Result<String, AdvancedCommonWebError> {
let extension = extract_extension_from_url(our_cdn_url, &ExtractExtensions::All)
.map(|ext| ext.without_period().to_string())
.unwrap_or_else(|| "png".to_string());
let file_bytes = http_download_url_to_bytes(our_cdn_url.as_str())
.await
.map_err(|err| {
warn!("Error downloading media from CDN for Seedance2Pro upload: {:?}", err);
AdvancedCommonWebError::from_error(err)
})?
.to_vec();
let prepare_result = prepare_file_upload(PrepareFileUploadArgs {
session,
extension,
host_override: None,
})
.await
.map_err(|err| {
warn!("Error preparing seedance2pro file upload: {:?}", err);
AdvancedCommonWebError::from_error(err)
})?;
let upload_result = upload_file(UploadFileArgs {
upload_url: prepare_result.upload_url,
file_bytes,
host_override: None,
})
.await
.map_err(|err| {
warn!("Error uploading file to seedance2pro: {:?}", err);
AdvancedCommonWebError::from_error(err)
})?;
Ok(upload_result.public_url)
}
@@ -1,6 +1,7 @@
pub mod omni_gen_video_generate_handler;
pub mod distill_video_request;
pub(crate) mod distill_helper;
mod execute_generation;
#[cfg(test)]
mod tests;
@@ -9,9 +9,10 @@ use url::Url;
use artcraft_api_defs::omni_gen::cost_and_generate_requests::omni_gen_video_cost_and_generate_request::OmniGenVideoCostAndGenerateRequest;
use artcraft_api_defs::omni_gen::generate_response::omni_gen_video_generate_response::OmniGenVideoGenerateResponse;
use artcraft_router::api::provider::Provider;
use enums::common::generation::common_video_model::CommonVideoModel;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation::common_generation_mode::CommonGenerationMode;
use enums::common::generation::common_model_type::CommonModelType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::visibility::Visibility;
@@ -21,6 +22,10 @@ use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job
insert_generic_inference_job_for_fal_queue_with_apriori_job_token,
InsertGenericInferenceForFalWithAprioriJobTokenArgs,
};
use mysql_queries::queries::generic_inference::seedance2pro::insert_generic_inference_job_for_seedance2pro_queue_with_apriori_job_token::{
insert_generic_inference_job_for_seedance2pro_queue_with_apriori_job_token,
InsertGenericInferenceForSeedance2ProWithAprioriJobTokenArgs,
};
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{
insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem,
@@ -37,6 +42,7 @@ use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use super::distill_video_request::distill_video_request;
use super::execute_generation::execute_generation;
/// Generate a video using the omni-gen unified endpoint.
#[utoipa::path(
@@ -149,12 +155,20 @@ pub async fn omni_gen_video_generate_handler(
}
};
// ==================== DETERMINE PROVIDER ==================== //
let execution_provider = match request.model {
Some(CommonVideoModel::Seedance2p0) => Provider::Seedance2Pro,
_ => Provider::Fal,
};
// ==================== DISTILL ==================== //
let distilled = distill_video_request(&request, media_file_hydration_map.as_ref())?;
let distilled = distill_video_request(&request, media_file_hydration_map.as_ref(), execution_provider)?;
info!(">>> Distilled cost: {:?}", distilled.cost);
info!(">>> Distilled plan: {:?}", distilled.plan());
info!(">>> Execution provider: {:?}", distilled.execution_provider);
// ==================== BILLING ==================== //
@@ -175,33 +189,12 @@ pub async fn omni_gen_video_generate_handler(
// ==================== EXECUTE GENERATION ==================== //
let fal_client = artcraft_router::client::router_fal_client::RouterFalClient::new(
server_state.fal.api_key.clone(),
server_state.fal.webhook_url.clone(),
);
let router_client = artcraft_router::client::router_client::RouterClient::Fal(fal_client);
let generation_response = distilled.plan().generate_video(&router_client)
.await
.map_err(|e| {
warn!("Video generation failed: {:?}", e);
AdvancedCommonWebError::from_error(e)
})?;
let external_job_id = match &generation_response {
artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse::Artcraft(p) => {
p.inference_job_token.as_str().to_string()
}
artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse::Muapi(p) => {
p.request_id.as_str().to_string()
}
artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse::Seedance2Pro(p) => {
p.order_id.clone()
}
artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse::Fal(p) => {
p.request_id.clone().unwrap_or_default()
}
};
let gen_result = execute_generation(
&distilled,
&request,
&server_state,
media_file_hydration_map.as_ref(),
).await?;
// ==================== DB TRANSACTION ==================== //
@@ -217,12 +210,6 @@ pub async fn omni_gen_video_generate_handler(
// -- Prompt --
let generation_mode = if request.start_frame_image_media_token.is_some() {
CommonGenerationMode::Keyframe
} else {
CommonGenerationMode::Text
};
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
@@ -232,7 +219,7 @@ pub async fn omni_gen_video_generate_handler(
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: request.negative_prompt.as_deref(),
maybe_other_args: None,
maybe_generation_mode: Some(generation_mode),
maybe_generation_mode: Some(gen_result.generation_mode),
maybe_aspect_ratio: None,
maybe_resolution: None,
maybe_batch_count: request.video_batch_count.map(|c| c as u8),
@@ -301,31 +288,88 @@ pub async fn omni_gen_video_generate_handler(
// -- Inference job --
let db_result = insert_generic_inference_job_for_fal_queue_with_apriori_job_token(
InsertGenericInferenceForFalWithAprioriJobTokenArgs {
apriori_job_token: &apriori_job_token,
uuid_idempotency_token: &idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: Some(user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
starting_job_status_override: None,
maybe_frontend_failure_category: None,
maybe_failure_reason: None,
phantom: Default::default(),
}
).await;
let job_token = if gen_result.is_seedance2pro {
// Seedance2Pro path: insert one job per order_id (batch support).
let fallback_ids = vec![gen_result.external_job_id.clone()];
let order_ids = gen_result.maybe_seedance_order_ids
.as_deref()
.unwrap_or(&fallback_ids);
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting inference job: {:?}", err);
return Err(AdvancedCommonWebError::from_error(err));
let mut all_job_tokens: Vec<InferenceJobToken> = Vec::with_capacity(order_ids.len());
for (i, order_id) in order_ids.iter().enumerate() {
let job_token = if i == 0 {
apriori_job_token.clone()
} else {
InferenceJobToken::generate()
};
let idempotency_str = if i == 0 {
idempotency_token.clone()
} else {
format!("{}-batch-{}", idempotency_token, i)
};
let db_result = insert_generic_inference_job_for_seedance2pro_queue_with_apriori_job_token(
InsertGenericInferenceForSeedance2ProWithAprioriJobTokenArgs {
apriori_job_token: &job_token,
uuid_idempotency_token: &idempotency_str,
maybe_external_third_party_id: order_id,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_wallet_ledger_entry_token: None,
maybe_creator_user_token: Some(user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}
).await;
match db_result {
Ok(token) => all_job_tokens.push(token),
Err(err) => {
warn!("Error inserting seedance2pro inference job (order_id={}): {:?}", order_id, err);
if i == 0 {
return Err(AdvancedCommonWebError::from_error(err));
}
}
}
}
all_job_tokens.first().cloned().ok_or_else(|| {
error!("No inference job token was created");
AdvancedCommonWebError::server_error_with_message("No inference job token was created")
})?
} else {
// Fal / other providers path.
let db_result = insert_generic_inference_job_for_fal_queue_with_apriori_job_token(
InsertGenericInferenceForFalWithAprioriJobTokenArgs {
apriori_job_token: &apriori_job_token,
uuid_idempotency_token: &idempotency_token,
maybe_external_third_party_id: &gen_result.external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: Some(user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
starting_job_status_override: None,
maybe_frontend_failure_category: None,
maybe_failure_reason: None,
phantom: Default::default(),
}
).await;
match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting inference job: {:?}", err);
return Err(AdvancedCommonWebError::from_error(err));
}
}
};
@@ -0,0 +1 @@
mod test_kinovi_seedance_2p0;
@@ -0,0 +1,459 @@
//! Tests for omni-gen video against the Seedance 2.0 model (Kinovi/Seedance2Pro provider).
//!
//! Seedance 2.0 is special: it uses the Seedance2Pro/Kinovi provider for execution
//! instead of Fal. The cost estimation uses Artcraft provider with media-token-form
//! references, while the execution plan uses URL-form references.
//!
//! Pricing: 16¢ per second × batch count.
//! - 4s/batch 1 = 64¢, 5s/batch 1 = 80¢, 15s/batch 1 = 240¢
//! - Batch 2 = 2×, Batch 4 = 4×
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use artcraft_api_defs::omni_gen::cost_and_generate_requests::omni_gen_video_cost_and_generate_request::OmniGenVideoCostAndGenerateRequest;
use artcraft_router::api::provider::Provider;
use artcraft_router::generate::generate_video::video_generation_plan::VideoGenerationPlan;
use enums::common::generation::common_aspect_ratio::CommonAspectRatio;
use enums::common::generation::common_video_model::CommonVideoModel;
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
use crate::http_server::endpoints::omni_gen::generate::video::distill_video_request::{
distill_video_request, DistilledVideoRequest,
};
// ── Helpers ────────────────────────────────────────────────────────────────
fn make_request(
prompt: Option<&str>,
aspect_ratio: Option<CommonAspectRatio>,
duration_seconds: Option<u16>,
video_batch_count: Option<u16>,
start_frame_token: Option<MediaFileToken>,
end_frame_token: Option<MediaFileToken>,
reference_image_tokens: Option<Vec<MediaFileToken>>,
reference_video_tokens: Option<Vec<MediaFileToken>>,
reference_audio_tokens: Option<Vec<MediaFileToken>>,
) -> OmniGenVideoCostAndGenerateRequest {
OmniGenVideoCostAndGenerateRequest {
idempotency_token: Some("11111111-1111-1111-1111-111111111111".to_string()),
model: Some(CommonVideoModel::Seedance2p0),
prompt: prompt.map(|s| s.to_string()),
negative_prompt: None,
start_frame_image_media_token: start_frame_token,
end_frame_image_media_token: end_frame_token,
reference_image_media_tokens: reference_image_tokens,
reference_video_media_tokens: reference_video_tokens,
reference_audio_media_tokens: reference_audio_tokens,
reference_character_tokens: None,
resolution: None,
aspect_ratio,
quality: None,
duration_seconds,
video_batch_count,
generate_audio: None,
}
}
fn text_only_request(
duration_seconds: Option<u16>,
video_batch_count: Option<u16>,
) -> OmniGenVideoCostAndGenerateRequest {
make_request(
Some("a cat dancing"),
None,
duration_seconds,
video_batch_count,
None, None, None, None, None,
)
}
fn fake_token(name: &str) -> MediaFileToken {
MediaFileToken::new_from_str(name)
}
fn build_hydration_map(tokens: &[&MediaFileToken]) -> HashMap<MediaFileToken, Url> {
tokens.iter().enumerate().map(|(i, t)| {
let url = Url::parse(&format!("https://cdn.example.com/media/{}.png", i)).unwrap();
((*t).clone(), url)
}).collect()
}
fn distill_text(request: &OmniGenVideoCostAndGenerateRequest) -> DistilledVideoRequest {
let empty: HashMap<MediaFileToken, Url> = HashMap::new();
distill_video_request(request, Some(&empty), Provider::Seedance2Pro)
.expect("distill should succeed for Seedance 2.0 (text)")
}
fn distill_with_map(
request: &OmniGenVideoCostAndGenerateRequest,
hydration: &HashMap<MediaFileToken, Url>,
) -> DistilledVideoRequest {
distill_video_request(request, Some(hydration), Provider::Seedance2Pro)
.expect("distill should succeed for Seedance 2.0 (with media)")
}
// ═══════════════════════════════════════════════════════════════════════════
// COST TESTS
// ═══════════════════════════════════════════════════════════════════════════
mod cost_tests {
use super::*;
#[test]
fn cost_is_present() {
let d = distill_text(&text_only_request(Some(5), None));
assert!(d.cost.cost_in_credits.is_some());
assert!(d.cost.cost_in_usd_cents.is_some());
}
#[test]
fn cost_is_not_free() {
let d = distill_text(&text_only_request(Some(5), None));
assert!(!d.cost.is_free);
}
// Pricing: 16¢/sec × batch_count
#[test]
fn cost_4s_batch_1() {
let d = distill_text(&text_only_request(Some(4), Some(1)));
assert_eq!(d.cost.cost_in_usd_cents, Some(64));
}
#[test]
fn cost_5s_batch_1() {
let d = distill_text(&text_only_request(Some(5), Some(1)));
assert_eq!(d.cost.cost_in_usd_cents, Some(80));
}
#[test]
fn cost_6s_batch_1() {
let d = distill_text(&text_only_request(Some(6), Some(1)));
assert_eq!(d.cost.cost_in_usd_cents, Some(96));
}
#[test]
fn cost_10s_batch_1() {
let d = distill_text(&text_only_request(Some(10), Some(1)));
assert_eq!(d.cost.cost_in_usd_cents, Some(160));
}
#[test]
fn cost_15s_batch_1() {
let d = distill_text(&text_only_request(Some(15), Some(1)));
assert_eq!(d.cost.cost_in_usd_cents, Some(240));
}
#[test]
fn cost_5s_batch_2() {
let d = distill_text(&text_only_request(Some(5), Some(2)));
assert_eq!(d.cost.cost_in_usd_cents, Some(160));
}
#[test]
fn cost_5s_batch_4() {
let d = distill_text(&text_only_request(Some(5), Some(4)));
assert_eq!(d.cost.cost_in_usd_cents, Some(320));
}
#[test]
fn cost_15s_batch_4() {
let d = distill_text(&text_only_request(Some(15), Some(4)));
assert_eq!(d.cost.cost_in_usd_cents, Some(960));
}
#[test]
fn cost_same_with_or_without_start_frame() {
// Cost should not change based on media presence — it's duration × batch.
let text_d = distill_text(&text_only_request(Some(5), Some(1)));
let start_token = fake_token("start_frame");
let map = build_hydration_map(&[&start_token]);
let req = make_request(
Some("a cat"), None, Some(5), Some(1),
Some(start_token), None, None, None, None,
);
let frame_d = distill_with_map(&req, &map);
assert_eq!(text_d.cost.cost_in_usd_cents, frame_d.cost.cost_in_usd_cents);
}
#[test]
fn cost_estimation_uses_media_tokens_not_urls() {
// The Artcraft cost plan should use token-form references.
// If it used URLs, the Artcraft plan builder would reject them.
// This test passes if distillation succeeds at all — the Artcraft
// builder would error out on URL-form refs.
let start_token = fake_token("start_frame");
let ref_img = fake_token("ref_img_1");
let ref_vid = fake_token("ref_vid_1");
let ref_aud = fake_token("ref_aud_1");
let map = build_hydration_map(&[&start_token, &ref_img, &ref_vid, &ref_aud]);
let req = make_request(
Some("test"), None, Some(5), Some(1),
Some(start_token), None,
Some(vec![ref_img]),
Some(vec![ref_vid]),
Some(vec![ref_aud]),
);
// If this succeeds, the Artcraft cost path handled tokens correctly.
let d = distill_with_map(&req, &map);
assert!(d.cost.cost_in_credits.is_some());
}
#[test]
fn default_duration_produces_valid_cost() {
// When duration_seconds is None, the model should use a default.
let d = distill_text(&text_only_request(None, None));
assert!(d.cost.cost_in_usd_cents.unwrap_or(0) > 0);
}
}
// ═══════════════════════════════════════════════════════════════════════════
// GENERATION PLAN TESTS
// ═══════════════════════════════════════════════════════════════════════════
mod generation_plan_tests {
use super::*;
#[test]
fn provider_is_seedance2pro() {
let d = distill_text(&text_only_request(Some(5), None));
assert!(
matches!(d.execution_provider, Provider::Seedance2Pro),
"expected Seedance2Pro provider, got {:?}", d.execution_provider
);
}
#[test]
fn plan_is_seedance2pro_variant() {
let d = distill_text(&text_only_request(Some(5), None));
assert!(
matches!(d.plan(), VideoGenerationPlan::Seedance2proSeedance2p0(_)),
"expected Seedance2proSeedance2p0 plan variant, got {:?}", d.plan()
);
}
#[test]
fn text_only_has_no_media_urls() {
let d = distill_text(&text_only_request(Some(5), None));
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert!(plan.start_frame_url.is_none());
assert!(plan.end_frame_url.is_none());
assert!(plan.reference_image_urls.is_none());
assert!(plan.reference_video_urls.is_none());
assert!(plan.reference_audio_urls.is_none());
} else {
panic!("wrong plan variant");
}
}
#[test]
fn text_only_prompt_is_passed_through() {
let d = distill_text(&text_only_request(Some(5), None));
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert_eq!(plan.prompt.as_deref(), Some("a cat dancing"));
} else {
panic!("wrong plan variant");
}
}
#[test]
fn start_frame_uses_url_not_token() {
let start_token = fake_token("start_frame");
let map = build_hydration_map(&[&start_token]);
let req = make_request(
Some("test"), None, Some(5), None,
Some(start_token), None, None, None, None,
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
let url = plan.start_frame_url.as_ref().expect("start_frame_url should be set");
assert!(url.starts_with("https://"), "expected URL, got: {}", url);
assert!(!url.contains("m_"), "URL should not look like a media token");
} else {
panic!("wrong plan variant");
}
}
#[test]
fn end_frame_uses_url_not_token() {
let end_token = fake_token("end_frame");
let map = build_hydration_map(&[&end_token]);
let req = make_request(
Some("test"), None, Some(5), None,
None, Some(end_token), None, None, None,
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
let url = plan.end_frame_url.as_ref().expect("end_frame_url should be set");
assert!(url.starts_with("https://"), "expected URL, got: {}", url);
} else {
panic!("wrong plan variant");
}
}
#[test]
fn start_and_end_frame_both_resolved() {
let start = fake_token("sf");
let end = fake_token("ef");
let map = build_hydration_map(&[&start, &end]);
let req = make_request(
Some("test"), None, Some(5), None,
Some(start), Some(end), None, None, None,
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert!(plan.start_frame_url.is_some());
assert!(plan.end_frame_url.is_some());
} else {
panic!("wrong plan variant");
}
}
#[test]
fn reference_images_use_urls() {
let img1 = fake_token("img1");
let img2 = fake_token("img2");
let map = build_hydration_map(&[&img1, &img2]);
let req = make_request(
Some("test"), None, Some(5), None,
None, None,
Some(vec![img1, img2]),
None, None,
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
let urls = plan.reference_image_urls.as_ref().expect("reference_image_urls should be set");
assert_eq!(urls.len(), 2);
for url in urls {
assert!(url.starts_with("https://"), "expected URL, got: {}", url);
}
} else {
panic!("wrong plan variant");
}
}
#[test]
fn reference_videos_use_urls() {
let vid1 = fake_token("vid1");
let map = build_hydration_map(&[&vid1]);
let req = make_request(
Some("test"), None, Some(5), None,
None, None, None,
Some(vec![vid1]),
None,
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
let urls = plan.reference_video_urls.as_ref().expect("reference_video_urls should be set");
assert_eq!(urls.len(), 1);
assert!(urls[0].starts_with("https://"));
} else {
panic!("wrong plan variant");
}
}
#[test]
fn reference_audio_use_urls() {
let aud1 = fake_token("aud1");
let map = build_hydration_map(&[&aud1]);
let req = make_request(
Some("test"), None, Some(5), None,
None, None, None, None,
Some(vec![aud1]),
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
let urls = plan.reference_audio_urls.as_ref().expect("reference_audio_urls should be set");
assert_eq!(urls.len(), 1);
assert!(urls[0].starts_with("https://"));
} else {
panic!("wrong plan variant");
}
}
#[test]
fn all_reference_types_together() {
let sf = fake_token("sf");
let ef = fake_token("ef");
let img = fake_token("img");
let vid = fake_token("vid");
let aud = fake_token("aud");
let map = build_hydration_map(&[&sf, &ef, &img, &vid, &aud]);
let req = make_request(
Some("full test"), None, Some(8), Some(2),
Some(sf), Some(ef),
Some(vec![img]),
Some(vec![vid]),
Some(vec![aud]),
);
let d = distill_with_map(&req, &map);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert!(plan.start_frame_url.is_some());
assert!(plan.end_frame_url.is_some());
assert_eq!(plan.reference_image_urls.as_ref().map(|v| v.len()), Some(1));
assert_eq!(plan.reference_video_urls.as_ref().map(|v| v.len()), Some(1));
assert_eq!(plan.reference_audio_urls.as_ref().map(|v| v.len()), Some(1));
assert!(plan.prompt.as_deref() == Some("full test"));
} else {
panic!("wrong plan variant");
}
}
#[test]
fn duration_is_clamped_to_valid_range() {
// Duration below 4 should clamp to 4.
let d = distill_text(&text_only_request(Some(1), None));
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert!(plan.duration_seconds >= 4, "expected >= 4, got {}", plan.duration_seconds);
} else {
panic!("wrong plan variant");
}
// Duration above 15 should clamp to 15.
let d = distill_text(&text_only_request(Some(99), None));
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
assert!(plan.duration_seconds <= 15, "expected <= 15, got {}", plan.duration_seconds);
} else {
panic!("wrong plan variant");
}
}
#[test]
fn missing_prompt_defaults_to_empty() {
let req = make_request(
None, None, Some(5), None,
None, None, None, None, None,
);
let d = distill_text(&req);
if let VideoGenerationPlan::Seedance2proSeedance2p0(plan) = d.plan() {
// Prompt should be None or empty string.
let prompt = plan.prompt.as_deref().unwrap_or("");
assert!(prompt.is_empty() || plan.prompt.is_none());
} else {
panic!("wrong plan variant");
}
}
}
}
@@ -1,3 +1,4 @@
mod test_seedance_1p5_pro;
mod test_veo_2;
mod test_veo_3;
mod kinovi;
@@ -19,6 +19,7 @@ mod tests {
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
use artcraft_router::api::provider::Provider;
use crate::http_server::endpoints::omni_gen::generate::video::distill_video_request::{
distill_video_request, DistilledVideoRequest,
};
@@ -75,7 +76,7 @@ mod tests {
fn distill_text(request: &OmniGenVideoCostAndGenerateRequest) -> DistilledVideoRequest {
let empty: HashMap<MediaFileToken, Url> = HashMap::new();
distill_video_request(request, Some(&empty))
distill_video_request(request, Some(&empty), Provider::Fal)
.expect("distill_video_request should succeed for Seedance 1.5 Pro (text)")
}
@@ -83,7 +84,7 @@ mod tests {
request: &OmniGenVideoCostAndGenerateRequest,
hydration: &HashMap<MediaFileToken, Url>,
) -> DistilledVideoRequest {
distill_video_request(request, Some(hydration))
distill_video_request(request, Some(hydration), Provider::Fal)
.expect("distill_video_request should succeed for Seedance 1.5 Pro (image)")
}
@@ -20,6 +20,7 @@ mod tests {
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
use artcraft_router::api::provider::Provider;
use crate::http_server::endpoints::omni_gen::generate::video::distill_video_request::{
distill_video_request, DistilledVideoRequest,
};
@@ -62,7 +63,7 @@ mod tests {
fn distill_text(request: &OmniGenVideoCostAndGenerateRequest) -> DistilledVideoRequest {
let empty: HashMap<MediaFileToken, Url> = HashMap::new();
distill_video_request(request, Some(&empty))
distill_video_request(request, Some(&empty), Provider::Fal)
.expect("distill_video_request should succeed for Veo 2 (text)")
}
@@ -70,7 +71,7 @@ mod tests {
request: &OmniGenVideoCostAndGenerateRequest,
hydration: &HashMap<MediaFileToken, Url>,
) -> DistilledVideoRequest {
distill_video_request(request, Some(hydration))
distill_video_request(request, Some(hydration), Provider::Fal)
.expect("distill_video_request should succeed for Veo 2 (image)")
}
@@ -24,6 +24,7 @@ mod tests {
use tokens::tokens::media_files::MediaFileToken;
use url::Url;
use artcraft_router::api::provider::Provider;
use crate::http_server::endpoints::omni_gen::generate::video::distill_video_request::{
distill_video_request, DistilledVideoRequest,
};
@@ -68,7 +69,7 @@ mod tests {
fn distill_text(request: &OmniGenVideoCostAndGenerateRequest) -> DistilledVideoRequest {
let empty: HashMap<MediaFileToken, Url> = HashMap::new();
distill_video_request(request, Some(&empty))
distill_video_request(request, Some(&empty), Provider::Fal)
.expect("distill should succeed for Veo 3 (text)")
}
@@ -76,7 +77,7 @@ mod tests {
request: &OmniGenVideoCostAndGenerateRequest,
hydration: &HashMap<MediaFileToken, Url>,
) -> DistilledVideoRequest {
distill_video_request(request, Some(hydration))
distill_video_request(request, Some(hydration), Provider::Fal)
.expect("distill should succeed for Veo 3 (image)")
}
@@ -18,7 +18,7 @@ pub(crate) fn check_advanced_common_web_error(
error: &AdvancedCommonWebError,
) -> bool {
if !error.is_server_error() {
// Non-500 errors (400, 401, 404, 402) are intentional — don't alert.
// Non-500 errors (400, 401, 403, 403, 404...) are intentional — don't alert.
return true;
}
@@ -50,6 +50,13 @@ pub(crate) fn check_advanced_common_web_error(
.set_session_token(metadata.session_token.clone())
.set_session_user_token(metadata.session_user_token.clone());
match error {
AdvancedCommonWebError::UncaughtServerErrorWithInternalMessage { internal_message, error } => {
builder = builder.set_extra_message(Some(internal_message.to_string()));
}
_ => {},
}
let notification = builder.build();
if let Err(err) = pager.enqueue_page(notification) {
@@ -16,7 +16,7 @@ pub(crate) fn check_status_code_fallback(
) {
match status_code {
500 => {
let summary = format!("HTTP 500: {} {}", method, path);
let summary = format!("[Legacy] HTTP 500: {} {}", method, path);
let description = format!(
"An untyped HTTP 500 response was returned (no typed error matched).\n\n\
Endpoint: {} {}\n\
@@ -17,6 +17,14 @@
/checkout* /index.html 200
/checkout/* /index.html 200
/create-image /index.html 200
/create-image/ /index.html 200
/create-image/* /index.html 200
/create-video /index.html 200
/create-video/ /index.html 200
/create-video/* /index.html 200
/library /index.html 200
/library/ /index.html 200
/library/* /index.html 200
@@ -4,8 +4,8 @@ import Media from "../pages/media";
import PressKit from "../pages/press-kit";
import Navbar from "../components/navbar";
import { ToastContainer } from "../components/toast/toast";
// import CreateImage from "../pages/create-image";
// import CreateVideo from "../pages/create-video";
import CreateImage from "../pages/create-image";
import CreateVideo from "../pages/create-video";
import Landing2 from "../pages/landing2";
import LandingSD2 from "../pages/landing-sd2";
import TutorialsPage from "../pages/tutorials";
@@ -30,8 +30,8 @@ export function App() {
<Routes>
<Route path="/" element={<Landing2 />} />
{/* <Route path="/create-image" element={<CreateImage />} />
<Route path="/create-video" element={<CreateVideo />} /> */}
<Route path="/create-image" element={<CreateImage />} />
<Route path="/create-video" element={<CreateVideo />} />
<Route path="/seedance-2" element={<LandingSD2 />} />
<Route path="/download" element={<Download />} />
<Route path="/media" element={<Media />} />
@@ -6,7 +6,7 @@ import {
type IconDefinition,
} from "@fortawesome/pro-solid-svg-icons";
import { Button } from "@storyteller/ui-button";
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
import { type PopoverItem } from "@storyteller/ui-popover";
import Seo from "../../components/seo";
import Footer from "../../components/footer";
@@ -49,8 +49,6 @@ export function CreateMediaPageShell({
emptyStateTitle,
emptyStateSubtitle,
bottomOffset,
modelItems,
onModelChange,
glowOrbs,
gridContent,
promptBox,
@@ -146,28 +144,6 @@ export function CreateMediaPageShell({
)}
{promptBox}
<div
className="animate-fade-in-up fixed bottom-3 left-6 z-20 hidden items-center gap-5 lg:flex"
style={{ animationDelay: "300ms" }}
>
<div
className="rounded-xl border border-white/10 px-3 py-2 shadow-lg backdrop-blur-xl"
style={{ backgroundColor: "rgba(30, 30, 33, 0.85)" }}
>
<PopoverMenu
items={modelItems}
onSelect={onModelChange}
mode="hoverSelect"
panelTitle="Select Model"
panelClassName="min-w-[300px]"
buttonClassName="bg-transparent border-0 shadow-none p-0 text-lg hover:bg-transparent text-white/80 hover:text-white"
showIconsInList
triggerLabel="Model"
maxListHeight={400}
/>
</div>
</div>
</div>
</div>
@@ -94,7 +94,7 @@ function jobToInProgress(job: Job): InProgressJob {
(m) => m.tauriId === modelType || m.id === modelType,
)
: undefined;
duration = model?.progressBarTime ?? (isVideo ? 120000 : 30000);
duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000);
taskDurationCache.set(job.job_token, duration);
}
@@ -24,8 +24,8 @@ import { TaskQueue } from "./task-queue";
const NAV_ITEMS = [
{ name: "Home", href: "/" },
// { name: "Image", href: "/create-image" },
// { name: "Video", href: "/create-video" },
{ name: "Image", href: "/create-image" },
{ name: "Video", href: "/create-video" },
{ name: "Tutorials", href: "/tutorials" },
{ name: "News", href: "/news" },
{ name: "FAQ", href: "/faq" },
@@ -519,7 +519,7 @@ function jobsToInProgress(jobs: Job[]): InProgressTask[] {
(m) => m.tauriId === modelType || m.id === modelType,
)
: undefined;
duration = model?.progressBarTime ?? (isVideo ? 120000 : 30000);
duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000);
taskDurationCache.set(j.job_token, duration);
}
@@ -633,7 +633,7 @@ export const MentionTextarea = forwardRef<HTMLDivElement, MentionTextareaProps>(
onBlur={onBlur}
className={twMerge(
className,
"outline-none whitespace-pre-wrap break-words overflow-y-auto",
"outline-none whitespace-pre-wrap break-words overflow-y-auto resize-y",
)}
/>
@@ -10,6 +10,8 @@ import {
import { twMerge } from "tailwind-merge";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faChevronDown,
faChevronUp,
faMusic,
faUserGroup,
faVideo,
@@ -143,8 +145,25 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
const highlightRef = useRef<HTMLDivElement>(null);
const mentionEditorRef = useRef<HTMLDivElement>(null);
const [isFocused, setIsFocused] = useState(false);
const [isExpanded, setIsExpanded] = useState(false);
const [showImagePrompts, setShowImagePrompts] = useState(false);
const EXPANDED_HEIGHT = "clamp(120px, calc(100vh - 700px), 500px)";
const toggleExpand = () => {
setIsExpanded((prev) => {
const next = !prev;
const height = next ? EXPANDED_HEIGHT : "auto";
if (textareaRef.current) {
textareaRef.current.style.height = height;
}
if (mentionEditorRef.current) {
mentionEditorRef.current.style.height = height;
}
return next;
});
};
// @-mention state
const [mentionOpen, setMentionOpen] = useState(false);
const [mentionFilter, setMentionFilter] = useState("");
@@ -167,13 +186,14 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
);
}, [mentionItems, mentionFilter]);
// Auto-resize textarea
// Auto-resize textarea (skip when expanded)
useEffect(() => {
if (isExpanded) return;
if (textareaRef.current) {
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
}, [prompt]);
}, [prompt, isExpanded]);
// Move caret to end on mount so autoFocus doesn't leave it at position 0
useEffect(() => {
@@ -402,7 +422,10 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
onChange={onPromptChange}
mentionItems={mentionItems}
placeholder={placeholder}
className="max-h-[5.5em] w-full text-white"
className={twMerge(
"promptbox-scrollbar min-h-[2.5em] w-full text-white",
isExpanded ? "max-h-[500px]" : "max-h-[5.5em]",
)}
colorMap={mentionColorMap}
onKeyDown={(e) => {
if (e.key === "Enter" && !e.shiftKey && !e.metaKey) {
@@ -419,7 +442,10 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
<div
ref={highlightRef}
aria-hidden
className="pointer-events-none absolute inset-0 max-h-[5.5em] overflow-y-auto whitespace-pre-wrap break-words text-sm text-white"
className={twMerge(
"pointer-events-none absolute inset-0 overflow-y-auto whitespace-pre-wrap break-words text-sm text-white",
isExpanded ? "max-h-[500px]" : "max-h-[5.5em]",
)}
>
{renderHighlightedPrompt()}
</div>
@@ -431,7 +457,8 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
autoFocus
placeholder={placeholder}
className={twMerge(
"max-h-[5.5em] w-full flex-1 resize-none overflow-y-auto bg-transparent text-md text-white placeholder-white/50 focus:outline-none",
"promptbox-scrollbar min-h-[2.5em] w-full flex-1 resize-y overflow-y-auto bg-transparent text-md text-white placeholder-white/50 focus:outline-none",
isExpanded ? "max-h-[500px]" : "max-h-[5.5em]",
hasMentionItems && "text-transparent caret-white",
)}
value={prompt}
@@ -515,9 +542,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
{/* Toolbar */}
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<div className="flex flex-wrap items-center gap-1.5 sm:gap-2">
{modelSelector && (
<div className="lg:hidden">{modelSelector}</div>
)}
{modelSelector}
{leftToolbar}
</div>
<div className="flex items-center gap-2 sm:shrink-0">
@@ -533,6 +558,25 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
</GenerateButton>
</div>
</div>
{/* Expand / Collapse toggle — hidden on small screens */}
<div className="absolute -bottom-1 left-1/2 hidden -translate-x-1/2 sm:block">
<Tooltip
content={isExpanded ? "Collapse" : "Expand"}
position="top"
>
<button
type="button"
onClick={toggleExpand}
className="px-3 py-0.5 text-white/30 transition-colors hover:text-white/90"
>
<FontAwesomeIcon
icon={isExpanded ? faChevronUp : faChevronDown}
className="text-xs"
/>
</button>
</Tooltip>
</div>
</div>
</div>
</div>
@@ -396,7 +396,7 @@ export default function Library() {
) : allItems.length === 0 && !loading ? (
<div className="flex flex-col items-center justify-center py-20">
<p className="text-white/40 text-sm mb-4">No items yet.</p>
{/* <div className="flex gap-3">
<div className="flex gap-3">
<Link to="/create-image">
<Button variant="primary" className="text-sm px-4 py-2">
Create Image
@@ -410,7 +410,7 @@ export default function Library() {
Create Video
</Button>
</Link>
</div> */}
</div>
</div>
) : (
<div className="space-y-6">
@@ -71,6 +71,30 @@
background-color: rgba(46, 46, 49, 0.85) !important;
}
/* Minimal thin scrollbar for prompt textareas */
.promptbox-scrollbar {
scrollbar-width: thin;
scrollbar-color: rgba(255, 255, 255, 0.2) transparent;
}
.promptbox-scrollbar::-webkit-scrollbar {
width: 4px;
}
.promptbox-scrollbar::-webkit-scrollbar-track {
background: transparent;
}
.promptbox-scrollbar::-webkit-scrollbar-thumb {
background-color: rgba(255, 255, 255, 0.2);
border-radius: 9999px;
}
.promptbox-scrollbar::-webkit-scrollbar-thumb:hover {
background-color: rgba(255, 255, 255, 0.35);
}
@keyframes fade-in-up {
from {
opacity: 0;
@@ -40,6 +40,9 @@
{
"path": "../../libs/components/gravatar/tsconfig.lib.json"
},
{
"path": "../../libs/components/slider-v2/tsconfig.lib.json"
},
{
"path": "../../libs/components/tab-selector/tsconfig.lib.json"
},
@@ -5,6 +5,9 @@
{
"path": "../../libs/components/gravatar"
},
{
"path": "../../libs/components/slider-v2"
},
{
"path": "../../libs/components/tab-selector"
},
@@ -3,6 +3,7 @@ import { UserInfo, ActiveSubscriptions } from "~/models";
import { authentication, flushAllBackgroundLoadedMedia } from "~/signals";
import { useCreditsState } from "@storyteller/credits";
import { useSubscriptionState } from "@storyteller/subscription";
import { clearGalleryCache } from "@storyteller/ui-gallery-modal";
export const updateAuthStatus = (newStatus: AUTH_STATUS) => {
console.log('updateAuthStatus()', newStatus)
@@ -40,4 +41,5 @@ export const setLogoutStates = () => {
flushAllBackgroundLoadedMedia();
useCreditsState.getState().reset();
useSubscriptionState.getState().reset();
clearGalleryCache();
};
@@ -1,4 +1,4 @@
import React, { useRef } from "react";
import React, { useEffect, useRef, useState } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faCheck,
@@ -7,6 +7,7 @@ import {
faTrashCan,
faUpload,
} from "@fortawesome/pro-solid-svg-icons";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { twMerge } from "tailwind-merge";
import galleryDnd from "./galleryDnd";
import { Tooltip } from "@storyteller/ui-tooltip";
@@ -57,6 +58,34 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
const imgRef = useRef<HTMLImageElement>(null);
const dragStarted = useRef(false);
// For freshly-completed videos the backend may still be generating the
// preview GIF, so the thumbnail URL 404s for a while. Show a spinner and
// refresh the image every 5s until it loads.
const isVideo = item.mediaClass === "video";
const [retryAttempt, setRetryAttempt] = useState(0);
const [videoLoaded, setVideoLoaded] = useState(false);
useEffect(() => {
setRetryAttempt(0);
setVideoLoaded(false);
}, [item.thumbnail]);
useEffect(() => {
if (!isVideo || videoLoaded || !item.thumbnail) return;
const t = setInterval(() => setRetryAttempt((n) => n + 1), 5000);
return () => clearInterval(t);
}, [isVideo, videoLoaded, item.thumbnail]);
const imgSrc =
isVideo && item.thumbnail && retryAttempt > 0
? `${item.thumbnail}${item.thumbnail.includes("?") ? "&" : "?"}_r=${retryAttempt}`
: item.thumbnail ?? undefined;
const handleImgError = (e: React.SyntheticEvent<HTMLImageElement>) => {
if (isVideo) return;
onImageError(e);
};
const handleDelete = () => {
showActionReminder({
reminderType: "default",
@@ -174,22 +203,33 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
<span className="text-white/60">Image not available</span>
</div>
) : (
<img
data-gallery-draggable-1="true"
// NB: "loading=lazy" is necessary to prevent loading GIGABYTES of images!
// It is a bit finnicky, too: you must include this attribute
// BEFORE the `src` attribute, or it won't work.
loading="lazy"
ref={imgRef}
src={item.thumbnail}
alt={item.label}
className={twMerge(
"h-full w-full bg-black/30",
imageFit === "contain" ? "object-contain" : "object-cover",
<>
<img
data-gallery-draggable-1="true"
// NB: "loading=lazy" is necessary to prevent loading GIGABYTES of images!
// It is a bit finnicky, too: you must include this attribute
// BEFORE the `src` attribute, or it won't work.
loading="lazy"
ref={imgRef}
src={imgSrc}
alt={item.label}
className={twMerge(
"h-full w-full bg-black/30",
imageFit === "contain" ? "object-contain" : "object-cover",
isVideo && !videoLoaded ? "opacity-0" : "",
)}
draggable={false}
onLoad={() => {
if (isVideo) setVideoLoaded(true);
}}
onError={handleImgError}
/>
{isVideo && !videoLoaded && (
<div className="absolute inset-0 flex items-center justify-center bg-black/30">
<LoadingSpinner className="h-6 w-6" />
</div>
)}
draggable={false}
onError={onImageError}
/>
</>
)}
{selected && (
<div className="absolute left-2 top-2 flex h-6 w-6 items-center justify-center rounded-full bg-primary">
@@ -70,17 +70,19 @@ interface GalleryCacheEntry {
}
const galleryCacheMap = new Map<string, GalleryCacheEntry>();
let cachedUsername: string | null = null;
const usersApiSingleton = new UsersApi();
/** Get cached username or fetch it once. */
async function getCachedUsername(): Promise<string | null> {
if (cachedUsername) return cachedUsername;
/** Clear the module-level gallery item cache. Call on logout so the next user doesn't see stale items. */
export const clearGalleryCache = () => {
galleryCacheMap.clear();
};
/** Fetch the current session's username fresh every time — the session may have changed (logout/login). */
async function fetchCurrentUsername(): Promise<string | null> {
try {
const session = await usersApiSingleton.GetSession();
if (session.success && session.data?.user) {
cachedUsername = session.data.user.username;
return cachedUsername;
return session.data.user.username;
}
} catch {
// ignore
@@ -229,6 +231,7 @@ interface GalleryModalProps {
onSelectItem?: (id: string) => void;
maxSelections?: number;
onUseSelected?: (selectedItems: GalleryItem[]) => void;
useSelectedLoading?: boolean;
onDownloadClicked?: (url: string, mediaClass?: string) => Promise<void>;
onAddToSceneClicked?: (
url: string,
@@ -363,6 +366,7 @@ export const GalleryModal = React.memo(
onSelectItem,
maxSelections = 4,
onUseSelected,
useSelectedLoading,
onDownloadClicked,
onAddToSceneClicked,
isOpen,
@@ -481,24 +485,52 @@ export const GalleryModal = React.memo(
failedImageUrls.current.add(url);
}, []);
// Fetch & cache username — uses module-level cache so subsequent opens are instant
// Ref to the latest refreshGallery so the async session-fetch effect can
// call it without taking a dependency on it. refreshGallery itself is
// defined lower in the file (it needs loadItems); we wire the ref via a
// layout-time assignment below.
const refreshGalleryRef = useRef<() => void>(() => {});
// Fetch the current session's username every time the modal opens.
// The modal is permanently mounted in TopBar, so this component survives
// logout → login. We re-verify who the backend considers the current user
// on every open and, if it's a different user, wipe all stale state before
// letting refreshGallery fire. This effect is the SOLE driver for refresh
// on modal-open — the separate filter/username effect below intentionally
// does NOT depend on modal-open signals, so it can't race this one.
useEffect(() => {
const getUsername = async () => {
if (username) return; // already have it
const modalIsOpen =
isOpen || (mode === "view" && galleryModalVisibleViewMode.value);
if (!modalIsOpen) return;
let cancelled = false;
(async () => {
setUsernameError(false);
const name = await getCachedUsername();
if (name) {
setInitialLoading(true); // keep skeleton alive until refreshGallery takes over
setUsername(name);
// refreshGallery (triggered by username state change) handles initialLoading
} else {
const name = await fetchCurrentUsername();
if (cancelled) return;
if (!name) {
setUsernameError(true);
setInitialLoading(false);
return;
}
if (name !== usernameRef.current) {
// Different user than before — wipe everything from the previous
// session. setUsername will trigger the filter/username effect
// below, which calls refreshGallery with the new identity.
galleryCacheMap.clear();
setAllItems([]);
setPageIndex(0);
setHasMore(true);
setInitialLoading(true);
setUsername(name);
} else {
// Same user reopening — just refresh (shows cache + background refetch).
refreshGalleryRef.current();
}
})();
return () => {
cancelled = true;
};
if (isOpen || (mode === "view" && galleryModalVisibleViewMode.value)) {
getUsername();
}
}, [mode, galleryModalVisibleViewMode.value, isOpen, username, usernameRetryCount]);
}, [mode, galleryModalVisibleViewMode.value, isOpen, usernameRetryCount]);
// Helper to build the cache key for the current filter
const getCacheKey = useCallback(
@@ -651,6 +683,7 @@ export const GalleryModal = React.memo(
);
// refresh logic — shows cached items immediately, then background-refreshes
// eslint-disable-next-line react-hooks/exhaustive-deps
const refreshGallery = useCallback(() => {
setItemsLoadError(null);
const cacheKey = `gallery_${activeFilterRef.current}`;
@@ -674,8 +707,17 @@ export const GalleryModal = React.memo(
}
}, [loadItems]);
// Keep the ref pointed at the latest refreshGallery so the session-fetch
// effect can call it without taking it as a dep (avoids tearing).
refreshGalleryRef.current = refreshGallery;
// Refresh on username change (new user logged in via the session-fetch
// effect) or on activeFilter change. This effect deliberately does NOT
// depend on modal-open signals — the session-fetch effect above is the
// sole driver for modal-open refreshes, which prevents a race where this
// effect would otherwise fire synchronously with a stale `username` and
// take the loading lock before the new session resolves.
useEffect(() => {
// Refresh every time the modal is opened
const modalIsOpen =
mode === "view"
? galleryModalVisibleViewMode.value
@@ -686,13 +728,7 @@ export const GalleryModal = React.memo(
refreshGallery();
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [
mode,
isOpen,
galleryModalVisibleViewMode.value,
username,
activeFilter,
]);
}, [username, activeFilter]);
// Auto-refresh when a generation completes while the gallery is open
const modalIsOpenRef = useRef(false);
@@ -1396,7 +1432,8 @@ export const GalleryModal = React.memo(
</div>
<Button
onClick={handleUseSelected}
disabled={selectedItemIds.length === 0}
disabled={selectedItemIds.length === 0 || useSelectedLoading}
loading={useSelectedLoading}
>
Use selected
</Button>
@@ -9,7 +9,7 @@
"path": "../../common"
},
{
"path": "../loading-spinner"
"path": "../../api"
},
{
"path": "../close-button"
@@ -24,10 +24,10 @@
"path": "../modal"
},
{
"path": "../../tauri-api"
"path": "../toaster"
},
{
"path": "../../api"
"path": "../../tauri-api"
},
{
"path": "../action-reminder-modal"
@@ -38,6 +38,9 @@
{
"path": "../tooltip"
},
{
"path": "../loading-spinner"
},
{
"path": "./tsconfig.lib.json"
},
@@ -50,7 +50,7 @@
"path": "../../common/tsconfig.lib.json"
},
{
"path": "../loading-spinner/tsconfig.lib.json"
"path": "../../api/tsconfig.lib.json"
},
{
"path": "../close-button/tsconfig.lib.json"
@@ -65,10 +65,10 @@
"path": "../modal/tsconfig.lib.json"
},
{
"path": "../../tauri-api/tsconfig.lib.json"
"path": "../toaster/tsconfig.lib.json"
},
{
"path": "../../api/tsconfig.lib.json"
"path": "../../tauri-api/tsconfig.lib.json"
},
{
"path": "../action-reminder-modal/tsconfig.lib.json"
@@ -78,6 +78,9 @@
},
{
"path": "../tooltip/tsconfig.lib.json"
},
{
"path": "../loading-spinner/tsconfig.lib.json"
}
]
}
@@ -210,6 +210,7 @@ export const ImagePromptRow = ({
id: string;
file: File;
} | null>(null);
const [isProcessingGallery, setIsProcessingGallery] = useState(false);
const referenceImagesRef = useRef(referenceImages);
useEffect(() => {
@@ -359,21 +360,22 @@ export const ImagePromptRow = ({
referenceVideosRef.current = referenceVideos;
}, [referenceVideos]);
const getVideoDuration = (file: File): Promise<number> =>
const getVideoDurationFromSrc = (src: string): Promise<number> =>
new Promise((resolve) => {
const video = document.createElement("video");
video.preload = "metadata";
video.onloadedmetadata = () => {
URL.revokeObjectURL(video.src);
resolve(Math.round(video.duration));
};
video.onerror = () => {
URL.revokeObjectURL(video.src);
resolve(0);
};
video.src = URL.createObjectURL(file);
video.onloadedmetadata = () => resolve(Math.round(video.duration));
video.onerror = () => resolve(0);
video.src = src;
});
const getVideoDuration = (file: File): Promise<number> => {
const src = URL.createObjectURL(file);
return getVideoDurationFromSrc(src).finally(() =>
URL.revokeObjectURL(src),
);
};
const totalVideoDuration = useMemo(
() => referenceVideos.reduce((sum, v) => sum + v.duration, 0),
[referenceVideos],
@@ -385,24 +387,30 @@ export const ImagePromptRow = ({
const files = Array.from(event.target.files || []);
if (files.length === 0) return;
const availableSlots = Math.max(0, maxVideoCount - referenceVideos.length);
// Snapshot the committed state at call time so removes that happened
// before this call are respected (don't re-read a stale ref).
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideoCount - baseVideos.length);
if (availableSlots <= 0) {
toast.error(
`Max ${maxVideoCount} videos / ${maxVideoRefDuration}s total`,
{ id: "video-ref-limit" },
);
if (videoFileInputRef.current) videoFileInputRef.current.value = "";
return;
}
const filesToProcess = files.slice(0, availableSlots);
let committed = baseVideos;
for (const file of filesToProcess) {
const duration = await getVideoDuration(file);
const currentTotal = referenceVideosRef.current.reduce(
(sum, v) => sum + v.duration,
0,
);
const currentTotal = committed.reduce((sum, v) => sum + v.duration, 0);
if (currentTotal + duration > maxVideoRefDuration) {
toast.error(
`Total video duration cannot exceed ${maxVideoRefDuration}s`,
{ id: "video-ref-limit" },
);
break;
}
@@ -426,7 +434,8 @@ export const ImagePromptRow = ({
duration,
};
setUploadingVideo(null);
setReferenceVideos?.([...referenceVideosRef.current, refVideo]);
committed = [...committed, refVideo];
setReferenceVideos?.(committed);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
@@ -445,7 +454,8 @@ export const ImagePromptRow = ({
duration,
};
setUploadingVideo(null);
setReferenceVideos?.([...referenceVideosRef.current, refVideo]);
committed = [...committed, refVideo];
setReferenceVideos?.(committed);
}
}
@@ -667,7 +677,12 @@ export const ImagePromptRow = ({
const handleImageSelect = (id: string) => {
setSelectedGalleryImages((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
const maxSelections = Math.max(1, maxImagePromptCount);
const maxSelections =
galleryTarget === "video"
? Math.max(1, maxVideoCount - referenceVideos.length)
: galleryTarget === "end"
? 1
: Math.max(1, maxImagePromptCount);
if (prev.length >= maxSelections) {
return maxSelections === 1 ? [id] : prev;
}
@@ -675,7 +690,73 @@ export const ImagePromptRow = ({
});
};
const handleGalleryImages = (selectedItems: GalleryItem[]) => {
const handleGalleryImages = async (selectedItems: GalleryItem[]) => {
if (galleryTarget === "video") {
// Snapshot committed state at call time to avoid re-reading a stale
// ref after removes.
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideoCount - baseVideos.length);
if (availableSlots <= 0) {
toast.error(
`Max ${maxVideoCount} videos / ${maxVideoRefDuration}s total`,
{ id: "video-ref-limit" },
);
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
return;
}
const itemsToProcess = selectedItems
.slice(0, availableSlots)
.filter((item): item is GalleryItem & { fullImage: string } =>
Boolean(item.fullImage),
);
setIsProcessingGallery(true);
try {
// Parallelize duration probes — sequential metadata loads was the
// source of the perceived lag on the "Use selected" click.
const durations = await Promise.all(
itemsToProcess.map((item) => getVideoDurationFromSrc(item.fullImage)),
);
const newVideos: RefVideo[] = [];
let currentTotal = baseVideos.reduce(
(sum, v) => sum + v.duration,
0,
);
let exceeded = false;
for (let i = 0; i < itemsToProcess.length; i++) {
const item = itemsToProcess[i]!;
const duration = durations[i]!;
if (currentTotal + duration > maxVideoRefDuration) {
exceeded = true;
break;
}
currentTotal += duration;
newVideos.push({
id: Math.random().toString(36).substring(7),
url: item.fullImage,
file: new File([], "library-video"),
mediaToken: item.id,
duration,
});
}
if (exceeded) {
toast.error(
`Total video duration cannot exceed ${maxVideoRefDuration}s`,
{ id: "video-ref-limit" },
);
}
if (newVideos.length > 0) {
setReferenceVideos?.([...baseVideos, ...newVideos]);
}
} finally {
setIsProcessingGallery(false);
}
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
return;
}
if (galleryTarget === "end") {
const item = selectedItems[0];
if (item && item.fullImage) {
@@ -1113,16 +1194,47 @@ export const ImagePromptRow = ({
)}
{referenceVideos.length < maxVideoCount &&
!uploadingVideo && (
<Button
variant="action"
className="bg-ui-controls/40 hover:bg-ui-controls/60 aspect-square w-full overflow-hidden rounded-lg w-14 border-dashed border-2 border-black/5 dark:border-white/25 transition-all"
onClick={handleUploadClickVideo}
<Tooltip
interactive={true}
position="top"
delay={100}
className="bg-ui-controls text-base-fg border border-ui-panel-border p-2 -mb-0.5"
closeOnClick={true}
content={
<div className="flex flex-col gap-1.5">
<Button
variant="primary"
onClick={handleUploadClickVideo}
icon={faPlus}
className="w-full"
>
Upload
</Button>
<Button
variant="action"
onClick={() => {
setGalleryTarget("video");
setIsGalleryModalOpen(true);
}}
icon={faImages}
className="w-full bg-base-fg/10 hover:bg-base-fg/20"
>
Pick from library
</Button>
</div>
}
>
<FontAwesomeIcon
icon={faPlus}
className="text-2xl opacity-80 text-base-fg"
/>
</Button>
<Button
variant="action"
className="bg-ui-controls/40 hover:bg-ui-controls/60 aspect-square w-full overflow-hidden rounded-lg w-14 border-dashed border-2 border-black/5 dark:border-white/25 transition-all"
onClick={handleUploadClickVideo}
>
<FontAwesomeIcon
icon={faPlus}
className="text-2xl opacity-80 text-base-fg"
/>
</Button>
</Tooltip>
)}
</div>
</div>
@@ -1198,17 +1310,23 @@ export const ImagePromptRow = ({
</div>
</div>
<GalleryModal
key={galleryTarget === "video" ? "video" : "image"}
isOpen={!!isGalleryModalOpen}
onClose={handleGalleryClose}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={handleImageSelect}
maxSelections={
galleryTarget === "end" ? 1 : Math.max(1, availableSlotsRender)
galleryTarget === "end"
? 1
: galleryTarget === "video"
? Math.max(1, maxVideoCount - referenceVideos.length)
: Math.max(1, availableSlotsRender)
}
onUseSelected={handleGalleryImages}
onDownloadClicked={downloadFileFromUrl}
forceFilter="image"
useSelectedLoading={isProcessingGallery}
forceFilter={galleryTarget === "video" ? "video" : "image"}
/>
</>
);
@@ -348,15 +348,26 @@ export const MentionTextarea = forwardRef<HTMLDivElement, MentionTextareaProps>(
}, [mentionItems, mentionState.isOpen, mentionState.query]);
// Build a regex that matches any known mention label (supports spaces in names)
// Sort longest-first so "@Pumpkin Head" matches before "@Pumpkin"
// Sort longest-first so "@Pumpkin Head" matches before "@Pumpkin".
// Case-insensitive so users can type "@image1" and still get it tagged
// when the canonical label is "@Image1".
const mentionRegex = useMemo(() => {
const labels = mentionItems.map((item) => item.label);
if (labels.length === 0) return null;
const sorted = [...labels].sort((a, b) => b.length - a.length);
const pattern = sorted.map((l) => escapeRegex(l)).join("|");
return new RegExp(`(${pattern})`, "g");
return new RegExp(`(${pattern})`, "gi");
}, [mentionItems]);
// Case-insensitive lookup table for colorMap so "@image1" finds "@Image1".
const lowerColorMap = useMemo(() => {
const m: Record<string, string> = {};
for (const [k, v] of Object.entries(colorMap)) {
m[k.toLowerCase()] = v;
}
return m;
}, [colorMap]);
// Build innerHTML with colored @mentions inline
const buildHTML = useCallback(
(text: string): string => {
@@ -376,7 +387,7 @@ export const MentionTextarea = forwardRef<HTMLDivElement, MentionTextareaProps>(
// biome-ignore lint/suspicious/noAssignInExpressions: --
while ((match = regex.exec(text)) !== null) {
const fullMatch = match[0];
const color = colorMap[fullMatch];
const color = lowerColorMap[fullMatch.toLowerCase()];
if (match.index > lastIndex) {
html += escapeHTML(text.slice(lastIndex, match.index));
@@ -402,7 +413,7 @@ export const MentionTextarea = forwardRef<HTMLDivElement, MentionTextareaProps>(
return html;
},
[colorMap, mentionRegex],
[lowerColorMap, mentionRegex],
);
// Sync DOM when value changes from parent (not from user input)
@@ -464,13 +475,20 @@ export const MentionTextarea = forwardRef<HTMLDivElement, MentionTextareaProps>(
}, []);
// Detect @mention trigger from cursor position
// Supports multi-word names by scanning back to the nearest @
// Supports multi-word names by scanning back to the nearest @.
// A valid trigger requires the char before @ to not be an ASCII
// identifier char (letter/digit/underscore) — this lets CJK characters,
// quotes, and punctuation all count as word boundaries, so users can
// type "@Image1" directly after Chinese text like 从@Image1.
const detectMention = useCallback(
(text: string, cursorPos: number) => {
// Find the last @ before cursor
const textBefore = text.slice(0, cursorPos);
const lastAt = textBefore.lastIndexOf("@");
if (lastAt !== -1 && (lastAt === 0 || /\s/.test(text[lastAt - 1]))) {
if (
lastAt !== -1 &&
(lastAt === 0 || !/[A-Za-z0-9_]/.test(text[lastAt - 1]))
) {
const query = text.slice(lastAt, cursorPos); // includes @
// Only open if there's no newline in the query
if (!query.includes("\n")) {
+85 -215
View File
File diff suppressed because it is too large Load Diff
+49 -12
View File
@@ -10,22 +10,60 @@ declare -r chrome_dir="/tmp/chrome/${entropy}"
# https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
# https://peter.sh/experiments/chromium-command-line-switches/
# NB: Creating the directory first seems to block process start.
# Perhaps this is the wrong chmod ownership flags.
#mkdir -p $CHROME_DIR
# Seed the profile with DevTools preferences before Chrome starts.
# This sets: dock position = bottom, default panel = network.
# Chrome stores these in <user-data-dir>/Default/Preferences.
seed_devtools_preferences() {
local prefs='{"devtools":{"preferences":{"currentDockState":"\"bottom\"","panel-selected-tab":"\"network\""}}}'
#--auto-open-devtools-for-tabs="true" \
#--devtools-flags="" \
# Seed both Default and Guest Profile — Chrome uses Guest Profile when
# launched with --guest, and Default otherwise.
for profile_name in "Default" "Guest Profile"; do
local prefs_dir="${chrome_dir}/${profile_name}"
mkdir -p "${prefs_dir}"
echo "${prefs}" > "${prefs_dir}/Preferences"
done
}
# TODO(bt): Math to determine better window placement.
# Flags reference:
# --auto-open-devtools-for-tabs Open DevTools automatically for every tab on launch.
# Dock position and default panel are set via seeded
# preferences (see seed_devtools_preferences above).
# --allow-running-insecure-content Allow HTTPS pages to load HTTP resources (mixed content).
# --disable-features=ChromeLabs, Disable Chrome Labs experiments UI and the
# DefaultBrowserPrompt feature-flag-based default browser prompt.
# --disable-site-isolation-trials Disable site isolation (allows cross-origin iframes
# to share process, useful for local dev).
# --disable-web-security Disable same-origin policy. Required for local dev
# where the frontend talks to a different-origin API.
# --ignore-certificate-errors Accept self-signed / invalid TLS certs.
# --no-default-browser-check Skip the "set as default browser" check on startup.
# --no-first-run Suppress the "Welcome to Google Chrome" first-run
# dialog (default browser checkbox, crash reporting, etc.).
# --install-autogenerated-theme Set a custom window color so the dev browser is
# visually distinct from your normal Chrome.
# --unsafely-treat-insecure-origin-as-secure
# Treat the given HTTP origin as secure. Required for
# APIs like getUserMedia() / navigator.mediaDevices
# which need a "secure context".
# --user-data-dir Use a throwaway profile directory so dev sessions
# don't pollute your real Chrome profile.
# --guest (Mac only) Launch in guest mode — no Google login screen.
# --window-position / --window-size Place the dev window predictably on screen.
launch_linux() {
seed_devtools_preferences
chromium \
--auto-open-devtools-for-tabs \
--allow-running-insecure-content \
--disable-features=ChromeLabs \
--disable-features=ChromeLabs,DefaultBrowserPrompt \
--disable-site-isolation-trials \
--disable-web-security \
--ignore-certificate-errors \
--no-default-browser-check \
--no-first-run \
--install-autogenerated-theme="100,150,255" \
--unsafely-treat-insecure-origin-as-secure="http://localhost:5173" \
--user-data-dir="${chrome_dir}" \
@@ -35,16 +73,15 @@ launch_linux() {
}
launch_mac() {
# See https://stackoverflow.com/a/58658101
# NB: '7001' port is being chosen because something on the system is hogging port 7000.
# chrome://flags/#unsafely-treat-insecure-origin-as-secure - allow microphone recording
# otherwise getUserMedia() / navigator.mediaDevices will not function
# --guest: no Google login screen
seed_devtools_preferences
open -na "Google Chrome" --args \
--disable-features=ChromeLabs \
--auto-open-devtools-for-tabs \
--disable-features=ChromeLabs,DefaultBrowserPrompt \
--disable-site-isolation-trials \
--disable-web-security \
--ignore-certificate-errors \
--no-default-browser-check \
--no-first-run \
--install-autogenerated-theme="100,150,255" \
--unsafely-treat-insecure-origin-as-secure="http://localhost:5173" \
--user-data-dir="${chrome_dir}" \