Compare commits

..
10 Commits
Author SHA1 Message Date
Brandon Thomas 046370c839 reuse lol 2026-02-20 15:45:21 +07:00
Brandon Thomas 6df8e03d97 cost estimate in credits 2026-02-20 14:48:03 +07:00
Brandon Thomas 1bc09a29e4 update endpoint 2026-02-20 14:31:59 +07:00
Brandon Thomas dfdfcaefeb need to create wallets for local testing 2026-02-20 14:25:31 +07:00
Brandon Thomas 5d1cfba7b4 cleanup 2026-02-20 14:13:20 +07:00
Brandon Thomas 03734a2961 all job tokens 2026-02-20 09:49:58 +07:00
Brandon Thomas 0d0595e557 test fix 2026-02-20 09:39:13 +07:00
Brandon Thomas 1b1c9b246c new handler 2026-02-20 09:34:21 +07:00
Brandon Thomas d5e0afc543 remove 2026-02-20 07:42:41 +07:00
Brandon Thomas 8be9e7ee30 cost estimates 2026-02-20 07:17:44 +07:00
51 changed files with 1157 additions and 2659 deletions
-1
View File
@@ -9,7 +9,6 @@ on:
# NB: Default-branch doesn't work, despite Github's documentation.
#- $default-branch
- main
- test
env:
IMAGE_NAME: storyteller-rust
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nUPDATE generic_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n internal_debugging_failure_reason = ?,\n frontend_failure_category = ?,\n retry_at = NULL\nWHERE token = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 5
},
"nullable": []
},
"hash": "15939605e0da2722693371799ba94c7d0cc2f77a5c5b37e346855996d16033ec"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO generic_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n job_type = ?,\n\n maybe_external_third_party = ?,\n maybe_external_third_party_id = ?,\n\n product_category = ?,\n inference_category = ?,\n\n maybe_model_type = NULL,\n maybe_model_token = NULL,\n\n maybe_input_source_token = NULL,\n maybe_input_source_token_type = NULL,\n\n maybe_download_url = NULL,\n maybe_cover_image_media_file_token = NULL,\n\n maybe_prompt_token = ?,\n\n maybe_raw_inference_text = NULL,\n\n maybe_inference_args = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n priority_level = 0,\n is_keepalive_required = FALSE,\n max_duration_seconds = 0,\n\n is_debug_request = FALSE,\n maybe_routing_tag = NULL,\n\n status = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 14
},
"nullable": []
},
"hash": "33aa8a1cf85bf1b8b68f4e2ab46541f25d7e8bb5eeb503c5ec14cb5e115f69ce"
}
@@ -1,91 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n jobs.maybe_external_third_party_id as `order_id`,\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n jobs.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\n AND jobs.status NOT IN ('complete_success', 'complete_failure')\n AND jobs.maybe_external_third_party_id IS NOT NULL\n\nLIMIT 25000\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "order_id",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 256
}
},
{
"ordinal": 2,
"name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 3,
"name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 4,
"name": "creator_ip_address",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 160
}
},
{
"ordinal": 5,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | ENUM",
"char_set": 224,
"max_size": 28
}
},
{
"ordinal": 6,
"name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 128
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
true,
true,
true,
false,
false,
true
]
},
"hash": "ca96942d8214edb84f163cf40131ceed873209e86718054e1502219fc81d2dc6"
}
Generated
+317 -402
View File
File diff suppressed because it is too large Load Diff
+8 -10
View File
@@ -117,7 +117,6 @@ members = [
"crates/service/job/email_sender_job",
"crates/service/job/es-update-job",
"crates/service/job/inference_job",
"crates/service/job/seedance2_pro_job",
"crates/service/job/tts_download_job",
"crates/service/job/tts_inference_job",
"crates/service/job/w2l_download_job",
@@ -261,24 +260,24 @@ actix-utils = "3.0.1"
actix-web = "4.7.0" # TODO(bt): We're behind and need to bump versions.
actix-web-actors = "4.3.1+deprecated"
actix-web-lab = "0.20.2"
anyhow = "1.0.102"
anyhow = "1.0.98"
base64 = "0.22.1"
bstr = "1.12.1"
bytes = "1.10.1"
chrono = { version = "0.4.41", features = ["serde"] }
cookie = "0.18.1"
env = "1.0.1"
env_logger = "0.11.9"
env_logger = "0.11.8"
futures = { version = "0.3.31" }
hex-literal = "1.0.0"
indexmap = "2.9.0"
log = "0.4.29"
log = "0.4.27"
md-5 = "0.10.6" # insecure but used for gravatar
once_cell = "1.21.3"
r2d2 = "0.8.10"
rand = "0.9.1"
redis = { version = "1.0.2", features = ["r2d2", "tokio-rustls-comp", "tokio-comp", "cluster-async", "aio"] }
regex = "1.12.3"
regex = "1.12.2"
reqwest = { version = "0.12.23", default-features = false, features = [ "charset", "http2", "system-proxy", "json", "cookies", "rustls-tls", "gzip" ] }
reqwest-websocket = { version = "0.5.0" }
serde = { version = "1.0.219", features = ["derive"] }
@@ -296,11 +295,11 @@ tempdir = "0.3.7"
tempfile = "3.20.0"
thiserror = "2.0.12"
tokio = { version = "1.45.1", features = ["macros"] }
toml = "0.8.23"
toml = "0.8.22"
url = { version = "2.5.4", features = ["serde"] }
uuid = "1.18.1"
# NB(1): wreq gets past fingerprinting, but we have to deal with platform-specific build issues.
# NB: wreq gets past fingerprinting, but we have to deal with platform-specific build issues.
#
# Mac:
# brew install --formula cmake pkg-config llvm
@@ -310,6 +309,5 @@ uuid = "1.18.1"
# choco install pkgconfiglite -y
# choco install llvm -y
# choco install nasm -y
wreq-util = "3.0.0-rc.10"
wreq = { version = "6.0.0-rc.28", features = ["json", "query", "multipart", "cookies", "ws", "stream", "gzip", "form"] }
wreq = { version = "6.0.0-rc.20", features = ["json", "multipart", "cookies", "ws", "stream"] }
wreq-util = "3.0.0-rc.6"
+1 -11
View File
@@ -33,7 +33,7 @@ RUN apt-get update \
FROM ubuntu:jammy as rust-base
# NB: This can be "stable" or another version.
ARG RUST_TOOLCHAIN="1.93.0"
ARG RUST_TOOLCHAIN="1.86.0"
WORKDIR /tmp
@@ -48,12 +48,8 @@ RUN apt-get update \
curl \
ffmpeg \
fontconfig \
git \
libclang-dev \
libfontconfig1-dev \
libssl-dev \
musl-tools \
perl \
pkg-config
# NB: Fix for fontconfig (servo-fontconfig-sys): https://github.com/alacritty/alacritty/issues/4423#issuecomment-727277235
@@ -137,12 +133,6 @@ RUN SQLX_OFFLINE=true \
--release \
--bin storyteller-web
RUN SQLX_OFFLINE=true \
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
$HOME/.cargo/bin/cargo build \
--release \
--bin seedance2-pro-job
RUN SQLX_OFFLINE=true \
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
$HOME/.cargo/bin/cargo build \
@@ -1,5 +1,4 @@
pub mod kling_2p5_turbo_pro_multi_function_video_gen;
pub mod seedance_2p0_multi_function_video_gen;
pub mod kling_2p6_pro_multi_function_video_gen;
pub mod sora_2_multi_function_video_gen;
pub mod sora_2_pro_multi_function_video_gen;
@@ -1,18 +0,0 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::seedance_2p0_multi_function_video_gen::{Seedance2p0MultiFunctionVideoGenRequest, Seedance2p0MultiFunctionVideoGenResponse, SEEDANCE_2P0_MULTI_FUNCTION_VIDEO_GEN_PATH};
pub async fn seedance_2p0_multi_function_video_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Seedance2p0MultiFunctionVideoGenRequest,
) -> Result<Seedance2p0MultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
SEEDANCE_2P0_MULTI_FUNCTION_VIDEO_GEN_PATH,
maybe_creds,
request,
).await?)
}
@@ -36,11 +36,11 @@ sha3.workspace = true
thiserror.workspace = true
#tokio.workspace = true
toml.workspace = true
url.workspace = true
tokio = { version = "1", features = ["full"] }
wreq-util.workspace = true
wreq.workspace = true
url.workspace = true
[dev-dependencies]
testing = { path = "../../lib/testing" }
+1 -1
View File
@@ -26,8 +26,8 @@ rand.workspace = true
serde.workspace = true
serde_derive.workspace = true
serde_json.workspace = true
wreq-util.workspace = true
wreq.workspace = true
wreq-util.workspace = true
url.workspace = true
[dev-dependencies]
@@ -117,7 +117,7 @@ pub async fn poll_orders(args: PollOrdersArgs<'_>) -> Result<PollOrdersResponse,
let cookie = args.session.cookies.as_str();
let request = client.get(GET_ORDERS_BASE_URL)
let response = client.get(GET_ORDERS_BASE_URL)
.query(&[("batch", "1"), ("input", input_json.as_str())])
.header("User-Agent", FIREFOX_USER_AGENT)
.header("Accept", "*/*")
@@ -133,10 +133,7 @@ pub async fn poll_orders(args: PollOrdersArgs<'_>) -> Result<PollOrdersResponse,
.header("Sec-Fetch-Site", "same-origin")
.header("Priority", "u=4")
.header("TE", "trailers")
.build()
.map_err(|err| Seedance2ProClientError::WreqClientError(err))?;
let response = client.execute(request)
.send()
.await
.map_err(|err| Seedance2ProGenericApiError::WreqError(err))?;
@@ -12,10 +12,8 @@ normal = ["workspace-build-acceleration"]
# Workspace hack (faster compile times)
workspace-build-acceleration.workspace = true
# Internal
easyenv = { path = "../../lib/easyenv" }
# External
easyenv = { path = "../../lib/easyenv" }
log.workspace = true
reqwest = { version = "0.12.4", default-features = false, features = [ "charset", "http2", "system-proxy", "rustls-tls" ] }
serde_json.workspace = true
@@ -1,62 +0,0 @@
use crate::core::commands::enqueue::generate_error::GenerateError;
use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::{EnqueueImageToVideoRequest, SoraOrientation};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::events::generation_events::common::{GenerationModel};
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use artcraft_api_defs::generate::video::multi_function::seedance_2p0_multi_function_video_gen::{Seedance2p0AspectRatio, Seedance2p0MultiFunctionVideoGenRequest};
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use idempotency::uuid::generate_random_uuid;
use log::{error, info};
use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
use storyteller_client::endpoints::generate::video::multi_function::seedance_2p0_multi_function_video_gen::seedance_2p0_multi_function_video_gen;
pub (super) async fn handle_artcraft_seedance_2p0(
request: &EnqueueImageToVideoRequest,
app_env_configs: &AppEnvConfigs,
creds: &StorytellerCredentialSet,
) -> Result<TaskEnqueueSuccess, GenerateError> {
let uuid_idempotency_token = generate_random_uuid();
let aspect_ratio = match request.sora_orientation {
Some(SoraOrientation::Portrait) => Seedance2p0AspectRatio::Portrait9x16,
Some(SoraOrientation::Landscape) => Seedance2p0AspectRatio::Landscape16x9,
None => Seedance2p0AspectRatio::Landscape16x9,
};
let request = Seedance2p0MultiFunctionVideoGenRequest {
uuid_idempotency_token,
prompt: request.prompt.clone(),
start_frame_media_token: request.image_media_token.clone(),
end_frame_media_token: request.end_frame_image_media_token.clone(),
reference_image_media_tokens: None,
aspect_ratio: Some(aspect_ratio),
duration_seconds: None, // TODO: Parameterize
batch_count: None,
};
let result = seedance_2p0_multi_function_video_gen(
&app_env_configs.storyteller_host,
Some(&creds),
request,
).await;
let job_token = match result {
Ok(enqueued) => {
info!("Successfully enqueued.");
enqueued.inference_job_token
}
Err(err) => {
error!("Failed to enqueue: {:?}", err);
return Err(GenerateError::from(err));
}
};
Ok(TaskEnqueueSuccess {
task_type: TaskType::VideoGeneration,
model: Some(GenerationModel::Seedance2p0),
provider: GenerationProvider::Artcraft,
provider_job_id: Some(job_token.to_string()),
})
}
@@ -6,7 +6,6 @@ use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_kl
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_kling_2p5_turbo_pro::handle_artcraft_kling_2p5_turbo_pro;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_kling_2p6_pro::handle_artcraft_kling_2p6_pro;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_seedance_1_lite::handle_artcraft_seedance_1_lite;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_seedance_2p0::handle_artcraft_seedance_2p0;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_sora2::handle_artcraft_sora2;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_sora2_pro::handle_artcraft_sora2_pro;
use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_veo2::handle_artcraft_veo2;
@@ -40,7 +39,6 @@ pub async fn handle_video_artcraft(
Some(VideoModel::Kling2p5TurboPro) => handle_artcraft_kling_2p5_turbo_pro(request, app_env_configs, &creds).await,
Some(VideoModel::Kling2p6Pro) => handle_artcraft_kling_2p6_pro(request, app_env_configs, &creds).await,
Some(VideoModel::Seedance10Lite) => handle_artcraft_seedance_1_lite(request, app_env_configs, &creds).await,
Some(VideoModel::Seedance2p0) => handle_artcraft_seedance_2p0(request, app_env_configs, &creds).await,
Some(VideoModel::Sora2) => handle_artcraft_sora2(request, app_env_configs, &creds).await,
Some(VideoModel::Sora2Pro) => handle_artcraft_sora2_pro(request, app_env_configs, &creds).await,
Some(VideoModel::Veo2) => handle_artcraft_veo2(request, app_env_configs, &creds).await,
@@ -5,7 +5,6 @@ mod handle_artcraft_kling_2p1_pro;
mod handle_artcraft_kling_2p5_turbo_pro;
mod handle_artcraft_kling_2p6_pro;
mod handle_artcraft_seedance_1_lite;
mod handle_artcraft_seedance_2p0;
mod handle_artcraft_sora2;
mod handle_artcraft_sora2_pro;
mod handle_artcraft_veo2;
@@ -56,9 +56,6 @@ pub enum VideoModel {
#[serde(rename = "seedance_1.0_lite")]
Seedance10Lite,
#[serde(rename = "seedance_2p0")]
Seedance2p0,
#[serde(rename = "sora_2")]
Sora2,
@@ -82,7 +82,6 @@ impl TaskEnqueueSuccess{
Some(GenerationModel::Kling2p5TurboPro) => Some(TaskModelType::Kling2p5TurboPro),
Some(GenerationModel::Kling2p6Pro) => Some(TaskModelType::Kling2p6Pro),
Some(GenerationModel::Seedance10Lite) => Some(TaskModelType::Seedance10Lite),
Some(GenerationModel::Seedance2p0) => Some(TaskModelType::Seedance2p0),
Some(GenerationModel::Sora2) => Some(TaskModelType::Sora2),
Some(GenerationModel::Sora2Pro) => Some(TaskModelType::Sora2Pro),
Some(GenerationModel::Veo2) => Some(TaskModelType::Veo2),
@@ -86,8 +86,6 @@ pub enum GenerationModel {
Sora2Pro,
#[serde(rename = "seedance_1.0_lite")]
Seedance10Lite,
#[serde(rename = "seedance_2p0")]
Seedance2p0,
#[serde(rename = "veo_2")]
Veo2,
#[serde(rename = "veo_3")]
@@ -16,5 +16,4 @@ pub fn add_artcraft_3d(cors: Cors, _is_production: bool) -> Cors {
.allowed_origin("http://localhost:5175") // If already started
.allowed_origin("https://macaroni-1.tailce84f.ts.net") // TODO
.allowed_origin("https://halide.tailce84f.ts.net") // TODO
.allowed_origin("https://brandons-macbook-pro.taild62114.ts.net") // TODO
}
@@ -22,8 +22,5 @@ workspace-build-acceleration.workspace = true
once_cell.workspace = true
wreq.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
wreq = { workspace = true, features = ["prefix-symbols"] }
[dev-dependencies]
# (None)
@@ -1,56 +0,0 @@
use anyhow::anyhow;
use sqlx::MySqlPool;
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
use enums::common::job_status_plus::JobStatusPlus;
use errors::AnyhowResult;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
pub struct MarkJobFailedByTokenArgs<'a> {
pub pool: &'a MySqlPool,
pub job_token: &'a InferenceJobToken,
pub maybe_public_failure_reason: Option<&'a str>,
pub internal_debugging_failure_reason: &'a str,
pub maybe_frontend_failure_category: Option<FrontendFailureCategory>,
}
/// Permanently mark an inference job as failed, looked up by its token.
/// Unlike `mark_generic_inference_job_failure`, this does not allow retries —
/// the job will always land in `complete_failure`.
pub async fn mark_job_failed_by_token(args: MarkJobFailedByTokenArgs<'_>) -> AnyhowResult<()> {
let maybe_public_failure_reason = args.maybe_public_failure_reason.map(|reason| {
let mut reason = reason.trim().to_string();
reason.truncate(512);
reason
});
let mut internal_debugging_failure_reason = args.internal_debugging_failure_reason.trim().to_string();
internal_debugging_failure_reason.truncate(512);
const FAILURE_STATUS: &str = JobStatusPlus::CompleteFailure.to_str();
let query_result = sqlx::query!(
r#"
UPDATE generic_inference_jobs
SET
status = ?,
failure_reason = ?,
internal_debugging_failure_reason = ?,
frontend_failure_category = ?,
retry_at = NULL
WHERE token = ?
"#,
FAILURE_STATUS,
maybe_public_failure_reason.as_deref(),
&internal_debugging_failure_reason,
args.maybe_frontend_failure_category,
args.job_token.as_str()
)
.execute(args.pool)
.await;
match query_result {
Err(err) => Err(anyhow!("error with query: {:?}", err)),
Ok(_r) => Ok(()),
}
}
@@ -3,7 +3,6 @@ pub mod count_untried_jobs_of_type;
pub mod list_available_generic_inference_jobs;
pub mod mark_generic_inference_job_completely_failed;
pub mod mark_generic_inference_job_failure;
pub mod mark_job_failed_by_token;
pub mod mark_generic_inference_job_pending_and_grab_lock;
pub mod mark_generic_inference_job_successfully_done;
pub mod reap_stale_fakeyou_jobs;
@@ -1,92 +0,0 @@
use anyhow::anyhow;
use log::warn;
use sqlx::MySqlPool;
use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::prompts::PromptToken;
use tokens::tokens::users::UserToken;
/// A Seedance2Pro job that is waiting for a result from the external API.
#[derive(Debug)]
pub struct PendingSeedance2ProJob {
pub job_token: InferenceJobToken,
/// The external order_id from seedance2-pro.com.
pub order_id: String,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_prompt_token: Option<PromptToken>,
}
#[derive(Debug, Default)]
struct RawRecord {
job_token: InferenceJobToken,
order_id: Option<String>,
maybe_creator_user_token: Option<UserToken>,
maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
creator_ip_address: String,
creator_set_visibility: Visibility,
maybe_prompt_token: Option<PromptToken>,
}
/// Returns all non-terminal Seedance2Pro jobs that have an associated order_id.
pub async fn list_pending_seedance2pro_jobs(pool: &MySqlPool) -> AnyhowResult<Vec<PendingSeedance2ProJob>> {
let records = sqlx::query_as!(
RawRecord,
r#"
SELECT
jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,
jobs.maybe_external_third_party_id as `order_id`,
jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,
jobs.creator_ip_address,
jobs.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`
FROM generic_inference_jobs as jobs
WHERE jobs.maybe_external_third_party = ?
AND jobs.status NOT IN ('complete_success', 'complete_failure')
AND jobs.maybe_external_third_party_id IS NOT NULL
LIMIT 25000
"#,
InferenceJobExternalThirdParty::Seedance2Pro.to_str(),
)
.fetch_all(pool)
.await
.map_err(|err| anyhow!("error querying pending seedance2pro jobs: {:?}", err))?;
let jobs = records
.into_iter()
.filter_map(|record| {
let order_id = match record.order_id {
Some(id) => id,
None => {
warn!("PendingSeedance2ProJob has no order_id, skipping");
return None;
}
};
Some(PendingSeedance2ProJob {
job_token: record.job_token,
order_id,
maybe_creator_user_token: record.maybe_creator_user_token,
maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token,
creator_ip_address: record.creator_ip_address,
creator_set_visibility: record.creator_set_visibility,
maybe_prompt_token: record.maybe_prompt_token,
})
})
.collect();
Ok(jobs)
}
@@ -1,2 +1 @@
pub mod insert_generic_inference_job_for_seedance2pro_queue_with_apriori_job_token;
pub mod list_pending_seedance2pro_jobs;
@@ -42,11 +42,6 @@ pub enum FrontendFailureCategory {
/// exactly what happened. We'll retry their workload in any case.
#[serde(rename = "retryable_worker_error")]
RetryableWorkerError,
/// Model content rules were violated
/// Eg. Seedance 2 will report: "your input text violates platform rules. please modify and try again"
#[serde(rename = "model_rules_violation")]
ModelRulesViolation,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -61,7 +56,6 @@ impl FrontendFailureCategory {
Self::KeepAliveElapsed => "keep_alive_elapsed",
Self::NotYetImplemented => "not_yet_implemented",
Self::RetryableWorkerError => "retryable_worker_error",
Self::ModelRulesViolation => "model_rules_violation",
}
}
@@ -71,7 +65,6 @@ impl FrontendFailureCategory {
"keep_alive_elapsed" => Ok(Self::KeepAliveElapsed),
"not_yet_implemented" => Ok(Self::NotYetImplemented),
"retryable_worker_error" => Ok(Self::RetryableWorkerError),
"model_rules_violation" => Ok(Self::ModelRulesViolation),
_ => Err(format!("invalid value: {:?}", value)),
}
}
@@ -84,7 +77,6 @@ impl FrontendFailureCategory {
Self::KeepAliveElapsed,
Self::NotYetImplemented,
Self::RetryableWorkerError,
Self::ModelRulesViolation,
])
}
}
@@ -103,7 +95,6 @@ mod tests {
assert_serialization(FrontendFailureCategory::KeepAliveElapsed, "keep_alive_elapsed");
assert_serialization(FrontendFailureCategory::NotYetImplemented, "not_yet_implemented");
assert_serialization(FrontendFailureCategory::RetryableWorkerError, "retryable_worker_error");
assert_serialization(FrontendFailureCategory::ModelRulesViolation, "model_rules_violation");
}
#[test]
@@ -112,7 +103,6 @@ mod tests {
assert_eq!(FrontendFailureCategory::KeepAliveElapsed.to_str(), "keep_alive_elapsed");
assert_eq!(FrontendFailureCategory::NotYetImplemented.to_str(), "not_yet_implemented");
assert_eq!(FrontendFailureCategory::RetryableWorkerError.to_str(), "retryable_worker_error");
assert_eq!(FrontendFailureCategory::ModelRulesViolation.to_str(), "model_rules_violation");
}
#[test]
@@ -121,18 +111,16 @@ mod tests {
assert_eq!(FrontendFailureCategory::from_str("keep_alive_elapsed").unwrap(), FrontendFailureCategory::KeepAliveElapsed);
assert_eq!(FrontendFailureCategory::from_str("not_yet_implemented").unwrap(), FrontendFailureCategory::NotYetImplemented);
assert_eq!(FrontendFailureCategory::from_str("retryable_worker_error").unwrap(), FrontendFailureCategory::RetryableWorkerError);
assert_eq!(FrontendFailureCategory::from_str("model_rules_violation").unwrap(), FrontendFailureCategory::ModelRulesViolation);
}
#[test]
fn all_variants() {
let mut variants = FrontendFailureCategory::all_variants();
assert_eq!(variants.len(), 5);
assert_eq!(variants.len(), 4);
assert_eq!(variants.pop_first(), Some(FrontendFailureCategory::FaceNotDetected));
assert_eq!(variants.pop_first(), Some(FrontendFailureCategory::KeepAliveElapsed));
assert_eq!(variants.pop_first(), Some(FrontendFailureCategory::NotYetImplemented));
assert_eq!(variants.pop_first(), Some(FrontendFailureCategory::RetryableWorkerError));
assert_eq!(variants.pop_first(), Some(FrontendFailureCategory::ModelRulesViolation));
assert_eq!(variants.pop_first(), None);
}
}
@@ -66,9 +66,6 @@ pub enum MediaFileOriginProductCategory {
#[serde(rename = "image_gen")]
ImageGeneration,
#[serde(rename = "video_gen")]
VideoGeneration,
// Media files for video filters
#[deprecated(note = "This isn't relevant product surface area anymore")]
#[serde(rename = "video_filter")]
@@ -101,7 +98,6 @@ impl MediaFileOriginProductCategory {
Self::ZeroShotVoice => "zs_voice",
Self::Mocap => "mocap",
Self::ImageGeneration => "image_gen",
Self::VideoGeneration => "video_gen",
Self::VideoFilter => "video_filter",
Self::Workflow => "workflow",
}
@@ -121,7 +117,6 @@ impl MediaFileOriginProductCategory {
"zs_voice" => Ok(Self::ZeroShotVoice),
"mocap" => Ok(Self::Mocap),
"image_gen" => Ok(Self::ImageGeneration),
"video_gen" => Ok(Self::VideoGeneration),
"video_filter" => Ok(Self::VideoFilter),
"workflow" => Ok(Self::Workflow),
_ => Err(format!("invalid value: {:?}", value)),
@@ -144,7 +139,6 @@ impl MediaFileOriginProductCategory {
Self::ZeroShotVoice,
Self::Mocap,
Self::ImageGeneration,
Self::VideoGeneration,
Self::VideoFilter,
Self::Workflow,
])
@@ -173,7 +167,6 @@ mod tests {
assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice");
assert_serialization(MediaFileOriginProductCategory::Mocap, "mocap");
assert_serialization(MediaFileOriginProductCategory::ImageGeneration, "image_gen");
assert_serialization(MediaFileOriginProductCategory::VideoGeneration, "video_gen");
assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter");
assert_serialization(MediaFileOriginProductCategory::Workflow, "workflow");
}
@@ -192,7 +185,6 @@ mod tests {
assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice");
assert_eq!(MediaFileOriginProductCategory::Mocap.to_str(), "mocap");
assert_eq!(MediaFileOriginProductCategory::ImageGeneration.to_str(), "image_gen");
assert_eq!(MediaFileOriginProductCategory::VideoGeneration.to_str(), "video_gen");
assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter");
assert_eq!(MediaFileOriginProductCategory::Workflow.to_str(), "workflow");
}
@@ -211,7 +203,6 @@ mod tests {
assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice);
assert_eq!(MediaFileOriginProductCategory::from_str("mocap").unwrap(), MediaFileOriginProductCategory::Mocap);
assert_eq!(MediaFileOriginProductCategory::from_str("image_gen").unwrap(),MediaFileOriginProductCategory::ImageGeneration);
assert_eq!(MediaFileOriginProductCategory::from_str("video_gen").unwrap(), MediaFileOriginProductCategory::VideoGeneration);
assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter);
assert_eq!(MediaFileOriginProductCategory::from_str("workflow").unwrap(), MediaFileOriginProductCategory::Workflow);
}
@@ -219,7 +210,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = MediaFileOriginProductCategory::all_variants();
assert_eq!(variants.len(), 15);
assert_eq!(variants.len(), 14);
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion));
@@ -232,7 +223,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Mocap));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageGeneration));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoGeneration));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Workflow));
assert_eq!(variants.pop_first(), None);
@@ -43,7 +43,7 @@ impl Default for JobStatusPlus {
}
impl JobStatusPlus {
pub const fn to_str(&self) -> &'static str {
pub fn to_str(&self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Started => "started",
@@ -65,8 +65,6 @@ pub enum TaskModelType {
Kling2p6Pro,
#[serde(rename = "seedance_1.0_lite")]
Seedance10Lite,
#[serde(rename = "seedance_2p0")]
Seedance2p0,
#[serde(rename = "sora_2")]
Sora2,
#[serde(rename = "sora_2_pro")]
@@ -128,7 +126,6 @@ impl TaskModelType {
Self::Kling2p5TurboPro => "kling_2p5_turbo_pro",
Self::Kling2p6Pro => "kling_2p6_pro",
Self::Seedance10Lite => "seedance_1.0_lite",
Self::Seedance2p0 => "seedance_2p0",
Self::Sora2 => "sora_2",
Self::Sora2Pro => "sora_2_pro",
Self::Veo2 => "veo_2",
@@ -172,7 +169,6 @@ impl TaskModelType {
"kling_2p5_turbo_pro" => Ok(Self::Kling2p5TurboPro),
"kling_2p6_pro" => Ok(Self::Kling2p6Pro),
"seedance_1.0_lite" => Ok(Self::Seedance10Lite),
"seedance_2p0" => Ok(Self::Seedance2p0),
"sora_2" => Ok(Self::Sora2),
"sora_2_pro" => Ok(Self::Sora2Pro),
"veo_2" => Ok(Self::Veo2),
@@ -219,7 +215,6 @@ impl TaskModelType {
Self::Kling2p5TurboPro,
Self::Kling2p6Pro,
Self::Seedance10Lite,
Self::Seedance2p0,
Self::Sora2,
Self::Sora2Pro,
Self::Veo2,
@@ -273,7 +268,6 @@ mod tests {
assert_serialization(TaskModelType::Kling2p5TurboPro, "kling_2p5_turbo_pro");
assert_serialization(TaskModelType::Kling2p6Pro, "kling_2p6_pro");
assert_serialization(TaskModelType::Seedance10Lite, "seedance_1.0_lite");
assert_serialization(TaskModelType::Seedance2p0, "seedance_2p0");
assert_serialization(TaskModelType::Sora2, "sora_2");
assert_serialization(TaskModelType::Sora2Pro, "sora_2_pro");
assert_serialization(TaskModelType::Veo2, "veo_2");
@@ -316,7 +310,6 @@ mod tests {
assert_eq!(TaskModelType::Kling2p5TurboPro.to_str(), "kling_2p5_turbo_pro");
assert_eq!(TaskModelType::Kling2p6Pro.to_str(), "kling_2p6_pro");
assert_eq!(TaskModelType::Seedance10Lite.to_str(), "seedance_1.0_lite");
assert_eq!(TaskModelType::Seedance2p0.to_str(), "seedance_2p0");
assert_eq!(TaskModelType::Sora2.to_str(), "sora_2");
assert_eq!(TaskModelType::Sora2Pro.to_str(), "sora_2_pro");
assert_eq!(TaskModelType::Veo2.to_str(), "veo_2");
@@ -359,7 +352,6 @@ mod tests {
assert_eq!(TaskModelType::from_str("kling_2p5_turbo_pro").unwrap(), TaskModelType::Kling2p5TurboPro);
assert_eq!(TaskModelType::from_str("kling_2p6_pro").unwrap(), TaskModelType::Kling2p6Pro);
assert_eq!(TaskModelType::from_str("seedance_1.0_lite").unwrap(), TaskModelType::Seedance10Lite);
assert_eq!(TaskModelType::from_str("seedance_2p0").unwrap(), TaskModelType::Seedance2p0);
assert_eq!(TaskModelType::from_str("sora_2").unwrap(), TaskModelType::Sora2);
assert_eq!(TaskModelType::from_str("sora_2_pro").unwrap(), TaskModelType::Sora2Pro);
assert_eq!(TaskModelType::from_str("veo_2").unwrap(), TaskModelType::Veo2);
@@ -388,7 +380,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = TaskModelType::all_variants();
assert_eq!(variants.len(), 36);
assert_eq!(variants.len(), 35);
// Image models
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell));
@@ -415,7 +407,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(TaskModelType::Kling2p5TurboPro));
assert_eq!(variants.pop_first(), Some(TaskModelType::Kling2p6Pro));
assert_eq!(variants.pop_first(), Some(TaskModelType::Seedance10Lite));
assert_eq!(variants.pop_first(), Some(TaskModelType::Seedance2p0));
assert_eq!(variants.pop_first(), Some(TaskModelType::Sora2));
assert_eq!(variants.pop_first(), Some(TaskModelType::Sora2Pro));
assert_eq!(variants.pop_first(), Some(TaskModelType::Veo2));
@@ -1,57 +0,0 @@
[package]
name = "seedance2-pro-job"
edition = "2021"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[[bin]]
name = "seedance2-pro-job"
path = "src/main.rs"
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { workspace = true }
# Internal
actix_helpers.workspace = true
bootstrap = { path = "../../../lib/bootstrap" }
bucket_paths = { path = "../../../lib/bucket_paths" }
cloud_storage.workspace = true
concurrency = { path = "../../../lib/concurrency" }
config = { path = "../../../lib/deprecated/config" }
easyenv = { path = "../../../lib/easyenv" }
enums.workspace = true
errors.workspace = true
hashing = { path = "../../../lib/files/hashing" }
jobs_common = { path = "../../../lib/jobs_common" }
mysql_queries = { path = "../../../schema/database/mysql_queries" }
seedance2pro = { path = "../../../api_clients/seedance2pro" }
server_environment = { path = "../../../lib/server_environment" }
tokens.workspace = true
# External
anyhow.workspace = true
log.workspace = true
sqlx = { version = "0.7.4", features = [ "mysql", "runtime-tokio-rustls", "chrono" ] }
# Runtime
tokio = { version = "1.40.0", features = ["macros", "sync"] }
# HTTP Client
reqwest.workspace = true
# HTTP Server
actix-web.workspace = true
# Serialization
serde.workspace = true
serde_derive.workspace = true
serde_json.workspace = true
@@ -1,4 +0,0 @@
# default .env configurations (these can be overridden with k8s)
# Development
SERVER_ENVIRONMENT='development'
@@ -1,101 +0,0 @@
use std::sync::Arc;
use actix_web::error::ResponseError;
use actix_web::http::header::ContentType;
use actix_web::http::StatusCode;
use actix_web::{web, HttpRequest, HttpResponse};
use log::error;
use actix_helpers::response_serializers::error_to_json_http_response::error_to_json_http_response;
use crate::http_server::http_server_shared_state::HttpServerSharedState;
#[derive(Serialize)]
pub struct HealthCheckResponse {
pub success: bool,
pub is_healthy: bool,
pub consecutive_failure_count: u64,
pub consecutive_success_count: u64,
pub total_failure_count: u64,
pub total_success_count: u64,
pub total_failure_ratio: f32,
pub total_success_ratio: f32,
}
#[derive(Debug, Serialize)]
pub enum HealthCheckError {
ServerError,
}
impl ResponseError for HealthCheckError {
fn status_code(&self) -> StatusCode {
match *self {
HealthCheckError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
error_to_json_http_response(self)
}
}
impl std::fmt::Display for HealthCheckError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn get_health_check_handler(
_http_request: HttpRequest,
server_state: web::Data<Arc<HttpServerSharedState>>,
) -> Result<HttpResponse, HealthCheckError> {
let job_stats = server_state
.job_stats
.get_status()
.map_err(|e| {
error!("Error serving health check status: {:?}", e);
HealthCheckError::ServerError
})?;
let total_tries = job_stats
.total_failure_count
.saturating_add(job_stats.total_success_count);
let total_success_ratio = if total_tries > 0 {
(job_stats.total_success_count as f32) / (total_tries as f32)
} else {
0.0
};
let total_failure_ratio = if total_tries > 0 {
1.0 - total_success_ratio
} else {
0.0
};
let is_healthy =
job_stats.consecutive_failure_count < server_state.consecutive_failure_unhealthy_threshold;
let response = HealthCheckResponse {
success: true,
is_healthy,
consecutive_failure_count: job_stats.consecutive_failure_count,
consecutive_success_count: job_stats.consecutive_success_count,
total_failure_count: job_stats.total_failure_count,
total_success_count: job_stats.total_success_count,
total_failure_ratio,
total_success_ratio,
};
let body = serde_json::to_string(&response).map_err(|_e| HealthCheckError::ServerError)?;
if is_healthy {
Ok(HttpResponse::Ok()
.content_type(ContentType::json())
.body(body))
} else {
Ok(HttpResponse::InternalServerError()
.content_type(ContentType::json())
.body(body))
}
}
@@ -1 +0,0 @@
pub mod health_check_handler;
@@ -1,7 +0,0 @@
use jobs_common::job_stats::JobStats;
#[derive(Clone)]
pub struct HttpServerSharedState {
pub job_stats: JobStats,
pub consecutive_failure_unhealthy_threshold: u64,
}
@@ -1,3 +0,0 @@
pub mod endpoints;
pub mod http_server_shared_state;
pub mod run_http_server;
@@ -1,67 +0,0 @@
use std::sync::Arc;
use actix_web::dev::Server;
use actix_web::middleware::Logger;
use actix_web::{web, App, HttpResponse, HttpServer};
use log::info;
use bootstrap::bootstrap::ContainerEnvironment;
use errors::AnyhowResult;
use jobs_common::job_stats::JobStats;
use crate::http_server::endpoints::health_check_handler::get_health_check_handler;
use crate::http_server::http_server_shared_state::HttpServerSharedState;
const DEFAULT_BIND_ADDRESS: &str = "0.0.0.0:12345";
const DEFAULT_NUM_WORKERS: usize = 2;
pub struct CreateServerArgs {
pub container_environment: ContainerEnvironment,
pub job_stats: JobStats,
}
pub fn run_http_server(args: CreateServerArgs) -> AnyhowResult<Server> {
let bind_address = easyenv::get_env_string_or_default("HTTP_BIND_ADDRESS", DEFAULT_BIND_ADDRESS);
let num_workers = easyenv::get_env_num("HTTP_NUM_WORKERS", DEFAULT_NUM_WORKERS)?;
let _hostname = args.container_environment.hostname.clone();
let server_state = HttpServerSharedState {
job_stats: args.job_stats.clone(),
consecutive_failure_unhealthy_threshold: easyenv::get_env_num(
"CONSECUTIVE_FAILURE_UNHEALTHY_THRESHOLD",
3,
)?,
};
let server_state_arc = web::Data::new(Arc::new(server_state));
info!("Starting HTTP service (for k8s health checking).");
let log_format = "[%{HOSTNAME}e] IP=[%{X-Forwarded-For}i] \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T";
let handle = HttpServer::new(move || {
App::new()
.app_data(server_state_arc.clone())
.wrap(Logger::new(&log_format).exclude("/_status"))
.service(
web::resource("/")
.route(web::get().to(|| HttpResponse::Ok()))
.route(web::head().to(|| HttpResponse::Ok())),
)
.service(
web::resource("/_status")
.route(web::get().to(get_health_check_handler))
.route(web::head().to(|| HttpResponse::Ok())),
)
})
.bind(bind_address)?
.workers(num_workers)
.run();
Ok(handle)
}
pub async fn launch_http_server(args: CreateServerArgs) -> AnyhowResult<()> {
run_http_server(args)?.await?;
Ok(())
}
@@ -1,143 +0,0 @@
use std::collections::HashMap;
use std::time::Duration;
use log::{error, info, warn};
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs};
use mysql_queries::queries::generic_inference::seedance2pro::list_pending_seedance2pro_jobs::list_pending_seedance2pro_jobs;
use seedance2pro::requests::poll_orders::poll_orders::{poll_orders, PollOrdersArgs, TaskStatus};
use crate::job::job_loop::process_completed_order::{mark_job_failed, process_completed_order};
use crate::job_dependencies::JobDependencies;
pub async fn main_loop(job_dependencies: JobDependencies) {
while !job_dependencies.application_shutdown.get() {
let result = run_poll_iteration(&job_dependencies).await;
if let Err(err) = result {
error!("Error in poll iteration: {:?}", err);
let _ = job_dependencies.job_stats.increment_failure_count();
}
tokio::time::sleep(Duration::from_millis(job_dependencies.poll_interval_millis)).await;
}
warn!("Seedance2Pro job runner main loop is shut down.");
}
async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
// 1. Query all non-terminal Seedance2Pro jobs from DB.
let pending_jobs = list_pending_seedance2pro_jobs(&deps.mysql_pool).await?;
if pending_jobs.is_empty() {
info!("No pending Seedance2Pro jobs.");
return Ok(());
}
info!("Found {} pending Seedance2Pro job(s).", pending_jobs.len());
// Build a lookup: order_id -> job
let mut job_by_order_id: HashMap<String, _> = pending_jobs
.into_iter()
.map(|job| (job.order_id.clone(), job))
.collect();
// 2. Poll all orders from seedance2pro API (exhausting pagination).
let mut all_orders = Vec::new();
let mut cursor: Option<u64> = None;
loop {
let response = poll_orders(PollOrdersArgs {
session: &deps.seedance2pro_session,
cursor,
})
.await
.map_err(|err| {
warn!("Error polling seedance2pro orders: {:?}", err);
anyhow::anyhow!("poll_orders failed: {:?}", err)
})?;
info!("Polled {} orders (cursor={:?})", response.orders.len(), cursor);
all_orders.extend(response.orders);
cursor = response.next_cursor;
if cursor.is_none() {
break;
}
}
// 3. Match API orders to DB jobs and process terminal ones.
for order in &all_orders {
let job = match job_by_order_id.remove(&order.order_id) {
Some(j) => j,
None => continue, // Not one of our pending jobs.
};
match &order.task_status {
TaskStatus::Completed => {
info!(
"Order {} completed, processing job {}",
order.order_id,
job.job_token.as_str()
);
if let Err(err) = process_completed_order(deps, &job, order).await {
warn!(
"Error processing completed order {}: {:?}",
order.order_id, err
);
let _ = deps.job_stats.increment_failure_count();
} else {
let _ = deps.job_stats.increment_success_count();
}
}
TaskStatus::Failed => {
let reason = order
.fail_reason
.as_deref()
.unwrap_or("unknown failure reason");
let reason_lower = reason.to_lowercase();
let platform_rules_violation = reason_lower.contains("violates") ||
reason_lower.contains("platform rules") ||
reason_lower.contains("please modify");
let frontend_failure_category = if platform_rules_violation {
Some(FrontendFailureCategory::ModelRulesViolation)
} else {
None
};
warn!(
"Order {} failed: {}. Marking job {} failed.",
order.order_id, reason, job.job_token.as_str()
);
let mark_failed_result = mark_job_failed_by_token(MarkJobFailedByTokenArgs {
pool: &deps.mysql_pool,
job_token: &job.job_token,
maybe_public_failure_reason: Some(reason),
internal_debugging_failure_reason: reason,
maybe_frontend_failure_category: frontend_failure_category,
}).await;
if let Err(err) = mark_failed_result {
error!(
"Error marking job {} as failed: {:?}",
job.job_token.as_str(),
err
);
}
}
TaskStatus::Pending | TaskStatus::Processing => {
// Still in progress — check again next poll.
}
TaskStatus::Unknown(unknown_status) => {
// NB: Keep polling it?
warn!("Unknown order status: {:?}", unknown_status);
}
}
}
Ok(())
}
@@ -1,2 +0,0 @@
pub mod main_loop;
pub mod process_completed_order;
@@ -1,160 +0,0 @@
use anyhow::anyhow;
use log::{error, info};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use errors::AnyhowResult;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs};
use mysql_queries::queries::generic_inference::seedance2pro::list_pending_seedance2pro_jobs::PendingSeedance2ProJob;
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_successfully_done_by_token::mark_generic_inference_job_successfully_done_by_token;
use seedance2pro::requests::poll_orders::poll_orders::OrderStatus;
use crate::job_dependencies::JobDependencies;
const PREFIX : &str = "artcraft_";
const SUFFIX : &str = ".mp4";
/// Download the completed video, upload to bucket, create media file record, and mark job done.
pub async fn process_completed_order(
deps: &JobDependencies,
job: &PendingSeedance2ProJob,
order: &OrderStatus,
) -> AnyhowResult<()> {
// Get the video URL.
let video_url = match &order.result_url {
Some(url) => url.as_str(),
None => {
// Fall back to the first result entry if the top-level result_url is missing.
match order.results.first() {
Some(result) => result.url.as_str(),
None => {
return Err(anyhow!(
"Completed order {} has no result_url and no results",
order.order_id
));
}
}
}
};
info!(
"Downloading video for order {} from: {}",
order.order_id, video_url
);
// Download the video bytes.
let video_bytes: Vec<u8> = reqwest::get(video_url)
.await
.map_err(|err| anyhow!("reqwest error downloading video: {:?}", err))?
.bytes()
.await
.map_err(|err| anyhow!("error reading video bytes: {:?}", err))?
.to_vec();
info!(
"Downloaded {} bytes for order {}",
video_bytes.len(),
order.order_id
);
// Hash the video.
let checksum = sha256_hash_bytes(&video_bytes)
.map_err(|err| anyhow!("error hashing video: {:?}", err))?;
// Build the bucket path.
let bucket_path = MediaFileBucketPath::generate_new(Some(PREFIX), Some(SUFFIX));
let object_path = bucket_path.get_full_object_path_str();
info!(
"Uploading video to public bucket at path: {}",
object_path
);
// Upload to public bucket.
deps
.public_bucket_client
.upload_file_with_content_type_process(object_path, &video_bytes, "video/mp4")
.await
.map_err(|err| anyhow!("error uploading video to bucket: {:?}", err))?;
info!(
"Uploaded video for order {}. Creating media file record.",
order.order_id
);
// Optionally extract frame dimensions from the order results.
let maybe_frame_width = order.results.first().map(|r| r.width);
let maybe_frame_height = order.results.first().map(|r| r.height);
// Insert media file record.
let media_file_token = MediaFileInsertBuilder::new()
.maybe_creator_user(job.maybe_creator_user_token.as_ref())
.maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref())
.creator_ip_address(&job.creator_ip_address)
.creator_set_visibility(job.creator_set_visibility)
.media_file_class(MediaFileClass::Video)
.media_file_type(MediaFileType::Mp4)
.media_file_origin_category(MediaFileOriginCategory::Inference)
.media_file_origin_product_category(MediaFileOriginProductCategory::VideoGeneration)
.mime_type("video/mp4")
.file_size_bytes(video_bytes.len() as u64)
.maybe_frame_width(maybe_frame_width)
.maybe_frame_height(maybe_frame_height)
.checksum_sha2(&checksum)
.maybe_prompt_token(job.maybe_prompt_token.as_ref())
.public_bucket_directory_hash(&bucket_path)
.insert_pool(&deps.mysql_pool)
.await
.map_err(|err| anyhow!("error inserting media file record: {:?}", err))?;
info!(
"Created media file {} for order {}. Marking job {} complete.",
media_file_token.as_str(),
order.order_id,
job.job_token.as_str()
);
// Mark inference job as successfully completed.
mark_generic_inference_job_successfully_done_by_token(
&deps.mysql_pool,
&job.job_token,
Some(InferenceResultType::MediaFile),
Some(media_file_token.as_str()),
None,
None,
)
.await
.map_err(|err| {
error!(
"Error marking job {} done: {:?}",
job.job_token.as_str(),
err
);
anyhow!("error marking job done: {:?}", err)
})?;
info!("Job {} completed successfully.", job.job_token.as_str());
Ok(())
}
/// Mark a job as permanently failed.
pub async fn mark_job_failed(
deps: &JobDependencies,
job: &PendingSeedance2ProJob,
fail_reason: &str,
) -> AnyhowResult<()> {
mark_job_failed_by_token(MarkJobFailedByTokenArgs {
pool: &deps.mysql_pool,
job_token: &job.job_token,
maybe_public_failure_reason: Some(fail_reason),
internal_debugging_failure_reason: fail_reason,
maybe_frontend_failure_category: None,
}).await
}
@@ -1 +0,0 @@
pub mod job_loop;
@@ -1,26 +0,0 @@
use cloud_storage::bucket_client::BucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_stats::JobStats;
use seedance2pro::creds::seedance2pro_session::Seedance2ProSession;
use server_environment::ServerEnvironment;
use sqlx::MySqlPool;
pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for storing generated videos.
pub public_bucket_client: BucketClient,
/// Session credentials for polling seedance2-pro.com.
pub seedance2pro_session: Seedance2ProSession,
pub server_environment: ServerEnvironment,
pub job_stats: JobStats,
/// How long to sleep between poll iterations (milliseconds).
pub poll_interval_millis: u64,
/// Set to `true` from another thread to trigger graceful shutdown.
pub application_shutdown: RelaxedAtomicBool,
}
@@ -1,142 +0,0 @@
// Never allow these
#![forbid(private_bounds)]
#![forbid(private_interfaces)]
#![forbid(unused_must_use)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
#[macro_use] extern crate serde_derive;
use std::time::Duration;
use anyhow::anyhow;
use log::{info, warn};
use sqlx::mysql::MySqlPoolOptions;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
use config::shared_constants::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
use jobs_common::job_stats::JobStats;
use seedance2pro::creds::seedance2pro_session::Seedance2ProSession;
use server_environment::ServerEnvironment;
use crate::http_server::run_http_server::{launch_http_server, CreateServerArgs};
use crate::job::job_loop::main_loop::main_loop;
use crate::job_dependencies::JobDependencies;
pub mod http_server;
pub mod job;
pub mod job_dependencies;
// Bucket config
const ENV_ACCESS_KEY: &str = "ACCESS_KEY";
const ENV_SECRET_KEY: &str = "SECRET_KEY";
const ENV_REGION_NAME: &str = "REGION_NAME";
const ENV_PUBLIC_BUCKET_NAME: &str = "PUBLIC_BUCKET_NAME";
const ENV_S3_ENDPOINT: &str = "S3_COMPATIBLE_ENDPOINT_URL";
const ENV_SEEDANCE2PRO_COOKIES : &str = "SEEDANCE2PRO_COOKIES";
#[actix_web::main]
async fn main() -> AnyhowResult<()> {
let container_environment = bootstrap(BootstrapArgs {
app_name: "seedance2-pro-job",
default_logging_override: Some(DEFAULT_RUST_LOG),
config_search_directories: &[".", "./config", "crates/service/job/seedance2_pro_job/config"],
ignore_legacy_dot_env_file: true,
})?;
info!("Hostname: {}", &container_environment.hostname);
let _k8s_node_name = easyenv::get_env_string_optional("K8S_NODE_NAME");
let _k8s_pod_name = easyenv::get_env_string_optional("K8S_POD_NAME");
let db_connection_string = easyenv::get_env_string_or_default(
"MYSQL_URL",
DEFAULT_MYSQL_CONNECTION_STRING,
);
info!("Connecting to database...");
let mysql_pool = MySqlPoolOptions::new()
.max_connections(2)
.connect(&db_connection_string)
.await?;
info!("Connected to MySQL.");
let server_environment = ServerEnvironment::from_str(
&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?,
)
.ok_or(anyhow!("invalid server environment"))?;
// Bucket setup
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
let s3_compatible_endpoint_url = easyenv::get_env_string_required(ENV_S3_ENDPOINT)?;
let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
"BUCKET_TIMEOUT_SECONDS",
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
&access_key,
&secret_key,
&region_name,
&public_bucket_name,
&s3_compatible_endpoint_url,
None,
Some(bucket_timeout),
)?;
// Seedance2Pro session from cookie string
let seedance2pro_cookies = easyenv::get_env_string_required(ENV_SEEDANCE2PRO_COOKIES)?;
let seedance2pro_session = Seedance2ProSession::from_cookies_string(seedance2pro_cookies);
// How often to poll for results (default: 15 seconds)
let poll_interval_millis: u64 = easyenv::get_env_num(
"SEEDANCE_POLL_INTERVAL_MILLIS",
5_000,
)?;
let application_shutdown = RelaxedAtomicBool::new(false);
let job_stats = JobStats::new();
let create_server_args = CreateServerArgs {
container_environment: container_environment.clone(),
job_stats: job_stats.clone(),
};
let job_dependencies = JobDependencies {
mysql_pool,
public_bucket_client,
seedance2pro_session,
server_environment,
job_stats,
poll_interval_millis,
application_shutdown: application_shutdown.clone(),
};
std::thread::spawn(move || {
let actix_runtime = actix_web::rt::System::new();
let http_server_handle = launch_http_server(create_server_args);
actix_runtime.block_on(http_server_handle)
.expect("HTTP server should not exit.");
warn!("Server thread is shut down.");
application_shutdown.set(true);
});
main_loop(job_dependencies).await;
Ok(())
}
@@ -52,9 +52,9 @@ email_sender= { path = "../../../lib/email_sender" }
enums.workspace = true
enums_public.workspace = true
errors.workspace = true
seedance2pro.workspace = true
fal_client.workspace = true
filesys.workspace = true
seedance2pro.workspace = true
google_sign_in = { path = "../../../api_clients/google_sign_in" }
hashing = { path = "../../../lib/files/hashing" }
http_server_common = { path = "../../../lib/deprecated/http_server_common" }
@@ -100,7 +100,7 @@ once_cell.workspace = true
rand.workspace = true
regex.workspace = true
strum_macros = "0.26.4"
time = "0.3.36"
time = "0.3.30"
url.workspace = true
# Money
@@ -115,11 +115,7 @@ language-tags = "0.3.2" # Parsing single tags richly
reqwest.workspace = true
# Email
resend-rs = { version = "0.21.0", default-features = false, features = ["rustls-tls"] }
# NB: Both ring (via reqwest 0.12/hyper-rustls) and aws-lc-rs (via quinn/resend-rs)
# are features of rustls 0.23 in this binary. rustls requires an explicit default
# provider when multiple crypto providers are compiled in.
rustls = { version = "0.23", features = ["ring"], default-features = false }
resend-rs = "0.21.0"
# Runtime
tokio.workspace = true
@@ -42,8 +42,7 @@ GC_ENABLED_PUBLIC_BUCKET_NAME="REPLACE_ME"
AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
# Fake API key for FAL
FAL_WEBHOOK_URL="https://brandons-macbook-pro.taild62114.ts.net/v1/webhooks/fal"
#FAL_WEBHOOK_URL="https://macaroni-1.tailce84f.ts.net/v1/webhooks/fal"
FAL_WEBHOOK_URL="https://macaroni-1.tailce84f.ts.net/v1/webhooks/fal"
#FAL_WEBHOOK_URL="https://halide.tailce84f.ts.net/v1/webhooks/fal"
#FAL_WEBHOOK_URL="https://s1.pancake-chimaera.ts.net/api/fal/webhook"
@@ -124,13 +124,6 @@ const LOG_FORMAT : &str =
#[actix_web::main]
async fn main() -> AnyhowResult<()> {
// NB: Both ring (via reqwest 0.12/hyper-rustls) and aws-lc-rs (via quinn/resend-rs)
// are compiled into this binary as rustls crypto providers. rustls 0.23 panics at
// runtime if it can't auto-select a single provider, so we install ring explicitly.
rustls::crypto::ring::default_provider()
.install_default()
.expect("Failed to install rustls crypto provider");
let container_environment = bootstrap(BootstrapArgs {
app_name: "storyteller-web",
default_logging_override: Some(DEFAULT_RUST_LOG),
@@ -1,14 +1,9 @@
import { useState, useRef, useEffect, useCallback, useMemo } from "react";
import { useShallow } from "zustand/react/shallow";
import { useState, useRef, useEffect, useCallback } from "react";
import { DRAW_LAYER_ID, INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface";
import "./App.css";
import PromptEditor from "./PromptEditor/PromptEditor";
import SideToolbar from "./components/ui/SideToolbar";
import {
AspectRatioType,
type SceneState,
useSceneStore,
} from "./stores/SceneState";
import { AspectRatioType, useSceneStore } from "./stores/SceneState";
import { useUndoRedoHotkeys } from "./hooks/useUndoRedoHotkeys";
import { useDeleteHotkeys } from "./hooks/useDeleteHotkeys";
import { useCopyPasteHotkeys } from "./hooks/useCopyPasteHotkeys";
@@ -23,26 +18,16 @@ import {
useSelectedImageModel,
useSelectedProviderForModel,
} from "@storyteller/ui-model-selector";
import {
EnqueueEditImage,
EnqueueEditImageRequest,
EnqueueEditImageResolution,
EnqueueEditImageSize,
EnqueueImageInpaint,
EnqueueImageInpaintRequest,
useCanvasBgRemovedEvent,
} from "@storyteller/tauri-api";
import { EnqueueEditImage, EnqueueEditImageRequest, EnqueueEditImageResolution, EnqueueEditImageSize, EnqueueImageInpaint, EnqueueImageInpaintRequest, useCanvasBgRemovedEvent } from "@storyteller/tauri-api";
import { HelpMenuButton } from "@storyteller/ui-help-menu";
import { GenerationProvider } from "@storyteller/api-enums";
import { HistoryStack, ImageBundle } from "../PageEdit/HistoryStack";
import {
BaseImageSelector,
BaseSelectorImage,
} from "../PageEdit/BaseImageSelector";
import { BaseImageSelector, BaseSelectorImage } from "../PageEdit/BaseImageSelector";
import { normalizeCanvas } from "~/Helpers/CanvasHelpers";
import { EncodeImageBitmapToBase64 } from "./utilities/EncodeImageBitmapToBase64";
import { RefImage, usePrompt2DStore } from "@storyteller/ui-promptbox";
import { PromptsApi } from "@storyteller/api";
import toast from "react-hot-toast";
const PAGE_ID: ModelPage = ModelPage.Canvas2D;
@@ -77,123 +62,26 @@ const PageDraw = () => {
const [isSelecting, setIsSelecting] = useState<boolean>(false);
const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
/*
* Scene store: use a selector + useShallow to avoid re-rendering on every store change.
* Without this, useSceneStore() subscribes to the whole store (e.g. moveNode during
* drag updates state constantly) and PageDraw would re-render every frame.
*
* useShallow compares the selected object by top-level keys: we only re-render when
* one of these values actually changes. Do NOT add:
* - cursorPosition / cursorVisible: updated every mouse move → would re-render constantly.
* - historyImageNodeMap: mutated in place in the store (reference never changes),
* so shallow would never see updates; fix the store to replace the Map if you need it.
* When adding new store fields here, prefer primitive/array/object refs that the store
* replaces (e.g. new array from .map()), not in-place mutations.
*
* IMPORTANT: The selector function is memoized to prevent infinite loops. If you modify
* this selector, ensure the function reference stays stable (use useMemo if needed).
*/
const selector = useMemo(
() => (state: SceneState) => ({
baseImageInfo: state.baseImageInfo,
baseImageBitmap: state.baseImageBitmap,
nodes: state.nodes,
selectedNodeIds: state.selectedNodeIds,
lineNodes: state.lineNodes,
activeTool: state.activeTool,
currentShape: state.currentShape,
fillColor: state.fillColor,
brushColor: state.brushColor,
brushSize: state.brushSize,
historyImageBundles: state.historyImageBundles,
getAspectRatioDimensions: state.getAspectRatioDimensions,
finishRemoveBackground: state.finishRemoveBackground,
createImageFromUrl: state.createImageFromUrl,
createImageFromFile: state.createImageFromFile,
setBaseImageInfo: state.setBaseImageInfo,
RESET: state.RESET,
clearLineNodes: state.clearLineNodes,
setNodes: state.setNodes,
removeHistoryImage: state.removeHistoryImage,
addHistoryImageBundle: state.addHistoryImageBundle,
setAspectRatioType: state.setAspectRatioType,
setActiveTool: state.setActiveTool,
selectNode: state.selectNode,
setCurrentShape: state.setCurrentShape,
setBrushColor: state.setBrushColor,
setBrushSize: state.setBrushSize,
setBrushOpacity: state.setBrushOpacity,
setFillColor: state.setFillColor,
toggleLock: state.toggleLock,
beginRemoveBackground: state.beginRemoveBackground,
bringToFront: state.bringToFront,
bringForward: state.bringForward,
sendBackward: state.sendBackward,
sendToBack: state.sendToBack,
copySelectedItems: state.copySelectedItems,
pasteItems: state.pasteItems,
deleteSelectedItems: state.deleteSelectedItems,
undo: state.undo,
redo: state.redo,
}),
[],
);
const {
baseImageInfo,
baseImageBitmap,
nodes,
selectedNodeIds,
lineNodes,
activeTool,
currentShape,
fillColor,
brushColor,
brushSize,
historyImageBundles,
getAspectRatioDimensions,
finishRemoveBackground,
createImageFromUrl,
createImageFromFile,
setBaseImageInfo,
RESET,
clearLineNodes,
setNodes,
removeHistoryImage,
addHistoryImageBundle,
setAspectRatioType,
setActiveTool,
selectNode,
setCurrentShape,
setBrushColor,
setBrushSize,
setBrushOpacity,
setFillColor,
toggleLock,
beginRemoveBackground,
bringToFront,
bringForward,
sendBackward,
sendToBack,
copySelectedItems,
pasteItems,
deleteSelectedItems,
undo,
redo,
} = useSceneStore(useShallow(selector));
const store = useSceneStore();
const promptStoreProvider = usePrompt2DStore;
const generationCount = promptStoreProvider((state) => state.generationCount);
const setGenerationCount = promptStoreProvider(
(state) => state.setGenerationCount,
);
const setGenerationCount = promptStoreProvider((state) => state.setGenerationCount);
const useSystemPrompt = promptStoreProvider((state) => state.useSystemPrompt);
const referenceImages = promptStoreProvider((state) => state.referenceImages);
const prompt = promptStoreProvider((state) => state.prompt);
const baseImageKonvaRef = useRef<Konva.Image>({} as Konva.Image);
const baseImageUrl = baseImageInfo?.url;
const baseImageUrl = store.baseImageInfo?.url;
const [pendingGenerations, setPendingGenerations] = useState<
{ id: string; count: number }[]
>([]);
const addHistoryImageBundle = useSceneStore(
(state) => state.addHistoryImageBundle,
);
const removeHistoryImage = useSceneStore((state) => state.removeHistoryImage);
const historyImageBundles = useSceneStore(
(state) => state.historyImageBundles,
);
const selectedImageModel: ImageModel | undefined =
useSelectedImageModel(PAGE_ID);
@@ -204,11 +92,11 @@ const PageDraw = () => {
const supportsMaskedInpainting =
selectedImageModel?.usesInpaintingMask ?? false;
useDeleteHotkeys({ onDelete: deleteSelectedItems });
useUndoRedoHotkeys({ undo, redo });
useDeleteHotkeys({ onDelete: store.deleteSelectedItems });
useUndoRedoHotkeys({ undo: store.undo, redo: store.redo });
useCopyPasteHotkeys({
onCopy: copySelectedItems,
onPaste: pasteItems,
onCopy: store.copySelectedItems,
onPaste: store.pasteItems,
});
useCanvasBgRemovedEvent(async (event) => {
@@ -225,19 +113,24 @@ const PageDraw = () => {
// const file = new File([blob], "generated_image.png", {
// type: blob.type,
// });
finishRemoveBackground(nodeId, event.media_token, event.image_cdn_url);
store.finishRemoveBackground(
nodeId,
event.media_token,
event.image_cdn_url,
);
});
// Create a function to use the left layer ref and download the bitmap from it
const getMaskArrayBuffer = async (): Promise<Uint8Array> => {
if (!stageRef.current || !baseImageKonvaRef.current) {
if (
!stageRef.current ||
!baseImageKonvaRef.current
) {
console.error("Stage or left panel ref is not available");
throw new Error("Stage or left panel or base image ref is not available");
}
const layer = stageRef.current
.getLayers()
.find((l) => l.id() === INPAINT_LAYER_ID)!;
const layer = stageRef.current.getLayers().find((l) => l.id() === INPAINT_LAYER_ID)!;
// Get the canvas area that's covered by the image/rectangle
const rect = baseImageKonvaRef.current;
@@ -302,7 +195,7 @@ const PageDraw = () => {
console.log("Creating image from URL:", imageUrl);
createImageFromUrl(stagePoint.x, stagePoint.y, imageUrl);
store.createImageFromUrl(stagePoint.x, stagePoint.y, imageUrl);
console.log(
`Created image "${item.label}" at stage position:`,
@@ -321,12 +214,12 @@ const PageDraw = () => {
handleGallery2DDrop as EventListener,
);
};
// Stable action ref; no need to depend on full store.
}, [createImageFromUrl]);
}, [store]);
const handleImageUpload = async (files: File[]): Promise<void> => {
// Determine current canvas dimensions from the store (according to aspect-ratio)
const { width: canvasW, height: canvasH } = getAspectRatioDimensions();
const { width: canvasW, height: canvasH } =
store.getAspectRatioDimensions();
// Target maximum size – 85 % of the canvas in each direction
const maxW = canvasW * 0.85;
@@ -347,20 +240,80 @@ const PageDraw = () => {
const x = (canvasW - finalW) / 2;
const y = (canvasH - finalH) / 2;
createImageFromFile(x, y, file, finalW, finalH);
store.createImageFromFile(x, y, file, finalW, finalH);
};
img.src = URL.createObjectURL(file);
}
};
const handleTauriEnqueue = async (image: ImageBitmap, aspectRatio: EnqueueEditImageSize | undefined, resolution: EnqueueEditImageResolution | undefined, subscriberId: string) => {
if (image === undefined) {
console.log("image is undefined");
return;
}
const api = new PromptsApi();
const base64Bitmap = await EncodeImageBitmapToBase64(image);
const byteString = atob(base64Bitmap);
const mimeString = "image/png";
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
const uuid = crypto.randomUUID(); // Generate a new UUID
const file = new File([ab], `${uuid}.png`, { type: mimeString });
const snapshotMediaToken = await api.uploadSceneSnapshot({
screenshot: file,
});
if (snapshotMediaToken.data === undefined) {
toast.error("Error: Unable to upload scene snapshot Please try again.");
return;
}
console.log("useSystemPrompt", useSystemPrompt);
console.log("Snapshot media token:", snapshotMediaToken.data);
const request: EnqueueEditImageRequest = {
model: selectedImageModel,
scene_image_media_token: snapshotMediaToken.data!,
image_media_tokens: referenceImages
.map((image) => image.mediaToken)
.filter((t) => t.length > 0),
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
image_count: generationCount,
aspect_ratio: aspectRatio,
image_resolution: resolution,
frontend_caller: "image_editor",
frontend_subscriber_id: subscriberId,
};
if (selectedProvider) {
request.provider = selectedProvider;
}
const generateResponse = await EnqueueEditImage(request);
console.log("generateResponse", generateResponse);
return generateResponse;
}
const getCompositeCanvasFile = useCallback(async (): Promise<File | null> => {
if (!stageRef.current || !baseImageKonvaRef.current || !baseImageBitmap) {
if (
!stageRef.current ||
!baseImageKonvaRef.current ||
!store.baseImageBitmap
) {
return null;
}
const editsLayer = stageRef.current
.getLayers()
.find((l) => l.id() === DRAW_LAYER_ID);
const editsLayer = stageRef.current.getLayers().find((l) => l.id() === DRAW_LAYER_ID);
if (!editsLayer) {
console.error("Edits layer not found");
@@ -375,7 +328,7 @@ const PageDraw = () => {
const ctx = canvas.getContext("2d");
if (!ctx) return null;
ctx.drawImage(baseImageBitmap, 0, 0, width, height);
ctx.drawImage(store.baseImageBitmap, 0, 0, width, height);
const markerLayerCanvas = editsLayer.toCanvas({
x: stageRef.current.x(),
@@ -394,7 +347,7 @@ const PageDraw = () => {
const blob = await canvas.convertToBlob({ type: "image/png" });
const uuid = crypto.randomUUID();
return new File([blob], `${uuid}.png`, { type: "image/png" });
}, [baseImageBitmap]);
}, [store.baseImageBitmap]);
const handleGenerate = useCallback(
async (
@@ -406,7 +359,7 @@ const PageDraw = () => {
selectedProvider?: GenerationProvider;
},
) => {
const editedImageToken = baseImageInfo?.mediaToken;
const editedImageToken = store.baseImageInfo?.mediaToken;
if (!editedImageToken) {
console.error("Base image is not available");
@@ -447,7 +400,7 @@ const PageDraw = () => {
}
};
const { width, height } = getAspectRatioDimensions();
const { width, height } = store.getAspectRatioDimensions();
const subscriberId: string =
crypto?.randomUUID?.() ??
`inpaint-${Date.now()}-${Math.random().toString(36).slice(2)}`;
@@ -565,9 +518,7 @@ const PageDraw = () => {
);
throw error;
}
},
[generationCount, getCompositeCanvasFile, selectedImageModel],
);
}, [generationCount, getCompositeCanvasFile, selectedImageModel, store]);
const onFitPressed = async () => {
// Get the stage and its container dimensions
@@ -579,8 +530,8 @@ const PageDraw = () => {
const containerHeight = stage.container().offsetHeight;
// Get canvas dimensions from store aspect ratio
const canvasW = getAspectRatioDimensions().width;
const canvasH = getAspectRatioDimensions().height;
const canvasW = store.getAspectRatioDimensions().width;
const canvasH = store.getAspectRatioDimensions().height;
// Add padding to ensure canvas doesn't touch the edges
const padding = 40;
@@ -609,17 +560,15 @@ const PageDraw = () => {
};
// When the model inpainting support changes, we need to auto-change the tool so it's not set to inpainting
// Note: setActiveTool is a stable Zustand action, so we don't need it in deps
useEffect(() => {
if (!supportsMaskedInpainting && activeTool === "inpaint") {
setActiveTool("select");
if (!supportsMaskedInpainting && store.activeTool === "inpaint") {
store.setActiveTool("select");
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [activeTool, supportsMaskedInpainting]);
}, [store, supportsMaskedInpainting]);
// Auto-fit canvas to screen on initial load
useEffect(() => {
if (!baseImageBitmap) {
if (!store.baseImageBitmap) {
return;
}
@@ -648,11 +597,11 @@ const PageDraw = () => {
autoFitCanvas();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [baseImageBitmap]);
}, [store.baseImageBitmap]);
// Display image selector on launch, otherwise hide it
// Also show loading state if info is set but image is loading
if (!baseImageInfo || !baseImageBitmap) {
if (!store.baseImageInfo || !store.baseImageBitmap) {
return (
<div
className={
@@ -664,9 +613,11 @@ const PageDraw = () => {
<BaseImageSelector
onImageSelect={(image: BaseSelectorImage) => {
addHistoryImageBundle({ images: [image] });
setBaseImageInfo(image);
store.setBaseImageInfo(image);
}}
showLoading={baseImageInfo !== null && baseImageBitmap === null}
showLoading={
store.baseImageInfo !== null && store.baseImageInfo === null
}
/>
</div>
</div>
@@ -678,28 +629,29 @@ const PageDraw = () => {
<>
<div className="fixed inset-0 -z-10 bg-ui-background" />
<div
className={`preserve-aspect-ratio fixed right-4 top-1/2 z-10 -translate-y-1/2 transform ${
isSelecting ? "pointer-events-none" : "pointer-events-auto"
}`}
className={`preserve-aspect-ratio fixed right-4 top-1/2 z-10 -translate-y-1/2 transform ${isSelecting ? "pointer-events-none" : "pointer-events-auto"
}`}
>
<HistoryStack
onClear={() => {
RESET();
store.RESET();
setPendingGenerations([]);
}}
imageBundles={historyImageBundles}
pendingPlaceholders={pendingGenerations}
blurredBackgroundUrl={baseImageUrl}
onImageSelect={(baseImage) => {
setBaseImageInfo(baseImage);
store.clearLineNodes();
store.setNodes([]);
store.setBaseImageInfo(baseImage);
}}
onImageRemove={(baseImage) => {
if (
pendingGenerations.length === 0 &&
historyImageBundles.length === 1 &&
historyImageBundles[0].images.length <= 1
store.historyImageBundles.length === 1 &&
store.historyImageBundles[0].images.length <= 1
) {
RESET();
store.RESET();
} else {
removeHistoryImage(baseImage);
}
@@ -710,13 +662,12 @@ const PageDraw = () => {
onResolvePending={(id: string) =>
setPendingGenerations((prev) => prev.filter((p) => p.id !== id))
}
selectedImageToken={baseImageInfo?.mediaToken}
selectedImageToken={store.baseImageInfo?.mediaToken}
/>
</div>
<div
className={`preserve-aspect-ratio fixed bottom-0 left-1/2 z-10 -translate-x-1/2 transform ${
isSelecting ? "pointer-events-none" : "pointer-events-auto"
}`}
className={`preserve-aspect-ratio fixed bottom-0 left-1/2 z-10 -translate-x-1/2 transform ${isSelecting ? "pointer-events-none" : "pointer-events-auto"
}`}
>
<PromptEditor
onAspectRatioChange={async (ratio: string) => {
@@ -734,7 +685,7 @@ const PageDraw = () => {
};
const aspectRatioType = ratioToType(ratio);
setAspectRatioType(aspectRatioType);
store.setAspectRatioType(aspectRatioType);
await new Promise((resolve) => requestAnimationFrame(resolve));
onFitPressed();
@@ -754,28 +705,28 @@ const PageDraw = () => {
<SideToolbar
className="fixed left-0 top-1/2 z-10 -translate-y-1/2 transform"
onSelect={(): void => {
setActiveTool("select");
store.setActiveTool("select");
}}
onActivateShapeTool={(
shape: "rectangle" | "circle" | "triangle",
): void => {
selectNode(null);
setCurrentShape(shape);
setActiveTool("shape");
selectNode(null);
store.selectNode(null);
store.setCurrentShape(shape);
store.setActiveTool("shape");
store.selectNode(null);
}}
onPaintBrush={(hex: string, size: number, opacity: number): void => {
setActiveTool("draw");
setBrushColor(hex);
setBrushSize(size);
setBrushOpacity(opacity);
store.setActiveTool("draw");
store.setBrushColor(hex);
store.setBrushSize(size);
store.setBrushOpacity(opacity);
}}
onCanvasBackground={(hex: string): void => {
console.log("Canvas background activated", { color: hex });
// Add background change logic here
// TODO: minor bug needs to update the preview panel
// Debounce also causes issues with real time color change.
setFillColor(hex);
store.setFillColor(hex);
}}
onUploadImage={(): void => {
// Create input element dynamically like in PromptEditor
@@ -814,14 +765,14 @@ const PageDraw = () => {
input.click();
}}
supportsMaskTool={supportsMaskedInpainting}
activeToolId={activeTool}
currentShape={currentShape}
activeToolId={store.activeTool}
currentShape={store.currentShape}
/>
<div className="relative z-0">
<ContextMenuContainer
onAction={(e, action) => {
if (action === "contextMenu") {
const hasSelection = selectedNodeIds.length > 0;
const hasSelection = store.selectedNodeIds.length > 0;
if (hasSelection) {
console.log("An item is selected.");
return true;
@@ -835,54 +786,51 @@ const PageDraw = () => {
onMenuAction={async (action) => {
switch (action) {
case "LOCK":
toggleLock(selectedNodeIds);
store.toggleLock(store.selectedNodeIds);
break;
case "REMOVE_BACKGROUND":
await beginRemoveBackground(selectedNodeIds);
await store.beginRemoveBackground(store.selectedNodeIds);
break;
case "BRING_TO_FRONT":
bringToFront(selectedNodeIds);
store.bringToFront(store.selectedNodeIds);
break;
case "BRING_FORWARD":
bringForward(selectedNodeIds);
store.bringForward(store.selectedNodeIds);
break;
case "SEND_BACKWARD":
sendBackward(selectedNodeIds);
store.sendBackward(store.selectedNodeIds);
break;
case "SEND_TO_BACK":
sendToBack(selectedNodeIds);
store.sendToBack(store.selectedNodeIds);
break;
case "DUPLICATE":
copySelectedItems();
pasteItems();
store.copySelectedItems();
store.pasteItems();
break;
case "DELETE":
deleteSelectedItems();
store.deleteSelectedItems();
break;
default:
// No action
}
}}
isLocked={selectedNodeIds.some((id: string) => {
const node = nodes.find((n: (typeof nodes)[number]) => n.id === id);
const lineNode = lineNodes.find(
(n: (typeof lineNodes)[number]) => n.id === id,
);
isLocked={store.selectedNodeIds.some((id) => {
const node = store.nodes.find((n) => n.id === id);
const lineNode = store.lineNodes.find((n) => n.id === id);
return (node?.locked || lineNode?.locked) ?? false;
})}
>
<PaintSurface
nodes={nodes}
lineNodes={lineNodes}
selectedNodeIds={selectedNodeIds}
nodes={store.nodes}
selectedNodeIds={store.selectedNodeIds}
onCanvasSizeChange={(width: number, height: number): void => {
canvasWidth.current = width;
canvasHeight.current = height;
}}
fillColor={fillColor}
activeTool={activeTool}
brushColor={brushColor}
brushSize={brushSize}
fillColor={store.fillColor}
activeTool={store.activeTool}
brushColor={store.brushColor}
brushSize={store.brushSize}
onSelectionChange={setIsSelecting}
stageRef={stageRef}
transformerRefs={transformerRefs}
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,6 @@
import { Vector2d } from "konva/lib/types";
import { useEffect } from "react";
import { DragState } from "~/pages/PageEdit/EditPaintSurface";
import { Vector2d } from 'konva/lib/types';
import { useEffect } from 'react';
import { DragState } from '~/pages/PageEdit/EditPaintSurface';
// This piece of code ensures that when you are selecting or dragging and go off the stage,
// it releases the controls mouse drag.
export const useGlobalMouseUp = (
@@ -10,12 +10,11 @@ export const useGlobalMouseUp = (
setIsSelecting: (value: boolean) => void,
isSelectingRef: React.MutableRefObject<boolean>,
setSelectionRect: (value: any) => void,
setLastPoint: (value: { x: number, y: number } | null) => void,
onSelectionChange?: (isSelecting: boolean) => void,
onBeforeReset?: () => void,
) => {
useEffect(() => {
const handleGlobalMouseUp = () => {
onBeforeReset?.();
// Always reset all states when mouse is released anywhere
setIsDragging(undefined);
setIsDrawing(false);
@@ -24,15 +23,16 @@ export const useGlobalMouseUp = (
isSelectingRef.current = false;
onSelectionChange?.(false);
setSelectionRect(null);
setLastPoint(null);
};
// Listen to mouseup on the entire window
window.addEventListener("mouseup", handleGlobalMouseUp);
window.addEventListener("touchend", handleGlobalMouseUp);
window.addEventListener('mouseup', handleGlobalMouseUp);
window.addEventListener('touchend', handleGlobalMouseUp);
return () => {
window.removeEventListener("mouseup", handleGlobalMouseUp);
window.removeEventListener("touchend", handleGlobalMouseUp);
window.removeEventListener('mouseup', handleGlobalMouseUp);
window.removeEventListener('touchend', handleGlobalMouseUp);
};
}, [onSelectionChange, onBeforeReset]);
};
}, [onSelectionChange]);
}
@@ -86,17 +86,11 @@ type SerializedNodeData = {
locked: boolean;
};
interface HistoryNodeData {
nodes: Node[];
lineNodes: LineNode[];
}
export interface SceneState {
interface SceneState {
// Nodes
nodes: Node[];
selectedNodeIds: string[];
lineNodes: LineNode[]; // Add lineNodes to state
historyImageNodeMap: Map<BaseSelectorImage, HistoryNodeData>;
// Clipboard
clipboard: (Node | LineNode)[]; // To store copied items
@@ -298,7 +292,6 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
nodes: [],
lineNodes: [],
selectedNodeIds: [],
historyImageNodeMap: new Map<BaseSelectorImage, HistoryNodeData>(),
clipboard: [],
history: [],
historyIndex: -1,
@@ -1594,40 +1587,15 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
},
setBaseImageInfo: (image: BaseSelectorImage | null) => {
// If the new base image is null, return early because nothing needs to be loaded.
set({ baseImageInfo: image });
if (!image) {
// Set the new base image info - setting null is allowed.
set({ baseImageInfo: null });
return;
}
// Save the current node data to the history image node map
const currentBaseImage = get().baseImageInfo;
if (currentBaseImage) {
get().historyImageNodeMap.set(currentBaseImage, {
nodes: get().nodes,
lineNodes: get().lineNodes,
});
}
// Set the new base image info
set({ baseImageInfo: image });
// Load the image bitmap now
// Load the image bitmap as well
const imgBitmap = new Image();
imgBitmap.onload = () => {
// Once the base image is set, check previous node data - or create new empty one
const previousNodeData = get().historyImageNodeMap.get(image);
if (previousNodeData) {
set({
nodes: previousNodeData.nodes,
lineNodes: previousNodeData.lineNodes,
});
} else {
set({ nodes: [], lineNodes: [] });
}
// Finally, set the base image bitmap
set({ baseImageBitmap: imgBitmap });
};
imgBitmap.onerror = (event) => {
@@ -1647,10 +1615,7 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
},
clearHistoryImages: () => {
set({
historyImageBundles: [],
historyImageNodeMap: new Map<BaseSelectorImage, HistoryNodeData>(),
});
set({ historyImageBundles: [] });
},
addHistoryImageBundle(bundle) {
@@ -1662,10 +1627,6 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
removeHistoryImage(image) {
console.log("Removing history image:", image);
set((state) => {
// Remove the history image node data
state.historyImageNodeMap.delete(image);
// Remove the image from the history image bundles
const updatedBundles = state.historyImageBundles
.map((bundle) => {
return {
@@ -90,6 +90,7 @@ export const EditPaintSurface = ({
/* 1️⃣ Track SplitPane percent so we can re-measure */
const [leftPct, setLeftPct] = useState(singlePaneMode ? 100 : 50);
const [isDrawing, setIsDrawing] = useState(false);
const [, setLastPoint] = useState<{ x: number; y: number } | null>(null);
const selectionRectRef = React.useRef<Konva.Rect>(null);
const [selectionRect, setSelectionRect] = useState<{
@@ -143,6 +144,7 @@ export const EditPaintSurface = ({
setIsSelecting,
isSelectingRef,
setSelectionRect,
setLastPoint,
onSelectionChange,
);
@@ -280,6 +282,7 @@ export const EditPaintSurface = ({
store.addLineNode(newLineNode, false);
setCurrentLineId(lineId);
setIsDrawing(true);
setLastPoint(stagePoint);
return;
}
@@ -448,6 +451,8 @@ export const EditPaintSurface = ({
lineNode.points(currentPoints);
lineNode.getLayer()?.batchDraw();
}
setLastPoint(stagePoint);
}
// Handle selection rectangle - Update directly through Konva
@@ -476,11 +481,11 @@ export const EditPaintSurface = ({
setSelectionRect((prev) =>
prev
? {
// Ensure prev is not null
...prev,
endX: clampedPoint.x,
endY: clampedPoint.y,
}
// Ensure prev is not null
...prev,
endX: clampedPoint.x,
endY: clampedPoint.y,
}
: null,
);
}
@@ -557,6 +562,7 @@ export const EditPaintSurface = ({
isSelectingRef.current = false;
onSelectionChange?.(false);
setSelectionRect(null);
setLastPoint(null);
setIsMiddleMousePressed(false); // Reset middle mouse state
};
@@ -632,8 +638,8 @@ export const EditPaintSurface = ({
if (!container) return;
const defaultCursor =
activeTool === "edit" ||
activeTool === "marker" ||
activeTool === "eraser"
activeTool === "marker" ||
activeTool === "eraser"
? "none"
: activeTool === "select"
? "grab"
@@ -687,7 +693,7 @@ export const EditPaintSurface = ({
const touch2 = touches[1];
return Math.sqrt(
Math.pow(touch2.clientX - touch1.clientX, 2) +
Math.pow(touch2.clientY - touch1.clientY, 2),
Math.pow(touch2.clientY - touch1.clientY, 2),
);
};
@@ -1467,11 +1473,11 @@ export const EditPaintSurface = ({
scaleY={previewScale}
x={0}
y={0}
// style={{
// background: '#f5f5f5',
// border: '1px solid #ddd',
// borderRadius: '8px',
// }}
// style={{
// background: '#f5f5f5',
// border: '1px solid #ddd',
// borderRadius: '8px',
// }}
>
<Layer>
<Image
@@ -127,37 +127,6 @@ export const VIDEO_MODELS: VideoModel[] = [
requiresImage: true,
progressBarTime: 100000,
}),
new VideoModel({
id: "seedance_2p0",
tauriId: "seedance_2p0",
fullName: "Seedance 2.0",
category: "video",
creator: ModelCreator.Bytedance,
selectorName: "Seedance 2.0",
selectorDescription: "High quality model",
selectorBadges: ["2 min."],
startFrame: true,
endFrame: true,
requiresImage: false,
sizeOptions: [
{
tauriValue: "landscape",
textLabel: "Landscape",
icon: SizeIconOption.Landscape,
},
{
tauriValue: "portrait",
textLabel: "Portrait",
icon: SizeIconOption.Portrait,
},
{
tauriValue: "square",
textLabel: "Square",
icon: SizeIconOption.Square,
},
],
progressBarTime: 100000,
}),
new VideoModel({
id: "sora_2",
tauriId: "sora_2",