mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-27 04:00:10 -04:00
Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6c2b59876 | ||
|
|
369c552096 | ||
|
|
13a5dad3a5 | ||
|
|
f8be6ea4c9 | ||
|
|
d659f8485c | ||
|
|
aec5e50f54 | ||
|
|
edd900d29b | ||
|
|
1ea94e4b42 | ||
|
|
32a7eb67dd | ||
|
|
5f70776afe | ||
|
|
a883f79c6d | ||
|
|
5c5a7d1182 | ||
|
|
d0e5a05253 | ||
|
|
72e052fe7e | ||
|
|
7471259d28 | ||
|
|
42e1787576 | ||
|
|
d3b0a0d78c | ||
|
|
8556ef0a6b | ||
|
|
6067849ef5 | ||
|
|
8836346150 | ||
|
|
86be8461ec | ||
|
|
3e7c67abce | ||
|
|
05097945f6 | ||
|
|
f8cd171b34 | ||
|
|
ffde0b06c0 | ||
|
|
3ff43fcf99 | ||
|
|
ebb0341347 | ||
|
|
ca4baf243c | ||
|
|
60e944d22b | ||
|
|
41cc84d99c | ||
|
|
7dda05e980 | ||
|
|
7f502e0cc2 | ||
|
|
3b839d50a0 | ||
|
|
6fffb3a734 | ||
|
|
e01b02e463 | ||
|
|
e2f1b218bc | ||
|
|
2482413a4a | ||
|
|
8250aa0a05 | ||
|
|
3d646b8f19 | ||
|
|
9ee43d4e4a | ||
|
|
40a6644c35 | ||
|
|
c6a0b43324 | ||
|
|
cdb16ff9f4 | ||
|
|
bd18c3ec64 | ||
|
|
4db39881ee | ||
|
|
b4da753346 | ||
|
|
3a8f44b0d1 | ||
|
|
7465d2a533 | ||
|
|
ca713ba1f9 | ||
|
|
e7dca6eedc |
+135
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n\n jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,\n jobs.maybe_debug_log_event_token as `maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "status: enums::common::job_status_plus::JobStatusPlus",
|
||||
"type_info": {
|
||||
"type": "String",
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
|
||||
"char_set": 224,
|
||||
"max_size": 76
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 64
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "external_third_party_id",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 256
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "MULTIPLE_KEY | BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "creator_ip_address",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"char_set": 224,
|
||||
"max_size": 160
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "created_at",
|
||||
"type_info": {
|
||||
"type": "Timestamp",
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"name": "updated_at",
|
||||
"type_info": {
|
||||
"type": "Timestamp",
|
||||
"flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 2
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "31d85cc0013f29e6db2d272ee6e709cca79b46496a2c222960f1a1419175f00b"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
$kind: collection
|
||||
name: edit
|
||||
order: 312
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
$kind: http-request
|
||||
name: Beeble SwitchX Edit Video
|
||||
url: "{{api_host}}/v1/generate/video/edit/beeble_switchx"
|
||||
method: POST
|
||||
headers:
|
||||
- key: Accept
|
||||
value: application/json
|
||||
- key: Session
|
||||
value: ""
|
||||
disabled: true
|
||||
body:
|
||||
type: json
|
||||
content: >-
|
||||
{
|
||||
"uuid_idempotency_token": "{{$randomUUID}}",
|
||||
//"source_video_media_token": "m_6vve5gffn2r4rajyvy8z2380x15kqc", // two corgis (dev)
|
||||
"source_video_media_token": "m_rkgdk7qw0a7gz4mrzfv0q78ax57kbx", // corgi cowboy (dev)
|
||||
//"source_video_media_token": "m_1k19k6dgdny253zag8g3jqf3bgemss", // corgi watermellon (prod)
|
||||
"reference_image_media_token": "m_pf310vdhc4v98n7r8mz70bz9geest3" // tornado (dev)
|
||||
//"reference_image_media_token": "m_z4y097t9pdmbyatf70z0qb8zvfphvm" // cactus (prod)
|
||||
//"prompt": "Stormy skies, there's a rotating tornado in the background"
|
||||
}
|
||||
order: 1778262498846
|
||||
@@ -9,7 +9,7 @@ body:
|
||||
// hanashi : password (dev)
|
||||
//"username_or_email": "hanashi",
|
||||
//"password": "hanashi"
|
||||
"username_or_email": "newuser2",
|
||||
"password": "newuser2"
|
||||
"username_or_email": "newuser4",
|
||||
"password": "newuser4"
|
||||
}
|
||||
order: 1000
|
||||
|
||||
@@ -18,6 +18,19 @@ pub enum BeebleSpecificApiError {
|
||||
|
||||
/// The generation job failed with a reported error message.
|
||||
GenerationFailed(String),
|
||||
|
||||
/// 400 Bad Request — the callback_url is not a valid publicly-reachable HTTPS URL.
|
||||
BadWebhookUrl {
|
||||
message: String,
|
||||
webhook_url: String,
|
||||
},
|
||||
|
||||
/// 400 Bad Request — the video has too many frames.
|
||||
VideoHasTooManyFrames {
|
||||
max_frames: u64,
|
||||
detected_frames: u64,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Error for BeebleSpecificApiError {}
|
||||
@@ -30,6 +43,12 @@ impl Display for BeebleSpecificApiError {
|
||||
Self::IdempotencyConflict => write!(f, "Beeble API: Idempotency conflict"),
|
||||
Self::RateLimited => write!(f, "Beeble API: Rate limited"),
|
||||
Self::GenerationFailed(msg) => write!(f, "Beeble API: Generation failed: {}", msg),
|
||||
Self::BadWebhookUrl { message, webhook_url } => {
|
||||
write!(f, "Beeble API: Bad webhook URL '{}': {}", webhook_url, message)
|
||||
}
|
||||
Self::VideoHasTooManyFrames { max_frames, detected_frames, message } => {
|
||||
write!(f, "Beeble API: Video has too many frames ({} detected, max {}): {}", detected_frames, max_frames, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,3 +5,4 @@ pub(crate) mod test_utils;
|
||||
pub mod creds;
|
||||
pub mod error;
|
||||
pub mod requests;
|
||||
pub mod webhook_api;
|
||||
|
||||
@@ -69,7 +69,12 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::test_utils::get_test_api_key::get_test_api_key;
|
||||
|
||||
const TEST_JOB_ID: &str = "swx_uZAcwMjRZpDesqfp3pKkF";
|
||||
//const TEST_JOB_ID: &str = "swx_uZAcwMjRZpDesqfp3pKkF"; // Production
|
||||
//const TEST_JOB_ID: &str = "swx_Ontb6KfbfdY24Cz8DZ9c8"; // Dev
|
||||
//const TEST_JOB_ID: &str = "swx_k5SmgdFcesuE05BE8VqFb"; // Dev
|
||||
//const TEST_JOB_ID: &str = "swx_snkg7Et8EOvWaQ0QgoZlZ"; // Dev
|
||||
const TEST_JOB_ID: &str = "swx_a1GBhEHwIfhzXfWEkEbcb"; // Dev
|
||||
//const TEST_JOB_ID: &str = "swx_9Cbr7coiW9ZOayIrKGe0v"; // Prod
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // manually test — requires real API key
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use crate::error::beeble_error::BeebleError;
|
||||
use crate::error::beeble_generic_api_error::BeebleGenericApiError;
|
||||
use crate::error::beeble_specific_api_error::BeebleSpecificApiError;
|
||||
|
||||
/// Additional context for error handling that depends on the original request.
|
||||
pub struct ErrorContext <'a> {
|
||||
pub maybe_callback_url: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Classify a non-success Beeble API response into the appropriate error.
|
||||
///
|
||||
/// Only call this when `!status.is_success()`. Always returns `Err`.
|
||||
pub fn handle_error_response<T>(
|
||||
status: StatusCode,
|
||||
response_body: &str,
|
||||
context: &ErrorContext,
|
||||
) -> Result<T, BeebleError> {
|
||||
match status.as_u16() {
|
||||
401 => Err(BeebleSpecificApiError::Unauthorized.into()),
|
||||
402 => Err(BeebleSpecificApiError::InsufficientCredits.into()),
|
||||
400 if response_body.contains("INVALID_CALLBACK_URL") => {
|
||||
let message = extract_error_message(response_body)
|
||||
.unwrap_or_else(|| "callback_url must be a valid, publicly-reachable HTTPS URL".to_string());
|
||||
Err(BeebleSpecificApiError::BadWebhookUrl {
|
||||
message,
|
||||
webhook_url: context.maybe_callback_url.unwrap_or_default().to_string(),
|
||||
}.into())
|
||||
}
|
||||
400 if response_body.contains("VIDEO_TOO_MANY_FRAMES") => {
|
||||
let message = extract_error_message(response_body)
|
||||
.unwrap_or_else(|| "Video has too many frames".to_string());
|
||||
let (detected, max) = extract_frame_counts(&message);
|
||||
Err(BeebleSpecificApiError::VideoHasTooManyFrames {
|
||||
max_frames: max,
|
||||
detected_frames: detected,
|
||||
message,
|
||||
}.into())
|
||||
}
|
||||
409 => Err(BeebleSpecificApiError::IdempotencyConflict.into()),
|
||||
429 => Err(BeebleSpecificApiError::RateLimited.into()),
|
||||
_ => {
|
||||
Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody {
|
||||
status_code: status,
|
||||
body: response_body.to_string(),
|
||||
}.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to extract the "message" field from a Beeble error response body.
|
||||
/// Beeble errors look like: `{"error":{"message":"...","code":"..."}}`
|
||||
fn extract_error_message(body: &str) -> Option<String> {
|
||||
let parsed: serde_json::Value = serde_json::from_str(body).ok()?;
|
||||
parsed.get("error")?.get("message")?.as_str().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
/// Parse frame counts from a message like "Video has 419 frames, which exceeds the maximum of 240."
|
||||
fn extract_frame_counts(message: &str) -> (u64, u64) {
|
||||
let numbers: Vec<u64> = message
|
||||
.split(|c: char| !c.is_ascii_digit())
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter_map(|s| s.parse().ok())
|
||||
.collect();
|
||||
|
||||
match numbers.as_slice() {
|
||||
[detected, max, ..] => (*detected, *max),
|
||||
[detected] => (*detected, 0),
|
||||
_ => (0, 0),
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod start_generation;
|
||||
pub(crate) mod handle_error_response;
|
||||
pub(crate) mod request_types;
|
||||
pub mod start_generation;
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::creds::beeble_api_key::BeebleApiKey;
|
||||
use crate::error::beeble_client_error::BeebleClientError;
|
||||
use crate::error::beeble_error::BeebleError;
|
||||
use crate::error::beeble_generic_api_error::BeebleGenericApiError;
|
||||
use crate::error::beeble_specific_api_error::BeebleSpecificApiError;
|
||||
use crate::requests::start_generation::handle_error_response::{handle_error_response, ErrorContext};
|
||||
use crate::requests::start_generation::request_types::*;
|
||||
|
||||
const BEEBLE_API_BASE_URL: &str = "https://api.beeble.ai/v1";
|
||||
@@ -110,7 +110,7 @@ pub async fn start_generation(args: StartGenerationArgs) -> Result<StartGenerati
|
||||
reference_image_uri: req.reference_image_uri,
|
||||
alpha_uri: req.alpha_uri,
|
||||
max_resolution: req.max_resolution,
|
||||
callback_url: req.callback_url,
|
||||
callback_url: req.callback_url.clone(),
|
||||
idempotency_key: req.idempotency_key,
|
||||
};
|
||||
|
||||
@@ -127,24 +127,17 @@ pub async fn start_generation(args: StartGenerationArgs) -> Result<StartGenerati
|
||||
.map_err(|err| BeebleGenericApiError::ReqwestError(err))?;
|
||||
|
||||
let status = response.status();
|
||||
|
||||
let response_body = response.text()
|
||||
.await
|
||||
.map_err(|err| BeebleGenericApiError::ReqwestError(err))?;
|
||||
|
||||
info!("Beeble start generation response: status={}", status);
|
||||
|
||||
match status.as_u16() {
|
||||
401 => return Err(BeebleSpecificApiError::Unauthorized.into()),
|
||||
402 => return Err(BeebleSpecificApiError::InsufficientCredits.into()),
|
||||
409 => return Err(BeebleSpecificApiError::IdempotencyConflict.into()),
|
||||
429 => return Err(BeebleSpecificApiError::RateLimited.into()),
|
||||
_ if !status.is_success() => {
|
||||
return Err(BeebleGenericApiError::UncategorizedBadResponseWithStatusAndBody {
|
||||
status_code: status,
|
||||
body: response_body,
|
||||
}.into());
|
||||
}
|
||||
_ => {}
|
||||
if !status.is_success() {
|
||||
return handle_error_response(status, &response_body, &ErrorContext {
|
||||
maybe_callback_url: req.callback_url.as_deref(),
|
||||
});
|
||||
}
|
||||
|
||||
let parsed: GenerationJobResponseBody = serde_json::from_str(&response_body)
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
use serde_derive::Deserialize;
|
||||
|
||||
/// Raw deserialized webhook payload from Beeble.
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct BeebleWebhookPayload {
|
||||
pub id: String,
|
||||
pub status: BeebleWebhookStatus,
|
||||
pub output: Option<BeebleWebhookOutput>,
|
||||
pub error: Option<String>,
|
||||
pub completed_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BeebleWebhookStatus {
|
||||
Completed,
|
||||
Failed,
|
||||
/// Catch unknown variants gracefully.
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct BeebleWebhookOutput {
|
||||
pub render: Option<String>,
|
||||
pub source: Option<String>,
|
||||
pub alpha: Option<String>,
|
||||
}
|
||||
|
||||
pub fn parse_beeble_webhook(raw_body: &str) -> Result<BeebleWebhookPayload, serde_json::Error> {
|
||||
serde_json::from_str(raw_body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SUCCESS_PAYLOAD: &str = r#"{
|
||||
"id": "swx_abc123",
|
||||
"status": "completed",
|
||||
"output": {
|
||||
"render": "https://cdn.beeble.ai/render.mp4",
|
||||
"source": "https://cdn.beeble.ai/source.mp4",
|
||||
"alpha": "https://cdn.beeble.ai/alpha.mp4"
|
||||
},
|
||||
"completed_at": "2026-02-23T10:05:00Z"
|
||||
}"#;
|
||||
|
||||
const FAILURE_PAYLOAD: &str = r#"{
|
||||
"id": "swx_abc123",
|
||||
"status": "failed",
|
||||
"completed_at": "2026-02-23T10:05:00Z",
|
||||
"error": "Processing failed"
|
||||
}"#;
|
||||
|
||||
const UNKNOWN_STATUS_PAYLOAD: &str = r#"{
|
||||
"id": "swx_xyz789",
|
||||
"status": "processing",
|
||||
"completed_at": null
|
||||
}"#;
|
||||
|
||||
const NULL_OUTPUT_PAYLOAD: &str = r#"{
|
||||
"id": "swx_abc123",
|
||||
"status": "completed",
|
||||
"output": null,
|
||||
"completed_at": "2026-02-23T10:05:00Z"
|
||||
}"#;
|
||||
|
||||
mod parse_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_success_payload() {
|
||||
let payload = parse_beeble_webhook(SUCCESS_PAYLOAD).unwrap();
|
||||
assert_eq!(payload.id, "swx_abc123");
|
||||
assert_eq!(payload.status, BeebleWebhookStatus::Completed);
|
||||
assert!(payload.error.is_none());
|
||||
|
||||
let output = payload.output.unwrap();
|
||||
assert_eq!(output.render.as_deref(), Some("https://cdn.beeble.ai/render.mp4"));
|
||||
assert_eq!(output.source.as_deref(), Some("https://cdn.beeble.ai/source.mp4"));
|
||||
assert_eq!(output.alpha.as_deref(), Some("https://cdn.beeble.ai/alpha.mp4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_failure_payload() {
|
||||
let payload = parse_beeble_webhook(FAILURE_PAYLOAD).unwrap();
|
||||
assert_eq!(payload.id, "swx_abc123");
|
||||
assert_eq!(payload.status, BeebleWebhookStatus::Failed);
|
||||
assert_eq!(payload.error.as_deref(), Some("Processing failed"));
|
||||
assert!(payload.output.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_unknown_status() {
|
||||
let payload = parse_beeble_webhook(UNKNOWN_STATUS_PAYLOAD).unwrap();
|
||||
assert_eq!(payload.id, "swx_xyz789");
|
||||
assert_eq!(payload.status, BeebleWebhookStatus::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_null_output() {
|
||||
let payload = parse_beeble_webhook(NULL_OUTPUT_PAYLOAD).unwrap();
|
||||
assert_eq!(payload.id, "swx_abc123");
|
||||
assert_eq!(payload.status, BeebleWebhookStatus::Completed);
|
||||
assert!(payload.output.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_json() {
|
||||
let result = parse_beeble_webhook("not valid json");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod beeble_webhook_payload;
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
use anyhow::anyhow;
|
||||
use chrono::{DateTime, Utc};
|
||||
use log::warn;
|
||||
use sqlx::pool::PoolConnection;
|
||||
use sqlx::{MySql, MySqlPool};
|
||||
|
||||
use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
|
||||
use enums::common::job_status_plus::JobStatusPlus;
|
||||
use errors::AnyhowResult;
|
||||
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
|
||||
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
|
||||
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
|
||||
use tokens::tokens::prompts::PromptToken;
|
||||
use tokens::tokens::users::UserToken;
|
||||
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BeebleJobDetails {
|
||||
pub job_token: InferenceJobToken,
|
||||
|
||||
pub status: JobStatusPlus,
|
||||
|
||||
pub external_third_party: InferenceJobExternalThirdParty,
|
||||
pub external_third_party_id: String,
|
||||
|
||||
pub maybe_creator_user_token: Option<UserToken>,
|
||||
pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
|
||||
pub creator_ip_address: String,
|
||||
|
||||
pub maybe_prompt_token: Option<PromptToken>,
|
||||
pub maybe_debug_log_event_token: Option<DebugLogEventToken>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct RawJobRecord {
|
||||
job_token: InferenceJobToken,
|
||||
|
||||
status: JobStatusPlus,
|
||||
|
||||
// NB: Nullable, but required to be filled for this query
|
||||
external_third_party: Option<InferenceJobExternalThirdParty>,
|
||||
|
||||
// NB: Nullable, but required to be filled for this query
|
||||
external_third_party_id: Option<String>,
|
||||
|
||||
maybe_creator_user_token: Option<UserToken>,
|
||||
maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
|
||||
creator_ip_address: String,
|
||||
|
||||
maybe_prompt_token: Option<PromptToken>,
|
||||
maybe_debug_log_event_token: Option<DebugLogEventToken>,
|
||||
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Returns Ok(None) when the record cannot be found.
|
||||
pub async fn get_inference_job_by_beeble_id(beeble_id: &str, mysql_pool: &MySqlPool)
|
||||
-> AnyhowResult<Option<BeebleJobDetails>>
|
||||
{
|
||||
let mut connection = mysql_pool.acquire().await?;
|
||||
get_inference_job_by_beeble_id_from_connection(beeble_id, &mut connection).await
|
||||
}
|
||||
|
||||
/// Returns Ok(None) when the record cannot be found.
|
||||
pub async fn get_inference_job_by_beeble_id_from_connection(beeble_id: &str, mysql_connection: &mut PoolConnection<MySql>)
|
||||
-> AnyhowResult<Option<BeebleJobDetails>>
|
||||
{
|
||||
let maybe_status = sqlx::query_as!(
|
||||
RawJobRecord,
|
||||
r#"
|
||||
SELECT
|
||||
jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,
|
||||
|
||||
jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,
|
||||
|
||||
jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,
|
||||
jobs.maybe_external_third_party_id as `external_third_party_id`,
|
||||
|
||||
jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
|
||||
jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,
|
||||
jobs.creator_ip_address,
|
||||
|
||||
jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,
|
||||
jobs.maybe_debug_log_event_token as `maybe_debug_log_event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,
|
||||
|
||||
jobs.created_at,
|
||||
jobs.updated_at
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
WHERE jobs.maybe_external_third_party = ?
|
||||
AND jobs.maybe_external_third_party_id = ?
|
||||
"#,
|
||||
InferenceJobExternalThirdParty::Beeble.to_str(),
|
||||
beeble_id
|
||||
)
|
||||
.fetch_one(&mut **mysql_connection)
|
||||
.await;
|
||||
|
||||
let record = match maybe_status {
|
||||
Ok(record) => record,
|
||||
Err(err) => match err {
|
||||
sqlx::Error::RowNotFound => return Ok(None),
|
||||
_ => {
|
||||
warn!("error querying job record: {:?}", err);
|
||||
return Err(anyhow!("error querying job record: {:?}", err));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let record = raw_record_to_public_result(record)?;
|
||||
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
fn raw_record_to_public_result(record: RawJobRecord) -> AnyhowResult<BeebleJobDetails> {
|
||||
Ok(BeebleJobDetails {
|
||||
job_token: record.job_token,
|
||||
status: record.status,
|
||||
external_third_party: record.external_third_party.ok_or_else(|| anyhow!("missing external_third_party"))?,
|
||||
external_third_party_id: record.external_third_party_id.ok_or_else(|| anyhow!("missing external_third_party_id"))?,
|
||||
maybe_creator_user_token: record.maybe_creator_user_token,
|
||||
maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token,
|
||||
creator_ip_address: record.creator_ip_address,
|
||||
maybe_prompt_token: record.maybe_prompt_token,
|
||||
maybe_debug_log_event_token: record.maybe_debug_log_event_token,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
})
|
||||
}
|
||||
@@ -1 +1,2 @@
|
||||
pub mod get_inference_job_by_beeble_id;
|
||||
pub mod insert_generic_inference_job_for_beeble_queue_with_apriori_job_token;
|
||||
|
||||
@@ -22,6 +22,7 @@ pub enum DebugLogType {
|
||||
FalRequest,
|
||||
KinoviRequest,
|
||||
FalWebhook,
|
||||
BeebleWebhook,
|
||||
}
|
||||
|
||||
impl_enum_display_and_debug_using_to_str!(DebugLogType);
|
||||
@@ -35,6 +36,7 @@ impl DebugLogType {
|
||||
Self::FalRequest => "fal_request",
|
||||
Self::KinoviRequest => "kinovi_request",
|
||||
Self::FalWebhook => "fal_webhook",
|
||||
Self::BeebleWebhook => "beeble_webhook",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +46,7 @@ impl DebugLogType {
|
||||
"fal_request" => Ok(Self::FalRequest),
|
||||
"kinovi_request" => Ok(Self::KinoviRequest),
|
||||
"fal_webhook" => Ok(Self::FalWebhook),
|
||||
"beeble_webhook" => Ok(Self::BeebleWebhook),
|
||||
_ => Err(EnumError::CouldNotConvertFromString(value.to_string())),
|
||||
}
|
||||
}
|
||||
@@ -54,6 +57,7 @@ impl DebugLogType {
|
||||
Self::FalRequest,
|
||||
Self::KinoviRequest,
|
||||
Self::FalWebhook,
|
||||
Self::BeebleWebhook,
|
||||
])
|
||||
}
|
||||
}
|
||||
@@ -74,6 +78,7 @@ mod tests {
|
||||
assert_serialization(DebugLogType::FalRequest, "fal_request");
|
||||
assert_serialization(DebugLogType::KinoviRequest, "kinovi_request");
|
||||
assert_serialization(DebugLogType::FalWebhook, "fal_webhook");
|
||||
assert_serialization(DebugLogType::BeebleWebhook, "beeble_webhook");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -82,6 +87,7 @@ mod tests {
|
||||
assert_eq!(DebugLogType::FalRequest.to_str(), "fal_request");
|
||||
assert_eq!(DebugLogType::KinoviRequest.to_str(), "kinovi_request");
|
||||
assert_eq!(DebugLogType::FalWebhook.to_str(), "fal_webhook");
|
||||
assert_eq!(DebugLogType::BeebleWebhook.to_str(), "beeble_webhook");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -90,6 +96,7 @@ mod tests {
|
||||
assert_eq!(DebugLogType::from_str("fal_request").unwrap(), DebugLogType::FalRequest);
|
||||
assert_eq!(DebugLogType::from_str("kinovi_request").unwrap(), DebugLogType::KinoviRequest);
|
||||
assert_eq!(DebugLogType::from_str("fal_webhook").unwrap(), DebugLogType::FalWebhook);
|
||||
assert_eq!(DebugLogType::from_str("beeble_webhook").unwrap(), DebugLogType::BeebleWebhook);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -106,11 +113,12 @@ mod tests {
|
||||
#[test]
|
||||
fn all_variants() {
|
||||
let mut variants = DebugLogType::all_variants();
|
||||
assert_eq!(variants.len(), 4);
|
||||
assert_eq!(variants.len(), 5);
|
||||
assert_eq!(variants.pop_first(), Some(DebugLogType::HttpRequest));
|
||||
assert_eq!(variants.pop_first(), Some(DebugLogType::FalRequest));
|
||||
assert_eq!(variants.pop_first(), Some(DebugLogType::KinoviRequest));
|
||||
assert_eq!(variants.pop_first(), Some(DebugLogType::FalWebhook));
|
||||
assert_eq!(variants.pop_first(), Some(DebugLogType::BeebleWebhook));
|
||||
assert_eq!(variants.pop_first(), None);
|
||||
}
|
||||
}
|
||||
|
||||
+11
-1
@@ -69,6 +69,9 @@ pub enum MediaFileOriginProductCategory {
|
||||
#[serde(rename = "video_gen")]
|
||||
VideoGeneration,
|
||||
|
||||
#[serde(rename = "video_edit")]
|
||||
VideoEdit,
|
||||
|
||||
#[serde(rename = "world_gen")]
|
||||
WorldGeneration,
|
||||
|
||||
@@ -105,6 +108,7 @@ impl MediaFileOriginProductCategory {
|
||||
Self::Mocap => "mocap",
|
||||
Self::ImageGeneration => "image_gen",
|
||||
Self::VideoGeneration => "video_gen",
|
||||
Self::VideoEdit => "video_edit",
|
||||
Self::WorldGeneration => "world_gen",
|
||||
Self::VideoFilter => "video_filter",
|
||||
Self::Workflow => "workflow",
|
||||
@@ -126,6 +130,7 @@ impl MediaFileOriginProductCategory {
|
||||
"mocap" => Ok(Self::Mocap),
|
||||
"image_gen" => Ok(Self::ImageGeneration),
|
||||
"video_gen" => Ok(Self::VideoGeneration),
|
||||
"video_edit" => Ok(Self::VideoEdit),
|
||||
"world_gen" => Ok(Self::WorldGeneration),
|
||||
"video_filter" => Ok(Self::VideoFilter),
|
||||
"workflow" => Ok(Self::Workflow),
|
||||
@@ -150,6 +155,7 @@ impl MediaFileOriginProductCategory {
|
||||
Self::Mocap,
|
||||
Self::ImageGeneration,
|
||||
Self::VideoGeneration,
|
||||
Self::VideoEdit,
|
||||
Self::WorldGeneration,
|
||||
Self::VideoFilter,
|
||||
Self::Workflow,
|
||||
@@ -180,6 +186,7 @@ mod tests {
|
||||
assert_serialization(MediaFileOriginProductCategory::Mocap, "mocap");
|
||||
assert_serialization(MediaFileOriginProductCategory::ImageGeneration, "image_gen");
|
||||
assert_serialization(MediaFileOriginProductCategory::VideoGeneration, "video_gen");
|
||||
assert_serialization(MediaFileOriginProductCategory::VideoEdit, "video_edit");
|
||||
assert_serialization(MediaFileOriginProductCategory::WorldGeneration, "world_gen");
|
||||
assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter");
|
||||
assert_serialization(MediaFileOriginProductCategory::Workflow, "workflow");
|
||||
@@ -200,6 +207,7 @@ mod tests {
|
||||
assert_eq!(MediaFileOriginProductCategory::Mocap.to_str(), "mocap");
|
||||
assert_eq!(MediaFileOriginProductCategory::ImageGeneration.to_str(), "image_gen");
|
||||
assert_eq!(MediaFileOriginProductCategory::VideoGeneration.to_str(), "video_gen");
|
||||
assert_eq!(MediaFileOriginProductCategory::VideoEdit.to_str(), "video_edit");
|
||||
assert_eq!(MediaFileOriginProductCategory::WorldGeneration.to_str(), "world_gen");
|
||||
assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter");
|
||||
assert_eq!(MediaFileOriginProductCategory::Workflow.to_str(), "workflow");
|
||||
@@ -220,6 +228,7 @@ mod tests {
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("mocap").unwrap(), MediaFileOriginProductCategory::Mocap);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("image_gen").unwrap(),MediaFileOriginProductCategory::ImageGeneration);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("video_gen").unwrap(), MediaFileOriginProductCategory::VideoGeneration);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("video_edit").unwrap(), MediaFileOriginProductCategory::VideoEdit);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("world_gen").unwrap(), MediaFileOriginProductCategory::WorldGeneration);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter);
|
||||
assert_eq!(MediaFileOriginProductCategory::from_str("workflow").unwrap(), MediaFileOriginProductCategory::Workflow);
|
||||
@@ -228,7 +237,7 @@ mod tests {
|
||||
#[test]
|
||||
fn all_variants() {
|
||||
let mut variants = MediaFileOriginProductCategory::all_variants();
|
||||
assert_eq!(variants.len(), 16);
|
||||
assert_eq!(variants.len(), 17);
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion));
|
||||
@@ -242,6 +251,7 @@ mod tests {
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Mocap));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageGeneration));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoGeneration));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoEdit));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::WorldGeneration));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter));
|
||||
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Workflow));
|
||||
|
||||
@@ -51,6 +51,8 @@ FAL_WEBHOOK_URL="https://brandons-macbook-pro.taild62114.ts.net/v1/webhooks/fal"
|
||||
#FAL_WEBHOOK_URL="https://halide.tailce84f.ts.net/v1/webhooks/fal"
|
||||
#FAL_WEBHOOK_URL="https://s1.pancake-chimaera.ts.net/api/fal/webhook"
|
||||
|
||||
BEEBLE_WEBHOOK_URL="https://brandons-macbook-pro.taild62114.ts.net/v1/webhooks/beeble"
|
||||
|
||||
STRIPE_ARTCRAFT_CHECKOUT_SUCCESS_URL="https://getartcraft.com/checkout_success"
|
||||
STRIPE_ARTCRAFT_CHECKOUT_CANCEL_URL="https://getartcraft.com/checkout_cancel"
|
||||
STRIPE_ARTCRAFT_PORTAL_RETURN_URL="https://getartcraft.com/portal_closed"
|
||||
|
||||
+19
-3
@@ -8,6 +8,8 @@ use sqlx::Acquire;
|
||||
use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::{
|
||||
BeebleSwitchXEditVideoRequest, BeebleSwitchXEditVideoResponse,
|
||||
};
|
||||
use beeble_client::error::beeble_error::BeebleError;
|
||||
use beeble_client::error::beeble_specific_api_error::BeebleSpecificApiError;
|
||||
use beeble_client::requests::create_upload_url::upload_bytes::upload_bytes_to_beeble;
|
||||
use beeble_client::requests::start_generation::start_generation::{
|
||||
start_generation, BeebleAlphaMode, BeebleGenerationType, StartGenerationArgs,
|
||||
@@ -214,12 +216,26 @@ pub async fn beeble_switchx_edit_video_gen_handler(
|
||||
reference_image_uri: maybe_reference_beeble_uri,
|
||||
alpha_uri: None,
|
||||
max_resolution: Some(1080),
|
||||
callback_url: None,
|
||||
callback_url: Some(server_state.beeble.webhook_url.clone()),
|
||||
idempotency_key: Some(request.uuid_idempotency_token.clone()),
|
||||
},
|
||||
}).await.map_err(|err| {
|
||||
error!("Beeble start_generation failed: {:?}", err);
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
warn!("Beeble start_generation failed: {:?}", err);
|
||||
match err {
|
||||
BeebleError::ApiSpecific(BeebleSpecificApiError::VideoHasTooManyFrames {
|
||||
max_frames,
|
||||
detected_frames,
|
||||
ref message,
|
||||
}) => {
|
||||
AdvancedCommonWebError::BadInputWithSimpleMessage(format!(
|
||||
"Video has too many frames: {} (max: {}); {}",
|
||||
detected_frames, max_frames, message
|
||||
))
|
||||
},
|
||||
_ => {
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
}
|
||||
}
|
||||
})?;
|
||||
|
||||
let external_job_id = &beeble_result.id;
|
||||
|
||||
+427
@@ -0,0 +1,427 @@
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_web::web::Bytes;
|
||||
use actix_web::web::Json;
|
||||
use actix_web::{web, HttpRequest};
|
||||
use log::{error, info, warn};
|
||||
use sqlx::pool::PoolConnection;
|
||||
use sqlx::MySql;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
use beeble_client::webhook_api::beeble_webhook_payload::{
|
||||
parse_beeble_webhook, BeebleWebhookPayload, BeebleWebhookStatus,
|
||||
};
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
|
||||
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info;
|
||||
use filesys::path_to_string::path_to_string;
|
||||
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
|
||||
use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
|
||||
use mysql_queries::queries::debug_logs::insert_debug_log::{insert_debug_log, InsertDebugLogArgs};
|
||||
use mysql_queries::queries::generic_inference::beeble::get_inference_job_by_beeble_id::{
|
||||
get_inference_job_by_beeble_id_from_connection, BeebleJobDetails,
|
||||
};
|
||||
use mysql_queries::queries::generic_inference::fal::mark_fal_generic_inference_job_successfully_done::{
|
||||
mark_fal_generic_inference_job_successfully_done, MarkJobArgs,
|
||||
};
|
||||
use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{
|
||||
mark_job_failed_by_token_from_connection, MarkJobFailedByTokenFromConnectionArgs,
|
||||
};
|
||||
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
|
||||
use pager::notification::notification_details_builder::NotificationDetailsBuilder;
|
||||
use pager::notification::notification_urgency::NotificationUrgency;
|
||||
use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
|
||||
const VIDEO_BUCKET_PREFIX: Option<&str> = Some("artcraft_");
|
||||
|
||||
// =============== Handler ===============
|
||||
|
||||
pub async fn beeble_webhook_handler(
|
||||
http_request: HttpRequest,
|
||||
request_body_bytes: Bytes,
|
||||
server_state: web::Data<Arc<ServerState>>,
|
||||
) -> Result<Json<SimpleGenericJsonSuccess>, AdvancedCommonWebError> {
|
||||
|
||||
// Step 1: Parse bytes into a UTF-8 string and log it.
|
||||
let raw_body = String::from_utf8(request_body_bytes.to_vec())
|
||||
.map_err(|err| {
|
||||
error!("Beeble webhook: could not decode request body to UTF-8: {:?}", err);
|
||||
enqueue_parse_error_alert(&server_state, &http_request, "UTF-8 decode failed", &err, None);
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
})?;
|
||||
|
||||
info!("Received Beeble webhook body: {}", raw_body);
|
||||
|
||||
// Step 2: Parse into BeebleWebhookPayload.
|
||||
let payload = parse_beeble_webhook(&raw_body)
|
||||
.map_err(|err| {
|
||||
error!("Beeble webhook: could not parse webhook payload: {:?}", err);
|
||||
enqueue_parse_error_alert(&server_state, &http_request, "JSON parse failed", &err, Some(&raw_body));
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
})?;
|
||||
|
||||
let job_id = payload.id.as_str();
|
||||
|
||||
info!("Beeble webhook job_id: {} (status: {:?})", job_id, payload.status);
|
||||
|
||||
// Step 3: Acquire a MySQL connection.
|
||||
let mut mysql_connection = server_state.mysql_pool.acquire().await?;
|
||||
|
||||
// Step 4: Look up the job by external third-party ID.
|
||||
let job = match get_inference_job_by_beeble_id_from_connection(job_id, &mut mysql_connection).await {
|
||||
Ok(Some(record)) => record,
|
||||
Ok(None) => {
|
||||
warn!("Could not find job record by Beeble job_id: {}", job_id);
|
||||
return Err(AdvancedCommonWebError::NotFound);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Error querying job record for Beeble job_id {}: {:?}", job_id, err);
|
||||
return Err(AdvancedCommonWebError::from_anyhow_error(err));
|
||||
}
|
||||
};
|
||||
|
||||
info!("Beeble webhook job record for job_id {}: {:?}", job_id, job);
|
||||
|
||||
// Insert debug log for the webhook payload.
|
||||
if let Some(debug_log_event_token) = &job.maybe_debug_log_event_token {
|
||||
if let Err(err) = insert_debug_log(InsertDebugLogArgs {
|
||||
apriori_debug_log_event_token: Some(debug_log_event_token),
|
||||
maybe_creator_user_token: job.maybe_creator_user_token.as_ref(),
|
||||
debug_log_type: DebugLogType::BeebleWebhook,
|
||||
message: &raw_body,
|
||||
mysql_executor: &mut *mysql_connection,
|
||||
phantom: Default::default(),
|
||||
}).await {
|
||||
warn!("Failed to insert Beeble webhook debug log: {:?}", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Branch on status.
|
||||
let result = match payload.status {
|
||||
BeebleWebhookStatus::Completed => {
|
||||
handle_completed(&server_state, &mut mysql_connection, job_id, &payload, &job).await
|
||||
}
|
||||
BeebleWebhookStatus::Failed => {
|
||||
handle_failed(&mut mysql_connection, job_id, &payload, &job).await
|
||||
}
|
||||
BeebleWebhookStatus::Unknown => {
|
||||
warn!("Beeble webhook received unknown status for job_id {}", job_id);
|
||||
Err(AdvancedCommonWebError::server_error_with_message(
|
||||
&format!("Beeble webhook received unknown status for job_id {}", job_id),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(ref err) = result {
|
||||
if err.is_server_error() {
|
||||
error!("Beeble webhook error for job_id {}: {:?}", job_id, err);
|
||||
|
||||
let description = format!(
|
||||
"Beeble webhook failed for job_id: {}\n\nError: {:?}\n\nWebhook JSON Payload: {}",
|
||||
job_id, err, raw_body,
|
||||
);
|
||||
|
||||
let mut builder = if let Some(cause_arc) = err.clone_cause_arc() {
|
||||
NotificationDetailsBuilder::from_error(cause_arc)
|
||||
.set_title(format!("Beeble webhook processing failed (job_id: {})", job_id))
|
||||
} else {
|
||||
NotificationDetailsBuilder::from_title(
|
||||
format!("Beeble webhook processing failed (job_id: {})", job_id)
|
||||
)
|
||||
};
|
||||
|
||||
builder = builder
|
||||
.set_description(Some(description))
|
||||
.set_third_party_id(Some(job_id.to_string()))
|
||||
.set_urgency(Some(NotificationUrgency::High))
|
||||
.set_http_method(Some(http_request.method().to_string()))
|
||||
.set_http_path(Some(http_request.path().to_string()));
|
||||
|
||||
if let Err(pager_err) = server_state.pager.enqueue_page(builder.build()) {
|
||||
error!("Failed to enqueue Beeble webhook pager alert: {:?}", pager_err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
// =============== Private helpers ===============
|
||||
|
||||
async fn handle_completed(
|
||||
server_state: &ServerState,
|
||||
mysql_connection: &mut PoolConnection<MySql>,
|
||||
job_id: &str,
|
||||
payload: &BeebleWebhookPayload,
|
||||
job: &BeebleJobDetails,
|
||||
) -> Result<Json<SimpleGenericJsonSuccess>, AdvancedCommonWebError> {
|
||||
let output = payload.output.as_ref().ok_or_else(|| {
|
||||
warn!("Beeble webhook completed but no output for job_id {}", job_id);
|
||||
AdvancedCommonWebError::server_error_with_message("Beeble webhook completed but output is missing")
|
||||
})?;
|
||||
|
||||
let render_url = output.render.as_deref().ok_or_else(|| {
|
||||
warn!("Beeble webhook completed but no render URL for job_id {}", job_id);
|
||||
AdvancedCommonWebError::server_error_with_message("Beeble webhook completed but render URL is missing")
|
||||
})?;
|
||||
|
||||
info!("Downloading Beeble render video from {} for job_id {}", render_url, job_id);
|
||||
|
||||
// Download and process the primary render video.
|
||||
let render_media_token = download_and_upload_video(render_url, job, server_state).await?;
|
||||
|
||||
info!("Render video uploaded with token {} for job_id {}", render_media_token, job_id);
|
||||
|
||||
// Download and process the alpha video if present.
|
||||
if let Some(ref alpha_url) = output.alpha {
|
||||
info!("Downloading Beeble alpha video from {} for job_id {}", alpha_url, job_id);
|
||||
match download_and_upload_video(alpha_url, job, server_state).await {
|
||||
Ok(alpha_token) => {
|
||||
info!("Alpha video uploaded with token {} for job_id {}", alpha_token, job_id);
|
||||
}
|
||||
Err(err) => {
|
||||
// Fail open for alpha — the render is the primary artifact.
|
||||
warn!("Failed to process alpha video for job_id {}: {:?}", job_id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mark the job as successfully done with the render video's media token.
|
||||
mark_fal_generic_inference_job_successfully_done(MarkJobArgs {
|
||||
job_token: &job.job_token,
|
||||
media_file_token: &render_media_token,
|
||||
maybe_batch_token: None,
|
||||
mysql_executor: &mut **mysql_connection,
|
||||
phantom: Default::default(),
|
||||
}).await.map_err(|err| {
|
||||
warn!("Error marking job as successfully done for job_id {}: {:?}", job_id, err);
|
||||
AdvancedCommonWebError::from_anyhow_error(err)
|
||||
})?;
|
||||
|
||||
info!("Job {} marked as successfully done for Beeble job_id {}", job.job_token, job_id);
|
||||
|
||||
Ok(SimpleGenericJsonSuccess::wrapped(true))
|
||||
}
|
||||
|
||||
async fn handle_failed(
|
||||
mysql_connection: &mut PoolConnection<MySql>,
|
||||
job_id: &str,
|
||||
payload: &BeebleWebhookPayload,
|
||||
job: &BeebleJobDetails,
|
||||
) -> Result<Json<SimpleGenericJsonSuccess>, AdvancedCommonWebError> {
|
||||
let public_failure_reason = payload.error
|
||||
.as_deref()
|
||||
.unwrap_or("Unknown Beeble error")
|
||||
.to_string();
|
||||
|
||||
let internal_failure_reason = format!(
|
||||
"Beeble job_id={}, error={:?}",
|
||||
job_id,
|
||||
payload.error,
|
||||
);
|
||||
|
||||
info!(
|
||||
"Marking job {} as failed for Beeble job_id {}. Reason: {}",
|
||||
job.job_token.as_str(),
|
||||
job_id,
|
||||
public_failure_reason,
|
||||
);
|
||||
|
||||
if let Err(err) = mark_job_failed_by_token_from_connection(MarkJobFailedByTokenFromConnectionArgs {
|
||||
mysql_connection,
|
||||
job_token: &job.job_token,
|
||||
maybe_public_failure_reason: Some(&public_failure_reason),
|
||||
internal_debugging_failure_reason: &internal_failure_reason,
|
||||
maybe_frontend_failure_category: Some(FrontendFailureCategory::GenerationFailed),
|
||||
}).await {
|
||||
error!(
|
||||
"Error marking job {} as failed for Beeble job_id {}: {:?}",
|
||||
job.job_token.as_str(),
|
||||
job_id,
|
||||
err,
|
||||
);
|
||||
return Err(AdvancedCommonWebError::from_anyhow_error(err));
|
||||
}
|
||||
|
||||
info!(
|
||||
"Job {} marked as failed for Beeble job_id {}.",
|
||||
job.job_token.as_str(),
|
||||
job_id,
|
||||
);
|
||||
|
||||
Ok(SimpleGenericJsonSuccess::wrapped(true))
|
||||
}
|
||||
|
||||
async fn download_and_upload_video(
|
||||
video_url: &str,
|
||||
job: &BeebleJobDetails,
|
||||
server_state: &ServerState,
|
||||
) -> Result<MediaFileToken, AdvancedCommonWebError> {
|
||||
// Download with a retry if the first attempt returns suspiciously few bytes.
|
||||
let mut file_bytes = http_download_url_to_bytes(video_url)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!("Failed to download video from {}: {:?}", video_url, err);
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
})?;
|
||||
|
||||
if file_bytes.len() <= 10 {
|
||||
warn!(
|
||||
"Downloaded only {} bytes from {} — retrying once",
|
||||
file_bytes.len(),
|
||||
video_url,
|
||||
);
|
||||
file_bytes = http_download_url_to_bytes(video_url)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!("Failed to download video on retry from {}: {:?}", video_url, err);
|
||||
AdvancedCommonWebError::from_error(err)
|
||||
})?;
|
||||
}
|
||||
|
||||
// Resolve mime type from magic bytes, with no fallback content_type from Beeble.
|
||||
let metadata = resolve_file_metadata(&file_bytes, None)
|
||||
.ok_or_else(|| {
|
||||
warn!(
|
||||
"Could not determine file type for Beeble video (bytes: {})",
|
||||
file_bytes.len(),
|
||||
);
|
||||
AdvancedCommonWebError::server_error_with_message(
|
||||
&format!("Could not determine file type for Beeble video (bytes: {})", file_bytes.len()))
|
||||
})?;
|
||||
|
||||
info!("File type: {}, extension: {:?}, source: {:?}",
|
||||
metadata.mime_type, metadata.file_extension, metadata.source);
|
||||
|
||||
let mime_type = metadata.mime_type.as_str();
|
||||
|
||||
let media_file_type = MediaFileType::try_from_mime_type(mime_type)
|
||||
.ok_or_else(|| {
|
||||
warn!("Unsupported media file type: {}", mime_type);
|
||||
AdvancedCommonWebError::server_error_with_message(
|
||||
&format!("Unsupported media file type: {}", mime_type))
|
||||
})?;
|
||||
|
||||
let extension_with_period = metadata.file_extension.extension_with_period();
|
||||
|
||||
let file_size_bytes = file_bytes.len();
|
||||
let file_hash = sha256_hash_bytes(&file_bytes)
|
||||
.map_err(|e| {
|
||||
warn!("Failed to hash video bytes: {:?}", e);
|
||||
AdvancedCommonWebError::from_anyhow_error(e)
|
||||
})?;
|
||||
|
||||
let mut maybe_duration_millis = None;
|
||||
let mut maybe_frame_width = None;
|
||||
let mut maybe_frame_height = None;
|
||||
|
||||
if let Ok(mut file) = NamedTempFile::new() {
|
||||
let _r = file.write_all(&file_bytes);
|
||||
|
||||
if let Ok(video_info) = ffprobe_get_info(&file.path()) {
|
||||
maybe_duration_millis = video_info.duration
|
||||
.as_ref()
|
||||
.map(|duration| duration.millis as u64);
|
||||
|
||||
maybe_frame_width = video_info.dimensions
|
||||
.as_ref()
|
||||
.map(|dims| dims.width as u32);
|
||||
|
||||
maybe_frame_height = video_info.dimensions
|
||||
.as_ref()
|
||||
.map(|dims| dims.height as u32);
|
||||
}
|
||||
}
|
||||
|
||||
let public_upload_path = MediaFileBucketPath::generate_new(VIDEO_BUCKET_PREFIX, Some(&extension_with_period));
|
||||
|
||||
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
|
||||
|
||||
server_state.public_bucket_client.upload_file_with_content_type_process(
|
||||
public_upload_path.get_full_object_path_str(),
|
||||
file_bytes.as_ref(),
|
||||
mime_type)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("Failed to upload video to bucket: {:?}", e);
|
||||
AdvancedCommonWebError::from_anyhow_error(e)
|
||||
})?;
|
||||
|
||||
let media_token = MediaFileInsertBuilder::new()
|
||||
.maybe_creator_user(job.maybe_creator_user_token.as_ref())
|
||||
.maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref())
|
||||
.creator_ip_address(&job.creator_ip_address)
|
||||
.public_bucket_directory_hash(&public_upload_path)
|
||||
.media_file_class(MediaFileClass::Video)
|
||||
.media_file_type(media_file_type)
|
||||
.media_file_origin_category(MediaFileOriginCategory::Inference)
|
||||
.media_file_origin_product_category(MediaFileOriginProductCategory::VideoEdit)
|
||||
.mime_type(mime_type)
|
||||
.file_size_bytes(file_size_bytes as u64)
|
||||
.maybe_frame_width(maybe_frame_width)
|
||||
.maybe_frame_height(maybe_frame_height)
|
||||
.maybe_duration_millis(maybe_duration_millis)
|
||||
.checksum_sha2(&file_hash)
|
||||
.maybe_prompt_token(job.maybe_prompt_token.as_ref())
|
||||
.insert_pool(&server_state.mysql_pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("Failed to insert video media file record: {:?}", e);
|
||||
AdvancedCommonWebError::from_error(e)
|
||||
})?;
|
||||
|
||||
info!("Video media file uploaded with token: {}", media_token);
|
||||
|
||||
// let thumbnail_task_result =
|
||||
// ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4)
|
||||
// .with_bucket(server_state.public_bucket_client.bucket_name().as_str())
|
||||
// .with_path(&*path_to_string(public_upload_path.to_full_object_pathbuf()))
|
||||
// .with_output_suffix("thumb")
|
||||
// .with_event_id(&media_token.to_string())
|
||||
// .send_all()
|
||||
// .await;
|
||||
// if let Err(err) = thumbnail_task_result {
|
||||
// // Fail open
|
||||
// error!("Failed to create some/all thumbnail tasks: {:?}", err);
|
||||
// }
|
||||
|
||||
Ok(media_token)
|
||||
}
|
||||
|
||||
/// Send a pager alert for early parse failures (before we have a job_id).
|
||||
fn enqueue_parse_error_alert<E: std::fmt::Debug>(
|
||||
server_state: &ServerState,
|
||||
http_request: &HttpRequest,
|
||||
context: &str,
|
||||
err: &E,
|
||||
maybe_raw_body: Option<&str>,
|
||||
) {
|
||||
let description = match maybe_raw_body {
|
||||
Some(body) => format!("Error: {:?}\n\nBeeble Webhook JSON Payload: {}", err, body),
|
||||
None => format!("Beeble Error: {:?}", err),
|
||||
};
|
||||
|
||||
let notification = NotificationDetailsBuilder::from_title(
|
||||
format!("Beeble webhook parse failure: {}", context))
|
||||
.set_description(Some(description))
|
||||
.set_urgency(Some(NotificationUrgency::High))
|
||||
.set_http_method(Some(http_request.method().to_string()))
|
||||
.set_http_path(Some(http_request.path().to_string()))
|
||||
.build();
|
||||
|
||||
if let Err(pager_err) = server_state.pager.enqueue_page(notification) {
|
||||
error!("Failed to enqueue Beeble webhook parse error alert: {:?}", pager_err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod beeble_webhook_handler;
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook;
|
||||
use crate::http_server::endpoints::webhooks::process_success::handle_successful_fal_webhook::handle_successful_fal_webhook;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::handle_successful_fal_webhook::handle_successful_fal_webhook;
|
||||
use crate::state::server_state::ServerState;
|
||||
use actix_web::web::Bytes;
|
||||
use actix_web::web::Json;
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod fal_webhook_handler;
|
||||
pub(super) mod process_failure;
|
||||
pub(super) mod process_success;
|
||||
+1
-1
@@ -4,4 +4,4 @@ mod process_images_payload;
|
||||
mod process_model_glb_payload;
|
||||
mod process_model_mesh_payload;
|
||||
mod process_video_payload;
|
||||
mod resolve_file_metadata;
|
||||
pub(in crate::http_server::endpoints::webhooks) mod resolve_file_metadata;
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
|
||||
use crate::http_server::endpoints::webhooks::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
|
||||
use crate::state::server_state::ServerState;
|
||||
use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
@@ -1,3 +1,2 @@
|
||||
pub mod fal_webhook_handler;
|
||||
pub (super) mod process_failure;
|
||||
pub (super) mod process_success;
|
||||
pub mod beeble;
|
||||
pub mod fal;
|
||||
|
||||
+6
-1
@@ -1,4 +1,5 @@
|
||||
use crate::http_server::endpoints::webhooks::fal_webhook_handler::fal_webhook_handler;
|
||||
use crate::http_server::endpoints::webhooks::beeble::beeble_webhook_handler::beeble_webhook_handler;
|
||||
use crate::http_server::endpoints::webhooks::fal::fal_webhook_handler::fal_webhook_handler;
|
||||
use actix_http::body::MessageBody;
|
||||
use actix_service::ServiceFactory;
|
||||
use actix_web::dev::{ServiceRequest, ServiceResponse};
|
||||
@@ -20,5 +21,9 @@ where
|
||||
.route(web::post().to(fal_webhook_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok()))
|
||||
)
|
||||
.service(web::resource("/beeble")
|
||||
.route(web::post().to(beeble_webhook_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok()))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -479,6 +479,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
api_key: beeble_client::creds::beeble_api_key::BeebleApiKey::new(
|
||||
easyenv::get_env_string_required("BEEBLE_API_KEY")?,
|
||||
),
|
||||
webhook_url: easyenv::get_env_string_required("BEEBLE_WEBHOOK_URL")?,
|
||||
},
|
||||
fal: FalData {
|
||||
api_key: fal_api_key,
|
||||
|
||||
@@ -328,6 +328,7 @@ pub struct FalData {
|
||||
#[derive(Clone)]
|
||||
pub struct BeebleData {
|
||||
pub api_key: BeebleApiKey,
|
||||
pub webhook_url: String,
|
||||
}
|
||||
|
||||
/// Seedance 2 Pro integration
|
||||
|
||||
@@ -343,10 +343,10 @@
|
||||
"markdownDescription": "Denies the fetch_send command without any pre-configured scope."
|
||||
},
|
||||
{
|
||||
"description": "This permission set configures what kind of\nfetch operations are available from the http plugin.\n\nThis enables all fetch operations but does not\nallow explicitly any origins to be fetched. This needs to\nbe manually configured before usage.\n\n#### Granted Permissions\n\nAll fetch operations are enabled.\n\n\n#### This default permission set includes:\n\n- `allow-fetch`\n- `allow-fetch-cancel`\n- `allow-fetch-read-body`\n- `allow-fetch-send`",
|
||||
"description": "This permission set configures what kind of\r\nfetch operations are available from the http plugin.\r\n\r\nThis enables all fetch operations but does not\r\nallow explicitly any origins to be fetched. This needs to\r\nbe manually configured before usage.\r\n\r\n#### Granted Permissions\r\n\r\nAll fetch operations are enabled.\r\n\r\n\n#### This default permission set includes:\n\n- `allow-fetch`\n- `allow-fetch-cancel`\n- `allow-fetch-read-body`\n- `allow-fetch-send`",
|
||||
"type": "string",
|
||||
"const": "default",
|
||||
"markdownDescription": "This permission set configures what kind of\nfetch operations are available from the http plugin.\n\nThis enables all fetch operations but does not\nallow explicitly any origins to be fetched. This needs to\nbe manually configured before usage.\n\n#### Granted Permissions\n\nAll fetch operations are enabled.\n\n\n#### This default permission set includes:\n\n- `allow-fetch`\n- `allow-fetch-cancel`\n- `allow-fetch-read-body`\n- `allow-fetch-send`"
|
||||
"markdownDescription": "This permission set configures what kind of\r\nfetch operations are available from the http plugin.\r\n\r\nThis enables all fetch operations but does not\r\nallow explicitly any origins to be fetched. This needs to\r\nbe manually configured before usage.\r\n\r\n#### Granted Permissions\r\n\r\nAll fetch operations are enabled.\r\n\r\n\n#### This default permission set includes:\n\n- `allow-fetch`\n- `allow-fetch-cancel`\n- `allow-fetch-read-body`\n- `allow-fetch-send`"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/background-change /index.html 200
|
||||
/background-change/ /index.html 200
|
||||
/background-change/* /index.html 200
|
||||
|
||||
/checkout_cancel /index.html 200
|
||||
/checkout_cancel/ /index.html 200
|
||||
/checkout_cancel/* /index.html 200
|
||||
|
||||
@@ -13,6 +13,7 @@ import Navbar from "../components/navbar";
|
||||
import { ToastContainer } from "../components/toast/toast";
|
||||
import CreateImage from "../pages/create-image";
|
||||
import CreateVideo from "../pages/create-video";
|
||||
import CreateVFX from "../pages/create-vfx";
|
||||
//import Landing2 from "../pages/landing2";
|
||||
import Landing3 from "../pages/landing3";
|
||||
import LandingSD2 from "../pages/landing-sd2";
|
||||
@@ -54,6 +55,7 @@ export function App() {
|
||||
<Route path="/landing3" element={<Landing3 />} />
|
||||
<Route path="/create-image" element={<CreateImage />} />
|
||||
<Route path="/create-video" element={<CreateVideo />} />
|
||||
<Route path="/background-change" element={<CreateVFX />} />
|
||||
<Route path="/seedance-2" element={<LandingSD2 />} />
|
||||
<Route path="/download" element={<Download />} />
|
||||
<Route path="/media" element={<Media />} />
|
||||
|
||||
@@ -42,6 +42,7 @@ const NAV_ITEMS: NavEntry[] = [
|
||||
{ name: "Home", href: "/" },
|
||||
{ name: "Image", href: "/create-image" },
|
||||
{ name: "Video", href: "/create-video" },
|
||||
{ name: "BG Change", href: "/background-change" },
|
||||
{
|
||||
name: "Resources",
|
||||
children: [
|
||||
@@ -72,6 +73,71 @@ function isEntryActive(pathname: string, entry: NavEntry): boolean {
|
||||
return isPathActive(pathname, entry.href);
|
||||
}
|
||||
|
||||
function CreditsChip({
|
||||
credits,
|
||||
align,
|
||||
onBuyCredits,
|
||||
onUpgrade,
|
||||
}: {
|
||||
credits: number;
|
||||
align: "center" | "end";
|
||||
onBuyCredits: () => void;
|
||||
onUpgrade: () => void;
|
||||
}) {
|
||||
return (
|
||||
<PopoverMenu
|
||||
position="bottom"
|
||||
align={align}
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon icon={faCoins} className="text-primary text-[11px]" />
|
||||
}
|
||||
triggerLabel={
|
||||
<span className="whitespace-nowrap text-sm font-medium">
|
||||
{credits.toLocaleString()}
|
||||
</span>
|
||||
}
|
||||
buttonClassName="h-8 px-3 ps-2.5 bg-white/[0.04] hover:bg-white/[0.08] border border-white/[0.08] shadow-none text-white/80 rounded-lg gap-1.5"
|
||||
panelClassName="mt-2 bg-[#1a1a1a] border border-white/[0.08] text-white rounded-xl"
|
||||
>
|
||||
{(close) => (
|
||||
<div className="w-72 max-w-[calc(100vw-24px)] p-3 text-white">
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<span className="text-sm font-medium text-white/70">
|
||||
Your credit balance
|
||||
</span>
|
||||
<button
|
||||
className="text-sm font-medium text-primary hover:text-primary-300 transition-colors"
|
||||
onClick={() => {
|
||||
close();
|
||||
onBuyCredits();
|
||||
}}
|
||||
>
|
||||
Buy credits
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-3xl font-semibold text-white tracking-tight">
|
||||
<FontAwesomeIcon icon={faCoins} className="text-xl text-primary" />
|
||||
{credits.toLocaleString()}
|
||||
</div>
|
||||
<div className="mt-3 flex gap-2">
|
||||
<Button
|
||||
variant="primary"
|
||||
className="h-9 grow"
|
||||
onClick={() => {
|
||||
close();
|
||||
onUpgrade();
|
||||
}}
|
||||
icon={faGem}
|
||||
>
|
||||
Upgrade
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</PopoverMenu>
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchCredits(): Promise<number | null> {
|
||||
try {
|
||||
const api = new CreditsApi();
|
||||
@@ -281,62 +347,12 @@ export default function Navbar() {
|
||||
</Link>
|
||||
|
||||
{credits !== null && (
|
||||
<PopoverMenu
|
||||
position="bottom"
|
||||
<CreditsChip
|
||||
credits={credits}
|
||||
align="center"
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon
|
||||
icon={faCoins}
|
||||
className="text-primary text-[11px]"
|
||||
/>
|
||||
}
|
||||
triggerLabel={
|
||||
<span className="whitespace-nowrap text-sm font-medium">
|
||||
{credits.toLocaleString()}
|
||||
</span>
|
||||
}
|
||||
buttonClassName="h-8 px-3 ps-2.5 bg-white/[0.04] hover:bg-white/[0.08] border border-white/[0.08] shadow-none text-white/80 rounded-lg gap-1.5"
|
||||
panelClassName="mt-2 bg-[#1a1a1a] border border-white/[0.08] text-white rounded-xl"
|
||||
>
|
||||
{(close) => (
|
||||
<div className="w-72 p-3 text-white">
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<span className="text-sm font-medium text-white/70">
|
||||
Your credit balance
|
||||
</span>
|
||||
<button
|
||||
className="text-sm font-medium text-primary hover:text-primary-300 transition-colors"
|
||||
onClick={() => {
|
||||
close();
|
||||
setCreditsModalOpen(true);
|
||||
}}
|
||||
>
|
||||
Buy credits
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-3xl font-medium text-white tracking-tight">
|
||||
<FontAwesomeIcon
|
||||
icon={faCoins}
|
||||
className="text-xl text-primary"
|
||||
/>
|
||||
{credits.toLocaleString()}
|
||||
</div>
|
||||
<div className="mt-3 flex gap-2">
|
||||
<Button
|
||||
variant="primary"
|
||||
className="h-9 grow"
|
||||
onClick={() => {
|
||||
close();
|
||||
navigate("/pricing");
|
||||
}}
|
||||
icon={faGem}
|
||||
>
|
||||
Upgrade
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</PopoverMenu>
|
||||
onBuyCredits={() => setCreditsModalOpen(true)}
|
||||
onUpgrade={() => navigate("/pricing")}
|
||||
/>
|
||||
)}
|
||||
|
||||
{hasPaidPlan === false && (
|
||||
@@ -488,8 +504,16 @@ export default function Navbar() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Mobile: hamburger + task queue */}
|
||||
<div className="flex items-center gap-2 lg:hidden">
|
||||
{/* Mobile: credits + task queue + hamburger */}
|
||||
<div className="flex items-center gap-1.5 lg:hidden">
|
||||
{user && credits !== null && (
|
||||
<CreditsChip
|
||||
credits={credits}
|
||||
align="end"
|
||||
onBuyCredits={() => setCreditsModalOpen(true)}
|
||||
onUpgrade={() => navigate("/pricing")}
|
||||
/>
|
||||
)}
|
||||
{user && <TaskQueue />}
|
||||
<DisclosureButton className="flex h-8 w-8 items-center justify-center rounded-lg text-white/70 hover:text-white hover:bg-white/[0.06] transition-colors">
|
||||
<span className="sr-only">Open main menu</span>
|
||||
@@ -640,15 +664,6 @@ export default function Navbar() {
|
||||
/>
|
||||
Discord
|
||||
</DisclosureButton>
|
||||
{credits !== null && (
|
||||
<span className="flex items-center gap-1.5 ml-auto text-[12px] font-medium text-white/80">
|
||||
<FontAwesomeIcon
|
||||
icon={faCoins}
|
||||
className="text-primary text-[10px]"
|
||||
/>
|
||||
{credits.toLocaleString()}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<img
|
||||
|
||||
@@ -23,7 +23,7 @@ import { Button } from "@storyteller/ui-button";
|
||||
import {
|
||||
getProviderDisplayName,
|
||||
getModelDisplayName,
|
||||
ALL_MODELS_LIST,
|
||||
findModelByKey,
|
||||
} from "@storyteller/model-list";
|
||||
import { CloseButton } from "@storyteller/ui-close-button";
|
||||
import dayjs from "dayjs";
|
||||
@@ -157,10 +157,10 @@ const PromptLine = ({
|
||||
style={
|
||||
marqueePlaying
|
||||
? {
|
||||
animation: "marquee 6.5s linear infinite",
|
||||
animationDelay: "0.5s",
|
||||
animationFillMode: "both",
|
||||
}
|
||||
animation: "marquee 6.5s linear infinite",
|
||||
animationDelay: "0.5s",
|
||||
animationFillMode: "both",
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
@@ -468,9 +468,7 @@ function getPrompt(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
return (
|
||||
cached?.maybe_positive_prompt ||
|
||||
job.request.maybe_raw_inference_text ||
|
||||
""
|
||||
cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || ""
|
||||
);
|
||||
}
|
||||
|
||||
@@ -483,8 +481,7 @@ function formatTitleParts(job: Job, promptsMap?: Map<string, Prompts>) {
|
||||
const taskTypeStr = job.request.inference_category?.toLowerCase() ?? "";
|
||||
const modelTypeStr =
|
||||
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
|
||||
const providerKey =
|
||||
cachedPrompt?.maybe_generation_provider ?? modelTypeStr;
|
||||
const providerKey = cachedPrompt?.maybe_generation_provider ?? modelTypeStr;
|
||||
|
||||
let kind: string | undefined;
|
||||
if (taskTypeStr.includes("video")) {
|
||||
@@ -532,18 +529,24 @@ function jobsToInProgress(
|
||||
const parts = formatTitleParts(j, promptsMap);
|
||||
const isVideo =
|
||||
j.request.inference_category?.toLowerCase().includes("video") ?? false;
|
||||
const modelType = j.request.maybe_model_type;
|
||||
|
||||
// Look up per-model estimated duration, cache per task
|
||||
const promptToken = j.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? (promptsMap.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
const modelType =
|
||||
cachedPrompt?.maybe_model_type ?? j.request.maybe_model_type;
|
||||
|
||||
// Look up per-model estimated duration from the model list, cache per
|
||||
// task. Cache only once `modelType` is resolved so a transient
|
||||
// "prompt not loaded yet" render doesn't lock in the wrong fallback.
|
||||
let duration = taskDurationCache.get(j.job_token);
|
||||
if (!duration) {
|
||||
const model = modelType
|
||||
? ALL_MODELS_LIST.find(
|
||||
(m) => m.tauriId === modelType || m.id === modelType,
|
||||
)
|
||||
: undefined;
|
||||
if (duration === undefined) {
|
||||
const model = findModelByKey(modelType);
|
||||
duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000);
|
||||
taskDurationCache.set(j.job_token, duration);
|
||||
if (modelType) {
|
||||
taskDurationCache.set(j.job_token, duration);
|
||||
}
|
||||
}
|
||||
|
||||
// Always use time-based fake progress (matching desktop app behavior)
|
||||
@@ -663,7 +666,8 @@ export const TaskQueue = () => {
|
||||
const promptTokens = useMemo(() => {
|
||||
const tokens: string[] = [];
|
||||
for (const j of jobs) {
|
||||
if (j.request.maybe_prompt_token) tokens.push(j.request.maybe_prompt_token);
|
||||
if (j.request.maybe_prompt_token)
|
||||
tokens.push(j.request.maybe_prompt_token);
|
||||
}
|
||||
return tokens;
|
||||
}, [jobs]);
|
||||
@@ -710,7 +714,7 @@ export const TaskQueue = () => {
|
||||
primaryActionText: "",
|
||||
primaryActionIcon: faTrashAlt,
|
||||
primaryActionBtnClassName: "",
|
||||
onConfirm: async () => { },
|
||||
onConfirm: async () => {},
|
||||
});
|
||||
|
||||
// ── Confirmation handlers ──────────────────────────────────────────
|
||||
@@ -814,7 +818,9 @@ export const TaskQueue = () => {
|
||||
const failedJobs = fetchedJobs.filter((j) =>
|
||||
FAILED_STATUSES.has(j.status.status),
|
||||
);
|
||||
const newCompletedIdSet = new Set(completedJobs.map((j) => j.job_token));
|
||||
const newCompletedIdSet = new Set(
|
||||
completedJobs.map((j) => j.job_token),
|
||||
);
|
||||
const newFailedIdSet = new Set(failedJobs.map((j) => j.job_token));
|
||||
|
||||
if (!initialLoadDoneRef.current) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { MediaUploadApi } from "@storyteller/api";
|
||||
import { EIntermediateFile, MediaUploadApi } from "@storyteller/api";
|
||||
import { UploaderState, UploaderStates } from "@storyteller/common";
|
||||
|
||||
export type UploadImageFn = (args: {
|
||||
@@ -21,6 +21,9 @@ export const uploadImage: UploadImageFn = async ({
|
||||
blob: assetFile,
|
||||
fileName: assetFile.name || `reference-${Date.now()}`,
|
||||
maybe_title: `ref_image_${title}`,
|
||||
// User-uploaded reference images should appear in their library, not be
|
||||
// hidden as intermediate system files.
|
||||
is_intermediate_system_file: EIntermediateFile.false,
|
||||
});
|
||||
|
||||
if (!response?.success || !response.data) {
|
||||
|
||||
@@ -18,7 +18,7 @@ if (import.meta.env.DEV) {
|
||||
window.location.origin,
|
||||
);
|
||||
// NB: This is for Brandon to test with storyteller-web locally:
|
||||
StorytellerApiHostStore.getInstance().setDevelopment();
|
||||
//StorytellerApiHostStore.getInstance().setDevelopment();
|
||||
} catch (e) {
|
||||
console.warn("Failed to set dev API host override", e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faSpinnerThird, faSparkles } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { Button } from "@storyteller/ui-button";
|
||||
import {
|
||||
PromptBoxVFX,
|
||||
TruchetPattern,
|
||||
VFXResultCard,
|
||||
useVFXStore,
|
||||
} from "@storyteller/ui-vfx";
|
||||
import Seo from "../../components/seo";
|
||||
import { useAuthCheck } from "../../components/generation-gallery";
|
||||
import { toast } from "../../components/toast/toast";
|
||||
import { uploadImage } from "../../components/prompt-box/upload-image";
|
||||
import { uploadVideo } from "../../components/prompt-box/upload-media";
|
||||
import {
|
||||
enqueueBackgroundChangeGeneration,
|
||||
listSessionBackgroundChangeJobs,
|
||||
startBackgroundChangePolling,
|
||||
} from "./generate-background-change-api";
|
||||
|
||||
export default function CreateVFX() {
|
||||
const { user, authChecked } = useAuthCheck();
|
||||
|
||||
const history = useVFXStore((s) => s.history);
|
||||
const startResult = useVFXStore((s) => s.startResult);
|
||||
const attachJobToken = useVFXStore((s) => s.attachJobToken);
|
||||
const completeResult = useVFXStore((s) => s.completeResult);
|
||||
const failResult = useVFXStore((s) => s.failResult);
|
||||
const dismissResult = useVFXStore((s) => s.dismissResult);
|
||||
const seedFromSession = useVFXStore((s) => s.seedFromSession);
|
||||
const updateMediaForResult = useVFXStore((s) => s.updateMediaForResult);
|
||||
const source = useVFXStore((s) => s.source);
|
||||
const reference = useVFXStore((s) => s.reference);
|
||||
const prompt = useVFXStore((s) => s.prompt);
|
||||
|
||||
const promptBoxRef = useRef<HTMLDivElement>(null);
|
||||
const [promptBoxHeight, setPromptBoxHeight] = useState(96);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const pollersRef = useRef(new Map<string, () => void>());
|
||||
|
||||
useEffect(() => {
|
||||
const el = promptBoxRef.current;
|
||||
if (!el || typeof ResizeObserver === "undefined") return;
|
||||
const update = () => setPromptBoxHeight(el.offsetHeight);
|
||||
update();
|
||||
const ro = new ResizeObserver(update);
|
||||
ro.observe(el);
|
||||
return () => ro.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const pollers = pollersRef.current;
|
||||
return () => {
|
||||
pollers.forEach((cancel) => cancel());
|
||||
pollers.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// On mount, reconcile against the server: pull recent SwitchX jobs from the
|
||||
// session so the page shows in-progress / completed / failed cards across
|
||||
// refreshes and devices, not just from localStorage. Then resume polling
|
||||
// for anything that's still pending.
|
||||
useEffect(() => {
|
||||
if (!user) return;
|
||||
|
||||
let cancelled = false;
|
||||
const pollers = pollersRef.current;
|
||||
|
||||
const startPollFor = (
|
||||
resultId: string,
|
||||
jobToken: string,
|
||||
) => {
|
||||
if (pollers.has(resultId)) return;
|
||||
const cancel = startBackgroundChangePolling(
|
||||
jobToken,
|
||||
(output) => {
|
||||
completeResult(resultId, output.cdn_url);
|
||||
pollers.delete(resultId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failResult(resultId, reason);
|
||||
pollers.delete(resultId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
pollers.set(resultId, cancel);
|
||||
};
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const sessionJobs = await listSessionBackgroundChangeJobs();
|
||||
if (cancelled) return;
|
||||
|
||||
const existing = useVFXStore.getState().history;
|
||||
const existingByToken = new Map(
|
||||
existing
|
||||
.filter((r) => r.inferenceJobToken)
|
||||
.map((r) => [r.inferenceJobToken!, r] as const),
|
||||
);
|
||||
|
||||
for (const job of sessionJobs) {
|
||||
const found = existingByToken.get(job.jobToken);
|
||||
if (found) {
|
||||
// Server may know more than we do (e.g. completed while we were
|
||||
// away). Reconcile terminal states.
|
||||
if (found.status === "pending" && job.status === "complete" && job.outputUrl) {
|
||||
completeResult(found.id, job.outputUrl);
|
||||
} else if (found.status === "pending" && job.status === "failed") {
|
||||
failResult(found.id, job.failureReason ?? "Generation failed");
|
||||
}
|
||||
// Patch in CDN media + prompt from the batch-prompt response so
|
||||
// dead blob: URLs (from a prior session) are replaced.
|
||||
updateMediaForResult(found.id, {
|
||||
source: job.source
|
||||
? {
|
||||
id: job.source.mediaToken,
|
||||
url: job.source.url,
|
||||
mediaToken: job.source.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
reference: job.reference
|
||||
? {
|
||||
id: job.reference.mediaToken,
|
||||
url: job.reference.url,
|
||||
mediaToken: job.reference.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
prompt: job.prompt,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Missing locally — seed it.
|
||||
const id =
|
||||
typeof crypto !== "undefined" && crypto.randomUUID
|
||||
? crypto.randomUUID()
|
||||
: Math.random().toString(36).slice(2);
|
||||
seedFromSession({
|
||||
id,
|
||||
status: job.status,
|
||||
prompt: job.prompt,
|
||||
resolution: "720p",
|
||||
source: job.source
|
||||
? {
|
||||
id: job.source.mediaToken,
|
||||
url: job.source.url,
|
||||
mediaToken: job.source.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
reference: job.reference
|
||||
? {
|
||||
id: job.reference.mediaToken,
|
||||
url: job.reference.url,
|
||||
mediaToken: job.reference.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
inferenceJobToken: job.jobToken,
|
||||
outputUrl: job.outputUrl,
|
||||
failureReason: job.failureReason,
|
||||
createdAt: job.createdAt,
|
||||
});
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
// Resume polling for everything still pending (local + freshly seeded).
|
||||
const pending = useVFXStore
|
||||
.getState()
|
||||
.history.filter((r) => r.status === "pending" && r.inferenceJobToken);
|
||||
for (const item of pending) {
|
||||
startPollFor(item.id, item.inferenceJobToken!);
|
||||
}
|
||||
} catch {
|
||||
// Network errors during session reconciliation are non-fatal — fall
|
||||
// back to local-only behavior by polling whatever localStorage had.
|
||||
if (cancelled) return;
|
||||
const pending = useVFXStore
|
||||
.getState()
|
||||
.history.filter((r) => r.status === "pending" && r.inferenceJobToken);
|
||||
for (const item of pending) {
|
||||
startPollFor(item.id, item.inferenceJobToken!);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [user, completeResult, failResult, seedFromSession, updateMediaForResult]);
|
||||
|
||||
const handleSubmit = useCallback(async () => {
|
||||
if (!source || !reference || isSubmitting) return;
|
||||
setIsSubmitting(true);
|
||||
const id = startResult();
|
||||
const enqueueResult = await enqueueBackgroundChangeGeneration({
|
||||
sourceVideoMediaToken: source.mediaToken,
|
||||
referenceImageMediaToken: reference.mediaToken,
|
||||
prompt: prompt.trim() || null,
|
||||
});
|
||||
setIsSubmitting(false);
|
||||
|
||||
if (!enqueueResult.success) {
|
||||
const message = enqueueResult.backendUnavailable
|
||||
? "Background change backend coming soon. Your inputs are saved."
|
||||
: enqueueResult.error;
|
||||
failResult(id, message);
|
||||
if (enqueueResult.backendUnavailable) {
|
||||
toast.success(message);
|
||||
} else {
|
||||
toast.error(message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
attachJobToken(id, enqueueResult.jobToken);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
const cancel = startBackgroundChangePolling(
|
||||
enqueueResult.jobToken,
|
||||
(output) => {
|
||||
completeResult(id, output.cdn_url);
|
||||
pollersRef.current.delete(id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failResult(id, reason);
|
||||
pollersRef.current.delete(id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
pollersRef.current.set(id, cancel);
|
||||
}, [
|
||||
source,
|
||||
reference,
|
||||
prompt,
|
||||
isSubmitting,
|
||||
startResult,
|
||||
attachJobToken,
|
||||
completeResult,
|
||||
failResult,
|
||||
]);
|
||||
|
||||
if (!authChecked) {
|
||||
return (
|
||||
<div className="flex h-screen items-center justify-center bg-[#101014]">
|
||||
<FontAwesomeIcon
|
||||
icon={faSpinnerThird}
|
||||
className="animate-spin text-4xl text-primary/80"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!user) {
|
||||
return (
|
||||
<div className="relative min-h-screen overflow-x-hidden bg-[#101014] text-white">
|
||||
<Seo
|
||||
title="Background Change - ArtCraft"
|
||||
description="Swap the backdrop of a video using a reference image."
|
||||
/>
|
||||
<div className="pointer-events-none absolute inset-x-0 top-0 z-0 flex justify-center">
|
||||
<div className="h-[600px] w-[600px] rounded-full bg-gradient-to-br from-primary/30 via-blue-500/20 to-teal-400/10 opacity-40 blur-[120px]" />
|
||||
</div>
|
||||
<div className="relative z-10 flex min-h-screen flex-col items-center justify-center px-4">
|
||||
<FontAwesomeIcon
|
||||
icon={faSparkles}
|
||||
className="mb-6 text-5xl text-white/20"
|
||||
/>
|
||||
<h1 className="mb-3 text-4xl font-bold">Background Change</h1>
|
||||
<p className="mb-8 max-w-md text-center text-lg text-white/60">
|
||||
Sign in to swap the backdrop of a video using a reference image.
|
||||
</p>
|
||||
<div className="flex gap-3">
|
||||
<Link to="/login">
|
||||
<Button
|
||||
variant="primary"
|
||||
className="bg-white px-6 py-2.5 font-semibold text-black shadow-md hover:bg-white/90"
|
||||
>
|
||||
Login
|
||||
</Button>
|
||||
</Link>
|
||||
<Link to="/signup">
|
||||
<Button
|
||||
variant="primary"
|
||||
className="px-6 py-2.5 font-semibold shadow-md"
|
||||
>
|
||||
Sign up
|
||||
</Button>
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="relative min-h-screen w-full bg-[#101014] text-white">
|
||||
<Seo
|
||||
title="Background Change - ArtCraft"
|
||||
description="Swap the backdrop of a video using a reference image."
|
||||
/>
|
||||
|
||||
{/* Truchet pattern is fixed to the viewport so it stays put as the
|
||||
page scrolls behind the (also fixed) navbar. */}
|
||||
<div
|
||||
aria-hidden
|
||||
className="pointer-events-none fixed inset-0 z-0"
|
||||
style={{
|
||||
maskImage:
|
||||
"radial-gradient(ellipse 70% 60% at 50% 45%, black 25%, transparent 85%)",
|
||||
WebkitMaskImage:
|
||||
"radial-gradient(ellipse 70% 60% at 50% 45%, black 25%, transparent 85%)",
|
||||
}}
|
||||
>
|
||||
<TruchetPattern
|
||||
intensity={0.7}
|
||||
className="absolute inset-0 h-full w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{history.length === 0 ? (
|
||||
<div
|
||||
className="relative z-10 flex min-h-screen items-center justify-center px-3 sm:px-6"
|
||||
style={{ paddingBottom: Math.max(promptBoxHeight + 32, 240) }}
|
||||
>
|
||||
<EmptyState
|
||||
title="No background changes yet"
|
||||
subtitle="Upload a source video and a reference image, then optionally add a prompt."
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
className="relative z-10"
|
||||
style={{
|
||||
paddingTop: 60 + 24,
|
||||
paddingBottom: Math.max(promptBoxHeight + 32, 240),
|
||||
}}
|
||||
>
|
||||
<div className="flex flex-col items-center gap-10 px-3 sm:px-6 sm:pt-[18px]">
|
||||
{history.map((r) => (
|
||||
<VFXResultCard
|
||||
key={r.id}
|
||||
data={{
|
||||
prompt: r.prompt,
|
||||
resolution: r.resolution,
|
||||
source: r.source,
|
||||
mask: r.mask,
|
||||
reference: r.reference,
|
||||
outputUrl: r.outputUrl,
|
||||
status: r.status,
|
||||
failureReason: r.failureReason,
|
||||
}}
|
||||
onDismiss={() => dismissResult(r.id)}
|
||||
className="w-[min(960px,calc(100vw-32px))]"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div
|
||||
aria-hidden
|
||||
className="pointer-events-none fixed bottom-0 left-0 right-0 z-20 h-72 bg-gradient-to-t from-[#101014] via-[#101014]/85 to-transparent"
|
||||
/>
|
||||
|
||||
<div
|
||||
ref={promptBoxRef}
|
||||
className="animate-fade-in-up pointer-events-none fixed bottom-2 left-1/2 z-30 -translate-x-1/2 sm:bottom-3"
|
||||
style={{ animationDelay: "150ms" }}
|
||||
>
|
||||
<div className="pointer-events-auto w-[min(620px,calc(100vw-32px))]">
|
||||
<PromptBoxVFX
|
||||
onSubmit={handleSubmit}
|
||||
isSubmitting={isSubmitting}
|
||||
uploadVideo={uploadVideo}
|
||||
uploadImage={uploadImage}
|
||||
onError={(msg) => toast.error(msg)}
|
||||
hideResolution
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface EmptyStateProps {
|
||||
title: string;
|
||||
subtitle: string;
|
||||
}
|
||||
|
||||
const EmptyState = ({ title, subtitle }: EmptyStateProps) => (
|
||||
<div className="flex max-w-md flex-col items-center gap-4 text-center">
|
||||
<div className="flex h-12 w-12 items-center justify-center rounded-2xl bg-white/5 ring-1 ring-white/10">
|
||||
<FontAwesomeIcon icon={faSparkles} className="text-2xl text-white/40" />
|
||||
</div>
|
||||
<h3 className="text-2xl font-bold text-white">{title}</h3>
|
||||
<p className="text-sm text-white/60 max-w-xs">{subtitle}</p>
|
||||
</div>
|
||||
);
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
import { JobsApi, PromptsApi } from "@storyteller/api";
|
||||
import {
|
||||
newIdempotencyToken,
|
||||
submitVFXJob,
|
||||
VFX_NOT_AVAILABLE_ERROR,
|
||||
} from "@storyteller/ui-vfx";
|
||||
|
||||
export interface GeneratedBackgroundChange {
|
||||
cdn_url: string;
|
||||
}
|
||||
|
||||
// ── Request params ───────────────────────────────────────────────────────
|
||||
|
||||
export interface GenerateBackgroundChangeParams {
|
||||
sourceVideoMediaToken: string;
|
||||
referenceImageMediaToken?: string | null;
|
||||
prompt?: string | null;
|
||||
}
|
||||
|
||||
export type EnqueueResult =
|
||||
| { success: true; jobToken: string }
|
||||
| { success: false; error: string; backendUnavailable?: boolean };
|
||||
|
||||
// ── Enqueue generation ───────────────────────────────────────────────────
|
||||
|
||||
export async function enqueueBackgroundChangeGeneration(
|
||||
params: GenerateBackgroundChangeParams,
|
||||
): Promise<EnqueueResult> {
|
||||
const trimmedPrompt = params.prompt?.trim();
|
||||
const response = await submitVFXJob({
|
||||
source_video_media_token: params.sourceVideoMediaToken,
|
||||
reference_image_media_token: params.referenceImageMediaToken ?? null,
|
||||
prompt: trimmedPrompt && trimmedPrompt.length > 0 ? trimmedPrompt : null,
|
||||
uuid_idempotency_token: newIdempotencyToken(),
|
||||
});
|
||||
|
||||
if (response.success) {
|
||||
return { success: true, jobToken: response.inference_job_token };
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: response.error_message ?? "Failed to submit background change job.",
|
||||
backendUnavailable: response.error_code_str === VFX_NOT_AVAILABLE_ERROR,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Poll for completion ──────────────────────────────────────────────────
|
||||
|
||||
export async function pollBackgroundChangeJobResult(
|
||||
jobToken: string,
|
||||
): Promise<{
|
||||
status: "pending" | "complete" | "failed";
|
||||
output?: GeneratedBackgroundChange;
|
||||
error?: string;
|
||||
}> {
|
||||
const jobsApi = new JobsApi();
|
||||
const response = await jobsApi.GetJobByToken({ token: jobToken });
|
||||
|
||||
if (!response.success || !response.data) {
|
||||
return { status: "pending" };
|
||||
}
|
||||
|
||||
const state = response.data;
|
||||
const statusStr = state.status?.status?.toLowerCase() ?? "";
|
||||
|
||||
if (statusStr === "complete_success" || statusStr === "complete") {
|
||||
const result = state.maybe_result as Record<string, unknown> | undefined;
|
||||
const cdnUrl =
|
||||
((result as any)?.media_links?.cdn_url as string | undefined) ??
|
||||
((result as any)?.cdn_url as string | undefined) ??
|
||||
undefined;
|
||||
|
||||
if (cdnUrl) {
|
||||
return { status: "complete", output: { cdn_url: cdnUrl } };
|
||||
}
|
||||
return {
|
||||
status: "failed",
|
||||
error: "Generation finished but returned no output URL.",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
statusStr.includes("fail") ||
|
||||
statusStr.includes("error") ||
|
||||
statusStr === "dead"
|
||||
) {
|
||||
return {
|
||||
status: "failed",
|
||||
error:
|
||||
state.status?.maybe_failure_message ??
|
||||
state.status?.maybe_extra_status_description ??
|
||||
"Generation failed",
|
||||
};
|
||||
}
|
||||
|
||||
return { status: "pending" };
|
||||
}
|
||||
|
||||
// ── Session sync ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Heuristic match for any Beeble SwitchX model variant the backend may emit.
|
||||
* Matches: `switch_x`, `switchx`, `beeble_switchx`, `beeble_switch_x`,
|
||||
* `Beeble.SwitchX`, etc.
|
||||
*/
|
||||
const isSwitchXModel = (modelType?: string | null): boolean => {
|
||||
if (!modelType) return false;
|
||||
const normalized = modelType.toLowerCase().replace(/\./g, "_");
|
||||
return normalized.includes("switchx") || normalized.includes("switch_x");
|
||||
};
|
||||
|
||||
export interface SessionMediaRef {
|
||||
url: string;
|
||||
mediaToken: string;
|
||||
}
|
||||
|
||||
export interface SessionBackgroundChangeJob {
|
||||
jobToken: string;
|
||||
status: "pending" | "complete" | "failed";
|
||||
prompt: string;
|
||||
source?: SessionMediaRef;
|
||||
reference?: SessionMediaRef;
|
||||
outputUrl?: string;
|
||||
failureReason?: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the user's recent jobs from `/v1/jobs/session` and filter down to
|
||||
* Beeble SwitchX (background-change) jobs. Used on mount so the page reflects
|
||||
* server-side state across refreshes and devices, not just localStorage.
|
||||
*
|
||||
* The model type may live on `job.request.maybe_model_type` OR on the prompt
|
||||
* (looked up by `maybe_prompt_token`) — that's the same fallback chain the
|
||||
* task queue / video page use, so we match here too.
|
||||
*/
|
||||
export async function listSessionBackgroundChangeJobs(): Promise<
|
||||
SessionBackgroundChangeJob[]
|
||||
> {
|
||||
const jobsApi = new JobsApi();
|
||||
const response = await jobsApi.ListRecentJobs();
|
||||
if (!response.success || !response.data) return [];
|
||||
|
||||
// Resolve any missing model_types via the prompts API.
|
||||
const promptTokens = Array.from(
|
||||
new Set(
|
||||
response.data
|
||||
.map((j) => j.request.maybe_prompt_token)
|
||||
.filter((t): t is string => !!t),
|
||||
),
|
||||
);
|
||||
const promptsByToken = new Map<
|
||||
string,
|
||||
{
|
||||
maybe_model_type?: string | null;
|
||||
maybe_positive_prompt?: string | null;
|
||||
maybe_context_images?: Array<{
|
||||
media_token: string;
|
||||
media_links: {
|
||||
cdn_url: string;
|
||||
maybe_video_previews?: unknown;
|
||||
};
|
||||
semantic: string;
|
||||
}> | null;
|
||||
}
|
||||
>();
|
||||
if (promptTokens.length > 0) {
|
||||
try {
|
||||
const promptsApi = new PromptsApi();
|
||||
const promptsResponse = await promptsApi.BatchGetPrompts({
|
||||
tokens: promptTokens,
|
||||
});
|
||||
if (promptsResponse.success && promptsResponse.data) {
|
||||
for (const p of promptsResponse.data) {
|
||||
promptsByToken.set(p.token, p);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal: fall back to job.request.maybe_model_type only.
|
||||
}
|
||||
}
|
||||
|
||||
const out: SessionBackgroundChangeJob[] = [];
|
||||
for (const job of response.data) {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? promptsByToken.get(promptToken)
|
||||
: undefined;
|
||||
const modelType =
|
||||
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type;
|
||||
if (!isSwitchXModel(modelType)) continue;
|
||||
|
||||
const statusStr = job.status?.status?.toLowerCase() ?? "";
|
||||
let status: SessionBackgroundChangeJob["status"];
|
||||
let outputUrl: string | undefined;
|
||||
let failureReason: string | undefined;
|
||||
|
||||
if (statusStr === "complete_success" || statusStr === "complete") {
|
||||
status = "complete";
|
||||
const result = job.maybe_result as unknown as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
outputUrl =
|
||||
((result as any)?.media_links?.cdn_url as string | undefined) ??
|
||||
((result as any)?.cdn_url as string | undefined);
|
||||
} else if (
|
||||
statusStr.includes("fail") ||
|
||||
statusStr.includes("error") ||
|
||||
statusStr === "dead" ||
|
||||
statusStr.includes("cancel")
|
||||
) {
|
||||
status = "failed";
|
||||
failureReason =
|
||||
job.status?.maybe_failure_message ??
|
||||
job.status?.maybe_extra_status_description ??
|
||||
"Generation failed";
|
||||
} else {
|
||||
status = "pending";
|
||||
}
|
||||
|
||||
// Extract source video + reference image from the prompt's context images.
|
||||
// We don't depend on exact semantic strings — videos are the items whose
|
||||
// media_links carry video previews, images are everything else.
|
||||
let source: SessionMediaRef | undefined;
|
||||
let reference: SessionMediaRef | undefined;
|
||||
const ctxImages = cachedPrompt?.maybe_context_images ?? [];
|
||||
for (const ci of ctxImages) {
|
||||
const isVideo = !!ci.media_links?.maybe_video_previews;
|
||||
if (isVideo && !source) {
|
||||
source = { url: ci.media_links.cdn_url, mediaToken: ci.media_token };
|
||||
} else if (!isVideo && !reference) {
|
||||
reference = {
|
||||
url: ci.media_links.cdn_url,
|
||||
mediaToken: ci.media_token,
|
||||
};
|
||||
}
|
||||
if (source && reference) break;
|
||||
}
|
||||
|
||||
const resolvedPrompt =
|
||||
cachedPrompt?.maybe_positive_prompt ??
|
||||
job.request.maybe_raw_inference_text ??
|
||||
"";
|
||||
|
||||
out.push({
|
||||
jobToken: job.job_token,
|
||||
status,
|
||||
prompt: resolvedPrompt,
|
||||
source,
|
||||
reference,
|
||||
outputUrl,
|
||||
failureReason,
|
||||
createdAt: new Date(job.created_at).getTime(),
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Polling controller ───────────────────────────────────────────────────
|
||||
|
||||
export function startBackgroundChangePolling(
|
||||
jobToken: string,
|
||||
onComplete: (output: GeneratedBackgroundChange) => void,
|
||||
onError: (reason: string) => void,
|
||||
intervalMs = 4000,
|
||||
maxAttempts = 75, // ~5 min at 4s — Beeble SwitchX expected runtime
|
||||
): () => void {
|
||||
let attempts = 0;
|
||||
let stopped = false;
|
||||
|
||||
const poll = async () => {
|
||||
if (stopped) return;
|
||||
attempts++;
|
||||
|
||||
try {
|
||||
const result = await pollBackgroundChangeJobResult(jobToken);
|
||||
if (stopped) return;
|
||||
|
||||
if (result.status === "complete" && result.output) {
|
||||
onComplete(result.output);
|
||||
return;
|
||||
}
|
||||
if (result.status === "failed") {
|
||||
onError(result.error ?? "Generation failed");
|
||||
return;
|
||||
}
|
||||
if (attempts >= maxAttempts) {
|
||||
onError("Generation timed out");
|
||||
return;
|
||||
}
|
||||
|
||||
setTimeout(poll, intervalMs);
|
||||
} catch {
|
||||
if (!stopped && attempts < maxAttempts) {
|
||||
setTimeout(poll, intervalMs * 2);
|
||||
} else if (!stopped) {
|
||||
onError("Network error during polling");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Start first poll after a short delay (video jobs take time to start)
|
||||
setTimeout(poll, 3000);
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from "./create-vfx";
|
||||
@@ -93,6 +93,9 @@
|
||||
},
|
||||
{
|
||||
"path": "../../libs/api/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/vfx/tsconfig.lib.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
{
|
||||
"path": "../../libs/components/slider-v2"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/vfx"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/gallery-modal"
|
||||
},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { BetaKey } from "~/models";
|
||||
import { Pagination } from "~/pages/PageEnigma/models";
|
||||
import type { Pagination } from "@storyteller/ui-pagescene";
|
||||
|
||||
export class BetaKeysApi extends ApiManager {
|
||||
public CreateBetaKey({
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
import type {
|
||||
MediaFile,
|
||||
MediaInfo,
|
||||
Pagination,
|
||||
PaginationInfinite,
|
||||
} from "~/pages/PageEnigma/models";
|
||||
} from "@storyteller/ui-pagescene";
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { authentication } from "~/signals";
|
||||
import type { Property } from "csstype";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { Prompts } from "~/pages/PageEnigma/models";
|
||||
import type { Prompts } from "@storyteller/ui-pagescene";
|
||||
|
||||
export class PromptsApi extends ApiManager {
|
||||
public GetPromptsByToken({
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { GenerateTtsAudioErrorType } from "~/pages/PageEnigma/enums";
|
||||
import { GenerateTtsAudioErrorType } from "@storyteller/ui-pagescene";
|
||||
import { Visibility } from "~/enums";
|
||||
|
||||
export interface GenerateTtsAudioRequest {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import {
|
||||
import type {
|
||||
Pagination,
|
||||
UserBookmarkBatch,
|
||||
UserBookmarkByEntity,
|
||||
UserBookmarkByUser,
|
||||
} from "~/pages/PageEnigma/models";
|
||||
} from "@storyteller/ui-pagescene";
|
||||
import { authentication } from "~/signals";
|
||||
|
||||
export enum ScopedEntityTypes {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { Pagination } from "~/pages/PageEnigma/models";
|
||||
import type { Pagination } from "@storyteller/ui-pagescene";
|
||||
import { authentication } from "~/signals";
|
||||
import { ZsDataset } from "~/models/Dataset";
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { ApiManager, ApiResponse } from "./ApiManager";
|
||||
import { authentication } from "~/signals";
|
||||
import { Weight } from "~/models";
|
||||
import { Pagination, PaginationInfinite } from "~/pages/PageEnigma/models";
|
||||
import type { Pagination, PaginationInfinite } from "@storyteller/ui-pagescene";
|
||||
import {
|
||||
FilterEngineCategories,
|
||||
FilterMediaClasses,
|
||||
|
||||
@@ -25,10 +25,9 @@ import {
|
||||
import { toast } from "react-hot-toast";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { useTabStore } from "../../pages/Stores/TabState";
|
||||
import { addObject } from "../../pages/PageEnigma/signals/objectGroup/addObject";
|
||||
import { set3DPageMounted } from "../../pages/PageEnigma/Editor/editor";
|
||||
import { addObject, getActiveEditor } from "@storyteller/ui-pagescene";
|
||||
import type { MediaItem } from "@storyteller/ui-pagescene";
|
||||
import { AssetType } from "~/enums";
|
||||
import type { MediaItem } from "../../pages/PageEnigma/models";
|
||||
import { SPLAT_MODELS } from "@storyteller/model-list";
|
||||
import {
|
||||
ClassyModelSelector,
|
||||
@@ -974,7 +973,6 @@ export const ImageTo3DExperience = ({
|
||||
variant="primary"
|
||||
className="min-w-[120px]"
|
||||
onClick={() => {
|
||||
set3DPageMounted(true);
|
||||
useTabStore.getState().setActiveTab("3D");
|
||||
setTimeout(() => {
|
||||
const mediaItem = {
|
||||
@@ -986,8 +984,13 @@ export const ImageTo3DExperience = ({
|
||||
} as MediaItem & {
|
||||
position: { x: number; y: number; z: number };
|
||||
};
|
||||
addObject(mediaItem);
|
||||
toast.success("Added to 3D scene");
|
||||
const editor = getActiveEditor();
|
||||
if (editor) {
|
||||
void addObject(editor, mediaItem);
|
||||
toast.success("Added to 3D scene");
|
||||
} else {
|
||||
toast.error("3D editor isn't ready yet");
|
||||
}
|
||||
}, 500);
|
||||
}}
|
||||
>
|
||||
|
||||
@@ -73,17 +73,6 @@ export function UploadModal3D(props: Props) {
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (uploaderState.status === UploaderStates.success) {
|
||||
// Automatically open the global Gallery modal after a successful upload
|
||||
galleryModalVisibleViewMode.value = true;
|
||||
galleryModalVisibleDuringDrag.value = true;
|
||||
|
||||
onSuccess(selectedCategory);
|
||||
onClose();
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [uploaderState.status]);
|
||||
|
||||
const UploaderModalContent = () => {
|
||||
switch (uploaderState.status) {
|
||||
@@ -130,8 +119,12 @@ export function UploadModal3D(props: Props) {
|
||||
<UploadSuccess
|
||||
title="3D model"
|
||||
onOk={() => {
|
||||
onClose();
|
||||
// Open the gallery so the user can drag the freshly
|
||||
// uploaded model onto the canvas.
|
||||
galleryModalVisibleViewMode.value = true;
|
||||
galleryModalVisibleDuringDrag.value = true;
|
||||
onSuccess(selectedCategory);
|
||||
onClose();
|
||||
}}
|
||||
/>
|
||||
);
|
||||
@@ -172,7 +165,8 @@ export function UploadModal3D(props: Props) {
|
||||
className="max-w-xl"
|
||||
showClose={true}
|
||||
>
|
||||
<UploaderModalContent />
|
||||
{/* Inline call — `<Comp />` would be a fresh component reference each render, remounting the dropzone mid-click and breaking the file picker. */}
|
||||
{UploaderModalContent()}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
+7
-13
@@ -48,17 +48,6 @@ export function UploadModalImage(props: Props) {
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (uploaderState.status === UploaderStates.success) {
|
||||
// Automatically open the global Gallery modal after a successful upload
|
||||
galleryModalVisibleViewMode.value = true;
|
||||
galleryModalVisibleDuringDrag.value = true;
|
||||
|
||||
onSuccess(selectedCategory);
|
||||
onClose();
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [uploaderState.status]);
|
||||
|
||||
const UploaderModalContent = () => {
|
||||
switch (uploaderState.status) {
|
||||
@@ -94,8 +83,12 @@ export function UploadModalImage(props: Props) {
|
||||
<UploadSuccess
|
||||
title="Image"
|
||||
onOk={() => {
|
||||
onClose();
|
||||
// Open the gallery so the user can drag the freshly
|
||||
// uploaded image onto the canvas.
|
||||
galleryModalVisibleViewMode.value = true;
|
||||
galleryModalVisibleDuringDrag.value = true;
|
||||
onSuccess(selectedCategory);
|
||||
onClose();
|
||||
}}
|
||||
/>
|
||||
);
|
||||
@@ -136,7 +129,8 @@ export function UploadModalImage(props: Props) {
|
||||
className="max-w-xl"
|
||||
showClose={true}
|
||||
>
|
||||
<UploaderModalContent />
|
||||
{/* Inline call — `<Comp />` would be a fresh component reference each render, remounting the dropzone mid-click and breaking the file picker. */}
|
||||
{UploaderModalContent()}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
+2
-1
@@ -113,7 +113,8 @@ export function UploadImageMediaModal({
|
||||
|
||||
return (
|
||||
<Modal isOpen={isOpen} onClose={onClose} title={title}>
|
||||
<UploaderModalContent />
|
||||
{/* Inline call — `<Comp />` would be a fresh component reference each render, remounting the dropzone mid-click and breaking the file picker. */}
|
||||
{UploaderModalContent()}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
+2
-1
@@ -100,7 +100,8 @@ export function UploadModalMedia({
|
||||
|
||||
return (
|
||||
<Modal isOpen={isOpen} onClose={onClose} title={title}>
|
||||
<UploaderModalContent />
|
||||
{/* Inline call — `<Comp />` would be a fresh component reference each render, remounting the dropzone mid-click and breaking the file picker. */}
|
||||
{UploaderModalContent()}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
-1
@@ -40,7 +40,6 @@ interface Props {
|
||||
};
|
||||
onClose: () => void;
|
||||
onUploadProgress: (newState: UploaderState) => void;
|
||||
onLocalBytes: (bytes: ArrayBuffer, shouldFlip: boolean) => void;
|
||||
}
|
||||
|
||||
export const UploadFilesSplat = ({
|
||||
|
||||
+8
-15
@@ -18,7 +18,7 @@ import { UploadFilesSplat } from "./UploadFilesSplat";
|
||||
|
||||
interface Props {
|
||||
onClose: () => void;
|
||||
onSuccess: (splatArrayBuffer: ArrayBuffer, shouldFlip: boolean) => void;
|
||||
onSuccess: (category: FilterEngineCategories) => void;
|
||||
isOpen: boolean;
|
||||
title: string;
|
||||
titleIcon: IconDefinition;
|
||||
@@ -52,17 +52,6 @@ export function UploadModalSplat(props: Props) {
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (uploaderState.status === UploaderStates.success) {
|
||||
// Automatically open the global Gallery modal after a successful upload
|
||||
// galleryModalVisibleViewMode.value = true;
|
||||
// galleryModalVisibleDuringDrag.value = true;
|
||||
|
||||
// onSuccess(selectedCategory);
|
||||
onClose();
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [uploaderState.status]);
|
||||
|
||||
const UploaderModalContent = () => {
|
||||
switch (uploaderState.status) {
|
||||
@@ -79,7 +68,6 @@ export function UploadModalSplat(props: Props) {
|
||||
}}
|
||||
onClose={onClose}
|
||||
onUploadProgress={updateUploaderState}
|
||||
onLocalBytes={(buffer, shouldFlip) => { onSuccess(buffer, shouldFlip); }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
@@ -103,8 +91,12 @@ export function UploadModalSplat(props: Props) {
|
||||
<UploadSuccess
|
||||
title="Splat"
|
||||
onOk={() => {
|
||||
// Open the gallery so the user can drag the freshly
|
||||
// uploaded splat onto the canvas.
|
||||
galleryModalVisibleViewMode.value = true;
|
||||
galleryModalVisibleDuringDrag.value = true;
|
||||
onSuccess(selectedCategory);
|
||||
onClose();
|
||||
// onSuccess(selectedCategory);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
@@ -145,7 +137,8 @@ export function UploadModalSplat(props: Props) {
|
||||
className="max-w-xl"
|
||||
showClose={true}
|
||||
>
|
||||
<UploaderModalContent />
|
||||
{/* Inline call — `<Comp />` would be a fresh component reference each render, remounting the dropzone mid-click and breaking the file picker. */}
|
||||
{UploaderModalContent()}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,32 +1,36 @@
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { faTriangleExclamation } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { useShallow } from "zustand/shallow";
|
||||
|
||||
import { Modal } from "@storyteller/ui-modal";
|
||||
import { Button } from "@storyteller/ui-button";
|
||||
import {
|
||||
showErrorDialog,
|
||||
errorDialogMessage,
|
||||
errorDialogTitle,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { usePageSceneStore } from "@storyteller/ui-pagescene";
|
||||
|
||||
export function ErrorDialog() {
|
||||
useSignals();
|
||||
const { showErrorDialog, errorDialogTitle, errorDialogMessage, setShowErrorDialog } =
|
||||
usePageSceneStore(
|
||||
useShallow((s) => ({
|
||||
showErrorDialog: s.showErrorDialog,
|
||||
errorDialogTitle: s.errorDialogTitle,
|
||||
errorDialogMessage: s.errorDialogMessage,
|
||||
setShowErrorDialog: s.setShowErrorDialog,
|
||||
})),
|
||||
);
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={errorDialogTitle.value}
|
||||
title={errorDialogTitle}
|
||||
titleIcon={faTriangleExclamation}
|
||||
titleIconClassName="text-brand-primary"
|
||||
isOpen={showErrorDialog.value}
|
||||
onClose={() => (showErrorDialog.value = false)}
|
||||
isOpen={showErrorDialog}
|
||||
onClose={() => setShowErrorDialog(false)}
|
||||
showClose={false}
|
||||
>
|
||||
<div>
|
||||
{errorDialogMessage.value}
|
||||
{errorDialogMessage}
|
||||
<div className="flex justify-end">
|
||||
<Button
|
||||
type="button"
|
||||
onClick={() => (showErrorDialog.value = false)}
|
||||
onClick={() => setShowErrorDialog(false)}
|
||||
variant="secondary"
|
||||
>
|
||||
Close
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { faFilm } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { Button } from "@storyteller/ui-button";
|
||||
import { generateMovieId, viewMyMovies } from "~/pages/PageEnigma/signals";
|
||||
|
||||
import { activeWorkflowJobs } from "~/signals";
|
||||
|
||||
export const MyMoviesButton = () => {
|
||||
useSignals();
|
||||
const activeCount = activeWorkflowJobs.value
|
||||
? activeWorkflowJobs.value.length
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
<Button
|
||||
variant="action"
|
||||
onClick={() => {
|
||||
generateMovieId.value = "";
|
||||
viewMyMovies.value = true;
|
||||
}}
|
||||
>
|
||||
<div className="relative flex items-center gap-2">
|
||||
{activeCount > 0 ? (
|
||||
<svg
|
||||
className="h-4 w-4 animate-spin text-white"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<circle
|
||||
className="opacity-25"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="4"
|
||||
></circle>
|
||||
<path
|
||||
className="opacity-75"
|
||||
fill="currentColor"
|
||||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
|
||||
></path>
|
||||
</svg>
|
||||
) : (
|
||||
<FontAwesomeIcon icon={faFilm} />
|
||||
)}
|
||||
<div>My Movies</div>
|
||||
</div>
|
||||
</Button>
|
||||
<div
|
||||
className={twMerge(
|
||||
"invisible absolute right-[-7px] top-[-7px] flex h-[20px] w-[20px] items-center justify-center rounded-full bg-brand-primary text-[13px] font-medium text-white",
|
||||
activeCount > 0 && "visible",
|
||||
)}
|
||||
>
|
||||
{activeCount}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -4,11 +4,7 @@ import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { faPencil, faSpinnerThird } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { scene, signalScene, authentication } from "~/signals";
|
||||
import {
|
||||
showErrorDialog,
|
||||
errorDialogMessage,
|
||||
errorDialogTitle,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { usePageSceneStore } from "@storyteller/ui-pagescene";
|
||||
import { Input } from "@storyteller/ui-input";
|
||||
import { MediaFilesApi } from "~/Classes/ApiManager/MediaFilesApi";
|
||||
import { getCreatorIcon, ModelCreator } from "@storyteller/model-list";
|
||||
@@ -36,9 +32,7 @@ export const SceneTitleInput = ({ pageName }: Props) => {
|
||||
};
|
||||
|
||||
const handleShowErrorDialog = (errorMessage: string) => {
|
||||
errorDialogTitle.value = "Error";
|
||||
errorDialogMessage.value = errorMessage;
|
||||
showErrorDialog.value = true;
|
||||
usePageSceneStore.getState().setErrorDialog("Error", errorMessage);
|
||||
};
|
||||
|
||||
const handleChangeSceneTitle = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
|
||||
@@ -82,11 +82,7 @@ import toast from "react-hot-toast";
|
||||
import { APP_DESCRIPTORS, goToApp } from "~/config/appMenu";
|
||||
import { useStoryboardStore } from "~/pages/PageStoryboard";
|
||||
import { useSceneStore } from "@storyteller/ui-pagedraw";
|
||||
import {
|
||||
is3DEditorInitialized,
|
||||
is3DSceneLoaded,
|
||||
set3DPageMounted,
|
||||
} from "~/pages/PageEnigma/Editor/editor";
|
||||
import { usePageSceneStore } from "@storyteller/ui-pagescene";
|
||||
import { useImageTo3DStore } from "~/pages/PageImageTo3DObject/ImageTo3DStore";
|
||||
import { useImageTo3DWorldStore } from "~/pages/PageImageTo3DWorld/ImageTo3DWorldStore";
|
||||
import { useRemoveBackgroundStore } from "~/pages/PageRemoveBackground/RemoveBackgroundStore";
|
||||
@@ -243,8 +239,8 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
|
||||
const tabStore = useTabStore();
|
||||
|
||||
const is3DSceneReady = is3DSceneLoaded.value;
|
||||
const is3DEditorReady = is3DEditorInitialized.value;
|
||||
const is3DSceneReady = usePageSceneStore((s) => s.is3DSceneLoaded);
|
||||
const is3DEditorReady = usePageSceneStore((s) => s.is3DEditorInitialized);
|
||||
const [disableSwitcher, setDisableSwitcher] = useState(false);
|
||||
const switcherThrottle = useRef(false);
|
||||
|
||||
@@ -580,6 +576,8 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
return "Image to 3D World";
|
||||
case "APPS":
|
||||
return "ArtCraft Apps";
|
||||
case "BACKGROUND_CHANGE":
|
||||
return "Background Change";
|
||||
default:
|
||||
return "Artcraft";
|
||||
}
|
||||
@@ -661,7 +659,6 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
setDisableSwitcher(true);
|
||||
|
||||
if (tabId === "APPS") {
|
||||
set3DPageMounted(false);
|
||||
useTabStore.getState().setActiveTab("APPS");
|
||||
setTimeout(() => {
|
||||
switcherThrottle.current = false;
|
||||
@@ -670,12 +667,8 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
return;
|
||||
}
|
||||
|
||||
// Disable 3d engine to prevent memory leak.
|
||||
if (tabId === "3D") {
|
||||
set3DPageMounted(true);
|
||||
} else {
|
||||
set3DPageMounted(false);
|
||||
}
|
||||
// PageScene's mount/unmount in MainApp drives the
|
||||
// engine lifecycle now — no manual flag flips needed.
|
||||
useTabStore.getState().setActiveTab(tabId as TabId);
|
||||
setTimeout(() => {
|
||||
// Clear the throttle
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
faWandMagicSparkles,
|
||||
faPenNib,
|
||||
faCrosshairs,
|
||||
faSparkles,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { useMemo } from "react";
|
||||
import {
|
||||
@@ -17,7 +18,6 @@ import {
|
||||
useStoryboardPageEnabled,
|
||||
} from "@storyteller/ui-settings-modal";
|
||||
import { useTabStore, TabId } from "~/pages/Stores/TabState";
|
||||
import { set3DPageMounted } from "~/pages/PageEnigma/Editor/editor";
|
||||
|
||||
export type AppId =
|
||||
| "IMAGE"
|
||||
@@ -32,7 +32,8 @@ export type AppId =
|
||||
| "IMAGE_TO_3D_WORLD"
|
||||
| "REMOVE_BACKGROUND"
|
||||
| "ANGLES"
|
||||
| "STORYBOARD";
|
||||
| "STORYBOARD"
|
||||
| "BACKGROUND_CHANGE";
|
||||
|
||||
export interface AppDescriptor {
|
||||
id: AppId;
|
||||
@@ -186,6 +187,16 @@ export const ALL_APPS: FullAppItem[] = [
|
||||
color: "bg-fuchsia-600/40",
|
||||
badge: "NEW",
|
||||
},
|
||||
{
|
||||
id: "background-change",
|
||||
label: "Background Change",
|
||||
description: "Swap the backdrop of a video using a reference image",
|
||||
icon: faSparkles,
|
||||
category: "edit",
|
||||
action: "BACKGROUND_CHANGE",
|
||||
color: "bg-orange-500/40",
|
||||
badge: "NEW",
|
||||
},
|
||||
{
|
||||
id: "2d-canvas",
|
||||
label: "Image Editor",
|
||||
@@ -215,9 +226,12 @@ export const useVisibleApps = (): FullAppItem[] => {
|
||||
const storyboardEnabled = useStoryboardPageEnabled();
|
||||
return useMemo(
|
||||
() =>
|
||||
ALL_APPS.filter((app) =>
|
||||
app.action === "STORYBOARD" ? storyboardEnabled : true,
|
||||
),
|
||||
ALL_APPS.filter((app) => {
|
||||
// Background Change is hidden in the desktop app for now.
|
||||
if (app.action === "BACKGROUND_CHANGE") return false;
|
||||
if (app.action === "STORYBOARD") return storyboardEnabled;
|
||||
return true;
|
||||
}),
|
||||
[storyboardEnabled],
|
||||
);
|
||||
};
|
||||
@@ -267,6 +281,7 @@ export const goToApp = (action?: string) => {
|
||||
"REMOVE_BACKGROUND",
|
||||
"ANGLES",
|
||||
"STORYBOARD",
|
||||
"BACKGROUND_CHANGE",
|
||||
].includes(action)
|
||||
) {
|
||||
if (action === "STORYBOARD") {
|
||||
@@ -274,11 +289,6 @@ export const goToApp = (action?: string) => {
|
||||
useExperimentalStore.getState();
|
||||
if (!enabled || !storyboardPageEnabled) return;
|
||||
}
|
||||
if (action === "3D") {
|
||||
set3DPageMounted(true);
|
||||
} else {
|
||||
set3DPageMounted(false);
|
||||
}
|
||||
useTabStore.getState().setActiveTab(action as TabId);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,6 +1 @@
|
||||
export enum AssetFilterOption {
|
||||
ALL,
|
||||
FEATURED,
|
||||
MINE,
|
||||
BOOKMARKED,
|
||||
}
|
||||
export { AssetFilterOption } from "@storyteller/ui-pagescene";
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
export enum AssetType {
|
||||
ANIMATION = "animation",
|
||||
AUDIO = "audio",
|
||||
CAMERA = "camera",
|
||||
CHARACTER = "character",
|
||||
EXPRESSION = "expression",
|
||||
OBJECT = "object",
|
||||
SKYBOX = "skybox",
|
||||
SHAPE = "shape",
|
||||
SPLAT = "splat",
|
||||
}
|
||||
// Single source of truth lives in @storyteller/ui-pagescene; this
|
||||
// re-export keeps the artcraft `~/enums` surface stable while the
|
||||
// 3D scene lib owns the canonical enum identity.
|
||||
export { AssetType } from "@storyteller/ui-pagescene";
|
||||
|
||||
@@ -1,7 +1 @@
|
||||
export enum ClipGroup {
|
||||
CHARACTER = "character",
|
||||
CAMERA = "camera",
|
||||
GLOBAL_AUDIO = "global_audio",
|
||||
OBJECT = "object",
|
||||
PROMPT_TRAVEL = "prompt",
|
||||
}
|
||||
export { ClipGroup } from "@storyteller/ui-pagescene";
|
||||
|
||||
@@ -1,31 +1,7 @@
|
||||
export enum FilterMediaClasses {
|
||||
AUDIO = "audio",
|
||||
IMAGE = "image",
|
||||
VIDEO = "video",
|
||||
DIMENSIONAL = "dimensional",
|
||||
}
|
||||
|
||||
export enum FilterMediaType {
|
||||
SCENE_JSON = "scene_json",
|
||||
GLB = "glb",
|
||||
GLTF = "gltf",
|
||||
PMX = "pmx",
|
||||
PMD = "pmd",
|
||||
VMD = "vmd",
|
||||
}
|
||||
|
||||
export enum FilterEngineCategories {
|
||||
ANIMATION = "animation",
|
||||
AUDIO = "audio",
|
||||
CHARACTER = "character",
|
||||
CREATURE = "creature",
|
||||
EXPRESSION = "expression",
|
||||
IMAGE_PLANE = "image_plane",
|
||||
LOCATION = "location",
|
||||
OBJECT = "object",
|
||||
SPLAT = "splat",
|
||||
SCENE = "scene",
|
||||
SET_DRESSING = "set_dressing",
|
||||
SKYBOX = "skybox",
|
||||
VIDEO_PLANE = "video_plane",
|
||||
}
|
||||
// Single source of truth lives in @storyteller/ui-pagescene; this
|
||||
// re-export keeps the artcraft `~/enums` surface stable.
|
||||
export {
|
||||
FilterEngineCategories,
|
||||
FilterMediaClasses,
|
||||
FilterMediaType,
|
||||
} from "@storyteller/ui-pagescene";
|
||||
|
||||
@@ -1,5 +1,2 @@
|
||||
export enum ToastTypes {
|
||||
SUCCESS,
|
||||
WARNING,
|
||||
ERROR,
|
||||
}
|
||||
// Single source of truth lives in @storyteller/ui-pagescene.
|
||||
export { ToastTypes } from "@storyteller/ui-pagescene";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { StrictMode, useEffect } from "react";
|
||||
import { useSignals, useSignalEffect } from "@preact/signals-react/runtime";
|
||||
import { BrowserRouter } from "react-router-dom";
|
||||
import { PageEnigma } from "./pages/PageEnigma/PageEnigma";
|
||||
import { MainApp } from "./pages/MainApp";
|
||||
import { GlobalFileDropHandler } from "./components/GlobalFileDropHandler/GlobalFileDropHandler";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import "./styles/normalize.css";
|
||||
@@ -13,6 +13,7 @@ import EnvironmentVariables from "~/Classes/EnvironmentVariables";
|
||||
import { pageHeight, pageWidth, persistLogin } from "~/signals";
|
||||
import { SyncStorytellerApiConfig } from "./api/SyncStorytellerApiConfig";
|
||||
import { posthog } from "posthog-js";
|
||||
import { SoundManager } from "@storyteller/soundboard";
|
||||
|
||||
config.autoAddCss = false; /* eslint-disable import/first */
|
||||
|
||||
@@ -84,6 +85,10 @@ const GlobalSettingsManager = ({ env }: { env: Record<string, string> }) => {
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
SoundManager.install();
|
||||
}, []);
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
@@ -94,7 +99,7 @@ createRoot(document.getElementById("root")!).render(
|
||||
<BrowserRouter>
|
||||
<GlobalSettingsManager env={ENV} />
|
||||
<div className="topbar-spacer" data-tauri-drag-region={true} />
|
||||
<PageEnigma />
|
||||
<MainApp />
|
||||
<GlobalFileDropHandler />
|
||||
</BrowserRouter>
|
||||
</StrictMode>
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
// Top-level shell for the artcraft app. Always-mounted chrome
|
||||
// (TopBar, login + pricing modals, toaster, Tauri event listeners,
|
||||
// background refresh hooks) lives here, and a single tab-driven
|
||||
// switch picks the active page below it.
|
||||
//
|
||||
// The 3D editor (PageScene) is one of the 13 tab branches — it
|
||||
// genuinely mounts/unmounts as the user moves on/off the "3D" tab,
|
||||
// so the Three.js engine constructs on entry and tears down on exit
|
||||
// via the EngineProvider's React lifecycle. Other pages render as
|
||||
// siblings, never wrapped by EngineProvider, so they have no
|
||||
// dependency on the 3D engine being present.
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import * as gpu from "detect-gpu";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
|
||||
import { TopBar } from "~/components";
|
||||
import { ErrorDialog } from "~/components";
|
||||
import { LoginModal, useLoginModalStore } from "@storyteller/ui-login-modal";
|
||||
import { toast, Toaster } from "@storyteller/ui-toaster";
|
||||
import {
|
||||
GalleryDragComponent,
|
||||
GalleryItem,
|
||||
onImageDrop,
|
||||
removeImageDropListener,
|
||||
} from "@storyteller/ui-gallery-modal";
|
||||
import {
|
||||
PricingModal,
|
||||
CreditsModal,
|
||||
useCreditsModalStore,
|
||||
} from "@storyteller/ui-pricing-modal";
|
||||
import {
|
||||
isActionReminderOpen,
|
||||
actionReminderProps,
|
||||
ActionReminderModal,
|
||||
} from "@storyteller/ui-action-reminder-modal";
|
||||
import {
|
||||
useFlashFileDownloadErrorEvent,
|
||||
useFlashUserInputErrorEvent,
|
||||
useGenerationCompleteEvent,
|
||||
useGenerationEnqueueFailureEvent,
|
||||
useGenerationEnqueueSuccessEvent,
|
||||
useGenerationFailedEvent,
|
||||
useMediaFileDeletedEvent,
|
||||
useTextToImageGenerationCompleteEvent,
|
||||
} from "@storyteller/tauri-events";
|
||||
import { SoundManager } from "@storyteller/soundboard";
|
||||
import { useStoryboardPageEnabled } from "@storyteller/ui-settings-modal";
|
||||
import {
|
||||
DomLevels,
|
||||
usePageSceneStore,
|
||||
} from "@storyteller/ui-pagescene";
|
||||
|
||||
import { useActiveJobs } from "~/hooks/useActiveJobs";
|
||||
import { useBackgroundLoadingMedia } from "~/hooks/useBackgroundLoadingMedia";
|
||||
import { UsersApi } from "~/Classes/ApiManager";
|
||||
import { authentication } from "~/signals";
|
||||
import { AUTH_STATUS } from "~/enums";
|
||||
import { useTabStore } from "./Stores/TabState";
|
||||
import { useTextToImageStore } from "./PageImage/TextToImageStore";
|
||||
|
||||
import { AppsIndexPage } from "./PageApps/AppsIndexPage";
|
||||
import PageDraw from "./PageDraw/PageDraw";
|
||||
import TextToImage from "./PageImage/TextToImage";
|
||||
import ImageToVideo from "./PageVideo/ImageToVideo";
|
||||
import { VideoFrameExtractor } from "./PageVideoFrameExtractor";
|
||||
import { VideoWatermarkRemover } from "./PageVideoWatermarkRemover";
|
||||
import { ImageWatermarkRemover } from "./PageImageWatermarkRemover";
|
||||
import { ImageTo3DObject } from "./PageImageTo3DObject";
|
||||
import { ImageTo3DWorld } from "./PageImageTo3DWorld";
|
||||
import { RemoveBackground } from "./PageRemoveBackground";
|
||||
import { Angles } from "./PageAngles";
|
||||
import { Storyboard } from "./PageStoryboard";
|
||||
import { PageBackgroundChange } from "./PageBackgroundChange";
|
||||
import { PageScene } from "./PageScene";
|
||||
import {
|
||||
topNavMediaId,
|
||||
topNavMediaUrl,
|
||||
} from "~/components/signaled/TopBar/TopBar";
|
||||
|
||||
interface Props {
|
||||
sceneToken?: string;
|
||||
}
|
||||
|
||||
export const MainApp = ({ sceneToken }: Props) => {
|
||||
useSignals();
|
||||
|
||||
// Background plumbing — should keep running regardless of which tab
|
||||
// is active.
|
||||
useActiveJobs();
|
||||
useBackgroundLoadingMedia();
|
||||
|
||||
// Tauri event listeners. Must always be mounted so generation/upload
|
||||
// completions are surfaced no matter which tab the user is on.
|
||||
useGenerationEnqueueSuccessEvent();
|
||||
useGenerationEnqueueFailureEvent();
|
||||
useGenerationCompleteEvent();
|
||||
useGenerationFailedEvent();
|
||||
|
||||
const completeBatch = useTextToImageStore((s) => s.completeBatch);
|
||||
useTextToImageGenerationCompleteEvent(async (event) => {
|
||||
completeBatch(
|
||||
event.generated_images || [],
|
||||
event.maybe_frontend_subscriber_id,
|
||||
);
|
||||
});
|
||||
|
||||
useFlashUserInputErrorEvent(async (event) => {
|
||||
console.log("Flash user input error event received:", event);
|
||||
toast.error(event.message);
|
||||
});
|
||||
|
||||
useFlashFileDownloadErrorEvent(async (event) => {
|
||||
console.log("Flash file download error event received:", event);
|
||||
toast.error(event.message || "File download failed");
|
||||
});
|
||||
|
||||
useMediaFileDeletedEvent(async (event) => {
|
||||
console.log("Media file deleted event received:", event);
|
||||
await SoundManager.playFileDeleted();
|
||||
toast.error("File deleted.");
|
||||
});
|
||||
|
||||
// Session probe (runs once per shell mount) and GPU detection.
|
||||
// Both are app-wide concerns, not 3D-only.
|
||||
useEffect(() => {
|
||||
const usersApi = new UsersApi();
|
||||
usersApi.GetSession().then((result) => {
|
||||
console.log(
|
||||
`User Info | Username: ${result.data?.user?.username}, Token: ${result.data?.user?.user_token}`,
|
||||
);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const [, setValidGpu] = useState("unknown");
|
||||
useEffect(() => {
|
||||
const { getGPUTier } = gpu;
|
||||
getGPUTier().then((gpuTier) => {
|
||||
console.log("GPU tier", gpuTier);
|
||||
let isValid = false;
|
||||
const fps = gpuTier.fps || 0;
|
||||
if (gpuTier.tier > 1) isValid = true;
|
||||
if (fps > 15) isValid = true;
|
||||
if (gpuTier.gpu === "apple gpu (Apple GPU)") isValid = true;
|
||||
setValidGpu(isValid ? "valid" : "error");
|
||||
});
|
||||
}, []);
|
||||
|
||||
const { triggerRecheck } = useLoginModalStore();
|
||||
const { isOpen: isCreditsOpen, closeModal: closeCreditsModal } =
|
||||
useCreditsModalStore();
|
||||
const disableHotkeyInput = usePageSceneStore((s) => s.disableHotkeyInput);
|
||||
const enableHotkeyInput = usePageSceneStore((s) => s.enableHotkeyInput);
|
||||
|
||||
const currentReminderModalProps = actionReminderProps.value;
|
||||
|
||||
return (
|
||||
<div className="w-screen">
|
||||
<TopBar
|
||||
loginSignUpPressed={() => {
|
||||
console.log("PRESSED");
|
||||
triggerRecheck();
|
||||
}}
|
||||
pageName="Edit Scene"
|
||||
/>
|
||||
<LoginModal
|
||||
videoSrc2D="/resources/videos/artcraft-canvas-demo.mp4"
|
||||
videoSrc3D="/resources/videos/artcraft-3d-demo.mp4"
|
||||
onOpenChange={(isOpen: boolean) => {
|
||||
if (isOpen) {
|
||||
disableHotkeyInput(DomLevels.DIALOGUE);
|
||||
} else {
|
||||
enableHotkeyInput(DomLevels.DIALOGUE);
|
||||
}
|
||||
}}
|
||||
onArtCraftAuthSuccess={(userInfo: any) => {
|
||||
authentication.status.value = AUTH_STATUS.LOGGED_IN;
|
||||
authentication.userInfo.value = userInfo;
|
||||
}}
|
||||
/>
|
||||
|
||||
<TabBody sceneToken={sceneToken} />
|
||||
|
||||
<GalleryDragComponent />
|
||||
<ErrorDialog />
|
||||
<Toaster offsetTop={70} offsetRight={12} zIndex={9999} />
|
||||
{currentReminderModalProps && (
|
||||
<ActionReminderModal
|
||||
isOpen={isActionReminderOpen.value}
|
||||
onClose={currentReminderModalProps.onClose}
|
||||
reminderType={currentReminderModalProps.reminderType}
|
||||
onPrimaryAction={currentReminderModalProps.onPrimaryAction}
|
||||
title={currentReminderModalProps.title}
|
||||
message={currentReminderModalProps.message}
|
||||
primaryActionText={currentReminderModalProps.primaryActionText}
|
||||
secondaryActionText={currentReminderModalProps.secondaryActionText}
|
||||
onSecondaryAction={currentReminderModalProps.onSecondaryAction}
|
||||
isLoading={currentReminderModalProps.isLoading}
|
||||
openAiLogo={currentReminderModalProps.openAiLogo}
|
||||
primaryActionIcon={currentReminderModalProps.primaryActionIcon}
|
||||
primaryActionBtnClassName={
|
||||
currentReminderModalProps.primaryActionBtnClassName
|
||||
}
|
||||
/>
|
||||
)}
|
||||
<PricingModal />
|
||||
<CreditsModal isOpen={isCreditsOpen} onClose={closeCreditsModal} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const TabBody = ({ sceneToken }: { sceneToken?: string }) => {
|
||||
const tabStore = useTabStore();
|
||||
const storyboardPageEnabled = useStoryboardPageEnabled();
|
||||
|
||||
// The 3D case stays unwrapped because Stage3DBody (lib) already
|
||||
// returns a <div> wrapper. Every other page is a fragment whose
|
||||
// top-level children may use position: fixed (e.g. PageDraw); the
|
||||
// wrapping <div> scopes them so they don't stack as siblings of
|
||||
// the TopBar at the MainApp root.
|
||||
switch (tabStore.activeTabId) {
|
||||
case "3D":
|
||||
return <PageScene sceneToken={sceneToken} />;
|
||||
case "APPS":
|
||||
return <div><AppsIndexPage /></div>;
|
||||
case "2D":
|
||||
return <div><PageDrawWithGalleryDrop /></div>;
|
||||
case "IMAGE":
|
||||
return (
|
||||
<div>
|
||||
<TextToImage
|
||||
imageMediaId={topNavMediaId.value}
|
||||
imageUrl={topNavMediaUrl.value}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
case "VIDEO":
|
||||
return <div><ImageToVideo /></div>;
|
||||
case "VIDEO_FRAME_EXTRACTOR":
|
||||
return <div><VideoFrameExtractor /></div>;
|
||||
case "VIDEO_WATERMARK_REMOVAL":
|
||||
return <div><VideoWatermarkRemover /></div>;
|
||||
case "IMAGE_WATERMARK_REMOVAL":
|
||||
return <div><ImageWatermarkRemover /></div>;
|
||||
case "IMAGE_TO_3D_OBJECT":
|
||||
return <div><ImageTo3DObject /></div>;
|
||||
case "IMAGE_TO_3D_WORLD":
|
||||
return <div><ImageTo3DWorld /></div>;
|
||||
case "REMOVE_BACKGROUND":
|
||||
return <div><RemoveBackground /></div>;
|
||||
case "ANGLES":
|
||||
return <div><Angles /></div>;
|
||||
case "STORYBOARD":
|
||||
return storyboardPageEnabled ? <div><Storyboard /></div> : null;
|
||||
case "BACKGROUND_CHANGE":
|
||||
return <div><PageBackgroundChange /></div>;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// Bridges gallery-modal's onImageDrop into PageDraw's existing
|
||||
// `gallery-2d-drop` window-event listener. Lives here (rather than
|
||||
// inside the pagedraw lib) so we don't have to add a gallery-modal
|
||||
// dep there. Mounted only when the 2D tab is active, so it doesn't
|
||||
// race the 3D-tab gallery handler.
|
||||
const PageDrawWithGalleryDrop = () => {
|
||||
useEffect(() => {
|
||||
const handler = onImageDrop(
|
||||
(item: GalleryItem, position: { x: number; y: number }) => {
|
||||
const canvasElement = document.querySelectorAll("canvas")[0];
|
||||
if (!canvasElement) return;
|
||||
const rect = canvasElement.getBoundingClientRect();
|
||||
const canvasX = position.x - rect.left;
|
||||
const canvasY = position.y - rect.top;
|
||||
if (
|
||||
canvasX < 0 ||
|
||||
canvasY < 0 ||
|
||||
canvasX > rect.width ||
|
||||
canvasY > rect.height
|
||||
) {
|
||||
return;
|
||||
}
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("gallery-2d-drop", {
|
||||
detail: { item, canvasPosition: { x: canvasX, y: canvasY } },
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
return () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
if (handler) removeImageDropListener(handler as any);
|
||||
};
|
||||
}, []);
|
||||
return <PageDraw />;
|
||||
};
|
||||
@@ -0,0 +1,203 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { UploadImageMedia, UploadVideoMedia } from "@storyteller/api";
|
||||
import {
|
||||
PromptBoxVFX,
|
||||
SubTabStrip,
|
||||
TruchetPattern,
|
||||
VFXResultCard,
|
||||
VFXShowcaseView,
|
||||
newIdempotencyToken,
|
||||
submitVFXJob,
|
||||
useVFXStore,
|
||||
VFX_NOT_AVAILABLE_ERROR,
|
||||
VFX_SHOWCASE,
|
||||
} from "@storyteller/ui-vfx";
|
||||
|
||||
export const PageBackgroundChange = () => {
|
||||
const subTab = useVFXStore((s) => s.subTab);
|
||||
const setSubTab = useVFXStore((s) => s.setSubTab);
|
||||
const history = useVFXStore((s) => s.history);
|
||||
const startResult = useVFXStore((s) => s.startResult);
|
||||
const failResult = useVFXStore((s) => s.failResult);
|
||||
const dismissResult = useVFXStore((s) => s.dismissResult);
|
||||
const loadFromShowcase = useVFXStore((s) => s.loadFromShowcase);
|
||||
const selectedShowcaseId = useVFXStore((s) => s.selectedShowcaseId);
|
||||
const source = useVFXStore((s) => s.source);
|
||||
const mask = useVFXStore((s) => s.mask);
|
||||
const reference = useVFXStore((s) => s.reference);
|
||||
const prompt = useVFXStore((s) => s.prompt);
|
||||
const resolution = useVFXStore((s) => s.resolution);
|
||||
|
||||
const promptBoxRef = useRef<HTMLDivElement>(null);
|
||||
const [promptBoxHeight, setPromptBoxHeight] = useState(96);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const el = promptBoxRef.current;
|
||||
if (!el || typeof ResizeObserver === "undefined") return;
|
||||
const update = () => setPromptBoxHeight(el.offsetHeight);
|
||||
update();
|
||||
const ro = new ResizeObserver(update);
|
||||
ro.observe(el);
|
||||
return () => ro.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleSubmit = useCallback(async () => {
|
||||
if (!source || !reference || isSubmitting) return;
|
||||
setIsSubmitting(true);
|
||||
const id = startResult();
|
||||
const trimmed = prompt.trim();
|
||||
const result = await submitVFXJob({
|
||||
source_video_media_token: source.mediaToken,
|
||||
reference_image_media_token: reference.mediaToken,
|
||||
prompt: trimmed.length > 0 ? trimmed : null,
|
||||
uuid_idempotency_token: newIdempotencyToken(),
|
||||
});
|
||||
setIsSubmitting(false);
|
||||
|
||||
if (!result.success) {
|
||||
const isExpected = result.error_code_str === VFX_NOT_AVAILABLE_ERROR;
|
||||
const message = isExpected
|
||||
? "Background change backend coming soon. Your inputs are saved."
|
||||
: result.error_message || "Failed to submit background change job.";
|
||||
failResult(id, message);
|
||||
if (isExpected) toast(message);
|
||||
else toast.error(message);
|
||||
}
|
||||
}, [source, reference, prompt, isSubmitting, startResult, failResult]);
|
||||
|
||||
const handleTryShowcase = useCallback(() => {
|
||||
const entry = VFX_SHOWCASE.find((e) => e.id === selectedShowcaseId);
|
||||
if (!entry) return;
|
||||
loadFromShowcase({
|
||||
prompt: entry.prompt,
|
||||
resolution: entry.resolution,
|
||||
source: {
|
||||
id: entry.source.mediaToken,
|
||||
url: entry.source.url,
|
||||
mediaToken: entry.source.mediaToken,
|
||||
},
|
||||
mask: entry.mask
|
||||
? {
|
||||
id: entry.mask.mediaToken,
|
||||
url: entry.mask.url,
|
||||
mediaToken: entry.mask.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
reference: entry.reference
|
||||
? {
|
||||
id: entry.reference.mediaToken,
|
||||
url: entry.reference.url,
|
||||
mediaToken: entry.reference.mediaToken,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
toast("Loaded showcase. Edit and Generate.");
|
||||
}, [selectedShowcaseId, loadFromShowcase]);
|
||||
|
||||
return (
|
||||
<div className="relative flex h-[calc(100vh-56px)] w-full flex-col bg-ui-background">
|
||||
<SubTabStrip activeTab={subTab} onChange={setSubTab} />
|
||||
|
||||
{subTab === "showcase" ? (
|
||||
<div
|
||||
className="min-h-0 flex-1 overflow-hidden"
|
||||
style={{ paddingBottom: Math.max(promptBoxHeight + 36, 240) }}
|
||||
>
|
||||
<VFXShowcaseView onTryThis={handleTryShowcase} />
|
||||
</div>
|
||||
) : history.length === 0 ? (
|
||||
<div
|
||||
className="relative flex flex-1 items-center justify-center px-6"
|
||||
style={{ paddingBottom: Math.max(promptBoxHeight + 36, 240) }}
|
||||
>
|
||||
<div
|
||||
aria-hidden
|
||||
className="pointer-events-none absolute inset-0 z-0"
|
||||
style={{
|
||||
maskImage:
|
||||
"radial-gradient(ellipse 60% 60% at 50% 45%, black 30%, transparent 85%)",
|
||||
WebkitMaskImage:
|
||||
"radial-gradient(ellipse 60% 60% at 50% 45%, black 30%, transparent 85%)",
|
||||
}}
|
||||
>
|
||||
<TruchetPattern
|
||||
intensity={0.8}
|
||||
className="absolute inset-0 h-full w-full"
|
||||
/>
|
||||
</div>
|
||||
<div className="relative z-10">
|
||||
<EmptyState
|
||||
title="No background changes yet"
|
||||
subtitle="Upload a source video and a reference image, then optionally add a prompt."
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
className="flex-1 overflow-y-auto"
|
||||
style={{ paddingBottom: Math.max(promptBoxHeight + 36, 240) }}
|
||||
>
|
||||
<div className="flex flex-col gap-10 px-6 pt-6">
|
||||
{history.map((r) => (
|
||||
<VFXResultCard
|
||||
key={r.id}
|
||||
data={{
|
||||
prompt: r.prompt,
|
||||
resolution: r.resolution,
|
||||
source: r.source,
|
||||
mask: r.mask,
|
||||
reference: r.reference,
|
||||
outputUrl: r.outputUrl,
|
||||
status: r.status,
|
||||
failureReason: r.failureReason,
|
||||
}}
|
||||
onDismiss={() => dismissResult(r.id)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div
|
||||
aria-hidden
|
||||
className="via-ui-background/85 pointer-events-none fixed bottom-0 left-0 right-0 z-20 h-72 bg-gradient-to-t from-ui-background to-transparent"
|
||||
/>
|
||||
|
||||
<div
|
||||
ref={promptBoxRef}
|
||||
className="pointer-events-none fixed bottom-4 left-1/2 z-30 -translate-x-1/2"
|
||||
>
|
||||
<div className="pointer-events-auto">
|
||||
<PromptBoxVFX
|
||||
onSubmit={handleSubmit}
|
||||
isSubmitting={isSubmitting}
|
||||
uploadVideo={UploadVideoMedia}
|
||||
uploadImage={UploadImageMedia}
|
||||
onError={(msg) => toast.error(msg)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
interface EmptyStateProps {
|
||||
title: string;
|
||||
subtitle: string;
|
||||
}
|
||||
|
||||
const EmptyState = ({ title, subtitle }: EmptyStateProps) => (
|
||||
<div className="flex max-w-md flex-col items-center gap-4 text-center">
|
||||
<div className="flex h-16 w-16 items-center justify-center rounded-2xl bg-base-fg/5 ring-1 ring-base-fg/10">
|
||||
<FontAwesomeIcon icon={faSparkles} className="text-2xl" />
|
||||
</div>
|
||||
<h3 className="text-2xl font-bold text-base-fg">{title}</h3>
|
||||
<p className="max-w-xs text-sm text-base-fg/60">{subtitle}</p>
|
||||
</div>
|
||||
);
|
||||
|
||||
export default PageBackgroundChange;
|
||||
@@ -0,0 +1 @@
|
||||
export { PageBackgroundChange } from "./PageBackgroundChange";
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { useMemo } from "react";
|
||||
import { useMemo } from "react";
|
||||
import {
|
||||
PageDraw as PageDrawLib,
|
||||
useSceneStore,
|
||||
@@ -97,7 +97,7 @@ const useTauriAdapter = (): PageDrawAdapter => {
|
||||
enqueueEditImage: async (req) => {
|
||||
const request: GenerateImageRequest = {
|
||||
model: req.model,
|
||||
scene_image_media_token: req.sceneImageMediaToken,
|
||||
canvas_image_media_token: req.canvasImageMediaToken,
|
||||
image_media_tokens: req.imageMediaTokens,
|
||||
prompt: req.prompt,
|
||||
enable_system_prompt:
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
import React from "react";
|
||||
import { MediaItem } from "~/pages/PageEnigma/models";
|
||||
import {
|
||||
addCharacter,
|
||||
addCharacterAnimation,
|
||||
addCharacterAudio,
|
||||
addCharacterExpression,
|
||||
addGlobalAudio,
|
||||
addObject,
|
||||
canDrop,
|
||||
currPosition,
|
||||
dragItem,
|
||||
overTimeline,
|
||||
timelineHeight,
|
||||
assetModalVisibleDuringDrag,
|
||||
reopenAfterDragSignal,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { pageHeight, pageWidth } from "~/signals";
|
||||
import { addShape } from "~/pages/PageEnigma/signals/shape";
|
||||
import { AssetType } from "~/enums";
|
||||
|
||||
class DndAsset {
|
||||
public dropId: string = "";
|
||||
public overElement: DOMRect | null = null;
|
||||
public dropOffset = 0;
|
||||
public initX = 0;
|
||||
public initY = 0;
|
||||
public notDropText = "";
|
||||
public isDragging: boolean = false;
|
||||
public dragThreshold: number = 5;
|
||||
|
||||
constructor() {
|
||||
this.onPointerMove = this.onPointerMove.bind(this);
|
||||
this.onPointerUp = this.onPointerUp.bind(this);
|
||||
}
|
||||
|
||||
onPointerDown(event: React.PointerEvent<HTMLDivElement>, item: MediaItem) {
|
||||
if (event.button === 0) {
|
||||
dragItem.value = item;
|
||||
currPosition.value = {
|
||||
currX: event.pageX,
|
||||
currY: event.pageY,
|
||||
};
|
||||
this.initX = event.pageX;
|
||||
this.initY = event.pageY;
|
||||
this.isDragging = false;
|
||||
canDrop.value = false;
|
||||
this.notDropText = "";
|
||||
assetModalVisibleDuringDrag.value = false;
|
||||
window.addEventListener("pointerup", this.onPointerUp);
|
||||
window.addEventListener("pointermove", this.onPointerMove);
|
||||
}
|
||||
}
|
||||
|
||||
endDrag() {
|
||||
if (dragItem.value) {
|
||||
dragItem.value = null;
|
||||
canDrop.value = false;
|
||||
this.overElement = null;
|
||||
overTimeline.value = false;
|
||||
this.notDropText = "";
|
||||
assetModalVisibleDuringDrag.value = reopenAfterDragSignal.value;
|
||||
}
|
||||
}
|
||||
|
||||
overCanvas(positionX: number, positionY: number) {
|
||||
if (positionY < 69) {
|
||||
return false;
|
||||
}
|
||||
if (positionY > pageHeight.value) {
|
||||
return false;
|
||||
}
|
||||
return positionX <= pageWidth.value;
|
||||
}
|
||||
|
||||
onPointerUp(event: PointerEvent) {
|
||||
window.removeEventListener("pointerup", this.onPointerUp);
|
||||
window.removeEventListener("pointermove", this.onPointerMove);
|
||||
|
||||
if (!this.isDragging) {
|
||||
// It's a click, not a drag
|
||||
assetModalVisibleDuringDrag.value = true;
|
||||
dragItem.value = null;
|
||||
currPosition.value = { currX: 0, currY: 0 };
|
||||
return;
|
||||
}
|
||||
|
||||
if (dragItem.value) {
|
||||
const positionX = event.pageX;
|
||||
const positionY = event.pageY;
|
||||
if (this.overCanvas(positionX, positionY)) {
|
||||
const mediaItem = dragItem.value;
|
||||
if (mediaItem.type === AssetType.CHARACTER) {
|
||||
addCharacter(dragItem.value);
|
||||
}
|
||||
// if (dragItem.value.type === AssetType.CAMERA) {
|
||||
// console.log("Dragged In Camera Type")
|
||||
// }
|
||||
/*
|
||||
FIXME:
|
||||
THIS IS A TEMPARARY SOLUTION TO A LONG PROBLEM WITH SKYBOXES
|
||||
UPDATE THIS WHEN UPLOADING SKYBOXES ARE FULLY IMPLEMENTED.
|
||||
THIS IS JUST TEMPARARY!!!
|
||||
*/
|
||||
if (mediaItem.type === AssetType.OBJECT || mediaItem.type === AssetType.SPLAT || mediaItem.type === AssetType.SKYBOX) {
|
||||
addObject(mediaItem);
|
||||
}
|
||||
|
||||
if (mediaItem.type === AssetType.SHAPE) {
|
||||
addShape(mediaItem);
|
||||
}
|
||||
|
||||
this.endDrag();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (canDrop.value && dragItem.value) {
|
||||
if (dragItem.value.type === AssetType.ANIMATION) {
|
||||
addCharacterAnimation({
|
||||
dragItem: dragItem.value,
|
||||
characterId: this.dropId,
|
||||
offset: this.dropOffset,
|
||||
});
|
||||
}
|
||||
if (dragItem.value.type === AssetType.EXPRESSION) {
|
||||
addCharacterExpression({
|
||||
dragItem: dragItem.value,
|
||||
characterId: this.dropId,
|
||||
offset: this.dropOffset,
|
||||
});
|
||||
}
|
||||
if (dragItem.value.type === AssetType.AUDIO) {
|
||||
addCharacterAudio({
|
||||
dragItem: dragItem.value,
|
||||
characterId: this.dropId,
|
||||
offset: this.dropOffset,
|
||||
});
|
||||
addGlobalAudio({
|
||||
dragItem: dragItem.value,
|
||||
audioId: this.dropId,
|
||||
offset: this.dropOffset,
|
||||
});
|
||||
}
|
||||
}
|
||||
this.endDrag();
|
||||
}
|
||||
|
||||
onPointerMove(event: MouseEvent) {
|
||||
if (dragItem.value) {
|
||||
event.stopPropagation();
|
||||
event.preventDefault();
|
||||
const deltaX = event.pageX - this.initX;
|
||||
const deltaY = event.pageY - this.initY;
|
||||
if (
|
||||
Math.abs(deltaX) > this.dragThreshold ||
|
||||
Math.abs(deltaY) > this.dragThreshold
|
||||
) {
|
||||
this.isDragging = true;
|
||||
}
|
||||
currPosition.value = {
|
||||
currX: this.initX + deltaX,
|
||||
currY: this.initY + deltaY,
|
||||
};
|
||||
overTimeline.value =
|
||||
event.pageY > pageHeight.value - timelineHeight.value;
|
||||
if (this.overElement) {
|
||||
const pos = this.overElement;
|
||||
const eventY = event.pageY;
|
||||
const inHeight = eventY >= pos.top && eventY <= pos.top + pos.height;
|
||||
const eventX = event.pageX;
|
||||
const inWidth = eventX >= pos.left && eventX <= pos.left + pos.width;
|
||||
|
||||
if (inHeight && inWidth) {
|
||||
return;
|
||||
}
|
||||
canDrop.value = false;
|
||||
this.dropId = "";
|
||||
this.overElement = null;
|
||||
this.notDropText = "";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dragAndDrop = new DndAsset();
|
||||
|
||||
export default dragAndDrop;
|
||||
@@ -1,12 +0,0 @@
|
||||
/**
|
||||
* This is designed to surface user customer facing messages as errors.
|
||||
* Errors shouldn't be 404 or something confusing should be
|
||||
*/
|
||||
class APIManagerResponseError extends Error {
|
||||
constructor(message?: string) {
|
||||
super(message);
|
||||
this.name = "APIManagerResponseError";
|
||||
}
|
||||
}
|
||||
|
||||
export { APIManagerResponseError };
|
||||
@@ -1,14 +0,0 @@
|
||||
/**
|
||||
* This is designed to surface user customer facing messages as errors.
|
||||
*/
|
||||
type Data = { [key: string]: any };
|
||||
class APIManagerResponseSuccess {
|
||||
public user_message: string;
|
||||
public data: Data | null;
|
||||
constructor(user_message: string = "", data: Data | null = null) {
|
||||
this.data = data;
|
||||
this.user_message = user_message;
|
||||
}
|
||||
}
|
||||
|
||||
export { APIManagerResponseSuccess };
|
||||
@@ -1,169 +0,0 @@
|
||||
import { SceneManager } from "./scene_manager_api";
|
||||
import * as THREE from "three";
|
||||
|
||||
export interface ICommand<T> {
|
||||
undo(): Promise<void>;
|
||||
redo(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface TransformData {
|
||||
object_uuid: string;
|
||||
position: THREE.Vector3;
|
||||
rotation: THREE.Euler;
|
||||
scale: THREE.Vector3;
|
||||
}
|
||||
|
||||
export interface UserData {
|
||||
object_uuid: string;
|
||||
color: string;
|
||||
visible: boolean;
|
||||
locked: boolean;
|
||||
}
|
||||
|
||||
export interface CreationData {
|
||||
object_uuid: string;
|
||||
name: string;
|
||||
position: THREE.Vector3;
|
||||
userdata: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface DeletionData {
|
||||
object_uuid: string;
|
||||
name: string;
|
||||
position: THREE.Vector3;
|
||||
userdata: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface SceneState {
|
||||
scene_objects: string[];
|
||||
scene_names: string[];
|
||||
scene_userdata: string[];
|
||||
scene_positions: string[];
|
||||
scene_rotations: string[];
|
||||
scene_scales: string[];
|
||||
}
|
||||
|
||||
export type CommandInputTypes = TransformData | CreationData | DeletionData;
|
||||
|
||||
export class CreationSceneItem implements ICommand<string> {
|
||||
private component: SceneManager;
|
||||
private creationData: CreationData;
|
||||
|
||||
public constructor(sceneManager: SceneManager, creationData: CreationData) {
|
||||
this.component = sceneManager;
|
||||
this.creationData = creationData;
|
||||
}
|
||||
|
||||
public async undo(): Promise<void> {
|
||||
this.component.delete(this.creationData.object_uuid);
|
||||
}
|
||||
|
||||
public async redo(): Promise<void> {
|
||||
const object = await this.component.create(
|
||||
this.creationData.userdata["media_id"],
|
||||
this.creationData.name,
|
||||
this.creationData.position,
|
||||
);
|
||||
object.position.copy(this.creationData.position);
|
||||
object.uuid = this.creationData.object_uuid;
|
||||
if (this.creationData.userdata["visible"] !== undefined) {
|
||||
object.visible = this.creationData.userdata["visible"];
|
||||
}
|
||||
this.component.scene.setColor(
|
||||
object.uuid,
|
||||
this.creationData.userdata["color"],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class DeletionSceneItem implements ICommand<string> {
|
||||
private component: SceneManager;
|
||||
private deletionData: DeletionData;
|
||||
|
||||
public constructor(sceneManager: SceneManager, creationData: DeletionData) {
|
||||
this.component = sceneManager;
|
||||
this.deletionData = creationData;
|
||||
}
|
||||
|
||||
public async undo(): Promise<void> {
|
||||
const object = await this.component.create(
|
||||
this.deletionData.userdata["media_id"],
|
||||
this.deletionData.name,
|
||||
this.deletionData.position,
|
||||
);
|
||||
object.uuid = this.deletionData.object_uuid;
|
||||
object.visible = this.deletionData.userdata["visible"];
|
||||
this.component.scene.setColor(
|
||||
object.uuid,
|
||||
this.deletionData.userdata["color"],
|
||||
);
|
||||
}
|
||||
|
||||
public async redo(): Promise<void> {
|
||||
this.component.delete(this.deletionData.object_uuid);
|
||||
}
|
||||
}
|
||||
|
||||
export class TransformSceneItem implements ICommand<string> {
|
||||
private component: SceneManager;
|
||||
private transformDataStart: TransformData;
|
||||
private transformDataEnd: TransformData;
|
||||
|
||||
public constructor(
|
||||
sceneManager: SceneManager,
|
||||
transformDataStart: TransformData,
|
||||
transformDataEnd: TransformData,
|
||||
) {
|
||||
this.component = sceneManager;
|
||||
this.transformDataStart = transformDataStart;
|
||||
this.transformDataEnd = transformDataEnd;
|
||||
}
|
||||
|
||||
public async undo(): Promise<void> {
|
||||
this.component.update(
|
||||
this.transformDataStart.object_uuid,
|
||||
this.transformDataStart.position,
|
||||
this.transformDataStart.rotation,
|
||||
this.transformDataStart.scale,
|
||||
);
|
||||
}
|
||||
|
||||
public async redo(): Promise<void> {
|
||||
this.component.update(
|
||||
this.transformDataEnd.object_uuid,
|
||||
this.transformDataEnd.position,
|
||||
this.transformDataEnd.rotation,
|
||||
this.transformDataEnd.scale,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class UserDataSceneItem implements ICommand<string> {
|
||||
private component: SceneManager;
|
||||
private startUserData: UserData;
|
||||
private endUserData: UserData;
|
||||
|
||||
public constructor(
|
||||
sceneManager: SceneManager,
|
||||
startUserData: UserData,
|
||||
endUserData: UserData,
|
||||
) {
|
||||
this.component = sceneManager;
|
||||
this.startUserData = startUserData;
|
||||
this.endUserData = endUserData;
|
||||
}
|
||||
|
||||
public async undo(): Promise<void> {
|
||||
this.component.scene.setColor(
|
||||
this.startUserData.object_uuid,
|
||||
this.startUserData.color,
|
||||
);
|
||||
}
|
||||
|
||||
public async redo(): Promise<void> {
|
||||
this.component.scene.setColor(
|
||||
this.endUserData.object_uuid,
|
||||
this.endUserData.color,
|
||||
);
|
||||
}
|
||||
}
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
export interface TimelineStateJson {
|
||||
version: number;
|
||||
timelineItemsJson: string;
|
||||
transformJson: string;
|
||||
animationJson: string;
|
||||
audioJson: string;
|
||||
lipsyncJson: string;
|
||||
emotionJson: string;
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
import { deepCopySerializableObjects as deepCopy } from "~/utilities";
|
||||
import { ClipUI } from "../../clips/clip_ui";
|
||||
import AudioEngine from "../Engines/audio_engine";
|
||||
import EmotionEngine from "../Engines/emotion_engine";
|
||||
import LipSyncEngine from "../Engines/lip_sync_engine";
|
||||
import { TimeLine } from "../timeline";
|
||||
import TransformEngine from "../Engines/transform_engine";
|
||||
import { TimelineStateJson } from "./EditorStateJSON/timeline_state_json";
|
||||
import { CharacterAnimationEngine } from "../Engines/CharacterAnimationEngine";
|
||||
|
||||
|
||||
export class TimelineState {
|
||||
version: number;
|
||||
timeline_items: ClipUI[] | undefined;
|
||||
transform_engine: TransformEngine | undefined;
|
||||
animation_engine: CharacterAnimationEngine | undefined;
|
||||
audio_engine: AudioEngine | undefined;
|
||||
lipsync_engine: LipSyncEngine | undefined;
|
||||
emotion_engine: EmotionEngine | undefined;
|
||||
|
||||
constructor({ editorVersion }: { editorVersion: number; }) {
|
||||
this.version = editorVersion;
|
||||
}
|
||||
|
||||
public async initializeFromTimeline(timeline: TimeLine): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
this.timeline_items = deepCopy(timeline.timeline_items);
|
||||
this.transform_engine = deepCopy(timeline.transform_engine);
|
||||
this.animation_engine = deepCopy(timeline.animation_engine);
|
||||
this.audio_engine = deepCopy(timeline.audio_engine);
|
||||
this.lipsync_engine = deepCopy(timeline.lipSync_engine);
|
||||
this.emotion_engine = deepCopy(timeline.emotion_engine);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
public async initializeFromTimelineStateJson(
|
||||
jsonObject: TimelineStateJson
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
console.log(jsonObject);
|
||||
//TODO: parse json object back into each engine object
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
public async toJSON() {
|
||||
const result: TimelineStateJson = {
|
||||
version: this.version,
|
||||
timelineItemsJson: JSON.stringify(this.timeline_items),
|
||||
transformJson: JSON.stringify(this.transform_engine),
|
||||
animationJson: JSON.stringify(this.animation_engine),
|
||||
audioJson: JSON.stringify(this.audio_engine),
|
||||
lipsyncJson: JSON.stringify(this.lipsync_engine),
|
||||
emotionJson: JSON.stringify(this.emotion_engine),
|
||||
};
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
-308
@@ -1,308 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { Retarget } from "../retargeting";
|
||||
import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
import { get_media_url } from "~/Classes/ApiHelpers";
|
||||
import { MMDLoader } from "three/examples/jsm/loaders/MMDLoader.js";
|
||||
import { ClipUI } from "../../clips/clip_ui";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
|
||||
export const START_FRAME_LABEL = "CharacterStartFrameLabel";
|
||||
|
||||
export class CharacterAnimationEngine implements Ijson {
|
||||
version: number;
|
||||
characterAnimations: Map<THREE.Object3D<THREE.Object3DEventMap>, ClipUI[]> = new Map();
|
||||
characterMixers = new Map<THREE.Object3D<THREE.Object3DEventMap>, THREE.AnimationMixer>();
|
||||
|
||||
constructor(version: number) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
toJSON(): unknown {
|
||||
// Return just an array of character object IDs
|
||||
const characterIds = Array.from(this.characterAnimations.keys()).map((character) => character.uuid);
|
||||
return { characters: characterIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* A single character model is capable of tracking multiple animations, in its own track
|
||||
* These tracks can be blended with weights or additively
|
||||
* The weight change can be done via custom interpolation logic
|
||||
*
|
||||
* @param objectUUID The UUID of the object.
|
||||
* @param clip The 3JS animation clip
|
||||
* @param clipUI The clip info
|
||||
*/
|
||||
addCharacterAnimation(character: THREE.Object3D, clip: THREE.AnimationClip, clipUI: ClipUI) {
|
||||
// If character doesn't exist, create a clip object
|
||||
// If it does exist, add a new animation to the existing character by extracting it
|
||||
// TODO: FIXME: Duplicate prevention?
|
||||
clip.name = clipUI.media_id;
|
||||
character.animations.push(clip);
|
||||
console.log("Adding animation track:")
|
||||
console.log(clip);
|
||||
|
||||
if (!this.characterAnimations.has(character)) {
|
||||
this.characterAnimations.set(character, []);
|
||||
this.#ensureMixerExists(character);
|
||||
}
|
||||
|
||||
// Add the clip info to the character map and sort the clips
|
||||
const characterClips = this.characterAnimations.get(character)!;
|
||||
characterClips.push(clipUI);
|
||||
characterClips.sort((a, b) => a.offset - b.offset);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param objectUUID The UUID of the object.
|
||||
* @param mediaId The media ID of the animation.
|
||||
* @param clipUI The clip info
|
||||
*/
|
||||
async addCharacterAnimationMedia(character: THREE.Object3D, mediaId: string, clipUI: ClipUI) {
|
||||
// If character doesn't exist, create a clip object
|
||||
// If it does exist, add a new animation to the existing character by extracting it
|
||||
// TODO: FIXME: Duplicate prevention?
|
||||
const animationTrack = await this.#load_animation(character, mediaId);
|
||||
this.addCharacterAnimation(character, animationTrack, clipUI);
|
||||
}
|
||||
|
||||
#ensureMixerExists(character: THREE.Object3D) {
|
||||
if (!this.characterMixers.has(character)) {
|
||||
this.characterMixers.set(character, new THREE.AnimationMixer(character));
|
||||
}
|
||||
}
|
||||
|
||||
getMixer(character: THREE.Object3D) {
|
||||
this.#ensureMixerExists(character);
|
||||
return this.characterMixers.get(character)!;
|
||||
}
|
||||
|
||||
#interpolateClips(character: THREE.Object3D, timestamp: number, maxTime: number) {
|
||||
const clips = this.characterAnimations.get(character);
|
||||
|
||||
const mixer = this.getMixer(character);
|
||||
|
||||
// Find the two clips we're interpolating between
|
||||
let prevClip: ClipUI | null = null;
|
||||
let nextClip: ClipUI | null = null;
|
||||
const firstClip = clips ? clips[0] : null;
|
||||
const lastClip = clips ? clips[clips.length - 1] : null;
|
||||
|
||||
// If we're before the first clip, interpolate it in from base pose
|
||||
// TODO: Replace with starting frame
|
||||
if (firstClip && firstClip.offset > timestamp) {
|
||||
prevClip = null;
|
||||
nextClip = firstClip;
|
||||
} else if (lastClip && (lastClip.offset + lastClip.length) < timestamp) { // After last clip, interpolate it out
|
||||
prevClip = lastClip;
|
||||
nextClip = null;
|
||||
} else if (clips) {
|
||||
let prevClipIndex = 0;
|
||||
|
||||
// Loop until the next clip is after the timestamp
|
||||
while (prevClipIndex < clips.length - 1 && clips[prevClipIndex + 1].offset < timestamp) {
|
||||
prevClipIndex++;
|
||||
}
|
||||
|
||||
prevClip = clips[prevClipIndex];
|
||||
nextClip = clips[prevClipIndex + 1];
|
||||
}
|
||||
|
||||
// If the clips exist, fetch their actions. If not, evaluation is either at the start or the end.
|
||||
// If evalution is on either sides, check the start/end frame tracks
|
||||
// Otherwise just mark it null
|
||||
const prevAnimationTrack = prevClip ?
|
||||
this.getCharacterAnimationTrack(character, prevClip.media_id) :
|
||||
this.getCharacterAnimationTrack(character, START_FRAME_LABEL);
|
||||
|
||||
// TODO: Replace with ending frame
|
||||
const nextAnimationTrack = nextClip ? this.getCharacterAnimationTrack(character, nextClip.media_id) : null;
|
||||
|
||||
const prevAction = prevAnimationTrack ? mixer.clipAction(prevAnimationTrack) : null;
|
||||
const nextAction = nextAnimationTrack ? mixer.clipAction(nextAnimationTrack) : null;
|
||||
|
||||
// Calculate the progress of timestamp from end of prev action to start of next action
|
||||
const left = (prevClip?.offset ?? 0) + (prevClip?.length ?? 0);
|
||||
const right = nextClip?.offset ?? maxTime;
|
||||
|
||||
// Simple Linear interpolation for now
|
||||
// TODO: Take an interpolation dependency, or better yet, write a transition engine
|
||||
const progress = (timestamp - left) / (right - left);
|
||||
prevAction?.setEffectiveWeight(1 - progress);
|
||||
nextAction?.setEffectiveWeight(progress);
|
||||
|
||||
// Make sure we hold that last frame for the previous action
|
||||
if (prevAction) {
|
||||
prevAction.clampWhenFinished = true;
|
||||
}
|
||||
|
||||
// The next action should stay at the first frame
|
||||
if (nextAction) {
|
||||
nextAction.paused = true;
|
||||
}
|
||||
|
||||
// Necessary to ensure the actions are active - the default is inactive, mixer won't do anything
|
||||
prevAction?.play();
|
||||
nextAction?.play();
|
||||
|
||||
// The clip time would still be relative to the previous clip
|
||||
const clipTime = timestamp - (prevClip?.offset ?? 0);
|
||||
mixer.setTime(clipTime / 1000);
|
||||
}
|
||||
|
||||
evaluateCharacter(character: THREE.Object3D, timestamp: number, maxTime: number) {
|
||||
const mixer = this.getMixer(character);
|
||||
const clips = this.characterAnimations.get(character);
|
||||
|
||||
// Find the clip we're in right now
|
||||
const currentClip = clips?.find((clip) => {
|
||||
return clip.offset <= timestamp && clip.offset + clip.length >= timestamp;
|
||||
});
|
||||
|
||||
// If timestamp not in any clip, do nothing.
|
||||
// If timestamp in clip, set mixer to the timestamp inside the clip
|
||||
if (!currentClip) {
|
||||
// Let the interpolation function handle this actions
|
||||
this.#interpolateClips(character, timestamp, maxTime)
|
||||
return;
|
||||
}
|
||||
|
||||
const clipTime = timestamp - currentClip.offset;
|
||||
const animationTrack = this.getCharacterAnimationTrack(character, currentClip.media_id);
|
||||
|
||||
// NOTE: This shouldn't really happen unless the UI was desynced at some point from the engine
|
||||
if (!animationTrack) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Play only this action
|
||||
const animationAction = mixer.clipAction(animationTrack);
|
||||
|
||||
// Since it's the only clip in this timestamp, make it full weight and make sure it's not paused (from interpolation or otherwise)
|
||||
animationAction.setEffectiveWeight(1);
|
||||
animationAction.paused = false;
|
||||
|
||||
// Necessary to ensure the actions are active - the default is inactive, mixer won't do anything
|
||||
animationAction.play();
|
||||
|
||||
mixer.setTime(clipTime / 1000);
|
||||
}
|
||||
|
||||
getCharacterAnimationTrack(character: THREE.Object3D, mediaId: string) {
|
||||
return character.animations.find((clip) => clip.name === mediaId);
|
||||
}
|
||||
|
||||
/** Evaluate all character animations at a given timestamp (milliseconds) */
|
||||
evaluate(timestamp: number, maxTime: number) {
|
||||
this.characterMixers.forEach((_, character) => {
|
||||
this.evaluateCharacter(character, timestamp, maxTime);
|
||||
})
|
||||
}
|
||||
|
||||
stopCharacter(character: THREE.Object3D) {
|
||||
const mixer = this.characterMixers.get(character);
|
||||
console.debug("Stopping all actions for character", character);
|
||||
console.debug(mixer);
|
||||
mixer?.stopAllAction();
|
||||
console.debug(mixer);
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.characterMixers.forEach((mixer) => {
|
||||
mixer.stopAllAction();
|
||||
});
|
||||
}
|
||||
|
||||
async #load_animation(character: THREE.Object3D, mediaId: string): Promise<THREE.AnimationClip> {
|
||||
// Get the file URL and extract the (first) animation track from it
|
||||
// TODO: Support for multiple animations in a single file?
|
||||
const url = await get_media_url(mediaId);
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
if (url.includes(".glb")) {
|
||||
const glbLoader = new GLTFLoader();
|
||||
|
||||
glbLoader.load(url, (gltf) => {
|
||||
const animationClip = gltf.animations[0];
|
||||
resolve(animationClip);
|
||||
});
|
||||
} else if (url.includes(".fbx")) {
|
||||
const fbxLoader = new FBXLoader();
|
||||
fbxLoader.load(url, (fbx) => {
|
||||
const animationClip_1 = fbx.animations[0];
|
||||
animationClip_1.tracks.forEach((track) => {
|
||||
const retarget = new Retarget();
|
||||
const retarget_value = retarget.retarget(track.name);
|
||||
track.name = retarget_value.bone;
|
||||
console.log(track);
|
||||
if (retarget_value.is_special) {
|
||||
// TODO: Revisit special properties later
|
||||
// this.special_properties.push(retarget_value);
|
||||
}
|
||||
});
|
||||
resolve(animationClip_1);
|
||||
});
|
||||
} else if (url.includes(".vmd")) {
|
||||
const mmdLoader = new MMDLoader();
|
||||
mmdLoader.loadAnimation(url, character as THREE.SkinnedMesh, (mmd) => {
|
||||
mmd.name = url;
|
||||
resolve(mmd as THREE.AnimationClip);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
removeCharacter(character: THREE.Object3D) {
|
||||
console.debug("Removing character from animation engine", character);
|
||||
this.characterAnimations.delete(character);
|
||||
this.characterMixers.delete(character);
|
||||
}
|
||||
|
||||
removeAnimation(character: THREE.Object3D, clip: ClipUI) {
|
||||
console.debug("Removing animation from character", character, clip);
|
||||
const clips = this.characterAnimations.get(character)!;
|
||||
|
||||
const clipIndex = clips.findIndex((c) => c.clip_uuid === clip.clip_uuid);
|
||||
if (clipIndex < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove the clip from character's tracked clips
|
||||
clips.splice(clipIndex, 1);
|
||||
|
||||
// Stop all character animations to cleanly reset
|
||||
this.stopCharacter(character);
|
||||
}
|
||||
|
||||
clearStartFrame(character: THREE.Object3D, resetTime?: number, maxTime = 1) {
|
||||
// Stop all character animations
|
||||
this.stopCharacter(character);
|
||||
|
||||
// Check the animations and remove any labelled START_FRAME_LABEL
|
||||
character.animations = character.animations.filter((clip) => clip.name !== START_FRAME_LABEL);
|
||||
|
||||
// If resetTime is passed, evaluate the character to that time
|
||||
if (resetTime) {
|
||||
this.evaluateCharacter(character, resetTime, maxTime);
|
||||
}
|
||||
}
|
||||
|
||||
/** resetTime: The time to reset the character to. This should ideally be the current timeline time. */
|
||||
async createStartFrameAnimation(character: THREE.Object3D, poseHelper: any, url: string, resetTime: number = 0, maxTime: number = 1) {
|
||||
//// Clear any existing start frame
|
||||
//// This will also reset the animation to default pose
|
||||
//this.clearStartFrame(character);
|
||||
|
||||
//// We must create the track data now after resetting to default
|
||||
//const poseData = await poseHelper.extractPoseData(url);
|
||||
//const tracks = poseHelper.inflatePoseDataToTracks(character, poseData);
|
||||
|
||||
//// Create the animation track from the rotation values and hip position
|
||||
//// then add it to the character
|
||||
//const startFrameClip = new THREE.AnimationClip(START_FRAME_LABEL, 0, tracks);
|
||||
//character.animations.push(startFrameClip);
|
||||
|
||||
//// Reset the character evaluation to what was passed in
|
||||
//this.evaluateCharacter(character, resetTime, maxTime);
|
||||
}
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
// A mapping between the model bones names and the Mediapipe output index
|
||||
export const MixamoPoseMap: Record<string, number> = {
|
||||
"mixamorigLeftArm": 11,
|
||||
"mixamorigRightArm": 12,
|
||||
"mixamorigLeftForeArm": 13,
|
||||
"mixamorigRightForeArm": 14,
|
||||
"mixamorigLeftHand": 15,
|
||||
"mixamorigRightHand": 16,
|
||||
"mixamorigLeftUpLeg": 23,
|
||||
"mixamorigRightUpLeg": 24,
|
||||
"mixamorigLeftLeg": 25,
|
||||
"mixamorigRightLeg": 26,
|
||||
"mixamorigLeftFoot": 27,
|
||||
"mixamorigRightFoot": 28,
|
||||
"mixamorigLeftHandThumb2": 21,
|
||||
"mixamorigRightHandThumb2": 22,
|
||||
}
|
||||
|
||||
export const MixamoInterpolationBoneNames = {
|
||||
Spine: "mixamorigSpine",
|
||||
Spine1: "mixamorigSpine1",
|
||||
Spine2: "mixamorigSpine2",
|
||||
Neck: "mixamorigNeck",
|
||||
LeftShoulderBlade: "mixamorigLeftShoulder",
|
||||
DefLeftUpperArm: "DEF-upper_armL001",
|
||||
DefLeftForeArm: "DEF-forearmL001",
|
||||
RightShoulderBlade: "mixamorigRightShoulder",
|
||||
DefRightUpperArm: "DEF-upper_armR001",
|
||||
DefRightForeArm: "DEF-forearmR001",
|
||||
DefLeftThigh: "DEF-thighL001",
|
||||
DefRightThigh: "DEF-thighR001",
|
||||
}
|
||||
|
||||
export const PoseMap = {
|
||||
NOSE: 0,
|
||||
LEFT_SHOULDER: 11,
|
||||
RIGHT_SHOULDER: 12,
|
||||
LEFT_ELBOW: 13,
|
||||
RIGHT_ELBOW: 14,
|
||||
LEFT_WRIST: 15,
|
||||
RIGHT_WRIST: 16,
|
||||
LEFT_HIP: 23,
|
||||
RIGHT_HIP: 24,
|
||||
LEFT_KNEE: 25,
|
||||
RIGHT_KNEE: 26,
|
||||
LEFT_ANKLE: 27,
|
||||
RIGHT_ANKLE: 28,
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { AnimationClip } from "../../clips/animation_clip";
|
||||
|
||||
export class AnimationEngine {
|
||||
clips: { [key: string]: AnimationClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.clips = {};
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
load_object(object_uuid: string, media_id: string, clip_name: string) {
|
||||
console.debug("load_object", object_uuid, media_id, clip_name);
|
||||
this.clips[object_uuid + media_id] = new AnimationClip(
|
||||
this.version,
|
||||
media_id,
|
||||
"glb",
|
||||
object_uuid,
|
||||
1.0,
|
||||
1.0,
|
||||
clip_name,
|
||||
0.0,
|
||||
0.0,
|
||||
);
|
||||
//this.clips[object_uuid + media_id]._load_animation();
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
import { AudioClip } from "../../clips/audio_clip";
|
||||
|
||||
class AudioEngine {
|
||||
clips: { [key: string]: AudioClip } = {};
|
||||
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
|
||||
version: number;
|
||||
playing: string[];
|
||||
last_frame: number;
|
||||
|
||||
constructor() {
|
||||
this.clips = {};
|
||||
this.playing = [];
|
||||
this.audio_sources = {};
|
||||
this.version = 1.0;
|
||||
this.last_frame = 0;
|
||||
}
|
||||
|
||||
// loads clips into the engine to cache
|
||||
loadClip(audio_media_id: string) {
|
||||
if (this.clips[audio_media_id] != null) {
|
||||
return;
|
||||
}
|
||||
this.clips[audio_media_id] = new AudioClip(
|
||||
this.version,
|
||||
audio_media_id,
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
|
||||
// plays from the timeline.
|
||||
playClip(audio_media_id: string) {
|
||||
if (this.clips[audio_media_id] == null) {
|
||||
this.loadClip(audio_media_id);
|
||||
}
|
||||
}
|
||||
|
||||
async step(audio_media_id: string, frame: number, offset: number) {
|
||||
const frameBuffer = 12;
|
||||
|
||||
const pos = frame - offset;
|
||||
|
||||
let doPlay = true;
|
||||
if (
|
||||
Math.abs(this.last_frame - frame) < frameBuffer - 1 &&
|
||||
this.last_frame !== 0
|
||||
) {
|
||||
doPlay = false;
|
||||
}
|
||||
if (pos <= 1) {
|
||||
doPlay = true;
|
||||
}
|
||||
|
||||
const clip = this.clips[audio_media_id];
|
||||
|
||||
if (doPlay && clip.audio_data?.audioContext) {
|
||||
const startTime = pos / 60.0;
|
||||
const endTime = frameBuffer / 60.0;
|
||||
clip.audio_data.source =
|
||||
clip.audio_data.audioContext.createBufferSource();
|
||||
clip.audio_data.source.buffer = clip.audio_data.audioBuffer;
|
||||
clip.audio_data.source.connect(clip.audio_data.audioContext.destination);
|
||||
clip.audio_data.source.start(0, startTime, endTime);
|
||||
this.last_frame = frame;
|
||||
}
|
||||
}
|
||||
|
||||
// stops the clips
|
||||
stopClip(audio_media_id: string) {
|
||||
const clip = this.clips[audio_media_id];
|
||||
if (clip.audio_data?.source) {
|
||||
clip.audio_data.source.stop();
|
||||
clip.audio_data.source.disconnect();
|
||||
} else {
|
||||
console.warn(
|
||||
`AudioManager: AudioClip with id "${audio_media_id}" not found.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default AudioEngine;
|
||||
@@ -1,24 +0,0 @@
|
||||
import { EmotionClip } from "../../clips/emotion_clip";
|
||||
|
||||
class EmotionEngine {
|
||||
clips: { [key: string]: EmotionClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.clips = {};
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
// loads clips into the engine to cache
|
||||
loadClip(object_uuid: string, csv_media_id: string) {
|
||||
if (this.clips[object_uuid + csv_media_id] != null) {
|
||||
return;
|
||||
}
|
||||
this.clips[object_uuid + csv_media_id] = new EmotionClip(
|
||||
this.version,
|
||||
csv_media_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export default EmotionEngine;
|
||||
@@ -1,22 +0,0 @@
|
||||
import { LipSyncClip } from "../../clips/lipsync_clip";
|
||||
|
||||
// needs to be apart of the editor window.
|
||||
export default class LipSyncEngine {
|
||||
clips: { [key: string]: LipSyncClip } = {};
|
||||
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
|
||||
version: number;
|
||||
|
||||
constructor() {
|
||||
this.clips = {};
|
||||
this.audio_sources = {};
|
||||
this.version = 1.0;
|
||||
}
|
||||
|
||||
load_object(object_uuid: string, audio_media_id: string) {
|
||||
this.clips[object_uuid + audio_media_id] = new LipSyncClip(
|
||||
this.version,
|
||||
audio_media_id,
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
import { TransformClip } from "../../clips/transform_clip";
|
||||
import * as THREE from "three";
|
||||
|
||||
class TransformEngine {
|
||||
clips: { [key: string]: TransformClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.version = version;
|
||||
this.clips = {};
|
||||
}
|
||||
|
||||
loadObject(object_uuid: string, clip_length: number = 2) {
|
||||
this.clips[object_uuid] = new TransformClip(
|
||||
this.version,
|
||||
object_uuid,
|
||||
clip_length,
|
||||
object_uuid,
|
||||
); // replace the last object_uuid with the media ID when its ready.
|
||||
return this.clips[object_uuid];
|
||||
}
|
||||
|
||||
addFrame(
|
||||
object_uuid: string,
|
||||
clip_length: number = 2,
|
||||
position: THREE.Vector3,
|
||||
rotation: THREE.Vector3,
|
||||
scale: THREE.Vector3,
|
||||
offset: number,
|
||||
keyframe_uuid: string,
|
||||
): boolean {
|
||||
if (this.clips[object_uuid] == null) {
|
||||
this.clips[object_uuid] = new TransformClip(
|
||||
this.version,
|
||||
object_uuid,
|
||||
clip_length,
|
||||
);
|
||||
this.clips[object_uuid].add_frame(
|
||||
position,
|
||||
rotation,
|
||||
scale,
|
||||
offset,
|
||||
keyframe_uuid,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
this.clips[object_uuid].add_frame(
|
||||
position,
|
||||
rotation,
|
||||
scale,
|
||||
offset,
|
||||
keyframe_uuid,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export default TransformEngine;
|
||||
@@ -1,32 +0,0 @@
|
||||
import { getArtStyle } from "~/enums/ArtStyle";
|
||||
import {
|
||||
cameraAspectRatio,
|
||||
cinematic,
|
||||
enginePreProcessing,
|
||||
faceDetail,
|
||||
globalIPAMediaToken,
|
||||
lipSync,
|
||||
styleStrength,
|
||||
upscale,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
|
||||
import Editor from "./editor";
|
||||
|
||||
export const getSceneGenerationMetaData = (
|
||||
editorEngine: Editor,
|
||||
): SceneGenerationMetaData => {
|
||||
// when this is called, editor engine is guarunteed by it's caller
|
||||
return {
|
||||
positivePrompt: editorEngine.positive_prompt,
|
||||
negativePrompt: editorEngine.negative_prompt,
|
||||
artisticStyle: getArtStyle(editorEngine.art_style.toString()),
|
||||
cameraAspectRatio: cameraAspectRatio.value,
|
||||
globalIPAMediaToken: globalIPAMediaToken.value || undefined,
|
||||
upscale: upscale.value,
|
||||
faceDetail: faceDetail.value,
|
||||
styleStrength: styleStrength.value,
|
||||
lipSync: lipSync.value,
|
||||
cinematic: cinematic.value,
|
||||
enginePreProcessing: enginePreProcessing.value,
|
||||
};
|
||||
};
|
||||
@@ -1,64 +0,0 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
export const uploadNewScene = async (file: File, sceneTitle: string) => {
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const endpoint = `${apiSchemeAndHost}/v1/media_files/upload/new_scene`;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("uuid_idempotency_token", uuidv4());
|
||||
formData.append("file", file);
|
||||
formData.append("maybe_title", sceneTitle);
|
||||
formData.append("maybe_visibility", "public");
|
||||
formData.append("engine_category", "scene");
|
||||
|
||||
return await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
credentials: "include",
|
||||
body: formData,
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((res) => {
|
||||
if (res && res.success) {
|
||||
return res;
|
||||
} else {
|
||||
return { success: false };
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
return { success: false };
|
||||
});
|
||||
};
|
||||
|
||||
export const updateExistingScene = async (file: File, sceneToken: string) => {
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const endpoint = `${apiSchemeAndHost}/v1/media_files/upload/saved_scene/${sceneToken}`;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("uuid_idempotency_token", uuidv4());
|
||||
formData.append("file", file);
|
||||
|
||||
return await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
credentials: "include",
|
||||
body: formData,
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((res) => {
|
||||
if (res && res.success) {
|
||||
return res;
|
||||
} else {
|
||||
return { success: false };
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
return { success: false };
|
||||
});
|
||||
};
|
||||
@@ -1,460 +0,0 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import * as THREE from "three";
|
||||
import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js";
|
||||
import { signalScene, startPollingActiveJobs } from "~/signals";
|
||||
import { updateExistingScene, uploadNewScene } from "./api_fetchers";
|
||||
import environmentVariables from "~/Classes/EnvironmentVariables";
|
||||
import { GetCdnOrigin } from "~/api/GetCdnOrigin";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
interface StylizeVideoParams {
|
||||
media_token: string;
|
||||
style: ArtStyle;
|
||||
positive_prompt: string;
|
||||
negative_prompt: string;
|
||||
visibility: Visibility;
|
||||
use_face_detailer?: boolean;
|
||||
use_upscaler?: boolean;
|
||||
use_strength?: number;
|
||||
use_lipsync?: boolean;
|
||||
use_cinematic?: boolean;
|
||||
use_global_ipa_media_token?: string | null;
|
||||
input_depth_file?: string | null;
|
||||
input_normal_file?: string | null;
|
||||
input_outline_file?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storyteller Studio API Manager
|
||||
* The source of truth of all these media items is the database in the cloud
|
||||
*/
|
||||
export class MediaFile {
|
||||
public_bucket_path: string;
|
||||
media_type: string;
|
||||
media_token: string;
|
||||
|
||||
constructor(
|
||||
public_bucket_path: string,
|
||||
media_type: string,
|
||||
media_token: string,
|
||||
) {
|
||||
this.media_token = media_token;
|
||||
this.public_bucket_path = public_bucket_path;
|
||||
this.media_type = media_type;
|
||||
}
|
||||
}
|
||||
|
||||
export enum ArtStyle {
|
||||
Anime2_5D = "anime_2_5d",
|
||||
Anime2DFlat = "anime_2d_flat",
|
||||
Cartoon3D = "cartoon_3d",
|
||||
ComicBook = "comic_book",
|
||||
AnimeGhibli = "anime_ghibli",
|
||||
InkPunk = "ink_punk",
|
||||
InkSplash = "ink_splash",
|
||||
InkBWStyle = "ink_bw_style",
|
||||
JojoStyle = "jojo_style",
|
||||
PaperOrigami = "paper_origami",
|
||||
PixelArt = "pixel_art",
|
||||
PopArt = "pop_art",
|
||||
Realistic1 = "realistic_1",
|
||||
Realistic2 = "realistic_2",
|
||||
AnimeRetroNeon = "anime_retro_neon",
|
||||
AnimeStandard = "anime_standard",
|
||||
HRGiger = "hr_giger",
|
||||
Simpsons = "simpsons",
|
||||
Carnage = "carnage",
|
||||
PastelCuteAnime = "pastel_cute_anime",
|
||||
Bloom = "bloom_lighting",
|
||||
Horror2_5D = "25d_horror",
|
||||
Creepy = "creepy",
|
||||
CreepyVHS = "creepy_vhs",
|
||||
TrailCamFootage = "trail_cam_footage",
|
||||
OldBWMovie = "old_black_white_movie",
|
||||
HorrorNoirBW = "horror_noir_black_white",
|
||||
TechnoNoirBW = "techno_noir_black_white",
|
||||
BW20s = "black_white_20s",
|
||||
AnimeCyberpunk = "cyberpunk_anime",
|
||||
Dragonball = "dragonball",
|
||||
RealisticMatrix = "realistic_matrix",
|
||||
RealisticCyberpunk = "realistic_cyberpunk",
|
||||
}
|
||||
|
||||
export enum Visibility {
|
||||
Public = "public",
|
||||
Hidden = "hidden",
|
||||
Private = "private",
|
||||
}
|
||||
|
||||
/**
|
||||
* This is designed to surface user customer facing messages as errors.
|
||||
*/
|
||||
// type Data = { [key: string]: any };
|
||||
// class APIManagerResponseSuccess {
|
||||
// public user_message: string;
|
||||
// public data: Data | null;
|
||||
// constructor(user_message: string = "", data: Data | null = null) {
|
||||
// this.data = data;
|
||||
// this.user_message = user_message;
|
||||
// }
|
||||
// }
|
||||
|
||||
/**
|
||||
* This is designed to surface user customer facing messages as errors.
|
||||
* Errors shouldn't be 404 or something confusing should be
|
||||
*/
|
||||
class APIManagerResponseError extends Error {
|
||||
constructor(message?: string) {
|
||||
super(message);
|
||||
this.name = "APIManagerResponseError";
|
||||
}
|
||||
}
|
||||
|
||||
export class APIManager {
|
||||
|
||||
constructor() {
|
||||
// NB(bt,2025-07-06): Cleaning this up. The code was duplicated a half dozen times.
|
||||
// I think all of the code in this file is legacy, but to be safe I'm continuing to update it.
|
||||
}
|
||||
|
||||
protected getApiSchemeAndHost() {
|
||||
return StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param scene The 3JS Scene we want to save.
|
||||
* @param scene_name The Scene name we want to display
|
||||
* @param scene_media_file_token If null we will we will create a new save or copy the scene, if provided we will overwrite the scene.
|
||||
* @returns APIManagerResponseMessage
|
||||
*/
|
||||
public async saveSceneState({
|
||||
saveJson,
|
||||
sceneTitle,
|
||||
sceneToken,
|
||||
sceneThumbnail,
|
||||
}: {
|
||||
saveJson: string;
|
||||
sceneTitle: string;
|
||||
sceneToken?: string;
|
||||
sceneThumbnail: Blob | undefined;
|
||||
}): Promise<string> {
|
||||
const file = new File([saveJson], `${sceneTitle}.glb`, {
|
||||
type: "application/json",
|
||||
});
|
||||
|
||||
const uploadSceneResponse = sceneToken
|
||||
? await updateExistingScene(file, sceneToken)
|
||||
: await uploadNewScene(file, sceneTitle);
|
||||
|
||||
// failed so catch this.
|
||||
if (uploadSceneResponse["success"] == false) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (sceneThumbnail) {
|
||||
const image_resp = await this.uploadMediaSceneThumbnail(
|
||||
sceneThumbnail,
|
||||
"render.png",
|
||||
);
|
||||
|
||||
if (image_resp["success"] == false) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (image_resp["media_file_token"]) {
|
||||
const image_token = image_resp["media_file_token"];
|
||||
const endpoint = `${this.getApiSchemeAndHost()}/v1/media_files/cover_image/${uploadSceneResponse["media_file_token"]}`;
|
||||
|
||||
await fetch(endpoint, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ cover_image_media_file_token: image_token }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(uploadSceneResponse);
|
||||
return uploadSceneResponse["media_file_token"];
|
||||
}
|
||||
|
||||
public async loadSceneState(
|
||||
scene_media_file_token: string | null,
|
||||
): Promise<any> {
|
||||
const url = `${this.getApiSchemeAndHost()}/v1/media_files/file/${scene_media_file_token}`;
|
||||
const response = await fetch(url);
|
||||
if (response.status > 200) {
|
||||
throw new APIManagerResponseError("Failed to load scene");
|
||||
}
|
||||
|
||||
const json = await response.json();
|
||||
if (json && json.media_file) {
|
||||
if (json.media_file.maybe_title === null) {
|
||||
console.warn(`Scene /w Token: ${scene_media_file_token} has no title`);
|
||||
}
|
||||
signalScene({
|
||||
title: json.media_file.maybe_title || "Untitled Scene",
|
||||
token: scene_media_file_token || undefined,
|
||||
ownerToken: json.media_file.maybe_creator_user.user_token,
|
||||
isModified: false,
|
||||
});
|
||||
}
|
||||
const bucket_path = json["media_file"]["public_bucket_path"];
|
||||
const media_base_url = GetCdnOrigin();
|
||||
//const media_url = `${media_base_url}/vocodes-public${bucket_path}`; // gets you a bucket path
|
||||
const media_url = `${media_base_url}${bucket_path}`; // gets you a bucket path
|
||||
|
||||
const file_response = await fetch(media_url);
|
||||
|
||||
if (!file_response.ok) {
|
||||
throw new APIManagerResponseError("Failed to download file");
|
||||
}
|
||||
// Convert the response from a blob to json text
|
||||
const blob = await file_response.blob();
|
||||
const json_result: string = await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => resolve(JSON.parse(reader.result as string));
|
||||
reader.onerror = reject;
|
||||
reader.readAsText(blob);
|
||||
});
|
||||
|
||||
return json_result;
|
||||
}
|
||||
|
||||
/**
|
||||
* This recieves the bucket path of a media file
|
||||
* @param media_file_token
|
||||
* @returns
|
||||
*/
|
||||
public async getMediaFile(media_file_token: string): Promise<string> {
|
||||
const url = `${this.getApiSchemeAndHost()}/v1/media_files/file/${media_file_token}`;
|
||||
const response = await fetch(url);
|
||||
const json = await JSON.parse(await response.text());
|
||||
const bucketPath = json["media_file"]["public_bucket_path"];
|
||||
const media_base_url = GetCdnOrigin();
|
||||
const media_url = `${media_base_url}${bucketPath}`; // gets you a bucket path
|
||||
return media_url;
|
||||
}
|
||||
|
||||
/**
|
||||
This will save the scene to keep ids positions.
|
||||
It will also give the file a name which will be a uuidv4()
|
||||
@param scene The 3JS Scene we want to make a file to be uploaded as multipart form.
|
||||
*/
|
||||
private async gltfExport(scene: THREE.Scene): Promise<File> {
|
||||
const gltfExporter = new GLTFExporter();
|
||||
const uuid = uuidv4();
|
||||
const result = await gltfExporter.parseAsync(scene);
|
||||
const file = new File([JSON.stringify(result)], `${uuid}.glb`, {
|
||||
type: "application/json",
|
||||
});
|
||||
return file;
|
||||
}
|
||||
|
||||
public async uploadMediaSceneThumbnail(blob: Blob | File, fileName: string) {
|
||||
const url = `${this.getApiSchemeAndHost()}/v1/media_files/upload/image`;
|
||||
const uuid = uuidv4();
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("uuid_idempotency_token", uuid);
|
||||
formData.append("is_intermediate_system_file", "true");
|
||||
formData.append("maybe_title", "Screenshot");
|
||||
formData.append("file", blob, fileName);
|
||||
//formData.append("source", "file");
|
||||
//formData.append("type", "video");
|
||||
//formData.append("source", "file");
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: formData,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new APIManagerResponseError("Upload Media Failed to send file");
|
||||
} else {
|
||||
const json_data = await response.json();
|
||||
//console.log(`uploadMedia: ${JSON.stringify(json_data)}`);
|
||||
return json_data;
|
||||
}
|
||||
}
|
||||
|
||||
public async uploadMedia({
|
||||
blob,
|
||||
fileName,
|
||||
title,
|
||||
styleName,
|
||||
maybe_scene_source_media_file_token,
|
||||
}: {
|
||||
blob: Blob;
|
||||
fileName: string;
|
||||
title: string;
|
||||
styleName?: string;
|
||||
maybe_scene_source_media_file_token: string | undefined;
|
||||
}) {
|
||||
// Promise<APIManagerResponseSuccess>
|
||||
//TODO: UPDATE ENDPOINT!!!!
|
||||
const url = `${this.getApiSchemeAndHost()}/v1/media_files/upload/new_video`;
|
||||
const uuid = uuidv4();
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("uuid_idempotency_token", uuid);
|
||||
|
||||
formData.append("file", blob, fileName);
|
||||
// formData.append("source", "file");
|
||||
// formData.append("type", "video");
|
||||
if (styleName) formData.append("maybe_style_name", styleName);
|
||||
formData.append("maybe_title", title);
|
||||
|
||||
// This signals to the backend to hide the video from view
|
||||
formData.append("is_intermediate_system_file", "true");
|
||||
if (maybe_scene_source_media_file_token !== undefined) {
|
||||
formData.append(
|
||||
"maybe_scene_source_media_file_token",
|
||||
maybe_scene_source_media_file_token,
|
||||
);
|
||||
}
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new APIManagerResponseError("Upload Media Failed to send file");
|
||||
} else {
|
||||
const json_data = await response.json();
|
||||
//console.log(`uploadMedia: ${JSON.stringify(json_data)}`);
|
||||
return json_data;
|
||||
}
|
||||
}
|
||||
|
||||
public async uploadMediaFrameGeneration(
|
||||
blob: Blob | File,
|
||||
fileName: string,
|
||||
style: string = "comic_book",
|
||||
positive_prompt: string,
|
||||
negative_prompt: string,
|
||||
): Promise<string> {
|
||||
const url = `${environmentVariables.values.FUNNEL_API}/preview/`;
|
||||
|
||||
const payload = {
|
||||
style: style,
|
||||
positive_prompt: positive_prompt,
|
||||
negative_prompt: negative_prompt,
|
||||
};
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("input_file", blob, fileName);
|
||||
formData.append("request", JSON.stringify(payload));
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new APIManagerResponseError("Upload Media Failed to send file");
|
||||
} else {
|
||||
return URL.createObjectURL(await response.blob());
|
||||
}
|
||||
}
|
||||
|
||||
public async getMediaBatch(media_tokens: string[]): Promise<MediaFile[]> {
|
||||
const tokens = media_tokens;
|
||||
const url = new URL(`${this.getApiSchemeAndHost()}/v1/media_files/batch`);
|
||||
tokens.forEach((token) => url.searchParams.append("tokens", token));
|
||||
const result = await fetch(url)
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
const result = data["media_files"].map((element: any) => {
|
||||
return new MediaFile(
|
||||
element["public_bucket_path"],
|
||||
element["media_type"],
|
||||
element["token"],
|
||||
);
|
||||
});
|
||||
return result;
|
||||
})
|
||||
.catch((error) => console.error("Error:", error));
|
||||
return result;
|
||||
}
|
||||
|
||||
public async stylizeVideo({
|
||||
media_token,
|
||||
style,
|
||||
positive_prompt,
|
||||
negative_prompt,
|
||||
visibility,
|
||||
use_face_detailer = false,
|
||||
use_upscaler = false,
|
||||
use_strength = 1.0,
|
||||
use_lipsync = false,
|
||||
use_cinematic = false,
|
||||
use_global_ipa_media_token = null,
|
||||
input_depth_file = null,
|
||||
input_normal_file = null,
|
||||
input_outline_file = null,
|
||||
}: StylizeVideoParams) {
|
||||
const uuid = uuidv4();
|
||||
|
||||
const data = {
|
||||
uuid_idempotency_token: uuid,
|
||||
style: style,
|
||||
input_file: media_token,
|
||||
prompt: positive_prompt,
|
||||
negative_prompt: negative_prompt,
|
||||
trim_start_millis: 0,
|
||||
trim_end_millis: 7000,
|
||||
enable_lipsync: use_lipsync,
|
||||
creator_set_visibility: visibility,
|
||||
use_face_detailer: use_face_detailer,
|
||||
use_upscaler: use_upscaler,
|
||||
use_strength: use_strength,
|
||||
use_lipsync: use_lipsync,
|
||||
global_ipa_media_token: use_global_ipa_media_token,
|
||||
use_cinematic: use_cinematic,
|
||||
input_depth_file: input_depth_file,
|
||||
input_normal_file: input_normal_file,
|
||||
input_outline_file: input_outline_file,
|
||||
};
|
||||
|
||||
const json_data = JSON.stringify(data);
|
||||
|
||||
const response = await fetch(
|
||||
`${this.getApiSchemeAndHost()}/v1/workflows/enqueue_studio`,
|
||||
{
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: json_data,
|
||||
},
|
||||
);
|
||||
|
||||
startPollingActiveJobs();
|
||||
|
||||
if (!response.ok) {
|
||||
// Handle HTTP error responses
|
||||
const errorData = await response.json();
|
||||
throw new Error(`API Error: ${response.status} ${errorData.message}`);
|
||||
}
|
||||
|
||||
// Assuming the response is JSON and matches the EnqueueVideoStyleTransferSuccessResponse interface
|
||||
return await response.json();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,445 +0,0 @@
|
||||
import {
|
||||
EventDispatcher,
|
||||
Quaternion,
|
||||
Vector3,
|
||||
Vector2,
|
||||
MathUtils,
|
||||
} from "three";
|
||||
import * as THREE from "three";
|
||||
import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
|
||||
import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
|
||||
|
||||
|
||||
export const isSafariOrWebKit = () => {
|
||||
const ua = navigator.userAgent;
|
||||
|
||||
const isWebKit = /AppleWebKit/i.test(ua) && !/Chrome|Chromium|Edg|OPR|CriOS|FxiOS/i.test(ua);
|
||||
const isSafari = /^((?!chrome|android|crios|fxios).)*safari/i.test(ua);
|
||||
|
||||
return isSafari || isWebKit;
|
||||
};
|
||||
|
||||
export const hasApplePay = () => {
|
||||
// @ts-expect-error Apple Pay is not defined in TypeScript
|
||||
return !!window.ApplePaySession;
|
||||
}
|
||||
|
||||
export const isSafari = () => {
|
||||
return isSafariOrWebKit() || hasApplePay();
|
||||
}
|
||||
|
||||
export const isPointerLockSupported = () => {
|
||||
return !isSafari();
|
||||
}
|
||||
|
||||
class FreeCam extends EventDispatcher {
|
||||
object: THREE.PerspectiveCamera;
|
||||
domElement: HTMLElement | Document;
|
||||
enabled = true;
|
||||
movementSpeed = 1.0;
|
||||
rollSpeed = 0.005;
|
||||
dragToLook = false;
|
||||
autoForward = false;
|
||||
dragging = false;
|
||||
xOffset = 0;
|
||||
yOffset = 0;
|
||||
|
||||
// Smoothing properties
|
||||
smoothingFactor = 0.2;
|
||||
currentVelocity = new Vector3();
|
||||
lastPosition = new Vector3();
|
||||
|
||||
lastMousePosition = new Vector2();
|
||||
mouseVelocity = new Vector2();
|
||||
tmpQuaternion = new Quaternion();
|
||||
|
||||
status = 0;
|
||||
|
||||
moveState = {
|
||||
up: 0,
|
||||
down: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
forward: 0,
|
||||
back: 0,
|
||||
pitchUp: 0,
|
||||
pitchDown: 0,
|
||||
yawLeft: 0,
|
||||
yawRight: 0,
|
||||
rollLeft: 0,
|
||||
rollRight: 0,
|
||||
};
|
||||
moveVector = new Vector3(0, 0, 0);
|
||||
rotationVector = new Vector3(0, 0, 0);
|
||||
|
||||
constructor(
|
||||
object: THREE.PerspectiveCamera,
|
||||
domElement: HTMLElement | Document,
|
||||
) {
|
||||
super();
|
||||
|
||||
this.object = object;
|
||||
this.domElement = domElement;
|
||||
this.lastPosition.copy(object.position);
|
||||
|
||||
this.keydown = this.keydown.bind(this);
|
||||
this.keyup = this.keyup.bind(this);
|
||||
this.mousedown = this.mousedown.bind(this);
|
||||
this.mouseup = this.mouseup.bind(this);
|
||||
this.mousemove = this.mousemove.bind(this);
|
||||
this.mousewheel = this.mousewheel.bind(this);
|
||||
this.reset = this.reset.bind(this);
|
||||
this.update = this.update.bind(this);
|
||||
this.updateMovementVector = this.updateMovementVector.bind(this);
|
||||
this.updateRotationVector = this.updateRotationVector.bind(this);
|
||||
this.getContainerDimensions = this.getContainerDimensions.bind(this);
|
||||
this.dispose = this.dispose.bind(this);
|
||||
|
||||
this.domElement.addEventListener("contextmenu", function (event) {
|
||||
event.preventDefault();
|
||||
});
|
||||
|
||||
this.updateMovementVector();
|
||||
this.updateRotationVector();
|
||||
}
|
||||
|
||||
attachEventListeners() {
|
||||
window.addEventListener("keydown", this.keydown);
|
||||
window.addEventListener("keyup", this.keyup);
|
||||
window.addEventListener("mousedown", this.mousedown);
|
||||
window.addEventListener("mouseup", this.mouseup);
|
||||
window.addEventListener("mousemove", this.mousemove);
|
||||
window.addEventListener("wheel", this.mousewheel);
|
||||
}
|
||||
|
||||
detachEventListeners() {
|
||||
window.removeEventListener("keydown", this.keydown);
|
||||
window.removeEventListener("keyup", this.keyup);
|
||||
window.removeEventListener("mousedown", this.mousedown);
|
||||
window.removeEventListener("mouseup", this.mouseup);
|
||||
window.removeEventListener("mousemove", this.mousemove);
|
||||
window.removeEventListener("wheel", this.mousewheel);
|
||||
}
|
||||
|
||||
keydown(event: KeyboardEvent) {
|
||||
if (!this.enabled || isPromptBoxFocused.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (event.code) {
|
||||
case "KeyW":
|
||||
this.moveState.forward = 1;
|
||||
break;
|
||||
case "KeyS":
|
||||
this.moveState.back = 1;
|
||||
break;
|
||||
|
||||
case "KeyA":
|
||||
this.moveState.left = 1;
|
||||
break;
|
||||
case "KeyD":
|
||||
this.moveState.right = 1;
|
||||
break;
|
||||
|
||||
case "KeyQ":
|
||||
this.moveState.down = 1;
|
||||
break;
|
||||
case "KeyE":
|
||||
this.moveState.up = 1;
|
||||
break;
|
||||
|
||||
case "ArrowUp":
|
||||
this.moveState.pitchUp = 1;
|
||||
break;
|
||||
case "ArrowDown":
|
||||
this.moveState.pitchDown = 1;
|
||||
break;
|
||||
|
||||
case "ArrowLeft":
|
||||
this.moveState.yawLeft = 1;
|
||||
break;
|
||||
case "ArrowRight":
|
||||
this.moveState.yawRight = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
this.updateMovementVector();
|
||||
this.updateRotationVector();
|
||||
}
|
||||
|
||||
keyup(event: KeyboardEvent) {
|
||||
if (!this.enabled || isPromptBoxFocused.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (event.code) {
|
||||
case "KeyW":
|
||||
this.moveState.forward = 0;
|
||||
break;
|
||||
case "KeyS":
|
||||
this.moveState.back = 0;
|
||||
break;
|
||||
|
||||
case "KeyA":
|
||||
this.moveState.left = 0;
|
||||
break;
|
||||
case "KeyD":
|
||||
this.moveState.right = 0;
|
||||
break;
|
||||
|
||||
case "ArrowUp":
|
||||
this.moveState.pitchUp = 0;
|
||||
break;
|
||||
case "ArrowDown":
|
||||
this.moveState.pitchDown = 0;
|
||||
break;
|
||||
|
||||
case "ArrowLeft":
|
||||
this.moveState.yawLeft = 0;
|
||||
break;
|
||||
case "ArrowRight":
|
||||
this.moveState.yawRight = 0;
|
||||
break;
|
||||
|
||||
case "KeyQ":
|
||||
this.moveState.down = 0;
|
||||
break;
|
||||
case "KeyE":
|
||||
this.moveState.up = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
this.updateMovementVector();
|
||||
this.updateRotationVector();
|
||||
}
|
||||
|
||||
mousedown(event: MouseEvent) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
if (event.button === 2) {
|
||||
this.dragging = true;
|
||||
this.xOffset = event.clientX;
|
||||
this.yOffset = event.clientY;
|
||||
this.resetVelocity();
|
||||
}
|
||||
}
|
||||
|
||||
mouseup(event: MouseEvent) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
if (event.button === 2) {
|
||||
this.xOffset = event.clientX;
|
||||
this.yOffset = event.clientY;
|
||||
this.resetVelocity();
|
||||
}
|
||||
this.dragging = false;
|
||||
}
|
||||
|
||||
mousemove(event: MouseEvent) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
if (this.dragging) {
|
||||
const mouseX = event.clientX - this.xOffset;
|
||||
const mouseY = event.clientY - this.yOffset;
|
||||
this.xOffset = event.clientX;
|
||||
this.yOffset = event.clientY;
|
||||
|
||||
if (Math.abs(mouseX + mouseY) > 0) {
|
||||
// Store last position before movement
|
||||
this.lastPosition.copy(this.object.position);
|
||||
|
||||
// Apply movement with smoothing
|
||||
const translationX = -mouseX * (this.movementSpeed * 0.01);
|
||||
const translationY = mouseY * (this.movementSpeed * 0.01);
|
||||
|
||||
this.currentVelocity.x = MathUtils.lerp(
|
||||
this.currentVelocity.x,
|
||||
translationX,
|
||||
this.smoothingFactor,
|
||||
);
|
||||
this.currentVelocity.y = MathUtils.lerp(
|
||||
this.currentVelocity.y,
|
||||
translationY,
|
||||
this.smoothingFactor,
|
||||
);
|
||||
|
||||
this.object.translateX(this.currentVelocity.x);
|
||||
this.object.translateY(this.currentVelocity.y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mousewheel(event: WheelEvent) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Store last position before zoom
|
||||
this.lastPosition.copy(this.object.position);
|
||||
|
||||
const zoomDelta = event.deltaY / 120;
|
||||
this.currentVelocity.z = MathUtils.lerp(
|
||||
this.currentVelocity.z,
|
||||
zoomDelta,
|
||||
this.smoothingFactor,
|
||||
);
|
||||
this.object.translateZ(this.currentVelocity.z);
|
||||
}
|
||||
|
||||
resetVelocity() {
|
||||
this.currentVelocity.set(0, 0, 0);
|
||||
this.lastPosition.copy(this.object.position);
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.moveState.forward = 0;
|
||||
this.moveState.back = 0;
|
||||
this.moveState.left = 0;
|
||||
this.moveState.right = 0;
|
||||
this.moveState.pitchUp = 0;
|
||||
this.moveState.pitchDown = 0;
|
||||
this.moveState.yawLeft = 0;
|
||||
this.moveState.yawRight = 0;
|
||||
this.moveState.down = 0;
|
||||
this.moveState.up = 0;
|
||||
|
||||
this.resetVelocity();
|
||||
|
||||
this.updateMovementVector();
|
||||
this.updateRotationVector();
|
||||
}
|
||||
|
||||
isStationary() {
|
||||
return (
|
||||
this.moveVector.distanceTo(new Vector3(0, 0, 0)) === 0 &&
|
||||
this.currentVelocity.distanceTo(new Vector3(0, 0, 0)) < 0.0001
|
||||
);
|
||||
}
|
||||
|
||||
isRotating() {
|
||||
return (
|
||||
this.moveState.pitchUp ||
|
||||
this.moveState.pitchDown ||
|
||||
this.moveState.yawLeft ||
|
||||
this.moveState.yawRight
|
||||
);
|
||||
}
|
||||
|
||||
update(delta: number) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip update if there's no movement, velocity, or rotation
|
||||
if (this.isStationary() && !this.isRotating()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const moveMulti = delta * this.movementSpeed;
|
||||
|
||||
// Apply WASD movement with smoothing
|
||||
const targetMovement = new Vector3(
|
||||
this.moveVector.x * moveMulti,
|
||||
this.moveVector.y * moveMulti,
|
||||
this.moveVector.z * moveMulti,
|
||||
);
|
||||
|
||||
// Smooth out the movement
|
||||
this.currentVelocity.lerp(targetMovement, this.smoothingFactor);
|
||||
|
||||
// Apply the smoothed movement
|
||||
this.object.translateX(this.currentVelocity.x);
|
||||
this.object.translateY(this.currentVelocity.y);
|
||||
this.object.translateZ(this.currentVelocity.z);
|
||||
|
||||
// Store last position
|
||||
this.lastPosition.copy(this.object.position);
|
||||
|
||||
|
||||
// Done with positions
|
||||
// Now handle rotations
|
||||
this.object.rotateX(this.rotationVector.x);
|
||||
this.object.rotateY(this.rotationVector.y);
|
||||
this.object.rotateZ(this.rotationVector.z);
|
||||
|
||||
// Update camera signals
|
||||
if (selectedCameraId.value) {
|
||||
const pos = this.object.position;
|
||||
const rot = this.object.rotation;
|
||||
|
||||
// Calculate lookAt point based on camera's forward direction
|
||||
const lookAtPoint = new Vector3(0, 0, -1);
|
||||
lookAtPoint.applyQuaternion(this.object.quaternion);
|
||||
lookAtPoint.add(pos);
|
||||
|
||||
cameras.value = cameras.value.map((cam) =>
|
||||
cam.id === selectedCameraId.value
|
||||
? {
|
||||
...cam,
|
||||
position: { x: pos.x, y: pos.y, z: pos.z },
|
||||
rotation: { x: rot.x, y: rot.y, z: rot.z },
|
||||
lookAt: { x: lookAtPoint.x, y: lookAtPoint.y, z: lookAtPoint.z },
|
||||
}
|
||||
: cam,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
updateMovementVector() {
|
||||
const forward =
|
||||
this.moveState.forward || (this.autoForward && !this.moveState.back)
|
||||
? 1
|
||||
: 0;
|
||||
|
||||
this.moveVector.x = -this.moveState.left + this.moveState.right;
|
||||
this.moveVector.y = -this.moveState.down + this.moveState.up;
|
||||
this.moveVector.z = -forward + this.moveState.back;
|
||||
}
|
||||
|
||||
updateRotationVector() {
|
||||
this.rotationVector.x = -this.moveState.pitchDown + this.moveState.pitchUp;
|
||||
this.rotationVector.y = -this.moveState.yawRight + this.moveState.yawLeft;
|
||||
this.rotationVector.z = -this.moveState.rollRight + this.moveState.rollLeft;
|
||||
this.rotationVector.multiplyScalar(this.rollSpeed);
|
||||
}
|
||||
|
||||
getContainerDimensions() {
|
||||
if (this.domElement !== document) {
|
||||
return {
|
||||
size: [
|
||||
(this.domElement as HTMLElement).offsetWidth,
|
||||
(this.domElement as HTMLElement).offsetHeight,
|
||||
],
|
||||
offset: [
|
||||
(this.domElement as HTMLElement).offsetLeft,
|
||||
(this.domElement as HTMLElement).offsetTop,
|
||||
],
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
size: [window.innerWidth, window.innerHeight],
|
||||
offset: [0, 0],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
dispose() {
|
||||
// window.addEventListener('contextmenu', function(event) {
|
||||
// event.preventDefault();
|
||||
// });
|
||||
|
||||
// this.domElement.removeEventListener("contextmenu", this.contextmenu);
|
||||
// this.domElement.removeEventListener("pointerdown", this.pointerdown);
|
||||
// this.domElement.removeEventListener("pointermove", this.pointermove);
|
||||
// this.domElement.removeEventListener("pointerup", this.pointerup);
|
||||
// this.domElement.removeEventListener("pointercancel", this.pointercancel);
|
||||
//
|
||||
window.removeEventListener("keydown", this.keydown);
|
||||
window.removeEventListener("keyup", this.keyup);
|
||||
}
|
||||
}
|
||||
|
||||
export { FreeCam };
|
||||
@@ -1,299 +0,0 @@
|
||||
import Scene from "./scene";
|
||||
import Editor from "./editor";
|
||||
import * as THREE from "three";
|
||||
import { XYZ } from "../datastructures/common";
|
||||
import { editorState } from "../signals/engine";
|
||||
import { EditorStates } from "../enums";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
|
||||
import { MediaItem } from "~/pages/PageEnigma/models";
|
||||
import { AssetType } from "~/enums";
|
||||
import { hideObjectPanel } from "../signals";
|
||||
|
||||
export class SceneUtils {
|
||||
scene: Scene;
|
||||
editor: Editor;
|
||||
constructor(editor: Editor, scene: Scene) {
|
||||
this.scene = scene;
|
||||
this.editor = editor;
|
||||
}
|
||||
|
||||
// If string is empty.
|
||||
isEmpty(value: string): boolean {
|
||||
return (
|
||||
value == null || (typeof value === "string" && value.trim().length === 0)
|
||||
);
|
||||
}
|
||||
|
||||
// For react to see if an object has lipsync capability or not.
|
||||
isObjectLipsync(object_uuid: string) {
|
||||
const object = this.scene.get_object_by_uuid(object_uuid);
|
||||
let hasLipsync = false;
|
||||
if (object) {
|
||||
object.traverse((c: THREE.Object3D) => {
|
||||
if (c instanceof THREE.Mesh) {
|
||||
if (c.morphTargetInfluences && c.morphTargetDictionary) {
|
||||
const blendShapeIndexE = c.morphTargetDictionary["E"];
|
||||
// console.log(c.morphTargetDictionary, blendShapeIndexE)
|
||||
if (blendShapeIndexE !== null) {
|
||||
hasLipsync = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return hasLipsync;
|
||||
}
|
||||
|
||||
// Returns if the object is locked or unlocked.
|
||||
isObjectLocked(object_uuid: string): boolean {
|
||||
const object = this.scene.get_object_by_uuid(object_uuid);
|
||||
if (object) {
|
||||
if (object.userData["locked"] == undefined) {
|
||||
object.userData["locked"] = false;
|
||||
}
|
||||
return object.userData["locked"];
|
||||
}
|
||||
//console.log("No object found.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Locks or unlocks and object and returns its new state,
|
||||
lockUnlockObject(object_uuid: string): boolean {
|
||||
const object = this.scene.get_object_by_uuid(object_uuid);
|
||||
if (object) {
|
||||
if (object.userData["locked"] == undefined) {
|
||||
object.userData["locked"] = false;
|
||||
}
|
||||
object.userData["locked"] = !object.userData["locked"];
|
||||
|
||||
if (object.userData["locked"]) {
|
||||
this.removeTransformControls(false);
|
||||
} else if (this.editor.control) {
|
||||
this.scene.scene.add(this.editor.control);
|
||||
if (this.editor.sceneManager?.selected_objects)
|
||||
this.editor.control.attach(
|
||||
this.editor.sceneManager?.selected_objects[0],
|
||||
);
|
||||
}
|
||||
|
||||
return object.userData["locked"];
|
||||
}
|
||||
//console.log("No object found.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Removes transform controls and publishes selected.
|
||||
removeTransformControls(remove_outline: boolean = true) {
|
||||
if (this.editor.control == undefined) {
|
||||
return;
|
||||
}
|
||||
if (this.editor.outlinePass == undefined) {
|
||||
return;
|
||||
}
|
||||
if (remove_outline) {
|
||||
this.editor.last_selected = this.editor.selected;
|
||||
this.editor.outlinePass.selectedObjects = [];
|
||||
this.editor.publishSelect();
|
||||
}
|
||||
this.editor.control.detach();
|
||||
this.editor.activeScene.scene.remove(this.editor.control);
|
||||
if (remove_outline) this.editor.outlinePass.selectedObjects = [];
|
||||
}
|
||||
|
||||
// TO UPDATE selected objects in the scene might want to add to the scene ...
|
||||
async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) {
|
||||
if (this.editor.sceneManager?.selected_objects) {
|
||||
const object = this.editor.sceneManager?.selected_objects[0];
|
||||
if (object != undefined || object != null) {
|
||||
object.position.x = position.x;
|
||||
object.position.y = position.y;
|
||||
object.position.z = position.z;
|
||||
object.rotation.x = THREE.MathUtils.degToRad(rotation.x);
|
||||
object.rotation.y = THREE.MathUtils.degToRad(rotation.y);
|
||||
object.rotation.z = THREE.MathUtils.degToRad(rotation.z);
|
||||
object.scale.x = scale.x;
|
||||
object.scale.y = scale.y;
|
||||
object.scale.z = scale.z;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switchCameraView() {
|
||||
this.editor.camera_person_mode = !this.editor.camera_person_mode;
|
||||
this.editor.cameraViewControls?.reset();
|
||||
if (this.editor.cam_obj) {
|
||||
if (this.editor.camera_person_mode && this.editor.camera) {
|
||||
this.editor.last_cam_pos.copy(this.editor.camera.position);
|
||||
this.editor.last_cam_rot.copy(this.editor.camera.rotation);
|
||||
|
||||
this.editor.camera.position.copy(this.editor.cam_obj.position);
|
||||
this.editor.camera.rotation.copy(this.editor.cam_obj.rotation);
|
||||
|
||||
if (this.editor.lockControls) {
|
||||
this.editor.activeScene.scene.add(
|
||||
this.editor.lockControls.getObject(),
|
||||
);
|
||||
}
|
||||
if (this.editor.cameraViewControls) {
|
||||
this.editor.cameraViewControls.enabled = true;
|
||||
}
|
||||
this.editor.cam_obj.scale.set(0, 0, 0);
|
||||
|
||||
this.removeTransformControls();
|
||||
this.editor.selected = this.editor.cam_obj;
|
||||
this.editor.publishSelect();
|
||||
this.editor.updateSelectedUI();
|
||||
editorState.value = EditorStates.CAMERA_VIEW;
|
||||
if (this.editor.activeScene.hot_items) {
|
||||
this.editor.activeScene.hot_items.forEach((element) => {
|
||||
element.visible = false;
|
||||
});
|
||||
}
|
||||
|
||||
// Make a better solution later but for now this is so that in camera mode when you right click it does not bring up the context menu and allows you to pan.
|
||||
setTimeout(
|
||||
() =>
|
||||
document
|
||||
.getElementById("letterbox")
|
||||
?.addEventListener("contextmenu", function (event) {
|
||||
event.preventDefault();
|
||||
}),
|
||||
250,
|
||||
);
|
||||
} else if (this.editor.camera) {
|
||||
this.editor.camera.position.copy(this.editor.last_cam_pos);
|
||||
this.editor.camera.rotation.copy(this.editor.last_cam_rot);
|
||||
if (this.editor.lockControls) {
|
||||
this.editor.activeScene.scene.remove(
|
||||
this.editor.lockControls.getObject(),
|
||||
);
|
||||
}
|
||||
this.editor.cam_obj.scale.set(1, 1, 1);
|
||||
if (this.editor.activeScene.hot_items) {
|
||||
this.editor.activeScene.hot_items.forEach((element) => {
|
||||
element.visible = true;
|
||||
});
|
||||
}
|
||||
|
||||
hideObjectPanel();
|
||||
editorState.value = EditorStates.EDIT;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the "check sum" of the editors selected object.
|
||||
getSelectedSum(): number {
|
||||
if (this.editor.sceneManager?.selected_objects === undefined) {
|
||||
return 0;
|
||||
}
|
||||
if (this.editor.sceneManager?.selected_objects.length <= 0) {
|
||||
return 0;
|
||||
}
|
||||
const posCombo =
|
||||
this.editor.sceneManager?.selected_objects[0].position.x +
|
||||
this.editor.sceneManager?.selected_objects[0].position.y +
|
||||
this.editor.sceneManager?.selected_objects[0].position.z;
|
||||
const rotCombo =
|
||||
this.editor.sceneManager?.selected_objects[0].rotation.x +
|
||||
this.editor.sceneManager?.selected_objects[0].rotation.y +
|
||||
this.editor.sceneManager?.selected_objects[0].rotation.z;
|
||||
const sclCombo =
|
||||
this.editor.sceneManager?.selected_objects[0].scale.x +
|
||||
this.editor.sceneManager?.selected_objects[0].scale.y +
|
||||
this.editor.sceneManager?.selected_objects[0].scale.z;
|
||||
return posCombo + rotCombo + sclCombo;
|
||||
}
|
||||
|
||||
/* Will add in the future
|
||||
|
||||
A good practice to remove 3D objects from Three.js scenes
|
||||
function removeObject3D(object3D) {
|
||||
if (!(object3D instanceof THREE.Object3D)) return false;
|
||||
|
||||
// for better memory management and performance
|
||||
if (object3D.geometry) object3D.geometry.dispose();
|
||||
|
||||
if (object3D.material) {
|
||||
if (object3D.material instanceof Array) {
|
||||
// for better memory management and performance
|
||||
object3D.material.forEach(material => material.dispose());
|
||||
} else {
|
||||
// for better memory management and performance
|
||||
object3D.material.dispose();
|
||||
}
|
||||
}
|
||||
object3D.removeFromParent(); // the parent might be the scene or another Object3D, but it is sure to be removed this way
|
||||
return true;
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
deleteObject(uuid: string) {
|
||||
const obj = this.scene.get_object_by_uuid(uuid);
|
||||
|
||||
if (!obj) {
|
||||
return
|
||||
}
|
||||
|
||||
this.removeTransformControls();
|
||||
if (obj.name === this.editor.camera_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Finally remove the object from the scene
|
||||
this.scene.scene.remove(obj);
|
||||
|
||||
obj.traverse(child => {
|
||||
(child as THREE.Mesh)?.geometry?.dispose()
|
||||
if (Array.isArray((child as THREE.Mesh).texture)) {
|
||||
(child as THREE.Mesh).texture.forEach(mat => mat.dispose());
|
||||
} else if ((child as THREE.Mesh).texture) {
|
||||
(child as THREE.Mesh).texture.dispose();
|
||||
}
|
||||
|
||||
if (Array.isArray((child as THREE.Mesh).material)) {
|
||||
(child as THREE.Mesh).material.forEach(mat => mat.dispose());
|
||||
} else if ((child as THREE.Mesh).material) {
|
||||
(child as THREE.Mesh).material.dispose();
|
||||
}
|
||||
})
|
||||
|
||||
if (Array.isArray((obj as THREE.Mesh).texture)) {
|
||||
(obj as THREE.Mesh).texture.forEach(mat => mat.dispose());
|
||||
} else if ((obj as THREE.Mesh).texture) {
|
||||
(obj as THREE.Mesh).texture.dispose();
|
||||
}
|
||||
|
||||
if (Array.isArray((obj as THREE.Mesh).material)) {
|
||||
(obj as THREE.Mesh).material.forEach(mat => mat.dispose());
|
||||
} else if ((obj as THREE.Mesh).material) {
|
||||
(obj as THREE.Mesh).material.dispose();
|
||||
}
|
||||
|
||||
if ((obj as THREE.Mesh).geometry) {
|
||||
(obj as THREE.Mesh).geometry.dispose()
|
||||
}
|
||||
|
||||
// TODO
|
||||
// FIXME: Timeline deletion is called twice, as well as a queue event.
|
||||
// Why is this necessary?
|
||||
this.editor.timeline.deleteObject(obj);
|
||||
Queue.publish({
|
||||
queueName: QueueNames.FROM_ENGINE,
|
||||
action: fromEngineActions.DELETE_OBJECT,
|
||||
data: {
|
||||
version: 1,
|
||||
type: AssetType.OBJECT,
|
||||
media_id: "",
|
||||
object_uuid: uuid,
|
||||
name: "",
|
||||
} as MediaItem,
|
||||
});
|
||||
this.editor.selected = undefined;
|
||||
this.editor.publishSelect();
|
||||
hideObjectPanel();
|
||||
this.editor.timeline.deleteObject(obj);
|
||||
}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
const BoundingFrequencyMasc = [0, 400, 560, 2400, 4800];
|
||||
const BoundingFrequencyFem = [0, 500, 700, 3000, 6000];
|
||||
const IndicesFrequencyFemale = [];
|
||||
const IndicesFrequencyMale = [];
|
||||
const FFT_SIZE = 1024;
|
||||
const samplingFrequency = 44100;
|
||||
|
||||
for (let m = 0; m < BoundingFrequencyMasc.length; m++) {
|
||||
IndicesFrequencyMale[m] = Math.round(
|
||||
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m],
|
||||
);
|
||||
}
|
||||
|
||||
for (let m = 0; m < BoundingFrequencyFem.length; m++) {
|
||||
IndicesFrequencyFemale[m] = Math.round(
|
||||
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m],
|
||||
);
|
||||
}
|
||||
|
||||
export class LipSync {
|
||||
constructor(face) {
|
||||
this.face = face;
|
||||
this.buffer = null;
|
||||
this.last_frame = 0;
|
||||
|
||||
// WE NEED THIS TO be run at RUNTIME
|
||||
// we have to modify this for our
|
||||
// this.ee = this.face.morphTargetDictionary["E"];
|
||||
// this.oh = this.face.morphTargetDictionary["O"];
|
||||
// this.ah = this.face.morphTargetDictionary["aa"];
|
||||
}
|
||||
|
||||
// for streaming sound?
|
||||
start(stream) {
|
||||
this.audioContext = new AudioContext();
|
||||
this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
}
|
||||
|
||||
// testing only
|
||||
startFromAudioFile(file) {
|
||||
if (!this.audioContext) this.audioContext = new AudioContext();
|
||||
|
||||
if (!this.userSpeechAnalyzer)
|
||||
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
|
||||
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
|
||||
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
|
||||
|
||||
if (this.mediaStreamSource) this.mediaStreamSource.stop();
|
||||
|
||||
this.audioContext.decodeAudioData(file).then((buffer) => {
|
||||
this.mediaStreamSource = this.audioContext.createBufferSource();
|
||||
this.mediaStreamSource.buffer = buffer;
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
this.mediaStreamSource.connect(this.audioContext.destination);
|
||||
this.mediaStreamSource.start();
|
||||
|
||||
// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
|
||||
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
|
||||
});
|
||||
}
|
||||
|
||||
startLipSyncFromAudioBuffer(buffer) {
|
||||
if (!this.audioContext) this.audioContext = new AudioContext();
|
||||
if (!this.userSpeechAnalyzer)
|
||||
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
|
||||
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
|
||||
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
|
||||
|
||||
if (this.mediaStreamSource) this.mediaStreamSource.stop();
|
||||
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.meter?.shutdown();
|
||||
this.meter = null;
|
||||
this.mediaStreamSource?.disconnect();
|
||||
return this.audioContext?.close().catch(() => {}) || Promise.resolve();
|
||||
}
|
||||
|
||||
update(frame, offset, rendering) {
|
||||
const frameBuffer = 12;
|
||||
|
||||
let pos = frame - offset;
|
||||
|
||||
if (Math.floor(pos) <= 1){
|
||||
this.last_frame = 0;
|
||||
}
|
||||
|
||||
let doPlay = true;
|
||||
if (
|
||||
Math.abs(this.last_frame - frame) < frameBuffer - 1 &&
|
||||
this.last_frame !== 0
|
||||
) {
|
||||
doPlay = false;
|
||||
}
|
||||
|
||||
if (doPlay) {
|
||||
const startTime = pos / 60.0;
|
||||
const endTime = frameBuffer / 60.0;
|
||||
this.mediaStreamSource = this.audioContext.createBufferSource();
|
||||
this.mediaStreamSource.buffer = this.buffer;
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
this.mediaStreamSource.connect(this.audioContext.destination);
|
||||
this.mediaStreamSource.start(0, startTime, endTime);
|
||||
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
|
||||
this.last_frame = frame;
|
||||
}
|
||||
|
||||
if (this.meter) {
|
||||
const { volume } = this.meter;
|
||||
let { ah, oh, ee } = this.update2();
|
||||
if (volume < 0.01) {
|
||||
ah = 0;
|
||||
oh = 0;
|
||||
ee = 0;
|
||||
}
|
||||
return { ah: ah, oh: oh, ee: ee };
|
||||
}
|
||||
return { ah: 0, oh: 0, ee: 0 };
|
||||
}
|
||||
|
||||
update2() {
|
||||
function getSensitivityMap(spectrum) {
|
||||
const sensitivity_threshold = 0.5;
|
||||
const stPSD = new Float32Array(spectrum.length);
|
||||
for (let i = 0; i < spectrum.length; i++) {
|
||||
stPSD[i] = sensitivity_threshold + (spectrum[i] + 20) / 140;
|
||||
}
|
||||
return stPSD;
|
||||
}
|
||||
|
||||
const spectrum = new Float32Array(
|
||||
this.userSpeechAnalyzer.frequencyBinCount,
|
||||
);
|
||||
// Populate frequency data for computing frequency intensities
|
||||
this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time
|
||||
// Populate time domain for calculating RMS
|
||||
// userSpeechAnalyzer.getFloatTimeDomainData(spectrum);
|
||||
// RMS (root mean square) is a better approximation of current input level than peak (just sampling this frame)
|
||||
// spectrumRMS = getRMS(spectrum);
|
||||
|
||||
const sensitivityPerPole = getSensitivityMap(spectrum);
|
||||
|
||||
// Lower and higher voices have different frequency domains, so we'll separate and max them
|
||||
const EnergyBinMasc = new Float32Array(BoundingFrequencyMasc.length);
|
||||
const EnergyBinFem = new Float32Array(BoundingFrequencyFem.length);
|
||||
|
||||
// Masc energy bins (groups of frequency-depending energy)
|
||||
for (let m = 0; m < BoundingFrequencyMasc.length - 1; m++) {
|
||||
for (
|
||||
let j = IndicesFrequencyMale[m];
|
||||
j <= IndicesFrequencyMale[m + 1];
|
||||
j++
|
||||
)
|
||||
if (sensitivityPerPole[j] > 0)
|
||||
EnergyBinMasc[m] += sensitivityPerPole[j];
|
||||
EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m];
|
||||
}
|
||||
|
||||
// Fem energy bin
|
||||
for (let m = 0; m < BoundingFrequencyFem.length - 1; m++) {
|
||||
for (
|
||||
let j = IndicesFrequencyMale[m];
|
||||
j <= IndicesFrequencyMale[m + 1];
|
||||
j++
|
||||
)
|
||||
if (sensitivityPerPole[j] > 0) EnergyBinFem[m] += sensitivityPerPole[j];
|
||||
EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m];
|
||||
EnergyBinFem[m] =
|
||||
EnergyBinFem[m] /
|
||||
(IndicesFrequencyFemale[m + 1] - IndicesFrequencyFemale[m]);
|
||||
}
|
||||
const oh =
|
||||
Math.max(EnergyBinFem[1], EnergyBinMasc[1]) > 0.2
|
||||
? 1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])
|
||||
: (1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])) *
|
||||
5 *
|
||||
Math.max(EnergyBinMasc[1], EnergyBinFem[1]);
|
||||
|
||||
const ah = 3 * Math.max(EnergyBinMasc[3], EnergyBinFem[3]);
|
||||
const ee =
|
||||
0.8 *
|
||||
(Math.max(EnergyBinMasc[1], EnergyBinFem[1]) -
|
||||
Math.max(EnergyBinMasc[3], EnergyBinFem[3]));
|
||||
return { oh, ee, ah };
|
||||
}
|
||||
|
||||
static createAudioMeter(audioContext) {
|
||||
const processor = audioContext.createScriptProcessor(512);
|
||||
processor.onaudioprocess = (event) => {
|
||||
const buf = event.inputBuffer.getChannelData(0);
|
||||
const bufLength = buf.length;
|
||||
let sum = 0;
|
||||
let x;
|
||||
|
||||
// eslint-disable-next-line no-plusplus
|
||||
for (let i = 0; i < bufLength; i++) {
|
||||
x = buf[i];
|
||||
if (Math.abs(x) >= processor.clipLevel) {
|
||||
processor.clipping = true;
|
||||
processor.lastClip = window.performance.now();
|
||||
}
|
||||
sum += x * x;
|
||||
}
|
||||
const rms = Math.sqrt(sum / bufLength);
|
||||
processor.volume = Math.max(rms, processor.volume * processor.averaging);
|
||||
};
|
||||
processor.clipping = false;
|
||||
processor.lastClip = 0;
|
||||
processor.volume = 0;
|
||||
processor.clipLevel = 0.98;
|
||||
processor.averaging = 0.95;
|
||||
processor.clipLag = 750;
|
||||
|
||||
processor.connect(audioContext.destination);
|
||||
|
||||
processor.checkClipping = () => {
|
||||
if (!processor.clipping) {
|
||||
return false;
|
||||
}
|
||||
if (processor.lastClip + processor.clipLag < window.performance.now()) {
|
||||
processor.clipping = false;
|
||||
}
|
||||
return processor.clipping;
|
||||
};
|
||||
|
||||
processor.shutdown = () => {
|
||||
processor.disconnect();
|
||||
processor.onaudioprocess = null;
|
||||
};
|
||||
|
||||
return processor;
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager";
|
||||
|
||||
import {
|
||||
ProcessStatus,
|
||||
StreamingProgressResponse,
|
||||
VideoAudioPreProcessor,
|
||||
} from "./VideoProcessor/video_audio_preprocessor";
|
||||
|
||||
class PreviewVideoGeneration {
|
||||
mediaUploadAPI: MediaUploadApi;
|
||||
videoAPI: VideoApi;
|
||||
videoAndAudioPreProcessor: VideoAudioPreProcessor;
|
||||
|
||||
onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void;
|
||||
|
||||
constructor(
|
||||
onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void,
|
||||
) {
|
||||
this.onProgress = onProgress;
|
||||
this.videoAndAudioPreProcessor = new VideoAudioPreProcessor(onProgress);
|
||||
this.mediaUploadAPI = new MediaUploadApi();
|
||||
this.videoAPI = new VideoApi();
|
||||
}
|
||||
|
||||
// Cancel any part of the processing
|
||||
public async cancel() {}
|
||||
|
||||
private async stageOneProcessing() {}
|
||||
|
||||
private async stageTwoProcessing() {}
|
||||
|
||||
private async processVideo() {
|
||||
// check if video is cached or not if not jump to 50%
|
||||
// collect frames and run preprocessing in the background.
|
||||
// start polling for percentage change and convert and remap to remainint %
|
||||
}
|
||||
|
||||
async clearFrameBuffers() {}
|
||||
}
|
||||
|
||||
export { PreviewVideoGeneration };
|
||||
@@ -1,97 +0,0 @@
|
||||
export class MoveAIResult {
|
||||
bone: string;
|
||||
is_special: boolean;
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
flip: boolean;
|
||||
only_y: boolean;
|
||||
|
||||
constructor(bone: string, is_special: boolean = false, x: number = 0, y: number = 0, z: number = 0,
|
||||
flip:boolean=false, only_y:boolean=false) {
|
||||
this.bone = bone;
|
||||
this.is_special = is_special;
|
||||
this.x = x;
|
||||
this.y = y;
|
||||
this.z = z;
|
||||
this.flip = flip;
|
||||
this.only_y = only_y;
|
||||
}
|
||||
}
|
||||
|
||||
export class Retarget {
|
||||
move_ai_options: { [Key: string]: MoveAIResult }
|
||||
constructor() {
|
||||
this.move_ai_options = {
|
||||
"_1Head.quaternion": new MoveAIResult("mixamorigHead.quaternion"),
|
||||
"_1Neck.quaternion": new MoveAIResult("mixamorigNeck.quaternion"),
|
||||
"_Spine.quaternion": new MoveAIResult("mixamorigSpine.quaternion"),
|
||||
"_1Spine.quaternion": new MoveAIResult("mixamorigSpine1.quaternion"),
|
||||
"_3Spine.quaternion": new MoveAIResult("mixamorigSpine2.quaternion"),
|
||||
|
||||
"_1Hips.quaternion": new MoveAIResult("mixamorigHips.quaternion", true, -90, 0, 0, false, true),
|
||||
"_1Hips.position": new MoveAIResult("mixamorigHips.position", true, 1, 0.05, 1),
|
||||
|
||||
"_1LeftFoot.quaternion": new MoveAIResult("mixamorigLeftFoot.quaternion", true, 0, 0, 0, true),
|
||||
"_1RightFoot.quaternion": new MoveAIResult("mixamorigRightFoot.quaternion", true, 0, 0, 0, true),
|
||||
|
||||
|
||||
"_1RightShoulder.quaternion": new MoveAIResult("mixamorigRightShoulder.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftShoulder.quaternion": new MoveAIResult("mixamorigLeftShoulder.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftArm.quaternion": new MoveAIResult("mixamorigLeftArm.quaternion", true, 0, 0, 0, false),
|
||||
"_1RightArm.quaternion": new MoveAIResult("mixamorigRightArm.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftForeArm.quaternion": new MoveAIResult("mixamorigLeftForeArm.quaternion"),
|
||||
"_1RightForeArm.quaternion": new MoveAIResult("mixamorigRightForeArm.quaternion"),
|
||||
|
||||
"_1LeftLeg.quaternion": new MoveAIResult("mixamorigLeftLeg.quaternion", true, 0, 0, 0, true),
|
||||
"_1RightLeg.quaternion": new MoveAIResult("mixamorigRightLeg.quaternion", true, 0, 0, 0, true),
|
||||
|
||||
"_1LeftUpLeg.quaternion": new MoveAIResult("mixamorigLeftUpLeg.quaternion", true, 0, 180, 0),
|
||||
"_1RightUpLeg.quaternion": new MoveAIResult("mixamorigRightUpLeg.quaternion", true, 0, 180, 0),
|
||||
|
||||
"_1LeftHand.quaternion": new MoveAIResult("mixamorigLeftHand.quaternion"),
|
||||
"_1RightHand.quaternion": new MoveAIResult("mixamorigRightHand.quaternion"),
|
||||
|
||||
"_1LeftHandThumb1.quaternion": new MoveAIResult("mixamorigLeftHandThumb1.quaternion"),
|
||||
"_1LeftHandThumb2.quaternion": new MoveAIResult("mixamorigLeftHandThumb2.quaternion"),
|
||||
"_1LeftHandThumb3.quaternion": new MoveAIResult("mixamorigLeftHandThumb3.quaternion"),
|
||||
"_1LeftHandIndex1.quaternion": new MoveAIResult("mixamorigLeftHandIndex1.quaternion"),
|
||||
"_1LeftHandIndex2.quaternion": new MoveAIResult("mixamorigLeftHandIndex2.quaternion"),
|
||||
"_1LeftHandIndex3.quaternion": new MoveAIResult("mixamorigLeftHandIndex3.quaternion"),
|
||||
"_1LeftHandMiddle1.quaternion": new MoveAIResult("mixamorigLeftHandMiddle1.quaternion"),
|
||||
"_1LeftHandMiddle2.quaternion": new MoveAIResult("mixamorigLeftHandMiddle2.quaternion"),
|
||||
"_1LeftHandMiddle3.quaternion": new MoveAIResult("mixamorigLeftHandMiddle3.quaternion"),
|
||||
"_1LeftHandRing1.quaternion": new MoveAIResult("mixamorigLeftHandRing1.quaternion"),
|
||||
"_1LeftHandRing2.quaternion": new MoveAIResult("mixamorigLeftHandRing2.quaternion"),
|
||||
"_1LeftHandRing3.quaternion": new MoveAIResult("mixamorigLeftHandRing3.quaternion"),
|
||||
"_1LeftHandPinky1.quaternion": new MoveAIResult("mixamorigLeftHandPinky1.quaternion"),
|
||||
"_1LeftHandPinky2.quaternion": new MoveAIResult("mixamorigLeftHandPinky2.quaternion"),
|
||||
"_1LeftHandPinky3.quaternion": new MoveAIResult("mixamorigLeftHandPinky3.quaternion"),
|
||||
"_1RightHandThumb1.quaternion": new MoveAIResult("mixamorigRightHandThumb1.quaternion"),
|
||||
"_1RightHandThumb2.quaternion": new MoveAIResult("mixamorigRightHandThumb2.quaternion"),
|
||||
"_1RightHandThumb3.quaternion": new MoveAIResult("mixamorigRightHandThumb3.quaternion"),
|
||||
"_1RightHandIndex1.quaternion": new MoveAIResult("mixamorigRightHandIndex1.quaternion"),
|
||||
"_1RightHandIndex2.quaternion": new MoveAIResult("mixamorigRightHandIndex2.quaternion"),
|
||||
"_1RightHandIndex3.quaternion": new MoveAIResult("mixamorigRightHandIndex3.quaternion"),
|
||||
"_1RightHandMiddle1.quaternion": new MoveAIResult("mixamorigRightHandMiddle1.quaternion"),
|
||||
"_1RightHandMiddle2.quaternion": new MoveAIResult("mixamorigRightHandMiddle2.quaternion"),
|
||||
"_1RightHandMiddle3.quaternion": new MoveAIResult("mixamorigRightHandMiddle3.quaternion"),
|
||||
"_1RightHandRing1.quaternion": new MoveAIResult("mixamorigRightHandRing1.quaternion"),
|
||||
"_1RightHandRing2.quaternion": new MoveAIResult("mixamorigRightHandRing2.quaternion"),
|
||||
"_1RightHandRing3.quaternion": new MoveAIResult("mixamorigRightHandRing3.quaternion"),
|
||||
"_1RightHandPinky1.quaternion": new MoveAIResult("mixamorigRightHandPinky1.quaternion"),
|
||||
"_1RightHandPinky2.quaternion": new MoveAIResult("mixamorigRightHandPinky2.quaternion"),
|
||||
"_1RightHandPinky3.quaternion": new MoveAIResult("mixamorigRightHandPinky3.quaternion"),
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
retarget(boneName: string): MoveAIResult {
|
||||
for (const [key, value] of Object.entries(this.move_ai_options)) {
|
||||
if (boneName == key) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return new MoveAIResult(boneName);
|
||||
}
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { SceneGenereationMetaData } from "../models/sceneGenerationMetadata";
|
||||
import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
|
||||
|
||||
import {
|
||||
setCameraAspectRatio,
|
||||
hideEditorLoader,
|
||||
restoreSceneGenerationMetadata,
|
||||
showEditorLoader,
|
||||
globalIPAMediaToken,
|
||||
} from "../signals";
|
||||
import Editor from "./editor";
|
||||
import { getArtStyle } from "~/enums";
|
||||
import { cameras, selectedCameraId, Camera } from "../signals/camera";
|
||||
|
||||
export type EditorInitializeConfig = {
|
||||
sceneToken: string;
|
||||
};
|
||||
|
||||
// need to move this into the
|
||||
export class SaveManager {
|
||||
editor: Editor;
|
||||
constructor(editor: Editor) {
|
||||
this.editor = editor;
|
||||
}
|
||||
|
||||
async computeHashForBrowser(data: string): Promise<string> {
|
||||
// Encode the string data to a Uint8Array
|
||||
const encoder = new TextEncoder();
|
||||
const encodedData = encoder.encode(data);
|
||||
|
||||
// Compute the hash using the SubtleCrypto.digest method
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", encodedData);
|
||||
|
||||
// Convert the ArrayBuffer to a hexadecimal string
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashHex = hashArray
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
return hashHex;
|
||||
}
|
||||
|
||||
public async computeSceneChecksum(): Promise<string> {
|
||||
const jsonString = await this.checkSumData();
|
||||
return this.computeHashForBrowser(jsonString);
|
||||
}
|
||||
|
||||
private async checkSumData(): Promise<string> {
|
||||
const proxyScene = new StoryTellerProxyScene(
|
||||
this.editor.version,
|
||||
this.editor.activeScene,
|
||||
);
|
||||
const scene_json = await proxyScene.saveToScene(this.editor.version);
|
||||
|
||||
const check_sum_data = {
|
||||
scene: scene_json,
|
||||
timeline: "",
|
||||
camera_data: {
|
||||
position: this.editor.camera?.position,
|
||||
rotation: this.editor.camera?.rotation,
|
||||
},
|
||||
};
|
||||
const jsonString = JSON.stringify(check_sum_data);
|
||||
return jsonString;
|
||||
}
|
||||
|
||||
// JSON structure should and can return snapshot
|
||||
public async saveData({
|
||||
sceneGenerationMetadata,
|
||||
}: {
|
||||
sceneTitle: string;
|
||||
sceneToken?: string;
|
||||
sceneGenerationMetadata: SceneGenereationMetaData;
|
||||
}): Promise<string> {
|
||||
const proxyScene = new StoryTellerProxyScene(
|
||||
this.editor.version,
|
||||
this.editor.activeScene,
|
||||
);
|
||||
const scene_json = await proxyScene.saveToScene(this.editor.version);
|
||||
|
||||
// Save all cameras data
|
||||
const camerasData = cameras.value.map((cam: Camera) => ({
|
||||
id: cam.id,
|
||||
label: cam.label,
|
||||
focalLength: cam.focalLength,
|
||||
position: cam.position,
|
||||
rotation: cam.rotation,
|
||||
lookAt: cam.lookAt,
|
||||
}));
|
||||
|
||||
const save_data = {
|
||||
version: this.editor.version,
|
||||
scene: scene_json,
|
||||
...sceneGenerationMetadata,
|
||||
timeline: "",
|
||||
skybox: this.editor.activeScene.skybox,
|
||||
camera_data: {
|
||||
position: this.editor.camera?.position,
|
||||
rotation: this.editor.camera?.rotation,
|
||||
},
|
||||
// Add cameras array and selected camera
|
||||
cameras: camerasData,
|
||||
selectedCameraId: selectedCameraId.value,
|
||||
};
|
||||
// take json scene and figure out checksum
|
||||
const jsonString = JSON.stringify(save_data);
|
||||
return jsonString;
|
||||
}
|
||||
|
||||
public getSceneJson({
|
||||
sceneGenerationMetadata,
|
||||
}: {
|
||||
sceneGenerationMetadata: SceneGenereationMetaData;
|
||||
}) {
|
||||
const proxyScene = new StoryTellerProxyScene(
|
||||
this.editor.version,
|
||||
this.editor.activeScene,
|
||||
);
|
||||
const scene_json = proxyScene.saveToScene(this.editor.version);
|
||||
console.log(scene_json);
|
||||
|
||||
// Save all cameras data
|
||||
const camerasData = cameras.value.map((cam: Camera) => ({
|
||||
id: cam.id,
|
||||
label: cam.label,
|
||||
focalLength: cam.focalLength,
|
||||
position: cam.position,
|
||||
rotation: cam.rotation,
|
||||
lookAt: cam.lookAt,
|
||||
}));
|
||||
|
||||
const save_data = {
|
||||
version: this.editor.version,
|
||||
scene: scene_json,
|
||||
...sceneGenerationMetadata,
|
||||
timeline: "",
|
||||
skybox: this.editor.activeScene.skybox,
|
||||
camera_data: {
|
||||
position: this.editor.camera?.position,
|
||||
rotation: this.editor.camera?.rotation,
|
||||
},
|
||||
// Add cameras array and selected camera
|
||||
cameras: camerasData,
|
||||
selectedCameraId: selectedCameraId.value,
|
||||
};
|
||||
|
||||
return save_data;
|
||||
}
|
||||
|
||||
// TODO Move this function into scene manager.
|
||||
public async saveScene({
|
||||
sceneTitle,
|
||||
sceneToken,
|
||||
sceneGenerationMetadata,
|
||||
}: {
|
||||
sceneTitle: string;
|
||||
sceneToken?: string;
|
||||
sceneGenerationMetadata: SceneGenereationMetaData;
|
||||
}): Promise<string> {
|
||||
this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out.
|
||||
// remove controls when saving scene.
|
||||
this.editor.utils.removeTransformControls();
|
||||
showEditorLoader();
|
||||
|
||||
const sceneJson = this.getSceneJson({
|
||||
sceneGenerationMetadata,
|
||||
});
|
||||
|
||||
// TODO turn scene information into and object ...
|
||||
let sceneThumbnail = undefined;
|
||||
|
||||
if (this.editor.renderer) {
|
||||
const imgData = this.editor.renderer.domElement.toDataURL();
|
||||
const response = await fetch(imgData); // Fetch the data URL
|
||||
sceneThumbnail = await response.blob(); // Convert to Blob
|
||||
}
|
||||
|
||||
const result = await this.editor.api_manager.saveSceneState({
|
||||
saveJson: JSON.stringify(sceneJson),
|
||||
sceneTitle,
|
||||
sceneToken,
|
||||
sceneThumbnail,
|
||||
});
|
||||
|
||||
// this means error going to have to fix the plumbing on this because not using
|
||||
// the api manager
|
||||
|
||||
hideEditorLoader();
|
||||
|
||||
this.editor.generating_preview = false; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR
|
||||
|
||||
console.debug("Save Scene Result: ", result);
|
||||
return result; // if this is an empty string it is an error. need to migrate to api manager.
|
||||
}
|
||||
|
||||
private async loadFromJson(scene_json: any) {
|
||||
const proxyScene = new StoryTellerProxyScene(
|
||||
this.editor.version,
|
||||
this.editor.activeScene,
|
||||
);
|
||||
|
||||
await proxyScene.loadFromSceneJson(
|
||||
scene_json["scene"],
|
||||
scene_json["skybox"],
|
||||
scene_json["version"],
|
||||
);
|
||||
|
||||
const camera_data = scene_json["camera_data"];
|
||||
if (camera_data && this.editor.camera) {
|
||||
const camera_position: THREE.Vector3 = camera_data["position"];
|
||||
const camera_rotation: THREE.Euler = camera_data["rotation"];
|
||||
|
||||
this.editor.camera.position.copy(camera_position);
|
||||
this.editor.camera.rotation.copy(camera_rotation);
|
||||
}
|
||||
|
||||
// Restore cameras data
|
||||
if (scene_json.cameras) {
|
||||
cameras.value = scene_json.cameras.map((cam: Camera) => ({
|
||||
id: cam.id,
|
||||
label: cam.label,
|
||||
focalLength: cam.focalLength,
|
||||
position: cam.position,
|
||||
rotation: cam.rotation,
|
||||
lookAt: cam.lookAt,
|
||||
}));
|
||||
}
|
||||
|
||||
// Restore selected camera
|
||||
if (scene_json.selectedCameraId) {
|
||||
selectedCameraId.value = scene_json.selectedCameraId;
|
||||
}
|
||||
|
||||
// For Remixing Scenes.
|
||||
// this calls the signal function to propagate the data to the UI
|
||||
restoreSceneGenerationMetadata(scene_json);
|
||||
// these propogate the values into the editor
|
||||
if (scene_json.globalIpAdapterImageMediaToken) {
|
||||
// this should be populated right after
|
||||
globalIPAMediaToken.value = scene_json.globalIpAdapterImageMediaToken;
|
||||
}
|
||||
if (scene_json.positivePrompt) {
|
||||
this.editor.positive_prompt = scene_json.positivePrompt;
|
||||
}
|
||||
if (scene_json.negativePrompt) {
|
||||
this.editor.negative_prompt = scene_json.negativePrompt;
|
||||
}
|
||||
if (scene_json.artisticStyle) {
|
||||
this.editor.art_style = getArtStyle(scene_json["artisticStyle"]);
|
||||
}
|
||||
if (scene_json.cameraAspectRatio) {
|
||||
//editor propagation
|
||||
this.editor.changeRenderCameraAspectRatio(scene_json.cameraAspectRatio);
|
||||
//signal propagation
|
||||
setCameraAspectRatio(scene_json.cameraAspectRatio);
|
||||
}
|
||||
|
||||
this.editor.version = scene_json["version"];
|
||||
this.editor.cam_obj = this.editor.activeScene.get_object_by_name(
|
||||
this.editor.camera_name,
|
||||
);
|
||||
|
||||
this.editor.cam_obj?.layers.set(1);
|
||||
this.editor.cam_obj?.children.forEach((child) => {
|
||||
child.layers.set(1);
|
||||
});
|
||||
|
||||
console.debug("Loading Timeline", scene_json["timeline"]);
|
||||
|
||||
this.editor.timeline.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async loadCache(cacheJson: string) {
|
||||
showEditorLoader();
|
||||
|
||||
const scene_json = JSON.parse(cacheJson);
|
||||
await this.loadFromJson(scene_json);
|
||||
|
||||
hideEditorLoader();
|
||||
this.editor.timeline.scrub({ currentTime: 0 });
|
||||
}
|
||||
|
||||
// TODO Refactor remove editor.
|
||||
public async loadScene(scene_media_token: string) {
|
||||
showEditorLoader();
|
||||
|
||||
this.editor.current_scene_media_token = scene_media_token;
|
||||
|
||||
const scene_json = await this.editor.api_manager
|
||||
.loadSceneState(this.editor.current_scene_media_token)
|
||||
.catch((err) => {
|
||||
hideEditorLoader();
|
||||
throw err;
|
||||
});
|
||||
|
||||
await this.loadFromJson(scene_json);
|
||||
|
||||
hideEditorLoader();
|
||||
// TODO figure out if this is a bug.
|
||||
this.editor.timeline.scrub({ currentTime: 0 });
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user