Compare commits

..
58 changed files with 298 additions and 870 deletions
@@ -29,10 +29,9 @@ pub enum ValueType {
pub enum PreferenceName {
PreferredDownloadDirectory,
PlaySounds,
EnqueueSuccessSound,
EnqueueFailureSound,
GenerationSuccessSound,
GenerationFailureSound,
GenerationEnqueueSound,
}
#[derive(Serialize)]
@@ -86,16 +85,6 @@ async fn update_prefs(
return Err(anyhow!("Invalid value: {:?}", request.value)),
}
}
PreferenceName::EnqueueSuccessSound => {
prefs.enqueue_success_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::EnqueueFailureSound => {
prefs.enqueue_failure_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::GenerationSuccessSound => {
prefs.generation_success_sound = request.value
.map(|val| string_value(&val))
@@ -106,6 +95,11 @@ async fn update_prefs(
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::GenerationEnqueueSound => {
prefs.generation_enqueue_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
}
app_prefs.set_clone(&prefs)?;
@@ -1,5 +1,3 @@
use std::ops::Add;
use chrono::{DateTime, TimeDelta, Utc};
use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload;
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
use crate::state::data_dir::app_data_root::AppDataRoot;
@@ -14,8 +12,6 @@ use openai_sora_client::requests::list_media::list_media::list_media;
use serde_derive::Serialize;
use tauri::{AppHandle, State};
const CACHE_PERIOD : TimeDelta = TimeDelta::milliseconds(1000 * 60 * 5); // 5 minutes
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct CheckSoraSessionCommand {
@@ -67,35 +63,10 @@ async fn do_check(
});
}
};
let mut needs_upgrade_or_check = false;
let stats = sora_creds_manager.get_credential_stats()?;
match stats.last_consecutive_credential_success {
None => {
info!("Session never checked, needs upgrade or status checking.");
needs_upgrade_or_check= true;
}
Some(last_success) => {
let now = chrono::Utc::now();
let expires = last_success.add(CACHE_PERIOD);
if now.gt(&expires) {
info!("Session success expired, needs upgrade or status checking.");
needs_upgrade_or_check = true;
}
}
}
let upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
let mut upgraded = false;
if needs_upgrade_or_check {
info!("Attempting to upgrade session...");
upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
info!("Attempting to check session...");
list_media(&creds).await?;
sora_creds_manager.record_credential_success()?;
}
list_media(&creds).await?;
if upgraded {
info!("Saving upgraded session to manager and disk...");
@@ -14,13 +14,8 @@ use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_g
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use std::fs::read_to_string;
use std::io::Cursor;
use std::time::Duration;
use tauri::{AppHandle, Manager, State};
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
#[tauri::command]
pub async fn sora_image_generation_command(
prompt: &str,
@@ -60,13 +55,8 @@ pub async fn generate_image(
let image_bytes = BASE64_STANDARD.decode(image)?;
let filename = "image.png".to_string();
let response = sora_media_upload_from_bytes(
image_bytes,
filename,
CredentialMigrationRef::New(&creds),
Some(SORA_IMAGE_UPLOAD_TIMEOUT))
let response = sora_media_upload_from_bytes(image_bytes, filename, CredentialMigrationRef::New(&creds))
.await
.map_err(|err| {
error!("Failed to upload image to Sora: {:?}", err);
@@ -82,7 +72,6 @@ pub async fn generate_image(
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: CredentialMigrationRef::New(&creds),
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
}).await
.map_err(|err| {
error!("Failed to call Sora image generation: {:?}", err);
@@ -25,22 +25,17 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
use openai_sora_client::sora_error::SoraError;
use serde_derive::{Deserialize, Serialize};
use std::fs::read_to_string;
use std::io::Cursor;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
use openai_sora_client::sora_error::SoraError;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use tokens::tokens::media_files::MediaFileToken;
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
#[derive(Deserialize)]
pub struct SoraImageRemixCommand {
/// Image media file; the engine or canvas snapshot (screenshot).
@@ -101,23 +96,23 @@ pub async fn sora_image_remix_command(
let mut status = CommandErrorStatus::ServerError;
let mut error_type = SoraImageRemixErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
let mut error_message = "A server error occurred.";
match (err) {
InnerError::SoraError(SoraError::TooManyConcurrentTasks) => {
error_type = SoraImageRemixErrorType::TooManyConcurrentTasks;
status = CommandErrorStatus::ServerError;
error_message = "You already have work in progress. Please wait for it to finish.";
error_message = "You already have work in progress. Please wait.";
},
InnerError::SoraError(SoraError::SoraUsernameNotYetCreated) => {
error_type = SoraImageRemixErrorType::SoraUsernameNotYetCreated;
status = CommandErrorStatus::BadRequest;
error_message = "You need to create a username on Sora.com to continue.";
error_message = "You need to create a username on Sora.com.";
},
InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => {
error_type = SoraImageRemixErrorType::SoraIsHavingProblems;
status = CommandErrorStatus::ServerError;
error_message = "Sora is having problems. Please wait a moment and then retry.";
error_message = "Sora is having problems. Please wait a moment before retrying.";
}
_ => {},
}
@@ -211,7 +206,6 @@ pub async fn generate_image(
image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest {
file_path,
credentials: &creds,
request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT),
}).await?;
if let Some(new_creds) = maybe_new_credentials {
@@ -236,7 +230,6 @@ pub async fn generate_image(
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: &creds,
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
}).await?;
if let Some(new_creds) = maybe_new_creds {
@@ -30,10 +30,5 @@ pub async fn sora_logout_command(
return Err("error clearing credentials from memory".into());
}
if let Err(err) = sora_creds_manager.reset_credential_stats() {
error!("Error resetting credential stats: {:?}", err);
return Err("error resetting credential stats".into());
}
Ok(().into())
}
@@ -2,5 +2,4 @@ pub mod read_sora_credentials_from_disk;
pub mod read_sora_credentials_from_disk_with_upgrade;
pub mod sora_credential_holder;
pub mod sora_credential_manager;
pub mod sora_credential_stats;
pub mod sora_task_queue;
@@ -1,29 +1,23 @@
use std::{fs, io};
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
use crate::state::sora::sora_credential_stats::SoraCredentialStats;
use anyhow::anyhow;
use errors::AnyhowResult;
use log::{error, info, warn};
use memory_store::clone_cell::CloneCell;
use openai_sora_client::creds::sora_cookies::SoraCookies;
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
use openai_sora_client::creds::sora_sentinel::SoraSentinel;
use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::Path;
use std::{fs, io};
use openai_sora_client::creds::sora_cookies::SoraCookies;
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
#[derive(Clone)]
pub struct SoraCredentialManager {
holder: SoraCredentialHolder,
app_data_root: AppDataRoot,
/// This is meant to help debug credential issues, keep credentials cached, fresh, etc.
// TODO: Put this into the same structure as the credentials and also persist to disk.
stats: CloneCell<SoraCredentialStats>,
}
impl SoraCredentialManager {
@@ -41,36 +35,9 @@ impl SoraCredentialManager {
Self {
holder,
app_data_root: app_data_root.clone(),
stats: CloneCell::with_owned(SoraCredentialStats::new()),
}
}
pub fn record_credential_success(&self) -> AnyhowResult<()> {
let mut stats = self.stats.get_clone()?;
stats.record_credential_success();
self.stats.set_owned(stats)?;
Ok(())
}
pub fn record_credential_failure(&self) -> AnyhowResult<()> {
let mut stats = self.stats.get_clone()?;
stats.record_credential_failure();
self.stats.set_owned(stats)?;
Ok(())
}
pub fn reset_credential_stats(&self) -> AnyhowResult<()> {
self.stats.set_owned(SoraCredentialStats::new())
}
pub fn get_credential_stats(&self) -> AnyhowResult<SoraCredentialStats> {
self.stats.get_clone()
}
pub fn set_credential_stats(&self, stats: SoraCredentialStats) -> AnyhowResult<()> {
self.stats.set_owned(stats)
}
pub fn set_credentials(&self, creds: &SoraCredentialSet) -> AnyhowResult<()> {
self.holder.set_credentials(creds)
}
@@ -1,47 +0,0 @@
/// This is meant to track 401s and other credential failures, not 500s.
#[derive(Clone)]
pub struct SoraCredentialStats {
pub last_credential_failure: Option<chrono::DateTime<chrono::Utc>>,
pub last_credential_success: Option<chrono::DateTime<chrono::Utc>>,
pub credential_success_count: u32,
pub credential_failure_count: u32,
/// This resets on each failure
pub last_consecutive_credential_success: Option<chrono::DateTime<chrono::Utc>>,
/// This resets on each failure
pub consecutive_credential_success_count: u32,
/// This resets on each success
pub consecutive_credential_failure_count: u32,
}
impl SoraCredentialStats {
pub fn new() -> Self {
Self {
last_credential_failure: None,
last_credential_success: None,
last_consecutive_credential_success: None,
credential_success_count: 0,
credential_failure_count: 0,
consecutive_credential_success_count: 0,
consecutive_credential_failure_count: 0,
}
}
pub fn record_credential_success(&mut self) {
let now = chrono::Utc::now();
self.last_credential_success = Some(now);
self.last_consecutive_credential_success = Some(now);
self.credential_success_count += 1;
self.consecutive_credential_success_count += 1;
self.consecutive_credential_failure_count = 0;
}
pub fn record_credential_failure(&mut self) {
let now = chrono::Utc::now();
self.last_credential_failure = Some(now);
self.last_consecutive_credential_success = None;
self.credential_failure_count += 1;
self.consecutive_credential_failure_count += 1;
self.consecutive_credential_success_count = 0;
}
}
+3 -37
View File
@@ -21,45 +21,11 @@
],
"security": {
"csp": {
"default-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://www.googletagmanager.com",
"ipc://localhost",
"tauri://localhost"
],
"connect-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://unpkg.com/detect-gpu@5.0.70/",
"https://www.googletagmanager.com",
"ipc://localhost",
"tauri://localhost"
],
"default-src": "*",
"connect-src": "*",
"font-src": "*",
"img-src": "* data: blob: 'unsafe-inline'",
"style-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://fonts.googleapis.com",
"ipc://localhost",
"tauri://localhost"
]
"style-src": "*"
}
}
},
@@ -1,4 +1,3 @@
use std::time::Duration;
use anyhow::anyhow;
use log::{info, warn};
use errors::AnyhowResult;
@@ -19,8 +18,6 @@ pub struct ImageRemixAutoRenewRequest<'a> {
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: &'a SoraCredentialSet,
/// This function can try several different requests. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image request with retry and session auto-renewal.
@@ -32,7 +29,6 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
image_size: request.image_size,
sora_media_tokens: request.sora_media_tokens.clone(), // NB: Clone because used again
credentials: CredentialMigrationRef::New(request.credentials),
request_timeout: request.request_timeout,
}).await;
let err = match result {
@@ -130,7 +126,6 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
image_size: request.image_size,
sora_media_tokens: request.sora_media_tokens,
credentials: CredentialMigrationRef::New(&new_creds),
request_timeout: request.request_timeout,
}).await;
match result {
@@ -14,16 +14,12 @@ use crate::sora_error::SoraError;
use anyhow::anyhow;
use errors::AnyhowResult;
use log::{error, info, warn};
use std::time::Duration;
pub struct ImageUploadFromBytesAutoRenewRequest<'a> {
pub file_bytes: Vec<u8>,
pub filename: String,
pub mime_type: &'a str,
pub credentials: &'a SoraCredentialSet,
/// This function can try to upload several times. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image upload with retry and session auto-renewal.
@@ -36,7 +32,6 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
request.file_bytes.clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
request.filename.clone(), // FIXME: Same
CredentialMigrationRef::New(request.credentials),
request.request_timeout,
).await;
let err = match result {
@@ -56,7 +51,6 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
request.file_bytes,
request.filename,
CredentialMigrationRef::New(&new_creds),
request.request_timeout,
).await?;
Ok((result, Some(new_creds)))
@@ -6,14 +6,10 @@ use crate::requests::upload::upload_media_http_request::{upload_media_http_reque
use crate::sora_error::SoraError;
use log::{info, warn};
use std::path::Path;
use std::time::Duration;
pub struct ImageUploadFromFileAutoRenewRequest<'a, P: AsRef<Path>> {
pub file_path: P,
pub credentials: &'a SoraCredentialSet,
/// This function can try to upload several times. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image upload with retry and session auto-renewal.
@@ -25,15 +21,10 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
let result = sora_media_upload_from_file(
request.file_path.as_ref().clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
CredentialMigrationRef::New(request.credentials),
/// This function can try to upload several times. This is the timeout for *individual* requests.
request.request_timeout,
).await;
let err = match result {
Ok(response) => return Ok((response, None)),
// We can't retry some errors.
Err(err @ SoraError::SoraUsernameNotYetCreated) => return Err(err),
// Retry all other errors.
Err(err) => err,
};
@@ -46,7 +37,6 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
let result = sora_media_upload_from_file(
request.file_path,
CredentialMigrationRef::New(&new_creds),
request.request_timeout,
).await?;
Ok((result, Some(new_creds)))
@@ -1,9 +1,8 @@
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::image_gen_status::TaskId;
use log::warn;
use crate::creds::credential_migration::CredentialMigrationRef;
use serde_derive::{Deserialize, Serialize};
use std::time::Duration;
use thiserror::Error;
use crate::requests::image_gen::image_gen_status::TaskId;
const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
@@ -153,11 +152,7 @@ impl std::fmt::Display for RawSoraErrorInner {
}
/// Don't expose the internal request implementation as there are only a few "correct" ways to call the API.
pub (crate) async fn image_gen_http_request(
sora_request: RawSoraImageGenRequest,
credentials: CredentialMigrationRef<'_>,
request_timeout: Option<Duration>,
) -> Result<RawSoraResponse, SoraImageGenError> {
pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: CredentialMigrationRef<'_>) -> Result<RawSoraResponse, SoraImageGenError> {
let client = reqwest::Client::new();
let mut cookie;
@@ -186,17 +181,12 @@ pub (crate) async fn image_gen_http_request(
}
}
let mut http_request = client.post(SORA_IMAGE_GEN_URL)
let http_request = client.post(SORA_IMAGE_GEN_URL)
.header("User-Agent", USER_AGENT)
.header("Cookie", &cookie)
.header("Authorization", &authorization_header)
.header("Content-Type", "application/json")
.header("OpenAI-Sentinel-Token", &sentinel);
if let Some(timeout) = request_timeout {
http_request = http_request.timeout(timeout);
}
let http_request = http_request.json(&sora_request).build()
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
@@ -251,8 +241,8 @@ pub (crate) async fn image_gen_http_request(
#[cfg(test)]
mod tests {
use crate::requests::image_gen::image_gen_http_request::RawSoraResponse;
use errors::AnyhowResult;
use crate::requests::image_gen::image_gen_http_request::RawSoraResponse;
#[test]
fn deserialize_task_id() -> AnyhowResult<()> {
@@ -1,8 +1,7 @@
use log::info;
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraImageGenError, VideoGenType};
use log::info;
use std::time::Duration;
pub struct SoraImageGenRemixRequest<'a> {
pub prompt: String,
@@ -10,7 +9,6 @@ pub struct SoraImageGenRemixRequest<'a> {
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: CredentialMigrationRef<'a>,
pub request_timeout: Option<Duration>,
}
/// The "remix" commands let you supply additional images as context.
@@ -41,7 +39,7 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu
// TODO: Replace this with debug level logging in the future.
info!("Sending remix request: {:?}", args);
let result = image_gen_http_request(args, request.credentials, request.request_timeout).await?;
let result = image_gen_http_request(args, request.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
@@ -82,7 +80,6 @@ mod tests {
image_size: ImageSize::Square,
sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()],
credentials: CredentialMigrationRef::Legacy(&creds),
request_timeout: None,
}).await?;
println!("task_id: {}", response.task_id);
@@ -1,15 +1,14 @@
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::credentials::SoraCredentials;
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType};
use errors::AnyhowResult;
use std::time::Duration;
use crate::creds::credential_migration::CredentialMigrationRef;
pub struct SoraImageGenSimpleRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub credentials: CredentialMigrationRef<'a>,
pub request_timeout: Option<Duration>,
}
pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
@@ -25,7 +24,7 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
};
// TODO: Error handling.
let result = image_gen_http_request(sora_request, args.credentials, args.request_timeout).await?;
let result = image_gen_http_request(sora_request, args.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
@@ -35,12 +34,12 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
#[cfg(test)]
mod tests {
use crate::credentials::SoraCredentials;
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::common::{ImageSize, NumImages};
use crate::requests::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
use crate::creds::credential_migration::CredentialMigrationRef;
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
@@ -65,7 +64,6 @@ mod tests {
num_images: NumImages::One,
image_size: ImageSize::Square,
credentials: CredentialMigrationRef::Legacy(&creds),
request_timeout: None,
}).await?;
println!("task_id: {}", response.task_id);
@@ -2,16 +2,10 @@ use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
use crate::sora_error::SoraError;
use std::path::PathBuf;
use std::time::Duration;
/// Upload bytes.
/// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference.
pub async fn sora_media_upload_from_bytes(
bytes: Vec<u8>,
file_name: String,
creds: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
pub async fn sora_media_upload_from_bytes(bytes: Vec<u8>, file_name: String, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
let file_path = PathBuf::from(&file_name);
// TODO: Read file magic bytes first, then fall back to this.
@@ -26,5 +20,5 @@ pub async fn sora_media_upload_from_bytes(
_ => "application/octet-stream",
};
Ok(upload_media_http_request(bytes, file_name, mime_type, creds, maybe_timeout).await?)
Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?)
}
@@ -2,7 +2,6 @@ use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
use crate::sora_error::SoraError;
use std::path::Path;
use std::time::Duration;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
@@ -10,11 +9,7 @@ use tokio::io::AsyncReadExt;
/// There's a better way to implement this.
const INITIAL_BUFFER_SIZE : usize = 1024*1024;
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(
file_path: P,
creds: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
let mut file = File::open(&file_path).await?;
let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE);
file.read_to_end(&mut buffer).await?;
@@ -36,7 +31,7 @@ pub async fn sora_media_upload_from_file<P: AsRef<Path>>(
_ => "application/octet-stream",
};
Ok(upload_media_http_request(buffer, filename, mime_type, creds, maybe_timeout).await?)
Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?)
}
#[cfg(test)]
@@ -68,7 +63,6 @@ mod tests {
let response = sora_media_upload_from_file(
image_path,
CredentialMigrationRef::Legacy(&creds),
None,
).await?;
println!("media: {:?}", response);
@@ -1,4 +1,3 @@
use std::time::Duration;
use crate::credentials::{SoraCredentials, USER_AGENT};
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::sora_error::SoraError;
@@ -70,7 +69,6 @@ pub (crate) async fn upload_media_http_request(
filename: String,
mime_type: &str,
credentials: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
// Create multipart form
@@ -99,15 +97,11 @@ pub (crate) async fn upload_media_http_request(
// Make API request
let client = Client::new();
let mut request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
let request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
.multipart(form)
.header("User-Agent", USER_AGENT)
.header("Cookie", &cookie)
.header("Authorization", &auth_header);
if let Some(timeout) = maybe_timeout {
request_builder = request_builder.timeout(timeout);
}
let response = request_builder.send().await?;
@@ -40,15 +40,7 @@ Message: {
|| message.contains("token_expired");
if cookie_expired {
return SoraError::UnauthorizedCookieOrBearerExpired;
}
let needs_onboarding =
message.contains("You must onboard before using this service")
|| message.contains("onboarding_required");
if needs_onboarding {
return SoraError::SoraUsernameNotYetCreated;
return SoraError::UnauthorizedCookieOrBearerExpired
}
let status_code = status.as_u16();
+3 -12
View File
@@ -1,20 +1,11 @@
<!doctype html>
<html lang="en">
<head>
<title>ArtCraft</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-Y4Y8KMLPQ7"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-Y4Y8KMLPQ7');
</script>
<title>ArtCraft 3D</title>
<base href="/" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
@@ -1,4 +1,4 @@
import { useContext, useEffect, useState } from "react";
import { useState } from "react";
import {
faChevronLeft,
faGear,
@@ -18,7 +18,6 @@ import { TabSelector, TabItem } from "@storyteller/ui-tab-selector";
import { Signal } from "@preact/signals-react";
import { useSignals } from "@preact/signals-react/runtime";
import { setLogoutStates } from "~/signals/authentication/utilities";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
function isEditorPath(path: string) {
if (path === "/") return true;
@@ -30,7 +29,6 @@ interface Props {
pageName: string;
appTabIdSignal: Signal<string>;
setAppTabId: (id: string) => void;
is3DInitSignal: Signal<boolean>;
}
const appTabs: TabItem[] = [
@@ -38,7 +36,7 @@ const appTabs: TabItem[] = [
{ id: "2D", label: "2D" },
];
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId, is3DInitSignal }: Props) => {
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
useSignals();
const currentLocation = getCurrentLocationWithoutParams(
@@ -49,22 +47,6 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId, is3DInitSignal }
const [activeLibraryTab, setActiveLibraryTab] = useState("my-media");
const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
const engine3D = useContext(EngineContext);
const handleTabChange = (tabId: string) => {
if (appTabIdSignal.peek() == "3D") {
engine3D?.unmountEngine();
}
setAppTabId(tabId);
}
const currentAppTabId = appTabIdSignal.value;
const is3DInit = is3DInitSignal.value;
useEffect(() => {
if (currentAppTabId == "3D" && is3DInit) {
engine3D?.remountEngine();
}
}, [currentAppTabId, engine3D, is3DInit]);
return (
<>
<header className="fixed left-0 top-0 z-[60] w-full border-b border-white/5 bg-ui-background">
@@ -86,13 +68,13 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId, is3DInitSignal }
Back to Editor
</ButtonLink>
)}
<TabSelector
{/* <TabSelector
tabs={appTabs}
activeTab={appTabIdSignal.value}
disabled={false}
onTabChange={handleTabChange}
onTabChange={(tabId: string) => setAppTabId(tabId)}
className="w-fit"
/>
/> */}
</div>
<div className="flex items-center justify-center gap-2 font-medium">
@@ -1,76 +0,0 @@
import { Object3D, ObjectLoader } from "three";
export class BoneJSONHelper {
model: Object3D;
constructor(model: Object3D) {
this.model = model;
}
private findHipBone(objectGroup: Object3D[]): Object3D | undefined {
let hipBone = undefined;
for (const child of objectGroup) {
if (child.type === "Bone" && child.name.toLowerCase().includes("mixamorighips")) {
hipBone = child;
break;
}
}
return hipBone;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public toJSON(): any {
if (!this.model.userData.isCharacter) {
return;
}
if (this.model.children.length < 1) {
return undefined;
}
// Find the hip bone
const hipBone = this.findHipBone(this.model.children[0].children);
if (!hipBone) {
console.error("No hip bone found in the model.");
return undefined;
}
return hipBone.toJSON();
}
/* Applies the bone tree in a recursive BFS manner */
private applyBoneTree(hipBone: Object3D, boneTree: Object3D): void {
hipBone.position.copy(boneTree.position);
hipBone.quaternion.copy(boneTree.quaternion);
hipBone.scale.copy(boneTree.scale);
const hipChildren = hipBone.children;
const treeChildren = boneTree.children;
// NOTE: Assumption made that the lengths match!
// This shouldn't be a problem unless the skeletal model is changed
for (let i = 0; i < hipChildren.length; i++) {
const hipChild = hipChildren[i];
const treeChild = treeChildren[i];
if (hipChild && treeChild) {
this.applyBoneTree(hipChild, treeChild);
}
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public poseFromBoneJSON(boneJSON: any): void {
const hipBone = this.findHipBone(this.model.children[0].children);
if (!hipBone) {
return;
}
const loader = new ObjectLoader();
const boneTree = loader.parse(boneJSON);
this.applyBoneTree(hipBone, boneTree);
}
}
@@ -3,7 +3,6 @@ import {
uploadNewScene as uploadNewSceneEndpoint,
updateExistingScene as updateExistingSceneEndpoint,
} from "~/api";
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
export const uploadNewScene = async (file: File, sceneTitle: string) => {
const endpoint = uploadNewSceneEndpoint;
@@ -57,7 +57,6 @@ import { BufferType, EngineFrameBuffers } from "./VideoProcessor/engine_buffer";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
import { is3DEditorInitialized, setIs3DEditorInitialized } from "~/signals/appTab";
export type EditorInitializeConfig = {
sceneToken: string;
@@ -198,11 +197,6 @@ class Editor {
processingHasFailed: boolean;
stats: Stats;
// Allows us to cancel the queued render
private renderEventToken: number;
private shouldRender: boolean;
private isMounted: boolean;
constructor() {
this.processingHasFailed = false;
console.log(
@@ -224,7 +218,7 @@ class Editor {
// life cycle fix
// Version and name.
this.version = 2.0;
this.version = 1.2;
// Clock, scene and camera essentials.
// global names
this.camera_name = "::CAM::";
@@ -262,8 +256,6 @@ class Editor {
this.lastFrameTime = 0;
this.last_selected_sum = 0;
this.selectedCanvas = false;
this.renderEventToken = -1;
this.shouldRender = false;
// Audio Engine Test.
this.render_camera_aspect_ratio = CameraAspectRatio.HORIZONTAL_3_2;
@@ -603,6 +595,13 @@ class Editor {
this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
this.mouse_controls = new MouseControls(
this.camera,
this.get_camera_person_mode.bind(this),
@@ -643,19 +642,10 @@ class Editor {
this.mouse_controls.sceneManager = this.sceneManager;
}
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
// Creates the main update loop.
//this.renderer.setAnimationLoop(this.updateLoop.bind(this));
// Attach freecam state controller
this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
// This will enable all event and render loops
// We'll disable it here so the UI events can control is manually
// this.remountEngine();
this.updateLoop();
if (!this.utils.isEmpty(sceneToken)) {
this.loadScene(sceneToken);
@@ -668,6 +658,24 @@ class Editor {
});
}
document.addEventListener("mouseover", (event) => {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
});
this._configurePostProcessingRaw();
loadingBarData.value = {
@@ -675,8 +683,6 @@ class Editor {
progress: 100,
};
loadingBarIsShowing.value = false;
setIs3DEditorInitialized(true);
}
public isMovable(): boolean {
@@ -687,32 +693,6 @@ class Editor {
document.body.appendChild(this.stats.dom);
}
private mouseOverEventHandler(event: MouseEvent) {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
}
public enableFreeCamControls() {
document.addEventListener("mouseover", this.mouseOverEventHandler);
}
public disableFreeCamControls() {
document.removeEventListener("mouseover", this.mouseOverEventHandler);
}
// Captures the scene without the grid
public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
if (!this.renderer?.domElement || !this.camera) {
@@ -1174,10 +1154,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
@@ -1186,10 +1166,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -1396,15 +1376,10 @@ class Editor {
}
// Basicly Unity 3D's update loop.
updateLoop() {
if (!this.shouldRender) {
console.debug("Stopping 3D render loop");
return;
}
async updateLoop() {
// Performance improvement: Handle frame cap
// Request the next render already - this is necessary so the loop doesn't stop if the fps cap is hit
this.renderEventToken = requestAnimationFrame(this.updateLoop.bind(this));
requestAnimationFrame(this.updateLoop.bind(this));
const frameTime = performance.now();
if (frameTime - this.lastFrameTime < 1000 / this.cap_fps) {
return;
@@ -1414,49 +1389,6 @@ class Editor {
this.renderSingleFrame();
}
startRenderLoop() {
if (this.shouldRender) {
console.warn("Render flag is already true");
return;
}
this.shouldRender = true;
this.updateLoop();
}
stopRenderLoop() {
this.shouldRender = false;
if (this.renderEventToken) {
cancelAnimationFrame(this.renderEventToken);
this.renderEventToken = -1;
}
}
remountEngine() {
if (!is3DEditorInitialized.peek()) {
console.log("3D mount: Wait for initialization");
return;
}
if (this.isMounted) {
console.log("3D already mounted, skipping");
return;
}
this.startRenderLoop();
this.sceneManager?.attachEventListeners();
this.enableFreeCamControls();
this.cameraViewControls?.attachEventListeners();
}
unmountEngine() {
this.stopRenderLoop();
this.sceneManager?.detachEventListeners();
this.disableFreeCamControls();
this.cameraViewControls?.detachEventListeners();
}
change_mode(type: "translate" | "rotate" | "scale") {
if (this.control == undefined) {
return;
@@ -10,14 +10,13 @@ import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
export const isSafariOrWebKit = () => {
export const isSafariUserAgent = () => {
const ua = navigator.userAgent;
const isWebKit = /AppleWebKit/i.test(ua) && !/Chrome|Chromium|Edg|OPR|CriOS|FxiOS/i.test(ua);
const isSafari = /^((?!chrome|android|crios|fxios).)*safari/i.test(ua);
return isSafari || isWebKit;
};
return isSafari;
}
export const hasApplePay = () => {
// @ts-expect-error Apple Pay is not defined in TypeScript
@@ -25,7 +24,7 @@ export const hasApplePay = () => {
}
export const isSafari = () => {
return isSafariOrWebKit() || hasApplePay();
return isSafariUserAgent() || hasApplePay();
}
export const isPointerLockSupported = () => {
@@ -84,10 +83,6 @@ class FreeCam extends EventDispatcher {
this.keydown = this.keydown.bind(this);
this.keyup = this.keyup.bind(this);
this.mousedown = this.mousedown.bind(this);
this.mouseup = this.mouseup.bind(this);
this.mousemove = this.mousemove.bind(this);
this.mousewheel = this.mousewheel.bind(this);
this.reset = this.reset.bind(this);
this.update = this.update.bind(this);
this.updateMovementVector = this.updateMovementVector.bind(this);
@@ -99,28 +94,18 @@ class FreeCam extends EventDispatcher {
event.preventDefault();
});
window.addEventListener("keydown", this.keydown.bind(this));
window.addEventListener("keyup", this.keyup.bind(this));
window.addEventListener("mousedown", this.mousedown.bind(this));
window.addEventListener("mouseup", this.mouseup.bind(this));
window.addEventListener("mousemove", this.mousemove.bind(this));
window.addEventListener("wheel", this.mousewheel.bind(this));
this.updateMovementVector();
this.updateRotationVector();
}
attachEventListeners() {
window.addEventListener("keydown", this.keydown);
window.addEventListener("keyup", this.keyup);
window.addEventListener("mousedown", this.mousedown);
window.addEventListener("mouseup", this.mouseup);
window.addEventListener("mousemove", this.mousemove);
window.addEventListener("wheel", this.mousewheel);
}
detachEventListeners() {
window.removeEventListener("keydown", this.keydown);
window.removeEventListener("keyup", this.keyup);
window.removeEventListener("mousedown", this.mousedown);
window.removeEventListener("mouseup", this.mouseup);
window.removeEventListener("mousemove", this.mousemove);
window.removeEventListener("wheel", this.mousewheel);
}
keydown(event: KeyboardEvent) {
if (!this.enabled || isPromptBoxFocused.value) {
return;
@@ -22,7 +22,6 @@ import {
poseMode,
showPoseControls,
} from "../signals/selectedMode";
import { Euler } from "three";
export enum KinMode {
FK,
@@ -66,7 +65,6 @@ export class MouseControls {
private fkHelper: FKHelper;
private isBoneDragged: boolean = false;
private ignoreNextClick: boolean = false;
private manualMouseLock = false;
constructor(
camera: THREE.PerspectiveCamera,
@@ -379,60 +377,6 @@ export class MouseControls {
}
}
handleMousePointerLock() {
if (this.isMouseClicked && this.lockControls) {
if (this.lockControls.isLocked == false) {
this.lockControls.lock();
}
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
this.lockControls.unlock();
}
}
}
handleMouseManualLock(event: MouseEvent) {
if (this.isMouseClicked && this.lockControls) {
// If the mouse is clicked and the lockControls is not locked, lock it
if (this.lockControls.isLocked == false) {
// Lock the mouse flag
this.lockControls.isLocked = true;
// Change the cursor to a dragging cursor
this.lockControls.domElement.style.cursor = "move";
console.log("Mouse locked manually");
return;
}
// If the mouse is clicked but controls also locked, move the camera with the mouse
const camera = this.lockControls.getObject();
const _euler = new Euler(0, 0, 0, 'YXZ');
const _MOUSE_SENSITIVITY = 0.002;
const pointerSpeed = 1.0;
const _PI_2 = Math.PI / 2;
const minPolarAngle = 0;
const maxPolarAngle = Math.PI;
_euler.setFromQuaternion(camera.quaternion);
_euler.y -= event.movementX * _MOUSE_SENSITIVITY * pointerSpeed;
_euler.x -= event.movementY * _MOUSE_SENSITIVITY * pointerSpeed;
_euler.x = Math.max(_PI_2 - maxPolarAngle, Math.min(_PI_2 - minPolarAngle, _euler.x));
camera.quaternion.setFromEuler(_euler);
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
// Unlock the mouse flag
this.lockControls.isLocked = false;
// Change the cursor back to default
this.lockControls.domElement.style.cursor = "default";
console.log("Mouse unlocked manually");
}
}
}
// Sets new mouse location usually used in raycasts.
onMouseMove(event: MouseEvent) {
if (this.canvReference == undefined) {
@@ -448,11 +392,18 @@ export class MouseControls {
// this causes an issue https://discourse.threejs.org/t/unable-to-use-pointer-lock-api/11092
if (!isPointerLockSupported()) {
this.handleMouseManualLock(event);
} else {
this.handleMousePointerLock();
return;
}
if (this.isMouseClicked && this.lockControls) {
if (this.lockControls.isLocked == false) {
this.lockControls.lock();
}
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
this.lockControls.unlock();
}
}
}
// When the mouse clicks the screen.
@@ -65,7 +65,6 @@ export class SceneManager implements SceneManagerAPI {
private lastSceneState: SceneState;
private copiedObject: THREE.Object3D | undefined;
private is_character: Function;
private devMode: boolean;
constructor(
version: number,
@@ -81,35 +80,14 @@ export class SceneManager implements SceneManagerAPI {
this.lastSceneState = this.getSceneState();
this.updateOutliner = updateOutliner;
this.is_character = is_character;
this.devMode = devMode;
this.onMouseMove = this.onMouseMove.bind(this);
this.onMouseClick = this.onMouseClick.bind(this);
this.onKeyDown = this.onKeyDown.bind(this);
this.onMouseDown = this.onMouseDown.bind(this);
this.onMouseUp = this.onMouseUp.bind(this);
}
public attachEventListeners() {
if (!this.devMode) {
return;
if (devMode) {
window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
window.addEventListener("click", this.onMouseClick.bind(this), false);
window.addEventListener("keydown", this.onKeyDown.bind(this), false);
window.addEventListener("mousedown", this.onMouseDown.bind(this), false);
window.addEventListener("mouseup", this.onMouseUp.bind(this), false);
}
window.addEventListener("mousemove", this.onMouseMove, false);
window.addEventListener("click", this.onMouseClick, false);
window.addEventListener("keydown", this.onKeyDown, false);
window.addEventListener("mousedown", this.onMouseDown, false);
window.addEventListener("mouseup", this.onMouseUp, false);
}
public detachEventListeners() {
if (!this.devMode) {
return;
}
window.removeEventListener("mousemove", this.onMouseMove, false);
window.removeEventListener("click", this.onMouseClick, false);
window.removeEventListener("keydown", this.onKeyDown, false);
window.removeEventListener("mousedown", this.onMouseDown, false);
window.removeEventListener("mouseup", this.onMouseUp, false);
}
public async undo() {
@@ -143,7 +121,7 @@ export class SceneManager implements SceneManagerAPI {
name: string,
position: THREE.Vector3,
): Promise<THREE.Object3D<THREE.Object3DEventMap>> {
if (media_token.includes("SKY::")) {
if (media_token.includes("SKY::")){
const token = media_token.replace("SKY::", "");
this.scene.updateSkybox(token);
}
@@ -265,16 +243,16 @@ export class SceneManager implements SceneManagerAPI {
}
public async copy() {
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
const object = this.mouse_controls.selected?.at(0)
if (object !== undefined) {
if (this.is_character(object.uuid) === false)
this.copiedObject = object;
if(object !== undefined ){
if(this.is_character(object.uuid) === false)
this.copiedObject = object;
}
}
public async paste() {
if (this.copiedObject && this.copiedObject.name != "::CAM::") {
if(this.copiedObject && this.copiedObject.name != "::CAM::") {
const userdata = this.copiedObject.userData;
const position = this.copiedObject.position.clone();
const rotation = this.copiedObject.rotation.clone();
@@ -286,7 +264,7 @@ export class SceneManager implements SceneManagerAPI {
const obj = await this.create(media_id, name, position);
this.scene.setColor(obj.uuid, color);
obj.position.copy(position.add(new THREE.Vector3(0.5, 0.0, 0.5)));
obj.position.copy(position.add(new THREE.Vector3(0.5,0.0,0.5)));
obj.rotation.copy(rotation);
obj.scale.copy(scale);
@@ -7,37 +7,23 @@ export const InstallSounds = () => {
return;
}
// Good for simple menu choices
r.setSoundOnce("click", new SoundEffect("resources/sound/smrpg_click.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_menu_beep_1", new SoundEffect("resources/sound/metroidprime_UI_15.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_beep_2", new SoundEffect("resources/sound/metroidprime_UI_14.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_select", new SoundEffect("resources/sound/metroidprime_UI_18.wav", { defaultVolume: 0.3 }));
// Good for simple immediate enqueue success
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.4 })); // DEFAULT
// Good for simple immediate enqueue
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for simple immediate failure
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.4 }));
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT
r.setSoundOnce("giant_shell_kick", new SoundEffect("resources/sound/smrpg_mario_giantshellkick.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("wrong", new SoundEffect("resources/sound/smrpg_wrong.wav", { defaultVolume: 0.4 }));
// Good for simple async success
r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT
r.setSoundOnce("extra_power", new SoundEffect("resources/sound/smrpg_character_extrapower.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for async errors
r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT
r.setSoundOnce("ghost", new SoundEffect("resources/sound/smrpg_ghost.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_alert", new SoundEffect("resources/sound/metroidprime_UI_52.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_shrill_alert", new SoundEffect("resources/sound/metroidprime_UI_51.wav", { defaultVolume: 0.2 }));
// Good for menus
r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 }));
r.setSoundOnce("scifi_menu_open", new SoundEffect("resources/sound/metroidprime_UI_12.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_close", new SoundEffect("resources/sound/metroidprime_UI_13.wav", { defaultVolume: 0.3 }));
// Good for special reward / celebration
r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 }));
@@ -28,7 +28,7 @@ import { PromptBox3D } from "@storyteller/ui-promptbox";
import { PopoverItem } from "@storyteller/ui-popover";
import { OnboardingHelper } from "./comps/OnboardingHelper";
import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDisplay";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
import { appTabId, setAppTabId } from "~/signals/appTab";
import { KonvaCanvasContainer } from "../Page2d/KonvaCanvasContainer";
import { KonvaRootComponent } from "../Page2d/KonvaRootComponent";
import {
@@ -251,8 +251,13 @@ export const PageEditor = () => {
});
};
const display3d = appTabId.value === "3D" ? "block" : "none";
const display2d = appTabId.value === "2D" ? "block" : "none";
const ContentFor2D = () => {
return (
<KonvaCanvasContainer>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
);
};
return (
<div className="w-screen">
@@ -260,109 +265,109 @@ export const PageEditor = () => {
pageName="Edit Scene"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
<>
<OnboardingHelper />
<div
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)", display: display3d }}
>
{/* Engine section/side panel */}
{appTabId.value === "3D" ? (
<>
<OnboardingHelper />
<div
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)" }}
>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
{/* Engine section/side panel */}
<div
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Top controls */}
<div
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
</div>
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
{/* Top controls */}
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<Outliner />
<PreviewEngineCamera />
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
</div>
<ControlPanelSceneObject />
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
>
<Outliner />
<PreviewEngineCamera />
</div>
<ControlPanelSceneObject />
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
console.log("editorEngine is not available");
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
console.log("editorEngine is not available");
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
</div>
</div>
</>
<KonvaCanvasContainer style={{ display: display2d }}>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
</>
) : (
<ContentFor2D />
)}
<LoginModal
onClose={() => { }}
onClose={() => {}}
videoSrc2D="/resources/videos/artcraft-canvas-demo.mp4"
videoSrc3D="/resources/videos/artcraft-3d-demo.mp4"
openAiLogo="/resources/images/openai-logo.png"
@@ -8,6 +8,7 @@ import { ErrorDialog, LoadingDots } from "~/components";
import { GenerateModals } from "~/pages/PageEnigma/comps/GenerateModals/GenerateModals";
import { Toaster } from "@storyteller/ui-toaster";
import { EditorLoadingBar } from "./comps/EditorLoadingBar";
// import { Wizard } from "~/pages/PageEnigma/Wizard/Wizard";
import { useSignals } from "@preact/signals-react/runtime";
import { useEffect, useState } from "react";
import * as gpu from "detect-gpu";
@@ -22,8 +23,8 @@ import {
showPrecisionSelector,
} from "./signals/precisionSelectorMenu";
import { InstallSounds } from "~/pages/PageEnigma/InstallSounds";
import { useImageGenerationFailureEvent } from "./TauriEvents/useImageGenerationFailureEvent";
import { useImageGenerationSuccessEvent } from "./TauriEvents/useImageGenerationSuccessEvent";
import { InstallImageGenerationFailure } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure";
import { InstallImageGenerationSuccess } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess";
export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
useSignals();
@@ -89,12 +90,11 @@ export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
});
});
useImageGenerationSuccessEvent();
useImageGenerationFailureEvent();
useEffect(() => {
console.log("installing event listeners");
InstallSounds();
InstallImageGenerationSuccess();
InstallImageGenerationFailure();
});
if (validGpu === "unknown") {
@@ -18,7 +18,7 @@ import { StyleSelection } from "~/pages/PageEnigma/comps/StyleSelection";
import { StyleMoreOptions } from "~/pages/PageEnigma/comps/StyleMoreOptions/StyleMoreOptions";
import Queue, { QueueNames } from "~/pages/PageEnigma/Queue";
import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
import { appTabId, setAppTabId } from "~/signals/appTab";
export const PageStyling = () => {
useSignals();
@@ -64,7 +64,6 @@ export const PageStyling = () => {
pageName="Stylization"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
<div className="mt-4 flex flex-col items-center gap-6">
<PreviewImages />
@@ -0,0 +1,24 @@
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
import { listen } from '@tauri-apps/api/event';
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type SoraImageGenerationFailed = {
prompt: string,
};
export const InstallImageGenerationFailure = () => {
listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Image generation failed!");
});
}
export const ImageGenerationFailure = () => {
}
@@ -0,0 +1,23 @@
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
import { listen } from '@tauri-apps/api/event';
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type ImageGenerationSuccess = {
media_file_token: string,
};
export const InstallImageGenerationSuccess = () => {
listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image generated!");
});
}
export const ImageGenerationSuccess = () => {
}
@@ -1,41 +0,0 @@
import { useEffect } from "react";
import { listen, UnlistenFn } from '@tauri-apps/api/event';
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type SoraImageGenerationFailed = {
prompt: string,
};
export const useImageGenerationFailureEvent = () => {
useEffect(() => {
let isUnmounted = false;
let unlisten: Promise<UnlistenFn>;
const setup = async () => {
unlisten = listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
console.log("Image generation failure event received:", event);
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Image generation failed!");
});
if (isUnmounted) {
unlisten.then(f => f()); // Unsubscribe if unmounted early.
}
};
setup();
return () => {
isUnmounted = true;
unlisten.then(f => f());
};
}, []);
}
@@ -1,41 +0,0 @@
import { useEffect } from "react";
import { listen, UnlistenFn } from '@tauri-apps/api/event';
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type ImageGenerationSuccess = {
media_file_token: string,
};
export const useImageGenerationSuccessEvent = () => {
useEffect(() => {
let isUnmounted = false;
let unlisten: Promise<UnlistenFn>;
const setup = async () => {
unlisten = listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
console.log("Image generation success event received:", event);
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image generated!");
});
if (isUnmounted) {
unlisten.then(f => f()); // Unsubscribe if unmounted early.
}
};
setup();
return () => {
isUnmounted = true;
unlisten.then(f => f());
};
}, []);
}
@@ -187,7 +187,7 @@ export const ControlsTopButtons = () => {
return (
<div className="flex flex-col gap-2 pl-2 pt-2">
<div className="flex gap-1.5">
<ButtonDropdown
{/* <ButtonDropdown
label="File"
icon={faFile}
className="shadow-xl"
@@ -222,9 +222,9 @@ export const ControlsTopButtons = () => {
},
{
disabled:
!(scene.value.isModified &&
(scene.value.ownerToken === undefined || scene.value.ownerToken ===
authentication.userInfo.value?.user_token)),
!scene.value.isModified ||
scene.value.ownerToken ===
authentication.userInfo.value?.user_token,
// save scene should be disabled if there are no changes
label: "Save scene",
description: "Ctrl+S",
@@ -285,7 +285,7 @@ export const ControlsTopButtons = () => {
},
},
]}
/>
/> */}
<Button
icon={outlinerIsShowing.value ? faCheckSquare : faSquare}
@@ -29,8 +29,6 @@ export interface ObjectJSON {
locked: boolean;
visible: boolean | undefined;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rigData: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
user_data: Record<string, any>;
}
@@ -52,8 +50,6 @@ export class StoryTellerProxy3DObject {
locked: boolean;
visible: boolean;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rigData: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
userData: Record<string, any>;
constructor(version: number, media_file_token: string) {
@@ -75,7 +71,6 @@ export class StoryTellerProxy3DObject {
this.specular = 0.5;
this.locked = false;
this.visible = true;
this.rigData = undefined;
this.userData = {};
this.userData.media_file_type = MediaFileType.None;
}
@@ -134,7 +129,6 @@ export class StoryTellerProxy3DObject {
locked: this.locked,
visible: this.visible,
user_data: this.userData,
rigData: this.rigData,
};
return json;
}
@@ -5,7 +5,6 @@ import {
ObjectJSON,
} from "./storyteller_proxy_3d_object";
import Scene from "../Editor/scene";
import { BoneJSONHelper } from "../Editor/KinHelpers/BoneJSONHelper";
interface LookUpDictionary {
[key: string]: StoryTellerProxy3DObject;
@@ -50,10 +49,6 @@ export class StoryTellerProxyScene {
proxyObject3D.locked = child.userData["locked"];
proxyObject3D.visible = child.visible;
proxyObject3D.userData = child.userData;
// Add rig data if present
const boneHelper = new BoneJSONHelper(child);
proxyObject3D.rigData = boneHelper.toJSON();
const json_data = await proxyObject3D.toJSON();
return json_data;
}
@@ -151,16 +146,6 @@ export class StoryTellerProxyScene {
new THREE.Vector3(-0.5, 1.5, 0),
version,
);
// Add any missing userdata back to the object
obj.userData = json_object.user_data;
// Add rig data if present
if (json_object.rigData) {
const boneHelper = new BoneJSONHelper(obj);
boneHelper.poseFromBoneJSON(json_object.rigData);
} else {
console.log("No rig data found");
}
} else if (token.includes("Point::")) {
const keyframe_uuid = token.replace("Point::", "");
obj = this.scene.createPoint(
@@ -10,8 +10,3 @@ export const setAppTabId = (newId: string) => {
appTabId.value = newId;
};
export const is3DEditorInitialized = signal(false);
export const setIs3DEditorInitialized = (isInitialized: boolean) => {
is3DEditorInitialized.value = isInitialized;
}
+8
View File
@@ -24,6 +24,11 @@
"allowJs": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"paths": {
"~/*": [
"apps/editor3d/app/src/*"
]
},
// Vite takes care of building everything, not tsc.
"noEmit": true
},
@@ -43,6 +48,9 @@
{
"path": "../../libs/components/toaster"
},
{
"path": "../../libs/components/demo-modal"
},
{
"path": "../../libs/components/login-modal"
},
@@ -110,7 +110,7 @@ export const PromptBox2D = ({
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
});
}, [prompt]);
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatioList((prev) =>
@@ -2,9 +2,6 @@
"files": [],
"include": [],
"references": [
{
"path": "../../tauri-api"
},
{
"path": "../camera-settings-modal"
},
@@ -43,9 +43,6 @@
"src/**/*.tsx"
],
"references": [
{
"path": "../../tauri-api/tsconfig.lib.json"
},
{
"path": "../camera-settings-modal/tsconfig.lib.json"
},
@@ -16,27 +16,16 @@ const SOUND_OPTIONS = [
{ value: "none", label: "None (Silent)" }, // NB: MUST BE FIRST
// Other options
{ label: "Click", value: "click" },
{ label: "Correct", value: "correct" },
{ label: "Crumble", value: "crumble" },
{ label: "Dialog Done", value: "done" },
{ label: "Dialog Next", value: "next" },
{ label: "Error Chirp", value: "error_chirp" },
{ label: "Flower", value: "flower" },
{ label: "Ghost", value: "ghost" },
{ label: "Sci-Fi Alert", value: "scifi_alert" },
{ label: "Sci-Fi Menu Beep 1", value: "scifi_menu_beep_1" },
{ label: "Sci-Fi Menu Beep 2", value: "scifi_menu_beep_2" },
{ label: "Sci-Fi Menu Close", value: "scifi_menu_close" },
{ label: "Sci-Fi Menu Open", value: "scifi_menu_open" },
{ label: "Sci-Fi Menu Select", value: "scifi_menu_select" },
{ label: "Sci-Fi Shrill Alert", value: "scifi_shrill_alert" },
{ label: "Select", value: "select" },
{ label: "Shell Kick", value: "giant_shell_kick" },
{ label: "Special Alert", value: "special_alert" },
{ label: "Special Flower", value: "special_flower" },
{ label: "Spike Throw", value: "spike_throw" },
{ label: "Wrong", value: "wrong" },
{ value: "correct", label: "Correct" },
{ value: "crumble", label: "Crumble" },
{ value: "done", label: "Done" },
{ value: "error_chirp", label: "Error Chirp" },
{ value: "flower", label: "Flower" },
{ value: "next", label: "Next" },
{ value: "select", label: "Select" },
{ value: "special_alert", label: "Special Alert" },
{ value: "special_flower", label: "Special Flower" },
{ value: "spike_throw", label: "Spike Throw" },
];
interface AudioSettingsPaneProps {}
+5 -4
View File
@@ -24,10 +24,11 @@
],
"baseUrl": ".",
"paths": {
"~/*": [
// Uncomment if 2D ever gets used
// "apps/editor2d/app/src/*",
"apps/editor3d/app/src/*"
"@frontend/login": [
"libs/login/src/index.ts"
],
"@frontend/soundboard": [
"libs/soundboard/src/index.ts"
]
}
}