Compare commits

..
Author SHA1 Message Date
Tanish Manku 942005282a Some bug fixes and improvements for FK 2025-04-04 00:38:31 -07:00
Tanish Manku 14474e68c2 Add basic Forward Kinematics support (#475)
There's still some glitches around selection and deletion.
2025-04-04 00:34:09 -07:00
Michael Chung 2f44da8ac7 Changed api calls to use file upload and uuid idempotency token and stuff works. 2025-04-03 23:10:23 -04:00
Brandon Thomas 3d67f86a19 set sora creds externally 2025-04-03 22:26:38 -04:00
Brandon Thomas 2913d91d0d fix media file downloading 2025-04-03 18:47:57 -04:00
ArEnScandMichael Chung ceb77505e1 Feature/gen (#474)
Co-authored-by: Michael Chung <michaelchung@michaels-mbp-2.home>
2025-04-03 18:01:29 -04:00
ArEnScandMichael Chung 0cad3fe7da Wire screenshot upload and generate to the engine. (#470)
Co-authored-by: Michael Chung <michaelchung@michaels-mbp-2.home>
2025-04-03 17:38:34 -04:00
Tanish Manku 1ced74040f Fix: Letterboxing blocking pointer events 2025-04-03 14:09:35 -07:00
Brandon ThomasandKasisnu c9dd545ae1 Use OpenAI Sora GPT 4o image gen (#472)
Co-authored-by: Kasisnu <5044120+kasisnu@users.noreply.github.com>
2025-04-03 17:07:20 -04:00
bflatastic 8ff91690c1 bring back objects, skybox, creatures tabs 2025-04-03 23:24:14 +07:00
BFlat 97bbe0fe07 add image ui (#471) 2025-04-03 23:01:02 +07:00
BFlat 9361634d83 final fix ratio default to 3:2 (#469) 2025-04-03 12:41:40 +07:00
bflatastic 7cf0363555 fixed ratio in render 3:2 2025-04-03 12:32:45 +07:00
bflatastic c2ee3b0ce1 render default to 3:2 ratio 2025-04-03 12:30:57 +07:00
BFlat ca6fc13171 aspect ratio update (#468)
- added 3:2, 2:3 ratios (defaulted to 3:2 [1200 x 800 px])
- added letterbox accordingly
- aspect ratio buttons to show icons for each ratio
- popover to close when item selected
2025-04-03 12:23:54 +07:00
33 changed files with 1363 additions and 297 deletions
@@ -234,17 +234,31 @@ impl BucketClient {
.map(|s| s.to_string())
.ok_or(anyhow!("could not convert object path to string"))?;
info!("downloading from bucket: {:?}", &object_path_str);
info!("creating file for bucket download: {:?}", filesystem_path.as_ref());
let mut output_file = tokio::fs::File::create(filesystem_path).await?;
let mut output_file = File::create(filesystem_path).await?;
let status_code = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await?;
let result = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await;
info!("downloading from bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
let status_code = match result {
Ok(status_code) => status_code,
Err(err) => {
return bail!("Error downloading from bucket (named '{}'): {:?}", &self.bucket.name, err)
}
};
match status_code {
404 => bail!("File not found in bucket: {}", &object_path_str),
_ => {},
404 => {
error!("File not found in bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
bail!("File not found in bucket: {}", &object_path_str)
},
_ => {
info!("download code: {}", status_code);
},
}
info!("download code: {}", status_code);
Ok(())
}
}
@@ -0,0 +1 @@
pub mod set_sora_secret_handler;
@@ -0,0 +1,137 @@
use std::fmt;
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web, Either};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Form, Json, Path};
use anyhow::anyhow;
use log::{error, log, warn};
use r2d2_redis::redis::Commands;
use utoipa::ToSchema;
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use mysql_queries::queries::media_files::edit::update_media_file_animation_type::update_media_file_animation_type;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use tokens::tokens::media_files::MediaFileToken;
use crate::state::server_state::ServerState;
use crate::util::redis_sora_secrets::get_sora_credentials;
use crate::util::redis_sora_secrets::set_sora_credential_field;
use crate::util::redis_sora_secrets::RedisSoraCredentialSubkey;
#[derive(Deserialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum SetSoraSecretType {
Bearer,
Cookie,
Sentinel
}
#[derive(Deserialize, ToSchema)]
pub struct SetSoraSecretRequest {
pub key: SetSoraSecretType,
pub value: String,
}
#[derive(Serialize, ToSchema)]
pub struct SetSoraSecretSuccessResponse {
pub success: bool,
pub bearer: String,
pub cookie: String,
pub sentinel: String,
}
// =============== Error Response ===============
#[derive(Debug, Serialize, ToSchema)]
pub enum SetSoraSecretError {
BadInput(String),
NotAuthorized,
ServerError,
}
impl ResponseError for SetSoraSecretError {
fn status_code(&self) -> StatusCode {
match *self {
SetSoraSecretError::BadInput(_) => StatusCode::BAD_REQUEST,
SetSoraSecretError::NotAuthorized => StatusCode::UNAUTHORIZED,
SetSoraSecretError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for SetSoraSecretError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// =============== Handler ===============
/// Set a Sora secret
#[utoipa::path(
post,
tag = "Control Plane",
path = "/v1/control_plane/sora_secret",
responses(
(status = 200, description = "Success", body = SetSoraSecretSuccessResponse),
(status = 400, description = "Bad input", body = SetSoraSecretError),
(status = 401, description = "Not authorized", body = SetSoraSecretError),
(status = 500, description = "Server error", body = SetSoraSecretError),
),
params(
("request" = SetSoraSecretRequest, description = "Payload for Request"),
("path" = MediaFileTokenPathInfo, description = "Path for Request")
)
)]
pub async fn set_sora_secret_handler(
http_request: HttpRequest,
request: Form<SetSoraSecretRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<SetSoraSecretSuccessResponse>, SetSoraSecretError> {
// TODO(bt,2025-04-03): Some form of authentication should be required for this endpoint.
let value = request.value.trim().to_string();
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
set_sora_credential_field(
&mut redis,
match request.key {
SetSoraSecretType::Bearer => RedisSoraCredentialSubkey::Bearer,
SetSoraSecretType::Cookie => RedisSoraCredentialSubkey::Cookie,
SetSoraSecretType::Sentinel => RedisSoraCredentialSubkey::Sentinel,
},
&value
).map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
let credentials = get_sora_credentials(&mut redis)
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
// TODO(bt,2025-04-03): Probably not smart to return, but useful for debugging.
Ok(Json(SetSoraSecretSuccessResponse {
success: true,
bearer: credentials.bearer_token,
cookie: credentials.cookie,
sentinel: credentials.sentinel.unwrap_or_else(|| "".to_string()),
}))
}
@@ -0,0 +1,68 @@
use actix_web::HttpRequest;
use anyhow::anyhow;
use log::{error, info};
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use openai_sora_client::credentials::SoraCredentials;
use crate::util::redis_sora_secrets::get_sora_credentials;
const SORA_BEARER_HEADER_NAME : &str = "sora-bearer";
const SORA_COOKIE_HEADER_NAME : &str = "sora-cookie";
const SORA_SENTINEL_HEADER_NAME : &str = "sora-sentinel";
pub fn get_sora_credentials_from_request(
http_request: &HttpRequest,
redis: &mut PooledConnection<RedisConnectionManager>,
) -> AnyhowResult<SoraCredentials> {
let sora_bearer = get_request_header_optional(&http_request, SORA_BEARER_HEADER_NAME);
let sora_cookie = get_request_header_optional(&http_request, SORA_COOKIE_HEADER_NAME);
let sora_sentinel = get_request_header_optional(&http_request, SORA_SENTINEL_HEADER_NAME);
match (&sora_bearer, &sora_cookie, &sora_sentinel) {
(Some(bearer), Some(cookie), Some(sentinel)) => {
info!("Sora credentials were present in HTTP request headers!");
return Ok(SoraCredentials {
bearer_token: bearer.to_string(),
cookie: cookie.to_string(),
sentinel: Some(sentinel.to_string()),
});
}
_ => {} // Fall through
}
let redis_result = get_sora_credentials(redis);
match redis_result {
Ok(mut credentials) => {
if let Some(bearer) = sora_bearer {
info!("override bearer value");
credentials.bearer_token = bearer;
}
if let Some(cookie) = sora_cookie {
info!("override cookie value");
credentials.cookie = cookie;
}
if let Some(sentinel) = sora_sentinel {
info!("override sentinel value");
credentials.sentinel = Some(sentinel);
}
return Ok(credentials)
},
Err(err) => {
error!("Failed to get Sora credentials from Redis; using env as backup. Error: {:?}", err);
// NB: Fall through
}
};
match SoraCredentials::from_env() {
Ok(creds) => Ok(creds),
Err(err) => {
error!("Failed to load Sora credentials from environment: {:?}", err);
Err(anyhow!("Failed to load Sora credentials from environment"))
}
}
}
@@ -1,2 +1,3 @@
pub mod upload;
pub mod prompt;
pub mod prompt;
pub mod get_sora_credentials_from_request;
@@ -11,13 +11,14 @@ use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use log::{debug, error};
use log::{debug, error, info};
use log::warn;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use openai_sora_client::credentials::SoraCredentials;
use serde::Deserialize;
use serde::Serialize;
use sqlx::MySqlPool;
use tempdir::TempDir;
use utoipa::ToSchema;
use web::Data;
@@ -47,10 +48,13 @@ use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::endpoints::image_studio::get_sora_credentials_from_request::get_sora_credentials_from_request;
use crate::http_server::endpoints::tts::get_tts_inference_job_status::GetTtsInferenceStatusError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::state::server_state::ServerState;
use crate::util::allowed_studio_access::allowed_studio_access;
use crate::util::redis_sora_secrets::get_sora_credentials;
/// This is the number of images (batch size) to generate for each request.
/// We should allow all users to have multiple images generated at once as this
@@ -68,10 +72,6 @@ const DEBUG_HEADER_NAME: &str = "enable-debug-mode";
/// This is useful for catching the live logs or intercepting the job.
const ROUTING_TAG_HEADER_NAME: &str = "routing-tag";
const SORA_BEARER_HEADER_NAME : &str = "sora-bearer";
const SORA_COOKIE_HEADER_NAME : &str = "sora-cookie";
const SORA_SENTINEL_HEADER_NAME : &str = "sora-sentinel";
#[derive(Deserialize, ToSchema)]
pub struct EnqueueStudioImageGenRequest {
pub uuid_idempotency_token: String,
@@ -195,7 +195,6 @@ pub async fn enqueue_studio_image_generation_handler(
|routing_tag| routing_tag.trim().to_string()
);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
@@ -220,7 +219,6 @@ pub async fn enqueue_studio_image_generation_handler(
// Get up IP address
let ip_address = get_request_ip(&http_request);
// ==================== HANDLE IDEMPOTENCY ==================== //
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
@@ -239,17 +237,14 @@ pub async fn enqueue_studio_image_generation_handler(
// TODO: Maybe this moves to a job.
let work_temp_dir = server_state.temp_dir_creator.new_tempdir("image_studio").unwrap();
let public_bucket_client = server_state.public_bucket_client.clone();
let private_bucket_client = server_state.private_bucket_client.clone();
let scene_media_token = request.snapshot_media_token.clone();
let scene_media_name = scene_media_token.to_string();
let scene_media_path = work_temp_dir.path().join(scene_media_name.clone());
let scene_media_path = do_download_media_file(
&scene_media_name,
let scene_media_path = query_and_download_media_file(
&scene_media_token,
&scene_media_path,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
@@ -265,13 +260,9 @@ pub async fn enqueue_studio_image_generation_handler(
let additional_images = request.maybe_additional_images.clone().unwrap_or_else(|| vec![]);
for media_file_token in &additional_images {
let media_file_name = media_file_token.to_string();
let media_file_path = work_temp_dir.path().join(media_file_name.clone());
let media_file_path = do_download_media_file(
&media_file_name,
let media_file_path = query_and_download_media_file(
&media_file_token,
&media_file_path,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
@@ -284,30 +275,18 @@ pub async fn enqueue_studio_image_generation_handler(
// ==================== HANDLE SORA CREDENTIALS ==================== //
let sora_bearer = get_request_header_optional(&http_request, SORA_BEARER_HEADER_NAME);
let sora_cookie = get_request_header_optional(&http_request, SORA_COOKIE_HEADER_NAME);
let sora_sentinel = get_request_header_optional(&http_request, SORA_SENTINEL_HEADER_NAME);
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
let sora_credentials;
match (sora_bearer, sora_cookie, sora_sentinel) {
(Some(bearer), Some(cookie), Some(sentinel)) => {
sora_credentials = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
}
_ => {
sora_credentials = match SoraCredentials::from_env() {
Ok(creds) => creds,
Err(err) => {
error!("Failed to load Sora credentials from environment: {:?}", err);
return Err(EnqueueImageGenRequestError::ServerError);
}
};
}
}
let sora_credentials = get_sora_credentials_from_request(&http_request, &mut redis)
.map_err(|e| {
error!("sora credential error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
// ==================== HANDLE SORA UPLOAD ==================== //
@@ -436,84 +415,51 @@ pub enum DownloadMediaFileError {
Other(anyhow::Error),
}
impl From<GetMediaFileError> for DownloadMediaFileError {
fn from(err: GetMediaFileError) -> Self {
match err {
GetMediaFileError::Other(e) => DownloadMediaFileError::Other(e),
GetMediaFileError::NotFound => DownloadMediaFileError::MediaFileNotFound,
}
}
}
impl From<anyhow::Error> for DownloadMediaFileError {
fn from(err: anyhow::Error) -> Self {
DownloadMediaFileError::Other(err)
}
}
async fn do_download_media_file(
media_file_name: &str,
async fn query_and_download_media_file(
media_file_token: &MediaFileToken,
download_directory: &Path,
download_dir: &TempDir,
bucket_client: &BucketClient,
mysql_pool: &MySqlPool,
) -> Result<(PathBuf), DownloadMediaFileError> {
let final_download_path = download_directory.join(format!("{}", media_file_name));
if final_download_path.exists() {
log::debug!("Deleting existing file at {:?}", final_download_path.display().to_string());
std::fs::remove_file(&final_download_path)
.map_err(|err| DownloadMediaFileError::IoError(err))?;
}
let bucket_object_path = get_media_file_details(&media_file_token, mysql_pool).await?;
log::info!("Downloading media from bucket path: {:?}", &bucket_object_path);
bucket_client.download_file_to_disk(
&bucket_object_path,
&final_download_path,
).await?;
Ok(final_download_path)
}
#[derive(Debug)]
enum GetMediaFileError {
NotFound,
Other(anyhow::Error),
}
async fn get_media_file_details(media_file_token: &MediaFileToken, mysql_pool: &MySqlPool) -> Result<PathBuf, GetMediaFileError> {
) -> Result<PathBuf, DownloadMediaFileError> {
let media_file_result = get_media_file(
&media_file_token,
false,
mysql_pool
).await;
let media_file_result = match media_file_result {
let media_file= match media_file_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media file: {:?}", media_file_token);
return Err(GetMediaFileError::NotFound)
warn!("could not find media file in database: {:?}", media_file_token);
return Err(DownloadMediaFileError::MediaFileNotFound)
}
Err(e) => {
error!("could not query media file: {:?}", e);
return Err(GetMediaFileError::Other(e))
return Err(DownloadMediaFileError::Other(e))
}
};
let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
&media_file_result.public_bucket_directory_hash,
media_file_result.maybe_public_bucket_prefix.as_deref(),
media_file_result.maybe_public_bucket_extension.as_deref());
&media_file.public_bucket_directory_hash,
media_file.maybe_public_bucket_prefix.as_deref(),
media_file.maybe_public_bucket_extension.as_deref());
Ok(media_file_bucket_path.to_full_object_pathbuf())
}
let extension = media_file.maybe_public_bucket_extension.clone().unwrap_or_else(|| ".jpg".to_string());
let download_filename = format!("download_file_{}.{}", media_file.token.as_str(), extension);
let download_file_path = download_dir.path().join(download_filename);
info!("Downloading from bucket...");
#[cfg(test)]
mod tests {
bucket_client.download_file_to_disk(
&media_file_bucket_path.to_full_object_pathbuf(),
&download_file_path,
).await?;
Ok(download_file_path)
}
@@ -5,6 +5,7 @@ pub mod app_state;
pub mod beta_keys;
pub mod categories;
pub mod comments;
pub mod control_plane;
pub mod conversion;
pub mod download_job;
pub mod engine;
@@ -12,6 +13,7 @@ pub mod events;
pub mod featured_items;
pub mod flags;
pub mod image_gen;
pub mod image_studio;
pub mod inference_job;
pub mod investor_demo;
pub mod leaderboard;
@@ -30,9 +32,9 @@ pub mod subscriptions;
pub mod tags;
pub mod trending;
pub mod tts;
pub mod users;
pub mod user_bookmarks;
pub mod user_ratings;
pub mod users;
pub mod vocoders;
pub mod voice_clone_requests;
pub mod voice_conversion;
@@ -40,4 +42,3 @@ pub mod voice_designer;
pub mod w2l;
pub mod weights;
pub mod workflows;
pub mod image_studio;
@@ -0,0 +1,24 @@
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
use crate::http_server::endpoints::control_plane::set_sora_secret_handler::set_sora_secret_handler;
pub fn add_control_plane_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/control_plane")
.service(web::resource("/sora_secret")
.route(web::post().to(set_sora_secret_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -123,6 +123,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp
use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler;
use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler;
use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler;
use crate::http_server::routes::add_control_plane_routes::add_control_plane_routes;
use crate::http_server::routes::add_image_studio_routes::add_image_studio_routes;
use crate::http_server::routes::add_studio_gen2_routes::add_studio_gen2_routes;
use crate::http_server::routes::beta_key_routes::add_beta_key_routes;
@@ -175,6 +176,7 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
app = add_image_studio_routes(app);
app = add_job_routes(app);
app = add_engine_routes(app); /* /v1/engine/... */
app = add_control_plane_routes(app); /* /v1/control_plane/... */
// app = add_image_gen_routes(app);
@@ -9,4 +9,5 @@ pub mod user_routes;
pub mod weights_routes;
pub mod workflow_routes;
mod tag_routes;
mod add_image_studio_routes;
mod add_image_studio_routes;
mod add_control_plane_routes;
@@ -11,6 +11,20 @@ pub struct ScopedTempDirCreator {
}
impl ScopedTempDirCreator {
pub fn auto_setup() -> Self {
// Production configuration
let maybe_tmp = easyenv::get_env_pathbuf_optional("TEMP_DIR");
// MacOS
// TODO: Only run this on macOS
let maybe_mac_tmp = easyenv::get_env_pathbuf_optional("TMPDIR");
let directory = maybe_tmp
.or(maybe_mac_tmp)
.unwrap_or_else(|| PathBuf::from("/tmp"));
Self::for_directory(directory)
}
pub fn for_directory<P: AsRef<Path>>(base_dir: P) -> Self {
Self {
@@ -464,7 +464,7 @@ async fn main() -> AnyhowResult<()> {
user_tokens: user_token_troll_bans,
ip_addresses: ip_address_troll_bans,
},
temp_dir_creator: ScopedTempDirCreator::for_directory(easyenv::get_env_pathbuf_or_default("TEMP_DIR", "/tmp")),
temp_dir_creator: ScopedTempDirCreator::auto_setup(),
google_sign_in_cert: GoogleSignInCert::new(),
};
@@ -11,5 +11,6 @@ pub mod enroll_in_studio;
pub mod generate_random_username;
pub mod placeholder_images;
pub mod read_toml_file_to_struct;
pub mod redis_sora_secrets;
pub mod title_to_url_slug;
pub mod troll_user_bans;
@@ -0,0 +1,67 @@
use std::collections::HashMap;
use anyhow::anyhow;
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::redis::Commands;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use openai_sora_client::credentials::SoraCredentials;
/// Redis key
const SORA_SECRET_REDIS_KEY : &str = "sora_secret";
// Fields within the HKEY
const BEARER_SUBKEY : &str = "bearer";
const COOKIE_SUBKEY : &str = "cookie";
const SENTINEL_SUBKEY : &str = "sentinel";
#[derive(Clone,Copy,Debug)]
pub enum RedisSoraCredentialSubkey {
Bearer,
Cookie,
Sentinel,
}
impl RedisSoraCredentialSubkey {
pub fn to_str(&self) -> &'static str {
match self {
RedisSoraCredentialSubkey::Bearer => BEARER_SUBKEY,
RedisSoraCredentialSubkey::Cookie => COOKIE_SUBKEY,
RedisSoraCredentialSubkey::Sentinel => SENTINEL_SUBKEY,
}
}
}
pub fn get_sora_credentials(
redis: &mut PooledConnection<RedisConnectionManager>
) -> AnyhowResult<SoraCredentials> {
let values : HashMap<String, String> = redis.hgetall(SORA_SECRET_REDIS_KEY)
.map_err(|e| anyhow!("Failed to get Sora credentials from Redis: {}", e))?;
let bearer = values.get(BEARER_SUBKEY);
let cookie = values.get(COOKIE_SUBKEY);
let sentinel = values.get(SENTINEL_SUBKEY);
match (bearer, cookie, sentinel) {
(Some(b), Some(c), Some(s)) => {
Ok(SoraCredentials {
bearer_token: b.to_string(),
cookie: c.to_string(),
sentinel: Some(s.to_string()),
})
}
_ => Err(anyhow!("redis sora credential values not present")),
}
}
pub fn set_sora_credential_field(
redis: &mut PooledConnection<RedisConnectionManager>,
subkey: RedisSoraCredentialSubkey,
value: &str,
) -> AnyhowResult<()> {
redis.hset(SORA_SECRET_REDIS_KEY, subkey.to_str(), value)
.map_err(|e| anyhow!("Failed to set Sora credentials in Redis: {}", e))?;
Ok(())
}
@@ -36,4 +36,174 @@ export class EngineApi extends ApiManager {
};
});
}
public async uploadSceneSnapshot({
screenshot,
sceneMediaToken,
}: {
screenshot: File; // base64 encoded PNG
sceneMediaToken?: string;
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
const formData = new FormData();
formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
if (sceneMediaToken) {
formData.append("scene_media_token", sceneMediaToken);
}
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token
const response = await fetch(endpoint, {
method: "POST",
headers: {
Accept: "application/json",
},
credentials: "include",
body: formData,
});
const postResponse = await response.json();
console.log(postResponse);
let result: { success: boolean; data?: string; errorMessage?: string };
if (postResponse.success) {
result = {
success: true,
data: postResponse.snapshot_media_token,
errorMessage: undefined,
};
} else {
result = {
success: false,
errorMessage: postResponse.BadInput,
};
}
return result;
}
public async enqueueImageGeneration({
prompt,
snapshotMediaToken,
additionalImages,
}: {
prompt: string;
snapshotMediaToken: string;
additionalImages?: string[];
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
const body = {
uuid_idempotency_token: uuidIdempotencyToken,
prompt,
snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
additional_images: additionalImages,
};
const postResponse = await this.post<
{
uuid_idempotency_token: string;
prompt: string;
snapshot_media_token: string;
additional_images?: string[];
},
{ success?: boolean; job_token?: string; BadInput?: string }
>({
endpoint,
body,
});
// Check if the response is successful
const isSuccess = postResponse.success ?? false;
// Prepare the result object
const result = {
success: isSuccess,
data: isSuccess ? postResponse.job_token : undefined,
errorMessage: isSuccess ? undefined : postResponse.BadInput,
};
return result;
}
public async pollJobSession(jobToken: string): Promise<
ApiResponse<{
status: string;
result: {
generated_images?: string[];
error?: string;
};
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
const response = await this.get<{
success?: boolean;
status?: string;
result?: {
generated_images?: string[];
error?: string;
};
BadInput?: string;
}>({
endpoint,
});
const success = response.success ?? false;
const status = response.status ?? "";
const result = response.result ?? {
generated_images: [],
error: undefined,
};
const errorMessage = response.BadInput;
return {
success,
data: { status, result },
errorMessage,
};
}
public async pollStudioSessionJobs(jobToken: string): Promise<
ApiResponse<{
status: string;
result: {
generated_images?: string[];
error?: string;
};
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
const response = await this.get<{
success?: boolean;
status?: string;
result?: {
generated_images?: string[];
error?: string;
};
BadInput?: string;
}>({
endpoint,
});
const success = response.success ?? false;
const status = response.status ?? "";
const result = response.result ?? {
generated_images: [],
error: undefined,
};
const errorMessage = response.BadInput;
return {
success,
data: { status, result },
errorMessage,
};
}
}
@@ -8,6 +8,7 @@ export interface PopoverItem {
selected: boolean;
icon?: ReactNode;
action?: string;
disabled?: boolean;
}
interface PopoverMenuProps {
@@ -42,11 +43,13 @@ export const PopoverMenu = ({
}: PopoverMenuProps) => {
const selectedItem = items.find((item) => item.selected);
const handleItemClick = (item: PopoverItem) => {
const handleItemClick = (item: PopoverItem, close: () => void) => {
if (mode === "button" && item.action && onAction) {
onAction(item.action);
close();
} else {
onSelect?.(item);
close();
}
};
@@ -60,67 +63,87 @@ export const PopoverMenu = ({
return (
<Popover className="relative">
<Popover.Button className={className}>
{triggerIcon}
{mode === "toggle" && selectedItem ? selectedItem.label : null}
{mode === "default" && triggerLabel ? triggerLabel : null}
</Popover.Button>
{({ close }) => (
<>
<Popover.Button className={className}>
{triggerIcon}
{mode === "toggle" && selectedItem ? selectedItem.label : null}
{mode === "default" && triggerLabel ? triggerLabel : null}
</Popover.Button>
<div className="absolute bottom-full left-0 transform-gpu">
<Transition
enter="transition duration-100 ease-out"
enterFrom="translate-y-2 opacity-0"
enterTo="translate-y-0 opacity-100"
leave="transition duration-100 ease-in"
leaveFrom="translate-y-0 opacity-100"
leaveTo="translate-y-2 opacity-0"
>
<Popover.Panel className="origin-bottom">
<div
className={twMerge(
"z-10 mb-2 min-w-48 rounded-lg bg-[#46464B] p-1.5 shadow-lg",
)}
<div className="absolute bottom-full left-0 transform-gpu">
<Transition
enter="transition duration-100 ease-out"
enterFrom="translate-y-2 opacity-0"
enterTo="translate-y-0 opacity-100"
leave="transition duration-100 ease-in"
leaveFrom="translate-y-0 opacity-100"
leaveTo="translate-y-2 opacity-0"
>
{panelTitle && (
<div className="mb-2 mt-0.5 px-1.5 text-sm font-normal text-white opacity-70">
{panelTitle}
</div>
)}
{mode === "default" && children ? (
<div className="text-sm text-white">{children}</div>
) : (
<div className="flex flex-col gap-0 text-sm text-white">
{items.map((item, index) => (
<Button
key={index}
className={`flex items-center justify-between border-transparent bg-transparent px-1.5 hover:bg-[#63636B]/60 ${mode === "toggle" && item.selected ? "hover:bg-[#63636B]" : "bg-transparent"}`}
onClick={() => handleItemClick(item)}
variant="secondary"
>
<div className="flex items-center gap-2">
{showIconsInList && item.icon}
{item.label}
</div>
{mode === "toggle" && (
<input type="radio" checked={item.selected} readOnly />
<Popover.Panel className="origin-bottom">
<div
className={twMerge(
"z-10 mb-2 min-w-48 rounded-lg bg-[#46464B] p-1.5 shadow-lg",
)}
>
{panelTitle && (
<div className="mb-2 mt-0.5 px-1.5 text-sm font-normal text-white opacity-70">
{panelTitle}
</div>
)}
{mode === "default" && children ? (
<div className="text-sm text-white">{children}</div>
) : (
<div className="flex flex-col gap-0 text-sm text-white">
{items.map((item, index) => (
<Button
key={index}
className={twMerge(
"flex items-center justify-between border-transparent bg-transparent px-1.5",
"hover:bg-[#63636B]/60",
mode === "toggle" && item.selected
? "hover:bg-[#63636B]"
: "bg-transparent",
item.disabled
? "!cursor-not-allowed opacity-50"
: "",
)}
onClick={() =>
!item.disabled && handleItemClick(item, close)
}
variant="secondary"
disabled={item.disabled}
>
<div className="flex items-center gap-2">
{showIconsInList && item.icon}
{item.label}
</div>
{mode === "toggle" && (
<input
type="radio"
checked={item.selected}
readOnly
/>
)}
</Button>
))}
{showAddButton && onAdd && (
<Button
variant="secondary"
className="mb-0.5 mt-2 border-none bg-[#7B7B84] py-1 hover:bg-[#8c8c96]"
onClick={onAdd}
>
+ Add
</Button>
)}
</Button>
))}
{showAddButton && onAdd && (
<Button
variant="secondary"
className="mb-0.5 mt-2 border-none bg-[#7B7B84] py-1 hover:bg-[#8c8c96]"
onClick={onAdd}
>
+ Add
</Button>
</div>
)}
</div>
)}
</div>
</Popover.Panel>
</Transition>
</div>
</Popover.Panel>
</Transition>
</div>
</>
)}
</Popover>
);
};
@@ -15,6 +15,8 @@ export const uploadImage = async ({
}) => {
const mediaUploadApi = new MediaUploadApi();
console.log("Uploading image:", assetFile);
progressCallback({ status: UploaderStates.uploadingImage });
const imageResponse = await mediaUploadApi.UploadImage({
@@ -24,7 +26,10 @@ export const uploadImage = async ({
maybe_title: "char_frame_" + title,
});
console.log("Image response:", imageResponse);
if (imageResponse == undefined) {
console.log("Error: Could not upload image!");
progressCallback({
status: UploaderStates.imageCreateError,
errorMessage: "Could not upload image!",
@@ -33,6 +38,7 @@ export const uploadImage = async ({
}
if (!imageResponse.success || !imageResponse.data) {
console.log("Error:", imageResponse.errorMessage);
progressCallback({
status: UploaderStates.imageCreateError,
errorMessage: imageResponse.errorMessage,
@@ -40,5 +46,9 @@ export const uploadImage = async ({
return;
}
progressCallback({ status: UploaderStates.success, data: imageResponse.data });
console.log("Upload successful:", imageResponse.data);
progressCallback({
status: UploaderStates.success,
data: imageResponse.data,
});
};
@@ -0,0 +1,146 @@
import { TransformControls } from "three/examples/jsm/Addons";
import * as THREE from "three";
export const FKHelperSphereName = "__FKHelperSphere__";
export class FKHelper {
private targetBoneSpheres: THREE.Object3D<THREE.Object3DEventMap>[];
private transformControls: TransformControls;
private raycaster: THREE.Raycaster;
private scene: THREE.Scene;
private skeletonHelper: THREE.SkeletonHelper | null = null;
private isControlInScene = false;
// Need the actual object to perform FK on
// This means the object must have bones
// Not all bones require FK right now
// Need a set of names to filter bones
// Use mixamo names
// Need a transform control (rotation mode) for the bones
// Need to disable the object transforms for bone transforms
// Need raycaster
constructor({
camera,
domElement,
scene,
onDragChange
}: {
camera: THREE.Camera;
domElement: HTMLElement;
scene: THREE.Scene;
onDragChange: (dragging: boolean) => void;
}) {
this.raycaster = new THREE.Raycaster();
this.scene = scene;
this.targetBoneSpheres = [];
this.transformControls = new TransformControls(camera, domElement);
this.transformControls.setSpace("local");
this.transformControls.setMode("rotate");
this.transformControls.setSize(0.5);
this.transformControls.addEventListener("dragging-changed", (event: any) => {
onDragChange(event.value);
});
}
// Set FK target
setTarget(target: THREE.Object3D) {
// TODO: Highlight all the necessary bones
this.clear();
target.traverse((child) => {
if (child.type !== "Bone") {
return;
}
// For each target bone, display a sphere to show the FK target and raycast intersect
const geometry = new THREE.SphereGeometry(1, 16, 16);
const material = new THREE.MeshBasicMaterial({ color: 0xff0000, depthTest: false, transparent: true });
const sphere = new THREE.Mesh(geometry, material);
sphere.name = FKHelperSphereName;
const uniformScale = 1.5;
sphere.scale.set(uniformScale, uniformScale, uniformScale);
child.add(sphere);
this.targetBoneSpheres.push(sphere);
});
this.skeletonHelper = new THREE.SkeletonHelper(target);
this.scene.add(this.skeletonHelper);
}
onMouseClick(mouse: THREE.Vector2) {
this.raycaster.setFromCamera(mouse, this.transformControls.camera);
const intersections = this.raycaster.intersectObjects(this.targetBoneSpheres, false);
if (intersections.length < 1) {
console.log("No intersections");
this.resetHighlight();
this.removeControls();
return;
}
// Attach the controls to the first bone
const firstBone = intersections[0];
this.highlightSphere(firstBone.object);
this.showControls(firstBone.object.parent!);
console.log("Showing bone", firstBone.object);
}
private showControls(object?: THREE.Object3D) {
if (object) {
this.transformControls.attach(object);
}
if (this.isControlInScene) {
return;
}
this.scene.add(this.transformControls);
this.isControlInScene = true;
}
private highlightSphere(sphere: THREE.Object3D) {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 0.2
});
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
sphere.material.opacity = 1;
}
private resetHighlight() {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 1;
});
}
private removeControls() {
this.transformControls.detach();
if (this.isControlInScene) {
this.scene.remove(this.transformControls);
this.isControlInScene = false;
}
}
private removeSpheres() {
this.targetBoneSpheres.forEach((sphere) => {
this.scene.remove(sphere);
sphere.parent?.remove(sphere);
// @ts-expect-error Geometry on object3d, but it's fine since this is a mesh
sphere.geometry.dispose();
});
this.targetBoneSpheres = [];
}
clear() {
this.removeControls();
this.removeSpheres();
if (this.skeletonHelper) {
this.scene.remove(this.skeletonHelper!);
}
}
}
@@ -66,7 +66,7 @@ class Editor {
camera: THREE.PerspectiveCamera | null = null;
render_camera: THREE.PerspectiveCamera | null = null;
render_camera_aspect_ratio: CameraAspectRatio =
CameraAspectRatio.HORIZONTAL_16_9;
CameraAspectRatio.HORIZONTAL_3_2;
renderer: THREE.WebGLRenderer | undefined;
rawRenderer: THREE.WebGLRenderer | undefined;
clock: THREE.Clock | undefined;
@@ -253,7 +253,7 @@ class Editor {
this.selectedCanvas = false;
// Audio Engine Test.
this.render_camera_aspect_ratio = CameraAspectRatio.HORIZONTAL_16_9;
this.render_camera_aspect_ratio = CameraAspectRatio.HORIZONTAL_3_2;
this.render_width = this.getRenderDimensions().width;
this.render_height = this.getRenderDimensions().height;
@@ -334,6 +334,20 @@ class Editor {
aspectRatio: 16 / 9,
};
}
case CameraAspectRatio.HORIZONTAL_3_2: {
return {
width: 1200,
height: 800,
aspectRatio: 3 / 2,
};
}
case CameraAspectRatio.VERTICAL_2_3: {
return {
width: 800,
height: 1200,
aspectRatio: 2 / 3,
};
}
case CameraAspectRatio.VERTICAL_9_16: {
return {
width: 720,
@@ -641,6 +655,35 @@ class Editor {
document.body.appendChild(this.stats.dom);
}
public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
const snapshot = this.renderer?.domElement.toDataURL("image/png");
if (!snapshot) {
console.error("Error: Snapshot could not be created.");
return null;
}
const base64Snapshot = snapshot.split(",")[1];
if (shouldDownload) {
const link = document.createElement("a");
link.download = "scene-snapshot.png";
link.href = snapshot;
link.click();
}
const byteString = atob(base64Snapshot);
const mimeString = "image/png";
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
const uuid = crypto.randomUUID(); // Generate a new UUID
const file = new File([ab], `${uuid}.png`, { type: mimeString });
return { base64Snapshot, file };
}
public async newScene(sceneTitleInput: string) {
this.activeScene.clear();
this.audio_engine = new AudioEngine();
@@ -917,6 +960,8 @@ class Editor {
}
deleteObject(uuid: string) {
this.mouse_controls?.clearFKVisuals();
this.mouse_controls?.removeTransformControls(true);
this.utils.deleteObject(uuid);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
@@ -948,16 +993,28 @@ class Editor {
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 1024
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_9_16
meraAspectRatio.VERTICAL_9_16
? 576
: 1000;
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 576
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_9_16
meraAspectRatio.VERTICAL_9_16
? 1024
: 1000;
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
this.rawRenderer.setSize(width, height);
@@ -14,6 +14,13 @@ import { TransformControls } from "./TransformControls";
import { SceneManager, SceneObject } from "./scene_manager_api";
import { FreeCam } from "./free_cam";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { FKHelper } from "./KinHelpers/FKHelper";
export enum KinMode {
FK,
IK,
NONE,
}
export class MouseControls {
camera: THREE.PerspectiveCamera | null;
@@ -47,8 +54,12 @@ export class MouseControls {
private isMovable: Function;
enable_stats: Function;
private kinMode: KinMode = KinMode.NONE;
private fkHelper: FKHelper;
private isBoneDragged: boolean = false;
constructor(
camera: THREE.PerspectiveCamera | null,
camera: THREE.PerspectiveCamera,
camera_person_mode: boolean,
cameraViewControls: FreeCam,
lockControls: PointerLockControls | undefined,
@@ -62,7 +73,7 @@ export class MouseControls {
mouse: THREE.Vector2 | undefined,
timeline_mouse: THREE.Vector2 | undefined,
raycaster: THREE.Raycaster | undefined,
control: TransformControls | undefined,
control: TransformControls,
outlinePass: OutlinePass | undefined,
scene: THREE.Scene,
publishSelect: Function,
@@ -101,6 +112,20 @@ export class MouseControls {
this.sceneManager = undefined;
this.isMovable = isMovable;
this.enable_stats = enable_stats;
this.fkHelper = new FKHelper({
camera: this.camera,
domElement: this.control.domElement,
scene: this.scene,
onDragChange: this.onFKControlsDragging.bind(this),
});
}
onFKControlsDragging(dragging: boolean) {
this.isBoneDragged = dragging;
}
clearFKVisuals() {
this.fkHelper.clear();
}
focus() {
@@ -125,9 +150,26 @@ export class MouseControls {
this.selected = [];
this.publishSelect();
}
this.hideTransformControls();
if (remove_outline) this.outlinePass.selectedObjects = [];
}
hideTransformControls() {
if (this.control == undefined) {
return;
}
this.control.detach();
this.scene.remove(this.control);
if (remove_outline) this.outlinePass.selectedObjects = [];
}
reattachTransformControls() {
if (this.control == undefined || this.selected == undefined) {
return;
}
this.control.attach(this.selected[0]);
this.scene.add(this.control);
}
selectObject(currentObject: THREE.Object3D) {
@@ -151,7 +193,7 @@ export class MouseControls {
}
onMouseDown(event: any) {
if ((event.button === 0 || event.button === 1) && this.isMovable()) {
if ((event.button === 0 || event.button === 1) && this.isMovable() && !this.isBoneDragged) {
this.isMouseClicked = true;
}
}
@@ -176,6 +218,35 @@ export class MouseControls {
return new Promise((resolve) => setTimeout(resolve, ms));
}
toggleFKMode() {
if (this.kinMode == KinMode.FK) {
this.fkHelper.clear();
this.kinMode = KinMode.NONE;
this.reattachTransformControls();
console.log("FK mode off");
return;
}
// Make sure we have an intersection
if (!(this.selected && this.selected.length > 0)) {
return;
}
// Make sure FK is supported only on character type objects
const firstSelection = this.selected[0];
if (!firstSelection.userData.isCharacter) {
return;
}
// FK is good to go
// Disable main transform controls
this.hideTransformControls();
this.kinMode = KinMode.FK;
this.fkHelper.setTarget(this.selected[0]);
console.log("FK mode on");
return;
}
async onkeydown(event: KeyboardEvent) {
if (hotkeysStatus.value.disabled) {
return;
@@ -198,7 +269,7 @@ export class MouseControls {
// transform
this.control?.setMode("translate");
return;
} else if (event.key === "r") {
} else if (event.key === "r" && !event.ctrlKey) {
// rotate
this.control?.setMode("rotate");
return;
@@ -206,6 +277,8 @@ export class MouseControls {
// scale
this.control?.setMode("scale");
return;
} else if (event.key === "k") {
this.toggleFKMode();
}
if ((event.ctrlKey || event.metaKey) && !this.isProcessing) {
@@ -298,6 +371,11 @@ export class MouseControls {
}
this.camera_last_pos.copy(camera_pos);
if (this.kinMode == KinMode.FK) {
this.fkHelper.onMouseClick(this.mouse);
return;
}
this.raycaster.setFromCamera(this.mouse, this.camera);
const interactable: any[] = [];
this.scene.children.forEach((child: THREE.Object3D) => {
@@ -287,10 +287,6 @@ export class TimeLine {
// testDeformBody(lastRig);
}
// Add a skeleton helper to the character
const skeletonHelper = new THREE.SkeletonHelper(obj);
this.scene.scene.add(skeletonHelper);
const object_uuid = obj.uuid;
this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.
@@ -113,7 +113,6 @@ export const PageEditor = () => {
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
<AspectRatioMenu />
</div>
</div>
@@ -86,7 +86,7 @@ export const Wizard = () => {
onNext: () => {
resetSceneGenerationMetadata();
editorEngine?.changeRenderCameraAspectRatio(
CameraAspectRatio.HORIZONTAL_16_9,
CameraAspectRatio.HORIZONTAL_3_2,
);
editorEngine?.newScene(textInput.value);
showWizard.value = "end";
@@ -18,13 +18,21 @@ export const AspectRatioMenu = () => {
? faRectangle
: cameraAspectRatio.value === CameraAspectRatio.VERTICAL_9_16
? faRectangleVertical
: faSquare;
: cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2
? faRectangle
: cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3
? faRectangleVertical
: faSquare;
const buttonText =
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9
? "16:9 Horizontal"
: cameraAspectRatio.value === CameraAspectRatio.VERTICAL_9_16
? "9:16 Vertical"
: "1:1 Squared";
: cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2
? "3:2 Horizontal"
: cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3
? "2:3 Vertical"
: "1:1 Squared";
const handleChangeAspectRatio = (newRatio: CameraAspectRatio) => {
Queue.publish({
@@ -52,6 +60,26 @@ export const AspectRatioMenu = () => {
onClick: () =>
handleChangeAspectRatio(CameraAspectRatio.HORIZONTAL_16_9),
},
{
label: "3:2",
icon: faRectangle,
className: "pl-4",
description: "Horizontal",
selected:
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2,
onClick: () =>
handleChangeAspectRatio(CameraAspectRatio.HORIZONTAL_3_2),
},
{
label: "2:3",
icon: faRectangleVertical,
className: "pl-4",
description: "Vertical",
selected:
cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3,
onClick: () =>
handleChangeAspectRatio(CameraAspectRatio.VERTICAL_2_3),
},
{
label: "9:16",
icon: faRectangleVertical,
@@ -67,7 +67,7 @@ export const ControlsTopButtons = () => {
const handleResetScene = () => {
resetSceneGenerationMetadata();
editorEngine?.changeRenderCameraAspectRatio(
CameraAspectRatio.HORIZONTAL_16_9,
CameraAspectRatio.HORIZONTAL_3_2,
);
};
@@ -1,5 +1,8 @@
import { useState, useRef, useEffect } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
import { useContext } from "react";
import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
import {
faPlus,
faCamera,
@@ -9,19 +12,61 @@ import {
faImages,
faDownload,
faSpinnerThird,
faTimes,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangleWide,
faRectangleVertical,
faSquare,
faRectangle,
} from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
import { Button } from "~/components";
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
import { cameraAspectRatio, disableHotkeyInput, DomLevels, enableHotkeyInput } from "~/pages/PageEnigma/signals";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
import { EngineApi } from "~/Classes/ApiManager/EngineApi";
import { scene } from "~/signals/scene";
import { UploaderStates } from "~/enums/UploaderStates";
export const PromptBox = () => {
useSignals();
const editorEngine = useContext(EngineContext);
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setisEnqueueing] = useState(false);
const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{ label: "16:9", selected: true },
{ label: "3:2", selected: false },
{ label: "2:3", selected: false },
{ label: "1:1", selected: false },
{
label: "3:2",
selected: true,
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "2:3",
selected: false,
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "1:1",
selected: false,
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
{
label: "16:9",
selected: false,
icon: <FontAwesomeIcon icon={faRectangleWide} className="h-4 w-4" />,
},
]);
const [cameraList, setCameraList] = useState<PopoverItem[]>([
{
@@ -45,6 +90,25 @@ export const PromptBox = () => {
}
}, [prompt]);
// Update aspect ratio list based on the current cameraAspectRatio signal
useEffect(() => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected:
(item.label === "16:9" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) ||
(item.label === "3:2" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) ||
(item.label === "2:3" &&
cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) ||
(item.label === "1:1" &&
cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1),
})),
);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [cameraAspectRatio.value]);
const handleCameraSelect = (selectedItem: PopoverItem) => {
setCameraList((prev) =>
prev.map((item) => ({
@@ -73,12 +137,70 @@ export const PromptBox = () => {
selected: item.label === selectedItem.label,
})),
);
// Map the selected label to the corresponding CameraAspectRatio enum value
let newRatio: CameraAspectRatio;
switch (selectedItem.label) {
case "16:9":
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
break;
case "3:2":
newRatio = CameraAspectRatio.HORIZONTAL_3_2;
break;
case "2:3":
newRatio = CameraAspectRatio.VERTICAL_2_3;
break;
case "1:1":
newRatio = CameraAspectRatio.SQUARE_1_1;
break;
default:
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
}
// Publish the change to the engine
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.CHANGE_CAMERA_ASPECT_RATIO,
data: newRatio,
});
};
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (file) {
console.log("File uploaded:", file);
const files = event.target.files;
if (files) {
Array.from(files).forEach((file) => {
const reader = new FileReader();
reader.onloadend = () => {
uploadImage({
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
console.debug("Upload progress:", newState.data);
if (newState.status === UploaderStates.success && newState.data) {
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: reader.result as string,
file,
mediaToken: newState.data || "",
};
setReferenceImages((prev) => [...prev, referenceImage]);
console.log("Reference image added:", referenceImage);
} else {
console.error("No data from uploadImage");
}
},
});
};
reader.readAsDataURL(file);
});
}
};
const handleRemoveReference = (id: string) => {
setReferenceImages((prev) => prev.filter((img) => img.id !== id));
if (fileInputRef.current) {
fileInputRef.current.value = "";
}
};
@@ -116,13 +238,37 @@ export const PromptBox = () => {
const handleEnqueue = async () => {
if (!prompt.trim()) return;
setisEnqueueing(true);
try {
// generate logic here
console.log("Enqueuing with prompt:", prompt);
await new Promise((resolve) => setTimeout(resolve, 2000)); // dummy delay here
} finally {
setisEnqueueing(false);
if (editorEngine) {
editorEngine.positive_prompt = prompt;
const engineApi = new EngineApi();
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const snapshotResult = await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file,
sceneMediaToken: "",
});
await engineApi.enqueueImageGeneration({
prompt: prompt,
snapshotMediaToken: snapshotResult.data || "",
additionalImages: referenceImages.map((image) => image.mediaToken),
});
}
setisEnqueueing(true);
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
);
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
setisEnqueueing(false);
}
}
};
@@ -133,101 +279,181 @@ export const PromptBox = () => {
}
};
return (
<div className="glass absolute bottom-4 left-1/2 w-[730px] -translate-x-1/2 rounded-xl p-4">
<input
type="file"
ref={fileInputRef}
className="hidden"
accept="image/*"
onChange={handleFileUpload}
/>
<div className="flex justify-center gap-2">
<PopoverMenu
mode="button"
panelTitle="Add Image"
items={[
{
label: "Upload from device",
selected: false,
icon: <FontAwesomeIcon icon={faUpload} className="h-4 w-4" />,
action: "upload",
},
{
label: "Choose from library",
selected: false,
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
action: "library",
},
]}
onAction={handleAction}
showIconsInList
buttonClassName="backdrop-blur-none bg-transparent hover:bg-transparent py-1.5 px-0 pr-0.5 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
triggerIcon={<FontAwesomeIcon icon={faPlus} className="text-xl" />}
/>
// Get the current aspect ratio icon based on the cameraAspectRatio signal
const getCurrentAspectRatioIcon = () => {
switch (cameraAspectRatio.value) {
case CameraAspectRatio.HORIZONTAL_16_9:
return faRectangleWide;
case CameraAspectRatio.HORIZONTAL_3_2:
return faRectangle;
case CameraAspectRatio.VERTICAL_2_3:
return faRectangleVertical;
case CameraAspectRatio.VERTICAL_9_16:
return faRectangleVertical;
case CameraAspectRatio.SQUARE_1_1:
return faSquare;
default:
return faRectangleWide;
}
};
<textarea
ref={textareaRef}
rows={1}
placeholder="Describe your image..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent px-2 pb-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
const handleSaveFrame = async () => {
if (editorEngine) {
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const engineApi = new EngineApi();
await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file || "",
sceneMediaToken: "",
});
}
}
};
const handleDownloadFrame = () => {
if (editorEngine) {
editorEngine.snapShotOfCurrentFrame(true);
}
};
return (
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
{referenceImages.length > 0 && (
<div className="flex w-full gap-2">
{referenceImages.map((image) => (
<div
key={image.id}
className="glass relative aspect-square w-20 rounded-lg"
>
<img
src={image.url}
alt="Reference"
className="h-full w-full rounded-lg object-cover"
/>
<button
onClick={() => handleRemoveReference(image.id)}
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/40 text-white backdrop-blur-md transition-colors hover:bg-black"
>
<FontAwesomeIcon icon={faTimes} className="h-2.5 w-2.5" />
</button>
</div>
))}
</div>
)}
<div className="glass w-[730px] rounded-xl p-4">
<input
type="file"
ref={fileInputRef}
className="hidden"
accept="image/*"
onChange={handleFileUpload}
multiple
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<div className="flex justify-center gap-2">
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
/>
<PopoverMenu
items={cameraList}
onSelect={handleCameraSelect}
onAdd={handleAddCamera}
triggerIcon={
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
}
showAddButton
mode="button"
panelTitle="Add Image"
items={[
{
label: "Upload from device",
selected: false,
icon: <FontAwesomeIcon icon={faUpload} className="h-4 w-4" />,
action: "upload",
},
{
label: "Choose from library",
selected: false,
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
action: "library",
disabled: true,
},
]}
onAction={handleAction}
showIconsInList
mode="toggle"
panelTitle="Camera"
buttonClassName="backdrop-blur-none bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
triggerIcon={<FontAwesomeIcon icon={faPlus} className="text-xl" />}
/>
<textarea
ref={textareaRef}
rows={1}
placeholder="Describe your image..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => {
console.log("disabling hotkeys")
disableHotkeyInput(DomLevels.INPUT);
}}
onBlur={() => {
console.log("enabling hotkeys")
enableHotkeyInput(DomLevels.INPUT);
}}
/>
</div>
<div className="flex items-center gap-2">
<Button
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faDownload}
>
Download frame
</Button>
<Button
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faSave}
>
Save frame
</Button>
<Button
className="flex items-center border-none bg-brand-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={!isEnqueueing ? faSparkles : undefined}
onClick={handleEnqueue}
disabled={isEnqueueing || !prompt.trim()}
>
{isEnqueueing ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
<PopoverMenu
items={cameraList}
onSelect={handleCameraSelect}
onAdd={handleAddCamera}
triggerIcon={
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
}
showAddButton
showIconsInList
mode="toggle"
panelTitle="Camera"
/>
</div>
<div className="flex items-center gap-2">
<Button
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faDownload}
onClick={handleDownloadFrame}
>
Download frame
</Button>
<Button
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faSave}
onClick={handleSaveFrame}
>
Save frame
</Button>
<Button
className="flex items-center border-none bg-brand-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={!isEnqueueing ? faSparkles : undefined}
onClick={handleEnqueue}
disabled={isEnqueueing || !prompt.trim()}
>
{isEnqueueing ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
</div>
</div>
</div>
</div>
@@ -51,7 +51,7 @@ export const Letterbox = ({
<div
id="letterbox"
className={twMerge(
"absolute left-0 top-0 flex h-full w-full justify-between",
"absolute left-0 top-0 flex h-full w-full justify-between pointer-events-none",
matteOri === MatteOrientation.TOP_BOTTOM ? "flex-col" : null,
)}
>
@@ -25,6 +25,16 @@ export const getMatteOrientation = ({
? MatteOrientation.TOP_BOTTOM
: MatteOrientation.LEFT_RIGHT;
}
case CameraAspectRatio.VERTICAL_2_3: {
return (height / 3) * 2 - width >= 0
? MatteOrientation.TOP_BOTTOM
: MatteOrientation.LEFT_RIGHT;
}
case CameraAspectRatio.HORIZONTAL_3_2: {
return (width / 3) * 2 - height >= 0
? MatteOrientation.LEFT_RIGHT
: MatteOrientation.TOP_BOTTOM;
}
case CameraAspectRatio.HORIZONTAL_16_9:
default: {
return (width / 16) * 9 - height >= 0
@@ -58,6 +68,12 @@ export const calcMatteWidth = ({
case CameraAspectRatio.VERTICAL_9_16: {
return (width - (height / 16) * 9) / 2;
}
case CameraAspectRatio.HORIZONTAL_3_2: {
return (width - (height / 2) * 3) / 2;
}
case CameraAspectRatio.VERTICAL_2_3: {
return (width - (height / 3) * 2) / 2;
}
default: {
return undefined;
}
@@ -88,6 +104,12 @@ export const calcMatteHeight = ({
case CameraAspectRatio.VERTICAL_9_16: {
return (height - (width / 9) * 16) / 2;
}
case CameraAspectRatio.HORIZONTAL_3_2: {
return (height - (width / 3) * 2) / 2;
}
case CameraAspectRatio.VERTICAL_2_3: {
return (height - (width / 2) * 3) / 2;
}
default: {
return undefined;
}
@@ -1,10 +1,17 @@
import { ReactNode } from "react";
import { FontAwesomeIconProps } from "@fortawesome/react-fontawesome";
import { faBrush, faCatSpace } from "@fortawesome/pro-solid-svg-icons";
import {
faAlienMonster,
faCatSpace,
faCloud,
faCube,
} from "@fortawesome/pro-solid-svg-icons";
import { TabTitles } from "~/enums";
import { StylizeTab } from "~/pages/PageEnigma/comps/SidePanelTabs/tabComps/StylizeTab/StylizeTab";
import { CartoonsTab } from "../SidePanelTabs/tabComps/CharactersComboTabs";
import { ObjectsTab } from "../SidePanelTabs/tabComps/ObjectsTab";
import { SkyboxesTab } from "../SidePanelTabs/tabComps/SkyboxesTab";
import { CreaturesTab } from "../SidePanelTabs/tabComps/CreaturesTab";
export interface TabItem {
icon: FontAwesomeIconProps["icon"];
@@ -19,9 +26,21 @@ export const tabList: TabItem[] = [
component: <CartoonsTab />,
},
// {
// icon: faBrush,
// title: TabTitles.STYLIZE,
// component: <StylizeTab />,
// }
{
icon: faCube,
title: TabTitles.OBJECTS,
component: <ObjectsTab />,
},
{
icon: faCloud,
title: TabTitles.SKYBOXES,
component: <SkyboxesTab />,
},
{
icon: faAlienMonster,
title: TabTitles.OBJECTS_CREATURES,
component: <CreaturesTab />,
},
];
@@ -8,4 +8,6 @@ export enum CameraAspectRatio {
HORIZONTAL_16_9 = "horizontal_16_9",
VERTICAL_9_16 = "vertical_9_16",
SQUARE_1_1 = "SQUARE_1_1",
HORIZONTAL_3_2 = "horizontal_3_2",
VERTICAL_2_3 = "vertical_2_3",
}
@@ -4,7 +4,7 @@ import { CameraAspectRatio, EditorStates } from "~/pages/PageEnigma/enums";
export const editorState = signal<EditorStates>(EditorStates.EDIT);
export const cameraAspectRatio = signal<CameraAspectRatio>(
CameraAspectRatio.HORIZONTAL_16_9,
CameraAspectRatio.HORIZONTAL_3_2,
);
export const setCameraAspectRatio = (newAspectRatio: CameraAspectRatio) => {
+12
View File
@@ -9,6 +9,7 @@
"@awesome.me/kit-fde2be5eb0": "^1.0.7",
"@fortawesome/fontawesome-svg-core": "^6.5.1",
"@fortawesome/free-brands-svg-icons": "^6.5.2",
"@fortawesome/pro-regular-svg-icons": "^6.5.1",
"@fortawesome/pro-solid-svg-icons": "^6.5.1",
"@fortawesome/react-fontawesome": "^0.2.0",
"@headlessui/react": "^1.7.18",
@@ -1423,6 +1424,17 @@
"node": ">=6"
}
},
"node_modules/@fortawesome/pro-regular-svg-icons": {
"version": "6.7.2",
"resolved": "https://npm.fontawesome.com/@fortawesome/pro-regular-svg-icons/-/6.7.2/pro-regular-svg-icons-6.7.2.tgz",
"integrity": "sha512-cJolK4K/s56mHAqOyno24SmjviHqSqG9ZfckBKCpWjJlgDexi61EfET5wWbmmEot6/BqasFjw9S6tudFQUL/PQ==",
"dependencies": {
"@fortawesome/fontawesome-common-types": "6.7.2"
},
"engines": {
"node": ">=6"
}
},
"node_modules/@fortawesome/pro-solid-svg-icons": {
"version": "6.7.2",
"resolved": "https://npm.fontawesome.com/@fortawesome/pro-solid-svg-icons/-/6.7.2/pro-solid-svg-icons-6.7.2.tgz",
+1
View File
@@ -21,6 +21,7 @@
"@awesome.me/kit-fde2be5eb0": "^1.0.7",
"@fortawesome/fontawesome-svg-core": "^6.5.1",
"@fortawesome/free-brands-svg-icons": "^6.5.2",
"@fortawesome/pro-regular-svg-icons": "^6.5.1",
"@fortawesome/pro-solid-svg-icons": "^6.5.1",
"@fortawesome/react-fontawesome": "^0.2.0",
"@headlessui/react": "^1.7.18",