proxy requests through tauri to avoid 3rd party cookie ban

There's an issue when the "main window thread" inquires about the
webview cookies shortly after app startup. When it attempts to dump the
cookies within a few milliseconds of app launch, it deadlocks and the
app never launches correctly. The entire webview goes blank and the app
freezes.

The timed waiting seems to fix this entirely.

Cargo.lock

@@ -2877,6 +2877,12 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
 
+[[package]]
+name = "data-url"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a"
+
 [[package]]
 name = "datetimes"
 version = "0.0.1"
@@ -6784,6 +6790,15 @@ dependencies = [
  "workspace-build-acceleration",
 ]
 
+[[package]]
+name = "memory_store"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "errors",
+ "workspace-build-acceleration",
+]
+
 [[package]]
 name = "metal"
 version = "0.27.0"
@@ -11280,6 +11295,7 @@ dependencies = [
  "anyhow",
  "backoff",
  "chrono",
+ "cookie 0.18.1",
  "enums",
  "enums_public",
  "errors",
@@ -12070,6 +12086,7 @@ dependencies = [
  "libloading 0.8.6",
  "log",
  "memory_caching",
+ "memory_store",
  "once_cell",
  "openai_sora_client",
  "primitives",
@@ -12084,6 +12101,7 @@ dependencies = [
  "subprocess_common",
  "tauri",
  "tauri-build",
+ "tauri-plugin-http",
  "tauri-plugin-log",
  "tempdir",
  "tempfile",
@@ -12173,6 +12191,53 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "tauri-plugin-fs"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88371e340ad2f07409a3b68294abe73f20bc9c1bc1b631a31dc37a3d0161f682"
+dependencies = [
+ "anyhow",
+ "dunce",
+ "glob",
+ "percent-encoding",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_repr",
+ "tauri",
+ "tauri-plugin",
+ "tauri-utils",
+ "thiserror 2.0.12",
+ "toml 0.8.20",
+ "url",
+ "uuid 1.8.0",
+]
+
+[[package]]
+name = "tauri-plugin-http"
+version = "2.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40dcd6c922a1885e1f0bcebc6768fec6e005bd4b9001c5d90a2f5d4cab297729"
+dependencies = [
+ "bytes 1.10.0",
+ "cookie_store",
+ "data-url",
+ "http 1.1.0",
+ "regex",
+ "reqwest 0.12.15",
+ "schemars",
+ "serde",
+ "serde_json",
+ "tauri",
+ "tauri-plugin",
+ "tauri-plugin-fs",
+ "thiserror 2.0.12",
+ "tokio",
+ "url",
+ "urlpattern",
+]
+
 [[package]]
 name = "tauri-plugin-log"
 version = "2.3.1"

Cargo.toml

@@ -39,6 +39,7 @@ members = [
   "crates/lib/bootstrap",
   "crates/lib/bucket_paths",
   "crates/lib/caching/memory_caching",
+  "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
   "crates/lib/concurrency",
   "crates/lib/cookies",
@@ -184,6 +185,7 @@ errors = { path = "crates/lib/errors" }
 filesys = { path = "crates/lib/files/filesys" }
 images = { path = "crates/lib/files/images" }
 memory_caching = { path = "crates/lib/caching/memory_caching" }
+memory_store = { path = "crates/lib/caching/memory_store" }
 mimetypes = { path = "crates/lib/files/mimetypes" }
 openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
 primitives = { path = "crates/lib/data/primitives" }

crates/desktop/tauri-artcraft/Cargo.toml

@@ -27,6 +27,7 @@ crockford = { workspace = true }
 errors = { workspace = true }
 filesys = { workspace = true }
 memory_caching = { workspace = true }
+memory_store = { workspace = true }
 openai_sora_client = { workspace = true }
 primitives = { workspace = true }
 storyteller_client = { workspace = true }
@@ -53,6 +54,7 @@ serde_derive = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 tauri = { version = "2.4.1", features = ["unstable"] }
+tauri-plugin-http = "2.0.0"
 tauri-plugin-log = "2.3.1"
 tempdir = "0.3.7"
 tempfile = "3.18.0"

crates/desktop/tauri-artcraft/capabilities/default.json

@@ -7,6 +7,14 @@
   ],
   "permissions": [
     "core:default",
-    "log:default"
+    "log:default",
+    {
+      "identifier": "http:default",
+      "allow": [
+        { "url": "https://*.storyteller.ai" },
+        { "url": "https://*.fakeyou.com" }
+      ],
+      "deny": [{ "url": "https://private.tauri.app" }]
+    }
   ]
 }

crates/desktop/tauri-artcraft/src/commands/mod.rs

@@ -1,2 +1,3 @@
 pub mod flip_image;
+pub mod platform_info_command;
 pub mod sora;

crates/desktop/tauri-artcraft/src/commands/platform_info_command.rs

@@ -0,0 +1,63 @@
+use crate::state::os_platform::OsPlatform;
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use image::codecs::png::{CompressionType, FilterType, PngEncoder};
+use image::ImageReader;
+use log::info;
+use serde_derive::Serialize;
+use std::io::Cursor;
+
+#[derive(Debug, Serialize)]
+pub struct PlatformInfoResponse {
+  pub os_platform: DetectedOs,
+  pub webview_runtime: WebviewRuntime,
+}
+
+#[derive(Copy, Clone, Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DetectedOs {
+  Windows,
+  #[serde(rename = "macos")]
+  MacOS,
+  Linux,
+  Unknown,
+}
+
+#[derive(Copy, Clone, Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum WebviewRuntime {
+  #[serde(rename = "webkit_safari")]
+  WebkitSafari,
+  #[serde(rename = "webkit_gtk")]
+  WebkitGtk,
+  #[serde(rename = "edge_webview_2")]
+  EdgeWebview2,
+  Unknown,
+}
+
+#[tauri::command]
+pub fn platform_info_command() -> Result<PlatformInfoResponse, String> {
+  info!("platform_info_command called...");
+
+  let os_platform = match OsPlatform::maybe_get() {
+    Some(OsPlatform::Linux) => DetectedOs::Linux,
+    Some(OsPlatform::MacOs) => DetectedOs::MacOS,
+    Some(OsPlatform::Windows) => DetectedOs::Windows,
+    None => DetectedOs::Unknown,
+  };
+
+  // These are the webviews that Tauri uses on each OS:
+  // https://tauri.app/reference/webview-versions/
+  let webview_runtime = match os_platform {
+    DetectedOs::Windows => WebviewRuntime::EdgeWebview2,
+    DetectedOs::MacOS => WebviewRuntime::WebkitSafari,
+    DetectedOs::Linux => WebviewRuntime::WebkitGtk,
+    DetectedOs::Unknown => WebviewRuntime::Unknown,
+  };
+
+  Ok(PlatformInfoResponse {
+    os_platform,
+    webview_runtime,
+  })
+}
+

crates/desktop/tauri-artcraft/src/lib.rs

@@ -4,21 +4,24 @@ pub mod state;
 pub mod threads;
 pub mod utils;
 
-use tauri::{Manager};
+use tauri::Manager;
 
 use crate::commands::flip_image::flip_image;
+use crate::commands::platform_info_command::platform_info_command;
 use crate::commands::sora::open_sora_login_command::open_sora_login_command;
-use crate::state::main_window_size::MainWindowSize;
 use crate::commands::sora::sora_image_generation_command::sora_image_generation_command;
 use crate::commands::sora::sora_image_remix_command::sora_image_remix_command;
 use crate::state::app_config::AppConfig;
+use crate::state::main_window_size::MainWindowSize;
 use crate::state::sora::sora_credential_manager::SoraCredentialManager;
 use crate::state::sora::sora_task_queue::SoraTaskQueue;
+use crate::state::storyteller::storyteller_credential_manager::StorytellerCredentialManager;
 use crate::threads::discord_presence_thread::discord_presence_thread;
-use crate::threads::main_window_thread::main_window_thread;
+use crate::threads::main_window_thread::main_window_thread::main_window_thread;
 use crate::threads::sora_session_login_thread::sora_session_login_thread;
 use crate::threads::sora_task_polling_thread::sora_task_polling_thread;
 
+use tauri_plugin_http;
 use tauri_plugin_log::Target;
 use tauri_plugin_log::TargetKind;
 
@@ -34,6 +37,10 @@ pub fn run() {
   let app_data_root = config.app_data_root.clone();
   let app_data_root_2 = config.app_data_root.clone();
 
+  println!("Attempting to read existing artcraft credentials...");
+  let storyteller_creds_manager = StorytellerCredentialManager::initialize_from_disk_infallible(&app_data_root);
+  let storyteller_creds_manager_2 = storyteller_creds_manager.clone();
+  
   println!("Attempting to read existing credentials...");
   let sora_creds_manager = SoraCredentialManager::initialize_from_disk_infallible(&app_data_root);
   let sora_creds_manager_2 = sora_creds_manager.clone();
@@ -45,6 +52,7 @@ pub fn run() {
   println!("Initializing backend runtime...");
 
   tauri::Builder::default()
+    .plugin(tauri_plugin_http::init())
     .plugin(tauri_plugin_log::Builder::new()
       .level(log::LevelFilter::Info)
       .targets(vec![
@@ -54,7 +62,7 @@ pub fn run() {
         }),
       ])
       .build())
-    .setup(|app| {
+    .setup(move |app| {
       // TODO(bt): This is broken on windows
       // log_environment_details();
 
@@ -88,7 +96,7 @@ pub fn run() {
       }
 
       tauri::async_runtime::spawn(sora_session_login_thread(app_2, app_data_root_2, sora_creds_manager_2));
-      tauri::async_runtime::spawn(main_window_thread(app_3, app_data_root_3));
+      tauri::async_runtime::spawn(main_window_thread(app_3, app_data_root_3, storyteller_creds_manager_2));
       tauri::async_runtime::spawn(sora_task_polling_thread(app_data_root_4, sora_creds_manager_3, sora_task_queue_2));
       tauri::async_runtime::spawn(discord_presence_thread());
 
@@ -101,6 +109,7 @@ pub fn run() {
     .invoke_handler(tauri::generate_handler![
       flip_image,
       open_sora_login_command,
+      platform_info_command,
       sora_image_generation_command,
       sora_image_remix_command,
     ])

crates/desktop/tauri-artcraft/src/state/app_startup_time.rs

@@ -0,0 +1,22 @@
+use chrono::{DateTime, TimeDelta, Utc};
+
+pub struct AppStartupTime {
+  startup_time: DateTime<Utc>,
+}
+
+impl AppStartupTime {
+  pub fn new() -> Self {
+    Self {
+      startup_time: Utc::now(),
+    }
+  }
+
+  pub fn get_startup_time(&self) -> DateTime<Utc> {
+    self.startup_time
+  }
+  
+  pub fn time_delta_since(&self) -> TimeDelta {
+    let now = Utc::now();
+    now.signed_duration_since(self.startup_time)
+  }
+}

crates/desktop/tauri-artcraft/src/state/data_dir/app_credentials_dir.rs

@@ -32,4 +32,12 @@ impl AppCredentialsDir {
   pub fn get_sora_sentinel_file_path(&self) -> PathBuf {
     self.path.join("sora_sentinel.txt")
   }
+
+  pub fn get_storyteller_avt_cookie_file_path(&self) -> PathBuf {
+    self.path.join("artcraft_avt.txt")
+  }
+  
+  pub fn get_storyteller_session_cookie_file_path(&self) -> PathBuf {
+    self.path.join("artcraft_session.txt")
+  }
 }

crates/desktop/tauri-artcraft/src/state/mod.rs

@@ -1,4 +1,5 @@
 pub mod app_config;
+pub mod app_startup_time;
 pub mod data_dir;
 pub mod expanduser;
 pub mod main_window_size;

crates/desktop/tauri-artcraft/src/state/os_platform.rs

@@ -7,14 +7,18 @@ pub enum OsPlatform {
 }
 
 impl OsPlatform {
-  pub fn get() -> OsPlatform {
+  pub fn maybe_get() -> Option<Self> {
     // Totally scientific.
     // https://doc.rust-lang.org/std/env/consts/constant.OS.html
     match std::env::consts::OS {
-      "linux" => OsPlatform::Linux,
-      "windows" => OsPlatform::Windows,
-      "macos" | "apple" => OsPlatform::MacOs,
-      _ => OsPlatform::Linux,
+      "linux" => Some(Self::Linux),
+      "windows" => Some(Self::Windows),
+      "macos" | "apple" => Some(Self::MacOs),
+      _ => None,
     }
   }
+
+  pub fn get() -> Self {
+    Self::maybe_get().unwrap_or(Self::Linux)
+  }
 }

crates/desktop/tauri-artcraft/src/state/storyteller/mod.rs

@@ -0,0 +1,3 @@
+pub mod read_storyteller_credentials_from_disk;
+pub mod storyteller_credential_holder;
+pub mod storyteller_credential_manager;

crates/desktop/tauri-artcraft/src/state/storyteller/read_storyteller_credentials_from_disk.rs

@@ -0,0 +1,39 @@
+use crate::state::data_dir::app_data_root::AppDataRoot;
+use errors::AnyhowResult;
+use std::fs::read_to_string;
+use storyteller_client::credentials::storyteller_avt_cookie::StorytellerAvtCookie;
+use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
+use storyteller_client::credentials::storyteller_session_cookie::StorytellerSessionCookie;
+
+pub fn read_storyteller_credentials_from_disk(app_data_root: &AppDataRoot) -> AnyhowResult<StorytellerCredentialSet> {
+  let creds_dir= app_data_root.credentials_dir();
+
+  let mut avt = None;
+
+  {
+    let avt_file = creds_dir.get_storyteller_avt_cookie_file_path();
+    if avt_file.exists() {
+      let contents = read_to_string(&avt_file)?
+          .trim()
+          .to_string();
+      avt = Some(StorytellerAvtCookie::new(contents));
+    }
+  }  
+
+  let mut session = None;
+
+  {
+    let session_file = creds_dir.get_storyteller_session_cookie_file_path();
+    if session_file.exists() {
+      let contents = read_to_string(&session_file)?
+          .trim()
+          .to_string();
+      session = Some(StorytellerSessionCookie::new(contents));
+    }
+  }
+
+  Ok(StorytellerCredentialSet::initialize(
+    avt,
+    session,
+  ))
+}

crates/desktop/tauri-artcraft/src/state/storyteller/storyteller_credential_holder.rs

@@ -0,0 +1,57 @@
+use anyhow::anyhow;
+use errors::AnyhowResult;
+use std::sync::{Arc, RwLock};
+use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
+
+/// Hold credentials for the application.
+#[derive(Clone)]
+pub struct StorytellerCredentialHolder {
+  credentials: Arc<RwLock<Option<StorytellerCredentialSet>>>,
+}
+
+impl StorytellerCredentialHolder {
+  pub fn new() -> Self {
+    Self {
+      credentials: Arc::new(RwLock::new(None)),
+    }
+  }
+
+  pub fn set_credentials(&self, credentials: &StorytellerCredentialSet) -> AnyhowResult<()> {
+    match self.credentials.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut creds) => {
+        *creds = Some(credentials.clone());
+        Ok(())
+      }
+    }
+  }
+
+  pub fn clear_credentials(&self) -> AnyhowResult<()> {
+    match self.credentials.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut creds) => {
+        *creds = None;
+        Ok(())
+      }
+    }
+  }
+
+  pub fn get_credentials(&self) -> AnyhowResult<Option<StorytellerCredentialSet>> {
+    match self.credentials.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(creds) => {
+        Ok(creds.clone())
+      }
+    }
+  }
+
+  pub fn get_credentials_required(&self) -> AnyhowResult<StorytellerCredentialSet> {
+    match self.credentials.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(creds) => match &*creds {
+        None => Err(anyhow!("Credentials not set")),
+        Some(creds) => Ok(creds.clone()),
+      }
+    }
+  }
+}

crates/desktop/tauri-artcraft/src/state/storyteller/storyteller_credential_manager.rs

@@ -0,0 +1,109 @@
+use crate::state::data_dir::app_data_root::AppDataRoot;
+use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
+use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
+use crate::state::storyteller::read_storyteller_credentials_from_disk::read_storyteller_credentials_from_disk;
+use crate::state::storyteller::storyteller_credential_holder::StorytellerCredentialHolder;
+use anyhow::anyhow;
+use errors::AnyhowResult;
+use log::{error, info, warn};
+use std::fs::OpenOptions;
+use std::io::Write;
+use storyteller_client::credentials::storyteller_avt_cookie::StorytellerAvtCookie;
+use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
+use storyteller_client::credentials::storyteller_session_cookie::StorytellerSessionCookie;
+
+#[derive(Clone)]
+pub struct StorytellerCredentialManager {
+  holder: StorytellerCredentialHolder,
+  app_data_root: AppDataRoot,
+}
+
+impl StorytellerCredentialManager {
+
+  pub fn initialize_from_disk_infallible(app_data_root: &AppDataRoot) -> Self {
+    let holder = StorytellerCredentialHolder::new();
+
+    match read_storyteller_credentials_from_disk(app_data_root) {
+      Err(err) => warn!("Failed to read credentials from disk: {:?}", err),
+      Ok(creds) => {
+        holder.set_credentials(&creds).expect("Failed to set credentials");
+      }
+    }
+
+    Self {
+      holder,
+      app_data_root: app_data_root.clone(),
+    }
+  }
+
+  pub fn set_credentials(&self, creds: &StorytellerCredentialSet) -> AnyhowResult<()> {
+    self.holder.set_credentials(creds)
+  }
+
+  pub fn clear_credentials(&self) -> AnyhowResult<()> {
+    self.holder.clear_credentials()?;
+    Ok(())
+  }
+
+  pub fn get_credentials(&self) -> AnyhowResult<Option<StorytellerCredentialSet>> {
+    self.holder.get_credentials()
+  }
+
+  pub fn get_credentials_required(&self) -> AnyhowResult<StorytellerCredentialSet> {
+    self.holder.get_credentials_required()
+  }
+
+  pub fn reset_from_disk(&self) -> AnyhowResult<StorytellerCredentialSet> {
+    let creds = read_storyteller_credentials_from_disk(&self.app_data_root)?;
+    self.holder.set_credentials(&creds)?;
+    Ok(creds)
+  }
+
+  pub fn persist_all_to_disk(&self) -> AnyhowResult<()> {
+    let credentials = self.holder.get_credentials_required()?;
+
+    info!("Persisting avt cookie to disk...");
+    if let Some(avt) = credentials.avt.as_ref() {
+      persist_avt_to_disk(&avt, &self.app_data_root)?;
+    }
+
+    info!("Persisting session cookie to disk...");
+    if let Some(session) = credentials.session.as_ref() {
+      persist_session_to_disk(&session, &self.app_data_root)?;
+    }
+
+    Ok(())
+  }
+}
+
+fn persist_session_to_disk(session: &StorytellerSessionCookie, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
+  let creds_dir = app_data_root.credentials_dir();
+  let filename = creds_dir.get_storyteller_session_cookie_file_path();
+
+  let mut file = OpenOptions::new()
+      .create(true)
+      .write(true)
+      .truncate(true)
+      .open(filename)?;
+
+  file.write_all(session.as_bytes())?;
+  file.flush()?;
+
+  Ok(())
+}
+
+fn persist_avt_to_disk(avt: &StorytellerAvtCookie, app_data_root: &AppDataRoot) -> AnyhowResult<()> {
+  let creds_dir = app_data_root.credentials_dir();
+  let filename = creds_dir.get_storyteller_avt_cookie_file_path();
+
+  let mut file = OpenOptions::new()
+      .create(true)
+      .write(true)
+      .truncate(true)
+      .open(filename)?;
+
+  file.write_all(avt.as_bytes())?;
+  file.flush()?;
+
+  Ok(())
+}

crates/desktop/tauri-artcraft/src/threads/main_window_thread.rs

@@ -1,48 +0,0 @@
-use crate::state::data_dir::app_data_root::AppDataRoot;
-use crate::state::main_window_size::MainWindowSize;
-use errors::AnyhowResult;
-use log::{error, info};
-use tauri::{AppHandle, Manager, Webview, Window};
-
-const MAIN_WINDOW_NAME : &str = "main";
-
-pub async fn main_window_thread(
-  app: AppHandle,
-  app_data_root: AppDataRoot,
-) -> ! {
-  loop {
-    for (window_name, window) in app.windows() {
-      if window_name == MAIN_WINDOW_NAME {
-        let result = handle_main_window(
-          &window,
-          &app_data_root,
-        ).await;
-        if let Err(err) = result {
-          error!("Error handling main window: {:?}", err);
-        }
-      }
-    }
-    tokio::time::sleep(std::time::Duration::from_millis(1_000)).await;
-  }
-}
-
-pub async fn handle_main_window(
-  window: &Window,
-  app_data_root: &AppDataRoot,
-) -> AnyhowResult<()> {
-  let mut window_size = MainWindowSize::from_window(window)?;
-
-  loop {
-    let old_size = window_size.to_physical_size();
-    let new_size = window.inner_size()?;
-
-    if !window_size.matches_physical_size(&new_size) {
-      info!("Window size changed from {:?} to {:?}", old_size, new_size);
-      window_size = MainWindowSize::from_window(&window)?;
-      info!("Saving window size configs to disk...");
-      window_size.persist_to_filesystem(app_data_root)?;
-    }
-
-    tokio::time::sleep(std::time::Duration::from_millis(10_000)).await;
-  }
-}

crates/desktop/tauri-artcraft/src/threads/main_window_thread/main_window_thread.rs

@@ -0,0 +1,60 @@
+use crate::state::data_dir::app_data_root::AppDataRoot;
+use crate::state::main_window_size::MainWindowSize;
+use crate::state::storyteller::storyteller_credential_manager::StorytellerCredentialManager;
+use crate::threads::main_window_thread::persist_storyteller_cookies_task::persist_storyteller_cookies_task;
+use crate::threads::main_window_thread::persist_window_resize_task::persist_window_resize_task;
+use errors::AnyhowResult;
+use log::{error, info};
+use memory_store::clone_slot::CloneSlot;
+use tauri::{AppHandle, Manager, Webview, Window};
+use crate::state::app_startup_time::AppStartupTime;
+
+const MAIN_WINDOW_NAME : &str = "main";
+
+pub async fn main_window_thread(
+  app: AppHandle,
+  app_data_root: AppDataRoot,
+  storyteller_creds_manager: StorytellerCredentialManager,
+) -> ! {
+  // TODO: Move these into some kind of dependency injection framework
+  let window_size_slot: CloneSlot<MainWindowSize> = CloneSlot::empty();
+  let app_startup_time = AppStartupTime::new();
+
+  loop {
+    for (window_name, window) in app.windows() {
+      if window_name == MAIN_WINDOW_NAME {
+        let result = handle_main_window(
+          &window,
+          &app_data_root,
+          &storyteller_creds_manager,
+          &window_size_slot,
+          &app_startup_time,
+        ).await;
+        if let Err(err) = result {
+          error!("Error handling main window: {:?}", err);
+        }
+      }
+    }
+    tokio::time::sleep(std::time::Duration::from_millis(1_000)).await;
+  }
+}
+
+pub async fn handle_main_window(
+  window: &Window,
+  app_data_root: &AppDataRoot,
+  storyteller_creds_manager: &StorytellerCredentialManager,
+  window_size_slot: &CloneSlot<MainWindowSize>,
+  app_startup_time: &AppStartupTime,
+) -> AnyhowResult<()> {
+  loop {
+    log_errors(persist_window_resize_task(window, app_data_root, window_size_slot).await);
+    log_errors(persist_storyteller_cookies_task(window, app_data_root, storyteller_creds_manager, app_startup_time).await);
+    tokio::time::sleep(std::time::Duration::from_millis(1_000)).await;
+  }
+}
+
+pub fn log_errors<T>(result: AnyhowResult<T>) {
+  if let Err(err) = result {
+    error!("Error persisting window size: {:?}", err);
+  }
+}

crates/desktop/tauri-artcraft/src/threads/main_window_thread/mod.rs

@@ -0,0 +1,3 @@
+pub mod main_window_thread;
+pub mod persist_storyteller_cookies_task;
+pub mod persist_window_resize_task;

crates/desktop/tauri-artcraft/src/threads/main_window_thread/persist_storyteller_cookies_task.rs

@@ -0,0 +1,119 @@
+use crate::state::data_dir::app_data_root::AppDataRoot;
+use crate::state::storyteller::storyteller_credential_manager::StorytellerCredentialManager;
+use crate::threads::sora_session_login_thread::LOGIN_WINDOW_NAME;
+use anyhow::anyhow;
+use errors::AnyhowResult;
+use log::{debug, error, info};
+use once_cell::sync::Lazy;
+use reqwest::Url;
+use std::fs;
+use chrono::TimeDelta;
+use storyteller_client::credentials::storyteller_avt_cookie::StorytellerAvtCookie;
+use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
+use storyteller_client::credentials::storyteller_session_cookie::StorytellerSessionCookie;
+use tauri::{AppHandle, Manager, Webview, Window};
+use crate::state::app_startup_time::AppStartupTime;
+
+const MAIN_WEBVIEW_NAME: &str = "main";
+
+const STORYTELLER_ROOT_COOKIE_URL_STR: &str = "https://api.storyteller.ai/";
+
+static STORYTELLER_ROOT_COOKIE_URL: Lazy<Url> = Lazy::new(|| {
+  Url::parse(STORYTELLER_ROOT_COOKIE_URL_STR).expect("URL should parse")
+});
+
+const AVT_COOKIE_NAME : &str = "visitor";
+const SESSION_COOKIE_NAME : &str = "session";
+
+/// There's some kind of race condition that we need to wait for before inspecting cookies.
+const RACE_CONDITION_WAIT_TIME: TimeDelta = TimeDelta::milliseconds(5000);
+
+pub async fn persist_storyteller_cookies_task(
+  window: &Window,
+  app_data_root: &AppDataRoot,
+  storyteller_credential_manager: &StorytellerCredentialManager,
+  app_startup_time: &AppStartupTime,
+) -> AnyhowResult<()> {
+
+  if app_startup_time.time_delta_since() < RACE_CONDITION_WAIT_TIME {
+    // NB:     There's an issue when the "main window thread" inquires about the
+    //     webview cookies shortly after app startup. When it attempts to dump the
+    //     cookies within a few milliseconds of app launch, it deadlocks and the
+    //     app never launches correctly. The entire webview goes blank and the app
+    //     freezes. This hack seems to fix the problem.
+    return Ok(())
+  }
+
+  for webview in window.webviews() {
+    let label = webview.label();
+    if label == MAIN_WEBVIEW_NAME {
+      persist_webview_cookies(&webview, app_data_root, storyteller_credential_manager).await?;
+      break;
+    }
+  }
+  
+  Ok(())
+}
+
+async fn persist_webview_cookies(
+  webview: &Webview,
+  app_data_root: &AppDataRoot,
+  storyteller_credential_manager: &StorytellerCredentialManager,
+) -> AnyhowResult<()> {
+  let current_webview_credentials = get_storyteller_cookies(webview).await?;
+
+  if current_webview_credentials.is_empty() {
+    // TODO: handle logout / cookie deletion
+    return Ok(());
+  }
+  
+  let mut replace_credentials = true;
+
+  let maybe_old_credentials = storyteller_credential_manager.get_credentials()?;
+
+  if let Some(old_credentials) = maybe_old_credentials {
+    if old_credentials.equals(&current_webview_credentials) {
+      replace_credentials = false;
+    }
+  }
+  
+  if replace_credentials {
+    info!("Writing ArtCraft credentials to disk...");
+    storyteller_credential_manager.set_credentials(&current_webview_credentials)?;
+    storyteller_credential_manager.persist_all_to_disk()?;
+  }
+  
+  Ok(())
+}
+
+pub async fn get_storyteller_cookies(webview: &Webview) -> AnyhowResult<StorytellerCredentialSet> {
+  debug!("Getting storyteller cookies...");
+
+  //// FIXME: THIS IS A TOTAL HACK. / DO NOT REMOVE.
+  //// NB: If we don't call a method on the webview, the very next call to get cookies will deadlock.
+  //if let Ok(url) = webview.url() {
+  //  debug!("For url: {:?}", url);
+  //}
+
+  let cookies = webview.cookies_for_url(STORYTELLER_ROOT_COOKIE_URL.clone())?;
+
+  info!("Got storyteller cookies: {:?}", cookies);
+  info!("Got storyteller cookies: {:?}", cookies.len());
+  info!("Got storyteller location: {:?}", webview.url());
+  
+  let mut avt_cookie = None;
+  let mut session_cookie = None;
+
+  for cookie in cookies.into_iter() {
+    if let Some(avt) = StorytellerAvtCookie::maybe_from_cookie(&cookie) {
+      avt_cookie = Some(avt);
+    } else if let Some(session) = StorytellerSessionCookie::maybe_from_cookie(&cookie) {
+      session_cookie = Some(session);
+    }
+  }
+
+  Ok(StorytellerCredentialSet::initialize(
+    avt_cookie,
+    session_cookie,
+  ))
+}

crates/desktop/tauri-artcraft/src/threads/main_window_thread/persist_window_resize_task.rs

@@ -0,0 +1,32 @@
+use crate::state::data_dir::app_data_root::AppDataRoot;
+use crate::state::main_window_size::MainWindowSize;
+use errors::AnyhowResult;
+use log::info;
+use memory_store::clone_slot::CloneSlot;
+use tauri::Window;
+
+pub async fn persist_window_resize_task(
+  window: &Window,
+  app_data_root: &AppDataRoot,
+  window_size_slot: &CloneSlot<MainWindowSize>,
+) -> AnyhowResult<()> {
+
+  let current_window_size = MainWindowSize::from_window(window)?;
+  let current_physical_size = window.inner_size()?;
+
+  let mut save_size_to_disk = true;
+  
+  if let Ok(Some(old_size)) = window_size_slot.get_clone() {
+    if old_size.matches_physical_size(&current_physical_size) {
+      save_size_to_disk = false;
+    }
+  }
+  
+  if save_size_to_disk {
+    info!("Saving window size configs to disk...");
+    current_window_size.persist_to_filesystem(app_data_root)?;
+    window_size_slot.set_clone(&current_window_size)?;
+  }
+
+  Ok(())
+}

crates/desktop/tauri-artcraft/src/threads/sora_session_login_thread.rs

@@ -1,8 +1,8 @@
 use crate::state::data_dir::app_data_root::AppDataRoot;
 use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
 use crate::state::sora::sora_credential_manager::SoraCredentialManager;
+use crate::utils::cookies::sora_webview_cookies::get_all_sora_cookies_as_string;
 use crate::utils::sora::initialize_sora_jwt_bearer_token::initialize_sora_jwt_bearer_token;
-use crate::utils::sora_webview_cookies::get_all_sora_cookies_as_string;
 use anyhow::anyhow;
 use chrono::{DateTime, NaiveDateTime, TimeDelta};
 use errors::AnyhowResult;

crates/desktop/tauri-artcraft/src/utils/cookies/mod.rs

@@ -0,0 +1 @@
+pub mod sora_webview_cookies;

crates/desktop/tauri-artcraft/src/utils/mod.rs

@@ -1,7 +1,8 @@
 pub mod clear_all_webview_cookies;
+pub mod cookies;
 pub mod get_url_file_extension;
 pub mod image;
 pub mod large_file_transfer;
+pub mod object_store;
 pub mod simple_http_download;
 pub mod sora;
-pub mod sora_webview_cookies;

crates/lib/api_clients/storyteller_client/Cargo.toml

@@ -25,6 +25,7 @@ tokens = { workspace = true }
 # External
 anyhow = { workspace = true }
 chrono = { workspace = true }
+cookie = { workspace = true }
 log = { workspace = true }
 reqwest = { workspace = true, features = ["multipart", "gzip"] }
 url = { workspace = true }

crates/lib/api_clients/storyteller_client/src/credentials/storyteller_avt_cookie.rs

@@ -1,3 +1,6 @@
+use cookie::Cookie;
+
+const AVT_COOKIE_NAME : &str = "visitor";
 
 /// The "anonymous visitor token" cookie allows users to interact with
 /// the service in a logged-out state with some continuity.
@@ -11,6 +14,14 @@ impl StorytellerAvtCookie {
     Self { cookie }
   }
 
+  pub fn maybe_from_cookie(cookie: &Cookie) -> Option<Self> {
+    if cookie.name() == AVT_COOKIE_NAME {
+      Some(Self::new(cookie.value().to_string()))
+    } else {
+      None
+    }
+  }
+
   pub fn as_str(&self) -> &str {
     &self.cookie
   }
@@ -22,4 +33,8 @@ impl StorytellerAvtCookie {
   pub fn to_string(&self) -> String {
     self.cookie.clone()
   }
+  
+  pub fn equals(&self, other: &Self) -> bool {
+    self.cookie == other.cookie
+  }
 }

crates/lib/api_clients/storyteller_client/src/credentials/storyteller_credential_set.rs

@@ -3,29 +3,67 @@ use crate::credentials::storyteller_session_cookie::StorytellerSessionCookie;
 
 #[derive(Clone)]
 pub struct StorytellerCredentialSet {
-  pub cookie: Option<StorytellerSessionCookie>,
   pub avt: Option<StorytellerAvtCookie>,
+  pub session: Option<StorytellerSessionCookie>,
 }
 
 impl StorytellerCredentialSet {
+  pub fn empty() -> Self {
+    Self {
+      avt: None,
+      session: None,
+    }
+  }
+  
   pub fn initialize(
-    cookie: Option<StorytellerSessionCookie>,
     avt: Option<StorytellerAvtCookie>,
+    session: Option<StorytellerSessionCookie>,
   ) -> Self {
-    Self { cookie, avt }
+    Self {
+      avt,
+      session,
+    }
   }
 
-  pub fn initialize_with_just_cookie(cookie: StorytellerSessionCookie) -> Self {
+  pub fn initialize_with_just_cookie(session: StorytellerSessionCookie) -> Self {
     Self {
-      cookie: Some(cookie),
       avt: None,
+      session: Some(session),
     }
   }
 
   pub fn initialize_with_just_avt(avt: StorytellerAvtCookie) -> Self {
     Self {
-      cookie: None,
       avt: Some(avt),
+      session: None,
     }
   }
+  
+  pub fn is_empty(&self) -> bool {
+    self.avt.is_none() && self.session.is_none()
+  }
+  
+  pub fn equals(&self, other: &Self) -> bool {
+    match (&self.avt, &other.avt) {
+      (None, None) => {} // Fallthrough
+      (Some(avt), Some(other_avt)) => {
+        if !avt.equals(other_avt) {
+          return false;
+        }
+      }
+      _ => return false,
+    }
+
+    match (&self.session, &other.session) {
+      (None, None) => {} // Fallthrough
+      (Some(session), Some(other_session)) => {
+        if !session.equals(other_session) {
+          return false;
+        }
+      }
+      _ => return false,
+    }
+
+    true
+  }
 }

crates/lib/api_clients/storyteller_client/src/credentials/storyteller_session_cookie.rs

@@ -1,3 +1,6 @@
+use cookie::Cookie;
+
+const SESSION_COOKIE_NAME : &str = "session";
 
 /// The primary FakeYou / Storyteller / Artcraft backend session identifier.
 #[derive(Clone)]
@@ -9,6 +12,14 @@ impl StorytellerSessionCookie {
   pub fn new(cookie: String) -> Self {
     Self { cookie }
   }
+  
+  pub fn maybe_from_cookie(cookie: &Cookie) -> Option<Self> {
+    if cookie.name() == SESSION_COOKIE_NAME {
+      Some(Self::new(cookie.value().to_string()))
+    } else {
+      None
+    }
+  }
 
   pub fn as_str(&self) -> &str {
     &self.cookie
@@ -21,4 +32,8 @@ impl StorytellerSessionCookie {
   pub fn to_string(&self) -> String {
     self.cookie.clone()
   }
+
+  pub fn equals(&self, other: &Self) -> bool {
+    self.cookie == other.cookie
+  }
 }

crates/lib/caching/memory_store/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "memory_store"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "memory_store"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[dependencies]
+# Workspace hack (faster compile times)
+workspace-build-acceleration = { workspace =  true }
+
+# Internal
+errors = { workspace = true }
+
+# External
+anyhow = { workspace = true }

crates/lib/caching/memory_store/src/clone_cell.rs

@@ -0,0 +1,54 @@
+use anyhow::anyhow;
+use errors::AnyhowResult;
+use std::sync::{Arc, RwLock};
+
+/// Store a threadsafe value that can be passed around and cloned.
+/// The object is interior mutable so it can be changed without mut/&mut references.
+/// This is useful for storing a value that may or may not be set between threads.
+/// 
+/// Unlike `CloneSlot`, this always holds an object.
+#[derive(Clone)]
+pub struct CloneCell<T: Clone> {
+  pub store: Arc<RwLock<T>>,
+}
+
+impl <T: Clone> CloneCell<T> {
+  pub fn with_clone(object: &T) -> Self {
+    Self {
+      store: Arc::new(RwLock::new(object.clone())),
+    }
+  }
+
+  pub fn with_owned(object: T) -> Self {
+    Self {
+      store: Arc::new(RwLock::new(object)),
+    }
+  }
+
+  pub fn get_clone(&self) -> AnyhowResult<T> {
+    match self.store.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(store) => Ok(store.clone()),
+    }
+  }
+  
+  pub fn set_clone(&self, value: &T) -> AnyhowResult<()> {
+    match self.store.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut store) => {
+        *store = value.clone();
+        Ok(())
+      }
+    }
+  }
+  
+  pub fn set_owned(&self, value: T) -> AnyhowResult<()> {
+    match self.store.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut store) => {
+        *store = value;
+        Ok(())
+      }
+    }
+  }
+}

crates/lib/caching/memory_store/src/clone_slot.rs

@@ -0,0 +1,77 @@
+use anyhow::anyhow;
+use errors::AnyhowResult;
+use std::sync::{Arc, RwLock};
+
+/// Store a threadsafe value that can be passed around and cloned.
+/// The object is interior mutable so it can be changed without mut/&mut references.
+/// This is useful for storing a value that may or may not be set between threads.
+/// 
+/// Unlike `CloneCell`, this holds an optional object.
+#[derive(Clone)]
+pub struct CloneSlot<T: Clone> {
+  pub store: Arc<RwLock<Option<T>>>,
+}
+
+impl <T: Clone> CloneSlot<T> {
+  pub fn empty() -> Self {
+    Self {
+      store: Arc::new(RwLock::new(None)),
+    }
+  }
+  
+  pub fn with_clone(object: &T) -> Self {
+    Self {
+      store: Arc::new(RwLock::new(Some(object.clone()))),
+    }
+  }
+
+  pub fn get_clone(&self) -> AnyhowResult<Option<T>> {
+    match self.store.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(store) => Ok(store.clone()),
+    }
+  }
+  
+  pub fn set_clone(&self, value: &T) -> AnyhowResult<()> {
+    match self.store.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut store) => {
+        *store = Some(value.clone());
+        Ok(())
+      }
+    }
+  }
+
+  pub fn clear(&self) -> AnyhowResult<()> {
+    match self.store.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut store) => {
+        *store = None; 
+        Ok(())
+      }
+    }
+  }
+
+  pub fn take(&self) -> AnyhowResult<Option<T>> {
+    match self.store.write() {
+      Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
+      Ok(mut store) => {
+        Ok(store.take())
+      }
+    }
+  }
+
+  pub fn is_some(&self) -> AnyhowResult<bool> {
+    match self.store.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(store) => Ok(store.is_some()),
+    }
+  }
+  
+  pub fn is_none(&self) -> AnyhowResult<bool> {
+    match self.store.read() {
+      Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
+      Ok(store) => Ok(store.is_none()),
+    }
+  }
+}

crates/lib/caching/memory_store/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod clone_cell;
+pub mod clone_slot;

frontend/apps/editor2d/src/Classes/ApiManager/BillingApi.ts

@@ -1,33 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { Subscription } from "./models/Billing";
-import { LoyaltyProgram } from "./enums/Billing";
-
-export class BillingApi extends ApiManager {
-  public async ListActiveSubscriptions(): Promise<
-    ApiResponse<{
-      active_subscriptions: Subscription[];
-      maybe_loyalty_program?: LoyaltyProgram;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/billing/active_subscriptions`;
-    return await this.get<{
-      success: boolean;
-      active_subscriptions?: Subscription[];
-      maybe_loyalty_program?: LoyaltyProgram;
-      error_message?: string;
-    }>({ endpoint: endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: {
-          active_subscriptions: response.active_subscriptions || [],
-          maybe_loyalty_program: response.maybe_loyalty_program,
-        },
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/CommentsApi.ts

@@ -1,91 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-
-export class CommentsApi extends ApiManager {
-  public CreateComment({
-    commentMarkdown,
-    entityToken,
-    entityType,
-    uuidIdempotencyToken,
-  }: {
-    commentMarkdown: string;
-    entityToken: string;
-    entityType: string;
-    uuidIdempotencyToken: string;
-  }): Promise<ApiResponse<string>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/new`;
-
-    const body = {
-      comment_markdown: commentMarkdown,
-      entity_token: entityToken,
-      entity_type: entityType,
-      uuid_idempotency_token: uuidIdempotencyToken,
-    };
-
-    return this.post<
-      {
-        comment_markdown: string;
-        entity_token: string;
-        entity_type: string;
-        uuid_idempotency_token: string;
-      },
-      {
-        success?: boolean;
-        comment_token?: string;
-        BadInput?: string;
-      }
-    >({ endpoint, body })
-      .then((response) => ({
-        success: response.success ?? false,
-        data: response.comment_token,
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public DeleteComment({
-    commentToken,
-  }: {
-    commentToken: string;
-  }): Promise<ApiResponse<undefined>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/delete/${commentToken}`;
-
-    return this.post<
-      { as_mod: boolean },
-      {
-        success?: boolean;
-        BadInput?: string;
-      }
-    >({ endpoint, body: { as_mod: true } })
-      .then((response) => ({
-        success: response.success ?? false,
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public ListCommentsByEntity({
-    entityType,
-    entityToken,
-  }: {
-    entityType: string;
-    entityToken: string;
-  }): Promise<ApiResponse<string[]>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/list/${entityType}/${entityToken}`;
-
-    return this.get<{
-      success: boolean;
-      comments: string[];
-    }>({ endpoint })
-      .then((response) => ({
-        success: true,
-        data: response.comments,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/JobsApi.ts

@@ -1,77 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { Job, JobPreview } from "./models/Job";
-
-export class JobsApi extends ApiManager {
-  public GetPreviewStatusByJobToken({
-    token,
-  }: {
-    token: string;
-  }): Promise<ApiResponse<JobPreview>> {
-    console.log(token);
-
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/workflows/preview_status/${token}`;
-
-    return this.get<ApiResponse<JobPreview>>({ endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: response.data,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public GetJobByToken({
-    token,
-  }: {
-    token: string;
-  }): Promise<ApiResponse<Job>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/job/${token}`;
-
-    return this.get<{
-      success: boolean;
-      state: Job;
-    }>({ endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: response.state,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public ListRecentJobs(): Promise<ApiResponse<Job[]>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session`;
-
-    return this.get<{
-      success: boolean;
-      jobs: Job[];
-      error_reason?: string;
-    }>({ endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: response.jobs,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public DeleteJobByToken(jobToken: string): Promise<ApiResponse<undefined>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/job/${jobToken}`;
-
-    return this.delete<
-      undefined,
-      {
-        success: boolean;
-      }
-    >({ endpoint })
-      .then((response) => ({
-        success: response.success,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/MiscApi.ts

@@ -1,26 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { StatusAlert } from "./models/StatusAlert";
-
-export class MiscApi extends ApiManager {
-  public GetStatusAlertCheck(): Promise<ApiResponse<StatusAlert>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/session`;
-    return this.get<{
-      success: boolean;
-      maybe_alert?: Omit<StatusAlert, "refresh_interval_millis">;
-      refresh_interval_millis: number;
-    }>({ endpoint: endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: {
-          ...response.maybe_alert,
-          refresh_interval_millis: response.refresh_interval_millis,
-        },
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.mesasge,
-        };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/ModerationApi.ts

@@ -1,42 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-
-export class ModerationApi extends ApiManager {
-  public async UpdateFeatureFlagsByToken({
-    usernameOrToken,
-    flags,
-  }: {
-    usernameOrToken: string;
-    flags: string[];
-  }): Promise<
-    ApiResponse<{
-      inference_job_token?: string;
-      inference_job_token_type?: string;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/moderation/user_feature_flags/${usernameOrToken}`;
-
-    const body = {
-      action: {
-        AddFlags: {
-          flags,
-        },
-      },
-    };
-
-    return this.post<
-      { action: { AddFlags: { flags: string[] } } },
-      { success?: boolean; BadInput?: string }
-    >({
-      endpoint,
-      body: body,
-    })
-      .then((response) => ({
-        success: response.success ?? false,
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => ({
-        success: false,
-        errorMessage: err.message,
-      }));
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/PromptsApi.ts

@@ -1,25 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { Prompts } from "./models/Prompts";
-
-export class PromptsApi extends ApiManager {
-  public GetPromptsByToken({
-    token,
-  }: {
-    token: string;
-  }): Promise<ApiResponse<Prompts>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/prompts/${token}`;
-
-    return this.get<{
-      success: boolean;
-      prompt: Prompts;
-      error_reason?: string;
-    }>({ endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: response.prompt,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/TtsApi.ts

@@ -1,64 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { GenerateTtsAudioErrorType } from "./enums/Tts";
-import { Visibility } from "./enums/Visibility";
-
-export interface GenerateTtsAudioRequest {
-  creator_set_visibility?: Visibility;
-  uuid_idempotency_token: string;
-  tts_model_token: string;
-  inference_text: string;
-  is_storyteller_demo?: boolean;
-}
-
-export interface GenerateTtsAudioResponse {
-  success?: boolean;
-  inference_job_token?: string;
-  inference_job_token_type?: string;
-  BadInput?: string;
-}
-
-export class TtsApi extends ApiManager {
-  public async GenerateTtsAudio(request: GenerateTtsAudioRequest): Promise<
-    ApiResponse<{
-      inference_job_token?: string;
-      inference_job_token_type?: string;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/tts/inference`;
-
-    const body = {
-      ...request,
-      ...(request.creator_set_visibility
-        ? {}
-        : { creator_set_visibility: Visibility.Public }),
-    };
-
-    return this.post<GenerateTtsAudioRequest, GenerateTtsAudioResponse>({
-      endpoint,
-      body: body,
-    })
-      .then((response) => {
-        if (!response.success) {
-          return {
-            success: false,
-            errorMessage: response.BadInput,
-            data: {},
-          };
-        }
-        return {
-          success: true,
-          data: {
-            inference_job_token: response.inference_job_token,
-            inference_job_token_type: response.inference_job_token_type,
-          },
-        };
-      })
-      .catch(() => {
-        return {
-          success: false,
-          errorMessage: GenerateTtsAudioErrorType.UnknownError,
-          data: {},
-        };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/UserBookmarksApi.ts

@@ -1,201 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import {
-  UserBookmarkBatch,
-  UserBookmarkByEntity,
-  UserBookmarkByUser,
-} from "./models/UserBookmark";
-import { Pagination } from "./models/Pagination";
-
-export enum ScopedEntityTypes {
-  USER = "user",
-  TTS_MODEL = "tts-model",
-  TTS_RESULT = "tts-result",
-  W2L_TEMPLATE = "w2l-template",
-  W2L_RESULT = "w2l-result",
-  MEDIA_FILE = "media_file",
-  MODEL_WEIGHT = "model_weight",
-  VOICE_CONVERSION_MODEL = "voice_conversion_model",
-  ZS_VOICE = "z_voice",
-}
-
-export enum ScopedWeightTypes {
-  HIFIGAN_TTl2 = "hifigan_tt2",
-  RVC_V2 = "rvc_v2",
-  SD_1_5 = "sd_1.5",
-  SDXL = "sdxl",
-  SO_VITS_SVC = "so_vits_svc",
-  TT2 = "tt2",
-  LORA = "loRA",
-  VALL_E = "vall_e",
-  COMFY_UI = "comfy_ui",
-}
-
-export enum ScopedMediaFileType {
-  IMAGE_GENERATION = "image_generation",
-  TEXT_TO_SPEECH = "text_to_speech",
-  VOCODER = "vocoder",
-  VOICE_CONVERSION = "voice_conversion",
-  WORKFLOW_CONFIG = "workflow_config",
-}
-
-interface ListUserBookmarksByUserRequest {
-  username: string;
-  sort_ascending?: boolean;
-  page_size?: number;
-  page_index?: number;
-  maybe_scoped_entity_type?: ScopedEntityTypes[];
-  maybe_scoped_weight_type?: ScopedWeightTypes[];
-  maybe_scoped_media_file_type?: ScopedMediaFileType[];
-}
-
-export class UserBookmarksApi extends ApiManager {
-  public CreateUserBookmark({
-    entityToken,
-    entityType,
-  }: {
-    entityToken: string;
-    entityType: string;
-  }): Promise<
-    ApiResponse<{
-      new_bookmark_count_for_entity?: number;
-      user_bookmark_token?: string;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/create`;
-    const body = {
-      entity_token: entityToken,
-      entity_type: entityType,
-    };
-
-    return this.post<
-      {
-        entity_token: string;
-        entity_type: string;
-      },
-      {
-        success?: boolean;
-        new_bookmark_count_for_entity?: number;
-        user_bookmark_token: string;
-        BadInput?: string;
-      }
-    >({ endpoint, body })
-      .then((response) => ({
-        success: response.success ?? false,
-        data: {
-          new_bookmark_count_for_entity: response.new_bookmark_count_for_entity,
-          user_bookmark_token: response.user_bookmark_token,
-        },
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public DeleteUserBookmark({
-    entityToken,
-  }: {
-    entityToken: string;
-  }): Promise<ApiResponse<undefined>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/delete/${entityToken}`;
-
-    return this.delete<
-      { as_mod: boolean },
-      {
-        success?: boolean;
-        BadInput?: string;
-      }
-    >({ endpoint, body: { as_mod: true } })
-      .then((response) => ({
-        success: response.success ?? false,
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public ListUserBookmarks(): Promise<ApiResponse<UserBookmarkBatch[]>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/batch`;
-
-    return this.get<{
-      success?: boolean;
-      bookmarks?: UserBookmarkBatch[];
-      BadInput?: string;
-    }>({ endpoint })
-      .then((response) => ({
-        success: response.success ?? false,
-        data: response.bookmarks,
-        errorMessage: response.BadInput,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public ListUserBookmarksByUser({
-    username,
-    sort_ascending,
-    page_size,
-    page_index,
-    maybe_scoped_entity_type,
-    maybe_scoped_weight_type,
-    maybe_scoped_media_file_type,
-  }: ListUserBookmarksByUserRequest): Promise<
-    ApiResponse<UserBookmarkByUser[], Pagination>
-  > {
-    const user = username;
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/list/${user}`;
-
-    const query = {
-      sort_ascending,
-      page_size,
-      page_index,
-      maybe_scoped_entity_type: maybe_scoped_entity_type
-        ? maybe_scoped_entity_type.join(",")
-        : undefined,
-      maybe_scoped_weight_type: maybe_scoped_weight_type
-        ? maybe_scoped_weight_type.join(",")
-        : undefined,
-      maybe_scoped_media_file_type: maybe_scoped_media_file_type
-        ? maybe_scoped_media_file_type.join(",")
-        : undefined,
-    };
-
-    return this.get<{
-      success?: boolean;
-      pagination?: Pagination;
-      results: UserBookmarkByUser[];
-    }>({ endpoint, query })
-      .then((response) => ({
-        success: response.success ?? false,
-        data: response.results,
-        pagination: response.pagination,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-
-  public ListUserBookmarksByEntity({
-    entityType,
-    entityToken,
-  }: {
-    entityType: string;
-    entityToken: string;
-  }): Promise<ApiResponse<UserBookmarkByEntity[]>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/list/${entityType}/${entityToken}`;
-
-    return this.get<{
-      success: boolean;
-      user_bookmarks: UserBookmarkByEntity[];
-    }>({ endpoint })
-      .then((response) => ({
-        success: true,
-        data: response.user_bookmarks,
-      }))
-      .catch((err) => {
-        return { success: false, errorMessage: err.message };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/UsersApi.ts

@@ -1,162 +0,0 @@
-import { ApiManager, ApiResponse } from "./ApiManager";
-import { UserInfo } from "./models/Users";
-
-export class UsersApi extends ApiManager {
-  public GetSession(): Promise<
-    ApiResponse<{
-      loggedIn: boolean;
-      user?: UserInfo;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/session`;
-    return this.get<{
-      success: boolean;
-      logged_in: boolean;
-      user?: UserInfo;
-      error_message?: string;
-    }>({ endpoint: endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: {
-          loggedIn: response.logged_in,
-          user: response.user,
-        },
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.mesasge,
-        };
-      });
-  }
-
-  public GetUserProfile(username: string): Promise<
-    ApiResponse<{
-      user?: UserInfo;
-    }>
-  > {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/user/${username}/profile`;
-    return this.get<{
-      success: boolean;
-      user?: UserInfo;
-      error_message?: string;
-    }>({ endpoint: endpoint })
-      .then((response) => ({
-        success: response.success,
-        data: {
-          user: response.user,
-        },
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          error_message: err.message,
-        };
-      });
-  }
-
-  public async Login({
-    usernameOrEmail,
-    password,
-  }: {
-    usernameOrEmail: string;
-    password: string;
-  }): Promise<ApiResponse<{ signedSession?: string }>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/login`;
-  
-    const body = {
-      username_or_email: usernameOrEmail,
-      password: password,
-    };
-    return await this.post<
-      { username_or_email: string; password: string },
-      {
-        success: boolean;
-        signed_session?: string;
-        error_message?: string;
-        error_type?: string;
-      }
-    >({
-      endpoint: endpoint,
-      body: body,
-    })
-      .then((response) => ({
-        success: response.success,
-        data: { signedSession: response.signed_session },
-        errorMessage: response.error_message,
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          error_message: err.message,
-        };
-      });
-  }
-
-  public Logout(): Promise<ApiResponse<null>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/logout`;
-    return this.post<null, { success: boolean; error_message?: string }>({
-      endpoint: endpoint,
-    })
-      .then((response) => ({
-        success: response.success,
-      }))
-      .catch((err) => {
-        return {
-          success: false,
-          errorMessage: err.message,
-        };
-      });
-  }
-
-  public async Signup({
-    username,
-    email,
-    password,
-    passwordConfirmation,
-  }: {
-    username: string;
-    email: string;
-    password: string;
-    passwordConfirmation: string;
-  }): Promise<ApiResponse<{ signedSession?: string }>> {
-    const endpoint = `${this.ApiTargets.BaseApi}/v1/create_account`;
-    const body = {
-      email_address: email,
-      password,
-      password_confirmation: passwordConfirmation,
-      username,
-    };
-    return await this.post<
-      {
-        username: string;
-        email_address: string;
-        password: string;
-        password_confirmation: string;
-      },
-      {
-        success: boolean;
-        signed_session?: string;
-        error_fields?: Record<string, string>;
-        error_message?: string;
-        error_type?: string;
-      }
-    >({
-      endpoint: endpoint,
-      body: body,
-    })
-      .then((response) => {
-        return {
-          success: response.success,
-          data: { signedSession: response.signed_session },
-          errorMessage: Object.values(response.error_fields ?? {}).join(","),
-        };
-      })
-      .catch((err) => {
-        return {
-          success: false,
-          error_message: err.message,
-        };
-      });
-  }
-}

frontend/apps/editor2d/src/Classes/ApiManager/index.ts

@@ -1,18 +1,9 @@
 // https://storyteller-docs.netlify.app/#/
 export * from "./AnalyticsApi";
 export * from "./BetaKeysApi";
-export * from "./BillingApi";
-export * from "./CommentsApi";
 export * from "./EngineApi";
 export * from "./FeaturedItemsApi";
-export * from "./JobsApi";
 export * from "./MediaFilesApi";
 export * from "./MediaUploadApi";
-export * from "./MiscApi";
-export * from "./ModerationApi";
-export * from "./PromptsApi";
-export * from "./TtsApi";
-export * from "./UserBookmarksApi";
-export * from "./UsersApi";
 export * from "./VideoApi";
 export * from "./WeightsApi";

frontend/apps/editor2d/src/GalleryRootComponent/GalleryRootComponent.tsx

@@ -3,6 +3,7 @@ import { useEffect, useState } from "react";
 import { Button } from "~/components/ui"
 
 const testUrl = "https://api.fakeyou.com/v1/media_files/list/user/hanashi?include_user_uploads=true&page_size=24&filter_media_classes=unknown";
+
 const getTestData = (callback: (data: any) => void) => {
   const netCall = async () => {
     // GET the test url

frontend/apps/editor2d/src/KonvaApp/RenderingPrimitives/RenderEngine.ts

@@ -1,607 +0,0 @@
-import Konva from "konva";
-import { Container } from "konva/lib/Container";
-import { Shape } from "konva/lib/Shape";
-import { Group } from "konva/lib/Group";
-
-import { DiffusionSharedWorkerClient } from "../SharedWorkers/Diffusion/DiffusionSharedWorkerClient";
-import {
-  // SharedWorkerRequest,
-  SharedWorkerResponse,
-} from "../WorkerPrimitives/SharedWorkerBase";
-import {
-  DiffusionSharedWorkerProgressData,
-  DiffusionSharedWorkerResponseData,
-  // DiffusionSharedWorker,
-  DiffusionSharedWorkerItemData,
-} from "../SharedWorkers/Diffusion/DiffusionSharedWorker";
-
-import { RenderingOptions } from "../Engine";
-import { FileUtilities } from "../FileUtilities/FileUtilities";
-import { ImageNode, VideoNode, TextNode } from "../Nodes";
-import { MediaNode } from "../types";
-
-import { RenderTask } from "./RenderTask";
-import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas";
-
-// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
-
-export class RenderEngine {
-  private videoNodes: VideoNode[];
-  private imageNodes: (ImageNode | TextNode)[];
-
-  private offScreenCanvas: OffscreenCanvas;
-  private context: OffscreenCanvasRenderingContext2D | null;
-
-  private isProcessing: boolean;
-
-  // private frames: ImageBitmap[];
-
-  // capturing composite within window
-  private bgLayerRef: Konva.Layer;
-  private mediaLayerRef: Konva.Layer;
-
-  private height: number;
-  private width: number;
-  private positionX: number;
-  private positionY: number;
-
-  private canUseSharedWorker: boolean;
-
-  private port: MessagePort | undefined;
-  private upperMaxFrames: number;
-
-  public captureCanvas: Konva.Rect;
-
-  public videoLoadingCanvas: VideoNode | undefined;
-
-  public fps: number = 24;
-  constructor({
-    width,
-    height,
-    bgLayerRef,
-    mediaLayerRef,
-    offScreenCanvas,
-    onRenderingSystemMessageRecieved,
-  }: {
-    width: number;
-    height: number;
-    bgLayerRef: Konva.Layer;
-    mediaLayerRef: Konva.Layer;
-    offScreenCanvas: OffscreenCanvas;
-    onRenderingSystemMessageRecieved: (
-      response: SharedWorkerResponse<
-        DiffusionSharedWorkerResponseData,
-        DiffusionSharedWorkerProgressData
-      >,
-    ) => void;
-  }) {
-    this.videoLoadingCanvas = undefined;
-    this.videoNodes = [];
-    this.imageNodes = [];
-
-    this.isProcessing = false;
-    this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved;
-    // TODO: Make this dynamic and update this on change of canvas.
-
-    this.width = width;
-    this.height = height;
-    this.positionX = window.innerWidth / 2 - this.width / 2;
-    this.positionY = window.innerHeight / 2 - this.height / 2;
-
-    this.offScreenCanvas = offScreenCanvas;
-    this.offScreenCanvas.width = this.width;
-    this.offScreenCanvas.height = this.height;
-    this.context = this.offScreenCanvas.getContext("2d");
-
-    // this.frames = [];
-
-    this.bgLayerRef = bgLayerRef;
-    this.mediaLayerRef = mediaLayerRef;
-
-    this.port = undefined;
-
-    this.fps = 24;
-
-    this.captureCanvas = new Konva.Rect({
-      name: "CaptureCanvas",
-      x: this.positionX,
-      y: this.positionY,
-      width: this.width,
-      height: this.height,
-      fill: "white",
-      stroke: "black",
-      strokeWidth: 1,
-      draggable: false,
-    });
-
-    this.upperMaxFrames = 7 * this.fps; // seconds by fps
-
-    this.bgLayerRef.add(this.captureCanvas);
-    // send back
-    this.captureCanvas.setZIndex(0);
-
-    this.canUseSharedWorker = false;
-    this.setupSharedWorker();
-
-    //this.debug();
-  }
-
-  private onRenderingSystemMessageRecieved: (
-    response: SharedWorkerResponse<
-      DiffusionSharedWorkerResponseData,
-      DiffusionSharedWorkerProgressData
-    >,
-  ) => void;
-
-  private diffusionWorker:
-    | DiffusionSharedWorkerClient<
-        DiffusionSharedWorkerItemData,
-        DiffusionSharedWorkerResponseData,
-        DiffusionSharedWorkerProgressData
-      >
-    | undefined;
-
-  async updateCaptureCanvas(
-    width: number | undefined,
-    height: number | undefined,
-  ) {
-    if (!this.captureCanvas) {
-      return;
-    }
-    if (width) {
-      this.width = width;
-    }
-    if (height) {
-      this.height = height;
-    }
-
-    // Ensures that all the nodes stag in the same place should
-    // there be a window resize.
-    // recompute the position
-    // to ensure that the position of this stays
-
-    const oldPositionX = this.positionX;
-    const oldPositionY = this.positionY;
-
-    // recompute the position
-    this.positionX = window.innerWidth / 2 - this.width / 2;
-    this.positionY = window.innerHeight / 2 - this.height / 2;
-
-    this.captureCanvas.setPosition({ x: this.positionX, y: this.positionY });
-    this.captureCanvas.size({ width: this.width, height: this.height });
-
-    // this is the change in positions
-    const deltaX = this.positionX - oldPositionX;
-    const deltaY = this.positionY - oldPositionY;
-
-    var children = this.mediaLayerRef.getChildren();
-    for (let i = 0; i < children.length; i++) {
-      let node = children[i];
-
-      // skip the capture canvas update.
-      if (node.name() === "CaptureCanvas") {
-        continue;
-      }
-      const pos = node.getPosition();
-      node.setPosition({
-        x: pos.x + deltaX,
-        y: pos.y + deltaY,
-      });
-    }
-
-    // update the context menu
-    // TODO: find selected node and update that position
-    // this.videoNodes.forEach((node) => {
-    //   node.updateContextComponents();
-    // });
-
-    this.mediaLayerRef.batchDraw();
-  }
-
-  debug() {
-    // DEBUG ONLY
-    const rectangle = new Konva.Rect({
-      x: this.positionX,
-      y: this.positionY,
-      width: 100,
-      height: 100,
-      fill: "green",
-      stroke: "black",
-      strokeWidth: 1,
-      draggable: false,
-    });
-    this.mediaLayerRef.add(rectangle);
-  }
-
-  async sendCanvasPayload(renderTask: RenderTask) {
-    if (!this.port) {
-      return console.log("Undefined Worker");
-    }
-    this.port.postMessage(renderTask);
-  }
-
-  setupSharedWorker() {
-    if (typeof SharedWorker !== "undefined") {
-      console.log("Shared Workers are supported in this browser.");
-      // Debug chrome://inspect/#workers
-      //  "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts",
-      this.diffusionWorker = new DiffusionSharedWorkerClient(
-        this.onRenderingSystemMessageRecieved,
-      );
-
-      this.canUseSharedWorker = true;
-    } else {
-      console.log("Shared Workers are not supported in this browser.");
-      // Handle the lack of Shared Worker support (e.g., fallback to another solution)
-      this.canUseSharedWorker = false;
-    }
-  }
-
-  public placeDebugRect(
-    x: number,
-    y: number,
-    width: number,
-    height: number,
-    layer: Konva.Layer,
-  ): void {
-    const rect = new Konva.Rect({
-      x: x,
-      y: y,
-      width: width, // Default width
-      height: height, // Default height
-      fill: "green",
-      draggable: false,
-    });
-
-    layer.add(rect);
-    layer.draw();
-  }
-
-  // This function uses a portion of the video layer to capture just the capture canvas.
-  // capture everything after seeking each video node.
-  renderPortionOfLayer(
-    layer: Konva.Layer,
-    x: number,
-    y: number,
-    width: number,
-    height: number,
-  ): HTMLCanvasElement {
-    const canvas = layer.toCanvas({
-      x: x,
-      y: y,
-      width: width,
-      height: height,
-    });
-    return canvas;
-  }
-
-  private findLongestVideoLength(): number {
-    let maxLength = 0;
-    // could simplify this logic to ensure that imageNodes has number of frames.
-    if (this.imageNodes.length > 0 && this.videoNodes.length === 0) {
-      return this.fps * 3; // three seconds determine whether to change this
-    }
-
-    this.videoNodes.forEach((node) => {
-      const videoLength = node.getNumberFrames();
-      if (videoLength > maxLength) {
-        maxLength = videoLength;
-      }
-    });
-    return maxLength;
-  }
-
-  public addNodes(node: MediaNode) {
-    if (node instanceof VideoNode) {
-      this.videoNodes.push(node);
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
-      this.imageNodes.push(node);
-    }
-  }
-
-  public removeNodes(node: MediaNode) {
-    if (node instanceof VideoNode) {
-      const index = this.videoNodes.indexOf(node);
-      if (index > -1) {
-        this.videoNodes.splice(index, 1);
-      }
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
-      const index = this.imageNodes.indexOf(node);
-      if (index > -1) {
-        this.imageNodes.splice(index, 1);
-      }
-    }
-  }
-
-  // Do a bunch of precondition checks and error out early on.
-  public async startProcessing(renderingOptions?: RenderingOptions) {
-    // Start processing and lock everything
-
-    this.isProcessing = true;
-
-    try {
-      // or not loaded
-      if (this.videoNodes.length + this.imageNodes.length < 1) {
-        throw Error("Must at least have 1 item on the board.");
-      }
-      // error out if nodes are not all loaded.
-      // todo remove when we have error handling + and ui
-      var failed = false;
-
-      // if there are atleast 1 video node images are all covered.
-      for (let i = 0; i < this.videoNodes.length; i++) {
-        const item = this.videoNodes[i];
-        if (!item.kNode) {
-          return;
-        }
-        item.kNode.listening(false);
-        item.unhighlight();
-        if (item.didFinishLoading == false) {
-          // error out and show error message
-          //this.startProcessing();
-          failed = true;
-          setTimeout(this.startProcessing.bind(this), 1000);
-          break;
-        }
-        item.setProcessing(true);
-      }
-
-      // todo remove
-      if (failed) {
-        // throw error
-        throw Error("Wait For Items to Finish Processing.");
-      }
-
-      //this.videoNodes.forEach((item: VideoNode) => {});
-
-      // find the longest video node
-      let numberOfFrames = this.findLongestVideoLength();
-      numberOfFrames = Math.min(numberOfFrames, this.upperMaxFrames);
-      console.log(`Number Of Frames: ${numberOfFrames}`);
-
-      await this.render(numberOfFrames, renderingOptions);
-    } catch (error) {
-      console.log(error);
-      throw error;
-    } finally {
-      this.isProcessing = false;
-
-      // enable all nodes again.
-      for (let i = 0; i < this.videoNodes.length; i++) {
-        const item = this.videoNodes[i];
-        if (!item.kNode) {
-          return;
-        }
-        item.setProcessing(false);
-        item.kNode.listening(true);
-      }
-    }
-  }
-
-  // public stopProcessing() {
-  //   this.isProcessing = false;
-  // }
-
-  // find the frame time given the frame number
-  private calculateFrameTime(frameNumber: number, frameRate: number): number {
-    return frameNumber / frameRate;
-  }
-
-  /**
-   *
-   * @param config
-   */
-  private async renderFrame(config: {
-    layerOfInterest: Konva.Layer; // layer where the element that you want to clip lives.
-    // XY and height of a captureCanvas ( region of interest )
-    x?: number; // x position of the region of interest
-    y?: number; // y position of the region of interest
-    width?: number; // size of the region of interest
-    height?: number; // size of the region of interest
-    pixelRatio?: number; // higher means higher quality
-    mimeType?: string; // image/jpeg or image/png
-    quality?: number; // 1.0 is the best.
-    test: boolean; // true == blob else Image Bitmap
-  }): Promise<ImageBitmap | Blob> {
-    try {
-      const box = config.layerOfInterest.getClientRect();
-      const stage = config.layerOfInterest.getStage();
-
-      const x = config.x !== undefined ? config.x : Math.floor(box.x);
-      const y = config.y !== undefined ? config.y : Math.floor(box.y);
-      const pixelRatio = config.pixelRatio || 1;
-
-      const container = config.layerOfInterest as Container<Group | Shape>;
-
-      const offScreenSceneCanvas = new OffScreenSceneCanvas({
-        width:
-          config.width || Math.ceil(box.width) || (stage ? stage.width() : 0),
-        height:
-          config.height ||
-          Math.ceil(box.height) ||
-          (stage ? stage.height() : 0),
-        pixelRatio: pixelRatio,
-      });
-
-      // TODO: fill canvas white properly
-      const context = offScreenSceneCanvas.getContext();
-      // context.fillStyle = "white";
-      // context.fillRect(
-      //   0,
-      //   0,
-      //   offScreenSceneCanvas.width,
-      //   offScreenSceneCanvas.height,
-      // );
-
-      const buffer = new OffScreenSceneCanvas({
-        width:
-          offScreenSceneCanvas.width / offScreenSceneCanvas.pixelRatio +
-          Math.abs(x),
-        height:
-          offScreenSceneCanvas.height / offScreenSceneCanvas.pixelRatio +
-          Math.abs(y),
-        pixelRatio: offScreenSceneCanvas.pixelRatio,
-      });
-
-      context.save();
-
-      if (x || y) {
-        context.translate(-1 * x, -1 * y);
-      }
-      container.drawScene(offScreenSceneCanvas, undefined, buffer);
-      // Not a type mistake ... DO NOT FIX
-      // @ts-ignore
-      const offscreenCanvas = offScreenSceneCanvas._canvas as OffscreenCanvas;
-      let result = undefined;
-
-      // if config.test is true, the result is downloaded to the local files
-      // config.test = true;
-      if (config.test) {
-        const blob = await offscreenCanvas.convertToBlob({
-          quality: config.quality ?? 1.0,
-          type: "image/jpeg",
-        });
-        await FileUtilities.blobToFileJpeg(blob, "1");
-        result = blob;
-      } else {
-        result = offscreenCanvas.transferToImageBitmap();
-      }
-      context.restore();
-      return result;
-    } catch (error) {
-      throw error;
-    }
-  }
-  /** 
-  find longest video
-  then seek through each node 1 step.
-  stop ignore stepping if the duration is less.
-  **/
-  private async render(
-    largestNumberOfFrames: number,
-    renderingOptions?: RenderingOptions,
-  ) {
-    if (!this.isProcessing) return;
-
-    // Stop all nodes first
-    console.log(`LargestNumberOfFrames:${largestNumberOfFrames}`);
-    for (let k = 0; k < this.videoNodes.length; k++) {
-      const videoNode = this.videoNodes[k];
-      if (videoNode.didFinishLoading === false) {
-        throw Error("Videos Did Not Finish Loading Please Try Again.");
-      }
-      await videoNode.reset();
-    }
-
-    // only pick nodes that intersect with the canvas on screen bounds to freeze.
-    for (let j = 0; j < largestNumberOfFrames; j++) {
-      // Seek Video Nodes first then draw
-      let frameTime = undefined;
-
-      for (let i = 0; i < this.videoNodes.length; i++) {
-        const currentVideoNode = this.videoNodes[i];
-        frameTime = this.calculateFrameTime(j, currentVideoNode.fps);
-        frameTime = parseFloat(frameTime.toFixed(2));
-        if (frameTime < currentVideoNode.duration) {
-          // console.log(`CurrentFrame:${j}`);
-          // console.log(`FrameTime:${frameTime}`);
-          // console.log(`Duration:${currentVideoNode.duration}`);
-          await currentVideoNode.seek(frameTime);
-        } // end of if context
-      } // End frame time
-      this.mediaLayerRef.draw();
-
-      // use main thread
-      if (this.canUseSharedWorker === false) {
-        // SCOPES the capture for the context
-        // Correct size for the mobile canvas.
-        this.offScreenCanvas.width = this.width;
-        this.offScreenCanvas.height = this.height;
-        if (this.context) {
-          // This crops it starting at position X / Y where the mobile canvas is
-          // Then picks the height and width range
-          // then we draw it at 0,0,width and height of the canvas
-          this.context.drawImage(
-            this.mediaLayerRef.canvas._canvas,
-            this.positionX,
-            this.positionY,
-            this.width,
-            this.height,
-            0,
-            0,
-            this.width,
-            this.height,
-          );
-          //TODO write the non webworker version
-          const blob = await this.offScreenCanvas.convertToBlob({
-            quality: 1.0,
-            type: "image/jpeg",
-          });
-          await FileUtilities.blobToFileJpeg(blob, "1");
-
-          break;
-        } // end of for each frame
-      } else {
-        // decode on the shared webworker.
-        if (!this.context) {
-          console.log("Context Didn't Initialize");
-          return;
-        }
-
-        if (!this.diffusionWorker) {
-          console.log("Didn't Initialize Diffusion");
-          return;
-        }
-
-        console.log(
-          `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`,
-        );
-
-        const bitmap = await this.renderFrame({
-          layerOfInterest: this.mediaLayerRef,
-          x: this.captureCanvas.x(),
-          y: this.captureCanvas.y(),
-          width: this.width,
-          height: this.height,
-          mimeType: "image/jpeg",
-          pixelRatio: 1,
-          quality: 1.0,
-          test: false,
-        });
-
-        const data: DiffusionSharedWorkerItemData = {
-          height: this.height,
-          width: this.width,
-          imageBitmap: bitmap as ImageBitmap,
-          frame: j,
-          totalFrames: largestNumberOfFrames,
-          prompt: renderingOptions,
-        };
-        console.log(
-          `Processing Frame:${j + 1} out of ${largestNumberOfFrames}`,
-        );
-        this.diffusionWorker.sendData(1, data, false);
-      }
-    } // end of largest number of frames loop
-
-    // finished looping
-    if (!this.diffusionWorker) {
-      console.log("Didn't Initialize Diffusion");
-      return;
-    }
-
-    const jobID = 1;
-    this.diffusionWorker.sendData(
-      jobID,
-      {
-        height: this.height,
-        width: this.width,
-        imageBitmap: undefined,
-        frame: -1,
-        totalFrames: largestNumberOfFrames,
-        prompt: renderingOptions,
-      },
-      true,
-    );
-  }
-}

frontend/apps/editor2d/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts

@@ -1,396 +0,0 @@
-// Example of this working.
-
-import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
-import { MediaUploadApi, VideoApi, MediaFilesApi } from "~/Classes/ApiManager";
-import { Visibility } from "~/Classes/ApiManager/enums/Visibility";
-import { v4 as uuidv4 } from "uuid";
-import { JobsApi } from "~/Classes/ApiManager";
-import {
-  ProgressData,
-  WorkResult,
-} from "~/KonvaApp/WorkerPrimitives/GenericWorker";
-
-import {
-  SharedWorkerBase,
-  SharedWorkerRequest,
-  SharedWorkerResponse,
-  ResponseType,
-} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
-import { JobStatus } from "~/Classes/ApiManager/enums/Job";
-import { RenderingOptions } from "~/KonvaApp/Engine";
-import { MediaFile } from "~/Classes/ApiManager/models/MediaFile";
-
-export interface DiffusionSharedWorkerProgressData {
-  url: string;
-  status: JobStatus;
-  progress: number;
-  mediaToken: string;
-}
-
-export interface DiffusionSharedWorkerItemData {
-  imageBitmap: ImageBitmap | undefined;
-  totalFrames: number;
-  frame: number;
-  height: number;
-  width: number;
-  prompt?: RenderingOptions;
-}
-
-export interface DiffusionSharedWorkerErrorData {
-  error: string;
-}
-export interface DiffusionSharedWorkerResponseData {
-  videoUrl: string;
-}
-
-export class DiffusionSharedWorker extends SharedWorkerBase<
-  DiffusionSharedWorkerItemData,
-  DiffusionSharedWorkerResponseData | MediaFile,
-  DiffusionSharedWorkerProgressData
-> {
-  public name: string;
-  public zipFileWriter: BlobWriter;
-  public zipWriter: ZipWriter<Blob>;
-  public imageType: string;
-  public totalFrames: number;
-
-  public offscreenCanvas: OffscreenCanvas | undefined;
-  public bitmapContext: ImageBitmapRenderingContext | undefined | null;
-
-  public mediaAPI: MediaUploadApi;
-  public mediaFilesAPI: MediaFilesApi;
-  public videoAPI: VideoApi;
-  public jobsAPI: JobsApi;
-  public blobs: Blob[];
-
-  constructor(port: MessagePort) {
-    super(port);
-    this.name = "DiffusionWorker";
-    this.setup(this.workFunction.bind(this), this.progressFunction.bind(this));
-    this.offscreenCanvas = undefined;
-    this.bitmapContext = undefined;
-    this.imageType = "image/jpeg";
-    this.zipFileWriter = new BlobWriter(this.imageType);
-    this.zipWriter = new ZipWriter(this.zipFileWriter);
-    this.totalFrames = 0;
-
-    this.videoAPI = new VideoApi();
-    this.jobsAPI = new JobsApi();
-    this.mediaAPI = new MediaUploadApi();
-    this.mediaFilesAPI = new MediaFilesApi();
-
-    this.blobs = [];
-
-    this.totalFrames = 0;
-  }
-
-  async zipBlobs(): Promise<Blob> {
-    for (let i = 0; i < this.blobs.length; i++) {
-      const blob = this.blobs[i];
-      const name = String(i).padStart(5, "0"); // '0009'
-      await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
-    }
-    const zipBlob = await this.zipWriter.close();
-
-    return zipBlob;
-  }
-
-  async reset() {
-    this.zipFileWriter = new BlobWriter(this.imageType);
-    this.zipWriter = new ZipWriter(this.zipFileWriter);
-    this.totalFrames = 0;
-    this.blobs = [];
-  }
-  // Data here will be shipped off for progressive loading
-  async workFunction(
-    isDoneStreaming: boolean,
-    item: DiffusionSharedWorkerItemData,
-    reportProgress: (data: DiffusionSharedWorkerProgressData) => void,
-  ): Promise<
-    [DiffusionSharedWorkerResponseData | MediaFile | undefined, boolean]
-  > {
-    // make request via api with options
-
-    try {
-      this.totalFrames = item.totalFrames;
-
-      if (this.offscreenCanvas === undefined) {
-        this.offscreenCanvas = new OffscreenCanvas(item.width, item.height);
-        this.bitmapContext = this.offscreenCanvas.getContext("bitmaprenderer");
-      }
-
-      if (!this.bitmapContext) {
-        console.log("Failed to create bitmap context.");
-        throw Error("Bitmap Rendering Context Not Availible.");
-      }
-
-      if (item.imageBitmap !== undefined) {
-        this.bitmapContext.transferFromImageBitmap(item.imageBitmap);
-
-        const blob = await this.offscreenCanvas.convertToBlob({
-          quality: 1.0,
-          type: this.imageType,
-        });
-
-        this.blobs.push(blob);
-        console.log("Length of blob");
-        console.log(this.blobs.length);
-      }
-
-      // progress
-      const aproxSteps = this.totalFrames;
-      const totalPercent = item.prompt ? 0.25 : 0.9;
-
-      const progressData: DiffusionSharedWorkerProgressData = {
-        url: "",
-        status: JobStatus.PENDING,
-        progress: (item.frame / aproxSteps) * totalPercent,
-        mediaToken: "",
-      };
-
-      console.log("Lets report progress.");
-      reportProgress(progressData); // once finished gives you up to 50%
-
-      if (isDoneStreaming === false) {
-        return [undefined, false];
-      }
-
-      console.log("Prepare to Zip");
-      const zipBlob = await this.zipBlobs();
-      console.log("Zipped", zipBlob);
-
-      const progressMedia: DiffusionSharedWorkerProgressData = {
-        url: "",
-        status: JobStatus.PENDING,
-        progress: ((item.frame + 0.5) / aproxSteps) * totalPercent,
-        mediaToken: "",
-      };
-
-      console.log("Lets report progress.");
-      reportProgress(progressMedia); // once finished gives you up to 50%
-
-      const response = await this.mediaAPI.UploadStudioShot({
-        maybe_title: "",
-        uuid_idempotency_token: uuidv4(),
-        blob: zipBlob,
-        fileName: "media.zip",
-        maybe_visibility: Visibility.Public,
-      });
-
-      const mediaToken = response.data;
-
-      if (!mediaToken) {
-        this.reset();
-        throw Error(`Server Failed Try Again: ${response.errorMessage}`);
-      }
-      if (!item.prompt) {
-        console.log("got video >>", response);
-        const videoRepsonse = await this.mediaFilesAPI.GetMediaFileByToken({
-          mediaFileToken: mediaToken,
-        });
-        this.reset();
-        return [videoRepsonse.data, true];
-      }
-      console.log(item.prompt);
-      const studioResponse = await this.videoAPI.EnqueueStudio({
-        enqueueVideo: {
-          disable_lcm: false,
-          enable_lipsync: item.prompt.lipSync,
-          input_file: mediaToken, // Replace with actual media token
-          negative_prompt: item.prompt.negativePrompt,
-          prompt: item.prompt.positivePrompt,
-          remove_watermark: false,
-          style: item.prompt.artstyle, // Map to the appropriate art style
-          frame_skip: 2,
-          travel_prompt: "",
-          trim_end_millis: 7000,
-          trim_start_millis: 0,
-          use_cinematic: item.prompt.cinematic,
-          use_face_detailer: item.prompt.faceDetail,
-          use_strength: item.prompt.styleStrength,
-          use_upscaler: item.prompt.upscale,
-          uuid_idempotency_token: uuidv4(),
-          global_ipa_media_token: "",
-          input_depth_file: "",
-          input_normal_file: "",
-          input_outline_file: "",
-          creator_set_visibility: Visibility.Public,
-        },
-      });
-
-      let resultURL = undefined;
-
-      if (studioResponse.success && studioResponse.data?.inference_job_token) {
-        console.log("Start Streaming Result");
-        if (!studioResponse.data.inference_job_token) {
-          await this.reset(); // reset on error
-          throw Error("No Job Token Returned Try Again");
-        }
-        const jobToken = studioResponse.data.inference_job_token;
-
-        // if error send it back through the pipe
-        let jobIsProcessing = true;
-
-        while (jobIsProcessing) {
-          const job = await this.jobsAPI.GetJobByToken({ token: jobToken });
-
-          // TODO: MAKE REQUEST FOR PREVIEW HERE
-          console.log(job);
-          if (!job.data) {
-            console.log("No Job Data resetting the zip state");
-            await this.reset();
-            console.log(job.data);
-            continue;
-          }
-          const status = job.data.status.status;
-          const progress = job.data.status.progress_percentage;
-
-          let computedProgress = 30;
-          if (progress === 0) {
-            computedProgress = 30;
-          }
-          computedProgress = 30 + progress * 0.7;
-
-          let renderProgressData: DiffusionSharedWorkerProgressData = {
-            url: "",
-            status: status,
-            progress: computedProgress / 100,
-            mediaToken: mediaToken,
-          };
-
-          renderProgressData.status = status;
-
-          switch (status) {
-            case JobStatus.PENDING:
-              console.log("Pending");
-              reportProgress(renderProgressData); // once finished gives you up to 50%
-              break;
-            case JobStatus.STARTED:
-              console.log("Started");
-              reportProgress(renderProgressData);
-              break;
-            case JobStatus.ATTEMPT_FAILED:
-              console.log("Attempt Failed");
-              reportProgress(renderProgressData);
-              break;
-            case JobStatus.COMPLETE_SUCCESS:
-              console.log("Complete Success");
-              renderProgressData.progress = 100;
-              jobIsProcessing = false;
-              if (!job.data.maybe_result.media_links.cdn_url) {
-                await this.reset();
-                throw Error("Server Failed To Return Result");
-              }
-              resultURL = job.data.maybe_result.media_links.cdn_url;
-              reportProgress(renderProgressData);
-              break;
-            case JobStatus.COMPLETE_FAILURE:
-              console.log("Complete Failure");
-              jobIsProcessing = false;
-              renderProgressData.progress = 0;
-              reportProgress(renderProgressData);
-              break;
-            case JobStatus.DEAD:
-              console.log("Dead");
-              jobIsProcessing = false;
-              reportProgress(renderProgressData);
-              throw Error("Server Failed to Process Please Try Again.");
-            case JobStatus.CANCELLED_BY_SYSTEM:
-              console.log("Cancelled by system");
-              jobIsProcessing = false;
-              reportProgress(renderProgressData);
-              break;
-            case JobStatus.CANCELLED_BY_USER:
-              console.log("Cancelled by user");
-              jobIsProcessing = false;
-              reportProgress(renderProgressData);
-              break;
-          }
-          await this.sleep(500);
-        } // end while loop
-      }
-      if (!resultURL) {
-        await this.reset(); // reset on error
-        throw Error("Media URL Result Missing");
-      }
-
-      const responseData: DiffusionSharedWorkerResponseData = {
-        videoUrl: resultURL,
-      };
-      await this.reset();
-      return [responseData, true];
-    } catch (error) {
-      await this.reset();
-      console.log(error);
-      throw error;
-    }
-  }
-
-  async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
-  progressFunction(progress: ProgressData<DiffusionSharedWorkerProgressData>) {
-    console.log(
-      `Progress Function  JobID:${progress.jobID} Data:${progress.data}`,
-    );
-
-    // send out to node as a worker response
-    this.send({
-      jobID: progress.jobID,
-      responseType: ResponseType.progress,
-      data: progress.data,
-    });
-  }
-
-  async reportResult(result: WorkResult<DiffusionSharedWorkerResponseData>) {
-    this.send({
-      jobID: result.jobID,
-      responseType: ResponseType.result,
-      data: result.data,
-    });
-  }
-
-  async errorFunction(
-    error: SharedWorkerResponse<
-      DiffusionSharedWorkerItemData,
-      DiffusionSharedWorkerResponseData
-    >,
-  ) {
-    this.send({
-      jobID: error.jobID,
-      responseType: ResponseType.error,
-      data: error.data?.toString(),
-    });
-  }
-
-  async receive(request: SharedWorkerRequest<DiffusionSharedWorkerItemData>) {
-    console.log("Received Request");
-    console.log(request);
-    this.submitWork({
-      jobID: request.jobID,
-      data: request.data,
-      isDoneStreaming: request.isDoneStreaming,
-    });
-  }
-}
-
-// This is a copy paste to create a worker now.
-//@ts-ignore
-self.onconnect = (e: any) => {
-  const port = e.ports[0];
-  console.log("DiffusionSharedWorker Started");
-  let worker: DiffusionSharedWorker | undefined = undefined;
-  let started = false;
-
-  if (started === false) {
-    started = true;
-    worker = new DiffusionSharedWorker(port);
-    worker.start();
-  }
-
-  // Response For Start.
-  const workerResult = "DiffusionSharedWorker Started";
-  port.postMessage(workerResult);
-  port.start(); // Required when using addEventListener. Otherwise called implicitly by onmessage setter.
-};

frontend/apps/editor2d/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts

@@ -1,88 +0,0 @@
-import { v4 as uuidv4 } from "uuid";
-import {
-  SharedWorkerRequest,
-  SharedWorkerResponse,
-} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
-import { ResponseType } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
-import DiffusionSharedWorker from "./DiffusionSharedWorker?sharedworker";
-
-export class DiffusionSharedWorkerClient<
-  DiffusionSharedWorkerItemData,
-  DiffusionSharedWorkerResponseData,
-  DiffusionSharedWorkerProgressData,
-> {
-  private port: MessagePort;
-  private sharedWorker: SharedWorker;
-  private messageReceived: (
-    response: SharedWorkerResponse<
-      DiffusionSharedWorkerResponseData,
-      DiffusionSharedWorkerProgressData
-    >,
-  ) => void;
-  constructor(
-    messageReceived: (
-      response: SharedWorkerResponse<
-        DiffusionSharedWorkerResponseData,
-        DiffusionSharedWorkerProgressData
-      >,
-    ) => void,
-  ) {
-    this.messageReceived = messageReceived;
-    try {
-      console.log("This is running a worker in production");
-      this.sharedWorker = new DiffusionSharedWorker({
-        name: "DiffusionWorker-" + uuidv4(),
-      });
-
-      this.sharedWorker.addEventListener("error", (value) => {
-        console.log("Shared worker ERROR:");
-        console.log(value);
-      });
-
-      this.port = this.sharedWorker.port;
-      this.port.onmessage = this.onMessage.bind(this);
-      this.port.start();
-
-      console.log("launched shared worker (?)");
-    } catch (error) {
-      console.log("ERROR with shared worker!");
-      console.log(error);
-      throw Error("Could Not Start Worker");
-    }
-  }
-
-  async onMessage(event: MessageEvent) {
-    // returns progress and returns result need to check error tomorrow then were good to go.
-    //console.log(`incoming`);
-    // console.log(event);
-    if (event.data.responseType === ResponseType.error) {
-      console.log(`DiffusionSharedWorkerClient Error`);
-      console.log(event.data);
-      this.messageReceived(event.data);
-    } else if (event.data.responseType === ResponseType.progress) {
-      // console.log(`DiffusionSharedWorkerClient Progress`);
-      //console.log(event.data);
-      this.messageReceived(event.data);
-    } else if (event.data.responseType === ResponseType.result) {
-      // console.log(`DiffusionSharedWorkerClient Result`);
-      //console.log(event.data);
-      this.messageReceived(event.data);
-    } else {
-      console.log(`DiffusionSharedWorkerClient Message Unknown:`);
-      console.log(event.data);
-    }
-  }
-
-  async sendData(
-    jobID: number,
-    data: DiffusionSharedWorkerItemData,
-    isDoneStreaming: boolean,
-  ) {
-    const payload: SharedWorkerRequest<DiffusionSharedWorkerItemData> = {
-      jobID: jobID,
-      data: data,
-      isDoneStreaming: isDoneStreaming,
-    };
-    this.port.postMessage(payload);
-  }
-}

frontend/apps/editor2d/src/app/pages/login.tsx

@@ -44,10 +44,10 @@ export const Login = () => {
 
       const usernameOrEmail = form.get("usernameOrEmail")?.toString();
       const password = form.get("password")?.toString();
-   
+
       if (usernameOrEmail && password) {
         let api = new UsersApi();
-        
+
         let response = await api.Login({
           usernameOrEmail,
           password,
@@ -99,7 +99,6 @@ export const Login = () => {
               shouldShowLoader && "opacity-0",
             )}
           >
-            
             <Input
               label="Username or Email"
               icon={faUser}

frontend/apps/editor2d/src/components/ImageGenerator.tsx

@@ -1,524 +0,0 @@
-import React, { useEffect, useState, useRef } from "react";
-import {
-  ServerClient,
-  fileToBase64,
-  base64ToImageUrl,
-} from "../PyServer/ServerClient";
-import {
-  ProgressUpdateMessage,
-  SetupResponse,
-  GenerateResponse,
-  StatusResponse,
-  UpdateSettingsResponse,
-} from "../PyServer/ServerTypes";
-
-// Default server URL
-const DEFAULT_SERVER_URL = "ws://localhost:8765";
-
-export const ImageGenerator: React.FC = () => {
-  // Server connection state
-  const [isConnected, setIsConnected] = useState(false);
-  const [serverUrl, setServerUrl] = useState(DEFAULT_SERVER_URL);
-  const clientRef = useRef<ServerClient | null>(null);
-
-  // Model state
-  const [modelPath, setModelPath] = useState("");
-  const [loraPath, setLoraPath] = useState("");
-  const [isModelInitialized, setIsModelInitialized] = useState(false);
-
-  // Generation parameters
-  const [sourceImage, setSourceImage] = useState<File | null>(null);
-  const [sourceImageUrl, setSourceImageUrl] = useState<string | null>(null);
-  const [prompt, setPrompt] = useState("");
-  const [loraStrength, setLoraStrength] = useState(1.0);
-  const [imageToImageStrength, setImageToImageStrength] = useState(0.75);
-  const [steps, setSteps] = useState(4);
-  const [guidanceScale, setGuidanceScale] = useState(1.0);
-  const [width, setWidth] = useState(1024);
-  const [height, setHeight] = useState(1024);
-
-  // Results and UI state
-  const [generatedImageUrl, setGeneratedImageUrl] = useState<string | null>(
-    null,
-  );
-  const [isLoading, setIsLoading] = useState(false);
-  const [progress, setProgress] = useState<ProgressUpdateMessage | null>(null);
-  const [errorMessage, setErrorMessage] = useState<string | null>(null);
-
-  // Initialize the client
-  useEffect(() => {
-    const client = new ServerClient(serverUrl, {
-      onOpen: () => {
-        console.log("Connected to server");
-        setIsConnected(true);
-        setErrorMessage(null);
-
-        // Request status to see if model is already initialized
-        client.requestStatus().catch((error) => {
-          console.error("Error requesting status:", error);
-        });
-      },
-
-      onClose: () => {
-        console.log("Disconnected from server");
-        setIsConnected(false);
-        setIsModelInitialized(false);
-      },
-
-      onError: (error) => {
-        console.error("Error:", error);
-        setErrorMessage(error.error);
-        setIsLoading(false);
-      },
-
-      onProgressUpdate: (update) => {
-        console.log("Progress:", update);
-        setProgress(update);
-
-        if (update.stage !== "error") {
-          setErrorMessage(null);
-        }
-      },
-
-      onSetupResponse: (response) => {
-        console.log("Setup response:", response);
-        setIsLoading(false);
-
-        if (response.success) {
-          setIsModelInitialized(true);
-          setErrorMessage(null);
-        } else {
-          setErrorMessage(response.error || "Unknown error during setup");
-        }
-      },
-
-      onUpdateSettingsResponse: (response) => {
-        console.log("Update settings response:", response);
-        setIsLoading(false);
-
-        if (response.success) {
-          setErrorMessage(null);
-        } else {
-          setErrorMessage(response.error || "Unknown error updating settings");
-        }
-      },
-
-      onGenerateResponse: (response) => {
-        console.log("Generate response:", response);
-        setIsLoading(false);
-
-        if (response.success && response.image) {
-          setGeneratedImageUrl(base64ToImageUrl(response.image));
-          setErrorMessage(null);
-        } else {
-          setErrorMessage(response.error || "Unknown error generating image");
-        }
-      },
-
-      onStatusResponse: (response) => {
-        console.log("Status response:", response);
-
-        if (response.success) {
-          setIsModelInitialized(response.model_initialized);
-
-          if (response.current_settings) {
-            // Update UI with current settings
-            setModelPath(response.current_settings.sdxl_checkpoint_path || "");
-            setLoraPath(response.current_settings.lora_path || "");
-
-            // Update generation parameters if they exist
-            if (response.current_settings.default_lora_strength !== undefined) {
-              setLoraStrength(response.current_settings.default_lora_strength);
-            }
-
-            if (
-              response.current_settings.default_image_to_image_strength !==
-              undefined
-            ) {
-              setImageToImageStrength(
-                response.current_settings.default_image_to_image_strength,
-              );
-            }
-
-            if (response.current_settings.default_steps !== undefined) {
-              setSteps(response.current_settings.default_steps);
-            }
-
-            if (
-              response.current_settings.default_guidance_scale !== undefined
-            ) {
-              setGuidanceScale(
-                response.current_settings.default_guidance_scale,
-              );
-            }
-
-            if (response.current_settings.default_width !== undefined) {
-              setWidth(response.current_settings.default_width);
-            }
-
-            if (response.current_settings.default_height !== undefined) {
-              setHeight(response.current_settings.default_height);
-            }
-          }
-        }
-      },
-    });
-
-    clientRef.current = client;
-
-    // Connect to the server
-    client.connect().catch((error) => {
-      console.error("Error connecting to server:", error);
-      setErrorMessage(`Failed to connect to server: ${error.message}`);
-    });
-
-    return () => {
-      client.disconnect();
-      clientRef.current = null;
-    };
-  }, [serverUrl]);
-
-  // Handle source image selection
-  const handleImageChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
-    const file = e.target.files?.[0];
-    if (file) {
-      setSourceImage(file);
-      setSourceImageUrl(URL.createObjectURL(file));
-    }
-  };
-
-  // Handle setup submission
-  const handleSetup = async () => {
-    if (!clientRef.current) return;
-
-    setIsLoading(true);
-    setErrorMessage(null);
-
-    try {
-      await clientRef.current.setup(modelPath, loraPath || undefined);
-    } catch (error) {
-      console.error("Error setting up model:", error);
-      setErrorMessage(`Error setting up model: ${(error as Error).message}`);
-      setIsLoading(false);
-    }
-  };
-
-  // Handle update settings
-  const handleUpdateSettings = async () => {
-    if (!clientRef.current) return;
-
-    setIsLoading(true);
-    setErrorMessage(null);
-
-    try {
-      await clientRef.current.updateSettings({
-        lora_path: loraPath || null,
-        default_lora_strength: loraStrength,
-        default_image_to_image_strength: imageToImageStrength,
-        default_steps: steps,
-        default_guidance_scale: guidanceScale,
-        default_width: width,
-        default_height: height,
-      });
-    } catch (error) {
-      console.error("Error updating settings:", error);
-      setErrorMessage(`Error updating settings: ${(error as Error).message}`);
-      setIsLoading(false);
-    }
-  };
-
-  // Handle generation
-  const handleGenerate = async () => {
-    if (!clientRef.current || !sourceImage) {
-      setErrorMessage("Please select a source image first");
-      return;
-    }
-
-    setIsLoading(true);
-    setErrorMessage(null);
-    setGeneratedImageUrl(null);
-
-    try {
-      const base64Image = await fileToBase64(sourceImage);
-
-      await clientRef.current.generate({
-        image: base64Image,
-        prompt,
-        lora_strength: loraStrength,
-        image_to_image_strength: imageToImageStrength,
-        num_inference_steps: steps,
-        guidance_scale: guidanceScale,
-        generated_image_width: width,
-        generated_image_height: height,
-      });
-    } catch (error) {
-      console.error("Error generating image:", error);
-      setErrorMessage(`Error generating image: ${(error as Error).message}`);
-      setIsLoading(false);
-    }
-  };
-
-  return (
-    <div className="mx-auto max-w-4xl p-6">
-      <h1 className="mb-6 text-2xl font-bold">SDXL Image Generator</h1>
-
-      {/* Connection and Setup */}
-      <div className="mb-6 rounded-lg bg-gray-100 p-4">
-        <h2 className="mb-2 text-xl font-semibold">Server Connection</h2>
-        <div className="mb-4 flex gap-2">
-          <input
-            type="text"
-            value={serverUrl}
-            onChange={(e) => setServerUrl(e.target.value)}
-            className="flex-grow rounded border p-2"
-            placeholder="WebSocket URL"
-          />
-          <button
-            onClick={() => clientRef.current?.connect()}
-            disabled={isConnected}
-            className="rounded bg-blue-500 px-4 py-2 text-white disabled:bg-gray-400"
-          >
-            Connect
-          </button>
-          <button
-            onClick={() => clientRef.current?.disconnect()}
-            disabled={!isConnected}
-            className="rounded bg-red-500 px-4 py-2 text-white disabled:bg-gray-400"
-          >
-            Disconnect
-          </button>
-        </div>
-
-        <div className="mb-2 flex items-center">
-          <span className="mr-2">Status:</span>
-          <span
-            className={`rounded px-2 py-1 text-white ${isConnected ? "bg-green-500" : "bg-red-500"}`}
-          >
-            {isConnected ? "Connected" : "Disconnected"}
-          </span>
-          <span className="ml-4 mr-2">Model:</span>
-          <span
-            className={`rounded px-2 py-1 text-white ${isModelInitialized ? "bg-green-500" : "bg-yellow-500"}`}
-          >
-            {isModelInitialized ? "Initialized" : "Not Initialized"}
-          </span>
-        </div>
-      </div>
-
-      {/* Model Setup */}
-      <div className="mb-6 rounded-lg bg-gray-100 p-4">
-        <h2 className="mb-2 text-xl font-semibold">Model Setup</h2>
-        <div className="space-y-4">
-          <div>
-            <label className="mb-1 block">SDXL Checkpoint Path:</label>
-            <input
-              type="text"
-              value={modelPath}
-              onChange={(e) => setModelPath(e.target.value)}
-              className="w-full rounded border p-2"
-              placeholder="/path/to/sdxl/model"
-            />
-          </div>
-          <div>
-            <label className="mb-1 block">LoRA Path (optional):</label>
-            <input
-              type="text"
-              value={loraPath}
-              onChange={(e) => setLoraPath(e.target.value)}
-              className="w-full rounded border p-2"
-              placeholder="/path/to/lora"
-            />
-          </div>
-          <div className="flex gap-2">
-            <button
-              onClick={handleSetup}
-              disabled={!isConnected || !modelPath || isLoading}
-              className="rounded bg-green-500 px-4 py-2 text-white disabled:bg-gray-400"
-            >
-              Initialize Model
-            </button>
-            <button
-              onClick={handleUpdateSettings}
-              disabled={!isConnected || !isModelInitialized || isLoading}
-              className="rounded bg-purple-500 px-4 py-2 text-white disabled:bg-gray-400"
-            >
-              Update Settings
-            </button>
-          </div>
-        </div>
-      </div>
-
-      {/* Generation Parameters */}
-      <div className="mb-6 rounded-lg bg-gray-100 p-4">
-        <h2 className="mb-2 text-xl font-semibold">Generation Parameters</h2>
-        <div className="grid grid-cols-1 gap-4 md:grid-cols-2">
-          <div>
-            <label className="mb-1 block">Prompt:</label>
-            <textarea
-              value={prompt}
-              onChange={(e) => setPrompt(e.target.value)}
-              className="h-20 w-full rounded border p-2"
-              placeholder="Enter your prompt..."
-            />
-          </div>
-
-          <div>
-            <label className="mb-1 block">Source Image:</label>
-            <input
-              type="file"
-              accept="image/*"
-              onChange={handleImageChange}
-              className="mb-2"
-            />
-            {sourceImageUrl && (
-              <img
-                src={sourceImageUrl}
-                alt="Source"
-                className="max-h-40 rounded border"
-              />
-            )}
-          </div>
-
-          <div>
-            <label className="mb-1 block">
-              LoRA Strength: {loraStrength.toFixed(2)}
-            </label>
-            <input
-              type="range"
-              min="0"
-              max="1"
-              step="0.05"
-              value={loraStrength}
-              onChange={(e) => setLoraStrength(parseFloat(e.target.value))}
-              className="w-full"
-            />
-          </div>
-
-          <div>
-            <label className="mb-1 block">
-              Image-to-Image Strength: {imageToImageStrength.toFixed(2)}
-            </label>
-            <input
-              type="range"
-              min="0"
-              max="1"
-              step="0.05"
-              value={imageToImageStrength}
-              onChange={(e) =>
-                setImageToImageStrength(parseFloat(e.target.value))
-              }
-              className="w-full"
-            />
-          </div>
-
-          <div>
-            <label className="mb-1 block">Steps: {steps}</label>
-            <input
-              type="range"
-              min="1"
-              max="20"
-              step="1"
-              value={steps}
-              onChange={(e) => setSteps(parseInt(e.target.value))}
-              className="w-full"
-            />
-          </div>
-
-          <div>
-            <label className="mb-1 block">
-              Guidance Scale: {guidanceScale.toFixed(2)}
-            </label>
-            <input
-              type="range"
-              min="0"
-              max="10"
-              step="0.1"
-              value={guidanceScale}
-              onChange={(e) => setGuidanceScale(parseFloat(e.target.value))}
-              className="w-full"
-            />
-          </div>
-
-          <div>
-            <label className="mb-1 block">Width:</label>
-            <select
-              value={width}
-              onChange={(e) => setWidth(parseInt(e.target.value))}
-              className="w-full rounded border p-2"
-            >
-              <option value="512">512</option>
-              <option value="768">768</option>
-              <option value="1024">1024</option>
-              <option value="1280">1280</option>
-            </select>
-          </div>
-
-          <div>
-            <label className="mb-1 block">Height:</label>
-            <select
-              value={height}
-              onChange={(e) => setHeight(parseInt(e.target.value))}
-              className="w-full rounded border p-2"
-            >
-              <option value="512">512</option>
-              <option value="768">768</option>
-              <option value="1024">1024</option>
-              <option value="1280">1280</option>
-            </select>
-          </div>
-        </div>
-
-        <button
-          onClick={handleGenerate}
-          disabled={
-            !isConnected || !isModelInitialized || !sourceImage || isLoading
-          }
-          className="mt-4 rounded bg-blue-500 px-6 py-2 text-white disabled:bg-gray-400"
-        >
-          Generate Image
-        </button>
-      </div>
-
-      {/* Progress and Results */}
-      {isLoading && progress && (
-        <div className="mb-6 rounded-lg bg-gray-100 p-4">
-          <h2 className="mb-2 text-xl font-semibold">Progress</h2>
-          <div className="mb-2">
-            <div className="mb-1 flex justify-between">
-              <span>{progress.stage}</span>
-              <span>{progress.progress}%</span>
-            </div>
-            <div className="h-2.5 w-full rounded-full bg-gray-300">
-              <div
-                className="h-2.5 rounded-full bg-blue-600"
-                style={{ width: `${progress.progress}%` }}
-              ></div>
-            </div>
-            {progress.file && (
-              <div className="mt-1 text-sm text-gray-600">{progress.file}</div>
-            )}
-          </div>
-        </div>
-      )}
-
-      {errorMessage && (
-        <div className="mb-6 rounded border border-red-400 bg-red-100 px-4 py-3 text-red-700">
-          <p>{errorMessage}</p>
-        </div>
-      )}
-
-      {generatedImageUrl && (
-        <div className="mb-6 rounded-lg bg-gray-100 p-4">
-          <h2 className="mb-2 text-xl font-semibold">Generated Image</h2>
-          <div className="flex justify-center">
-            <img
-              src={generatedImageUrl}
-              alt="Generated"
-              className="max-w-full rounded border"
-            />
-          </div>
-        </div>
-      )}
-    </div>
-  );
-};

frontend/apps/editor2d/src/components/PromptBox.tsx

@@ -2,17 +2,12 @@ import { useState, useRef, useEffect } from "react";
 import { invoke } from "@tauri-apps/api/core";
 import { useSignals } from "@preact/signals-react/runtime";
 import { toast } from "sonner";
-import {
-  UploaderStates,
-  // UploaderState,
-  uploadImage,
-} from "~/components/UploadImage";
+import { UploaderStates, uploadImage } from "~/components/UploadImage";
 
 import { PopoverMenu } from "~/components/reusable/Popover";
-import { Tooltip } from "~/components/reusable/Tooltip";
-import { Button } from "~/components/reusable/Button";
+import { Tooltip } from "@storyteller/ui-tooltip";
+import { Button } from "@storyteller/ui-button";
 import { ToggleButton } from "~/components/reusable/ToggleButton";
-import { Toaster } from "~/components/ui/Toast";
 import { Modal } from "@storyteller/ui-modal";
 import {
   faMessageXmark,
@@ -25,7 +20,6 @@ import {
   faImages,
 } from "@fortawesome/pro-solid-svg-icons";
 import {
-  faRectangleWide,
   faRectangleVertical,
   faSquare,
   faRectangle,
@@ -34,7 +28,7 @@ import { getCanvasRenderBitmap } from "~/signal/canvasRenderBitmap";
 import { EncodeImageBitmapToBase64 } from "~/utilities/EncodeImageBitmapToBase64";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { PopoverItem } from "~/components/reusable/Popover";
-import { Api, ApiManager } from "~/KonvaApp/Api";
+import { Api } from "~/KonvaApp/Api";
 import { useJobContext } from "~/components/JobContext";
 import { IsDesktopApp } from "@storyteller/tauri-utils";
 import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";

frontend/apps/editor2d/src/components/features/ToolbarUserProfile/ProfileDropdown.tsx

@@ -44,8 +44,12 @@ export function ProfileDropdown() {
           size={34}
           username={userInfo.value.display_name}
           email_hash={userInfo.value.email_gravatar_hash}
-          avatarIndex={Number(userInfo.value.core_info.default_avatar.image_index) || 0}
-          backgroundIndex={Number(userInfo.value.core_info.default_avatar.color_index) || 0}
+          avatarIndex={
+            Number(userInfo.value.core_info.default_avatar.image_index) || 0
+          }
+          backgroundIndex={
+            Number(userInfo.value.core_info.default_avatar.color_index) || 0
+          }
         />
         <FontAwesomeIcon icon={faChevronDown} />
       </MenuButton>

frontend/apps/editor2d/src/signals/authentication/fetchers.ts

@@ -1,6 +1,6 @@
 import { signals } from "./auth_signals";
 import { UsersApi } from "@storyteller/api";
-import { BillingApi } from "@ApiManager/BillingApi";
+import { BillingApi } from "@storyteller/api";
 
 import {
   updateActiveSubscriptions,

frontend/apps/editor2d/src/signals/authentication/index.ts

@@ -1,7 +1,6 @@
 import { signals } from "./auth_signals";
-import * as fetchers from "./fetchers";
 import * as enums from "./enums";
-
+import * as fetchers from "./fetchers";
 export const authentication = {
   signals,
   fetchers,

frontend/apps/editor3d/app/src/Classes/ApiManager/ApiManager.ts

@@ -1,6 +1,8 @@
 import { GetBuildEnvironment } from "~/BuildEnvironment";
 import environmentVariables from "../EnvironmentVariables";
 import { Configs } from "~/configs";
+//import { fetch } from '@tauri-apps/plugin-http'
+import { FetchProxy as fetch } from "@storyteller/tauri-utils";
 
 type NonNullableObject<T extends object> = NonNullable<T>;
 

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts

@@ -549,7 +549,7 @@ class Editor {
     );
     this.cameraViewControls.movementSpeed = 1.15;
     this.cameraViewControls.domElement = this.renderer.domElement;
-    this.cameraViewControls.rollSpeed = Math.PI / 24;
+    this.cameraViewControls.rollSpeed = Math.PI / 180;
     this.cameraViewControls.autoForward = false;
     this.cameraViewControls.dragToLook = true;
     this.cameraViewControls.enabled = true;

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/free_cam.ts

@@ -9,6 +9,28 @@ import * as THREE from "three";
 import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
 import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
 
+
+export const isSafariUserAgent = () => {
+  const ua = navigator.userAgent;
+
+  const isSafari = /^((?!chrome|android|crios|fxios).)*safari/i.test(ua);
+
+  return isSafari;
+}
+
+export const hasApplePay = () => {
+  // @ts-expect-error Apple Pay is not defined in TypeScript
+  return !!window.ApplePaySession;
+}
+
+export const isSafari = () => {
+  return isSafariUserAgent() || hasApplePay();
+}
+
+export const isPointerLockSupported = () => {
+  return !isSafari();
+}
+
 class FreeCam extends EventDispatcher {
   object: THREE.PerspectiveCamera;
   domElement: HTMLElement | Document;
@@ -64,6 +86,7 @@ class FreeCam extends EventDispatcher {
     this.reset = this.reset.bind(this);
     this.update = this.update.bind(this);
     this.updateMovementVector = this.updateMovementVector.bind(this);
+    this.updateRotationVector = this.updateRotationVector.bind(this);
     this.getContainerDimensions = this.getContainerDimensions.bind(this);
     this.dispose = this.dispose.bind(this);
 
@@ -80,6 +103,7 @@ class FreeCam extends EventDispatcher {
     window.addEventListener("wheel", this.mousewheel.bind(this));
 
     this.updateMovementVector();
+    this.updateRotationVector();
   }
 
   keydown(event: KeyboardEvent) {
@@ -125,6 +149,7 @@ class FreeCam extends EventDispatcher {
     }
 
     this.updateMovementVector();
+    this.updateRotationVector();
   }
 
   keyup(event: KeyboardEvent) {
@@ -170,6 +195,7 @@ class FreeCam extends EventDispatcher {
     }
 
     this.updateMovementVector();
+    this.updateRotationVector();
   }
 
   mousedown(event: MouseEvent) {
@@ -268,6 +294,7 @@ class FreeCam extends EventDispatcher {
     this.resetVelocity();
 
     this.updateMovementVector();
+    this.updateRotationVector();
   }
 
   isStationary() {
@@ -277,13 +304,22 @@ class FreeCam extends EventDispatcher {
     );
   }
 
+  isRotating() {
+    return (
+      this.moveState.pitchUp ||
+      this.moveState.pitchDown ||
+      this.moveState.yawLeft ||
+      this.moveState.yawRight
+    );
+  }
+
   update(delta: number) {
     if (!this.enabled) {
       return;
     }
 
-    // Skip update if there's no movement and velocity
-    if (this.isStationary()) {
+    // Skip update if there's no movement, velocity, or rotation
+    if (this.isStationary() && !this.isRotating()) {
       return;
     }
 
@@ -307,6 +343,13 @@ class FreeCam extends EventDispatcher {
     // Store last position
     this.lastPosition.copy(this.object.position);
 
+
+    // Done with positions
+    // Now handle rotations
+    this.object.rotateX(this.rotationVector.x);
+    this.object.rotateY(this.rotationVector.y);
+    this.object.rotateZ(this.rotationVector.z);
+
     // Update camera signals
     if (selectedCameraId.value) {
       const pos = this.object.position;
@@ -320,11 +363,11 @@ class FreeCam extends EventDispatcher {
       cameras.value = cameras.value.map((cam) =>
         cam.id === selectedCameraId.value
           ? {
-              ...cam,
-              position: { x: pos.x, y: pos.y, z: pos.z },
-              rotation: { x: rot.x, y: rot.y, z: rot.z },
-              lookAt: { x: lookAtPoint.x, y: lookAtPoint.y, z: lookAtPoint.z },
-            }
+            ...cam,
+            position: { x: pos.x, y: pos.y, z: pos.z },
+            rotation: { x: rot.x, y: rot.y, z: rot.z },
+            lookAt: { x: lookAtPoint.x, y: lookAtPoint.y, z: lookAtPoint.z },
+          }
           : cam,
       );
     }
@@ -341,6 +384,13 @@ class FreeCam extends EventDispatcher {
     this.moveVector.z = -forward + this.moveState.back;
   }
 
+  updateRotationVector() {
+    this.rotationVector.x = -this.moveState.pitchDown + this.moveState.pitchUp;
+    this.rotationVector.y = -this.moveState.yawRight + this.moveState.yawLeft;
+    this.rotationVector.z = -this.moveState.rollRight + this.moveState.rollLeft;
+    this.rotationVector.multiplyScalar(this.rollSpeed);
+  }
+
   getContainerDimensions() {
     if (this.domElement !== document) {
       return {

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/keybinds_controls.ts

@@ -14,7 +14,7 @@ import {
 } from "three/examples/jsm/Addons.js";
 import { TransformControls } from "./TransformControls";
 import { SceneManager, SceneObject } from "./scene_manager_api";
-import { FreeCam } from "./free_cam";
+import { FreeCam, isPointerLockSupported } from "./free_cam";
 import Stats from "three/examples/jsm/libs/stats.module.js";
 import { FKHelper } from "./KinHelpers/FKHelper";
 import {
@@ -391,6 +391,10 @@ export class MouseControls {
     this.timeline_mouse = this.mouse;
 
     // this causes an issue  https://discourse.threejs.org/t/unable-to-use-pointer-lock-api/11092
+    if (!isPointerLockSupported()) {
+      return;
+    }
+
     if (this.isMouseClicked && this.lockControls) {
       if (this.lockControls.isLocked == false) {
         this.lockControls.lock();

frontend/apps/editor3d/package.json

@@ -45,6 +45,7 @@
     "@remix-run/react": "^2.8.1",
     "@remix-run/router": "^1.15.3",
     "@remix-run/serve": "^2.8.1",
+    "@tauri-apps/plugin-http": "^2.4.3",
     "@tweenjs/tween.js": "^18.6.4",
     "@types/lodash": "^4.17.6",
     "@types/react-virtualized": "^9.22.2",

frontend/libs/api/src/lib/ApiManager.ts

@@ -1,5 +1,7 @@
 // import { bool } from "@techstark/opencv-js";
 import { API_TARGETS } from "./enums/Api.js";
+//import { fetch } from '@tauri-apps/plugin-http'
+import { FetchProxy as fetch } from "@storyteller/tauri-utils";
 
 type NonNullableObject<T extends object> = NonNullable<T>;
 

frontend/libs/tauri-utils/src/lib/FetchProxy.ts

@@ -0,0 +1,27 @@
+import { IsDesktopApp } from "./IsDesktopApp";
+import { fetch as tauriFetch } from '@tauri-apps/plugin-http'
+
+// TODO: Match the function signature in typescript annotations:
+// export declare function fetch(input: URL | Request | string, init?: RequestInit & ClientOptions): Promise<Response>;
+
+/// Third party cookies are blocked on Mac, which means all of our API calls
+/// to our backend fail. Fortunately Tauri exposes a drop-in mixin for the 
+/// Javascript fetch API that can proxy all requests to Rust, and then onto
+/// the downstream service. It maintains its own cookie jar and avoids the 
+/// 3rd party cookie issue.
+///
+/// This function basically determines if we're in Tauri or on the Web. If the
+/// former, we replace Fetch with the Tauri fetch. If the latter, we dispatch
+/// to the normal browser fetch.
+
+export function FetchProxy(args: any) : Promise<Response> {
+  if (IsDesktopApp()) {
+    // Tauri proxy fetch
+    return tauriFetch(args);
+  } else {
+    // Browser native fetch
+    return fetch(args);
+  }
+}
+
+(window as any).FetchProxy = FetchProxy;

frontend/libs/tauri-utils/src/lib/tauri-utils.tsx

@@ -1 +1,2 @@
 export { IsDesktopApp } from './IsDesktopApp';
+export { FetchProxy } from './FetchProxy';

frontend/package-lock.json

@@ -1272,6 +1272,7 @@
         "@remix-run/react": "^2.8.1",
         "@remix-run/router": "^1.15.3",
         "@remix-run/serve": "^2.8.1",
+        "@tauri-apps/plugin-http": "^2.4.3",
         "@tweenjs/tween.js": "^18.6.4",
         "@types/lodash": "^4.17.6",
         "@types/react-virtualized": "^9.22.2",
@@ -11135,6 +11136,15 @@
         "node": ">= 10"
       }
     },
+    "node_modules/@tauri-apps/plugin-http": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-http/-/plugin-http-2.4.3.tgz",
+      "integrity": "sha512-Us8X+FikzpaZRNr4kH4HLwyXascHbM42p6LxAqRTQnHPrrqp1usaH4vxWAZalPvTbHJ3gBEMJPHusFJgtjGJjA==",
+      "license": "MIT OR Apache-2.0",
+      "dependencies": {
+        "@tauri-apps/api": "^2.0.0"
+      }
+    },
     "node_modules/@techstark/opencv-js": {
       "version": "4.10.0-release.1",
       "resolved": "https://registry.npmjs.org/@techstark/opencv-js/-/opencv-js-4.10.0-release.1.tgz",