Compare commits

...
27 changed files with 292 additions and 73 deletions
@@ -17,7 +17,15 @@ pub struct AppPreferencesPayload {
/// Play sounds on events.
pub play_sounds: bool,
/// Key pointing to file; defined in the frontend code.
/// Defined for enqueue since image enqueue can be async
pub enqueue_success_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
/// Defined for enqueue since image enqueue can be async
pub enqueue_failure_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
pub generation_success_sound: Option<String>,
@@ -25,6 +33,7 @@ pub struct AppPreferencesPayload {
pub generation_failure_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
#[deprecated]
pub generation_enqueue_sound: Option<String>,
}
@@ -52,6 +61,8 @@ async fn get_prefs(app_prefs: &AppPreferencesManager) -> AnyhowResult<AppPrefere
Ok(AppPreferencesPayload {
preferred_download_directory: prefs.preferred_download_directory,
play_sounds: prefs.play_sounds,
enqueue_success_sound: prefs.enqueue_success_sound,
enqueue_failure_sound: prefs.enqueue_failure_sound,
generation_success_sound: prefs.generation_success_sound,
generation_failure_sound: prefs.generation_failure_sound,
generation_enqueue_sound: prefs.generation_enqueue_sound,
@@ -26,6 +26,9 @@ pub struct CommandSuccessResponseWrapper<T: Serialize> {
/// Both "success" and "error" types have a `status` field for the frontend.
pub status: CommandSuccessStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub success_message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub payload: Option<T>,
}
@@ -36,7 +39,10 @@ pub struct CommandErrorResponseWrapper<T: Serialize> {
pub status: CommandErrorStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<T>,
pub error_message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_details: Option<T>,
}
@@ -44,6 +50,7 @@ impl CommandSuccessResponseWrapper<()> {
pub fn empty() -> Self {
CommandSuccessResponseWrapper {
status: CommandSuccessStatus::Success,
success_message: None,
payload: None,
}
}
@@ -61,6 +68,7 @@ impl From<()> for CommandSuccessResponseWrapper<()> {
fn from(_: ()) -> Self {
CommandSuccessResponseWrapper {
status: CommandSuccessStatus::Success,
success_message: None,
payload: None,
}
}
@@ -72,16 +80,28 @@ impl <T> From<T> for CommandSuccessResponseWrapper<T> where T: Serialize + Seria
fn from(val: T) -> Self {
CommandSuccessResponseWrapper {
status: CommandSuccessStatus::Success,
success_message: None,
payload: Some(val),
}
}
}
impl From<&str> for CommandSuccessResponseWrapper<String> {
fn from(msg: &str) -> Self {
CommandSuccessResponseWrapper {
status: CommandSuccessStatus::Success,
success_message: Some(msg.to_string()),
payload: None,
}
}
}
impl From<&str> for CommandErrorResponseWrapper<String> {
fn from(value: &str) -> Self {
CommandErrorResponseWrapper {
status: CommandErrorStatus::BadRequest, // NB: Default to 500 error.
error: Some(value.to_string()),
error_message: Some(value.to_string()),
error_details: None,
}
}
}
@@ -1,3 +1,4 @@
use crate::commands::command_response_wrapper::CommandResult;
use crate::events::sendable_event_trait::SendableEvent;
use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
@@ -56,7 +57,7 @@ pub async fn sora_image_remix_command(
app_data_root: State<'_, AppDataRoot>,
sora_creds_manager: State<'_, SoraCredentialManager>,
sora_task_queue: State<'_, SoraTaskQueue>,
) -> Result<String, String> {
) -> CommandResult<(), String> {
info!("image_generation_command called; scene media token: {:?}, additional images: {:?}",
request.snapshot_media_token, request.maybe_additional_images);
@@ -75,7 +76,7 @@ pub async fn sora_image_remix_command(
error!("Failed to emit event: {:?}", err);
}
return Err("there was an error".to_string());
return Err("there was an error".into());
}
let event = SoraImageEnqueueSuccessEvent {};
@@ -85,7 +86,7 @@ pub async fn sora_image_remix_command(
error!("Failed to emit event: {:?}", err);
}
Ok("success".to_string())
Ok(().into())
}
pub async fn generate_image(
@@ -9,6 +9,14 @@ pub struct AppPreferences {
/// Play sounds on events.
pub play_sounds: bool,
/// Key pointing to file; defined in the frontend code.
/// Defined for enqueue since image enqueue can be async
pub enqueue_success_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
/// Defined for enqueue since image enqueue can be async
pub enqueue_failure_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
pub generation_success_sound: Option<String>,
@@ -17,6 +25,7 @@ pub struct AppPreferences {
pub generation_failure_sound: Option<String>,
/// Key pointing to file; defined in the frontend code.
#[deprecated]
pub generation_enqueue_sound: Option<String>,
}
@@ -26,7 +35,9 @@ impl Default for AppPreferences {
preferred_download_directory: PreferredDownloadDirectory::System(SystemDownloadDirectory::Downloads),
play_sounds: true,
// NB: These are defined in the frontend.
generation_success_sound: Some("flower".to_string()),
enqueue_success_sound: Some("done".to_string()),
enqueue_failure_sound: Some("spike_throw".to_string()),
generation_success_sound: Some("special_flower".to_string()),
generation_failure_sound: Some("crumble".to_string()),
generation_enqueue_sound: Some("done".to_string()),
}
@@ -3,13 +3,39 @@ import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
// TODO: This is maintained in two places. Here and MiscSettingsPane.
export const InstallSounds = () => {
const r = SoundRegistry.getInstance();
if (r.hasSound("crumble")) {
if (r.hasSound("decline_normal")) {
return;
}
r.setSoundOnce("flower", new SoundEffect("resources/sound/smrpg_flower.wav", { defaultVolume: 0.1 }));
r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.3 }));
// Good for simple immediate enqueue
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for simple immediate failure
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT
// Good for simple async success
r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for async errors
r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT
r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 }));
// Good for menus
r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 }));
r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 }));
// Good for special reward / celebration
r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 }));
r.setSoundOnce("flower", new SoundEffect("resources/sound/smrpg_flower.wav", { defaultVolume: 0.1 }));
// Misc / ungrouped
r.setSoundOnce("accept_chirp", new SoundEffect("resources/sound/goldensun_101.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("accept_normal_level_1", new SoundEffect("resources/sound/goldensun_173.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("accept_normal_level_2", new SoundEffect("resources/sound/goldensun_174.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("accept_normal_level_3", new SoundEffect("resources/sound/goldensun_175.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("decline_chirp", new SoundEffect("resources/sound/goldensun_102.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("decline_normal", new SoundEffect("resources/sound/goldensun_113.wav", { defaultVolume: 0.2 }));
(window as any).sounds = r;
}
@@ -9,13 +9,9 @@ type SoraImageGenerationFailed = {
export const InstallImageGenerationFailure = () => {
listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
console.log("sora-image-generation-failed event", event);
const prefs = await GetAppPreferences();
console.log("prefs", prefs);
const soundName = prefs.preferences?.generation_failure_sound;
console.log("soundName", soundName);
if (soundName !== undefined) {
console.log("playing sound", soundName);
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
@@ -9,13 +9,9 @@ type ImageGenerationSuccess = {
export const InstallImageGenerationSuccess = () => {
listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
console.log("sora-image-generation-complete event", event);
const prefs = await GetAppPreferences();
console.log("prefs", prefs);
const soundName = prefs.preferences?.generation_success_sound;
console.log("soundName", soundName);
if (soundName !== undefined) {
console.log("playing sound", soundName);
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
@@ -46,6 +46,8 @@ import Queue from "~/pages/PageEnigma/Queue/Queue";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
import { PromptsApi } from "@storyteller/api";
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api";
import { CommandSuccessStatus } from "@storyteller/tauri-api";
import {
cameras,
@@ -522,19 +524,54 @@ export const PromptBox = () => {
//sceneMediaToken: "",
});
const generateResponse = await invoke("sora_image_remix_command", {
request: {
snapshot_media_token: snapshotResult.data!,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken,
),
maybe_number_of_samples: 1,
},
console.log("snapshotResult", snapshotResult)
const generateResponse = await SoraImageRemix({
snapshot_media_token: snapshotResult.data!,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken,
),
maybe_number_of_samples: 1,
});
console.log("response", generateResponse);
console.log("generateResponse", generateResponse)
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image added to queue");
setisEnqueueing(false);
return;
} else {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Failed to enqueue image");
setisEnqueueing(false);
return;
}
// const generateResponse = await invoke("sora_image_remix_command", {
// request: {
// snapshot_media_token: snapshotResult.data!,
// disable_system_prompt: !useSystemPrompt,
// prompt: prompt,
// maybe_additional_images: referenceImages.map(
// (image) => image.mediaToken,
// ),
// maybe_number_of_samples: 1,
// },
// });
//if (generateResponse.errorMessage) {
// handleError(response.errorMessage);
@@ -2,6 +2,9 @@
"files": [],
"include": [],
"references": [
{
"path": "../../api"
},
{
"path": "../../tauri-api"
},
@@ -11,6 +14,9 @@
{
"path": "../button"
},
{
"path": "../input"
},
{
"path": "./tsconfig.lib.json"
},
@@ -43,6 +43,9 @@
"src/**/*.tsx"
],
"references": [
{
"path": "../../api/tsconfig.lib.json"
},
{
"path": "../../tauri-api/tsconfig.lib.json"
},
@@ -51,6 +54,9 @@
},
{
"path": "../button/tsconfig.lib.json"
},
{
"path": "../input/tsconfig.lib.json"
}
]
}
@@ -13,12 +13,19 @@ import { Label } from "@storyteller/ui-label";
// TODO: This is maintained in two places. Here and InstallSounds.
const SOUND_OPTIONS = [
{ value: "none", label: "None (Silent)" },
{ value: "flower", label: "Flower" },
{ value: "none", label: "None (Silent)" }, // NB: MUST BE FIRST
// Other options
{ value: "correct", label: "Correct" },
{ value: "next", label: "Next" },
{ value: "done", label: "Done" },
{ value: "crumble", label: "Crumble" },
{ value: "done", label: "Done" },
{ value: "error_chirp", label: "Error Chirp" },
{ value: "flower", label: "Flower" },
{ value: "next", label: "Next" },
{ value: "select", label: "Select" },
{ value: "special_alert", label: "Special Alert" },
{ value: "special_flower", label: "Special Flower" },
{ value: "spike_throw", label: "Spike Throw" },
];
interface AudioSettingsPaneProps {}
@@ -38,9 +45,10 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
const playSounds = preferences?.play_sounds || false;
const successSound = orNone(preferences?.generation_success_sound);
const failureSound = orNone(preferences?.generation_failure_sound);
const enqueueSound = orNone(preferences?.generation_enqueue_sound);
const enqueueSuccessSound = orNone(preferences?.enqueue_success_sound);
const enqueueFailureSound = orNone(preferences?.enqueue_failure_sound);
const generationSuccessSound = orNone(preferences?.generation_success_sound);
const generationFailureSound = orNone(preferences?.generation_failure_sound);
const reloadPreferences = async () => {
const prefs = await GetAppPreferences();
@@ -56,6 +64,26 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
await reloadPreferences();
};
const setEnqueueSuccessSound = async (val: string) => {
let sendVal = val === "none" ? undefined : val;
await UpdateAppPreferences({
preference: PreferenceName.EnqueueSuccessSound,
value: sendVal,
});
SoundRegistry.getInstance().playSound(val);
await reloadPreferences();
};
const setEnqueueFailureSound = async (val: string) => {
let sendVal = val === "none" ? undefined : val;
await UpdateAppPreferences({
preference: PreferenceName.EnqueueFailureSound,
value: sendVal,
});
SoundRegistry.getInstance().playSound(val);
await reloadPreferences();
};
const setSuccessSound = async (val: string) => {
let sendVal = val === "none" ? undefined : val;
await UpdateAppPreferences({
@@ -76,16 +104,6 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
await reloadPreferences();
};
const setEnqueueSound = async (val: string) => {
let sendVal = val === "none" ? undefined : val;
await UpdateAppPreferences({
preference: PreferenceName.GenerationEnqueueSound,
value: sendVal,
});
SoundRegistry.getInstance().playSound(val);
await reloadPreferences();
};
const playSound = (val?: string) => {
if (val !== undefined && val !== "none") {
SoundRegistry.getInstance().playSound(val);
@@ -103,11 +121,49 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
</div>
<div className="space-y-1">
<Label htmlFor="success-sound">Success Sound</Label>
<Label htmlFor="success-sound">Enqueue Success Sound</Label>
<div className="flex items-center gap-2">
<Select
id="success-sound"
value={successSound}
value={enqueueSuccessSound}
onChange={(val: SelectValue) => setEnqueueSuccessSound(val as string)}
options={SOUND_OPTIONS}
className="grow"
/>
<Button
variant="primary"
className="w-[40px] h-[40px]"
icon={faPlay}
onClick={() => playSound(enqueueSuccessSound)}
/>
</div>
</div>
<div className="space-y-1">
<Label htmlFor="success-sound">Enqueue Failure Sound</Label>
<div className="flex items-center gap-2">
<Select
id="success-sound"
value={enqueueFailureSound}
onChange={(val: SelectValue) => setEnqueueFailureSound(val as string)}
options={SOUND_OPTIONS}
className="grow"
/>
<Button
variant="primary"
className="w-[40px] h-[40px]"
icon={faPlay}
onClick={() => playSound(enqueueFailureSound)}
/>
</div>
</div>
<div className="space-y-1">
<Label htmlFor="success-sound">Generation Success Sound</Label>
<div className="flex items-center gap-2">
<Select
id="success-sound"
value={generationSuccessSound}
onChange={(val: SelectValue) => setSuccessSound(val as string)}
options={SOUND_OPTIONS}
className="grow"
@@ -116,17 +172,17 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
variant="primary"
className="w-[40px] h-[40px]"
icon={faPlay}
onClick={() => playSound(successSound)}
onClick={() => playSound(generationSuccessSound)}
/>
</div>
</div>
<div className="space-y-1">
<Label htmlFor="failure-sound">Failure Sound</Label>
<Label htmlFor="failure-sound">Generation Failure Sound</Label>
<div className="flex items-center gap-2">
<Select
id="failure-sound"
value={failureSound}
value={generationFailureSound}
onChange={(val: SelectValue) => setFailureSound(val as string)}
options={SOUND_OPTIONS}
className="grow"
@@ -135,29 +191,11 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
variant="primary"
className="w-[40px] h-[40px]"
icon={faPlay}
onClick={() => playSound(failureSound)}
onClick={() => playSound(generationFailureSound)}
/>
</div>
</div>
<div className="space-y-1">
<Label htmlFor="enqueue-sound">Enqueue Sound</Label>
<div className="flex items-center gap-2">
<Select
id="enqueue-sound"
value={enqueueSound}
onChange={(val: SelectValue) => setEnqueueSound(val as string)}
options={SOUND_OPTIONS}
className="grow"
/>
<Button
variant="primary"
className="w-[40px] h-[40px]"
icon={faPlay}
onClick={() => playSound(enqueueSound)}
/>
</div>
</div>
</div>
</>
);
+3 -1
View File
@@ -1,5 +1,7 @@
export * from "./lib/common/CommandStatus.js";
export * from "./lib/settings/GetAppPreferences.js";
export * from "./lib/settings/UpdateAppPreference.js";
export * from "./lib/sora/CheckSoraSession.js";
export * from "./lib/sora/useSoraLoginListener.js";
export * from "./lib/sora/LogoutSoraSession.js";
export * from "./lib/sora/SoraImageRemix.js";
export * from "./lib/sora/useSoraLoginListener.js";
@@ -0,0 +1,17 @@
export enum CommandSuccessStatus {
Success = "success",
}
export enum CommandErrorStatus {
BadRequest = "bad_request",
Unauthorized = "unauthorized",
ServerError = "server_error",
}
export type CommandStatus = CommandSuccessStatus | CommandErrorStatus;
export interface CommandResult {
status: CommandStatus;
}
@@ -13,6 +13,8 @@ export interface AppPreferencesPayload {
// Sound names to play.
// These are keys, not filenames, and are defined by the frontend.
enqueue_success_sound?: string,
enqueue_failure_sound?: string,
generation_success_sound?: string,
generation_failure_sound?: string,
generation_enqueue_sound?: string,
@@ -10,6 +10,8 @@ export interface UpdateAppPreferencesRequest {
export enum PreferenceName {
PreferredDownloadDirectory = "preferred_download_directory",
PlaySounds = "play_sounds",
EnqueueSuccessSound = "enqueue_success_sound",
EnqueueFailureSound = "enqueue_failure_sound",
GenerationSuccessSound = "generation_success_sound",
GenerationFailureSound = "generation_failure_sound",
GenerationEnqueueSound = "generation_enqueue_sound",
@@ -0,0 +1,48 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
export interface SoraImageRemixRequest {
// The media token of the visible screen snapshot
snapshot_media_token: string,
// Whether to disable (our) system prompt.
disable_system_prompt: boolean,
// The text prompt
prompt: string,
// Additional images (media tokens) to use as reference.
maybe_additional_images: string[],
// Typically "1"
maybe_number_of_samples: number,
}
export interface SoraImageRemixResult extends CommandResult {
// TODO: Status fields (they're not on the backend yet)
}
// Returns the Success and Error variants directly.
// Throws on Network/Tauri errors.
export const SoraImageRemix = async (request: SoraImageRemixRequest) : Promise<SoraImageRemixResult> => {
try {
return await invoke("sora_image_remix_command", {
request: {
snapshot_media_token: request.snapshot_media_token,
disable_system_prompt: request.disable_system_prompt,
prompt: request.prompt,
maybe_additional_images: request.maybe_additional_images,
maybe_number_of_samples: request.maybe_number_of_samples,
},
}) as SoraImageRemixResult;
} catch (error) {
let maybeTypedError = error as CommandResult;
if ("status" in maybeTypedError) {
return maybeTypedError
} else {
// Something else with Tauri, network, etc.
throw error;
}
}
}