Compare commits

...
19 changed files with 14 additions and 2363 deletions
@@ -1,33 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { Subscription } from "./models/Billing";
import { LoyaltyProgram } from "./enums/Billing";
export class BillingApi extends ApiManager {
public async ListActiveSubscriptions(): Promise<
ApiResponse<{
active_subscriptions: Subscription[];
maybe_loyalty_program?: LoyaltyProgram;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/billing/active_subscriptions`;
return await this.get<{
success: boolean;
active_subscriptions?: Subscription[];
maybe_loyalty_program?: LoyaltyProgram;
error_message?: string;
}>({ endpoint: endpoint })
.then((response) => ({
success: response.success,
data: {
active_subscriptions: response.active_subscriptions || [],
maybe_loyalty_program: response.maybe_loyalty_program,
},
}))
.catch((err) => {
return {
success: false,
errorMessage: err.message,
};
});
}
}
@@ -1,91 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
export class CommentsApi extends ApiManager {
public CreateComment({
commentMarkdown,
entityToken,
entityType,
uuidIdempotencyToken,
}: {
commentMarkdown: string;
entityToken: string;
entityType: string;
uuidIdempotencyToken: string;
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/new`;
const body = {
comment_markdown: commentMarkdown,
entity_token: entityToken,
entity_type: entityType,
uuid_idempotency_token: uuidIdempotencyToken,
};
return this.post<
{
comment_markdown: string;
entity_token: string;
entity_type: string;
uuid_idempotency_token: string;
},
{
success?: boolean;
comment_token?: string;
BadInput?: string;
}
>({ endpoint, body })
.then((response) => ({
success: response.success ?? false,
data: response.comment_token,
errorMessage: response.BadInput,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public DeleteComment({
commentToken,
}: {
commentToken: string;
}): Promise<ApiResponse<undefined>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/delete/${commentToken}`;
return this.post<
{ as_mod: boolean },
{
success?: boolean;
BadInput?: string;
}
>({ endpoint, body: { as_mod: true } })
.then((response) => ({
success: response.success ?? false,
errorMessage: response.BadInput,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public ListCommentsByEntity({
entityType,
entityToken,
}: {
entityType: string;
entityToken: string;
}): Promise<ApiResponse<string[]>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/comments/list/${entityType}/${entityToken}`;
return this.get<{
success: boolean;
comments: string[];
}>({ endpoint })
.then((response) => ({
success: true,
data: response.comments,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
}
@@ -1,77 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { Job, JobPreview } from "./models/Job";
export class JobsApi extends ApiManager {
public GetPreviewStatusByJobToken({
token,
}: {
token: string;
}): Promise<ApiResponse<JobPreview>> {
console.log(token);
const endpoint = `${this.ApiTargets.BaseApi}/v1/workflows/preview_status/${token}`;
return this.get<ApiResponse<JobPreview>>({ endpoint })
.then((response) => ({
success: response.success,
data: response.data,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public GetJobByToken({
token,
}: {
token: string;
}): Promise<ApiResponse<Job>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/job/${token}`;
return this.get<{
success: boolean;
state: Job;
}>({ endpoint })
.then((response) => ({
success: response.success,
data: response.state,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public ListRecentJobs(): Promise<ApiResponse<Job[]>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session`;
return this.get<{
success: boolean;
jobs: Job[];
error_reason?: string;
}>({ endpoint })
.then((response) => ({
success: response.success,
data: response.jobs,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public DeleteJobByToken(jobToken: string): Promise<ApiResponse<undefined>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/job/${jobToken}`;
return this.delete<
undefined,
{
success: boolean;
}
>({ endpoint })
.then((response) => ({
success: response.success,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
}
@@ -1,26 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { StatusAlert } from "./models/StatusAlert";
export class MiscApi extends ApiManager {
public GetStatusAlertCheck(): Promise<ApiResponse<StatusAlert>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/session`;
return this.get<{
success: boolean;
maybe_alert?: Omit<StatusAlert, "refresh_interval_millis">;
refresh_interval_millis: number;
}>({ endpoint: endpoint })
.then((response) => ({
success: response.success,
data: {
...response.maybe_alert,
refresh_interval_millis: response.refresh_interval_millis,
},
}))
.catch((err) => {
return {
success: false,
errorMessage: err.mesasge,
};
});
}
}
@@ -1,42 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
export class ModerationApi extends ApiManager {
public async UpdateFeatureFlagsByToken({
usernameOrToken,
flags,
}: {
usernameOrToken: string;
flags: string[];
}): Promise<
ApiResponse<{
inference_job_token?: string;
inference_job_token_type?: string;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/moderation/user_feature_flags/${usernameOrToken}`;
const body = {
action: {
AddFlags: {
flags,
},
},
};
return this.post<
{ action: { AddFlags: { flags: string[] } } },
{ success?: boolean; BadInput?: string }
>({
endpoint,
body: body,
})
.then((response) => ({
success: response.success ?? false,
errorMessage: response.BadInput,
}))
.catch((err) => ({
success: false,
errorMessage: err.message,
}));
}
}
@@ -1,25 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { Prompts } from "./models/Prompts";
export class PromptsApi extends ApiManager {
public GetPromptsByToken({
token,
}: {
token: string;
}): Promise<ApiResponse<Prompts>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/prompts/${token}`;
return this.get<{
success: boolean;
prompt: Prompts;
error_reason?: string;
}>({ endpoint })
.then((response) => ({
success: response.success,
data: response.prompt,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
}
@@ -1,64 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { GenerateTtsAudioErrorType } from "./enums/Tts";
import { Visibility } from "./enums/Visibility";
export interface GenerateTtsAudioRequest {
creator_set_visibility?: Visibility;
uuid_idempotency_token: string;
tts_model_token: string;
inference_text: string;
is_storyteller_demo?: boolean;
}
export interface GenerateTtsAudioResponse {
success?: boolean;
inference_job_token?: string;
inference_job_token_type?: string;
BadInput?: string;
}
export class TtsApi extends ApiManager {
public async GenerateTtsAudio(request: GenerateTtsAudioRequest): Promise<
ApiResponse<{
inference_job_token?: string;
inference_job_token_type?: string;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/tts/inference`;
const body = {
...request,
...(request.creator_set_visibility
? {}
: { creator_set_visibility: Visibility.Public }),
};
return this.post<GenerateTtsAudioRequest, GenerateTtsAudioResponse>({
endpoint,
body: body,
})
.then((response) => {
if (!response.success) {
return {
success: false,
errorMessage: response.BadInput,
data: {},
};
}
return {
success: true,
data: {
inference_job_token: response.inference_job_token,
inference_job_token_type: response.inference_job_token_type,
},
};
})
.catch(() => {
return {
success: false,
errorMessage: GenerateTtsAudioErrorType.UnknownError,
data: {},
};
});
}
}
@@ -1,201 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import {
UserBookmarkBatch,
UserBookmarkByEntity,
UserBookmarkByUser,
} from "./models/UserBookmark";
import { Pagination } from "./models/Pagination";
export enum ScopedEntityTypes {
USER = "user",
TTS_MODEL = "tts-model",
TTS_RESULT = "tts-result",
W2L_TEMPLATE = "w2l-template",
W2L_RESULT = "w2l-result",
MEDIA_FILE = "media_file",
MODEL_WEIGHT = "model_weight",
VOICE_CONVERSION_MODEL = "voice_conversion_model",
ZS_VOICE = "z_voice",
}
export enum ScopedWeightTypes {
HIFIGAN_TTl2 = "hifigan_tt2",
RVC_V2 = "rvc_v2",
SD_1_5 = "sd_1.5",
SDXL = "sdxl",
SO_VITS_SVC = "so_vits_svc",
TT2 = "tt2",
LORA = "loRA",
VALL_E = "vall_e",
COMFY_UI = "comfy_ui",
}
export enum ScopedMediaFileType {
IMAGE_GENERATION = "image_generation",
TEXT_TO_SPEECH = "text_to_speech",
VOCODER = "vocoder",
VOICE_CONVERSION = "voice_conversion",
WORKFLOW_CONFIG = "workflow_config",
}
interface ListUserBookmarksByUserRequest {
username: string;
sort_ascending?: boolean;
page_size?: number;
page_index?: number;
maybe_scoped_entity_type?: ScopedEntityTypes[];
maybe_scoped_weight_type?: ScopedWeightTypes[];
maybe_scoped_media_file_type?: ScopedMediaFileType[];
}
export class UserBookmarksApi extends ApiManager {
public CreateUserBookmark({
entityToken,
entityType,
}: {
entityToken: string;
entityType: string;
}): Promise<
ApiResponse<{
new_bookmark_count_for_entity?: number;
user_bookmark_token?: string;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/create`;
const body = {
entity_token: entityToken,
entity_type: entityType,
};
return this.post<
{
entity_token: string;
entity_type: string;
},
{
success?: boolean;
new_bookmark_count_for_entity?: number;
user_bookmark_token: string;
BadInput?: string;
}
>({ endpoint, body })
.then((response) => ({
success: response.success ?? false,
data: {
new_bookmark_count_for_entity: response.new_bookmark_count_for_entity,
user_bookmark_token: response.user_bookmark_token,
},
errorMessage: response.BadInput,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public DeleteUserBookmark({
entityToken,
}: {
entityToken: string;
}): Promise<ApiResponse<undefined>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/delete/${entityToken}`;
return this.delete<
{ as_mod: boolean },
{
success?: boolean;
BadInput?: string;
}
>({ endpoint, body: { as_mod: true } })
.then((response) => ({
success: response.success ?? false,
errorMessage: response.BadInput,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public ListUserBookmarks(): Promise<ApiResponse<UserBookmarkBatch[]>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/batch`;
return this.get<{
success?: boolean;
bookmarks?: UserBookmarkBatch[];
BadInput?: string;
}>({ endpoint })
.then((response) => ({
success: response.success ?? false,
data: response.bookmarks,
errorMessage: response.BadInput,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public ListUserBookmarksByUser({
username,
sort_ascending,
page_size,
page_index,
maybe_scoped_entity_type,
maybe_scoped_weight_type,
maybe_scoped_media_file_type,
}: ListUserBookmarksByUserRequest): Promise<
ApiResponse<UserBookmarkByUser[], Pagination>
> {
const user = username;
const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/list/${user}`;
const query = {
sort_ascending,
page_size,
page_index,
maybe_scoped_entity_type: maybe_scoped_entity_type
? maybe_scoped_entity_type.join(",")
: undefined,
maybe_scoped_weight_type: maybe_scoped_weight_type
? maybe_scoped_weight_type.join(",")
: undefined,
maybe_scoped_media_file_type: maybe_scoped_media_file_type
? maybe_scoped_media_file_type.join(",")
: undefined,
};
return this.get<{
success?: boolean;
pagination?: Pagination;
results: UserBookmarkByUser[];
}>({ endpoint, query })
.then((response) => ({
success: response.success ?? false,
data: response.results,
pagination: response.pagination,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
public ListUserBookmarksByEntity({
entityType,
entityToken,
}: {
entityType: string;
entityToken: string;
}): Promise<ApiResponse<UserBookmarkByEntity[]>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/user_bookmarks/list/${entityType}/${entityToken}`;
return this.get<{
success: boolean;
user_bookmarks: UserBookmarkByEntity[];
}>({ endpoint })
.then((response) => ({
success: true,
data: response.user_bookmarks,
}))
.catch((err) => {
return { success: false, errorMessage: err.message };
});
}
}
@@ -1,162 +0,0 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { UserInfo } from "./models/Users";
export class UsersApi extends ApiManager {
public GetSession(): Promise<
ApiResponse<{
loggedIn: boolean;
user?: UserInfo;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/session`;
return this.get<{
success: boolean;
logged_in: boolean;
user?: UserInfo;
error_message?: string;
}>({ endpoint: endpoint })
.then((response) => ({
success: response.success,
data: {
loggedIn: response.logged_in,
user: response.user,
},
}))
.catch((err) => {
return {
success: false,
errorMessage: err.mesasge,
};
});
}
public GetUserProfile(username: string): Promise<
ApiResponse<{
user?: UserInfo;
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/user/${username}/profile`;
return this.get<{
success: boolean;
user?: UserInfo;
error_message?: string;
}>({ endpoint: endpoint })
.then((response) => ({
success: response.success,
data: {
user: response.user,
},
}))
.catch((err) => {
return {
success: false,
error_message: err.message,
};
});
}
public async Login({
usernameOrEmail,
password,
}: {
usernameOrEmail: string;
password: string;
}): Promise<ApiResponse<{ signedSession?: string }>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/login`;
const body = {
username_or_email: usernameOrEmail,
password: password,
};
return await this.post<
{ username_or_email: string; password: string },
{
success: boolean;
signed_session?: string;
error_message?: string;
error_type?: string;
}
>({
endpoint: endpoint,
body: body,
})
.then((response) => ({
success: response.success,
data: { signedSession: response.signed_session },
errorMessage: response.error_message,
}))
.catch((err) => {
return {
success: false,
error_message: err.message,
};
});
}
public Logout(): Promise<ApiResponse<null>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/logout`;
return this.post<null, { success: boolean; error_message?: string }>({
endpoint: endpoint,
})
.then((response) => ({
success: response.success,
}))
.catch((err) => {
return {
success: false,
errorMessage: err.message,
};
});
}
public async Signup({
username,
email,
password,
passwordConfirmation,
}: {
username: string;
email: string;
password: string;
passwordConfirmation: string;
}): Promise<ApiResponse<{ signedSession?: string }>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/create_account`;
const body = {
email_address: email,
password,
password_confirmation: passwordConfirmation,
username,
};
return await this.post<
{
username: string;
email_address: string;
password: string;
password_confirmation: string;
},
{
success: boolean;
signed_session?: string;
error_fields?: Record<string, string>;
error_message?: string;
error_type?: string;
}
>({
endpoint: endpoint,
body: body,
})
.then((response) => {
return {
success: response.success,
data: { signedSession: response.signed_session },
errorMessage: Object.values(response.error_fields ?? {}).join(","),
};
})
.catch((err) => {
return {
success: false,
error_message: err.message,
};
});
}
}
@@ -1,18 +1,9 @@
// https://storyteller-docs.netlify.app/#/
export * from "./AnalyticsApi";
export * from "./BetaKeysApi";
export * from "./BillingApi";
export * from "./CommentsApi";
export * from "./EngineApi";
export * from "./FeaturedItemsApi";
export * from "./JobsApi";
export * from "./MediaFilesApi";
export * from "./MediaUploadApi";
export * from "./MiscApi";
export * from "./ModerationApi";
export * from "./PromptsApi";
export * from "./TtsApi";
export * from "./UserBookmarksApi";
export * from "./UsersApi";
export * from "./VideoApi";
export * from "./WeightsApi";
@@ -1,607 +0,0 @@
import Konva from "konva";
import { Container } from "konva/lib/Container";
import { Shape } from "konva/lib/Shape";
import { Group } from "konva/lib/Group";
import { DiffusionSharedWorkerClient } from "../SharedWorkers/Diffusion/DiffusionSharedWorkerClient";
import {
// SharedWorkerRequest,
SharedWorkerResponse,
} from "../WorkerPrimitives/SharedWorkerBase";
import {
DiffusionSharedWorkerProgressData,
DiffusionSharedWorkerResponseData,
// DiffusionSharedWorker,
DiffusionSharedWorkerItemData,
} from "../SharedWorkers/Diffusion/DiffusionSharedWorker";
import { RenderingOptions } from "../Engine";
import { FileUtilities } from "../FileUtilities/FileUtilities";
import { ImageNode, VideoNode, TextNode } from "../Nodes";
import { MediaNode } from "../types";
import { RenderTask } from "./RenderTask";
import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas";
// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
export class RenderEngine {
private videoNodes: VideoNode[];
private imageNodes: (ImageNode | TextNode)[];
private offScreenCanvas: OffscreenCanvas;
private context: OffscreenCanvasRenderingContext2D | null;
private isProcessing: boolean;
// private frames: ImageBitmap[];
// capturing composite within window
private bgLayerRef: Konva.Layer;
private mediaLayerRef: Konva.Layer;
private height: number;
private width: number;
private positionX: number;
private positionY: number;
private canUseSharedWorker: boolean;
private port: MessagePort | undefined;
private upperMaxFrames: number;
public captureCanvas: Konva.Rect;
public videoLoadingCanvas: VideoNode | undefined;
public fps: number = 24;
constructor({
width,
height,
bgLayerRef,
mediaLayerRef,
offScreenCanvas,
onRenderingSystemMessageRecieved,
}: {
width: number;
height: number;
bgLayerRef: Konva.Layer;
mediaLayerRef: Konva.Layer;
offScreenCanvas: OffscreenCanvas;
onRenderingSystemMessageRecieved: (
response: SharedWorkerResponse<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>,
) => void;
}) {
this.videoLoadingCanvas = undefined;
this.videoNodes = [];
this.imageNodes = [];
this.isProcessing = false;
this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved;
// TODO: Make this dynamic and update this on change of canvas.
this.width = width;
this.height = height;
this.positionX = window.innerWidth / 2 - this.width / 2;
this.positionY = window.innerHeight / 2 - this.height / 2;
this.offScreenCanvas = offScreenCanvas;
this.offScreenCanvas.width = this.width;
this.offScreenCanvas.height = this.height;
this.context = this.offScreenCanvas.getContext("2d");
// this.frames = [];
this.bgLayerRef = bgLayerRef;
this.mediaLayerRef = mediaLayerRef;
this.port = undefined;
this.fps = 24;
this.captureCanvas = new Konva.Rect({
name: "CaptureCanvas",
x: this.positionX,
y: this.positionY,
width: this.width,
height: this.height,
fill: "white",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
this.upperMaxFrames = 7 * this.fps; // seconds by fps
this.bgLayerRef.add(this.captureCanvas);
// send back
this.captureCanvas.setZIndex(0);
this.canUseSharedWorker = false;
this.setupSharedWorker();
//this.debug();
}
private onRenderingSystemMessageRecieved: (
response: SharedWorkerResponse<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>,
) => void;
private diffusionWorker:
| DiffusionSharedWorkerClient<
DiffusionSharedWorkerItemData,
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>
| undefined;
async updateCaptureCanvas(
width: number | undefined,
height: number | undefined,
) {
if (!this.captureCanvas) {
return;
}
if (width) {
this.width = width;
}
if (height) {
this.height = height;
}
// Ensures that all the nodes stag in the same place should
// there be a window resize.
// recompute the position
// to ensure that the position of this stays
const oldPositionX = this.positionX;
const oldPositionY = this.positionY;
// recompute the position
this.positionX = window.innerWidth / 2 - this.width / 2;
this.positionY = window.innerHeight / 2 - this.height / 2;
this.captureCanvas.setPosition({ x: this.positionX, y: this.positionY });
this.captureCanvas.size({ width: this.width, height: this.height });
// this is the change in positions
const deltaX = this.positionX - oldPositionX;
const deltaY = this.positionY - oldPositionY;
var children = this.mediaLayerRef.getChildren();
for (let i = 0; i < children.length; i++) {
let node = children[i];
// skip the capture canvas update.
if (node.name() === "CaptureCanvas") {
continue;
}
const pos = node.getPosition();
node.setPosition({
x: pos.x + deltaX,
y: pos.y + deltaY,
});
}
// update the context menu
// TODO: find selected node and update that position
// this.videoNodes.forEach((node) => {
// node.updateContextComponents();
// });
this.mediaLayerRef.batchDraw();
}
debug() {
// DEBUG ONLY
const rectangle = new Konva.Rect({
x: this.positionX,
y: this.positionY,
width: 100,
height: 100,
fill: "green",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
this.mediaLayerRef.add(rectangle);
}
async sendCanvasPayload(renderTask: RenderTask) {
if (!this.port) {
return console.log("Undefined Worker");
}
this.port.postMessage(renderTask);
}
setupSharedWorker() {
if (typeof SharedWorker !== "undefined") {
console.log("Shared Workers are supported in this browser.");
// Debug chrome://inspect/#workers
// "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts",
this.diffusionWorker = new DiffusionSharedWorkerClient(
this.onRenderingSystemMessageRecieved,
);
this.canUseSharedWorker = true;
} else {
console.log("Shared Workers are not supported in this browser.");
// Handle the lack of Shared Worker support (e.g., fallback to another solution)
this.canUseSharedWorker = false;
}
}
public placeDebugRect(
x: number,
y: number,
width: number,
height: number,
layer: Konva.Layer,
): void {
const rect = new Konva.Rect({
x: x,
y: y,
width: width, // Default width
height: height, // Default height
fill: "green",
draggable: false,
});
layer.add(rect);
layer.draw();
}
// This function uses a portion of the video layer to capture just the capture canvas.
// capture everything after seeking each video node.
renderPortionOfLayer(
layer: Konva.Layer,
x: number,
y: number,
width: number,
height: number,
): HTMLCanvasElement {
const canvas = layer.toCanvas({
x: x,
y: y,
width: width,
height: height,
});
return canvas;
}
private findLongestVideoLength(): number {
let maxLength = 0;
// could simplify this logic to ensure that imageNodes has number of frames.
if (this.imageNodes.length > 0 && this.videoNodes.length === 0) {
return this.fps * 3; // three seconds determine whether to change this
}
this.videoNodes.forEach((node) => {
const videoLength = node.getNumberFrames();
if (videoLength > maxLength) {
maxLength = videoLength;
}
});
return maxLength;
}
public addNodes(node: MediaNode) {
if (node instanceof VideoNode) {
this.videoNodes.push(node);
} else if (node instanceof ImageNode || node instanceof TextNode) {
this.imageNodes.push(node);
}
}
public removeNodes(node: MediaNode) {
if (node instanceof VideoNode) {
const index = this.videoNodes.indexOf(node);
if (index > -1) {
this.videoNodes.splice(index, 1);
}
} else if (node instanceof ImageNode || node instanceof TextNode) {
const index = this.imageNodes.indexOf(node);
if (index > -1) {
this.imageNodes.splice(index, 1);
}
}
}
// Do a bunch of precondition checks and error out early on.
public async startProcessing(renderingOptions?: RenderingOptions) {
// Start processing and lock everything
this.isProcessing = true;
try {
// or not loaded
if (this.videoNodes.length + this.imageNodes.length < 1) {
throw Error("Must at least have 1 item on the board.");
}
// error out if nodes are not all loaded.
// todo remove when we have error handling + and ui
var failed = false;
// if there are atleast 1 video node images are all covered.
for (let i = 0; i < this.videoNodes.length; i++) {
const item = this.videoNodes[i];
if (!item.kNode) {
return;
}
item.kNode.listening(false);
item.unhighlight();
if (item.didFinishLoading == false) {
// error out and show error message
//this.startProcessing();
failed = true;
setTimeout(this.startProcessing.bind(this), 1000);
break;
}
item.setProcessing(true);
}
// todo remove
if (failed) {
// throw error
throw Error("Wait For Items to Finish Processing.");
}
//this.videoNodes.forEach((item: VideoNode) => {});
// find the longest video node
let numberOfFrames = this.findLongestVideoLength();
numberOfFrames = Math.min(numberOfFrames, this.upperMaxFrames);
console.log(`Number Of Frames: ${numberOfFrames}`);
await this.render(numberOfFrames, renderingOptions);
} catch (error) {
console.log(error);
throw error;
} finally {
this.isProcessing = false;
// enable all nodes again.
for (let i = 0; i < this.videoNodes.length; i++) {
const item = this.videoNodes[i];
if (!item.kNode) {
return;
}
item.setProcessing(false);
item.kNode.listening(true);
}
}
}
// public stopProcessing() {
// this.isProcessing = false;
// }
// find the frame time given the frame number
private calculateFrameTime(frameNumber: number, frameRate: number): number {
return frameNumber / frameRate;
}
/**
*
* @param config
*/
private async renderFrame(config: {
layerOfInterest: Konva.Layer; // layer where the element that you want to clip lives.
// XY and height of a captureCanvas ( region of interest )
x?: number; // x position of the region of interest
y?: number; // y position of the region of interest
width?: number; // size of the region of interest
height?: number; // size of the region of interest
pixelRatio?: number; // higher means higher quality
mimeType?: string; // image/jpeg or image/png
quality?: number; // 1.0 is the best.
test: boolean; // true == blob else Image Bitmap
}): Promise<ImageBitmap | Blob> {
try {
const box = config.layerOfInterest.getClientRect();
const stage = config.layerOfInterest.getStage();
const x = config.x !== undefined ? config.x : Math.floor(box.x);
const y = config.y !== undefined ? config.y : Math.floor(box.y);
const pixelRatio = config.pixelRatio || 1;
const container = config.layerOfInterest as Container<Group | Shape>;
const offScreenSceneCanvas = new OffScreenSceneCanvas({
width:
config.width || Math.ceil(box.width) || (stage ? stage.width() : 0),
height:
config.height ||
Math.ceil(box.height) ||
(stage ? stage.height() : 0),
pixelRatio: pixelRatio,
});
// TODO: fill canvas white properly
const context = offScreenSceneCanvas.getContext();
// context.fillStyle = "white";
// context.fillRect(
// 0,
// 0,
// offScreenSceneCanvas.width,
// offScreenSceneCanvas.height,
// );
const buffer = new OffScreenSceneCanvas({
width:
offScreenSceneCanvas.width / offScreenSceneCanvas.pixelRatio +
Math.abs(x),
height:
offScreenSceneCanvas.height / offScreenSceneCanvas.pixelRatio +
Math.abs(y),
pixelRatio: offScreenSceneCanvas.pixelRatio,
});
context.save();
if (x || y) {
context.translate(-1 * x, -1 * y);
}
container.drawScene(offScreenSceneCanvas, undefined, buffer);
// Not a type mistake ... DO NOT FIX
// @ts-ignore
const offscreenCanvas = offScreenSceneCanvas._canvas as OffscreenCanvas;
let result = undefined;
// if config.test is true, the result is downloaded to the local files
// config.test = true;
if (config.test) {
const blob = await offscreenCanvas.convertToBlob({
quality: config.quality ?? 1.0,
type: "image/jpeg",
});
await FileUtilities.blobToFileJpeg(blob, "1");
result = blob;
} else {
result = offscreenCanvas.transferToImageBitmap();
}
context.restore();
return result;
} catch (error) {
throw error;
}
}
/**
find longest video
then seek through each node 1 step.
stop ignore stepping if the duration is less.
**/
private async render(
largestNumberOfFrames: number,
renderingOptions?: RenderingOptions,
) {
if (!this.isProcessing) return;
// Stop all nodes first
console.log(`LargestNumberOfFrames:${largestNumberOfFrames}`);
for (let k = 0; k < this.videoNodes.length; k++) {
const videoNode = this.videoNodes[k];
if (videoNode.didFinishLoading === false) {
throw Error("Videos Did Not Finish Loading Please Try Again.");
}
await videoNode.reset();
}
// only pick nodes that intersect with the canvas on screen bounds to freeze.
for (let j = 0; j < largestNumberOfFrames; j++) {
// Seek Video Nodes first then draw
let frameTime = undefined;
for (let i = 0; i < this.videoNodes.length; i++) {
const currentVideoNode = this.videoNodes[i];
frameTime = this.calculateFrameTime(j, currentVideoNode.fps);
frameTime = parseFloat(frameTime.toFixed(2));
if (frameTime < currentVideoNode.duration) {
// console.log(`CurrentFrame:${j}`);
// console.log(`FrameTime:${frameTime}`);
// console.log(`Duration:${currentVideoNode.duration}`);
await currentVideoNode.seek(frameTime);
} // end of if context
} // End frame time
this.mediaLayerRef.draw();
// use main thread
if (this.canUseSharedWorker === false) {
// SCOPES the capture for the context
// Correct size for the mobile canvas.
this.offScreenCanvas.width = this.width;
this.offScreenCanvas.height = this.height;
if (this.context) {
// This crops it starting at position X / Y where the mobile canvas is
// Then picks the height and width range
// then we draw it at 0,0,width and height of the canvas
this.context.drawImage(
this.mediaLayerRef.canvas._canvas,
this.positionX,
this.positionY,
this.width,
this.height,
0,
0,
this.width,
this.height,
);
//TODO write the non webworker version
const blob = await this.offScreenCanvas.convertToBlob({
quality: 1.0,
type: "image/jpeg",
});
await FileUtilities.blobToFileJpeg(blob, "1");
break;
} // end of for each frame
} else {
// decode on the shared webworker.
if (!this.context) {
console.log("Context Didn't Initialize");
return;
}
if (!this.diffusionWorker) {
console.log("Didn't Initialize Diffusion");
return;
}
console.log(
`context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`,
);
const bitmap = await this.renderFrame({
layerOfInterest: this.mediaLayerRef,
x: this.captureCanvas.x(),
y: this.captureCanvas.y(),
width: this.width,
height: this.height,
mimeType: "image/jpeg",
pixelRatio: 1,
quality: 1.0,
test: false,
});
const data: DiffusionSharedWorkerItemData = {
height: this.height,
width: this.width,
imageBitmap: bitmap as ImageBitmap,
frame: j,
totalFrames: largestNumberOfFrames,
prompt: renderingOptions,
};
console.log(
`Processing Frame:${j + 1} out of ${largestNumberOfFrames}`,
);
this.diffusionWorker.sendData(1, data, false);
}
} // end of largest number of frames loop
// finished looping
if (!this.diffusionWorker) {
console.log("Didn't Initialize Diffusion");
return;
}
const jobID = 1;
this.diffusionWorker.sendData(
jobID,
{
height: this.height,
width: this.width,
imageBitmap: undefined,
frame: -1,
totalFrames: largestNumberOfFrames,
prompt: renderingOptions,
},
true,
);
}
}
@@ -1,396 +0,0 @@
// Example of this working.
import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
import { MediaUploadApi, VideoApi, MediaFilesApi } from "~/Classes/ApiManager";
import { Visibility } from "~/Classes/ApiManager/enums/Visibility";
import { v4 as uuidv4 } from "uuid";
import { JobsApi } from "~/Classes/ApiManager";
import {
ProgressData,
WorkResult,
} from "~/KonvaApp/WorkerPrimitives/GenericWorker";
import {
SharedWorkerBase,
SharedWorkerRequest,
SharedWorkerResponse,
ResponseType,
} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
import { JobStatus } from "~/Classes/ApiManager/enums/Job";
import { RenderingOptions } from "~/KonvaApp/Engine";
import { MediaFile } from "~/Classes/ApiManager/models/MediaFile";
export interface DiffusionSharedWorkerProgressData {
url: string;
status: JobStatus;
progress: number;
mediaToken: string;
}
export interface DiffusionSharedWorkerItemData {
imageBitmap: ImageBitmap | undefined;
totalFrames: number;
frame: number;
height: number;
width: number;
prompt?: RenderingOptions;
}
export interface DiffusionSharedWorkerErrorData {
error: string;
}
export interface DiffusionSharedWorkerResponseData {
videoUrl: string;
}
export class DiffusionSharedWorker extends SharedWorkerBase<
DiffusionSharedWorkerItemData,
DiffusionSharedWorkerResponseData | MediaFile,
DiffusionSharedWorkerProgressData
> {
public name: string;
public zipFileWriter: BlobWriter;
public zipWriter: ZipWriter<Blob>;
public imageType: string;
public totalFrames: number;
public offscreenCanvas: OffscreenCanvas | undefined;
public bitmapContext: ImageBitmapRenderingContext | undefined | null;
public mediaAPI: MediaUploadApi;
public mediaFilesAPI: MediaFilesApi;
public videoAPI: VideoApi;
public jobsAPI: JobsApi;
public blobs: Blob[];
constructor(port: MessagePort) {
super(port);
this.name = "DiffusionWorker";
this.setup(this.workFunction.bind(this), this.progressFunction.bind(this));
this.offscreenCanvas = undefined;
this.bitmapContext = undefined;
this.imageType = "image/jpeg";
this.zipFileWriter = new BlobWriter(this.imageType);
this.zipWriter = new ZipWriter(this.zipFileWriter);
this.totalFrames = 0;
this.videoAPI = new VideoApi();
this.jobsAPI = new JobsApi();
this.mediaAPI = new MediaUploadApi();
this.mediaFilesAPI = new MediaFilesApi();
this.blobs = [];
this.totalFrames = 0;
}
async zipBlobs(): Promise<Blob> {
for (let i = 0; i < this.blobs.length; i++) {
const blob = this.blobs[i];
const name = String(i).padStart(5, "0"); // '0009'
await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
}
const zipBlob = await this.zipWriter.close();
return zipBlob;
}
async reset() {
this.zipFileWriter = new BlobWriter(this.imageType);
this.zipWriter = new ZipWriter(this.zipFileWriter);
this.totalFrames = 0;
this.blobs = [];
}
// Data here will be shipped off for progressive loading
async workFunction(
isDoneStreaming: boolean,
item: DiffusionSharedWorkerItemData,
reportProgress: (data: DiffusionSharedWorkerProgressData) => void,
): Promise<
[DiffusionSharedWorkerResponseData | MediaFile | undefined, boolean]
> {
// make request via api with options
try {
this.totalFrames = item.totalFrames;
if (this.offscreenCanvas === undefined) {
this.offscreenCanvas = new OffscreenCanvas(item.width, item.height);
this.bitmapContext = this.offscreenCanvas.getContext("bitmaprenderer");
}
if (!this.bitmapContext) {
console.log("Failed to create bitmap context.");
throw Error("Bitmap Rendering Context Not Availible.");
}
if (item.imageBitmap !== undefined) {
this.bitmapContext.transferFromImageBitmap(item.imageBitmap);
const blob = await this.offscreenCanvas.convertToBlob({
quality: 1.0,
type: this.imageType,
});
this.blobs.push(blob);
console.log("Length of blob");
console.log(this.blobs.length);
}
// progress
const aproxSteps = this.totalFrames;
const totalPercent = item.prompt ? 0.25 : 0.9;
const progressData: DiffusionSharedWorkerProgressData = {
url: "",
status: JobStatus.PENDING,
progress: (item.frame / aproxSteps) * totalPercent,
mediaToken: "",
};
console.log("Lets report progress.");
reportProgress(progressData); // once finished gives you up to 50%
if (isDoneStreaming === false) {
return [undefined, false];
}
console.log("Prepare to Zip");
const zipBlob = await this.zipBlobs();
console.log("Zipped", zipBlob);
const progressMedia: DiffusionSharedWorkerProgressData = {
url: "",
status: JobStatus.PENDING,
progress: ((item.frame + 0.5) / aproxSteps) * totalPercent,
mediaToken: "",
};
console.log("Lets report progress.");
reportProgress(progressMedia); // once finished gives you up to 50%
const response = await this.mediaAPI.UploadStudioShot({
maybe_title: "",
uuid_idempotency_token: uuidv4(),
blob: zipBlob,
fileName: "media.zip",
maybe_visibility: Visibility.Public,
});
const mediaToken = response.data;
if (!mediaToken) {
this.reset();
throw Error(`Server Failed Try Again: ${response.errorMessage}`);
}
if (!item.prompt) {
console.log("got video >>", response);
const videoRepsonse = await this.mediaFilesAPI.GetMediaFileByToken({
mediaFileToken: mediaToken,
});
this.reset();
return [videoRepsonse.data, true];
}
console.log(item.prompt);
const studioResponse = await this.videoAPI.EnqueueStudio({
enqueueVideo: {
disable_lcm: false,
enable_lipsync: item.prompt.lipSync,
input_file: mediaToken, // Replace with actual media token
negative_prompt: item.prompt.negativePrompt,
prompt: item.prompt.positivePrompt,
remove_watermark: false,
style: item.prompt.artstyle, // Map to the appropriate art style
frame_skip: 2,
travel_prompt: "",
trim_end_millis: 7000,
trim_start_millis: 0,
use_cinematic: item.prompt.cinematic,
use_face_detailer: item.prompt.faceDetail,
use_strength: item.prompt.styleStrength,
use_upscaler: item.prompt.upscale,
uuid_idempotency_token: uuidv4(),
global_ipa_media_token: "",
input_depth_file: "",
input_normal_file: "",
input_outline_file: "",
creator_set_visibility: Visibility.Public,
},
});
let resultURL = undefined;
if (studioResponse.success && studioResponse.data?.inference_job_token) {
console.log("Start Streaming Result");
if (!studioResponse.data.inference_job_token) {
await this.reset(); // reset on error
throw Error("No Job Token Returned Try Again");
}
const jobToken = studioResponse.data.inference_job_token;
// if error send it back through the pipe
let jobIsProcessing = true;
while (jobIsProcessing) {
const job = await this.jobsAPI.GetJobByToken({ token: jobToken });
// TODO: MAKE REQUEST FOR PREVIEW HERE
console.log(job);
if (!job.data) {
console.log("No Job Data resetting the zip state");
await this.reset();
console.log(job.data);
continue;
}
const status = job.data.status.status;
const progress = job.data.status.progress_percentage;
let computedProgress = 30;
if (progress === 0) {
computedProgress = 30;
}
computedProgress = 30 + progress * 0.7;
let renderProgressData: DiffusionSharedWorkerProgressData = {
url: "",
status: status,
progress: computedProgress / 100,
mediaToken: mediaToken,
};
renderProgressData.status = status;
switch (status) {
case JobStatus.PENDING:
console.log("Pending");
reportProgress(renderProgressData); // once finished gives you up to 50%
break;
case JobStatus.STARTED:
console.log("Started");
reportProgress(renderProgressData);
break;
case JobStatus.ATTEMPT_FAILED:
console.log("Attempt Failed");
reportProgress(renderProgressData);
break;
case JobStatus.COMPLETE_SUCCESS:
console.log("Complete Success");
renderProgressData.progress = 100;
jobIsProcessing = false;
if (!job.data.maybe_result.media_links.cdn_url) {
await this.reset();
throw Error("Server Failed To Return Result");
}
resultURL = job.data.maybe_result.media_links.cdn_url;
reportProgress(renderProgressData);
break;
case JobStatus.COMPLETE_FAILURE:
console.log("Complete Failure");
jobIsProcessing = false;
renderProgressData.progress = 0;
reportProgress(renderProgressData);
break;
case JobStatus.DEAD:
console.log("Dead");
jobIsProcessing = false;
reportProgress(renderProgressData);
throw Error("Server Failed to Process Please Try Again.");
case JobStatus.CANCELLED_BY_SYSTEM:
console.log("Cancelled by system");
jobIsProcessing = false;
reportProgress(renderProgressData);
break;
case JobStatus.CANCELLED_BY_USER:
console.log("Cancelled by user");
jobIsProcessing = false;
reportProgress(renderProgressData);
break;
}
await this.sleep(500);
} // end while loop
}
if (!resultURL) {
await this.reset(); // reset on error
throw Error("Media URL Result Missing");
}
const responseData: DiffusionSharedWorkerResponseData = {
videoUrl: resultURL,
};
await this.reset();
return [responseData, true];
} catch (error) {
await this.reset();
console.log(error);
throw error;
}
}
async sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
progressFunction(progress: ProgressData<DiffusionSharedWorkerProgressData>) {
console.log(
`Progress Function JobID:${progress.jobID} Data:${progress.data}`,
);
// send out to node as a worker response
this.send({
jobID: progress.jobID,
responseType: ResponseType.progress,
data: progress.data,
});
}
async reportResult(result: WorkResult<DiffusionSharedWorkerResponseData>) {
this.send({
jobID: result.jobID,
responseType: ResponseType.result,
data: result.data,
});
}
async errorFunction(
error: SharedWorkerResponse<
DiffusionSharedWorkerItemData,
DiffusionSharedWorkerResponseData
>,
) {
this.send({
jobID: error.jobID,
responseType: ResponseType.error,
data: error.data?.toString(),
});
}
async receive(request: SharedWorkerRequest<DiffusionSharedWorkerItemData>) {
console.log("Received Request");
console.log(request);
this.submitWork({
jobID: request.jobID,
data: request.data,
isDoneStreaming: request.isDoneStreaming,
});
}
}
// This is a copy paste to create a worker now.
//@ts-ignore
self.onconnect = (e: any) => {
const port = e.ports[0];
console.log("DiffusionSharedWorker Started");
let worker: DiffusionSharedWorker | undefined = undefined;
let started = false;
if (started === false) {
started = true;
worker = new DiffusionSharedWorker(port);
worker.start();
}
// Response For Start.
const workerResult = "DiffusionSharedWorker Started";
port.postMessage(workerResult);
port.start(); // Required when using addEventListener. Otherwise called implicitly by onmessage setter.
};
@@ -1,88 +0,0 @@
import { v4 as uuidv4 } from "uuid";
import {
SharedWorkerRequest,
SharedWorkerResponse,
} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
import { ResponseType } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
import DiffusionSharedWorker from "./DiffusionSharedWorker?sharedworker";
export class DiffusionSharedWorkerClient<
DiffusionSharedWorkerItemData,
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData,
> {
private port: MessagePort;
private sharedWorker: SharedWorker;
private messageReceived: (
response: SharedWorkerResponse<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>,
) => void;
constructor(
messageReceived: (
response: SharedWorkerResponse<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>,
) => void,
) {
this.messageReceived = messageReceived;
try {
console.log("This is running a worker in production");
this.sharedWorker = new DiffusionSharedWorker({
name: "DiffusionWorker-" + uuidv4(),
});
this.sharedWorker.addEventListener("error", (value) => {
console.log("Shared worker ERROR:");
console.log(value);
});
this.port = this.sharedWorker.port;
this.port.onmessage = this.onMessage.bind(this);
this.port.start();
console.log("launched shared worker (?)");
} catch (error) {
console.log("ERROR with shared worker!");
console.log(error);
throw Error("Could Not Start Worker");
}
}
async onMessage(event: MessageEvent) {
// returns progress and returns result need to check error tomorrow then were good to go.
//console.log(`incoming`);
// console.log(event);
if (event.data.responseType === ResponseType.error) {
console.log(`DiffusionSharedWorkerClient Error`);
console.log(event.data);
this.messageReceived(event.data);
} else if (event.data.responseType === ResponseType.progress) {
// console.log(`DiffusionSharedWorkerClient Progress`);
//console.log(event.data);
this.messageReceived(event.data);
} else if (event.data.responseType === ResponseType.result) {
// console.log(`DiffusionSharedWorkerClient Result`);
//console.log(event.data);
this.messageReceived(event.data);
} else {
console.log(`DiffusionSharedWorkerClient Message Unknown:`);
console.log(event.data);
}
}
async sendData(
jobID: number,
data: DiffusionSharedWorkerItemData,
isDoneStreaming: boolean,
) {
const payload: SharedWorkerRequest<DiffusionSharedWorkerItemData> = {
jobID: jobID,
data: data,
isDoneStreaming: isDoneStreaming,
};
this.port.postMessage(payload);
}
}
@@ -44,10 +44,10 @@ export const Login = () => {
const usernameOrEmail = form.get("usernameOrEmail")?.toString();
const password = form.get("password")?.toString();
if (usernameOrEmail && password) {
let api = new UsersApi();
let response = await api.Login({
usernameOrEmail,
password,
@@ -99,7 +99,6 @@ export const Login = () => {
shouldShowLoader && "opacity-0",
)}
>
<Input
label="Username or Email"
icon={faUser}
@@ -1,524 +0,0 @@
import React, { useEffect, useState, useRef } from "react";
import {
ServerClient,
fileToBase64,
base64ToImageUrl,
} from "../PyServer/ServerClient";
import {
ProgressUpdateMessage,
SetupResponse,
GenerateResponse,
StatusResponse,
UpdateSettingsResponse,
} from "../PyServer/ServerTypes";
// Default server URL
const DEFAULT_SERVER_URL = "ws://localhost:8765";
export const ImageGenerator: React.FC = () => {
// Server connection state
const [isConnected, setIsConnected] = useState(false);
const [serverUrl, setServerUrl] = useState(DEFAULT_SERVER_URL);
const clientRef = useRef<ServerClient | null>(null);
// Model state
const [modelPath, setModelPath] = useState("");
const [loraPath, setLoraPath] = useState("");
const [isModelInitialized, setIsModelInitialized] = useState(false);
// Generation parameters
const [sourceImage, setSourceImage] = useState<File | null>(null);
const [sourceImageUrl, setSourceImageUrl] = useState<string | null>(null);
const [prompt, setPrompt] = useState("");
const [loraStrength, setLoraStrength] = useState(1.0);
const [imageToImageStrength, setImageToImageStrength] = useState(0.75);
const [steps, setSteps] = useState(4);
const [guidanceScale, setGuidanceScale] = useState(1.0);
const [width, setWidth] = useState(1024);
const [height, setHeight] = useState(1024);
// Results and UI state
const [generatedImageUrl, setGeneratedImageUrl] = useState<string | null>(
null,
);
const [isLoading, setIsLoading] = useState(false);
const [progress, setProgress] = useState<ProgressUpdateMessage | null>(null);
const [errorMessage, setErrorMessage] = useState<string | null>(null);
// Initialize the client
useEffect(() => {
const client = new ServerClient(serverUrl, {
onOpen: () => {
console.log("Connected to server");
setIsConnected(true);
setErrorMessage(null);
// Request status to see if model is already initialized
client.requestStatus().catch((error) => {
console.error("Error requesting status:", error);
});
},
onClose: () => {
console.log("Disconnected from server");
setIsConnected(false);
setIsModelInitialized(false);
},
onError: (error) => {
console.error("Error:", error);
setErrorMessage(error.error);
setIsLoading(false);
},
onProgressUpdate: (update) => {
console.log("Progress:", update);
setProgress(update);
if (update.stage !== "error") {
setErrorMessage(null);
}
},
onSetupResponse: (response) => {
console.log("Setup response:", response);
setIsLoading(false);
if (response.success) {
setIsModelInitialized(true);
setErrorMessage(null);
} else {
setErrorMessage(response.error || "Unknown error during setup");
}
},
onUpdateSettingsResponse: (response) => {
console.log("Update settings response:", response);
setIsLoading(false);
if (response.success) {
setErrorMessage(null);
} else {
setErrorMessage(response.error || "Unknown error updating settings");
}
},
onGenerateResponse: (response) => {
console.log("Generate response:", response);
setIsLoading(false);
if (response.success && response.image) {
setGeneratedImageUrl(base64ToImageUrl(response.image));
setErrorMessage(null);
} else {
setErrorMessage(response.error || "Unknown error generating image");
}
},
onStatusResponse: (response) => {
console.log("Status response:", response);
if (response.success) {
setIsModelInitialized(response.model_initialized);
if (response.current_settings) {
// Update UI with current settings
setModelPath(response.current_settings.sdxl_checkpoint_path || "");
setLoraPath(response.current_settings.lora_path || "");
// Update generation parameters if they exist
if (response.current_settings.default_lora_strength !== undefined) {
setLoraStrength(response.current_settings.default_lora_strength);
}
if (
response.current_settings.default_image_to_image_strength !==
undefined
) {
setImageToImageStrength(
response.current_settings.default_image_to_image_strength,
);
}
if (response.current_settings.default_steps !== undefined) {
setSteps(response.current_settings.default_steps);
}
if (
response.current_settings.default_guidance_scale !== undefined
) {
setGuidanceScale(
response.current_settings.default_guidance_scale,
);
}
if (response.current_settings.default_width !== undefined) {
setWidth(response.current_settings.default_width);
}
if (response.current_settings.default_height !== undefined) {
setHeight(response.current_settings.default_height);
}
}
}
},
});
clientRef.current = client;
// Connect to the server
client.connect().catch((error) => {
console.error("Error connecting to server:", error);
setErrorMessage(`Failed to connect to server: ${error.message}`);
});
return () => {
client.disconnect();
clientRef.current = null;
};
}, [serverUrl]);
// Handle source image selection
const handleImageChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (file) {
setSourceImage(file);
setSourceImageUrl(URL.createObjectURL(file));
}
};
// Handle setup submission
const handleSetup = async () => {
if (!clientRef.current) return;
setIsLoading(true);
setErrorMessage(null);
try {
await clientRef.current.setup(modelPath, loraPath || undefined);
} catch (error) {
console.error("Error setting up model:", error);
setErrorMessage(`Error setting up model: ${(error as Error).message}`);
setIsLoading(false);
}
};
// Handle update settings
const handleUpdateSettings = async () => {
if (!clientRef.current) return;
setIsLoading(true);
setErrorMessage(null);
try {
await clientRef.current.updateSettings({
lora_path: loraPath || null,
default_lora_strength: loraStrength,
default_image_to_image_strength: imageToImageStrength,
default_steps: steps,
default_guidance_scale: guidanceScale,
default_width: width,
default_height: height,
});
} catch (error) {
console.error("Error updating settings:", error);
setErrorMessage(`Error updating settings: ${(error as Error).message}`);
setIsLoading(false);
}
};
// Handle generation
const handleGenerate = async () => {
if (!clientRef.current || !sourceImage) {
setErrorMessage("Please select a source image first");
return;
}
setIsLoading(true);
setErrorMessage(null);
setGeneratedImageUrl(null);
try {
const base64Image = await fileToBase64(sourceImage);
await clientRef.current.generate({
image: base64Image,
prompt,
lora_strength: loraStrength,
image_to_image_strength: imageToImageStrength,
num_inference_steps: steps,
guidance_scale: guidanceScale,
generated_image_width: width,
generated_image_height: height,
});
} catch (error) {
console.error("Error generating image:", error);
setErrorMessage(`Error generating image: ${(error as Error).message}`);
setIsLoading(false);
}
};
return (
<div className="mx-auto max-w-4xl p-6">
<h1 className="mb-6 text-2xl font-bold">SDXL Image Generator</h1>
{/* Connection and Setup */}
<div className="mb-6 rounded-lg bg-gray-100 p-4">
<h2 className="mb-2 text-xl font-semibold">Server Connection</h2>
<div className="mb-4 flex gap-2">
<input
type="text"
value={serverUrl}
onChange={(e) => setServerUrl(e.target.value)}
className="flex-grow rounded border p-2"
placeholder="WebSocket URL"
/>
<button
onClick={() => clientRef.current?.connect()}
disabled={isConnected}
className="rounded bg-blue-500 px-4 py-2 text-white disabled:bg-gray-400"
>
Connect
</button>
<button
onClick={() => clientRef.current?.disconnect()}
disabled={!isConnected}
className="rounded bg-red-500 px-4 py-2 text-white disabled:bg-gray-400"
>
Disconnect
</button>
</div>
<div className="mb-2 flex items-center">
<span className="mr-2">Status:</span>
<span
className={`rounded px-2 py-1 text-white ${isConnected ? "bg-green-500" : "bg-red-500"}`}
>
{isConnected ? "Connected" : "Disconnected"}
</span>
<span className="ml-4 mr-2">Model:</span>
<span
className={`rounded px-2 py-1 text-white ${isModelInitialized ? "bg-green-500" : "bg-yellow-500"}`}
>
{isModelInitialized ? "Initialized" : "Not Initialized"}
</span>
</div>
</div>
{/* Model Setup */}
<div className="mb-6 rounded-lg bg-gray-100 p-4">
<h2 className="mb-2 text-xl font-semibold">Model Setup</h2>
<div className="space-y-4">
<div>
<label className="mb-1 block">SDXL Checkpoint Path:</label>
<input
type="text"
value={modelPath}
onChange={(e) => setModelPath(e.target.value)}
className="w-full rounded border p-2"
placeholder="/path/to/sdxl/model"
/>
</div>
<div>
<label className="mb-1 block">LoRA Path (optional):</label>
<input
type="text"
value={loraPath}
onChange={(e) => setLoraPath(e.target.value)}
className="w-full rounded border p-2"
placeholder="/path/to/lora"
/>
</div>
<div className="flex gap-2">
<button
onClick={handleSetup}
disabled={!isConnected || !modelPath || isLoading}
className="rounded bg-green-500 px-4 py-2 text-white disabled:bg-gray-400"
>
Initialize Model
</button>
<button
onClick={handleUpdateSettings}
disabled={!isConnected || !isModelInitialized || isLoading}
className="rounded bg-purple-500 px-4 py-2 text-white disabled:bg-gray-400"
>
Update Settings
</button>
</div>
</div>
</div>
{/* Generation Parameters */}
<div className="mb-6 rounded-lg bg-gray-100 p-4">
<h2 className="mb-2 text-xl font-semibold">Generation Parameters</h2>
<div className="grid grid-cols-1 gap-4 md:grid-cols-2">
<div>
<label className="mb-1 block">Prompt:</label>
<textarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
className="h-20 w-full rounded border p-2"
placeholder="Enter your prompt..."
/>
</div>
<div>
<label className="mb-1 block">Source Image:</label>
<input
type="file"
accept="image/*"
onChange={handleImageChange}
className="mb-2"
/>
{sourceImageUrl && (
<img
src={sourceImageUrl}
alt="Source"
className="max-h-40 rounded border"
/>
)}
</div>
<div>
<label className="mb-1 block">
LoRA Strength: {loraStrength.toFixed(2)}
</label>
<input
type="range"
min="0"
max="1"
step="0.05"
value={loraStrength}
onChange={(e) => setLoraStrength(parseFloat(e.target.value))}
className="w-full"
/>
</div>
<div>
<label className="mb-1 block">
Image-to-Image Strength: {imageToImageStrength.toFixed(2)}
</label>
<input
type="range"
min="0"
max="1"
step="0.05"
value={imageToImageStrength}
onChange={(e) =>
setImageToImageStrength(parseFloat(e.target.value))
}
className="w-full"
/>
</div>
<div>
<label className="mb-1 block">Steps: {steps}</label>
<input
type="range"
min="1"
max="20"
step="1"
value={steps}
onChange={(e) => setSteps(parseInt(e.target.value))}
className="w-full"
/>
</div>
<div>
<label className="mb-1 block">
Guidance Scale: {guidanceScale.toFixed(2)}
</label>
<input
type="range"
min="0"
max="10"
step="0.1"
value={guidanceScale}
onChange={(e) => setGuidanceScale(parseFloat(e.target.value))}
className="w-full"
/>
</div>
<div>
<label className="mb-1 block">Width:</label>
<select
value={width}
onChange={(e) => setWidth(parseInt(e.target.value))}
className="w-full rounded border p-2"
>
<option value="512">512</option>
<option value="768">768</option>
<option value="1024">1024</option>
<option value="1280">1280</option>
</select>
</div>
<div>
<label className="mb-1 block">Height:</label>
<select
value={height}
onChange={(e) => setHeight(parseInt(e.target.value))}
className="w-full rounded border p-2"
>
<option value="512">512</option>
<option value="768">768</option>
<option value="1024">1024</option>
<option value="1280">1280</option>
</select>
</div>
</div>
<button
onClick={handleGenerate}
disabled={
!isConnected || !isModelInitialized || !sourceImage || isLoading
}
className="mt-4 rounded bg-blue-500 px-6 py-2 text-white disabled:bg-gray-400"
>
Generate Image
</button>
</div>
{/* Progress and Results */}
{isLoading && progress && (
<div className="mb-6 rounded-lg bg-gray-100 p-4">
<h2 className="mb-2 text-xl font-semibold">Progress</h2>
<div className="mb-2">
<div className="mb-1 flex justify-between">
<span>{progress.stage}</span>
<span>{progress.progress}%</span>
</div>
<div className="h-2.5 w-full rounded-full bg-gray-300">
<div
className="h-2.5 rounded-full bg-blue-600"
style={{ width: `${progress.progress}%` }}
></div>
</div>
{progress.file && (
<div className="mt-1 text-sm text-gray-600">{progress.file}</div>
)}
</div>
</div>
)}
{errorMessage && (
<div className="mb-6 rounded border border-red-400 bg-red-100 px-4 py-3 text-red-700">
<p>{errorMessage}</p>
</div>
)}
{generatedImageUrl && (
<div className="mb-6 rounded-lg bg-gray-100 p-4">
<h2 className="mb-2 text-xl font-semibold">Generated Image</h2>
<div className="flex justify-center">
<img
src={generatedImageUrl}
alt="Generated"
className="max-w-full rounded border"
/>
</div>
</div>
)}
</div>
);
};
@@ -2,17 +2,12 @@ import { useState, useRef, useEffect } from "react";
import { invoke } from "@tauri-apps/api/core";
import { useSignals } from "@preact/signals-react/runtime";
import { toast } from "sonner";
import {
UploaderStates,
// UploaderState,
uploadImage,
} from "~/components/UploadImage";
import { UploaderStates, uploadImage } from "~/components/UploadImage";
import { PopoverMenu } from "~/components/reusable/Popover";
import { Tooltip } from "~/components/reusable/Tooltip";
import { Button } from "~/components/reusable/Button";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button } from "@storyteller/ui-button";
import { ToggleButton } from "~/components/reusable/ToggleButton";
import { Toaster } from "~/components/ui/Toast";
import { Modal } from "@storyteller/ui-modal";
import {
faMessageXmark,
@@ -25,7 +20,6 @@ import {
faImages,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangleWide,
faRectangleVertical,
faSquare,
faRectangle,
@@ -34,7 +28,7 @@ import { getCanvasRenderBitmap } from "~/signal/canvasRenderBitmap";
import { EncodeImageBitmapToBase64 } from "~/utilities/EncodeImageBitmapToBase64";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { PopoverItem } from "~/components/reusable/Popover";
import { Api, ApiManager } from "~/KonvaApp/Api";
import { Api } from "~/KonvaApp/Api";
import { useJobContext } from "~/components/JobContext";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
@@ -44,8 +44,12 @@ export function ProfileDropdown() {
size={34}
username={userInfo.value.display_name}
email_hash={userInfo.value.email_gravatar_hash}
avatarIndex={Number(userInfo.value.core_info.default_avatar.image_index) || 0}
backgroundIndex={Number(userInfo.value.core_info.default_avatar.color_index) || 0}
avatarIndex={
Number(userInfo.value.core_info.default_avatar.image_index) || 0
}
backgroundIndex={
Number(userInfo.value.core_info.default_avatar.color_index) || 0
}
/>
<FontAwesomeIcon icon={faChevronDown} />
</MenuButton>
@@ -1,6 +1,6 @@
import { signals } from "./auth_signals";
import { UsersApi } from "@storyteller/api";
import { BillingApi } from "@ApiManager/BillingApi";
import { BillingApi } from "@storyteller/api";
import {
updateActiveSubscriptions,
@@ -1,7 +1,6 @@
import { signals } from "./auth_signals";
import * as fetchers from "./fetchers";
import * as enums from "./enums";
import * as fetchers from "./fetchers";
export const authentication = {
signals,
fetchers,