Some bug fixes and improvements for FK

frontend/editor3d/app/pages/PageEnigma/Editor/KinHelpers/FKHelper.ts

@@ -0,0 +1,146 @@
+import { TransformControls } from "three/examples/jsm/Addons";
+import * as THREE from "three";
+
+export const FKHelperSphereName = "__FKHelperSphere__";
+
+export class FKHelper {
+
+  private targetBoneSpheres: THREE.Object3D<THREE.Object3DEventMap>[];
+  private transformControls: TransformControls;
+  private raycaster: THREE.Raycaster;
+  private scene: THREE.Scene;
+
+  private skeletonHelper: THREE.SkeletonHelper | null = null;
+  private isControlInScene = false;
+
+  // Need the actual object to perform FK on
+  // This means the object must have bones
+  // Not all bones require FK right now
+  // Need a set of names to filter bones
+  // Use mixamo names
+  // Need a transform control (rotation mode) for the bones
+  // Need to disable the object transforms for bone transforms
+  // Need raycaster
+  constructor({
+    camera,
+    domElement,
+    scene,
+    onDragChange
+  }: {
+    camera: THREE.Camera;
+    domElement: HTMLElement;
+    scene: THREE.Scene;
+    onDragChange: (dragging: boolean) => void;
+  }) {
+    this.raycaster = new THREE.Raycaster();
+    this.scene = scene;
+    this.targetBoneSpheres = [];
+    this.transformControls = new TransformControls(camera, domElement);
+
+    this.transformControls.setSpace("local");
+    this.transformControls.setMode("rotate");
+    this.transformControls.setSize(0.5);
+    this.transformControls.addEventListener("dragging-changed", (event: any) => {
+      onDragChange(event.value);
+    });
+  }
+
+  // Set FK target
+  setTarget(target: THREE.Object3D) {
+    // TODO: Highlight all the necessary bones
+    this.clear();
+    target.traverse((child) => {
+      if (child.type !== "Bone") {
+        return;
+      }
+
+      // For each target bone, display a sphere to show the FK target and raycast intersect
+      const geometry = new THREE.SphereGeometry(1, 16, 16);
+      const material = new THREE.MeshBasicMaterial({ color: 0xff0000, depthTest: false, transparent: true });
+      const sphere = new THREE.Mesh(geometry, material);
+      sphere.name = FKHelperSphereName;
+      const uniformScale = 1.5;
+      sphere.scale.set(uniformScale, uniformScale, uniformScale);
+      child.add(sphere);
+      this.targetBoneSpheres.push(sphere);
+    });
+
+    this.skeletonHelper = new THREE.SkeletonHelper(target);
+    this.scene.add(this.skeletonHelper);
+  }
+
+  onMouseClick(mouse: THREE.Vector2) {
+    this.raycaster.setFromCamera(mouse, this.transformControls.camera);
+    const intersections = this.raycaster.intersectObjects(this.targetBoneSpheres, false);
+
+    if (intersections.length < 1) {
+      console.log("No intersections");
+      this.resetHighlight();
+      this.removeControls();
+      return;
+    }
+
+    // Attach the controls to the first bone
+    const firstBone = intersections[0];
+    this.highlightSphere(firstBone.object);
+    this.showControls(firstBone.object.parent!);
+    console.log("Showing bone", firstBone.object);
+  }
+
+  private showControls(object?: THREE.Object3D) {
+    if (object) {
+      this.transformControls.attach(object);
+    }
+
+    if (this.isControlInScene) {
+      return;
+    }
+
+    this.scene.add(this.transformControls);
+    this.isControlInScene = true;
+  }
+
+  private highlightSphere(sphere: THREE.Object3D) {
+    this.targetBoneSpheres.forEach((s) => {
+      // @ts-expect-error Material on object3d, but it's fine since this is a mesh
+      s.material.opacity = 0.2
+    });
+    // @ts-expect-error Material on object3d, but it's fine since this is a mesh
+    sphere.material.opacity = 1;
+  }
+
+  private resetHighlight() {
+    this.targetBoneSpheres.forEach((s) => {
+      // @ts-expect-error Material on object3d, but it's fine since this is a mesh
+      s.material.opacity = 1;
+    });
+  }
+
+  private removeControls() {
+    this.transformControls.detach();
+
+    if (this.isControlInScene) {
+      this.scene.remove(this.transformControls);
+      this.isControlInScene = false;
+    }
+  }
+
+  private removeSpheres() {
+    this.targetBoneSpheres.forEach((sphere) => {
+      this.scene.remove(sphere);
+      sphere.parent?.remove(sphere);
+      // @ts-expect-error Geometry on object3d, but it's fine since this is a mesh
+      sphere.geometry.dispose();
+    });
+    this.targetBoneSpheres = [];
+  }
+
+  clear() {
+    this.removeControls();
+    this.removeSpheres();
+
+    if (this.skeletonHelper) {
+      this.scene.remove(this.skeletonHelper!);
+    }
+  }
+}

frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts

@@ -931,6 +931,8 @@ class Editor {
   }
 
   deleteObject(uuid: string) {
+    this.mouse_controls?.clearFKVisuals();
+    this.mouse_controls?.removeTransformControls(true);
     this.utils.deleteObject(uuid);
     if (this.outliner_feature_flag) {
       const result = this.sceneManager?.render_outliner(
@@ -962,26 +964,26 @@ class Editor {
           this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
             ? 1024
             : this.render_camera_aspect_ratio ===
-                CameraAspectRatio.VERTICAL_9_16
+              meraAspectRatio.VERTICAL_9_16
               ? 576
               : this.render_camera_aspect_ratio ===
-                  CameraAspectRatio.HORIZONTAL_3_2
+                meraAspectRatio.HORIZONTAL_3_2
                 ? 900
                 : this.render_camera_aspect_ratio ===
-                    CameraAspectRatio.VERTICAL_2_3
+                  meraAspectRatio.VERTICAL_2_3
                   ? 600
                   : 1000;
         const height =
           this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
             ? 576
             : this.render_camera_aspect_ratio ===
-                CameraAspectRatio.VERTICAL_9_16
+              meraAspectRatio.VERTICAL_9_16
               ? 1024
               : this.render_camera_aspect_ratio ===
-                  CameraAspectRatio.HORIZONTAL_3_2
+                meraAspectRatio.HORIZONTAL_3_2
                 ? 600
                 : this.render_camera_aspect_ratio ===
-                    CameraAspectRatio.VERTICAL_2_3
+                  meraAspectRatio.VERTICAL_2_3
                   ? 900
                   : 1000;
 

frontend/editor3d/app/pages/PageEnigma/Editor/keybinds_controls.ts

@@ -14,6 +14,13 @@ import { TransformControls } from "./TransformControls";
 import { SceneManager, SceneObject } from "./scene_manager_api";
 import { FreeCam } from "./free_cam";
 import Stats from "three/examples/jsm/libs/stats.module.js";
+import { FKHelper } from "./KinHelpers/FKHelper";
+
+export enum KinMode {
+  FK,
+  IK,
+  NONE,
+}
 
 export class MouseControls {
   camera: THREE.PerspectiveCamera | null;
@@ -47,8 +54,12 @@ export class MouseControls {
   private isMovable: Function;
   enable_stats: Function;
 
+  private kinMode: KinMode = KinMode.NONE;
+  private fkHelper: FKHelper;
+  private isBoneDragged: boolean = false;
+
   constructor(
-    camera: THREE.PerspectiveCamera | null,
+    camera: THREE.PerspectiveCamera,
     camera_person_mode: boolean,
     cameraViewControls: FreeCam,
     lockControls: PointerLockControls | undefined,
@@ -62,7 +73,7 @@ export class MouseControls {
     mouse: THREE.Vector2 | undefined,
     timeline_mouse: THREE.Vector2 | undefined,
     raycaster: THREE.Raycaster | undefined,
-    control: TransformControls | undefined,
+    control: TransformControls,
     outlinePass: OutlinePass | undefined,
     scene: THREE.Scene,
     publishSelect: Function,
@@ -101,6 +112,20 @@ export class MouseControls {
     this.sceneManager = undefined;
     this.isMovable = isMovable;
     this.enable_stats = enable_stats;
+    this.fkHelper = new FKHelper({
+      camera: this.camera,
+      domElement: this.control.domElement,
+      scene: this.scene,
+      onDragChange: this.onFKControlsDragging.bind(this),
+    });
+  }
+
+  onFKControlsDragging(dragging: boolean) {
+    this.isBoneDragged = dragging;
+  }
+
+  clearFKVisuals() {
+    this.fkHelper.clear();
   }
 
   focus() {
@@ -125,9 +150,26 @@ export class MouseControls {
       this.selected = [];
       this.publishSelect();
     }
+    this.hideTransformControls();
+    if (remove_outline) this.outlinePass.selectedObjects = [];
+  }
+
+  hideTransformControls() {
+    if (this.control == undefined) {
+      return;
+    }
+
     this.control.detach();
     this.scene.remove(this.control);
-    if (remove_outline) this.outlinePass.selectedObjects = [];
+  }
+
+  reattachTransformControls() {
+    if (this.control == undefined || this.selected == undefined) {
+      return;
+    }
+
+    this.control.attach(this.selected[0]);
+    this.scene.add(this.control);
   }
 
   selectObject(currentObject: THREE.Object3D) {
@@ -151,7 +193,7 @@ export class MouseControls {
   }
 
   onMouseDown(event: any) {
-    if ((event.button === 0 || event.button === 1) && this.isMovable()) {
+    if ((event.button === 0 || event.button === 1) && this.isMovable() && !this.isBoneDragged) {
       this.isMouseClicked = true;
     }
   }
@@ -176,6 +218,35 @@ export class MouseControls {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
 
+  toggleFKMode() {
+    if (this.kinMode == KinMode.FK) {
+      this.fkHelper.clear();
+      this.kinMode = KinMode.NONE;
+      this.reattachTransformControls();
+      console.log("FK mode off");
+      return;
+    }
+
+    // Make sure we have an intersection
+    if (!(this.selected && this.selected.length > 0)) {
+      return;
+    }
+
+    // Make sure FK is supported only on character type objects
+    const firstSelection = this.selected[0];
+    if (!firstSelection.userData.isCharacter) {
+      return;
+    }
+
+    // FK is good to go
+    // Disable main transform controls
+    this.hideTransformControls();
+    this.kinMode = KinMode.FK;
+    this.fkHelper.setTarget(this.selected[0]);
+    console.log("FK mode on");
+    return;
+  }
+
   async onkeydown(event: KeyboardEvent) {
     if (hotkeysStatus.value.disabled) {
       return;
@@ -198,7 +269,7 @@ export class MouseControls {
       // transform
       this.control?.setMode("translate");
       return;
-    } else if (event.key === "r") {
+    } else if (event.key === "r" && !event.ctrlKey) {
       // rotate
       this.control?.setMode("rotate");
       return;
@@ -206,6 +277,8 @@ export class MouseControls {
       // scale
       this.control?.setMode("scale");
       return;
+    } else if (event.key === "k") {
+      this.toggleFKMode();
     }
 
     if ((event.ctrlKey || event.metaKey) && !this.isProcessing) {
@@ -298,6 +371,11 @@ export class MouseControls {
     }
     this.camera_last_pos.copy(camera_pos);
 
+    if (this.kinMode == KinMode.FK) {
+      this.fkHelper.onMouseClick(this.mouse);
+      return;
+    }
+
     this.raycaster.setFromCamera(this.mouse, this.camera);
     const interactable: any[] = [];
     this.scene.children.forEach((child: THREE.Object3D) => {

frontend/editor3d/app/pages/PageEnigma/Editor/timeline.ts

@@ -287,10 +287,6 @@ export class TimeLine {
         // testDeformBody(lastRig);
       }
 
-      // Add a skeleton helper to the character
-      const skeletonHelper = new THREE.SkeletonHelper(obj);
-      this.scene.scene.add(skeletonHelper);
-
       const object_uuid = obj.uuid;
 
       this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.

frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx

@@ -22,7 +22,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
 import { Button } from "~/components";
 import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
-import { cameraAspectRatio } from "~/pages/PageEnigma/signals";
+import { cameraAspectRatio, disableHotkeyInput, DomLevels, enableHotkeyInput } from "~/pages/PageEnigma/signals";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
 import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
@@ -325,6 +325,14 @@ export const PromptBox = () => {
             onChange={handleChange}
             onPaste={handlePaste}
             onKeyDown={handleKeyDown}
+            onFocus={() => {
+              console.log("disabling hotkeys")
+              disableHotkeyInput(DomLevels.INPUT);
+            }}
+            onBlur={() => {
+              console.log("enabling hotkeys")
+              enableHotkeyInput(DomLevels.INPUT);
+            }}
           />
         </div>
         <div className="mt-2 flex items-center justify-between gap-2">