From 24603bc7513920a40170e308e57a6ad73cb933aa Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sat, 22 Oct 2022 15:58:03 -0700 Subject: [PATCH 01/11] Fix the Gizmos ray direction on left hand. --- Assets/Scripts/Animator/HandLandmarksController.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 7409383..19eaff9 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -78,6 +78,9 @@ void OnDrawGizmos() { Gizmos.DrawSphere(_target.position, 0.005f); Gizmos.color = Color.yellow; Vector3 direction = _target.TransformDirection(Vector3.forward) * 1; + if (handType == HandType.LeftHand) { + direction = -direction; + } Gizmos.DrawRay(_target.position, direction); } Gizmos.color = Color.red; From 5cf49e359638bd7beab37e683aea7a90712921b2 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sat, 22 Oct 2022 16:06:34 -0700 Subject: [PATCH 02/11] Add Kalman filter to stablize hand landmarks. --- Assets/Scripts/Animator/HandLandmarksController.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 19eaff9..8753b51 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -40,6 +40,7 @@ public class HandLandmarksController : MonoBehaviour { private GameObject[] _handLandmarks = new GameObject[_landmarksNum]; private float _screenRatio = 1.0f; + private KalmanFilter[] _kalmanFilters = new KalmanFilter[_landmarksNum]; void Start() { // Note: HandPose use camera perspective to determine left and right hand, which is mirrored @@ -51,6 +52,7 @@ void Start() { for (int i = 0; i < _landmarksNum; i++) { _handLandmarks[i] = new GameObject($"HandLandmark{i}"); _handLandmarks[i].transform.parent = transform; + _kalmanFilters[i] = new KalmanFilter(0.125f, 1f); } _screenRatio = 1.0f * ScreenWidth / ScreenHeight; } @@ -67,7 +69,7 @@ void Update() { for (int i = 1; i < HandLandmarkList.Landmark.Count; i++) { NormalizedLandmark landmark = HandLandmarkList.Landmark[i]; Vector3 tip = Vector3.Scale(ToVector(landmark) - ToVector(landmark0), scale); - _handLandmarks[i].transform.localPosition = tip; + _handLandmarks[i].transform.localPosition = _kalmanFilters[i].Update(tip); } } } From 0ceba3c16574560136ce2a6c7b551421ee264c80 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sat, 22 Oct 2022 16:37:24 -0700 Subject: [PATCH 03/11] Stop following the hands if flipped backwards. --- .../Animator/HandLandmarksController.cs | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 8753b51..21acac4 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -41,6 +41,10 @@ public class HandLandmarksController : MonoBehaviour { private GameObject[] _handLandmarks = new GameObject[_landmarksNum]; private float _screenRatio = 1.0f; private KalmanFilter[] _kalmanFilters = new KalmanFilter[_landmarksNum]; + // The forward direction of the wrist. Computed by OrthoNormalize of the positions of index + // finger, middle finger and wrist. + private Vector3 _forwardVector; + private Quaternion _wristRotation = Quaternion.Euler(0, 0, 0); void Start() { // Note: HandPose use camera perspective to determine left and right hand, which is mirrored @@ -58,9 +62,6 @@ void Start() { } void Update() { - transform.position = _target.transform.position; - _target.rotation = ComputeWristRotation(); - if (HandLandmarkList != null) { NormalizedLandmark landmark0 = HandLandmarkList.Landmark[0]; NormalizedLandmark landmark1 = HandLandmarkList.Landmark[1]; @@ -72,6 +73,13 @@ void Update() { _handLandmarks[i].transform.localPosition = _kalmanFilters[i].Update(tip); } } + + ComputeWristRotation(); + + if (AreHandsFacingForward()) { + transform.position = _target.transform.position; + _target.rotation = _wristRotation; + } } void OnDrawGizmos() { @@ -96,7 +104,7 @@ private Vector3 ToVector(NormalizedLandmark landmark) { return new Vector3(landmark.X, landmark.Y, landmark.Z); } - private Quaternion ComputeWristRotation() { + private void ComputeWristRotation() { var wristTransform = transform; var indexFinger = _handLandmarks[5].transform.position; var middleFinger = _handLandmarks[9].transform.position; @@ -104,8 +112,16 @@ private Quaternion ComputeWristRotation() { var vectorToMiddle = middleFinger - wristTransform.position; var vectorToIndex = indexFinger - wristTransform.position; Vector3.OrthoNormalize(ref vectorToMiddle, ref vectorToIndex); - Vector3 normalVector = Vector3.Cross(vectorToIndex, vectorToMiddle); - return Quaternion.LookRotation(normalVector, vectorToIndex); + _forwardVector = Vector3.Cross(vectorToIndex, vectorToMiddle); + _wristRotation = Quaternion.LookRotation(_forwardVector, vectorToIndex); + } + + private bool AreHandsFacingForward() { + // TODO: normalizing before angle computation can be removed? + return (handType == HandType.LeftHand && + Vector3.Angle(_forwardVector.normalized, Vector3.forward) < 90.0f || + handType == HandType.RightHand && + Vector3.Angle(_forwardVector.normalized, Vector3.forward) > 90.0f); } } } From 3e270e13e570f573ab5559be1e538e3fe94bdd70 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sat, 22 Oct 2022 16:37:24 -0700 Subject: [PATCH 04/11] Stop following the hands if flipped backwards. --- Assets/Scripts/Animator/HandLandmarksController.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 21acac4..5d8abb3 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -118,10 +118,9 @@ private void ComputeWristRotation() { private bool AreHandsFacingForward() { // TODO: normalizing before angle computation can be removed? - return (handType == HandType.LeftHand && - Vector3.Angle(_forwardVector.normalized, Vector3.forward) < 90.0f || - handType == HandType.RightHand && - Vector3.Angle(_forwardVector.normalized, Vector3.forward) > 90.0f); + var wristAngle = Vector3.Angle(_forwardVector.normalized, Vector3.forward); + return (handType == HandType.LeftHand && wristAngle < 90.0f || + handType == HandType.RightHand && wristAngle > 90.0f); } } } From a22e6e8a2d7dd8b71411178f541a8022a9dff173 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sun, 23 Oct 2022 15:20:52 -0700 Subject: [PATCH 05/11] Finger animation --- Assets/Scenes/VTuber.unity | 30 +++- .../Animator/HandLandmarksController.cs | 146 ++++++++++++++++-- 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/Assets/Scenes/VTuber.unity b/Assets/Scenes/VTuber.unity index 0e03d44..f7b35c8 100644 --- a/Assets/Scenes/VTuber.unity +++ b/Assets/Scenes/VTuber.unity @@ -38,7 +38,7 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.44657874, g: 0.49641275, b: 0.5748172, a: 1} + m_IndirectSpecularColor: {r: 0.44657815, g: 0.49641192, b: 0.57481617, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -992,10 +992,38 @@ PrefabInstance: propertyPath: anim value: objectReference: {fileID: 364035618} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.w + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.y + value: 0.70710677 + objectReference: {fileID: 0} - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} propertyPath: anim value: objectReference: {fileID: 364035618} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.w + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.y + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.z + value: 0 + objectReference: {fileID: 0} - target: {fileID: 2091460211759904244, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} propertyPath: m_Name value: IKRoot diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 5d8abb3..b57c49a 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -15,6 +15,7 @@ using System.Collections; using System.Collections.Generic; using UnityEngine; +using UnityEngine.Assertions; using Color = UnityEngine.Color; using Mediapipe; @@ -22,6 +23,26 @@ namespace SeedUnityVRKit { public enum HandType { LeftHand, RightHand } + + public class HandLandmarks { + public const int ThumbProximal = 0; + public const int ThumbIntermediate = 1; + public const int ThumbDistal = 2; + public const int IndexProximal = 3; + public const int IndexIntermediate = 4; + public const int IndexDistal = 5; + public const int MiddleProximal = 6; + public const int MiddleIntermediate = 7; + public const int MiddleDistal = 8; + public const int RingProximal = 9; + public const int RingIntermediate = 10; + public const int RingDistal = 11; + public const int LittleProximal = 12; + public const int LittleIntermediate = 13; + public const int LittleDistal = 14; + public const int Total = 15; + }; + public class HandLandmarksController : MonoBehaviour { [Tooltip("Set it to the character's animator game object.")] public Animator anim; @@ -45,6 +66,8 @@ public class HandLandmarksController : MonoBehaviour { // finger, middle finger and wrist. private Vector3 _forwardVector; private Quaternion _wristRotation = Quaternion.Euler(0, 0, 0); + private Transform[] _fingerTargets = new Transform[HandLandmarks.Total]; + public Quaternion InitFingerRotation = Quaternion.Euler(0, 0, 0); void Start() { // Note: HandPose use camera perspective to determine left and right hand, which is mirrored @@ -59,6 +82,7 @@ void Start() { _kalmanFilters[i] = new KalmanFilter(0.125f, 1f); } _screenRatio = 1.0f * ScreenWidth / ScreenHeight; + assignFingerTargets(); } void Update() { @@ -75,11 +99,10 @@ void Update() { } ComputeWristRotation(); + transform.position = _target.transform.position; + _target.rotation = _wristRotation; - if (AreHandsFacingForward()) { - transform.position = _target.transform.position; - _target.rotation = _wristRotation; - } + ComputeFingerRotation(); } void OnDrawGizmos() { @@ -98,6 +121,14 @@ void OnDrawGizmos() { if (handLandmark != null) Gizmos.DrawSphere(handLandmark.transform.position, 0.005f); } + + var bone = _fingerTargets[HandLandmarks.RingIntermediate]; + if (bone != null) { + Gizmos.color = Color.blue; + Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.forward)); + Gizmos.color = Color.red; + Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.up)); + } } private Vector3 ToVector(NormalizedLandmark landmark) { @@ -116,11 +147,108 @@ private void ComputeWristRotation() { _wristRotation = Quaternion.LookRotation(_forwardVector, vectorToIndex); } - private bool AreHandsFacingForward() { - // TODO: normalizing before angle computation can be removed? - var wristAngle = Vector3.Angle(_forwardVector.normalized, Vector3.forward); - return (handType == HandType.LeftHand && wristAngle < 90.0f || - handType == HandType.RightHand && wristAngle > 90.0f); + private void ComputeFingerRotation() { + var rotationTable = new(int fingerId, int landmarkId1, int landmarkId2)[] { + (0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 5, 6), (4, 6, 7), + (5, 7, 8), (6, 9, 10), (7, 10, 11), (8, 11, 12), (9, 13, 14), + (10, 14, 15), (11, 15, 16), (12, 17, 18), (13, 18, 19), (14, 19, 20) + }; + var wristTransform = transform; + var middleFinger = _handLandmarks[9].transform.position; + var vectorToMiddle = middleFinger - wristTransform.position; + foreach (var (fingerId, landmarkId1, landmarkId2) in rotationTable) { + var lookAt = _handLandmarks[landmarkId2].transform.position - + _handLandmarks[landmarkId1].transform.position; + var upwards = GetNormal(transform.position, _handLandmarks[landmarkId1].transform.position, + _handLandmarks[landmarkId2].transform.position); + _fingerTargets[fingerId].rotation = + Quaternion.LookRotation(lookAt, upwards) * InitFingerRotation; + } + } + + private void assignFingerTargets() { + // The output of HandPose detection is mirrored here. + if (handType == HandType.LeftHand) { + _fingerTargets[HandLandmarks.ThumbProximal] = + anim.GetBoneTransform(HumanBodyBones.RightThumbProximal); + _fingerTargets[HandLandmarks.ThumbIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightThumbIntermediate); + _fingerTargets[HandLandmarks.ThumbDistal] = + anim.GetBoneTransform(HumanBodyBones.RightThumbDistal); + _fingerTargets[HandLandmarks.IndexProximal] = + anim.GetBoneTransform(HumanBodyBones.RightIndexProximal); + _fingerTargets[HandLandmarks.IndexIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightIndexIntermediate); + _fingerTargets[HandLandmarks.IndexDistal] = + anim.GetBoneTransform(HumanBodyBones.RightIndexDistal); + _fingerTargets[HandLandmarks.MiddleProximal] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleProximal); + _fingerTargets[HandLandmarks.MiddleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleIntermediate); + _fingerTargets[HandLandmarks.MiddleDistal] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleDistal); + _fingerTargets[HandLandmarks.RingProximal] = + anim.GetBoneTransform(HumanBodyBones.RightRingProximal); + _fingerTargets[HandLandmarks.RingIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightRingIntermediate); + _fingerTargets[HandLandmarks.RingDistal] = + anim.GetBoneTransform(HumanBodyBones.RightRingDistal); + _fingerTargets[HandLandmarks.LittleProximal] = + anim.GetBoneTransform(HumanBodyBones.RightLittleProximal); + _fingerTargets[HandLandmarks.LittleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightLittleIntermediate); + _fingerTargets[HandLandmarks.LittleDistal] = + anim.GetBoneTransform(HumanBodyBones.RightLittleDistal); + } else { + _fingerTargets[HandLandmarks.ThumbProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbProximal); + _fingerTargets[HandLandmarks.ThumbIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbIntermediate); + _fingerTargets[HandLandmarks.ThumbDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbDistal); + _fingerTargets[HandLandmarks.IndexProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexProximal); + _fingerTargets[HandLandmarks.IndexIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexIntermediate); + _fingerTargets[HandLandmarks.IndexDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexDistal); + _fingerTargets[HandLandmarks.MiddleProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleProximal); + _fingerTargets[HandLandmarks.MiddleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleIntermediate); + _fingerTargets[HandLandmarks.MiddleDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleDistal); + _fingerTargets[HandLandmarks.RingProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftRingProximal); + _fingerTargets[HandLandmarks.RingIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftRingIntermediate); + _fingerTargets[HandLandmarks.RingDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftRingDistal); + _fingerTargets[HandLandmarks.LittleProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleProximal); + _fingerTargets[HandLandmarks.LittleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleIntermediate); + _fingerTargets[HandLandmarks.LittleDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleDistal); + } + } + + // + // Get the normal to a triangle from the three corner points, a, b and c. + // See https://docs.unity3d.com/ScriptReference/Vector3.Cross.html. + // + // + // An assertion is thrown if the sides from input vectors are parallel. + // + private Vector3 GetNormal(Vector3 a, Vector3 b, Vector3 c) { + // Find vectors corresponding to two of the sides of the triangle. + Vector3 side1 = b - a; + Vector3 side2 = c - a; + + // Cross the vectors to get a perpendicular vector, then normalize it. + Vector3 result = Vector3.Cross(side1, side2).normalized; + Assert.AreNotEqual(Vector3.zero, result, "The sides from input vectors are parallel."); + return result; } } } From 186e015d3ce81cea315a376f7ca5031fcd51b01e Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sun, 23 Oct 2022 23:19:54 -0700 Subject: [PATCH 06/11] Remove unused computation --- Assets/Scripts/Animator/HandLandmarksController.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index b57c49a..af421c0 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -153,9 +153,6 @@ private void ComputeFingerRotation() { (5, 7, 8), (6, 9, 10), (7, 10, 11), (8, 11, 12), (9, 13, 14), (10, 14, 15), (11, 15, 16), (12, 17, 18), (13, 18, 19), (14, 19, 20) }; - var wristTransform = transform; - var middleFinger = _handLandmarks[9].transform.position; - var vectorToMiddle = middleFinger - wristTransform.position; foreach (var (fingerId, landmarkId1, landmarkId2) in rotationTable) { var lookAt = _handLandmarks[landmarkId2].transform.position - _handLandmarks[landmarkId1].transform.position; From 02d0f54208917c05d9443a67fce36d4586fb33a6 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Mon, 31 Oct 2022 00:10:36 -0700 Subject: [PATCH 07/11] Fix finger rotation computation. --- Assets/Scripts/Animator/HandLandmarksController.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index af421c0..af62f20 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -67,7 +67,6 @@ public class HandLandmarksController : MonoBehaviour { private Vector3 _forwardVector; private Quaternion _wristRotation = Quaternion.Euler(0, 0, 0); private Transform[] _fingerTargets = new Transform[HandLandmarks.Total]; - public Quaternion InitFingerRotation = Quaternion.Euler(0, 0, 0); void Start() { // Note: HandPose use camera perspective to determine left and right hand, which is mirrored @@ -154,12 +153,13 @@ private void ComputeFingerRotation() { (10, 14, 15), (11, 15, 16), (12, 17, 18), (13, 18, 19), (14, 19, 20) }; foreach (var (fingerId, landmarkId1, landmarkId2) in rotationTable) { - var lookAt = _handLandmarks[landmarkId2].transform.position - - _handLandmarks[landmarkId1].transform.position; - var upwards = GetNormal(transform.position, _handLandmarks[landmarkId1].transform.position, - _handLandmarks[landmarkId2].transform.position); - _fingerTargets[fingerId].rotation = - Quaternion.LookRotation(lookAt, upwards) * InitFingerRotation; + Vector3 fingerDirection = _handLandmarks[landmarkId2].transform.position - + _handLandmarks[landmarkId1].transform.position; + Vector3 right = + GetNormal(transform.position, _handLandmarks[landmarkId1].transform.position, + _handLandmarks[landmarkId2].transform.position); + Vector3 forward = Vector3.Cross(right, fingerDirection); + _fingerTargets[fingerId].rotation = Quaternion.LookRotation(forward, right); } } From 4aeec23edd8c2523a8efff8099b458f46d83625e Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Sun, 23 Oct 2022 23:08:11 -0700 Subject: [PATCH 08/11] Update some gizmo --- .../Animator/HandLandmarksController.cs | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index af62f20..09abe09 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -109,24 +109,47 @@ void OnDrawGizmos() { Gizmos.color = Color.red; Gizmos.DrawSphere(_target.position, 0.005f); Gizmos.color = Color.yellow; - Vector3 direction = _target.TransformDirection(Vector3.forward) * 1; + Vector3 direction = _target.TransformDirection(Vector3.forward) * 0.1f; if (handType == HandType.LeftHand) { direction = -direction; } Gizmos.DrawRay(_target.position, direction); } + // drawGizmoFingerJointsSpheres(); + // drawGizmoFingerSkeleton(); + + drawGizmoFingerJointsForward(); + } + + private void drawGizmoFingerJointsSpheres() { Gizmos.color = Color.red; foreach (var handLandmark in _handLandmarks) { if (handLandmark != null) Gizmos.DrawSphere(handLandmark.transform.position, 0.005f); } + } - var bone = _fingerTargets[HandLandmarks.RingIntermediate]; - if (bone != null) { - Gizmos.color = Color.blue; - Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.forward)); - Gizmos.color = Color.red; - Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.up)); + private void drawGizmoFingerSkeleton() { + Gizmos.color = Color.white; + Gizmos.DrawLine(_handLandmarks[0].transform.position, _handLandmarks[1].transform.position); + Gizmos.DrawLine(_handLandmarks[1].transform.position, _handLandmarks[2].transform.position); + Gizmos.DrawLine(_handLandmarks[2].transform.position, _handLandmarks[3].transform.position); + Gizmos.DrawLine(_handLandmarks[3].transform.position, _handLandmarks[4].transform.position); + Gizmos.DrawLine(_handLandmarks[0].transform.position, _handLandmarks[5].transform.position); + Gizmos.DrawLine(_handLandmarks[5].transform.position, _handLandmarks[6].transform.position); + Gizmos.DrawLine(_handLandmarks[6].transform.position, _handLandmarks[7].transform.position); + Gizmos.DrawLine(_handLandmarks[7].transform.position, _handLandmarks[8].transform.position); + } + + private void drawGizmoFingerJointsForward() { + if (_fingerTargets != null) { + for (int i = 9; i < 12; i++) { + var bone = _fingerTargets[i]; + if (bone != null) { + Gizmos.color = Color.blue; + Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.forward) * 0.1f); + } + } } } From b4c512ea6d07914c98a366ca02efcc8b95f56297 Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Mon, 31 Oct 2022 00:26:15 -0700 Subject: [PATCH 09/11] Fix case --- Assets/Scripts/Animator/HandLandmarksController.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 09abe09..466902c 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -115,13 +115,13 @@ void OnDrawGizmos() { } Gizmos.DrawRay(_target.position, direction); } - // drawGizmoFingerJointsSpheres(); - // drawGizmoFingerSkeleton(); + // DrawGizmoFingerJointsSpheres(); + // DrawGizmoFingerSkeleton(); - drawGizmoFingerJointsForward(); + DrawGizmoFingerJointsForward(); } - private void drawGizmoFingerJointsSpheres() { + private void DrawGizmoFingerJointsSpheres() { Gizmos.color = Color.red; foreach (var handLandmark in _handLandmarks) { if (handLandmark != null) @@ -129,7 +129,7 @@ private void drawGizmoFingerJointsSpheres() { } } - private void drawGizmoFingerSkeleton() { + private void DrawGizmoFingerSkeleton() { Gizmos.color = Color.white; Gizmos.DrawLine(_handLandmarks[0].transform.position, _handLandmarks[1].transform.position); Gizmos.DrawLine(_handLandmarks[1].transform.position, _handLandmarks[2].transform.position); @@ -141,7 +141,7 @@ private void drawGizmoFingerSkeleton() { Gizmos.DrawLine(_handLandmarks[7].transform.position, _handLandmarks[8].transform.position); } - private void drawGizmoFingerJointsForward() { + private void DrawGizmoFingerJointsForward() { if (_fingerTargets != null) { for (int i = 9; i < 12; i++) { var bone = _fingerTargets[i]; From cc6767aece94a458b5d417df663cbc9c85ec918b Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Mon, 31 Oct 2022 00:33:35 -0700 Subject: [PATCH 10/11] Revert previous changes to ComputeWristRotation. --- Assets/Scripts/Animator/HandLandmarksController.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 466902c..fc766af 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -62,9 +62,6 @@ public class HandLandmarksController : MonoBehaviour { private GameObject[] _handLandmarks = new GameObject[_landmarksNum]; private float _screenRatio = 1.0f; private KalmanFilter[] _kalmanFilters = new KalmanFilter[_landmarksNum]; - // The forward direction of the wrist. Computed by OrthoNormalize of the positions of index - // finger, middle finger and wrist. - private Vector3 _forwardVector; private Quaternion _wristRotation = Quaternion.Euler(0, 0, 0); private Transform[] _fingerTargets = new Transform[HandLandmarks.Total]; @@ -81,7 +78,7 @@ void Start() { _kalmanFilters[i] = new KalmanFilter(0.125f, 1f); } _screenRatio = 1.0f * ScreenWidth / ScreenHeight; - assignFingerTargets(); + AssignFingerTargets(); } void Update() { @@ -165,8 +162,8 @@ private void ComputeWristRotation() { var vectorToMiddle = middleFinger - wristTransform.position; var vectorToIndex = indexFinger - wristTransform.position; Vector3.OrthoNormalize(ref vectorToMiddle, ref vectorToIndex); - _forwardVector = Vector3.Cross(vectorToIndex, vectorToMiddle); - _wristRotation = Quaternion.LookRotation(_forwardVector, vectorToIndex); + Vector3 normalVector = Vector3.Cross(vectorToIndex, vectorToMiddle); + _wristRotation = Quaternion.LookRotation(normalVector, vectorToIndex); } private void ComputeFingerRotation() { @@ -186,7 +183,7 @@ private void ComputeFingerRotation() { } } - private void assignFingerTargets() { + private void AssignFingerTargets() { // The output of HandPose detection is mirrored here. if (handType == HandType.LeftHand) { _fingerTargets[HandLandmarks.ThumbProximal] = From e20e0670621358fb9c68272d8f9076d9b25c1c4d Mon Sep 17 00:00:00 2001 From: Henry Ou Date: Mon, 31 Oct 2022 00:33:35 -0700 Subject: [PATCH 11/11] Revert previous changes to ComputeWristRotation. --- Assets/Scripts/Animator/HandLandmarksController.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index fc766af..5df88a5 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -62,7 +62,6 @@ public class HandLandmarksController : MonoBehaviour { private GameObject[] _handLandmarks = new GameObject[_landmarksNum]; private float _screenRatio = 1.0f; private KalmanFilter[] _kalmanFilters = new KalmanFilter[_landmarksNum]; - private Quaternion _wristRotation = Quaternion.Euler(0, 0, 0); private Transform[] _fingerTargets = new Transform[HandLandmarks.Total]; void Start() { @@ -94,9 +93,8 @@ void Update() { } } - ComputeWristRotation(); transform.position = _target.transform.position; - _target.rotation = _wristRotation; + _target.rotation = ComputeWristRotation(); ComputeFingerRotation(); } @@ -154,7 +152,7 @@ private Vector3 ToVector(NormalizedLandmark landmark) { return new Vector3(landmark.X, landmark.Y, landmark.Z); } - private void ComputeWristRotation() { + private Quaternion ComputeWristRotation() { var wristTransform = transform; var indexFinger = _handLandmarks[5].transform.position; var middleFinger = _handLandmarks[9].transform.position; @@ -163,7 +161,7 @@ private void ComputeWristRotation() { var vectorToIndex = indexFinger - wristTransform.position; Vector3.OrthoNormalize(ref vectorToMiddle, ref vectorToIndex); Vector3 normalVector = Vector3.Cross(vectorToIndex, vectorToMiddle); - _wristRotation = Quaternion.LookRotation(normalVector, vectorToIndex); + return Quaternion.LookRotation(normalVector, vectorToIndex); } private void ComputeFingerRotation() {