diff --git a/Assets/Scenes/VTuber.unity b/Assets/Scenes/VTuber.unity index 0e03d44..f7b35c8 100644 --- a/Assets/Scenes/VTuber.unity +++ b/Assets/Scenes/VTuber.unity @@ -38,7 +38,7 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.44657874, g: 0.49641275, b: 0.5748172, a: 1} + m_IndirectSpecularColor: {r: 0.44657815, g: 0.49641192, b: 0.57481617, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -992,10 +992,38 @@ PrefabInstance: propertyPath: anim value: objectReference: {fileID: 364035618} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.w + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 468427542, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.y + value: 0.70710677 + objectReference: {fileID: 0} - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} propertyPath: anim value: objectReference: {fileID: 364035618} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.w + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.y + value: 0.70710677 + objectReference: {fileID: 0} + - target: {fileID: 866042038, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} + propertyPath: InitFingerRotation.z + value: 0 + objectReference: {fileID: 0} - target: {fileID: 2091460211759904244, guid: 1639ff504c5034ce2a726caf017d9bcf, type: 3} propertyPath: m_Name value: IKRoot diff --git a/Assets/Scripts/Animator/HandLandmarksController.cs b/Assets/Scripts/Animator/HandLandmarksController.cs index 7409383..5df88a5 100644 --- a/Assets/Scripts/Animator/HandLandmarksController.cs +++ b/Assets/Scripts/Animator/HandLandmarksController.cs @@ -15,6 +15,7 @@ using System.Collections; using System.Collections.Generic; using UnityEngine; +using UnityEngine.Assertions; using Color = UnityEngine.Color; using Mediapipe; @@ -22,6 +23,26 @@ namespace SeedUnityVRKit { public enum HandType { LeftHand, RightHand } + + public class HandLandmarks { + public const int ThumbProximal = 0; + public const int ThumbIntermediate = 1; + public const int ThumbDistal = 2; + public const int IndexProximal = 3; + public const int IndexIntermediate = 4; + public const int IndexDistal = 5; + public const int MiddleProximal = 6; + public const int MiddleIntermediate = 7; + public const int MiddleDistal = 8; + public const int RingProximal = 9; + public const int RingIntermediate = 10; + public const int RingDistal = 11; + public const int LittleProximal = 12; + public const int LittleIntermediate = 13; + public const int LittleDistal = 14; + public const int Total = 15; + }; + public class HandLandmarksController : MonoBehaviour { [Tooltip("Set it to the character's animator game object.")] public Animator anim; @@ -40,6 +61,8 @@ public class HandLandmarksController : MonoBehaviour { private GameObject[] _handLandmarks = new GameObject[_landmarksNum]; private float _screenRatio = 1.0f; + private KalmanFilter[] _kalmanFilters = new KalmanFilter[_landmarksNum]; + private Transform[] _fingerTargets = new Transform[HandLandmarks.Total]; void Start() { // Note: HandPose use camera perspective to determine left and right hand, which is mirrored @@ -51,14 +74,13 @@ void Start() { for (int i = 0; i < _landmarksNum; i++) { _handLandmarks[i] = new GameObject($"HandLandmark{i}"); _handLandmarks[i].transform.parent = transform; + _kalmanFilters[i] = new KalmanFilter(0.125f, 1f); } _screenRatio = 1.0f * ScreenWidth / ScreenHeight; + AssignFingerTargets(); } void Update() { - transform.position = _target.transform.position; - _target.rotation = ComputeWristRotation(); - if (HandLandmarkList != null) { NormalizedLandmark landmark0 = HandLandmarkList.Landmark[0]; NormalizedLandmark landmark1 = HandLandmarkList.Landmark[1]; @@ -67,9 +89,14 @@ void Update() { for (int i = 1; i < HandLandmarkList.Landmark.Count; i++) { NormalizedLandmark landmark = HandLandmarkList.Landmark[i]; Vector3 tip = Vector3.Scale(ToVector(landmark) - ToVector(landmark0), scale); - _handLandmarks[i].transform.localPosition = tip; + _handLandmarks[i].transform.localPosition = _kalmanFilters[i].Update(tip); } } + + transform.position = _target.transform.position; + _target.rotation = ComputeWristRotation(); + + ComputeFingerRotation(); } void OnDrawGizmos() { @@ -77,9 +104,19 @@ void OnDrawGizmos() { Gizmos.color = Color.red; Gizmos.DrawSphere(_target.position, 0.005f); Gizmos.color = Color.yellow; - Vector3 direction = _target.TransformDirection(Vector3.forward) * 1; + Vector3 direction = _target.TransformDirection(Vector3.forward) * 0.1f; + if (handType == HandType.LeftHand) { + direction = -direction; + } Gizmos.DrawRay(_target.position, direction); } + // DrawGizmoFingerJointsSpheres(); + // DrawGizmoFingerSkeleton(); + + DrawGizmoFingerJointsForward(); + } + + private void DrawGizmoFingerJointsSpheres() { Gizmos.color = Color.red; foreach (var handLandmark in _handLandmarks) { if (handLandmark != null) @@ -87,6 +124,30 @@ void OnDrawGizmos() { } } + private void DrawGizmoFingerSkeleton() { + Gizmos.color = Color.white; + Gizmos.DrawLine(_handLandmarks[0].transform.position, _handLandmarks[1].transform.position); + Gizmos.DrawLine(_handLandmarks[1].transform.position, _handLandmarks[2].transform.position); + Gizmos.DrawLine(_handLandmarks[2].transform.position, _handLandmarks[3].transform.position); + Gizmos.DrawLine(_handLandmarks[3].transform.position, _handLandmarks[4].transform.position); + Gizmos.DrawLine(_handLandmarks[0].transform.position, _handLandmarks[5].transform.position); + Gizmos.DrawLine(_handLandmarks[5].transform.position, _handLandmarks[6].transform.position); + Gizmos.DrawLine(_handLandmarks[6].transform.position, _handLandmarks[7].transform.position); + Gizmos.DrawLine(_handLandmarks[7].transform.position, _handLandmarks[8].transform.position); + } + + private void DrawGizmoFingerJointsForward() { + if (_fingerTargets != null) { + for (int i = 9; i < 12; i++) { + var bone = _fingerTargets[i]; + if (bone != null) { + Gizmos.color = Color.blue; + Gizmos.DrawRay(bone.position, bone.TransformDirection(Vector3.forward) * 0.1f); + } + } + } + } + private Vector3 ToVector(NormalizedLandmark landmark) { return new Vector3(landmark.X, landmark.Y, landmark.Z); } @@ -102,5 +163,107 @@ private Quaternion ComputeWristRotation() { Vector3 normalVector = Vector3.Cross(vectorToIndex, vectorToMiddle); return Quaternion.LookRotation(normalVector, vectorToIndex); } + + private void ComputeFingerRotation() { + var rotationTable = new(int fingerId, int landmarkId1, int landmarkId2)[] { + (0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 5, 6), (4, 6, 7), + (5, 7, 8), (6, 9, 10), (7, 10, 11), (8, 11, 12), (9, 13, 14), + (10, 14, 15), (11, 15, 16), (12, 17, 18), (13, 18, 19), (14, 19, 20) + }; + foreach (var (fingerId, landmarkId1, landmarkId2) in rotationTable) { + Vector3 fingerDirection = _handLandmarks[landmarkId2].transform.position - + _handLandmarks[landmarkId1].transform.position; + Vector3 right = + GetNormal(transform.position, _handLandmarks[landmarkId1].transform.position, + _handLandmarks[landmarkId2].transform.position); + Vector3 forward = Vector3.Cross(right, fingerDirection); + _fingerTargets[fingerId].rotation = Quaternion.LookRotation(forward, right); + } + } + + private void AssignFingerTargets() { + // The output of HandPose detection is mirrored here. + if (handType == HandType.LeftHand) { + _fingerTargets[HandLandmarks.ThumbProximal] = + anim.GetBoneTransform(HumanBodyBones.RightThumbProximal); + _fingerTargets[HandLandmarks.ThumbIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightThumbIntermediate); + _fingerTargets[HandLandmarks.ThumbDistal] = + anim.GetBoneTransform(HumanBodyBones.RightThumbDistal); + _fingerTargets[HandLandmarks.IndexProximal] = + anim.GetBoneTransform(HumanBodyBones.RightIndexProximal); + _fingerTargets[HandLandmarks.IndexIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightIndexIntermediate); + _fingerTargets[HandLandmarks.IndexDistal] = + anim.GetBoneTransform(HumanBodyBones.RightIndexDistal); + _fingerTargets[HandLandmarks.MiddleProximal] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleProximal); + _fingerTargets[HandLandmarks.MiddleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleIntermediate); + _fingerTargets[HandLandmarks.MiddleDistal] = + anim.GetBoneTransform(HumanBodyBones.RightMiddleDistal); + _fingerTargets[HandLandmarks.RingProximal] = + anim.GetBoneTransform(HumanBodyBones.RightRingProximal); + _fingerTargets[HandLandmarks.RingIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightRingIntermediate); + _fingerTargets[HandLandmarks.RingDistal] = + anim.GetBoneTransform(HumanBodyBones.RightRingDistal); + _fingerTargets[HandLandmarks.LittleProximal] = + anim.GetBoneTransform(HumanBodyBones.RightLittleProximal); + _fingerTargets[HandLandmarks.LittleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.RightLittleIntermediate); + _fingerTargets[HandLandmarks.LittleDistal] = + anim.GetBoneTransform(HumanBodyBones.RightLittleDistal); + } else { + _fingerTargets[HandLandmarks.ThumbProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbProximal); + _fingerTargets[HandLandmarks.ThumbIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbIntermediate); + _fingerTargets[HandLandmarks.ThumbDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftThumbDistal); + _fingerTargets[HandLandmarks.IndexProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexProximal); + _fingerTargets[HandLandmarks.IndexIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexIntermediate); + _fingerTargets[HandLandmarks.IndexDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftIndexDistal); + _fingerTargets[HandLandmarks.MiddleProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleProximal); + _fingerTargets[HandLandmarks.MiddleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleIntermediate); + _fingerTargets[HandLandmarks.MiddleDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftMiddleDistal); + _fingerTargets[HandLandmarks.RingProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftRingProximal); + _fingerTargets[HandLandmarks.RingIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftRingIntermediate); + _fingerTargets[HandLandmarks.RingDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftRingDistal); + _fingerTargets[HandLandmarks.LittleProximal] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleProximal); + _fingerTargets[HandLandmarks.LittleIntermediate] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleIntermediate); + _fingerTargets[HandLandmarks.LittleDistal] = + anim.GetBoneTransform(HumanBodyBones.LeftLittleDistal); + } + } + + // + // Get the normal to a triangle from the three corner points, a, b and c. + // See https://docs.unity3d.com/ScriptReference/Vector3.Cross.html. + // + // + // An assertion is thrown if the sides from input vectors are parallel. + // + private Vector3 GetNormal(Vector3 a, Vector3 b, Vector3 c) { + // Find vectors corresponding to two of the sides of the triangle. + Vector3 side1 = b - a; + Vector3 side2 = c - a; + + // Cross the vectors to get a perpendicular vector, then normalize it. + Vector3 result = Vector3.Cross(side1, side2).normalized; + Assert.AreNotEqual(Vector3.zero, result, "The sides from input vectors are parallel."); + return result; + } } }