diff --git a/android/app/src/main/java/com/offlinefaceauth/TFLiteFrameProcessorRunner.java b/android/app/src/main/java/com/offlinefaceauth/TFLiteFrameProcessorRunner.java index 46367a4..8ad37b0 100644 --- a/android/app/src/main/java/com/offlinefaceauth/TFLiteFrameProcessorRunner.java +++ b/android/app/src/main/java/com/offlinefaceauth/TFLiteFrameProcessorRunner.java @@ -127,8 +127,9 @@ private static boolean processBufferInternal( @Nullable Image image) { final long inferenceStartedNs = System.nanoTime(); - final FaceCandidate faceCandidate = - findFaceCandidateFast(image, yBuffer, width, height, stride); + final FaceCandidate faceCandidate = image != null && image.getPlanes().length >= 3 + ? findFaceCandidate(image, yBuffer, width, height, stride) + : findFaceCandidateFast(image, yBuffer, width, height, stride); if (!faceCandidate.faceLike) { final float inferenceMs = (System.nanoTime() - inferenceStartedNs) / 1_000_000.0f; @@ -180,9 +181,9 @@ private static FaceCandidate findFaceCandidateFast( if (!frameHasEnoughTexture(yBuffer, width, height, stride)) { return FaceCandidate.none(); } - // Try single-orientation Android detector with front camera default (270°) + // Try the legacy detector across supported frame orientations. final FaceCandidate detectorCandidate = - detectFaceCandidateAtOrientation(yBuffer, width, height, stride, 270); + detectFaceCandidateWithAndroidDetector(yBuffer, width, height, stride); if (detectorCandidate.faceLike) { return detectorCandidate; } @@ -383,8 +384,10 @@ private static FaceMeshOutput runFaceMesh( } final boolean confidenceAllowsFace = !hasPresenceScore || presenceScore >= FACE_PRESENCE_HARD_REJECT_THRESHOLD; + final boolean geometryAllowsFace = landmarksLookLikeFace(landmarks, width, height); final boolean facePresent = - confidenceAllowsFace && landmarksLookLikeFace(landmarks, width, height); + geometryAllowsFace && + (confidenceAllowsFace || candidate.confidence >= STRONG_FACE_CANDIDATE_CONFIDENCE); return new FaceMeshOutput(landmarks, facePresent, presenceScore); } @@ -1036,7 +1039,7 @@ private static boolean landmarksLookLikeFace(float[] values, int width, int heig eyeRatio(values, 362, 385, 387, 263, 373, 380)); final float mar = mouthRatio(values); return boxToEye >= 1.35f && boxToEye <= 3.60f && - ear >= 0.03f && ear <= 0.60f && + ear >= 0.03f && ear <= 0.85f && mar >= 0.01f && mar <= 1.20f; } diff --git a/cpp/landmarks/LivenessFSM.cpp b/cpp/landmarks/LivenessFSM.cpp index 93e8665..e82d121 100644 --- a/cpp/landmarks/LivenessFSM.cpp +++ b/cpp/landmarks/LivenessFSM.cpp @@ -1,4 +1,5 @@ #include "LivenessFSM.h" +#include #include #include @@ -108,6 +109,7 @@ void LivenessFSM::StartChallenge(LivenessChallenge challenge, smileStartedAt_ = now; blinkWasClosed_ = false; blinkBaselineCaptured_ = false; + blinkMinEarDuringClosure_ = 0.0f; smileBaselineCaptured_ = false; challengeSatisfied_ = false; turnBaselineCaptured_ = false; @@ -126,6 +128,7 @@ void LivenessFSM::Reset(Clock::time_point now) { smileStartedAt_ = now; baselineYaw_ = 0.0f; baselineEar_ = 0.0f; + blinkMinEarDuringClosure_ = 0.0f; baselineMar_ = 0.0f; blinkWasClosed_ = false; blinkBaselineCaptured_ = false; @@ -211,27 +214,52 @@ void LivenessFSM::EvaluateBlink(const LivenessInput& input) { return; } - const float dynamicClosedEar = std::max( - thresholds_.blinkClosedEar, - baselineEar_ * 0.78f); - const float dynamicOpenEar = - std::max(dynamicClosedEar + 0.015f, baselineEar_ * 0.88f); + const bool scaledEar = + baselineEar_ > thresholds_.blinkOpenEar * thresholds_.blinkScaledEarMultiplier; + const bool relativeEar = baselineEar_ < thresholds_.blinkClosedEar; + const float dynamicClosedEar = scaledEar + ? baselineEar_ - std::max(thresholds_.blinkScaledMinCloseDrop, + baselineEar_ * thresholds_.blinkScaledCloseDropRatio) + : relativeEar + ? baselineEar_ - std::max(thresholds_.blinkRelativeMinCloseDrop, + baselineEar_ * thresholds_.blinkRelativeCloseDropRatio) + : std::max(thresholds_.blinkClosedEar, baselineEar_ * 0.78f); + const float dynamicOpenEar = scaledEar + ? baselineEar_ - std::max(thresholds_.blinkScaledMinCloseDrop * 0.5f, + baselineEar_ * thresholds_.blinkScaledOpenDropRatio) + : relativeEar + ? dynamicClosedEar + std::max(thresholds_.blinkRelativeMinRecovery, + baselineEar_ * thresholds_.blinkRelativeRecoveryRatio) + : std::max( + dynamicClosedEar + thresholds_.blinkRecoveryEar, + std::min(thresholds_.blinkOpenEar, baselineEar_ * 0.88f)); + const float recoveryDelta = scaledEar + ? std::max(thresholds_.blinkScaledMinCloseDrop, + baselineEar_ * thresholds_.blinkScaledCloseDropRatio) + : relativeEar + ? std::max(thresholds_.blinkRelativeMinRecovery, + baselineEar_ * thresholds_.blinkRelativeRecoveryRatio) + : thresholds_.blinkRecoveryEar; if (!blinkWasClosed_ && ear <= dynamicClosedEar) { blinkWasClosed_ = true; blinkClosedAt_ = input.timestamp; + blinkMinEarDuringClosure_ = ear; reason_ = "blink closed"; return; } if (blinkWasClosed_) { + blinkMinEarDuringClosure_ = std::min(blinkMinEarDuringClosure_, ear); if (input.timestamp - blinkClosedAt_ > thresholds_.blinkWindow) { blinkWasClosed_ = false; + blinkMinEarDuringClosure_ = 0.0f; reason_ = "blink window expired"; return; } - if (ear >= dynamicOpenEar) { + if (ear >= dynamicOpenEar || + ear - blinkMinEarDuringClosure_ >= recoveryDelta) { Pass("blink challenge passed"); } } @@ -280,7 +308,7 @@ void LivenessFSM::EvaluateTurn(const LivenessInput& input, bool left) { const float delta = input.metrics.yaw - baselineYaw_; const bool passed = left ? delta <= -thresholds_.yawDeltaDegrees : delta >= thresholds_.yawDeltaDegrees; - if (passed && input.timestamp - challengeStartedAt_ <= thresholds_.turnWindow) { + if (passed) { Pass(left ? "turn left challenge passed" : "turn right challenge passed"); } } diff --git a/cpp/landmarks/LivenessFSM.h b/cpp/landmarks/LivenessFSM.h index 6740d1b..998b355 100644 --- a/cpp/landmarks/LivenessFSM.h +++ b/cpp/landmarks/LivenessFSM.h @@ -29,14 +29,23 @@ enum class LivenessChallenge : uint8_t { struct LivenessThresholds { float blinkClosedEar{0.21f}; float blinkOpenEar{0.28f}; + float blinkRecoveryEar{0.04f}; + float blinkScaledEarMultiplier{1.5f}; + float blinkScaledCloseDropRatio{0.01f}; + float blinkScaledOpenDropRatio{0.004f}; + float blinkScaledMinCloseDrop{0.006f}; + float blinkRelativeCloseDropRatio{0.06f}; + float blinkRelativeMinCloseDrop{0.008f}; + float blinkRelativeRecoveryRatio{0.035f}; + float blinkRelativeMinRecovery{0.006f}; std::chrono::milliseconds blinkWindow{800}; float smileMar{0.45f}; std::chrono::milliseconds smileSustain{600}; - float yawDeltaDegrees{8.0f}; + float yawDeltaDegrees{6.0f}; std::chrono::milliseconds turnWindow{2000}; std::chrono::milliseconds challengeTimeout{4000}; std::chrono::milliseconds failResetDelay{2000}; - std::chrono::milliseconds faceDropoutTolerance{300}; + std::chrono::milliseconds faceDropoutTolerance{1000}; }; struct LivenessInput { @@ -99,6 +108,7 @@ class LivenessFSM { std::chrono::steady_clock::time_point smileStartedAt_; float baselineYaw_{0.0f}; float baselineEar_{0.0f}; + float blinkMinEarDuringClosure_{0.0f}; float baselineMar_{0.0f}; bool blinkWasClosed_{false}; bool blinkBaselineCaptured_{false}; diff --git a/package-lock.json b/package-lock.json index 1f71079..e36c627 100644 --- a/package-lock.json +++ b/package-lock.json @@ -122,6 +122,7 @@ "node_modules/@aws-sdk/client-s3": { "version": "3.1060.0", "license": "Apache-2.0", + "peer": true, "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", @@ -517,6 +518,7 @@ "node_modules/@babel/core": { "version": "7.29.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -546,6 +548,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@nicolo-ribaudo/eslint-scope-5-internals": "5.1.1-v1", "eslint-visitor-keys": "^2.1.0", @@ -2105,6 +2108,7 @@ "node_modules/@babel/preset-env": { "version": "7.29.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/compat-data": "^7.29.7", "@babel/helper-compilation-targets": "^7.29.7", @@ -2604,6 +2608,7 @@ "node_modules/@jest/environment": { "version": "29.7.0", "license": "MIT", + "peer": true, "dependencies": { "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", @@ -3914,6 +3919,7 @@ "version": "5.62.0", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -4103,6 +4109,7 @@ "version": "3.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chalk": "^4.1.0", "pngjs": "^7.0.0", @@ -4150,6 +4157,7 @@ "node_modules/acorn": { "version": "8.16.0", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4699,6 +4707,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -4765,6 +4774,7 @@ "node >=0.10.0" ], "license": "MIT", + "peer": true, "bin": { "bunyan": "bin/bunyan" }, @@ -5515,6 +5525,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@wix-pilot/core": "^3.4.2", "@wix-pilot/detox": "^1.0.13", @@ -5991,6 +6002,7 @@ "version": "8.57.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6515,6 +6527,7 @@ "version": "29.7.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/expect-utils": "^29.7.0", "jest-get-type": "^29.6.3", @@ -7923,6 +7936,7 @@ "version": "29.7.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -10023,6 +10037,7 @@ "version": "2.8.8", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -10178,6 +10193,7 @@ "node_modules/react": { "version": "18.2.0", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -10200,6 +10216,7 @@ "node_modules/react-native": { "version": "0.73.6", "license": "MIT", + "peer": true, "dependencies": { "@jest/create-cache-key-function": "^29.6.3", "@react-native-community/cli": "12.3.6", @@ -10285,6 +10302,7 @@ "node_modules/react-native-worklets-core": { "version": "0.4.0", "license": "MIT", + "peer": true, "dependencies": { "string-hash-64": "^1.0.3" }, @@ -11683,6 +11701,7 @@ "version": "5.0.4", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/App.tsx b/src/App.tsx index 6774253..441a9cc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -71,6 +71,20 @@ type NativeBridgeModule = { type NativeChallenge = 'NONE' | 'BLINK' | 'SMILE' | 'TURN_LEFT' | 'TURN_RIGHT'; const MODEL_PATH = '/sdcard/Download/mobilefacenet.tflite'; +const LIVENESS_STATE_NAMES = [ + 'IDLE', + 'DETECTED', + 'CHALLENGE_ACTIVE', + 'LIVENESS_PASS', + 'LIVENESS_FAIL', +] as const; +const LIVENESS_CHALLENGE_NAMES = [ + 'NONE', + 'BLINK', + 'SMILE', + 'TURN_LEFT', + 'TURN_RIGHT', +] as const; const styles = StyleSheet.create({ root: {flex: 1, backgroundColor: '#0f172a'}, @@ -184,6 +198,10 @@ function formatResult(result: OfflineFaceAuthResult): string { const preview = isUsableEmbedding(result) ? embeddingArray.slice(0, 16).map((value) => value.toFixed(6)) : []; + const livenessStateName = + LIVENESS_STATE_NAMES[result.livenessState ?? 0] ?? 'UNKNOWN'; + const livenessChallengeName = + LIVENESS_CHALLENGE_NAMES[result.livenessChallenge ?? 0] ?? 'UNKNOWN'; return JSON.stringify( { accepted: result.accepted, @@ -197,7 +215,9 @@ function formatResult(result: OfflineFaceAuthResult): string { faceMeshThreadCount: result.faceMeshThreadCount, mobileFaceNetThreadCount: result.mobileFaceNetThreadCount, livenessState: result.livenessState, + livenessStateName, livenessChallenge: result.livenessChallenge, + livenessChallengeName, faceDetected: result.faceDetected, ear: result.ear, mar: result.mar, diff --git a/src/components/camera/CameraView.tsx b/src/components/camera/CameraView.tsx index c87788d..4d2e1e1 100644 --- a/src/components/camera/CameraView.tsx +++ b/src/components/camera/CameraView.tsx @@ -1,4 +1,4 @@ -import React, {useEffect, useMemo, useRef, useState} from 'react'; +import React, {useCallback, useEffect, useMemo, useRef, useState} from 'react'; import {StyleSheet, Text, View} from 'react-native'; import { Camera, @@ -161,17 +161,59 @@ export function CameraView({ const telemetry = useNativeLivenessTelemetry(isActive && hasPermission); const previousState = useRef(telemetry.state); const previousChallenge = useRef(telemetry.challenge); + const frameDeliveryDiagnosticLogged = useRef(false); + const [previewInitialized, setPreviewInitialized] = useState(false); const frameProcessor = useFrameProcessor((frame) => { 'worklet'; nayanFrameProcessorPlugin?.call(frame); }, [nayanFrameProcessorPlugin]); + const handleInitialized = useCallback(() => { + setPreviewInitialized(true); + onPreviewReady?.(); + }, [onPreviewReady]); + useEffect(() => { if (!hasPermission) { requestPermission(); } }, [hasPermission, requestPermission]); + useEffect(() => { + if (!isActive || !hasPermission || !previewInitialized) { + frameDeliveryDiagnosticLogged.current = false; + return undefined; + } + if ( + nayanFrameProcessorPlugin == null || + frameDeliveryDiagnosticLogged.current + ) { + return undefined; + } + + frameDeliveryDiagnosticLogged.current = true; + const initialCount = getLatestFrameResult()?.framesProcessed ?? 0; + // eslint-disable-next-line no-console + console.log( + '[DIAG] Camera preview initialized. Initial frames_processed:', + initialCount, + ); + + const timeout = setTimeout(() => { + const count = getLatestFrameResult()?.framesProcessed ?? 0; + // eslint-disable-next-line no-console + console.log('[DIAG] frames_processed after active preview:', count); + if (count === initialCount) { + // eslint-disable-next-line no-console + console.log( + '[DIAG] FrameProcessorPlugin has not produced a processed frame since preview initialization.', + ); + } + }, 2000); + + return () => clearTimeout(timeout); + }, [hasPermission, isActive, nayanFrameProcessorPlugin, previewInitialized]); + useEffect(() => { if (telemetry.challenge !== previousChallenge.current) { if (telemetry.challenge === 'BLINK') { @@ -223,7 +265,7 @@ export function CameraView({ pixelFormat="yuv" isActive={isActive && hasPermission} frameProcessor={frameProcessor} - onInitialized={onPreviewReady} + onInitialized={handleInitialized} /> { - const count = getLatestFrameResult()?.framesProcessed ?? 0; - // eslint-disable-next-line no-console - console.log('[DIAG] frames_processed after 1s:', count); - if (count === initialCount) { - // eslint-disable-next-line no-console - console.error( - '[FATAL] FrameProcessorPlugin not receiving frames. Check worklet registration.', - ); - } - }, 1000); return installed; }