diff --git a/CLAUDE.md b/CLAUDE.md index b2f8c46..adb2248 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -202,6 +202,38 @@ unchanged and is the default + fallback. See `docs/plans/CLIENT_SIDE_ML_PLAN.md` data sent is a derived, non-invertible 512-d embedding, over TLS, stored encrypted (Fernet) — data minimization, NOT "biometric data never leaves the device". +## Client-side VOICE embedding (GPU-less, audit H3, flag-gated `VITE_CLIENT_SIDE_VOICE_EMBEDDING`, default OFF) + +VOICE twin of the face client-embedding path. When ON, the browser computes the +**256-d Resemblyzer speaker embedding** locally and uploads **only the vector** — +the raw audio never leaves the device. Flag OFF (default) = byte-identical legacy +audio upload (`{ voiceData }`). + +- **Module:** `src/features/biometrics/voice-embedding/` — `clientVoiceEmbeddingFlag.ts` + (mirrors `clientEmbeddingFlag.ts`), `voicePreprocess.ts` (WAV decode → dBFS + normalize → mel partials), `speakerEmbedder.ts` (`SpeakerEmbedder` ONNX class, + lazy `import('onnxruntime-web')`, WASM EP, reuses `embedding/modelCache.getModel`), + `embedCapturedVoice.ts` (orchestrator, returns `number[256] | null`, never throws), + `prefetchVoiceModel.ts` (mirrors `prefetchFacenetModel`, wired into `VoiceStep` mount). +- **Integration:** `MfaStepRenderer.tsx` VOICE case — when the flag is ON, compute the + vector and `verifyStep(VOICE, { embedding })`; **on ANY failure FALL BACK to + `{ voiceData }`** (unlike FACE which hard-fails — the voice preprocessing is scaffold + + the server accepts either when its flag is ON, so a gap must never block login). +- **Model delivery:** the FP32 `resemblyzer-.onnx` (~5.7 MB) must be hosted at + `app.fivucsas.com/models/` and fetched at RUNTIME only when ON. Do **NOT** add it to + `public/models/manifest.json` (a build-prefetch entry FATALs `fetch-models`/deploy — + the exact facenet failure). INT8 rejected (onnxruntime-web WASM lacks quant ops); FP32 ships. +- **⚠️ SCAFFOLD — NOT parity-validated.** The ONNX export + inference are EXACT + (torch↔ONNX cosine 1.0), but the JS preprocessing does NOT reproduce Resemblyzer's + WebRTC-VAD silence trim (no bit-exact JS port) and uses a librosa-APPROXIMATE mel. + Skipping the VAD shifts the embedding ≈0.11 cosine vs server — risky near the 0.65 + threshold. **Keep this flag OFF** until the client mel+VAD are validated to parity. + Full contract + canary steps: biometric-processor `docs/design/VOICE_CLIENT_EMBEDDING_SPEC.md`. +- **Ordering caveat:** flip the Identity Core `app.auth.client-side-voice-embedding` flag + **before** `VITE_CLIENT_SIDE_VOICE_EMBEDDING` (web-ON + identity-OFF would send an + embedding the server-OFF path rejects — though the renderer's fallback to `{ voiceData }` + softens this). + ## Puzzle as a first-class auth-flow layer (feature-flagged `app.auth.puzzle-layer`, default OFF) The Biometric Puzzle is composable in the `AuthFlowBuilder` as its own LAYER (allowed diff --git a/src/features/auth/components/steps/VoiceStep.tsx b/src/features/auth/components/steps/VoiceStep.tsx index 2f2d0a8..e124115 100644 --- a/src/features/auth/components/steps/VoiceStep.tsx +++ b/src/features/auth/components/steps/VoiceStep.tsx @@ -17,6 +17,7 @@ import { import { motion } from 'framer-motion' import { useTranslation } from 'react-i18next' import { useVoiceRecorder } from '@/lib/biometric-engine/hooks/useVoiceRecorder' +import { scheduleVoiceModelPrefetch } from '@features/biometrics/voice-embedding/prefetchVoiceModel' import StepLayout from './StepLayout' import { stepItemVariants as itemVariants } from './stepMotion' @@ -70,6 +71,12 @@ export default function VoiceStep({ onSubmit, loading, error }: VoiceStepProps) const [displayDuration, setDisplayDuration] = useState(0) const autoStopTriggered = useRef(false) + // GPU-less VOICE (audit H3): warm the speaker-encoder model on mount so the + // in-browser embedding is ready by submit time. No-op unless the + // VITE_CLIENT_SIDE_VOICE_EMBEDDING flag is ON (mirrors FaceCaptureStep's + // scheduleFacenetPrefetch). Cleanup cancels a still-pending idle schedule. + useEffect(() => scheduleVoiceModelPrefetch(), []) + // Auto-stop after MAX_RECORDING_SECONDS. useEffect(() => { if (!isRecording) return diff --git a/src/features/auth/login-shared/MfaStepRenderer.tsx b/src/features/auth/login-shared/MfaStepRenderer.tsx index 12cec6e..cd89351 100644 --- a/src/features/auth/login-shared/MfaStepRenderer.tsx +++ b/src/features/auth/login-shared/MfaStepRenderer.tsx @@ -43,6 +43,8 @@ import PasswordStep from '../components/steps/PasswordStep' import PuzzleStep from './steps/PuzzleStep' import { isClientSideEmbeddingEnabled } from '@features/biometrics/embedding/clientEmbeddingFlag' import { embedCapturedFace } from '@features/biometrics/embedding/embedCapturedFace' +import { isClientSideVoiceEmbeddingEnabled } from '@features/biometrics/voice-embedding/clientVoiceEmbeddingFlag' +import { embedCapturedVoice } from '@features/biometrics/voice-embedding/embedCapturedVoice' import type { PuzzleConfig } from '@domain/models/AuthMethod' /** @@ -258,6 +260,25 @@ export default function MfaStepRenderer({ } catch (vadErr) { console.warn('[MfaStepRenderer] VAD check failed, proceeding:', vadErr) } + + // GPU-less VOICE (audit H3): when the client-side-voice- + // embedding flag is ON, compute the 256-d speaker vector + // in-browser and submit ONLY the vector (the raw audio + // never leaves the device). On ANY failure we fall back to + // uploading the audio so a scaffold/model-hosting gap never + // blocks the login (the server with the voice flag OFF + // would reject an embedding anyway; with it ON it accepts + // either). Flag OFF ⇒ byte-identical legacy audio upload. + if (isClientSideVoiceEmbeddingEnabled()) { + const embedding = await embedCapturedVoice(voiceData) + if (embedding) { + verifyStep(AuthMethodType.VOICE, { embedding }) + return + } + console.warn( + '[MfaStepRenderer] client voice embedding unavailable; falling back to audio upload', + ) + } verifyStep(AuthMethodType.VOICE, { voiceData }) }} loading={loading} diff --git a/src/features/auth/login-shared/__tests__/MfaStepRendererVoiceEmbedding.test.tsx b/src/features/auth/login-shared/__tests__/MfaStepRendererVoiceEmbedding.test.tsx new file mode 100644 index 0000000..a27becc --- /dev/null +++ b/src/features/auth/login-shared/__tests__/MfaStepRendererVoiceEmbedding.test.tsx @@ -0,0 +1,138 @@ +/** + * MfaStepRenderer — VOICE submit path: client-side embedding vs legacy audio. + * + * Audit H3 (GPU-less voice). Behind the `client-side-voice-embedding` flag + * (mirrors the server flag `app.auth.client-side-voice-embedding`), the VOICE + * submit uploads the 256-d speaker embedding vector instead of the recorded + * audio — the raw audio then never leaves the device. + * + * Contract verified here (deterministic — embedder + flag are mocked, no ONNX): + * - Flag OFF → verifyStep(VOICE, { voiceData }) exactly as the legacy path did. + * - Flag ON → verifyStep(VOICE, { embedding: number[256] }) and NOT { voiceData }. + * - Flag ON + embedding fails → FALLS BACK to { voiceData } (the preprocessing + * is scaffold + the server accepts either when ON, so a model/preproc gap must + * never block the login — unlike FACE which hard-fails). + * + * The VoiceStep internals (mic, recorder) are mocked to a single button that + * hands a fixed captured WAV data-URL up to onSubmit, so this suite asserts ONLY + * the renderer's submit-path routing. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import '../../../../i18n' +import { AuthMethodType } from '@features/auth/constants' + +// Inert biometric engine (the VOICE VAD path reads it; keep the VAD absent so the +// gate is skipped and we test ONLY the embedding routing). +vi.mock('@/lib/biometric-engine/core/BiometricEngine', () => ({ + BiometricEngine: { + getInstance: () => ({ + initialize: () => Promise.resolve(), + voiceVAD: null, + }), + }, +})) + +const CAPTURED_VOICE = 'data:audio/wav;base64,AAAA' + +// Mock VoiceStep down to a single Submit button so we drive ONLY the renderer's +// onSubmit routing — no mic / MediaRecorder in jsdom. +vi.mock('@features/auth/components/steps/VoiceStep', () => ({ + __esModule: true, + default: ({ onSubmit }: { onSubmit: (voiceData: string) => void }) => ( + + ), +})) + +// Mock the client-side voice embedding flag + embed function so the test is +// deterministic and the flag value is injectable. +const flagState = { enabled: false } +const fakeEmbedding = Array.from({ length: 256 }, (_, i) => (i % 7) / 7) +const embedCapturedVoiceMock = vi.fn(async () => fakeEmbedding as number[] | null) + +vi.mock('@features/biometrics/voice-embedding/clientVoiceEmbeddingFlag', () => ({ + isClientSideVoiceEmbeddingEnabled: () => flagState.enabled, +})) +vi.mock('@features/biometrics/voice-embedding/embedCapturedVoice', () => ({ + embedCapturedVoice: (...args: unknown[]) => embedCapturedVoiceMock(...(args as [])), +})) + +import MfaStepRenderer from '../MfaStepRenderer' + +function renderVoiceStep() { + const verifyStep = vi.fn() + const onError = vi.fn() + render( + [0]['httpClient']} + onAuthenticated={vi.fn()} + onBack={vi.fn()} + loading={false} + onError={onError} + />, + ) + return { verifyStep, onError } +} + +describe('MfaStepRenderer — VOICE submit: client embedding vs legacy audio', () => { + beforeEach(() => { + flagState.enabled = false + embedCapturedVoiceMock.mockClear() + embedCapturedVoiceMock.mockResolvedValue(fakeEmbedding) + }) + + it('flag OFF (default): uploads the captured { voiceData } unchanged, never embeds', async () => { + flagState.enabled = false + const { verifyStep } = renderVoiceStep() + + await userEvent.click(screen.getByRole('button', { name: 'voice-submit' })) + + await waitFor(() => expect(verifyStep).toHaveBeenCalledTimes(1)) + expect(verifyStep).toHaveBeenCalledWith(AuthMethodType.VOICE, { voiceData: CAPTURED_VOICE }) + expect(embedCapturedVoiceMock).not.toHaveBeenCalled() + }) + + it('flag ON: uploads { embedding: number[256] } and NOT { voiceData }', async () => { + flagState.enabled = true + const { verifyStep } = renderVoiceStep() + + await userEvent.click(screen.getByRole('button', { name: 'voice-submit' })) + + await waitFor(() => expect(verifyStep).toHaveBeenCalledTimes(1)) + expect(embedCapturedVoiceMock).toHaveBeenCalledWith(CAPTURED_VOICE) + + const [method, payload] = verifyStep.mock.calls[0] + expect(method).toBe(AuthMethodType.VOICE) + expect(payload).toHaveProperty('embedding') + expect(payload).not.toHaveProperty('voiceData') + expect(Array.isArray((payload as { embedding: number[] }).embedding)).toBe(true) + expect((payload as { embedding: number[] }).embedding).toHaveLength(256) + }) + + it('flag ON but embedding fails: falls back to { voiceData } (never blocks the login)', async () => { + flagState.enabled = true + embedCapturedVoiceMock.mockResolvedValue(null) + const { verifyStep, onError } = renderVoiceStep() + + await userEvent.click(screen.getByRole('button', { name: 'voice-submit' })) + + await waitFor(() => expect(verifyStep).toHaveBeenCalledTimes(1)) + // Unlike FACE, VOICE falls back to uploading the audio (scaffold preproc + + // the server accepts either when ON), so the login is never blocked. + expect(verifyStep).toHaveBeenCalledWith(AuthMethodType.VOICE, { voiceData: CAPTURED_VOICE }) + expect(onError).not.toHaveBeenCalled() + }) +}) diff --git a/src/features/biometrics/voice-embedding/__tests__/clientVoiceEmbeddingFlag.test.ts b/src/features/biometrics/voice-embedding/__tests__/clientVoiceEmbeddingFlag.test.ts new file mode 100644 index 0000000..6ab278f --- /dev/null +++ b/src/features/biometrics/voice-embedding/__tests__/clientVoiceEmbeddingFlag.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { isClientSideVoiceEmbeddingEnabled } from '../clientVoiceEmbeddingFlag' + +describe('isClientSideVoiceEmbeddingEnabled', () => { + afterEach(() => { + vi.unstubAllEnvs() + }) + + it("is OFF by default (env unset)", () => { + vi.stubEnv('VITE_CLIENT_SIDE_VOICE_EMBEDDING', '') + expect(isClientSideVoiceEmbeddingEnabled()).toBe(false) + }) + + it("is ON only for the exact string 'true'", () => { + vi.stubEnv('VITE_CLIENT_SIDE_VOICE_EMBEDDING', 'true') + expect(isClientSideVoiceEmbeddingEnabled()).toBe(true) + }) + + it("is OFF for any non-'true' value", () => { + vi.stubEnv('VITE_CLIENT_SIDE_VOICE_EMBEDDING', '1') + expect(isClientSideVoiceEmbeddingEnabled()).toBe(false) + vi.stubEnv('VITE_CLIENT_SIDE_VOICE_EMBEDDING', 'TRUE') + expect(isClientSideVoiceEmbeddingEnabled()).toBe(false) + }) +}) diff --git a/src/features/biometrics/voice-embedding/__tests__/speakerEmbedder.test.ts b/src/features/biometrics/voice-embedding/__tests__/speakerEmbedder.test.ts new file mode 100644 index 0000000..00d9e8c --- /dev/null +++ b/src/features/biometrics/voice-embedding/__tests__/speakerEmbedder.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect, vi } from 'vitest' +import { + SpeakerEmbedder, + l2Normalize, + VOICE_EMBEDDING_DIMENSION, +} from '../speakerEmbedder' +import { PARTIAL_N_FRAMES, MEL_N_CHANNELS, TARGET_SAMPLE_RATE } from '../voicePreprocess' + +/** + * A fake onnxruntime-web module. The fake model returns a FIXED unit-ish vector + * per call so we can assert the embedder's orchestration: one inference per + * partial, (1,160,40) input shape, mean-of-partials then final L2-norm. + */ +function makeFakeOrt(onRun: (dims: readonly number[]) => Float32Array) { + const runs: Array = [] + const ort = { + env: { wasm: {} as { wasmPaths?: string } }, + Tensor: class { + type: string + data: Float32Array + dims: readonly number[] + constructor(type: string, data: Float32Array, dims: readonly number[]) { + this.type = type + this.data = data + this.dims = dims + } + }, + InferenceSession: { + create: vi.fn(async () => ({ + inputNames: ['mels'], + outputNames: ['embeds'], + run: vi.fn(async (feeds: Record) => { + const dims = feeds['mels'].dims + runs.push(dims) + return { embeds: { data: onRun(dims) } } + }), + })), + }, + } + return { ort, runs } +} + +function oneSecondTone(): Float32Array { + const wav = new Float32Array(TARGET_SAMPLE_RATE) + for (let i = 0; i < wav.length; i++) wav[i] = 0.3 * Math.sin((2 * Math.PI * 220 * i) / TARGET_SAMPLE_RATE) + return wav +} + +describe('l2Normalize', () => { + it('returns a unit-norm vector', () => { + const out = l2Normalize(new Float32Array([3, 4])) + expect(Math.hypot(out[0], out[1])).toBeCloseTo(1, 6) + }) + it('passes a zero vector through without NaN', () => { + const out = l2Normalize(new Float32Array([0, 0])) + expect(out.every((v) => Number.isFinite(v))).toBe(true) + }) +}) + +describe('SpeakerEmbedder.embed', () => { + it('feeds (1,160,40) per partial and returns an L2-normalized 256-d vector', async () => { + // Fixed per-call vector → mean == that vector → normalized. + const fixed = new Float32Array(VOICE_EMBEDDING_DIMENSION) + for (let i = 0; i < fixed.length; i++) fixed[i] = (i % 7) - 3 + const { ort, runs } = makeFakeOrt(() => fixed.slice()) + + // sha256 null → skip model fetch; pass a dummy URL. + const embedder = new SpeakerEmbedder(ort as never, 'dummy://model', undefined, null) + const vec = await embedder.embed(oneSecondTone()) + + expect(vec.length).toBe(VOICE_EMBEDDING_DIMENSION) + // Result is L2-normalized. + let sumSq = 0 + for (const v of vec) sumSq += v * v + expect(Math.sqrt(sumSq)).toBeCloseTo(1, 5) + + // At least one inference, each with the (1,160,40) partial shape. + expect(runs.length).toBeGreaterThanOrEqual(1) + for (const dims of runs) { + expect(dims).toEqual([1, PARTIAL_N_FRAMES, MEL_N_CHANNELS]) + } + }) + + it('reuses the ONNX session across calls (loads once)', async () => { + const fixed = new Float32Array(VOICE_EMBEDDING_DIMENSION).fill(0.5) + const { ort } = makeFakeOrt(() => fixed.slice()) + const embedder = new SpeakerEmbedder(ort as never, 'dummy://model', undefined, null) + + await embedder.embed(oneSecondTone()) + await embedder.embed(oneSecondTone()) + + expect(ort.InferenceSession.create).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/features/biometrics/voice-embedding/__tests__/voicePreprocess.test.ts b/src/features/biometrics/voice-embedding/__tests__/voicePreprocess.test.ts new file mode 100644 index 0000000..c0c30c9 --- /dev/null +++ b/src/features/biometrics/voice-embedding/__tests__/voicePreprocess.test.ts @@ -0,0 +1,127 @@ +import { describe, it, expect } from 'vitest' +import { + decodeWavBytes, + normalizeVolumeIncreaseOnly, + wavToMelSpectrogram, + computePartialSlices, + buildMelPartials, + MEL_N_CHANNELS, + PARTIAL_N_FRAMES, + TARGET_SAMPLE_RATE, +} from '../voicePreprocess' + +/** Build a canonical 16 kHz mono 16-bit PCM WAV ArrayBuffer from Float32 samples. */ +function makeWav(samples: Float32Array): ArrayBuffer { + const headerSize = 44 + const dataSize = samples.length * 2 + const buf = new ArrayBuffer(headerSize + dataSize) + const view = new DataView(buf) + const writeStr = (off: number, s: string) => { + for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i)) + } + writeStr(0, 'RIFF') + view.setUint32(4, 36 + dataSize, true) + writeStr(8, 'WAVE') + writeStr(12, 'fmt ') + view.setUint32(16, 16, true) + view.setUint16(20, 1, true) // PCM + view.setUint16(22, 1, true) // mono + view.setUint32(24, TARGET_SAMPLE_RATE, true) + view.setUint32(28, TARGET_SAMPLE_RATE * 2, true) + view.setUint16(32, 2, true) + view.setUint16(34, 16, true) + writeStr(36, 'data') + view.setUint32(40, dataSize, true) + for (let i = 0; i < samples.length; i++) { + const v = Math.max(-1, Math.min(1, samples[i])) + view.setInt16(headerSize + i * 2, Math.round(v * 32767), true) + } + return buf +} + +describe('decodeWavBytes', () => { + it('round-trips 16-bit PCM samples to Float32 in [-1, 1]', () => { + const input = new Float32Array([0, 0.5, -0.5, 1, -1]) + const decoded = decodeWavBytes(makeWav(input)) + expect(decoded).not.toBeNull() + expect(decoded!.length).toBe(input.length) + // 16-bit quantization tolerance. + for (let i = 0; i < input.length; i++) { + expect(decoded![i]).toBeCloseTo(input[i], 2) + } + }) + + it('returns null for non-WAV bytes', () => { + const junk = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]).buffer + expect(decodeWavBytes(junk)).toBeNull() + }) + + it('returns null for too-short input', () => { + expect(decodeWavBytes(new ArrayBuffer(10))).toBeNull() + }) +}) + +describe('normalizeVolumeIncreaseOnly', () => { + it('amplifies a quiet clip toward -30 dBFS', () => { + const quiet = new Float32Array(1000).fill(0.001) + const out = normalizeVolumeIncreaseOnly(quiet) + // The gain must be > 1 (quiet clip boosted). + expect(out[0]).toBeGreaterThan(quiet[0]) + }) + + it('never attenuates an already-loud clip (increase_only)', () => { + const loud = new Float32Array(1000).fill(0.9) + const out = normalizeVolumeIncreaseOnly(loud) + // Already louder than -30 dBFS → returned unchanged (same reference). + expect(out).toBe(loud) + }) + + it('leaves a silent clip unchanged (no divide-by-zero)', () => { + const silent = new Float32Array(500) + const out = normalizeVolumeIncreaseOnly(silent) + expect(out).toBe(silent) + }) +}) + +describe('wavToMelSpectrogram', () => { + it('produces 40-channel mel frames at ~10ms hop', () => { + // 1 second of a 440 Hz tone. + const n = TARGET_SAMPLE_RATE + const wav = new Float32Array(n) + for (let i = 0; i < n; i++) wav[i] = 0.3 * Math.sin((2 * Math.PI * 440 * i) / TARGET_SAMPLE_RATE) + const mel = wavToMelSpectrogram(wav) + // center=True → ~ n/hop + 1 frames. + expect(mel.length).toBeGreaterThan(95) + expect(mel[0].length).toBe(MEL_N_CHANNELS) + // All non-negative (power mel). + for (const frame of mel) for (const v of frame) expect(v).toBeGreaterThanOrEqual(0) + }) +}) + +describe('computePartialSlices', () => { + it('returns at least one 160-frame slice for a short clip', () => { + const slices = computePartialSlices(TARGET_SAMPLE_RATE) // 1s + expect(slices.length).toBeGreaterThanOrEqual(1) + for (const [start, end] of slices) { + expect(end - start).toBe(PARTIAL_N_FRAMES) + } + }) + + it('produces multiple partials for a longer clip', () => { + const slices = computePartialSlices(TARGET_SAMPLE_RATE * 5) // 5s + expect(slices.length).toBeGreaterThan(1) + }) +}) + +describe('buildMelPartials', () => { + it('builds (n_partials, 160, 40) mel partials', () => { + const wav = new Float32Array(TARGET_SAMPLE_RATE * 2) + for (let i = 0; i < wav.length; i++) wav[i] = 0.2 * Math.sin(i / 20) + const partials = buildMelPartials(wav) + expect(partials.length).toBeGreaterThanOrEqual(1) + for (const p of partials) { + expect(p.length).toBe(PARTIAL_N_FRAMES) + expect(p[0].length).toBe(MEL_N_CHANNELS) + } + }) +}) diff --git a/src/features/biometrics/voice-embedding/clientVoiceEmbeddingFlag.ts b/src/features/biometrics/voice-embedding/clientVoiceEmbeddingFlag.ts new file mode 100644 index 0000000..afc61ae --- /dev/null +++ b/src/features/biometrics/voice-embedding/clientVoiceEmbeddingFlag.ts @@ -0,0 +1,29 @@ +/** + * clientVoiceEmbeddingFlag — client gate for the GPU-less "upload the speaker + * embedding, not the audio" VOICE login path (audit H3). + * + * Mirrors the server flag `app.auth.client-side-voice-embedding`. Default OFF: + * when off, the VOICE submit uploads the recorded audio exactly as it always has + * (byte-identical legacy behaviour). When on, the browser computes the 256-d + * Resemblyzer speaker embedding locally and uploads only the vector — the raw + * audio never leaves the device. + * + * Build-time env flag ONLY (`VITE_CLIENT_SIDE_VOICE_EMBEDDING`), baked at build + * time; it is not driven from login-config (a future per-tenant item). + * + * Read via a function (not a module-load constant) so the value is injectable / + * mockable in tests and is evaluated at call time rather than frozen at import. + * + * ⚠️ SCAFFOLD CAVEAT: the browser audio preprocessing (Resemblyzer + * `preprocess_wav` WebRTC-VAD silence trim + librosa power-mel) is NOT yet + * validated to parity with the server. Keep this OFF until the mel + VAD are + * validated — see biometric-processor `docs/design/VOICE_CLIENT_EMBEDDING_SPEC.md`. + */ + +/** + * True when the client-side-voice-embedding path is enabled for this build. + * Controlled by `VITE_CLIENT_SIDE_VOICE_EMBEDDING` (string `'true'` to enable). + */ +export function isClientSideVoiceEmbeddingEnabled(): boolean { + return import.meta.env.VITE_CLIENT_SIDE_VOICE_EMBEDDING === 'true' +} diff --git a/src/features/biometrics/voice-embedding/embedCapturedVoice.ts b/src/features/biometrics/voice-embedding/embedCapturedVoice.ts new file mode 100644 index 0000000..dd36018 --- /dev/null +++ b/src/features/biometrics/voice-embedding/embedCapturedVoice.ts @@ -0,0 +1,71 @@ +/** + * embedCapturedVoice — turn a captured VOICE data-URL (16 kHz mono WAV) into a + * 256-d Resemblyzer speaker embedding entirely in the browser, so the raw audio + * never leaves the device (audit H3, GPU-less). + * + * Input is the SAME base64 WAV data-URL the legacy VOICE path uploads (from + * `VoiceStep` → `useVoiceRecorder.wav16k`). We decode it to a Float32 PCM array, + * apply the Resemblyzer volume normalization, build mel partials and run the + * VoiceEncoder ONNX (onnxruntime-web, WASM EP) → mean of partial embeddings → + * L2-normalize. + * + * Resilient by design: returns `null` on ANY failure (no/unhosted model, ORT + * load error, non-WAV input, inference error) so the VOICE submit path can fall + * back to the legacy audio upload and never block a login. + * + * ⚠️ SCAFFOLD: the model + inference are exact, but the JS preprocessing + * (`voicePreprocess`) is not parity-validated against the server (no WebRTC-VAD + * trim, librosa-approximate mel). Keep `VITE_CLIENT_SIDE_VOICE_EMBEDDING` OFF + * until validated — see biometric-processor `docs/design/VOICE_CLIENT_EMBEDDING_SPEC.md`. + */ + +import { SpeakerEmbedder, VOICE_EMBEDDING_DIMENSION } from './speakerEmbedder' +import { decodeWav16kDataUrl, normalizeVolumeIncreaseOnly } from './voicePreprocess' + +/** + * onnxruntime-web ships its runtime WASM separately from the JS. Without + * `wasmPaths`, ORT resolves it against the current origin — and the SPA rewrite + * returns index.html for any non-existent path, so the browser sees HTML instead + * of WASM and dies. Point ORT at the pinned jsdelivr copy (CSP already allows + * cdn.jsdelivr.net). Same approach as `embedCapturedFace` / `VoiceVAD`. + */ +const ORT_WASM_PATHS = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.18.0/dist/' + +/** Lazily-constructed singleton embedder (loads the ONNX session once, reuses). */ +let embedder: SpeakerEmbedder | null = null + +async function getEmbedder(): Promise { + if (embedder) return embedder + // Dynamic import keeps the WASM runtime out of the main bundle (code-split + // exactly like embedCapturedFace / VoiceVAD). + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore — onnxruntime-web is loaded at runtime + const ort = await import('onnxruntime-web') + embedder = new SpeakerEmbedder(ort as never, undefined, ORT_WASM_PATHS) + return embedder +} + +/** + * Compute the client-side Resemblyzer speaker embedding for a captured VOICE + * data-URL (16 kHz mono WAV). + * + * @param voiceDataUrl base64 WAV data-URL (as the legacy path uploads). + * @returns A 256-number embedding array, or `null` on any failure. + */ +export async function embedCapturedVoice(voiceDataUrl: string): Promise { + try { + const wav = decodeWav16kDataUrl(voiceDataUrl) + if (!wav || wav.length === 0) return null + + const normalized = normalizeVolumeIncreaseOnly(wav) + + const instance = await getEmbedder() + const vec = await instance.embed(normalized) + if (!vec || vec.length !== VOICE_EMBEDDING_DIMENSION) return null + + return Array.from(vec) + } catch { + // Resilient: never block the VOICE login on a client-embedding failure. + return null + } +} diff --git a/src/features/biometrics/voice-embedding/prefetchVoiceModel.ts b/src/features/biometrics/voice-embedding/prefetchVoiceModel.ts new file mode 100644 index 0000000..8cc5d53 --- /dev/null +++ b/src/features/biometrics/voice-embedding/prefetchVoiceModel.ts @@ -0,0 +1,63 @@ +/** + * prefetchVoiceModel — warm the Resemblyzer voice-encoder model cache BEFORE the + * VOICE step needs it, so by submit time the model is already on the device and + * the in-browser embedding is fast. Mirrors `prefetchFacenetModel`. + * + * Flag-gated (no-op unless `VITE_CLIENT_SIDE_VOICE_EMBEDDING` is ON), de-duped + * (concurrent calls share the in-flight promise), and never throws — a failed + * prefetch is harmless because `embedCapturedVoice` retries the download on + * submit (and falls back to audio upload if even that fails). + */ + +import { getModel } from '@features/biometrics/embedding/modelCache' +import { isClientSideVoiceEmbeddingEnabled } from './clientVoiceEmbeddingFlag' +import { DEFAULT_VOICE_MODEL_URL, DEFAULT_VOICE_MODEL_SHA256 } from './speakerEmbedder' + +/** In-flight prefetch promise, so repeat/concurrent calls don't re-download. */ +let inFlight: Promise | null = null + +/** + * Warm the voice-encoder model cache (download-once + SHA256-verify) when the + * flag is ON. No-op when OFF. Never throws. + */ +export function prefetchVoiceModel(): Promise { + if (!isClientSideVoiceEmbeddingEnabled()) return Promise.resolve() + if (inFlight) return inFlight + + inFlight = getModel(DEFAULT_VOICE_MODEL_URL, DEFAULT_VOICE_MODEL_SHA256) + .then(() => undefined) + .catch(() => { + // Swallow: a failed prefetch is harmless — embedCapturedVoice retries + // on submit. Reset so a later attempt can warm the cache again. + inFlight = null + }) + + return inFlight +} + +/** + * Schedule {@link prefetchVoiceModel} on the browser idle queue (falling back to + * a short `setTimeout` where `requestIdleCallback` is unavailable). Returns a + * cleanup function that cancels a still-pending schedule — wire it as a React + * effect cleanup so an unmount before idle does not fire a stray download. + */ +export function scheduleVoiceModelPrefetch(): () => void { + if (!isClientSideVoiceEmbeddingEnabled()) return () => {} + + const win = window as Window & { + requestIdleCallback?: (cb: () => void) => number + cancelIdleCallback?: (handle: number) => void + } + + if (typeof win.requestIdleCallback === 'function') { + const handle = win.requestIdleCallback(() => { + void prefetchVoiceModel() + }) + return () => win.cancelIdleCallback?.(handle) + } + + const timer = setTimeout(() => { + void prefetchVoiceModel() + }, 1) + return () => clearTimeout(timer) +} diff --git a/src/features/biometrics/voice-embedding/speakerEmbedder.ts b/src/features/biometrics/voice-embedding/speakerEmbedder.ts new file mode 100644 index 0000000..8202624 --- /dev/null +++ b/src/features/biometrics/voice-embedding/speakerEmbedder.ts @@ -0,0 +1,166 @@ +/** + * speakerEmbedder — run the Resemblyzer VoiceEncoder ONNX model in the browser + * via onnxruntime-web (WASM execution provider) and return an L2-normalized 256-d + * speaker embedding. + * + * Mirrors `facenetEmbedder.ts`: lazily loads the ONNX session, reuses it, + * download-once SHA256-verified model bytes via `modelCache.getModel`, WASM EP. + * + * The model (`scripts/export_resemblyzer_onnx.py` in biometric-processor) maps a + * batch of mel partials `(batch, n_frames, 40)` → `(batch, 256)` unit-norm + * partial embeddings. The full utterance embedding is the L2-normed MEAN of the + * partial embeddings (Resemblyzer `embed_utterance`), computed here in JS. + * + * ── Model format: ship FP32 ────────────────────────────────────────────────── + * The model is tiny (~5.7 MB), so ship FP32. Do NOT INT8-quantize (onnxruntime-web + * WASM lacks the dynamic-quant ops, same finding as facenet512). The model is + * fetched + SHA256-verified at RUNTIME (NOT via the build-time manifest — adding + * it to public/models/manifest.json before hosting FATALs the build). + * + * ⚠️ SCAFFOLD: the model export + this inference are exact, but the JS audio + * preprocessing that feeds it (`voicePreprocess`) is not parity-validated against + * the server — see `voicePreprocess.ts` + the spec. Keep the flag OFF. + */ + +import { getModel } from '@features/biometrics/embedding/modelCache' +import { buildMelPartials, MEL_N_CHANNELS, PARTIAL_N_FRAMES } from './voicePreprocess' + +/** Length of a Resemblyzer speaker embedding. */ +export const VOICE_EMBEDDING_DIMENSION = 256 + +/** + * Default model location (same-origin `/models/`, hosted at + * app.fivucsas.com/models/ alongside facenet512). Override for tests. + * + * NOTE: the `` filename + SHA constant below are placeholders for the + * actually-hosted artifact — the real values come from running + * `scripts/export_resemblyzer_onnx.py` (it prints the sha256). They are wired + * here so the runtime path is complete; the model must be hosted before the flag + * is enabled (see the spec). Until hosted, `embedCapturedVoice` returns null + * (model fetch fails → swallowed) and the VOICE submit falls back to audio. + */ +export const DEFAULT_VOICE_MODEL_URL = '/models/resemblyzer-voice-encoder.onnx' + +/** SHA256 of the FP32 model, verified at runtime by `getModel`. */ +export const DEFAULT_VOICE_MODEL_SHA256 = + '8538a560f1e7c03c5f8157f6771beba83b3f8ba52b7791eb372b31d75d86a29d' + +/** Minimal structural type for the bits of onnxruntime-web we use. */ +interface OrtLike { + env: { wasm: { wasmPaths?: string; numThreads?: number } } + Tensor: new (type: 'float32', data: Float32Array, dims: readonly number[]) => unknown + InferenceSession: { + create: ( + source: string | ArrayBuffer, + options: { executionProviders: string[] }, + ) => Promise + } +} +interface OrtSession { + inputNames: string[] + outputNames: string[] + run: (feeds: Record) => Promise> + release?: () => Promise +} + +/** L2-normalize a vector. */ +export function l2Normalize(vec: Float32Array): Float32Array { + let sumSq = 0 + for (let i = 0; i < vec.length; i++) sumSq += vec[i] * vec[i] + const norm = Math.sqrt(sumSq) + if (norm === 0) return vec.slice() + const out = new Float32Array(vec.length) + for (let i = 0; i < vec.length; i++) out[i] = vec[i] / norm + return out +} + +/** Flatten one (160, 40) partial into the (1, 160, 40) row-major float buffer. */ +function flattenPartial(partial: Float32Array[]): Float32Array { + const out = new Float32Array(PARTIAL_N_FRAMES * MEL_N_CHANNELS) + for (let f = 0; f < PARTIAL_N_FRAMES; f++) { + const frame = partial[f] + for (let c = 0; c < MEL_N_CHANNELS; c++) { + out[f * MEL_N_CHANNELS + c] = frame ? frame[c] : 0 + } + } + return out +} + +/** + * Browser speaker embedder. Lazily loads the ONNX session on first use and reuses + * it. Construct with an injected `ort` module (dynamic `import('onnxruntime-web')`) + * so the heavy WASM runtime is code-split out of the main bundle and tests can + * supply the module without a bundler. Mirrors `FacenetEmbedder`. + */ +export class SpeakerEmbedder { + private session: OrtSession | null = null + private loading: Promise | null = null + + constructor( + private readonly ort: OrtLike, + private readonly modelUrl: string = DEFAULT_VOICE_MODEL_URL, + private readonly wasmPaths: string | undefined = undefined, + private readonly modelSha256: string | null = DEFAULT_VOICE_MODEL_SHA256, + ) {} + + /** Load (or reuse) the ONNX session. */ + async load(): Promise { + if (this.session) return this.session + if (this.loading) return this.loading + + this.loading = (async () => { + if (this.wasmPaths) this.ort.env.wasm.wasmPaths = this.wasmPaths + const source: string | ArrayBuffer = + this.modelSha256 != null + ? await getModel(this.modelUrl, this.modelSha256) + : this.modelUrl + const session = await this.ort.InferenceSession.create(source, { + executionProviders: ['wasm'], + }) + this.session = session + return session + })() + return this.loading + } + + /** + * Embed a 16 kHz mono Float32 PCM waveform → L2-normalized 256-d Float32Array. + * Reproduces Resemblyzer `embed_utterance`: build mel partials, run each + * through the model (batch=1), average the partial embeddings, L2-normalize. + */ + async embed(wav: Float32Array): Promise { + const session = await this.load() + const inputName = session.inputNames[0] + const outputName = session.outputNames[0] + + const partials = buildMelPartials(wav) + if (partials.length === 0) { + throw new Error('voice: no mel partials produced from waveform') + } + + const sum = new Float32Array(VOICE_EMBEDDING_DIMENSION) + for (const partial of partials) { + const tensorData = flattenPartial(partial) + const feeds: Record = { + [inputName]: new this.ort.Tensor('float32', tensorData, [ + 1, + PARTIAL_N_FRAMES, + MEL_N_CHANNELS, + ]), + } + const outputs = await session.run(feeds) + const raw = outputs[outputName].data + for (let i = 0; i < VOICE_EMBEDDING_DIMENSION; i++) sum[i] += raw[i] + } + // Mean of partials, then final L2-norm (Resemblyzer embed_utterance). + for (let i = 0; i < VOICE_EMBEDDING_DIMENSION; i++) sum[i] /= partials.length + return l2Normalize(sum) + } + + /** Release the ONNX session. */ + async dispose(): Promise { + await this.session?.release?.() + this.session = null + this.loading = null + } +} diff --git a/src/features/biometrics/voice-embedding/voicePreprocess.ts b/src/features/biometrics/voice-embedding/voicePreprocess.ts new file mode 100644 index 0000000..7b39fd8 --- /dev/null +++ b/src/features/biometrics/voice-embedding/voicePreprocess.ts @@ -0,0 +1,314 @@ +/** + * voicePreprocess — turn 16 kHz mono PCM into the mel-spectrogram partials the + * Resemblyzer VoiceEncoder ONNX model expects. + * + * ⚠️ SCAFFOLD — NOT PARITY-VALIDATED. The Resemblyzer reference preprocessing is + * `preprocess_wav` (dBFS normalize + WebRTC-VAD silence trim) → + * `wav_to_mel_spectrogram` (librosa power-mel, n_fft 400 / hop 160 / 40 mels) → + * partial slicing. This module reproduces the dBFS normalize, the mel + * spectrogram, and the partial slicing, but DOES NOT reproduce the WebRTC VAD + * silence trim (webrtcvad is a C extension with no bit-exact JS port). Skipping + * the VAD shifts the embedding ≈0.11 cosine on clean audio vs the server, which + * is risky near the 0.65 accept threshold. The mel here is a straightforward + * Hann-window STFT + Slaney mel filterbank that APPROXIMATES librosa but has not + * been validated frame-for-frame against the Python reference. + * + * Therefore the whole client voice-embedding path ships behind a default-OFF + * flag and is documented as scaffold-only. Full load-bearing contract + the + * exact steps to make this parity-correct + the canary plan are in + * biometric-processor `docs/design/VOICE_CLIENT_EMBEDDING_SPEC.md`. + * + * Constants below are the Resemblyzer hparams.py values (load-bearing). + */ + +export const TARGET_SAMPLE_RATE = 16_000 +/** Mel-filterbank channels (model input feature dim). */ +export const MEL_N_CHANNELS = 40 +/** 25 ms window @ 16 kHz. */ +export const N_FFT = 400 +/** 10 ms hop @ 16 kHz. */ +export const HOP_LENGTH = 160 +/** A partial utterance is 160 frames (1.6 s). */ +export const PARTIAL_N_FRAMES = 160 +/** Resemblyzer embed_utterance defaults. */ +export const PARTIAL_RATE = 1.3 +export const MIN_COVERAGE = 0.75 +/** Resemblyzer normalize_volume target. */ +const TARGET_DBFS = -30 +const INT16_MAX = 32767 + +/** + * Decode a base64 data-URL of a 16 kHz mono PCM WAV (as produced by + * `audioToWav16k.encodeToWav16kMono`) into a Float32 sample array in [-1, 1]. + * + * Returns null on any parse failure (non-WAV, truncated, unsupported bit depth) + * so callers can fall back to uploading the audio. + */ +export function decodeWav16kDataUrl(dataUrl: string): Float32Array | null { + try { + const commaIdx = dataUrl.indexOf(',') + if (commaIdx < 0 || !dataUrl.startsWith('data:')) return null + const binary = atob(dataUrl.slice(commaIdx + 1)) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return decodeWavBytes(bytes.buffer) + } catch { + return null + } +} + +/** + * Decode a canonical 44-byte-header PCM WAV ArrayBuffer (16-bit mono, 16 kHz) + * into a Float32 sample array in [-1, 1]. Only the 16-bit PCM mono shape the + * web-app's `encodeToWav16kMono` produces is supported; anything else → null. + */ +export function decodeWavBytes(buffer: ArrayBuffer): Float32Array | null { + const view = new DataView(buffer) + if (buffer.byteLength < 44) return null + // 'RIFF' .... 'WAVE' + if (view.getUint32(0, false) !== 0x52494646) return null // "RIFF" + if (view.getUint32(8, false) !== 0x57415645) return null // "WAVE" + + const bitsPerSample = view.getUint16(34, true) + const numChannels = view.getUint16(22, true) + if (bitsPerSample !== 16) return null + + // Find the 'data' chunk (it is usually at offset 36, but scan to be safe). + let offset = 12 + let dataOffset = -1 + let dataLen = 0 + while (offset + 8 <= buffer.byteLength) { + const chunkId = view.getUint32(offset, false) + const chunkSize = view.getUint32(offset + 4, true) + if (chunkId === 0x64617461) { + // "data" + dataOffset = offset + 8 + dataLen = chunkSize + break + } + offset += 8 + chunkSize + (chunkSize & 1) + } + if (dataOffset < 0) return null + + const available = Math.min(dataLen, buffer.byteLength - dataOffset) + const totalSamples = Math.floor(available / 2) + const frames = Math.floor(totalSamples / Math.max(1, numChannels)) + const out = new Float32Array(frames) + for (let i = 0; i < frames; i++) { + // Downmix to mono by averaging channels (the encoder writes mono, so + // numChannels is 1, but this stays correct if that ever changes). + let acc = 0 + for (let c = 0; c < numChannels; c++) { + acc += view.getInt16(dataOffset + (i * numChannels + c) * 2, true) + } + out[i] = acc / numChannels / 32768 + } + return out +} + +/** + * Resemblyzer `normalize_volume(wav, -30, increase_only=True)`: scale the + * waveform up to -30 dBFS, never attenuating (a clip already louder is left + * unchanged). Pure + exact — validate to ~1e-6 against the Python reference. + */ +export function normalizeVolumeIncreaseOnly(wav: Float32Array): Float32Array { + let sumSq = 0 + for (let i = 0; i < wav.length; i++) { + const s = wav[i] * INT16_MAX + sumSq += s * s + } + if (wav.length === 0) return wav + const rms = Math.sqrt(sumSq / wav.length) + if (rms === 0) return wav + const waveDbfs = 20 * Math.log10(rms / INT16_MAX) + const dbfsChange = TARGET_DBFS - waveDbfs + if (dbfsChange < 0) return wav // increase-only + const gain = Math.pow(10, dbfsChange / 20) + const out = new Float32Array(wav.length) + for (let i = 0; i < wav.length; i++) out[i] = wav[i] * gain + return out +} + +// ── Mel spectrogram (librosa-approximate, SCAFFOLD) ────────────────────────── + +let melCache: Float32Array[] | null = null + +/** Periodic Hann window of length `n` (librosa default for melspectrogram). */ +function hannWindow(n: number): Float32Array { + const w = new Float32Array(n) + for (let i = 0; i < n; i++) { + w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / n)) + } + return w +} + +/** Hz → Slaney mel (librosa default, htk=False). */ +function hzToMel(hz: number): number { + const fMin = 0 + const fSp = 200 / 3 + let mel = (hz - fMin) / fSp + const minLogHz = 1000 + const minLogMel = (minLogHz - fMin) / fSp + const logstep = Math.log(6.4) / 27 + if (hz >= minLogHz) mel = minLogMel + Math.log(hz / minLogHz) / logstep + return mel +} + +/** Slaney mel → Hz (inverse of {@link hzToMel}). */ +function melToHz(mel: number): number { + const fMin = 0 + const fSp = 200 / 3 + let hz = fMin + fSp * mel + const minLogHz = 1000 + const minLogMel = (minLogHz - fMin) / fSp + const logstep = Math.log(6.4) / 27 + if (mel >= minLogMel) hz = minLogHz * Math.exp(logstep * (mel - minLogMel)) + return hz +} + +/** + * Slaney-normalized mel filterbank, shape (MEL_N_CHANNELS, N_FFT/2 + 1). + * Approximates `librosa.filters.mel(sr, n_fft, n_mels, htk=False, norm='slaney')`. + */ +function melFilterbank(): Float32Array[] { + if (melCache) return melCache + const nFreqs = N_FFT / 2 + 1 + const fMax = TARGET_SAMPLE_RATE / 2 + const fftFreqs = new Float32Array(nFreqs) + for (let i = 0; i < nFreqs; i++) fftFreqs[i] = (i * TARGET_SAMPLE_RATE) / N_FFT + + const melMin = hzToMel(0) + const melMax = hzToMel(fMax) + const melPoints = new Float32Array(MEL_N_CHANNELS + 2) + for (let i = 0; i < melPoints.length; i++) { + melPoints[i] = melToHz(melMin + ((melMax - melMin) * i) / (MEL_N_CHANNELS + 1)) + } + + const bank: Float32Array[] = [] + for (let m = 0; m < MEL_N_CHANNELS; m++) { + const fLeft = melPoints[m] + const fCenter = melPoints[m + 1] + const fRight = melPoints[m + 2] + const row = new Float32Array(nFreqs) + // Slaney area normalization: 2 / (f[m+2] - f[m]). + const enorm = 2 / (fRight - fLeft) + for (let k = 0; k < nFreqs; k++) { + const f = fftFreqs[k] + const lower = (f - fLeft) / (fCenter - fLeft) + const upper = (fRight - f) / (fRight - fCenter) + const weight = Math.max(0, Math.min(lower, upper)) + row[k] = weight * enorm + } + bank.push(row) + } + melCache = bank + return bank +} + +/** Naive DFT power spectrum for one Hann-windowed frame (length N_FFT). */ +function framePowerSpectrum(frame: Float32Array, window: Float32Array): Float32Array { + const nFreqs = N_FFT / 2 + 1 + const power = new Float32Array(nFreqs) + for (let k = 0; k < nFreqs; k++) { + let re = 0 + let im = 0 + for (let n = 0; n < N_FFT; n++) { + const angle = (-2 * Math.PI * k * n) / N_FFT + const x = frame[n] * window[n] + re += x * Math.cos(angle) + im += x * Math.sin(angle) + } + power[k] = re * re + im * im + } + return power +} + +/** + * Power mel spectrogram, shape (n_frames, MEL_N_CHANNELS). Reproduces + * `librosa.feature.melspectrogram(power=2.0)` with center=True (reflect pad by + * N_FFT/2) + Hann window. NOT log-mel. + * + * SCAFFOLD: this is an O(n·N_FFT^2) naive DFT — correct in shape but slow and + * not validated frame-for-frame against librosa's FFT. A production port should + * use an FFT and validate the matrix to a tight tolerance (see the spec). + */ +export function wavToMelSpectrogram(wav: Float32Array): Float32Array[] { + const window = hannWindow(N_FFT) + const bank = melFilterbank() + const pad = Math.floor(N_FFT / 2) + + // center=True → reflect-pad by N_FFT/2 on both ends. + const padded = new Float32Array(wav.length + 2 * pad) + for (let i = 0; i < pad; i++) padded[i] = wav[Math.min(pad - i, wav.length - 1)] ?? 0 + padded.set(wav, pad) + for (let i = 0; i < pad; i++) { + padded[pad + wav.length + i] = wav[wav.length - 2 - i] ?? 0 + } + + const nFrames = 1 + Math.floor((padded.length - N_FFT) / HOP_LENGTH) + const mels: Float32Array[] = [] + const frame = new Float32Array(N_FFT) + for (let t = 0; t < nFrames; t++) { + const start = t * HOP_LENGTH + frame.set(padded.subarray(start, start + N_FFT)) + const power = framePowerSpectrum(frame, window) + const mel = new Float32Array(MEL_N_CHANNELS) + for (let m = 0; m < MEL_N_CHANNELS; m++) { + const row = bank[m] + let acc = 0 + for (let k = 0; k < row.length; k++) acc += row[k] * power[k] + mel[m] = acc + } + mels.push(mel) + } + return mels +} + +/** + * Resemblyzer `compute_partial_slices`: the frame-index ranges [start, end) of + * each 160-frame partial. Pure index arithmetic ported directly from + * voice_encoder.py. + */ +export function computePartialSlices(nSamples: number): Array<[number, number]> { + const samplesPerFrame = Math.floor((TARGET_SAMPLE_RATE * 10) / 1000) // = HOP_LENGTH + const nFrames = Math.floor((nSamples - 1) / samplesPerFrame + 1) + const frameStep = Math.max( + 1, + Math.floor(Math.round((TARGET_SAMPLE_RATE / PARTIAL_RATE) / samplesPerFrame)), + ) + + const slices: Array<[number, number]> = [] + for (let i = 0; ; i += frameStep) { + const melRangeStart = i + const melRangeEnd = i + PARTIAL_N_FRAMES + if (melRangeEnd > nFrames) { + // Last partial: keep it only if coverage >= MIN_COVERAGE (else drop), + // but always keep at least one slice. + const coverage = (nFrames - melRangeStart) / PARTIAL_N_FRAMES + if (coverage < MIN_COVERAGE && slices.length > 0) break + slices.push([melRangeStart, melRangeEnd]) + break + } + slices.push([melRangeStart, melRangeEnd]) + } + return slices +} + +/** + * Build the (n_partials, 160, 40) mel partials Resemblyzer feeds to the model. + * Each partial is exactly PARTIAL_N_FRAMES frames; the mel is zero-padded at the + * end when the last partial runs past the available frames. + */ +export function buildMelPartials(wav: Float32Array): Float32Array[][] { + const mel = wavToMelSpectrogram(wav) + const slices = computePartialSlices(wav.length) + const partials: Float32Array[][] = [] + for (const [start, end] of slices) { + const frames: Float32Array[] = [] + for (let f = start; f < end; f++) { + frames.push(mel[f] ?? new Float32Array(MEL_N_CHANNELS)) + } + partials.push(frames) + } + return partials +} diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 7d38166..227ea03 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -1699,7 +1699,8 @@ "reRecord": "Re-record", "submit": "Submit", "noVoiceDetected": "No voice detected. Please speak clearly and try again.", - "noSpeechDetected": "No speech detected. Please try again and speak clearly." + "noSpeechDetected": "No speech detected. Please try again and speak clearly.", + "preparingSecure": "Preparing secure verification…" }, "fingerprint": { "title": "Device Authentication", diff --git a/src/i18n/locales/tr.json b/src/i18n/locales/tr.json index 7f4b2ac..17baede 100644 --- a/src/i18n/locales/tr.json +++ b/src/i18n/locales/tr.json @@ -1699,7 +1699,8 @@ "reRecord": "Tekrar Kaydet", "submit": "Gönder", "noVoiceDetected": "Ses algılanmadı. Lütfen mikrofona net bir şekilde konuşun ve tekrar deneyin.", - "noSpeechDetected": "Konuşma algılanamadı. Lütfen tekrar deneyin ve net konuşun." + "noSpeechDetected": "Konuşma algılanamadı. Lütfen tekrar deneyin ve net konuşun.", + "preparingSecure": "Güvenli doğrulama hazırlanıyor…" }, "fingerprint": { "title": "Cihaz Kimlik Doğrulaması",