From 121278a362354280b6a2fd0f8643136134f366bc Mon Sep 17 00:00:00 2001 From: cukas Date: Fri, 19 Jun 2026 16:28:07 +0200 Subject: [PATCH] feat(rag): add vector store adapter conformance --- packages/cli/src/commands/rag.ts | 130 ++++++- packages/cli/tests/rag.test.ts | 40 ++ packages/core/src/index.ts | 16 + packages/core/src/rag-adapter-conformance.ts | 348 ++++++++++++++++++ .../tests/rag-adapter-conformance.test.ts | 122 ++++++ 5 files changed, 653 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/rag-adapter-conformance.ts create mode 100644 packages/core/tests/rag-adapter-conformance.test.ts diff --git a/packages/cli/src/commands/rag.ts b/packages/cli/src/commands/rag.ts index 4aef3e08..f9d0a8d2 100644 --- a/packages/cli/src/commands/rag.ts +++ b/packages/cli/src/commands/rag.ts @@ -1,18 +1,28 @@ import { + BUILTIN_RAG_VECTOR_STORE_MANIFESTS, + builtinRagVectorStoreManifest, + createInMemoryRagVectorStoreForConformance, evaluateRagEvalDocumentAsync, evaluateRagEvalDocumentFromDeclaredSourcesAsync, type RagChunkInput, type RagEvalDocumentReport, type RagRetrieveDocumentReport, + type RagVectorStoreConformanceContext, + type RagVectorStoreConformanceReport, + type RagVectorStoreKind, ragRetrieveCorpusSourceSummary, retrieveRagDocument, + runRagVectorStoreConformance, } from '@kernlang/core'; -import { existsSync, readFileSync } from 'fs'; -import { resolve } from 'path'; +import { LocalPersistentRagVectorStoreAdapter } from '@kernlang/core/node'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import { join, resolve } from 'path'; const USAGE = 'Usage: kern rag eval [--corpus ] [--openai-api-key ]\n' + - ' kern rag retrieve --query [--param name=value] (local embed models only)'; + ' kern rag retrieve --query [--param name=value] (local embed models only)\n' + + ' kern rag conformance [--adapter memory|local-persistent] [--json]'; /** `kern rag …` — run a RAG spec's contracts in the toolchain (dbt-test shape). */ export async function runRag(args: string[]): Promise { @@ -25,10 +35,76 @@ export async function runRag(args: string[]): Promise { runRagRetrieve(args.slice(2)); return; } + if (sub === 'conformance') { + runRagConformance(args.slice(2)); + return; + } console.error(USAGE); process.exit(1); } +function runRagConformance(args: string[]): void { + const { adapterFlagPresent, adapterName, json, unexpectedArgs, unknownFlags } = parseRagConformanceArgs(args); + if (unknownFlags.length > 0) fail(`unknown flag for conformance: ${unknownFlags[0]}.\n${USAGE}`); + if (unexpectedArgs.length > 0) fail(`unexpected argument for conformance: ${unexpectedArgs[0]}.\n${USAGE}`); + if (adapterFlagPresent && (!adapterName?.trim() || adapterName.trim().startsWith('-'))) { + fail(`missing value for --adapter.\n${USAGE}`); + } + const manifests = adapterName + ? [builtinRagVectorStoreManifest(adapterName) ?? fail(`unknown RAG adapter '${adapterName}'.\n${USAGE}`)] + : BUILTIN_RAG_VECTOR_STORE_MANIFESTS; + + const reports: RagVectorStoreConformanceReport[] = []; + for (const manifest of manifests) { + const tmp = mkdtempSync(join(tmpdir(), `kern-rag-${manifest.name}-conformance-`)); + try { + reports.push( + runRagVectorStoreConformance({ + manifest, + createStore: (context) => createConformanceStore(manifest.adapterKind, context, tmp), + }), + ); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + } + + const passed = reports.every((report) => report.passed); + if (json) { + console.log(JSON.stringify({ passed, reports }, null, 2)); + } else { + console.log('kern rag conformance'); + for (const report of reports) { + const mark = report.passed ? '✓' : '✗'; + console.log( + ` ${mark} ${report.manifest.name} (${report.summary.passed} passed, ${report.summary.failed} failed, ${report.summary.skipped} skipped)`, + ); + for (const entry of report.cases) { + if (entry.status === 'failed') console.log(` ✗ ${entry.name}: ${entry.message ?? 'failed'}`); + if (entry.status === 'skipped') console.log(` - ${entry.name}: ${entry.message ?? 'skipped'}`); + } + } + } + process.exit(passed ? 0 : 1); +} + +function createConformanceStore( + adapterKind: RagVectorStoreKind, + context: RagVectorStoreConformanceContext, + directory: string, +) { + if (adapterKind === 'memory') return createInMemoryRagVectorStoreForConformance(context); + if (adapterKind === 'local-persistent') { + return new LocalPersistentRagVectorStoreAdapter({ + directory, + fileName: `${context.namespace}.json`, + fingerprint: context.fingerprint, + dims: context.dims, + }); + } + fail(`unknown RAG adapter kind '${adapterKind}'.`); +} + function runRagRetrieve(args: string[]): void { const { filePath, paramError, paramFlagPresent, query, queryFlagPresent, queryParams, unknownFlags } = parseRagRetrieveArgs(args); @@ -186,6 +262,54 @@ function parseRagEvalArgs(args: readonly string[]): ParsedRagEvalArgs { return { filePath, corpusPath, corpusFlagPresent, openAiApiKeyFlag, openAiKeyFlagPresent }; } +interface ParsedRagConformanceArgs { + readonly adapterName?: string; + readonly adapterFlagPresent: boolean; + readonly json: boolean; + readonly unexpectedArgs: readonly string[]; + readonly unknownFlags: readonly string[]; +} + +function parseRagConformanceArgs(args: readonly string[]): ParsedRagConformanceArgs { + let adapterName: string | undefined; + let adapterFlagPresent = false; + let json = false; + const unexpectedArgs: string[] = []; + const unknownFlags: string[] = []; + + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + if (arg === '--json') { + json = true; + continue; + } + if (arg === '--adapter') { + adapterFlagPresent = true; + adapterName = args[i + 1]; + i += 1; + continue; + } + if (arg.startsWith('--adapter=')) { + adapterFlagPresent = true; + adapterName = arg.slice('--adapter='.length); + continue; + } + if (arg.startsWith('-')) { + unknownFlags.push(arg); + continue; + } + unexpectedArgs.push(arg); + } + + return { + ...(adapterName !== undefined ? { adapterName } : {}), + adapterFlagPresent, + json, + unexpectedArgs, + unknownFlags, + }; +} + interface ParsedRagRetrieveArgs { readonly filePath?: string; readonly query?: string; diff --git a/packages/cli/tests/rag.test.ts b/packages/cli/tests/rag.test.ts index e99e64f5..12b71fd6 100644 --- a/packages/cli/tests/rag.test.ts +++ b/packages/cli/tests/rag.test.ts @@ -205,4 +205,44 @@ describe('kern rag', () => { expect(result.stderr).toContain('./docs/**/*.md'); expect(result.stderr).toContain('matched no files'); }); + + test('runs built-in RAG adapter conformance checks', () => { + const result = run(['rag', 'conformance'], dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('kern rag conformance'); + expect(result.stdout).toContain('✓ memory'); + expect(result.stdout).toContain('✓ local-persistent'); + }); + + test('emits JSON for RAG adapter conformance checks', () => { + const result = run(['rag', 'conformance', '--adapter', 'memory', '--json'], dir); + expect(result.status).toBe(0); + const parsed = JSON.parse(result.stdout) as { + readonly passed: boolean; + readonly reports: readonly [ + { readonly manifest: { readonly name: string }; readonly summary: { readonly failed: number } }, + ]; + }; + expect(parsed.passed).toBe(true); + expect(parsed.reports[0].manifest.name).toBe('memory'); + expect(parsed.reports[0].summary.failed).toBe(0); + }); + + test('rejects unknown RAG conformance adapters', () => { + const result = run(['rag', 'conformance', '--adapter', 'pinecone'], dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain("unknown RAG adapter 'pinecone'"); + }); + + test('rejects RAG conformance adapter flag without a value', () => { + const result = run(['rag', 'conformance', '--adapter'], dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('missing value for --adapter'); + }); + + test('rejects unexpected RAG conformance positional arguments', () => { + const result = run(['rag', 'conformance', 'memory'], dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('unexpected argument for conformance: memory'); + }); }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e5c6be62..fcb43144 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -648,6 +648,22 @@ export { validatePortablePredicateAST, } from './portable-predicate.js'; export { parsePortableNonNegativeIntLiteral, parsePortablePathSegments } from './portable-route-collection.js'; +export type { + RagAdapterPersistence, + RagVectorStoreAdapterCapabilities, + RagVectorStoreAdapterManifest, + RagVectorStoreConformanceCaseResult, + RagVectorStoreConformanceContext, + RagVectorStoreConformanceOptions, + RagVectorStoreConformanceReport, + RagVectorStoreConformanceStatus, +} from './rag-adapter-conformance.js'; +export { + BUILTIN_RAG_VECTOR_STORE_MANIFESTS, + builtinRagVectorStoreManifest, + createInMemoryRagVectorStoreForConformance, + runRagVectorStoreConformance, +} from './rag-adapter-conformance.js'; export type { RagAssertionKind } from './rag-assertions.js'; export { RAG_ASSERTION_KIND_SET, RAG_ASSERTION_KINDS } from './rag-assertions.js'; export type { RagProviderEmbeddingOptions, RagSupportedEmbedModel } from './rag-embed-resolver.js'; diff --git a/packages/core/src/rag-adapter-conformance.ts b/packages/core/src/rag-adapter-conformance.ts new file mode 100644 index 00000000..1704e87a --- /dev/null +++ b/packages/core/src/rag-adapter-conformance.ts @@ -0,0 +1,348 @@ +import { + DeterministicHashEmbedder, + type Embedder, + embedderFingerprint, + InMemoryPgVectorRagStore, + type RagVectorStoreAdapter, + type RagVectorStoreKind, + type RagVectorStoreMetric, +} from './rag-embedding.js'; +import type { RagChunkInput } from './rag-runtime.js'; + +export type RagAdapterPersistence = 'ephemeral' | 'durable'; +export type RagVectorStoreConformanceStatus = 'passed' | 'failed' | 'skipped'; + +export interface RagVectorStoreAdapterCapabilities { + readonly upsert: boolean; + readonly upsertMany: boolean; + readonly search: boolean; + readonly snapshot: boolean; + readonly clear: boolean; +} + +export interface RagVectorStoreAdapterManifest { + readonly name: string; + readonly kind: 'vectorStore'; + readonly adapterKind: RagVectorStoreKind; + readonly version: string; + readonly metrics: readonly RagVectorStoreMetric[]; + readonly maxDimensions: number; + readonly persistence: RagAdapterPersistence; + readonly capabilities: RagVectorStoreAdapterCapabilities; +} + +export interface RagVectorStoreConformanceCaseResult { + readonly name: string; + readonly status: RagVectorStoreConformanceStatus; + readonly message?: string; +} + +export interface RagVectorStoreConformanceReport { + readonly manifest: RagVectorStoreAdapterManifest; + readonly passed: boolean; + readonly cases: readonly RagVectorStoreConformanceCaseResult[]; + readonly summary: { + readonly passed: number; + readonly failed: number; + readonly skipped: number; + }; +} + +export interface RagVectorStoreConformanceContext { + readonly fingerprint: string; + readonly dims: number; + readonly namespace: string; +} + +export interface RagVectorStoreConformanceOptions { + readonly manifest: RagVectorStoreAdapterManifest; + readonly createStore: (context: RagVectorStoreConformanceContext) => RagVectorStoreAdapter; + readonly embedder?: Embedder; + /** Optional namespace seed for programmatic callers that need reproducible backing file names. */ + readonly runId?: string; +} + +const CONFORMANCE_CORPUS: readonly RagChunkInput[] = [ + { id: 'refunds', text: 'refund refunds policy window thirty days money back', source: 'docs/refunds.md' }, + { id: 'shipping', text: 'shipping delivery courier tracking parcel transit', source: 'docs/shipping.md' }, + { id: 'returns', text: 'return exchange store credit receipt policy window', source: 'docs/returns.md' }, +]; +const CONFORMANCE_CORPUS_IDS = new Set(CONFORMANCE_CORPUS.map((chunk) => chunk.id)); +const CONFORMANT_RAG_VECTOR_STORE_KINDS: readonly RagVectorStoreKind[] = ['memory', 'local-persistent']; + +export const BUILTIN_RAG_VECTOR_STORE_MANIFESTS: readonly RagVectorStoreAdapterManifest[] = [ + { + name: 'memory', + kind: 'vectorStore', + adapterKind: 'memory', + version: '1.0.0', + metrics: ['cosine'], + maxDimensions: 4096, + persistence: 'ephemeral', + capabilities: { + upsert: true, + upsertMany: true, + search: true, + snapshot: true, + clear: true, + }, + }, + { + name: 'local-persistent', + kind: 'vectorStore', + adapterKind: 'local-persistent', + version: '1.0.0', + metrics: ['cosine'], + maxDimensions: 4096, + persistence: 'durable', + capabilities: { + upsert: true, + upsertMany: true, + search: true, + snapshot: true, + clear: true, + }, + }, +] as const; + +let conformanceRunSequence = 0; + +export function builtinRagVectorStoreManifest(name: string): RagVectorStoreAdapterManifest | undefined { + return BUILTIN_RAG_VECTOR_STORE_MANIFESTS.find((manifest) => manifest.name === name); +} + +export function createInMemoryRagVectorStoreForConformance( + context: RagVectorStoreConformanceContext, +): RagVectorStoreAdapter { + return new InMemoryPgVectorRagStore(context.fingerprint, context.dims); +} + +export function runRagVectorStoreConformance( + options: RagVectorStoreConformanceOptions, +): RagVectorStoreConformanceReport { + const embedder = options.embedder ?? new DeterministicHashEmbedder({ dims: 64 }); + const context = { + fingerprint: embedderFingerprint(embedder, 'cosine'), + dims: embedder.dims, + }; + const runNamespace = safeConformanceNamespace(options.runId ?? defaultConformanceRunNamespace()); + const contextFor = (name: string): RagVectorStoreConformanceContext => ({ + ...context, + namespace: `${runNamespace}-${safeConformanceNamespace(name)}`, + }); + const cases: RagVectorStoreConformanceCaseResult[] = []; + + runCase(cases, 'manifest-shape', () => assertManifest(options.manifest)); + runCase(cases, 'manifest-matches-adapter', () => { + if (options.manifest.maxDimensions < context.dims) { + throw new Error(`manifest maxDimensions ${options.manifest.maxDimensions} is below tested dims ${context.dims}.`); + } + usingStore(options.createStore(contextFor('manifest-matches-adapter')), (store) => { + if (store.kind !== options.manifest.adapterKind) { + throw new Error( + `manifest adapterKind '${options.manifest.adapterKind}' does not match store kind '${store.kind}'.`, + ); + } + if (store.metric !== 'cosine') throw new Error(`expected cosine metric, got '${store.metric}'.`); + if (store.dims !== context.dims) throw new Error(`expected store dims ${context.dims}, got ${store.dims}.`); + if (store.fingerprint !== context.fingerprint) + throw new Error('store fingerprint does not match conformance context.'); + }); + }); + runCase(cases, 'persistence-matches-adapter', () => { + const observed = observeStorePersistence(options, contextFor('persistence-matches-adapter'), embedder); + if (observed !== options.manifest.persistence) { + throw new Error(`manifest persistence '${options.manifest.persistence}' does not match observed '${observed}'.`); + } + }); + runCase(cases, 'empty-search-returns-empty-list', () => { + usingStore(options.createStore(contextFor('empty-search-returns-empty-list')), (store) => { + const result = store.search('refund policy', embedder.embed('refund policy'), { topK: 3 }); + if (result.chunks.length !== 0) throw new Error(`expected 0 chunks, got ${result.chunks.length}.`); + }); + }); + runCase(cases, 'upsert-search-ranks-related-chunk', () => { + usingStore(options.createStore(contextFor('upsert-search-ranks-related-chunk')), (store) => { + store.upsertMany( + CONFORMANCE_CORPUS.map((chunk) => ({ + chunk, + vector: embedder.embed(chunk.text), + })), + ); + const result = store.search('refund policy money back', embedder.embed('refund policy money back'), { topK: 1 }); + if (result.chunks[0]?.id !== 'refunds') { + throw new Error(`expected top chunk 'refunds', got '${result.chunks[0]?.id ?? ''}'.`); + } + }); + }); + runCase(cases, 'topk-is-respected', () => { + usingStore(options.createStore(contextFor('topk-is-respected')), (store) => { + store.upsertMany( + CONFORMANCE_CORPUS.map((chunk) => ({ + chunk, + vector: embedder.embed(chunk.text), + })), + ); + const result = store.search('policy return refund shipping', embedder.embed('policy return refund shipping'), { + topK: 2, + }); + if (result.chunks.length !== 2) throw new Error(`expected exactly 2 chunks, got ${result.chunks.length}.`); + const unexpected = result.chunks.find((chunk) => !CONFORMANCE_CORPUS_IDS.has(chunk.id)); + if (unexpected) throw new Error(`returned unknown chunk '${unexpected.id}'.`); + }); + }); + runCase(cases, 'dimension-mismatch-fails-closed', () => { + usingStore(options.createStore(contextFor('dimension-mismatch-fails-closed')), (store) => { + expectThrow(() => store.upsert(CONFORMANCE_CORPUS[0], new Float64Array(context.dims + 1)), /dimensions/u); + }); + }); + runCase(cases, 'fingerprint-mismatch-fails-closed', () => { + usingStore(options.createStore(contextFor('fingerprint-mismatch-fails-closed')), (store) => { + const mismatchedFingerprint = `${context.fingerprint}:mismatch`; + expectThrow( + () => store.search('refund policy', embedder.embed('refund policy'), {}, mismatchedFingerprint), + /fingerprint mismatch/u, + ); + }); + }); + runCase(cases, 'snapshot-is-deterministic-and-sorted', () => { + usingStore(options.createStore(contextFor('snapshot-is-deterministic-and-sorted')), (store) => { + store.upsertMany( + [CONFORMANCE_CORPUS[1], CONFORMANCE_CORPUS[0]].map((chunk) => ({ + chunk, + vector: embedder.embed(chunk.text), + })), + ); + const ids = store.snapshot().entries.map((entry) => entry.chunk.id); + if (ids.join(',') !== 'refunds,shipping') throw new Error(`snapshot order was ${ids.join(',')}.`); + }); + }); + runCase(cases, 'clear-removes-indexed-vectors', () => { + usingStore(options.createStore(contextFor('clear-removes-indexed-vectors')), (store) => { + store.upsert(CONFORMANCE_CORPUS[0], embedder.embed(CONFORMANCE_CORPUS[0].text)); + store.clear(); + const result = store.search('refund policy', embedder.embed('refund policy')); + if (result.chunks.length !== 0) + throw new Error(`expected clear() to remove chunks, got ${result.chunks.length}.`); + }); + }); + if (options.manifest.persistence === 'durable') { + runCase(cases, 'durable-round-trip', () => { + const durableContext = contextFor('durable-round-trip'); + usingStore(options.createStore(durableContext), (first) => { + first.upsertMany( + CONFORMANCE_CORPUS.slice(0, 2).map((chunk) => ({ + chunk, + vector: embedder.embed(chunk.text), + })), + ); + }); + usingStore(options.createStore(durableContext), (reopened) => { + const result = reopened.search('refund policy shipping', embedder.embed('refund policy shipping'), { topK: 2 }); + const ids = new Set(result.chunks.map((chunk) => chunk.id)); + if (!ids.has('refunds') || !ids.has('shipping')) { + throw new Error(`expected durable store to reload 'refunds' and 'shipping', got '${[...ids].join(',')}'.`); + } + }); + }); + } else { + cases.push({ name: 'durable-round-trip', status: 'skipped', message: 'adapter persistence is ephemeral' }); + } + + const summary = conformanceSummary(cases); + return { + manifest: options.manifest, + passed: summary.failed === 0, + cases, + summary, + }; +} + +function assertManifest(manifest: RagVectorStoreAdapterManifest): void { + if (!manifest.name.trim()) throw new Error('manifest name must be non-empty.'); + if (!manifest.version.trim()) throw new Error('manifest version must be non-empty.'); + if (manifest.kind !== 'vectorStore') throw new Error('manifest kind must be vectorStore.'); + if (!CONFORMANT_RAG_VECTOR_STORE_KINDS.includes(manifest.adapterKind)) { + throw new Error(`manifest adapterKind '${manifest.adapterKind}' is not supported by this conformance profile.`); + } + if (!manifest.metrics.includes('cosine')) throw new Error('manifest must support cosine metric.'); + if (!Number.isInteger(manifest.maxDimensions) || manifest.maxDimensions <= 0) { + throw new Error('manifest maxDimensions must be a positive integer.'); + } + for (const capability of ['upsert', 'upsertMany', 'search', 'snapshot', 'clear'] as const) { + if (manifest.capabilities[capability] !== true) + throw new Error(`manifest capability '${capability}' must be true.`); + } +} + +function runCase(cases: RagVectorStoreConformanceCaseResult[], name: string, fn: () => void): void { + try { + fn(); + cases.push({ name, status: 'passed' }); + } catch (error) { + cases.push({ name, status: 'failed', message: error instanceof Error ? error.message : String(error) }); + } +} + +function usingStore(store: RagVectorStoreAdapter, fn: (store: RagVectorStoreAdapter) => void): void { + let failure: unknown; + try { + fn(store); + } catch (error) { + failure = error; + } + try { + store.close(); + } catch (error) { + // Preserve the body failure as the conformance signal; close errors are secondary cleanup noise. + if (failure === undefined) throw error; + } + if (failure !== undefined) throw failure; +} + +function observeStorePersistence( + options: RagVectorStoreConformanceOptions, + context: RagVectorStoreConformanceContext, + embedder: Embedder, +): RagAdapterPersistence { + usingStore(options.createStore(context), (store) => { + store.upsert(CONFORMANCE_CORPUS[0], embedder.embed(CONFORMANCE_CORPUS[0].text)); + }); + let persisted = false; + usingStore(options.createStore(context), (reopened) => { + const result = reopened.search('refund policy', embedder.embed('refund policy'), { topK: 1 }); + persisted = result.chunks[0]?.id === CONFORMANCE_CORPUS[0].id; + }); + return persisted ? 'durable' : 'ephemeral'; +} + +function defaultConformanceRunNamespace(): string { + conformanceRunSequence += 1; + const random = Math.random().toString(36).slice(2, 8); + return `run-${Date.now().toString(36)}-${conformanceRunSequence.toString(36)}-${random}`; +} + +function expectThrow(fn: () => void, pattern: RegExp): void { + try { + fn(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!pattern.test(message)) throw new Error(`expected error matching ${pattern}, got '${message}'.`); + return; + } + throw new Error(`expected error matching ${pattern}, but no error was thrown.`); +} + +function safeConformanceNamespace(name: string): string { + return name.replace(/[^a-z0-9_-]+/giu, '-').replace(/^-+|-+$/gu, '') || 'case'; +} + +function conformanceSummary( + cases: readonly RagVectorStoreConformanceCaseResult[], +): RagVectorStoreConformanceReport['summary'] { + return { + passed: cases.filter((entry) => entry.status === 'passed').length, + failed: cases.filter((entry) => entry.status === 'failed').length, + skipped: cases.filter((entry) => entry.status === 'skipped').length, + }; +} diff --git a/packages/core/tests/rag-adapter-conformance.test.ts b/packages/core/tests/rag-adapter-conformance.test.ts new file mode 100644 index 00000000..4a1ab93d --- /dev/null +++ b/packages/core/tests/rag-adapter-conformance.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + builtinRagVectorStoreManifest, + createInMemoryRagVectorStoreForConformance, + type RagVectorStoreConformanceContext, + runRagVectorStoreConformance, +} from '../src/index.js'; +import { LocalPersistentRagVectorStoreAdapter } from '../src/rag-embedding-node.js'; + +describe('RAG vector store adapter conformance', () => { + test('built-in memory adapter passes deterministic conformance with durable case skipped', () => { + const manifest = builtinRagVectorStoreManifest('memory'); + expect(manifest).toBeDefined(); + + const report = runRagVectorStoreConformance({ + manifest: manifest!, + createStore: createInMemoryRagVectorStoreForConformance, + }); + + expect(report.passed).toBe(true); + expect(report.summary.failed).toBe(0); + expect(report.cases.some((entry) => entry.name === 'durable-round-trip' && entry.status === 'skipped')).toBe(true); + }); + + test('built-in local persistent adapter passes durable conformance', () => { + const manifest = builtinRagVectorStoreManifest('local-persistent'); + expect(manifest).toBeDefined(); + const dir = mkdtempSync(join(tmpdir(), 'kern-rag-conformance-')); + try { + const report = runRagVectorStoreConformance({ + manifest: manifest!, + createStore: (context) => + new LocalPersistentRagVectorStoreAdapter({ + directory: dir, + fileName: `${context.namespace}.json`, + fingerprint: context.fingerprint, + dims: context.dims, + }), + }); + + expect(report.passed).toBe(true); + expect(report.cases.some((entry) => entry.name === 'durable-round-trip' && entry.status === 'passed')).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('repeated local persistent conformance runs isolate backing files by default', () => { + const manifest = builtinRagVectorStoreManifest('local-persistent'); + expect(manifest).toBeDefined(); + const dir = mkdtempSync(join(tmpdir(), 'kern-rag-conformance-')); + try { + const createStore = (context: RagVectorStoreConformanceContext) => + new LocalPersistentRagVectorStoreAdapter({ + directory: dir, + fileName: `${context.namespace}.json`, + fingerprint: context.fingerprint, + dims: context.dims, + }); + + const first = runRagVectorStoreConformance({ manifest: manifest!, createStore }); + const second = runRagVectorStoreConformance({ manifest: manifest!, createStore }); + + expect(first.passed).toBe(true); + expect(second.passed).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('manifest shape failures are reported instead of thrown', () => { + const manifest = { + ...builtinRagVectorStoreManifest('memory')!, + metrics: [], + }; + + const report = runRagVectorStoreConformance({ + manifest, + createStore: createInMemoryRagVectorStoreForConformance, + }); + + expect(report.passed).toBe(false); + expect(report.cases.some((entry) => entry.name === 'manifest-shape' && entry.status === 'failed')).toBe(true); + }); + + test('manifest adapter mismatches are reported instead of thrown', () => { + const manifest = { + ...builtinRagVectorStoreManifest('memory')!, + adapterKind: 'local-persistent' as const, + }; + + const report = runRagVectorStoreConformance({ + manifest, + createStore: createInMemoryRagVectorStoreForConformance, + }); + + expect(report.passed).toBe(false); + expect(report.cases.some((entry) => entry.name === 'manifest-matches-adapter' && entry.status === 'failed')).toBe( + true, + ); + }); + + test('manifest persistence mismatches are reported instead of thrown', () => { + const manifest = { + ...builtinRagVectorStoreManifest('memory')!, + persistence: 'durable' as const, + }; + + const report = runRagVectorStoreConformance({ + manifest, + createStore: createInMemoryRagVectorStoreForConformance, + }); + + expect(report.passed).toBe(false); + expect( + report.cases.some((entry) => entry.name === 'persistence-matches-adapter' && entry.status === 'failed'), + ).toBe(true); + }); +});